From a2638026016e652ec68ccc3333d85c6218da5ca1 Mon Sep 17 00:00:00 2001 From: "[OXID-PS] Keywan Ghadami" Date: Sun, 29 Jul 2018 15:10:37 +0200 Subject: [PATCH 1/3] Rename htdocs/packages.php to src/RegistryBuilder.php --- htdocs/packages.php => src/RegistryBuilder.php | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename htdocs/packages.php => src/RegistryBuilder.php (100%) diff --git a/htdocs/packages.php b/src/RegistryBuilder.php similarity index 100% rename from htdocs/packages.php rename to src/RegistryBuilder.php From ed188b2efa663cf8246737737d3b02b67ba7bb59 Mon Sep 17 00:00:00 2001 From: "[OXID-PS] Keywan Ghadami" Date: Sun, 29 Jul 2018 01:13:03 +0200 Subject: [PATCH 2/3] webhooks & refactoring - using composer autoloader - new settings -- ip protection -- webhook settings --- composer.json | 5 + confs/.htaccess | 2 + confs/samples/gitlab.ini | 16 ++ htdocs/packages.php | 15 ++ htdocs/webhook.php | 10 + src/Auth.php | 37 +++ src/AuthWebhook.php | 39 ++++ src/Builder.php | 1 + src/Config.php | 46 ++++ src/RegistryBuilder.php | 481 +++++++++++++++++++++++---------------- 10 files changed, 451 insertions(+), 201 deletions(-) create mode 100644 confs/.htaccess create mode 100644 htdocs/packages.php create mode 100644 htdocs/webhook.php create mode 100644 src/Auth.php create mode 100644 src/AuthWebhook.php create mode 100644 src/Builder.php create mode 100644 src/Config.php diff --git a/composer.json b/composer.json index d1516fb..01ae5d1 100644 --- a/composer.json +++ b/composer.json @@ -2,5 +2,10 @@ "require": { "m4tthumphrey/php-gitlab-api": "^9.0", "php-http/guzzle6-adapter": "^1.1" + }, + "autoload": { + "psr-4": { + "GitlabComposer\\": "src" + } } } diff --git a/confs/.htaccess b/confs/.htaccess new file mode 100644 index 0000000..76e8380 --- /dev/null +++ b/confs/.htaccess @@ -0,0 +1,2 @@ +Satisfy All +Deny from all diff --git a/confs/samples/gitlab.ini b/confs/samples/gitlab.ini index 21475ed..0503c06 100644 --- a/confs/samples/gitlab.ini +++ b/confs/samples/gitlab.ini @@ -5,6 +5,22 @@ endpoint="http://gitlab.example.com/api/v4/" api_key="ASDFGHJKL12345678" method="ssh" +;automaticly create a webhook in the project so every new tag will availible in the composer registry +;your API Key must belong to a user with maintaner rights or higher to use this setting +;create_webhook="true" + +;You can set manually a URL if you do not want to relay to automaticlly getting the url during the call to package.json/package.php +;this must be set on public systems for security reason +;webhook_url="https://yourdomain/webhook.php" + +;ip white listing use it only as additional security layer +;allowed_client_ips[]="" +;allowed_client_ips[]="" +;allowed_webhook_ips[]="" +;allowed_webhook_ips[]="" + +;MANDETORY setting to allow webhooks put a random secrete string here +;webhook_token= ; You can restrict to some gitlab groups: ;groups[]="one_group" diff --git a/htdocs/packages.php b/htdocs/packages.php new file mode 100644 index 0000000..e6cf5cb --- /dev/null +++ b/htdocs/packages.php @@ -0,0 +1,15 @@ +getConfs(); + $a = new Auth(); + $a->setConfig($confs); + $a->auth(); + $Cr = new RegistryBuilder(); + $Cr->setConfig($confs); + $Cr->outputFile(); +} catch (\Exception $ex) { + print $ex; +} diff --git a/htdocs/webhook.php b/htdocs/webhook.php new file mode 100644 index 0000000..76a2706 --- /dev/null +++ b/htdocs/webhook.php @@ -0,0 +1,10 @@ +getConfs(); +$a=new AuthWebhook(); +$a->setConfig($confs); +$a->auth(); +$Cr=new RegistryBuilder(); +$Cr->setConfig($confs); +$Cr->update(); diff --git a/src/Auth.php b/src/Auth.php new file mode 100644 index 0000000..185d350 --- /dev/null +++ b/src/Auth.php @@ -0,0 +1,37 @@ +confs['allowed_client_ips']; + } + + public function auth(){ + $ips = $this->getAllowedIps(); + if ($ips) { + if (!isset($_SERVER['REMOTE_ADDR'])){ + return true; + } + $REMOTE_ADDR = $_SERVER['REMOTE_ADDR']; + if (in_array($REMOTE_ADDR, $ips)){ + return true; + } + exit($REMOTE_ADDR . ' is not allowed to access'); + } + } + + public function setConfig($confs){ + $this->confs = $confs; + } +} \ No newline at end of file diff --git a/src/AuthWebhook.php b/src/AuthWebhook.php new file mode 100644 index 0000000..16500f8 --- /dev/null +++ b/src/AuthWebhook.php @@ -0,0 +1,39 @@ +confs['allowed_webhook_ips']; + } + + + public function auth(){ + if (!$this->confs['webhook_token']) { + http_response_code(500); + exit("webhook_token is not configured in gitlab.ini, please add it to the composer-gitlab config file"); + } + if (!$_SERVER['HTTP_X-Gitlab-Token'] == $this->confs['webhook_token']){ + http_response_code(403); + exit("X-Gitlab-Token is not allowed to access"); + } + return parent::auth(); + } +} \ No newline at end of file diff --git a/src/Builder.php b/src/Builder.php new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/src/Builder.php @@ -0,0 +1 @@ + diff --git a/src/Config.php b/src/Config.php new file mode 100644 index 0000000..532348c --- /dev/null +++ b/src/Config.php @@ -0,0 +1,46 @@ +confs['allowed_client_ips']) ? $this->confs['allowed_client_ips'] : false; + $confs['allowed_webhook_ips'] = isset($this->confs['allowed_webhook_ips']) ? $this->confs['allowed_webhook_ips'] : false; + + $this->confs = $confs; + } + public function getConfs(){ + return $this->confs; + } +} \ No newline at end of file diff --git a/src/RegistryBuilder.php b/src/RegistryBuilder.php index 9a7d560..55c3856 100644 --- a/src/RegistryBuilder.php +++ b/src/RegistryBuilder.php @@ -1,249 +1,328 @@ = $mtime) { - header('HTTP/1.0 304 Not Modified'); - } else { - readfile($file); + protected $confs; + + public function setConfig($confs){ + $this->confs = $confs; } - die(); -}; - -// See ../confs/samples/gitlab.ini -$config_file = __DIR__ . '/../confs/gitlab.ini'; -if (!file_exists($config_file)) { - header('HTTP/1.0 500 Internal Server Error'); - die('confs/gitlab.ini missing'); -} -$confs = parse_ini_file($config_file); -$client = Client::create($confs['endpoint']); -$client->authenticate($confs['api_key'], Client::AUTH_URL_TOKEN); + /** + * Output a json file, sending max-age header + */ + function outputFile() + { + $file = $this->packages_file; + if (!file_exists($this->packages_file)){ + $this->build(); + } + $mtime = filemtime($file); -$groups = $client->api('groups'); -$projects = $client->api('projects'); -$repos = $client->api('repositories'); + header('Content-Type: application/json'); + header('Last-Modified: ' . gmdate('r', $mtime)); + header('Cache-Control: max-age=0'); -$validMethods = array('ssh', 'http'); -if (isset($confs['method']) && in_array($confs['method'], $validMethods)) { - define('method', $confs['method']); -} else { - define('method', 'ssh'); -} + if (!empty($_SERVER['HTTP_IF_MODIFIED_SINCE']) && ($since = strtotime($_SERVER['HTTP_IF_MODIFIED_SINCE'])) && $since >= $mtime) { + header('HTTP/1.0 304 Not Modified'); + } else { + readfile($file); + } + + } -$allow_package_name_mismatches = !empty($confs['allow_package_name_mismatch']); -/** - * Retrieves some information about a project's composer.json - * - * @param array $project - * @param string $ref commit id - * @return array|false - */ -$fetch_composer = function($project, $ref) use ($repos, $allow_package_name_mismatches) { - try { - $c = $repos->getFile($project['id'], 'composer.json', $ref); - if(!isset($c['content'])) { + /** + * Retrieves some information about a project's composer.json + * + * @param array $project + * @param string $ref commit id + * @return array|false + */ + public function fetch_composer($project, $ref) { + $repos = $this->repos; + $allow_package_name_mismatches = $this->confs['allow_package_name_mismatch']; + try { + $c = $repos->getFile($project['id'], 'composer.json', $ref); + + if (!isset($c['content'])) { + return false; + } + + $composer = json_decode(base64_decode($c['content']), true); + + if (empty($composer['name']) || (!$allow_package_name_mismatches && strcasecmp($composer['name'], $project['path_with_namespace']) !== 0)) { + return false; // packages must have a name and must match + } + + return $composer; + } catch (RuntimeException $e) { + //fwrite(STDERR, $e->getMessage() . $project['id'] . $ref); return false; } + } - $composer = json_decode(base64_decode($c['content']), true); + /** + * Retrieves some information about a project for a specific ref + * + * @param array $project + * @param array $ref commit id + * @return array [$version => ['name' => $name, 'version' => $version, 'source' => [...]]] + */ + public function fetch_ref($project, $ref) { - if (empty($composer['name']) || (!$allow_package_name_mismatches && strcasecmp($composer['name'], $project['path_with_namespace']) !== 0)) { - return false; // packages must have a name and must match + static $ref_cache = []; + + $ref_key = md5(serialize($project) . serialize($ref)); + + if (!isset($ref_cache[$ref_key])) { + if (preg_match('/^v?\d+\.\d+(\.\d+)*(\-(dev|patch|alpha|beta|RC)\d*)?$/', $ref['name'])) { + $version = $ref['name']; + } else { + $version = 'dev-' . $ref['name']; + } + if (($data = $this->fetch_composer($project, $ref['commit']['id'])) !== false) { + $data['version'] = $version; + $data['source'] = [ + 'url' => $project[method . '_url_to_repo'], + 'type' => 'git', + 'reference' => $ref['commit']['id'], + ]; + + $ref_cache[$ref_key] = [$version => $data]; + } else { + $ref_cache[$ref_key] = []; + } } - return $composer; - } catch (RuntimeException $e) { - return false; + return $ref_cache[$ref_key]; } -}; -/** - * Retrieves some information about a project for a specific ref - * - * @param array $project - * @param string $ref commit id - * @return array [$version => ['name' => $name, 'version' => $version, 'source' => [...]]] - */ -$fetch_ref = function($project, $ref) use ($fetch_composer) { + protected $repos; + /** + * @var $projects Projects + */ + protected $projects; - static $ref_cache = []; - $ref_key = md5(serialize($project) . serialize($ref)); + /** + * update from a webhook + */ + public function update() { + //get post data + $data = json_decode(file_get_contents('php://input'), true); + $client = $this->getClient(); + $this->repos = $repos = $client->api('repositories'); + $project = $data['project']; + $project[method . '_url_to_repo'] = $project[method . '_url']; - if (!isset($ref_cache[$ref_key])) { - if (preg_match('/^v?\d+\.\d+(\.\d+)*(\-(dev|patch|alpha|beta|RC)\d*)?$/', $ref['name'])) { - $version = $ref['name']; - } else { - $version = 'dev-' . $ref['name']; - } + $ref_name = $data['ref']; + $ref_name = str_replace('refs/tags/','', $ref_name); + $ref_name = str_replace('refs/heads/','', $ref_name); - if (($data = $fetch_composer($project, $ref['commit']['id'])) !== false) { - $data['version'] = $version; - $data['source'] = [ - 'url' => $project[method . '_url_to_repo'], - 'type' => 'git', - 'reference' => $ref['commit']['id'], - ]; + $ref = ['name'=>$ref_name, 'commit' => ['id'=> $data['checkout_sha']]]; - $ref_cache[$ref_key] = [$version => $data]; - } else { - $ref_cache[$ref_key] = []; + + $file = __DIR__ . "/../cache/{$project['path_with_namespace']}.json"; + $datas = json_decode(file_get_contents($file),true); + foreach ($this->fetch_ref($project, $ref) as $version => $data) { + $datas[$version] = $data; } + + file_put_contents($file,json_encode($datas,JSON_PRETTY_PRINT)); + unlink($this->packages_file); + $this->build(); } - return $ref_cache[$ref_key]; -}; - -/** - * Retrieves some information about a project for all refs - * @param array $project - * @return array Same as $fetch_ref, but for all refs - */ -$fetch_refs = function($project) use ($fetch_ref, $repos) { - $datas = array(); - try { - foreach (array_merge($repos->branches($project['id']), $repos->tags($project['id'])) as $ref) { - foreach ($fetch_ref($project, $ref) as $version => $data) { - $datas[$version] = $data; + public function build() + { + $confs = $this->confs; + $client = $this->getClient(); + + $groups = $client->api('groups'); + /** + * @var $projects Projects + */ + $projects = $client->api('projects'); + $this->projects = $projects; + $this->repos = $repos = $client->api('repositories'); + + /** + * Retrieves some information about a project for all refs + * @param array $project + * @return array Same as $fetch_ref, but for all refs + */ + $fetch_refs = function ($project) use ($repos) { + $datas = array(); + try { + foreach (array_merge($repos->branches($project['id']), $repos->tags($project['id'])) as $ref) { + foreach ($this->fetch_ref($project, $ref) as $version => $data) { + $datas[$version] = $data; + } + } + } catch (RuntimeException $e) { + // The repo has no commits — skipping it. } - } - } catch (RuntimeException $e) { - // The repo has no commits — skipping it. - } - return $datas; -}; - -/** - * Caching layer on top of $fetch_refs - * Uses last_activity_at from the $project array, so no invalidation is needed - * - * @param array $project - * @return array Same as $fetch_refs - */ -$load_data = function($project) use ($fetch_refs) { - $file = __DIR__ . "/../cache/{$project['path_with_namespace']}.json"; - $mtime = strtotime($project['last_activity_at']); - - if (!is_dir(dirname($file))) { - mkdir(dirname($file), 0777, true); - } + return $datas; + }; + + /** + * Caching layer on top of $fetch_refs + * Uses last_activity_at from the $project array, so no invalidation is needed + * + * @param array $project + * @return array Same as $fetch_refs + */ + $load_data = function ($project) use ($fetch_refs) { + $file = __DIR__ . "/../cache/{$project['path_with_namespace']}.json"; + $mtime = strtotime($project['last_activity_at']); + + if (!is_dir(dirname($file))) { + mkdir(dirname($file), 0777, true); + } - if (file_exists($file) && filemtime($file) >= $mtime) { - if (filesize($file) > 0) { - return json_decode(file_get_contents($file)); - } else { - return false; - } - } elseif ($data = $fetch_refs($project)) { - file_put_contents($file, json_encode($data)); - touch($file, $mtime); + if (file_exists($file) && filemtime($file) >= $mtime) { + if (filesize($file) > 0) { + return json_decode(file_get_contents($file)); + } else { + return false; + } + } elseif ($data = $fetch_refs($project)) { + if ($data) { + if ($this->confs['create_webhook']) { + $webhook_url = $this->confs['webhook_url']; + $id = $project['id']; + $allHooks = $this->projects->hooks($id); + $hookExists = false; + foreach ($allHooks as $hook) { + if ($hook['url'] == $webhook_url) { + $hookExists = true; + break; + } + } + if (!$hookExists) { + $arguments['tag_push_events'] = true; + if ($this->confs['webhook_token']) { + $arguments['token'] = $this->confs['webhook_token']; + } + $this->projects->addHook($id, $webhook_url, $arguments); + } + } + } + file_put_contents($file, json_encode($data,JSON_PRETTY_PRINT)); + touch($file, $mtime); - return $data; - } else { - $f = fopen($file, 'w'); - fclose($f); - touch($file, $mtime); + return $data; + } else { + $f = fopen($file, 'w'); + fclose($f); + touch($file, $mtime); - return false; - } -}; - -/** - * Determine the name to use for the package. - * - * @param array $project - * @return string The name of the project - */ -$get_package_name = function($project) use ($allow_package_name_mismatches, $fetch_ref, $repos) { - if ($allow_package_name_mismatches) { - $ref = $fetch_ref($project, $repos->branch($project['id'], $project['default_branch'])); - return reset($ref)['name']; - } + return false; + } + }; + + /** + * Determine the name to use for the package. + * + * @param array $project + * @return string The name of the project + */ + $get_package_name = function ($project) use ($repos) { + $allow_package_name_mismatches = $this->confs['allow_package_name_mismatch']; + if ($allow_package_name_mismatches) { + $ref = $this->fetch_ref($project, $repos->branch($project['id'], $project['default_branch'])); + return reset($ref)['name']; + } - return $project['path_with_namespace']; -}; - -// Load projects -$all_projects = array(); -$mtime = 0; -if (!empty($confs['groups'])) { - // We have to get projects from specifics groups - foreach ($groups->all(array('page' => 1, 'per_page' => 100)) as $group) { - if (!in_array($group['name'], $confs['groups'], true)) { - continue; - } - for ($page = 1; count($p = $groups->projects($group['id'], array('page' => $page, 'per_page' => 100))); $page++) { - foreach ($p as $project) { - $all_projects[] = $project; - $mtime = max($mtime, strtotime($project['last_activity_at'])); + return $project['path_with_namespace']; + }; + + // Load projects + $all_projects = array(); + $mtime = 0; + if (!empty($confs['groups'])) { + // We have to get projects from specifics groups + foreach ($groups->all(array('page' => 1, 'per_page' => 100)) as $group) { + if (!in_array($group['name'], $confs['groups'], true)) { + continue; + } + for ($page = 1; count($p = $groups->projects($group['id'], array('page' => $page, 'per_page' => 100))); $page++) { + foreach ($p as $project) { + $all_projects[] = $project; + $mtime = max($mtime, strtotime($project['last_activity_at'])); + } + } + } + } else { + // We have to get all accessible projects + $me = $client->api('users')->me(); + for ($page = 1; count($p = $projects->all(array('page' => $page, 'per_page' => 100))); $page++) { + foreach ($p as $project) { + $all_projects[] = $project; + $mtime = max($mtime, strtotime($project['last_activity_at'])); + } } } - } -} else { - // We have to get all accessible projects - $me = $client->api('users')->me(); - for ($page = 1; count($p = $projects->all(array('page' => $page, 'per_page' => 100))); $page++) { - foreach ($p as $project) { - $all_projects[] = $project; - $mtime = max($mtime, strtotime($project['last_activity_at'])); - } - } -} -// Regenerate packages_file is needed -if (!file_exists($packages_file) || filemtime($packages_file) < $mtime) { - $packages = array(); - foreach ($all_projects as $project) { - if (($package = $load_data($project)) && ($package_name = $get_package_name($project))) { - $packages[$package_name] = $package; - } - } - if ( file_exists( $static_file ) ) { - $static_packages = json_decode( file_get_contents( $static_file ) ); - foreach ( $static_packages as $name => $package ) { - foreach ( $package as $version => $root ) { - if ( isset( $root->extra ) ) { - $source = '_source'; - while ( isset( $root->extra->{$source} ) ) { - $source = '_' . $source; - } - $root->extra->{$source} = 'static'; + // Regenerate packages_file is needed + if (!file_exists($this->packages_file) || filemtime($this->packages_file) < $mtime) { + $packages = array(); + foreach ($all_projects as $project) { + if (($package = $load_data($project)) && ($package_name = $get_package_name($project))) { + $packages[$package_name] = $package; } - else { - $root->extra = array( - '_source' => 'static', - ); + } + if (file_exists($this->static_file)) { + $static_packages = json_decode(file_get_contents($this->static_file)); + foreach ($static_packages as $name => $package) { + foreach ($package as $version => $root) { + if (isset($root->extra)) { + $source = '_source'; + while (isset($root->extra->{$source})) { + $source = '_' . $source; + } + $root->extra->{$source} = 'static'; + } else { + $root->extra = array( + '_source' => 'static', + ); + } + } + $packages[$name] = $package; } } - $packages[$name] = $package; + $data = json_encode(array( + 'packages' => array_filter($packages), + ), JSON_PRETTY_PRINT); + + file_put_contents($this->packages_file, $data); } } - $data = json_encode(array( - 'packages' => array_filter($packages), - )); - file_put_contents($packages_file, $data); -} + /** + * @param $confs + * @return Client + */ + public function getClient() + { + $confs = $this->confs; + $client = Client::create($confs['endpoint']); + $client->authenticate($confs['api_key'], Client::AUTH_URL_TOKEN); + return $client; + } -$outputFile($packages_file); +} From c86138274c7ede7b3831499197fe45042995b080 Mon Sep 17 00:00:00 2001 From: Keywan Ghadami Date: Wed, 1 Aug 2018 17:40:00 +0200 Subject: [PATCH 3/3] use more general setting --- confs/samples/gitlab.ini | 4 ++-- src/AuthWebhook.php | 2 +- src/Config.php | 4 +++- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/confs/samples/gitlab.ini b/confs/samples/gitlab.ini index 0503c06..6710d53 100644 --- a/confs/samples/gitlab.ini +++ b/confs/samples/gitlab.ini @@ -10,8 +10,8 @@ method="ssh" ;create_webhook="true" ;You can set manually a URL if you do not want to relay to automaticlly getting the url during the call to package.json/package.php -;this must be set on public systems for security reason -;webhook_url="https://yourdomain/webhook.php" +;this must be set on public systems for security reason to avoid abuse usage of the webhook endpoint +;base_url="https://yourdomain/" ;ip white listing use it only as additional security layer ;allowed_client_ips[]="" diff --git a/src/AuthWebhook.php b/src/AuthWebhook.php index 16500f8..2e41cd7 100644 --- a/src/AuthWebhook.php +++ b/src/AuthWebhook.php @@ -30,7 +30,7 @@ public function auth(){ http_response_code(500); exit("webhook_token is not configured in gitlab.ini, please add it to the composer-gitlab config file"); } - if (!$_SERVER['HTTP_X-Gitlab-Token'] == $this->confs['webhook_token']){ + if (!$_SERVER['HTTP_X_GITLAB_TOKEN'] == $this->confs['webhook_token']){ http_response_code(403); exit("X-Gitlab-Token is not allowed to access"); } diff --git a/src/Config.php b/src/Config.php index 532348c..dc933b0 100644 --- a/src/Config.php +++ b/src/Config.php @@ -29,9 +29,11 @@ public function __construct(){ } $confs['allow_package_name_mismatch'] = !empty($confs['allow_package_name_mismatch']); - $confs['webhook_url'] = isset( $confs['webhook_url']) ? $confs['webhook_url'] : str_replace('packages.json', 'webhook.php', + $confs['base_url'] = isset( $confs['base_url']) ? $confs['base_url'] : str_replace('packages.json', '', "https://$_SERVER[HTTP_HOST]$_SERVER[REQUEST_URI]"); + $confs['webhook_url'] = $confs['base_url'] . 'webhook.php'; + $confs['webhook_token'] = isset($confs['webhook_token']) ? $confs['webhook_token'] : false; $confs['create_webhook'] = !empty($confs['create_webhook']);