From ad195fd6f228ed9c8cf304ffe52c32f82be07048 Mon Sep 17 00:00:00 2001 From: "Charles R. (hugo)" Date: Tue, 19 Apr 2016 10:19:51 +0200 Subject: [PATCH 01/96] Replace /home/git with pillar:lookup:root_dir variable --- gitlab/files/gitlab-gitlab.yml | 10 ++++++---- gitlab/files/gitlab-logrotate | 6 ++++-- gitlab/files/gitlab-nginx | 6 ++++-- gitlab/files/gitlab-nginx-ssl | 8 +++++--- gitlab/files/gitlab-shell-config.yml | 8 +++++--- gitlab/files/gitlab-unicorn.rb | 12 ++++++----- gitlab/gitlab-shell.sls | 8 +++++--- gitlab/gitlab.sls | 30 +++++++++++++++------------- gitlab/user.sls | 6 ++++-- 9 files changed, 56 insertions(+), 38 deletions(-) diff --git a/gitlab/files/gitlab-gitlab.yml b/gitlab/files/gitlab-gitlab.yml index c50120f..0a0887b 100644 --- a/gitlab/files/gitlab-gitlab.yml +++ b/gitlab/files/gitlab-gitlab.yml @@ -7,6 +7,8 @@ # 2. Replace gitlab -> host with your domain # 3. Replace gitlab -> email_from +{% set root_dir = salt['pillar.get']('gitlab:lookup:root_dir', '/home/git') %} + production: &base # # 1. GitLab app settings @@ -180,7 +182,7 @@ production: &base # GitLab Satellites satellites: # Relative paths are relative to Rails.root (default: tmp/repo_satellites/) - path: /home/git/gitlab-satellites/ + path: {{ root_dir }}/gitlab-satellites/ ## Backup settings backup: @@ -189,11 +191,11 @@ production: &base ## GitLab Shell settings gitlab_shell: - path: /home/git/gitlab-shell/ + path: {{ root_dir }}/gitlab-shell/ # REPOS_PATH MUST NOT BE A SYMLINK!!! - repos_path: /home/git/repositories/ - hooks_path: /home/git/gitlab-shell/hooks/ + repos_path: {{ root_dir }}/repositories/ + hooks_path: {{ root_dir }}/gitlab-shell/hooks/ # Git over HTTP upload_pack: true diff --git a/gitlab/files/gitlab-logrotate b/gitlab/files/gitlab-logrotate index 6df8685..351f5b3 100644 --- a/gitlab/files/gitlab-logrotate +++ b/gitlab/files/gitlab-logrotate @@ -1,7 +1,9 @@ # GitLab logrotate settings # based on: http://stackoverflow.com/a/4883967 -/home/git/gitlab/log/*.log { +{% set root_dir = salt['pillar.get']('gitlab:lookup:root_dir', '/home/git') %} + +{{ root_dir }}/gitlab/log/*.log { weekly missingok rotate 52 @@ -11,7 +13,7 @@ copytruncate } -/home/git/gitlab-shell/gitlab-shell.log { +{{ root_dir }}/gitlab-shell/gitlab-shell.log { weekly missingok rotate 52 diff --git a/gitlab/files/gitlab-nginx b/gitlab/files/gitlab-nginx index 9c23dd3..82f71eb 100644 --- a/gitlab/files/gitlab-nginx +++ b/gitlab/files/gitlab-nginx @@ -2,15 +2,17 @@ # Maintainer: @randx # App Version: 5.0 +{% set root_dir = salt['pillar.get']('gitlab:lookup:root_dir', '/home/git') %} + upstream gitlab { - server unix:/home/git/gitlab/tmp/sockets/gitlab.socket; + server unix:{{ root_dir }}/gitlab/tmp/sockets/gitlab.socket; } server { listen *:80 default_server; # e.g., listen 192.168.1.1:80; In most cases *:80 is a good idea server_name {{ grains['fqdn'] }}; # e.g., server_name source.example.com; server_tokens off; # don't show the version number, a security best practice - root /home/git/gitlab/public; + root {{ root_dir }}/gitlab/public; # Set value of client_max_body_size to at least the value of git.max_size in gitlab.yml client_max_body_size 5m; diff --git a/gitlab/files/gitlab-nginx-ssl b/gitlab/files/gitlab-nginx-ssl index 60d8f87..0f4f770 100644 --- a/gitlab/files/gitlab-nginx-ssl +++ b/gitlab/files/gitlab-nginx-ssl @@ -18,13 +18,15 @@ # 2) Enable https "https: true" # 3) Update ssl for gravatar "ssl_url: https://secure.gravatar.com/avatar/%{hash}?s=%{size}&d=mm" +{% set root_dir = salt['pillar.get']('gitlab:lookup:root_dir', '/home/git') %} + upstream gitlab { ## Uncomment if you have set up puma/unicorn to listen on a unix socket (recommended). - server unix:/home/git/gitlab/tmp/sockets/gitlab.socket; + server unix:{{ root_dir }}/gitlab/tmp/sockets/gitlab.socket; ## Uncomment if puma/unicorn are configured to listen on a tcp port. - ## Check the port number in /home/git/gitlab/config/{puma.rb/unicorn.rb} + ## Check the port number in {{ root_dir }}/gitlab/config/{puma.rb/unicorn.rb} # server 127.0.0.1:9292; } @@ -41,7 +43,7 @@ server { listen 443 ssl; server_name {{ grains['fqdn'] }}; server_tokens off; - root /home/git/gitlab/public; + root {{ root_dir }}/gitlab/public; ssl on; ssl_certificate /etc/nginx/gitlab.crt; diff --git a/gitlab/files/gitlab-shell-config.yml b/gitlab/files/gitlab-shell-config.yml index f6e0da9..2aac25b 100644 --- a/gitlab/files/gitlab-shell-config.yml +++ b/gitlab/files/gitlab-shell-config.yml @@ -1,6 +1,8 @@ # GitLab user. git by default user: git +{% set root_dir = salt['pillar.get']('gitlab:lookup:root_dir', '/home/git') %} + # Url to gitlab instance. Used for api calls. Should end with a slash. {%- if salt['pillar.get']('gitlab:https', false) %} gitlab_url: "https://{{ grains['fqdn'] }}/" @@ -23,10 +25,10 @@ http_settings: # Give the canonicalized absolute pathname, # REPOS_PATH MUST NOT CONTAIN ANY SYMLINK!!! # Check twice that none of the components is a symlink, including "/home". -repos_path: "/home/git/repositories" +repos_path: "{{ root_dir }}/repositories" # File used as authorized_keys for gitlab user -auth_file: "/home/git/.ssh/authorized_keys" +auth_file: "{{ root_dir }}/.ssh/authorized_keys" # Redis settings used for pushing commit notices to gitlab redis: @@ -38,7 +40,7 @@ redis: # Log file. # Default is gitlab-shell.log in the root directory. -# log_file: "/home/git/gitlab-shell/gitlab-shell.log" +# log_file: "{{ root_dir }}/gitlab-shell/gitlab-shell.log" # Log level. INFO by default log_level: {{ salt['pillar.get']('gitlab:shell:log_level', 'INFO') }} diff --git a/gitlab/files/gitlab-unicorn.rb b/gitlab/files/gitlab-unicorn.rb index cc23145..00dd7ae 100644 --- a/gitlab/files/gitlab-unicorn.rb +++ b/gitlab/files/gitlab-unicorn.rb @@ -8,6 +8,8 @@ # See http://unicorn.bogomips.org/Unicorn/Configurator.html for complete # documentation. +{% set root_dir = salt['pillar.get']('gitlab:lookup:root_dir', '/home/git') %} + # Uncomment and customize the last line to run in a non-root path # WARNING: We recommend creating a FQDN to host GitLab in a root path instead of this. # Note that four settings need to be changed for this to work. @@ -32,24 +34,24 @@ # Help ensure your application will always spawn in the symlinked # "current" directory that Capistrano sets up. -working_directory "/home/git/gitlab" # available in 0.94.0+ +working_directory "{{ root_dir }}/gitlab" # available in 0.94.0+ # listen on both a Unix domain socket and a TCP port, # we use a shorter backlog for quicker failover when busy -listen "/home/git/gitlab/tmp/sockets/gitlab.socket", :backlog => 64 +listen "{{ root_dir }}/gitlab/tmp/sockets/gitlab.socket", :backlog => 64 listen "127.0.0.1:8080", :tcp_nopush => true # nuke workers after 30 seconds instead of 60 seconds (the default) timeout {{ salt['pillar.get']('gitlab:unicorn:timeout', 30) }} # feel free to point this anywhere accessible on the filesystem -pid "/home/git/gitlab/tmp/pids/unicorn.pid" +pid "{{ root_dir }}/gitlab/tmp/pids/unicorn.pid" # By default, the Unicorn logger will write to stderr. # Additionally, some applications/frameworks log to stderr or stdout, # so prevent them from going to /dev/null when daemonized here: -stderr_path "/home/git/gitlab/log/unicorn.stderr.log" -stdout_path "/home/git/gitlab/log/unicorn.stdout.log" +stderr_path "{{ root_dir }}/gitlab/log/unicorn.stderr.log" +stdout_path "{{ root_dir }}/gitlab/log/unicorn.stdout.log" # combine Ruby 2.0.0dev or REE with "preload_app true" for memory savings # http://rubyenterpriseedition.com/faq.html#adapt_apps_for_cow diff --git a/gitlab/gitlab-shell.sls b/gitlab/gitlab-shell.sls index 559c442..70ed162 100644 --- a/gitlab/gitlab-shell.sls +++ b/gitlab/gitlab-shell.sls @@ -2,11 +2,13 @@ include: - gitlab.user - gitlab.ruby +{% set root_dir = salt['pillar.get']('gitlab:lookup:root_dir', '/home/git') %} + gitlab-shell-git: git.latest: - name: https://gitlab.com/gitlab-org/gitlab-shell.git - rev: {{ salt['pillar.get']('gitlab:shell_version') }} - - target: /home/git/gitlab-shell + - target: {{ root_dir }}/gitlab-shell - user: git - require: - pkg: gitlab-deps @@ -17,7 +19,7 @@ gitlab-shell-git: # https://gitlab.com/gitlab-org/gitlab-shell/blob/master/config.yml.example gitlab-shell-config: file.managed: - - name: /home/git/gitlab-shell/config.yml + - name: {{ root_dir }}/gitlab-shell/config.yml - source: salt://gitlab/files/gitlab-shell-config.yml - template: jinja - user: git @@ -29,7 +31,7 @@ gitlab-shell-config: gitlab-shell: cmd.wait: - user: git - - cwd: /home/git/gitlab-shell + - cwd: {{ root_dir }}/gitlab-shell - name: ./bin/install - shell: /bin/bash - watch: diff --git a/gitlab/gitlab.sls b/gitlab/gitlab.sls index 3e23e13..62e9f9c 100644 --- a/gitlab/gitlab.sls +++ b/gitlab/gitlab.sls @@ -1,12 +1,14 @@ include: - gitlab.ruby +{% set root_dir = salt['pillar.get']('gitlab:lookup:root_dir', '/home/git') %} + gitlab-git: git.latest: - name: https://gitlab.com/gitlab-org/gitlab-ce.git - rev: {{ salt['pillar.get']('gitlab:gitlab_version') }} - user: git - - target: /home/git/gitlab + - target: {{ root_dir }}/gitlab - require: - pkg: gitlab-deps - pkg: git @@ -17,7 +19,7 @@ gitlab-git: # https://gitlab.com/gitlab-org/gitlab-ce/blob/master/config/gitlab.yml.example gitlab-config: file.managed: - - name: /home/git/gitlab/config/gitlab.yml + - name: {{ root_dir }}/gitlab/config/gitlab.yml - source: salt://gitlab/files/gitlab-gitlab.yml - template: jinja - user: git @@ -30,7 +32,7 @@ gitlab-config: # https://gitlab.com/gitlab-org/gitlab-ce/blob/master/config/database.yml.postgresql gitlab-db-config: file.managed: - - name: /home/git/gitlab/config/database.yml + - name: {{ root_dir }}/gitlab/config/database.yml - source: salt://gitlab/files/gitlab-database.yml - template: jinja - user: git @@ -43,7 +45,7 @@ gitlab-db-config: # https://gitlab.com/gitlab-org/gitlab-ce/blob/master/config/unicorn.rb.example unicorn-config: file.managed: - - name: /home/git/gitlab/config/unicorn.rb + - name: {{ root_dir }}/gitlab/config/unicorn.rb - source: salt://gitlab/files/gitlab-unicorn.rb - template: jinja - user: git @@ -56,7 +58,7 @@ unicorn-config: # https://gitlab.com/gitlab-org/gitlab-ce/blob/master/config/initializers/rack_attack.rb.example rack_attack-config: file.managed: - - name: /home/git/gitlab/config/initializers/rack_attack.rb + - name: {{ root_dir }}/gitlab/config/initializers/rack_attack.rb - source: salt://gitlab/files/gitlab-rack_attack.rb - user: git - group: git @@ -67,7 +69,7 @@ rack_attack-config: git-config: file.managed: - - name: /home/git/.gitconfig + - name: {{ root_dir }}/.gitconfig - source: salt://gitlab/files/gitlab-gitconfig - template: jinja - user: git @@ -77,7 +79,7 @@ git-config: - user: git-user {% for dir in ['gitlab-satellites', 'gitlab/tmp/pids', 'gitlab/tmp/sockets', 'gitlab/public/uploads'] %} -/home/git/{{ dir }}: +{{ root_dir }}/{{ dir }}: file.directory: - user: git - group: git @@ -90,7 +92,7 @@ git-config: gitlab-initialize: cmd.wait: - user: git - - cwd: /home/git/gitlab + - cwd: {{ root_dir }}/gitlab - name: echo yes | bundle exec rake gitlab:setup RAILS_ENV=production - shell: /bin/bash - unless: psql -U {{ salt['pillar.get']('gitlab:db_user') }} {{ salt['pillar.get']('gitlab:db_name') }} -c 'select * from users;' @@ -105,7 +107,7 @@ gitlab-initialize: gitlab-gems: cmd.wait: - user: git - - cwd: /home/git/gitlab + - cwd: {{ root_dir }}/gitlab - name: bundle install --deployment --without development test mysql aws - shell: /bin/bash - watch: @@ -120,7 +122,7 @@ gitlab-gems: gitlab-migrate-db: cmd.wait: - user: git - - cwd: /home/git/gitlab + - cwd: {{ root_dir }}/gitlab - name: bundle exec rake db:migrate RAILS_ENV=production - shell: /bin/bash - watch: @@ -133,7 +135,7 @@ gitlab-migrate-db: gitlab-recompile-assets: cmd.wait: - user: git - - cwd: /home/git/gitlab + - cwd: {{ root_dir }}/gitlab - name: bundle exec rake assets:clean assets:precompile RAILS_ENV=production - shell: /bin/bash - watch: @@ -144,7 +146,7 @@ gitlab-recompile-assets: gitlab-clear-cache: cmd.wait: - user: git - - cwd: /home/git/gitlab + - cwd: {{ root_dir }}/gitlab - name: bundle exec rake cache:clear RAILS_ENV=production - shell: /bin/bash - watch: @@ -156,7 +158,7 @@ gitlab-clear-cache: gitlab-stash: cmd.wait: - user: git - - cwd: /home/git/gitlab + - cwd: {{ root_dir }}/gitlab - name: git stash - watch: - git: gitlab-git @@ -176,7 +178,7 @@ gitlab-default: gitlab-service: file.symlink: - name: /etc/init.d/gitlab - - target: /home/git/gitlab/lib/support/init.d/gitlab + - target: {{ root_dir }}/gitlab/lib/support/init.d/gitlab - require: - git: gitlab-git service: diff --git a/gitlab/user.sls b/gitlab/user.sls index 7d145f0..182279f 100644 --- a/gitlab/user.sls +++ b/gitlab/user.sls @@ -1,14 +1,16 @@ +{% set root_dir = salt['pillar.get']('gitlab:lookup:root_dir', '/home/git') %} + git-user: user.present: - name : git - system: True - shell: /bin/bash - fullname: GitLab - - home: /home/git + - home: {{ root_dir }} git-home: file.directory: - - name: /home/git + - name: {{ root_dir }} - user: git - group: git - mode: 750 From 64bbf8783d7bbfe93ea26eaf8329f042deb81241 Mon Sep 17 00:00:00 2001 From: "Charles R. (hugo)" Date: Tue, 19 Apr 2016 10:30:21 +0200 Subject: [PATCH 02/96] Nginx conf file come from pillar lookup data --- gitlab/nginx.sls | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/gitlab/nginx.sls b/gitlab/nginx.sls index 2429b0f..228fb11 100644 --- a/gitlab/nginx.sls +++ b/gitlab/nginx.sls @@ -6,6 +6,9 @@ {% set nginx_path = '/etc/nginx/conf.d' %} {% endif %} +{% set nginx_conf = salt['pillar.get']('backupmanager:lookup:nginx_conf', 'salt://gitlab/files/gitlab-nginx') %} +{% set nginx_ssl_conf = salt['pillar.get']('backupmanager:lookup:nginx_ssl_conf', 'salt://gitlab/files/gitlab-nginx-ssl') %} + nginx: pkg.installed: [] service.running: @@ -30,7 +33,7 @@ nginx: gitlab-nginx: file.managed: - name: {{ nginx_path }}/gitlab.conf - - source: salt://gitlab/files/gitlab-nginx-ssl + - source: {{ nginx_ssl_conf }} - template: jinja - user: root - group: root @@ -66,7 +69,7 @@ nginx-ssl-cert: gitlab-nginx: file.managed: - name: {{ nginx_path }}/gitlab.conf - - source: salt://gitlab/files/gitlab-nginx + - source: {{ nginx_conf }} - template: jinja - user: root - group: root From 0b9e5769c361642b2553b6ad746b6bade32b9a39 Mon Sep 17 00:00:00 2001 From: "Charles R. (hugo)" Date: Thu, 21 Apr 2016 17:27:38 +0200 Subject: [PATCH 03/96] Add specific conf for gitlab-workhorse --- gitlab/gitlab-workhorse.sls | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 gitlab/gitlab-workhorse.sls diff --git a/gitlab/gitlab-workhorse.sls b/gitlab/gitlab-workhorse.sls new file mode 100644 index 0000000..cebb067 --- /dev/null +++ b/gitlab/gitlab-workhorse.sls @@ -0,0 +1,32 @@ + +{% set root_dir = salt['pillar.get']('gitlab:lookup:root_dir', '/home/git') %} +{% set lib_dir = salt['pillar.get']('gitlab:lookup:lib_dir', root_dir ~ '/libraries') %} + +gitlab-workhorse-git: + git.latest: + - name: https://gitlab.com/gitlab-org/gitlab-workhorse.git + - rev: {{ salt['pillar.get']('gitlab:workhorse_version') }} + - target: {{ lib_dir }}/gitlab-workhorse.git + - user: git + - require: + - pkg: gitlab-deps + - pkg: git + - sls: gitlab.ruby + - file: git-home + +{{ root_dir }}/gitlab-workhorse: + file.directory: + - user: git + - group: git + - mode: 750 + +gitlab-workhorse-make: + cmd.run: + - user: git + - cwd: {{ lib_dir }}/gitlab-workhorse.git + - name: make install PREFIX={{ root_dir }}/gitlab-workhorse + - shell: /bin/bash + - require: + - git: gitlab-workhorse-git + - file: {{ root_dir }}/gitlab-workhorse + From d4b019ee175f858aa5166f72c415d5d3947373fb Mon Sep 17 00:00:00 2001 From: "Charles R. (hugo)" Date: Thu, 21 Apr 2016 17:28:00 +0200 Subject: [PATCH 04/96] Add initd --- gitlab/files/initd | 399 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 399 insertions(+) create mode 100755 gitlab/files/initd diff --git a/gitlab/files/initd b/gitlab/files/initd new file mode 100755 index 0000000..b978adb --- /dev/null +++ b/gitlab/files/initd @@ -0,0 +1,399 @@ +#! /bin/sh + +# GITLAB +# Maintainer: @randx +# Authors: rovanion.luckey@gmail.com, @randx + +### BEGIN INIT INFO +# Provides: gitlab +# Required-Start: $local_fs $remote_fs $network $syslog redis-server +# Required-Stop: $local_fs $remote_fs $network $syslog +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: GitLab git repository management +# Description: GitLab git repository management +# chkconfig: - 85 14 +### END INIT INFO + + +### +# DO NOT EDIT THIS FILE! +# This file will be overwritten on update. +# Instead add/change your variables in /etc/default/gitlab +# An example defaults file can be found in lib/support/init.d/gitlab.default.example +### + +{%- set root_dir = salt['pillar.get']('gitlab:lookup:root_dir', '/home/git') %} +{%- set var_dir = salt['pillar.get']('gitlab:lookup:var_dir', '/home/$app_user') -%} + +### Environment variables +RAILS_ENV="production" + +# Script variable names should be lower-case not to conflict with +# internal /bin/sh variables such as PATH, EDITOR or SHELL. +app_user="git" +app_root="{{ root_dir }}/gitlab" +pid_path="{{ var_dir }}/pids" +socket_path="{{ var_dir }}/sockets" +rails_socket="$socket_path/gitlab.socket" +web_server_pid_path="$pid_path/unicorn.pid" +sidekiq_pid_path="$pid_path/sidekiq.pid" +mail_room_enabled=false +mail_room_pid_path="$pid_path/mail_room.pid" +gitlab_workhorse_dir=$(cd {{ root_dir }}/gitlab-workhorse/bin 2> /dev/null && pwd) +gitlab_workhorse_pid_path="$pid_path/gitlab-workhorse.pid" +gitlab_workhorse_options="-listenUmask 0 -listenNetwork unix -listenAddr $socket_path/gitlab-workhorse.socket -authBackend http://127.0.0.1:8080 -authSocket $rails_socket -documentRoot $app_root/public" +gitlab_workhorse_log="{{ var_dir }}/logs/gitlab-workhorse.log" +shell_path="/bin/bash" + +# Read configuration variable file if it is present +test -f /etc/default/gitlab && . /etc/default/gitlab + +# Switch to the app_user if it is not he/she who is running the script. +if [ `whoami` != "$app_user" ]; then + eval su - "$app_user" -c $(echo \")$shell_path -l -c \'$0 "$@"\'$(echo \"); exit; +fi + +# Switch to the gitlab path, exit on failure. +if ! cd "$app_root" ; then + echo "Failed to cd into $app_root, exiting!"; exit 1 +fi + + +### Init Script functions + +## Gets the pids from the files +check_pids(){ + if ! mkdir -p "$pid_path"; then + echo "Could not create the path $pid_path needed to store the pids." + exit 1 + fi + # If there exists a file which should hold the value of the Unicorn pid: read it. + if [ -f "$web_server_pid_path" ]; then + wpid=$(cat "$web_server_pid_path") + else + wpid=0 + fi + if [ -f "$sidekiq_pid_path" ]; then + spid=$(cat "$sidekiq_pid_path") + else + spid=0 + fi + if [ -f "$gitlab_workhorse_pid_path" ]; then + hpid=$(cat "$gitlab_workhorse_pid_path") + else + hpid=0 + fi + if [ "$mail_room_enabled" = true ]; then + if [ -f "$mail_room_pid_path" ]; then + mpid=$(cat "$mail_room_pid_path") + else + mpid=0 + fi + fi +} + +## Called when we have started the two processes and are waiting for their pid files. +wait_for_pids(){ + # We are sleeping a bit here mostly because sidekiq is slow at writing its pid + i=0; + while [ ! -f $web_server_pid_path ] || [ ! -f $sidekiq_pid_path ] || [ ! -f $gitlab_workhorse_pid_path ] || { [ "$mail_room_enabled" = true ] && [ ! -f $mail_room_pid_path ]; }; do + sleep 0.1; + i=$((i+1)) + if [ $((i%10)) = 0 ]; then + echo -n "." + elif [ $((i)) = 301 ]; then + echo "Waited 30s for the processes to write their pids, something probably went wrong." + exit 1; + fi + done + echo +} + +# We use the pids in so many parts of the script it makes sense to always check them. +# Only after start() is run should the pids change. Sidekiq sets its own pid. +check_pids + + +## Checks whether the different parts of the service are already running or not. +check_status(){ + check_pids + # If the web server is running kill -0 $wpid returns true, or rather 0. + # Checks of *_status should only check for == 0 or != 0, never anything else. + if [ $wpid -ne 0 ]; then + kill -0 "$wpid" 2>/dev/null + web_status="$?" + else + web_status="-1" + fi + if [ $spid -ne 0 ]; then + kill -0 "$spid" 2>/dev/null + sidekiq_status="$?" + else + sidekiq_status="-1" + fi + if [ $hpid -ne 0 ]; then + kill -0 "$hpid" 2>/dev/null + gitlab_workhorse_status="$?" + else + gitlab_workhorse_status="-1" + fi + if [ "$mail_room_enabled" = true ]; then + if [ $mpid -ne 0 ]; then + kill -0 "$mpid" 2>/dev/null + mail_room_status="$?" + else + mail_room_status="-1" + fi + fi + if [ $web_status = 0 ] && [ $sidekiq_status = 0 ] && [ $gitlab_workhorse_status = 0 ] && { [ "$mail_room_enabled" != true ] || [ $mail_room_status = 0 ]; }; then + gitlab_status=0 + else + # http://refspecs.linuxbase.org/LSB_4.1.0/LSB-Core-generic/LSB-Core-generic/iniscrptact.html + # code 3 means 'program is not running' + gitlab_status=3 + fi +} + +## Check for stale pids and remove them if necessary. +check_stale_pids(){ + check_status + # If there is a pid it is something else than 0, the service is running if + # *_status is == 0. + if [ "$wpid" != "0" ] && [ "$web_status" != "0" ]; then + echo "Removing stale Unicorn web server pid. This is most likely caused by the web server crashing the last time it ran." + if ! rm "$web_server_pid_path"; then + echo "Unable to remove stale pid, exiting." + exit 1 + fi + fi + if [ "$spid" != "0" ] && [ "$sidekiq_status" != "0" ]; then + echo "Removing stale Sidekiq job dispatcher pid. This is most likely caused by Sidekiq crashing the last time it ran." + if ! rm "$sidekiq_pid_path"; then + echo "Unable to remove stale pid, exiting" + exit 1 + fi + fi + if [ "$hpid" != "0" ] && [ "$gitlab_workhorse_status" != "0" ]; then + echo "Removing stale gitlab-workhorse pid. This is most likely caused by gitlab-workhorse crashing the last time it ran." + if ! rm "$gitlab_workhorse_pid_path"; then + echo "Unable to remove stale pid, exiting" + exit 1 + fi + fi + if [ "$mail_room_enabled" = true ] && [ "$mpid" != "0" ] && [ "$mail_room_status" != "0" ]; then + echo "Removing stale MailRoom job dispatcher pid. This is most likely caused by MailRoom crashing the last time it ran." + if ! rm "$mail_room_pid_path"; then + echo "Unable to remove stale pid, exiting" + exit 1 + fi + fi +} + +## If no parts of the service is running, bail out. +exit_if_not_running(){ + check_stale_pids + if [ "$web_status" != "0" ] && [ "$sidekiq_status" != "0" ] && [ "$gitlab_workhorse_status" != "0" ] && { [ "$mail_room_enabled" != true ] || [ "$mail_room_status" != "0" ]; }; then + echo "GitLab is not running." + exit + fi +} + +## Starts Unicorn and Sidekiq if they're not running. +start_gitlab() { + check_stale_pids + + if [ "$web_status" != "0" ]; then + echo "Starting GitLab Unicorn" + fi + if [ "$sidekiq_status" != "0" ]; then + echo "Starting GitLab Sidekiq" + fi + if [ "$gitlab_workhorse_status" != "0" ]; then + echo "Starting gitlab-workhorse" + fi + if [ "$mail_room_enabled" = true ] && [ "$mail_room_status" != "0" ]; then + echo "Starting GitLab MailRoom" + fi + + # Then check if the service is running. If it is: don't start again. + if [ "$web_status" = "0" ]; then + echo "The Unicorn web server already running with pid $wpid, not restarting." + else + # Remove old socket if it exists + rm -f "$rails_socket" 2>/dev/null + # Start the web server + RAILS_ENV=$RAILS_ENV bin/web start + fi + + # If sidekiq is already running, don't start it again. + if [ "$sidekiq_status" = "0" ]; then + echo "The Sidekiq job dispatcher is already running with pid $spid, not restarting" + else + RAILS_ENV=$RAILS_ENV bin/background_jobs start & + fi + + if [ "$gitlab_workhorse_status" = "0" ]; then + echo "The gitlab-workhorse is already running with pid $spid, not restarting" + else + # No need to remove a socket, gitlab-workhorse does this itself. + # Because gitlab-workhorse has multiple executables we need to fix + # the PATH. + $app_root/bin/daemon_with_pidfile $gitlab_workhorse_pid_path \ + /usr/bin/env PATH=$gitlab_workhorse_dir:$PATH \ + gitlab-workhorse $gitlab_workhorse_options \ + >> $gitlab_workhorse_log 2>&1 & + fi + + if [ "$mail_room_enabled" = true ]; then + # If MailRoom is already running, don't start it again. + if [ "$mail_room_status" = "0" ]; then + echo "The MailRoom email processor is already running with pid $mpid, not restarting" + else + RAILS_ENV=$RAILS_ENV bin/mail_room start & + fi + fi + + # Wait for the pids to be planted + wait_for_pids + # Finally check the status to tell wether or not GitLab is running + print_status +} + +## Asks Unicorn, Sidekiq and MailRoom if they would be so kind as to stop, if not kills them. +stop_gitlab() { + exit_if_not_running + + if [ "$web_status" = "0" ]; then + echo "Shutting down GitLab Unicorn" + RAILS_ENV=$RAILS_ENV bin/web stop + fi + if [ "$sidekiq_status" = "0" ]; then + echo "Shutting down GitLab Sidekiq" + RAILS_ENV=$RAILS_ENV bin/background_jobs stop + fi + if [ "$gitlab_workhorse_status" = "0" ]; then + echo "Shutting down gitlab-workhorse" + kill -- $(cat $gitlab_workhorse_pid_path) + fi + if [ "$mail_room_enabled" = true ] && [ "$mail_room_status" = "0" ]; then + echo "Shutting down GitLab MailRoom" + RAILS_ENV=$RAILS_ENV bin/mail_room stop + fi + + # If something needs to be stopped, lets wait for it to stop. Never use SIGKILL in a script. + while [ "$web_status" = "0" ] || [ "$sidekiq_status" = "0" ] || [ "$gitlab_workhorse_status" = "0" ] || { [ "$mail_room_enabled" = true ] && [ "$mail_room_status" = "0" ]; }; do + sleep 1 + check_status + printf "." + if [ "$web_status" != "0" ] && [ "$sidekiq_status" != "0" ] && [ "$gitlab_workhorse_status" != "0" ] && { [ "$mail_room_enabled" != true ] || [ "$mail_room_status" != "0" ]; }; then + printf "\n" + break + fi + done + + sleep 1 + # Cleaning up unused pids + rm "$web_server_pid_path" 2>/dev/null + # rm "$sidekiq_pid_path" 2>/dev/null # Sidekiq seems to be cleaning up its own pid. + rm -f "$gitlab_workhorse_pid_path" + if [ "$mail_room_enabled" = true ]; then + rm "$mail_room_pid_path" 2>/dev/null + fi + + print_status +} + +## Prints the status of GitLab and its components. +print_status() { + check_status + if [ "$web_status" != "0" ] && [ "$sidekiq_status" != "0" ] && [ "$gitlab_workhorse_status" != "0" ] && { [ "$mail_room_enabled" != true ] || [ "$mail_room_status" != "0" ]; }; then + echo "GitLab is not running." + return + fi + if [ "$web_status" = "0" ]; then + echo "The GitLab Unicorn web server with pid $wpid is running." + else + printf "The GitLab Unicorn web server is \033[31mnot running\033[0m.\n" + fi + if [ "$sidekiq_status" = "0" ]; then + echo "The GitLab Sidekiq job dispatcher with pid $spid is running." + else + printf "The GitLab Sidekiq job dispatcher is \033[31mnot running\033[0m.\n" + fi + if [ "$gitlab_workhorse_status" = "0" ]; then + echo "The gitlab-workhorse with pid $hpid is running." + else + printf "The gitlab-workhorse is \033[31mnot running\033[0m.\n" + fi + if [ "$mail_room_enabled" = true ]; then + if [ "$mail_room_status" = "0" ]; then + echo "The GitLab MailRoom email processor with pid $mpid is running." + else + printf "The GitLab MailRoom email processor is \033[31mnot running\033[0m.\n" + fi + fi + if [ "$web_status" = "0" ] && [ "$sidekiq_status" = "0" ] && [ "$gitlab_workhorse_status" = "0" ] && { [ "$mail_room_enabled" != true ] || [ "$mail_room_status" = "0" ]; }; then + printf "GitLab and all its components are \033[32mup and running\033[0m.\n" + fi +} + +## Tells unicorn to reload its config and Sidekiq to restart +reload_gitlab(){ + exit_if_not_running + if [ "$wpid" = "0" ];then + echo "The GitLab Unicorn Web server is not running thus its configuration can't be reloaded." + exit 1 + fi + printf "Reloading GitLab Unicorn configuration... " + RAILS_ENV=$RAILS_ENV bin/web reload + echo "Done." + + echo "Restarting GitLab Sidekiq since it isn't capable of reloading its config..." + RAILS_ENV=$RAILS_ENV bin/background_jobs restart + + if [ "$mail_room_enabled" != true ]; then + echo "Restarting GitLab MailRoom since it isn't capable of reloading its config..." + RAILS_ENV=$RAILS_ENV bin/mail_room restart + fi + + wait_for_pids + print_status +} + +## Restarts Sidekiq and Unicorn. +restart_gitlab(){ + check_status + if [ "$web_status" = "0" ] || [ "$sidekiq_status" = "0" ] || [ "$gitlab_workhorse" = "0" ] || { [ "$mail_room_enabled" = true ] && [ "$mail_room_status" = "0" ]; }; then + stop_gitlab + fi + start_gitlab +} + + +### Finally the input handling. + +case "$1" in + start) + start_gitlab + ;; + stop) + stop_gitlab + ;; + restart) + restart_gitlab + ;; + reload|force-reload) + reload_gitlab + ;; + status) + print_status + exit $gitlab_status + ;; + *) + echo "Usage: service gitlab {start|stop|restart|reload|status}" + exit 1 + ;; +esac + +exit From 87d63cd80b6aad3f06fdc2d993ba07e0ff0709cc Mon Sep 17 00:00:00 2001 From: "Charles R. (hugo)" Date: Mon, 2 May 2016 14:34:52 +0200 Subject: [PATCH 05/96] Fully dynamic gitlab database.yml file --- gitlab/files/gitlab-database.yml | 64 ++++++++------------------------ 1 file changed, 16 insertions(+), 48 deletions(-) diff --git a/gitlab/files/gitlab-database.yml b/gitlab/files/gitlab-database.yml index d052257..d502aa7 100644 --- a/gitlab/files/gitlab-database.yml +++ b/gitlab/files/gitlab-database.yml @@ -1,50 +1,18 @@ -# -# PRODUCTION -# -production: - adapter: postgresql - encoding: unicode - database: {{ salt['pillar.get']('gitlab:db_name') }} - pool: 10 - username: {{ salt['pillar.get']('gitlab:db_user') }} - password: {{ salt['pillar.get']('gitlab:db_pass') }} - # host: localhost - # port: 5432 - # socket: /tmp/postgresql.sock +{%- set db_name = salt['pillar.get']('gitlab:db:name') %} +{%- set db_user = salt['pillar.get']('postgres:databases:' ~ db_name ~ ':owner') %} +{%- set db_pass = salt['pillar.get']('postgres:users:' ~ db_user ~ ':password') %} -# -# Development specific -# -development: - adapter: postgresql - encoding: unicode - database: gitlabhq_development - pool: 5 - username: postgres - password: - # socket: /tmp/postgresql.sock - -# -# Staging specific -# -staging: - adapter: postgresql - encoding: unicode - database: gitlabhq_staging - pool: 5 - username: postgres - password: - # socket: /tmp/postgresql.sock - -# Warning: The database defined as "test" will be erased and -# re-generated from your development database when you run "rake". -# Do not set this db to the same as development or production. -test: &test - adapter: postgresql - encoding: unicode - database: gitlabhq_test - pool: 5 - username: postgres - password: - # socket: /tmp/postgresql.sock +{%- for name, infos in salt['pillar.get']('gitlab:databases', {}).items() %} +{%- set db_user = salt['pillar.get']('postgres:databases:' ~ infos['name'] ~ ':owner') %} +{{ name }}: + adapter: {{ infos['engine'] }} + database: {{ infos['name'] }} + username: {{ db_user }} + password: {{ salt['pillar.get']('postgres:users:' ~ db_user ~ ':password') }} + encoding: {% if 'encoding' in infos %}{{ infos['encoding'] }}{% else %}unicode{% endif %} + pool: {% if 'pool' in infos %}{{ infos['pool'] }}{% else %}5{% endif %} +{%- if 'host' in infos %} + host: {{ infos['host'] }} +{% endif %} +{%- endfor %} From b513b94b679a996f979d8ec1d2b274ef391234ff Mon Sep 17 00:00:00 2001 From: "Charles R. (hugo)" Date: Mon, 2 May 2016 14:35:59 +0200 Subject: [PATCH 06/96] Remove unused gitlab/postgresql file --- gitlab/postgresql.sls | 19 ------------------- 1 file changed, 19 deletions(-) delete mode 100644 gitlab/postgresql.sls diff --git a/gitlab/postgresql.sls b/gitlab/postgresql.sls deleted file mode 100644 index 177cfc5..0000000 --- a/gitlab/postgresql.sls +++ /dev/null @@ -1,19 +0,0 @@ -include: - - postgresql - -gitlab-db: - postgres_user.present: - - name: {{ salt['pillar.get']('gitlab:db_user') }} - - password: {{ salt['pillar.get']('gitlab:db_pass') }} - - require: - - pkg: postgresql-server - - service: postgresql-server - postgres_database.present: - - name: {{ salt['pillar.get']('gitlab:db_name') }} - - owner: {{ salt['pillar.get']('gitlab:db_user') }} - - template: template1 - - require: - - file: gitlab-service - - pkg: postgresql-server - - service: postgresql-server - - postgres_user: gitlab-db From ff8c44407c517491bf3ed5a68c004d5861686163 Mon Sep 17 00:00:00 2001 From: "Charles R. (hugo)" Date: Mon, 2 May 2016 14:36:15 +0200 Subject: [PATCH 07/96] Fully dynamic path / Misc --- gitlab/files/gitlab-default | 10 ++- gitlab/files/gitlab-gitlab.yml | 14 ++-- gitlab/files/gitlab-logrotate | 5 +- gitlab/files/gitlab-nginx | 24 ++++-- gitlab/files/gitlab-nginx-ssl | 11 ++- gitlab/files/gitlab-shell-config.yml | 6 +- gitlab/files/gitlab-unicorn.rb | 11 ++- gitlab/files/initd | 12 +-- gitlab/gitlab-shell.sls | 15 +++- gitlab/gitlab.sls | 114 +++++++++++++++++++-------- gitlab/init.sls | 3 +- gitlab/nginx.sls | 4 +- gitlab/packages.sls | 7 +- gitlab/test.sls | 25 ++++++ 14 files changed, 185 insertions(+), 76 deletions(-) create mode 100644 gitlab/test.sls diff --git a/gitlab/files/gitlab-default b/gitlab/files/gitlab-default index 00a44e5..eb5dc25 100644 --- a/gitlab/files/gitlab-default +++ b/gitlab/files/gitlab-default @@ -1,6 +1,9 @@ # Copy this lib/support/init.d/gitlab.default.example file to # /etc/default/gitlab in order for it to apply to your system. +{% set root_dir = salt['pillar.get']('gitlab:lookup:root_dir', '/home/git') %} +{% set pids_dir = salt['pillar.get']('gitlab:lookup:pids_dir', root_dir ~ '/var/pids') %} +{% set sockets_dir = salt['pillar.get']('gitlab:lookup:sockets_dir', root_dir ~ '/var/sockets') %} # RAILS_ENV defines the type of installation that is running. # Normal values are "production", "test" and "development". @@ -12,22 +15,21 @@ app_user="git" # app_root defines the folder in which gitlab and it's components are installed. # The default is "/home/$app_user/gitlab" -app_root="/home/$app_user/gitlab" +app_root="{{ root_dir }}/gitlab" {% if salt['pillar.get']('gitlab:use_rvm', false) %} # Load RVM variables . /home/$app_user/.rvm/scripts/rvm - {% endif %} # pid_path defines a folder in which the gitlab and it's components place their pids. # This variable is also used below to define the relevant pids for the gitlab components. # The default is "$app_root/tmp/pids" -pid_path="$app_root/tmp/pids" +pid_path="{{ pids_dir }}" # socket_path defines the folder in which gitlab places the sockets #The default is "$app_root/tmp/sockets" -socket_path="$app_root/tmp/sockets" +socket_path="{{ sockets_dir }}" # web_server_pid_path defines the path in which to create the pid file fo the web_server # The default is "$pid_path/unicorn.pid" diff --git a/gitlab/files/gitlab-gitlab.yml b/gitlab/files/gitlab-gitlab.yml index 0a0887b..efd49f0 100644 --- a/gitlab/files/gitlab-gitlab.yml +++ b/gitlab/files/gitlab-gitlab.yml @@ -8,6 +8,8 @@ # 3. Replace gitlab -> email_from {% set root_dir = salt['pillar.get']('gitlab:lookup:root_dir', '/home/git') %} +{% set lib_dir = salt['pillar.get']('gitlab:lookup:lib_dir', root_dir ~ '/libraries') %} +{% set repositories = salt['pillar.get']('gitlab:lookup:repositories', root_dir ~ '/repositories') %} production: &base # @@ -180,9 +182,9 @@ production: &base # ========================== # GitLab Satellites - satellites: - # Relative paths are relative to Rails.root (default: tmp/repo_satellites/) - path: {{ root_dir }}/gitlab-satellites/ + #satellites: + # # Relative paths are relative to Rails.root (default: tmp/repo_satellites/) + # path: {{ lib_dir }}/gitlab-satellites/ ## Backup settings backup: @@ -191,11 +193,11 @@ production: &base ## GitLab Shell settings gitlab_shell: - path: {{ root_dir }}/gitlab-shell/ + path: {{ lib_dir }}/gitlab-shell.git/ # REPOS_PATH MUST NOT BE A SYMLINK!!! - repos_path: {{ root_dir }}/repositories/ - hooks_path: {{ root_dir }}/gitlab-shell/hooks/ + repos_path: {{ repositories }}/ + hooks_path: {{ repositories }}/hooks/ # Git over HTTP upload_pack: true diff --git a/gitlab/files/gitlab-logrotate b/gitlab/files/gitlab-logrotate index 351f5b3..cfe3589 100644 --- a/gitlab/files/gitlab-logrotate +++ b/gitlab/files/gitlab-logrotate @@ -2,8 +2,9 @@ # based on: http://stackoverflow.com/a/4883967 {% set root_dir = salt['pillar.get']('gitlab:lookup:root_dir', '/home/git') %} +{% set logs_dir = salt['pillar.get']('gitlab:lookup:logs_dir', root_dir ~ '/var/logs') %} -{{ root_dir }}/gitlab/log/*.log { +{{ logs_dir }}/*.log { weekly missingok rotate 52 @@ -13,7 +14,7 @@ copytruncate } -{{ root_dir }}/gitlab-shell/gitlab-shell.log { +{{ logs_dir }}/gitlab-shell.log { weekly missingok rotate 52 diff --git a/gitlab/files/gitlab-nginx b/gitlab/files/gitlab-nginx index 82f71eb..e55f9e8 100644 --- a/gitlab/files/gitlab-nginx +++ b/gitlab/files/gitlab-nginx @@ -1,11 +1,10 @@ -# GITLAB -# Maintainer: @randx -# App Version: 5.0 {% set root_dir = salt['pillar.get']('gitlab:lookup:root_dir', '/home/git') %} +{% set sockets_dir = salt['pillar.get']('gitlab:lookup:sockets_dir', root_dir ~ '/var/sockets') %} +{% set logs_dir = salt['pillar.get']('gitlab:lookup:logs_dir', root_dir ~ '/var/logs') %} -upstream gitlab { - server unix:{{ root_dir }}/gitlab/tmp/sockets/gitlab.socket; +upstream gitlab-workhorse { + server unix:{{ sockets_dir }}/gitlab-workhorse.socket fail_timeout=0; } server { @@ -18,8 +17,8 @@ server { client_max_body_size 5m; # individual nginx logs for this gitlab vhost - access_log /var/log/nginx/gitlab_access.log; - error_log /var/log/nginx/gitlab_error.log; + access_log {{ logs_dir }}/nginx_access.log; + error_log {{ logs_dir }}/nginx_error.log; location / { # serve static files from defined root folder;. @@ -39,7 +38,16 @@ server { proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_pass http://gitlab; + proxy_pass http://gitlab-workhorse; + } + + error_page 404 /404.html; + error_page 422 /422.html; + error_page 500 /500.html; + error_page 502 /502.html; + location ~ ^/(404|422|500|502)\.html$ { + root {{ root_dir }}/gitlab/public; + internal; } } diff --git a/gitlab/files/gitlab-nginx-ssl b/gitlab/files/gitlab-nginx-ssl index 0f4f770..e806449 100644 --- a/gitlab/files/gitlab-nginx-ssl +++ b/gitlab/files/gitlab-nginx-ssl @@ -19,11 +19,13 @@ # 3) Update ssl for gravatar "ssl_url: https://secure.gravatar.com/avatar/%{hash}?s=%{size}&d=mm" {% set root_dir = salt['pillar.get']('gitlab:lookup:root_dir', '/home/git') %} +{% set sockets_dir = salt['pillar.get']('gitlab:lookup:sockets_dir', root_dir ~ '/var/sockets') %} +{% set logs_dir = salt['pillar.get']('gitlab:lookup:logs_dir', root_dir ~ '/var/logs') %} -upstream gitlab { +upstream gitlab-workhorse { ## Uncomment if you have set up puma/unicorn to listen on a unix socket (recommended). - server unix:{{ root_dir }}/gitlab/tmp/sockets/gitlab.socket; + server unix:{{ sockets_dir }}/gitlab.socket; ## Uncomment if puma/unicorn are configured to listen on a tcp port. ## Check the port number in {{ root_dir }}/gitlab/config/{puma.rb/unicorn.rb} @@ -39,6 +41,7 @@ server { root /nowhere; # this doesn't have to be a valid path since we are redirecting, you don't have to change it. rewrite ^ https://$server_name$request_uri permanent; } + server { listen 443 ssl; server_name {{ grains['fqdn'] }}; @@ -53,8 +56,8 @@ server { ssl_prefer_server_ciphers on; # individual nginx logs for this gitlab vhost - access_log /var/log/nginx/gitlab_access.log; - error_log /var/log/nginx/gitlab_error.log; + access_log {{ logs_dir }}/nginx_access.log; + error_log {{ logs_dir }}/nginx_error.log; location / { # serve static files from defined root folder;. diff --git a/gitlab/files/gitlab-shell-config.yml b/gitlab/files/gitlab-shell-config.yml index 2aac25b..e525071 100644 --- a/gitlab/files/gitlab-shell-config.yml +++ b/gitlab/files/gitlab-shell-config.yml @@ -2,6 +2,8 @@ user: git {% set root_dir = salt['pillar.get']('gitlab:lookup:root_dir', '/home/git') %} +{% set repo_dir = salt['pillar.get']('gitlab:lookup:repo_dir', root_dir ~ '/repositories') %} +{% set logs_dir = salt['pillar.get']('gitlab:lookup:logs_dir', root_dir ~ '/logs') %} # Url to gitlab instance. Used for api calls. Should end with a slash. {%- if salt['pillar.get']('gitlab:https', false) %} @@ -25,7 +27,7 @@ http_settings: # Give the canonicalized absolute pathname, # REPOS_PATH MUST NOT CONTAIN ANY SYMLINK!!! # Check twice that none of the components is a symlink, including "/home". -repos_path: "{{ root_dir }}/repositories" +repos_path: "{{ repo_dir }}" # File used as authorized_keys for gitlab user auth_file: "{{ root_dir }}/.ssh/authorized_keys" @@ -40,7 +42,7 @@ redis: # Log file. # Default is gitlab-shell.log in the root directory. -# log_file: "{{ root_dir }}/gitlab-shell/gitlab-shell.log" +log_file: "{{ logs_dir }}/gitlab-shell.log" # Log level. INFO by default log_level: {{ salt['pillar.get']('gitlab:shell:log_level', 'INFO') }} diff --git a/gitlab/files/gitlab-unicorn.rb b/gitlab/files/gitlab-unicorn.rb index 00dd7ae..d1eaaf5 100644 --- a/gitlab/files/gitlab-unicorn.rb +++ b/gitlab/files/gitlab-unicorn.rb @@ -9,6 +9,9 @@ # documentation. {% set root_dir = salt['pillar.get']('gitlab:lookup:root_dir', '/home/git') %} +{% set sockets_dir = salt['pillar.get']('gitlab:lookup:sockets_dir', root_dir ~ '/var/sockets') %} +{% set pids_dir = salt['pillar.get']('gitlab:lookup:pids_dir', root_dir ~ '/var/pids') %} +{% set logs_dir = salt['pillar.get']('gitlab:lookup:logs_dir', root_dir ~ '/var/logs') %} # Uncomment and customize the last line to run in a non-root path # WARNING: We recommend creating a FQDN to host GitLab in a root path instead of this. @@ -38,20 +41,20 @@ # listen on both a Unix domain socket and a TCP port, # we use a shorter backlog for quicker failover when busy -listen "{{ root_dir }}/gitlab/tmp/sockets/gitlab.socket", :backlog => 64 +listen "{{ sockets_dir }}/gitlab.socket", :backlog => 64 listen "127.0.0.1:8080", :tcp_nopush => true # nuke workers after 30 seconds instead of 60 seconds (the default) timeout {{ salt['pillar.get']('gitlab:unicorn:timeout', 30) }} # feel free to point this anywhere accessible on the filesystem -pid "{{ root_dir }}/gitlab/tmp/pids/unicorn.pid" +pid "{{ pids_dir }}/unicorn.pid" # By default, the Unicorn logger will write to stderr. # Additionally, some applications/frameworks log to stderr or stdout, # so prevent them from going to /dev/null when daemonized here: -stderr_path "{{ root_dir }}/gitlab/log/unicorn.stderr.log" -stdout_path "{{ root_dir }}/gitlab/log/unicorn.stdout.log" +stderr_path "{{ logs_dir }}/unicorn.stderr.log" +stdout_path "{{ logs_dir }}/unicorn.stdout.log" # combine Ruby 2.0.0dev or REE with "preload_app true" for memory savings # http://rubyenterpriseedition.com/faq.html#adapt_apps_for_cow diff --git a/gitlab/files/initd b/gitlab/files/initd index b978adb..fd5a591 100755 --- a/gitlab/files/initd +++ b/gitlab/files/initd @@ -23,8 +23,10 @@ # An example defaults file can be found in lib/support/init.d/gitlab.default.example ### -{%- set root_dir = salt['pillar.get']('gitlab:lookup:root_dir', '/home/git') %} -{%- set var_dir = salt['pillar.get']('gitlab:lookup:var_dir', '/home/$app_user') -%} +{% set root_dir = salt['pillar.get']('gitlab:lookup:root_dir', '/home/git') %} +{% set pids_dir = salt['pillar.get']('gitlab:lookup:pids_dir', root_dir ~ '/var/pids') %} +{% set sockets_dir = salt['pillar.get']('gitlab:lookup:sockets_dir', root_dir ~ '/var/sockets') %} +{% set logs_dir = salt['pillar.get']('gitlab:lookup:logs_dir', root_dir ~ '/var/logs') %} ### Environment variables RAILS_ENV="production" @@ -33,8 +35,8 @@ RAILS_ENV="production" # internal /bin/sh variables such as PATH, EDITOR or SHELL. app_user="git" app_root="{{ root_dir }}/gitlab" -pid_path="{{ var_dir }}/pids" -socket_path="{{ var_dir }}/sockets" +pid_path="{{ pids_dir }}" +socket_path="{{ sockets_dir }}" rails_socket="$socket_path/gitlab.socket" web_server_pid_path="$pid_path/unicorn.pid" sidekiq_pid_path="$pid_path/sidekiq.pid" @@ -43,7 +45,7 @@ mail_room_pid_path="$pid_path/mail_room.pid" gitlab_workhorse_dir=$(cd {{ root_dir }}/gitlab-workhorse/bin 2> /dev/null && pwd) gitlab_workhorse_pid_path="$pid_path/gitlab-workhorse.pid" gitlab_workhorse_options="-listenUmask 0 -listenNetwork unix -listenAddr $socket_path/gitlab-workhorse.socket -authBackend http://127.0.0.1:8080 -authSocket $rails_socket -documentRoot $app_root/public" -gitlab_workhorse_log="{{ var_dir }}/logs/gitlab-workhorse.log" +gitlab_workhorse_log="{{ logs_dir }}/gitlab-workhorse.log" shell_path="/bin/bash" # Read configuration variable file if it is present diff --git a/gitlab/gitlab-shell.sls b/gitlab/gitlab-shell.sls index 70ed162..95e1eac 100644 --- a/gitlab/gitlab-shell.sls +++ b/gitlab/gitlab-shell.sls @@ -3,12 +3,13 @@ include: - gitlab.ruby {% set root_dir = salt['pillar.get']('gitlab:lookup:root_dir', '/home/git') %} +{% set lib_dir = salt['pillar.get']('gitlab:lookup:lib_dir', root_dir ~ '/libraries') %} gitlab-shell-git: git.latest: - name: https://gitlab.com/gitlab-org/gitlab-shell.git - rev: {{ salt['pillar.get']('gitlab:shell_version') }} - - target: {{ root_dir }}/gitlab-shell + - target: {{ lib_dir }}/gitlab-shell.git - user: git - require: - pkg: gitlab-deps @@ -19,7 +20,7 @@ gitlab-shell-git: # https://gitlab.com/gitlab-org/gitlab-shell/blob/master/config.yml.example gitlab-shell-config: file.managed: - - name: {{ root_dir }}/gitlab-shell/config.yml + - name: {{ lib_dir }}/gitlab-shell.git/config.yml - source: salt://gitlab/files/gitlab-shell-config.yml - template: jinja - user: git @@ -31,10 +32,18 @@ gitlab-shell-config: gitlab-shell: cmd.wait: - user: git - - cwd: {{ root_dir }}/gitlab-shell + - cwd: {{ lib_dir }}/gitlab-shell.git - name: ./bin/install - shell: /bin/bash - watch: - git: gitlab-shell-git - require: - file: gitlab-shell-config + +gitlab-shell-chmod-bin: + file.directory: + - name: {{ lib_dir }}/gitlab-shell.git/bin + - file_mode: 0770 + - recurse: + - mode + diff --git a/gitlab/gitlab.sls b/gitlab/gitlab.sls index 62e9f9c..6695ac9 100644 --- a/gitlab/gitlab.sls +++ b/gitlab/gitlab.sls @@ -1,7 +1,16 @@ include: + - postgres - gitlab.ruby {% set root_dir = salt['pillar.get']('gitlab:lookup:root_dir', '/home/git') %} +{% set repositories = salt['pillar.get']('gitlab:lookup:repositories', root_dir ~ '/repositories') %} +{% set sockets_dir = salt['pillar.get']('gitlab:lookup:sockets_dir', root_dir ~ '/var/sockets') %} +{% set pids_dir = salt['pillar.get']('gitlab:lookup:pids_dir', root_dir ~ '/var/pids') %} +{% set logs_dir = salt['pillar.get']('gitlab:lookup:logs_dir', root_dir ~ '/var/logs') %} +{% set uploads_dir = salt['pillar.get']('gitlab:lookup:uploads_dir', root_dir ~ '/var/uploads') %} + +{% set active_db = salt['pillar.get']('gitlab:databases:production', 'paf') %} +{% set db_user, db_user_infos = salt['pillar.get']('postgres:users').items()[0] %} gitlab-git: git.latest: @@ -78,37 +87,38 @@ git-config: - require: - user: git-user -{% for dir in ['gitlab-satellites', 'gitlab/tmp/pids', 'gitlab/tmp/sockets', 'gitlab/public/uploads'] %} -{{ root_dir }}/{{ dir }}: +git-var-mkdir: file.directory: + - name: {{ root_dir }}/var - user: git - group: git - mode: 750 - - require: - - user: git-user - - git: gitlab-git -{% endfor %} -gitlab-initialize: - cmd.wait: +# pids_dir +{% for dir in [ sockets_dir, logs_dir ] %} +git-{{ dir }}-mkdir: + file.directory: + - name: {{ dir }} - user: git - - cwd: {{ root_dir }}/gitlab - - name: echo yes | bundle exec rake gitlab:setup RAILS_ENV=production - - shell: /bin/bash - - unless: psql -U {{ salt['pillar.get']('gitlab:db_user') }} {{ salt['pillar.get']('gitlab:db_name') }} -c 'select * from users;' - - watch: - - git: gitlab-git - - require: - - cmd: gitlab-gems - - postgres_database: gitlab-db + - group: git + - mode: 750 +{% endfor %} + +# Hardcoded in gitlab, so, we have to create symlink +gitlab-pids_dir-symlink: + file.symlink: + - name: {{ pids_dir }} + - target: {{ root_dir }}/gitlab/tmp/pids + require: + - file: gitlab-config # When code changes, trigger upgrade procedure # Based on https://gitlab.com/gitlab-org/gitlab-ce/blob/master/lib/gitlab/upgrader.rb gitlab-gems: - cmd.wait: + cmd.run: - user: git - cwd: {{ root_dir }}/gitlab - - name: bundle install --deployment --without development test mysql aws + - name: bundle install --deployment --without development test mysql aws kerberos - shell: /bin/bash - watch: - git: gitlab-git @@ -119,6 +129,19 @@ gitlab-gems: - file: rack_attack-config - sls: gitlab.ruby +gitlab-initialize: + cmd.run: + - user: git + - cwd: {{ root_dir }}/gitlab + - name: force=yes bundle exec rake gitlab:setup RAILS_ENV=production + - shell: /bin/bash + - unless: PGPASSWORD={{ db_user_infos.password }} psql -h {{ active_db.host }} -U {{ db_user }} {{ active_db.name }} -c 'select * from users;' + - watch: + - git: gitlab-git + - require: + - cmd: gitlab-gems + - file: gitlab-db-config + gitlab-migrate-db: cmd.wait: - user: git @@ -130,7 +153,7 @@ gitlab-migrate-db: - require: - cmd: gitlab-gems - cmd: gitlab-initialize - - postgres_database: gitlab-db + - file: gitlab-db-config gitlab-recompile-assets: cmd.wait: @@ -175,33 +198,58 @@ gitlab-default: - group: root - mode: 644 -gitlab-service: +# https://gitlab.com/gitlab-org/gitlab-ce/blob/master/lib/support/logrotate/gitlab +gitlab-logwatch: + file.managed: + - name: /etc/logrotate.d/gitlab + - source: salt://gitlab/files/gitlab-logrotate + - template: jinja + - user: root + - group: root + - mode: 644 + +gitlab-respositories-dir: + file.directory: + - name: {{ repositories }} + - user: git + - group: git + - file_mode: 0660 + - dir_mode: 2770 + +gitlab-uploads-dir: + file.directory: + - name: {{ root_dir }}/gitlab/public/uploads + - dir_mode: 0700 + +gitlab-uploads-symlink: file.symlink: + - name: {{ uploads_dir }} + - target: {{ root_dir }}/gitlab/public/uploads + - require: + - file: git-var-mkdir + +gitlab-service: + file.managed: - name: /etc/init.d/gitlab - - target: {{ root_dir }}/gitlab/lib/support/init.d/gitlab + - source: salt://gitlab/files/initd + - mode: 0755 + - template: jinja - require: - git: gitlab-git service: - name: gitlab - running - enable: True + - reload: True - require: - - cmd: gitlab-initialize + - file: gitlab-service +# - cmd: gitlab-initialize + - file: gitlab-pids_dir-symlink - watch: - git: gitlab-git - cmd: gitlab-clear-cache - file: gitlab-config - file: gitlab-db-config - file: gitlab-default - - file: gitlab-service - file: rack_attack-config - file: unicorn-config - -# https://gitlab.com/gitlab-org/gitlab-ce/blob/master/lib/support/logrotate/gitlab -gitlab-logwatch: - file.managed: - - name: /etc/logrotate.d/gitlab - - source: salt://gitlab/files/gitlab-logrotate - - user: root - - group: root - - mode: 644 diff --git a/gitlab/init.sls b/gitlab/init.sls index 12d7de6..692ffaf 100644 --- a/gitlab/init.sls +++ b/gitlab/init.sls @@ -1,13 +1,12 @@ include: - - postgresql {% if grains['os_family'] == 'RedHat' %} - gitlab.repos {% endif %} - gitlab.packages - redis - - gitlab.postgresql - gitlab.user - gitlab.ruby - gitlab.gitlab-shell + - gitlab.gitlab-workhorse - gitlab.gitlab - gitlab.nginx diff --git a/gitlab/nginx.sls b/gitlab/nginx.sls index 228fb11..2c2c198 100644 --- a/gitlab/nginx.sls +++ b/gitlab/nginx.sls @@ -6,8 +6,8 @@ {% set nginx_path = '/etc/nginx/conf.d' %} {% endif %} -{% set nginx_conf = salt['pillar.get']('backupmanager:lookup:nginx_conf', 'salt://gitlab/files/gitlab-nginx') %} -{% set nginx_ssl_conf = salt['pillar.get']('backupmanager:lookup:nginx_ssl_conf', 'salt://gitlab/files/gitlab-nginx-ssl') %} +{% set nginx_conf = salt['pillar.get']('gitlab:lookup:nginx_conf', 'salt://gitlab/files/gitlab-nginx') %} +{% set nginx_ssl_conf = salt['pillar.get']('gitlab:lookup:nginx_ssl_conf', 'salt://gitlab/files/gitlab-nginx-ssl') %} nginx: pkg.installed: [] diff --git a/gitlab/packages.sls b/gitlab/packages.sls index 053d9c8..68273bf 100644 --- a/gitlab/packages.sls +++ b/gitlab/packages.sls @@ -74,6 +74,8 @@ gitlab-deps: - build-essential - checkinstall - curl + - cmake + - golang - libcurl4-openssl-dev - libffi-dev - libgdbm-dev @@ -86,11 +88,14 @@ gitlab-deps: - libyaml-dev - logrotate - openssh-server + - nodejs + - pkg-config - python - python-docutils + - rake - redis-server - zlib1g-dev - {% if salt['pillar.get']('gitlab:db_engine', 'postgresql') == 'postgresql' %} + {% if salt['pillar.get']('gitlab:db:engine', 'postgresql') == 'postgresql' %} - libpq-dev {% endif %} {% endif %} diff --git a/gitlab/test.sls b/gitlab/test.sls new file mode 100644 index 0000000..a2547bb --- /dev/null +++ b/gitlab/test.sls @@ -0,0 +1,25 @@ +{% set root_dir = salt['pillar.get']('gitlab:lookup:root_dir', '/home/git') %} + +{% set active_db = salt['pillar.get']('gitlab:databases:production', 'paf') %} +{% set user, user_infos = salt['pillar.get']('postgres:users').items()[0] %} + +/tmp/test: + file.managed: + - source: salt://gitlab/files/test + - template: jinja + - user: root + - group: root + - mode: 644 + +gitlab-initialize: + cmd.run: + - user: git + - cwd: {{ root_dir }}/gitlab + - name: force=yes bundle exec rake gitlab:setup RAILS_ENV=production + - shell: /bin/bash + - unless: PGPASSWORD={{ user_infos.password }} psql -h {{ active_db.host }} -U {{ user }} {{ active_db.name }} -c 'select * from users;' +# - watch: +# - git: gitlab-git +# - require: +# - cmd: gitlab-gems +# - file: gitlab-db-config From 54156835a172d5968bdd4a3c451b9327b1d9db42 Mon Sep 17 00:00:00 2001 From: "Charles R. (hugo)" Date: Mon, 2 May 2016 14:44:59 +0200 Subject: [PATCH 08/96] Update Readme and pillar.example --- README.rst | 18 +++++++--------- pillar.example | 57 +++++++++++++++++++++++++++++++------------------- 2 files changed, 42 insertions(+), 33 deletions(-) diff --git a/README.rst b/README.rst index 3d45436..d0ad715 100644 --- a/README.rst +++ b/README.rst @@ -1,6 +1,13 @@ gitlab-formula ============== +Modification from original formula : + +* No hardcoded path : possibilty to install gitlab where you want +* Use of Postgresql formula +* Original initd script from Gitlab setup +* Lot of little things... + SaltStack formula to install GitLab Salt state for installing GitLab - https://gitlab.com/gitlab-org/gitlab-ce @@ -8,17 +15,6 @@ Salt state for installing GitLab - https://gitlab.com/gitlab-org/gitlab-ce Following original install docs ( https://gitlab.com/gitlab-org/gitlab-ce/blob/6-5-stable/doc/install/installation.md ) as close as possible, with some exceptions: * ruby 1.9.3 is enough for it to work, so I'm using system packages for that -* Using PostgreSQL "because". - -Currently tested only on Debian, further systems planned. - -Initial work done for CentOS but doesn't work there yet, heavily inspired by https://github.com/gitlabhq/gitlab-recipes/tree/master/install/centos - -There are some initial bits for supporting RVM, but it's not working yet. - -I chose to use PostgreSQL "because", planning to make this tunable via pillar. - -I assume you're running gitlab under your node's FQDN, not under another name. Attempt made to have most settings tunable via pillars. diff --git a/pillar.example b/pillar.example index 874cad6..606f8e3 100644 --- a/pillar.example +++ b/pillar.example @@ -1,12 +1,33 @@ gitlab: + lookup: + root_dir: /opt/git + lib_dir: /opt/git/libraries + repo_dir: /opt/git/repositories + pids_dir: /opt/git/var/pids + sockets_dir: /opt/git/var/sockets + logs_dir: /opt/git/var/logs + uploads_dir: /opt/git/var/uploads use_rvm: False rvm_ruby: 2.1.0 - shell_version: v1.8.0 - gitlab_version: 6-5-stable - db_engine: postgresql - db_name: 'gitlabhq_production' - db_user: 'git' - db_pass: 'MyVerySecretGitLabPassword' + shell_version: v2.7.2 + gitlab_version: 8-7-stable + workhorse_version: v0.7.1 + databases: + production: &production + engine: postgresql + name: gitlab + pool: 10 + host: localhost + development: + engine: postgresql + name: gitlab + staging: + engine: postgresql + name: gitlab + test: + engine: postgresql + name: gitlab + active_database: *production gravatar: enabled: false ldap: @@ -43,21 +64,13 @@ gitlab: unicorn: worker_processes: 2 timeout: 30 - https: True - ssl_key: | - -----BEGIN PRIVATE KEY----- - ABC= - -----END PRIVATE KEY----- - ssl_cert: | - -----BEGIN CERTIFICATE----- - ABC= - -----END CERTIFICATE----- + https: false #runner - runner: - downloadpath: "https://s3-eu-west-1.amazonaws.com/downloads-packages/ubuntu-14.04/gitlab-runner_5.2.0~omnibus.1-1_amd64.deb" - #(default debian wget https://s3-eu-west-1.amazonaws.com/downloads-packages/ubuntu-14.04/gitlab-runner_5.2.0~omnibus.1-1_amd64.deb ) - username: "gitlab-runner" - # default: gitlab-runner - url: "url gitlab-ci" - token: "token gitlab-ci" +# runner: +# downloadpath: "https://s3-eu-west-1.amazonaws.com/downloads-packages/ubuntu-14.04/gitlab-runner_5.2.0~omnibus.1-1_amd64.deb" +# #(default debian wget https://s3-eu-west-1.amazonaws.com/downloads-packages/ubuntu-14.04/gitlab-runner_5.2.0~omnibus.1-1_amd64.deb ) +# username: "gitlab-runner" +# # default: gitlab-runner +# url: "url gitlab-ci" +# token: "token gitlab-ci" From 0773c89d507aceffbcb097200c9c80e4c4c8987b Mon Sep 17 00:00:00 2001 From: "Charles R. (hugo)" Date: Mon, 2 May 2016 17:38:59 +0200 Subject: [PATCH 09/96] Added proxy --- README.rst | 1 + gitlab/git.sls | 9 +++++++++ gitlab/gitlab-shell.sls | 18 ++++++++++++++++++ gitlab/gitlab-workhorse.sls | 16 ++++++++++++++++ gitlab/gitlab.sls | 16 ++++++++++++++++ gitlab/ruby.sls | 3 +++ pillar.example | 3 +++ 7 files changed, 66 insertions(+) create mode 100644 gitlab/git.sls diff --git a/README.rst b/README.rst index d0ad715..71f4e3b 100644 --- a/README.rst +++ b/README.rst @@ -6,6 +6,7 @@ Modification from original formula : * No hardcoded path : possibilty to install gitlab where you want * Use of Postgresql formula * Original initd script from Gitlab setup +* Proxy for git.latest / gem * Lot of little things... SaltStack formula to install GitLab diff --git a/gitlab/git.sls b/gitlab/git.sls new file mode 100644 index 0000000..661ff40 --- /dev/null +++ b/gitlab/git.sls @@ -0,0 +1,9 @@ + +{% if salt['pillar.get']('gitlab:proxy:enabled', false) %} +gitproxy: + git.config: + - name: http.proxy + - value: {{ salt['pillar.get']('gitlab:proxy:address') }} + - is_global: True +{% endif %} + diff --git a/gitlab/gitlab-shell.sls b/gitlab/gitlab-shell.sls index 95e1eac..26f67b9 100644 --- a/gitlab/gitlab-shell.sls +++ b/gitlab/gitlab-shell.sls @@ -5,6 +5,21 @@ include: {% set root_dir = salt['pillar.get']('gitlab:lookup:root_dir', '/home/git') %} {% set lib_dir = salt['pillar.get']('gitlab:lookup:lib_dir', root_dir ~ '/libraries') %} +{% if salt['pillar.get']('gitlab:proxy:enabled', false) %} +gitlab-shell-git-present: + git.present: + - name: {{ lib_dir }}/gitlab-shell.git + - bare: False + +gitlab-shell-git-proxy: + git.config: + - name: http.proxy + - value: {{ salt['pillar.get']('gitlab:proxy:address') }} + - repo: {{ lib_dir }}/gitlab-shell.git +# - require: +# - git: gitlab-shell-git-present +{% endif %} + gitlab-shell-git: git.latest: - name: https://gitlab.com/gitlab-org/gitlab-shell.git @@ -16,6 +31,9 @@ gitlab-shell-git: - pkg: git - sls: gitlab.ruby - file: git-home + {% if salt['pillar.get']('gitlab:proxy:enabled', false) %} + - git: gitlab-shell-git-proxy + {% endif %} # https://gitlab.com/gitlab-org/gitlab-shell/blob/master/config.yml.example gitlab-shell-config: diff --git a/gitlab/gitlab-workhorse.sls b/gitlab/gitlab-workhorse.sls index cebb067..17d7944 100644 --- a/gitlab/gitlab-workhorse.sls +++ b/gitlab/gitlab-workhorse.sls @@ -2,6 +2,19 @@ {% set root_dir = salt['pillar.get']('gitlab:lookup:root_dir', '/home/git') %} {% set lib_dir = salt['pillar.get']('gitlab:lookup:lib_dir', root_dir ~ '/libraries') %} +{% if salt['pillar.get']('gitlab:proxy:enabled', false) %} +gitlab-workhorse-git-present: + git.present: + - name: {{ lib_dir }}/gitlab-workhorse.git + - bare: False + +gitlab-workhorse-git-proxy: + git.config: + - name: http.proxy + - value: {{ salt['pillar.get']('gitlab:proxy:address') }} + - repo: {{ lib_dir }}/gitlab-workhorse.git +{% endif %} + gitlab-workhorse-git: git.latest: - name: https://gitlab.com/gitlab-org/gitlab-workhorse.git @@ -13,6 +26,9 @@ gitlab-workhorse-git: - pkg: git - sls: gitlab.ruby - file: git-home + {% if salt['pillar.get']('gitlab:proxy:enabled', false) %} + - git: gitlab-workhorse-git-proxy + {% endif %} {{ root_dir }}/gitlab-workhorse: file.directory: diff --git a/gitlab/gitlab.sls b/gitlab/gitlab.sls index 6695ac9..bf8c807 100644 --- a/gitlab/gitlab.sls +++ b/gitlab/gitlab.sls @@ -12,6 +12,19 @@ include: {% set active_db = salt['pillar.get']('gitlab:databases:production', 'paf') %} {% set db_user, db_user_infos = salt['pillar.get']('postgres:users').items()[0] %} +{% if salt['pillar.get']('gitlab:proxy:enabled', false) %} +gitlab-git-present: + git.present: + - name: {{ root_dir }}/gitlab + - bare: False + +gitlab-git-proxy: + git.config: + - name: http.proxy + - value: {{ salt['pillar.get']('gitlab:proxy:address') }} + - repo: {{ root_dir }}/gitlab +{% endif %} + gitlab-git: git.latest: - name: https://gitlab.com/gitlab-org/gitlab-ce.git @@ -24,6 +37,9 @@ gitlab-git: - sls: gitlab.ruby - cmd: gitlab-shell - user: git-user + {% if salt['pillar.get']('gitlab:proxy:enabled', false) %} + - git: gitlab-git-proxy + {% endif %} # https://gitlab.com/gitlab-org/gitlab-ce/blob/master/config/gitlab.yml.example gitlab-config: diff --git a/gitlab/ruby.sls b/gitlab/ruby.sls index 847cc30..7c7459f 100644 --- a/gitlab/ruby.sls +++ b/gitlab/ruby.sls @@ -22,6 +22,9 @@ gitlab-ruby: - name: bundler - require: - pkg: gitlab-ruby + {% if salt['pillar.get']('gitlab:proxy:enabled', false) %} + - proxy: {{ salt['pillar.get']('gitlab:proxy:address') }} + {% endif %} {% elif grains['os_family'] == 'RedHat' %} pkg.installed: - pkgs: diff --git a/pillar.example b/pillar.example index 606f8e3..5df8b3b 100644 --- a/pillar.example +++ b/pillar.example @@ -7,6 +7,9 @@ gitlab: sockets_dir: /opt/git/var/sockets logs_dir: /opt/git/var/logs uploads_dir: /opt/git/var/uploads + proxy: + enabled: false + address: http://ourproxy:port use_rvm: False rvm_ruby: 2.1.0 shell_version: v2.7.2 From df53da978bef5c110d0aa2a27b52a25103435bf2 Mon Sep 17 00:00:00 2001 From: "Charles R. (hugo)" Date: Tue, 3 May 2016 16:46:51 +0200 Subject: [PATCH 10/96] Install from archives option --- README.rst | 1 + gitlab/gitlab-shell.sls | 68 +++++++++++------- gitlab/gitlab-workhorse.sls | 47 +++++++----- gitlab/gitlab.sls | 139 ++++++++++++++++++++++++++---------- gitlab/test.sls | 2 +- pillar.example | 15 ++++ 6 files changed, 190 insertions(+), 82 deletions(-) diff --git a/README.rst b/README.rst index 71f4e3b..a82992c 100644 --- a/README.rst +++ b/README.rst @@ -7,6 +7,7 @@ Modification from original formula : * Use of Postgresql formula * Original initd script from Gitlab setup * Proxy for git.latest / gem +* Install package from archive * Lot of little things... SaltStack formula to install GitLab diff --git a/gitlab/gitlab-shell.sls b/gitlab/gitlab-shell.sls index 26f67b9..7c814fc 100644 --- a/gitlab/gitlab-shell.sls +++ b/gitlab/gitlab-shell.sls @@ -5,63 +5,77 @@ include: {% set root_dir = salt['pillar.get']('gitlab:lookup:root_dir', '/home/git') %} {% set lib_dir = salt['pillar.get']('gitlab:lookup:lib_dir', root_dir ~ '/libraries') %} -{% if salt['pillar.get']('gitlab:proxy:enabled', false) %} -gitlab-shell-git-present: - git.present: - - name: {{ lib_dir }}/gitlab-shell.git - - bare: False +{% set shell_dir = lib_dir ~ "/gitlab-shell" %} -gitlab-shell-git-proxy: - git.config: - - name: http.proxy - - value: {{ salt['pillar.get']('gitlab:proxy:address') }} - - repo: {{ lib_dir }}/gitlab-shell.git -# - require: -# - git: gitlab-shell-git-present +{% if salt['pillar.get']('gitlab:archives:enabled', false) %} + {% set shell_dir_content = shell_dir ~ '/' ~ salt['pillar.get']('gitlab:archives:sources:shell:content') %} +{% else %} + {% set shell_dir_content = shell_dir %} {% endif %} -gitlab-shell-git: +gitlab-shell-fetcher: +{% if salt['pillar.get']('gitlab:archives:enabled', false) %} + archive.extracted: + - name: {{ shell_dir }} + - source: {{ salt['pillar.get']('gitlab:archives:sources:shell:url') }} + - source_hash: md5={{ salt['pillar.get']('gitlab:archives:sources:shell:md5') }} + - archive_format: tar + - if_missing: {{ shell_dir_content }} + - keep: True + file.directory: + - name: {{ shell_dir }} + - user: git + - recurse: + - user +{% else %} git.latest: - name: https://gitlab.com/gitlab-org/gitlab-shell.git - rev: {{ salt['pillar.get']('gitlab:shell_version') }} - - target: {{ lib_dir }}/gitlab-shell.git + - target: {{ shell_dir }} - user: git + - force: True - require: - pkg: gitlab-deps - pkg: git - sls: gitlab.ruby - file: git-home - {% if salt['pillar.get']('gitlab:proxy:enabled', false) %} - - git: gitlab-shell-git-proxy - {% endif %} +{% endif %} # https://gitlab.com/gitlab-org/gitlab-shell/blob/master/config.yml.example gitlab-shell-config: file.managed: - - name: {{ lib_dir }}/gitlab-shell.git/config.yml + - name: {{ shell_dir_content }}/config.yml - source: salt://gitlab/files/gitlab-shell-config.yml - template: jinja - user: git - group: git - mode: 644 - require: - - git: gitlab-shell-git + {% if salt['pillar.get']('gitlab:archives:enabled', false) %} + - archive: gitlab-shell-fetcher + {% else %} + - git: gitlab-shell-fetcher + {% endif %} gitlab-shell: cmd.wait: - user: git - - cwd: {{ lib_dir }}/gitlab-shell.git + - cwd: {{ shell_dir_content }} - name: ./bin/install - shell: /bin/bash - watch: - - git: gitlab-shell-git + {% if salt['pillar.get']('gitlab:archives:enabled', false) %} + - archive: gitlab-shell-fetcher + {% else %} + - git: gitlab-shell-fetcher + {% endif %} - require: - file: gitlab-shell-config -gitlab-shell-chmod-bin: - file.directory: - - name: {{ lib_dir }}/gitlab-shell.git/bin - - file_mode: 0770 - - recurse: - - mode +#gitlab-shell-chmod-bin: +# file.directory: +# - name: {{ shell_dir }}/bin +# - file_mode: 0770 +# - recurse: +# - mode diff --git a/gitlab/gitlab-workhorse.sls b/gitlab/gitlab-workhorse.sls index 17d7944..2ead12c 100644 --- a/gitlab/gitlab-workhorse.sls +++ b/gitlab/gitlab-workhorse.sls @@ -2,33 +2,41 @@ {% set root_dir = salt['pillar.get']('gitlab:lookup:root_dir', '/home/git') %} {% set lib_dir = salt['pillar.get']('gitlab:lookup:lib_dir', root_dir ~ '/libraries') %} -{% if salt['pillar.get']('gitlab:proxy:enabled', false) %} -gitlab-workhorse-git-present: - git.present: - - name: {{ lib_dir }}/gitlab-workhorse.git - - bare: False +{% set workhorse_dir = lib_dir ~ "/gitlab-workhorse" %} -gitlab-workhorse-git-proxy: - git.config: - - name: http.proxy - - value: {{ salt['pillar.get']('gitlab:proxy:address') }} - - repo: {{ lib_dir }}/gitlab-workhorse.git +{% if salt['pillar.get']('gitlab:archives:enabled', false) %} + {% set workhorse_dir_content = workhorse_dir ~ '/' ~ salt['pillar.get']('gitlab:archives:sources:workhorse:content') %} +{% else %} + {% set workhorse_dir_content = workhorse_dir %} {% endif %} -gitlab-workhorse-git: +gitlab-workhorse-fetcher: +{% if salt['pillar.get']('gitlab:archives:enabled', false) %} + archive.extracted: + - name: {{ workhorse_dir }} + - source: {{ salt['pillar.get']('gitlab:archives:sources:workhorse:url') }} + - source_hash: md5={{ salt['pillar.get']('gitlab:archives:sources:workhorse:md5') }} + - archive_format: tar + - if_missing: {{ workhorse_dir_content }} + - keep: True + file.directory: + - name: {{ workhorse_dir }} + - user: git + - recurse: + - user +{% else %} git.latest: - name: https://gitlab.com/gitlab-org/gitlab-workhorse.git - rev: {{ salt['pillar.get']('gitlab:workhorse_version') }} - - target: {{ lib_dir }}/gitlab-workhorse.git + - target: {{ workhorse_dir }} - user: git + - force: True - require: - pkg: gitlab-deps - pkg: git - sls: gitlab.ruby - file: git-home - {% if salt['pillar.get']('gitlab:proxy:enabled', false) %} - - git: gitlab-workhorse-git-proxy - {% endif %} +{% endif %} {{ root_dir }}/gitlab-workhorse: file.directory: @@ -39,10 +47,15 @@ gitlab-workhorse-git: gitlab-workhorse-make: cmd.run: - user: git - - cwd: {{ lib_dir }}/gitlab-workhorse.git + - cwd: {{ workhorse_dir_content }} - name: make install PREFIX={{ root_dir }}/gitlab-workhorse - shell: /bin/bash - require: - - git: gitlab-workhorse-git + {% if salt['pillar.get']('gitlab:archives:enabled', false) %} + - archive: gitlab-workhorse-fetcher + - file: gitlab-workhorse-fetcher + {% else %} + - git: gitlab-workhorse-fetcher + {% endif %} - file: {{ root_dir }}/gitlab-workhorse diff --git a/gitlab/gitlab.sls b/gitlab/gitlab.sls index bf8c807..f65f4bd 100644 --- a/gitlab/gitlab.sls +++ b/gitlab/gitlab.sls @@ -8,38 +8,55 @@ include: {% set pids_dir = salt['pillar.get']('gitlab:lookup:pids_dir', root_dir ~ '/var/pids') %} {% set logs_dir = salt['pillar.get']('gitlab:lookup:logs_dir', root_dir ~ '/var/logs') %} {% set uploads_dir = salt['pillar.get']('gitlab:lookup:uploads_dir', root_dir ~ '/var/uploads') %} +{% set lib_dir = salt['pillar.get']('gitlab:lookup:lib_dir', root_dir ~ '/libraries') %} {% set active_db = salt['pillar.get']('gitlab:databases:production', 'paf') %} {% set db_user, db_user_infos = salt['pillar.get']('postgres:users').items()[0] %} -{% if salt['pillar.get']('gitlab:proxy:enabled', false) %} -gitlab-git-present: - git.present: - - name: {{ root_dir }}/gitlab - - bare: False - -gitlab-git-proxy: - git.config: - - name: http.proxy - - value: {{ salt['pillar.get']('gitlab:proxy:address') }} - - repo: {{ root_dir }}/gitlab +{% set gitlab_dir = root_dir ~ "/gitlab" %} +{% if salt['pillar.get']('gitlab:archives:enabled', false) %} + {% set gitlab_dir_content = lib_dir ~ '/gitlab/' ~ salt['pillar.get']('gitlab:archives:sources:gitlab:content') %} +{% else %} + {% set gitlab_dir_content = gitlab_dir %} {% endif %} -gitlab-git: +{% if salt['pillar.get']('gitlab:archives:enabled', false) %} +gitlab-fetcher: + archive.extracted: + - name: {{ lib_dir }}/gitlab + - source: {{ salt['pillar.get']('gitlab:archives:sources:gitlab:url') }} + - source_hash: md5={{ salt['pillar.get']('gitlab:archives:sources:gitlab:md5') }} + - archive_format: tar + - if_missing: {{ gitlab_dir_content }} + - keep: True + file.directory: + - name: {{ gitlab_dir_content }} + - user: git + - group: git + - recurse: + - user + +gitlab-lib-symlink: + file.symlink: + - name: {{ gitlab_dir }} + - target: {{ gitlab_dir_content }} + require: + - file: gitlab-fetcher +{% else %} +gitlab-fetcher: git.latest: - name: https://gitlab.com/gitlab-org/gitlab-ce.git - rev: {{ salt['pillar.get']('gitlab:gitlab_version') }} - user: git - - target: {{ root_dir }}/gitlab + - target: {{ gitlab_dir }} + - force: True - require: - pkg: gitlab-deps - pkg: git - sls: gitlab.ruby - cmd: gitlab-shell - user: git-user - {% if salt['pillar.get']('gitlab:proxy:enabled', false) %} - - git: gitlab-git-proxy - {% endif %} +{% endif %} # https://gitlab.com/gitlab-org/gitlab-ce/blob/master/config/gitlab.yml.example gitlab-config: @@ -51,7 +68,11 @@ gitlab-config: - group: git - mode: 640 - require: - - git: gitlab-git + {% if salt['pillar.get']('gitlab:archives:enabled', false) %} + - archive: gitlab-fetcher + {% else %} + - git: gitlab-fetcher + {% endif %} - user: git-user # https://gitlab.com/gitlab-org/gitlab-ce/blob/master/config/database.yml.postgresql @@ -64,7 +85,11 @@ gitlab-db-config: - group: git - mode: 640 - require: - - git: gitlab-git + {% if salt['pillar.get']('gitlab:archives:enabled', false) %} + - archive: gitlab-fetcher + {% else %} + - git: gitlab-fetcher + {% endif %} - user: git-user # https://gitlab.com/gitlab-org/gitlab-ce/blob/master/config/unicorn.rb.example @@ -77,7 +102,11 @@ unicorn-config: - group: git - mode: 640 - require: - - git: gitlab-git + {% if salt['pillar.get']('gitlab:archives:enabled', false) %} + - archive: gitlab-fetcher + {% else %} + - git: gitlab-fetcher + {% endif %} - user: git-user # https://gitlab.com/gitlab-org/gitlab-ce/blob/master/config/initializers/rack_attack.rb.example @@ -89,7 +118,11 @@ rack_attack-config: - group: git - mode: 640 - require: - - git: gitlab-git + {% if salt['pillar.get']('gitlab:archives:enabled', false) %} + - archive: gitlab-fetcher + {% else %} + - git: gitlab-fetcher + {% endif %} - user: git-user git-config: @@ -124,7 +157,7 @@ git-{{ dir }}-mkdir: gitlab-pids_dir-symlink: file.symlink: - name: {{ pids_dir }} - - target: {{ root_dir }}/gitlab/tmp/pids + - target: {{ gitlab_dir }}/tmp/pids require: - file: gitlab-config @@ -133,11 +166,15 @@ gitlab-pids_dir-symlink: gitlab-gems: cmd.run: - user: git - - cwd: {{ root_dir }}/gitlab + - cwd: {{ gitlab_dir }} - name: bundle install --deployment --without development test mysql aws kerberos - shell: /bin/bash - watch: - - git: gitlab-git + {% if salt['pillar.get']('gitlab:archives:enabled', false) %} + - archive: gitlab-fetcher + {% else %} + - git: gitlab-fetcher + {% endif %} - require: - file: gitlab-db-config - file: gitlab-config @@ -148,12 +185,16 @@ gitlab-gems: gitlab-initialize: cmd.run: - user: git - - cwd: {{ root_dir }}/gitlab + - cwd: {{ gitlab_dir }} - name: force=yes bundle exec rake gitlab:setup RAILS_ENV=production - shell: /bin/bash - unless: PGPASSWORD={{ db_user_infos.password }} psql -h {{ active_db.host }} -U {{ db_user }} {{ active_db.name }} -c 'select * from users;' - watch: - - git: gitlab-git + {% if salt['pillar.get']('gitlab:archives:enabled', false) %} + - archive: gitlab-fetcher + {% else %} + - git: gitlab-fetcher + {% endif %} - require: - cmd: gitlab-gems - file: gitlab-db-config @@ -161,11 +202,15 @@ gitlab-initialize: gitlab-migrate-db: cmd.wait: - user: git - - cwd: {{ root_dir }}/gitlab + - cwd: {{ gitlab_dir }} - name: bundle exec rake db:migrate RAILS_ENV=production - shell: /bin/bash - watch: - - git: gitlab-git + {% if salt['pillar.get']('gitlab:archives:enabled', false) %} + - archive: gitlab-fetcher + {% else %} + - git: gitlab-fetcher + {% endif %} - require: - cmd: gitlab-gems - cmd: gitlab-initialize @@ -174,22 +219,30 @@ gitlab-migrate-db: gitlab-recompile-assets: cmd.wait: - user: git - - cwd: {{ root_dir }}/gitlab + - cwd: {{ gitlab_dir }} - name: bundle exec rake assets:clean assets:precompile RAILS_ENV=production - shell: /bin/bash - watch: - - git: gitlab-git + {% if salt['pillar.get']('gitlab:archives:enabled', false) %} + - archive: gitlab-fetcher + {% else %} + - git: gitlab-fetcher + {% endif %} - require: - cmd: gitlab-migrate-db gitlab-clear-cache: cmd.wait: - user: git - - cwd: {{ root_dir }}/gitlab + - cwd: {{ gitlab_dir }} - name: bundle exec rake cache:clear RAILS_ENV=production - shell: /bin/bash - watch: - - git: gitlab-git + {% if salt['pillar.get']('gitlab:archives:enabled', false) %} + - archive: gitlab-fetcher + {% else %} + - git: gitlab-fetcher + {% endif %} - require: - cmd: gitlab-recompile-assets @@ -197,10 +250,14 @@ gitlab-clear-cache: gitlab-stash: cmd.wait: - user: git - - cwd: {{ root_dir }}/gitlab + - cwd: {{ gitlab_dir }} - name: git stash - watch: - - git: gitlab-git + {% if salt['pillar.get']('gitlab:archives:enabled', false) %} + - archive: gitlab-fetcher + {% else %} + - git: gitlab-fetcher + {% endif %} - require: - cmd: gitlab-clear-cache @@ -234,13 +291,13 @@ gitlab-respositories-dir: gitlab-uploads-dir: file.directory: - - name: {{ root_dir }}/gitlab/public/uploads + - name: {{ gitlab_dir }}/public/uploads - dir_mode: 0700 gitlab-uploads-symlink: file.symlink: - name: {{ uploads_dir }} - - target: {{ root_dir }}/gitlab/public/uploads + - target: {{ gitlab_dir }}/public/uploads - require: - file: git-var-mkdir @@ -251,7 +308,11 @@ gitlab-service: - mode: 0755 - template: jinja - require: - - git: gitlab-git + {% if salt['pillar.get']('gitlab:archives:enabled', false) %} + - archive: gitlab-fetcher + {% else %} + - git: gitlab-fetcher + {% endif %} service: - name: gitlab - running @@ -262,7 +323,11 @@ gitlab-service: # - cmd: gitlab-initialize - file: gitlab-pids_dir-symlink - watch: - - git: gitlab-git + {% if salt['pillar.get']('gitlab:archives:enabled', false) %} + - archive: gitlab-fetcher + {% else %} + - git: gitlab-fetcher + {% endif %} - cmd: gitlab-clear-cache - file: gitlab-config - file: gitlab-db-config diff --git a/gitlab/test.sls b/gitlab/test.sls index a2547bb..0e8d03a 100644 --- a/gitlab/test.sls +++ b/gitlab/test.sls @@ -19,7 +19,7 @@ gitlab-initialize: - shell: /bin/bash - unless: PGPASSWORD={{ user_infos.password }} psql -h {{ active_db.host }} -U {{ user }} {{ active_db.name }} -c 'select * from users;' # - watch: -# - git: gitlab-git +# - git: gitlab-fetcher # - require: # - cmd: gitlab-gems # - file: gitlab-db-config diff --git a/pillar.example b/pillar.example index 5df8b3b..d69c355 100644 --- a/pillar.example +++ b/pillar.example @@ -10,6 +10,21 @@ gitlab: proxy: enabled: false address: http://ourproxy:port + archives: + enabled: false + sources: + workhorse: + url: https://gitlab.com/gitlab-org/gitlab-workhorse/repository/archive.tar.gz?ref=0.7.2 + md5: c0b266285bf54ed7cef806f25a7e8aa9 + content: gitlab-workhorse-0.7.2-7a2c97cb8f98a2af9b8ec80fcafc2721ef4a8e97 + shell: + url: https://gitlab.com/gitlab-org/gitlab-shell/repository/archive.tar.gz?ref=v2.7.2 + md5: 1835e858d9285ac9941eec320e2ecac6 + content: gitlab-shell-v2.7.2-c615ca4650dfb4b17bf073276024f886e52b74a3 + gitlab: + url: https://gitlab.com/gitlab-org/gitlab-ce/repository/archive.tar.gz?ref=v8.7.1 + md5: 684a0c36cd96bb5766d75400955786b1 + content: gitlab-ce-v8.7.1-93187cbe200908d649b2fd11dff549da95033211 use_rvm: False rvm_ruby: 2.1.0 shell_version: v2.7.2 From 7aa734527bbb35befe1b1445a7e3283dd7ea771f Mon Sep 17 00:00:00 2001 From: "Charles R. (hugo)" Date: Tue, 3 May 2016 16:54:15 +0200 Subject: [PATCH 11/96] Update to gitlab 8.7.2 --- pillar.example | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pillar.example b/pillar.example index d69c355..38ec8f9 100644 --- a/pillar.example +++ b/pillar.example @@ -13,6 +13,10 @@ gitlab: archives: enabled: false sources: + gitlab: + url: https://gitlab.com/gitlab-org/gitlab-ce/repository/archive.tar.gz?ref=v8.7.2 + md5: 0a63803fd87766d034cf3ff4d4133fce + content: gitlab-ce-v8.7.2-213974bc9bfca79136db32d47f4854b02e0e7fc2 workhorse: url: https://gitlab.com/gitlab-org/gitlab-workhorse/repository/archive.tar.gz?ref=0.7.2 md5: c0b266285bf54ed7cef806f25a7e8aa9 @@ -21,10 +25,6 @@ gitlab: url: https://gitlab.com/gitlab-org/gitlab-shell/repository/archive.tar.gz?ref=v2.7.2 md5: 1835e858d9285ac9941eec320e2ecac6 content: gitlab-shell-v2.7.2-c615ca4650dfb4b17bf073276024f886e52b74a3 - gitlab: - url: https://gitlab.com/gitlab-org/gitlab-ce/repository/archive.tar.gz?ref=v8.7.1 - md5: 684a0c36cd96bb5766d75400955786b1 - content: gitlab-ce-v8.7.1-93187cbe200908d649b2fd11dff549da95033211 use_rvm: False rvm_ruby: 2.1.0 shell_version: v2.7.2 From 2bb7594cbd5c84705a954b05db1901a0d36b39cc Mon Sep 17 00:00:00 2001 From: "Charles R. (hugo)" Date: Tue, 3 May 2016 17:42:42 +0200 Subject: [PATCH 12/96] Add proxy on bundle install --- gitlab/gitlab.sls | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/gitlab/gitlab.sls b/gitlab/gitlab.sls index f65f4bd..28afd6e 100644 --- a/gitlab/gitlab.sls +++ b/gitlab/gitlab.sls @@ -186,7 +186,11 @@ gitlab-initialize: cmd.run: - user: git - cwd: {{ gitlab_dir }} + {% if salt['pillar.get']('gitlab:proxy:enabled', false) %} + - name: HTTP_PROXY={{ salt['pillar.get']('gitlab:proxy:address') }} force=yes bundle exec rake gitlab:setup RAILS_ENV=production + {% else %} - name: force=yes bundle exec rake gitlab:setup RAILS_ENV=production + {% endif %} - shell: /bin/bash - unless: PGPASSWORD={{ db_user_infos.password }} psql -h {{ active_db.host }} -U {{ db_user }} {{ active_db.name }} -c 'select * from users;' - watch: From abbbdbd0dfbcab03c707c990175e850c97bc6ee6 Mon Sep 17 00:00:00 2001 From: "Charles R. (hugo)" Date: Tue, 3 May 2016 18:11:23 +0200 Subject: [PATCH 13/96] Rename gitlab.url to gitlab.source --- gitlab/gitlab-shell.sls | 2 +- gitlab/gitlab-workhorse.sls | 2 +- gitlab/gitlab.sls | 2 +- pillar.example | 6 +++--- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/gitlab/gitlab-shell.sls b/gitlab/gitlab-shell.sls index 7c814fc..d6ad9f1 100644 --- a/gitlab/gitlab-shell.sls +++ b/gitlab/gitlab-shell.sls @@ -17,7 +17,7 @@ gitlab-shell-fetcher: {% if salt['pillar.get']('gitlab:archives:enabled', false) %} archive.extracted: - name: {{ shell_dir }} - - source: {{ salt['pillar.get']('gitlab:archives:sources:shell:url') }} + - source: {{ salt['pillar.get']('gitlab:archives:sources:shell:source') }} - source_hash: md5={{ salt['pillar.get']('gitlab:archives:sources:shell:md5') }} - archive_format: tar - if_missing: {{ shell_dir_content }} diff --git a/gitlab/gitlab-workhorse.sls b/gitlab/gitlab-workhorse.sls index 2ead12c..b12870b 100644 --- a/gitlab/gitlab-workhorse.sls +++ b/gitlab/gitlab-workhorse.sls @@ -14,7 +14,7 @@ gitlab-workhorse-fetcher: {% if salt['pillar.get']('gitlab:archives:enabled', false) %} archive.extracted: - name: {{ workhorse_dir }} - - source: {{ salt['pillar.get']('gitlab:archives:sources:workhorse:url') }} + - source: {{ salt['pillar.get']('gitlab:archives:sources:workhorse:source') }} - source_hash: md5={{ salt['pillar.get']('gitlab:archives:sources:workhorse:md5') }} - archive_format: tar - if_missing: {{ workhorse_dir_content }} diff --git a/gitlab/gitlab.sls b/gitlab/gitlab.sls index 28afd6e..48ab7a9 100644 --- a/gitlab/gitlab.sls +++ b/gitlab/gitlab.sls @@ -24,7 +24,7 @@ include: gitlab-fetcher: archive.extracted: - name: {{ lib_dir }}/gitlab - - source: {{ salt['pillar.get']('gitlab:archives:sources:gitlab:url') }} + - source: {{ salt['pillar.get']('gitlab:archives:sources:gitlab:source') }} - source_hash: md5={{ salt['pillar.get']('gitlab:archives:sources:gitlab:md5') }} - archive_format: tar - if_missing: {{ gitlab_dir_content }} diff --git a/pillar.example b/pillar.example index 38ec8f9..ca66ca5 100644 --- a/pillar.example +++ b/pillar.example @@ -14,15 +14,15 @@ gitlab: enabled: false sources: gitlab: - url: https://gitlab.com/gitlab-org/gitlab-ce/repository/archive.tar.gz?ref=v8.7.2 + source: https://gitlab.com/gitlab-org/gitlab-ce/repository/archive.tar.gz?ref=v8.7.2 md5: 0a63803fd87766d034cf3ff4d4133fce content: gitlab-ce-v8.7.2-213974bc9bfca79136db32d47f4854b02e0e7fc2 workhorse: - url: https://gitlab.com/gitlab-org/gitlab-workhorse/repository/archive.tar.gz?ref=0.7.2 + source: https://gitlab.com/gitlab-org/gitlab-workhorse/repository/archive.tar.gz?ref=0.7.2 md5: c0b266285bf54ed7cef806f25a7e8aa9 content: gitlab-workhorse-0.7.2-7a2c97cb8f98a2af9b8ec80fcafc2721ef4a8e97 shell: - url: https://gitlab.com/gitlab-org/gitlab-shell/repository/archive.tar.gz?ref=v2.7.2 + source: https://gitlab.com/gitlab-org/gitlab-shell/repository/archive.tar.gz?ref=v2.7.2 md5: 1835e858d9285ac9941eec320e2ecac6 content: gitlab-shell-v2.7.2-c615ca4650dfb4b17bf073276024f886e52b74a3 use_rvm: False From 6ede75b36e9e8f819f97652bdf99182dec3e1a4f Mon Sep 17 00:00:00 2001 From: "Charles R. (hugo)" Date: Wed, 4 May 2016 11:57:33 +0200 Subject: [PATCH 14/96] Fix proxy problem with gem and problem with gitlab shell path --- gitlab/files/gitlab-gitlab.yml | 2 +- gitlab/gitlab.sls | 12 +++++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/gitlab/files/gitlab-gitlab.yml b/gitlab/files/gitlab-gitlab.yml index efd49f0..9d9c737 100644 --- a/gitlab/files/gitlab-gitlab.yml +++ b/gitlab/files/gitlab-gitlab.yml @@ -193,7 +193,7 @@ production: &base ## GitLab Shell settings gitlab_shell: - path: {{ lib_dir }}/gitlab-shell.git/ + path: {{ lib_dir }}/gitlab-shell/ # REPOS_PATH MUST NOT BE A SYMLINK!!! repos_path: {{ repositories }}/ diff --git a/gitlab/gitlab.sls b/gitlab/gitlab.sls index 48ab7a9..f2d3fa3 100644 --- a/gitlab/gitlab.sls +++ b/gitlab/gitlab.sls @@ -20,6 +20,12 @@ include: {% set gitlab_dir_content = gitlab_dir %} {% endif %} +{% if salt['pillar.get']('gitlab:proxy:enabled', false) %} + {% set proxy = 'HTTP_PROXY=' ~ salt['pillar.get']('gitlab:proxy:address') %} +{% else %} + {% set proxy = '' %} +{% endif %} + {% if salt['pillar.get']('gitlab:archives:enabled', false) %} gitlab-fetcher: archive.extracted: @@ -167,7 +173,7 @@ gitlab-gems: cmd.run: - user: git - cwd: {{ gitlab_dir }} - - name: bundle install --deployment --without development test mysql aws kerberos + - name: {{ proxy }} bundle install --deployment --without development test mysql aws kerberos - shell: /bin/bash - watch: {% if salt['pillar.get']('gitlab:archives:enabled', false) %} @@ -186,11 +192,7 @@ gitlab-initialize: cmd.run: - user: git - cwd: {{ gitlab_dir }} - {% if salt['pillar.get']('gitlab:proxy:enabled', false) %} - - name: HTTP_PROXY={{ salt['pillar.get']('gitlab:proxy:address') }} force=yes bundle exec rake gitlab:setup RAILS_ENV=production - {% else %} - name: force=yes bundle exec rake gitlab:setup RAILS_ENV=production - {% endif %} - shell: /bin/bash - unless: PGPASSWORD={{ db_user_infos.password }} psql -h {{ active_db.host }} -U {{ db_user }} {{ active_db.name }} -c 'select * from users;' - watch: From a09a365ae983afe092e185d027cd3a2dd182fe44 Mon Sep 17 00:00:00 2001 From: "Charles R. (hugo)" Date: Wed, 4 May 2016 15:38:25 +0200 Subject: [PATCH 15/96] Fix shell path --- gitlab/files/gitlab-gitlab.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/gitlab/files/gitlab-gitlab.yml b/gitlab/files/gitlab-gitlab.yml index 9d9c737..864c288 100644 --- a/gitlab/files/gitlab-gitlab.yml +++ b/gitlab/files/gitlab-gitlab.yml @@ -11,6 +11,13 @@ {% set lib_dir = salt['pillar.get']('gitlab:lookup:lib_dir', root_dir ~ '/libraries') %} {% set repositories = salt['pillar.get']('gitlab:lookup:repositories', root_dir ~ '/repositories') %} +{% set shell_dir = lib_dir ~ "/gitlab-shell" %} +{% if salt['pillar.get']('gitlab:archives:enabled', false) %} + {% set shell_dir_content = shell_dir ~ '/' ~ salt['pillar.get']('gitlab:archives:sources:shell:content') %} +{% else %} + {% set shell_dir_content = shell_dir %} +{% endif %} + production: &base # # 1. GitLab app settings @@ -193,7 +200,7 @@ production: &base ## GitLab Shell settings gitlab_shell: - path: {{ lib_dir }}/gitlab-shell/ + path: {{ shell_dir_content }} # REPOS_PATH MUST NOT BE A SYMLINK!!! repos_path: {{ repositories }}/ From e2e4a65e16408d614cfd4105869ce9d0f567d31d Mon Sep 17 00:00:00 2001 From: "Charles R. (hugo)" Date: Wed, 4 May 2016 15:44:23 +0200 Subject: [PATCH 16/96] Replace fqdn with hostname variable --- gitlab/files/gitlab-gitconfig | 2 +- gitlab/files/gitlab-gitlab.yml | 6 +++--- gitlab/files/gitlab-nginx | 2 +- gitlab/files/gitlab-nginx-ssl | 4 ++-- gitlab/files/gitlab-shell-config.yml | 4 ++-- pillar.example | 1 + 6 files changed, 10 insertions(+), 9 deletions(-) diff --git a/gitlab/files/gitlab-gitconfig b/gitlab/files/gitlab-gitconfig index 2ac30d0..7c25e0c 100644 --- a/gitlab/files/gitlab-gitconfig +++ b/gitlab/files/gitlab-gitconfig @@ -1,6 +1,6 @@ [user] name = GitLab - email = gitlab@{{ grains['fqdn'] }} + email = gitlab@{{ salt['pillar.get']('gitlab:hostname', grains['fqdn']) }} [core] autocrlf = input diff --git a/gitlab/files/gitlab-gitlab.yml b/gitlab/files/gitlab-gitlab.yml index 864c288..8bfa9da 100644 --- a/gitlab/files/gitlab-gitlab.yml +++ b/gitlab/files/gitlab-gitlab.yml @@ -26,7 +26,7 @@ production: &base ## GitLab settings gitlab: ## Web server settings - host: {{ grains['fqdn'] }} + host: {{ salt['pillar.get']('gitlab:hostname', grains['fqdn']) }} {%- if salt['pillar.get']('gitlab:https', false) %} port: 443 https: true @@ -51,10 +51,10 @@ production: &base ## Email settings # Email address used in the "From" field in mails sent by GitLab - email_from: gitlab@{{ grains['fqdn'] }} + email_from: gitlab@{{ salt['pillar.get']('gitlab:hostname', grains['fqdn']) }} # Email address of your support contact (default: same as email_from) - support_email: support@{{ grains['fqdn'] }} + support_email: support@{{ salt['pillar.get']('gitlab:hostname', grains['fqdn']) }} ## User settings default_projects_limit: 10 diff --git a/gitlab/files/gitlab-nginx b/gitlab/files/gitlab-nginx index e55f9e8..6440551 100644 --- a/gitlab/files/gitlab-nginx +++ b/gitlab/files/gitlab-nginx @@ -9,7 +9,7 @@ upstream gitlab-workhorse { server { listen *:80 default_server; # e.g., listen 192.168.1.1:80; In most cases *:80 is a good idea - server_name {{ grains['fqdn'] }}; # e.g., server_name source.example.com; + server_name {{ salt['pillar.get']('gitlab:hostname', grains['fqdn']) }}; # e.g., server_name source.example.com; server_tokens off; # don't show the version number, a security best practice root {{ root_dir }}/gitlab/public; diff --git a/gitlab/files/gitlab-nginx-ssl b/gitlab/files/gitlab-nginx-ssl index e806449..8fc3fbc 100644 --- a/gitlab/files/gitlab-nginx-ssl +++ b/gitlab/files/gitlab-nginx-ssl @@ -36,7 +36,7 @@ upstream gitlab-workhorse { # Replace {{ grains['fqdn'] }} with your FQDN. server { listen *:80; - server_name {{ grains['fqdn'] }}; + server_name {{ salt['pillar.get']('gitlab:hostname', grains['fqdn']) }}; server_tokens off; root /nowhere; # this doesn't have to be a valid path since we are redirecting, you don't have to change it. rewrite ^ https://$server_name$request_uri permanent; @@ -44,7 +44,7 @@ server { server { listen 443 ssl; - server_name {{ grains['fqdn'] }}; + server_name {{ salt['pillar.get']('gitlab:hostname', grains['fqdn']) }}; server_tokens off; root {{ root_dir }}/gitlab/public; diff --git a/gitlab/files/gitlab-shell-config.yml b/gitlab/files/gitlab-shell-config.yml index e525071..7cdbeaa 100644 --- a/gitlab/files/gitlab-shell-config.yml +++ b/gitlab/files/gitlab-shell-config.yml @@ -7,9 +7,9 @@ user: git # Url to gitlab instance. Used for api calls. Should end with a slash. {%- if salt['pillar.get']('gitlab:https', false) %} -gitlab_url: "https://{{ grains['fqdn'] }}/" +gitlab_url: "https://{{ salt['pillar.get']('gitlab:hostname', grains['fqdn']) }}/" {% else %} -gitlab_url: "http://{{ grains['fqdn'] }}/" +gitlab_url: "http://{{ salt['pillar.get']('gitlab:hostname', grains['fqdn']) }}/" {% endif %} http_settings: diff --git a/pillar.example b/pillar.example index ca66ca5..4a9a1af 100644 --- a/pillar.example +++ b/pillar.example @@ -7,6 +7,7 @@ gitlab: sockets_dir: /opt/git/var/sockets logs_dir: /opt/git/var/logs uploads_dir: /opt/git/var/uploads + hostname: localhost proxy: enabled: false address: http://ourproxy:port From 1cb183b0240b219de687dd8e6f0e3e9ddf500b8b Mon Sep 17 00:00:00 2001 From: "Charles R. (hugo)" Date: Mon, 16 May 2016 18:11:05 +0200 Subject: [PATCH 17/96] Remove Nginx configuration -> Use of nginx-formula --- README.rst | 2 +- gitlab/files/gitlab-nginx | 53 ---------------------- gitlab/files/gitlab-nginx-ssl | 83 ----------------------------------- gitlab/init.sls | 1 - gitlab/nginx.sls | 81 ---------------------------------- 5 files changed, 1 insertion(+), 219 deletions(-) delete mode 100644 gitlab/files/gitlab-nginx delete mode 100644 gitlab/files/gitlab-nginx-ssl delete mode 100644 gitlab/nginx.sls diff --git a/README.rst b/README.rst index a82992c..b790a7c 100644 --- a/README.rst +++ b/README.rst @@ -4,7 +4,7 @@ gitlab-formula Modification from original formula : * No hardcoded path : possibilty to install gitlab where you want -* Use of Postgresql formula +* Use of Postgresql / Nginx formulas * Original initd script from Gitlab setup * Proxy for git.latest / gem * Install package from archive diff --git a/gitlab/files/gitlab-nginx b/gitlab/files/gitlab-nginx deleted file mode 100644 index 6440551..0000000 --- a/gitlab/files/gitlab-nginx +++ /dev/null @@ -1,53 +0,0 @@ - -{% set root_dir = salt['pillar.get']('gitlab:lookup:root_dir', '/home/git') %} -{% set sockets_dir = salt['pillar.get']('gitlab:lookup:sockets_dir', root_dir ~ '/var/sockets') %} -{% set logs_dir = salt['pillar.get']('gitlab:lookup:logs_dir', root_dir ~ '/var/logs') %} - -upstream gitlab-workhorse { - server unix:{{ sockets_dir }}/gitlab-workhorse.socket fail_timeout=0; -} - -server { - listen *:80 default_server; # e.g., listen 192.168.1.1:80; In most cases *:80 is a good idea - server_name {{ salt['pillar.get']('gitlab:hostname', grains['fqdn']) }}; # e.g., server_name source.example.com; - server_tokens off; # don't show the version number, a security best practice - root {{ root_dir }}/gitlab/public; - - # Set value of client_max_body_size to at least the value of git.max_size in gitlab.yml - client_max_body_size 5m; - - # individual nginx logs for this gitlab vhost - access_log {{ logs_dir }}/nginx_access.log; - error_log {{ logs_dir }}/nginx_error.log; - - location / { - # serve static files from defined root folder;. - # @gitlab is a named location for the upstream fallback, see below - try_files $uri $uri/index.html $uri.html @gitlab; - } - - # if a file, which is not found in the root folder is requested, - # then the proxy pass the request to the upsteam (gitlab unicorn) - location @gitlab { - proxy_read_timeout 300; # Some requests take more than 30 seconds. - proxy_connect_timeout 300; # Some requests take more than 30 seconds. - proxy_redirect off; - - proxy_set_header X-Forwarded-Proto $scheme; - proxy_set_header Host $http_host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - - proxy_pass http://gitlab-workhorse; - } - - error_page 404 /404.html; - error_page 422 /422.html; - error_page 500 /500.html; - error_page 502 /502.html; - location ~ ^/(404|422|500|502)\.html$ { - root {{ root_dir }}/gitlab/public; - internal; - } -} - diff --git a/gitlab/files/gitlab-nginx-ssl b/gitlab/files/gitlab-nginx-ssl deleted file mode 100644 index 8fc3fbc..0000000 --- a/gitlab/files/gitlab-nginx-ssl +++ /dev/null @@ -1,83 +0,0 @@ -# GITLAB -# Contributors: yin8086, sashkab, orkoden, axilleas -# App Version: 5.4 - 6.0 - -# Modified from nginx http version -# Modified from http://blog.phusion.nl/2012/04/21/tutorial-setting-up-gitlab-on-debian-6/ - -# You need to run openssl to generate a self-signed ssl certificate. -# cd /etc/nginx/ -# sudo openssl req -new -x509 -nodes -days 3560 -out gitlab.crt -keyout gitlab.key -# sudo chmod o-r gitlab.key -# Also you need to edit gitlab-shell config. -# 1) Set "gitlab_url" param in gitlab-shell/config.yml to https://{{ grains['fqdn'] }} -# 2) Set "ca_file" to /etc/nginx/gitlab.crt -# 3) Set "self_signed_cert" to true -# You also need to edit gitlab/config/gitlab.yml -# 1) Define port for http "port: 443" -# 2) Enable https "https: true" -# 3) Update ssl for gravatar "ssl_url: https://secure.gravatar.com/avatar/%{hash}?s=%{size}&d=mm" - -{% set root_dir = salt['pillar.get']('gitlab:lookup:root_dir', '/home/git') %} -{% set sockets_dir = salt['pillar.get']('gitlab:lookup:sockets_dir', root_dir ~ '/var/sockets') %} -{% set logs_dir = salt['pillar.get']('gitlab:lookup:logs_dir', root_dir ~ '/var/logs') %} - -upstream gitlab-workhorse { - - ## Uncomment if you have set up puma/unicorn to listen on a unix socket (recommended). - server unix:{{ sockets_dir }}/gitlab.socket; - - ## Uncomment if puma/unicorn are configured to listen on a tcp port. - ## Check the port number in {{ root_dir }}/gitlab/config/{puma.rb/unicorn.rb} - # server 127.0.0.1:9292; -} - -# This is a normal HTTP host which redirects all traffic to the HTTPS host. -# Replace {{ grains['fqdn'] }} with your FQDN. -server { - listen *:80; - server_name {{ salt['pillar.get']('gitlab:hostname', grains['fqdn']) }}; - server_tokens off; - root /nowhere; # this doesn't have to be a valid path since we are redirecting, you don't have to change it. - rewrite ^ https://$server_name$request_uri permanent; -} - -server { - listen 443 ssl; - server_name {{ salt['pillar.get']('gitlab:hostname', grains['fqdn']) }}; - server_tokens off; - root {{ root_dir }}/gitlab/public; - - ssl on; - ssl_certificate /etc/nginx/gitlab.crt; - ssl_certificate_key /etc/nginx/gitlab.key; - ssl_protocols SSLv3 TLSv1 TLSv1.2; - ssl_ciphers AES:HIGH:!ADH:!MD5; - ssl_prefer_server_ciphers on; - - # individual nginx logs for this gitlab vhost - access_log {{ logs_dir }}/nginx_access.log; - error_log {{ logs_dir }}/nginx_error.log; - - location / { - # serve static files from defined root folder;. - # @gitlab is a named location for the upstream fallback, see below - try_files $uri $uri/index.html $uri.html @gitlab; - } - - # if a file, which is not found in the root folder is requested, - # then the proxy pass the request to the upsteam (gitlab unicorn) - location @gitlab { - proxy_read_timeout 300; # https://github.com/gitlabhq/gitlabhq/issues/694 - proxy_connect_timeout 300; # https://github.com/gitlabhq/gitlabhq/issues/694 - proxy_redirect off; - - proxy_set_header X-Forwarded-Proto https; - proxy_set_header X-Forwarded-Ssl on; - proxy_set_header Host $http_host; - proxy_set_header X-Real-IP $remote_addr; - - proxy_pass http://gitlab; - } -} - diff --git a/gitlab/init.sls b/gitlab/init.sls index 692ffaf..09a244e 100644 --- a/gitlab/init.sls +++ b/gitlab/init.sls @@ -9,4 +9,3 @@ include: - gitlab.gitlab-shell - gitlab.gitlab-workhorse - gitlab.gitlab - - gitlab.nginx diff --git a/gitlab/nginx.sls b/gitlab/nginx.sls deleted file mode 100644 index 2c2c198..0000000 --- a/gitlab/nginx.sls +++ /dev/null @@ -1,81 +0,0 @@ -{% if grains['os_family'] == 'Debian' %} -{% set nginx_user = 'www-data' %} -{% set nginx_path = '/etc/nginx/sites-enabled' %} -{% elif grains['os_family'] == 'RedHat' %} -{% set nginx_user = 'nginx' %} -{% set nginx_path = '/etc/nginx/conf.d' %} -{% endif %} - -{% set nginx_conf = salt['pillar.get']('gitlab:lookup:nginx_conf', 'salt://gitlab/files/gitlab-nginx') %} -{% set nginx_ssl_conf = salt['pillar.get']('gitlab:lookup:nginx_ssl_conf', 'salt://gitlab/files/gitlab-nginx-ssl') %} - -nginx: - pkg.installed: [] - service.running: - - enable: True - - require: - - pkg: nginx - - user: nginx - - watch: - - file: gitlab-nginx - file.absent: - - name: {{ nginx_path }}/default.conf - user.present: - - name: {{ nginx_user }} - - groups: - - git - - require: - - pkg: nginx - -{%- if salt['pillar.get']('gitlab:https', false) %} - -# https://gitlab.com/gitlab-org/gitlab-recipes/blob/master/web-server/nginx/gitlab-ssl -gitlab-nginx: - file.managed: - - name: {{ nginx_path }}/gitlab.conf - - source: {{ nginx_ssl_conf }} - - template: jinja - - user: root - - group: root - - mode: 644 - - require: - - pkg: nginx - - file: nginx-ssl-key - - file: nginx-ssl-cert - -nginx-ssl-key: - file.managed: - - name: /etc/nginx/gitlab.key - - user: root - - group: {{ nginx_user }} - - mode: 640 - - contents_pillar: gitlab:ssl_key - - watch_in: - - service: nginx - -nginx-ssl-cert: - file.managed: - - name: /etc/nginx/gitlab.crt - - user: root - - group: {{ nginx_user }} - - mode: 644 - - contents_pillar: gitlab:ssl_cert - - watch_in: - - service: nginx - -{% else %} - -# https://gitlab.com/gitlab-org/gitlab-ce/blob/master/lib/support/nginx/gitlab -gitlab-nginx: - file.managed: - - name: {{ nginx_path }}/gitlab.conf - - source: {{ nginx_conf }} - - template: jinja - - user: root - - group: root - - mode: 644 - - require: - - pkg: nginx - -{% endif %} - From 43a47e09c574bbbb1c9712a1837cab7359697012 Mon Sep 17 00:00:00 2001 From: "Charles R. (hugo)" Date: Tue, 17 May 2016 13:18:48 +0200 Subject: [PATCH 18/96] Fix hooks path --- gitlab/files/gitlab-gitlab.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/gitlab/files/gitlab-gitlab.yml b/gitlab/files/gitlab-gitlab.yml index 8bfa9da..e5f237c 100644 --- a/gitlab/files/gitlab-gitlab.yml +++ b/gitlab/files/gitlab-gitlab.yml @@ -204,7 +204,8 @@ production: &base # REPOS_PATH MUST NOT BE A SYMLINK!!! repos_path: {{ repositories }}/ - hooks_path: {{ repositories }}/hooks/ + + hooks_path: {{ shell_dir_content }}/hooks/ # Git over HTTP upload_pack: true From bdd079f3f2677548ca4169b2dd3c7b94fbbc8ce2 Mon Sep 17 00:00:00 2001 From: "Charles R. (hugo)" Date: Tue, 17 May 2016 14:57:06 +0200 Subject: [PATCH 19/96] Add databases key (secrets.yml) --- gitlab/files/gitlab-secrets.yml | 8 ++++++++ gitlab/gitlab.sls | 15 +++++++++++++++ pillar.example | 1 + 3 files changed, 24 insertions(+) create mode 100644 gitlab/files/gitlab-secrets.yml diff --git a/gitlab/files/gitlab-secrets.yml b/gitlab/files/gitlab-secrets.yml new file mode 100644 index 0000000..db8811e --- /dev/null +++ b/gitlab/files/gitlab-secrets.yml @@ -0,0 +1,8 @@ + +{%- for name, infos in salt['pillar.get']('gitlab:databases', {}).items() %} +{% if 'key' in infos %} +{{ name }}: + db_key_base: {{ infos['key'] }} +{% endif %} + +{%- endfor %} diff --git a/gitlab/gitlab.sls b/gitlab/gitlab.sls index f2d3fa3..d7626a7 100644 --- a/gitlab/gitlab.sls +++ b/gitlab/gitlab.sls @@ -98,6 +98,21 @@ gitlab-db-config: {% endif %} - user: git-user +gitlab-db-secrets: + file.managed: + - name: {{ root_dir }}/gitlab/config/secrets.yml + - source: salt://gitlab/files/gitlab-secrets.yml + - template: jinja + - user: git + - group: git + - mode: 600 + - require: + {% if salt['pillar.get']('gitlab:archives:enabled', false) %} + - archive: gitlab-fetcher + {% else %} + - git: gitlab-fetcher + {% endif %} + # https://gitlab.com/gitlab-org/gitlab-ce/blob/master/config/unicorn.rb.example unicorn-config: file.managed: diff --git a/pillar.example b/pillar.example index 4a9a1af..baba449 100644 --- a/pillar.example +++ b/pillar.example @@ -37,6 +37,7 @@ gitlab: name: gitlab pool: 10 host: localhost + key: YOUR_KEY development: engine: postgresql name: gitlab From 9ea7ab55c363a4377f6257bce80ec364aa9e6791 Mon Sep 17 00:00:00 2001 From: Gilles Dartiguelongue Date: Mon, 6 Jun 2016 11:56:47 +0200 Subject: [PATCH 20/96] Drop redundant logrotate configuration Generates unwanted cron output: /etc/cron.daily/logrotate: error: gitlab:17 duplicate log entry for /opt/git/var/logs/gitlab-shell.log --- gitlab/files/gitlab-logrotate | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/gitlab/files/gitlab-logrotate b/gitlab/files/gitlab-logrotate index cfe3589..7771379 100644 --- a/gitlab/files/gitlab-logrotate +++ b/gitlab/files/gitlab-logrotate @@ -13,14 +13,3 @@ notifempty copytruncate } - -{{ logs_dir }}/gitlab-shell.log { - weekly - missingok - rotate 52 - compress - delaycompress - notifempty - copytruncate -} - From 77b842ccbd7c3e9f3c2a39860456f441121ffbcd Mon Sep 17 00:00:00 2001 From: "Charles R. (hugo)" Date: Mon, 20 Jun 2016 14:06:49 +0200 Subject: [PATCH 21/96] Add on changes on Workhorse make state --- gitlab/gitlab-workhorse.sls | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/gitlab/gitlab-workhorse.sls b/gitlab/gitlab-workhorse.sls index b12870b..f8000d1 100644 --- a/gitlab/gitlab-workhorse.sls +++ b/gitlab/gitlab-workhorse.sls @@ -50,12 +50,11 @@ gitlab-workhorse-make: - cwd: {{ workhorse_dir_content }} - name: make install PREFIX={{ root_dir }}/gitlab-workhorse - shell: /bin/bash - - require: + - onchanges: {% if salt['pillar.get']('gitlab:archives:enabled', false) %} - archive: gitlab-workhorse-fetcher - file: gitlab-workhorse-fetcher {% else %} - git: gitlab-workhorse-fetcher {% endif %} - - file: {{ root_dir }}/gitlab-workhorse From 10b09d983e938480be5592d8dcfec82a4dc4887b Mon Sep 17 00:00:00 2001 From: "Charles R. (hugo)" Date: Mon, 20 Jun 2016 14:54:29 +0200 Subject: [PATCH 22/96] Git stash only when gitlab fetcher method is git --- gitlab/gitlab.sls | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/gitlab/gitlab.sls b/gitlab/gitlab.sls index d7626a7..5d9ff92 100644 --- a/gitlab/gitlab.sls +++ b/gitlab/gitlab.sls @@ -267,6 +267,7 @@ gitlab-clear-cache: - require: - cmd: gitlab-recompile-assets +{% if not salt['pillar.get']('gitlab:archives:enabled', false) %} # Needed to be able to update tree via git gitlab-stash: cmd.wait: @@ -274,13 +275,10 @@ gitlab-stash: - cwd: {{ gitlab_dir }} - name: git stash - watch: - {% if salt['pillar.get']('gitlab:archives:enabled', false) %} - - archive: gitlab-fetcher - {% else %} - git: gitlab-fetcher - {% endif %} - require: - cmd: gitlab-clear-cache +{% endif %} # https://gitlab.com/gitlab-org/gitlab-ce/blob/master/lib/support/init.d/gitlab.default.example gitlab-default: From 6eb27adf41b8dd29bc18fa936c6b10abfebcc299 Mon Sep 17 00:00:00 2001 From: "Charles R. (hugo)" Date: Tue, 21 Jun 2016 11:07:59 +0200 Subject: [PATCH 23/96] Replace watch with onchanges --- gitlab/gitlab.sls | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/gitlab/gitlab.sls b/gitlab/gitlab.sls index 5d9ff92..8efe799 100644 --- a/gitlab/gitlab.sls +++ b/gitlab/gitlab.sls @@ -190,7 +190,7 @@ gitlab-gems: - cwd: {{ gitlab_dir }} - name: {{ proxy }} bundle install --deployment --without development test mysql aws kerberos - shell: /bin/bash - - watch: + - onchanges: {% if salt['pillar.get']('gitlab:archives:enabled', false) %} - archive: gitlab-fetcher {% else %} @@ -226,7 +226,7 @@ gitlab-migrate-db: - cwd: {{ gitlab_dir }} - name: bundle exec rake db:migrate RAILS_ENV=production - shell: /bin/bash - - watch: + - onchanges: {% if salt['pillar.get']('gitlab:archives:enabled', false) %} - archive: gitlab-fetcher {% else %} @@ -243,7 +243,7 @@ gitlab-recompile-assets: - cwd: {{ gitlab_dir }} - name: bundle exec rake assets:clean assets:precompile RAILS_ENV=production - shell: /bin/bash - - watch: + - onchanges: {% if salt['pillar.get']('gitlab:archives:enabled', false) %} - archive: gitlab-fetcher {% else %} @@ -258,7 +258,7 @@ gitlab-clear-cache: - cwd: {{ gitlab_dir }} - name: bundle exec rake cache:clear RAILS_ENV=production - shell: /bin/bash - - watch: + - onchanges: {% if salt['pillar.get']('gitlab:archives:enabled', false) %} - archive: gitlab-fetcher {% else %} From fd58d50eee9d81045233a7f5869c0483ccd8e7a2 Mon Sep 17 00:00:00 2001 From: "Charles R. (hugo)" Date: Wed, 22 Jun 2016 12:02:15 +0200 Subject: [PATCH 24/96] Change on dependances states --- gitlab/gitlab.sls | 33 ++++++--------------------------- 1 file changed, 6 insertions(+), 27 deletions(-) diff --git a/gitlab/gitlab.sls b/gitlab/gitlab.sls index 8efe799..9488515 100644 --- a/gitlab/gitlab.sls +++ b/gitlab/gitlab.sls @@ -221,7 +221,7 @@ gitlab-initialize: - file: gitlab-db-config gitlab-migrate-db: - cmd.wait: + cmd.run: - user: git - cwd: {{ gitlab_dir }} - name: bundle exec rake db:migrate RAILS_ENV=production @@ -233,30 +233,14 @@ gitlab-migrate-db: - git: gitlab-fetcher {% endif %} - require: - - cmd: gitlab-gems - - cmd: gitlab-initialize - file: gitlab-db-config + - cmd: gitlab-gems -gitlab-recompile-assets: - cmd.wait: - - user: git - - cwd: {{ gitlab_dir }} - - name: bundle exec rake assets:clean assets:precompile RAILS_ENV=production - - shell: /bin/bash - - onchanges: - {% if salt['pillar.get']('gitlab:archives:enabled', false) %} - - archive: gitlab-fetcher - {% else %} - - git: gitlab-fetcher - {% endif %} - - require: - - cmd: gitlab-migrate-db - -gitlab-clear-cache: - cmd.wait: +gitlab-recompile-assets-cache: + cmd.run: - user: git - cwd: {{ gitlab_dir }} - - name: bundle exec rake cache:clear RAILS_ENV=production + - name: bundle exec rake assets:clean assets:precompile cache:clear RAILS_ENV=production - shell: /bin/bash - onchanges: {% if salt['pillar.get']('gitlab:archives:enabled', false) %} @@ -264,8 +248,6 @@ gitlab-clear-cache: {% else %} - git: gitlab-fetcher {% endif %} - - require: - - cmd: gitlab-recompile-assets {% if not salt['pillar.get']('gitlab:archives:enabled', false) %} # Needed to be able to update tree via git @@ -274,10 +256,8 @@ gitlab-stash: - user: git - cwd: {{ gitlab_dir }} - name: git stash - - watch: + - onchanges: - git: gitlab-fetcher - - require: - - cmd: gitlab-clear-cache {% endif %} # https://gitlab.com/gitlab-org/gitlab-ce/blob/master/lib/support/init.d/gitlab.default.example @@ -347,7 +327,6 @@ gitlab-service: {% else %} - git: gitlab-fetcher {% endif %} - - cmd: gitlab-clear-cache - file: gitlab-config - file: gitlab-db-config - file: gitlab-default From f3bd1f601bcd6d2db8f89a999d113d6340c9b339 Mon Sep 17 00:00:00 2001 From: "Charles R. (hugo)" Date: Wed, 22 Jun 2016 14:54:04 +0200 Subject: [PATCH 25/96] Chown only if fetcher state change --- gitlab/gitlab.sls | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/gitlab/gitlab.sls b/gitlab/gitlab.sls index 9488515..a556d67 100644 --- a/gitlab/gitlab.sls +++ b/gitlab/gitlab.sls @@ -35,12 +35,16 @@ gitlab-fetcher: - archive_format: tar - if_missing: {{ gitlab_dir_content }} - keep: True + +gitlab-chown: file.directory: - name: {{ gitlab_dir_content }} - user: git - group: git - recurse: - user + - onchanges: + - archive: gitlab-fetcher gitlab-lib-symlink: file.symlink: From 7c7f0fadf887675029c7daa648f33e2882c56c1c Mon Sep 17 00:00:00 2001 From: "Charles R. (hugo)" Date: Thu, 23 Jun 2016 11:17:54 +0200 Subject: [PATCH 26/96] Modification on uploads path --- gitlab/gitlab.sls | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/gitlab/gitlab.sls b/gitlab/gitlab.sls index a556d67..d68ecee 100644 --- a/gitlab/gitlab.sls +++ b/gitlab/gitlab.sls @@ -292,15 +292,10 @@ gitlab-respositories-dir: - file_mode: 0660 - dir_mode: 2770 -gitlab-uploads-dir: - file.directory: - - name: {{ gitlab_dir }}/public/uploads - - dir_mode: 0700 - gitlab-uploads-symlink: file.symlink: - - name: {{ uploads_dir }} - - target: {{ gitlab_dir }}/public/uploads + - name: {{ gitlab_dir }}/public/uploads + - target: {{ uploads_dir }} - require: - file: git-var-mkdir From 1c8a77e0b49b255cdf145e04cb62f2db29552bc6 Mon Sep 17 00:00:00 2001 From: "Charles R. (hugo)" Date: Thu, 23 Jun 2016 15:28:52 +0200 Subject: [PATCH 27/96] Create gitlab-shell link --- gitlab/files/gitlab-gitlab.yml | 2 +- gitlab/gitlab-shell.sls | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/gitlab/files/gitlab-gitlab.yml b/gitlab/files/gitlab-gitlab.yml index e5f237c..ba84008 100644 --- a/gitlab/files/gitlab-gitlab.yml +++ b/gitlab/files/gitlab-gitlab.yml @@ -200,7 +200,7 @@ production: &base ## GitLab Shell settings gitlab_shell: - path: {{ shell_dir_content }} + path: {{ root_dir }}/gitlab-shell # REPOS_PATH MUST NOT BE A SYMLINK!!! repos_path: {{ repositories }}/ diff --git a/gitlab/gitlab-shell.sls b/gitlab/gitlab-shell.sls index d6ad9f1..8936d27 100644 --- a/gitlab/gitlab-shell.sls +++ b/gitlab/gitlab-shell.sls @@ -41,6 +41,13 @@ gitlab-shell-fetcher: - file: git-home {% endif %} +gitlab-shell-symlink: + file.symlink: + - name: {{ root_dir }}/gitlab-shell + - target: {{ shell_dir_content }} + - require: + - file: git-var-mkdir + # https://gitlab.com/gitlab-org/gitlab-shell/blob/master/config.yml.example gitlab-shell-config: file.managed: From e84309529a5207f37c37c8d420abe0752d10a013 Mon Sep 17 00:00:00 2001 From: "Charles R. (hugo)" Date: Thu, 23 Jun 2016 16:48:57 +0200 Subject: [PATCH 28/96] Enhance speed --- gitlab/gitlab-shell.sls | 8 +++++++- gitlab/gitlab-workhorse.sls | 9 +++++++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/gitlab/gitlab-shell.sls b/gitlab/gitlab-shell.sls index 8936d27..e5d5e75 100644 --- a/gitlab/gitlab-shell.sls +++ b/gitlab/gitlab-shell.sls @@ -13,8 +13,8 @@ include: {% set shell_dir_content = shell_dir %} {% endif %} -gitlab-shell-fetcher: {% if salt['pillar.get']('gitlab:archives:enabled', false) %} +gitlab-shell-fetcher: archive.extracted: - name: {{ shell_dir }} - source: {{ salt['pillar.get']('gitlab:archives:sources:shell:source') }} @@ -22,12 +22,18 @@ gitlab-shell-fetcher: - archive_format: tar - if_missing: {{ shell_dir_content }} - keep: True + +gitlab-shell-chown: file.directory: - name: {{ shell_dir }} - user: git + - group: git - recurse: - user + - onchanges: + - archive: gitlab-shell-fetcher {% else %} +gitlab-shell-fetcher: git.latest: - name: https://gitlab.com/gitlab-org/gitlab-shell.git - rev: {{ salt['pillar.get']('gitlab:shell_version') }} diff --git a/gitlab/gitlab-workhorse.sls b/gitlab/gitlab-workhorse.sls index f8000d1..6be988f 100644 --- a/gitlab/gitlab-workhorse.sls +++ b/gitlab/gitlab-workhorse.sls @@ -10,8 +10,8 @@ {% set workhorse_dir_content = workhorse_dir %} {% endif %} -gitlab-workhorse-fetcher: {% if salt['pillar.get']('gitlab:archives:enabled', false) %} +gitlab-workhorse-fetcher: archive.extracted: - name: {{ workhorse_dir }} - source: {{ salt['pillar.get']('gitlab:archives:sources:workhorse:source') }} @@ -19,12 +19,18 @@ gitlab-workhorse-fetcher: - archive_format: tar - if_missing: {{ workhorse_dir_content }} - keep: True + +gitlab-workhorse-chown: file.directory: - name: {{ workhorse_dir }} - user: git + - group: git - recurse: - user + - onchanges: + - archive: gitlab-workhorse-fetcher {% else %} +gitlab-workhorse-fetcher: git.latest: - name: https://gitlab.com/gitlab-org/gitlab-workhorse.git - rev: {{ salt['pillar.get']('gitlab:workhorse_version') }} @@ -53,7 +59,6 @@ gitlab-workhorse-make: - onchanges: {% if salt['pillar.get']('gitlab:archives:enabled', false) %} - archive: gitlab-workhorse-fetcher - - file: gitlab-workhorse-fetcher {% else %} - git: gitlab-workhorse-fetcher {% endif %} From e624a6889b9606eff1d5166e0f15d0e258d1f13c Mon Sep 17 00:00:00 2001 From: "Charles R. (hugo)" Date: Thu, 23 Jun 2016 17:55:57 +0200 Subject: [PATCH 29/96] No symlink in Shell installation path --- gitlab/gitlab-shell.sls | 38 +++++++++++++++++++++++++++++++------- 1 file changed, 31 insertions(+), 7 deletions(-) diff --git a/gitlab/gitlab-shell.sls b/gitlab/gitlab-shell.sls index e5d5e75..c82efec 100644 --- a/gitlab/gitlab-shell.sls +++ b/gitlab/gitlab-shell.sls @@ -47,13 +47,6 @@ gitlab-shell-fetcher: - file: git-home {% endif %} -gitlab-shell-symlink: - file.symlink: - - name: {{ root_dir }}/gitlab-shell - - target: {{ shell_dir_content }} - - require: - - file: git-var-mkdir - # https://gitlab.com/gitlab-org/gitlab-shell/blob/master/config.yml.example gitlab-shell-config: file.managed: @@ -92,3 +85,34 @@ gitlab-shell: # - recurse: # - mode + +{% if salt['pillar.get']('gitlab:archives:enabled', false) %} +{# + Symlink is not good because Shell run 'File.expand_path' on + Shell installation path and convert it to absolute version... +#} + +{# +gitlab-shell-symlink: + file.symlink: + - name: {{ root_dir }}/gitlab-shell + - target: {{ shell_dir_content }} + - require: + - file: git-var-mkdir +#} + +gitlab-shell-mkdir: + file.directory: + - name: {{ root_dir }}/gitlab-shell + - user: git + - group: git + +gitlab-shell-copy: + cmd.run: + - user: git + - cwd: {{ shell_dir_content }} + - name: cp -r {{ shell_dir_content }}/* {{ root_dir }}/gitlab-shell/ + - shell: /bin/bash + - onchanges: + - archive: gitlab-shell-fetcher +{% endif %} From 8c0132460a1a8fd6ee68e9cfd7b686c7d3af172e Mon Sep 17 00:00:00 2001 From: "Charles R. (hugo)" Date: Tue, 2 Aug 2016 10:25:38 +0200 Subject: [PATCH 30/96] Load git sls only if archive mode is not enabled --- gitlab/init.sls | 3 +++ gitlab/packages.sls | 2 -- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/gitlab/init.sls b/gitlab/init.sls index 09a244e..d93465a 100644 --- a/gitlab/init.sls +++ b/gitlab/init.sls @@ -2,6 +2,9 @@ include: {% if grains['os_family'] == 'RedHat' %} - gitlab.repos {% endif %} + {% if not salt['pillar.get']('gitlab:archives:enabled', false) %} + - gitlab.git + {% endif %} - gitlab.packages - redis - gitlab.user diff --git a/gitlab/packages.sls b/gitlab/packages.sls index 68273bf..2581974 100644 --- a/gitlab/packages.sls +++ b/gitlab/packages.sls @@ -1,5 +1,3 @@ -include: - - git gitlab-deps: pkg.installed: From 3fdcdb2a9ddbe356ad716dc8f6f0aa9ecbd48c9f Mon Sep 17 00:00:00 2001 From: "Charles R. (hugo)" Date: Tue, 2 Aug 2016 14:15:14 +0200 Subject: [PATCH 31/96] Upload directory creation --- gitlab/gitlab.sls | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/gitlab/gitlab.sls b/gitlab/gitlab.sls index d68ecee..68e78d9 100644 --- a/gitlab/gitlab.sls +++ b/gitlab/gitlab.sls @@ -169,7 +169,7 @@ git-var-mkdir: - mode: 750 # pids_dir -{% for dir in [ sockets_dir, logs_dir ] %} +{% for dir in [ sockets_dir, logs_dir, uploads_dir ] %} git-{{ dir }}-mkdir: file.directory: - name: {{ dir }} @@ -178,6 +178,11 @@ git-{{ dir }}-mkdir: - mode: 750 {% endfor %} +gitlab-uploads_dir-mode: + file.directory: + - name: {{ uploads_dir }} + - mode: 700 + # Hardcoded in gitlab, so, we have to create symlink gitlab-pids_dir-symlink: file.symlink: From f23b1f5136a812f639c10b27ab03f65dcb5497ef Mon Sep 17 00:00:00 2001 From: "Charles R. (hugo)" Date: Mon, 7 Nov 2016 14:40:31 +0100 Subject: [PATCH 32/96] Add secret key configuration (link between gitlab and gitlab-shell) --- README.rst | 1 + gitlab/files/gitlab-gitlab.yml | 4 ++++ gitlab/files/gitlab-shell-config.yml | 4 ++++ gitlab/gitlab.sls | 8 ++++++++ pillar.example | 2 ++ 5 files changed, 19 insertions(+) diff --git a/README.rst b/README.rst index b790a7c..69c2d0e 100644 --- a/README.rst +++ b/README.rst @@ -8,6 +8,7 @@ Modification from original formula : * Original initd script from Gitlab setup * Proxy for git.latest / gem * Install package from archive +* Configuration keys for secret_key and secret_file (gitlab_shell_secret) * Lot of little things... SaltStack formula to install GitLab diff --git a/gitlab/files/gitlab-gitlab.yml b/gitlab/files/gitlab-gitlab.yml index ba84008..bc50ec3 100644 --- a/gitlab/files/gitlab-gitlab.yml +++ b/gitlab/files/gitlab-gitlab.yml @@ -207,6 +207,10 @@ production: &base hooks_path: {{ shell_dir_content }}/hooks/ + # File that contains the secret key for verifying access for gitlab-shell. + # Default is '.gitlab_shell_secret' relative to Rails.root (i.e. root of the GitLab app). + secret_file: "{{ salt['pillar.get']('gitlab:secret_file', '/opt/git/.gitlab_shell_secret') }}" + # Git over HTTP upload_pack: true receive_pack: true diff --git a/gitlab/files/gitlab-shell-config.yml b/gitlab/files/gitlab-shell-config.yml index 7cdbeaa..87237f4 100644 --- a/gitlab/files/gitlab-shell-config.yml +++ b/gitlab/files/gitlab-shell-config.yml @@ -32,6 +32,10 @@ repos_path: "{{ repo_dir }}" # File used as authorized_keys for gitlab user auth_file: "{{ root_dir }}/.ssh/authorized_keys" +# File that contains the secret key for verifying access to GitLab. +# Default is .gitlab_shell_secret in the root directory. +secret_file: "{{ salt['pillar.get']('gitlab:secret_file', '/opt/git/.gitlab_shell_secret') }}" + # Redis settings used for pushing commit notices to gitlab redis: bin: /usr/bin/redis-cli diff --git a/gitlab/gitlab.sls b/gitlab/gitlab.sls index 68e78d9..540daa1 100644 --- a/gitlab/gitlab.sls +++ b/gitlab/gitlab.sls @@ -336,3 +336,11 @@ gitlab-service: - file: gitlab-default - file: rack_attack-config - file: unicorn-config + +gitlab-secret_file: + file.managed: + - name: {{ salt['pillar.get']('gitlab:secret_file', '/opt/git/.gitlab_shell_secret') }} + - user: git + - group: git + - mode: 640 + - contents_pillar: gitlab:secret_key diff --git a/pillar.example b/pillar.example index baba449..6d458c7 100644 --- a/pillar.example +++ b/pillar.example @@ -31,6 +31,8 @@ gitlab: shell_version: v2.7.2 gitlab_version: 8-7-stable workhorse_version: v0.7.1 + secret_key: 0123456789abcdef0123456789abcde + secret_file: "/opt/git/.gitlab_shell_secret" databases: production: &production engine: postgresql From bb481b60b76efe40ce5828598bf4a2d84a40f576 Mon Sep 17 00:00:00 2001 From: "Charles R. (hugo)" Date: Mon, 5 Dec 2016 11:06:34 +0100 Subject: [PATCH 33/96] Fix: Create and clean gitlab-shell directory only when new archive fetched --- gitlab/gitlab-shell.sls | 3 +++ 1 file changed, 3 insertions(+) diff --git a/gitlab/gitlab-shell.sls b/gitlab/gitlab-shell.sls index c82efec..856f0a8 100644 --- a/gitlab/gitlab-shell.sls +++ b/gitlab/gitlab-shell.sls @@ -106,6 +106,9 @@ gitlab-shell-mkdir: - name: {{ root_dir }}/gitlab-shell - user: git - group: git + - clean: true + - onchanges: + - archive: gitlab-shell-fetcher gitlab-shell-copy: cmd.run: From f7cca7e29dd944d2d70ed942b5988369b35c6e51 Mon Sep 17 00:00:00 2001 From: Gilles Dartiguelongue Date: Thu, 31 Aug 2017 16:01:17 +0200 Subject: [PATCH 34/96] Update git configuration for GitLab 8.17 --- gitlab/files/gitlab-gitconfig | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/gitlab/files/gitlab-gitconfig b/gitlab/files/gitlab-gitconfig index 7c25e0c..c43520d 100644 --- a/gitlab/files/gitlab-gitconfig +++ b/gitlab/files/gitlab-gitconfig @@ -3,4 +3,5 @@ email = gitlab@{{ salt['pillar.get']('gitlab:hostname', grains['fqdn']) }} [core] autocrlf = input - +[repack] + writeBitmaps = true From 189ede0cd042a63ae69e767f2f74f97cc6b90d18 Mon Sep 17 00:00:00 2001 From: Gilles Dartiguelongue Date: Thu, 31 Aug 2017 15:21:31 +0200 Subject: [PATCH 35/96] Move shell secret handling to gitlab:shell Reuse existing structure to make room for gitlab secrets. --- gitlab/files/gitlab-shell-config.yml | 2 +- gitlab/gitlab.sls | 4 ++-- pillar.example | 5 +++-- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/gitlab/files/gitlab-shell-config.yml b/gitlab/files/gitlab-shell-config.yml index 87237f4..1719658 100644 --- a/gitlab/files/gitlab-shell-config.yml +++ b/gitlab/files/gitlab-shell-config.yml @@ -34,7 +34,7 @@ auth_file: "{{ root_dir }}/.ssh/authorized_keys" # File that contains the secret key for verifying access to GitLab. # Default is .gitlab_shell_secret in the root directory. -secret_file: "{{ salt['pillar.get']('gitlab:secret_file', '/opt/git/.gitlab_shell_secret') }}" +secret_file: "{{ salt['pillar.get']('gitlab:shell:secret:path', '/opt/git/.gitlab_shell_secret') }}" # Redis settings used for pushing commit notices to gitlab redis: diff --git a/gitlab/gitlab.sls b/gitlab/gitlab.sls index 540daa1..a6ce458 100644 --- a/gitlab/gitlab.sls +++ b/gitlab/gitlab.sls @@ -339,8 +339,8 @@ gitlab-service: gitlab-secret_file: file.managed: - - name: {{ salt['pillar.get']('gitlab:secret_file', '/opt/git/.gitlab_shell_secret') }} + - name: {{ salt['pillar.get']('gitlab:shell:secret:path', '/opt/git/.gitlab_shell_secret') }} + - contents_pillar: gitlab:shell:secret:value - user: git - group: git - mode: 640 - - contents_pillar: gitlab:secret_key diff --git a/pillar.example b/pillar.example index 6d458c7..9048597 100644 --- a/pillar.example +++ b/pillar.example @@ -31,8 +31,6 @@ gitlab: shell_version: v2.7.2 gitlab_version: 8-7-stable workhorse_version: v0.7.1 - secret_key: 0123456789abcdef0123456789abcde - secret_file: "/opt/git/.gitlab_shell_secret" databases: production: &production engine: postgresql @@ -82,6 +80,9 @@ gitlab: #{% elif grains['os_family'] == 'Debian' %} #ca_path: /etc/ssl/certs #{% endif %} + secret: + value: 0123456789abcdef0123456789abcde + path: "/opt/git/.gitlab_shell_secret" unicorn: worker_processes: 2 From b9ce8d58753bebbbd2e102adc7bef7858f6baf5d Mon Sep 17 00:00:00 2001 From: Gilles Dartiguelongue Date: Thu, 31 Aug 2017 15:29:34 +0200 Subject: [PATCH 36/96] Add extensible support for new database secrets The keys added to pillar.example are the one currently missing. Newer releases of GitLab appear to require even more. --- gitlab/files/gitlab-secrets.yml | 11 ++++++----- pillar.example | 5 ++++- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/gitlab/files/gitlab-secrets.yml b/gitlab/files/gitlab-secrets.yml index db8811e..cd77006 100644 --- a/gitlab/files/gitlab-secrets.yml +++ b/gitlab/files/gitlab-secrets.yml @@ -1,8 +1,9 @@ - +--- {%- for name, infos in salt['pillar.get']('gitlab:databases', {}).items() %} -{% if 'key' in infos %} +{%- if 'secrets' in infos %} {{ name }}: - db_key_base: {{ infos['key'] }} -{% endif %} - + {%- for key, value in infos['secrets'].items() %} + {{ key }}: {{ value|yaml_dquote }} + {%- endfor %} +{%- endif %} {%- endfor %} diff --git a/pillar.example b/pillar.example index 9048597..988bf18 100644 --- a/pillar.example +++ b/pillar.example @@ -37,7 +37,10 @@ gitlab: name: gitlab pool: 10 host: localhost - key: YOUR_KEY + secrets: + db_key_base: + secret_key_base: + otp_key_base: development: engine: postgresql name: gitlab From 7cd3483a76cc109c34686b79ca8fc7ca5c406a86 Mon Sep 17 00:00:00 2001 From: Gilles Dartiguelongue Date: Thu, 31 Aug 2017 16:32:25 +0200 Subject: [PATCH 37/96] Add new installation step for GitLab 8.17+ --- gitlab/gitlab.sls | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/gitlab/gitlab.sls b/gitlab/gitlab.sls index 540daa1..bf72d3c 100644 --- a/gitlab/gitlab.sls +++ b/gitlab/gitlab.sls @@ -245,6 +245,30 @@ gitlab-migrate-db: - file: gitlab-db-config - cmd: gitlab-gems +gitlab-npm-install: + {# npm is stubborn and will not respect proxy settings and git insteadOf #} + file.replace: + - name: {{ gitlab_dir }}/package.json + - pattern: '"jquery-ui": "github:.*#1.11.4",' + - repl: '"jquery-ui": "https://github.com/jquery/jquery-ui/archive/1.11.4.tar.gz",' + cmd.run: + - name: npm install --production + - user: git + - cwd: {{ gitlab_dir }} + {%- if salt['pillar.get']('gitlab:proxy:address') %} + - env: + - HTTP_PROXY: {{ pillar.gitlab.proxy.address }} + - HTTPS_PROXY: {{ pillar.gitlab.proxy.address }} + {%- endif %} + - onchanges: + {% if salt['pillar.get']('gitlab:archives:enabled', false) %} + - archive: gitlab-fetcher + {% else %} + - git: gitlab-fetcher + {% endif %} + - require: + - cmd: gitlab-migrate-db + gitlab-recompile-assets-cache: cmd.run: - user: git @@ -257,6 +281,8 @@ gitlab-recompile-assets-cache: {% else %} - git: gitlab-fetcher {% endif %} + - require: + - cmd: gitlab-npm-install {% if not salt['pillar.get']('gitlab:archives:enabled', false) %} # Needed to be able to update tree via git From 52ba8134a864f41bff63bbcdfa1bc47682c05181 Mon Sep 17 00:00:00 2001 From: Gilles Dartiguelongue Date: Thu, 31 Aug 2017 17:40:28 +0200 Subject: [PATCH 38/96] Synchronize configuration file with gitlab 8.17 branch --- gitlab/files/gitlab-gitlab.yml | 550 ++++++++++++++++++++++++++------- 1 file changed, 441 insertions(+), 109 deletions(-) diff --git a/gitlab/files/gitlab-gitlab.yml b/gitlab/files/gitlab-gitlab.yml index bc50ec3..f4b6d46 100644 --- a/gitlab/files/gitlab-gitlab.yml +++ b/gitlab/files/gitlab-gitlab.yml @@ -2,10 +2,24 @@ # GitLab application config file # # # # # # # # # # # # # # # # # # # # +########################### NOTE ##################################### +# This file should not receive new settings. All configuration options # +# * are being moved to ApplicationSetting model! # +# If a setting requires an application restart say so in that screen. # +# If you change this file in a Merge Request, please also create # +# a MR on https://gitlab.com/gitlab-org/omnibus-gitlab/merge_requests # +######################################################################## +# +# # How to use: -# 1. copy file as gitlab.yml -# 2. Replace gitlab -> host with your domain -# 3. Replace gitlab -> email_from +# 1. Copy file as gitlab.yml +# 2. Update gitlab -> host with your fully qualified domain name +# 3. Update gitlab -> email_from +# 4. If you installed Git from source, change git -> bin_path to /usr/local/bin/git +# IMPORTANT: If Git was installed in a different location use that instead. +# You can check with `which git`. If a wrong path of Git is specified, it will +# result in various issues such as failures of GitLab CI builds. +# 5. Review this configuration file for other settings you may want to adjust {% set root_dir = salt['pillar.get']('gitlab:lookup:root_dir', '/home/git') %} {% set lib_dir = salt['pillar.get']('gitlab:lookup:lib_dir', root_dir ~ '/libraries') %} @@ -35,100 +49,203 @@ production: &base https: false {% endif %} - # Uncomment and customize the last line to run in a non-root path - # WARNING: We recommend creating a FQDN to host GitLab in a root path instead of this. - # Note that four settings need to be changed for this to work. - # 1) In your application.rb file: config.relative_url_root = "/gitlab" - # 2) In your gitlab.yml file: relative_url_root: /gitlab - # 3) In your unicorn.rb: ENV['RAILS_RELATIVE_URL_ROOT'] = "/gitlab" - # 4) In ../gitlab-shell/config.yml: gitlab_url: "http://127.0.0.1/gitlab" - # To update the path, run: sudo -u git -H bundle exec rake assets:precompile RAILS_ENV=production + # Uncommment this line below if your ssh host is different from HTTP/HTTPS one + # (you'd obviously need to replace ssh.host_example.com with your own host). + # Otherwise, ssh host will be set to the `host:` value above + # ssh_host: ssh.host_example.com + + # Relative URL support + # WARNING: We recommend using an FQDN to host GitLab in a root path instead + # of using a relative URL. + # Documentation: http://doc.gitlab.com/ce/install/relative_url.html + # Uncomment and customize the following line to run in a non-root path # # relative_url_root: /gitlab + # Trusted Proxies + # Customize if you have GitLab behind a reverse proxy which is running on a different machine. + # Add the IP address for your reverse proxy to the list, otherwise users will appear signed in from that address. + trusted_proxies: + # Examples: + #- 192.168.1.0/24 + #- 192.168.2.1 + #- 2001:0db8::/32 + # Uncomment and customize if you can't use the default user to run GitLab (default: 'git') # user: git + ## Date & Time settings + # Uncomment and customize if you want to change the default time zone of GitLab application. + # To see all available zones, run `bundle exec rake time:zones:all RAILS_ENV=production` + # time_zone: 'UTC' + ## Email settings + # Uncomment and set to false if you need to disable email sending from GitLab (default: true) + # email_enabled: true # Email address used in the "From" field in mails sent by GitLab email_from: gitlab@{{ salt['pillar.get']('gitlab:hostname', grains['fqdn']) }} + email_display_name: GitLab + email_reply_to: noreply@{{ salt['pillar.get']('gitlab:hostname', grains['fqdn']) }} + email_subject_suffix: '' - # Email address of your support contact (default: same as email_from) - support_email: support@{{ salt['pillar.get']('gitlab:hostname', grains['fqdn']) }} + # Email server smtp settings are in config/initializers/smtp_settings.rb.sample - ## User settings - default_projects_limit: 10 # default_can_create_group: false # default: true # username_changing_enabled: false # default: true - User can change her username/namespace - ## Default theme - ## BASIC = 1 - ## MARS = 2 - ## MODERN = 3 - ## GRAY = 4 - ## COLOR = 5 + ## Default theme ID + ## 1 - Graphite + ## 2 - Charcoal + ## 3 - Green + ## 4 - Gray + ## 5 - Violet + ## 6 - Blue # default_theme: 2 # default: 2 - - ## Users management - # default: false - Account passwords are not sent via the email if signup is enabled. - # signup_enabled: true - - # Restrict setting visibility levels for non-admin users. - # The default is to allow all levels. - #restricted_visibility_levels: [ "public" ] - ## Automatic issue closing # If a commit message matches this regular expression, all issues referenced from the matched text will be closed. # This happens when the commit is pushed or merged into the default branch of a project. # When not specified the default issue_closing_pattern as specified below will be used. - # issue_closing_pattern: '([Cc]lose[sd]|[Ff]ixe[sd]) +#\d+' + # Tip: you can test your closing pattern at http://rubular.com. + # issue_closing_pattern: '((?:[Cc]los(?:e[sd]?|ing)|[Ff]ix(?:e[sd]|ing)?|[Rr]esolv(?:e[sd]?|ing))(:?) +(?:(?:issues? +)?%{issue_ref}(?:(?:, *| +and +)?)|([A-Z][A-Z0-9_]+-\d+))+)' ## Default project features settings default_projects_features: issues: true merge_requests: true wiki: true - wall: false snippets: false - visibility_level: "private" # can be "private" | "internal" | "public" + builds: true + container_registry: true + + ## Webhook settings + # Number of seconds to wait for HTTP response after sending webhook HTTP POST request (default: 10) + # webhook_timeout: 10 + + ## Repository downloads directory + # When a user clicks e.g. 'Download zip' on a project, a temporary zip file is created in the following directory. + # The default is 'shared/cache/archive/' relative to the root of the Rails app. + # repository_downloads_path: shared/cache/archive/ + + ## Reply by email + # Allow users to comment on issues and merge requests by replying to notification emails. + # For documentation on how to set this up, see http://doc.gitlab.com/ce/administration/reply_by_email.html + incoming_email: + enabled: false + + # The email address including the `%{key}` placeholder that will be replaced to reference the item being replied to. + # The placeholder can be omitted but if present, it must appear in the "user" part of the address (before the `@`). + address: "gitlab-incoming+%{key}@gmail.com" + + # Email account username + # With third party providers, this is usually the full email address. + # With self-hosted email servers, this is usually the user part of the email address. + user: "gitlab-incoming@gmail.com" + # Email account password + password: "[REDACTED]" + + # IMAP server host + host: "imap.gmail.com" + # IMAP server port + port: 993 + # Whether the IMAP server uses SSL + ssl: true + # Whether the IMAP server uses StartTLS + start_tls: false + + # The mailbox where incoming mail will end up. Usually "inbox". + mailbox: "inbox" + # The IDLE command timeout. + idle_timeout: 60 + + ## Build Artifacts + artifacts: + enabled: true + # The location where build artifacts are stored (default: shared/artifacts). + # path: shared/artifacts + + ## Git LFS + lfs: + enabled: true + # The location where LFS objects are stored (default: shared/lfs-objects). + # storage_path: shared/lfs-objects + + ## GitLab Pages + pages: + enabled: false + # The location where pages are stored (default: shared/pages). + # path: shared/pages + + # The domain under which the pages are served: + # http://group.example.com/project + # or project path can be a group page: group.example.com + host: example.com + port: 80 # Set to 443 if you serve the pages with HTTPS + https: false # Set to true if you serve the pages with HTTPS + # external_http: "1.1.1.1:80" # If defined, enables custom domain support in GitLab Pages + # external_https: "1.1.1.1:443" # If defined, enables custom domain and certificate support in GitLab Pages + + ## Mattermost + ## For enabling Add to Mattermost button + mattermost: + enabled: false + host: 'https://mattermost.example.com' - ## External issues trackers - issues_tracker: - # redmine: - # title: "Redmine" - # ## If not nil, link 'Issues' on project page will be replaced with this - # ## Use placeholders: - # ## :project_id - GitLab project identifier - # ## :issues_tracker_id - Project Name or Id in external issue tracker - # project_url: "http://redmine.sample/projects/:issues_tracker_id" + ## Gravatar + ## For Libravatar see: http://doc.gitlab.com/ce/customization/libravatar.html + gravatar: + # gravatar urls: possible placeholders: %{hash} %{size} %{email} + # plain_url: "http://..." # default: http://www.gravatar.com/avatar/%{hash}?s=%{size}&d=identicon + # ssl_url: "https://..." # default: https://secure.gravatar.com/avatar/%{hash}?s=%{size}&d=identicon + + ## Auxiliary jobs + # Periodically executed jobs, to self-heal Gitlab, do external synchronizations, etc. + # Please read here for more information: https://github.com/ondrejbartas/sidekiq-cron#adding-cron-job + cron_jobs: + # Flag stuck CI builds as failed + stuck_ci_builds_worker: + cron: "0 0 * * *" + # Remove expired build artifacts + expire_build_artifacts_worker: + cron: "50 * * * *" + # Periodically run 'git fsck' on all repositories. If started more than + # once per hour you will have concurrent 'git fsck' jobs. + repository_check_worker: + cron: "20 * * * *" + # Send admin emails once a week + admin_email_worker: + cron: "0 0 * * 0" + + # Remove outdated repository archives + repository_archive_cache_worker: + cron: "0 * * * *" + + registry: + # enabled: true + # host: registry.example.com + # port: 5005 + # api_url: http://localhost:5000/ # internal address to the registry, will be used by GitLab to directly communicate with API + # key: config/registry.key + # path: shared/registry + # issuer: gitlab-issuer + + # + # 2. GitLab CI settings + # ========================== + + gitlab_ci: + # Default project notifications settings: # - # ## If not nil, links from /#\d/ entities from commit messages will replaced with this - # ## Use placeholders: - # ## :project_id - GitLab project identifier - # ## :issues_tracker_id - Project Name or Id in external issue tracker - # ## :id - Issue id (from commit messages) - # issues_url: "http://redmine.sample/issues/:id" + # Send emails only on broken builds (default: true) + # all_broken_builds: true # - # ## If not nil, linkis to creating new issues will be replaced with this - # ## Use placeholders: - # ## :project_id - GitLab project identifier - # ## :issues_tracker_id - Project Name or Id in external issue tracker - # new_issue_url: "http://redmine.sample/projects/:issues_tracker_id/issues/new" - # - # jira: - # title: "Atlassian Jira" - # project_url: "http://jira.sample/issues/?jql=project=:issues_tracker_id" - # issues_url: "http://jira.sample/browse/:id" - # new_issue_url: "http://jira.sample/secure/CreateIssue.jspa" + # Add pusher to recipients list (default: false) + # add_pusher: true - ## Gravatar - gravatar: - enabled: {{ salt['pillar.get']('gitlab:gravatar:enabled', true) }} # Use user avatar image from Gravatar.com (default: true) - # plain_url: "http://..." # default: http://www.gravatar.com/avatar/%{hash}?s=%{size}&d=mm - # ssl_url: "https://..." # default: https://secure.gravatar.com/avatar/%{hash}?s=%{size}&d=mm + # The location where build traces are stored (default: builds/). Relative paths are relative to Rails.root + # builds_path: builds/ # - # 2. Auth settings + # 3. Auth settings # ========================== ## LDAP settings @@ -136,75 +253,264 @@ production: &base # bundle exec rake gitlab:ldap:check RAILS_ENV=production ldap: enabled: {{ salt['pillar.get']('gitlab:ldap:enabled', false) }} - host: '{{ salt['pillar.get']('gitlab:ldap:host', '') }}' - base: '{{ salt['pillar.get']('gitlab:ldap:base', '') }}' - port: {{ salt['pillar.get']('gitlab:ldap:port', 636) }} - uid: '{{ salt['pillar.get']('gitlab:ldap:uid', 'sAMAccountName') }}' - method: '{{ salt['pillar.get']('gitlab:ldap:method', 'ssl') }}' # "ssl" or "plain" - bind_dn: '{{ salt['pillar.get']('gitlab:ldap:bind_dn', '') }}' - password: '{{ salt['pillar.get']('gitlab:ldap:password', '') }}' - # If allow_username_or_email_login is enabled, GitLab will ignore everything - # after the first '@' in the LDAP username submitted by the user on login. - # - # Example: - # - the user enters 'jane.doe@example.com' and 'p@ssw0rd' as LDAP credentials; - # - GitLab queries the LDAP server with 'jane.doe' and 'p@ssw0rd'. - # - # If you are using "uid: 'userPrincipalName'" on ActiveDirectory you need to - # disable this setting, because the userPrincipalName contains an '@'. - allow_username_or_email_login: '{{ salt['pillar.get']('gitlab:ldap:allow_username_or_email_login', true) }}' + servers: + ########################################################################## + # + # Since GitLab 7.4, LDAP servers get ID's (below the ID is 'main'). GitLab + # Enterprise Edition now supports connecting to multiple LDAP servers. + # + # If you are updating from the old (pre-7.4) syntax, you MUST give your + # old server the ID 'main'. + # + ########################################################################## + main: # 'main' is the GitLab 'provider ID' of this LDAP server + ## label + # + # A human-friendly name for your LDAP server. It is OK to change the label later, + # for instance if you find out it is too large to fit on the web page. + # + # Example: 'Paris' or 'Acme, Ltd.' + label: {{ salt['pillar.get']('gitlab:ldap:label', 'LDAP') }} + + host: '{{ salt['pillar.get']('gitlab:ldap:host', '') }}' + port: {{ salt['pillar.get']('gitlab:ldap:port', 636) }} + uid: '{{ salt['pillar.get']('gitlab:ldap:uid', 'sAMAccountName') }}' + method: '{{ salt['pillar.get']('gitlab:ldap:method', 'ssl') }}' # "tls" or "ssl" or "plain" + bind_dn: '{{ salt['pillar.get']('gitlab:ldap:bind_dn', '') }}' + password: '{{ salt['pillar.get']('gitlab:ldap:password', '') }}' + + # Set a timeout, in seconds, for LDAP queries. This helps avoid blocking + # a request if the LDAP server becomes unresponsive. + # A value of 0 means there is no timeout. + timeout: 10 + + # This setting specifies if LDAP server is Active Directory LDAP server. + # For non AD servers it skips the AD specific queries. + # If your LDAP server is not AD, set this to false. + active_directory: {{ salt['pillar.get']('gitlab:ldap:active_directory', false) }} + + # If allow_username_or_email_login is enabled, GitLab will ignore everything + # after the first '@' in the LDAP username submitted by the user on login. + # + # Example: + # - the user enters 'jane.doe@example.com' and 'p@ssw0rd' as LDAP credentials; + # - GitLab queries the LDAP server with 'jane.doe' and 'p@ssw0rd'. + # + # If you are using "uid: 'userPrincipalName'" on ActiveDirectory you need to + # disable this setting, because the userPrincipalName contains an '@'. + allow_username_or_email_login: {{ salt['pillar.get']('gitlab:ldap:allow_username_or_email_login', true) }} + + # To maintain tight control over the number of active users on your GitLab installation, + # enable this setting to keep new users blocked until they have been cleared by the admin + # (default: false). + block_auto_created_users: {{ salt['pillar.get']('gitlab:ldap:block_auto_created_users', true) }} + + # Base where we can search for users + # + # Ex. ou=People,dc=gitlab,dc=example + # + base: {{ salt['pillar.get']('gitlab:ldap:base', '') }} + + # Filter LDAP users + # + # Format: RFC 4515 http://tools.ietf.org/search/rfc4515 + # Ex. (employeeType=developer) + # + # Note: GitLab does not support omniauth-ldap's custom filter syntax. + # + user_filter: {{ salt['pillar.get']('gitlab:ldap:user_filter', '') }} + + # LDAP attributes that GitLab will use to create an account for the LDAP user. + # The specified attribute can either be the attribute name as a string (e.g. 'mail'), + # or an array of attribute names to try in order (e.g. ['mail', 'email']). + # Note that the user's LDAP login will always be the attribute specified as `uid` above. + attributes: + # The username will be used in paths for the user's own projects + # (like `gitlab.example.com/username/project`) and when mentioning + # them in issues, merge request and comments (like `@username`). + # If the attribute specified for `username` contains an email address, + # the GitLab username will be the part of the email address before the '@'. + username: ['uid', 'userid', 'sAMAccountName'] + email: ['mail', 'email', 'userPrincipalName'] + + # If no full name could be found at the attribute specified for `name`, + # the full name is determined using the attributes specified for + # `first_name` and `last_name`. + name: 'cn' + first_name: 'givenName' + last_name: 'sn' + + # GitLab EE only: add more LDAP servers + # Choose an ID made of a-z and 0-9 . This ID will be stored in the database + # so that GitLab can remember which LDAP server a user belongs to. + # uswest2: + # label: + # host: + # .... + ## OmniAuth settings omniauth: # Allow login via Twitter, Google, etc. using OmniAuth providers enabled: {{ salt['pillar.get']('gitlab:omniauth:enabled', false) }} + # Uncomment this to automatically sign in with a specific omniauth provider's without + # showing GitLab's sign-in page (default: show the GitLab sign-in page) + # auto_sign_in_with_provider: saml + # CAUTION! - # This allows users to login without having a user account first (default: false). + # This allows users to login without having a user account first. Define the allowed providers + # using an array, e.g. ["saml", "twitter"], or as true/false to allow all providers or none. # User accounts will be created automatically when authentication was successful. allow_single_sign_on: {{ salt['pillar.get']('gitlab:omniauth:allow_single_sign_on', false) }} + # Locks down those users until they have been cleared by the admin (default: true). block_auto_created_users: {{ salt['pillar.get']('gitlab:omniauth:block_auto_created_users', true) }} + # Look up new users in LDAP servers. If a match is found (same uid), automatically + # link the omniauth identity with the LDAP account. (default: false) + auto_link_ldap_user: false + + # Allow users with existing accounts to login and auto link their account via SAML + # login, without having to do a manual login first and manually add SAML + # (default: false) + auto_link_saml_user: false + + # Set different Omniauth providers as external so that all users creating accounts + # via these providers will not be able to have access to internal projects. You + # will need to use the full name of the provider, like `google_oauth2` for Google. + # Refer to the examples below for the full names of the supported providers. + # (default: []) + external_providers: [] ## Auth providers # Uncomment the following lines and fill in the data of the auth provider you want to use # If your favorite auth provider is not listed you can use others: - # see https://github.com/gitlabhq/gitlab-public-wiki/wiki/Working-custom-omniauth-provider-configurations + # see https://github.com/gitlabhq/gitlab-public-wiki/wiki/Custom-omniauth-provider-configurations # The 'app_id' and 'app_secret' parameters are always passed as the first two # arguments, followed by optional 'args' which can be either a hash or an array. + # Documentation for this is available at http://doc.gitlab.com/ce/integration/omniauth.html providers: - # - { name: 'google_oauth2', app_id: 'YOUR APP ID', - # app_secret: 'YOUR APP SECRET', - # args: { access_type: 'offline', approval_prompt: '' } } - # - { name: 'twitter', app_id: 'YOUR APP ID', - # app_secret: 'YOUR APP SECRET'} - # - { name: 'github', app_id: 'YOUR APP ID', - # app_secret: 'YOUR APP SECRET', + # See omniauth-cas3 for more configuration details + # - { name: 'cas3', + # label: 'cas3', + # args: { + # url: 'https://sso.example.com', + # disable_ssl_verification: false, + # login_url: '/cas/login', + # service_validate_url: '/cas/p3/serviceValidate', + # logout_url: '/cas/logout'} } + # - { name: 'authentiq', + # # for client credentials (client ID and secret), go to https://www.authentiq.com/ + # app_id: 'YOUR_CLIENT_ID', + # app_secret: 'YOUR_CLIENT_SECRET', + # args: { + # scope: 'aq:name email~rs address aq:push' + # # redirect_uri parameter is optional except when 'gitlab.host' in this file is set to 'localhost' + # # redirect_uri: 'YOUR_REDIRECT_URI' + # } + # } + # - { name: 'github', + # app_id: 'YOUR_APP_ID', + # app_secret: 'YOUR_APP_SECRET', + # url: "https://github.com/", + # verify_ssl: true, # args: { scope: 'user:email' } } - + # - { name: 'bitbucket', + # app_id: 'YOUR_APP_ID', + # app_secret: 'YOUR_APP_SECRET' } + # - { name: 'gitlab', + # app_id: 'YOUR_APP_ID', + # app_secret: 'YOUR_APP_SECRET', + # args: { scope: 'api' } } + # - { name: 'google_oauth2', + # app_id: 'YOUR_APP_ID', + # app_secret: 'YOUR_APP_SECRET', + # args: { access_type: 'offline', approval_prompt: '' } } + # - { name: 'facebook', + # app_id: 'YOUR_APP_ID', + # app_secret: 'YOUR_APP_SECRET' } + # - { name: 'twitter', + # app_id: 'YOUR_APP_ID', + # app_secret: 'YOUR_APP_SECRET' } + # + # - { name: 'saml', + # label: 'Our SAML Provider', + # groups_attribute: 'Groups', + # external_groups: ['Contractors', 'Freelancers'], + # args: { + # assertion_consumer_service_url: 'https://gitlab.example.com/users/auth/saml/callback', + # idp_cert_fingerprint: '43:51:43:a1:b5:fc:8b:b7:0a:3a:a9:b1:0f:66:73:a8', + # idp_sso_target_url: 'https://login.example.com/idp', + # issuer: 'https://gitlab.example.com', + # name_identifier_format: 'urn:oasis:names:tc:SAML:2.0:nameid-format:transient' + # } } + # + # - { name: 'crowd', + # args: { + # crowd_server_url: 'CROWD SERVER URL', + # application_name: 'YOUR_APP_NAME', + # application_password: 'YOUR_APP_PASSWORD' } } + # + # - { name: 'auth0', + # args: { + # client_id: 'YOUR_AUTH0_CLIENT_ID', + # client_secret: 'YOUR_AUTH0_CLIENT_SECRET', + # namespace: 'YOUR_AUTH0_DOMAIN' } } + + # SSO maximum session duration in seconds. Defaults to CAS default of 8 hours. + # cas3: + # session_duration: 28800 + + # Shared file storage settings + shared: + # path: /mnt/gitlab # Default: shared # - # 3. Advanced settings + # 4. Advanced settings # ========================== # GitLab Satellites - #satellites: - # # Relative paths are relative to Rails.root (default: tmp/repo_satellites/) - # path: {{ lib_dir }}/gitlab-satellites/ + # + # Note for maintainers: keep the satellites.path setting until GitLab 9.0 at + # least. This setting is fed to 'rm -rf' in + # db/migrate/20151023144219_remove_satellites.rb + satellites: + path: {{ lib_dir }}/gitlab-satellites/ + + ## Repositories settings + repositories: + # Paths where repositories can be stored. Give the canonicalized absolute pathname. + # IMPORTANT: None of the path components may be symlink, because + # gitlab-shell invokes Dir.pwd inside the repository path and that results + # real path not the symlink. + storages: # You must have at least a `default` storage path. + default: {{ repositories }} ## Backup settings backup: path: "tmp/backups" # Relative paths are relative to Rails.root (default: tmp/backups/) + # archive_permissions: 0640 # Permissions for the resulting backup.tar file (default: 0600) # keep_time: 604800 # default: 0 (forever) (in seconds) + # pg_schema: public # default: nil, it means that all schemas will be backed up + # upload: + # # Fog storage connection settings, see http://fog.io/storage/ . + # connection: + # provider: AWS + # region: eu-west-1 + # aws_access_key_id: AKIAKIAKI + # aws_secret_access_key: 'secret123' + # # The remote 'directory' to store your backups. For S3, this would be the bucket name. + # remote_directory: 'my.s3.bucket' + # # Use multipart uploads when file size reaches 100MB, see + # # http://docs.aws.amazon.com/AmazonS3/latest/dev/uploadobjusingmpu.html + # multipart_chunk_size: 104857600 + # # Turns on AWS Server-Side Encryption with Amazon S3-Managed Keys for backups, this is optional + # # encryption: 'AES256' ## GitLab Shell settings gitlab_shell: - path: {{ root_dir }}/gitlab-shell - - # REPOS_PATH MUST NOT BE A SYMLINK!!! - repos_path: {{ repositories }}/ - + path: {{ root_dir }}/gitlab-shell/ hooks_path: {{ shell_dir_content }}/hooks/ # File that contains the secret key for verifying access for gitlab-shell. @@ -223,24 +529,51 @@ production: &base # Use the default values unless you really know what you are doing git: bin_path: /usr/bin/git - # Max size of a git object (e.g. a commit), in bytes + # The next value is the maximum memory size grit can use + # Given in number of bytes per git object (e.g. a commit) # This value can be increased if you have very large commits - max_size: 5242880 # 5.megabytes + max_size: 20971520 # 20.megabytes # Git timeout to read a commit, in seconds timeout: 10 + ## Webpack settings + # If enabled, this will tell rails to serve frontend assets from the webpack-dev-server running + # on a given port instead of serving directly from /assets/webpack. This is only indended for use + # in development. + webpack: + # dev_server: + # enabled: true + # host: localhost + # port: 3808 + # - # 4. Extra customization + # 5. Extra customization # ========================== extra: ## Google analytics. Uncomment if you want it # google_analytics_id: '_your_tracking_id' - ## Text under sign-in page (Markdown enabled) - # sign_in_text: | - # ![Company Logo](http://www.companydomain.com/logo.png) - # [Learn more about CompanyName](http://www.companydomain.com/) + ## Piwik analytics. + # piwik_url: '_your_piwik_url' + # piwik_site_id: '_your_piwik_site_id' + + rack_attack: + git_basic_auth: + # Rack Attack IP banning enabled + # enabled: true + # + # Whitelist requests from 127.0.0.1 for web proxies (NGINX/Apache) with incorrect headers + # ip_whitelist: ["127.0.0.1"] + # + # Limit the number of Git HTTP authentication attempts per IP + # maxretry: 10 + # + # Reset the auth attempt counter per IP after 60 seconds + # findtime: 60 + # + # Ban an IP for one hour (3600s) after too many auth attempts + # bantime: 3600 development: <<: *base @@ -256,4 +589,3 @@ test: staging: <<: *base - From c302faacce1f470f8440a86b83868c64f69512ad Mon Sep 17 00:00:00 2001 From: Gilles Dartiguelongue Date: Thu, 31 Aug 2017 17:59:42 +0200 Subject: [PATCH 39/96] Synchronize init script with gitlab 8.17 branch --- gitlab/files/gitlab-default | 59 ++++++++++++++++++++++++++- gitlab/files/initd | 80 ++++++++++++++++++++++++++++++------- 2 files changed, 123 insertions(+), 16 deletions(-) diff --git a/gitlab/files/gitlab-default b/gitlab/files/gitlab-default index eb5dc25..fb0506f 100644 --- a/gitlab/files/gitlab-default +++ b/gitlab/files/gitlab-default @@ -4,6 +4,7 @@ {% set root_dir = salt['pillar.get']('gitlab:lookup:root_dir', '/home/git') %} {% set pids_dir = salt['pillar.get']('gitlab:lookup:pids_dir', root_dir ~ '/var/pids') %} {% set sockets_dir = salt['pillar.get']('gitlab:lookup:sockets_dir', root_dir ~ '/var/sockets') %} +{% set logs_dir = salt['pillar.get']('gitlab:lookup:logs_dir', root_dir ~ '/var/log') %} # RAILS_ENV defines the type of installation that is running. # Normal values are "production", "test" and "development". @@ -13,7 +14,7 @@ RAILS_ENV="production" # The default is "git". app_user="git" -# app_root defines the folder in which gitlab and it's components are installed. +# app_root defines the folder in which gitlab and its components are installed. # The default is "/home/$app_user/gitlab" app_root="{{ root_dir }}/gitlab" @@ -22,7 +23,7 @@ app_root="{{ root_dir }}/gitlab" . /home/$app_user/.rvm/scripts/rvm {% endif %} -# pid_path defines a folder in which the gitlab and it's components place their pids. +# pid_path defines a folder in which the gitlab and its components place their pids. # This variable is also used below to define the relevant pids for the gitlab components. # The default is "$app_root/tmp/pids" pid_path="{{ pids_dir }}" @@ -39,3 +40,57 @@ web_server_pid_path="$pid_path/unicorn.pid" # The default is "$pid_path/sidekiq.pid" sidekiq_pid_path="$pid_path/sidekiq.pid" +# The directory where the gitlab-workhorse binaries are. Usually +# /home/git/gitlab-workhorse . +gitlab_workhorse_dir=$(cd {{ root_dir }}/gitlab-workhorse/bin && pwd) +gitlab_workhorse_pid_path="$pid_path/gitlab-workhorse.pid" + +# The -listenXxx settings determine where gitlab-workhorse +# listens for connections from the web server. By default it listens to a +# socket. To listen on TCP connections (needed by Apache) change to: +# '-listenNetwork tcp -listenAddr 127.0.0.1:8181' +# +# The -authBackend setting tells gitlab-workhorse where it can reach Unicorn. +# For relative URL support change to: +# '-authBackend http://127.0.0.1/8080/gitlab' +# Read more in http://doc.gitlab.com/ce/install/relative_url.html +gitlab_workhorse_options="-listenUmask 0 -listenNetwork unix -listenAddr $socket_path/gitlab-workhorse.socket -authBackend http://127.0.0.1:8080 -authSocket $socket_path/gitlab.socket -documentRoot $app_root/public" +gitlab_workhorse_log="{{ logs_dir }}/gitlab-workhorse.log" + +# The GitLab Pages Daemon needs either a separate IP address on which it will +# listen or use different ports than 80 or 443 that will be forwarded to GitLab +# Pages Daemon. +# +# To enable HTTP support for custom domains add the `-listen-http` directive +# in `gitlab_pages_options` below. +# The value of -listen-http must be set to `gitlab.yml > pages > external_http` +# as well. For example: +# +# -listen-http 1.1.1.1:80 +# +# To enable HTTPS support for custom domains add the `-listen-https`, +# `-root-cert` and `-root-key` directives in `gitlab_pages_options` below. +# The value of -listen-https must be set to `gitlab.yml > pages > external_https` +# as well. For example: +# +# -listen-https 1.1.1.1:443 -root-cert /path/to/example.com.crt -root-key /path/to/example.com.key +# +# The -pages-domain must be specified the same as in `gitlab.yml > pages > host`. +# Set `gitlab_pages_enabled=true` if you want to enable the Pages feature. +gitlab_pages_enabled=false +gitlab_pages_options="-pages-domain example.com -pages-root $app_root/shared/pages -listen-proxy 127.0.0.1:8090" +gitlab_pages_log="{{ logs_dir }}/gitlab-pages.log" + +# mail_room_enabled specifies whether mail_room, which is used to process incoming email, is enabled. +# This is required for the Reply by email feature. +# The default is "false" +mail_room_enabled=false + +# mail_room_pid_path defines the path in which to create the pid file for mail_room +# The default is "$pid_path/mail_room.pid" +mail_room_pid_path="$pid_path/mail_room.pid" + +# shell_path defines the path of shell for "$app_user" in case you are using +# shell other than "bash" +# The default is "/bin/bash" +shell_path="/bin/bash" diff --git a/gitlab/files/initd b/gitlab/files/initd index fd5a591..07d9d27 100755 --- a/gitlab/files/initd +++ b/gitlab/files/initd @@ -46,6 +46,11 @@ gitlab_workhorse_dir=$(cd {{ root_dir }}/gitlab-workhorse/bin 2> /dev/null && pw gitlab_workhorse_pid_path="$pid_path/gitlab-workhorse.pid" gitlab_workhorse_options="-listenUmask 0 -listenNetwork unix -listenAddr $socket_path/gitlab-workhorse.socket -authBackend http://127.0.0.1:8080 -authSocket $rails_socket -documentRoot $app_root/public" gitlab_workhorse_log="{{ logs_dir }}/gitlab-workhorse.log" +gitlab_pages_enabled=false +gitlab_pages_dir=$(cd {{ root_dir }}/gitlab-pages/bin 2> /dev/null && pwd) +gitlab_pages_pid_path="$pid_path/gitlab-pages.pid" +gitlab_pages_options="-pages-domain example.com -pages-root $app_root/shared/pages -listen-proxy 127.0.0.1:8090" +gitlab_pages_log="{{ logs_dir }}/gitlab-pages.log" shell_path="/bin/bash" # Read configuration variable file if it is present @@ -93,13 +98,20 @@ check_pids(){ mpid=0 fi fi + if [ "$gitlab_pages_enabled" = true ]; then + if [ -f "$gitlab_pages_pid_path" ]; then + gppid=$(cat "$gitlab_pages_pid_path") + else + gppid=0 + fi + fi } ## Called when we have started the two processes and are waiting for their pid files. wait_for_pids(){ # We are sleeping a bit here mostly because sidekiq is slow at writing its pid i=0; - while [ ! -f $web_server_pid_path ] || [ ! -f $sidekiq_pid_path ] || [ ! -f $gitlab_workhorse_pid_path ] || { [ "$mail_room_enabled" = true ] && [ ! -f $mail_room_pid_path ]; }; do + while [ ! -f $web_server_pid_path ] || [ ! -f $sidekiq_pid_path ] || [ ! -f $gitlab_workhorse_pid_path ] || { [ "$mail_room_enabled" = true ] && [ ! -f $mail_room_pid_path ]; } || { [ "$gitlab_pages_enabled" = true ] && [ ! -f $gitlab_pages_pid_path ]; }; do sleep 0.1; i=$((i+1)) if [ $((i%10)) = 0 ]; then @@ -148,7 +160,15 @@ check_status(){ mail_room_status="-1" fi fi - if [ $web_status = 0 ] && [ $sidekiq_status = 0 ] && [ $gitlab_workhorse_status = 0 ] && { [ "$mail_room_enabled" != true ] || [ $mail_room_status = 0 ]; }; then + if [ "$gitlab_pages_enabled" = true ]; then + if [ $gppid -ne 0 ]; then + kill -0 "$gppid" 2>/dev/null + gitlab_pages_status="$?" + else + gitlab_pages_status="-1" + fi + fi + if [ $web_status = 0 ] && [ $sidekiq_status = 0 ] && [ $gitlab_workhorse_status = 0 ] && { [ "$mail_room_enabled" != true ] || [ $mail_room_status = 0 ]; } && { [ "$gitlab_pages_enabled" != true ] || [ $gitlab_pages_status = 0 ]; }; then gitlab_status=0 else # http://refspecs.linuxbase.org/LSB_4.1.0/LSB-Core-generic/LSB-Core-generic/iniscrptact.html @@ -177,7 +197,7 @@ check_stale_pids(){ fi fi if [ "$hpid" != "0" ] && [ "$gitlab_workhorse_status" != "0" ]; then - echo "Removing stale gitlab-workhorse pid. This is most likely caused by gitlab-workhorse crashing the last time it ran." + echo "Removing stale GitLab Workhorse pid. This is most likely caused by GitLab Workhorse crashing the last time it ran." if ! rm "$gitlab_workhorse_pid_path"; then echo "Unable to remove stale pid, exiting" exit 1 @@ -190,12 +210,19 @@ check_stale_pids(){ exit 1 fi fi + if [ "$gitlab_pages_enabled" = true ] && [ "$gppid" != "0" ] && [ "$gitlab_pages_status" != "0" ]; then + echo "Removing stale GitLab Pages job dispatcher pid. This is most likely caused by GitLab Pages crashing the last time it ran." + if ! rm "$gitlab_pages_pid_path"; then + echo "Unable to remove stale pid, exiting" + exit 1 + fi + fi } ## If no parts of the service is running, bail out. exit_if_not_running(){ check_stale_pids - if [ "$web_status" != "0" ] && [ "$sidekiq_status" != "0" ] && [ "$gitlab_workhorse_status" != "0" ] && { [ "$mail_room_enabled" != true ] || [ "$mail_room_status" != "0" ]; }; then + if [ "$web_status" != "0" ] && [ "$sidekiq_status" != "0" ] && [ "$gitlab_workhorse_status" != "0" ] && { [ "$mail_room_enabled" != true ] || [ "$mail_room_status" != "0" ]; } && { [ "$gitlab_pages_enabled" != true ] || [ "$gitlab_pages_status" != "0" ]; }; then echo "GitLab is not running." exit fi @@ -212,11 +239,14 @@ start_gitlab() { echo "Starting GitLab Sidekiq" fi if [ "$gitlab_workhorse_status" != "0" ]; then - echo "Starting gitlab-workhorse" + echo "Starting GitLab Workhorse" fi if [ "$mail_room_enabled" = true ] && [ "$mail_room_status" != "0" ]; then echo "Starting GitLab MailRoom" fi + if [ "$gitlab_pages_enabled" = true ] && [ "$gitlab_pages_status" != "0" ]; then + echo "Starting GitLab Pages" + fi # Then check if the service is running. If it is: don't start again. if [ "$web_status" = "0" ]; then @@ -236,7 +266,7 @@ start_gitlab() { fi if [ "$gitlab_workhorse_status" = "0" ]; then - echo "The gitlab-workhorse is already running with pid $spid, not restarting" + echo "The GitLab Workhorse is already running with pid $spid, not restarting" else # No need to remove a socket, gitlab-workhorse does this itself. # Because gitlab-workhorse has multiple executables we need to fix @@ -256,6 +286,16 @@ start_gitlab() { fi fi + if [ "$gitlab_pages_enabled" = true ]; then + if [ "$gitlab_pages_status" = "0" ]; then + echo "The GitLab Pages is already running with pid $spid, not restarting" + else + $app_root/bin/daemon_with_pidfile $gitlab_pages_pid_path \ + $gitlab_pages_dir/gitlab-pages $gitlab_pages_options \ + >> $gitlab_pages_log 2>&1 & + fi + fi + # Wait for the pids to be planted wait_for_pids # Finally check the status to tell wether or not GitLab is running @@ -275,20 +315,24 @@ stop_gitlab() { RAILS_ENV=$RAILS_ENV bin/background_jobs stop fi if [ "$gitlab_workhorse_status" = "0" ]; then - echo "Shutting down gitlab-workhorse" + echo "Shutting down GitLab Workhorse" kill -- $(cat $gitlab_workhorse_pid_path) fi if [ "$mail_room_enabled" = true ] && [ "$mail_room_status" = "0" ]; then echo "Shutting down GitLab MailRoom" RAILS_ENV=$RAILS_ENV bin/mail_room stop fi + if [ "$gitlab_pages_status" = "0" ]; then + echo "Shutting down gitlab-pages" + kill -- $(cat $gitlab_pages_pid_path) + fi # If something needs to be stopped, lets wait for it to stop. Never use SIGKILL in a script. - while [ "$web_status" = "0" ] || [ "$sidekiq_status" = "0" ] || [ "$gitlab_workhorse_status" = "0" ] || { [ "$mail_room_enabled" = true ] && [ "$mail_room_status" = "0" ]; }; do + while [ "$web_status" = "0" ] || [ "$sidekiq_status" = "0" ] || [ "$gitlab_workhorse_status" = "0" ] || { [ "$mail_room_enabled" = true ] && [ "$mail_room_status" = "0" ]; } || { [ "$gitlab_pages_enabled" = true ] && [ "$gitlab_pages_status" = "0" ]; }; do sleep 1 check_status printf "." - if [ "$web_status" != "0" ] && [ "$sidekiq_status" != "0" ] && [ "$gitlab_workhorse_status" != "0" ] && { [ "$mail_room_enabled" != true ] || [ "$mail_room_status" != "0" ]; }; then + if [ "$web_status" != "0" ] && [ "$sidekiq_status" != "0" ] && [ "$gitlab_workhorse_status" != "0" ] && { [ "$mail_room_enabled" != true ] || [ "$mail_room_status" != "0" ]; } && { [ "$gitlab_pages_enabled" != true ] || [ "$gitlab_pages_status" != "0" ]; }; then printf "\n" break fi @@ -302,6 +346,7 @@ stop_gitlab() { if [ "$mail_room_enabled" = true ]; then rm "$mail_room_pid_path" 2>/dev/null fi + rm -f "$gitlab_pages_pid_path" print_status } @@ -309,7 +354,7 @@ stop_gitlab() { ## Prints the status of GitLab and its components. print_status() { check_status - if [ "$web_status" != "0" ] && [ "$sidekiq_status" != "0" ] && [ "$gitlab_workhorse_status" != "0" ] && { [ "$mail_room_enabled" != true ] || [ "$mail_room_status" != "0" ]; }; then + if [ "$web_status" != "0" ] && [ "$sidekiq_status" != "0" ] && [ "$gitlab_workhorse_status" != "0" ] && { [ "$mail_room_enabled" != true ] || [ "$mail_room_status" != "0" ]; } && { [ "$gitlab_pages_enabled" != true ] || [ "$gitlab_pages_status" != "0" ]; }; then echo "GitLab is not running." return fi @@ -324,9 +369,9 @@ print_status() { printf "The GitLab Sidekiq job dispatcher is \033[31mnot running\033[0m.\n" fi if [ "$gitlab_workhorse_status" = "0" ]; then - echo "The gitlab-workhorse with pid $hpid is running." + echo "The GitLab Workhorse with pid $hpid is running." else - printf "The gitlab-workhorse is \033[31mnot running\033[0m.\n" + printf "The GitLab Workhorse is \033[31mnot running\033[0m.\n" fi if [ "$mail_room_enabled" = true ]; then if [ "$mail_room_status" = "0" ]; then @@ -335,7 +380,14 @@ print_status() { printf "The GitLab MailRoom email processor is \033[31mnot running\033[0m.\n" fi fi - if [ "$web_status" = "0" ] && [ "$sidekiq_status" = "0" ] && [ "$gitlab_workhorse_status" = "0" ] && { [ "$mail_room_enabled" != true ] || [ "$mail_room_status" = "0" ]; }; then + if [ "$gitlab_pages_enabled" = true ]; then + if [ "$gitlab_pages_status" = "0" ]; then + echo "The GitLab Pages with pid $mpid is running." + else + printf "The GitLab Pages is \033[31mnot running\033[0m.\n" + fi + fi + if [ "$web_status" = "0" ] && [ "$sidekiq_status" = "0" ] && [ "$gitlab_workhorse_status" = "0" ] && { [ "$mail_room_enabled" != true ] || [ "$mail_room_status" = "0" ]; } && { [ "$gitlab_pages_enabled" != true ] || [ "$gitlab_pages_status" = "0" ]; }; then printf "GitLab and all its components are \033[32mup and running\033[0m.\n" fi } @@ -366,7 +418,7 @@ reload_gitlab(){ ## Restarts Sidekiq and Unicorn. restart_gitlab(){ check_status - if [ "$web_status" = "0" ] || [ "$sidekiq_status" = "0" ] || [ "$gitlab_workhorse" = "0" ] || { [ "$mail_room_enabled" = true ] && [ "$mail_room_status" = "0" ]; }; then + if [ "$web_status" = "0" ] || [ "$sidekiq_status" = "0" ] || [ "$gitlab_workhorse" = "0" ] || { [ "$mail_room_enabled" = true ] && [ "$mail_room_status" = "0" ]; } || { [ "$gitlab_pages_enabled" = true ] && [ "$gitlab_pages_status" = "0" ]; }; then stop_gitlab fi start_gitlab From f1e3831b6d57a18fa8458c5fbd96c7d242bfa807 Mon Sep 17 00:00:00 2001 From: Gilles Dartiguelongue Date: Fri, 1 Sep 2017 14:29:52 +0200 Subject: [PATCH 40/96] Update assets update command --- gitlab/gitlab.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitlab/gitlab.sls b/gitlab/gitlab.sls index bf72d3c..ab8e401 100644 --- a/gitlab/gitlab.sls +++ b/gitlab/gitlab.sls @@ -273,7 +273,7 @@ gitlab-recompile-assets-cache: cmd.run: - user: git - cwd: {{ gitlab_dir }} - - name: bundle exec rake assets:clean assets:precompile cache:clear RAILS_ENV=production + - name: bundle exec rake gitlab:assets:clean gitlab:assets:compile cache:clear RAILS_ENV=production - shell: /bin/bash - onchanges: {% if salt['pillar.get']('gitlab:archives:enabled', false) %} From afb2cf5d52845796057a0661e3b14d8f1107a499 Mon Sep 17 00:00:00 2001 From: Gilles Dartiguelongue Date: Fri, 1 Sep 2017 15:33:23 +0200 Subject: [PATCH 41/96] Add repository for installing nodejs >=4.3 As required by upgrade notes. --- gitlab/files/nodesource.gpg.key | 52 +++++++++++++++++++++++++++++++++ gitlab/repos.sls | 27 +++++++++++++++++ 2 files changed, 79 insertions(+) create mode 100644 gitlab/files/nodesource.gpg.key diff --git a/gitlab/files/nodesource.gpg.key b/gitlab/files/nodesource.gpg.key new file mode 100644 index 0000000..1dc1d10 --- /dev/null +++ b/gitlab/files/nodesource.gpg.key @@ -0,0 +1,52 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- +Version: GnuPG v1 +Comment: GPGTools - https://gpgtools.org + +mQINBFObJLYBEADkFW8HMjsoYRJQ4nCYC/6Eh0yLWHWfCh+/9ZSIj4w/pOe2V6V+ +W6DHY3kK3a+2bxrax9EqKe7uxkSKf95gfns+I9+R+RJfRpb1qvljURr54y35IZgs +fMG22Np+TmM2RLgdFCZa18h0+RbH9i0b+ZrB9XPZmLb/h9ou7SowGqQ3wwOtT3Vy +qmif0A2GCcjFTqWW6TXaY8eZJ9BCEqW3k/0Cjw7K/mSy/utxYiUIvZNKgaG/P8U7 +89QyvxeRxAf93YFAVzMXhoKxu12IuH4VnSwAfb8gQyxKRyiGOUwk0YoBPpqRnMmD +Dl7SdmY3oQHEJzBelTMjTM8AjbB9mWoPBX5G8t4u47/FZ6PgdfmRg9hsKXhkLJc7 +C1btblOHNgDx19fzASWX+xOjZiKpP6MkEEzq1bilUFul6RDtxkTWsTa5TGixgCB/ +G2fK8I9JL/yQhDc6OGY9mjPOxMb5PgUlT8ox3v8wt25erWj9z30QoEBwfSg4tzLc +Jq6N/iepQemNfo6Is+TG+JzI6vhXjlsBm/Xmz0ZiFPPObAH/vGCY5I6886vXQ7ft +qWHYHT8jz/R4tigMGC+tvZ/kcmYBsLCCI5uSEP6JJRQQhHrCvOX0UaytItfsQfLm +EYRd2F72o1yGh3yvWWfDIBXRmaBuIGXGpajC0JyBGSOWb9UxMNZY/2LJEwARAQAB +tB9Ob2RlU291cmNlIDxncGdAbm9kZXNvdXJjZS5jb20+iQI4BBMBAgAiBQJTmyS2 +AhsDBgsJCAcDAgYVCAIJCgsEFgIDAQIeAQIXgAAKCRAWVaCraFdigHTmD/9OKhUy +jJ+h8gMRg6ri5EQxOExccSRU0i7UHktecSs0DVC4lZG9AOzBe+Q36cym5Z1di6JQ +kHl69q3zBdV3KTW+H1pdmnZlebYGz8paG9iQ/wS9gpnSeEyx0Enyi167Bzm0O4A1 +GK0prkLnz/yROHHEfHjsTgMvFwAnf9uaxwWgE1d1RitIWgJpAnp1DZ5O0uVlsPPm +XAhuBJ32mU8S5BezPTuJJICwBlLYECGb1Y65Cil4OALU7T7sbUqfLCuaRKxuPtcU +VnJ6/qiyPygvKZWhV6Od0Yxlyed1kftMJyYoL8kPHfeHJ+vIyt0s7cropfiwXoka +1iJB5nKyt/eqMnPQ9aRpqkm9ABS/r7AauMA/9RALudQRHBdWIzfIg0Mlqb52yyTI +IgQJHNGNX1T3z1XgZhI+Vi8SLFFSh8x9FeUZC6YJu0VXXj5iz+eZmk/nYjUt4Mtc +pVsVYIB7oIDIbImODm8ggsgrIzqxOzQVP1zsCGek5U6QFc9GYrQ+Wv3/fG8hfkDn +xXLww0OGaEQxfodm8cLFZ5b8JaG3+Yxfe7JkNclwvRimvlAjqIiW5OK0vvfHco+Y +gANhQrlMnTx//IdZssaxvYytSHpPZTYw+qPEjbBJOLpoLrz8ZafN1uekpAqQjffI +AOqW9SdIzq/kSHgl0bzWbPJPw86XzzftewjKNbkCDQRTmyS2ARAAxSSdQi+WpPQZ +fOflkx9sYJa0cWzLl2w++FQnZ1Pn5F09D/kPMNh4qOsyvXWlekaV/SseDZtVziHJ +Km6V8TBG3flmFlC3DWQfNNFwn5+pWSB8WHG4bTA5RyYEEYfpbekMtdoWW/Ro8Kmh +41nuxZDSuBJhDeFIp0ccnN2Lp1o6XfIeDYPegyEPSSZqrudfqLrSZhStDlJgXjea +JjW6UP6txPtYaaila9/Hn6vF87AQ5bR2dEWB/xRJzgNwRiax7KSU0xca6xAuf+TD +xCjZ5pp2JwdCjquXLTmUnbIZ9LGV54UZ/MeiG8yVu6pxbiGnXo4Ekbk6xgi1ewLi +vGmz4QRfVklV0dba3Zj0fRozfZ22qUHxCfDM7ad0eBXMFmHiN8hg3IUHTO+UdlX/ +aH3gADFAvSVDv0v8t6dGc6XE9Dr7mGEFnQMHO4zhM1HaS2Nh0TiL2tFLttLbfG5o +QlxCfXX9/nasj3K9qnlEg9G3+4T7lpdPmZRRe1O8cHCI5imVg6cLIiBLPO16e0fK +yHIgYswLdrJFfaHNYM/SWJxHpX795zn+iCwyvZSlLfH9mlegOeVmj9cyhN/VOmS3 +QRhlYXoA2z7WZTNoC6iAIlyIpMTcZr+ntaGVtFOLS6fwdBqDXjmSQu66mDKwU5Ek +fNlbyrpzZMyFCDWEYo4AIR/18aGZBYUAEQEAAYkCHwQYAQIACQUCU5sktgIbDAAK +CRAWVaCraFdigIPQEACcYh8rR19wMZZ/hgYv5so6Y1HcJNARuzmffQKozS/rxqec +0xM3wceL1AIMuGhlXFeGd0wRv/RVzeZjnTGwhN1DnCDy1I66hUTgehONsfVanuP1 +PZKoL38EAxsMzdYgkYH6T9a4wJH/IPt+uuFTFFy3o8TKMvKaJk98+Jsp2X/QuNxh +qpcIGaVbtQ1bn7m+k5Qe/fz+bFuUeXPivafLLlGc6KbdgMvSW9EVMO7yBy/2JE15 +ZJgl7lXKLQ31VQPAHT3an5IV2C/ie12eEqZWlnCiHV/wT+zhOkSpWdrheWfBT+ac +hR4jDH80AS3F8jo3byQATJb3RoCYUCVc3u1ouhNZa5yLgYZ/iZkpk5gKjxHPudFb +DdWjbGflN9k17VCf4Z9yAb9QMqHzHwIGXrb7ryFcuROMCLLVUp07PrTrRxnO9A/4 +xxECi0l/BzNxeU1gK88hEaNjIfviPR/h6Gq6KOcNKZ8rVFdwFpjbvwHMQBWhrqfu +G3KaePvbnObKHXpfIKoAM7X2qfO+IFnLGTPyhFTcrl6vZBTMZTfZiC1XDQLuGUnd +sckuXINIU3DFWzZGr0QrqkuE/jyr7FXeUJj9B7cLo+s/TXo+RaVfi3kOc9BoxIvy +/qiNGs/TKy2/Ujqp/affmIMoMXSozKmga81JSwkADO1JMgUy6dApXz9kP4EE3g== +=CLGF +-----END PGP PUBLIC KEY BLOCK----- diff --git a/gitlab/repos.sls b/gitlab/repos.sls index 47335de..6ae3d2f 100644 --- a/gitlab/repos.sls +++ b/gitlab/repos.sls @@ -19,4 +19,31 @@ ruby-scl: - require_in: - pkg: gitlab-ruby {% endif %} + +{% elif grains['os_family'] == 'Debian' %} +{# TODO: Handling of packages should be moved to map.jinja #} +{# Gitlab 8.17+ requires nodejs-4.3+ but is not available before Debian 9 or Ubuntu 16.10 #} +gitlab-nodejs-repo-mgmt-pkgs: + pkg.installed: + - names: + - python-apt + - apt-transport-https + - require_in: + - pkgrepo: gitlab-nodejs-repo + +gitlab-nodejs-repo: + pkgrepo.managed: + - name: deb https://deb.nodesource.com/node_4.x {{ grains.oscodename|lower }} main + - file: /etc/apt/sources.list.d/nodesource_4.list + - key_url: salt://gitlab/files/nodesource.gpg.key + +gitlab-nodejs-preference: + file.managed: + - name: /etc/apt/preferences.d/90_nodesource + - contents: | + Package: nodejs + Pin: release o=Node source,l=Node source + Pin-Priority: 901 + - require_in: + - sls: gitlab.packages {% endif %} From 7e7182921a69869786eaf9b35cd71919294699cf Mon Sep 17 00:00:00 2001 From: Gilles Dartiguelongue Date: Fri, 1 Sep 2017 15:40:06 +0200 Subject: [PATCH 42/96] Install yarn As documented in migration procedure. --- gitlab/files/dl.yarn.com.key | 118 +++++++++++++++++++++++++++++++++++ gitlab/gitlab.sls | 2 - gitlab/init.sls | 2 - gitlab/packages.sls | 1 + gitlab/repos.sls | 9 +++ 5 files changed, 128 insertions(+), 4 deletions(-) create mode 100644 gitlab/files/dl.yarn.com.key diff --git a/gitlab/files/dl.yarn.com.key b/gitlab/files/dl.yarn.com.key new file mode 100644 index 0000000..3cf9b34 --- /dev/null +++ b/gitlab/files/dl.yarn.com.key @@ -0,0 +1,118 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mQINBFf0j5oBEADS6cItqCbf4lOLICohq2aHqM5I1jsz3DC4ddIU5ONbKXP1t0wk +FEUPRzd6m80cTo7Q02Bw7enh4J6HvM5XVBSSGKENP6XAsiOZnY9nkXlcQAPFRnCn +CjEfoOPZ0cBKjn2IpIXXcC+7xh4p1yruBpOsCbT6BuzA+Nm9j4cpRjdRdWSSmdID +TyMZClmYm/NIfCPduYvNZxZXhW3QYeieP7HIonhZSHVu/jauEUyHLVsieUIvAOJI +cXYpwLlrw0yy4flHe1ORJzuA7EZ4eOWCuKf1PgowEnVSS7Qp7lksCuljtfXgWelB +XGJlAMD90mMbsNpQPF8ywQ2wjECM8Q6BGUcQuGMDBtFihobb+ufJxpUOm4uDt0y4 +zaw+MVSi+a56+zvY0VmMGVyJstldPAcUlFYBDsfC9+zpzyrAqRY+qFWOT2tj29R5 +ZNYvUUjEmA/kXPNIwmEr4oj7PVjSTUSpwoKamFFE6Bbha1bzIHpdPIRYc6cEulp3 +dTOWfp+Cniiblp9gwz3HeXOWu7npTTvJBnnyRSVtQgRnZrrtRt3oLZgmj2fpZFCE +g8VcnQOb0iFcIM7VlWL0QR4SOz36/GFyezZkGsMlJwIGjXkqGhcEHYVDpg0nMoq1 +qUvizxv4nKLanZ5jKrV2J8V09PbL+BERIi6QSeXhXQIui/HfV5wHXC6DywARAQAB +tBxZYXJuIFBhY2thZ2luZyA8eWFybkBkYW4uY3g+iQI5BBMBCAAjBQJX9I+aAhsD +BwsJCAcDAgEGFQgCCQoLBBYCAwECHgECF4AACgkQFkawG4blAxB52Q/9FcyGIEK2 +QamDhookuoUGGYjIeN+huQPWmc6mLPEKS2Vahk5jnJKVtAFiaqINiUtt/1jZuhF2 +bVGITvZK79kM6lg42xQcnhypzQPgkN7GQ/ApYqeKqCh1wV43KzT/CsJ9TrI0SC34 +qYHTEXXUprAuwQitgAJNi5QMdMtauCmpK+Xtl/72aetvL8jMFElOobeGwKgfLo9+ +We2EkKhSwyiy3W5TYI1UlV+evyyT+N0pmhRUSH6sJpzDnVYYPbCWa2b+0D/PHjXi +edKcely/NvqyVGoWZ+j41wkp5Q0wK2ybURS1ajfaKt0OcMhRf9XCfeXAQvU98mEk +FlfPaq0CXsjOy8eJXDeoc1dwxjDi2YbfHel0CafjrNp6qIFG9v3JxPUU19hG9lxD +Iv7VXftvMpjJCo/J4Qk+MOv7KsabgXg1iZHmllyyH3TY4AA4VA+mlceiiOHdXbKk +Q3BfS1jdXPV+2kBfqM4oWANArlrFTqtop8PPsDNqh/6SrVsthr7WTvC5q5h/Lmxy +Krm4Laf7JJMvdisfAsBbGZcR0Xv/Vw9cf2OIEzeOWbj5xul0kHT1vHhVNrBNanfe +t79RTDGESPbqz+bTS7olHWctl6TlwxA0/qKlI/PzXfOg63Nqy15woq9buca+uTcS +ccYO5au+g4Z70IEeQHsq5SC56qDR5/FvYyu5Ag0EV/SPmgEQANDSEMBKp6ER86y+ +udfKdSLP9gOv6hPsAgCHhcvBsks+ixeX9U9KkK7vj/1q6wodKf9oEbbdykHgIIB1 +lzY1l7u7/biAtQhTjdEZPh/dt3vjogrJblUEC0rt+fZe325ociocS4Bt9I75Ttkd +nWgkE4uOBJsSllpUbqfLBfYR58zz2Rz1pkBqRTkmJFetVNYErYi2tWbeJ59GjUN7 +w1K3GhxqbMbgx4dF5+rjGs+KI9k6jkGeeQHqhDk+FU70oLVLuH2Dmi9IFjklKmGa +3BU7VpNxvDwdoV7ttRYEBcBnPOmL24Sn4Xhe2MDCqgJwwyohd9rk8neV7GtavVea +Tv6bnzi1iJRgDld51HFWG8X+y55i5cYWaiXHdHOAG1+t35QUrczm9+sgkiKSk1II +TlEFsfwRl16NTCMGzjP5kGCm/W+yyyvBMw7CkENQcd23fMsdaQ/2UNYJau2PoRH/ +m+IoRehIcmE0npKeLVTDeZNCzpmfY18T542ibK49kdjZiK6G/VyBhIbWEFVu5Ll9 ++8GbcO9ucYaaeWkFS8Hg0FZafMk59VxKiICKLZ5he/C4f0UssXdyRYU6C5BH8UTC +QLg0z8mSSL+Wb2iFVPrn39Do7Zm8ry6LBCmfCf3pI99Q/1VaLDauorooJV3rQ5kC +JEiAeqQtLOvyoXIex1VbzlRUXmElABEBAAGJAh8EGAEIAAkFAlf0j5oCGwwACgkQ +FkawG4blAxAUUQ//afD0KLHjClHsA/dFiW+5qVzI8kPMHwO1QcUjeXrB6I3SluOT +rLSPhOsoS72yAaU9hFuq8g9ecmFrl3Skp/U4DHZXioEmozyZRp7eVsaHTewlfaOb +6g7+v52ktYdomcp3BM5v/pPZCnB5rLrH2KaUWbpY6V6tqtCHbF7zftDqcBENJDXf +hiCqS19J08GZFjDEqGDrEj3YEmEXZMN7PcXEISPIz6NYI6rw4yVH8AXfQW6vpPzm +ycHwI0QsVW2NQdcZ6zZt+phm6shNUbN2iDdg3BJICmIvQf8qhO3bOh0Bwc11FLHu +MKuGVxnWN82HyIsuUB7WDLBHEOtg61Zf1nAF1PQK52YuQz3EWI4LL9OqVqfSTY1J +jqIfj+u1PY2UHrxZfxlz1M8pXb1grozjKQ5aNqBKRrcMZNx71itR5rv18qGjGR2i +Sciu/xah7zAroEQrx72IjYt03tbk/007CvUlUqFIFB8kY1bbfX8JAA+TxelUniUR +2CY8eom5HnaPpKE3kGXZ0jWkudbWb7uuWcW1FE/bO+VtexpBL3SoXmwbVMGnJIEi +Uvy8m6ez0kzLXzJ/4K4b8bDO4NjFX2ocKdzLA89Z95KcZUxEG0O7kaDCu0x3BEge +uArJLecD5je2/2HXAdvkOAOUi6Gc/LiJrtInc0vUFsdqWCUK5Ao/MKvdMFW5Ag0E +V/SP2AEQALRcYv/hiv1n3VYuJbFnEfMkGwkdBYLGo3hiHKY8xrsFVePl9SkL8aqd +C310KUFNI42gGY/lz54RUHOqfMszTdafFrmwU18ECWGo4oG9qEutIKG7fkxcvk2M +tgsOMZFJqVDS1a9I4QTIkv1ellLBhVub9S7vhe/0jDjXs9IyOBpYQrpCXAm6SypC +fpqkDJ4qt/yFheATcm3s8ZVTsk2hiz2jnbqfvpte3hr3XArDjZXr3mGAp3YY9JFT +zVBOhyhT/92e6tURz8a/+IrMJzhSyIDel9L+2sHHo9E+fA3/h3lg2mo6EZmRTuvE +v9GXf5xeP5lSCDwS6YBXevJ8OSPlocC8Qm8ziww6dy/23XTxPg4YTkdf42i7VOpS +pa7EvBGne8YrmUzfbrxyAArK05lo56ZWb9ROgTnqM62wfvrCbEqSHidN3WQQEhMH +N7vtXeDPhAd8vaDhYBk4A/yWXIwgIbMczYf7Pl7oY3bXlQHb0KW/y7N3OZCr5mPW +94VLLH/v+T5R4DXaqTWeWtDGXLih7uXrG9vdlyrULEW+FDSpexKFUQe83a+Vkp6x +GX7FdMC9tNKYnPeRYqPF9UQEJg+MSbfkHSAJgky+bbacz+eqacLXMNCEk2LXFV1B +66u2EvSkGZiH7+6BNOar84I3qJrU7LBD7TmKBDHtnRr9JXrAxee3ABEBAAGJBEQE +GAEIAA8FAlf0j9gCGwIFCQHhM4ACKQkQFkawG4blAxDBXSAEGQEIAAYFAlf0j9gA +CgkQ0QH3iZ1B88PaoA//VuGdF5sjxRIOAOYqXypOD9/Kd7lYyxmtCwnvKdM7f8O5 +iD8oR2Pk1RhYHjpkfMRVjMkaLfxIRXfGQsWfKN2Zsa4zmTuNy7H6X26XW3rkFWpm +dECz1siGRvcpL6NvwLPIPQe7tST72q03u1H7bcyLGk0sTppgMoBND7yuaBTBZkAO +WizR+13x7FV+Y2j430Ft/DOe/NTc9dAlp6WmF5baOZClULfFzCTf9OcS2+bo68oP +gwWwnciJHSSLm6WRjsgoDxo5f3xBJs0ELKCr4jMwpSOTYqbDgEYOQTmHKkX8ZeQA +7mokc9guA0WK+DiGZis85lU95mneyJ2RuYcz6/VDwvT84ooe1swVkC2palDqBMwg +jZSTzbcUVqZRRnSDCe9jtpvF48WK4ZRiqtGO6Avzg1ZwMmWSr0zHQrLrUMTq/62W +KxLyj2oPxgptRg589hIwXVxJRWQjFijvK/xSjRMLgg73aNTq6Ojh98iyKAQ3HfzW +6iXBLLuGfvxflFednUSdWorr38MspcFvjFBOly+NDSjPHamNQ2h19iHLrYT7t4ve +nU9PvC+ORvXGxTN8mQR9btSdienQ8bBuU/mg/c417w6WbY7tkkqHqUuQC9LoaVdC +QFeE/SKGNe+wWN/EKi0QhXR9+UgWA41Gddi83Bk5deuTwbUeYkMDeUlOq3yyemcG +VxAA0PSktXnJgUj63+cdXu7ustVqzMjVJySCKSBtwJOge5aayonCNxz7KwoPO34m +Gdr9P4iJfc9kjawNV79aQ5aUH9uU2qFlbZOdO8pHOTjy4E+J0wbJb3VtzCJc1Eaa +83kZLFtJ45Fv2WQQ2Nv3Fo+yqAtkOkaBZv9Yq0UTaDkSYE9MMzHDVFx11TT21NZD +xu2QiIiqBcZfqJtIFHN5jONjwPG08xLAQKfUNROzclZ1h4XYUT+TWouopmpNeay5 +JSNcp5LsC2Rn0jSFuZGPJ1rBwB9vSFVA/GvOj8qEdfhjN3XbqPLVdOeChKuhlK0/ +sOLZZG91SHmT5SjP2zM6QKKSwNgHX4xZt4uugSZiY13+XqnrOGO9zRH8uumhsQmI +eFEdT27fsXTDTkWPI2zlHTltQjH1iebqqM9gfa2KUt671WyoL1yLhWrgePvDE+He +r002OslvvW6aAIIBki3FntPDqdIH89EEB4UEGqiA1eIZ6hGaQfinC7/IOkkm/mEa +qdeoI6NRS521/yf7i34NNj3IaL+rZQFbVWdbTEzAPtAs+bMJOHQXSGZeUUFrEQ/J +ael6aNg7mlr7cacmDwZWYLoCfY4w9GW6JHi6i63np8EA34CXecfor7cAX4XfaokB +XjyEkrnfV6OWYS7f01JJOcqYANhndxz1Ph8bxoRPelf5q+W5Ag0EWBU7dwEQAL1p +wH4prFMFMNV7MJPAwEug0Mxf3OsTBtCBnBYNvgFB+SFwKQLyDXUujuGQudjqQPCz +/09MOJPwGCOi0uA0BQScJ5JAfOq33qXi1iXCj9akeCfZXCOWtG3Izc3ofS6uee7K +fWUF1hNyA3PUwpRtM2pll+sQEO3y/EN7xYGUOM0mlCawrYGtxSNMlWBlMk/y5HK9 +upz+iHwUaEJ4PjV+P4YmDq0PnPvXE4qhTIvxx0kO5oZF0tAJCoTg1HE7o99/xq9Z +rejDR1JJj6btNw1YFQsRDLxRZv4rL9He10lmLhiQE8QN7zOWzyJbRP++tWY2d2zE +yFzvsOsGPbBqLDNkbb9d8Bfvp+udG13sHAEtRzI2UWe5SEdVHobAgu5l+m10WlsN +TG/L0gJe1eD1bwceWlnSrbqw+y+pam9YKWqdu18ETN6CeAbNo4w7honRkcRdZyoG +p9zZf3o1bGBBMla6RbLuJBoRDOy2Ql7B+Z87N0td6KlHI6X8fNbatbtsXR7qLUBP +5oRb6nXX4+DnTMDbvFpE2zxnkg+C354Tw5ysyHhM6abB2+zCXcZ3holeyxC+BUrO +gGPyLH/s01mg2zmttwC1UbkaGkQ6SwCoQoFEVq9Dp96B6PgZxhEw0GMrKRw53LoX +4rZif9Exv6qUFsGY8U9daEdDPF5UHYe7t/nPpfW3ABEBAAGJBD4EGAEIAAkFAlgV +O3cCGwICKQkQFkawG4blAxDBXSAEGQEIAAYFAlgVO3cACgkQRsITDf0kl/VynQ/+ +P3Vksu4fno26vA7ml9bzV3mu/X/gzU1HqySqYv9Zwzk2o512Z4QkoT/8lRepIG7v +AFRQzPn56Pz/vpMfiMDaf6thxs8wpv4y3m+rcQIQKO4sN3wwFPPbvM8wGoY6fGav +IkLKKIXy1BpzRGltGduf0c29+ycvzccQpyuTrZk4Zl73kLyBS8fCt+MZWejMMolD +uuLJiHbXci6+Pdi3ImabyStbNnJYmSyruNHcLHlgIbyugTiAcdTy0Bi/z8MfeYwj +VAwEkX4b2NwtuweYLzupBOTv0SqYCmBduZObkS5LHMZ+5Yh9Hfrd04uMdO5cIiy0 +AsGehTRC3Xyaea7Qk993rNcGEzX7LNB1GB2BXSq9FYPb+q0ewf8k8Lr9E0WG0dvD +OaJSkSGedgdA1QzvTgpAAkVWsXlksShVf4NVskxNUGDRaPLeRB+IV/5jO+kRsFuO +g5Tlkn6cgu1+Bn5gIfv0ny9K7TeC697gRQIcK8db1t8XidgSKbRmsSYEaRCy3c9x +w2/N7DLU/Js3gV8FUd7cZpaYN+k/erMdyfqLA7oFd+HLbA5Du/971yF8/6Bof8zp +jB9+QPRIARpcROEcQXz09dtl8wW8M0r09xpna+0Jk6JxF+stD97+hzikQXIxUtCX +j35ps9USSxv1cuz0MaFdWGW13OugtN4bQ2DNgelbTDUEKg//YTbBl9oGYQxHv9S5 +qvZVNvV3DuI18E5VW5ddyo/JfW24+Tukli/ZjPQYnMOP86nnIqo/LPGb4nV1uWL4 +KhmOCbH7t43+TkAwdwoxLjYP7iOqQp9VRPFjomUfvtmLjHp4r3cVEt5QeJEZLiSC +zSKMjPKqRMo5nNs3Et+/FyWCMRYdSggwhBfkbKKo44H9pmL3bTLqyir7EJAcArla +zjKMyZqRsK3gZfQgoASN5xAhemVWHnnecVSAqrOW599EBkc7Kf6lXjTVHtHN02vX +YYRZ16zrEjrfwb23LR+lAxSfWxLDovKLBg2SPbpduEv1GxyEFgF7v9fco4aQbuh/ +fOGvA8nuXkC5nI6ukw4c4zwmJ5+SNQthFUYKWLd4hR4qrCoJkMEWZmsCRtqxjVCJ +/i9ygRJHOGAWaam7bS+U7pdmq2mgF+qTxb2vX6mSzI3q3M7drGUA3EdaZo1hPA5u +kWi7tMCGqPQmtUFRnUvHPzCDuXLYT8lRxhTxDi3T5MXdIUlAUTcNpwG8Ill0xkGc +pMlh0D5p44GEdMFfJiXw6AUETHcqC2qZr2rP9kpzvVlapIrsPRg/DU+s70YnccI3 +iMCVm4/WrghFeK232zkjiwRVOm+IEWBlDFrm4MMjfguUeneYbK9WhqJnss9nc4QK +Vhzuyn3GTtg1w/T6CaYVXBjcHFk= +=HsHU +-----END PGP PUBLIC KEY BLOCK----- diff --git a/gitlab/gitlab.sls b/gitlab/gitlab.sls index ab8e401..8edb014 100644 --- a/gitlab/gitlab.sls +++ b/gitlab/gitlab.sls @@ -281,8 +281,6 @@ gitlab-recompile-assets-cache: {% else %} - git: gitlab-fetcher {% endif %} - - require: - - cmd: gitlab-npm-install {% if not salt['pillar.get']('gitlab:archives:enabled', false) %} # Needed to be able to update tree via git diff --git a/gitlab/init.sls b/gitlab/init.sls index d93465a..e2bf1f3 100644 --- a/gitlab/init.sls +++ b/gitlab/init.sls @@ -1,7 +1,5 @@ include: - {% if grains['os_family'] == 'RedHat' %} - gitlab.repos - {% endif %} {% if not salt['pillar.get']('gitlab:archives:enabled', false) %} - gitlab.git {% endif %} diff --git a/gitlab/packages.sls b/gitlab/packages.sls index 2581974..7129210 100644 --- a/gitlab/packages.sls +++ b/gitlab/packages.sls @@ -92,6 +92,7 @@ gitlab-deps: - python-docutils - rake - redis-server + - yarn - zlib1g-dev {% if salt['pillar.get']('gitlab:db:engine', 'postgresql') == 'postgresql' %} - libpq-dev diff --git a/gitlab/repos.sls b/gitlab/repos.sls index 6ae3d2f..940ff8a 100644 --- a/gitlab/repos.sls +++ b/gitlab/repos.sls @@ -30,6 +30,7 @@ gitlab-nodejs-repo-mgmt-pkgs: - apt-transport-https - require_in: - pkgrepo: gitlab-nodejs-repo + - pkgrepo: gitlab-yarn-repo gitlab-nodejs-repo: pkgrepo.managed: @@ -46,4 +47,12 @@ gitlab-nodejs-preference: Pin-Priority: 901 - require_in: - sls: gitlab.packages + +gitlab-yarn-repo: + pkgrepo.managed: + - name: deb https://dl.yarnpkg.com/debian/ stable main + - file: /etc/apt/sources.list.d/yarn.list + - key_url: salt://gitlab/files/dl.yarn.com.key + - require_in: + - sls: gitlab.packages {% endif %} From 97d9224da6171785437ceb05c38bb80476bf31ce Mon Sep 17 00:00:00 2001 From: Gilles Dartiguelongue Date: Fri, 1 Sep 2017 15:48:54 +0200 Subject: [PATCH 43/96] Update gitlab configuration for 9.0 --- gitlab/files/gitlab-gitlab.yml | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/gitlab/files/gitlab-gitlab.yml b/gitlab/files/gitlab-gitlab.yml index f4b6d46..676cf95 100644 --- a/gitlab/files/gitlab-gitlab.yml +++ b/gitlab/files/gitlab-gitlab.yml @@ -92,14 +92,6 @@ production: &base # default_can_create_group: false # default: true # username_changing_enabled: false # default: true - User can change her username/namespace - ## Default theme ID - ## 1 - Graphite - ## 2 - Charcoal - ## 3 - Green - ## 4 - Gray - ## 5 - Violet - ## 6 - Blue - # default_theme: 2 # default: 2 ## Automatic issue closing # If a commit message matches this regular expression, all issues referenced from the matched text will be closed. @@ -201,9 +193,9 @@ production: &base # Periodically executed jobs, to self-heal Gitlab, do external synchronizations, etc. # Please read here for more information: https://github.com/ondrejbartas/sidekiq-cron#adding-cron-job cron_jobs: - # Flag stuck CI builds as failed - stuck_ci_builds_worker: - cron: "0 0 * * *" + # Flag stuck CI jobs as failed + stuck_ci_jobs_worker: + cron: "0 * * * *" # Remove expired build artifacts expire_build_artifacts_worker: cron: "50 * * * *" @@ -465,6 +457,16 @@ production: &base shared: # path: /mnt/gitlab # Default: shared + # Gitaly settings + gitaly: + # The socket_path setting is optional and obsolete. When this is set + # GitLab assumes it can reach a Gitaly services via a Unix socket at + # this path. When this is commented out GitLab will not use Gitaly. + # + # This setting is obsolete because we expect it to be moved under + # repositories/storages in GitLab 9.1. + # + # socket_path: tmp/sockets/private/gitaly.socket # # 4. Advanced settings @@ -485,7 +487,8 @@ production: &base # gitlab-shell invokes Dir.pwd inside the repository path and that results # real path not the symlink. storages: # You must have at least a `default` storage path. - default: {{ repositories }} + default: + path: {{ repositories }} ## Backup settings backup: @@ -507,6 +510,8 @@ production: &base # multipart_chunk_size: 104857600 # # Turns on AWS Server-Side Encryption with Amazon S3-Managed Keys for backups, this is optional # # encryption: 'AES256' + # # Specifies Amazon S3 storage class to use for backups, this is optional + # # storage_class: 'STANDARD' ## GitLab Shell settings gitlab_shell: From 18440d0b19cd3a83d2e2c7874363470031ff34fe Mon Sep 17 00:00:00 2001 From: Gilles Dartiguelongue Date: Fri, 1 Sep 2017 15:46:38 +0200 Subject: [PATCH 44/96] Update init script and configuration for 9.0 --- gitlab/files/gitlab-default | 4 +++ gitlab/files/initd | 67 ++++++++++++++++++++++++++++++++----- 2 files changed, 63 insertions(+), 8 deletions(-) diff --git a/gitlab/files/gitlab-default b/gitlab/files/gitlab-default index fb0506f..b950f60 100644 --- a/gitlab/files/gitlab-default +++ b/gitlab/files/gitlab-default @@ -94,3 +94,7 @@ mail_room_pid_path="$pid_path/mail_room.pid" # shell other than "bash" # The default is "/bin/bash" shell_path="/bin/bash" + +# This variable controls whether the init script starts/stops Gitaly +gitaly_enabled=false +gitaly_log="{{ logs_dir }}/gitaly.log" diff --git a/gitlab/files/initd b/gitlab/files/initd index 07d9d27..6e4a429 100755 --- a/gitlab/files/initd +++ b/gitlab/files/initd @@ -52,6 +52,10 @@ gitlab_pages_pid_path="$pid_path/gitlab-pages.pid" gitlab_pages_options="-pages-domain example.com -pages-root $app_root/shared/pages -listen-proxy 127.0.0.1:8090" gitlab_pages_log="{{ logs_dir }}/gitlab-pages.log" shell_path="/bin/bash" +gitaly_enabled=false +gitaly_dir=$(cd {{ root_dir }}/gitaly/bin 2> /dev/null && pwd) +gitaly_pid_path="$pid_path/gitaly.pid" +gitaly_log="{{ logs_dir }}/gitaly.log" # Read configuration variable file if it is present test -f /etc/default/gitlab && . /etc/default/gitlab @@ -105,13 +109,20 @@ check_pids(){ gppid=0 fi fi + if [ "$gitaly_enabled" = true ]; then + if [ -f "$gitaly_pid_path" ]; then + gapid=$(cat "$gitaly_pid_path") + else + gapid=0 + fi + fi } ## Called when we have started the two processes and are waiting for their pid files. wait_for_pids(){ # We are sleeping a bit here mostly because sidekiq is slow at writing its pid i=0; - while [ ! -f $web_server_pid_path ] || [ ! -f $sidekiq_pid_path ] || [ ! -f $gitlab_workhorse_pid_path ] || { [ "$mail_room_enabled" = true ] && [ ! -f $mail_room_pid_path ]; } || { [ "$gitlab_pages_enabled" = true ] && [ ! -f $gitlab_pages_pid_path ]; }; do + while [ ! -f $web_server_pid_path ] || [ ! -f $sidekiq_pid_path ] || [ ! -f $gitlab_workhorse_pid_path ] || { [ "$mail_room_enabled" = true ] && [ ! -f $mail_room_pid_path ]; } || { [ "$gitlab_pages_enabled" = true ] && [ ! -f $gitlab_pages_pid_path ]; } || { [ "$gitaly_enabled" = true ] && [ ! -f $gitaly_pid_path ]; }; do sleep 0.1; i=$((i+1)) if [ $((i%10)) = 0 ]; then @@ -168,7 +179,15 @@ check_status(){ gitlab_pages_status="-1" fi fi - if [ $web_status = 0 ] && [ $sidekiq_status = 0 ] && [ $gitlab_workhorse_status = 0 ] && { [ "$mail_room_enabled" != true ] || [ $mail_room_status = 0 ]; } && { [ "$gitlab_pages_enabled" != true ] || [ $gitlab_pages_status = 0 ]; }; then + if [ "$gitaly_enabled" = true ]; then + if [ $gapid -ne 0 ]; then + kill -0 "$gapid" 2>/dev/null + gitaly_status="$?" + else + gitaly_status="-1" + fi + fi + if [ $web_status = 0 ] && [ $sidekiq_status = 0 ] && [ $gitlab_workhorse_status = 0 ] && { [ "$mail_room_enabled" != true ] || [ $mail_room_status = 0 ]; } && { [ "$gitlab_pages_enabled" != true ] || [ $gitlab_pages_status = 0 ]; } && { [ "$gitaly_enabled" != true ] || [ $gitaly_status = 0 ]; }; then gitlab_status=0 else # http://refspecs.linuxbase.org/LSB_4.1.0/LSB-Core-generic/LSB-Core-generic/iniscrptact.html @@ -217,12 +236,19 @@ check_stale_pids(){ exit 1 fi fi + if [ "$gitaly_enabled" = true ] && [ "$gapid" != "0" ] && [ "$gitaly_status" != "0" ]; then + echo "Removing stale Gitaly pid. This is most likely caused by Gitaly crashing the last time it ran." + if ! rm "$gitaly_pid_path"; then + echo "Unable to remove stale pid, exiting" + exit 1 + fi + fi } ## If no parts of the service is running, bail out. exit_if_not_running(){ check_stale_pids - if [ "$web_status" != "0" ] && [ "$sidekiq_status" != "0" ] && [ "$gitlab_workhorse_status" != "0" ] && { [ "$mail_room_enabled" != true ] || [ "$mail_room_status" != "0" ]; } && { [ "$gitlab_pages_enabled" != true ] || [ "$gitlab_pages_status" != "0" ]; }; then + if [ "$web_status" != "0" ] && [ "$sidekiq_status" != "0" ] && [ "$gitlab_workhorse_status" != "0" ] && { [ "$mail_room_enabled" != true ] || [ "$mail_room_status" != "0" ]; } && { [ "$gitlab_pages_enabled" != true ] || [ "$gitlab_pages_status" != "0" ]; } && { [ "$gitaly_enabled" != true ] || [ "$gitaly_status" != "0" ]; }; then echo "GitLab is not running." exit fi @@ -247,6 +273,9 @@ start_gitlab() { if [ "$gitlab_pages_enabled" = true ] && [ "$gitlab_pages_status" != "0" ]; then echo "Starting GitLab Pages" fi + if [ "$gitaly_enabled" = true ] && [ "$gitaly_status" != "0" ]; then + echo "Starting Gitaly" + fi # Then check if the service is running. If it is: don't start again. if [ "$web_status" = "0" ]; then @@ -296,6 +325,16 @@ start_gitlab() { fi fi + if [ "$gitaly_enabled" = true ]; then + if [ "$gitaly_status" = "0" ]; then + echo "Gitaly is already running with pid $gapid, not restarting" + else + $app_root/bin/daemon_with_pidfile $gitaly_pid_path \ + $app_root/bin/with_env $gitaly_dir/env \ + $gitaly_dir/gitaly >> $gitaly_log 2>&1 & + fi + fi + # Wait for the pids to be planted wait_for_pids # Finally check the status to tell wether or not GitLab is running @@ -326,13 +365,17 @@ stop_gitlab() { echo "Shutting down gitlab-pages" kill -- $(cat $gitlab_pages_pid_path) fi + if [ "$gitaly_status" = "0" ]; then + echo "Shutting down Gitaly" + kill -- $(cat $gitaly_pid_path) + fi # If something needs to be stopped, lets wait for it to stop. Never use SIGKILL in a script. - while [ "$web_status" = "0" ] || [ "$sidekiq_status" = "0" ] || [ "$gitlab_workhorse_status" = "0" ] || { [ "$mail_room_enabled" = true ] && [ "$mail_room_status" = "0" ]; } || { [ "$gitlab_pages_enabled" = true ] && [ "$gitlab_pages_status" = "0" ]; }; do + while [ "$web_status" = "0" ] || [ "$sidekiq_status" = "0" ] || [ "$gitlab_workhorse_status" = "0" ] || { [ "$mail_room_enabled" = true ] && [ "$mail_room_status" = "0" ]; } || { [ "$gitlab_pages_enabled" = true ] && [ "$gitlab_pages_status" = "0" ]; } || { [ "$gitaly_enabled" = true ] && [ "$gitaly_status" = "0" ]; }; do sleep 1 check_status printf "." - if [ "$web_status" != "0" ] && [ "$sidekiq_status" != "0" ] && [ "$gitlab_workhorse_status" != "0" ] && { [ "$mail_room_enabled" != true ] || [ "$mail_room_status" != "0" ]; } && { [ "$gitlab_pages_enabled" != true ] || [ "$gitlab_pages_status" != "0" ]; }; then + if [ "$web_status" != "0" ] && [ "$sidekiq_status" != "0" ] && [ "$gitlab_workhorse_status" != "0" ] && { [ "$mail_room_enabled" != true ] || [ "$mail_room_status" != "0" ]; } && { [ "$gitlab_pages_enabled" != true ] || [ "$gitlab_pages_status" != "0" ]; } && { [ "$gitaly_enabled" != true ] || [ "$gitaly_status" != "0" ]; }; then printf "\n" break fi @@ -347,6 +390,7 @@ stop_gitlab() { rm "$mail_room_pid_path" 2>/dev/null fi rm -f "$gitlab_pages_pid_path" + rm -f "$gitaly_pid_path" print_status } @@ -354,7 +398,7 @@ stop_gitlab() { ## Prints the status of GitLab and its components. print_status() { check_status - if [ "$web_status" != "0" ] && [ "$sidekiq_status" != "0" ] && [ "$gitlab_workhorse_status" != "0" ] && { [ "$mail_room_enabled" != true ] || [ "$mail_room_status" != "0" ]; } && { [ "$gitlab_pages_enabled" != true ] || [ "$gitlab_pages_status" != "0" ]; }; then + if [ "$web_status" != "0" ] && [ "$sidekiq_status" != "0" ] && [ "$gitlab_workhorse_status" != "0" ] && { [ "$mail_room_enabled" != true ] || [ "$mail_room_status" != "0" ]; } && { [ "$gitlab_pages_enabled" != true ] || [ "$gitlab_pages_status" != "0" ]; } && { [ "$gitaly_enabled" != true ] || [ "$gitaly_status" != "0" ]; }; then echo "GitLab is not running." return fi @@ -387,7 +431,14 @@ print_status() { printf "The GitLab Pages is \033[31mnot running\033[0m.\n" fi fi - if [ "$web_status" = "0" ] && [ "$sidekiq_status" = "0" ] && [ "$gitlab_workhorse_status" = "0" ] && { [ "$mail_room_enabled" != true ] || [ "$mail_room_status" = "0" ]; } && { [ "$gitlab_pages_enabled" != true ] || [ "$gitlab_pages_status" = "0" ]; }; then + if [ "$gitaly_enabled" = true ]; then + if [ "$gitaly_status" = "0" ]; then + echo "Gitaly with pid $gapid is running." + else + printf "Gitaly is \033[31mnot running\033[0m.\n" + fi + fi + if [ "$web_status" = "0" ] && [ "$sidekiq_status" = "0" ] && [ "$gitlab_workhorse_status" = "0" ] && { [ "$mail_room_enabled" != true ] || [ "$mail_room_status" = "0" ]; } && { [ "$gitlab_pages_enabled" != true ] || [ "$gitlab_pages_status" = "0" ]; } && { [ "$gitaly_enabled" != true ] || [ "$gitaly_status" = "0" ]; }; then printf "GitLab and all its components are \033[32mup and running\033[0m.\n" fi } @@ -418,7 +469,7 @@ reload_gitlab(){ ## Restarts Sidekiq and Unicorn. restart_gitlab(){ check_status - if [ "$web_status" = "0" ] || [ "$sidekiq_status" = "0" ] || [ "$gitlab_workhorse" = "0" ] || { [ "$mail_room_enabled" = true ] && [ "$mail_room_status" = "0" ]; } || { [ "$gitlab_pages_enabled" = true ] && [ "$gitlab_pages_status" = "0" ]; }; then + if [ "$web_status" = "0" ] || [ "$sidekiq_status" = "0" ] || [ "$gitlab_workhorse" = "0" ] || { [ "$mail_room_enabled" = true ] && [ "$mail_room_status" = "0" ]; } || { [ "$gitlab_pages_enabled" = true ] && [ "$gitlab_pages_status" = "0" ]; } || { [ "$gitaly_enabled" = true ] && [ "$gitaly_status" = "0" ]; }; then stop_gitlab fi start_gitlab From 0956234dab3894c6b31e0ed9d8afec392aa9a8dd Mon Sep 17 00:00:00 2001 From: Gilles Dartiguelongue Date: Fri, 1 Sep 2017 15:51:42 +0200 Subject: [PATCH 45/96] Add re2 dependency --- gitlab/packages.sls | 1 + 1 file changed, 1 insertion(+) diff --git a/gitlab/packages.sls b/gitlab/packages.sls index 7129210..e335af3 100644 --- a/gitlab/packages.sls +++ b/gitlab/packages.sls @@ -79,6 +79,7 @@ gitlab-deps: - libgdbm-dev - libicu-dev - libncurses5-dev + - libre2-dev - libreadline-dev - libssl-dev - libxml2-dev From 2bd49b962da65f40592dd1a26a4d0b92396f0bcb Mon Sep 17 00:00:00 2001 From: Gilles Dartiguelongue Date: Fri, 1 Sep 2017 15:54:00 +0200 Subject: [PATCH 46/96] Update assets refresh command npm was replaced by yarn for managing javascript dependencies. --- gitlab/gitlab.sls | 26 +------------------------- 1 file changed, 1 insertion(+), 25 deletions(-) diff --git a/gitlab/gitlab.sls b/gitlab/gitlab.sls index 8edb014..f77f728 100644 --- a/gitlab/gitlab.sls +++ b/gitlab/gitlab.sls @@ -245,35 +245,11 @@ gitlab-migrate-db: - file: gitlab-db-config - cmd: gitlab-gems -gitlab-npm-install: - {# npm is stubborn and will not respect proxy settings and git insteadOf #} - file.replace: - - name: {{ gitlab_dir }}/package.json - - pattern: '"jquery-ui": "github:.*#1.11.4",' - - repl: '"jquery-ui": "https://github.com/jquery/jquery-ui/archive/1.11.4.tar.gz",' - cmd.run: - - name: npm install --production - - user: git - - cwd: {{ gitlab_dir }} - {%- if salt['pillar.get']('gitlab:proxy:address') %} - - env: - - HTTP_PROXY: {{ pillar.gitlab.proxy.address }} - - HTTPS_PROXY: {{ pillar.gitlab.proxy.address }} - {%- endif %} - - onchanges: - {% if salt['pillar.get']('gitlab:archives:enabled', false) %} - - archive: gitlab-fetcher - {% else %} - - git: gitlab-fetcher - {% endif %} - - require: - - cmd: gitlab-migrate-db - gitlab-recompile-assets-cache: cmd.run: - user: git - cwd: {{ gitlab_dir }} - - name: bundle exec rake gitlab:assets:clean gitlab:assets:compile cache:clear RAILS_ENV=production + - name: bundle exec rake yarn:install gitlab:assets:clean gitlab:assets:compile cache:clear RAILS_ENV=production NODE_ENV=production - shell: /bin/bash - onchanges: {% if salt['pillar.get']('gitlab:archives:enabled', false) %} From 6bc960165a3b8ea036f78f69aff8e837a3e2e57b Mon Sep 17 00:00:00 2001 From: Gilles Dartiguelongue Date: Fri, 1 Sep 2017 18:36:31 +0200 Subject: [PATCH 47/96] Split yarn install to its own state as it requires network access --- gitlab/gitlab.sls | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/gitlab/gitlab.sls b/gitlab/gitlab.sls index f77f728..5e321e7 100644 --- a/gitlab/gitlab.sls +++ b/gitlab/gitlab.sls @@ -245,11 +245,30 @@ gitlab-migrate-db: - file: gitlab-db-config - cmd: gitlab-gems +gitlab-yarn-install: + cmd.run: + - name: bundle exec rake yarn:install + - user: git + - cwd: {{ gitlab_dir }} + - env: + - RAILS_ENV: production + - NODE_ENV: production + {%- if salt['pillar.get']('gitlab:proxy:address') %} + - HTTP_PROXY: http://{{ pillar.gitlab.proxy.address }} + - HTTPS_PROXY: http://{{ pillar.gitlab.proxy.address }} + {%- endif %} + - onchanges: + {% if salt['pillar.get']('gitlab:archives:enabled', false) %} + - archive: gitlab-fetcher + {% else %} + - git: gitlab-fetcher + {% endif %} + gitlab-recompile-assets-cache: cmd.run: - user: git - cwd: {{ gitlab_dir }} - - name: bundle exec rake yarn:install gitlab:assets:clean gitlab:assets:compile cache:clear RAILS_ENV=production NODE_ENV=production + - name: bundle exec rake gitlab:assets:clean gitlab:assets:compile cache:clear RAILS_ENV=production NODE_ENV=production - shell: /bin/bash - onchanges: {% if salt['pillar.get']('gitlab:archives:enabled', false) %} @@ -257,6 +276,8 @@ gitlab-recompile-assets-cache: {% else %} - git: gitlab-fetcher {% endif %} + - require: + - cmd: gitlab-yarn-install {% if not salt['pillar.get']('gitlab:archives:enabled', false) %} # Needed to be able to update tree via git From f48606fd493ca528496afe3fa7543a018c11b817 Mon Sep 17 00:00:00 2001 From: Gilles Dartiguelongue Date: Mon, 4 Sep 2017 10:35:22 +0200 Subject: [PATCH 48/96] Install libssl headers compatible with gems required in 9.0 --- gitlab/packages.sls | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/gitlab/packages.sls b/gitlab/packages.sls index e335af3..665619d 100644 --- a/gitlab/packages.sls +++ b/gitlab/packages.sls @@ -81,7 +81,11 @@ gitlab-deps: - libncurses5-dev - libre2-dev - libreadline-dev + {%- if (grains['os'] == 'Ubuntu' and grains['os_releaseinfo'][0] >= 17 or grains['os'] == 'Debian' and grains['os_releaseinfo'][0] >= 9) %} + - libssl1.0-dev + {%- else %} - libssl-dev + {%- endif %} - libxml2-dev - libxslt1-dev - libyaml-dev From 8934c38248221627ceeb65edc8f5c890094e51b8 Mon Sep 17 00:00:00 2001 From: Gilles Dartiguelongue Date: Mon, 4 Sep 2017 10:37:43 +0200 Subject: [PATCH 49/96] Set bundler version restrictions https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/11787 --- gitlab/ruby.sls | 1 + 1 file changed, 1 insertion(+) diff --git a/gitlab/ruby.sls b/gitlab/ruby.sls index 7c7459f..2b92356 100644 --- a/gitlab/ruby.sls +++ b/gitlab/ruby.sls @@ -20,6 +20,7 @@ gitlab-ruby: - ruby-dev gem.installed: - name: bundler + - version: ">= 1.14, <15.0" - require: - pkg: gitlab-ruby {% if salt['pillar.get']('gitlab:proxy:enabled', false) %} From 53916c115157effaec8188862e3676a2d0b052fb Mon Sep 17 00:00:00 2001 From: Gilles Dartiguelongue Date: Mon, 4 Sep 2017 10:54:12 +0200 Subject: [PATCH 50/96] Fix usage of make install variables --- gitlab/gitlab-workhorse.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitlab/gitlab-workhorse.sls b/gitlab/gitlab-workhorse.sls index 6be988f..bddba04 100644 --- a/gitlab/gitlab-workhorse.sls +++ b/gitlab/gitlab-workhorse.sls @@ -54,7 +54,7 @@ gitlab-workhorse-make: cmd.run: - user: git - cwd: {{ workhorse_dir_content }} - - name: make install PREFIX={{ root_dir }}/gitlab-workhorse + - name: make install DESTDIR={{ root_dir }}/gitlab-workhorse PREFIX= - shell: /bin/bash - onchanges: {% if salt['pillar.get']('gitlab:archives:enabled', false) %} From e86863da149b14167bc3d429a7c5576d5863a8e4 Mon Sep 17 00:00:00 2001 From: Gilles Dartiguelongue Date: Mon, 4 Sep 2017 11:25:24 +0200 Subject: [PATCH 51/96] Fix libssl-dev conditional --- gitlab/packages.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitlab/packages.sls b/gitlab/packages.sls index 665619d..0cd6c5d 100644 --- a/gitlab/packages.sls +++ b/gitlab/packages.sls @@ -81,7 +81,7 @@ gitlab-deps: - libncurses5-dev - libre2-dev - libreadline-dev - {%- if (grains['os'] == 'Ubuntu' and grains['os_releaseinfo'][0] >= 17 or grains['os'] == 'Debian' and grains['os_releaseinfo'][0] >= 9) %} + {%- if (grains['os'] == 'Ubuntu' and grains['osrelease_info'][0] >= 17) or (grains['os'] == 'Debian' and grains['osrelease_info'][0] >= 9) %} - libssl1.0-dev {%- else %} - libssl-dev From 4f1a49d0e7171d1a54dc8d0c2a481f8ca0f2caeb Mon Sep 17 00:00:00 2001 From: Gilles Dartiguelongue Date: Thu, 14 Sep 2017 15:52:20 +0200 Subject: [PATCH 52/96] Gitlab 9.0+ requires ruby >=2.3 This was not modified with 9-0-support branch because the minions had recent enough ruby but let's add some protection from installing with unsupported versions of ruby and remove RedHat packages support as I couldn't find a source of RPMs for ruby2.3+ on either for RHEL 6 or 7 based distributions. --- gitlab/repos.sls | 13 ------------- gitlab/ruby.sls | 14 ++++---------- pillar.example | 2 +- 3 files changed, 5 insertions(+), 24 deletions(-) diff --git a/gitlab/repos.sls b/gitlab/repos.sls index 940ff8a..d47d4f4 100644 --- a/gitlab/repos.sls +++ b/gitlab/repos.sls @@ -7,19 +7,6 @@ PUIAS_6_computational: - gpgkey: http://springdale.math.ias.edu/data/puias/6/x86_64/os/RPM-GPG-KEY-puias - mirrorlist: http://puias.math.ias.edu/data/puias/computational/$releasever/$basearch/mirrorlist -{% if not salt['pillar.get']('gilab:use_rvm', false) %} -include: - - gitlab.ruby - -ruby-scl: - pkgrepo.managed: - - humanname: Ruby 1.9.3 Dynamic Software Collection - - gpgcheck: 0 - - baseurl: http://people.redhat.com/bkabrda/ruby193-rhel-6/ - - require_in: - - pkg: gitlab-ruby -{% endif %} - {% elif grains['os_family'] == 'Debian' %} {# TODO: Handling of packages should be moved to map.jinja #} {# Gitlab 8.17+ requires nodejs-4.3+ but is not available before Debian 9 or Ubuntu 16.10 #} diff --git a/gitlab/ruby.sls b/gitlab/ruby.sls index 2b92356..c7601d2 100644 --- a/gitlab/ruby.sls +++ b/gitlab/ruby.sls @@ -1,7 +1,7 @@ gitlab-ruby: {% if salt['pillar.get']('gitlab:use_rvm', false) %} rvm.installed: - - name: ruby-{{ salt['pillar.get']('gitlab:rvm_ruby', '2.1.0') }} + - name: ruby-{{ salt['pillar.get']('gitlab:rvm_ruby', '2.3.3') }} - default: True - user: git - require: @@ -9,15 +9,15 @@ gitlab-ruby: - pkg: rvm-deps gem.installed: - user: git - - ruby: ruby-2.1.0 + - ruby: ruby-2.3.3 - require: - rvm: gitlab-ruby {% else %} {% if grains['os_family'] == 'Debian' %} pkg.installed: - pkgs: - - ruby - - ruby-dev + - ruby: ">=2.3" + - ruby-dev: ">=2.3" gem.installed: - name: bundler - version: ">= 1.14, <15.0" @@ -26,11 +26,5 @@ gitlab-ruby: {% if salt['pillar.get']('gitlab:proxy:enabled', false) %} - proxy: {{ salt['pillar.get']('gitlab:proxy:address') }} {% endif %} - {% elif grains['os_family'] == 'RedHat' %} - pkg.installed: - - pkgs: - - ruby193-ruby - - ruby193-ruby-devel - - ruby193-rubygem-bundler {% endif %} {% endif %} diff --git a/pillar.example b/pillar.example index 988bf18..4c30cf8 100644 --- a/pillar.example +++ b/pillar.example @@ -27,7 +27,7 @@ gitlab: md5: 1835e858d9285ac9941eec320e2ecac6 content: gitlab-shell-v2.7.2-c615ca4650dfb4b17bf073276024f886e52b74a3 use_rvm: False - rvm_ruby: 2.1.0 + rvm_ruby: 2.3.3 shell_version: v2.7.2 gitlab_version: 8-7-stable workhorse_version: v0.7.1 From 38286f2956b263f66d6ea9fd577b3d46088bc719 Mon Sep 17 00:00:00 2001 From: Gilles Dartiguelongue Date: Thu, 14 Sep 2017 17:29:25 +0200 Subject: [PATCH 53/96] Fix bundler version lock --- gitlab/ruby.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitlab/ruby.sls b/gitlab/ruby.sls index c7601d2..b61dc3b 100644 --- a/gitlab/ruby.sls +++ b/gitlab/ruby.sls @@ -20,7 +20,7 @@ gitlab-ruby: - ruby-dev: ">=2.3" gem.installed: - name: bundler - - version: ">= 1.14, <15.0" + - version: ">= 1.14, <1.15" - require: - pkg: gitlab-ruby {% if salt['pillar.get']('gitlab:proxy:enabled', false) %} From c6453cc6786785cd70a2a9de7857833f7819043d Mon Sep 17 00:00:00 2001 From: Gilles Dartiguelongue Date: Thu, 14 Sep 2017 17:38:10 +0200 Subject: [PATCH 54/96] Add missing dependency on yarn to make sure gitlab is installed --- gitlab/gitlab.sls | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gitlab/gitlab.sls b/gitlab/gitlab.sls index 7396047..b4fe9d5 100644 --- a/gitlab/gitlab.sls +++ b/gitlab/gitlab.sls @@ -263,6 +263,8 @@ gitlab-yarn-install: {% else %} - git: gitlab-fetcher {% endif %} + - require: + - cmd: gitlab-gems gitlab-recompile-assets-cache: cmd.run: From abeab19d641b8159d1789648ccda150d10f120a9 Mon Sep 17 00:00:00 2001 From: Gilles Dartiguelongue Date: Thu, 14 Sep 2017 20:15:53 +0200 Subject: [PATCH 55/96] Fix HTTP proxy env variables in gitlab-yarn-install --- gitlab/gitlab.sls | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gitlab/gitlab.sls b/gitlab/gitlab.sls index b4fe9d5..8b9e178 100644 --- a/gitlab/gitlab.sls +++ b/gitlab/gitlab.sls @@ -254,8 +254,8 @@ gitlab-yarn-install: - RAILS_ENV: production - NODE_ENV: production {%- if salt['pillar.get']('gitlab:proxy:address') %} - - HTTP_PROXY: http://{{ pillar.gitlab.proxy.address }} - - HTTPS_PROXY: http://{{ pillar.gitlab.proxy.address }} + - HTTP_PROXY: {{ pillar.gitlab.proxy.address }} + - HTTPS_PROXY: {{ pillar.gitlab.proxy.address }} {%- endif %} - onchanges: {% if salt['pillar.get']('gitlab:archives:enabled', false) %} From 1e51fffbc7fa675499c5e32c5920def13df3be1c Mon Sep 17 00:00:00 2001 From: Gilles Dartiguelongue Date: Thu, 14 Sep 2017 20:24:04 +0200 Subject: [PATCH 56/96] Pass environment variable through adequate state keyword --- gitlab/gitlab.sls | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/gitlab/gitlab.sls b/gitlab/gitlab.sls index 8b9e178..d2f5b33 100644 --- a/gitlab/gitlab.sls +++ b/gitlab/gitlab.sls @@ -20,12 +20,6 @@ include: {% set gitlab_dir_content = gitlab_dir %} {% endif %} -{% if salt['pillar.get']('gitlab:proxy:enabled', false) %} - {% set proxy = 'HTTP_PROXY=' ~ salt['pillar.get']('gitlab:proxy:address') %} -{% else %} - {% set proxy = '' %} -{% endif %} - {% if salt['pillar.get']('gitlab:archives:enabled', false) %} gitlab-fetcher: archive.extracted: @@ -197,8 +191,12 @@ gitlab-gems: cmd.run: - user: git - cwd: {{ gitlab_dir }} - - name: {{ proxy }} bundle install --deployment --without development test mysql aws kerberos - - shell: /bin/bash + - name: bundle install --deployment --without development test mysql aws kerberos + - env: + {%- if salt['pillar.get']('gitlab:proxy:address') %} + - HTTP_PROXY: {{ pillar.gitlab.proxy.address }} + - HTTPS_PROXY: {{ pillar.gitlab.proxy.address }} + {%- endif %} - onchanges: {% if salt['pillar.get']('gitlab:archives:enabled', false) %} - archive: gitlab-fetcher @@ -216,8 +214,11 @@ gitlab-initialize: cmd.run: - user: git - cwd: {{ gitlab_dir }} - - name: force=yes bundle exec rake gitlab:setup RAILS_ENV=production + - name: bundle exec rake gitlab:setup - shell: /bin/bash + - env: + - force: yes + - RAILS_ENV: production - unless: PGPASSWORD={{ db_user_infos.password }} psql -h {{ active_db.host }} -U {{ db_user }} {{ active_db.name }} -c 'select * from users;' - watch: {% if salt['pillar.get']('gitlab:archives:enabled', false) %} @@ -233,8 +234,10 @@ gitlab-migrate-db: cmd.run: - user: git - cwd: {{ gitlab_dir }} - - name: bundle exec rake db:migrate RAILS_ENV=production + - name: bundle exec rake db:migrate - shell: /bin/bash + - env: + - RAILS_ENV: production - onchanges: {% if salt['pillar.get']('gitlab:archives:enabled', false) %} - archive: gitlab-fetcher @@ -270,8 +273,11 @@ gitlab-recompile-assets-cache: cmd.run: - user: git - cwd: {{ gitlab_dir }} - - name: bundle exec rake gitlab:assets:clean gitlab:assets:compile cache:clear RAILS_ENV=production NODE_ENV=production + - name: bundle exec rake gitlab:assets:clean gitlab:assets:compile cache:clear - shell: /bin/bash + - env: + - RAILS_ENV: production + - NODE_ENV: production - onchanges: {% if salt['pillar.get']('gitlab:archives:enabled', false) %} - archive: gitlab-fetcher From 29c973e366f4079f36aba4f23ffd8e5c675f27c3 Mon Sep 17 00:00:00 2001 From: Gilles Dartiguelongue Date: Thu, 14 Sep 2017 20:25:20 +0200 Subject: [PATCH 57/96] Drop unneeded explicit shell setting These commands just execute ruby code, not some complex bash-specific shell script. --- gitlab/gitlab.sls | 3 --- 1 file changed, 3 deletions(-) diff --git a/gitlab/gitlab.sls b/gitlab/gitlab.sls index d2f5b33..7d294c6 100644 --- a/gitlab/gitlab.sls +++ b/gitlab/gitlab.sls @@ -215,7 +215,6 @@ gitlab-initialize: - user: git - cwd: {{ gitlab_dir }} - name: bundle exec rake gitlab:setup - - shell: /bin/bash - env: - force: yes - RAILS_ENV: production @@ -235,7 +234,6 @@ gitlab-migrate-db: - user: git - cwd: {{ gitlab_dir }} - name: bundle exec rake db:migrate - - shell: /bin/bash - env: - RAILS_ENV: production - onchanges: @@ -274,7 +272,6 @@ gitlab-recompile-assets-cache: - user: git - cwd: {{ gitlab_dir }} - name: bundle exec rake gitlab:assets:clean gitlab:assets:compile cache:clear - - shell: /bin/bash - env: - RAILS_ENV: production - NODE_ENV: production From d6f9e7867db7889853225428fb4d67af9fa2c699 Mon Sep 17 00:00:00 2001 From: Gilles Dartiguelongue Date: Fri, 1 Sep 2017 15:48:54 +0200 Subject: [PATCH 58/96] Update gitlab configuration for 9.1 --- gitlab/files/gitlab-gitlab.yml | 32 ++++++++++++-------------------- 1 file changed, 12 insertions(+), 20 deletions(-) diff --git a/gitlab/files/gitlab-gitlab.yml b/gitlab/files/gitlab-gitlab.yml index 676cf95..df49dd8 100644 --- a/gitlab/files/gitlab-gitlab.yml +++ b/gitlab/files/gitlab-gitlab.yml @@ -39,7 +39,7 @@ production: &base ## GitLab settings gitlab: - ## Web server settings + ## Web server settings (note: host is the FQDN, do not include http://) host: {{ salt['pillar.get']('gitlab:hostname', grains['fqdn']) }} {%- if salt['pillar.get']('gitlab:https', false) %} port: 443 @@ -105,7 +105,7 @@ production: &base issues: true merge_requests: true wiki: true - snippets: false + snippets: true builds: true container_registry: true @@ -173,8 +173,8 @@ production: &base host: example.com port: 80 # Set to 443 if you serve the pages with HTTPS https: false # Set to true if you serve the pages with HTTPS - # external_http: "1.1.1.1:80" # If defined, enables custom domain support in GitLab Pages - # external_https: "1.1.1.1:443" # If defined, enables custom domain and certificate support in GitLab Pages + # external_http: ["1.1.1.1:80", "[2001::1]:80"] # If defined, enables custom domain support in GitLab Pages + # external_https: ["1.1.1.1:443", "[2001::1]:443"] # If defined, enables custom domain and certificate support in GitLab Pages ## Mattermost ## For enabling Add to Mattermost button @@ -196,6 +196,9 @@ production: &base # Flag stuck CI jobs as failed stuck_ci_jobs_worker: cron: "0 * * * *" + # Execute scheduled triggers + trigger_schedule_worker: + cron: "0 */12 * * *" # Remove expired build artifacts expire_build_artifacts_worker: cron: "50 * * * *" @@ -459,27 +462,15 @@ production: &base # Gitaly settings gitaly: - # The socket_path setting is optional and obsolete. When this is set - # GitLab assumes it can reach a Gitaly services via a Unix socket at - # this path. When this is commented out GitLab will not use Gitaly. - # - # This setting is obsolete because we expect it to be moved under - # repositories/storages in GitLab 9.1. - # - # socket_path: tmp/sockets/private/gitaly.socket + # This setting controls whether GitLab uses Gitaly (new component + # introduced in 9.0). Eventually Gitaly use will become mandatory and + # this option will disappear. + enabled: false # # 4. Advanced settings # ========================== - # GitLab Satellites - # - # Note for maintainers: keep the satellites.path setting until GitLab 9.0 at - # least. This setting is fed to 'rm -rf' in - # db/migrate/20151023144219_remove_satellites.rb - satellites: - path: {{ lib_dir }}/gitlab-satellites/ - ## Repositories settings repositories: # Paths where repositories can be stored. Give the canonicalized absolute pathname. @@ -489,6 +480,7 @@ production: &base storages: # You must have at least a `default` storage path. default: path: {{ repositories }} + gitaly_address: unix:{{ root_dir }}/gitlab/tmp/sockets/private/gitaly.socket # TCP connections are supported too (e.g. tcp://host:port) ## Backup settings backup: From ce7d5276d27efbfb75be784f3e64fbc792da6d70 Mon Sep 17 00:00:00 2001 From: Gilles Dartiguelongue Date: Fri, 1 Sep 2017 15:46:38 +0200 Subject: [PATCH 59/96] Update init script and configuration for 9.1 --- gitlab/files/gitlab-default | 4 ++-- gitlab/files/initd | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/gitlab/files/gitlab-default b/gitlab/files/gitlab-default index b950f60..c074aed 100644 --- a/gitlab/files/gitlab-default +++ b/gitlab/files/gitlab-default @@ -66,14 +66,14 @@ gitlab_workhorse_log="{{ logs_dir }}/gitlab-workhorse.log" # The value of -listen-http must be set to `gitlab.yml > pages > external_http` # as well. For example: # -# -listen-http 1.1.1.1:80 +# -listen-http 1.1.1.1:80 -listen-http [2001::1]:80 # # To enable HTTPS support for custom domains add the `-listen-https`, # `-root-cert` and `-root-key` directives in `gitlab_pages_options` below. # The value of -listen-https must be set to `gitlab.yml > pages > external_https` # as well. For example: # -# -listen-https 1.1.1.1:443 -root-cert /path/to/example.com.crt -root-key /path/to/example.com.key +# -listen-https 1.1.1.1:443 -listen-http [2001::1]:443 -root-cert /path/to/example.com.crt -root-key /path/to/example.com.key # # The -pages-domain must be specified the same as in `gitlab.yml > pages > host`. # Set `gitlab_pages_enabled=true` if you want to enable the Pages feature. diff --git a/gitlab/files/initd b/gitlab/files/initd index 6e4a429..0c98487 100755 --- a/gitlab/files/initd +++ b/gitlab/files/initd @@ -330,8 +330,7 @@ start_gitlab() { echo "Gitaly is already running with pid $gapid, not restarting" else $app_root/bin/daemon_with_pidfile $gitaly_pid_path \ - $app_root/bin/with_env $gitaly_dir/env \ - $gitaly_dir/gitaly >> $gitaly_log 2>&1 & + $gitaly_dir/gitaly $gitaly_dir/config.toml >> $gitaly_log 2>&1 & fi fi From e27f64b27e6856955640ff34532ee2d48e9666a8 Mon Sep 17 00:00:00 2001 From: Gilles Dartiguelongue Date: Fri, 15 Sep 2017 13:37:21 +0200 Subject: [PATCH 60/96] Update minimum version requirements --- gitlab/packages.sls | 6 +++--- gitlab/repos.sls | 26 ++++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/gitlab/packages.sls b/gitlab/packages.sls index 0cd6c5d..29f5ac1 100644 --- a/gitlab/packages.sls +++ b/gitlab/packages.sls @@ -73,7 +73,7 @@ gitlab-deps: - checkinstall - curl - cmake - - golang + - golang: ">=1.8" - libcurl4-openssl-dev - libffi-dev - libgdbm-dev @@ -91,13 +91,13 @@ gitlab-deps: - libyaml-dev - logrotate - openssh-server - - nodejs + - nodejs: ">=4.3" - pkg-config - python - python-docutils - rake - redis-server - - yarn + - yarn: ">=0.17" - zlib1g-dev {% if salt['pillar.get']('gitlab:db:engine', 'postgresql') == 'postgresql' %} - libpq-dev diff --git a/gitlab/repos.sls b/gitlab/repos.sls index d47d4f4..91068e7 100644 --- a/gitlab/repos.sls +++ b/gitlab/repos.sls @@ -9,6 +9,32 @@ PUIAS_6_computational: {% elif grains['os_family'] == 'Debian' %} {# TODO: Handling of packages should be moved to map.jinja #} +{# Gitlab 9.2+ requires golang-1.8+ which requires backports on Debian 9 and Ubuntu 16.04 #} +{%- set distro = grains.oscodename %} +gitlab-distro-backports: + file.managed: + - name: /etc/apt/preferences.d/55_gitlab_req_backports + {%- if grains.os == "Ubuntu" %} + - contents: | + Package: golang + Pin: release o=Ubuntu,a={{ distro }}-backports + Pin-Priority: 800 + {%- else %} + - contents: | + Package: golang + Pin: release o=Debian Backports,a={{ distro }}-backports + Pin-Priority: 800 + {%- endif %} + pkgrepo.managed: + {%- if grains.os == "Ubuntu" %} + - name: deb http://archive.ubuntu.com/ubuntu {{ distro }}-backports main + {%- else %} + - name: deb http://httpredir.debian.org/debian {{ distro }}-backports main + {%- endif %} + - file: /etc/apt/sources.list.d/gitlab_req_backports.list + - require_in: + - sls: gitlab.packages + {# Gitlab 8.17+ requires nodejs-4.3+ but is not available before Debian 9 or Ubuntu 16.10 #} gitlab-nodejs-repo-mgmt-pkgs: pkg.installed: From 7324780e631ec2bcba6dc14916da8a690532cfd8 Mon Sep 17 00:00:00 2001 From: Gilles Dartiguelongue Date: Fri, 1 Sep 2017 15:48:54 +0200 Subject: [PATCH 61/96] Update gitlab configuration for 9.2 --- gitlab/files/gitlab-gitlab.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/gitlab/files/gitlab-gitlab.yml b/gitlab/files/gitlab-gitlab.yml index df49dd8..cb3fb6a 100644 --- a/gitlab/files/gitlab-gitlab.yml +++ b/gitlab/files/gitlab-gitlab.yml @@ -197,7 +197,7 @@ production: &base stuck_ci_jobs_worker: cron: "0 * * * *" # Execute scheduled triggers - trigger_schedule_worker: + pipeline_schedule_worker: cron: "0 */12 * * *" # Remove expired build artifacts expire_build_artifacts_worker: @@ -518,9 +518,17 @@ production: &base upload_pack: true receive_pack: true + # Git import/fetch timeout + # git_timeout: 800 + # If you use non-standard ssh port you need to specify it # ssh_port: 22 + workhorse: + # File that contains the secret key for verifying access for gitlab-workhorse. + # Default is '.gitlab_workhorse_secret' relative to Rails.root (i.e. root of the GitLab app). + # secret_file: {{ root_dir }}/gitlab/.gitlab_workhorse_secret + ## Git settings # CAUTION! # Use the default values unless you really know what you are doing From ed2c6f7d8f13cdd06d2c8e5caef9d65b4f6ac045 Mon Sep 17 00:00:00 2001 From: Gilles Dartiguelongue Date: Fri, 15 Sep 2017 14:04:43 +0200 Subject: [PATCH 62/96] Add translation generation installation step --- gitlab/gitlab.sls | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/gitlab/gitlab.sls b/gitlab/gitlab.sls index 7d294c6..27e4dea 100644 --- a/gitlab/gitlab.sls +++ b/gitlab/gitlab.sls @@ -246,6 +246,22 @@ gitlab-migrate-db: - file: gitlab-db-config - cmd: gitlab-gems +gitlab-build-translations: + cmd.run: + - user: git + - cwd: {{ gitlab_dir }} + - name: bundle exec rake gettext:pack gettext:po_to_json + - env: + - RAILS_ENV: production + - onchanges: + {% if salt['pillar.get']('gitlab:archives:enabled', false) %} + - archive: gitlab-fetcher + {% else %} + - git: gitlab-fetcher + {% endif %} + - require: + - cmd: gitlab-gems + gitlab-yarn-install: cmd.run: - name: bundle exec rake yarn:install @@ -265,7 +281,7 @@ gitlab-yarn-install: - git: gitlab-fetcher {% endif %} - require: - - cmd: gitlab-gems + - cmd: gitlab-build-translations gitlab-recompile-assets-cache: cmd.run: From 85f866c8e24f831e1c1bf0483bc32312957660c1 Mon Sep 17 00:00:00 2001 From: Gilles Dartiguelongue Date: Fri, 15 Sep 2017 14:33:54 +0200 Subject: [PATCH 63/96] Move gitlab-shell secret file management to gitlab-shell.sls --- gitlab/gitlab-shell.sls | 8 ++++++++ gitlab/gitlab.sls | 8 -------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/gitlab/gitlab-shell.sls b/gitlab/gitlab-shell.sls index 856f0a8..8b536c7 100644 --- a/gitlab/gitlab-shell.sls +++ b/gitlab/gitlab-shell.sls @@ -119,3 +119,11 @@ gitlab-shell-copy: - onchanges: - archive: gitlab-shell-fetcher {% endif %} + +gitlab-shell-secret_file: + file.managed: + - name: {{ salt['pillar.get']('gitlab:shell:secret:path', root_dir ~ '/.gitlab_shell_secret') }} + - contents_pillar: gitlab:shell:secret:value + - user: git + - group: git + - mode: 640 diff --git a/gitlab/gitlab.sls b/gitlab/gitlab.sls index 27e4dea..f9680e1 100644 --- a/gitlab/gitlab.sls +++ b/gitlab/gitlab.sls @@ -378,11 +378,3 @@ gitlab-service: - file: gitlab-default - file: rack_attack-config - file: unicorn-config - -gitlab-secret_file: - file.managed: - - name: {{ salt['pillar.get']('gitlab:shell:secret:path', '/opt/git/.gitlab_shell_secret') }} - - contents_pillar: gitlab:shell:secret:value - - user: git - - group: git - - mode: 640 From 332758b54f2ed3e360763925cdd87cf6efcb6510 Mon Sep 17 00:00:00 2001 From: Gilles Dartiguelongue Date: Fri, 15 Sep 2017 14:35:13 +0200 Subject: [PATCH 64/96] Add gitlab-workhorse secret file management Added in 8.12. --- gitlab/files/initd | 3 ++- gitlab/gitlab-workhorse.sls | 7 +++++++ pillar.example | 4 ++++ 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/gitlab/files/initd b/gitlab/files/initd index 0c98487..4b065a9 100755 --- a/gitlab/files/initd +++ b/gitlab/files/initd @@ -27,6 +27,7 @@ {% set pids_dir = salt['pillar.get']('gitlab:lookup:pids_dir', root_dir ~ '/var/pids') %} {% set sockets_dir = salt['pillar.get']('gitlab:lookup:sockets_dir', root_dir ~ '/var/sockets') %} {% set logs_dir = salt['pillar.get']('gitlab:lookup:logs_dir', root_dir ~ '/var/logs') %} +{% set workhorse_secret = salt['pillar.get']('gitlab:workhorse:secret:path', root_dir ~ '/.gitlab_workhorse_secret') %} ### Environment variables RAILS_ENV="production" @@ -44,7 +45,7 @@ mail_room_enabled=false mail_room_pid_path="$pid_path/mail_room.pid" gitlab_workhorse_dir=$(cd {{ root_dir }}/gitlab-workhorse/bin 2> /dev/null && pwd) gitlab_workhorse_pid_path="$pid_path/gitlab-workhorse.pid" -gitlab_workhorse_options="-listenUmask 0 -listenNetwork unix -listenAddr $socket_path/gitlab-workhorse.socket -authBackend http://127.0.0.1:8080 -authSocket $rails_socket -documentRoot $app_root/public" +gitlab_workhorse_options="-listenUmask 0 -listenNetwork unix -listenAddr $socket_path/gitlab-workhorse.socket -authBackend http://127.0.0.1:8080 -authSocket $rails_socket -secretPath {{ workhorse_secret }} -documentRoot $app_root/public" gitlab_workhorse_log="{{ logs_dir }}/gitlab-workhorse.log" gitlab_pages_enabled=false gitlab_pages_dir=$(cd {{ root_dir }}/gitlab-pages/bin 2> /dev/null && pwd) diff --git a/gitlab/gitlab-workhorse.sls b/gitlab/gitlab-workhorse.sls index bddba04..ec4aaac 100644 --- a/gitlab/gitlab-workhorse.sls +++ b/gitlab/gitlab-workhorse.sls @@ -63,3 +63,10 @@ gitlab-workhorse-make: - git: gitlab-workhorse-fetcher {% endif %} +gitlab-workhorse-secret_file: + file.managed: + - name: {{ salt['pillar.get']('gitlab:shell:workhorse:path', root_dir ~ '/.gitlab_workhorse_secret') }} + - contents_pillar: gitlab:workhorse:secret:value + - user: git + - group: git + - mode: 640 diff --git a/pillar.example b/pillar.example index 4c30cf8..bdbdfe0 100644 --- a/pillar.example +++ b/pillar.example @@ -86,6 +86,10 @@ gitlab: secret: value: 0123456789abcdef0123456789abcde path: "/opt/git/.gitlab_shell_secret" + workhorse + secret: + value: 0123456789abcdef0123456789zyxwvu + path: "/opt/git/.gitlab_workhorse_secret" unicorn: worker_processes: 2 From d76a0990cf5c513f1716b6e0a04be2dcbf04765b Mon Sep 17 00:00:00 2001 From: Gilles Dartiguelongue Date: Fri, 15 Sep 2017 16:52:52 +0200 Subject: [PATCH 65/96] Update yarn repository GPG key --- gitlab/files/dl.yarn.com.key | 39 ++++++++++++++++++++++++++++++++++-- 1 file changed, 37 insertions(+), 2 deletions(-) diff --git a/gitlab/files/dl.yarn.com.key b/gitlab/files/dl.yarn.com.key index 3cf9b34..e1cb0b6 100644 --- a/gitlab/files/dl.yarn.com.key +++ b/gitlab/files/dl.yarn.com.key @@ -1,4 +1,5 @@ -----BEGIN PGP PUBLIC KEY BLOCK----- +Version: GnuPG v2 mQINBFf0j5oBEADS6cItqCbf4lOLICohq2aHqM5I1jsz3DC4ddIU5ONbKXP1t0wk FEUPRzd6m80cTo7Q02Bw7enh4J6HvM5XVBSSGKENP6XAsiOZnY9nkXlcQAPFRnCn @@ -113,6 +114,40 @@ fOGvA8nuXkC5nI6ukw4c4zwmJ5+SNQthFUYKWLd4hR4qrCoJkMEWZmsCRtqxjVCJ kWi7tMCGqPQmtUFRnUvHPzCDuXLYT8lRxhTxDi3T5MXdIUlAUTcNpwG8Ill0xkGc pMlh0D5p44GEdMFfJiXw6AUETHcqC2qZr2rP9kpzvVlapIrsPRg/DU+s70YnccI3 iMCVm4/WrghFeK232zkjiwRVOm+IEWBlDFrm4MMjfguUeneYbK9WhqJnss9nc4QK -Vhzuyn3GTtg1w/T6CaYVXBjcHFk= -=HsHU +Vhzuyn3GTtg1w/T6CaYVXBjcHFm5Ag0EWbWWowEQALCiEk5Ic40W7/v5hqYNjrRl +xTE/1axOhhzt8eCB7eOeNOMQKwabYxqBceNmol/guzlnFqLtbaA6yZQkzz/K3eNw +WQg7CfXO3+p/dN0HtktPfdCk+kY/t7StKRjINW6S9xk9KshiukmdiDq8JKS0Hgxq +phBB3tDjmo6/RiaOEFMoUlXKSU+BYYpBpLKg53P8F/8nIsK2aZJyk8XuBd0UXKI+ +N1gfCfzoDWnYHs73LQKcjrTaZQauT81J7+TeWoLI28vkVxyjvTXAyjSBnhxTYfwU +NGSoawEXyJ1uKCwhIpklxcCMI9Hykg7sKNsvmJ4uNcRJ7cSRfb0g5DR9dLhR+eEv +Fd+o4PblKk16AI48N8Zg1dLlJuV2cAtl0oBPk+tnbZukvkS5n1IzTSmiiPIXvK2t +506VtfFEw4iZrJWf2Q9//TszBM3r1FPATLH7EAeG5P8RV+ri7L7NvzP6ZQClRDUs +xeimCSe8v/t0OpheCVMlM9TpVcKGMw8ig/WEodoLOP4iqBs4BKR7fuydjDqbU0k/ +sdJTltp7IIdK1e49POIQ7pt+SUrsq/HnPW4woLC1WjouBWyr2M7/a0SldPidZ2BU +AK7O9oXosidZMJT7dBp3eHrspY4bdkSxsd0nshj0ndtqNktxkrSFRkoFpMz0J/M3 +Q93CjdHuTLpTHQEWjm/7ABEBAAGJBEQEGAEIAA8FAlm1lqMCGwIFCQJ2LQACKQkQ +FkawG4blAxDBXSAEGQEIAAYFAlm1lqMACgkQ4HTRbrb/TeMpDQ//eOIsCWY2gYOG +ACw42JzMVvuTDrgRT4hMhgHCGeKzn1wFL1EsbSQV4Z6pYvnNayuEakgIz14wf4UF +s5u1ehfBwatmakSQJn32ANcAvI0INAkLEoqqy81mROjMc9FFrOkdqjcN7yN0BzH9 +jNYL/gsvmOOwOu+dIH3C1Lgei844ZR1BZK1900mohuRwcji0sdROMcrKrGjqd4yb +6f7yl0wbdAxA3IHT3TFGczC7Y41P2OEpaJeVIZZgxkgQsJ14qK/QGpdKvmZAQpjH +BipeO/H+qxyOT5Y+f15VLWGOOVL090+ZdtF7h3m4X2+L7xWsFIgdOprfO60gq3e7 +9YFfgNBYU5BGtJGFGlJ0sGtnpzx5QCRka0j/1E5lIu00sW3WfGItFd48hW6wHClo +yoi7pBR7xqSEoU/U5o7+nC8wHFrDYyqcyO9Q3mZDw4LvlgnyMOM+qLv/fNgO9USE +4T30eSvc0t/5p1hCKNvyxHFghdRSJqn70bm6MQY+kd6+B/k62Oy8eCwRt4PR+LQE +IPnxN7xGuNpVO1oMyhhO41osYruMrodzw81icBRKYFlSuDOQ5jlcSajc6TvF22y+ +VXy7nx1q/CN4tzB/ryUASU+vXS8/QNM6qI/QbbgBy7VtHqDbs2KHp4cP0j9KYQzM +rKwtRwfHqVrwFLkCp61EHwSlPsEFiglpMg/8DQ92O4beY0n7eSrilwEdJg89Ieep +TBm1QYiLM33qWLR9CABYAIiDG7qxviHozVfX6kUwbkntVpyHAXSbWrM3kD6jPs3u +/dimLKVyd29AVrBSn9FC04EjtDWsj1KB7HrFN4oo9o0JLSnXeJb8FnPf3MitaKlt +vj/kZhegozIs+zvpzuri0LvoB4fNA0T4eAmxkGkZBB+mjNCrUHIakyPZVzWGL0QG +sfK1Q9jvw0OErqHJYX8A1wLre/HkBne+e5ezS6Mc7kFW33Y1arfbHFNAe12juPsO +xqK76qNilUbQpPtNvWP3FTpbkAdodMLq/gQ+M5yHwPe8SkpZ8wYCfcwEemz/P+4Q +hQB8tbYbpcPxJ+aQjVjcHpsLdrlSY3JL/gqockR7+97GrCzqXbgvsqiWr16Zyn6m +xYWEHn9HXMh3b+2IYKFFXHffbIBq/mfibDnZtQBrZpn2uyh6F2ZuOsZh0LTD7RL5 +3KV3fi90nS00Gs1kbMkPycL1JLqvYQDpllE2oZ1dKDYkwivGyDQhRNfERL6Jkjyi +SxfZ2c84r2HPgnJTi/WBplloQkM+2NfXrBo6kLHSC6aBndRKk2UmUhrUluGcQUyf +zYRFH5kVueIYfDaBPus9gb+sjnViFRpqVjefwlXSJEDHWP3Cl2cuo2mJjeDghj40 +0U6pjSUW3bIC/PI= +=BxMn -----END PGP PUBLIC KEY BLOCK----- From 76d3f0619d4ddb1fed3d9cd2a8f4c88245840f77 Mon Sep 17 00:00:00 2001 From: Gilles Dartiguelongue Date: Fri, 15 Sep 2017 18:21:12 +0200 Subject: [PATCH 66/96] Make gitlab-shell secret file default respect root_dir --- gitlab/files/gitlab-gitlab.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitlab/files/gitlab-gitlab.yml b/gitlab/files/gitlab-gitlab.yml index cb3fb6a..8910274 100644 --- a/gitlab/files/gitlab-gitlab.yml +++ b/gitlab/files/gitlab-gitlab.yml @@ -512,7 +512,7 @@ production: &base # File that contains the secret key for verifying access for gitlab-shell. # Default is '.gitlab_shell_secret' relative to Rails.root (i.e. root of the GitLab app). - secret_file: "{{ salt['pillar.get']('gitlab:secret_file', '/opt/git/.gitlab_shell_secret') }}" + secret_file: "{{ salt['pillar.get']('gitlab:secret_file', root_dir ~ '/.gitlab_shell_secret') }}" # Git over HTTP upload_pack: true From 4c395dfb695226e1c0abb22ec2131fc5255b24d1 Mon Sep 17 00:00:00 2001 From: Gilles Dartiguelongue Date: Fri, 1 Sep 2017 15:48:54 +0200 Subject: [PATCH 67/96] Update gitlab configuration for 9.3 --- gitlab/files/gitlab-gitlab.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/gitlab/files/gitlab-gitlab.yml b/gitlab/files/gitlab-gitlab.yml index 8910274..20c3733 100644 --- a/gitlab/files/gitlab-gitlab.yml +++ b/gitlab/files/gitlab-gitlab.yml @@ -185,7 +185,7 @@ production: &base ## Gravatar ## For Libravatar see: http://doc.gitlab.com/ce/customization/libravatar.html gravatar: - # gravatar urls: possible placeholders: %{hash} %{size} %{email} + # gravatar urls: possible placeholders: %{hash} %{size} %{email} %{username} # plain_url: "http://..." # default: http://www.gravatar.com/avatar/%{hash}?s=%{size}&d=identicon # ssl_url: "https://..." # default: https://secure.gravatar.com/avatar/%{hash}?s=%{size}&d=identicon @@ -198,7 +198,7 @@ production: &base cron: "0 * * * *" # Execute scheduled triggers pipeline_schedule_worker: - cron: "0 */12 * * *" + cron: "19 * * * *" # Remove expired build artifacts expire_build_artifacts_worker: cron: "50 * * * *" @@ -353,6 +353,10 @@ production: &base # showing GitLab's sign-in page (default: show the GitLab sign-in page) # auto_sign_in_with_provider: saml + # Sync user's email address from the specified Omniauth provider every time the user logs + # in (default: nil). And consequently make this field read-only. + # sync_email_from_provider: cas3 + # CAUTION! # This allows users to login without having a user account first. Define the allowed providers # using an array, e.g. ["saml", "twitter"], or as true/false to allow all providers or none. From 72238069f48690f8b26bf1e3055f4cc4e04f9808 Mon Sep 17 00:00:00 2001 From: Gilles Dartiguelongue Date: Fri, 1 Sep 2017 15:46:38 +0200 Subject: [PATCH 68/96] Update init script and configuration for 9.3 --- gitlab/files/gitlab-default | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gitlab/files/gitlab-default b/gitlab/files/gitlab-default index c074aed..17efd7a 100644 --- a/gitlab/files/gitlab-default +++ b/gitlab/files/gitlab-default @@ -98,3 +98,5 @@ shell_path="/bin/bash" # This variable controls whether the init script starts/stops Gitaly gitaly_enabled=false gitaly_log="{{ logs_dir }}/gitaly.log" +gitaly_pid_path="$pid_path/gitaly.pid" +gitaly_log="{{ logs_dir }}/log/gitaly.log" From f796c2554228137af9f6c2ad64e0535a27c7bc83 Mon Sep 17 00:00:00 2001 From: Gilles Dartiguelongue Date: Mon, 18 Sep 2017 10:28:41 +0200 Subject: [PATCH 69/96] Compile gitlab-shell Otherwise gitaly-* helpers are not present. Why it doesn't use make for compile/install is a mystery. --- gitlab/gitlab-shell.sls | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/gitlab/gitlab-shell.sls b/gitlab/gitlab-shell.sls index 8b536c7..d79dcae 100644 --- a/gitlab/gitlab-shell.sls +++ b/gitlab/gitlab-shell.sls @@ -63,6 +63,18 @@ gitlab-shell-config: - git: gitlab-shell-fetcher {% endif %} +gitlab-shell-compile: + cmd.run: + - user: git + - cwd: {{ shell_dir_content }} + - name: ./bin/compile + - onchanges: + {% if salt['pillar.get']('gitlab:archives:enabled', false) %} + - archive: gitlab-shell-fetcher + {% else %} + - git: gitlab-shell-fetcher + {% endif %} + gitlab-shell: cmd.wait: - user: git @@ -77,6 +89,7 @@ gitlab-shell: {% endif %} - require: - file: gitlab-shell-config + - cmd: gitlab-shell-compile #gitlab-shell-chmod-bin: # file.directory: From 2cad8d228cda64738ada9ff07b3accb98bdbd5cb Mon Sep 17 00:00:00 2001 From: Gilles Dartiguelongue Date: Wed, 18 Oct 2017 11:40:03 +0200 Subject: [PATCH 70/96] Add initial gitaly support --- gitlab/files/gitaly-config.toml | 35 ++++++++++++++ gitlab/files/gitlab-gitlab.yml | 3 +- gitlab/gitaly.sls | 81 +++++++++++++++++++++++++++++++++ 3 files changed, 118 insertions(+), 1 deletion(-) create mode 100644 gitlab/files/gitaly-config.toml create mode 100644 gitlab/gitaly.sls diff --git a/gitlab/files/gitaly-config.toml b/gitlab/files/gitaly-config.toml new file mode 100644 index 0000000..f7512ea --- /dev/null +++ b/gitlab/files/gitaly-config.toml @@ -0,0 +1,35 @@ +{%- set root_dir = salt['pillar.get']('gitlab:lookup:root_dir', '/home/git') %} +{%- set sockets_dir = salt['pillar.get']('gitlab:lookup:sockets_dir', root_dir ~ '/var/sockets') %} +{%- set repositories = salt['pillar.get']('gitlab:lookup:repositories', root_dir ~ '/repositories') %} +# Example Gitaly configuration file + +socket_path = "{{ sockets_dir }}/private/gitaly.socket" + +# # Optional: listen on a TCP socket. This is insecure (no authentication) +# listen_addr = "localhost:9999" +# + +# # Optional: export metrics via Prometheus +# prometheus_listen_addr = "localhost:9236" +# + +[[storage]] +name = "default" +path = "{{ repositories }}" + +# # You can optionally configure more storages for this Gitaly instance to serve up +# +# [[storage]] +# name = "other_storage" +# path = "/mnt/other_storage/repositories" +# + +# # You can optionally configure Gitaly to output JSON-formatted log messages to stdout +# [logging] +# format = "json" +# # Additionally exceptions can be reported to Sentry +# sentry_dsn = "https://:@sentry.io/" + +# # You can optionally configure Gitaly to record histogram latencies on GRPC method calls +# [prometheus] +# grpc_latency_buckets = [0.001, 0.005, 0.025, 0.1, 0.5, 1.0, 10.0, 30.0, 60.0, 300.0, 1500.0] diff --git a/gitlab/files/gitlab-gitlab.yml b/gitlab/files/gitlab-gitlab.yml index 20c3733..a1be60f 100644 --- a/gitlab/files/gitlab-gitlab.yml +++ b/gitlab/files/gitlab-gitlab.yml @@ -24,6 +24,7 @@ {% set root_dir = salt['pillar.get']('gitlab:lookup:root_dir', '/home/git') %} {% set lib_dir = salt['pillar.get']('gitlab:lookup:lib_dir', root_dir ~ '/libraries') %} {% set repositories = salt['pillar.get']('gitlab:lookup:repositories', root_dir ~ '/repositories') %} +{% set sockets_dir = salt['pillar.get']('gitlab:lookup:sockets_dir', root_dir ~ '/var/sockets') %} {% set shell_dir = lib_dir ~ "/gitlab-shell" %} {% if salt['pillar.get']('gitlab:archives:enabled', false) %} @@ -484,7 +485,7 @@ production: &base storages: # You must have at least a `default` storage path. default: path: {{ repositories }} - gitaly_address: unix:{{ root_dir }}/gitlab/tmp/sockets/private/gitaly.socket # TCP connections are supported too (e.g. tcp://host:port) + gitaly_address: unix:{{ sockets_dir }}/private/gitaly.socket # TCP connections are supported too (e.g. tcp://host:port) ## Backup settings backup: diff --git a/gitlab/gitaly.sls b/gitlab/gitaly.sls new file mode 100644 index 0000000..8b70963 --- /dev/null +++ b/gitlab/gitaly.sls @@ -0,0 +1,81 @@ +{%- set root_dir = salt['pillar.get']('gitlab:lookup:root_dir', '/home/git') %} +{%- set sockets_dir = salt['pillar.get']('gitlab:lookup:sockets_dir', root_dir ~ '/var/sockets') %} +{%- set lib_dir = salt['pillar.get']('gitlab:lookup:lib_dir', root_dir ~ '/libraries') %} + +{%- set gitaly_dir = lib_dir ~ "/gitaly" %} + +{%- if salt['pillar.get']('gitlab:archives:enabled', false) %} + {%- set gitaly_dir_content = gitaly_dir ~ '/' ~ salt['pillar.get']('gitlab:archives:sources:gitaly:content') %} +{%- else %} + {%- set gitaly_dir_content = gitaly_dir %} +{%- endif %} + +{%- if salt['pillar.get']('gitlab:archives:enabled', false) %} +gitaly-fetcher: + archive.extracted: + - name: {{ gitaly_dir }} + - source: {{ salt['pillar.get']('gitlab:archives:sources:gitaly:source') }} + - source_hash: md5={{ salt['pillar.get']('gitlab:archives:sources:gitaly:md5') }} + - archive_format: tar + - if_missing: {{ gitaly_dir_content }} + - keep: True + +gitaly-chown: + file.directory: + - name: {{ gitaly_dir }} + - user: git + - group: git + - recurse: + - user + - onchanges: + - archive: gitaly-fetcher +{%- else %} +gitaly-fetcher: + git.latest: + - name: https://gitlab.com/gitlab-org/gitaly.git + - rev: {{ salt['pillar.get']('gitlab:gitaly_version') }} + - target: {{ gitaly_dir_content }} + - user: git + - force: True + - require: + - pkg: gitlab-deps + - pkg: git + - sls: gitlab.ruby + - file: git-home +{%- endif %} + +# https://gitlab.com/gitlab-org/gitaly/blob/master/config.toml.example +gitaly-config: + file.managed: + - name: {{ gitaly_dir_content }}/config.toml + - source: salt://gitlab/files/gitaly-config.toml + - template: jinja + - user: git + - group: git + - mode: 644 + - require: + - gitaly-fetcher + +gitaly-private-sockets-dir: + file.directory: + - name: {{ sockets_dir }}/private + - user: git + - group: git + - mode: 700 + +gitaly-bin-dir: + file.directory: + - name: {{ root_dir }}/gitaly + - user: git + - group: git + - mode: 750 + +gitaly-make: + cmd.run: + - name: make build install DESTDIR={{ root_dir }}/gitaly PREFIX= + - user: git + - cwd: {{ gitaly_dir_content }} + - onchanges: + - gitaly-fetcher + - require: + - file: gitaly-bin-dir From 1d3d107381e174e2cf40827311a7e2ad02cb5b96 Mon Sep 17 00:00:00 2001 From: Gilles Dartiguelongue Date: Wed, 18 Oct 2017 11:42:24 +0200 Subject: [PATCH 71/96] Switch gitaly on Defaulted to true in GitLab 9.3 and becoming mandatory in 9.4. --- gitlab/files/gitlab-default | 2 +- gitlab/files/gitlab-gitlab.yml | 2 +- gitlab/files/initd | 2 +- gitlab/init.sls | 1 + 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/gitlab/files/gitlab-default b/gitlab/files/gitlab-default index 17efd7a..920193d 100644 --- a/gitlab/files/gitlab-default +++ b/gitlab/files/gitlab-default @@ -96,7 +96,7 @@ mail_room_pid_path="$pid_path/mail_room.pid" shell_path="/bin/bash" # This variable controls whether the init script starts/stops Gitaly -gitaly_enabled=false +gitaly_enabled=true gitaly_log="{{ logs_dir }}/gitaly.log" gitaly_pid_path="$pid_path/gitaly.pid" gitaly_log="{{ logs_dir }}/log/gitaly.log" diff --git a/gitlab/files/gitlab-gitlab.yml b/gitlab/files/gitlab-gitlab.yml index a1be60f..cce3335 100644 --- a/gitlab/files/gitlab-gitlab.yml +++ b/gitlab/files/gitlab-gitlab.yml @@ -470,7 +470,7 @@ production: &base # This setting controls whether GitLab uses Gitaly (new component # introduced in 9.0). Eventually Gitaly use will become mandatory and # this option will disappear. - enabled: false + enabled: true # # 4. Advanced settings diff --git a/gitlab/files/initd b/gitlab/files/initd index 4b065a9..689e1a6 100755 --- a/gitlab/files/initd +++ b/gitlab/files/initd @@ -53,7 +53,7 @@ gitlab_pages_pid_path="$pid_path/gitlab-pages.pid" gitlab_pages_options="-pages-domain example.com -pages-root $app_root/shared/pages -listen-proxy 127.0.0.1:8090" gitlab_pages_log="{{ logs_dir }}/gitlab-pages.log" shell_path="/bin/bash" -gitaly_enabled=false +gitaly_enabled=true gitaly_dir=$(cd {{ root_dir }}/gitaly/bin 2> /dev/null && pwd) gitaly_pid_path="$pid_path/gitaly.pid" gitaly_log="{{ logs_dir }}/gitaly.log" diff --git a/gitlab/init.sls b/gitlab/init.sls index e2bf1f3..ba1c4b8 100644 --- a/gitlab/init.sls +++ b/gitlab/init.sls @@ -9,4 +9,5 @@ include: - gitlab.ruby - gitlab.gitlab-shell - gitlab.gitlab-workhorse + - gitlab.gitaly - gitlab.gitlab From 1553ae58a723cca16e3f79626ede05187d26c58e Mon Sep 17 00:00:00 2001 From: Gilles Dartiguelongue Date: Wed, 18 Oct 2017 13:19:30 +0200 Subject: [PATCH 72/96] Fix golang repository handling for Ubuntu apt pkg backend doesn't deal well with >= version specifications. Try to workaround it. --- gitlab/packages.sls | 14 +++++++++++++- gitlab/repos.sls | 10 +++++----- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/gitlab/packages.sls b/gitlab/packages.sls index 29f5ac1..75af8a0 100644 --- a/gitlab/packages.sls +++ b/gitlab/packages.sls @@ -1,3 +1,14 @@ +{%- if grains.os_family == 'Debian' %} +# aptpkg does not deal with >= versions +gitlab-golang-deps: + pkg.installed: + - pkgs: + - golang + - golang-1.8 + {%- if grains.os == "Ubuntu" and grains.osrelease_info[0] < 17 %} + - fromrepo: artful + {%- endif %} +{%- endif %} gitlab-deps: pkg.installed: @@ -73,7 +84,6 @@ gitlab-deps: - checkinstall - curl - cmake - - golang: ">=1.8" - libcurl4-openssl-dev - libffi-dev - libgdbm-dev @@ -102,6 +112,8 @@ gitlab-deps: {% if salt['pillar.get']('gitlab:db:engine', 'postgresql') == 'postgresql' %} - libpq-dev {% endif %} + - require: + - pkg: gitlab-golang-deps {% endif %} {% if salt['pillar.get']('gitlab:use_rvm', False) %} diff --git a/gitlab/repos.sls b/gitlab/repos.sls index 91068e7..5a50aa1 100644 --- a/gitlab/repos.sls +++ b/gitlab/repos.sls @@ -9,15 +9,15 @@ PUIAS_6_computational: {% elif grains['os_family'] == 'Debian' %} {# TODO: Handling of packages should be moved to map.jinja #} -{# Gitlab 9.2+ requires golang-1.8+ which requires backports on Debian 9 and Ubuntu 16.04 #} +{# Gitlab 9.2+ requires golang-1.8+ which requires backports on Debian 9 and Artful repositories on Ubuntu #} {%- set distro = grains.oscodename %} gitlab-distro-backports: file.managed: - name: /etc/apt/preferences.d/55_gitlab_req_backports - {%- if grains.os == "Ubuntu" %} + {%- if grains.os == "Ubuntu" and grains.osrelease_info[0] < 17 %} - contents: | Package: golang - Pin: release o=Ubuntu,a={{ distro }}-backports + Pin: release o=Ubuntu,a=artful Pin-Priority: 800 {%- else %} - contents: | @@ -26,8 +26,8 @@ gitlab-distro-backports: Pin-Priority: 800 {%- endif %} pkgrepo.managed: - {%- if grains.os == "Ubuntu" %} - - name: deb http://archive.ubuntu.com/ubuntu {{ distro }}-backports main + {%- if grains.os == "Ubuntu" and grains.osrelease_info[0] < 17 %} + - name: deb http://archive.ubuntu.com/ubuntu artful main {%- else %} - name: deb http://httpredir.debian.org/debian {{ distro }}-backports main {%- endif %} From 40e69d8d5286e7b8d6b091244156dec9ac28bf07 Mon Sep 17 00:00:00 2001 From: Gilles Dartiguelongue Date: Wed, 18 Oct 2017 15:57:30 +0200 Subject: [PATCH 73/96] Fix Pin-Priority of golang --- gitlab/repos.sls | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gitlab/repos.sls b/gitlab/repos.sls index 5a50aa1..760eec2 100644 --- a/gitlab/repos.sls +++ b/gitlab/repos.sls @@ -18,12 +18,12 @@ gitlab-distro-backports: - contents: | Package: golang Pin: release o=Ubuntu,a=artful - Pin-Priority: 800 + Pin-Priority: 901 {%- else %} - contents: | Package: golang Pin: release o=Debian Backports,a={{ distro }}-backports - Pin-Priority: 800 + Pin-Priority: 901 {%- endif %} pkgrepo.managed: {%- if grains.os == "Ubuntu" and grains.osrelease_info[0] < 17 %} From c7136faada29275dce8f8cff596f30f05ea63a15 Mon Sep 17 00:00:00 2001 From: Gilles Dartiguelongue Date: Wed, 18 Oct 2017 16:10:51 +0200 Subject: [PATCH 74/96] Try to fix dependency issue Salt is complaining about missing IDs again during highstate. --- gitlab/packages.sls | 4 ++++ gitlab/repos.sls | 6 ------ 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/gitlab/packages.sls b/gitlab/packages.sls index 75af8a0..3f88dff 100644 --- a/gitlab/packages.sls +++ b/gitlab/packages.sls @@ -8,6 +8,8 @@ gitlab-golang-deps: {%- if grains.os == "Ubuntu" and grains.osrelease_info[0] < 17 %} - fromrepo: artful {%- endif %} + - require: + - pkgrepo: gitlab-distro-backports {%- endif %} gitlab-deps: @@ -113,6 +115,8 @@ gitlab-deps: - libpq-dev {% endif %} - require: + - pkgrepo: gitlab-nodejs-repo + - pkgrepo: gitlab-yarn-repo - pkg: gitlab-golang-deps {% endif %} diff --git a/gitlab/repos.sls b/gitlab/repos.sls index 760eec2..a0bb624 100644 --- a/gitlab/repos.sls +++ b/gitlab/repos.sls @@ -32,8 +32,6 @@ gitlab-distro-backports: - name: deb http://httpredir.debian.org/debian {{ distro }}-backports main {%- endif %} - file: /etc/apt/sources.list.d/gitlab_req_backports.list - - require_in: - - sls: gitlab.packages {# Gitlab 8.17+ requires nodejs-4.3+ but is not available before Debian 9 or Ubuntu 16.10 #} gitlab-nodejs-repo-mgmt-pkgs: @@ -58,14 +56,10 @@ gitlab-nodejs-preference: Package: nodejs Pin: release o=Node source,l=Node source Pin-Priority: 901 - - require_in: - - sls: gitlab.packages gitlab-yarn-repo: pkgrepo.managed: - name: deb https://dl.yarnpkg.com/debian/ stable main - file: /etc/apt/sources.list.d/yarn.list - key_url: salt://gitlab/files/dl.yarn.com.key - - require_in: - - sls: gitlab.packages {% endif %} From 8baae9f5367f6cc69f2fb615c170217b255580e2 Mon Sep 17 00:00:00 2001 From: Gilles Dartiguelongue Date: Wed, 18 Oct 2017 16:21:39 +0200 Subject: [PATCH 75/96] Fix paths for gitaly --- gitlab/files/gitlab-default | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gitlab/files/gitlab-default b/gitlab/files/gitlab-default index 920193d..10fca61 100644 --- a/gitlab/files/gitlab-default +++ b/gitlab/files/gitlab-default @@ -97,6 +97,6 @@ shell_path="/bin/bash" # This variable controls whether the init script starts/stops Gitaly gitaly_enabled=true -gitaly_log="{{ logs_dir }}/gitaly.log" +gitaly_dir=$(cd {{ root_dir }}/gitaly/bin && pwd) gitaly_pid_path="$pid_path/gitaly.pid" -gitaly_log="{{ logs_dir }}/log/gitaly.log" +gitaly_log="{{ logs_dir }}/gitaly.log" From f2b437c4e0401b7ffd41e623e5144efaf040215e Mon Sep 17 00:00:00 2001 From: Gilles Dartiguelongue Date: Wed, 18 Oct 2017 16:23:34 +0200 Subject: [PATCH 76/96] Move gitaly configuration file where binary is --- gitlab/gitaly.sls | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/gitlab/gitaly.sls b/gitlab/gitaly.sls index 8b70963..ee6015a 100644 --- a/gitlab/gitaly.sls +++ b/gitlab/gitaly.sls @@ -44,18 +44,6 @@ gitaly-fetcher: - file: git-home {%- endif %} -# https://gitlab.com/gitlab-org/gitaly/blob/master/config.toml.example -gitaly-config: - file.managed: - - name: {{ gitaly_dir_content }}/config.toml - - source: salt://gitlab/files/gitaly-config.toml - - template: jinja - - user: git - - group: git - - mode: 644 - - require: - - gitaly-fetcher - gitaly-private-sockets-dir: file.directory: - name: {{ sockets_dir }}/private @@ -79,3 +67,18 @@ gitaly-make: - gitaly-fetcher - require: - file: gitaly-bin-dir + +# https://gitlab.com/gitlab-org/gitaly/blob/master/config.toml.example +# gitaly looks for configuration in the same directory it is running from +gitaly-config: + file.managed: + - name: {{ root_dir }}/gitaly/bin/config.toml + - source: salt://gitlab/files/gitaly-config.toml + - template: jinja + - user: git + - group: git + - mode: 644 + - require: + - gitaly-fetcher + - file: gitaly-bin-dir + - cmd: gitaly-make \ No newline at end of file From a5b2d73f36d292b05de2c395a0266eed22ac4a9a Mon Sep 17 00:00:00 2001 From: Gilles Dartiguelongue Date: Wed, 18 Oct 2017 18:49:23 +0200 Subject: [PATCH 77/96] Update gitlab config for 9.4 support --- gitlab/files/gitaly-config.toml | 4 +++ gitlab/files/gitlab-gitlab.yml | 63 ++++++++++++++++++++++++++++----- 2 files changed, 58 insertions(+), 9 deletions(-) diff --git a/gitlab/files/gitaly-config.toml b/gitlab/files/gitaly-config.toml index f7512ea..76d901f 100644 --- a/gitlab/files/gitaly-config.toml +++ b/gitlab/files/gitaly-config.toml @@ -13,6 +13,10 @@ socket_path = "{{ sockets_dir }}/private/gitaly.socket" # prometheus_listen_addr = "localhost:9236" # +# # Git executable settings +# [git] +# bin_path = "/usr/bin/git" + [[storage]] name = "default" path = "{{ repositories }}" diff --git a/gitlab/files/gitlab-gitlab.yml b/gitlab/files/gitlab-gitlab.yml index cce3335..7664436 100644 --- a/gitlab/files/gitlab-gitlab.yml +++ b/gitlab/files/gitlab-gitlab.yml @@ -245,7 +245,8 @@ production: &base # ========================== ## LDAP settings - # You can inspect a sample of the LDAP users with login access by running: + # You can test connections and inspect a sample of the LDAP users with login + # access by running: # bundle exec rake gitlab:ldap:check RAILS_ENV=production ldap: enabled: {{ salt['pillar.get']('gitlab:ldap:enabled', false) }} @@ -268,17 +269,48 @@ production: &base # Example: 'Paris' or 'Acme, Ltd.' label: {{ salt['pillar.get']('gitlab:ldap:label', 'LDAP') }} + # Example: 'ldap.mydomain.com' host: '{{ salt['pillar.get']('gitlab:ldap:host', '') }}' port: {{ salt['pillar.get']('gitlab:ldap:port', 636) }} uid: '{{ salt['pillar.get']('gitlab:ldap:uid', 'sAMAccountName') }}' - method: '{{ salt['pillar.get']('gitlab:ldap:method', 'ssl') }}' # "tls" or "ssl" or "plain" + + # Examples: 'america\\momo' or 'CN=Gitlab Git,CN=Users,DC=mydomain,DC=com' bind_dn: '{{ salt['pillar.get']('gitlab:ldap:bind_dn', '') }}' password: '{{ salt['pillar.get']('gitlab:ldap:password', '') }}' + # Encryption method. The "method" key is deprecated in favor of + # "encryption". + # + # Examples: "start_tls" or "simple_tls" or "plain" + # + # Deprecated values: "tls" was replaced with "start_tls" and "ssl" was + # replaced with "simple_tls". + # + encryption: {{ salt['pillar.get']('gitlab:ldap:encryption', 'plain') }} + + # Enables SSL certificate verification if encryption method is + # "start_tls" or "simple_tls". (Defaults to false for backward- + # compatibility) + verify_certificates: {{ salt['pillar.get']('gitlab:ldap:verify_certificates', false) }} + + # Specifies the path to a file containing a PEM-format CA certificate, + # e.g. if you need to use an internal CA. + # + # Example: '/etc/ca.pem' + # + ca_file: {{ salt['pillar.get']('gitlab:ldap:ca_file', '') }} + + # Specifies the SSL version for OpenSSL to use, if the OpenSSL default + # is not appropriate. + # + # Example: 'TLSv1_1' + # + ssl_version: {{ salt['pillar.get']('gitlab:ldap:ssl_version', '') }} + # Set a timeout, in seconds, for LDAP queries. This helps avoid blocking # a request if the LDAP server becomes unresponsive. # A value of 0 means there is no timeout. - timeout: 10 + timeout: {{ salt['pillar.get']('gitlab:ldap:timeout', 10) }} # This setting specifies if LDAP server is Active Directory LDAP server. # For non AD servers it skips the AD specific queries. @@ -303,17 +335,20 @@ production: &base # Base where we can search for users # - # Ex. ou=People,dc=gitlab,dc=example + # Ex. 'ou=People,dc=gitlab,dc=example' or 'DC=mydomain,DC=com' # base: {{ salt['pillar.get']('gitlab:ldap:base', '') }} # Filter LDAP users # - # Format: RFC 4515 http://tools.ietf.org/search/rfc4515 + # Format: RFC 4515 https://tools.ietf.org/search/rfc4515 # Ex. (employeeType=developer) # # Note: GitLab does not support omniauth-ldap's custom filter syntax. # + # Example for getting only specific users: + # '(&(objectclass=user)(|(samaccountname=momo)(samaccountname=toto)))' + # user_filter: {{ salt['pillar.get']('gitlab:ldap:user_filter', '') }} # LDAP attributes that GitLab will use to create an account for the LDAP user. @@ -467,10 +502,10 @@ production: &base # Gitaly settings gitaly: - # This setting controls whether GitLab uses Gitaly (new component - # introduced in 9.0). Eventually Gitaly use will become mandatory and - # this option will disappear. - enabled: true + # Default Gitaly authentication token. Can be overriden per storage. Can + # be left blank when Gitaly is running locally on a Unix socket, which + # is the normal way to deploy Gitaly. + token: # # 4. Advanced settings @@ -486,6 +521,7 @@ production: &base default: path: {{ repositories }} gitaly_address: unix:{{ sockets_dir }}/private/gitaly.socket # TCP connections are supported too (e.g. tcp://host:port) + # gitaly_token: 'special token' # Optional: override global gitaly.token for this storage. ## Backup settings backup: @@ -556,6 +592,15 @@ production: &base # host: localhost # port: 3808 + ## Monitoring + # Built in monitoring settings + monitoring: + # Time between sampling of unicorn socket metrics, in seconds + # unicorn_sampler_interval: 10 + # IP whitelist to access monitoring endpoints + ip_whitelist: + - 127.0.0.0/8 + # # 5. Extra customization # ========================== From 9e7e738d37ec5ea661fccb615816ac89627c86cf Mon Sep 17 00:00:00 2001 From: Gilles Dartiguelongue Date: Wed, 18 Oct 2017 18:49:23 +0200 Subject: [PATCH 78/96] Update gitlab config for 9.5 support --- gitlab/files/gitaly-config.toml | 8 ++++++++ gitlab/files/gitlab-gitlab.yml | 17 ++++++++++++++--- gitlab/files/gitlab-logrotate | 9 ++++----- gitlab/files/initd | 6 +++--- 4 files changed, 29 insertions(+), 11 deletions(-) diff --git a/gitlab/files/gitaly-config.toml b/gitlab/files/gitaly-config.toml index 76d901f..30e7f43 100644 --- a/gitlab/files/gitaly-config.toml +++ b/gitlab/files/gitaly-config.toml @@ -37,3 +37,11 @@ path = "{{ repositories }}" # # You can optionally configure Gitaly to record histogram latencies on GRPC method calls # [prometheus] # grpc_latency_buckets = [0.001, 0.005, 0.025, 0.1, 0.5, 1.0, 10.0, 30.0, 60.0, 300.0, 1500.0] + +[gitaly-ruby] +# The directory where gitaly-ruby is installed +dir = "{{ root_dir }}/gitaly/ruby" + +[gitlab-shell] +# The directory where gitlab-shell is installed +dir = "{{ root_dir }}/gitlab-shell" diff --git a/gitlab/files/gitlab-gitlab.yml b/gitlab/files/gitlab-gitlab.yml index 7664436..3958ec9 100644 --- a/gitlab/files/gitlab-gitlab.yml +++ b/gitlab/files/gitlab-gitlab.yml @@ -435,13 +435,13 @@ production: &base # service_validate_url: '/cas/p3/serviceValidate', # logout_url: '/cas/logout'} } # - { name: 'authentiq', - # # for client credentials (client ID and secret), go to https://www.authentiq.com/ + # # for client credentials (client ID and secret), go to https://www.authentiq.com/developers # app_id: 'YOUR_CLIENT_ID', # app_secret: 'YOUR_CLIENT_SECRET', # args: { # scope: 'aq:name email~rs address aq:push' - # # redirect_uri parameter is optional except when 'gitlab.host' in this file is set to 'localhost' - # # redirect_uri: 'YOUR_REDIRECT_URI' + # # callback_url parameter is optional except when 'gitlab.host' in this file is set to 'localhost' + # # callback_url: 'YOUR_CALLBACK_URL' # } # } # - { name: 'github', @@ -522,6 +522,11 @@ production: &base path: {{ repositories }} gitaly_address: unix:{{ sockets_dir }}/private/gitaly.socket # TCP connections are supported too (e.g. tcp://host:port) # gitaly_token: 'special token' # Optional: override global gitaly.token for this storage. + failure_count_threshold: 10 # number of failures before stopping attempts + failure_wait_time: 30 # Seconds after an access failure before allowing access again + failure_reset_time: 1800 # Time in seconds to expire failures + storage_timeout: 30 # Time in seconds to wait before aborting a storage access attempt + ## Backup settings backup: @@ -601,6 +606,12 @@ production: &base ip_whitelist: - 127.0.0.0/8 + # Sidekiq exporter is webserver built in to Sidekiq to expose Prometheus metrics + sidekiq_exporter: + # enabled: true + # address: localhost + # port: 3807 + # # 5. Extra customization # ========================== diff --git a/gitlab/files/gitlab-logrotate b/gitlab/files/gitlab-logrotate index 7771379..acb5b50 100644 --- a/gitlab/files/gitlab-logrotate +++ b/gitlab/files/gitlab-logrotate @@ -1,15 +1,14 @@ # GitLab logrotate settings # based on: http://stackoverflow.com/a/4883967 -{% set root_dir = salt['pillar.get']('gitlab:lookup:root_dir', '/home/git') %} -{% set logs_dir = salt['pillar.get']('gitlab:lookup:logs_dir', root_dir ~ '/var/logs') %} +{%- set root_dir = salt['pillar.get']('gitlab:lookup:root_dir', '/home/git') %} +{%- set logs_dir = salt['pillar.get']('gitlab:lookup:logs_dir', root_dir ~ '/var/logs') %} {{ logs_dir }}/*.log { - weekly + daily missingok - rotate 52 + rotate 90 compress - delaycompress notifempty copytruncate } diff --git a/gitlab/files/initd b/gitlab/files/initd index 689e1a6..8d078ba 100755 --- a/gitlab/files/initd +++ b/gitlab/files/initd @@ -296,7 +296,7 @@ start_gitlab() { fi if [ "$gitlab_workhorse_status" = "0" ]; then - echo "The GitLab Workhorse is already running with pid $spid, not restarting" + echo "The GitLab Workhorse is already running with pid $hpid, not restarting" else # No need to remove a socket, gitlab-workhorse does this itself. # Because gitlab-workhorse has multiple executables we need to fix @@ -318,7 +318,7 @@ start_gitlab() { if [ "$gitlab_pages_enabled" = true ]; then if [ "$gitlab_pages_status" = "0" ]; then - echo "The GitLab Pages is already running with pid $spid, not restarting" + echo "The GitLab Pages is already running with pid $gppid, not restarting" else $app_root/bin/daemon_with_pidfile $gitlab_pages_pid_path \ $gitlab_pages_dir/gitlab-pages $gitlab_pages_options \ @@ -426,7 +426,7 @@ print_status() { fi if [ "$gitlab_pages_enabled" = true ]; then if [ "$gitlab_pages_status" = "0" ]; then - echo "The GitLab Pages with pid $mpid is running." + echo "The GitLab Pages with pid $gppid is running." else printf "The GitLab Pages is \033[31mnot running\033[0m.\n" fi From 1b6b7d4c7bc1ee52f05ad402f2ac48a51cb3818f Mon Sep 17 00:00:00 2001 From: Gilles Dartiguelongue Date: Thu, 19 Oct 2017 13:25:37 +0200 Subject: [PATCH 79/96] Move out init.d script modifications Everything is handled in default file as intended for this init system. Also fixes gitlab:check output. --- gitlab/files/gitlab-default | 13 +++++++------ gitlab/files/initd | 25 ++++++++++--------------- 2 files changed, 17 insertions(+), 21 deletions(-) diff --git a/gitlab/files/gitlab-default b/gitlab/files/gitlab-default index 10fca61..285feb5 100644 --- a/gitlab/files/gitlab-default +++ b/gitlab/files/gitlab-default @@ -1,11 +1,11 @@ +{%- set root_dir = salt['pillar.get']('gitlab:lookup:root_dir', '/home/git') %} +{%- set pids_dir = salt['pillar.get']('gitlab:lookup:pids_dir', root_dir ~ '/var/pids') %} +{%- set sockets_dir = salt['pillar.get']('gitlab:lookup:sockets_dir', root_dir ~ '/var/sockets') %} +{%- set logs_dir = salt['pillar.get']('gitlab:lookup:logs_dir', root_dir ~ '/var/log') %} +{%- set workhorse_secret = salt['pillar.get']('gitlab:shell:workhorse:path', root_dir ~ '/.gitlab_workhorse_secret') %} # Copy this lib/support/init.d/gitlab.default.example file to # /etc/default/gitlab in order for it to apply to your system. -{% set root_dir = salt['pillar.get']('gitlab:lookup:root_dir', '/home/git') %} -{% set pids_dir = salt['pillar.get']('gitlab:lookup:pids_dir', root_dir ~ '/var/pids') %} -{% set sockets_dir = salt['pillar.get']('gitlab:lookup:sockets_dir', root_dir ~ '/var/sockets') %} -{% set logs_dir = salt['pillar.get']('gitlab:lookup:logs_dir', root_dir ~ '/var/log') %} - # RAILS_ENV defines the type of installation that is running. # Normal values are "production", "test" and "development". RAILS_ENV="production" @@ -54,7 +54,7 @@ gitlab_workhorse_pid_path="$pid_path/gitlab-workhorse.pid" # For relative URL support change to: # '-authBackend http://127.0.0.1/8080/gitlab' # Read more in http://doc.gitlab.com/ce/install/relative_url.html -gitlab_workhorse_options="-listenUmask 0 -listenNetwork unix -listenAddr $socket_path/gitlab-workhorse.socket -authBackend http://127.0.0.1:8080 -authSocket $socket_path/gitlab.socket -documentRoot $app_root/public" +gitlab_workhorse_options="-listenUmask 0 -listenNetwork unix -listenAddr $socket_path/gitlab-workhorse.socket -authBackend http://127.0.0.1:8080 -authSocket $socket_path/gitlab.socket -secretPath {{ workhorse_secret }} -documentRoot $app_root/public" gitlab_workhorse_log="{{ logs_dir }}/gitlab-workhorse.log" # The GitLab Pages Daemon needs either a separate IP address on which it will @@ -78,6 +78,7 @@ gitlab_workhorse_log="{{ logs_dir }}/gitlab-workhorse.log" # The -pages-domain must be specified the same as in `gitlab.yml > pages > host`. # Set `gitlab_pages_enabled=true` if you want to enable the Pages feature. gitlab_pages_enabled=false +gitlab_pages_dir=$(cd {{ root_dir }}/gitlab-pages/bin && pwd) gitlab_pages_options="-pages-domain example.com -pages-root $app_root/shared/pages -listen-proxy 127.0.0.1:8090" gitlab_pages_log="{{ logs_dir }}/gitlab-pages.log" diff --git a/gitlab/files/initd b/gitlab/files/initd index 8d078ba..2f2de08 100755 --- a/gitlab/files/initd +++ b/gitlab/files/initd @@ -23,11 +23,6 @@ # An example defaults file can be found in lib/support/init.d/gitlab.default.example ### -{% set root_dir = salt['pillar.get']('gitlab:lookup:root_dir', '/home/git') %} -{% set pids_dir = salt['pillar.get']('gitlab:lookup:pids_dir', root_dir ~ '/var/pids') %} -{% set sockets_dir = salt['pillar.get']('gitlab:lookup:sockets_dir', root_dir ~ '/var/sockets') %} -{% set logs_dir = salt['pillar.get']('gitlab:lookup:logs_dir', root_dir ~ '/var/logs') %} -{% set workhorse_secret = salt['pillar.get']('gitlab:workhorse:secret:path', root_dir ~ '/.gitlab_workhorse_secret') %} ### Environment variables RAILS_ENV="production" @@ -35,28 +30,28 @@ RAILS_ENV="production" # Script variable names should be lower-case not to conflict with # internal /bin/sh variables such as PATH, EDITOR or SHELL. app_user="git" -app_root="{{ root_dir }}/gitlab" -pid_path="{{ pids_dir }}" -socket_path="{{ sockets_dir }}" +app_root="/home/$app_user/gitlab" +pid_path="$app_root/tmp/pids" +socket_path="$app_root/tmp/sockets" rails_socket="$socket_path/gitlab.socket" web_server_pid_path="$pid_path/unicorn.pid" sidekiq_pid_path="$pid_path/sidekiq.pid" mail_room_enabled=false mail_room_pid_path="$pid_path/mail_room.pid" -gitlab_workhorse_dir=$(cd {{ root_dir }}/gitlab-workhorse/bin 2> /dev/null && pwd) +gitlab_workhorse_dir=$(cd $app_root/../gitlab-workhorse 2> /dev/null && pwd) gitlab_workhorse_pid_path="$pid_path/gitlab-workhorse.pid" -gitlab_workhorse_options="-listenUmask 0 -listenNetwork unix -listenAddr $socket_path/gitlab-workhorse.socket -authBackend http://127.0.0.1:8080 -authSocket $rails_socket -secretPath {{ workhorse_secret }} -documentRoot $app_root/public" -gitlab_workhorse_log="{{ logs_dir }}/gitlab-workhorse.log" +gitlab_workhorse_options="-listenUmask 0 -listenNetwork unix -listenAddr $socket_path/gitlab-workhorse.socket -authBackend http://127.0.0.1:8080 -authSocket $rails_socket -documentRoot $app_root/public" +gitlab_workhorse_log="$app_root/log/gitlab-workhorse.log" gitlab_pages_enabled=false -gitlab_pages_dir=$(cd {{ root_dir }}/gitlab-pages/bin 2> /dev/null && pwd) +gitlab_pages_dir=$(cd $app_root/../gitlab-pages 2> /dev/null && pwd) gitlab_pages_pid_path="$pid_path/gitlab-pages.pid" gitlab_pages_options="-pages-domain example.com -pages-root $app_root/shared/pages -listen-proxy 127.0.0.1:8090" -gitlab_pages_log="{{ logs_dir }}/gitlab-pages.log" +gitlab_pages_log="$app_root/log/gitlab-pages.log" shell_path="/bin/bash" gitaly_enabled=true -gitaly_dir=$(cd {{ root_dir }}/gitaly/bin 2> /dev/null && pwd) +gitaly_dir=$(cd $app_root/../gitaly 2> /dev/null && pwd) gitaly_pid_path="$pid_path/gitaly.pid" -gitaly_log="{{ logs_dir }}/gitaly.log" +gitaly_log="$app_root/log/gitaly.log" # Read configuration variable file if it is present test -f /etc/default/gitlab && . /etc/default/gitlab From 4a28f768423cb2524ed5aee0c5b9fd764b87de22 Mon Sep 17 00:00:00 2001 From: Gilles Dartiguelongue Date: Thu, 19 Oct 2017 13:27:11 +0200 Subject: [PATCH 80/96] Fix gitlab-shell hooks check This has been bothering me for long enough. --- gitlab/files/gitlab-gitlab.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitlab/files/gitlab-gitlab.yml b/gitlab/files/gitlab-gitlab.yml index 3958ec9..2d8cd5d 100644 --- a/gitlab/files/gitlab-gitlab.yml +++ b/gitlab/files/gitlab-gitlab.yml @@ -554,7 +554,7 @@ production: &base ## GitLab Shell settings gitlab_shell: path: {{ root_dir }}/gitlab-shell/ - hooks_path: {{ shell_dir_content }}/hooks/ + hooks_path: {{ root_dir }}/gitlab-shell/hooks/ # File that contains the secret key for verifying access for gitlab-shell. # Default is '.gitlab_shell_secret' relative to Rails.root (i.e. root of the GitLab app). From 384d019c5d89906620cf8bc139a04ab28f5bd34d Mon Sep 17 00:00:00 2001 From: Gilles Dartiguelongue Date: Thu, 19 Oct 2017 14:36:15 +0200 Subject: [PATCH 81/96] gitaly now requires proxy for installation --- gitlab/gitaly.sls | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/gitlab/gitaly.sls b/gitlab/gitaly.sls index ee6015a..58dc691 100644 --- a/gitlab/gitaly.sls +++ b/gitlab/gitaly.sls @@ -63,6 +63,11 @@ gitaly-make: - name: make build install DESTDIR={{ root_dir }}/gitaly PREFIX= - user: git - cwd: {{ gitaly_dir_content }} + - env: + {%- if salt['pillar.get']('gitlab:proxy:address') %} + - HTTP_PROXY: {{ pillar.gitlab.proxy.address }} + - HTTPS_PROXY: {{ pillar.gitlab.proxy.address }} + {%- endif %} - onchanges: - gitaly-fetcher - require: @@ -81,4 +86,4 @@ gitaly-config: - require: - gitaly-fetcher - file: gitaly-bin-dir - - cmd: gitaly-make \ No newline at end of file + - cmd: gitaly-make From 6a78d32d7530aa49af018a2ea8c6b01597d0883e Mon Sep 17 00:00:00 2001 From: Gilles Dartiguelongue Date: Thu, 19 Oct 2017 14:40:28 +0200 Subject: [PATCH 82/96] Fix gitaly ruby configuration --- gitlab/files/gitaly-config.toml | 5 +---- gitlab/gitaly.sls | 6 ++++++ 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/gitlab/files/gitaly-config.toml b/gitlab/files/gitaly-config.toml index 30e7f43..e73ac87 100644 --- a/gitlab/files/gitaly-config.toml +++ b/gitlab/files/gitaly-config.toml @@ -1,6 +1,3 @@ -{%- set root_dir = salt['pillar.get']('gitlab:lookup:root_dir', '/home/git') %} -{%- set sockets_dir = salt['pillar.get']('gitlab:lookup:sockets_dir', root_dir ~ '/var/sockets') %} -{%- set repositories = salt['pillar.get']('gitlab:lookup:repositories', root_dir ~ '/repositories') %} # Example Gitaly configuration file socket_path = "{{ sockets_dir }}/private/gitaly.socket" @@ -40,7 +37,7 @@ path = "{{ repositories }}" [gitaly-ruby] # The directory where gitaly-ruby is installed -dir = "{{ root_dir }}/gitaly/ruby" +dir = "{{ gitaly_dir_content }}/ruby" [gitlab-shell] # The directory where gitlab-shell is installed diff --git a/gitlab/gitaly.sls b/gitlab/gitaly.sls index 58dc691..4928f85 100644 --- a/gitlab/gitaly.sls +++ b/gitlab/gitaly.sls @@ -1,4 +1,5 @@ {%- set root_dir = salt['pillar.get']('gitlab:lookup:root_dir', '/home/git') %} +{%- set repositories = salt['pillar.get']('gitlab:lookup:repositories', root_dir ~ '/repositories') %} {%- set sockets_dir = salt['pillar.get']('gitlab:lookup:sockets_dir', root_dir ~ '/var/sockets') %} {%- set lib_dir = salt['pillar.get']('gitlab:lookup:lib_dir', root_dir ~ '/libraries') %} @@ -83,6 +84,11 @@ gitaly-config: - user: git - group: git - mode: 644 + - context: + root_dir: {{ root_dir }} + sockets_dir: {{ sockets_dir }} + repositories: {{ repositories }} + gitaly_dir_content: {{ gitaly_dir_content }} - require: - gitaly-fetcher - file: gitaly-bin-dir From ed1d0a1ba3ccc6e2a380f8716f9dc0fe22971a24 Mon Sep 17 00:00:00 2001 From: Gilles Dartiguelongue Date: Thu, 19 Oct 2017 16:10:51 +0200 Subject: [PATCH 83/96] Add redis configuration This appears needed for migration and was not documentation, see [1]. [1] https://gitlab.com/gitlab-org/gitlab-ce/issues/38423 --- gitlab/files/gitlab-resque.yml | 34 ++++++++++++++++++++++++++++++++++ gitlab/gitlab.sls | 12 ++++++++++++ 2 files changed, 46 insertions(+) create mode 100644 gitlab/files/gitlab-resque.yml diff --git a/gitlab/files/gitlab-resque.yml b/gitlab/files/gitlab-resque.yml new file mode 100644 index 0000000..b69c77d --- /dev/null +++ b/gitlab/files/gitlab-resque.yml @@ -0,0 +1,34 @@ +# If you change this file in a Merge Request, please also create +# a Merge Request on https://gitlab.com/gitlab-org/omnibus-gitlab/merge_requests +# +development: + url: redis://localhost:6379 + # sentinels: + # - + # host: localhost + # port: 26380 # point to sentinel, not to redis port + # - + # host: slave2 + # port: 26381 # point to sentinel, not to redis port +test: + url: redis://localhost:6379 +production: + # Redis (single instance) + url: {{ salt['pillar.get']('gitlab:databases:production:redis:url', 'redis://localhost:6379') }} + ## + # Redis + Sentinel (for HA) + # + # Please read instructions carefully before using it as you may lose data: + # http://redis.io/topics/sentinel + # + # You must specify a list of a few sentinels that will handle client connection + # please read here for more information: https://docs.gitlab.com/ce/administration/high_availability/redis.html + ## + # url: redis://master:6379 + # sentinels: + # - + # host: slave1 + # port: 26379 # point to sentinel, not to redis port + # - + # host: slave2 + # port: 26379 # point to sentinel, not to redis port diff --git a/gitlab/gitlab.sls b/gitlab/gitlab.sls index f9680e1..6dfb142 100644 --- a/gitlab/gitlab.sls +++ b/gitlab/gitlab.sls @@ -96,6 +96,17 @@ gitlab-db-config: {% endif %} - user: git-user +gitlab-redis-config: + file.managed: + - name: {{ root_dir }}/gitlab/config/resque.yml + - source: salt://gitlab/files/gitlab-resque.yml + - template: jinja + - user: git + - group: git + - mode: 640 + - require: + - user: git-user + gitlab-db-secrets: file.managed: - name: {{ root_dir }}/gitlab/config/secrets.yml @@ -244,6 +255,7 @@ gitlab-migrate-db: {% endif %} - require: - file: gitlab-db-config + - file: gitlab-redis-config - cmd: gitlab-gems gitlab-build-translations: From 3db24b2cb68afa51b30ae975812fff55bd281e3a Mon Sep 17 00:00:00 2001 From: Gilles Dartiguelongue Date: Mon, 30 Oct 2017 16:10:53 +0100 Subject: [PATCH 84/96] Fix path to workhorse secret --- gitlab/files/gitlab-gitlab.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/gitlab/files/gitlab-gitlab.yml b/gitlab/files/gitlab-gitlab.yml index 2d8cd5d..0a6d211 100644 --- a/gitlab/files/gitlab-gitlab.yml +++ b/gitlab/files/gitlab-gitlab.yml @@ -25,6 +25,7 @@ {% set lib_dir = salt['pillar.get']('gitlab:lookup:lib_dir', root_dir ~ '/libraries') %} {% set repositories = salt['pillar.get']('gitlab:lookup:repositories', root_dir ~ '/repositories') %} {% set sockets_dir = salt['pillar.get']('gitlab:lookup:sockets_dir', root_dir ~ '/var/sockets') %} +{%- set workhorse_secret = salt['pillar.get']('gitlab:shell:workhorse:path', root_dir ~ '/.gitlab_workhorse_secret') %} {% set shell_dir = lib_dir ~ "/gitlab-shell" %} {% if salt['pillar.get']('gitlab:archives:enabled', false) %} @@ -573,7 +574,7 @@ production: &base workhorse: # File that contains the secret key for verifying access for gitlab-workhorse. # Default is '.gitlab_workhorse_secret' relative to Rails.root (i.e. root of the GitLab app). - # secret_file: {{ root_dir }}/gitlab/.gitlab_workhorse_secret + secret_file: {{ workhorse_secret }} ## Git settings # CAUTION! From ed26ccbe56c2d1154d2d619dd7ee1dd9b68854de Mon Sep 17 00:00:00 2001 From: Gilles Dartiguelongue Date: Fri, 1 Sep 2017 15:48:54 +0200 Subject: [PATCH 85/96] Update gitlab configuration for 10.0 --- gitlab/files/gitlab-gitlab.yml | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/gitlab/files/gitlab-gitlab.yml b/gitlab/files/gitlab-gitlab.yml index 0a6d211..dd0b774 100644 --- a/gitlab/files/gitlab-gitlab.yml +++ b/gitlab/files/gitlab-gitlab.yml @@ -94,6 +94,13 @@ production: &base # default_can_create_group: false # default: true # username_changing_enabled: false # default: true - User can change her username/namespace + ## Default theme ID + ## 1 - Indigo + ## 2 - Dark + ## 3 - Light + ## 4 - Blue + ## 5 - Green + # default_theme: 1 # default: 1 ## Automatic issue closing # If a commit message matches this regular expression, all issues referenced from the matched text will be closed. @@ -290,8 +297,7 @@ production: &base encryption: {{ salt['pillar.get']('gitlab:ldap:encryption', 'plain') }} # Enables SSL certificate verification if encryption method is - # "start_tls" or "simple_tls". (Defaults to false for backward- - # compatibility) + # "start_tls" or "simple_tls". Defaults to true. verify_certificates: {{ salt['pillar.get']('gitlab:ldap:verify_certificates', false) }} # Specifies the path to a file containing a PEM-format CA certificate, @@ -390,9 +396,16 @@ production: &base # showing GitLab's sign-in page (default: show the GitLab sign-in page) # auto_sign_in_with_provider: saml - # Sync user's email address from the specified Omniauth provider every time the user logs - # in (default: nil). And consequently make this field read-only. - # sync_email_from_provider: cas3 + # Sync user's profile from the specified Omniauth providers every time the user logs in (default: empty). + # Define the allowed providers using an array, e.g. ["cas3", "saml", "twitter"], + # or as true/false to allow all providers or none. + # sync_profile_from_provider: [] + + # Select which info to sync from the providers above. (default: email). + # Define the synced profile info using an array. Available options are "name", "email" and "location" + # e.g. ["name", "email", "location"] or as true to sync all available. + # This consequently will make the selected attributes read-only. + # sync_profile_attributes: true # CAUTION! # This allows users to login without having a user account first. Define the allowed providers From f096672eda08acffd7f8cc0c60b57c8228ee8508 Mon Sep 17 00:00:00 2001 From: Gilles Dartiguelongue Date: Mon, 27 Nov 2017 16:19:50 +0100 Subject: [PATCH 86/96] Fix passing proxy to yarn Yarn did not support standard environment variables until very recently. How did I get this to work in the first place... https://github.com/yarnpkg/yarn/pull/4849 --- gitlab/gitlab.sls | 1 + 1 file changed, 1 insertion(+) diff --git a/gitlab/gitlab.sls b/gitlab/gitlab.sls index 6dfb142..0333c99 100644 --- a/gitlab/gitlab.sls +++ b/gitlab/gitlab.sls @@ -283,6 +283,7 @@ gitlab-yarn-install: - RAILS_ENV: production - NODE_ENV: production {%- if salt['pillar.get']('gitlab:proxy:address') %} + - YARN_PROXY: {{ pillar.gitlab.proxy.address }} - HTTP_PROXY: {{ pillar.gitlab.proxy.address }} - HTTPS_PROXY: {{ pillar.gitlab.proxy.address }} {%- endif %} From a571f60daef2a2355e6c65d2a0d6834fa9093bbe Mon Sep 17 00:00:00 2001 From: Gilles Dartiguelongue Date: Fri, 1 Sep 2017 15:48:54 +0200 Subject: [PATCH 87/96] Update gitlab configuration for 10.1 --- gitlab/files/gitlab-gitlab.yml | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/gitlab/files/gitlab-gitlab.yml b/gitlab/files/gitlab-gitlab.yml index dd0b774..c421311 100644 --- a/gitlab/files/gitlab-gitlab.yml +++ b/gitlab/files/gitlab-gitlab.yml @@ -107,7 +107,7 @@ production: &base # This happens when the commit is pushed or merged into the default branch of a project. # When not specified the default issue_closing_pattern as specified below will be used. # Tip: you can test your closing pattern at http://rubular.com. - # issue_closing_pattern: '((?:[Cc]los(?:e[sd]?|ing)|[Ff]ix(?:e[sd]|ing)?|[Rr]esolv(?:e[sd]?|ing))(:?) +(?:(?:issues? +)?%{issue_ref}(?:(?:, *| +and +)?)|([A-Z][A-Z0-9_]+-\d+))+)' + # issue_closing_pattern: '((?:[Cc]los(?:e[sd]?|ing)|[Ff]ix(?:e[sd]|ing)?|[Rr]esolv(?:e[sd]?|ing)|[Ii]mplement(?:s|ed|ing)?)(:?) +(?:(?:issues? +)?%{issue_ref}(?:(?:, *| +and +)?)|([A-Z][A-Z0-9_]+-\d+))+)' ## Default project features settings default_projects_features: @@ -182,6 +182,7 @@ production: &base host: example.com port: 80 # Set to 443 if you serve the pages with HTTPS https: false # Set to true if you serve the pages with HTTPS + artifacts_server: true # external_http: ["1.1.1.1:80", "[2001::1]:80"] # If defined, enables custom domain support in GitLab Pages # external_https: ["1.1.1.1:443", "[2001::1]:443"] # If defined, enables custom domain and certificate support in GitLab Pages @@ -516,6 +517,8 @@ production: &base # Gitaly settings gitaly: + # Path to the directory containing Gitaly client executables. + client_path: {{ root_dir }}/gitaly # Default Gitaly authentication token. Can be overriden per storage. Can # be left blank when Gitaly is running locally on a Unix socket, which # is the normal way to deploy Gitaly. @@ -536,11 +539,6 @@ production: &base path: {{ repositories }} gitaly_address: unix:{{ sockets_dir }}/private/gitaly.socket # TCP connections are supported too (e.g. tcp://host:port) # gitaly_token: 'special token' # Optional: override global gitaly.token for this storage. - failure_count_threshold: 10 # number of failures before stopping attempts - failure_wait_time: 30 # Seconds after an access failure before allowing access again - failure_reset_time: 1800 # Time in seconds to expire failures - storage_timeout: 30 # Time in seconds to wait before aborting a storage access attempt - ## Backup settings backup: @@ -594,12 +592,6 @@ production: &base # Use the default values unless you really know what you are doing git: bin_path: /usr/bin/git - # The next value is the maximum memory size grit can use - # Given in number of bytes per git object (e.g. a commit) - # This value can be increased if you have very large commits - max_size: 20971520 # 20.megabytes - # Git timeout to read a commit, in seconds - timeout: 10 ## Webpack settings # If enabled, this will tell rails to serve frontend assets from the webpack-dev-server running From 6c0716cc83a87eccf763ab0b5a044fed722807a9 Mon Sep 17 00:00:00 2001 From: Gilles Dartiguelongue Date: Fri, 1 Sep 2017 15:48:54 +0200 Subject: [PATCH 88/96] Update gitaly configuration for gitlab 10.1 --- gitlab/files/gitaly-config.toml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/gitlab/files/gitaly-config.toml b/gitlab/files/gitaly-config.toml index e73ac87..a152c5a 100644 --- a/gitlab/files/gitaly-config.toml +++ b/gitlab/files/gitaly-config.toml @@ -42,3 +42,8 @@ dir = "{{ gitaly_dir_content }}/ruby" [gitlab-shell] # The directory where gitlab-shell is installed dir = "{{ root_dir }}/gitlab-shell" + +# # You can adjust the concurrency of each RPC endpoint +# [[concurrency]] +# rpc = "/gitaly.RepositoryService/GarbageCollect" +# max_per_repo = 1 From b0125048622b7095b41a5bbb9aa72473ae776f84 Mon Sep 17 00:00:00 2001 From: Gilles Dartiguelongue Date: Fri, 1 Sep 2017 15:48:54 +0200 Subject: [PATCH 89/96] Update gitlab configuration for 10.2 --- gitlab/files/gitlab-gitlab.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitlab/files/gitlab-gitlab.yml b/gitlab/files/gitlab-gitlab.yml index c421311..98dee2d 100644 --- a/gitlab/files/gitlab-gitlab.yml +++ b/gitlab/files/gitlab-gitlab.yml @@ -518,7 +518,7 @@ production: &base # Gitaly settings gitaly: # Path to the directory containing Gitaly client executables. - client_path: {{ root_dir }}/gitaly + client_path: {{ root_dir }}/gitaly/bin # Default Gitaly authentication token. Can be overriden per storage. Can # be left blank when Gitaly is running locally on a Unix socket, which # is the normal way to deploy Gitaly. From b3ed987d6fa82db9635b58a271750ad7ab3567ee Mon Sep 17 00:00:00 2001 From: Gilles Dartiguelongue Date: Mon, 19 Feb 2018 14:36:42 +0100 Subject: [PATCH 90/96] Fix highstate noise due to mode changes of uploads_dir --- gitlab/gitlab.sls | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/gitlab/gitlab.sls b/gitlab/gitlab.sls index 0333c99..d6c1932 100644 --- a/gitlab/gitlab.sls +++ b/gitlab/gitlab.sls @@ -174,7 +174,7 @@ git-var-mkdir: - mode: 750 # pids_dir -{% for dir in [ sockets_dir, logs_dir, uploads_dir ] %} +{% for dir in [ sockets_dir, logs_dir ] %} git-{{ dir }}-mkdir: file.directory: - name: {{ dir }} @@ -183,9 +183,11 @@ git-{{ dir }}-mkdir: - mode: 750 {% endfor %} -gitlab-uploads_dir-mode: +gitlab-uploads_dir-mkdir: file.directory: - name: {{ uploads_dir }} + - user: git + - group: git - mode: 700 # Hardcoded in gitlab, so, we have to create symlink From adfc5c5ac76123e7f7fc08b78970754ab08447d8 Mon Sep 17 00:00:00 2001 From: Gilles Dartiguelongue Date: Mon, 19 Feb 2018 14:38:06 +0100 Subject: [PATCH 91/96] Fix gitlab-uploads-symlink state dependencies Make sure it is setup before starting service as well. --- gitlab/gitlab.sls | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/gitlab/gitlab.sls b/gitlab/gitlab.sls index d6c1932..8c975c3 100644 --- a/gitlab/gitlab.sls +++ b/gitlab/gitlab.sls @@ -190,6 +190,14 @@ gitlab-uploads_dir-mkdir: - group: git - mode: 700 +gitlab-uploads_dir-symlink: + file.symlink: + - name: {{ gitlab_dir }}/public/uploads + - target: {{ uploads_dir }} + - require: + - file: gitlab-config + - file: gitlab-uploads_dir-mkdir + # Hardcoded in gitlab, so, we have to create symlink gitlab-pids_dir-symlink: file.symlink: @@ -354,13 +362,6 @@ gitlab-respositories-dir: - file_mode: 0660 - dir_mode: 2770 -gitlab-uploads-symlink: - file.symlink: - - name: {{ gitlab_dir }}/public/uploads - - target: {{ uploads_dir }} - - require: - - file: git-var-mkdir - gitlab-service: file.managed: - name: /etc/init.d/gitlab @@ -381,7 +382,8 @@ gitlab-service: - require: - file: gitlab-service # - cmd: gitlab-initialize - - file: gitlab-pids_dir-symlink + - file: gitlab-pids_dir-symlink + - file: gitlab-uploads_dir-symlink - watch: {% if salt['pillar.get']('gitlab:archives:enabled', false) %} - archive: gitlab-fetcher From eb0450237685d38f5f0a844208540ce46e4f29dd Mon Sep 17 00:00:00 2001 From: Gilles Dartiguelongue Date: Mon, 19 Feb 2018 16:27:19 +0100 Subject: [PATCH 92/96] Update requirements for nodejs --- gitlab/packages.sls | 2 +- gitlab/repos.sls | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/gitlab/packages.sls b/gitlab/packages.sls index 3f88dff..5e03c22 100644 --- a/gitlab/packages.sls +++ b/gitlab/packages.sls @@ -103,7 +103,7 @@ gitlab-deps: - libyaml-dev - logrotate - openssh-server - - nodejs: ">=4.3" + - nodejs: ">=6" - pkg-config - python - python-docutils diff --git a/gitlab/repos.sls b/gitlab/repos.sls index a0bb624..66ef4c9 100644 --- a/gitlab/repos.sls +++ b/gitlab/repos.sls @@ -33,7 +33,7 @@ gitlab-distro-backports: {%- endif %} - file: /etc/apt/sources.list.d/gitlab_req_backports.list -{# Gitlab 8.17+ requires nodejs-4.3+ but is not available before Debian 9 or Ubuntu 16.10 #} +{# Gitlab 10.3+ requires nodejs-6+ but is not available in Debian 10 and not before Ubuntu 17.10 #} gitlab-nodejs-repo-mgmt-pkgs: pkg.installed: - names: @@ -45,8 +45,8 @@ gitlab-nodejs-repo-mgmt-pkgs: gitlab-nodejs-repo: pkgrepo.managed: - - name: deb https://deb.nodesource.com/node_4.x {{ grains.oscodename|lower }} main - - file: /etc/apt/sources.list.d/nodesource_4.list + - name: deb https://deb.nodesource.com/node_6.x {{ grains.oscodename|lower }} main + - file: /etc/apt/sources.list.d/nodesource_6.list - key_url: salt://gitlab/files/nodesource.gpg.key gitlab-nodejs-preference: From 1575860990f4065a8e269c35df183c89c23ff05c Mon Sep 17 00:00:00 2001 From: Gilles Dartiguelongue Date: Fri, 1 Sep 2017 15:48:54 +0200 Subject: [PATCH 93/96] Update gitlab configuration for 10.3 --- gitlab/files/gitlab-gitlab.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/gitlab/files/gitlab-gitlab.yml b/gitlab/files/gitlab-gitlab.yml index 98dee2d..54c7989 100644 --- a/gitlab/files/gitlab-gitlab.yml +++ b/gitlab/files/gitlab-gitlab.yml @@ -400,6 +400,7 @@ production: &base # Sync user's profile from the specified Omniauth providers every time the user logs in (default: empty). # Define the allowed providers using an array, e.g. ["cas3", "saml", "twitter"], # or as true/false to allow all providers or none. + # When authenticating using LDAP, the user's email is always synced. # sync_profile_from_provider: [] # Select which info to sync from the providers above. (default: email). @@ -576,8 +577,8 @@ production: &base upload_pack: true receive_pack: true - # Git import/fetch timeout - # git_timeout: 800 + # Git import/fetch timeout, in seconds. Defaults to 3 hours. + # git_timeout: 10800 # If you use non-standard ssh port you need to specify it # ssh_port: 22 From 8d30658a6b223f7c54cd63ef49817d3c550b1328 Mon Sep 17 00:00:00 2001 From: Gilles Dartiguelongue Date: Mon, 19 Feb 2018 17:23:03 +0100 Subject: [PATCH 94/96] Fix version specifiers in gitlab.packages Previous syntax was just broken and did not work at all. latest will require salt 2017.7 but will prevent requirement being satisfied by previously installed released on formula update. --- gitlab/packages.sls | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gitlab/packages.sls b/gitlab/packages.sls index 5e03c22..921a73b 100644 --- a/gitlab/packages.sls +++ b/gitlab/packages.sls @@ -103,13 +103,13 @@ gitlab-deps: - libyaml-dev - logrotate - openssh-server - - nodejs: ">=6" + - nodejs: latest - pkg-config - python - python-docutils - rake - redis-server - - yarn: ">=0.17" + - yarn: latest - zlib1g-dev {% if salt['pillar.get']('gitlab:db:engine', 'postgresql') == 'postgresql' %} - libpq-dev From afc087d81d4dd2a884de7b7dd73964ebf3221ab4 Mon Sep 17 00:00:00 2001 From: Gilles Dartiguelongue Date: Mon, 23 Apr 2018 16:18:30 +0200 Subject: [PATCH 95/96] Allow compiling nokogiri against system librairies Keeping this optional for now as I don't know if it is needed on non-Debian distributions. --- gitlab/gitaly.sls | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/gitlab/gitaly.sls b/gitlab/gitaly.sls index 4928f85..1f74c73 100644 --- a/gitlab/gitaly.sls +++ b/gitlab/gitaly.sls @@ -59,6 +59,15 @@ gitaly-bin-dir: - group: git - mode: 750 +{% if pillar.gitlab.nokogiri_system_libs|default(False) %} +gitaly-bundle-config: + cmd.run: + - name: bundle config build.nokogiri --use-system-libraries --with-xml2-config=/usr/bin/xml2-config --with-xslt-config=/usr/bin/xslt-config + - user: git + - cwd: {{ gitaly_dir_content }} + - onlyif: bundle config build.nokogiri |grep -q "not configured" +{% endif %} + gitaly-make: cmd.run: - name: make build install DESTDIR={{ root_dir }}/gitaly PREFIX= @@ -72,6 +81,9 @@ gitaly-make: - onchanges: - gitaly-fetcher - require: + {% if pillar.gitlab.nokogiri_system_libs|default(False) %} + - cmd: gitaly-bundle-config + {% endif %} - file: gitaly-bin-dir # https://gitlab.com/gitlab-org/gitaly/blob/master/config.toml.example From 1b9353ba2485046242bc8b01bb83eebf55aa2b08 Mon Sep 17 00:00:00 2001 From: Gilles Dartiguelongue Date: Tue, 24 Apr 2018 15:53:06 +0200 Subject: [PATCH 96/96] Update configuration template for 10.5 series --- gitlab/files/gitlab-gitlab.yml | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/gitlab/files/gitlab-gitlab.yml b/gitlab/files/gitlab-gitlab.yml index 98dee2d..b8c9bf6 100644 --- a/gitlab/files/gitlab-gitlab.yml +++ b/gitlab/files/gitlab-gitlab.yml @@ -170,6 +170,12 @@ production: &base # The location where LFS objects are stored (default: shared/lfs-objects). # storage_path: shared/lfs-objects + ## Uploads (attachments, avatars, etc...) + uploads: + # The location where uploads objects are stored (default: public/). + # storage_path: public/ + # base_dir: uploads/-/system + ## GitLab Pages pages: enabled: false @@ -193,14 +199,16 @@ production: &base host: 'https://mattermost.example.com' ## Gravatar - ## For Libravatar see: http://doc.gitlab.com/ce/customization/libravatar.html + ## If using gravatar.com, there's nothing to change here. For Libravatar + ## you'll need to provide the custom URLs. For more information, + ## see: https://docs.gitlab.com/ee/customization/libravatar.html gravatar: - # gravatar urls: possible placeholders: %{hash} %{size} %{email} %{username} - # plain_url: "http://..." # default: http://www.gravatar.com/avatar/%{hash}?s=%{size}&d=identicon + # Gravatar/Libravatar URLs: possible placeholders: %{hash} %{size} %{email} %{username} + # plain_url: "http://..." # default: https://www.gravatar.com/avatar/%{hash}?s=%{size}&d=identicon # ssl_url: "https://..." # default: https://secure.gravatar.com/avatar/%{hash}?s=%{size}&d=identicon ## Auxiliary jobs - # Periodically executed jobs, to self-heal Gitlab, do external synchronizations, etc. + # Periodically executed jobs, to self-heal GitLab, do external synchronizations, etc. # Please read here for more information: https://github.com/ondrejbartas/sidekiq-cron#adding-cron-job cron_jobs: # Flag stuck CI jobs as failed @@ -224,6 +232,10 @@ production: &base repository_archive_cache_worker: cron: "0 * * * *" + # Verify custom GitLab Pages domains + pages_domain_verification_cron_worker: + cron: "*/15 * * * *" + registry: # enabled: true # host: registry.example.com @@ -379,6 +391,9 @@ production: &base first_name: 'givenName' last_name: 'sn' + # If lowercase_usernames is enabled, GitLab will lower case the username. + lowercase_usernames: false + # GitLab EE only: add more LDAP servers # Choose an ID made of a-z and 0-9 . This ID will be stored in the database # so that GitLab can remember which LDAP server a user belongs to. @@ -400,6 +415,7 @@ production: &base # Sync user's profile from the specified Omniauth providers every time the user logs in (default: empty). # Define the allowed providers using an array, e.g. ["cas3", "saml", "twitter"], # or as true/false to allow all providers or none. + # When authenticating using LDAP, the user's email is always synced. # sync_profile_from_provider: [] # Select which info to sync from the providers above. (default: email). @@ -576,8 +592,8 @@ production: &base upload_pack: true receive_pack: true - # Git import/fetch timeout - # git_timeout: 800 + # Git import/fetch timeout, in seconds. Defaults to 3 hours. + # git_timeout: 10800 # If you use non-standard ssh port you need to specify it # ssh_port: 22