Merge pull request #113635 from talyz/gitlab-improvements

nixos/gitlab: Clean config dir, switch to puma, split PreStart script and more
2021-03-01 13:37:53 +01:00
parent dea1d7921e ca725e7fcd
commit e72375464b
6 changed files with 322 additions and 227 deletions
@@ -1,69 +0,0 @@
-worker_processes 3
-
-listen ENV["UNICORN_PATH"] + "/tmp/sockets/gitlab.socket", :backlog => 1024
-listen "/run/gitlab/gitlab.socket", :backlog => 1024
-
-working_directory ENV["GITLAB_PATH"]
-
-pid ENV["UNICORN_PATH"] + "/tmp/pids/unicorn.pid"
-
-timeout 60
-
-# combine Ruby 2.0.0dev or REE with "preload_app true" for memory savings
-# http://rubyenterpriseedition.com/faq.html#adapt_apps_for_cow
-preload_app true
-GC.respond_to?(:copy_on_write_friendly=) and
-  GC.copy_on_write_friendly = true
-
-check_client_connection false
-
-before_fork do |server, worker|
-  # the following is highly recommended for Rails + "preload_app true"
-  # as there's no need for the master process to hold a connection
-  defined?(ActiveRecord::Base) and
-    ActiveRecord::Base.connection.disconnect!
-
-  # The following is only recommended for memory/DB-constrained
-  # installations.  It is not needed if your system can house
-  # twice as many worker_processes as you have configured.
-  #
-  # This allows a new master process to incrementally
-  # phase out the old master process with SIGTTOU to avoid a
-  # thundering herd (especially in the "preload_app false" case)
-  # when doing a transparent upgrade.  The last worker spawned
-  # will then kill off the old master process with a SIGQUIT.
-  old_pid = "#{server.config[:pid]}.oldbin"
-  if old_pid != server.pid
-    begin
-      sig = (worker.nr + 1) >= server.worker_processes ? :QUIT : :TTOU
-      Process.kill(sig, File.read(old_pid).to_i)
-    rescue Errno::ENOENT, Errno::ESRCH
-    end
-  end
-
-  # Throttle the master from forking too quickly by sleeping.  Due
-  # to the implementation of standard Unix signal handlers, this
-  # helps (but does not completely) prevent identical, repeated signals
-  # from being lost when the receiving process is busy.
-  # sleep 1
-end
-
-after_fork do |server, worker|
-  # per-process listener ports for debugging/admin/migrations
-  # addr = "127.0.0.1:#{9293 + worker.nr}"
-  # server.listen(addr, :tries => -1, :delay => 5, :tcp_nopush => true)
-
-  # the following is *required* for Rails + "preload_app true",
-  defined?(ActiveRecord::Base) and
-    ActiveRecord::Base.establish_connection
-
-  # reset prometheus client, this will cause any opened metrics files to be closed
-  defined?(::Prometheus::Client.reinitialize_on_pid_change) &&
-    Prometheus::Client.reinitialize_on_pid_change
-
-  # if preload_app is true, then you may also want to check and
-  # restart any other shared sockets/descriptors such as Memcached,
-  # and Redis.  TokyoCabinet file handles are safe to reuse
-  # between any number of forked children (assuming your kernel
-  # correctly implements pread()/pwrite() system calls)
-end
@@ -142,7 +142,7 @@ let

  gitlabEnv = {
    HOME = "${cfg.statePath}/home";
-    UNICORN_PATH = "${cfg.statePath}/";
+    PUMA_PATH = "${cfg.statePath}/";
    GITLAB_PATH = "${cfg.packages.gitlab}/share/gitlab/";
    SCHEMA = "${cfg.statePath}/db/structure.sql";
    GITLAB_UPLOADS_PATH = "${cfg.statePath}/uploads";
@@ -424,7 +424,7 @@ in {

        port = mkOption {
          type = types.int;
-          default = 465;
+          default = 25;
          description = "Port of the SMTP server for Gitlab.";
        };

@@ -641,6 +641,11 @@ in {

    environment.systemPackages = [ pkgs.git gitlab-rake gitlab-rails cfg.packages.gitlab-shell ];

+    systemd.targets.gitlab = {
+      description = "Common target for all GitLab services.";
+      wantedBy = [ "multi-user.target" ];
+    };
+
    # Redis is required for the sidekiq queue runner.
    services.redis.enable = mkDefault true;

@@ -655,36 +660,45 @@ in {
    # here.
    systemd.services.gitlab-postgresql = let pgsql = config.services.postgresql; in mkIf databaseActuallyCreateLocally {
      after = [ "postgresql.service" ];
-      wantedBy = [ "multi-user.target" ];
-      path = [ pgsql.package ];
+      bindsTo = [ "postgresql.service" ];
+      wantedBy = [ "gitlab.target" ];
+      partOf = [ "gitlab.target" ];
+      path = [
+        pgsql.package
+        pkgs.util-linux
+      ];
      script = ''
        set -eu

-        PSQL="${pkgs.util-linux}/bin/runuser -u ${pgsql.superUser} -- psql --port=${toString pgsql.port}"
+        PSQL() {
+            psql --port=${toString pgsql.port} "$@"
+        }

-        $PSQL -tAc "SELECT 1 FROM pg_database WHERE datname = '${cfg.databaseName}'" | grep -q 1 || $PSQL -tAc 'CREATE DATABASE "${cfg.databaseName}" OWNER "${cfg.databaseUsername}"'
-        current_owner=$($PSQL -tAc "SELECT pg_catalog.pg_get_userbyid(datdba) FROM pg_catalog.pg_database WHERE datname = '${cfg.databaseName}'")
+        PSQL -tAc "SELECT 1 FROM pg_database WHERE datname = '${cfg.databaseName}'" | grep -q 1 || PSQL -tAc 'CREATE DATABASE "${cfg.databaseName}" OWNER "${cfg.databaseUsername}"'
+        current_owner=$(PSQL -tAc "SELECT pg_catalog.pg_get_userbyid(datdba) FROM pg_catalog.pg_database WHERE datname = '${cfg.databaseName}'")
        if [[ "$current_owner" != "${cfg.databaseUsername}" ]]; then
-            $PSQL -tAc 'ALTER DATABASE "${cfg.databaseName}" OWNER TO "${cfg.databaseUsername}"'
+            PSQL -tAc 'ALTER DATABASE "${cfg.databaseName}" OWNER TO "${cfg.databaseUsername}"'
            if [[ -e "${config.services.postgresql.dataDir}/.reassigning_${cfg.databaseName}" ]]; then
                echo "Reassigning ownership of database ${cfg.databaseName} to user ${cfg.databaseUsername} failed on last boot. Failing..."
                exit 1
            fi
            touch "${config.services.postgresql.dataDir}/.reassigning_${cfg.databaseName}"
-            $PSQL "${cfg.databaseName}" -tAc "REASSIGN OWNED BY \"$current_owner\" TO \"${cfg.databaseUsername}\""
+            PSQL "${cfg.databaseName}" -tAc "REASSIGN OWNED BY \"$current_owner\" TO \"${cfg.databaseUsername}\""
            rm "${config.services.postgresql.dataDir}/.reassigning_${cfg.databaseName}"
        fi
-        $PSQL '${cfg.databaseName}' -tAc "CREATE EXTENSION IF NOT EXISTS pg_trgm"
-        $PSQL '${cfg.databaseName}' -tAc "CREATE EXTENSION IF NOT EXISTS btree_gist;"
+        PSQL '${cfg.databaseName}' -tAc "CREATE EXTENSION IF NOT EXISTS pg_trgm"
+        PSQL '${cfg.databaseName}' -tAc "CREATE EXTENSION IF NOT EXISTS btree_gist;"
      '';

      serviceConfig = {
+        User = pgsql.superUser;
        Type = "oneshot";
+        RemainAfterExit = true;
      };
    };

    # Use postfix to send out mails.
-    services.postfix.enable = mkDefault true;
+    services.postfix.enable = mkDefault (cfg.smtp.enable && cfg.smtp.address == "localhost");

    users.users.${cfg.user} =
      { group = cfg.group;
@@ -703,7 +717,6 @@ in {
      "d ${cfg.statePath} 0750 ${cfg.user} ${cfg.group} -"
      "d ${cfg.statePath}/builds 0750 ${cfg.user} ${cfg.group} -"
      "d ${cfg.statePath}/config 0750 ${cfg.user} ${cfg.group} -"
-      "d ${cfg.statePath}/config/initializers 0750 ${cfg.user} ${cfg.group} -"
      "d ${cfg.statePath}/db 0750 ${cfg.user} ${cfg.group} -"
      "d ${cfg.statePath}/log 0750 ${cfg.user} ${cfg.group} -"
      "d ${cfg.statePath}/repositories 2770 ${cfg.user} ${cfg.group} -"
@@ -726,13 +739,156 @@ in {
      "L+ /run/gitlab/uploads - - - - ${cfg.statePath}/uploads"

      "L+ /run/gitlab/shell-config.yml - - - - ${pkgs.writeText "config.yml" (builtins.toJSON gitlabShellConfig)}"
-
-      "L+ ${cfg.statePath}/config/unicorn.rb - - - - ${./defaultUnicornConfig.rb}"
    ];

+
+    systemd.services.gitlab-config = {
+      wantedBy = [ "gitlab.target" ];
+      partOf = [ "gitlab.target" ];
+      path = with pkgs; [
+        jq
+        openssl
+        replace
+        git
+      ];
+      serviceConfig = {
+        Type = "oneshot";
+        User = cfg.user;
+        Group = cfg.group;
+        TimeoutSec = "infinity";
+        Restart = "on-failure";
+        WorkingDirectory = "${cfg.packages.gitlab}/share/gitlab";
+        RemainAfterExit = true;
+
+        ExecStartPre = let
+          preStartFullPrivileges = ''
+            shopt -s dotglob nullglob
+            set -eu
+
+            chown --no-dereference '${cfg.user}':'${cfg.group}' '${cfg.statePath}'/*
+            if [[ -n "$(ls -A '${cfg.statePath}'/config/)" ]]; then
+              chown --no-dereference '${cfg.user}':'${cfg.group}' '${cfg.statePath}'/config/*
+            fi
+          '';
+        in "+${pkgs.writeShellScript "gitlab-pre-start-full-privileges" preStartFullPrivileges}";
+
+        ExecStart = pkgs.writeShellScript "gitlab-config" ''
+          set -eu
+
+          umask u=rwx,g=rx,o=
+
+          cp -f ${cfg.packages.gitlab}/share/gitlab/VERSION ${cfg.statePath}/VERSION
+          rm -rf ${cfg.statePath}/db/*
+          rm -f ${cfg.statePath}/lib
+          find '${cfg.statePath}/config/' -maxdepth 1 -mindepth 1 -type d -execdir rm -rf {} \;
+          cp -rf --no-preserve=mode ${cfg.packages.gitlab}/share/gitlab/config.dist/* ${cfg.statePath}/config
+          cp -rf --no-preserve=mode ${cfg.packages.gitlab}/share/gitlab/db/* ${cfg.statePath}/db
+          ln -sf ${extraGitlabRb} ${cfg.statePath}/config/initializers/extra-gitlab.rb
+
+          ${cfg.packages.gitlab-shell}/bin/install
+
+          ${optionalString cfg.smtp.enable ''
+              install -m u=rw ${smtpSettings} ${cfg.statePath}/config/initializers/smtp_settings.rb
+              ${optionalString (cfg.smtp.passwordFile != null) ''
+                  smtp_password=$(<'${cfg.smtp.passwordFile}')
+                  replace-literal -e '@smtpPassword@' "$smtp_password" '${cfg.statePath}/config/initializers/smtp_settings.rb'
+              ''}
+          ''}
+
+          (
+            umask u=rwx,g=,o=
+
+            openssl rand -hex 32 > ${cfg.statePath}/gitlab_shell_secret
+
+            rm -f '${cfg.statePath}/config/database.yml'
+
+            ${if cfg.databasePasswordFile != null then ''
+                export db_password="$(<'${cfg.databasePasswordFile}')"
+
+                if [[ -z "$db_password" ]]; then
+                  >&2 echo "Database password was an empty string!"
+                  exit 1
+                fi
+
+                jq <${pkgs.writeText "database.yml" (builtins.toJSON databaseConfig)} \
+                   '.production.password = $ENV.db_password' \
+                   >'${cfg.statePath}/config/database.yml'
+              ''
+              else ''
+                jq <${pkgs.writeText "database.yml" (builtins.toJSON databaseConfig)} \
+                   >'${cfg.statePath}/config/database.yml'
+              ''
+            }
+
+            ${utils.genJqSecretsReplacementSnippet
+                gitlabConfig
+                "${cfg.statePath}/config/gitlab.yml"
+            }
+
+            rm -f '${cfg.statePath}/config/secrets.yml'
+
+            export secret="$(<'${cfg.secrets.secretFile}')"
+            export db="$(<'${cfg.secrets.dbFile}')"
+            export otp="$(<'${cfg.secrets.otpFile}')"
+            export jws="$(<'${cfg.secrets.jwsFile}')"
+            jq -n '{production: {secret_key_base: $ENV.secret,
+                    otp_key_base: $ENV.otp,
+                    db_key_base: $ENV.db,
+                    openid_connect_signing_key: $ENV.jws}}' \
+               > '${cfg.statePath}/config/secrets.yml'
+          )
+
+          # We remove potentially broken links to old gitlab-shell versions
+          rm -Rf ${cfg.statePath}/repositories/**/*.git/hooks
+
+          git config --global core.autocrlf "input"
+        '';
+      };
+    };
+
+    systemd.services.gitlab-db-config = {
+      after = [ "gitlab-config.service" "gitlab-postgresql.service" "postgresql.service" ];
+      bindsTo = [
+        "gitlab-config.service"
+      ] ++ optional (cfg.databaseHost == "") "postgresql.service"
+        ++ optional databaseActuallyCreateLocally "gitlab-postgresql.service";
+      wantedBy = [ "gitlab.target" ];
+      partOf = [ "gitlab.target" ];
+      serviceConfig = {
+        Type = "oneshot";
+        User = cfg.user;
+        Group = cfg.group;
+        TimeoutSec = "infinity";
+        Restart = "on-failure";
+        WorkingDirectory = "${cfg.packages.gitlab}/share/gitlab";
+        RemainAfterExit = true;
+
+        ExecStart = pkgs.writeShellScript "gitlab-db-config" ''
+          set -eu
+          umask u=rwx,g=rx,o=
+
+          initial_root_password="$(<'${cfg.initialRootPasswordFile}')"
+          ${gitlab-rake}/bin/gitlab-rake gitlab:db:configure GITLAB_ROOT_PASSWORD="$initial_root_password" \
+                                                             GITLAB_ROOT_EMAIL='${cfg.initialRootEmail}' > /dev/null
+        '';
+      };
+    };
+
    systemd.services.gitlab-sidekiq = {
-      after = [ "network.target" "redis.service" "gitlab.service" ];
-      wantedBy = [ "multi-user.target" ];
+      after = [
+        "network.target"
+        "redis.service"
+        "postgresql.service"
+        "gitlab-config.service"
+        "gitlab-db-config.service"
+      ];
+      bindsTo = [
+        "redis.service"
+        "gitlab-config.service"
+        "gitlab-db-config.service"
+      ] ++ optional (cfg.databaseHost == "") "postgresql.service";
+      wantedBy = [ "gitlab.target" ];
+      partOf = [ "gitlab.target" ];
      environment = gitlabEnv;
      path = with pkgs; [
        postgresqlPackage
@@ -758,9 +914,10 @@ in {
    };

    systemd.services.gitaly = {
-      after = [ "network.target" "gitlab.service" ];
-      bindsTo = [ "gitlab.service" ];
-      wantedBy = [ "multi-user.target" ];
+      after = [ "network.target" "gitlab-config.service" ];
+      bindsTo = [ "gitlab-config.service" ];
+      wantedBy = [ "gitlab.target" ];
+      partOf = [ "gitlab.target" ];
      path = with pkgs; [
        openssh
        procps  # See https://gitlab.com/gitlab-org/gitaly/issues/1562
@@ -783,8 +940,10 @@ in {

    systemd.services.gitlab-pages = mkIf (gitlabConfig.production.pages.enabled or false) {
      description = "GitLab static pages daemon";
-      after = [ "network.target" "redis.service" "gitlab.service" ]; # gitlab.service creates configs
-      wantedBy = [ "multi-user.target" ];
+      after = [ "network.target" "gitlab-config.service" ];
+      bindsTo = [ "gitlab-config.service" ];
+      wantedBy = [ "gitlab.target" ];
+      partOf = [ "gitlab.target" ];

      path = [ pkgs.unzip ];

@@ -803,7 +962,8 @@ in {

    systemd.services.gitlab-workhorse = {
      after = [ "network.target" ];
-      wantedBy = [ "multi-user.target" ];
+      wantedBy = [ "gitlab.target" ];
+      partOf = [ "gitlab.target" ];
      path = with pkgs; [
        exiftool
        git
@@ -832,8 +992,10 @@ in {

    systemd.services.gitlab-mailroom = mkIf (gitlabConfig.production.incoming_email.enabled or false) {
      description = "GitLab incoming mail daemon";
-      after = [ "network.target" "redis.service" "gitlab.service" ]; # gitlab.service creates configs
-      wantedBy = [ "multi-user.target" ];
+      after = [ "network.target" "redis.service" "gitlab-config.service" ];
+      bindsTo = [ "gitlab-config.service" ];
+      wantedBy = [ "gitlab.target" ];
+      partOf = [ "gitlab.target" ];
      environment = gitlabEnv;
      serviceConfig = {
        Type = "simple";
@@ -842,15 +1004,26 @@ in {

        User = cfg.user;
        Group = cfg.group;
-        ExecStart = "${cfg.packages.gitlab.rubyEnv}/bin/bundle exec mail_room -c ${cfg.packages.gitlab}/share/gitlab/config.dist/mail_room.yml";
+        ExecStart = "${cfg.packages.gitlab.rubyEnv}/bin/bundle exec mail_room -c ${cfg.statePath}/config/mail_room.yml";
        WorkingDirectory = gitlabEnv.HOME;
      };
    };

    systemd.services.gitlab = {
-      after = [ "gitlab-workhorse.service" "network.target" "gitlab-postgresql.service" "redis.service" ];
-      requires = [ "gitlab-sidekiq.service" ];
-      wantedBy = [ "multi-user.target" ];
+      after = [
+        "gitlab-workhorse.service"
+        "network.target"
+        "redis.service"
+        "gitlab-config.service"
+        "gitlab-db-config.service"
+      ];
+      bindsTo = [
+        "redis.service"
+        "gitlab-config.service"
+        "gitlab-db-config.service"
+      ] ++ optional (cfg.databaseHost == "") "postgresql.service";
+      wantedBy = [ "gitlab.target" ];
+      partOf = [ "gitlab.target" ];
      environment = gitlabEnv;
      path = with pkgs; [
        postgresqlPackage
@@ -868,96 +1041,7 @@ in {
        TimeoutSec = "infinity";
        Restart = "on-failure";
        WorkingDirectory = "${cfg.packages.gitlab}/share/gitlab";
-        ExecStartPre = let
-          preStartFullPrivileges = ''
-            shopt -s dotglob nullglob
-            set -eu
-
-            chown --no-dereference '${cfg.user}':'${cfg.group}' '${cfg.statePath}'/*
-            chown --no-dereference '${cfg.user}':'${cfg.group}' '${cfg.statePath}'/config/*
-          '';
-          preStart = ''
-            set -eu
-
-            cp -f ${cfg.packages.gitlab}/share/gitlab/VERSION ${cfg.statePath}/VERSION
-            rm -rf ${cfg.statePath}/db/*
-            rm -rf ${cfg.statePath}/config/initializers/*
-            rm -f ${cfg.statePath}/lib
-            cp -rf --no-preserve=mode ${cfg.packages.gitlab}/share/gitlab/config.dist/* ${cfg.statePath}/config
-            cp -rf --no-preserve=mode ${cfg.packages.gitlab}/share/gitlab/db/* ${cfg.statePath}/db
-            ln -sf ${extraGitlabRb} ${cfg.statePath}/config/initializers/extra-gitlab.rb
-
-            ${cfg.packages.gitlab-shell}/bin/install
-
-            ${optionalString cfg.smtp.enable ''
-              install -m u=rw ${smtpSettings} ${cfg.statePath}/config/initializers/smtp_settings.rb
-              ${optionalString (cfg.smtp.passwordFile != null) ''
-                smtp_password=$(<'${cfg.smtp.passwordFile}')
-                ${pkgs.replace}/bin/replace-literal -e '@smtpPassword@' "$smtp_password" '${cfg.statePath}/config/initializers/smtp_settings.rb'
-              ''}
-            ''}
-
-            (
-              umask u=rwx,g=,o=
-
-              ${pkgs.openssl}/bin/openssl rand -hex 32 > ${cfg.statePath}/gitlab_shell_secret
-
-              if [[ -h '${cfg.statePath}/config/database.yml' ]]; then
-                rm '${cfg.statePath}/config/database.yml'
-              fi
-
-              ${if cfg.databasePasswordFile != null then ''
-                  export db_password="$(<'${cfg.databasePasswordFile}')"
-
-                  if [[ -z "$db_password" ]]; then
-                    >&2 echo "Database password was an empty string!"
-                    exit 1
-                  fi
-
-                  ${pkgs.jq}/bin/jq <${pkgs.writeText "database.yml" (builtins.toJSON databaseConfig)} \
-                                    '.production.password = $ENV.db_password' \
-                                    >'${cfg.statePath}/config/database.yml'
-                ''
-                else ''
-                  ${pkgs.jq}/bin/jq <${pkgs.writeText "database.yml" (builtins.toJSON databaseConfig)} \
-                                    >'${cfg.statePath}/config/database.yml'
-                ''
-              }
-
-              ${utils.genJqSecretsReplacementSnippet
-                  gitlabConfig
-                  "${cfg.statePath}/config/gitlab.yml"
-              }
-
-              if [[ -h '${cfg.statePath}/config/secrets.yml' ]]; then
-                rm '${cfg.statePath}/config/secrets.yml'
-              fi
-
-              export secret="$(<'${cfg.secrets.secretFile}')"
-              export db="$(<'${cfg.secrets.dbFile}')"
-              export otp="$(<'${cfg.secrets.otpFile}')"
-              export jws="$(<'${cfg.secrets.jwsFile}')"
-              ${pkgs.jq}/bin/jq -n '{production: {secret_key_base: $ENV.secret,
-                                                  otp_key_base: $ENV.otp,
-                                                  db_key_base: $ENV.db,
-                                                  openid_connect_signing_key: $ENV.jws}}' \
-                                > '${cfg.statePath}/config/secrets.yml'
-            )
-
-            initial_root_password="$(<'${cfg.initialRootPasswordFile}')"
-            ${gitlab-rake}/bin/gitlab-rake gitlab:db:configure GITLAB_ROOT_PASSWORD="$initial_root_password" \
-                                                               GITLAB_ROOT_EMAIL='${cfg.initialRootEmail}' > /dev/null
-
-            # We remove potentially broken links to old gitlab-shell versions
-            rm -Rf ${cfg.statePath}/repositories/**/*.git/hooks
-
-            ${pkgs.git}/bin/git config --global core.autocrlf "input"
-          '';
-        in [
-          "+${pkgs.writeShellScript "gitlab-pre-start-full-privileges" preStartFullPrivileges}"
-          "${pkgs.writeShellScript "gitlab-pre-start" preStart}"
-        ];
-        ExecStart = "${cfg.packages.gitlab.rubyEnv}/bin/unicorn -c ${cfg.statePath}/config/unicorn.rb -E production";
+        ExecStart = "${cfg.packages.gitlab.rubyEnv}/bin/puma -C ${cfg.statePath}/config/puma.rb -e production";
      };

    };