nixos: Add gitlab and gitlab-shell

I had to make several adjustments to make it work with nixos:

* Replace relative config file lookups with ENV variable.
* Modify gitlab-shell to not clear then environment when running
  pre-receive.
* Modify gitlab-shell to write some environment variables into
  the .authorized_keys file to make sure gitlab-shell reads the
  correct config file.
* Log unicorn output to syslog.
  I tried various ways of adding a syslog package but the bundler would
  not pick them up. Please fix in a better way if possible.
* Gitlab-runner program wrapper.
  This is useful to run e.g. backups etc. with the correct
  environment set up.
This commit is contained in:
Thomas Hunger
2014-12-12 18:01:29 +01:00
committed by Jaka Hudoklin
parent b7eba773dc
commit 59995e168c
13 changed files with 3085 additions and 0 deletions
+2
View File
@@ -172,6 +172,7 @@
kubernetes = 162;
peerflix = 163;
chronos = 164;
gitlab = 165;
# When adding a uid, make sure it doesn't match an existing gid. And don't use uids above 399!
@@ -308,6 +309,7 @@
bosun = 157;
kubernetes = 158;
fleet = 159;
gitlab = 160;
# When adding a gid, make sure it doesn't match an existing uid. And don't use gids above 399!
+1
View File
@@ -176,6 +176,7 @@
./services/misc/etcd.nix
./services/misc/felix.nix
./services/misc/folding-at-home.nix
./services/misc/gitlab.nix
./services/misc/gitolite.nix
./services/misc/gpsd.nix
./services/misc/mesos-master.nix
@@ -0,0 +1,206 @@
# The following was taken from github.com/crohr/syslogger and is BSD
# licensed.
require 'syslog'
require 'logger'
require 'thread'
class Syslogger
VERSION = "1.6.0"
attr_reader :level, :ident, :options, :facility, :max_octets
attr_accessor :formatter
MAPPING = {
Logger::DEBUG => Syslog::LOG_DEBUG,
Logger::INFO => Syslog::LOG_INFO,
Logger::WARN => Syslog::LOG_WARNING,
Logger::ERROR => Syslog::LOG_ERR,
Logger::FATAL => Syslog::LOG_CRIT,
Logger::UNKNOWN => Syslog::LOG_ALERT
}
#
# Initializes default options for the logger
# <tt>ident</tt>:: the name of your program [default=$0].
# <tt>options</tt>:: syslog options [default=<tt>Syslog::LOG_PID | Syslog::LOG_CONS</tt>].
# Correct values are:
# LOG_CONS : writes the message on the console if an error occurs when sending the message;
# LOG_NDELAY : no delay before sending the message;
# LOG_PERROR : messages will also be written on STDERR;
# LOG_PID : adds the process number to the message (just after the program name)
# <tt>facility</tt>:: the syslog facility [default=nil] Correct values include:
# Syslog::LOG_DAEMON
# Syslog::LOG_USER
# Syslog::LOG_SYSLOG
# Syslog::LOG_LOCAL2
# Syslog::LOG_NEWS
# etc.
#
# Usage:
# logger = Syslogger.new("my_app", Syslog::LOG_PID | Syslog::LOG_CONS, Syslog::LOG_LOCAL0)
# logger.level = Logger::INFO # use Logger levels
# logger.warn "warning message"
# logger.debug "debug message"
#
def initialize(ident = $0, options = Syslog::LOG_PID | Syslog::LOG_CONS, facility = nil)
@ident = ident
@options = options || (Syslog::LOG_PID | Syslog::LOG_CONS)
@facility = facility
@level = Logger::INFO
@mutex = Mutex.new
@formatter = Logger::Formatter.new
end
%w{debug info warn error fatal unknown}.each do |logger_method|
# Accepting *args as message could be nil.
# Default params not supported in ruby 1.8.7
define_method logger_method.to_sym do |*args, &block|
return true if @level > Logger.const_get(logger_method.upcase)
message = args.first || block && block.call
add(Logger.const_get(logger_method.upcase), message)
end
unless logger_method == 'unknown'
define_method "#{logger_method}?".to_sym do
@level <= Logger.const_get(logger_method.upcase)
end
end
end
# Log a message at the Logger::INFO level. Useful for use with Rack::CommonLogger
def write(msg)
add(Logger::INFO, msg)
end
# Logs a message at the Logger::INFO level.
def <<(msg)
add(Logger::INFO, msg)
end
# Low level method to add a message.
# +severity+:: the level of the message. One of Logger::DEBUG, Logger::INFO, Logger::WARN, Logger::ERROR, Logger::FATAL, Logger::UNKNOWN
# +message+:: the message string.
# If nil, the method will call the block and use the result as the message string.
# If both are nil or no block is given, it will use the progname as per the behaviour of both the standard Ruby logger, and the Rails BufferedLogger.
# +progname+:: optionally, overwrite the program name that appears in the log message.
def add(severity, message = nil, progname = nil, &block)
if message.nil? && block.nil? && !progname.nil?
message, progname = progname, nil
end
progname ||= @ident
@mutex.synchronize do
Syslog.open(progname, @options, @facility) do |s|
s.mask = Syslog::LOG_UPTO(MAPPING[@level])
communication = clean(message || block && block.call)
if self.max_octets
buffer = "#{tags_text}"
communication.bytes do |byte|
buffer.concat(byte)
# if the last byte we added is potentially part of an escape, we'll go ahead and add another byte
if buffer.bytesize >= self.max_octets && !['%'.ord,'\\'.ord].include?(byte)
s.log(MAPPING[severity],buffer)
buffer = ""
end
end
s.log(MAPPING[severity],buffer) unless buffer.empty?
else
s.log(MAPPING[severity],"#{tags_text}#{communication}")
end
end
end
end
# Set the max octets of the messages written to the log
def max_octets=(max_octets)
@max_octets = max_octets
end
# Sets the minimum level for messages to be written in the log.
# +level+:: one of <tt>Logger::DEBUG</tt>, <tt>Logger::INFO</tt>, <tt>Logger::WARN</tt>, <tt>Logger::ERROR</tt>, <tt>Logger::FATAL</tt>, <tt>Logger::UNKNOWN</tt>
def level=(level)
level = Logger.const_get(level.to_s.upcase) if level.is_a?(Symbol)
unless level.is_a?(Fixnum)
raise ArgumentError.new("Invalid logger level `#{level.inspect}`")
end
@level = level
end
# Sets the ident string passed along to Syslog
def ident=(ident)
@ident = ident
end
# Tagging code borrowed from ActiveSupport gem
def tagged(*tags)
new_tags = push_tags(*tags)
yield self
ensure
pop_tags(new_tags.size)
end
def push_tags(*tags)
tags.flatten.reject{ |i| i.respond_to?(:empty?) ? i.empty? : !i }.tap do |new_tags|
current_tags.concat new_tags
end
end
def pop_tags(size = 1)
current_tags.pop size
end
def clear_tags!
current_tags.clear
end
protected
# Borrowed from SyslogLogger.
def clean(message)
message = message.to_s.dup
message.strip! # remove whitespace
message.gsub!(/\n/, '\\n') # escape newlines
message.gsub!(/%/, '%%') # syslog(3) freaks on % (printf)
message.gsub!(/\e\[[^m]*m/, '') # remove useless ansi color codes
message
end
private
def tags_text
tags = current_tags
if tags.any?
tags.collect { |tag| "[#{tag}] " }.join
end
end
def current_tags
Thread.current[:syslogger_tagged_logging_tags] ||= []
end
end
worker_processes 2
working_directory ENV["GITLAB_PATH"]
pid ENV["UNICORN_PATH"] + "/tmp/pids/unicorn.pid"
listen ENV["UNICORN_PATH"] + "/tmp/sockets/gitlab.socket", :backlog => 1024
listen "127.0.0.1:8080", :tcp_nopush => true
timeout 60
logger Syslogger.new
preload_app true
GC.respond_to?(:copy_on_write_friendly=) and
GC.copy_on_write_friendly = true
check_client_connection false
after_fork do |server, worker|
defined?(ActiveRecord::Base) and
ActiveRecord::Base.establish_connection
end
+283
View File
@@ -0,0 +1,283 @@
{ config, lib, pkgs, ... }:
# TODO: support non-postgresql
with lib;
let
cfg = config.services.gitlab;
ruby = pkgs.ruby;
rubyLibs = pkgs.rubyLibs;
databaseYml = ''
production:
adapter: postgresql
database: ${cfg.databaseName}
host: ${cfg.databaseHost}
password: ${cfg.databasePassword}
username: ${cfg.databaseUsername}
encoding: utf8
'';
gitlabShellYml = ''
user: gitlab
gitlab_url: "http://localhost:8080/"
http_settings:
self_signed_cert: false
repos_path: "${cfg.stateDir}/repositories"
log_file: "${cfg.stateDir}/log/gitlab-shell.log"
redis:
bin: ${pkgs.redis}/bin/redis-cli
host: 127.0.0.1
port: 6379
database: 0
namespace: resque:gitlab
'';
unicornConfig = builtins.readFile ./defaultUnicornConfig.rb;
gitlab-runner = pkgs.stdenv.mkDerivation rec {
name = "gitlab-runner";
buildInputs = [ pkgs.gitlab pkgs.rubyLibs.bundler pkgs.makeWrapper ];
phases = "installPhase fixupPhase";
buildPhase = "";
installPhase = ''
mkdir -p $out/bin
makeWrapper ${rubyLibs.bundler}/bin/bundle $out/bin/gitlab-runner\
--set RAKEOPT '"-f ${pkgs.gitlab}/share/gitlab/Rakefile"'\
--set UNICORN_PATH "${cfg.stateDir}/"\
--set GITLAB_PATH "${pkgs.gitlab}/share/gitlab/"\
--set GITLAB_APPLICATION_LOG_PATH "${cfg.stateDir}/log/application.log"\
--set GITLAB_SATELLITES_PATH "${cfg.stateDir}/satellites"\
--set GITLAB_SHELL_PATH "${pkgs.gitlab-shell}"\
--set GITLAB_REPOSITORIES_PATH "${cfg.stateDir}/repositories"\
--set GITLAB_SHELL_HOOKS_PATH "${cfg.stateDir}/shell/hooks"\
--set BUNDLE_GEMFILE "${pkgs.gitlab}/share/gitlab/Gemfile"\
--set GITLAB_EMAIL_FROM "${cfg.emailFrom}"\
--set GITLAB_SHELL_CONFIG_PATH "${cfg.stateDir}/shell/config.yml"\
--set GITLAB_SHELL_SECRET_PATH "${cfg.stateDir}/config/gitlab_shell_secret"\
--set GITLAB_HOST "${cfg.host}"\
--set GITLAB_BACKUP_PATH"${cfg.backupPath}"\
--set RAILS_ENV "production"
'';
};
in {
options = {
services.gitlab = {
enable = mkOption {
type = types.bool;
default = false;
description = ''
Enable the gitlab service.
'';
};
satelliteDir = mkOption {
type = types.str;
default = "/var/gitlab/git-satellites";
description = "Directory to store checked out git trees requires for operation.";
};
stateDir = mkOption {
type = types.str;
default = "/var/gitlab/state";
description = "The state directory, logs are stored here.";
};
backupPath = mkOption {
type = types.str;
default = cfg.stateDir + "/backup";
description = "Path for backups.";
};
databaseHost = mkOption {
type = types.str;
default = "127.0.0.1";
description = "Database hostname";
};
databasePassword = mkOption {
type = types.str;
default = "";
description = "Database user password";
};
databaseName = mkOption {
type = types.str;
default = "gitlab";
description = "Database name";
};
databaseUsername = mkOption {
type = types.str;
default = "gitlab";
description = "Database user";
};
emailFrom = mkOption {
type = types.str;
default = "example@example.org";
description = "The source address for emails sent by gitlab.";
};
host = mkOption {
type = types.str;
default = config.networking.hostName;
description = "The gitlab host name. Used e.g. for copy-paste URLs.";
};
};
};
config = mkIf cfg.enable {
environment.systemPackages = [ gitlab-runner pkgs.gitlab-shell ];
assertions = [
{ assertion = cfg.databasePassword != "";
message = "databasePassword must be set";
}
];
# Redis is required for the sidekiq queue runner.
services.redis.enable = mkDefault true;
# We use postgres as the main data store.
services.postgresql.enable = mkDefault true;
# Use postfix to send out mails.
services.postfix.enable = mkDefault true;
users.extraUsers = [
{ name = "gitlab";
group = "gitlab";
home = "${cfg.stateDir}/home";
shell = "${pkgs.bash}/bin/bash";
uid = config.ids.uids.gitlab;
} ];
users.extraGroups = [
{ name = "gitlab";
gid = config.ids.gids.gitlab;
} ];
systemd.services.gitlab-sidekiq = {
after = [ "network.target" "redis.service" ];
wantedBy = [ "multi-user.target" ];
environment.HOME = "${cfg.stateDir}/home";
environment.UNICORN_PATH = "${cfg.stateDir}/";
environment.GITLAB_PATH = "${pkgs.gitlab}/share/gitlab/";
environment.GITLAB_APPLICATION_LOG_PATH = "${cfg.stateDir}/log/application.log";
environment.GITLAB_SATELLITES_PATH = "${cfg.stateDir}/satellites";
environment.GITLAB_SHELL_PATH = "${pkgs.gitlab-shell}";
environment.GITLAB_REPOSITORIES_PATH = "${cfg.stateDir}/repositories";
environment.GITLAB_SHELL_HOOKS_PATH = "${cfg.stateDir}/shell/hooks";
environment.BUNDLE_GEMFILE = "${pkgs.gitlab}/share/gitlab/Gemfile";
environment.GITLAB_EMAIL_FROM = "${cfg.emailFrom}";
environment.GITLAB_SHELL_CONFIG_PATH = "${cfg.stateDir}/shell/config.yml";
environment.GITLAB_SHELL_SECRET_PATH = "${cfg.stateDir}/config/gitlab_shell_secret";
environment.GITLAB_HOST = "${cfg.host}";
environment.GITLAB_DATABASE_HOST = "${cfg.databaseHost}";
environment.GITLAB_DATABASE_PASSWORD = "${cfg.databasePassword}";
environment.RAILS_ENV = "production";
path = with pkgs; [
config.services.postgresql.package
gitAndTools.git
ruby
openssh
];
serviceConfig = {
Type = "simple";
User = "gitlab";
Group = "gitlab";
TimeoutSec = "300";
WorkingDirectory = "${pkgs.gitlab}/share/gitlab";
ExecStart="${rubyLibs.bundler}/bin/bundle exec \"sidekiq -q post_receive -q mailer -q system_hook -q project_web_hook -q gitlab_shell -q common -q default -e production -P ${cfg.stateDir}/tmp/sidekiq.pid\"";
};
};
systemd.services.gitlab = {
after = [ "network.target" "postgresql.service" "redis.service" ];
wantedBy = [ "multi-user.target" ];
environment.HOME = "${cfg.stateDir}/home";
environment.UNICORN_PATH = "${cfg.stateDir}/";
environment.GITLAB_PATH = "${pkgs.gitlab}/share/gitlab/";
environment.GITLAB_APPLICATION_LOG_PATH = "${cfg.stateDir}/log/application.log";
environment.GITLAB_SATELLITES_PATH = "${cfg.stateDir}/satellites";
environment.GITLAB_SHELL_PATH = "${pkgs.gitlab-shell}";
environment.GITLAB_REPOSITORIES_PATH = "${cfg.stateDir}/repositories";
environment.GITLAB_SHELL_HOOKS_PATH = "${cfg.stateDir}/shell/hooks";
environment.BUNDLE_GEMFILE = "${pkgs.gitlab}/share/gitlab/Gemfile";
environment.GITLAB_EMAIL_FROM = "${cfg.emailFrom}";
environment.GITLAB_HOST = "${cfg.host}";
environment.GITLAB_DATABASE_HOST = "${cfg.databaseHost}";
environment.GITLAB_DATABASE_PASSWORD = "${cfg.databasePassword}";
environment.RAILS_ENV = "production";
path = with pkgs; [
config.services.postgresql.package
gitAndTools.git
ruby
openssh
];
preStart = ''
# TODO: use env vars
mkdir -p ${cfg.stateDir}
mkdir -p ${cfg.stateDir}/log
mkdir -p ${cfg.stateDir}/satellites
mkdir -p ${cfg.stateDir}/repositories
mkdir -p ${cfg.stateDir}/shell/hooks
mkdir -p ${cfg.stateDir}/tmp/pids
mkdir -p ${cfg.stateDir}/tmp/sockets
rm -rf ${cfg.stateDir}/config
mkdir -p ${cfg.stateDir}/config
# TODO: What exactly is gitlab-shell doing with the secret?
head -c 20 /dev/urandom > ${cfg.stateDir}/config/gitlab_shell_secret
mkdir -p ${cfg.stateDir}/home/.ssh
touch ${cfg.stateDir}/home/.ssh/authorized_keys
cp -rf ${pkgs.gitlab}/share/gitlab/config ${cfg.stateDir}/
cp ${pkgs.gitlab}/share/gitlab/VERSION ${cfg.stateDir}/VERSION
ln -fs ${pkgs.writeText "database.yml" databaseYml} ${cfg.stateDir}/config/database.yml
ln -fs ${pkgs.writeText "unicorn.rb" unicornConfig} ${cfg.stateDir}/config/unicorn.rb
chown -R gitlab:gitlab ${cfg.stateDir}/
chmod -R 755 ${cfg.stateDir}/
if [ "${cfg.databaseHost}" = "127.0.0.1" ]; then
if ! test -e "${cfg.stateDir}/db-created"; then
psql postgres -c "CREATE ROLE gitlab WITH LOGIN NOCREATEDB NOCREATEROLE NOCREATEUSER ENCRYPTED PASSWORD '${cfg.databasePassword}'"
${config.services.postgresql.package}/bin/createdb --owner gitlab gitlab || true
touch "${cfg.stateDir}/db-created"
# force=yes disables the manual-interaction yes/no prompt
# which breaks without an stdin.
force=yes ${rubyLibs.bundler}/bin/bundle exec rake -f ${pkgs.gitlab}/share/gitlab/Rakefile gitlab:setup RAILS_ENV=production
fi
fi
# Install the shell required to push repositories
ln -fs ${pkgs.writeText "config.yml" gitlabShellYml} ${cfg.stateDir}/shell/config.yml
export GITLAB_SHELL_CONFIG_PATH=""${cfg.stateDir}/shell/config.yml
${pkgs.gitlab-shell}/bin/install
# Change permissions in the last step because some of the
# intermediary scripts like to create directories as root.
chown -R gitlab:gitlab ${cfg.stateDir}/
chmod -R 755 ${cfg.stateDir}/
'';
serviceConfig = {
PermissionsStartOnly = true; # preStart must be run as root
Type = "simple";
User = "gitlab";
Group = "gitlab";
TimeoutSec = "300";
WorkingDirectory = "${pkgs.gitlab}/share/gitlab";
ExecStart="${rubyLibs.bundler}/bin/bundle exec \"unicorn -c ${cfg.stateDir}/config/unicorn.rb -E production\"";
};
};
};
}