public/nixpkgs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/master' into staging-next

Browse Source 
Conflicts:
  pkgs/applications/terminal-emulators/alacritty/default.nix
  pkgs/servers/clickhouse/default.nix
This commit is contained in:
Jonathan Ringer
2021-05-20 09:12:42 -07:00
parent 24f13758b9 8aac6d8daf
commit 14f3686af1
120 changed files with 7535 additions and 2303 deletions
Download Patch File Download Diff File Expand all files Collapse all files
nixos/lib/build-vms.nix
+7
View File
@@ -36,6 +36,13 @@ rec {
[ ../modules/virtualisation/qemu-vm.nix
../modules/testing/test-instrumentation.nix # !!! should only get added for automated test runs
{ key = "no-manual"; documentation.nixos.enable = false; }
{ key = "no-revision";
# Make the revision metadata constant, in order to avoid needless retesting.
# The human version (e.g. 21.05-pre) is left as is, because it is useful
# for external modules that test with e.g. nixosTest and rely on that
# version number.
config.system.nixos.revision = "constant-nixos-revision";
}
{ key = "nodes"; _module.args.nodes = nodes; }
] ++ optional minimal ../modules/testing/minimal-kernel.nix;
};
nixos/lib/test-driver/test-driver.py
+27 -6
View File
@@ -3,6 +3,7 @@ from contextlib import contextmanager, _GeneratorContextManager
from queue import Queue, Empty
from typing import Tuple, Any, Callable, Dict, Iterator, Optional, List, Iterable
from xml.sax.saxutils import XMLGenerator
from colorama import Style
import queue
import io
import _thread
@@ -151,6 +152,8 @@ class Logger:
self.xml.startDocument()
self.xml.startElement("logfile", attrs={})
self._print_serial_logs = True
def close(self) -> None:
self.xml.endElement("logfile")
self.xml.endDocument()
@@ -174,15 +177,21 @@ class Logger:
self.drain_log_queue()
self.log_line(message, attributes)
def enqueue(self, message: Dict[str, str]) -> None:
self.queue.put(message)
def log_serial(self, message: str, machine: str) -> None:
self.enqueue({"msg": message, "machine": machine, "type": "serial"})
if self._print_serial_logs:
eprint(Style.DIM + "{} # {}".format(machine, message) + Style.RESET_ALL)
def enqueue(self, item: Dict[str, str]) -> None:
self.queue.put(item)
def drain_log_queue(self) -> None:
try:
while True:
item = self.queue.get_nowait()
attributes = {"machine": item["machine"], "type": "serial"}
self.log_line(self.sanitise(item["msg"]), attributes)
msg = self.sanitise(item["msg"])
del item["msg"]
self.log_line(msg, item)
except Empty:
pass
@@ -327,6 +336,9 @@ class Machine:
def log(self, msg: str) -> None:
self.logger.log(msg, {"machine": self.name})
def log_serial(self, msg: str) -> None:
self.logger.log_serial(msg, self.name)
def nested(self, msg: str, attrs: Dict[str, str] = {}) -> _GeneratorContextManager:
my_attrs = {"machine": self.name}
my_attrs.update(attrs)
@@ -784,8 +796,7 @@ class Machine:
# Ignore undecodable bytes that may occur in boot menus
line = _line.decode(errors="ignore").replace("\r", "").rstrip()
self.last_lines.put(line)
eprint("{} # {}".format(self.name, line))
self.logger.enqueue({"msg": line, "machine": self.name})
self.log_serial(line)
_thread.start_new_thread(process_serial_output, ())
@@ -927,6 +938,16 @@ def run_tests() -> None:
machine.execute("sync")
def serial_stdout_on() -> None:
global log
log._print_serial_logs = True
def serial_stdout_off() -> None:
global log
log._print_serial_logs = False
@contextmanager
def subtest(name: str) -> Iterator[None]:
with log.nested(name):
nixos/lib/testing-python.nix
+1 -1
View File
@@ -25,7 +25,7 @@ rec {
name = "nixos-test-driver";
nativeBuildInputs = [ makeWrapper ];
buildInputs = [ (python3.withPackages (p: [ p.ptpython ])) ];
buildInputs = [ (python3.withPackages (p: [ p.ptpython p.colorama ])) ];
checkInputs = with python3Packages; [ pylint black mypy ];
dontUnpack = true;
nixos/modules/services/cluster/kubernetes/addon-manager.nix
+1 -1
View File
@@ -62,7 +62,7 @@ in
'';
};
enable = mkEnableOption "Whether to enable Kubernetes addon manager.";
enable = mkEnableOption "Kubernetes addon manager.";
};
###### implementation
nixos/modules/services/databases/clickhouse.nix
+1
View File
@@ -42,6 +42,7 @@ with lib;
User = "clickhouse";
Group = "clickhouse";
ConfigurationDirectory = "clickhouse-server";
AmbientCapabilities = "CAP_SYS_NICE";
StateDirectory = "clickhouse";
LogsDirectory = "clickhouse";
ExecStart = "${pkgs.clickhouse}/bin/clickhouse-server --config-file=${pkgs.clickhouse}/etc/clickhouse-server/config.xml";
nixos/modules/services/network-filesystems/netatalk.nix
+40 -95
View File
@@ -3,43 +3,10 @@
with lib;
let
cfg = config.services.netatalk;
extmapFile = pkgs.writeText "extmap.conf" cfg.extmap;
afpToString = x: if builtins.typeOf x == "bool"
then boolToString x
else toString x;
volumeConfig = name:
let vol = getAttr name cfg.volumes; in
"[${name}]\n " + (toString (
map
(key: "${key} = ${afpToString (getAttr key vol)}\n")
(attrNames vol)
));
afpConf = ''[Global]
extmap file = ${extmapFile}
afp port = ${toString cfg.port}
${cfg.extraConfig}
${if cfg.homes.enable then ''[Homes]
${optionalString (cfg.homes.path != "") "path = ${cfg.homes.path}"}
basedir regex = ${cfg.homes.basedirRegex}
${cfg.homes.extraConfig}
'' else ""}
${toString (map volumeConfig (attrNames cfg.volumes))}
'';
afpConfFile = pkgs.writeText "afp.conf" afpConf;
in
{
settingsFormat = pkgs.formats.ini { };
afpConfFile = settingsFormat.generate "afp.conf" cfg.settings;
in {
options = {
services.netatalk = {
@@ -51,61 +18,24 @@ in
description = "TCP port to be used for AFP.";
};
extraConfig = mkOption {
type = types.lines;
default = "";
example = "uam list = uams_guest.so";
description = ''
Lines of configuration to add to the <literal>[Global]</literal> section.
See <literal>man apf.conf</literal> for more information.
'';
};
homes = {
enable = mkOption {
type = types.bool;
default = false;
description = "Enable sharing of the UNIX server user home directories.";
};
path = mkOption {
type = types.str;
default = "";
example = "afp-data";
description = "Share not the whole user home but this subdirectory path.";
};
basedirRegex = mkOption {
example = "/home";
type = types.str;
description = "Regex which matches the parent directory of the user homes.";
};
extraConfig = mkOption {
type = types.lines;
default = "";
description = ''
Lines of configuration to add to the <literal>[Homes]</literal> section.
See <literal>man apf.conf</literal> for more information.
'';
};
};
volumes = mkOption {
settings = mkOption {
inherit (settingsFormat) type;
default = { };
type = types.attrsOf (types.attrsOf types.unspecified);
description =
''
Set of AFP volumes to export.
See <literal>man apf.conf</literal> for more information.
'';
example = literalExample ''
{ srv =
{ path = "/srv";
"read only" = true;
"hosts allow" = "10.1.0.0/16 10.2.1.100 2001:0db8:1234::/48";
};
}
example = {
Global = { "uam list" = "uams_guest.so"; };
Homes = {
path = "afp-data";
"basedir regex" = "/home";
};
example-volume = {
path = "/srv/volume";
"read only" = true;
};
};
description = ''
Configuration for Netatalk. See
<citerefentry><refentrytitle>afp.conf</refentrytitle>
<manvolnum>5</manvolnum></citerefentry>.
'';
};
@@ -114,18 +44,33 @@ in
default = "";
description = ''
File name extension mappings.
See <literal>man extmap.conf</literal> for more information.
See <citerefentry><refentrytitle>extmap.conf</refentrytitle>
<manvolnum>5</manvolnum></citerefentry>. for more information.
'';
};
};
};
imports = (map (option:
mkRemovedOptionModule [ "services" "netatalk" option ]
"This option was removed in favor of `services.netatalk.settings`.") [
"extraConfig"
"homes"
"volumes"
]);
config = mkIf cfg.enable {
services.netatalk.settings.Global = {
"afp port" = toString cfg.port;
"extmap file" = "${pkgs.writeText "extmap.conf" cfg.extmap}";
};
systemd.services.netatalk = {
description = "Netatalk AFP fileserver for Macintosh clients";
unitConfig.Documentation = "man:afp.conf(5) man:netatalk(8) man:afpd(8) man:cnid_metad(8) man:cnid_dbd(8)";
unitConfig.Documentation =
"man:afp.conf(5) man:netatalk(8) man:afpd(8) man:cnid_metad(8) man:cnid_dbd(8)";
after = [ "network.target" "avahi-daemon.service" ];
wantedBy = [ "multi-user.target" ];
@@ -135,12 +80,12 @@ in
Type = "forking";
GuessMainPID = "no";
PIDFile = "/run/lock/netatalk";
ExecStartPre = "${pkgs.coreutils}/bin/mkdir -m 0755 -p /var/lib/netatalk/CNID";
ExecStart = "${pkgs.netatalk}/sbin/netatalk -F ${afpConfFile}";
ExecStart = "${pkgs.netatalk}/sbin/netatalk -F ${afpConfFile}";
ExecReload = "${pkgs.coreutils}/bin/kill -HUP $MAINPID";
ExecStop = "${pkgs.coreutils}/bin/kill -TERM $MAINPID";
ExecStop = "${pkgs.coreutils}/bin/kill -TERM $MAINPID";
Restart = "always";
RestartSec = 1;
StateDirectory = [ "netatalk/CNID" ];
};
};
nixos/modules/services/networking/nsd.nix
+19 -3
View File
@@ -20,6 +20,15 @@ let
mkZoneFileName = name: if name == "." then "root" else name;
# replaces include: directives for keys with fake keys for nsd-checkconf
injectFakeKeys = keys: concatStrings
(mapAttrsToList
(keyName: keyOptions: ''
fakeKey="$(${pkgs.bind}/bin/tsig-keygen -a ${escapeShellArgs [ keyOptions.algorithm keyName ]} | grep -oP "\s*secret \"\K.*(?=\";)")"
sed "s@^\s*include:\s*\"${stateDir}/private/${keyName}\"\$@secret: $fakeKey@" -i $out/nsd.conf
'')
keys);
nsdEnv = pkgs.buildEnv {
name = "nsd-env";
@@ -34,9 +43,9 @@ let
echo "|- checking zone '$out/zones/$zoneFile'"
${nsdPkg}/sbin/nsd-checkzone "$zoneFile" "$zoneFile" || {
if grep -q \\\\\\$ "$zoneFile"; then
echo zone "$zoneFile" contains escaped dollar signes \\\$
echo Escaping them is not needed any more. Please make shure \
to unescape them where they prefix a variable name
echo zone "$zoneFile" contains escaped dollar signs \\\$
echo Escaping them is not needed any more. Please make sure \
to unescape them where they prefix a variable name.
fi
exit 1
@@ -44,7 +53,14 @@ let
done
echo "checking configuration file"
# Save original config file including key references...
cp $out/nsd.conf{,.orig}
# ...inject mock keys into config
${injectFakeKeys cfg.keys}
# ...do the checkconf
${nsdPkg}/sbin/nsd-checkconf $out/nsd.conf
# ... and restore original config file.
mv $out/nsd.conf{.orig,}
'';
};
nixos/tests/all-tests.nix
+1
View File
@@ -303,6 +303,7 @@ in
openarena = handleTest ./openarena.nix {};
openldap = handleTest ./openldap.nix {};
opensmtpd = handleTest ./opensmtpd.nix {};
opensmtpd-rspamd = handleTest ./opensmtpd-rspamd.nix {};
openssh = handleTest ./openssh.nix {};
openstack-image-metadata = (handleTestOn ["x86_64-linux"] ./openstack-image.nix {}).metadata or {};
openstack-image-userdata = (handleTestOn ["x86_64-linux"] ./openstack-image.nix {}).userdata or {};
nixos/tests/clickhouse.nix
+1
View File
@@ -4,6 +4,7 @@ import ./make-test-python.nix ({ pkgs, ... }: {
machine = {
services.clickhouse.enable = true;
virtualisation.memorySize = 4096;
};
testScript =
nixos/tests/nsd.nix
+9
View File
@@ -43,6 +43,10 @@ in import ./make-test-python.nix ({ pkgs, ...} : {
services.nsd.enable = true;
services.nsd.rootServer = true;
services.nsd.interfaces = lib.mkForce [];
services.nsd.keys."tsig.example.com." = {
algorithm = "hmac-sha256";
keyFile = pkgs.writeTextFile { name = "tsig.example.com."; text = "aR3FJA92+bxRSyosadsJ8Aeeav5TngQW/H/EF9veXbc="; };
};
services.nsd.zones."example.com.".data = ''
@ SOA ns.example.com noc.example.com 666 7200 3600 1209600 3600
ipv4 A 1.2.3.4
@@ -51,6 +55,7 @@ in import ./make-test-python.nix ({ pkgs, ...} : {
ns A 192.168.0.1
ns AAAA dead:beef::1
'';
services.nsd.zones."example.com.".provideXFR = [ "0.0.0.0 tsig.example.com." ];
services.nsd.zones."deleg.example.com.".data = ''
@ SOA ns.example.com noc.example.com 666 7200 3600 1209600 3600
@ A 9.8.7.6
@@ -71,6 +76,10 @@ in import ./make-test-python.nix ({ pkgs, ...} : {
clientv6.wait_for_unit("network.target")
server.wait_for_unit("nsd.service")
with subtest("server tsig.example.com."):
expected_tsig = " secret: \"aR3FJA92+bxRSyosadsJ8Aeeav5TngQW/H/EF9veXbc=\"\n"
tsig=server.succeed("cat /var/lib/nsd/private/tsig.example.com.")
assert expected_tsig == tsig, f"Expected /var/lib/nsd/private/tsig.example.com. to contain '{expected_tsig}', but found '{tsig}'"
def assert_host(type, rr, query, expected):
self = clientv4 if type == 4 else clientv6
nixos/tests/opensmtpd-rspamd.nix
+142
View File
@@ -0,0 +1,142 @@
import ./make-test-python.nix {
name = "opensmtpd-rspamd";
nodes = {
smtp1 = { pkgs, ... }: {
imports = [ common/user-account.nix ];
networking = {
firewall.allowedTCPPorts = [ 25 143 ];
useDHCP = false;
interfaces.eth1.ipv4.addresses = pkgs.lib.mkOverride 0 [
{ address = "192.168.1.1"; prefixLength = 24; }
];
};
environment.systemPackages = [ pkgs.opensmtpd ];
services.opensmtpd = {
enable = true;
extraServerArgs = [ "-v" ];
serverConfiguration = ''
listen on 0.0.0.0
action dovecot_deliver mda \
"${pkgs.dovecot}/libexec/dovecot/deliver -d %{user.username}"
match from any for local action dovecot_deliver
action do_relay relay
# DO NOT DO THIS IN PRODUCTION!
# Setting up authentication requires a certificate which is painful in
# a test environment, but THIS WOULD BE DANGEROUS OUTSIDE OF A
# WELL-CONTROLLED ENVIRONMENT!
match from any for any action do_relay
'';
};
services.dovecot2 = {
enable = true;
enableImap = true;
mailLocation = "maildir:~/mail";
protocols = [ "imap" ];
};
};
smtp2 = { pkgs, ... }: {
imports = [ common/user-account.nix ];
virtualisation.memorySize = 512;
networking = {
firewall.allowedTCPPorts = [ 25 143 ];
useDHCP = false;
interfaces.eth1.ipv4.addresses = pkgs.lib.mkOverride 0 [
{ address = "192.168.1.2"; prefixLength = 24; }
];
};
environment.systemPackages = [ pkgs.opensmtpd ];
services.rspamd = {
enable = true;
locals."worker-normal.inc".text = ''
bind_socket = "127.0.0.1:11333";
'';
};
services.opensmtpd = {
enable = true;
extraServerArgs = [ "-v" ];
serverConfiguration = ''
filter rspamd proc-exec "${pkgs.opensmtpd-filter-rspamd}/bin/filter-rspamd"
listen on 0.0.0.0 filter rspamd
action dovecot_deliver mda \
"${pkgs.dovecot}/libexec/dovecot/deliver -d %{user.username}"
match from any for local action dovecot_deliver
'';
};
services.dovecot2 = {
enable = true;
enableImap = true;
mailLocation = "maildir:~/mail";
protocols = [ "imap" ];
};
};
client = { pkgs, ... }: {
networking = {
useDHCP = false;
interfaces.eth1.ipv4.addresses = pkgs.lib.mkOverride 0 [
{ address = "192.168.1.3"; prefixLength = 24; }
];
};
environment.systemPackages = let
sendTestMail = pkgs.writeScriptBin "send-a-test-mail" ''
#!${pkgs.python3.interpreter}
import smtplib, sys
with smtplib.SMTP('192.168.1.1') as smtp:
smtp.sendmail('alice@[192.168.1.1]', 'bob@[192.168.1.2]', """
From: alice@smtp1
To: bob@smtp2
Subject: Test
Hello World
Here goes the spam test
XJS*C4JDBQADN1.NSBN3*2IDNEN*GTUBE-STANDARD-ANTI-UBE-TEST-EMAIL*C.34X
""")
'';
checkMailBounced = pkgs.writeScriptBin "check-mail-bounced" ''
#!${pkgs.python3.interpreter}
import imaplib
with imaplib.IMAP4('192.168.1.1', 143) as imap:
imap.login('alice', 'foobar')
imap.select()
status, refs = imap.search(None, 'ALL')
assert status == 'OK'
assert len(refs) == 1
status, msg = imap.fetch(refs[0], 'BODY[TEXT]')
assert status == 'OK'
content = msg[0][1]
print("===> content:", content)
assert b"An error has occurred while attempting to deliver a message" in content
'';
in [ sendTestMail checkMailBounced ];
};
};
testScript = ''
start_all()
client.wait_for_unit("network-online.target")
smtp1.wait_for_unit("opensmtpd")
smtp2.wait_for_unit("opensmtpd")
smtp2.wait_for_unit("rspamd")
smtp2.wait_for_unit("dovecot2")
# To prevent sporadic failures during daemon startup, make sure
# services are listening on their ports before sending requests
smtp1.wait_for_open_port(25)
smtp2.wait_for_open_port(25)
smtp2.wait_for_open_port(143)
smtp2.wait_for_open_port(11333)
client.succeed("send-a-test-mail")
smtp1.wait_until_fails("smtpctl show queue | egrep .")
client.succeed("check-mail-bounced >&2")
'';
meta.timeout = 1800;
}