public/nixpkgs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

nixos/kubernetes: docker -> containerd

Browse Source 
also, nixos/containerd: module init
This commit is contained in:
Johan Thomsen
2021-03-07 12:51:14 +10:00
committed by zowoq
parent 7da62867be
commit 7b5c38e973
12 changed files with 129 additions and 185 deletions
Download Patch File Download Diff File Expand all files Collapse all files
nixos/doc/manual/release-notes/rl-2105.xml
+9
View File
@@ -788,6 +788,15 @@ self: super:
and use Maturin as their build tool.
</para>
</listitem>
<listitem>
<para>
Kubernetes has <link xlink:href="https://kubernetes.io/blog/2020/12/02/dont-panic-kubernetes-and-docker/">deprecated docker</link> as container runtime.
As a consequence, the Kubernetes module now has support for configuration of custom remote container runtimes and enables containerd by default.
Note that containerd is more strict regarding container image OCI-compliance.
As an example, images with CMD or ENTRYPOINT defined as strings (not lists) will fail on containerd, while working fine on docker.
Please test your setup and container images with containerd prior to upgrading.
</para>
</listitem>
</itemizedlist>
</section>
</section>
nixos/modules/module-list.nix
+1
View File
@@ -1053,6 +1053,7 @@
./testing/service-runner.nix
./virtualisation/anbox.nix
./virtualisation/container-config.nix
./virtualisation/containerd.nix
./virtualisation/containers.nix
./virtualisation/nixos-containers.nix
./virtualisation/oci-containers.nix
nixos/modules/services/cluster/kubernetes/apiserver.nix
-2
View File
@@ -260,7 +260,6 @@ in
account token issuer. The issuer will sign issued ID tokens with this
private key.
'';
default = top.serviceAccountSigningKeyFile;
type = path;
};
@@ -272,7 +271,6 @@ in
different files. If unspecified, --tls-private-key-file is used.
Must be specified when --service-account-signing-key is provided
'';
default = top.serviceAccountKeyFile;
type = path;
};
nixos/modules/services/cluster/kubernetes/default.nix
+25 -8
View File
@@ -5,6 +5,29 @@ with lib;
let
cfg = config.services.kubernetes;
defaultContainerdConfigFile = pkgs.writeText "containerd.toml" ''
version = 2
root = "/var/lib/containerd/daemon"
state = "/var/run/containerd/daemon"
oom_score = 0
[grpc]
address = "/var/run/containerd/containerd.sock"
[plugins."io.containerd.grpc.v1.cri"]
sandbox_image = "pause:latest"
[plugins."io.containerd.grpc.v1.cri".cni]
bin_dir = "/opt/cni/bin"
max_conf_num = 0
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
runtime_type = "io.containerd.runc.v2"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes."io.containerd.runc.v2".options]
SystemdCgroup = true
'';
mkKubeConfig = name: conf: pkgs.writeText "${name}-kubeconfig" (builtins.toJSON {
apiVersion = "v1";
kind = "Config";
@@ -222,14 +245,9 @@ in {
})
(mkIf cfg.kubelet.enable {
virtualisation.docker = {
virtualisation.containerd = {
enable = mkDefault true;
# kubernetes needs access to logs
logDriver = mkDefault "json-file";
# iptables must be disabled for kubernetes
extraOptions = "--iptables=false --ip-masq=false";
configFile = mkDefault defaultContainerdConfigFile;
};
})
@@ -269,7 +287,6 @@ in {
users.users.kubernetes = {
uid = config.ids.uids.kubernetes;
description = "Kubernetes user";
extraGroups = [ "docker" ];
group = "kubernetes";
home = cfg.dataDir;
createHome = true;
nixos/modules/services/cluster/kubernetes/flannel.nix
+2 -38
View File
@@ -8,16 +8,6 @@ let
# we want flannel to use kubernetes itself as configuration backend, not direct etcd
storageBackend = "kubernetes";
# needed for flannel to pass options to docker
mkDockerOpts = pkgs.runCommand "mk-docker-opts" {
buildInputs = [ pkgs.makeWrapper ];
} ''
mkdir -p $out
# bashInteractive needed for `compgen`
makeWrapper ${pkgs.bashInteractive}/bin/bash $out/mk-docker-opts --add-flags "${pkgs.kubernetes}/bin/mk-docker-opts.sh"
'';
in
{
###### interface
@@ -43,43 +33,17 @@ in
cniVersion = "0.3.1";
delegate = {
isDefaultGateway = true;
bridge = "docker0";
bridge = "mynet";
};
}];
};
systemd.services.mk-docker-opts = {
description = "Pre-Docker Actions";
path = with pkgs; [ gawk gnugrep ];
script = ''
${mkDockerOpts}/mk-docker-opts -d /run/flannel/docker
systemctl restart docker
'';
serviceConfig.Type = "oneshot";
};
systemd.paths.flannel-subnet-env = {
wantedBy = [ "flannel.service" ];
pathConfig = {
PathModified = "/run/flannel/subnet.env";
Unit = "mk-docker-opts.service";
};
};
systemd.services.docker = {
environment.DOCKER_OPTS = "-b none";
serviceConfig.EnvironmentFile = "-/run/flannel/docker";
};
# read environment variables generated by mk-docker-opts
virtualisation.docker.extraOptions = "$DOCKER_OPTS";
networking = {
firewall.allowedUDPPorts = [
8285 # flannel udp
8472 # flannel vxlan
];
dhcpcd.denyInterfaces = [ "docker*" "flannel*" ];
dhcpcd.denyInterfaces = [ "mynet*" "flannel*" ];
};
services.kubernetes.pki.certs = {
nixos/modules/services/cluster/kubernetes/kubelet.nix
+20 -7
View File
@@ -23,7 +23,7 @@ let
name = "pause";
tag = "latest";
contents = top.package.pause;
config.Cmd = "/bin/pause";
config.Cmd = ["/bin/pause"];
};
kubeconfig = top.lib.mkKubeConfig "kubelet" cfg.kubeconfig;
@@ -134,7 +134,7 @@ in
containerRuntimeEndpoint = mkOption {
description = "Endpoint at which to find the container runtime api interface/socket";
type = str;
default = "unix:///var/run/docker/containerd/containerd.sock";
default = "unix:///var/run/containerd/containerd.sock";
};
enable = mkEnableOption "Kubernetes kubelet.";
@@ -247,16 +247,24 @@ in
###### implementation
config = mkMerge [
(mkIf cfg.enable {
environment.etc."cni/net.d".source = cniConfig;
services.kubernetes.kubelet.seedDockerImages = [infraContainer];
boot.kernel.sysctl = {
"net.bridge.bridge-nf-call-iptables" = 1;
"net.ipv4.ip_forward" = 1;
"net.bridge.bridge-nf-call-ip6tables" = 1;
};
systemd.services.kubelet = {
description = "Kubernetes Kubelet Service";
wantedBy = [ "kubernetes.target" ];
after = [ "network.target" "kube-apiserver.service" "sockets.target" ];
after = [ "containerd.service" "network.target" "kube-apiserver.service" ];
path = with pkgs; [
gitMinimal
openssh
docker
util-linux
iproute
ethtool
@@ -266,8 +274,12 @@ in
] ++ lib.optional config.boot.zfs.enabled config.boot.zfs.package ++ top.path;
preStart = ''
${concatMapStrings (img: ''
echo "Seeding docker image: ${img}"
docker load <${img}
echo "Seeding container image: ${img}"
${if (lib.hasSuffix "gz" img) then
''${pkgs.gzip}/bin/zcat "${img}" | ${pkgs.containerd}/bin/ctr -n k8s.io image import -''
else
''${pkgs.coreutils}/bin/cat "${img}" | ${pkgs.containerd}/bin/ctr -n k8s.io image import -''
}
'') cfg.seedDockerImages}
rm /opt/cni/bin/* || true
@@ -320,6 +332,7 @@ in
${optionalString (cfg.verbosity != null) "--v=${toString cfg.verbosity}"} \
--container-runtime=${cfg.containerRuntime} \
--container-runtime-endpoint=${cfg.containerRuntimeEndpoint} \
--cgroup-driver=systemd \
${cfg.extraOpts}
'';
WorkingDirectory = top.dataDir;
@@ -329,7 +342,7 @@ in
# Allways include cni plugins
services.kubernetes.kubelet.cni.packages = [pkgs.cni-plugins];
boot.kernelModules = ["br_netfilter"];
boot.kernelModules = ["br_netfilter" "overlay"];
services.kubernetes.kubelet.hostname = with config.networking;
mkDefault (hostName + optionalString (domain != null) ".${domain}");
nixos/modules/services/networking/flannel.nix
+2 -4
View File
@@ -162,10 +162,7 @@ in {
NODE_NAME = cfg.nodeName;
};
path = [ pkgs.iptables ];
preStart = ''
mkdir -p /run/flannel
touch /run/flannel/docker
'' + optionalString (cfg.storageBackend == "etcd") ''
preStart = optionalString (cfg.storageBackend == "etcd") ''
echo "setting network configuration"
until ${pkgs.etcdctl}/bin/etcdctl set /coreos.com/network/config '${builtins.toJSON networkConfig}'
do
@@ -177,6 +174,7 @@ in {
ExecStart = "${cfg.package}/bin/flannel";
Restart = "always";
RestartSec = "10s";
RuntimeDirectory = "flannel";
};
};
nixos/modules/virtualisation/containerd.nix
+60
View File
@@ -0,0 +1,60 @@
{ pkgs, lib, config, ... }:
let
cfg = config.virtualisation.containerd;
containerdConfigChecked = pkgs.runCommand "containerd-config-checked.toml" { nativeBuildInputs = [pkgs.containerd]; } ''
containerd -c ${cfg.configFile} config dump >/dev/null
ln -s ${cfg.configFile} $out
'';
in
{
options.virtualisation.containerd = with lib.types; {
enable = lib.mkEnableOption "containerd container runtime";
configFile = lib.mkOption {
default = null;
description = "path to containerd config file";
type = nullOr path;
};
args = lib.mkOption {
default = {};
description = "extra args to append to the containerd cmdline";
type = attrsOf str;
};
};
config = lib.mkIf cfg.enable {
virtualisation.containerd.args.config = lib.mkIf (cfg.configFile != null) (toString containerdConfigChecked);
environment.systemPackages = [pkgs.containerd];
systemd.services.containerd = {
description = "containerd - container runtime";
wantedBy = [ "multi-user.target" ];
after = [ "network.target" ];
path = with pkgs; [
containerd
runc
iptables
];
serviceConfig = {
ExecStart = ''${pkgs.containerd}/bin/containerd ${lib.concatStringsSep " " (lib.cli.toGNUCommandLine {} cfg.args)}'';
Delegate = "yes";
KillMode = "process";
Type = "notify";
Restart = "always";
RestartSec = "5";
StartLimitBurst = "8";
StartLimitIntervalSec = "120s";
# "limits" defined below are adopted from upstream: https://github.com/containerd/containerd/blob/master/containerd.service
LimitNPROC = "infinity";
LimitCORE = "infinity";
LimitNOFILE = "infinity";
TasksMax = "infinity";
OOMScoreAdjust = "-999";
};
};
};
}
nixos/tests/kubernetes/dns.nix
+7 -8
View File
@@ -34,7 +34,7 @@ let
name = "redis";
tag = "latest";
contents = [ pkgs.redis pkgs.bind.host ];
config.Entrypoint = "/bin/redis-server";
config.Entrypoint = ["/bin/redis-server"];
};
probePod = pkgs.writeText "probe-pod.json" (builtins.toJSON {
@@ -55,12 +55,11 @@ let
name = "probe";
tag = "latest";
contents = [ pkgs.bind.host pkgs.busybox ];
config.Entrypoint = "/bin/tail";
config.Entrypoint = ["/bin/tail"];
};
extraConfiguration = { config, pkgs, ... }: {
extraConfiguration = { config, pkgs, lib, ... }: {
environment.systemPackages = [ pkgs.bind.host ];
# virtualisation.docker.extraOptions = "--dns=${config.services.kubernetes.addons.dns.clusterIp}";
services.dnsmasq.enable = true;
services.dnsmasq.servers = [
"/cluster.local/${config.services.kubernetes.addons.dns.clusterIp}#53"
@@ -77,7 +76,7 @@ let
# prepare machine1 for test
machine1.wait_until_succeeds("kubectl get node machine1.${domain} | grep -w Ready")
machine1.wait_until_succeeds(
"docker load < ${redisImage}"
"${pkgs.gzip}/bin/zcat ${redisImage} | ${pkgs.containerd}/bin/ctr -n k8s.io image import -"
)
machine1.wait_until_succeeds(
"kubectl create -f ${redisPod}"
@@ -86,7 +85,7 @@ let
"kubectl create -f ${redisService}"
)
machine1.wait_until_succeeds(
"docker load < ${probeImage}"
"${pkgs.gzip}/bin/zcat ${probeImage} | ${pkgs.containerd}/bin/ctr -n k8s.io image import -"
)
machine1.wait_until_succeeds(
"kubectl create -f ${probePod}"
@@ -118,7 +117,7 @@ let
# prepare machines for test
machine1.wait_until_succeeds("kubectl get node machine2.${domain} | grep -w Ready")
machine2.wait_until_succeeds(
"docker load < ${redisImage}"
"${pkgs.gzip}/bin/zcat ${redisImage} | ${pkgs.containerd}/bin/ctr -n k8s.io image import -"
)
machine1.wait_until_succeeds(
"kubectl create -f ${redisPod}"
@@ -127,7 +126,7 @@ let
"kubectl create -f ${redisService}"
)
machine2.wait_until_succeeds(
"docker load < ${probeImage}"
"${pkgs.gzip}/bin/zcat ${probeImage} | ${pkgs.containerd}/bin/ctr -n k8s.io image import -"
)
machine1.wait_until_succeeds(
"kubectl create -f ${probePod}"
nixos/tests/kubernetes/rbac.nix
+3 -3
View File
@@ -85,7 +85,7 @@ let
name = "kubectl";
tag = "latest";
contents = [ kubectl pkgs.busybox kubectlPod2 ];
config.Entrypoint = "/bin/sh";
config.Entrypoint = ["/bin/sh"];
};
base = {
@@ -97,7 +97,7 @@ let
machine1.wait_until_succeeds("kubectl get node machine1.my.zyx | grep -w Ready")
machine1.wait_until_succeeds(
"docker load < ${kubectlImage}"
"${pkgs.gzip}/bin/zcat ${kubectlImage} | ${pkgs.containerd}/bin/ctr -n k8s.io image import -"
)
machine1.wait_until_succeeds(
@@ -134,7 +134,7 @@ let
machine1.wait_until_succeeds("kubectl get node machine2.my.zyx | grep -w Ready")
machine2.wait_until_succeeds(
"docker load < ${kubectlImage}"
"${pkgs.gzip}/bin/zcat ${kubectlImage} | ${pkgs.containerd}/bin/ctr -n k8s.io image import -"
)
machine1.wait_until_succeeds(