nixos/kubernetes: docker -> containerd
also, nixos/containerd: module init
This commit is contained in:
Johan Thomsen
@@ -260,7 +260,6 @@ in
|
||||
account token issuer. The issuer will sign issued ID tokens with this
|
||||
private key.
|
||||
'';
|
||||
default = top.serviceAccountSigningKeyFile;
|
||||
type = path;
|
||||
};
|
||||
|
||||
@@ -272,7 +271,6 @@ in
|
||||
different files. If unspecified, --tls-private-key-file is used.
|
||||
Must be specified when --service-account-signing-key is provided
|
||||
'';
|
||||
default = top.serviceAccountKeyFile;
|
||||
type = path;
|
||||
};
|
||||
|
||||
|
||||
@@ -5,6 +5,29 @@ with lib;
|
||||
let
|
||||
cfg = config.services.kubernetes;
|
||||
|
||||
defaultContainerdConfigFile = pkgs.writeText "containerd.toml" ''
|
||||
version = 2
|
||||
root = "/var/lib/containerd/daemon"
|
||||
state = "/var/run/containerd/daemon"
|
||||
oom_score = 0
|
||||
|
||||
[grpc]
|
||||
address = "/var/run/containerd/containerd.sock"
|
||||
|
||||
[plugins."io.containerd.grpc.v1.cri"]
|
||||
sandbox_image = "pause:latest"
|
||||
|
||||
[plugins."io.containerd.grpc.v1.cri".cni]
|
||||
bin_dir = "/opt/cni/bin"
|
||||
max_conf_num = 0
|
||||
|
||||
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
|
||||
runtime_type = "io.containerd.runc.v2"
|
||||
|
||||
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes."io.containerd.runc.v2".options]
|
||||
SystemdCgroup = true
|
||||
'';
|
||||
|
||||
mkKubeConfig = name: conf: pkgs.writeText "${name}-kubeconfig" (builtins.toJSON {
|
||||
apiVersion = "v1";
|
||||
kind = "Config";
|
||||
@@ -222,14 +245,9 @@ in {
|
||||
})
|
||||
|
||||
(mkIf cfg.kubelet.enable {
|
||||
virtualisation.docker = {
|
||||
virtualisation.containerd = {
|
||||
enable = mkDefault true;
|
||||
|
||||
# kubernetes needs access to logs
|
||||
logDriver = mkDefault "json-file";
|
||||
|
||||
# iptables must be disabled for kubernetes
|
||||
extraOptions = "--iptables=false --ip-masq=false";
|
||||
configFile = mkDefault defaultContainerdConfigFile;
|
||||
};
|
||||
})
|
||||
|
||||
@@ -269,7 +287,6 @@ in {
|
||||
users.users.kubernetes = {
|
||||
uid = config.ids.uids.kubernetes;
|
||||
description = "Kubernetes user";
|
||||
extraGroups = [ "docker" ];
|
||||
group = "kubernetes";
|
||||
home = cfg.dataDir;
|
||||
createHome = true;
|
||||
|
||||
@@ -8,16 +8,6 @@ let
|
||||
|
||||
# we want flannel to use kubernetes itself as configuration backend, not direct etcd
|
||||
storageBackend = "kubernetes";
|
||||
|
||||
# needed for flannel to pass options to docker
|
||||
mkDockerOpts = pkgs.runCommand "mk-docker-opts" {
|
||||
buildInputs = [ pkgs.makeWrapper ];
|
||||
} ''
|
||||
mkdir -p $out
|
||||
|
||||
# bashInteractive needed for `compgen`
|
||||
makeWrapper ${pkgs.bashInteractive}/bin/bash $out/mk-docker-opts --add-flags "${pkgs.kubernetes}/bin/mk-docker-opts.sh"
|
||||
'';
|
||||
in
|
||||
{
|
||||
###### interface
|
||||
@@ -43,43 +33,17 @@ in
|
||||
cniVersion = "0.3.1";
|
||||
delegate = {
|
||||
isDefaultGateway = true;
|
||||
bridge = "docker0";
|
||||
bridge = "mynet";
|
||||
};
|
||||
}];
|
||||
};
|
||||
|
||||
systemd.services.mk-docker-opts = {
|
||||
description = "Pre-Docker Actions";
|
||||
path = with pkgs; [ gawk gnugrep ];
|
||||
script = ''
|
||||
${mkDockerOpts}/mk-docker-opts -d /run/flannel/docker
|
||||
systemctl restart docker
|
||||
'';
|
||||
serviceConfig.Type = "oneshot";
|
||||
};
|
||||
|
||||
systemd.paths.flannel-subnet-env = {
|
||||
wantedBy = [ "flannel.service" ];
|
||||
pathConfig = {
|
||||
PathModified = "/run/flannel/subnet.env";
|
||||
Unit = "mk-docker-opts.service";
|
||||
};
|
||||
};
|
||||
|
||||
systemd.services.docker = {
|
||||
environment.DOCKER_OPTS = "-b none";
|
||||
serviceConfig.EnvironmentFile = "-/run/flannel/docker";
|
||||
};
|
||||
|
||||
# read environment variables generated by mk-docker-opts
|
||||
virtualisation.docker.extraOptions = "$DOCKER_OPTS";
|
||||
|
||||
networking = {
|
||||
firewall.allowedUDPPorts = [
|
||||
8285 # flannel udp
|
||||
8472 # flannel vxlan
|
||||
];
|
||||
dhcpcd.denyInterfaces = [ "docker*" "flannel*" ];
|
||||
dhcpcd.denyInterfaces = [ "mynet*" "flannel*" ];
|
||||
};
|
||||
|
||||
services.kubernetes.pki.certs = {
|
||||
|
||||
@@ -23,7 +23,7 @@ let
|
||||
name = "pause";
|
||||
tag = "latest";
|
||||
contents = top.package.pause;
|
||||
config.Cmd = "/bin/pause";
|
||||
config.Cmd = ["/bin/pause"];
|
||||
};
|
||||
|
||||
kubeconfig = top.lib.mkKubeConfig "kubelet" cfg.kubeconfig;
|
||||
@@ -134,7 +134,7 @@ in
|
||||
containerRuntimeEndpoint = mkOption {
|
||||
description = "Endpoint at which to find the container runtime api interface/socket";
|
||||
type = str;
|
||||
default = "unix:///var/run/docker/containerd/containerd.sock";
|
||||
default = "unix:///var/run/containerd/containerd.sock";
|
||||
};
|
||||
|
||||
enable = mkEnableOption "Kubernetes kubelet.";
|
||||
@@ -247,16 +247,24 @@ in
|
||||
###### implementation
|
||||
config = mkMerge [
|
||||
(mkIf cfg.enable {
|
||||
|
||||
environment.etc."cni/net.d".source = cniConfig;
|
||||
|
||||
services.kubernetes.kubelet.seedDockerImages = [infraContainer];
|
||||
|
||||
boot.kernel.sysctl = {
|
||||
"net.bridge.bridge-nf-call-iptables" = 1;
|
||||
"net.ipv4.ip_forward" = 1;
|
||||
"net.bridge.bridge-nf-call-ip6tables" = 1;
|
||||
};
|
||||
|
||||
systemd.services.kubelet = {
|
||||
description = "Kubernetes Kubelet Service";
|
||||
wantedBy = [ "kubernetes.target" ];
|
||||
after = [ "network.target" "kube-apiserver.service" "sockets.target" ];
|
||||
after = [ "containerd.service" "network.target" "kube-apiserver.service" ];
|
||||
path = with pkgs; [
|
||||
gitMinimal
|
||||
openssh
|
||||
docker
|
||||
util-linux
|
||||
iproute
|
||||
ethtool
|
||||
@@ -266,8 +274,12 @@ in
|
||||
] ++ lib.optional config.boot.zfs.enabled config.boot.zfs.package ++ top.path;
|
||||
preStart = ''
|
||||
${concatMapStrings (img: ''
|
||||
echo "Seeding docker image: ${img}"
|
||||
docker load <${img}
|
||||
echo "Seeding container image: ${img}"
|
||||
${if (lib.hasSuffix "gz" img) then
|
||||
''${pkgs.gzip}/bin/zcat "${img}" | ${pkgs.containerd}/bin/ctr -n k8s.io image import -''
|
||||
else
|
||||
''${pkgs.coreutils}/bin/cat "${img}" | ${pkgs.containerd}/bin/ctr -n k8s.io image import -''
|
||||
}
|
||||
'') cfg.seedDockerImages}
|
||||
|
||||
rm /opt/cni/bin/* || true
|
||||
@@ -320,6 +332,7 @@ in
|
||||
${optionalString (cfg.verbosity != null) "--v=${toString cfg.verbosity}"} \
|
||||
--container-runtime=${cfg.containerRuntime} \
|
||||
--container-runtime-endpoint=${cfg.containerRuntimeEndpoint} \
|
||||
--cgroup-driver=systemd \
|
||||
${cfg.extraOpts}
|
||||
'';
|
||||
WorkingDirectory = top.dataDir;
|
||||
@@ -329,7 +342,7 @@ in
|
||||
# Allways include cni plugins
|
||||
services.kubernetes.kubelet.cni.packages = [pkgs.cni-plugins];
|
||||
|
||||
boot.kernelModules = ["br_netfilter"];
|
||||
boot.kernelModules = ["br_netfilter" "overlay"];
|
||||
|
||||
services.kubernetes.kubelet.hostname = with config.networking;
|
||||
mkDefault (hostName + optionalString (domain != null) ".${domain}");
|
||||
|
||||
Reference in New Issue
Block a user