nixos/kubernetes: major module refactor
- All kubernetes components have been seperated into different files - All TLS-enabled ports have been deprecated and disabled by default - EasyCert option added to support automatic cluster PKI-bootstrap - RBAC has been enforced for all cluster components by default - NixOS kubernetes test cases make use of easyCerts to setup PKI
This commit is contained in:
Johan Thomsen
@@ -10,7 +10,6 @@ let
|
||||
mkKubernetesBaseTest =
|
||||
{ name, domain ? "my.zyx", test, machines
|
||||
, pkgs ? import <nixpkgs> { inherit system; }
|
||||
, certs ? import ./certs.nix { inherit pkgs; externalDomain = domain; kubelets = attrNames machines; }
|
||||
, extraConfiguration ? null }:
|
||||
let
|
||||
masterName = head (filter (machineName: any (role: role == "master") machines.${machineName}.roles) (attrNames machines));
|
||||
@@ -20,6 +19,10 @@ let
|
||||
${master.ip} api.${domain}
|
||||
${concatMapStringsSep "\n" (machineName: "${machines.${machineName}.ip} ${machineName}.${domain}") (attrNames machines)}
|
||||
'';
|
||||
kubectl = with pkgs; runCommand "wrap-kubectl" { buildInputs = [ makeWrapper ]; } ''
|
||||
mkdir -p $out/bin
|
||||
makeWrapper ${pkgs.kubernetes}/bin/kubectl $out/bin/kubectl --set KUBECONFIG "/etc/kubernetes/cluster-admin.kubeconfig"
|
||||
'';
|
||||
in makeTest {
|
||||
inherit name;
|
||||
|
||||
@@ -27,6 +30,7 @@ let
|
||||
{ config, pkgs, lib, nodes, ... }:
|
||||
mkMerge [
|
||||
{
|
||||
boot.postBootCommands = "rm -fr /var/lib/kubernetes/secrets /tmp/shared/*";
|
||||
virtualisation.memorySize = mkDefault 1536;
|
||||
virtualisation.diskSize = mkDefault 4096;
|
||||
networking = {
|
||||
@@ -45,34 +49,26 @@ let
|
||||
};
|
||||
};
|
||||
programs.bash.enableCompletion = true;
|
||||
environment.variables = {
|
||||
ETCDCTL_CERT_FILE = "${certs.worker}/etcd-client.pem";
|
||||
ETCDCTL_KEY_FILE = "${certs.worker}/etcd-client-key.pem";
|
||||
ETCDCTL_CA_FILE = "${certs.worker}/ca.pem";
|
||||
ETCDCTL_PEERS = "https://etcd.${domain}:2379";
|
||||
};
|
||||
environment.systemPackages = [ kubectl ];
|
||||
services.flannel.iface = "eth1";
|
||||
services.kubernetes.apiserver.advertiseAddress = master.ip;
|
||||
services.kubernetes = {
|
||||
addons.dashboard.enable = true;
|
||||
|
||||
easyCerts = true;
|
||||
inherit (machine) roles;
|
||||
apiserver = {
|
||||
securePort = 443;
|
||||
advertiseAddress = master.ip;
|
||||
};
|
||||
masterAddress = "${masterName}.${config.networking.domain}";
|
||||
};
|
||||
}
|
||||
(optionalAttrs (any (role: role == "master") machine.roles) {
|
||||
networking.firewall.allowedTCPPorts = [
|
||||
2379 2380 # etcd
|
||||
443 # kubernetes apiserver
|
||||
];
|
||||
services.etcd = {
|
||||
enable = true;
|
||||
certFile = "${certs.master}/etcd.pem";
|
||||
keyFile = "${certs.master}/etcd-key.pem";
|
||||
trustedCaFile = "${certs.master}/ca.pem";
|
||||
peerClientCertAuth = true;
|
||||
listenClientUrls = ["https://0.0.0.0:2379"];
|
||||
listenPeerUrls = ["https://0.0.0.0:2380"];
|
||||
advertiseClientUrls = ["https://etcd.${config.networking.domain}:2379"];
|
||||
initialCluster = ["${masterName}=https://etcd.${config.networking.domain}:2380"];
|
||||
initialAdvertisePeerUrls = ["https://etcd.${config.networking.domain}:2380"];
|
||||
};
|
||||
})
|
||||
(import ./kubernetes-common.nix { inherit (machine) roles; inherit pkgs config certs; })
|
||||
(optionalAttrs (machine ? "extraConfiguration") (machine.extraConfiguration { inherit config pkgs lib nodes; }))
|
||||
(optionalAttrs (extraConfiguration != null) (extraConfiguration { inherit config pkgs lib nodes; }))
|
||||
]
|
||||
|
||||
@@ -1,219 +0,0 @@
|
||||
{
|
||||
pkgs ? import <nixpkgs> {},
|
||||
externalDomain ? "myawesomecluster.cluster.yourdomain.net",
|
||||
serviceClusterIp ? "10.0.0.1",
|
||||
kubelets,
|
||||
...
|
||||
}:
|
||||
let
|
||||
runWithCFSSL = name: cmd:
|
||||
let secrets = pkgs.runCommand "${name}-cfss.json" {
|
||||
buildInputs = [ pkgs.cfssl pkgs.jq ];
|
||||
outputs = [ "out" "cert" "key" "csr" ];
|
||||
}
|
||||
''
|
||||
(
|
||||
echo "${cmd}"
|
||||
cfssl ${cmd} > tmp
|
||||
cat tmp | jq -r .key > $key
|
||||
cat tmp | jq -r .cert > $cert
|
||||
cat tmp | jq -r .csr > $csr
|
||||
|
||||
touch $out
|
||||
) 2>&1 | fold -w 80 -s
|
||||
'';
|
||||
in {
|
||||
key = secrets.key;
|
||||
cert = secrets.cert;
|
||||
csr = secrets.csr;
|
||||
};
|
||||
|
||||
writeCFSSL = content:
|
||||
pkgs.runCommand content.name {
|
||||
buildInputs = [ pkgs.cfssl pkgs.jq ];
|
||||
} ''
|
||||
mkdir -p $out
|
||||
cd $out
|
||||
|
||||
json=${pkgs.lib.escapeShellArg (builtins.toJSON content)}
|
||||
|
||||
# for a given $field in the $json, treat the associated value as a
|
||||
# file path and substitute the contents thereof into the $json
|
||||
# object.
|
||||
expandFileField() {
|
||||
local field=$1
|
||||
if jq -e --arg field "$field" 'has($field)'; then
|
||||
local path="$(echo "$json" | jq -r ".$field")"
|
||||
json="$(echo "$json" | jq --arg val "$(cat "$path")" ".$field = \$val")"
|
||||
fi
|
||||
}
|
||||
|
||||
expandFileField key
|
||||
expandFileField ca
|
||||
expandFileField cert
|
||||
|
||||
echo "$json" | cfssljson -bare ${content.name}
|
||||
'';
|
||||
|
||||
noCSR = content: pkgs.lib.filterAttrs (n: v: n != "csr") content;
|
||||
noKey = content: pkgs.lib.filterAttrs (n: v: n != "key") content;
|
||||
|
||||
writeFile = content:
|
||||
if pkgs.lib.isDerivation content
|
||||
then content
|
||||
else pkgs.writeText "content" (builtins.toJSON content);
|
||||
|
||||
createServingCertKey = { ca, cn, hosts? [], size ? 2048, name ? cn }:
|
||||
noCSR (
|
||||
(runWithCFSSL name "gencert -ca=${writeFile ca.cert} -ca-key=${writeFile ca.key} -profile=server -config=${writeFile ca.config} ${writeFile {
|
||||
CN = cn;
|
||||
hosts = hosts;
|
||||
key = { algo = "rsa"; inherit size; };
|
||||
}}") // { inherit name; }
|
||||
);
|
||||
|
||||
createClientCertKey = { ca, cn, groups ? [], size ? 2048, name ? cn }:
|
||||
noCSR (
|
||||
(runWithCFSSL name "gencert -ca=${writeFile ca.cert} -ca-key=${writeFile ca.key} -profile=client -config=${writeFile ca.config} ${writeFile {
|
||||
CN = cn;
|
||||
names = map (group: {O = group;}) groups;
|
||||
hosts = [""];
|
||||
key = { algo = "rsa"; inherit size; };
|
||||
}}") // { inherit name; }
|
||||
);
|
||||
|
||||
createSigningCertKey = { C ? "xx", ST ? "x", L ? "x", O ? "x", OU ? "x", CN ? "ca", emailAddress ? "x", expiry ? "43800h", size ? 2048, name ? CN }:
|
||||
(noCSR (runWithCFSSL CN "genkey -initca ${writeFile {
|
||||
key = { algo = "rsa"; inherit size; };
|
||||
names = [{ inherit C ST L O OU CN emailAddress; }];
|
||||
}}")) // {
|
||||
inherit name;
|
||||
config.signing = {
|
||||
default.expiry = expiry;
|
||||
profiles = {
|
||||
server = {
|
||||
inherit expiry;
|
||||
usages = [
|
||||
"signing"
|
||||
"key encipherment"
|
||||
"server auth"
|
||||
];
|
||||
};
|
||||
client = {
|
||||
inherit expiry;
|
||||
usages = [
|
||||
"signing"
|
||||
"key encipherment"
|
||||
"client auth"
|
||||
];
|
||||
};
|
||||
peer = {
|
||||
inherit expiry;
|
||||
usages = [
|
||||
"signing"
|
||||
"key encipherment"
|
||||
"server auth"
|
||||
"client auth"
|
||||
];
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
ca = createSigningCertKey {};
|
||||
|
||||
kube-apiserver = createServingCertKey {
|
||||
inherit ca;
|
||||
cn = "kube-apiserver";
|
||||
hosts = ["kubernetes.default" "kubernetes.default.svc" "localhost" "api.${externalDomain}" serviceClusterIp];
|
||||
};
|
||||
|
||||
kubelet = createServingCertKey {
|
||||
inherit ca;
|
||||
cn = "kubelet";
|
||||
hosts = ["*.${externalDomain}"];
|
||||
};
|
||||
|
||||
service-accounts = createServingCertKey {
|
||||
inherit ca;
|
||||
cn = "kube-service-accounts";
|
||||
};
|
||||
|
||||
etcd = createServingCertKey {
|
||||
inherit ca;
|
||||
cn = "etcd";
|
||||
hosts = ["etcd.${externalDomain}"];
|
||||
};
|
||||
|
||||
etcd-client = createClientCertKey {
|
||||
inherit ca;
|
||||
cn = "etcd-client";
|
||||
};
|
||||
|
||||
kubelet-client = createClientCertKey {
|
||||
inherit ca;
|
||||
cn = "kubelet-client";
|
||||
groups = ["system:masters"];
|
||||
};
|
||||
|
||||
apiserver-client = {
|
||||
kubelet = hostname: createClientCertKey {
|
||||
inherit ca;
|
||||
name = "apiserver-client-kubelet-${hostname}";
|
||||
cn = "system:node:${hostname}.${externalDomain}";
|
||||
groups = ["system:nodes"];
|
||||
};
|
||||
|
||||
kube-proxy = createClientCertKey {
|
||||
inherit ca;
|
||||
name = "apiserver-client-kube-proxy";
|
||||
cn = "system:kube-proxy";
|
||||
groups = ["system:kube-proxy" "system:nodes"];
|
||||
};
|
||||
|
||||
kube-controller-manager = createClientCertKey {
|
||||
inherit ca;
|
||||
name = "apiserver-client-kube-controller-manager";
|
||||
cn = "system:kube-controller-manager";
|
||||
groups = ["system:masters"];
|
||||
};
|
||||
|
||||
kube-scheduler = createClientCertKey {
|
||||
inherit ca;
|
||||
name = "apiserver-client-kube-scheduler";
|
||||
cn = "system:kube-scheduler";
|
||||
groups = ["system:kube-scheduler"];
|
||||
};
|
||||
|
||||
admin = createClientCertKey {
|
||||
inherit ca;
|
||||
cn = "admin";
|
||||
groups = ["system:masters"];
|
||||
};
|
||||
};
|
||||
in {
|
||||
master = pkgs.buildEnv {
|
||||
name = "master-keys";
|
||||
paths = [
|
||||
(writeCFSSL (noKey ca))
|
||||
(writeCFSSL kube-apiserver)
|
||||
(writeCFSSL kubelet-client)
|
||||
(writeCFSSL apiserver-client.kube-controller-manager)
|
||||
(writeCFSSL apiserver-client.kube-scheduler)
|
||||
(writeCFSSL service-accounts)
|
||||
(writeCFSSL etcd)
|
||||
];
|
||||
};
|
||||
|
||||
worker = pkgs.buildEnv {
|
||||
name = "worker-keys";
|
||||
paths = [
|
||||
(writeCFSSL (noKey ca))
|
||||
(writeCFSSL kubelet)
|
||||
(writeCFSSL apiserver-client.kube-proxy)
|
||||
(writeCFSSL etcd-client)
|
||||
] ++ map (hostname: writeCFSSL (apiserver-client.kubelet hostname)) kubelets;
|
||||
};
|
||||
|
||||
admin = writeCFSSL apiserver-client.admin;
|
||||
}
|
||||
@@ -71,7 +71,7 @@ let
|
||||
|
||||
base = {
|
||||
name = "dns";
|
||||
inherit domain certs extraConfiguration;
|
||||
inherit domain extraConfiguration;
|
||||
};
|
||||
|
||||
singleNodeTest = {
|
||||
@@ -99,8 +99,11 @@ let
|
||||
|
||||
multiNodeTest = {
|
||||
test = ''
|
||||
# Node token exchange
|
||||
$machine1->waitUntilSucceeds("cp -f /var/lib/cfssl/apitoken.secret /tmp/shared/apitoken.secret");
|
||||
$machine2->waitUntilSucceeds("cat /tmp/shared/apitoken.secret | nixos-kubernetes-node-join");
|
||||
|
||||
# prepare machines for test
|
||||
$machine1->waitUntilSucceeds("kubectl get node machine1.${domain} | grep -w Ready");
|
||||
$machine1->waitUntilSucceeds("kubectl get node machine2.${domain} | grep -w Ready");
|
||||
$machine2->execute("docker load < ${redisImage}");
|
||||
$machine1->waitUntilSucceeds("kubectl create -f ${redisPod}");
|
||||
|
||||
@@ -1,57 +0,0 @@
|
||||
{ roles, config, pkgs, certs }:
|
||||
with pkgs.lib;
|
||||
let
|
||||
base = {
|
||||
inherit roles;
|
||||
flannel.enable = true;
|
||||
addons.dashboard.enable = true;
|
||||
|
||||
caFile = "${certs.master}/ca.pem";
|
||||
apiserver = {
|
||||
tlsCertFile = "${certs.master}/kube-apiserver.pem";
|
||||
tlsKeyFile = "${certs.master}/kube-apiserver-key.pem";
|
||||
kubeletClientCertFile = "${certs.master}/kubelet-client.pem";
|
||||
kubeletClientKeyFile = "${certs.master}/kubelet-client-key.pem";
|
||||
serviceAccountKeyFile = "${certs.master}/kube-service-accounts.pem";
|
||||
};
|
||||
etcd = {
|
||||
servers = ["https://etcd.${config.networking.domain}:2379"];
|
||||
certFile = "${certs.worker}/etcd-client.pem";
|
||||
keyFile = "${certs.worker}/etcd-client-key.pem";
|
||||
};
|
||||
kubeconfig = {
|
||||
server = "https://api.${config.networking.domain}";
|
||||
};
|
||||
kubelet = {
|
||||
tlsCertFile = "${certs.worker}/kubelet.pem";
|
||||
tlsKeyFile = "${certs.worker}/kubelet-key.pem";
|
||||
hostname = "${config.networking.hostName}.${config.networking.domain}";
|
||||
kubeconfig = {
|
||||
certFile = "${certs.worker}/apiserver-client-kubelet-${config.networking.hostName}.pem";
|
||||
keyFile = "${certs.worker}/apiserver-client-kubelet-${config.networking.hostName}-key.pem";
|
||||
};
|
||||
};
|
||||
controllerManager = {
|
||||
serviceAccountKeyFile = "${certs.master}/kube-service-accounts-key.pem";
|
||||
kubeconfig = {
|
||||
certFile = "${certs.master}/apiserver-client-kube-controller-manager.pem";
|
||||
keyFile = "${certs.master}/apiserver-client-kube-controller-manager-key.pem";
|
||||
};
|
||||
};
|
||||
scheduler = {
|
||||
kubeconfig = {
|
||||
certFile = "${certs.master}/apiserver-client-kube-scheduler.pem";
|
||||
keyFile = "${certs.master}/apiserver-client-kube-scheduler-key.pem";
|
||||
};
|
||||
};
|
||||
proxy = {
|
||||
kubeconfig = {
|
||||
certFile = "${certs.worker}/apiserver-client-kube-proxy.pem";
|
||||
keyFile = "${certs.worker}//apiserver-client-kube-proxy-key.pem";
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
in {
|
||||
services.kubernetes = base;
|
||||
}
|
||||
@@ -105,7 +105,7 @@ let
|
||||
|
||||
$machine1->waitUntilSucceeds("kubectl get pod kubectl | grep Running");
|
||||
|
||||
$machine1->succeed("kubectl exec -ti kubectl -- kubectl get pods");
|
||||
$machine1->waitUntilSucceeds("kubectl exec -ti kubectl -- kubectl get pods");
|
||||
$machine1->fail("kubectl exec -ti kubectl -- kubectl create -f /kubectl-pod-2.json");
|
||||
$machine1->fail("kubectl exec -ti kubectl -- kubectl delete pods -l name=kubectl");
|
||||
'';
|
||||
@@ -113,7 +113,10 @@ let
|
||||
|
||||
multinode = base // {
|
||||
test = ''
|
||||
$machine1->waitUntilSucceeds("kubectl get node machine1.my.zyx | grep -w Ready");
|
||||
# Node token exchange
|
||||
$machine1->waitUntilSucceeds("cp -f /var/lib/cfssl/apitoken.secret /tmp/shared/apitoken.secret");
|
||||
$machine2->waitUntilSucceeds("cat /tmp/shared/apitoken.secret | nixos-kubernetes-node-join");
|
||||
|
||||
$machine1->waitUntilSucceeds("kubectl get node machine2.my.zyx | grep -w Ready");
|
||||
|
||||
$machine2->execute("docker load < ${kubectlImage}");
|
||||
@@ -125,7 +128,7 @@ let
|
||||
|
||||
$machine1->waitUntilSucceeds("kubectl get pod kubectl | grep Running");
|
||||
|
||||
$machine1->succeed("kubectl exec -ti kubectl -- kubectl get pods");
|
||||
$machine1->waitUntilSucceeds("kubectl exec -ti kubectl -- kubectl get pods");
|
||||
$machine1->fail("kubectl exec -ti kubectl -- kubectl create -f /kubectl-pod-2.json");
|
||||
$machine1->fail("kubectl exec -ti kubectl -- kubectl delete pods -l name=kubectl");
|
||||
'';
|
||||
|
||||
Reference in New Issue
Block a user