kubernetes module: flannel support, minor fixes
- add flannel support - remove deprecated authorizationRBACSuperAdmin option - rename from deprecated poratalNet to serviceClusterIpRange - add nodeIp option for kubelet - kubelet, add br_netfilter to kernelModules - enable firewall by default - enable dns by default on node and on master - disable iptables for docker by default on nodes - dns, restart on failure - update tests and other minor changes
This commit is contained in:
Jaka Hudoklin
committed by
Robin Gloster
Robin Gloster
parent
8e14e978c8
commit
7dfeac88ac
@@ -1,4 +1,5 @@
|
||||
{ config, pkgs, certs, servers }:
|
||||
|
||||
let
|
||||
etcd_key = "${certs}/etcd-key.pem";
|
||||
etcd_cert = "${certs}/etcd.pem";
|
||||
@@ -9,8 +10,6 @@ let
|
||||
worker_key = "${certs}/worker-key.pem";
|
||||
worker_cert = "${certs}/worker.pem";
|
||||
|
||||
mkDockerOpts = "${pkgs.kubernetes.src}/cluster/centos/node/bin/mk-docker-opts.sh";
|
||||
|
||||
rootCaFile = pkgs.writeScript "rootCaFile.pem" ''
|
||||
${pkgs.lib.readFile "${certs}/ca.pem"}
|
||||
|
||||
@@ -26,16 +25,9 @@ in
|
||||
environment.systemPackages = with pkgs; [ netcat bind etcd.bin ];
|
||||
|
||||
networking = {
|
||||
firewall = {
|
||||
enable = true;
|
||||
allowedTCPPorts = [
|
||||
10250 80 443
|
||||
];
|
||||
allowedUDPPorts = [
|
||||
8285 # flannel udp
|
||||
8472 # flannel vxlan
|
||||
];
|
||||
};
|
||||
firewall.allowedTCPPorts = [
|
||||
10250 # kubelet
|
||||
];
|
||||
extraHosts = ''
|
||||
# register "external" domains
|
||||
${servers.master} etcd.kubernetes.nixos.xyz
|
||||
@@ -43,42 +35,7 @@ in
|
||||
${mkHosts}
|
||||
'';
|
||||
};
|
||||
virtualisation.docker.extraOptions = ''
|
||||
--iptables=false $DOCKER_OPTS
|
||||
'';
|
||||
|
||||
# lets create environment file for docker startup - network stuff
|
||||
systemd.services."pre-docker" = {
|
||||
description = "Pre-Docker Actions";
|
||||
wantedBy = [ "flannel.service" ];
|
||||
before = [ "docker.service" ];
|
||||
after = [ "flannel.service" ];
|
||||
path = [ pkgs.gawk pkgs.gnugrep ];
|
||||
script = ''
|
||||
mkdir -p /run/flannel
|
||||
# bashInteractive needed for `compgen`
|
||||
${pkgs.bashInteractive}/bin/bash ${mkDockerOpts} -d /run/flannel/docker
|
||||
cat /run/flannel/docker # just for debugging
|
||||
|
||||
# allow container to host communication for DNS traffic
|
||||
${pkgs.iptables}/bin/iptables -I nixos-fw -p tcp -m tcp -i docker0 --dport 53 -j nixos-fw-accept
|
||||
${pkgs.iptables}/bin/iptables -I nixos-fw -p udp -m udp -i docker0 --dport 53 -j nixos-fw-accept
|
||||
'';
|
||||
serviceConfig.Type = "simple";
|
||||
};
|
||||
systemd.services.docker.serviceConfig.EnvironmentFile = "/run/flannel/docker";
|
||||
|
||||
services.flannel = {
|
||||
enable = true;
|
||||
network = "10.2.0.0/16";
|
||||
iface = "eth1";
|
||||
etcd = {
|
||||
endpoints = ["https://etcd.kubernetes.nixos.xyz:2379"];
|
||||
keyFile = etcd_client_key;
|
||||
certFile = etcd_client_cert;
|
||||
caFile = ca_pem;
|
||||
};
|
||||
};
|
||||
services.flannel.iface = "eth1";
|
||||
environment.variables = {
|
||||
ETCDCTL_CERT_FILE = "${etcd_client_cert}";
|
||||
ETCDCTL_KEY_FILE = "${etcd_client_key}";
|
||||
@@ -88,20 +45,10 @@ in
|
||||
|
||||
services.kubernetes = {
|
||||
kubelet = {
|
||||
networkPlugin = "cni";
|
||||
cni.config = [{
|
||||
name = "mynet";
|
||||
type = "flannel";
|
||||
delegate = {
|
||||
isDefaultGateway = true;
|
||||
bridge = "docker0";
|
||||
};
|
||||
}];
|
||||
tlsKeyFile = worker_key;
|
||||
tlsCertFile = worker_cert;
|
||||
hostname = "${config.networking.hostName}.nixos.xyz";
|
||||
extraOpts = "--node-ip ${config.networking.primaryIPAddress}";
|
||||
clusterDns = config.networking.primaryIPAddress;
|
||||
nodeIp = config.networking.primaryIPAddress;
|
||||
};
|
||||
etcd = {
|
||||
servers = ["https://etcd.kubernetes.nixos.xyz:2379"];
|
||||
@@ -110,22 +57,16 @@ in
|
||||
caFile = ca_pem;
|
||||
};
|
||||
kubeconfig = {
|
||||
server = "https://kubernetes.nixos.xyz:4443";
|
||||
server = "https://kubernetes.nixos.xyz";
|
||||
caFile = rootCaFile;
|
||||
certFile = worker_cert;
|
||||
keyFile = worker_key;
|
||||
};
|
||||
flannel.enable = true;
|
||||
|
||||
# make sure you cover kubernetes.apiserver.portalNet and flannel networks
|
||||
clusterCidr = "10.0.0.0/8";
|
||||
|
||||
dns.enable = true;
|
||||
dns.port = 4453;
|
||||
};
|
||||
|
||||
services.dnsmasq.enable = true;
|
||||
services.dnsmasq.servers = ["/${config.services.kubernetes.dns.domain}/127.0.0.1#4453"];
|
||||
|
||||
virtualisation.docker.enable = true;
|
||||
virtualisation.docker.storageDriver = "overlay";
|
||||
}
|
||||
|
||||
@@ -25,7 +25,7 @@ in
|
||||
allowPing = true;
|
||||
allowedTCPPorts = [
|
||||
2379 2380 # etcd
|
||||
4443 # kubernetes
|
||||
443 # kubernetes apiserver
|
||||
];
|
||||
};
|
||||
};
|
||||
@@ -43,14 +43,13 @@ in
|
||||
initialCluster = ["master=https://etcd.kubernetes.nixos.xyz:2380"];
|
||||
initialAdvertisePeerUrls = ["https://etcd.kubernetes.nixos.xyz:2380"];
|
||||
};
|
||||
|
||||
services.kubernetes = {
|
||||
roles = ["master"];
|
||||
scheduler.leaderElect = true;
|
||||
controllerManager.leaderElect = true;
|
||||
controllerManager.rootCaFile = rootCaFile;
|
||||
controllerManager.serviceAccountKeyFile = apiserver_key;
|
||||
apiserver = {
|
||||
securePort = 4443;
|
||||
publicAddress = "192.168.1.1";
|
||||
advertiseAddress = "192.168.1.1";
|
||||
tlsKeyFile = apiserver_key;
|
||||
@@ -59,9 +58,6 @@ in
|
||||
kubeletClientCaFile = rootCaFile;
|
||||
kubeletClientKeyFile = worker_key;
|
||||
kubeletClientCertFile = worker_cert;
|
||||
portalNet = "10.1.10.0/24"; # --service-cluster-ip-range
|
||||
runtimeConfig = "";
|
||||
/*extraOpts = "--v=2";*/
|
||||
};
|
||||
};
|
||||
}
|
||||
|
||||
@@ -8,7 +8,6 @@ let
|
||||
servers.master = "192.168.1.1";
|
||||
servers.one = "192.168.1.10";
|
||||
servers.two = "192.168.1.20";
|
||||
servers.three = "192.168.1.30";
|
||||
|
||||
certs = import ./certs.nix { inherit servers; };
|
||||
|
||||
@@ -39,7 +38,7 @@ let
|
||||
clusters = [{
|
||||
name = "local";
|
||||
cluster.certificate-authority = "/ca.pem";
|
||||
cluster.server = "https://${servers.master}:4443/";
|
||||
cluster.server = "https://${servers.master}";
|
||||
}];
|
||||
users = [{
|
||||
name = "kubelet";
|
||||
@@ -61,11 +60,9 @@ let
|
||||
$master->waitUntilSucceeds("kubectl get node master.nixos.xyz | grep Ready");
|
||||
$master->waitUntilSucceeds("kubectl get node one.nixos.xyz | grep Ready");
|
||||
$master->waitUntilSucceeds("kubectl get node two.nixos.xyz | grep Ready");
|
||||
$master->waitUntilSucceeds("kubectl get node three.nixos.xyz | grep Ready");
|
||||
|
||||
$one->execute("docker load < ${kubectlImage}");
|
||||
$two->execute("docker load < ${kubectlImage}");
|
||||
$three->execute("docker load < ${kubectlImage}");
|
||||
|
||||
$master->waitUntilSucceeds("kubectl create -f ${kubectlPod} || kubectl apply -f ${kubectlPod}");
|
||||
|
||||
@@ -116,19 +113,6 @@ in makeTest {
|
||||
}
|
||||
(import ./kubernetes-common.nix { inherit pkgs config certs servers; })
|
||||
];
|
||||
|
||||
three =
|
||||
{ config, pkgs, lib, nodes, ... }:
|
||||
mkMerge [
|
||||
{
|
||||
virtualisation.memorySize = 768;
|
||||
virtualisation.diskSize = 4096;
|
||||
networking.interfaces.eth1.ip4 = mkForce [{address = servers.three; prefixLength = 24;}];
|
||||
networking.primaryIPAddress = mkForce servers.three;
|
||||
services.kubernetes.roles = ["node"];
|
||||
}
|
||||
(import ./kubernetes-common.nix { inherit pkgs config certs servers; })
|
||||
];
|
||||
};
|
||||
|
||||
testScript = ''
|
||||
|
||||
@@ -46,37 +46,30 @@ let
|
||||
$kubernetes->waitUntilSucceeds("kubectl get pod redis | grep Running");
|
||||
$kubernetes->succeed("nc -z \$\(dig redis.default.svc.cluster.local +short\) 6379");
|
||||
'';
|
||||
in {
|
||||
# This test runs kubernetes on a single node
|
||||
singlenode = makeTest {
|
||||
name = "kubernetes-singlenode";
|
||||
in makeTest {
|
||||
name = "kubernetes-singlenode";
|
||||
|
||||
nodes = {
|
||||
kubernetes =
|
||||
{ config, pkgs, lib, nodes, ... }:
|
||||
{
|
||||
virtualisation.memorySize = 768;
|
||||
virtualisation.diskSize = 2048;
|
||||
nodes = {
|
||||
kubernetes =
|
||||
{ config, pkgs, lib, nodes, ... }:
|
||||
{
|
||||
virtualisation.memorySize = 768;
|
||||
virtualisation.diskSize = 2048;
|
||||
|
||||
programs.bash.enableCompletion = true;
|
||||
environment.systemPackages = with pkgs; [ netcat bind ];
|
||||
programs.bash.enableCompletion = true;
|
||||
environment.systemPackages = with pkgs; [ netcat bind ];
|
||||
|
||||
services.kubernetes.roles = ["master" "node"];
|
||||
services.kubernetes.dns.port = 4453;
|
||||
virtualisation.docker.extraOptions = "--iptables=false --ip-masq=false -b cbr0";
|
||||
services.kubernetes.roles = ["master" "node"];
|
||||
services.kubernetes.dns.port = 4453;
|
||||
|
||||
networking.bridges.cbr0.interfaces = [];
|
||||
networking.interfaces.cbr0 = {};
|
||||
|
||||
services.dnsmasq.enable = true;
|
||||
services.dnsmasq.servers = ["/${config.services.kubernetes.dns.domain}/127.0.0.1#4453"];
|
||||
};
|
||||
};
|
||||
|
||||
testScript = ''
|
||||
startAll;
|
||||
|
||||
${testSimplePod}
|
||||
'';
|
||||
services.dnsmasq.enable = true;
|
||||
services.dnsmasq.servers = ["/${config.services.kubernetes.dns.domain}/127.0.0.1#4453"];
|
||||
};
|
||||
};
|
||||
|
||||
testScript = ''
|
||||
startAll;
|
||||
|
||||
${testSimplePod}
|
||||
'';
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user