nixos/hadoop: add hadoop module (hdfs, yarn)

2018-06-29 23:14:02 -04:00
parent 978397d358
commit 0c10b2baa6
9 changed files with 352 additions and 0 deletions
@@ -0,0 +1,54 @@
+import ../make-test.nix ({pkgs, ...}: {
+  nodes = {
+    namenode = {pkgs, config, ...}: {
+      services.hadoop = {
+        package = pkgs.hadoop_3_1;
+        hdfs.namenode.enabled = true;
+        coreSite = {
+          "fs.defaultFS" = "hdfs://namenode:8020";
+        };
+        hdfsSite = {
+          "dfs.replication" = 1;
+          "dfs.namenode.rpc-bind-host" = "0.0.0.0";
+          "dfs.namenode.http-bind-host" = "0.0.0.0";
+        };
+      };
+      networking.firewall.allowedTCPPorts = [
+        9870 # namenode.http-address
+        8020 # namenode.rpc-address
+      ];
+    };
+    datanode = {pkgs, config, ...}: {
+      services.hadoop = {
+        package = pkgs.hadoop_3_1;
+        hdfs.datanode.enabled = true;
+        coreSite = {
+          "fs.defaultFS" = "hdfs://namenode:8020";
+        };
+      };
+      networking.firewall.allowedTCPPorts = [
+        9864 # datanode.http.address
+        9866 # datanode.address
+        9867 # datanode.ipc.address
+      ];
+    };
+  };
+
+  testScript = ''
+    startAll
+
+    $namenode->waitForUnit("hdfs-namenode");
+    $namenode->waitForUnit("network.target");
+    $namenode->waitForOpenPort(8020);
+    $namenode->waitForOpenPort(9870);
+
+    $datanode->waitForUnit("hdfs-datanode");
+    $datanode->waitForUnit("network.target");
+    $datanode->waitForOpenPort(9864);
+    $datanode->waitForOpenPort(9866);
+    $datanode->waitForOpenPort(9867);
+
+    $namenode->succeed("curl http://namenode:9870");
+    $datanode->succeed("curl http://datanode:9864");
+  '';
+})
@@ -0,0 +1,46 @@
+import ../make-test.nix ({pkgs, ...}: {
+  nodes = {
+    resourcemanager = {pkgs, config, ...}: {
+      services.hadoop.package = pkgs.hadoop_3_1;
+      services.hadoop.yarn.resourcemanager.enabled = true;
+      services.hadoop.yarnSite = {
+        "yarn.resourcemanager.scheduler.class" = "org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler";
+      };
+      networking.firewall.allowedTCPPorts = [
+        8088 # resourcemanager.webapp.address
+        8031 # resourcemanager.resource-tracker.address
+      ];
+    };
+    nodemanager = {pkgs, config, ...}: {
+      services.hadoop.package = pkgs.hadoop_3_1;
+      services.hadoop.yarn.nodemanager.enabled = true;
+      services.hadoop.yarnSite = {
+        "yarn.resourcemanager.hostname" = "resourcemanager";
+        "yarn.nodemanager.log-dirs" = "/tmp/userlogs";
+        "yarn.nodemanager.address" = "0.0.0.0:8041";
+      };
+      networking.firewall.allowedTCPPorts = [
+        8042 # nodemanager.webapp.address
+        8041 # nodemanager.address
+      ];
+    };
+
+  };
+
+  testScript = ''
+    startAll;
+
+    $resourcemanager->waitForUnit("yarn-resourcemanager");
+    $resourcemanager->waitForUnit("network.target");
+    $resourcemanager->waitForOpenPort(8031);
+    $resourcemanager->waitForOpenPort(8088);
+
+    $nodemanager->waitForUnit("yarn-nodemanager");
+    $nodemanager->waitForUnit("network.target");
+    $nodemanager->waitForOpenPort(8042);
+    $nodemanager->waitForOpenPort(8041);
+
+    $resourcemanager->succeed("curl http://localhost:8088");
+    $nodemanager->succeed("curl http://localhost:8042");
+  '';
+})