public/nixpkgs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #32822 from LumiGuide/elk6

Browse Source 
ELK: 5.6.1 -> 5.6.5 & add ELK 6.1.0
This commit is contained in:
Sarah Brofeldt
2018-01-14 10:40:50 +01:00
committed by GitHub
parent 08492b31d1 e8e88c43f6
commit ee4e6ebbfa
13 changed files with 313 additions and 92 deletions
Download Patch File Download Diff File Expand all files Collapse all files
nixos/modules/services/search/elasticsearch.nix
+5 -3
View File
@@ -6,6 +6,7 @@ let
cfg = config.services.elasticsearch;
es5 = builtins.compareVersions (builtins.parseDrvName cfg.package.name).version "5" >= 0;
es6 = builtins.compareVersions (builtins.parseDrvName cfg.package.name).version "6" >= 0;
esConfig = ''
network.host: ${cfg.listenAddress}
@@ -92,8 +93,6 @@ in {
node.name: "elasticsearch"
node.master: true
node.data: false
index.number_of_shards: 5
index.number_of_replicas: 1
'';
};
@@ -165,7 +164,10 @@ in {
path = [ pkgs.inetutils ];
environment = {
ES_HOME = cfg.dataDir;
ES_JAVA_OPTS = toString ([ "-Des.path.conf=${configDir}" ] ++ cfg.extraJavaOptions);
ES_JAVA_OPTS = toString ( optional (!es6) [ "-Des.path.conf=${configDir}" ]
++ cfg.extraJavaOptions);
} // optionalAttrs es6 {
ES_PATH_CONF = configDir;
};
serviceConfig = {
ExecStart = "${cfg.package}/bin/elasticsearch ${toString cfg.extraCmdLineOptions}";
nixos/release.nix
+1 -1
View File
@@ -260,7 +260,7 @@ in rec {
tests.etcd = hydraJob (import tests/etcd.nix { system = "x86_64-linux"; });
tests.ec2-nixops = hydraJob (import tests/ec2.nix { system = "x86_64-linux"; }).boot-ec2-nixops;
tests.ec2-config = hydraJob (import tests/ec2.nix { system = "x86_64-linux"; }).boot-ec2-config;
tests.elk = hydraJob (import tests/elk.nix { system = "x86_64-linux"; });
tests.elk = callSubTests tests/elk.nix { system = "x86_64-linux"; };
tests.env = callTest tests/env.nix {};
tests.ferm = callTest tests/ferm.nix {};
tests.firefox = callTest tests/firefox.nix {};
nixos/tests/elk.nix
+92 -80
View File
@@ -1,95 +1,107 @@
# Test the ELK stack: Elasticsearch, Logstash and Kibana.
import ./make-test.nix ({ pkgs, ...} :
{ system ? builtins.currentSystem }:
with import ../lib/testing.nix { inherit system; };
with pkgs.lib;
let
esUrl = "http://localhost:9200";
in {
name = "ELK";
meta = with pkgs.stdenv.lib.maintainers; {
maintainers = [ eelco chaoflow offline basvandijk ];
};
nodes = {
one =
{ config, pkgs, ... }: {
# Not giving the machine at least 2060MB results in elasticsearch failing with the following error:
#
# OpenJDK 64-Bit Server VM warning:
# INFO: os::commit_memory(0x0000000085330000, 2060255232, 0)
# failed; error='Cannot allocate memory' (errno=12)
#
# There is insufficient memory for the Java Runtime Environment to continue.
# Native memory allocation (mmap) failed to map 2060255232 bytes for committing reserved memory.
#
# When setting this to 2500 I got "Kernel panic - not syncing: Out of
# memory: compulsory panic_on_oom is enabled" so lets give it even a
# bit more room:
virtualisation.memorySize = 3000;
mkElkTest = name : elk : makeTest {
inherit name;
meta = with pkgs.stdenv.lib.maintainers; {
maintainers = [ eelco chaoflow offline basvandijk ];
};
nodes = {
one =
{ config, pkgs, ... }: {
# Not giving the machine at least 2060MB results in elasticsearch failing with the following error:
#
# OpenJDK 64-Bit Server VM warning:
# INFO: os::commit_memory(0x0000000085330000, 2060255232, 0)
# failed; error='Cannot allocate memory' (errno=12)
#
# There is insufficient memory for the Java Runtime Environment to continue.
# Native memory allocation (mmap) failed to map 2060255232 bytes for committing reserved memory.
#
# When setting this to 2500 I got "Kernel panic - not syncing: Out of
# memory: compulsory panic_on_oom is enabled" so lets give it even a
# bit more room:
virtualisation.memorySize = 3000;
# For querying JSON objects returned from elasticsearch and kibana.
environment.systemPackages = [ pkgs.jq ];
# For querying JSON objects returned from elasticsearch and kibana.
environment.systemPackages = [ pkgs.jq ];
services = {
logstash = {
enable = true;
package = pkgs.logstash5;
inputConfig = ''
exec { command => "echo -n flowers" interval => 1 type => "test" }
exec { command => "echo -n dragons" interval => 1 type => "test" }
'';
filterConfig = ''
if [message] =~ /dragons/ {
drop {}
}
'';
outputConfig = ''
file {
path => "/tmp/logstash.out"
codec => line { format => "%{message}" }
}
elasticsearch {
hosts => [ "${esUrl}" ]
}
'';
};
services = {
logstash = {
enable = true;
package = elk.logstash;
inputConfig = ''
exec { command => "echo -n flowers" interval => 1 type => "test" }
exec { command => "echo -n dragons" interval => 1 type => "test" }
'';
filterConfig = ''
if [message] =~ /dragons/ {
drop {}
}
'';
outputConfig = ''
file {
path => "/tmp/logstash.out"
codec => line { format => "%{message}" }
}
elasticsearch {
hosts => [ "${esUrl}" ]
}
'';
};
elasticsearch = {
enable = true;
package = pkgs.elasticsearch5;
};
elasticsearch = {
enable = true;
package = elk.elasticsearch;
};
kibana = {
enable = true;
package = pkgs.kibana5;
elasticsearch.url = esUrl;
kibana = {
enable = true;
package = elk.kibana;
elasticsearch.url = esUrl;
};
};
};
};
};
};
testScript = ''
startAll;
testScript = ''
startAll;
$one->waitForUnit("elasticsearch.service");
$one->waitForUnit("elasticsearch.service");
# Continue as long as the status is not "red". The status is probably
# "yellow" instead of "green" because we are using a single elasticsearch
# node which elasticsearch considers risky.
#
# TODO: extend this test with multiple elasticsearch nodes and see if the status turns "green".
$one->waitUntilSucceeds("curl --silent --show-error '${esUrl}/_cluster/health' | jq .status | grep -v red");
# Continue as long as the status is not "red". The status is probably
# "yellow" instead of "green" because we are using a single elasticsearch
# node which elasticsearch considers risky.
#
# TODO: extend this test with multiple elasticsearch nodes and see if the status turns "green".
$one->waitUntilSucceeds("curl --silent --show-error '${esUrl}/_cluster/health' | jq .status | grep -v red");
# Perform some simple logstash tests.
$one->waitForUnit("logstash.service");
$one->waitUntilSucceeds("cat /tmp/logstash.out | grep flowers");
$one->waitUntilSucceeds("cat /tmp/logstash.out | grep -v dragons");
# Perform some simple logstash tests.
$one->waitForUnit("logstash.service");
$one->waitUntilSucceeds("cat /tmp/logstash.out | grep flowers");
$one->waitUntilSucceeds("cat /tmp/logstash.out | grep -v dragons");
# See if kibana is healthy.
$one->waitForUnit("kibana.service");
$one->waitUntilSucceeds("curl --silent --show-error 'http://localhost:5601/api/status' | jq .status.overall.state | grep green");
# See if kibana is healthy.
$one->waitForUnit("kibana.service");
$one->waitUntilSucceeds("curl --silent --show-error 'http://localhost:5601/api/status' | jq .status.overall.state | grep green");
# See if logstash messages arive in elasticsearch.
$one->waitUntilSucceeds("curl --silent --show-error '${esUrl}/_search' -H 'Content-Type: application/json' -d '{\"query\" : { \"match\" : { \"message\" : \"flowers\"}}}' | jq .hits.total | grep -v 0");
$one->waitUntilSucceeds("curl --silent --show-error '${esUrl}/_search' -H 'Content-Type: application/json' -d '{\"query\" : { \"match\" : { \"message\" : \"dragons\"}}}' | jq .hits.total | grep 0");
'';
})
# See if logstash messages arive in elasticsearch.
$one->waitUntilSucceeds("curl --silent --show-error '${esUrl}/_search' -H 'Content-Type: application/json' -d '{\"query\" : { \"match\" : { \"message\" : \"flowers\"}}}' | jq .hits.total | grep -v 0");
$one->waitUntilSucceeds("curl --silent --show-error '${esUrl}/_search' -H 'Content-Type: application/json' -d '{\"query\" : { \"match\" : { \"message\" : \"dragons\"}}}' | jq .hits.total | grep 0");
'';
};
in mapAttrs mkElkTest {
"ELK-5" = {
elasticsearch = pkgs.elasticsearch5;
logstash = pkgs.logstash5;
kibana = pkgs.kibana5;
};
"ELK-6" = {
elasticsearch = pkgs.elasticsearch6;
logstash = pkgs.logstash6;
kibana = pkgs.kibana6;
};
}