diff --git a/nixos/doc/manual/release-notes/rl-1809.xml b/nixos/doc/manual/release-notes/rl-1809.xml index d831f851446..8ee2a5f1623 100644 --- a/nixos/doc/manual/release-notes/rl-1809.xml +++ b/nixos/doc/manual/release-notes/rl-1809.xml @@ -190,6 +190,39 @@ $ nix-instantiate -E '(import <nixpkgsunstable> {}).gitFull' which indicates that the nix output hash will be used as tag. + + + The ELK stack: elasticsearch, logstash and kibana + has been upgraded from 2.* to 6.3.*. + The 2.* versions have been unsupported since last year + so they have been removed. You can still use the 5.* versions under the names + elasticsearch5, logstash5 and + kibana5. + + + The elastic beats: + filebeat, heartbeat, + metricbeat and packetbeat + have had the same treatment: they now target 6.3.* as well. + The 5.* versions are available under the names: + filebeat5, heartbeat5, + metricbeat5 and packetbeat5 + + + The ELK-6.3 stack now comes with + X-Pack by default. + Since X-Pack is licensed under the + Elastic License + the ELK packages now have an unfree license. To use them you need to specify + allowUnfree = true; in your nixpkgs configuration. + + + Fortunately there is also a free variant of the ELK stack without X-Pack. + The packages are available under the names: + elasticsearch-oss, logstash-oss and + kibana-oss. + + Options diff --git a/nixos/modules/rename.nix b/nixos/modules/rename.nix index f032f10e455..b51dcd2976f 100644 --- a/nixos/modules/rename.nix +++ b/nixos/modules/rename.nix @@ -256,6 +256,7 @@ with lib; (mkRemovedOptionModule [ "fonts" "fontconfig" "forceAutohint" ] "") (mkRemovedOptionModule [ "fonts" "fontconfig" "renderMonoTTFAsBitmap" ] "") (mkRemovedOptionModule [ "virtualisation" "xen" "qemu" ] "You don't need this option anymore, it will work without it.") + (mkRemovedOptionModule [ "services" "logstash" "enableWeb" ] "The web interface was removed from logstash") (mkRemovedOptionModule [ "boot" "zfs" "enableLegacyCrypto" ] "The corresponding package was removed from nixpkgs.") # ZSH diff --git a/nixos/modules/services/logging/logstash.nix b/nixos/modules/services/logging/logstash.nix index 28d89a7463a..aa019d855ea 100644 --- a/nixos/modules/services/logging/logstash.nix +++ b/nixos/modules/services/logging/logstash.nix @@ -4,25 +4,12 @@ with lib; let cfg = config.services.logstash; - atLeast54 = versionAtLeast (builtins.parseDrvName cfg.package.name).version "5.4"; pluginPath = lib.concatStringsSep ":" cfg.plugins; havePluginPath = lib.length cfg.plugins > 0; ops = lib.optionalString; - verbosityFlag = - if atLeast54 - then "--log.level " + cfg.logLevel - else { - debug = "--debug"; - info = "--verbose"; - warn = ""; # intentionally empty - error = "--quiet"; - fatal = "--silent"; - }."${cfg.logLevel}"; + verbosityFlag = "--log.level " + cfg.logLevel; - pluginsPath = - if atLeast54 - then "--path.plugins ${pluginPath}" - else "--pluginpath ${pluginPath}"; + pluginsPath = "--path.plugins ${pluginPath}"; logstashConf = pkgs.writeText "logstash.conf" '' input { @@ -63,7 +50,7 @@ in type = types.package; default = pkgs.logstash; defaultText = "pkgs.logstash"; - example = literalExample "pkgs.logstash"; + example = literalExample "pkgs.logstash5"; description = "Logstash package to use."; }; @@ -95,12 +82,6 @@ in description = "The quantity of filter workers to run."; }; - enableWeb = mkOption { - type = types.bool; - default = false; - description = "Enable the logstash web interface."; - }; - listenAddress = mkOption { type = types.str; default = "127.0.0.1"; @@ -174,16 +155,6 @@ in ###### implementation config = mkIf cfg.enable { - assertions = [ - { assertion = atLeast54 -> !cfg.enableWeb; - message = '' - The logstash web interface is only available for versions older than 5.4. - So either set services.logstash.enableWeb = false, - or set services.logstash.package to an older logstash. - ''; - } - ]; - systemd.services.logstash = with pkgs; { description = "Logstash Daemon"; wantedBy = [ "multi-user.target" ]; @@ -193,14 +164,12 @@ in ExecStartPre = ''${pkgs.coreutils}/bin/mkdir -p "${cfg.dataDir}" ; ${pkgs.coreutils}/bin/chmod 700 "${cfg.dataDir}"''; ExecStart = concatStringsSep " " (filter (s: stringLength s != 0) [ "${cfg.package}/bin/logstash" - (ops (!atLeast54) "agent") "-w ${toString cfg.filterWorkers}" (ops havePluginPath pluginsPath) "${verbosityFlag}" "-f ${logstashConf}" - (ops atLeast54 "--path.settings ${logstashSettingsDir}") - (ops atLeast54 "--path.data ${cfg.dataDir}") - (ops cfg.enableWeb "-- web -a ${cfg.listenAddress} -p ${cfg.port}") + "--path.settings ${logstashSettingsDir}" + "--path.data ${cfg.dataDir}" ]); }; }; diff --git a/nixos/modules/services/search/elasticsearch.nix b/nixos/modules/services/search/elasticsearch.nix index b0831dcd1ca..6b688c640d5 100644 --- a/nixos/modules/services/search/elasticsearch.nix +++ b/nixos/modules/services/search/elasticsearch.nix @@ -5,22 +5,14 @@ with lib; let cfg = config.services.elasticsearch; - es5 = builtins.compareVersions (builtins.parseDrvName cfg.package.name).version "5" >= 0; - es6 = builtins.compareVersions (builtins.parseDrvName cfg.package.name).version "6" >= 0; + es6 = builtins.compareVersions cfg.package.version "6" >= 0; esConfig = '' network.host: ${cfg.listenAddress} cluster.name: ${cfg.cluster_name} - ${if es5 then '' - http.port: ${toString cfg.port} - transport.tcp.port: ${toString cfg.tcp_port} - '' else '' - network.port: ${toString cfg.port} - network.tcp.port: ${toString cfg.tcp_port} - # TODO: find a way to enable security manager - security.manager.enabled: false - ''} + http.port: ${toString cfg.port} + transport.tcp.port: ${toString cfg.tcp_port} ${cfg.extraConf} ''; @@ -32,7 +24,7 @@ let text = esConfig; }; - loggingConfigFilename = if es5 then "log4j2.properties" else "logging.yml"; + loggingConfigFilename = "log4j2.properties"; loggingConfigFile = pkgs.writeTextFile { name = loggingConfigFilename; text = cfg.logging; @@ -41,8 +33,7 @@ let esPlugins = pkgs.buildEnv { name = "elasticsearch-plugins"; paths = cfg.plugins; - # Elasticsearch 5.x won't start when the plugins directory does not exist - postBuild = if es5 then "${pkgs.coreutils}/bin/mkdir -p $out/plugins" else ""; + postBuild = "${pkgs.coreutils}/bin/mkdir -p $out/plugins"; }; in { @@ -58,8 +49,8 @@ in { package = mkOption { description = "Elasticsearch package to use."; - default = pkgs.elasticsearch2; - defaultText = "pkgs.elasticsearch2"; + default = pkgs.elasticsearch; + defaultText = "pkgs.elasticsearch"; type = types.package; }; @@ -100,30 +91,18 @@ in { logging = mkOption { description = "Elasticsearch logging configuration."; - default = - if es5 then '' - logger.action.name = org.elasticsearch.action - logger.action.level = info + default = '' + logger.action.name = org.elasticsearch.action + logger.action.level = info - appender.console.type = Console - appender.console.name = console - appender.console.layout.type = PatternLayout - appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%m%n + appender.console.type = Console + appender.console.name = console + appender.console.layout.type = PatternLayout + appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%m%n - rootLogger.level = info - rootLogger.appenderRef.console.ref = console - '' else '' - rootLogger: INFO, console - logger: - action: INFO - com.amazonaws: WARN - appender: - console: - type: console - layout: - type: consolePattern - conversionPattern: "[%d{ISO8601}][%-5p][%-25c] %m%n" - ''; + rootLogger.level = info + rootLogger.appenderRef.console.ref = console + ''; type = types.str; }; @@ -204,9 +183,9 @@ in { cp ${elasticsearchYml} ${configDir}/elasticsearch.yml # Make sure the logging configuration for old elasticsearch versions is removed: - rm -f ${if es5 then "${configDir}/logging.yml" else "${configDir}/log4j2.properties"} + rm -f "${configDir}/logging.yml" cp ${loggingConfigFile} ${configDir}/${loggingConfigFilename} - ${optionalString es5 "mkdir -p ${configDir}/scripts"} + mkdir -p ${configDir}/scripts ${optionalString es6 "cp ${cfg.package}/config/jvm.options ${configDir}/jvm.options"} if [ "$(id -u)" = 0 ]; then chown -R elasticsearch:elasticsearch ${cfg.dataDir}; fi diff --git a/nixos/modules/services/search/kibana.nix b/nixos/modules/services/search/kibana.nix index 5885a72c662..ca36bba58c0 100644 --- a/nixos/modules/services/search/kibana.nix +++ b/nixos/modules/services/search/kibana.nix @@ -5,43 +5,7 @@ with lib; let cfg = config.services.kibana; - atLeast54 = versionAtLeast (builtins.parseDrvName cfg.package.name).version "5.4"; - - cfgFile = if atLeast54 then cfgFile5 else cfgFile4; - - cfgFile4 = pkgs.writeText "kibana.json" (builtins.toJSON ( - (filterAttrsRecursive (n: v: v != null) ({ - host = cfg.listenAddress; - port = cfg.port; - ssl_cert_file = cfg.cert; - ssl_key_file = cfg.key; - - kibana_index = cfg.index; - default_app_id = cfg.defaultAppId; - - elasticsearch_url = cfg.elasticsearch.url; - kibana_elasticsearch_username = cfg.elasticsearch.username; - kibana_elasticsearch_password = cfg.elasticsearch.password; - kibana_elasticsearch_cert = cfg.elasticsearch.cert; - kibana_elasticsearch_key = cfg.elasticsearch.key; - ca = cfg.elasticsearch.ca; - - bundled_plugin_ids = [ - "plugins/dashboard/index" - "plugins/discover/index" - "plugins/doc/index" - "plugins/kibana/index" - "plugins/markdown_vis/index" - "plugins/metric_vis/index" - "plugins/settings/index" - "plugins/table_vis/index" - "plugins/vis_types/index" - "plugins/visualize/index" - ]; - } // cfg.extraConf) - ))); - - cfgFile5 = pkgs.writeText "kibana.json" (builtins.toJSON ( + cfgFile = pkgs.writeText "kibana.json" (builtins.toJSON ( (filterAttrsRecursive (n: v: v != null) ({ server.host = cfg.listenAddress; server.port = cfg.port; diff --git a/pkgs/development/tools/misc/kibana/6.x.nix b/pkgs/development/tools/misc/kibana/6.x.nix deleted file mode 100644 index 1fd52f43557..00000000000 --- a/pkgs/development/tools/misc/kibana/6.x.nix +++ /dev/null @@ -1,75 +0,0 @@ -{ elk6Version -, enableUnfree ? true -, stdenv -, makeWrapper -, fetchzip -, fetchurl -, nodejs -, coreutils -, which -}: - -with stdenv.lib; -let - inherit (builtins) elemAt; - info = splitString "-" stdenv.system; - arch = elemAt info 0; - plat = elemAt info 1; - shas = - if enableUnfree - then { - "x86_64-linux" = "1kk97ggpzmblhqm6cfd2sv5940f58h323xcyg6rba1njj7lzanv0"; - "x86_64-darwin" = "1xvwffk8d8br92h0laf4b1m76kvki6cj0pbgcvirfcj1r70vk6c3"; - } - else { - "x86_64-linux" = "0m81ki1v61gpwb3s6zf84azqrirlm9pdfx65g3xmvdp3d3wii5ly"; - "x86_64-darwin" = "0zh9p6vsq1d0gh6ks7z6bh8sbhn6rm4jshjcfp3c9k7n2qa8vv9b"; - }; - - # For the correct phantomjs version see: - # https://github.com/elastic/kibana/blob/master/x-pack/plugins/reporting/server/browsers/phantom/paths.js - phantomjs = rec { - name = "phantomjs-${version}-linux-x86_64"; - version = "2.1.1"; - src = fetchzip { - inherit name; - url = "https://github.com/Medium/phantomjs/releases/download/v${version}/${name}.tar.bz2"; - sha256 = "0g2dqjzr2daz6rkd6shj6rrlw55z4167vqh7bxadl8jl6jk7zbfv"; - }; - }; - -in stdenv.mkDerivation rec { - name = "kibana-${optionalString (!enableUnfree) "oss-"}${version}"; - version = elk6Version; - - src = fetchurl { - url = "https://artifacts.elastic.co/downloads/kibana/${name}-${plat}-${arch}.tar.gz"; - sha256 = shas."${stdenv.system}" or (throw "Unknown architecture"); - }; - - buildInputs = [ makeWrapper ]; - - installPhase = '' - mkdir -p $out/libexec/kibana $out/bin - mv * $out/libexec/kibana/ - rm -r $out/libexec/kibana/node - makeWrapper $out/libexec/kibana/bin/kibana $out/bin/kibana \ - --prefix PATH : "${stdenv.lib.makeBinPath [ nodejs coreutils which ]}" - sed -i 's@NODE=.*@NODE=${nodejs}/bin/node@' $out/libexec/kibana/bin/kibana - '' + - # phantomjs is needed in the unfree version. When phantomjs doesn't exist in - # $out/libexec/kibana/data kibana will try to download and unpack it during - # runtime which will fail because the nix store is read-only. So we make sure - # it already exist in the nix store. - optionalString enableUnfree '' - ln -s ${phantomjs.src} $out/libexec/kibana/data/${phantomjs.name} - ''; - - meta = { - description = "Visualize logs and time-stamped data"; - homepage = http://www.elasticsearch.org/overview/kibana; - license = if enableUnfree then licenses.elastic else licenses.asl20; - maintainers = with maintainers; [ offline rickynils basvandijk ]; - platforms = with platforms; unix; - }; -} diff --git a/pkgs/development/tools/misc/kibana/default.nix b/pkgs/development/tools/misc/kibana/default.nix index fc1ac932338..1fd52f43557 100644 --- a/pkgs/development/tools/misc/kibana/default.nix +++ b/pkgs/development/tools/misc/kibana/default.nix @@ -1,26 +1,49 @@ -{ stdenv, makeWrapper, fetchurl, nodejs, coreutils, which }: +{ elk6Version +, enableUnfree ? true +, stdenv +, makeWrapper +, fetchzip +, fetchurl +, nodejs +, coreutils +, which +}: with stdenv.lib; let inherit (builtins) elemAt; - archOverrides = { - "i686" = "x86"; - }; info = splitString "-" stdenv.system; - arch = (elemAt info 0); - elasticArch = archOverrides."${arch}" or arch; + arch = elemAt info 0; plat = elemAt info 1; - shas = { - "x86_64-linux" = "1wnnrhhpgc58s09p99cmi8r2jmwsd5lmh2inb0k8nmizz5v1sjz0"; - "i686-linux" = "0sdx59jlfrf7r9793xpn2vxaxjdczgn3qfw8yny03dcs6fjaxi2y"; - "x86_64-darwin" = "0rmp536kn001g52lxngpj6x6d0j3qj0r11d4djbz7h6s5ml03kza"; + shas = + if enableUnfree + then { + "x86_64-linux" = "1kk97ggpzmblhqm6cfd2sv5940f58h323xcyg6rba1njj7lzanv0"; + "x86_64-darwin" = "1xvwffk8d8br92h0laf4b1m76kvki6cj0pbgcvirfcj1r70vk6c3"; + } + else { + "x86_64-linux" = "0m81ki1v61gpwb3s6zf84azqrirlm9pdfx65g3xmvdp3d3wii5ly"; + "x86_64-darwin" = "0zh9p6vsq1d0gh6ks7z6bh8sbhn6rm4jshjcfp3c9k7n2qa8vv9b"; + }; + + # For the correct phantomjs version see: + # https://github.com/elastic/kibana/blob/master/x-pack/plugins/reporting/server/browsers/phantom/paths.js + phantomjs = rec { + name = "phantomjs-${version}-linux-x86_64"; + version = "2.1.1"; + src = fetchzip { + inherit name; + url = "https://github.com/Medium/phantomjs/releases/download/v${version}/${name}.tar.bz2"; + sha256 = "0g2dqjzr2daz6rkd6shj6rrlw55z4167vqh7bxadl8jl6jk7zbfv"; + }; }; + in stdenv.mkDerivation rec { - name = "kibana-${version}"; - version = "4.6.5"; + name = "kibana-${optionalString (!enableUnfree) "oss-"}${version}"; + version = elk6Version; src = fetchurl { - url = "https://download.elastic.co/kibana/kibana/${name}-${plat}-${elasticArch}.tar.gz"; + url = "https://artifacts.elastic.co/downloads/kibana/${name}-${plat}-${arch}.tar.gz"; sha256 = shas."${stdenv.system}" or (throw "Unknown architecture"); }; @@ -33,13 +56,20 @@ in stdenv.mkDerivation rec { makeWrapper $out/libexec/kibana/bin/kibana $out/bin/kibana \ --prefix PATH : "${stdenv.lib.makeBinPath [ nodejs coreutils which ]}" sed -i 's@NODE=.*@NODE=${nodejs}/bin/node@' $out/libexec/kibana/bin/kibana + '' + + # phantomjs is needed in the unfree version. When phantomjs doesn't exist in + # $out/libexec/kibana/data kibana will try to download and unpack it during + # runtime which will fail because the nix store is read-only. So we make sure + # it already exist in the nix store. + optionalString enableUnfree '' + ln -s ${phantomjs.src} $out/libexec/kibana/data/${phantomjs.name} ''; meta = { description = "Visualize logs and time-stamped data"; homepage = http://www.elasticsearch.org/overview/kibana; - license = licenses.asl20; - maintainers = with maintainers; [ offline rickynils ]; + license = if enableUnfree then licenses.elastic else licenses.asl20; + maintainers = with maintainers; [ offline rickynils basvandijk ]; platforms = with platforms; unix; }; } diff --git a/pkgs/misc/logging/beats/5.x.nix b/pkgs/misc/logging/beats/5.x.nix index 94d5449100d..e92613025ff 100644 --- a/pkgs/misc/logging/beats/5.x.nix +++ b/pkgs/misc/logging/beats/5.x.nix @@ -23,10 +23,10 @@ let beat = package : extraArgs : buildGoPackage (rec { }; } // extraArgs); in { - filebeat = beat "filebeat" {meta.description = "Lightweight shipper for logfiles";}; - heartbeat = beat "heartbeat" {meta.description = "Lightweight shipper for uptime monitoring";}; - metricbeat = beat "metricbeat" {meta.description = "Lightweight shipper for metrics";}; - packetbeat = beat "packetbeat" { + filebeat5 = beat "filebeat" {meta.description = "Lightweight shipper for logfiles";}; + heartbeat5 = beat "heartbeat" {meta.description = "Lightweight shipper for uptime monitoring";}; + metricbeat5 = beat "metricbeat" {meta.description = "Lightweight shipper for metrics";}; + packetbeat5 = beat "packetbeat" { buildInputs = [ libpcap ]; meta.description = "Network packet analyzer that ships data to Elasticsearch"; meta.longDescription = '' diff --git a/pkgs/misc/logging/beats/6.x.nix b/pkgs/misc/logging/beats/6.x.nix index 111bc449d88..59d7379e06c 100644 --- a/pkgs/misc/logging/beats/6.x.nix +++ b/pkgs/misc/logging/beats/6.x.nix @@ -23,10 +23,10 @@ let beat = package : extraArgs : buildGoPackage (rec { }; } // extraArgs); in { - filebeat = beat "filebeat" {meta.description = "Lightweight shipper for logfiles";}; - heartbeat = beat "heartbeat" {meta.description = "Lightweight shipper for uptime monitoring";}; - metricbeat = beat "metricbeat" {meta.description = "Lightweight shipper for metrics";}; - packetbeat = beat "packetbeat" { + filebeat6 = beat "filebeat" {meta.description = "Lightweight shipper for logfiles";}; + heartbeat6 = beat "heartbeat" {meta.description = "Lightweight shipper for uptime monitoring";}; + metricbeat6 = beat "metricbeat" {meta.description = "Lightweight shipper for metrics";}; + packetbeat6 = beat "packetbeat" { buildInputs = [ libpcap ]; meta.description = "Network packet analyzer that ships data to Elasticsearch"; meta.longDescription = '' diff --git a/pkgs/servers/search/elasticsearch/2.x.nix b/pkgs/servers/search/elasticsearch/2.x.nix deleted file mode 100644 index 28244d1b3dc..00000000000 --- a/pkgs/servers/search/elasticsearch/2.x.nix +++ /dev/null @@ -1,40 +0,0 @@ -{ stdenv, fetchurl, makeWrapper, jre, utillinux }: - -with stdenv.lib; - -stdenv.mkDerivation rec { - version = "2.4.4"; - name = "elasticsearch-${version}"; - - src = fetchurl { - url = "https://download.elasticsearch.org/elasticsearch/release/org/elasticsearch/distribution/tar/elasticsearch/${version}/${name}.tar.gz"; - sha256 = "1qjq04sfqb35pf2xpvr8j5p27chfxpjp8ymrp1h5bfk5rbk9444q"; - }; - - patches = [ ./es-home-2.x.patch ./es-classpath-2.x.patch ]; - - buildInputs = [ makeWrapper jre utillinux ]; - - installPhase = '' - mkdir -p $out - cp -R bin config lib modules $out - - # don't want to have binary with name plugin - mv $out/bin/plugin $out/bin/elasticsearch-plugin - wrapProgram $out/bin/elasticsearch \ - --prefix ES_CLASSPATH : "$out/lib/${name}.jar":"$out/lib/*" \ - --prefix PATH : "${utillinux}/bin" \ - --set JAVA_HOME "${jre}" - wrapProgram $out/bin/elasticsearch-plugin --set JAVA_HOME "${jre}" - ''; - - meta = { - description = "Open Source, Distributed, RESTful Search Engine"; - license = licenses.asl20; - platforms = platforms.unix; - maintainers = [ - maintainers.offline - maintainers.markWot - ]; - }; -} diff --git a/pkgs/servers/search/elasticsearch/6.x.nix b/pkgs/servers/search/elasticsearch/6.x.nix deleted file mode 100644 index 84872649c49..00000000000 --- a/pkgs/servers/search/elasticsearch/6.x.nix +++ /dev/null @@ -1,65 +0,0 @@ -{ elk6Version -, enableUnfree ? true -, stdenv -, fetchurl -, makeWrapper -, jre_headless -, utillinux -, autoPatchelfHook -, zlib -}: - -with stdenv.lib; - -stdenv.mkDerivation (rec { - version = elk6Version; - name = "elasticsearch-${optionalString (!enableUnfree) "oss-"}${version}"; - - src = fetchurl { - url = "https://artifacts.elastic.co/downloads/elasticsearch/${name}.tar.gz"; - sha256 = - if enableUnfree - then "0960ak602pm95p2mha9cb1mrwdky8pfw3y89r2v4zpr5n730hmnh" - else "1i4i1ai75bf8k0zd1qf8x0bavrm8rcw13xdim443zza09w95ypk4"; - }; - - patches = [ ./es-home-6.x.patch ]; - - postPatch = '' - sed -i "s|ES_CLASSPATH=\"\$ES_HOME/lib/\*\"|ES_CLASSPATH=\"$out/lib/*\"|" ./bin/elasticsearch-env - ''; - - buildInputs = [ makeWrapper jre_headless utillinux ]; - - installPhase = '' - mkdir -p $out - cp -R bin config lib modules plugins $out - - chmod -x $out/bin/*.* - - wrapProgram $out/bin/elasticsearch \ - --prefix PATH : "${utillinux}/bin/" \ - --set JAVA_HOME "${jre_headless}" - - wrapProgram $out/bin/elasticsearch-plugin --set JAVA_HOME "${jre_headless}" - ''; - - passthru = { inherit enableUnfree; }; - - meta = { - description = "Open Source, Distributed, RESTful Search Engine"; - license = if enableUnfree then licenses.elastic else licenses.asl20; - platforms = platforms.unix; - maintainers = with maintainers; [ apeschar basvandijk ]; - }; -} // optionalAttrs enableUnfree { - dontPatchELF = true; - nativeBuildInputs = [ autoPatchelfHook ]; - runtimeDependencies = [ zlib ]; - postFixup = '' - for exe in $(find $out/modules/x-pack/x-pack-ml/platform/linux-x86_64/bin -executable -type f); do - echo "patching $exe..." - patchelf --set-interpreter "$(cat $NIX_CC/nix-support/dynamic-linker)" "$exe" - done - ''; -}) diff --git a/pkgs/servers/search/elasticsearch/default.nix b/pkgs/servers/search/elasticsearch/default.nix index f4780603f50..84872649c49 100644 --- a/pkgs/servers/search/elasticsearch/default.nix +++ b/pkgs/servers/search/elasticsearch/default.nix @@ -1,40 +1,65 @@ -{ stdenv, fetchurl, makeWrapper, jre, utillinux }: +{ elk6Version +, enableUnfree ? true +, stdenv +, fetchurl +, makeWrapper +, jre_headless +, utillinux +, autoPatchelfHook +, zlib +}: with stdenv.lib; -stdenv.mkDerivation rec { - name = "elasticsearch-1.7.2"; +stdenv.mkDerivation (rec { + version = elk6Version; + name = "elasticsearch-${optionalString (!enableUnfree) "oss-"}${version}"; src = fetchurl { - url = "https://download.elastic.co/elasticsearch/elasticsearch/${name}.tar.gz"; - sha256 = "1lix4asvx1lbc227gzsrws3xqbcbqaal7v10w60kch0c4xg970bg"; + url = "https://artifacts.elastic.co/downloads/elasticsearch/${name}.tar.gz"; + sha256 = + if enableUnfree + then "0960ak602pm95p2mha9cb1mrwdky8pfw3y89r2v4zpr5n730hmnh" + else "1i4i1ai75bf8k0zd1qf8x0bavrm8rcw13xdim443zza09w95ypk4"; }; - patches = [ ./es-home.patch ]; + patches = [ ./es-home-6.x.patch ]; - buildInputs = [ makeWrapper jre utillinux ]; + postPatch = '' + sed -i "s|ES_CLASSPATH=\"\$ES_HOME/lib/\*\"|ES_CLASSPATH=\"$out/lib/*\"|" ./bin/elasticsearch-env + ''; + + buildInputs = [ makeWrapper jre_headless utillinux ]; installPhase = '' mkdir -p $out - cp -R bin config lib $out + cp -R bin config lib modules plugins $out - # don't want to have binary with name plugin - mv $out/bin/plugin $out/bin/elasticsearch-plugin + chmod -x $out/bin/*.* - # set ES_CLASSPATH and JAVA_HOME wrapProgram $out/bin/elasticsearch \ - --prefix ES_CLASSPATH : "$out/lib/${name}.jar":"$out/lib/*":"$out/lib/sigar/*" \ - --prefix PATH : "${utillinux}/bin" \ - --set JAVA_HOME "${jre}" - wrapProgram $out/bin/elasticsearch-plugin \ - --prefix ES_CLASSPATH : "$out/lib/${name}.jar":"$out/lib/*":"$out/lib/sigar/*" \ - --set JAVA_HOME "${jre}" + --prefix PATH : "${utillinux}/bin/" \ + --set JAVA_HOME "${jre_headless}" + + wrapProgram $out/bin/elasticsearch-plugin --set JAVA_HOME "${jre_headless}" ''; + passthru = { inherit enableUnfree; }; + meta = { description = "Open Source, Distributed, RESTful Search Engine"; - license = licenses.asl20; + license = if enableUnfree then licenses.elastic else licenses.asl20; platforms = platforms.unix; - maintainers = [ maintainers.offline ]; + maintainers = with maintainers; [ apeschar basvandijk ]; }; -} +} // optionalAttrs enableUnfree { + dontPatchELF = true; + nativeBuildInputs = [ autoPatchelfHook ]; + runtimeDependencies = [ zlib ]; + postFixup = '' + for exe in $(find $out/modules/x-pack/x-pack-ml/platform/linux-x86_64/bin -executable -type f); do + echo "patching $exe..." + patchelf --set-interpreter "$(cat $NIX_CC/nix-support/dynamic-linker)" "$exe" + done + ''; +}) diff --git a/pkgs/servers/search/elasticsearch/es-classpath-2.x.patch b/pkgs/servers/search/elasticsearch/es-classpath-2.x.patch deleted file mode 100644 index 46a3f0be71b..00000000000 --- a/pkgs/servers/search/elasticsearch/es-classpath-2.x.patch +++ /dev/null @@ -1,38 +0,0 @@ -diff -rupN a/bin/elasticsearch b/bin/elasticsearch ---- a/bin/elasticsearch 2017-02-08 18:32:28.000298543 -0500 -+++ b/bin/elasticsearch 2017-02-08 19:10:45.692916675 -0500 -@@ -81,12 +81,7 @@ ES_HOME=`cd "$ES_HOME"; pwd` - # If an include wasn't specified in the environment, then search for one... - if [ "x$ES_INCLUDE" = "x" ]; then - # Locations (in order) to use when searching for an include file. -- for include in /usr/share/elasticsearch/elasticsearch.in.sh \ -- /usr/local/share/elasticsearch/elasticsearch.in.sh \ -- /opt/elasticsearch/elasticsearch.in.sh \ -- ~/.elasticsearch.in.sh \ -- "$ES_HOME/bin/elasticsearch.in.sh" \ -- "`dirname "$0"`"/elasticsearch.in.sh; do -+ for include in "`dirname "$0"`"/elasticsearch.in.sh; do - if [ -r "$include" ]; then - . "$include" - break -diff -rupN a/bin/elasticsearch.in.sh b/bin/elasticsearch.in.sh ---- a/bin/elasticsearch.in.sh 2017-02-08 18:32:28.000298543 -0500 -+++ b/bin/elasticsearch.in.sh 2017-02-08 18:33:46.816634599 -0500 -@@ -1,17 +1,5 @@ - #!/bin/sh - --# check in case a user was using this mechanism --if [ "x$ES_CLASSPATH" != "x" ]; then -- cat >&2 << EOF --Error: Don't modify the classpath with ES_CLASSPATH. Best is to add --additional elements via the plugin mechanism, or if code must really be --added to the main classpath, add jars to lib/ (unsupported). --EOF -- exit 1 --fi -- --ES_CLASSPATH="$ES_HOME/lib/elasticsearch-2.4.4.jar:$ES_HOME/lib/*" -- - if [ "x$ES_MIN_MEM" = "x" ]; then - ES_MIN_MEM=256m - fi diff --git a/pkgs/servers/search/elasticsearch/es-home-2.x.patch b/pkgs/servers/search/elasticsearch/es-home-2.x.patch deleted file mode 100644 index 0c80e865114..00000000000 --- a/pkgs/servers/search/elasticsearch/es-home-2.x.patch +++ /dev/null @@ -1,31 +0,0 @@ -diff -rupN a/bin/elasticsearch b/bin/elasticsearch ---- a/bin/elasticsearch 2015-11-18 21:48:18.000000000 +0100 -+++ b/bin/elasticsearch 2015-12-04 00:52:21.032475098 +0100 -@@ -72,7 +72,11 @@ while [ -h "$SCRIPT" ] ; do - done - - # determine elasticsearch home --ES_HOME=`dirname "$SCRIPT"`/.. -+ -+if [ -z "$ES_HOME" ]; then -+ echo "You must set the ES_HOME var" >&2 -+ exit 1 -+fi - - # make ELASTICSEARCH_HOME absolute - ES_HOME=`cd "$ES_HOME"; pwd` -diff -rupN a/bin/plugin b/bin/plugin ---- a/bin/plugin 2015-11-18 21:48:18.000000000 +0100 -+++ b/bin/plugin 2015-12-04 00:52:55.947453619 +0100 -@@ -17,7 +17,10 @@ while [ -h "$SCRIPT" ] ; do - done - - # determine elasticsearch home --ES_HOME=`dirname "$SCRIPT"`/.. -+if [ -z "$ES_HOME" ]; then -+ echo "You must set the ES_HOME var" >&2 -+ exit 1 -+fi - - # make ELASTICSEARCH_HOME absolute - ES_HOME=`cd "$ES_HOME"; pwd` diff --git a/pkgs/servers/search/elasticsearch/es-home.patch b/pkgs/servers/search/elasticsearch/es-home.patch deleted file mode 100644 index 2f2018fd69f..00000000000 --- a/pkgs/servers/search/elasticsearch/es-home.patch +++ /dev/null @@ -1,37 +0,0 @@ -diff -rupN a/bin/elasticsearch b/bin/elasticsearch ---- a/bin/elasticsearch 2015-08-05 17:52:05.740819671 +0200 -+++ b/bin/elasticsearch 2015-08-05 17:22:34.664657364 +0200 -@@ -83,7 +83,10 @@ while [ -h "$SCRIPT" ] ; do - done - - # determine elasticsearch home --ES_HOME=`dirname "$SCRIPT"`/.. -+if [ -z "$ES_HOME" ]; then -+ echo "You must set the ES_HOME var" >&2 -+ exit 1 -+fi - - # make ELASTICSEARCH_HOME absolute - ES_HOME=`cd "$ES_HOME"; pwd` -diff -rupN a/bin/plugin b/bin/plugin ---- a/bin/plugin 2015-08-05 17:57:07.903088815 +0200 -+++ b/bin/plugin 2015-08-05 17:57:38.979808139 +0200 -@@ -16,7 +16,10 @@ while [ -h "$SCRIPT" ] ; do - done - - # determine elasticsearch home --ES_HOME=`dirname "$SCRIPT"`/.. -+if [ -z "$ES_HOME" ]; then -+ echo "You must set the ES_HOME var" >&2 -+ exit 1 -+fi - - # make ELASTICSEARCH_HOME absolute - ES_HOME=`cd "$ES_HOME"; pwd` -@@ -105,4 +105,4 @@ - - export HOSTNAME=`hostname -s` - --eval "$JAVA" $JAVA_OPTS $ES_JAVA_OPTS -Xmx64m -Xms16m -Delasticsearch -Des.path.home=\""$ES_HOME"\" $properties -cp \""$ES_HOME/lib/*"\" org.elasticsearch.plugins.PluginManager $args -\ No newline at end of file -+eval "$JAVA" $JAVA_OPTS $ES_JAVA_OPTS -Xmx64m -Xms16m -Delasticsearch -Des.path.home="$ES_HOME" $properties -cp "$ES_CLASSPATH/lib/*" org.elasticsearch.plugins.PluginManager $args diff --git a/pkgs/servers/search/elasticsearch/plugins.nix b/pkgs/servers/search/elasticsearch/plugins.nix index b93d8c2203d..b3141ee807d 100644 --- a/pkgs/servers/search/elasticsearch/plugins.nix +++ b/pkgs/servers/search/elasticsearch/plugins.nix @@ -1,4 +1,4 @@ -{ pkgs, stdenv, fetchurl, unzip, elasticsearch }: +{ pkgs, stdenv, fetchurl, unzip, elasticsearch-oss, javaPackages, elk6Version }: with pkgs.lib; @@ -6,8 +6,9 @@ let esPlugin = a@{ pluginName, installPhase ? '' - mkdir -p $out/bin - ES_HOME=$out ${elasticsearch}/bin/elasticsearch-plugin --install ${pluginName} --url file://$src + mkdir -p $out/config + mkdir -p $out/plugins + ES_HOME=$out ${elasticsearch-oss}/bin/elasticsearch-plugin install --batch -v file://$src '', ... }: @@ -16,33 +17,19 @@ let unpackPhase = "true"; buildInputs = [ unzip ]; meta = a.meta // { - platforms = elasticsearch.meta.platforms; + platforms = elasticsearch-oss.meta.platforms; maintainers = (a.meta.maintainers or []) ++ [ maintainers.offline ]; }; }); in { - elasticsearch_river_jdbc = esPlugin rec { - name = "elasticsearch-river-jdbc-${version}"; - pluginName = "elasticsearch-river-jdbc"; - version = "1.5.0.5"; - src = fetchurl { - url = "http://xbib.org/repository/org/xbib/elasticsearch/plugin/elasticsearch-river-jdbc/${version}/${name}-plugin.zip"; - sha256 = "1p75l3vcnb90ar4j3dci2xf8dqnqyy31kc1r075fa2xqlsxgigcp"; - }; - meta = { - homepage = https://github.com/jprante/elasticsearch-river-jdbc; - description = "Plugin to fetch data from JDBC sources for indexing into Elasticsearch"; - license = licenses.asl20; - }; - }; elasticsearch_analysis_lemmagen = esPlugin rec { name = "elasticsearch-analysis-lemmagen-${version}"; pluginName = "elasticsearch-analysis-lemmagen"; - version = "0.1"; + version = "${elk6Version}"; src = fetchurl { url = "https://github.com/vhyza/elasticsearch-analysis-lemmagen/releases/download/v${version}/${name}-plugin.zip"; - sha256 = "bf7bf5ce3ccdd3afecd0e18cd6fce1ef56f824e41f4ef50553ae598caa5c366d"; + sha256 = "1m4z05wixjrq4nlbdjyhvprkrwxjym8aba18scmzfn25fhbjgvkz"; }; meta = { homepage = https://github.com/vhyza/elasticsearch-analysis-lemmagen; @@ -51,68 +38,28 @@ in { }; }; - elasticsearch_http_basic = stdenv.mkDerivation rec { - name = "elasticsearch-http-basic-${version}"; - version = "1.5.0"; - - src = fetchurl { - url = "https://github.com/Asquera/elasticsearch-http-basic/releases/download/v${version}/${name}.jar"; - sha256 = "0fif6sbn2ich39lrgm039y9d5bxkylx9pvly04wss8rdhspvdskb"; + discovery-ec2 = esPlugin { + name = "elasticsearch-discovery-ec2-${version}"; + pluginName = "discovery-ec2"; + version = "${elk6Version}"; + src = pkgs.fetchurl { + url = "https://artifacts.elastic.co/downloads/elasticsearch-plugins/discovery-ec2/discovery-ec2-${elk6Version}.zip"; + sha256 = "1i7ksy69132sr84h51lamgq967yz3a3dw0b54nckxpqwad9pcpj0"; }; - - phases = ["installPhase"]; - installPhase = "install -D $src $out/plugins/http-basic/${name}.jar"; - meta = { - homepage = https://github.com/Asquera/elasticsearch-http-basic; - description = "HTTP Basic Authentication for Elasticsearch"; - license = licenses.mit; - platforms = elasticsearch.meta.platforms; - }; - }; - - elasticsearch_river_twitter = esPlugin rec { - name = pname + "-" + version; - pname = "elasticsearch-river-twitter"; - pluginName = "elasticsearch/" + pname + "/" + version; - version = "2.5.0"; - - src = fetchurl { - url = "http://download.elasticsearch.org/elasticsearch/${pname}/${name}.zip"; - sha256 = "0851yrmyrpp6whyxk34ykcj7b28f90w0nvkrhvl49dwqgr5s4mn4"; - }; - - meta = { - homepage = https://github.com/elasticsearch/elasticsearch-river-twitter; - description = "Twitter River Plugin for ElasticSearch"; + homepage = https://github.com/elastic/elasticsearch/tree/master/plugins/discovery-ec2; + description = "The EC2 discovery plugin uses the AWS API for unicast discovery."; license = licenses.asl20; - maintainers = [ maintainers.edwtjo ]; - platforms = elasticsearch.meta.platforms; - }; - }; - - elasticsearch_kopf = esPlugin rec { - name = "elasticsearch-kopf-${version}"; - pluginName = "elasticsearch-kopf"; - version = "2.1.1"; - src = fetchurl { - url = "https://github.com/lmenezes/elasticsearch-kopf/archive/v${version}.zip"; - sha256 = "1nwwd92g0jxhfpkxb1a9z5a62naa1y7hvlx400dm6mwwav3mrf4v"; - }; - meta = { - homepage = https://github.com/lmenezes/elasticsearch-kopf; - description = "Web administration tool for ElasticSearch"; - license = licenses.mit; }; }; search_guard = esPlugin rec { name = "elastic-search-guard-${version}"; pluginName = "search-guard"; - version = "0.5"; - src = fetchurl { - url = "https://github.com/floragunncom/search-guard/releases/download/v${version}/${pluginName}-${version}.zip"; - sha256 = "1zima4jmq1rrcqxhlrp2xian80vp244d2splby015n5cgqrp39fl"; + version = "${elk6Version}-22.3"; + src = fetchurl rec { + url = "mirror://maven/com/floragunn/search-guard-6/${version}/search-guard-6-${version}.zip"; + sha256 = "1r71h4h9bmxak1mq5gpm19xq5ji1gry1kp3sjmm8azy4ykdqdncx"; }; meta = { homepage = https://github.com/floragunncom/search-guard; diff --git a/pkgs/tools/misc/logstash/6.x.nix b/pkgs/tools/misc/logstash/6.x.nix deleted file mode 100644 index 4f15ba90d39..00000000000 --- a/pkgs/tools/misc/logstash/6.x.nix +++ /dev/null @@ -1,53 +0,0 @@ -{ elk6Version -, enableUnfree ? true -, stdenv -, fetchurl -, makeWrapper -, jre -}: - -with stdenv.lib; - -stdenv.mkDerivation rec { - version = elk6Version; - name = "logstash-${optionalString (!enableUnfree) "oss-"}${version}"; - - src = fetchurl { - url = "https://artifacts.elastic.co/downloads/logstash/${name}.tar.gz"; - sha256 = - if enableUnfree - then "0yx9hpiav4d5z1b52x2h5i0iknqs9lmxy8vmz0wkb23mjiz8njdr" - else "1ir8pbq706mxr56k5cgc9ajn2jp603zrqj66dimx6xxf2nfamw0w"; - }; - - dontBuild = true; - dontPatchELF = true; - dontStrip = true; - dontPatchShebangs = true; - - buildInputs = [ - makeWrapper jre - ]; - - installPhase = '' - mkdir -p $out - cp -r {Gemfile*,modules,vendor,lib,bin,config,data,logstash-core,logstash-core-plugin-api} $out - - patchShebangs $out/bin/logstash - patchShebangs $out/bin/logstash-plugin - - wrapProgram $out/bin/logstash \ - --set JAVA_HOME "${jre}" - - wrapProgram $out/bin/logstash-plugin \ - --set JAVA_HOME "${jre}" - ''; - - meta = with stdenv.lib; { - description = "Logstash is a data pipeline that helps you process logs and other event data from a variety of systems"; - homepage = https://www.elastic.co/products/logstash; - license = if enableUnfree then licenses.elastic else licenses.asl20; - platforms = platforms.unix; - maintainers = with maintainers; [ wjlroe offline basvandijk ]; - }; -} diff --git a/pkgs/tools/misc/logstash/default.nix b/pkgs/tools/misc/logstash/default.nix index 079e5c7a93b..4f15ba90d39 100644 --- a/pkgs/tools/misc/logstash/default.nix +++ b/pkgs/tools/misc/logstash/default.nix @@ -1,12 +1,23 @@ -{ stdenv, fetchurl, makeWrapper, jre }: +{ elk6Version +, enableUnfree ? true +, stdenv +, fetchurl +, makeWrapper +, jre +}: + +with stdenv.lib; stdenv.mkDerivation rec { - version = "2.4.0"; - name = "logstash-${version}"; + version = elk6Version; + name = "logstash-${optionalString (!enableUnfree) "oss-"}${version}"; src = fetchurl { - url = "https://download.elasticsearch.org/logstash/logstash/logstash-${version}.tar.gz"; - sha256 = "1k27hb6q1r26rp3y9pb2ry92kicw83mi352dzl2y4h0gbif46b32"; + url = "https://artifacts.elastic.co/downloads/logstash/${name}.tar.gz"; + sha256 = + if enableUnfree + then "0yx9hpiav4d5z1b52x2h5i0iknqs9lmxy8vmz0wkb23mjiz8njdr" + else "1ir8pbq706mxr56k5cgc9ajn2jp603zrqj66dimx6xxf2nfamw0w"; }; dontBuild = true; @@ -20,14 +31,14 @@ stdenv.mkDerivation rec { installPhase = '' mkdir -p $out - cp -r {Gemfile*,vendor,lib,bin} $out + cp -r {Gemfile*,modules,vendor,lib,bin,config,data,logstash-core,logstash-core-plugin-api} $out + + patchShebangs $out/bin/logstash + patchShebangs $out/bin/logstash-plugin wrapProgram $out/bin/logstash \ --set JAVA_HOME "${jre}" - wrapProgram $out/bin/rspec \ - --set JAVA_HOME "${jre}" - wrapProgram $out/bin/logstash-plugin \ --set JAVA_HOME "${jre}" ''; @@ -35,8 +46,8 @@ stdenv.mkDerivation rec { meta = with stdenv.lib; { description = "Logstash is a data pipeline that helps you process logs and other event data from a variety of systems"; homepage = https://www.elastic.co/products/logstash; - license = licenses.asl20; + license = if enableUnfree then licenses.elastic else licenses.asl20; platforms = platforms.unix; - maintainers = [ maintainers.wjlroe maintainers.offline ]; + maintainers = with maintainers; [ wjlroe offline basvandijk ]; }; } diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 39070259738..514f5ac551b 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -849,23 +849,23 @@ with pkgs; bchunk = callPackage ../tools/cd-dvd/bchunk { }; - inherit (callPackages ../misc/logging/beats/5.x.nix { }) - filebeat - heartbeat - metricbeat - packetbeat; - - inherit (let beats6 = callPackages ../misc/logging/beats/6.x.nix { }; in { - filebeat6 = beats6.filebeat; - heartbeat6 = beats6.heartbeat; - metricbeat6 = beats6.metricbeat; - packetbeat6 = beats6.packetbeat; - }) + inherit (callPackages ../misc/logging/beats/6.x.nix { }) filebeat6 heartbeat6 metricbeat6 packetbeat6; + filebeat = filebeat6; + heartbeat = heartbeat6; + metricbeat = metricbeat6; + packetbeat = packetbeat6; + + inherit (callPackages ../misc/logging/beats/5.x.nix { }) + filebeat5 + heartbeat5 + metricbeat5 + packetbeat5; + bfr = callPackage ../tools/misc/bfr { }; bibtool = callPackage ../tools/misc/bibtool { }; @@ -2303,13 +2303,13 @@ with pkgs; elk5Version = "5.6.9"; elk6Version = "6.3.2"; - elasticsearch = callPackage ../servers/search/elasticsearch { }; - elasticsearch2 = callPackage ../servers/search/elasticsearch/2.x.nix { }; elasticsearch5 = callPackage ../servers/search/elasticsearch/5.x.nix { }; - elasticsearch6 = callPackage ../servers/search/elasticsearch/6.x.nix { }; - elasticsearch6-oss = callPackage ../servers/search/elasticsearch/6.x.nix { + elasticsearch6 = callPackage ../servers/search/elasticsearch { }; + elasticsearch6-oss = callPackage ../servers/search/elasticsearch { enableUnfree = false; }; + elasticsearch = elasticsearch6; + elasticsearch-oss = elasticsearch6-oss; elasticsearchPlugins = recurseIntoAttrs ( callPackage ../servers/search/elasticsearch/plugins.nix { } @@ -3424,12 +3424,13 @@ with pkgs; keyfuzz = callPackage ../tools/inputmethods/keyfuzz { }; - kibana = callPackage ../development/tools/misc/kibana { }; kibana5 = callPackage ../development/tools/misc/kibana/5.x.nix { }; - kibana6 = callPackage ../development/tools/misc/kibana/6.x.nix { }; - kibana6-oss = callPackage ../development/tools/misc/kibana/6.x.nix { + kibana6 = callPackage ../development/tools/misc/kibana/default.nix { }; + kibana6-oss = callPackage ../development/tools/misc/kibana/default.nix { enableUnfree = false; }; + kibana = kibana6; + kibana-oss = kibana6-oss; kismet = callPackage ../applications/networking/sniffers/kismet { }; @@ -3506,12 +3507,12 @@ with pkgs; lockfileProgs = callPackage ../tools/misc/lockfile-progs { }; - logstash = callPackage ../tools/misc/logstash { }; logstash5 = callPackage ../tools/misc/logstash/5.x.nix { }; - logstash6 = callPackage ../tools/misc/logstash/6.x.nix { }; - logstash6-oss = callPackage ../tools/misc/logstash/6.x.nix { + logstash6 = callPackage ../tools/misc/logstash { }; + logstash6-oss = callPackage ../tools/misc/logstash { enableUnfree = false; }; + logstash = logstash6; logstash-contrib = callPackage ../tools/misc/logstash/contrib.nix { };