Merge branch 'master' into staging-next

Comments on conflicts:
- llvm: d6f401e1 vs. 469ecc70 - docs for 6 and 7 say the default is
  to build all targets, so we should be fine
- some pypi hashes: they were equivalent, just base16 vs. base32

pkgs/development/tools/build-managers/alibuild/default.nix

@@ -9,12 +9,9 @@ python.pkgs.buildPythonApplication rec {
     sha256 = "1mnh0h9m96p78b9ln1gbl4lw1mgl16qbyfi9fj2l13p3nxaq1sib";
   };
 
-  argparse = null;
-
   doCheck = false;
   propagatedBuildInputs = [
     python.pkgs.requests
-    python.pkgs.argparse
     python.pkgs.pyyaml
   ];
 

pkgs/development/tools/build-managers/bam/default.nix

@@ -1,15 +1,17 @@
-{ stdenv, fetchurl, lua5, python }:
+{ stdenv, fetchFromGitHub, lua5_3, python }:
 
 stdenv.mkDerivation rec {
   name = "bam-${version}";
-  version = "0.4.0";
+  version = "0.5.1";
 
-  src = fetchurl {
-    url = "http://github.com/downloads/matricks/bam/${name}.tar.bz2";
-    sha256 = "0z90wvyd4nfl7mybdrv9dsd4caaikc6fxw801b72gqi1m9q0c0sn";
+  src = fetchFromGitHub {
+    owner = "matricks";
+    repo = "bam";
+    rev = "v${version}";
+    sha256 = "13br735ig7lygvzyfd15fc2rdygrqm503j6xj5xkrl1r7w2wipq6";
   };
 
-  buildInputs = [ lua5 python ];
+  buildInputs = [ lua5_3 python ];
 
   buildPhase = ''${stdenv.shell} make_unix.sh'';
 
@@ -29,7 +31,7 @@ stdenv.mkDerivation rec {
       raskin
     ];
     platforms = platforms.linux;
-    license = licenses.free;
+    license = licenses.zlib;
     downloadPage = "http://matricks.github.com/bam/";
   };
 }

pkgs/development/tools/build-managers/bazel/bash-tools-test.nix

@@ -0,0 +1,42 @@
+{ stdenv, writeText, runCommandCC, bazel }:
+
+# Tests that certain executables are available in bazel-executed bash shells.
+
+let
+  WORKSPACE = writeText "WORKSPACE" ''
+    workspace(name = "our_workspace")
+  '';
+
+  fileIn = writeText "input.txt" ''
+  one
+  two
+  three
+  '';
+
+  fileBUILD = writeText "BUILD" ''
+    genrule(
+      name = "tool_usage",
+      srcs = [ ":input.txt" ],
+      outs = [ "output.txt" ],
+      cmd = "cat $(location :input.txt) | gzip - | gunzip - | awk '/t/' > $@",
+    )
+  '';
+
+  runLocal = name: script: runCommandCC name { preferLocalBuild = true; } script;
+
+  workspaceDir = runLocal "our_workspace" ''
+    mkdir $out
+    cp ${WORKSPACE} $out/WORKSPACE
+    cp ${fileIn} $out/input.txt
+    cp ${fileBUILD} $out/BUILD
+  '';
+
+  testBazel = runLocal "bazel-test-bash-tools" ''
+    export HOME=$(mktemp -d)
+    cp -r ${workspaceDir} wd && chmod +w wd && cd wd
+    ${bazel}/bin/bazel build :tool_usage
+    cp bazel-genfiles/output.txt $out
+    echo "Testing content" && [ "$(cat $out | wc -l)" == "2" ] && echo "OK"
+  '';
+
+in testBazel

pkgs/development/tools/build-managers/bazel/default.nix

@@ -1,10 +1,11 @@
 { stdenv, callPackage, lib, fetchurl, fetchpatch, runCommand, makeWrapper
-, jdk, zip, unzip, bash, writeCBin, coreutils
-, which, python, perl, gawk, gnused, gnugrep, findutils
+, zip, unzip, bash, writeCBin, coreutils
+, which, python, perl, gawk, gnused, gnutar, gnugrep, gzip, findutils
 # Apple dependencies
 , cctools, clang, libcxx, CoreFoundation, CoreServices, Foundation
 # Allow to independently override the jdks used to build and run respectively
-, buildJdk ? jdk, runJdk ? jdk
+, buildJdk, runJdk
+, buildJdkName
 # Always assume all markers valid (don't redownload dependencies).
 # Also, don't clean up environment variables.
 , enableNixHacks ? false
@@ -23,12 +24,43 @@ let
     for i in ${builtins.toString srcDeps}; do cp $i $out/$(stripHash $i); done
   '';
 
-  defaultShellPath = lib.makeBinPath [ bash coreutils findutils gawk gnugrep gnused which unzip ];
+  defaultShellPath = lib.makeBinPath
+    # Keep this list conservative. For more exotic tools, prefer to use
+    # @rules_nixpkgs to pull in tools from the nix repository. Example:
+    #
+    # WORKSPACE:
+    #
+    #     nixpkgs_git_repository(
+    #         name = "nixpkgs",
+    #         revision = "def5124ec8367efdba95a99523dd06d918cb0ae8",
+    #     )
+    #
+    #     # This defines an external Bazel workspace.
+    #     nixpkgs_package(
+    #         name = "bison",
+    #         repositories = { "nixpkgs": "@nixpkgs//:default.nix" },
+    #     )
+    #
+    # some/BUILD.bazel:
+    #
+    #     genrule(
+    #        ...
+    #        cmd = "$(location @bison//:bin/bison) -other -args",
+    #        tools = [
+    #            ...
+    #            "@bison//:bin/bison",
+    #        ],
+    #     )
+    #
+    [ bash coreutils findutils gawk gnugrep gnutar gnused gzip which unzip ];
+
+  # Java toolchain used for the build and tests
+  javaToolchain = "@bazel_tools//tools/jdk:toolchain_host${buildJdkName}";
 
 in
 stdenv.mkDerivation rec {
 
-  version = "0.21.0";
+  version = "0.22.0";
 
   meta = with lib; {
     homepage = "https://github.com/bazelbuild/bazel/";
@@ -38,16 +70,21 @@ stdenv.mkDerivation rec {
     platforms = platforms.linux ++ platforms.darwin;
   };
 
-  # additional tests that check bazel’s functionality
+  # Additional tests that check bazel’s functionality. Execute
+  #
+  #     nix-build . -A bazel.tests
+  #
+  # in the nixpkgs checkout root to exercise them locally.
   passthru.tests = {
-    python_bin_path = callPackage ./python-bin-path-test.nix {};
+    pythonBinPath = callPackage ./python-bin-path-test.nix {};
+    bashTools = callPackage ./bash-tools-test.nix {};
   };
 
   name = "bazel-${version}";
 
   src = fetchurl {
-    url = "https://github.com/bazelbuild/bazel/releases/download/${version}/bazel-${version}-dist.zip";
-    sha256 = "1d3x0f1hzaiqq00pd65bks7v8kbv57m13jsing7y0y9id0g87jvc";
+    url = "https://github.com/bazelbuild/bazel/releases/download/${version}/${name}-dist.zip";
+    sha256 = "0hannnvia8rvmi2v5d97j1f6wv0m1kxkd5hq4aqp0dqjr0ka4q38";
   };
 
   sourceRoot = ".";
@@ -125,9 +162,11 @@ stdenv.mkDerivation rec {
 
     genericPatches = ''
       # Substitute python's stub shebang to plain python path. (see TODO add pr URL)
+      # See also `postFixup` where python is added to $out/nix-support
       substituteInPlace src/main/java/com/google/devtools/build/lib/bazel/rules/python/python_stub_template.txt\
           --replace "/usr/bin/env python" "${python}/bin/python" \
           --replace "NIX_STORE_PYTHON_PATH" "${python}/bin/python" \
+
       # substituteInPlace is rather slow, so prefilter the files with grep
       grep -rlZ /bin src/main/java/com/google/devtools | while IFS="" read -r -d "" path; do
         # If you add more replacements here, you must change the grep above!
@@ -142,11 +181,6 @@ stdenv.mkDerivation rec {
       substituteInPlace scripts/bootstrap/compile.sh \
           --replace /bin/sh ${customBash}/bin/bash
 
-      # We only build with JDK8 for now, since JDK11 does not compile bazel
-      substituteInPlace tools/jdk/default_java_toolchain.bzl \
-        --replace '"jvm_opts": JDK9_JVM_OPTS' \
-                  '"jvm_opts": JDK8_JVM_OPTS'
-
       # add nix environment vars to .bazelrc
       cat >> .bazelrc <<EOF
       build --experimental_distdir=${distDir}
@@ -156,6 +190,7 @@ stdenv.mkDerivation rec {
       build --linkopt="-Wl,$(echo $NIX_LDFLAGS | sed -e 's/ /" --linkopt="-Wl,/g')"
       build --host_linkopt="-Wl,$(echo $NIX_LDFLAGS | sed -e 's/ /" --host_linkopt="-Wl,/g')"
       build --host_javabase='@local_jdk//:jdk'
+      build --host_java_toolchain='${javaToolchain}'
       EOF
 
       # add the same environment vars to compile.sh
@@ -164,6 +199,7 @@ stdenv.mkDerivation rec {
           -e "/\$command \\\\$/a --linkopt=\"-Wl,$(echo $NIX_LDFLAGS | sed -e 's/ /" --linkopt=\"-Wl,/g')\" \\\\" \
           -e "/\$command \\\\$/a --host_linkopt=\"-Wl,$(echo $NIX_LDFLAGS | sed -e 's/ /" --host_linkopt=\"-Wl,/g')\" \\\\" \
           -e "/\$command \\\\$/a --host_javabase='@local_jdk//:jdk' \\\\" \
+          -e "/\$command \\\\$/a --host_java_toolchain='${javaToolchain}' \\\\" \
           -i scripts/bootstrap/compile.sh
 
       # --experimental_strict_action_env (which will soon become the
@@ -233,7 +269,9 @@ stdenv.mkDerivation rec {
     export TEST_TMPDIR=$(pwd)
 
     hello_test () {
-      $out/bin/bazel test --test_output=errors \
+      $out/bin/bazel test \
+        --test_output=errors \
+        --java_toolchain='${javaToolchain}' \
         examples/cpp:hello-success_test \
         examples/java-native/src/test/java/com/example/myproject:hello
     }
@@ -262,7 +300,10 @@ stdenv.mkDerivation rec {
   # Save paths to hardcoded dependencies so Nix can detect them.
   postFixup = ''
     mkdir -p $out/nix-support
-    echo "${customBash} ${defaultShellPath}" > $out/nix-support/depends
+    echo "${customBash} ${defaultShellPath}" >> $out/nix-support/depends
+    # The templates get tar’d up into a .jar,
+    # so nix can’t detect python is needed in the runtime closure
+    echo "${python}" >> $out/nix-support/depends
   '';
 
   dontStrip = true;

pkgs/development/tools/build-managers/bazel/python-bin-path-test.nix

@@ -42,8 +42,10 @@ let
 
   testBazel = runLocal "bazel-test-builtin-rules" ''
     export HOME=$(mktemp -d)
-    cp -r ${workspaceDir}/* .
-    ${bazel}/bin/bazel --output_base=/tmp/bazel-tests/wd\
+    # Note https://github.com/bazelbuild/bazel/issues/5763#issuecomment-456374609
+    # about why to create a subdir for the workspace.
+    cp -r ${workspaceDir} wd && chmod u+w wd && cd wd
+    ${bazel}/bin/bazel \
       test \
         --test_output=errors \
         --host_javabase='@local_jdk//:jdk' \

pkgs/development/tools/build-managers/bloop/default.nix

@@ -2,7 +2,7 @@
 
 let
   baseName = "bloop";
-  version = "1.2.3";
+  version = "1.2.5";
   deps = stdenv.mkDerivation {
     name = "${baseName}-deps-${version}";
     buildCommand = ''
@@ -16,7 +16,7 @@ let
     '';
     outputHashMode = "recursive";
     outputHashAlgo = "sha256";
-    outputHash     = "0d0q4rzz21afzfclm3sjp940wk7p8cllbxsidr6rg3r1qqhzawlr";
+    outputHash     = "19373fyb0g7irrdzb1vsjmyv5xj84qwbcfb6lm076px7wfyn0w1c";
   };
 in
 stdenv.mkDerivation rec {

pkgs/development/tools/build-managers/scons/default.nix

@@ -7,8 +7,8 @@ in {
     version = "3.0.1";
     sha256 = "0wzid419mlwqw9llrg8gsx4nkzhqy16m4m40r0xnh6cwscw5wir4";
   };
-  scons_3_0_3 = mkScons {
-    version = "3.0.3";
-    sha256 = "1wwn0534d83ryfxjihvqk2ncj8wh5210pi3jxjd2cvjqa9mpkv6q";
+  scons_latest = mkScons {
+    version = "3.0.4";
+    sha256 = "06lv3pmdz5l23rx3kqsi1k712bdl36i942hgbjh209s94mpb7f72";
   };
 }