From fa1bbe69fb1f774a20b57cbde66fa9253927b976 Mon Sep 17 00:00:00 2001 From: Jan Tojnar Date: Sun, 2 Jun 2019 08:09:17 +0200 Subject: [PATCH 1/9] maintainers/scripts/update.nix: Do not store updateScripts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit For update script parallelization, we have started calling builtins.toJSON on updateScripts, which triggers evaluation of paths and therefore their copying to Nix store. This breaks update scripts that assume that they exist in nixpkgs like dwarf-fortress. https://github.com/NixOS/nixpkgs/issues/61935 Let’s stringify the paths before JSONification to prevent the evaluation. --- maintainers/scripts/update.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maintainers/scripts/update.nix b/maintainers/scripts/update.nix index 120cd5552f4..61afb64f9fc 100755 --- a/maintainers/scripts/update.nix +++ b/maintainers/scripts/update.nix @@ -122,7 +122,7 @@ let packageData = package: { name = package.name; pname = (builtins.parseDrvName package.name).name; - updateScript = pkgs.lib.toList package.updateScript; + updateScript = map builtins.toString (pkgs.lib.toList package.updateScript); }; in pkgs.stdenv.mkDerivation { From ea3ed0c3ae6a2f987910b53cfdbad06cb220aa26 Mon Sep 17 00:00:00 2001 From: Jan Tojnar Date: Sun, 2 Jun 2019 08:19:19 +0200 Subject: [PATCH 2/9] doc: fix passthru.updateScript help command --- doc/stdenv.xml | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/doc/stdenv.xml b/doc/stdenv.xml index 00d1ec87098..da72086640f 100644 --- a/doc/stdenv.xml +++ b/doc/stdenv.xml @@ -715,13 +715,7 @@ passthru.updateScript = [ ../../update.sh pname "--requested-release=unstable" ] For information about how to run the updates, execute - - nix-shell - - maintainers/scripts/update.nix - - - . + nix-shell maintainers/scripts/update.nix. From 36b2012447b936a1a638e2ddae17507bab8fc839 Mon Sep 17 00:00:00 2001 From: Jan Tojnar Date: Sun, 2 Jun 2019 08:55:29 +0200 Subject: [PATCH 3/9] maintainers/scripts/update.nix: ignore overlays It makes little sense for update.nix to try to update overlays; for most people, they will point to a read-only repository most of the cases. --- maintainers/scripts/update.nix | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/maintainers/scripts/update.nix b/maintainers/scripts/update.nix index 61afb64f9fc..2d1f152ed77 100755 --- a/maintainers/scripts/update.nix +++ b/maintainers/scripts/update.nix @@ -20,7 +20,9 @@ let in [x] ++ nubOn f xs; - pkgs = import ./../../default.nix { }; + pkgs = import ./../../default.nix { + overlays = []; + }; packagesWith = cond: return: set: nubOn (pkg: pkg.updateScript) From c3dd31f177b532e7e94b74afca7241778c23af6f Mon Sep 17 00:00:00 2001 From: Jan Tojnar Date: Sun, 2 Jun 2019 09:03:41 +0200 Subject: [PATCH 4/9] maintainers/scripts/update.nix: handle null paths MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previously we did not handle non-existant paths making the program crash. Let’s show a proper error. --- maintainers/scripts/update.nix | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/maintainers/scripts/update.nix b/maintainers/scripts/update.nix index 2d1f152ed77..b980514cc09 100755 --- a/maintainers/scripts/update.nix +++ b/maintainers/scripts/update.nix @@ -69,9 +69,12 @@ let let attrSet = pkgs.lib.attrByPath (pkgs.lib.splitString "." path) null pkgs; in - packagesWith (name: pkg: builtins.hasAttr "updateScript" pkg) - (name: pkg: pkg) - attrSet; + if attrSet == null then + builtins.throw "Attribute path `${path}` does not exists." + else + packagesWith (name: pkg: builtins.hasAttr "updateScript" pkg) + (name: pkg: pkg) + attrSet; packageByName = name: let From e98c804cc92cd3412ae6877e9014b3193a5aa3e6 Mon Sep 17 00:00:00 2001 From: Jan Tojnar Date: Sun, 2 Jun 2019 09:07:18 +0200 Subject: [PATCH 5/9] maintainers/scripts/update.nix: Clean up Make calling update.py a little nicer --- maintainers/scripts/update.nix | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/maintainers/scripts/update.nix b/maintainers/scripts/update.nix index b980514cc09..04723cb8a36 100755 --- a/maintainers/scripts/update.nix +++ b/maintainers/scripts/update.nix @@ -130,6 +130,14 @@ let updateScript = map builtins.toString (pkgs.lib.toList package.updateScript); }; + packagesJson = pkgs.writeText "packages.json" (builtins.toJSON (map packageData packages)); + + optionalArgs = + pkgs.lib.optional (max-workers != null) "--max-workers=${max-workers}" + ++ pkgs.lib.optional (keep-going == "true") "--keep-going"; + + args = [ packagesJson ] ++ optionalArgs; + in pkgs.stdenv.mkDerivation { name = "nixpkgs-update-script"; buildCommand = '' @@ -144,6 +152,6 @@ in pkgs.stdenv.mkDerivation { ''; shellHook = '' unset shellHook # do not contaminate nested shells - exec ${pkgs.python3.interpreter} ${./update.py} ${pkgs.writeText "packages.json" (builtins.toJSON (map packageData packages))}${pkgs.lib.optionalString (max-workers != null) " --max-workers=${max-workers}"}${pkgs.lib.optionalString (keep-going == "true") " --keep-going"} + exec ${pkgs.python3.interpreter} ${./update.py} ${builtins.concatStringsSep " " args} ''; } From 1b507bea4d181fb1fc15ddc6d3ac01aa5f8ab664 Mon Sep 17 00:00:00 2001 From: Jan Tojnar Date: Fri, 12 Apr 2019 18:54:13 +0200 Subject: [PATCH 6/9] common-updater-scripts: keep the unescaped oldVersion Previously, we escaped the old version in place for use in sed commands, and then had to use that in error messages. We can do better. --- pkgs/common-updater/scripts/update-source-version | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/pkgs/common-updater/scripts/update-source-version b/pkgs/common-updater/scripts/update-source-version index bc2c68feaea..e1eece1fbf4 100755 --- a/pkgs/common-updater/scripts/update-source-version +++ b/pkgs/common-updater/scripts/update-source-version @@ -109,13 +109,13 @@ if [ "$oldVersion" = "$newVersion" ]; then fi # Escape regex metacharacter that are allowed in store path names -oldVersion=$(echo "$oldVersion" | sed -re 's|[.+]|\\&|g') -oldUrl=$(echo "$oldUrl" | sed -re 's|[${}.+]|\\&|g') +oldVersionEscaped=$(echo "$oldVersion" | sed -re 's|[.+]|\\&|g') +oldUrlEscaped=$(echo "$oldUrl" | sed -re 's|[${}.+]|\\&|g') -if [ $(grep -c -E "^\s*(let\b)?\s*$versionKey\s*=\s*\"$oldVersion\"" "$nixFile") = 1 ]; then - pattern="/\b$versionKey\b\s*=/ s|\"$oldVersion\"|\"$newVersion\"|" -elif [ $(grep -c -E "^\s*(let\b)?\s*name\s*=\s*\"[^\"]+-$oldVersion\"" "$nixFile") = 1 ]; then - pattern="/\bname\b\s*=/ s|-$oldVersion\"|-$newVersion\"|" +if [ $(grep -c -E "^\s*(let\b)?\s*$versionKey\s*=\s*\"$oldVersionEscaped\"" "$nixFile") = 1 ]; then + pattern="/\b$versionKey\b\s*=/ s|\"$oldVersionEscaped\"|\"$newVersion\"|" +elif [ $(grep -c -E "^\s*(let\b)?\s*name\s*=\s*\"[^\"]+-$oldVersionEscaped\"" "$nixFile") = 1 ]; then + pattern="/\bname\b\s*=/ s|-$oldVersionEscaped\"|-$newVersion\"|" else die "Couldn't figure out where out where to patch in new version in '$attr'!" fi @@ -128,7 +128,7 @@ fi # Replace new URL if [ -n "$newUrl" ]; then - sed -i "$nixFile" -re "s|\"$oldUrl\"|\"$newUrl\"|" + sed -i "$nixFile" -re "s|\"$oldUrlEscaped\"|\"$newUrl\"|" if cmp -s "$nixFile" "$nixFile.bak"; then die "Failed to replace source URL '$oldUrl' to '$newUrl' in '$attr'!" From 5751988b556ef354b3689e82406ddc9df8d39547 Mon Sep 17 00:00:00 2001 From: Jan Tojnar Date: Fri, 12 Apr 2019 18:39:58 +0200 Subject: [PATCH 7/9] common-updater-scripts: simplify fetchgit fix No need to try evaluate the URL twice as introduced in the original fix https://github.com/NixOS/nixpkgs/commit/cce486838b6c52dedfec1c93f60ebadb72c48431 --- pkgs/common-updater/scripts/update-source-version | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/pkgs/common-updater/scripts/update-source-version b/pkgs/common-updater/scripts/update-source-version index e1eece1fbf4..591ffa6ca5b 100755 --- a/pkgs/common-updater/scripts/update-source-version +++ b/pkgs/common-updater/scripts/update-source-version @@ -86,14 +86,10 @@ if [ $(grep -c "$oldHash" "$nixFile") != 1 ]; then die "Couldn't locate old source hash '$oldHash' (or it appeared more than once) in '$nixFile'!" fi -oldUrl=$(nix-instantiate $systemArg --eval -E "with import ./. {}; builtins.elemAt $attr.src.drvAttrs.urls 0" | tr -d '"') +oldUrl=$(nix-instantiate $systemArg --eval -E "with import ./. {}; builtins.elemAt ($attr.src.drvAttrs.urls or [ $attr.src.url ]) 0" | tr -d '"') if [ -z "$oldUrl" ]; then - oldUrl=$(nix-instantiate $systemArg --eval -E "with import ./. {}; $attr.src.url" | tr -d '"') - - if [ -z "$oldUrl" ]; then - die "Couldn't evaluate source url from '$attr.src'!" - fi + die "Couldn't evaluate source url from '$attr.src'!" fi drvName=$(nix-instantiate $systemArg --eval -E "with import ./. {}; (builtins.parseDrvName $attr.name).name" | tr -d '"') From 9f6a9ce3689a1bbaf271f31b93bd595fc8185187 Mon Sep 17 00:00:00 2001 From: Jan Tojnar Date: Sun, 2 Jun 2019 09:40:55 +0200 Subject: [PATCH 8/9] gnome3.updateScript: clean up PATH grep was not actually properly included by common-updater-scripts before 01050586980a81e525428eb9e571a8b040f96b26 but I am not sure why would we ever need to add coreutils to PATH. --- pkgs/desktops/gnome-3/update.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgs/desktops/gnome-3/update.nix b/pkgs/desktops/gnome-3/update.nix index 2ee72d46993..f42b6723950 100644 --- a/pkgs/desktops/gnome-3/update.nix +++ b/pkgs/desktops/gnome-3/update.nix @@ -1,4 +1,4 @@ -{ stdenv, lib, writeScript, python3, common-updater-scripts, coreutils, gnugrep, gnused }: +{ stdenv, lib, writeScript, python3, common-updater-scripts }: { packageName, attrPath ? packageName, versionPolicy ? "odd-unstable" }: let @@ -9,7 +9,7 @@ let package_name="$1" attr_path="$2" version_policy="$3" - PATH=${lib.makeBinPath [ common-updater-scripts coreutils gnugrep gnused python ]} + PATH=${lib.makeBinPath [ common-updater-scripts python ]} latest_tag=$(python "${./find-latest-version.py}" "$package_name" "$version_policy" "stable") update-source-version "$attr_path" "$latest_tag" ''; From 431dc973a5b99de1f96365a373706440a8017385 Mon Sep 17 00:00:00 2001 From: Jan Tojnar Date: Mon, 3 Jun 2019 00:15:29 +0200 Subject: [PATCH 9/9] doc: document PWD for passthru.updateScript Make it clear that PWD is not fixed --- doc/stdenv.xml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/doc/stdenv.xml b/doc/stdenv.xml index da72086640f..a14d78afe71 100644 --- a/doc/stdenv.xml +++ b/doc/stdenv.xml @@ -709,9 +709,12 @@ passthru.updateScript = writeScript "update-zoom-us" '' passthru.updateScript = [ ../../update.sh pname "--requested-release=unstable" ]; - Note that the update scripts will be run in parallel by default; you - should avoid running git commit or any other commands - that cannot handle that. + + + The script will be usually run from the root of the Nixpkgs repository + but you should not rely on that. Also note that the update scripts will + be run in parallel by default; you should avoid running git + commit or any other commands that cannot handle that. For information about how to run the updates, execute