public/nixpkgs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

hydra: 2020-02-06 -> 2020-03-{24,27}

Browse Source 
Upgrades Hydra to the latest master/flake branch. To perform this
upgrade, it's needed to do a non-trivial db-migration which provides a
massive performance-improvement[1].

The basic ideas behind multi-step upgrades of services between NixOS versions
have been gathered already[2]. For further context it's recommended to
read this first.

Basically, the following steps are needed:

* Upgrade to a non-breaking version of Hydra with the db-changes
  (columns are still nullable here). If `system.stateVersion` is set to
  something older than 20.03, the package will be selected
  automatically, otherwise `pkgs.hydra-migration` needs to be used.

* Run `hydra-backfill-ids` on the server.

* Deploy either `pkgs.hydra-unstable` (for Hydra master) or
  `pkgs.hydra-flakes` (for flakes-support) to activate the optimization.

The steps are also documented in the release-notes and in the module
using `warnings`.

`pkgs.hydra` has been removed as latest Hydra doesn't compile with
`pkgs.nixStable` and to ensure a graceful migration using the newly
introduced packages.

To verify the approach, a simple vm-test has been added which verifies
the migration steps.

[1] https://github.com/NixOS/hydra/pull/711
[2] https://github.com/NixOS/nixpkgs/pull/82353#issuecomment-598269471
This commit is contained in:
Maximilian Bosch
2020-03-28 23:33:25 +01:00
parent ea5c800175
commit bd5324c4fc
9 changed files with 444 additions and 231 deletions
Download Patch File Download Diff File Expand all files Collapse all files
nixos/tests/hydra/common.nix
+47
View File
@@ -0,0 +1,47 @@
{ system, ... }:
{
baseConfig = { pkgs, ... }: let
trivialJob = pkgs.writeTextDir "trivial.nix" ''
{ trivial = builtins.derivation {
name = "trivial";
system = "${system}";
builder = "/bin/sh";
allowSubstitutes = false;
preferLocalBuild = true;
args = ["-c" "echo success > $out; exit 0"];
};
}
'';
createTrivialProject = pkgs.stdenv.mkDerivation {
name = "create-trivial-project";
dontUnpack = true;
buildInputs = [ pkgs.makeWrapper ];
installPhase = "install -m755 -D ${./create-trivial-project.sh} $out/bin/create-trivial-project.sh";
postFixup = ''
wrapProgram "$out/bin/create-trivial-project.sh" --prefix PATH ":" ${pkgs.stdenv.lib.makeBinPath [ pkgs.curl ]} --set EXPR_PATH ${trivialJob}
'';
};
in {
virtualisation.memorySize = 2048;
time.timeZone = "UTC";
environment.systemPackages = [ createTrivialProject pkgs.jq ];
services.hydra = {
enable = true;
# Hydra needs those settings to start up, so we add something not harmfull.
hydraURL = "example.com";
notificationSender = "example@example.com";
extraConfig = ''
email_notification = 1
'';
};
services.postfix.enable = true;
nix = {
buildMachines = [{
hostName = "localhost";
systems = [ system ];
}];
binaryCaches = [];
};
};
}
nixos/tests/hydra/db-migration.nix
+86
View File
@@ -0,0 +1,86 @@
{ system ? builtins.currentSystem, ... }:
let inherit (import ./common.nix { inherit system; }) baseConfig; in
{ mig = import ../make-test-python.nix ({ pkgs, lib, ... }: {
name = "hydra-db-migration";
meta = with pkgs.stdenv.lib.maintainers; {
maintainers = [ ma27 ];
};
nodes = {
original = { pkgs, lib, ... }: {
imports = [ baseConfig ];
# An older version of Hydra before the db change
# for testing purposes.
services.hydra.package = pkgs.hydra-migration.overrideAttrs (old: {
inherit (old) pname;
version = "2020-02-06";
src = pkgs.fetchFromGitHub {
owner = "NixOS";
repo = "hydra";
rev = "2b4f14963b16b21ebfcd6b6bfa7832842e9b2afc";
sha256 = "16q0cffcsfx5pqd91n9k19850c1nbh4vvbd9h8yi64ihn7v8bick";
};
});
};
migration_phase1 = { pkgs, lib, ... }: {
imports = [ baseConfig ];
services.hydra.package = pkgs.hydra-migration;
};
finished = { pkgs, lib, ... }: {
imports = [ baseConfig ];
services.hydra.package = pkgs.hydra-unstable;
};
};
testScript = { nodes, ... }: let
next = nodes.migration_phase1.config.system.build.toplevel;
finished = nodes.finished.config.system.build.toplevel;
in ''
original.start()
original.wait_for_unit("multi-user.target")
original.wait_for_unit("postgresql.service")
original.wait_for_unit("hydra-init.service")
original.require_unit_state("hydra-queue-runner.service")
original.require_unit_state("hydra-evaluator.service")
original.require_unit_state("hydra-notify.service")
original.succeed("hydra-create-user admin --role admin --password admin")
original.wait_for_open_port(3000)
original.succeed("create-trivial-project.sh")
original.wait_until_succeeds(
'curl -L -s http://localhost:3000/build/1 -H "Accept: application/json" | jq .buildstatus | xargs test 0 -eq'
)
out = original.succeed("su -l postgres -c 'psql -d hydra <<< \"\\d+ jobs\" -A'")
assert "jobset_id" not in out
original.succeed(
"${next}/bin/switch-to-configuration test >&2"
)
original.wait_for_unit("hydra-init.service")
out = original.succeed("su -l postgres -c 'psql -d hydra <<< \"\\d+ jobs\" -A'")
assert "jobset_id|integer|||" in out
original.succeed("hydra-backfill-ids")
original.succeed(
"${finished}/bin/switch-to-configuration test >&2"
)
original.wait_for_unit("hydra-init.service")
out = original.succeed("su -l postgres -c 'psql -d hydra <<< \"\\d+ jobs\" -A'")
assert "jobset_id|integer||not null|" in out
original.wait_until_succeeds(
'curl -L -s http://localhost:3000/build/1 -H "Accept: application/json" | jq .buildstatus | xargs test 0 -eq'
)
original.shutdown()
'';
});
}
nixos/tests/hydra/default.nix
+47 -92
View File
@@ -3,102 +3,57 @@
, pkgs ? import ../../.. { inherit system config; }
}:
with import ../../lib/testing-python.nix { inherit system pkgs; };
with pkgs.lib;
let
trivialJob = pkgs.writeTextDir "trivial.nix" ''
{ trivial = builtins.derivation {
name = "trivial";
system = "${system}";
builder = "/bin/sh";
allowSubstitutes = false;
preferLocalBuild = true;
args = ["-c" "echo success > $out; exit 0"];
};
}
'';
inherit (import ./common.nix { inherit system; }) baseConfig;
createTrivialProject = pkgs.stdenv.mkDerivation {
name = "create-trivial-project";
dontUnpack = true;
buildInputs = [ pkgs.makeWrapper ];
installPhase = "install -m755 -D ${./create-trivial-project.sh} $out/bin/create-trivial-project.sh";
postFixup = ''
wrapProgram "$out/bin/create-trivial-project.sh" --prefix PATH ":" ${pkgs.stdenv.lib.makeBinPath [ pkgs.curl ]} --set EXPR_PATH ${trivialJob}
hydraPkgs = {
inherit (pkgs) hydra-migration hydra-unstable hydra-flakes;
};
makeHydraTest = with pkgs.lib; name: package: makeTest {
name = "hydra-${name}";
meta = with pkgs.stdenv.lib.maintainers; {
maintainers = [ pstn lewo ma27 ];
};
machine = { pkgs, lib, ... }: {
imports = [ baseConfig ];
services.hydra = { inherit package; };
};
testScript = ''
# let the system boot up
machine.wait_for_unit("multi-user.target")
# test whether the database is running
machine.wait_for_unit("postgresql.service")
# test whether the actual hydra daemons are running
machine.wait_for_unit("hydra-init.service")
machine.require_unit_state("hydra-queue-runner.service")
machine.require_unit_state("hydra-evaluator.service")
machine.require_unit_state("hydra-notify.service")
machine.succeed("hydra-create-user admin --role admin --password admin")
# create a project with a trivial job
machine.wait_for_open_port(3000)
# make sure the build as been successfully built
machine.succeed("create-trivial-project.sh")
machine.wait_until_succeeds(
'curl -L -s http://localhost:3000/build/1 -H "Accept: application/json" | jq .buildstatus | xargs test 0 -eq'
)
machine.wait_until_succeeds(
'journalctl -eu hydra-notify.service -o cat | grep -q "sending mail notification to hydra@localhost"'
)
'';
};
callTest = f: f { inherit system pkgs; };
hydraPkgs = {
inherit (pkgs) nixStable nixUnstable nixFlakes;
};
tests = pkgs.lib.flip pkgs.lib.mapAttrs hydraPkgs (name: nix:
callTest (import ../make-test-python.nix ({ pkgs, lib, ... }:
{
name = "hydra-with-${name}";
meta = with pkgs.stdenv.lib.maintainers; {
maintainers = [ pstn lewo ma27 ];
};
machine = { pkgs, ... }:
{
virtualisation.memorySize = 1024;
time.timeZone = "UTC";
environment.systemPackages = [ createTrivialProject pkgs.jq ];
services.hydra = {
enable = true;
#Hydra needs those settings to start up, so we add something not harmfull.
hydraURL = "example.com";
notificationSender = "example@example.com";
package = pkgs.hydra.override { inherit nix; };
extraConfig = ''
email_notification = 1
'';
};
services.postfix.enable = true;
nix = {
buildMachines = [{
hostName = "localhost";
systems = [ system ];
}];
binaryCaches = [];
};
};
testScript = ''
# let the system boot up
machine.wait_for_unit("multi-user.target")
# test whether the database is running
machine.wait_for_unit("postgresql.service")
# test whether the actual hydra daemons are running
machine.wait_for_unit("hydra-init.service")
machine.require_unit_state("hydra-queue-runner.service")
machine.require_unit_state("hydra-evaluator.service")
machine.require_unit_state("hydra-notify.service")
machine.succeed("hydra-create-user admin --role admin --password admin")
# create a project with a trivial job
machine.wait_for_open_port(3000)
# make sure the build as been successfully built
machine.succeed("create-trivial-project.sh")
machine.wait_until_succeeds(
'curl -L -s http://localhost:3000/build/1 -H "Accept: application/json" | jq .buildstatus | xargs test 0 -eq'
)
machine.wait_until_succeeds(
'journalctl -eu hydra-notify.service -o cat | grep -q "sending mail notification to hydra@localhost"'
)
'';
})));
in
tests
mapAttrs makeHydraTest hydraPkgs