nixos/postgresqlBackup: Only replace backup when successful

Previously, a failed backup would always overwrite ${db}.sql.gz,
because the bash `>` redirect truncates the file; even if the
backup was going to fail.
On the next run, the ${db}.prev.sql.gz backup would be
overwritten by the bad ${db}.sql.gz.

Now, if the backup fails, the ${db}.in-progress.sql.gz is in an
unknown state, but ${db}.sql.gz will not be written.
On the next run, ${db}.prev.sql.gz (our only good backup) will
not be overwritten because ${db}.sql.gz does not exist.

(cherry picked from commit 81c8189a841728a813bcde8604b80427fcf33522)

nixos/modules/services/backup/postgresql-backup.nix

@@ -17,6 +17,8 @@ let
       path = [ pkgs.coreutils pkgs.gzip config.services.postgresql.package ];
 
       script = ''
+        set -e -o pipefail
+
         umask 0077 # ensure backup is only readable by postgres user
 
         if [ -e ${cfg.location}/${db}.sql.gz ]; then
@@ -24,7 +26,9 @@ let
         fi
 
         ${dumpCmd} | \
-          gzip -c > ${cfg.location}/${db}.sql.gz
+          gzip -c > ${cfg.location}/${db}.in-progress.sql.gz
+
+        mv ${cfg.location}/${db}.in-progress.sql.gz ${cfg.location}/${db}.sql.gz
       '';
 
       serviceConfig = {

nixos/tests/postgresql.nix

@@ -73,8 +73,30 @@ let
           machine.succeed(
               "systemctl start ${backupService}.service",
               "zcat /var/backup/postgresql/${backupName}.sql.gz | grep '<test>ok</test>'",
+              "ls -hal /var/backup/postgresql/ >/dev/console",
               "stat -c '%a' /var/backup/postgresql/${backupName}.sql.gz | grep 600",
           )
+      with subtest("Backup service fails gracefully"):
+          # Sabotage the backup process
+          machine.succeed("rm /run/postgresql/.s.PGSQL.5432")
+          machine.fail(
+              "systemctl start ${backupService}.service",
+          )
+          machine.succeed(
+              "ls -hal /var/backup/postgresql/ >/dev/console",
+              "zcat /var/backup/postgresql/${backupName}.prev.sql.gz | grep '<test>ok</test>'",
+              "stat /var/backup/postgresql/${backupName}.in-progress.sql.gz",
+          )
+          # In a previous version, the second run would overwrite prev.sql.gz,
+          # so we test a second run as well.
+          machine.fail(
+              "systemctl start ${backupService}.service",
+          )
+          machine.succeed(
+              "stat /var/backup/postgresql/${backupName}.in-progress.sql.gz",
+              "zcat /var/backup/postgresql/${backupName}.prev.sql.gz | grep '<test>ok</test>'",
+          )
+
 
       with subtest("Initdb works"):
           machine.succeed("sudo -u postgres initdb -D /tmp/testpostgres2")