nixos/acme: Restructure module

- Use an acme user and group, allow group override only
- Use hashes to determine when certs actually need to regenerate
- Avoid running lego more than necessary
- Harden permissions
- Support "systemctl clean" for cert regeneration
- Support reuse of keys between some configuration changes
- Permissions fix services solves for previously root owned certs
- Add a note about multiple account creation and emails
- Migrate extraDomains to a list
- Deprecate user option
- Use minica for self-signed certs
- Rewrite all tests

I thought of a few more cases where things may go wrong,
and added tests to cover them. In particular, the web server
reload services were depending on the target - which stays alive,
meaning that the renewal timer wouldn't be triggering a reload
and old certs would stay on the web servers.

I encountered some problems ensuring that the reload took place
without accidently triggering it as part of the test. The sync
commands I added ended up being essential and I'm not sure why,
it seems like either node.succeed ends too early or there's an
oddity of the vm's filesystem I'm not aware of.

- Fix duplicate systemd rules on reload services

Since useACMEHost is not unique to every vhost, if one cert
was reused many times it would create duplicate entries in
${server}-config-reload.service for wants, before and
ConditionPathExists

nixos/modules/services/web-servers/apache-httpd/default.nix

@@ -6,6 +6,8 @@ let
 
   cfg = config.services.httpd;
 
+  certs = config.security.acme.certs;
+
   runtimeDir = "/run/httpd";
 
   pkg = cfg.package.out;
@@ -26,6 +28,13 @@ let
 
   vhosts = attrValues cfg.virtualHosts;
 
+  # certName is used later on to determine systemd service names.
+  acmeEnabledVhosts = map (hostOpts: hostOpts // {
+    certName = if hostOpts.useACMEHost != null then hostOpts.useACMEHost else hostOpts.hostName;
+  }) (filter (hostOpts: hostOpts.enableACME || hostOpts.useACMEHost != null) vhosts);
+
+  dependentCertNames = unique (map (hostOpts: hostOpts.certName) acmeEnabledVhosts);
+
   mkListenInfo = hostOpts:
     if hostOpts.listen != [] then hostOpts.listen
     else (
@@ -125,13 +134,13 @@ let
 
       useACME = hostOpts.enableACME || hostOpts.useACMEHost != null;
       sslCertDir =
-        if hostOpts.enableACME then config.security.acme.certs.${hostOpts.hostName}.directory
-        else if hostOpts.useACMEHost != null then config.security.acme.certs.${hostOpts.useACMEHost}.directory
+        if hostOpts.enableACME then certs.${hostOpts.hostName}.directory
+        else if hostOpts.useACMEHost != null then certs.${hostOpts.useACMEHost}.directory
         else abort "This case should never happen.";
 
-      sslServerCert = if useACME then "${sslCertDir}/full.pem" else hostOpts.sslServerCert;
+      sslServerCert = if useACME then "${sslCertDir}/fullchain.pem" else hostOpts.sslServerCert;
       sslServerKey = if useACME then "${sslCertDir}/key.pem" else hostOpts.sslServerKey;
-      sslServerChain = if useACME then "${sslCertDir}/fullchain.pem" else hostOpts.sslServerChain;
+      sslServerChain = if useACME then "${sslCertDir}/chain.pem" else hostOpts.sslServerChain;
 
       acmeChallenge = optionalString useACME ''
         Alias /.well-known/acme-challenge/ "${hostOpts.acmeRoot}/.well-known/acme-challenge/"
@@ -347,7 +356,6 @@ let
       cat ${php.phpIni} > $out
       echo "$options" >> $out
     '';
-
 in
 
 
@@ -647,14 +655,17 @@ in
       wwwrun.gid = config.ids.gids.wwwrun;
     };
 
-    security.acme.certs = mapAttrs (name: hostOpts: {
-      user = cfg.user;
-      group = mkDefault cfg.group;
-      email = if hostOpts.adminAddr != null then hostOpts.adminAddr else cfg.adminAddr;
-      webroot = hostOpts.acmeRoot;
-      extraDomains = genAttrs hostOpts.serverAliases (alias: null);
-      postRun = "systemctl reload httpd.service";
-    }) (filterAttrs (name: hostOpts: hostOpts.enableACME) cfg.virtualHosts);
+    security.acme.certs = let
+      acmePairs = map (hostOpts: nameValuePair hostOpts.hostName {
+        group = mkDefault cfg.group;
+        webroot = hostOpts.acmeRoot;
+        extraDomainNames = hostOpts.serverAliases;
+        # Use the vhost-specific email address if provided, otherwise let
+        # security.acme.email or security.acme.certs.<cert>.email be used.
+        email = mkOverride 2000 (if hostOpts.adminAddr != null then hostOpts.adminAddr else cfg.adminAddr);
+      # Filter for enableACME-only vhosts. Don't want to create dud certs
+      }) (filter (hostOpts: hostOpts.useACMEHost == null) acmeEnabledVhosts);
+    in listToAttrs acmePairs;
 
     environment.systemPackages = [
       apachectl
@@ -724,16 +735,12 @@ in
           "Z '${cfg.logDir}' - ${svc.User} ${svc.Group}"
         ];
 
-    systemd.services.httpd =
-      let
-        vhostsACME = filter (hostOpts: hostOpts.enableACME) vhosts;
-      in
-      { description = "Apache HTTPD";
-
+    systemd.services.httpd = {
+        description = "Apache HTTPD";
         wantedBy = [ "multi-user.target" ];
-        wants = concatLists (map (hostOpts: [ "acme-${hostOpts.hostName}.service" "acme-selfsigned-${hostOpts.hostName}.service" ]) vhostsACME);
-        after = [ "network.target" "fs.target" ] ++ map (hostOpts: "acme-selfsigned-${hostOpts.hostName}.service") vhostsACME;
-        before = map (hostOpts: "acme-${hostOpts.hostName}.service") vhostsACME;
+        wants = concatLists (map (certName: [ "acme-finished-${certName}.target" ]) dependentCertNames);
+        after = [ "network.target" ] ++ map (certName: "acme-selfsigned-${certName}.service") dependentCertNames;
+        before = map (certName: "acme-${certName}.service") dependentCertNames;
 
         path = [ pkg pkgs.coreutils pkgs.gnugrep ];
 
@@ -767,5 +774,31 @@ in
         };
       };
 
+    # postRun hooks on cert renew can't be used to restart Apache since renewal
+    # runs as the unprivileged acme user. sslTargets are added to wantedBy + before
+    # which allows the acme-finished-$cert.target to signify the successful updating
+    # of certs end-to-end.
+    systemd.services.httpd-config-reload = let
+      sslServices = map (certName: "acme-${certName}.service") dependentCertNames;
+      sslTargets = map (certName: "acme-finished-${certName}.target") dependentCertNames;
+    in mkIf (sslServices != []) {
+      wantedBy = sslServices ++ [ "multi-user.target" ];
+      # Before the finished targets, after the renew services.
+      # This service might be needed for HTTP-01 challenges, but we only want to confirm
+      # certs are updated _after_ config has been reloaded.
+      before = sslTargets;
+      after = sslServices;
+      # Block reloading if not all certs exist yet.
+      # Happens when config changes add new vhosts/certs.
+      unitConfig.ConditionPathExists = map (certName: certs.${certName}.directory + "/fullchain.pem") dependentCertNames;
+      serviceConfig = {
+        Type = "oneshot";
+        TimeoutSec = 60;
+        ExecCondition = "/run/current-system/systemd/bin/systemctl -q is-active httpd.service";
+        ExecStartPre = "${pkg}/bin/apachectl configtest";
+        ExecStart = "/run/current-system/systemd/bin/systemctl reload httpd.service";
+      };
+    };
+
   };
 }