nixfmt test

nixos/modules/module-list.nix

@@ -247,7 +247,7 @@
   ./services/computing/slurm/slurm.nix
   ./services/continuous-integration/buildbot/master.nix
   ./services/continuous-integration/buildbot/worker.nix
-  ./services/continuous-integration/buildkite-agent.nix
+  ./services/continuous-integration/buildkite-agents.nix
   ./services/continuous-integration/hail.nix
   ./services/continuous-integration/hydra/default.nix
   ./services/continuous-integration/gitlab-runner.nix

nixos/modules/services/backup/borgbackup.nix

@@ -98,6 +98,24 @@ let
       inherit (cfg) startAt;
     };
 
+  # utility function around makeWrapper
+  mkWrapperDrv = {
+      original, name, set ? {}, setDefault ? {}
+    }:
+    pkgs.runCommandNoCC "${name}-wrapper" {
+      buildInputs = [ pkgs.makeWrapper ];
+    } (with lib; ''
+      makeWrapper "${original}" "$out/bin/${name}" \
+        ${concatStringsSep " \\\n " (mapAttrsToList (name: value: ''--set ${name} "${value}"'') set)} \
+        ${concatStringsSep " \\\n " (mapAttrsToList (name: value: ''--set-default ${name} "${value}"'') setDefault)}
+    '');
+
+  mkBorgWrapper = name: cfg: mkWrapperDrv {
+    original = "${pkgs.borgbackup}/bin/borg";
+    name = "borg-job-${name}";
+    set = { BORG_REPO = cfg.repo; } // (mkPassEnv cfg) // cfg.environment;
+  };
+
   # Paths listed in ReadWritePaths must exist before service is started
   mkActivationScript = name: cfg:
     let
@@ -169,7 +187,11 @@ in {
   ###### interface
 
   options.services.borgbackup.jobs = mkOption {
-    description = "Deduplicating backups using BorgBackup.";
+    description = ''
+      Deduplicating backups using BorgBackup.
+      Adding a job will cause a borg-job-NAME wrapper to be added
+      to your system path, so that you can perform maintenance easily.
+    '';
     default = { };
     example = literalExample ''
       {
@@ -610,6 +632,6 @@ in {
 
       users = mkMerge (mapAttrsToList mkUsersConfig repos);
 
-      environment.systemPackages = with pkgs; [ borgbackup ];
+      environment.systemPackages = with pkgs; [ borgbackup ] ++ (mapAttrsToList mkBorgWrapper jobs);
     });
 }

nixos/modules/services/continuous-integration/buildkite-agent.nix → nixos/modules/services/continuous-integration/buildkite-agents.nix

@@ -3,7 +3,7 @@
 with lib;
 
 let
-  cfg = config.services.buildkite-agent;
+  cfg = config.services.buildkite-agents;
 
   mkHookOption = { name, description, example ? null }: {
     inherit name;
@@ -15,7 +15,7 @@ let
   };
   mkHookOptions = hooks: listToAttrs (map mkHookOption hooks);
 
-  hooksDir = let
+  hooksDir = cfg: let
     mkHookEntry = name: value: ''
       cat > $out/${name} <<'EOF'
       #! ${pkgs.runtimeShell}
@@ -29,12 +29,13 @@ let
     ${concatStringsSep "\n" (mapAttrsToList mkHookEntry (filterAttrs (n: v: v != null) cfg.hooks))}
   '';
 
-in
-
-{
-  options = {
-    services.buildkite-agent = {
-      enable = mkEnableOption "buildkite-agent";
+  buildkiteOptions = { name ? "", config, ... }:
+  { options = {
+      enable = mkOption {
+        default = true;
+        type = types.bool;
+        description = "Whether to enable this buildkite agent";
+      };
 
       package = mkOption {
         default = pkgs.buildkite-agent;
@@ -43,10 +44,20 @@ in
         type = types.package;
       };
 
-      dataDir = mkOption {
-        default = "/var/lib/buildkite-agent";
-        description = "The workdir for the agent";
-        type = types.str;
+      userName = mkOption {
+        readOnly = true;
+        default = "buildkite-agent-${name}";
+        description = ''
+          Username of the systemd service this will run as.
+        '';
+      };
+
+      statePath = mkOption {
+        readOnly = true;
+        default = "/var/lib/buildkite-agent-${name}";
+        description = ''
+          Absolute path to the buildkite-agent's state directory
+        '';
       };
 
       runtimePackages = mkOption {
@@ -68,19 +79,18 @@ in
 
       name = mkOption {
         type = types.str;
-        default = "%hostname-%n";
+        default = "%hostname-${name}-%n";
         description = ''
-          The name of the agent.
+          The name of the agent as seen in the buildkite dashboard.
         '';
       };
 
-      meta-data = mkOption {
-        type = types.str;
-        default = "";
-        example = "queue=default,docker=true,ruby2=true";
+      tags = mkOption {
+        type = (types.attrsOf types.str);
+        default = {};
+        example = { queue = "default"; docker = "true"; ruby2 = "true"; };
         description = ''
-          Meta data for the agent. This is a comma-separated list of
-          <code>key=value</code> pairs.
+          Meta data for the agent.
         '';
       };
 
@@ -93,26 +103,29 @@ in
         '';
       };
 
-      openssh =
-        { privateKeyPath = mkOption {
-            type = types.path;
-            description = ''
-              Private agent key.
+      extraSetup = mkOption {
+        type = types.lines;
+        default = "";
+        example = "touch $HOME/test";
+        description = ''
+          Extra commands to while setting up the buildkite dir and config.
+          The directory ownership will be fixed up afterwards.
+        '';
+      };
 
-              A run-time path to the key file, which is supposed to be provisioned
-              outside of Nix store.
-            '';
-          };
-          publicKeyPath = mkOption {
-            type = types.path;
-            description = ''
-              Public agent key.
-
-              A run-time path to the key file, which is supposed to be provisioned
-              outside of Nix store.
-            '';
-          };
-        };
+      sshKeyPath = mkOption {
+        type = types.nullOr types.path;
+        ## NB: maximum care is taken so that secrets (ssh keys and the CI token)
+        ## don't end up in the Nix store.
+        apply = final: if final == null then null else toString final;
+        default = null;
+        description = ''
+          Private agent SSH key.
+
+          A runtime path to the key file, which is supposed to be provisioned
+          outside of Nix store.
+        '';
+      };
 
       hooks = mkHookOptions [
         { name = "checkout";
@@ -173,80 +186,107 @@ in
 
       hooksPath = mkOption {
         type = types.path;
-        default = hooksDir;
-        defaultText = "generated from services.buildkite-agent.hooks";
+        default = hooksDir config;
+        defaultText = "generated from services.buildkite-agents.<name>.hooks";
         description = ''
           Path to the directory storing the hooks.
-          Consider using <option>services.buildkite-agent.hooks.&lt;name&gt;</option>
+          Consider using <option>services.buildkite-agents.&lt;name&gt;.hooks.&lt;name&gt;</option>
           instead.
         '';
       };
-    };
+
+      shell = mkOption {
+        type = types.str;
+        default = "${pkgs.bash}/bin/bash -e -c";
+        description = ''
+          Command that buildkite-agent 3 will execute when it spawns a shell.
+        '';
+      };
   };
+};
 
-  config = mkIf config.services.buildkite-agent.enable {
-    users.users.buildkite-agent =
-      { name = "buildkite-agent";
-        home = cfg.dataDir;
+  enabledAgents = lib.filterAttrs (n: v: v.enable) cfg;
+  mapAgents = function: lib.mkMerge (lib.mapAttrsToList function enabledAgents);
+in {
+  imports = [
+    (mkRemovedOptionModule [ "services" "buildkite-agent"] "services.buildkite-agent has been moved to an attribute set at services.buildkite-agents")
+  ];
+
+  options.services.buildkite-agents = mkOption {
+    type = types.attrsOf (types.submodule buildkiteOptions);
+    default = {};
+    description = ''
+      Attribute set of buildkite agents.
+
+      The attribute key is combined with the hostname and a unique integer to
+      create the final agent name. This can be overridden by setting the `name`
+      attribute.
+    '';
+  };
+
+  config.users.users = mapAgents (name: cfg: {
+    "${cfg.userName}" =
+      { home = cfg.statePath;
         createHome = true;
         description = "Buildkite agent user";
         extraGroups = [ "keys" ];
       };
+    });
 
-    environment.systemPackages = [ cfg.package ];
-
-    systemd.services.buildkite-agent =
+  config.systemd.services = mapAgents (name: cfg: {
+    "buildkite-${name}" =
       { description = "Buildkite Agent";
         wantedBy = [ "multi-user.target" ];
         after = [ "network.target" ];
-        path = cfg.runtimePackages ++ [ pkgs.coreutils ];
+        path = cfg.runtimePackages ++ [ cfg.package pkgs.coreutils ];
         environment = config.networking.proxy.envVars // {
-          HOME = cfg.dataDir;
+          HOME = cfg.statePath;
           NIX_REMOTE = "daemon";
+          BUILDKITE_SHELL = cfg.shell;
         };
 
-        ## NB: maximum care is taken so that secrets (ssh keys and the CI token)
-        ##     don't end up in the Nix store.
-        preStart = let
-          sshDir = "${cfg.dataDir}/.ssh";
-        in
-          ''
-            mkdir -m 0700 -p "${sshDir}"
-            cp -f "${toString cfg.openssh.privateKeyPath}" "${sshDir}/id_rsa"
-            cp -f "${toString cfg.openssh.publicKeyPath}"  "${sshDir}/id_rsa.pub"
-            chmod 600 "${sshDir}"/id_rsa*
-
-            cat > "${cfg.dataDir}/buildkite-agent.cfg" <<EOF
+
+        serviceConfig = let
+          sshDir = "${cfg.statePath}/.ssh";
+          tagStr = lib.concatStringsSep "," (lib.mapAttrsToList (k: v: "${k}=${v}") cfg.tags);
+          preStart = ''
+            ${optionalString (cfg.sshKeyPath != null) ''
+              mkdir -p "${sshDir}"
+              chmod 700 "${sshDir}"
+              cp -f "${cfg.sshKeyPath}" "${sshDir}/id_rsa"
+              chmod 600 "${sshDir}/id_rsa"
+            ''}
+
+            cat > "${cfg.statePath}/buildkite-agent.cfg" <<EOF
             token="$(cat ${toString cfg.tokenPath})"
             name="${cfg.name}"
-            meta-data="${cfg.meta-data}"
-            build-path="${cfg.dataDir}/builds"
+            tags="${tagStr}"
+            build-path="${cfg.statePath}/builds"
             hooks-path="${cfg.hooksPath}"
             ${cfg.extraConfig}
             EOF
-          '';
-
-        serviceConfig =
-          { ExecStart = "${pkgs.buildkite-agent}/bin/buildkite-agent start --config /var/lib/buildkite-agent/buildkite-agent.cfg";
-            User = "buildkite-agent";
+            ${cfg.extraSetup}
+            chown -R $USER $HOME
+          ''; in
+          { ExecStart = "${cfg.package}/bin/buildkite-agent start --config ${cfg.statePath}/buildkite-agent.cfg";
+            ExecStartPre = "+${pkgs.writeShellScript "bk-agent-prestart" preStart}";
+            User = cfg.userName;
             RestartSec = 5;
             Restart = "on-failure";
             TimeoutSec = 10;
+            # set a long timeout to give buildkite-agent a chance to finish current builds
+            TimeoutStopSec = "2 min";
+            KillMode = "mixed";
+            StateDirectory = cfg.userName;
           };
       };
-
-    assertions = [
-      { assertion = cfg.hooksPath == hooksDir || all (v: v == null) (attrValues cfg.hooks);
+  });
+  config.assertions = mapAgents (name: cfg: [
+      { assertion = cfg.hooksPath == hooksDir cfg || all isNull (attrValues cfg.hooks);
         message = ''
-          Options `services.buildkite-agent.hooksPath' and
-          `services.buildkite-agent.hooks.<name>' are mutually exclusive.
+          Options `services.buildkite-agents.<name>.hooksPath' and
+          `services.buildkite-agents.<name>.hooks.<name>' are mutually exclusive.
         '';
       }
-    ];
-  };
-  imports = [
-    (mkRenamedOptionModule [ "services" "buildkite-agent" "token" ]                [ "services" "buildkite-agent" "tokenPath" ])
-    (mkRenamedOptionModule [ "services" "buildkite-agent" "openssh" "privateKey" ] [ "services" "buildkite-agent" "openssh" "privateKeyPath" ])
-    (mkRenamedOptionModule [ "services" "buildkite-agent" "openssh" "publicKey" ]  [ "services" "buildkite-agent" "openssh" "publicKeyPath" ])
-  ];
+  ]);
 }

nixos/modules/services/monitoring/prometheus/alertmanager.nix

@@ -18,7 +18,7 @@ let
     in checkedConfig yml;
 
   cmdlineArgs = cfg.extraFlags ++ [
-    "--config.file ${alertmanagerYml}"
+    "--config.file /tmp/alert-manager-substituted.yaml"
     "--web.listen-address ${cfg.listenAddress}:${toString cfg.port}"
     "--log.level ${cfg.logLevel}"
     ] ++ (optional (cfg.webExternalUrl != null)
@@ -118,6 +118,16 @@ in {
           Extra commandline options when launching the Alertmanager.
         '';
       };
+
+      environmentFile = mkOption {
+        type = types.nullOr types.path;
+        default = null;
+        example = "/root/alertmanager.env";
+        description = ''
+          File to load as environment file. Useful to insert secrets
+          into the configuration (via substituteAll).
+        '';
+      };
     };
   };
 
@@ -135,9 +145,14 @@ in {
       systemd.services.alertmanager = {
         wantedBy = [ "multi-user.target" ];
         after    = [ "network.target" ];
+        preStart = ''
+          (source ${(pkgs.substituteAll {}).substitute-lib}
+           substituteAll "${alertmanagerYml}" /tmp/alert-manager-substituted.yaml)
+        '';
         serviceConfig = {
           Restart  = "always";
           DynamicUser = true;
+          EnvironmentFile = lib.mkIf (cfg.environmentFile != null) cfg.environmentFile;
           WorkingDirectory = "/tmp";
           ExecStart = "${cfg.package}/bin/alertmanager" +
             optionalString (length cmdlineArgs != 0) (" \\\n  " +