diff --git a/nixos/modules/module-list.nix b/nixos/modules/module-list.nix
index fb6bc8e1efe6c..2809ded7fad27 100644
--- a/nixos/modules/module-list.nix
+++ b/nixos/modules/module-list.nix
@@ -247,7 +247,7 @@
./services/computing/slurm/slurm.nix
./services/continuous-integration/buildbot/master.nix
./services/continuous-integration/buildbot/worker.nix
- ./services/continuous-integration/buildkite-agent.nix
+ ./services/continuous-integration/buildkite-agents.nix
./services/continuous-integration/hail.nix
./services/continuous-integration/hydra/default.nix
./services/continuous-integration/gitlab-runner.nix
diff --git a/nixos/modules/services/backup/borgbackup.nix b/nixos/modules/services/backup/borgbackup.nix
index 2ad116a7872ad..9b279319c6e52 100644
--- a/nixos/modules/services/backup/borgbackup.nix
+++ b/nixos/modules/services/backup/borgbackup.nix
@@ -98,6 +98,24 @@ let
inherit (cfg) startAt;
};
+ # utility function around makeWrapper
+ mkWrapperDrv = {
+ original, name, set ? {}, setDefault ? {}
+ }:
+ pkgs.runCommandNoCC "${name}-wrapper" {
+ buildInputs = [ pkgs.makeWrapper ];
+ } (with lib; ''
+ makeWrapper "${original}" "$out/bin/${name}" \
+ ${concatStringsSep " \\\n " (mapAttrsToList (name: value: ''--set ${name} "${value}"'') set)} \
+ ${concatStringsSep " \\\n " (mapAttrsToList (name: value: ''--set-default ${name} "${value}"'') setDefault)}
+ '');
+
+ mkBorgWrapper = name: cfg: mkWrapperDrv {
+ original = "${pkgs.borgbackup}/bin/borg";
+ name = "borg-job-${name}";
+ set = { BORG_REPO = cfg.repo; } // (mkPassEnv cfg) // cfg.environment;
+ };
+
# Paths listed in ReadWritePaths must exist before service is started
mkActivationScript = name: cfg:
let
@@ -169,7 +187,11 @@ in {
###### interface
options.services.borgbackup.jobs = mkOption {
- description = "Deduplicating backups using BorgBackup.";
+ description = ''
+ Deduplicating backups using BorgBackup.
+ Adding a job will cause a borg-job-NAME wrapper to be added
+ to your system path, so that you can perform maintenance easily.
+ '';
default = { };
example = literalExample ''
{
@@ -610,6 +632,6 @@ in {
users = mkMerge (mapAttrsToList mkUsersConfig repos);
- environment.systemPackages = with pkgs; [ borgbackup ];
+ environment.systemPackages = with pkgs; [ borgbackup ] ++ (mapAttrsToList mkBorgWrapper jobs);
});
}
diff --git a/nixos/modules/services/continuous-integration/buildkite-agent.nix b/nixos/modules/services/continuous-integration/buildkite-agents.nix
similarity index 53%
rename from nixos/modules/services/continuous-integration/buildkite-agent.nix
rename to nixos/modules/services/continuous-integration/buildkite-agents.nix
index 12cc3d2b1ccce..91b372d4efa3d 100644
--- a/nixos/modules/services/continuous-integration/buildkite-agent.nix
+++ b/nixos/modules/services/continuous-integration/buildkite-agents.nix
@@ -3,7 +3,7 @@
with lib;
let
- cfg = config.services.buildkite-agent;
+ cfg = config.services.buildkite-agents;
mkHookOption = { name, description, example ? null }: {
inherit name;
@@ -15,7 +15,7 @@ let
};
mkHookOptions = hooks: listToAttrs (map mkHookOption hooks);
- hooksDir = let
+ hooksDir = cfg: let
mkHookEntry = name: value: ''
cat > $out/${name} <<'EOF'
#! ${pkgs.runtimeShell}
@@ -29,12 +29,13 @@ let
${concatStringsSep "\n" (mapAttrsToList mkHookEntry (filterAttrs (n: v: v != null) cfg.hooks))}
'';
-in
-
-{
- options = {
- services.buildkite-agent = {
- enable = mkEnableOption "buildkite-agent";
+ buildkiteOptions = { name ? "", config, ... }:
+ { options = {
+ enable = mkOption {
+ default = true;
+ type = types.bool;
+ description = "Whether to enable this buildkite agent";
+ };
package = mkOption {
default = pkgs.buildkite-agent;
@@ -43,10 +44,20 @@ in
type = types.package;
};
- dataDir = mkOption {
- default = "/var/lib/buildkite-agent";
- description = "The workdir for the agent";
- type = types.str;
+ userName = mkOption {
+ readOnly = true;
+ default = "buildkite-agent-${name}";
+ description = ''
+ Username of the systemd service this will run as.
+ '';
+ };
+
+ statePath = mkOption {
+ readOnly = true;
+ default = "/var/lib/buildkite-agent-${name}";
+ description = ''
+ Absolute path to the buildkite-agent's state directory
+ '';
};
runtimePackages = mkOption {
@@ -68,19 +79,18 @@ in
name = mkOption {
type = types.str;
- default = "%hostname-%n";
+ default = "%hostname-${name}-%n";
description = ''
- The name of the agent.
+ The name of the agent as seen in the buildkite dashboard.
'';
};
- meta-data = mkOption {
- type = types.str;
- default = "";
- example = "queue=default,docker=true,ruby2=true";
+ tags = mkOption {
+ type = (types.attrsOf types.str);
+ default = {};
+ example = { queue = "default"; docker = "true"; ruby2 = "true"; };
description = ''
- Meta data for the agent. This is a comma-separated list of
- key=value pairs.
+ Meta data for the agent.
'';
};
@@ -93,26 +103,29 @@ in
'';
};
- openssh =
- { privateKeyPath = mkOption {
- type = types.path;
- description = ''
- Private agent key.
+ extraSetup = mkOption {
+ type = types.lines;
+ default = "";
+ example = "touch $HOME/test";
+ description = ''
+ Extra commands to while setting up the buildkite dir and config.
+ The directory ownership will be fixed up afterwards.
+ '';
+ };
- A run-time path to the key file, which is supposed to be provisioned
- outside of Nix store.
- '';
- };
- publicKeyPath = mkOption {
- type = types.path;
- description = ''
- Public agent key.
-
- A run-time path to the key file, which is supposed to be provisioned
- outside of Nix store.
- '';
- };
- };
+ sshKeyPath = mkOption {
+ type = types.nullOr types.path;
+ ## NB: maximum care is taken so that secrets (ssh keys and the CI token)
+ ## don't end up in the Nix store.
+ apply = final: if final == null then null else toString final;
+ default = null;
+ description = ''
+ Private agent SSH key.
+
+ A runtime path to the key file, which is supposed to be provisioned
+ outside of Nix store.
+ '';
+ };
hooks = mkHookOptions [
{ name = "checkout";
@@ -173,80 +186,107 @@ in
hooksPath = mkOption {
type = types.path;
- default = hooksDir;
- defaultText = "generated from services.buildkite-agent.hooks";
+ default = hooksDir config;
+ defaultText = "generated from services.buildkite-agents..hooks";
description = ''
Path to the directory storing the hooks.
- Consider using
+ Consider using
instead.
'';
};
- };
+
+ shell = mkOption {
+ type = types.str;
+ default = "${pkgs.bash}/bin/bash -e -c";
+ description = ''
+ Command that buildkite-agent 3 will execute when it spawns a shell.
+ '';
+ };
};
+};
- config = mkIf config.services.buildkite-agent.enable {
- users.users.buildkite-agent =
- { name = "buildkite-agent";
- home = cfg.dataDir;
+ enabledAgents = lib.filterAttrs (n: v: v.enable) cfg;
+ mapAgents = function: lib.mkMerge (lib.mapAttrsToList function enabledAgents);
+in {
+ imports = [
+ (mkRemovedOptionModule [ "services" "buildkite-agent"] "services.buildkite-agent has been moved to an attribute set at services.buildkite-agents")
+ ];
+
+ options.services.buildkite-agents = mkOption {
+ type = types.attrsOf (types.submodule buildkiteOptions);
+ default = {};
+ description = ''
+ Attribute set of buildkite agents.
+
+ The attribute key is combined with the hostname and a unique integer to
+ create the final agent name. This can be overridden by setting the `name`
+ attribute.
+ '';
+ };
+
+ config.users.users = mapAgents (name: cfg: {
+ "${cfg.userName}" =
+ { home = cfg.statePath;
createHome = true;
description = "Buildkite agent user";
extraGroups = [ "keys" ];
};
+ });
- environment.systemPackages = [ cfg.package ];
-
- systemd.services.buildkite-agent =
+ config.systemd.services = mapAgents (name: cfg: {
+ "buildkite-${name}" =
{ description = "Buildkite Agent";
wantedBy = [ "multi-user.target" ];
after = [ "network.target" ];
- path = cfg.runtimePackages ++ [ pkgs.coreutils ];
+ path = cfg.runtimePackages ++ [ cfg.package pkgs.coreutils ];
environment = config.networking.proxy.envVars // {
- HOME = cfg.dataDir;
+ HOME = cfg.statePath;
NIX_REMOTE = "daemon";
+ BUILDKITE_SHELL = cfg.shell;
};
- ## NB: maximum care is taken so that secrets (ssh keys and the CI token)
- ## don't end up in the Nix store.
- preStart = let
- sshDir = "${cfg.dataDir}/.ssh";
- in
- ''
- mkdir -m 0700 -p "${sshDir}"
- cp -f "${toString cfg.openssh.privateKeyPath}" "${sshDir}/id_rsa"
- cp -f "${toString cfg.openssh.publicKeyPath}" "${sshDir}/id_rsa.pub"
- chmod 600 "${sshDir}"/id_rsa*
-
- cat > "${cfg.dataDir}/buildkite-agent.cfg" < "${cfg.statePath}/buildkite-agent.cfg" <' are mutually exclusive.
+ Options `services.buildkite-agents..hooksPath' and
+ `services.buildkite-agents..hooks.' are mutually exclusive.
'';
}
- ];
- };
- imports = [
- (mkRenamedOptionModule [ "services" "buildkite-agent" "token" ] [ "services" "buildkite-agent" "tokenPath" ])
- (mkRenamedOptionModule [ "services" "buildkite-agent" "openssh" "privateKey" ] [ "services" "buildkite-agent" "openssh" "privateKeyPath" ])
- (mkRenamedOptionModule [ "services" "buildkite-agent" "openssh" "publicKey" ] [ "services" "buildkite-agent" "openssh" "publicKeyPath" ])
- ];
+ ]);
}
diff --git a/nixos/modules/services/monitoring/prometheus/alertmanager.nix b/nixos/modules/services/monitoring/prometheus/alertmanager.nix
index 11d85e9c4fc3a..1e935ac5b4316 100644
--- a/nixos/modules/services/monitoring/prometheus/alertmanager.nix
+++ b/nixos/modules/services/monitoring/prometheus/alertmanager.nix
@@ -18,7 +18,7 @@ let
in checkedConfig yml;
cmdlineArgs = cfg.extraFlags ++ [
- "--config.file ${alertmanagerYml}"
+ "--config.file /tmp/alert-manager-substituted.yaml"
"--web.listen-address ${cfg.listenAddress}:${toString cfg.port}"
"--log.level ${cfg.logLevel}"
] ++ (optional (cfg.webExternalUrl != null)
@@ -118,6 +118,16 @@ in {
Extra commandline options when launching the Alertmanager.
'';
};
+
+ environmentFile = mkOption {
+ type = types.nullOr types.path;
+ default = null;
+ example = "/root/alertmanager.env";
+ description = ''
+ File to load as environment file. Useful to insert secrets
+ into the configuration (via substituteAll).
+ '';
+ };
};
};
@@ -135,9 +145,14 @@ in {
systemd.services.alertmanager = {
wantedBy = [ "multi-user.target" ];
after = [ "network.target" ];
+ preStart = ''
+ (source ${(pkgs.substituteAll {}).substitute-lib}
+ substituteAll "${alertmanagerYml}" /tmp/alert-manager-substituted.yaml)
+ '';
serviceConfig = {
Restart = "always";
DynamicUser = true;
+ EnvironmentFile = lib.mkIf (cfg.environmentFile != null) cfg.environmentFile;
WorkingDirectory = "/tmp";
ExecStart = "${cfg.package}/bin/alertmanager" +
optionalString (length cmdlineArgs != 0) (" \\\n " +
diff --git a/nixos/modules/services/web-servers/nginx/vhost-options.nix b/nixos/modules/services/web-servers/nginx/vhost-options.nix
index 15b933c984a6d..3446650abee54 100644
--- a/nixos/modules/services/web-servers/nginx/vhost-options.nix
+++ b/nixos/modules/services/web-servers/nginx/vhost-options.nix
@@ -119,13 +119,15 @@ with lib;
};
sslCertificate = mkOption {
- type = types.path;
+ type = types.nullOr types.path;
+ default = null;
example = "/var/host.cert";
description = "Path to server SSL certificate.";
};
sslCertificateKey = mkOption {
- type = types.path;
+ type = types.nullOr types.path;
+ default = null;
example = "/var/host.key";
description = "Path to server SSL certificate key.";
};
diff --git a/nixos/modules/virtualisation/docker-containers.nix b/nixos/modules/virtualisation/docker-containers.nix
index 59b0943f591f1..1e7616e92bc83 100644
--- a/nixos/modules/virtualisation/docker-containers.nix
+++ b/nixos/modules/virtualisation/docker-containers.nix
@@ -10,11 +10,21 @@ let
options = {
image = mkOption {
- type = types.str;
+ type = with types; str;
description = "Docker image to run.";
example = "library/hello-world";
};
+ imageFile = mkOption {
+ type = with types; nullOr package;
+ default = null;
+ description = ''
+ Path to an image file to load instead of pulling from a registry.
+ If defined, do not pull from registry.
+ '';
+ example = literalExample "pkgs.dockerTools.buildDockerImage {...};";
+ };
+
cmd = mkOption {
type = with types; listOf str;
default = [];
@@ -153,6 +163,13 @@ let
example = "/var/lib/hello_world";
};
+ containerDependencies = mkOption {
+ type = with types; listOf str;
+ default = [];
+ description = ''
+ '';
+ };
+
extraDockerOptions = mkOption {
type = with types; listOf str;
default = [];
@@ -164,15 +181,18 @@ let
};
};
- mkService = name: container: {
+ mkService = name: container: let
+ mkAfter = map (x: "docker-${x}.service") container.containerDependencies;
+ in {
wantedBy = [ "multi-user.target" ];
- after = [ "docker.service" "docker.socket" ];
- requires = [ "docker.service" "docker.socket" ];
+ after = [ "docker.service" "docker.socket" ] ++ mkAfter;
+ requires = [ "docker.service" "docker.socket" ] ++ mkAfter;
+
serviceConfig = {
ExecStart = concatStringsSep " \\\n " ([
"${pkgs.docker}/bin/docker run"
"--rm"
- "--name=%n"
+ "--name=${name}"
"--log-driver=${container.log-driver}"
] ++ optional (container.entrypoint != null)
"--entrypoint=${escapeShellArg container.entrypoint}"
@@ -185,9 +205,15 @@ let
++ [container.image]
++ map escapeShellArg container.cmd
);
- ExecStartPre = "-${pkgs.docker}/bin/docker rm -f %n";
- ExecStop = "${pkgs.docker}/bin/docker stop %n";
- ExecStopPost = "-${pkgs.docker}/bin/docker rm -f %n";
+
+ ExecStartPre = ["-${pkgs.docker}/bin/docker rm -f ${name}"
+ "-${pkgs.docker}/bin/docker image prune -f"] ++
+ (if (container.imageFile != null)
+ then ["${pkgs.docker}/bin/docker load -i ${container.imageFile}"]
+ else ["${pkgs.docker}/bin/docker pull ${container.image}"]);
+
+ ExecStop = "${pkgs.docker}/bin/docker stop ${name}";
+ ExecStopPost = "-${pkgs.docker}/bin/docker rm -f ${name}";
### There is no generalized way of supporting `reload` for docker
### containers. Some containers may respond well to SIGHUP sent to their
diff --git a/nixos/modules/virtualisation/docker.nix b/nixos/modules/virtualisation/docker.nix
index 7d196a46276ac..5168ea501112f 100644
--- a/nixos/modules/virtualisation/docker.nix
+++ b/nixos/modules/virtualisation/docker.nix
@@ -9,6 +9,67 @@ let
cfg = config.virtualisation.docker;
proxy_env = config.networking.proxy.envVars;
+ inherit (builtins) attrNames;
+
+ mkUncreateMaybe = networks: volumes: ''
+ set -euo pipefail
+
+ nexisting=$(${pkgs.coreutils}/bin/mktemp)
+ nwanted=$(${pkgs.coreutils}/bin/mktemp)
+ vexisting=$(${pkgs.coreutils}/bin/mktemp)
+ vwanted=$(${pkgs.coreutils}/bin/mktemp)
+
+ cleanup() {
+ rm -f "$nexisting" "$nwanted" "$vexisting" "$vwanted"
+ }
+ trap cleanup EXIT
+
+ ${pkgs.docker}/bin/docker network ls --format '{{.Name}}' > "$nexisting"
+ echo -e "bridge\nhost\nnone\n${concatStringsSep "\n" networks}" > "$nwanted"
+
+ ${pkgs.docker}/bin/docker volume ls --format '{{.Name}}' > "$vexisting"
+ echo -e "${concatStringsSep "\n" volumes}" > "$vwanted"
+
+ nsuperfluous="$(${pkgs.gnugrep}/bin/grep -vxF -f $nwanted $nexisting || true)"
+ vsuperfluous="$(${pkgs.gnugrep}/bin/grep -vxF -f $vwanted $vexisting || true)"
+
+ while read -r net; do
+ if [[ ! -z "$net" ]]; then
+ echo -n "Removed superfluous Docker network: "
+ ${pkgs.docker}/bin/docker network rm "$net" || true
+ fi
+ done <<< "$nsuperfluous"
+
+ while read -r vol; do
+ if [[ ! -z "$vol" ]]; then
+ echo -n "Removed superfluous Docker volume: "
+ ${pkgs.docker}/bin/docker volume rm "$vol" || true
+ fi
+ done <<< "$vsuperfluous"
+ '';
+
+ mkNetworkOpts = opts: concatStringsSep " "
+ ([ "--driver=${opts.driver}" ]
+ ++ optional (cfg ? subnet && cfg.subnet != null) "--subnet=${opts.subnet}"
+ ++ optional (cfg ? ip-range && cfg.ip-range != null) "--ip-range=${opts.ip-range}"
+ ++ optional (cfg ? gateway && cfg.gateway != null) "--gateway=${opts.gateway}"
+ ++ optional (cfg ? ipv6 && cfg.ipv6) "--ipv6"
+ ++ optional (cfg ? internal && cfg.internal) "--internal");
+
+
+ mkNetwork = name: opts: ''
+ if [[ $(${pkgs.docker}/bin/docker network ls --quiet --filter name=${name} | wc -c) -eq 0 ]]; then
+ echo "*** docker network create ${mkNetworkOpts opts} ${name}"
+ ${pkgs.docker}/bin/docker network create ${mkNetworkOpts opts} ${name}
+ fi
+ '';
+
+ mkVolume = name: ''
+ if [[ $(${pkgs.docker}/bin/docker volume ls --quiet --filter name=${name} | wc -c) -eq 0 ]]; then
+ echo "*** docker volume create ${name}"
+ ${pkgs.docker}/bin/docker volume create ${name}
+ fi
+ '';
in
{
@@ -93,6 +154,16 @@ in
'';
};
+ logLevel =
+ mkOption {
+ type = types.enum ["debug" "info" "warn" "error" "fatal"];
+ default = "info";
+ description =
+ ''
+ This option determines the log level for the Docker daemon.
+ '';
+ };
+
extraOptions =
mkOption {
type = types.separatedString " ";
@@ -144,6 +215,90 @@ in
Docker package to be used in the module.
'';
};
+
+ volumes = mkOption {
+ default = [];
+ type = types.listOf types.str;
+ example = [ "volume_1" "volume_2" ];
+ description = ''
+ A list of named volumes that should be created.
+ '';
+ };
+
+
+ networks = mkOption {
+ default = {};
+ type = types.attrsOf (types.submodule {
+ options = {
+ driver = mkOption {
+ default = "bridge";
+ type = types.str;
+ example = "overlay";
+ description = ''
+ Driver to manage the network. One of bridge, or overlay.
+ '';
+ };
+
+ subnet = mkOption {
+ default = null;
+ type = types.nullOr types.str;
+ example = "172.28.0.0/16";
+ description = ''
+ Subnet in CIDR format that represents a network segment.
+ '';
+ };
+
+ ip-range = mkOption {
+ default = null;
+ type = types.nullOr types.str;
+ example = "172.28.5.0/24";
+ description = ''
+ Allocate container ip from a sub-range.
+ '';
+ };
+
+ gateway = mkOption {
+ default = null;
+ type = types.nullOr types.str;
+ example = "172.28.5.254";
+ description = ''
+ IPv4 or IPv6 Gateway for the master subnet.
+ '';
+ };
+
+ ipv6 = mkOption {
+ default = false;
+ type = types.bool;
+ example = true;
+ description = ''
+ Enable IPv6 networking.
+ '';
+ };
+
+ internal = mkOption {
+ default = false;
+ type = types.bool;
+ example = true;
+ description = ''
+ Restrict external access to the network.
+ '';
+ };
+ };
+ });
+
+ example = {
+ my-network = {
+ driver = "bridge";
+ subnet = "172.28.0.0/16";
+ ip-range = "172.28.5.0/24";
+ gateway = "172.28.5.254";
+ };
+ };
+
+ description = ''
+ A list of named networks to be created.
+ '';
+ };
};
###### implementation
@@ -157,6 +312,11 @@ in
systemd.services.docker = {
wantedBy = optional cfg.enableOnBoot "multi-user.target";
environment = proxy_env;
+
+ postStart = mkUncreateMaybe (attrNames cfg.networks) cfg.volumes
+ + concatStrings (mapAttrsToList mkNetwork cfg.networks)
+ + concatStrings (map mkVolume cfg.volumes);
+
serviceConfig = {
ExecStart = [
""
@@ -165,11 +325,13 @@ in
--group=docker \
--host=fd:// \
--log-driver=${cfg.logDriver} \
+ --log-level=${cfg.logLevel} \
${optionalString (cfg.storageDriver != null) "--storage-driver=${cfg.storageDriver}"} \
${optionalString cfg.liveRestore "--live-restore" } \
${optionalString cfg.enableNvidia "--add-runtime nvidia=${pkgs.nvidia-docker}/bin/nvidia-container-runtime" } \
${cfg.extraOptions}
''];
+
ExecReload=[
""
"${pkgs.procps}/bin/kill -s HUP $MAINPID"
diff --git a/nixos/tests/all-tests.nix b/nixos/tests/all-tests.nix
index 5643da99e5570..433086c4712f2 100644
--- a/nixos/tests/all-tests.nix
+++ b/nixos/tests/all-tests.nix
@@ -35,6 +35,7 @@ in
boot-stage1 = handleTest ./boot-stage1.nix {};
borgbackup = handleTest ./borgbackup.nix {};
buildbot = handleTest ./buildbot.nix {};
+ buildkite-agents = handleTest ./buildkite-agents.nix {};
cadvisor = handleTestOn ["x86_64-linux"] ./cadvisor.nix {};
cassandra = handleTest ./cassandra.nix {};
ceph = handleTestOn ["x86_64-linux"] ./ceph.nix {};
diff --git a/nixos/tests/buildkite-agents.nix b/nixos/tests/buildkite-agents.nix
new file mode 100644
index 0000000000000..83ed29cb68de3
--- /dev/null
+++ b/nixos/tests/buildkite-agents.nix
@@ -0,0 +1,28 @@
+import ./make-test.nix ({ lib, ... } : {
+ name = "buildkite-agents";
+ meta = with lib.maintainers; {
+ maintainers = [ earvstedt ];
+ };
+
+ machine = { pkgs, ... }: {
+ services.buildkite-agents = {
+ foo = {
+ extraConfig = "debug=true";
+ hooks.environment = "export SECRET_VAR=`head -1 /run/keys/secret`";
+ tokenPath = (pkgs.writeText "my-token" "1234");
+ };
+ bar = {
+ sshKeyPath = (import ./ssh-keys.nix pkgs).snakeOilPrivateKey;
+ tokenPath = (pkgs.writeText "my-token" "5678");
+ };
+ };
+ };
+
+ testScript = ''
+ # we can't wait on the unit to start up, as we obviously can't connect to buildkite,
+ # but we can look whether files are set up correctly
+ $machine->waitForFile("/var/lib/buildkite-foo/buildkite-agent.cfg");
+ $machine->waitForFile("/var/lib/buildkite-bar/buildkite-agent.cfg");
+ $machine->waitForFile("/var/lib/buildkite-bar/.ssh/id_rsa");
+ '';
+})
diff --git a/nixos/tests/docker.nix b/nixos/tests/docker.nix
index d67b2f8743d80..7932e4d177dcf 100644
--- a/nixos/tests/docker.nix
+++ b/nixos/tests/docker.nix
@@ -10,8 +10,19 @@ import ./make-test.nix ({ pkgs, ...} : {
docker =
{ pkgs, ... }:
{
- virtualisation.docker.enable = true;
- virtualisation.docker.package = pkgs.docker;
+ virtualisation.docker = {
+ enable = true;
+ package = pkgs.docker;
+ volumes = [ "thevolume" ];
+ networks.thenetwork = {
+ driver = "bridge";
+ subnet = "172.28.0.0/16";
+ ip-range = "172.28.5.0/24";
+ gateway = "172.28.5.254";
+ };
+
+ logLevel = "warn";
+ };
users.users = {
noprivs = {
@@ -41,6 +52,15 @@ import ./make-test.nix ({ pkgs, ...} : {
$docker->fail("sudo -u noprivs docker ps");
$docker->succeed("docker stop sleeping");
+ $docker->succeed("docker volume ls | grep thevolume");
+ $docker->succeed("docker network ls | grep thenetwork");
+
+ $docker->succeed("docker volume create superfluousvolume");
+ $docker->succeed("docker network create superfluousnetwork");
+ $docker->systemctl("restart docker");
+ $docker->waitForUnit("docker.service");
+ $docker->fail("docker volume ls | grep superfluous");
+
# Must match version twice to ensure client and server versions are correct
$docker->succeed('[ $(docker version | grep ${pkgs.docker.version} | wc -l) = "2" ]');
'';
diff --git a/pkgs/build-support/substitute/substitute-all.nix b/pkgs/build-support/substitute/substitute-all.nix
index 57b160bbe9014..2d4d394b5b022 100644
--- a/pkgs/build-support/substitute/substitute-all.nix
+++ b/pkgs/build-support/substitute/substitute-all.nix
@@ -7,6 +7,7 @@ stdenvNoCC.mkDerivation ({
name = if args ? name then args.name else baseNameOf (toString args.src);
builder = ./substitute-all.sh;
inherit (args) src;
+ passthru.substitute-lib = ./substitute-lib.sh;
preferLocalBuild = true;
allowSubstitutes = false;
} // args)
diff --git a/pkgs/build-support/substitute/substitute-lib.sh b/pkgs/build-support/substitute/substitute-lib.sh
new file mode 100644
index 0000000000000..19e82fd3ffe4e
--- /dev/null
+++ b/pkgs/build-support/substitute/substitute-lib.sh
@@ -0,0 +1,120 @@
+substituteStream() {
+ local var=$1
+ local description=$2
+ shift 2
+
+ while (( "$#" )); do
+ case "$1" in
+ --replace)
+ pattern="$2"
+ replacement="$3"
+ shift 3
+ local savedvar
+ savedvar="${!var}"
+ eval "$var"'=${'"$var"'//"$pattern"/"$replacement"}'
+ if [ "$pattern" != "$replacement" ]; then
+ if [ "${!var}" == "$savedvar" ]; then
+ echo "substituteStream(): WARNING: pattern '$pattern' doesn't match anything in $description" >&2
+ fi
+ fi
+ ;;
+
+ --subst-var)
+ local varName="$2"
+ shift 2
+ # check if the used nix attribute name is a valid bash name
+ if ! [[ "$varName" =~ ^[a-zA-Z_][a-zA-Z0-9_]*$ ]]; then
+ echo "substituteStream(): ERROR: substitution variables must be valid Bash names, \"$varName\" isn't." >&2
+ return 1
+ fi
+ if [ -z ${!varName+x} ]; then
+ echo "substituteStream(): ERROR: variable \$$varName is unset" >&2
+ return 1
+ fi
+ pattern="@$varName@"
+ replacement="${!varName}"
+ eval "$var"'=${'"$var"'//"$pattern"/"$replacement"}'
+ ;;
+
+ --subst-var-by)
+ pattern="@$2@"
+ replacement="$3"
+ eval "$var"'=${'"$var"'//"$pattern"/"$replacement"}'
+ shift 3
+ ;;
+
+ *)
+ echo "substituteStream(): ERROR: Invalid command line argument: $1" >&2
+ return 1
+ ;;
+ esac
+ done
+
+ printf "%s" "${!var}"
+}
+
+consumeEntire() {
+ # read returns non-0 on EOF, so we want read to fail
+ if IFS='' read -r -N 0 $1; then
+ echo "consumeEntire(): ERROR: Input null bytes, won't process" >&2
+ return 1
+ fi
+}
+
+substitute() {
+ local input="$1"
+ local output="$2"
+ shift 2
+
+ if [ ! -f "$input" ]; then
+ echo "substitute(): ERROR: file '$input' does not exist" >&2
+ return 1
+ fi
+
+ local content
+ consumeEntire content < "$input"
+
+ if [ -e "$output" ]; then chmod +w "$output"; fi
+ substituteStream content "file '$input'" "$@" > "$output"
+}
+
+substituteInPlace() {
+ local fileName="$1"
+ shift
+ substitute "$fileName" "$fileName" "$@"
+}
+
+_allFlags() {
+ for varName in $(awk 'BEGIN { for (v in ENVIRON) if (v ~ /^[a-z][a-zA-Z0-9_]*$/) print v }'); do
+ if (( "${NIX_DEBUG:-0}" >= 1 )); then
+ printf "@%s@ -> %q\n" "${varName}" "${!varName}"
+ fi
+ args+=("--subst-var" "$varName")
+ done
+}
+
+substituteAllStream() {
+ local -a args=()
+ _allFlags
+
+ substituteStream "$1" "$2" "${args[@]}"
+}
+
+# Substitute all environment variables that start with a lowercase character and
+# are valid Bash names.
+substituteAll() {
+ local input="$1"
+ local output="$2"
+
+ local -a args=()
+ _allFlags
+
+ substitute "$input" "$output" "${args[@]}"
+}
+
+
+substituteAllInPlace() {
+ local fileName="$1"
+ shift
+ substituteAll "$fileName" "$fileName" "$@"
+}
diff --git a/pkgs/development/tools/continuous-integration/buildkite-agent/2.x.nix b/pkgs/development/tools/continuous-integration/buildkite-agent/2.x.nix
deleted file mode 100644
index 6a73e2581822e..0000000000000
--- a/pkgs/development/tools/continuous-integration/buildkite-agent/2.x.nix
+++ /dev/null
@@ -1,12 +0,0 @@
-{ callPackage, fetchFromGitHub, ... } @ args:
-
-callPackage ./generic.nix (args // rec {
- src = fetchFromGitHub {
- owner = "buildkite";
- repo = "agent";
- rev = "v${version}";
- sha256 = "07065hhhb418w5qlqnyiap45r59paysysbwz1l7dmaw3j4q8m8rg";
- };
- version = "2.6.10";
- hasBootstrapScript = true;
-})
diff --git a/pkgs/development/tools/continuous-integration/buildkite-agent/3.x.nix b/pkgs/development/tools/continuous-integration/buildkite-agent/3.x.nix
deleted file mode 100644
index e8266c2efe2cd..0000000000000
--- a/pkgs/development/tools/continuous-integration/buildkite-agent/3.x.nix
+++ /dev/null
@@ -1,15 +0,0 @@
-{ bash, callPackage, fetchFromGitHub, ... } @ args:
-
-callPackage ./generic.nix (args // rec {
- src = fetchFromGitHub {
- owner = "buildkite";
- repo = "agent";
- rev = "v${version}";
- sha256 = "0sr1rxl92d4wdipl66f1yymx5bmyj1y85v6k22v57rzr6yhyfmsf";
- };
- version = "3.8.4";
- hasBootstrapScript = false;
- postPatch = ''
- substituteInPlace bootstrap/shell/shell.go --replace /bin/bash ${bash}/bin/bash
- '';
-})
diff --git a/pkgs/development/tools/continuous-integration/buildkite-agent/default.nix b/pkgs/development/tools/continuous-integration/buildkite-agent/default.nix
new file mode 100644
index 0000000000000..98698712a4993
--- /dev/null
+++ b/pkgs/development/tools/continuous-integration/buildkite-agent/default.nix
@@ -0,0 +1,48 @@
+{ fetchFromGitHub, stdenv, buildGoPackage,
+ makeWrapper, coreutils, git, openssh, bash, gnused, gnugrep }:
+buildGoPackage rec {
+ name = "buildkite-agent-${version}";
+ version = "3.8.4";
+
+ goPackagePath = "github.com/buildkite/agent";
+
+ src = fetchFromGitHub {
+ owner = "buildkite";
+ repo = "agent";
+ rev = "v${version}";
+ sha256 = "0sr1rxl92d4wdipl66f1yymx5bmyj1y85v6k22v57rzr6yhyfmsf";
+ };
+ postPatch = ''
+ substituteInPlace bootstrap/shell/shell.go --replace /bin/bash ${bash}/bin/bash
+ '';
+
+ nativeBuildInputs = [ makeWrapper ];
+
+ # on Linux, the TMPDIR is /build which is the same prefix as this package
+ # remove once #35068 is merged
+ noAuditTmpdir = stdenv.isLinux;
+
+ postInstall = ''
+ # Fix binary name
+ mv $bin/bin/{agent,buildkite-agent}
+
+ # These are runtime dependencies
+ wrapProgram $bin/bin/buildkite-agent \
+ --prefix PATH : '${stdenv.lib.makeBinPath [ openssh git coreutils gnused gnugrep ]}'
+ '';
+
+ meta = with stdenv.lib; {
+ description = "Build runner for buildkite.com";
+ longDescription = ''
+ The buildkite-agent is a small, reliable, and cross-platform build runner
+ that makes it easy to run automated builds on your own infrastructure.
+ It’s main responsibilities are polling buildkite.com for work, running
+ build jobs, reporting back the status code and output log of the job,
+ and uploading the job's artifacts.
+ '';
+ homepage = https://buildkite.com/docs/agent;
+ license = licenses.mit;
+ maintainers = with maintainers; [ pawelpacana zimbatm rvl ];
+ platforms = platforms.unix;
+ };
+}
diff --git a/pkgs/tools/misc/txr-copy/default.nix b/pkgs/tools/misc/txr-copy/default.nix
new file mode 100644
index 0000000000000..3a41ddd2a7874
--- /dev/null
+++ b/pkgs/tools/misc/txr-copy/default.nix
@@ -0,0 +1,41 @@
+{ stdenv, fetchurl, bison, flex, libffi }:
+
+stdenv.mkDerivation rec {
+ pname = "txr";
+ version = "225";
+
+ src = fetchurl {
+ url = "http://www.kylheku.com/cgit/txr/snapshot/${pname}-${version}.tar.bz2"; sha256 = "07vh0rmvjr2sir15l3ppp2pnp2d849dg17rzykkzqyk3d5rwfxyj";
+ };
+
+ nativeBuildInputs = [ bison flex ];
+ buildInputs = [ libffi ];
+
+ enableParallelBuilding = true;
+
+ doCheck = true;
+ checkTarget = "tests";
+
+ # Remove failing test-- mentions 'usr/bin' so probably related :)
+ preCheck = "rm -rf tests/017";
+
+ postInstall = ''
+ d=$out/share/vim-plugins/txr
+ mkdir -p $d/{syntax,ftdetect}
+
+ cp {tl,txr}.vim $d/syntax/
+
+ cat > $d/ftdetect/txr.vim <