aboutsummaryrefslogtreecommitdiffstats
path: root/pkgs/populate/tests
diff options
context:
space:
mode:
authortv <tv@krebsco.de>2026-11-14 00:33:20 +0000
committertv <tv@krebsco.de>2026-11-14 01:48:35 +0100
commitdd166e5a4d4349241aa3408c8d7f7340cb0a90fb (patch)
tree0955739060de93d99c6bc7828adc377815ad02a8 /pkgs/populate/tests
parenta4540a584d762b000b21cb7e61987c414eb72a1d (diff)
tests: populate's tests of its own, krops' using them
populate's tests are in pkgs/populate/tests, independent of krops. They run the program, with the sources as JSON from populate's Nix library, and target strings for the remote targets, leaving the user and whether the target is local to populate, so they cover its command line too. Phases expecting populate to refuse its arguments, e.g. a source name or a target path that can't be used, expect the program's refusal, with refused, the message, since nothing checks them during evaluation there. builder.nix is what builds the scenarios, which krops' tests use as well, with a function running the phases, and whether that checks the arguments during evaluation. vm.nix takes a module for both machines, for what else the scenarios run, and fixtures.nix and smoke.nix are what both use of the fixtures and the smoke test. krops' tests are deploy, writeDeploy's arguments to nixos-rebuild, with the stand-in for nixos-rebuild in the module, and populate, populate's scenarios that expect refusals, which pkgs.populate makes evaluation fail with, and those of populate's smoke test, all through pkgs.populate with target attribute sets as mkTarget returns them. -A only selects from all of populate's scenarios and krops' own.
Diffstat (limited to 'pkgs/populate/tests')
-rw-r--r--pkgs/populate/tests/builder.nix249
-rw-r--r--pkgs/populate/tests/default.nix135
-rw-r--r--pkgs/populate/tests/fixtures.nix7
-rwxr-xr-xpkgs/populate/tests/fixtures.sh498
-rw-r--r--pkgs/populate/tests/lib.sh118
-rw-r--r--pkgs/populate/tests/rsync.nix50
-rw-r--r--pkgs/populate/tests/scenarios.nix2174
-rw-r--r--pkgs/populate/tests/smoke.nix21
-rw-r--r--pkgs/populate/tests/vm.nix203
9 files changed, 3455 insertions, 0 deletions
diff --git a/pkgs/populate/tests/builder.nix b/pkgs/populate/tests/builder.nix
new file mode 100644
index 0000000..26a360b
--- /dev/null
+++ b/pkgs/populate/tests/builder.nix
@@ -0,0 +1,249 @@
+# The scenario builder of populate's tests, which krops' tests use as well.
+# A scenario prepares the target directory, then runs its phases. Each phase
+# runs populate, or something running it, expecting it to succeed, to fail
+# with a message, or to refuse its arguments, and checks the result on the
+# target. Refusals checked during evaluation get checked here and don't run.
+{
+ # nixpkgs' lib.
+ lib,
+ # evalSource of populate's Nix library.
+ evalSource,
+ # Function from a phase's arguments, { source, backup, force, prune,
+ # deploy, targetKind, targetPath }, to the command running the phase.
+ # source is what evalSource returns; deploy is null, or arguments for
+ # running populate in a deployment.
+ run,
+ # Whether the command checks populate's arguments during evaluation, so
+ # phases expecting a refusal expect evaluation to fail, instead of
+ # populate refusing when it runs.
+ checksDuringEvaluation,
+ # Where the deploying machine serves /fixtures/www via HTTP.
+ url,
+ # URL of the Git repository /fixtures/git/gitsrc.git.
+ gitUrl,
+ # Host name of the remote target.
+ remoteHost,
+ # Absolute path of age on the deploying machine.
+ agePath,
+ # HEAD of /fixtures/gitsrc, and its parent.
+ gitHead,
+ gitFirst,
+ # Version of rsync on the deploying machine.
+ rsyncVersion,
+}:
+
+let
+ inherit (lib)
+ attrNames concatMap concatStringsSep foldl' isBool mapAttrsToList
+ optional optionalAttrs optionals optionalString;
+
+ prelude = ". ${./lib.sh}\n";
+
+ cartesian = attrs: foldl'
+ (acc: name: concatMap
+ (combination: map
+ (value: combination // { ${name} = value; })
+ attrs.${name})
+ acc)
+ [{}]
+ (attrNames attrs);
+
+ flagsName = flags: concatStringsSep "-" (mapAttrsToList
+ (name: value: name + (if isBool value then (if value then "1" else "0") else value))
+ flags);
+
+ alt = { alt.file = "/fixtures/alt"; };
+
+ # A path on a target of kind targetKind, as populate names it in what it
+ # prints.
+ targetName = targetKind: path: {
+ local = "";
+ remote = "${remoteHost}:";
+ sudo = "deploy@${remoteHost}:";
+ }.${targetKind} + path;
+
+ # How a source draws inputs from other sources: not at all, by rewriting
+ # the lock file of its flake, or by a generated flake.
+ inputModes = [ "none" "rewrite" "generated" ];
+
+ # The options of a source drawing inputs in mode.
+ drawInputs = mode: inputs: optionalAttrs (mode != "none") ({
+ inherit inputs;
+ } // optionalAttrs (mode == "generated") {
+ rewriteLock = false;
+ });
+
+ # The flake of source x that has its inputs drawn in mode.
+ flakeOf = mode:
+ if mode == "generated" then ''"$T"/.populate/flakes/x'' else ''"$T"/x'';
+
+ # Builds a scenario from a name, a target kind, and phases with sources.
+ # Isolated scenarios start with no target directories and a collected store
+ # on both machines, so no other scenario's store paths or roots stand in for
+ # missing ones. An exhaustive scenario covers a combination that no code
+ # path tells apart from the ones the other scenarios cover, so it only runs
+ # in the exhaustive tests.
+ scenario = {
+ name, targetKind, node ? null, isolate ? false, exhaustive ? false,
+ prepare ? "", prepareDeployer ? null, phases,
+ }: let
+ path = "/var/krops/${name}";
+ vars = "T=${path}\n";
+ # A phase expecting evaluation to fail, if checksDuringEvaluation, or else
+ # populate to refuse with the message refused, has a control: sources that
+ # differ only in what makes evaluation fail, or a function returning them
+ # for the phase's sources, and that must evaluate. So the phase doesn't
+ # pass for failing for an unrelated reason, e.g. a misspelt option.
+ # targetPath deploys to another path than the scenario's target
+ # directory, and controlTargetPath is the control for a path that makes
+ # evaluation fail. env sets environment variables for populate, as
+ # arguments of env. interrupt ends populate after that many seconds,
+ # which it's expected not to finish before. output lists what a
+ # succeeding populate is expected to print, and unexpected what populate
+ # is expected not to print, succeeding or not. expect.fail is what a
+ # failing populate is expected to print, and expect.status, if given,
+ # its exit status.
+ phase = {
+ sources, backup ? false, prune ? false, deploy ? null,
+ targetPath ? path, env ? "", interrupt ? null, output ? [],
+ unexpected ? [],
+ expect ? "success", refused ? null, control ? null, controlTargetPath ? null,
+ check ? "", mutate ? null, checkDeployer ? null, mutateDeployer ? null,
+ }: let
+ command = sources: targetPath: run {
+ source = evalSource [ sources ];
+ inherit backup prune deploy targetKind targetPath;
+ force = true;
+ };
+ evaluates = sources: targetPath:
+ (builtins.tryEval
+ (builtins.seq (toString (command sources targetPath)) true)).success;
+ script = text: if text == null then null else prelude + vars + text;
+ # Without checks during evaluation, populate refuses when it runs.
+ expect' =
+ if expect == "eval-error" && !checksDuringEvaluation then
+ if refused == null then
+ throw "scenario ${name}: expecting a refusal needs refused, what populate says"
+ else { fail = refused; }
+ else expect;
+ in
+ if expect' == "eval-error" then
+ if control == null && controlTargetPath == null then
+ throw "scenario ${name}: expecting evaluation to fail needs a control"
+ else if evaluates sources targetPath then
+ throw "scenario ${name}: populate evaluates, but is expected to fail"
+ else if !(
+ if controlTargetPath != null then evaluates sources controlTargetPath
+ else evaluates
+ (if lib.isFunction control then control sources else control)
+ targetPath
+ ) then
+ throw "scenario ${name}: the control doesn't evaluate either"
+ else
+ null
+ else {
+ populate = toString (command sources targetPath);
+ inherit env interrupt output unexpected;
+ expect = if expect' == "success" then null else expect'.fail;
+ status = if expect' == "success" then null else expect'.status or null;
+ # No phase gets around checking the canaries.
+ check = script (check + "\ncanary_check\n");
+ mutate = script mutate;
+ checkDeployer = script checkDeployer;
+ mutateDeployer = script mutateDeployer;
+ };
+ in {
+ inherit name isolate exhaustive;
+ # Whether a phase expects a refusal, which krops' tests check during
+ # evaluation.
+ refusals = lib.any (phase: phase.expect or null == "eval-error") phases;
+ node =
+ if node != null then node
+ else if targetKind == "local" then "deployer"
+ else "target";
+ prepare = prelude + vars + ''
+ rm -fR "$T"
+ '' + prepare;
+ prepareDeployer =
+ if prepareDeployer == null then null else prelude + vars + prepareDeployer;
+ # For all machines: before the scenario, and after it for the ones its
+ # checks don't run on.
+ canarySetup = prelude + "canary_setup\n";
+ canaryCheck = prelude + "canary_check\n";
+ phases = lib.filter (x: x != null) (map phase phases);
+ };
+
+ # Same sources twice, so the second phase checks redeploying.
+ twice = phase: [ phase phase ];
+
+ targetKinds = [ "local" "remote" "sudo" ];
+
+ # Scenarios for all combinations of options and target kinds. f gets the
+ # options and the scenario's name, target kind, and target directory, and
+ # returns the scenario's arguments, or null for invalid combinations.
+ matrix = type: options: f:
+ concatMap
+ (targetKind: lib.filter (x: x != null) (map
+ (flags: let
+ name = "${type}-${targetKind}" +
+ optionalString (flags != {}) "-${flagsName flags}";
+ args = f flags { inherit name targetKind; path = "/var/krops/${name}"; };
+ in
+ if args == null then null
+ else scenario ({ inherit name targetKind; } // args))
+ (cartesian options)))
+ targetKinds;
+
+ # Checks that the source at $T/x is a link to a store path, or to the
+ # subdirectory sub therein, or a writable directory if materialized, and
+ # evaluates the attribute attr of the flake there, or of its generated
+ # flake, after collecting garbage.
+ checkFlake = {
+ attr, expected, subdir ? false, generated ? false, materialized ? false,
+ }: (if materialized then ''
+ test -d "$T"/x && ! test -L "$T"/x || fail "x isn't a directory"
+ test -w "$T"/x/flake.nix || fail "flake.nix isn't writable"
+ '' else ''
+ link=$(readlink "$T"/x)
+ ${if subdir then ''
+ expect_eq subdirectory "$(basename "$link")" sub
+ link=$(dirname "$link")
+ '' else ""}
+ expect_eq "store directory" "$(dirname "$link")" /nix/store
+ '') + ''
+ test ! -e "$T"/.populate/flake-uploads || fail "upload directory remains"
+ gc
+ expect_eq ${attr} "$(nix_eval ${
+ if generated then ''"$T"/.populate/flakes/x'' else ''"$T"/x''
+ }#${attr})" ${lib.escapeShellArg expected}
+ '';
+
+ copyingWithSudo = flags: ctx:
+ ctx.targetKind == "sudo" && !(flags.fetchOnTarget or false);
+
+ # The sources with fetchOnTarget set for all flake and flake-input sources,
+ # the control for copying flakes to targets using sudo.
+ withFetchOnTarget = lib.mapAttrs (_: lib.mapAttrs (type: options:
+ if lib.elem type [ "flake" "flake-input" ]
+ then options // { fetchOnTarget = true; }
+ else options));
+
+ # A phase copying flakes to a target using sudo, which isn't supported,
+ # and gets refused during evaluation.
+ refuseCopyingWithSudo = {
+ expect = "eval-error";
+ refused = "copying flakes to targets with sudo is not supported";
+ control = withFetchOnTarget;
+ };
+
+in {
+ inherit lib url gitUrl remoteHost agePath gitHead gitFirst rsyncVersion;
+ inherit (lib)
+ attrNames concatMap concatStringsSep foldl' isBool mapAttrsToList
+ optional optionalAttrs optionals optionalString;
+ inherit
+ prelude cartesian flagsName alt targetName
+ inputModes drawInputs flakeOf scenario twice
+ targetKinds matrix checkFlake copyingWithSudo withFetchOnTarget
+ refuseCopyingWithSudo;
+}
diff --git a/pkgs/populate/tests/default.nix b/pkgs/populate/tests/default.nix
new file mode 100644
index 0000000..7b37bb3
--- /dev/null
+++ b/pkgs/populate/tests/default.nix
@@ -0,0 +1,135 @@
+# NixOS VM tests for populate: one test per source type, one for features
+# spanning sources, one for edge cases of flakes, and one for staying inside
+# the target directory. Each runs the program, with the sources as JSON
+# from populate's Nix library, and target strings, which leave the user and
+# whether the target is local to populate to decide. Build all with
+# nix-build in this directory, or one with, e.g., nix-build -A flake. The
+# exhaustive tests add the combinations of options that no code path tells
+# apart from the ones the others cover; build them with nix-build -A
+# exhaustive, or one with, e.g., nix-build -A exhaustive.flake. The smoke
+# test runs a few scenarios covering each source type and each kind of
+# target in a single VM test, for checking changes quickly; build it with
+# nix-build -A smoke. nix-build -A only --argstr only REGEX runs the
+# scenarios whose names match the regular expression REGEX in a single VM
+# test, e.g. --argstr only 'lock-hooks-.*'.
+{ pkgs ? import <nixpkgs> {}, only ? null }:
+
+let
+ inherit (pkgs) lib;
+
+ populate-unwrapped = pkgs.callPackage ../. {};
+ populateLib = import ../lib { inherit lib; };
+
+ inherit (import ./fixtures.nix) gitHead gitFirst;
+
+ remoteHost = "target";
+
+ # rsync versions the scenarios depending on rsync's version run with.
+ # callPackage adds override and overrideDerivation, which aren't versions.
+ rsyncs = lib.filterAttrs
+ (_: lib.isDerivation)
+ (pkgs.callPackage ./rsync.nix {});
+
+ # A script running populate for a phase, with the tools it runs in front
+ # of PATH, and the sources on a file descriptor of their own, so the
+ # commands populate runs keep standard input. Store paths in the sources
+ # stay dependencies of the script.
+ runWith = rsync: {
+ source, backup, force, prune, deploy, targetKind, targetPath,
+ }:
+ if deploy != null then throw "populate's tests don't deploy" else
+ pkgs.writers.writeDash "populate" /* sh */ ''
+ set -efu
+ PATH=${lib.makeBinPath (with pkgs; [
+ coreutils findutils git nix openssh pass passage rsync
+ ])}''${PATH:+:$PATH}
+ export PATH
+ exec ${populate-unwrapped}/bin/populate ${lib.escapeShellArgs (
+ lib.optional backup "--backup" ++
+ lib.optional force "--force" ++
+ lib.optional prune "--prune" ++
+ [ "/dev/fd/3" ] ++
+ {
+ local = [];
+ remote = [ "root@${remoteHost}" ];
+ sudo = [ "deploy@${remoteHost}" "--sudo" ];
+ }.${targetKind} ++
+ [ "--path" targetPath ]
+ )} 3<<'EOF'
+ ${builtins.toJSON source}
+ EOF
+ '';
+
+ # The scenarios, with populate using rsync.
+ scenariosWith = rsync: import ./scenarios.nix {
+ inherit lib;
+ inherit (populateLib) evalSource;
+ run = runWith rsync;
+ checksDuringEvaluation = false;
+ url = "http://deployer:8000";
+ gitUrl = "git://deployer/gitsrc.git";
+ inherit remoteHost;
+ agePath = "${pkgs.age}/bin/age";
+ inherit gitHead gitFirst;
+ rsyncVersion = rsync.version;
+ };
+ scenarios = scenariosWith pkgs.rsync;
+
+ vm = name: scenarios: rsync: import ./vm.nix {
+ inherit pkgs gitHead scenarios rsync;
+ name = "populate-${name}";
+ };
+
+ # VM tests of those of the groups of scenarios that are, or aren't,
+ # exhaustive, named after the groups with prefix in front.
+ testsOf = exhaustive: prefix: lib.mapAttrs
+ (name: scenarios: vm (prefix + name) scenarios pkgs.rsync)
+ (lib.filterAttrs (_: scenarios: scenarios != [])
+ (builtins.mapAttrs
+ (_: builtins.filter (scenario: scenario.exhaustive == exhaustive))
+ scenarios));
+
+ allScenarios = lib.concatLists (builtins.attrValues scenarios);
+
+ # The scenarios named names, failing if one is missing.
+ named = what: names: let
+ selected = builtins.filter (scenario: builtins.elem scenario.name names) allScenarios;
+ missing = lib.subtractLists (map (scenario: scenario.name) selected) names;
+ in
+ if missing != [] then
+ throw "tests: ${what} names missing scenarios: ${toString missing}"
+ else
+ selected;
+
+ # The scenarios whose names match the regular expression only.
+ onlyScenarios = let
+ selected = builtins.filter
+ (scenario: builtins.match only scenario.name != null)
+ allScenarios;
+ in
+ if only == null then
+ throw "tests: -A only needs --argstr only REGEX"
+ else if selected == [] then
+ throw "tests: no scenario's name matches ${builtins.toJSON only}"
+ else
+ selected;
+in
+ testsOf false ""
+ # Not built by nix-build either, which runs these scenarios anyway.
+ // {
+ smoke.smoke = vm "smoke" (named "smoke test" (import ./smoke.nix)) pkgs.rsync;
+ only.only = vm "only" onlyScenarios pkgs.rsync;
+ }
+ # An attribute set without recurseForDerivations, so nix-build doesn't
+ # build them.
+ // { exhaustive = testsOf true "exhaustive-"; }
+ // lib.mapAttrs'
+ (version: rsync: {
+ name = "rsync-${builtins.replaceStrings [ "." ] [ "-" ] version}";
+ value = vm "rsync-${version}"
+ (builtins.filter
+ (scenario: lib.hasInfix "insecureLinks" scenario.name)
+ (scenariosWith rsync).file)
+ rsync;
+ })
+ rsyncs
diff --git a/pkgs/populate/tests/fixtures.nix b/pkgs/populate/tests/fixtures.nix
new file mode 100644
index 0000000..c834898
--- /dev/null
+++ b/pkgs/populate/tests/fixtures.nix
@@ -0,0 +1,7 @@
+# What tests know about the fixtures fixtures.sh creates: HEAD of
+# /fixtures/gitsrc, which fixtures.sh creates deterministically, and its
+# parent. fixtures.sh fails if HEAD differs, naming the actual one.
+{
+ gitHead = "e42d44ef9cdb76ba5a84dff4c89c38222703bf0c";
+ gitFirst = "8d47013fa0888948cf0ffcc4e3c69a7b5f180562";
+}
diff --git a/pkgs/populate/tests/fixtures.sh b/pkgs/populate/tests/fixtures.sh
new file mode 100755
index 0000000..85701a1
--- /dev/null
+++ b/pkgs/populate/tests/fixtures.sh
@@ -0,0 +1,498 @@
+#! /bin/sh
+# Creates the fixtures the tests deploy from in /fixtures.
+#
+# usage: fixtures.sh URL GIT_HEAD
+#
+# URL is where /fixtures/www gets served via HTTP, e.g. http://deployer:8000.
+# The tarball flakes there get locked against it, so it must be served while
+# this script runs. /fixtures/git gets served by git daemon. GIT_HEAD is the
+# commit the tests expect as HEAD of /fixtures/gitsrc.
+set -efu
+
+url=$1
+git_head=$2
+
+nix() {
+ command nix --extra-experimental-features 'nix-command flakes' "$@"
+}
+
+export GIT_AUTHOR_NAME=krops GIT_AUTHOR_EMAIL=spam@krebsco.de
+export GIT_COMMITTER_NAME=krops GIT_COMMITTER_EMAIL=spam@krebsco.de
+export GIT_AUTHOR_DATE='1000000000 +0000' GIT_COMMITTER_DATE='1000000000 +0000'
+# Configuration such as commit.gpgsign would change the commits.
+export GIT_CONFIG_GLOBAL=/dev/null GIT_CONFIG_NOSYSTEM=1
+
+# Commits everything in the current directory. Commits are deterministic, so
+# the tests can refer to them by hash.
+commit() {
+ git add -A
+ git commit -q -m "$1"
+}
+
+tarball() {
+ tar -C /fixtures/www/src --sort=name --mtime=@1 --owner=0 --group=0 \
+ --numeric-owner -czf /fixtures/www/"$1".tar.gz "$1"
+}
+
+rm -fR /fixtures
+mkdir -p /fixtures/www/src
+
+# Tarball flakes, fetched via HTTP: inner, dep (with input inner), mono (with
+# a flake in subdirectory sub), and remote (with input dep).
+mkdir /fixtures/www/src/inner
+cat > /fixtures/www/src/inner/flake.nix <<'NIX'
+{ outputs = _: { who = "inner"; }; }
+NIX
+tarball inner
+
+mkdir /fixtures/www/src/dep
+cat > /fixtures/www/src/dep/flake.nix <<NIX
+{
+ inputs.inner.url = "$url/inner.tar.gz";
+ outputs = { inner, ... }: { greeting = "dep+\${inner.who}"; };
+}
+NIX
+(cd /fixtures/www/src/dep && nix flake lock)
+tarball dep
+
+mkdir -p /fixtures/www/src/mono/sub
+echo mono > /fixtures/www/src/mono/README
+cat > /fixtures/www/src/mono/sub/flake.nix <<'NIX'
+{ outputs = _: { who = "sub"; }; }
+NIX
+tarball mono
+
+mkdir /fixtures/www/src/remote
+cat > /fixtures/www/src/remote/flake.nix <<NIX
+{
+ inputs.dep.url = "$url/dep.tar.gz";
+ outputs = { dep, ... }: { msg = "remote:\${dep.greeting}"; };
+}
+NIX
+(cd /fixtures/www/src/remote && nix flake lock)
+tarball remote
+
+# top: a flake in a Git work tree, with inputs dep and sub (a flake in a
+# subdirectory).
+mkdir /fixtures/top
+cat > /fixtures/top/flake.nix <<NIX
+{
+ inputs.dep.url = "$url/dep.tar.gz";
+ inputs.sub.url = "$url/mono.tar.gz?dir=sub";
+ outputs = { dep, sub, ... }: { msg = "top:\${dep.greeting},\${sub.who}"; };
+}
+NIX
+(cd /fixtures/top && nix flake lock && git init -q && commit top)
+
+# mono: a Git work tree with a flake in subdirectory sub.
+cp -R /fixtures/www/src/mono /fixtures/mono
+(cd /fixtures/mono && git init -q && commit mono)
+
+# alt: a flake to draw inputs from.
+mkdir /fixtures/alt
+cat > /fixtures/alt/flake.nix <<'NIX'
+{ outputs = _: { greeting = "alt"; who = "alt"; }; }
+NIX
+
+# usegone: a flake with an input locked to a path that doesn't exist anymore.
+mkdir /fixtures/gone /fixtures/usegone
+cat > /fixtures/gone/flake.nix <<'NIX'
+{ outputs = _: {}; }
+NIX
+cat > /fixtures/usegone/flake.nix <<'NIX'
+{
+ inputs.gone.url = "path:/fixtures/gone";
+ outputs = _: {};
+}
+NIX
+(cd /fixtures/usegone && nix flake lock)
+rm -R /fixtures/gone
+
+# kropsremote: a registry entry for remote, so it has an indirect reference.
+nix registry add kropsremote "$url/remote.tar.gz"
+
+# monoin: a Git work tree with a flake in subdirectory sub that has an input.
+mkdir -p /fixtures/monoin/sub
+echo monoin > /fixtures/monoin/README
+cat > /fixtures/monoin/sub/flake.nix <<NIX
+{
+ inputs.dep.url = "$url/dep.tar.gz";
+ outputs = { dep, ... }: { msg = "monoin:\${dep.greeting}"; };
+}
+NIX
+(cd /fixtures/monoin/sub && nix flake lock)
+(cd /fixtures/monoin && git init -q && commit monoin)
+
+# dirty: a copy of top with an uncommitted change and an untracked file.
+cp -R /fixtures/top /fixtures/dirty
+sed -i 's/top:/dirty:/' /fixtures/dirty/flake.nix
+echo untracked > /fixtures/dirty/untracked
+
+# unlocked: a flake whose lock file is missing.
+mkdir /fixtures/unlocked
+cat > /fixtures/unlocked/flake.nix <<NIX
+{
+ inputs.inner.url = "$url/inner.tar.gz";
+ outputs = { inner, ... }: { msg = inner.who; };
+}
+NIX
+
+# uselocal: a flake with an input locked to a path on the deploying machine.
+mkdir /fixtures/uselocal
+cat > /fixtures/uselocal/flake.nix <<'NIX'
+{
+ inputs.a.url = "path:/fixtures/alt";
+ outputs = { a, ... }: { msg = "uselocal:${a.who}"; };
+}
+NIX
+(cd /fixtures/uselocal && nix flake lock)
+
+# follow: a flake whose input dep draws its input inner from the flake's own
+# input inner, which is inner2, so the lock file records a follows path, and
+# dep's own lock file pins another inner than the flake's.
+mkdir /fixtures/www/src/inner2
+cat > /fixtures/www/src/inner2/flake.nix <<'NIX'
+{ outputs = _: { who = "inner2"; }; }
+NIX
+tarball inner2
+mkdir /fixtures/follow
+cat > /fixtures/follow/flake.nix <<NIX
+{
+ inputs.inner.url = "$url/inner2.tar.gz";
+ inputs.dep.url = "$url/dep.tar.gz";
+ inputs.dep.inputs.inner.follows = "inner";
+ outputs = { dep, ... }: { msg = "follow:\${dep.greeting}"; };
+}
+NIX
+(cd /fixtures/follow && nix flake lock)
+
+# nofollow: a flake that removes an input of its input, so the lock file
+# records an empty follows path.
+mkdir /fixtures/nofollow
+cat > /fixtures/nofollow/flake.nix <<NIX
+{
+ inputs.dep.url = "$url/dep.tar.gz";
+ inputs.dep.inputs.inner.follows = "";
+ outputs = _: { msg = "nofollow"; };
+}
+NIX
+(cd /fixtures/nofollow && nix flake lock)
+
+# cyc: a flake whose inputs cyca and cycb follow each other.
+for name in cyca cycb; do
+ other=$(test $name = cyca && echo cycb || echo cyca)
+ mkdir /fixtures/www/src/$name
+ cat > /fixtures/www/src/$name/flake.nix <<NIX
+{
+ inputs.other.url = "$url/$other.tar.gz";
+ outputs = _: { who = "$name"; };
+}
+NIX
+ tarball $name
+done
+mkdir /fixtures/cyc
+cat > /fixtures/cyc/flake.nix <<NIX
+{
+ inputs.a.url = "$url/cyca.tar.gz";
+ inputs.b.url = "$url/cycb.tar.gz";
+ inputs.a.inputs.other.follows = "b";
+ inputs.b.inputs.other.follows = "a";
+ outputs = { a, b, ... }: { msg = "cyc:\${a.who},\${b.who}"; };
+}
+NIX
+(cd /fixtures/cyc && nix flake lock)
+
+# rel: a flake in a Git work tree with an input in a subdirectory, referred
+# to by a relative path.
+mkdir -p /fixtures/rel/lib
+cat > /fixtures/rel/lib/flake.nix <<'NIX'
+{ outputs = _: { who = "lib"; }; }
+NIX
+cat > /fixtures/rel/flake.nix <<'NIX'
+{
+ inputs.lib.url = "path:./lib";
+ outputs = { lib, ... }: { msg = "rel:${lib.who}"; };
+}
+NIX
+(cd /fixtures/rel && nix flake lock && git init -q && commit rel)
+
+# two, c1, c2: a flake with two inputs, and two flakes to draw them from.
+mkdir /fixtures/two /fixtures/c1 /fixtures/c2
+cat > /fixtures/two/flake.nix <<NIX
+{
+ inputs.p.url = "$url/inner.tar.gz";
+ inputs.q.url = "$url/inner.tar.gz";
+ outputs = { p, q, ... }: { greeting = "two:\${p.who}+\${q.who}"; };
+}
+NIX
+(cd /fixtures/two && nix flake lock)
+cat > /fixtures/c1/flake.nix <<'NIX'
+{ outputs = _: { who = "c1"; }; }
+NIX
+cat > /fixtures/c2/flake.nix <<'NIX'
+{ outputs = _: { who = "c2"; }; }
+NIX
+
+# usedata, data: a flake with an input that isn't a flake, and a directory
+# to draw it from.
+mkdir /fixtures/usedata /fixtures/data
+cat > /fixtures/usedata/flake.nix <<'NIX'
+{
+ inputs.data = { url = "path:/nonexistent"; flake = false; };
+ outputs = { data, ... }: { msg = "data:${builtins.readFile (data + "/value")}"; };
+}
+NIX
+printf hello > /fixtures/data/value
+
+# pipe2: a command for the pipe source type whose output the tests change.
+cat > /fixtures/pipe2 <<'SH'
+#! /bin/sh
+cat /fixtures/pipe2-data
+SH
+chmod +x /fixtures/pipe2
+echo piped > /fixtures/pipe2-data
+
+# alt2: like alt, with other outputs, to change a source to.
+mkdir /fixtures/alt2
+cat > /fixtures/alt2/flake.nix <<'NIX'
+{ outputs = _: { greeting = "alt2"; who = "alt2"; }; }
+NIX
+
+# filtered, filtered-link: a directory with an .rsync-filter, and a symlink to
+# it that neither root nor the user deploying owns, through which rsync 3.5.0
+# and later can't read that file.
+mkdir /fixtures/filtered
+printf -- '- /garbage\n' > /fixtures/filtered/.rsync-filter
+echo keep > /fixtures/filtered/keep
+echo garbage > /fixtures/filtered/garbage
+ln -s filtered /fixtures/filtered-link
+chown -h nobody /fixtures/filtered-link
+
+# noflake: a directory without flake.nix.
+mkdir /fixtures/noflake
+echo noflake > /fixtures/noflake/README
+
+# filesrc: a copy of top, with files for testing the file source type's
+# options. same has the same size and modification time as the file the
+# tests put on the target beforehand, while skip-ex and skip-f have the same
+# size but an older modification time than theirs, so rsync's quick check
+# tells them apart.
+mkdir /fixtures/filesrc
+cp /fixtures/top/flake.nix /fixtures/top/flake.lock /fixtures/filesrc
+echo keep > /fixtures/filesrc/keep
+echo src > /fixtures/filesrc/skip-ex
+echo src > /fixtures/filesrc/skip-f
+echo AAAA > /fixtures/filesrc/same
+touch -d @1000000000 /fixtures/filesrc/same
+touch -d @999999999 /fixtures/filesrc/skip-ex /fixtures/filesrc/skip-f
+
+# gitsrc: a Git repository with two commits, served by git daemon. Its flake
+# has an input that only exists when drawn from a source.
+mkdir /fixtures/gitsrc
+cat > /fixtures/gitsrc/flake.nix <<'NIX'
+{
+ inputs.dep.url = "path:/nonexistent";
+ outputs = { dep, ... }: { msg = "git:${dep.greeting}"; };
+}
+NIX
+(cd /fixtures/gitsrc && git init -q -b master && commit one)
+echo two > /fixtures/gitsrc/two
+(cd /fixtures/gitsrc && commit two)
+if test "$(git -C /fixtures/gitsrc rev-parse HEAD)" != "$git_head"; then
+ >&2 printf 'error: HEAD of /fixtures/gitsrc is %s, but the tests expect %s\n' \
+ "$(git -C /fixtures/gitsrc rev-parse HEAD)" "$git_head"
+ exit 1
+fi
+mkdir /fixtures/git
+git clone -q --bare /fixtures/gitsrc /fixtures/git/gitsrc.git
+git -C /fixtures/git/gitsrc.git config uploadpack.allowReachableSHA1InWant true
+
+# withsub: a Git repository with gitsrc as its submodule sub, served by git
+# daemon like gitsrc, which the tests reach as git://deployer/gitsrc.git.
+mkdir /fixtures/withsub
+cat > /fixtures/withsub/.gitmodules <<'GIT'
+[submodule "sub"]
+ path = sub
+ url = git://deployer/gitsrc.git
+GIT
+(
+ cd /fixtures/withsub
+ git init -q -b master
+ git update-index --add --cacheinfo 160000,"$git_head",sub
+ git add .gitmodules
+ git commit -q -m withsub
+)
+git clone -q --bare /fixtures/withsub /fixtures/git/withsub.git
+
+# pass: password stores with x/a, without and with Git.
+if ! gpg --list-secret-keys spam@krebsco.de >/dev/null 2>&1; then
+ gpg --batch --passphrase '' --quick-gen-key spam@krebsco.de default default never
+fi
+for store in pass pass-git; do
+ export PASSWORD_STORE_DIR=/fixtures/$store
+ pass init spam@krebsco.de >/dev/null
+ if test $store = pass-git; then
+ pass git init >/dev/null
+ fi
+ printf 'secret-a\n' | pass insert -m x/a >/dev/null
+done
+unset PASSWORD_STORE_DIR
+
+# passage: a passage store with x/a.
+mkdir -p /fixtures/passage/store
+age-keygen -o /fixtures/passage/identities 2>/dev/null
+age-keygen -y /fixtures/passage/identities > /fixtures/passage/store/.age-recipients
+printf 'secret-a\n' |
+ PASSAGE_DIR=/fixtures/passage/store \
+ PASSAGE_IDENTITIES_FILE=/fixtures/passage/identities \
+ passage insert -m x/a >/dev/null
+
+# pipe: a command for the pipe source type.
+cat > /fixtures/pipe <<'SH'
+#! /bin/sh
+echo piped
+SH
+chmod +x /fixtures/pipe
+
+# pipe-slow: a command that takes longer than the tests wait for it.
+cat > /fixtures/pipe-slow <<'SH'
+#! /bin/sh
+sleep 60
+echo piped
+SH
+chmod +x /fixtures/pipe-slow
+
+# patches: hello and hello2 change dep's greeting, hello2 only after hello;
+# sub changes the flake in mono's subdirectory; bad applies to nothing.
+mkdir -p /fixtures/patches
+# Prints a patch of file $1 in the directory $2, replacing $3 by $4.
+patch_of() {
+ sed "s/$3/$4/" "$2/$1" > /fixtures/patches/new
+ diff -u --label a/"$1" --label b/"$1" "$2/$1" /fixtures/patches/new || test $? = 1
+ rm /fixtures/patches/new
+}
+patch_of flake.nix /fixtures/www/src/dep 'dep+' 'hello+' > /fixtures/patches/hello.patch
+mkdir /fixtures/patches/dep-hello
+sed 's/dep+/hello+/' /fixtures/www/src/dep/flake.nix > /fixtures/patches/dep-hello/flake.nix
+patch_of flake.nix /fixtures/patches/dep-hello 'hello+' 'hello2+' > /fixtures/patches/hello2.patch
+rm -R /fixtures/patches/dep-hello
+patch_of flake.nix /fixtures/www/src/mono/sub '"sub"' '"patched"' > /fixtures/patches/sub.patch
+cat > /fixtures/patches/bad.patch <<'PATCH'
+--- a/flake.nix
++++ b/flake.nix
+@@ -1 +1 @@
+-nothing like this
++anything
+PATCH
+
+# broken: a flake with an input that can't be fetched anymore.
+mkdir /fixtures/www/src/vanishing
+cat > /fixtures/www/src/vanishing/flake.nix <<'NIX'
+{ outputs = _: { who = "vanishing"; }; }
+NIX
+tarball vanishing
+mkdir /fixtures/broken
+cat > /fixtures/broken/flake.nix <<NIX
+{
+ inputs.vanishing.url = "$url/vanishing.tar.gz";
+ outputs = { vanishing, ... }: { msg = "broken:\${vanishing.who}"; };
+}
+NIX
+(cd /fixtures/broken && nix flake lock)
+
+# pinned, usepinned: a tarball flake whose own lock file pins an input that
+# can't be fetched anymore, and a flake that draws that input from its own
+# input instead, so nothing it evaluates needs the pin.
+mkdir /fixtures/www/src/pinned
+cat > /fixtures/www/src/pinned/flake.nix <<NIX
+{
+ inputs.inner.url = "$url/vanishing.tar.gz";
+ outputs = { inner, ... }: { greeting = "pinned+\${inner.who}"; };
+}
+NIX
+(cd /fixtures/www/src/pinned && nix flake lock)
+tarball pinned
+mkdir /fixtures/usepinned
+cat > /fixtures/usepinned/flake.nix <<NIX
+{
+ inputs.inner.url = "$url/inner.tar.gz";
+ inputs.dep.url = "$url/pinned.tar.gz";
+ inputs.dep.inputs.inner.follows = "inner";
+ outputs = { dep, ... }: { msg = "usepinned:\${dep.greeting}"; };
+}
+NIX
+(cd /fixtures/usepinned && nix flake lock)
+rm /fixtures/www/vanishing.tar.gz
+
+# stale, usestale: a tarball flake whose lock file lacks an input its
+# flake.nix has, and a flake with it as input, whose own lock file is
+# complete.
+mkdir /fixtures/www/src/stale
+cat > /fixtures/www/src/stale/flake.nix <<NIX
+{
+ inputs.inner.url = "$url/inner.tar.gz";
+ outputs = { inner, ... }: { greeting = "stale+\${inner.who}"; };
+}
+NIX
+(cd /fixtures/www/src/stale && nix flake lock)
+cat > /fixtures/www/src/stale/flake.nix <<NIX
+{
+ inputs.inner.url = "$url/inner.tar.gz";
+ inputs.extra.url = "$url/inner2.tar.gz";
+ outputs = { inner, extra, ... }: { greeting = "stale+\${inner.who}+\${extra.who}"; };
+}
+NIX
+tarball stale
+mkdir /fixtures/usestale
+cat > /fixtures/usestale/flake.nix <<NIX
+{
+ inputs.dep.url = "$url/stale.tar.gz";
+ outputs = { dep, ... }: { msg = "usestale:\${dep.greeting}"; };
+}
+NIX
+(cd /fixtures/usestale && nix flake lock)
+
+# offline: a flake with an input of its own, which the tests make
+# unreachable between deploys.
+mkdir /fixtures/www/src/gap
+cat > /fixtures/www/src/gap/flake.nix <<'NIX'
+{ outputs = _: { who = "gap"; }; }
+NIX
+tarball gap
+mkdir /fixtures/offline
+cat > /fixtures/offline/flake.nix <<NIX
+{
+ inputs.gap.url = "$url/gap.tar.gz";
+ outputs = { gap, ... }: { msg = "offline:\${gap.who}"; };
+}
+NIX
+(cd /fixtures/offline && nix flake lock)
+
+# c3: a flake with an input, to draw another flake's input from.
+mkdir /fixtures/c3
+cat > /fixtures/c3/flake.nix <<NIX
+{
+ inputs.z.url = "$url/inner.tar.gz";
+ outputs = { z, ... }: { who = "c3(\${z.who})"; };
+}
+NIX
+(cd /fixtures/c3 && nix flake lock)
+
+# filtered-parent, filtered-parent-link: like filtered, but the filter file
+# applies to a subdirectory, which is what gets synced.
+mkdir -p /fixtures/filtered-parent/sub
+printf -- '- garbage\n' > /fixtures/filtered-parent/.rsync-filter
+echo keep > /fixtures/filtered-parent/sub/keep
+echo garbage > /fixtures/filtered-parent/sub/garbage
+ln -s filtered-parent /fixtures/filtered-parent-link
+chown -h nobody /fixtures/filtered-parent-link
+
+# filtered-name, filtered-name-link: like filtered, but the filter file is in
+# a subdirectory whose name has a byte that rsync prints escaped in the C
+# locale.
+mkdir -p /fixtures/filtered-name/bär
+printf -- '- garbage\n' > /fixtures/filtered-name/bär/.rsync-filter
+echo keep > /fixtures/filtered-name/bär/keep
+echo garbage > /fixtures/filtered-name/bär/garbage
+ln -s filtered-name /fixtures/filtered-name-link
+chown -h nobody /fixtures/filtered-name-link
diff --git a/pkgs/populate/tests/lib.sh b/pkgs/populate/tests/lib.sh
new file mode 100644
index 0000000..a8f2594
--- /dev/null
+++ b/pkgs/populate/tests/lib.sh
@@ -0,0 +1,118 @@
+# Helpers for preparing and checking scenarios, sourced on the machine
+# populate deploys to.
+set -efu
+
+fail() {
+ printf 'FAIL: %s\n' "$*" >&2
+ exit 1
+}
+
+# usage: expect_eq WHAT ACTUAL EXPECTED
+expect_eq() {
+ if test "$2" != "$3"; then
+ fail "$1: expected '$3', got '$2'"
+ fi
+}
+
+nix() {
+ command nix --extra-experimental-features 'nix-command flakes' "$@"
+}
+
+# Evaluates a flake output without network access to the fixtures, so all
+# inputs must be in the store. An empty cache directory keeps Nix from
+# restoring inputs from its fetcher cache.
+nix_eval() {
+ nix_eval_cache=$(mktemp -d)
+ nix_eval_status=0
+ env \
+ XDG_CACHE_HOME="$nix_eval_cache" \
+ http_proxy=http://127.0.0.1:9 https_proxy=http://127.0.0.1:9 \
+ HTTP_PROXY=http://127.0.0.1:9 HTTPS_PROXY=http://127.0.0.1:9 \
+ all_proxy=http://127.0.0.1:9 ALL_PROXY=http://127.0.0.1:9 \
+ no_proxy= NO_PROXY= \
+ nix --extra-experimental-features 'nix-command flakes' eval --raw "$@" ||
+ nix_eval_status=$?
+ rm -fR "$nix_eval_cache"
+ return "$nix_eval_status"
+}
+
+# Evaluates a flake output, fetching missing inputs.
+nix_eval_online() {
+ nix eval --raw "$@"
+}
+
+# Collects garbage if KROPS_TEST_GC is set, so evaluating afterwards checks
+# that delivered store paths are registered as garbage collector roots.
+gc() {
+ if test -n "${KROPS_TEST_GC-}"; then
+ nix-store --gc >/dev/null
+ fi
+}
+
+# Canaries are files outside the target directories, which populate must
+# never change, whatever the target directories contain. Some scenarios
+# point links at them. The manifest lists every entry with its type, mode,
+# modification time, and link target, and every file with its checksum.
+# Entries beside the target directories whose names start like the
+# canaries' count as well, so files created next to them show.
+canary_manifest() {
+ set -- /var/krops-outside $(
+ find /var/krops -mindepth 1 -maxdepth 1 \
+ \( -name 'canary*' -o -name 'sibling*' \) 2>/dev/null
+ )
+ {
+ find "$@" -printf '%y %m %T@ %p -> %l\n'
+ find "$@" -type f -exec sha256sum {} +
+ } 2>&1 | sort
+}
+
+# Creates the canaries, unless they are there unchanged: a file beside the
+# target directories, a directory beside them, and a directory elsewhere
+# with files, a read-only directory, and a Git work tree with a changed and
+# an untracked file, which resetting and cleaning would lose.
+canary_setup() {
+ if test -e /var/krops-outside.manifest &&
+ canary_manifest | cmp -s - /var/krops-outside.manifest; then
+ return
+ fi
+ rm -fR /var/krops/canary /var/krops/sibling /var/krops-outside
+ mkdir -p /var/krops/sibling /var/krops-outside/dir/sub
+ echo canary > /var/krops/canary
+ echo sibling > /var/krops/sibling/file
+ echo outside > /var/krops-outside/file
+ echo outside > /var/krops-outside/dir/file
+ echo outside > /var/krops-outside/dir/sub/file
+ chmod a-w /var/krops-outside/dir/sub/file /var/krops-outside/dir/sub
+ git init -q /var/krops-outside/repo
+ (
+ cd /var/krops-outside/repo
+ echo tracked > tracked
+ git add tracked
+ GIT_CONFIG_GLOBAL=/dev/null GIT_CONFIG_NOSYSTEM=1 \
+ git -c user.name=krops -c user.email=spam@krebsco.de commit -q -m canary
+ echo changed > tracked
+ echo untracked > untracked
+ )
+ canary_manifest > /var/krops-outside.manifest
+}
+
+# Fails if a canary changed, or if a store path that an entry of the target
+# directory $T links to doesn't match its hash anymore.
+canary_check() {
+ if ! canary_manifest | cmp -s - /var/krops-outside.manifest; then
+ canary_manifest | diff /var/krops-outside.manifest - >&2 || :
+ fail "something outside the target directory changed"
+ fi
+ if test -n "${T-}" && test -d "$T"; then
+ ls -A "$T" | while IFS= read -r canary_name; do
+ canary_link=$(readlink "$T/$canary_name" 2>/dev/null) || continue
+ case $canary_link in
+ /nix/store/*)
+ nix-store --verify-path \
+ "$(printf '%s\n' "$canary_link" | cut -d/ -f1-4)" ||
+ fail "$T/$canary_name links to a store path that changed"
+ ;;
+ esac
+ done
+ fi
+}
diff --git a/pkgs/populate/tests/rsync.nix b/pkgs/populate/tests/rsync.nix
new file mode 100644
index 0000000..6c14048
--- /dev/null
+++ b/pkgs/populate/tests/rsync.nix
@@ -0,0 +1,50 @@
+# rsync versions whose behaviour around .rsync-filter files differs: 3.4.1
+# reads them like the files it transfers, and 3.5.0 opens them more
+# strictly. The tests run the scenarios depending on that with each,
+# whatever version nixpkgs has. rsync's own test suite doesn't run: these
+# builds only need to behave like the releases.
+{ lib, stdenv, fetchurl, fetchpatch, perl, updateAutotoolsGnuConfigScriptsHook
+, acl, libiconv, popt, zlib
+}:
+
+let
+ rsync = { version, hash, patches ? [] }: stdenv.mkDerivation {
+ pname = "rsync";
+ inherit version patches;
+ src = fetchurl {
+ # signed with key 9FEF 112D CE19 A0DC 7E88 2CB8 1BB2 4997 A853 5F6F
+ url = "mirror://samba/rsync/src/rsync-${version}.tar.gz";
+ inherit hash;
+ };
+ nativeBuildInputs = [ perl updateAutotoolsGnuConfigScriptsHook ];
+ # With ACLs and extended attributes, which backups use (rsync -aAX).
+ buildInputs = [ acl libiconv popt zlib ];
+ configureFlags = [
+ "--disable-lz4"
+ "--disable-openssl"
+ "--disable-xxhash"
+ "--disable-zstd"
+ "--enable-ipv6"
+ "--with-included-zlib=no"
+ "--with-nobody-group=nogroup"
+ ];
+ enableParallelBuilding = true;
+ meta.mainProgram = "rsync";
+ };
+in {
+ "3.4.1" = rsync {
+ version = "3.4.1";
+ hash = "sha256-KSS8s6Hti1UfwQH3QLnw/gogKxFQJ2R89phQ1l/YjFI=";
+ patches = [
+ # Compilation with GCC 15.
+ (fetchpatch {
+ url = "https://github.com/RsyncProject/rsync/commit/a4b926dcdce96b0f2cc0dc7744e95747b233500a.patch";
+ hash = "sha256-UiEQJ+p2gtIDYNJqnxx4qKgItKIZzCpkHnvsgoxBmSE=";
+ })
+ ];
+ };
+ "3.5.0" = rsync {
+ version = "3.5.0";
+ hash = "sha256-x//R72U+mVQPZh5HywC3+crR7muXI5mxb5PWcmVuDTM=";
+ };
+}
diff --git a/pkgs/populate/tests/scenarios.nix b/pkgs/populate/tests/scenarios.nix
new file mode 100644
index 0000000..d00ebcb
--- /dev/null
+++ b/pkgs/populate/tests/scenarios.nix
@@ -0,0 +1,2174 @@
+# Test scenarios for populate: every combination of each source type's
+# options, delivered to the local machine, to a remote target as root, and
+# to a remote target using sudo, plus scenarios for features spanning
+# sources. The arguments are those of builder.nix, which builds them.
+args:
+
+with import ./builder.nix args;
+
+{
+
+ derivation = matrix "derivation" {} (_: _: {
+ phases = twice {
+ sources.x.derivation = ''
+ derivation {
+ name = "krops-test";
+ system = builtins.currentSystem;
+ builder = "/bin/sh";
+ args = [ "-c" "echo built > $out" ];
+ }
+ '';
+ check = ''
+ gc
+ expect_eq x "$(cat "$T"/x)" built
+ '';
+ };
+ });
+
+ file = let
+ f = flags: _: {
+ # Drawing inputs happens after rsync, so it doesn't depend on rsync's
+ # options.
+ exhaustive = flags.inputs != "none" &&
+ (flags.checksum || flags.deleteExcluded || flags.filters);
+ prepare = ''
+ mkdir -p "$T"/x
+ echo old > "$T"/x/skip-ex
+ echo old > "$T"/x/skip-f
+ echo stale > "$T"/x/stale
+ echo BBBB > "$T"/x/same
+ touch -d @1000000000 "$T"/x/same
+ '';
+ phases = twice {
+ sources = {
+ x.file = {
+ path = "/fixtures/filesrc";
+ useChecksum = flags.checksum;
+ inherit (flags) deleteExcluded;
+ exclude = optional (flags.exclude or false) "skip-ex";
+ filters = optional flags.filters { type = "exclude"; pattern = "skip-f"; };
+ } // drawInputs flags.inputs { dep.follows = "alt"; };
+ } // optionalAttrs (flags.inputs != "none") alt;
+ check = let
+ excluded = file: set: ''
+ ${if !set then ''
+ expect_eq ${file} "$(cat "$T"/x/${file})" src
+ '' else if flags.deleteExcluded then ''
+ test ! -e "$T"/x/${file} || fail "${file} not deleted"
+ '' else ''
+ expect_eq ${file} "$(cat "$T"/x/${file})" old
+ ''}
+ '';
+ in ''
+ expect_eq keep "$(cat "$T"/x/keep)" keep
+ ${excluded "skip-ex" (flags.exclude or false)}
+ ${excluded "skip-f" flags.filters}
+ ${if flags.deleteExcluded then ''
+ test ! -e "$T"/x/stale || fail "stale not deleted"
+ '' else ''
+ expect_eq stale "$(cat "$T"/x/stale)" stale
+ ''}
+ expect_eq same "$(cat "$T"/x/same)" ${if flags.checksum then "AAAA" else "BBBB"}
+ '' + optionalString (flags.inputs != "none") ''
+ # Inputs not drawn from sources aren't delivered, so this needs the
+ # network and can't collect garbage before.
+ expect_eq msg "$(nix_eval_online ${flakeOf flags.inputs}#msg)" top:alt,sub
+ '';
+ };
+ };
+ in
+ matrix "file" {
+ checksum = [ false true ];
+ deleteExcluded = [ false true ];
+ filters = [ false true ];
+ inputs = inputModes;
+ } f
+ ++ [
+ # The deprecated exclude option, once, because it warns.
+ (scenario (let
+ args = f {
+ checksum = false;
+ deleteExcluded = true;
+ exclude = true;
+ filters = false;
+ inputs = "none";
+ } null;
+ in args // {
+ name = "file-local-exclude";
+ targetKind = "local";
+ phases = [ (builtins.head args.phases) ];
+ }))
+ ]
+ # A source whose .rsync-filter rsync 3.5.0 and later can't read, because
+ # it's reached through a symlink that neither root nor the user deploying
+ # owns. (Running as root, the scenarios can't use the other common
+ # cause, a directory on the way that the user may enter but not read.)
+ ++ concatMap (targetKind: map (insecureLinks: let
+ affected = lib.versionAtLeast rsyncVersion "3.5.0";
+ excluded = !affected || insecureLinks == true;
+ in scenario {
+ name = "file-${targetKind}-insecureLinks" +
+ (if insecureLinks == null then "Null" else if insecureLinks then "True" else "False");
+ inherit targetKind;
+ prepare = ''
+ mkdir -p "$T"/x
+ echo old > "$T"/x/garbage
+ '';
+ phases = twice ({
+ sources.x.file = { path = "/fixtures/filtered-link"; inherit insecureLinks; };
+ } // (if affected && insecureLinks == null then {
+ expect.fail = "can't read /fixtures/filtered-link/.rsync-filter, so it would sync /fixtures/filtered-link/ without it.";
+ } else {
+ output = optional affected (
+ if insecureLinks
+ then "can't read /fixtures/filtered-link/.rsync-filter; it reads it with --insecure-links anyway, because insecureLinks is true."
+ else "can't read /fixtures/filtered-link/.rsync-filter, so it syncs /fixtures/filtered-link/ without it, because insecureLinks is false."
+ );
+ check = ''
+ expect_eq keep "$(cat "$T"/x/keep)" keep
+ ${if excluded then ''
+ test ! -e "$T"/x/garbage || fail "garbage not excluded"
+ '' else ''
+ expect_eq garbage "$(cat "$T"/x/garbage)" garbage
+ ''}
+ '';
+ }));
+ }) [ null true false ]) targetKinds
+ ++ concatMap (targetKind: let
+ affected = lib.versionAtLeast rsyncVersion "3.5.0";
+ unreadable = name: { path, file, dir ? "x", env ? "" }: scenario {
+ name = "file-${targetKind}-insecureLinks${name}";
+ inherit targetKind;
+ prepare = ''
+ mkdir -p "$T"/${dir}
+ echo old > "$T"/${dir}/garbage
+ '';
+ phases = twice ({
+ sources.x.file = { inherit path; };
+ inherit env;
+ } // (if affected then {
+ expect.fail = "can't read ${file}, so it would sync ${path}/ without it.";
+ } else {
+ check = ''
+ expect_eq keep "$(cat "$T"/${dir}/keep)" keep
+ test ! -e "$T"/${dir}/garbage || fail "garbage not excluded"
+ '';
+ }));
+ };
+ in [
+ # The filter file is in the directory above what gets synced, where it
+ # applies as well.
+ (unreadable "Parent" {
+ path = "/fixtures/filtered-parent-link/sub";
+ file = "/fixtures/filtered-parent-link/.rsync-filter";
+ })
+ # The filter file is in a directory whose name rsync prints escaped
+ # in the C locale.
+ (unreadable "Name" {
+ path = "/fixtures/filtered-name-link";
+ file = "/fixtures/filtered-name-link/bär/.rsync-filter";
+ dir = "x/bär";
+ env = "LC_ALL=C";
+ })
+ ]) targetKinds;
+
+ flake = matrix "flake" {
+ kind = [ "git" "path" "subdir" "remote" ];
+ fetchOnTarget = [ false true ];
+ inputs = inputModes;
+ materialize = [ false true ];
+ } (flags: ctx: let
+ flake = {
+ git = "/fixtures/top";
+ path = "path:/fixtures/top";
+ subdir = "git+file:/fixtures/mono?dir=sub";
+ remote = "${url}/remote.tar.gz";
+ }.${flags.kind};
+ msg = {
+ git = "top:${if flags.inputs != "none" then "alt" else "dep+inner"},sub";
+ path = msg.git;
+ remote = "remote:${if flags.inputs != "none" then "alt" else "dep+inner"}";
+ };
+ in
+ # The flake in a subdirectory has no inputs to draw from sources.
+ if flags.kind == "subdir" && flags.inputs != "none" then null else {
+ isolate = true;
+ # Materializing copies what got delivered, whichever way, so it depends
+ # on how inputs get drawn and on subdirectories, not on how the flake got
+ # there.
+ exhaustive = flags.materialize && !lib.elem flags.kind [ "git" "subdir" ];
+ phases = twice ({
+ sources = {
+ x.flake = {
+ url = flake;
+ inherit (flags) fetchOnTarget materialize;
+ } // drawInputs flags.inputs { dep.follows = "alt"; };
+ } // optionalAttrs (flags.inputs != "none") alt;
+ } // (if copyingWithSudo flags ctx then refuseCopyingWithSudo else {
+ check = if flags.kind == "subdir" then checkFlake {
+ attr = "who"; expected = "sub"; subdir = true;
+ materialized = flags.materialize;
+ } else checkFlake {
+ attr = "msg"; expected = msg.${flags.kind};
+ generated = flags.inputs == "generated";
+ materialized = flags.materialize;
+ };
+ }));
+ });
+
+ flake-input = concatMap
+ (name: matrix "flake-input-${lib.replaceStrings ["/"] ["."] name}" ({
+ fetchOnTarget = [ false true ];
+ materialize = [ false true ];
+ } // optionalAttrs (name == "dep") {
+ inputs = inputModes;
+ }) (flags: ctx: let
+ inputs = flags.inputs or "none";
+ in {
+ isolate = true;
+ # Materializing copies what got delivered, whichever input it is.
+ exhaustive = flags.materialize && name != "dep";
+ phases = twice ({
+ sources = {
+ x.flake-input = {
+ url = "/fixtures/top";
+ inherit name;
+ inherit (flags) fetchOnTarget materialize;
+ } // drawInputs inputs { inner.follows = "alt"; };
+ } // optionalAttrs (inputs != "none") alt;
+ } // (if copyingWithSudo flags ctx then refuseCopyingWithSudo else {
+ check = {
+ dep = checkFlake {
+ attr = "greeting";
+ expected = if inputs != "none" then "dep+alt" else "dep+inner";
+ generated = inputs == "generated";
+ materialized = flags.materialize;
+ };
+ "dep/inner" = checkFlake {
+ attr = "who"; expected = "inner"; materialized = flags.materialize;
+ };
+ sub = checkFlake {
+ attr = "who"; expected = "sub"; subdir = true;
+ materialized = flags.materialize;
+ };
+ }.${name};
+ }));
+ }))
+ [ "dep" "dep/inner" "sub" ];
+
+ git = matrix "git" {
+ shallow = [ false true ];
+ fetchAlways = [ false true ];
+ cleanExclude = [ false true ];
+ inputs = inputModes;
+ } (flags: _: let
+ check = ''
+ expect_eq HEAD "$(git -C "$T"/x rev-parse HEAD)" ${gitHead}
+ expect_eq commits "$(git -C "$T"/x rev-list --count HEAD)" ${
+ if flags.shallow then "1" else "2"
+ }
+ test -e "$T"/x/two || fail "two missing"
+ '' + optionalString (flags.inputs == "generated") ''
+ grep -q '"git+file:' "$T"/.populate/flakes/x/flake.nix ||
+ fail "the Git work tree isn't referred to as git+file:"
+ '' + optionalString (flags.inputs != "none") ''
+ gc
+ expect_eq msg "$(nix_eval ${flakeOf flags.inputs}#msg)" git:alt
+ '';
+ sources = ref: {
+ x.git = {
+ url = gitUrl;
+ inherit ref;
+ inherit (flags) shallow fetchAlways;
+ clean.exclude = optional flags.cleanExclude "keepme";
+ } // drawInputs flags.inputs { dep.follows = "alt"; };
+ } // optionalAttrs (flags.inputs != "none") alt;
+ in {
+ # Drawing inputs happens after Git, so it doesn't depend on Git's
+ # options.
+ exhaustive = flags.inputs != "none" &&
+ (flags.shallow || flags.fetchAlways || flags.cleanExclude);
+ phases = [
+ # The older commit first, so deploying HEAD changes the work tree, and
+ # fetches if fetchAlways is set or the clone is shallow.
+ {
+ sources = sources gitFirst;
+ check = ''
+ expect_eq HEAD "$(git -C "$T"/x rev-parse HEAD)" ${gitFirst}
+ expect_eq commits "$(git -C "$T"/x rev-list --count HEAD)" 1
+ test ! -e "$T"/x/two || fail "two present"
+ '';
+ }
+ {
+ sources = sources gitHead;
+ inherit check;
+ mutate = ''
+ echo junk > "$T"/x/junk
+ echo keepme > "$T"/x/keepme
+ '';
+ }
+ {
+ sources = sources gitHead;
+ check = check + ''
+ test ! -e "$T"/x/junk || fail "junk not cleaned"
+ ${if flags.cleanExclude then ''
+ test -e "$T"/x/keepme || fail "keepme cleaned"
+ '' else ''
+ test ! -e "$T"/x/keepme || fail "keepme not cleaned"
+ ''}
+ '';
+ }
+ ];
+ })
+ ++ map (targetKind: scenario (let
+ # The commit the first phase adds on top of gitHead, deterministic like
+ # the ones fixtures.sh creates.
+ gitThird = "c2e0f9d727337617a548284552909abe0f196d05";
+ in {
+ # A full clone fetches a commit that appears on the server afterwards,
+ # also without fetchAlways.
+ name = "git-${targetKind}-fetch";
+ inherit targetKind;
+ prepareDeployer = ''
+ git -C /fixtures/git/gitsrc.git update-ref -d refs/heads/extra
+ '';
+ phases = [
+ {
+ sources.x.git = { url = gitUrl; ref = gitHead; };
+ check = ''
+ expect_eq HEAD "$(git -C "$T"/x rev-parse HEAD)" ${gitHead}
+ '';
+ mutateDeployer = ''
+ export GIT_AUTHOR_NAME=krops GIT_AUTHOR_EMAIL=spam@krebsco.de
+ export GIT_COMMITTER_NAME=krops GIT_COMMITTER_EMAIL=spam@krebsco.de
+ export GIT_AUTHOR_DATE='1000000000 +0000' GIT_COMMITTER_DATE='1000000000 +0000'
+ export GIT_CONFIG_GLOBAL=/dev/null GIT_CONFIG_NOSYSTEM=1
+ cd /fixtures/gitsrc
+ git checkout -q -B extra ${gitHead}
+ echo three > three
+ git add -A
+ git commit -q -m three
+ git push -q -f /fixtures/git/gitsrc.git extra
+ git checkout -q master
+ expect_eq "third commit" "$(git rev-parse extra)" ${gitThird}
+ '';
+ }
+ {
+ sources.x.git = { url = gitUrl; ref = gitThird; };
+ check = ''
+ expect_eq HEAD "$(git -C "$T"/x rev-parse HEAD)" ${gitThird}
+ test -e "$T"/x/three || fail "three missing"
+ '';
+ }
+ ];
+ })) targetKinds
+ # Branches and tags follow the server: a branch that moves between two
+ # deploys, and a tag, with full and shallow clones.
+ ++ concatMap (targetKind: map (shallow: scenario {
+ name = "git-${targetKind}-shallow${if shallow then "1" else "0"}-refs";
+ inherit targetKind;
+ prepareDeployer = ''
+ git -C /fixtures/git/gitsrc.git update-ref refs/heads/moving ${gitFirst}
+ git -C /fixtures/git/gitsrc.git update-ref refs/tags/first ${gitFirst}
+ '';
+ phases = [
+ {
+ sources.x.git = { url = gitUrl; ref = "moving"; inherit shallow; };
+ check = ''
+ expect_eq HEAD "$(git -C "$T"/x rev-parse HEAD)" ${gitFirst}
+ '';
+ mutateDeployer = ''
+ git -C /fixtures/git/gitsrc.git update-ref refs/heads/moving ${gitHead}
+ '';
+ }
+ {
+ sources.x.git = { url = gitUrl; ref = "moving"; inherit shallow; };
+ check = ''
+ expect_eq HEAD "$(git -C "$T"/x rev-parse HEAD)" ${gitHead}
+ '';
+ }
+ {
+ sources.x.git = { url = gitUrl; ref = "first"; inherit shallow; };
+ check = ''
+ expect_eq HEAD "$(git -C "$T"/x rev-parse HEAD)" ${gitFirst}
+ '';
+ }
+ ];
+ }) [ false true ]) targetKinds
+
+ # A git source stops drawing inputs: its lock file, which Nix created and
+ # staged, goes, so the work tree is as the commit has it.
+ ++ map (targetKind: scenario {
+ name = "git-${targetKind}-dropInputs";
+ inherit targetKind;
+ phases = [
+ {
+ sources = alt // {
+ x.git = { url = gitUrl; ref = gitHead; inputs.dep.follows = "alt"; };
+ };
+ check = ''
+ expect_eq msg "$(nix_eval_online git+file:"$T"/x#msg)" git:alt
+ '';
+ }
+ {
+ sources = alt // { x.git = { url = gitUrl; ref = gitHead; }; };
+ check = ''
+ expect_eq status "$(git -C "$T"/x status --porcelain)" ""
+ '';
+ }
+ ];
+ }) targetKinds;
+
+ pass = matrix "pass" {
+ store = [ "pass" "pass-git" ];
+ } (flags: _: {
+ phases = twice {
+ sources.x.pass = { dir = "/fixtures/${flags.store}"; name = "x"; };
+ check = ''
+ expect_eq a "$(cat "$T"/x/a)" secret-a
+ '';
+ };
+ });
+
+ passage = matrix "passage" {
+ age = [ "default" "path" ];
+ dir = [ "store" "subdir" ];
+ } (flags: _: {
+ phases = twice {
+ sources.x.passage = {
+ dir = "/fixtures/passage/store" + optionalString (flags.dir == "subdir") "/x";
+ identities_file = "/fixtures/passage/identities";
+ } // optionalAttrs (flags.age == "path") {
+ age = agePath;
+ };
+ check = ''
+ expect_eq a "$(cat "$T"/x/${optionalString (flags.dir == "store") "x/"}a)" secret-a
+ '';
+ };
+ });
+
+ pipe = matrix "pipe" {} (_: _: {
+ phases = twice {
+ sources.x.pipe = "/fixtures/pipe";
+ check = ''
+ expect_eq x "$(cat "$T"/x)" piped
+ '';
+ };
+ });
+
+ symlink = matrix "symlink" {
+ absolute = [ false true ];
+ inputs = [ false true ];
+ } (flags: ctx: let
+ target = optionalString flags.absolute "${ctx.path}/" + "topcopy";
+ in {
+ phases = twice {
+ sources = {
+ x.symlink = {
+ inherit target;
+ inputs = optionalAttrs flags.inputs { dep.follows = "alt"; };
+ };
+ topcopy.file = "/fixtures/filesrc";
+ } // optionalAttrs flags.inputs alt;
+ check = ''
+ expect_eq link "$(readlink "$T"/x)" ${target}
+ test -e "$T"/x/flake.nix || fail "flake.nix missing"
+ '' + optionalString flags.inputs ''
+ expect_eq msg "$(nix_eval_online "$T"/.populate/flakes/x#msg)" top:alt,sub
+ '';
+ };
+ });
+
+ # Features spanning several sources.
+ features = concatMap (targetKind: let
+ s = name: args: scenario ({
+ name = "${name}-${targetKind}";
+ inherit targetKind;
+ isolate = true;
+ } // args);
+ copying = targetKind != "sudo";
+ fetchOnTarget = !copying;
+ # A scenario for each way of drawing inputs, which f gets as mode, along
+ # with draw, which adds inputs drawn that way to a source's options, and
+ # flake, which gives the flake to evaluate for a source.
+ modes = name: f: map
+ (mode: s "${name}-${mode}" (f {
+ draw = drawInputs mode;
+ flake = n:
+ if mode == "generated" then ''"$T"/.populate/flakes/${n}'' else ''"$T"/${n}'';
+ inherit mode;
+ }))
+ [ "rewrite" "generated" ];
+ in lib.flatten [
+ # A source draws an input from a source that draws inputs itself.
+ (modes "chain" ({ draw, flake, ... }: {
+ phases = twice {
+ sources = alt // {
+ a.flake = { url = "/fixtures/top"; inherit fetchOnTarget; } // draw { dep.follows = "b"; };
+ b.flake-input = {
+ url = "/fixtures/top"; name = "dep"; inherit fetchOnTarget;
+ } // draw { inner.follows = "alt"; };
+ };
+ check = ''
+ gc
+ expect_eq msg "$(nix_eval ${flake "a"}#msg)" top:dep+alt,sub
+ '';
+ };
+ }))
+ # An input of an input drawn from a source gets drawn from another
+ # source.
+ (modes "nested" ({ draw, flake, ... }: {
+ phases = twice {
+ sources = alt // {
+ x.flake = {
+ url = "/fixtures/top"; inherit fetchOnTarget;
+ } // draw { dep = { follows = "b"; inputs.inner.follows = "alt"; }; };
+ b.flake-input = { url = "/fixtures/top"; name = "dep"; inherit fetchOnTarget; };
+ };
+ check = ''
+ gc
+ expect_eq msg "$(nix_eval ${flake "x"}#msg)" top:dep+alt,sub
+ '';
+ };
+ }))
+ # The lock file gets rewritten or generated again when a source inputs
+ # get drawn from changes.
+ (modes "relock" ({ draw, flake, ... }: let
+ x.file = { path = "/fixtures/filesrc"; } // draw { dep.follows = "a"; };
+ in {
+ phases = [
+ {
+ sources = { inherit x; a.file = "/fixtures/alt"; };
+ check = ''
+ expect_eq msg "$(nix_eval_online ${flake "x"}#msg)" top:alt,sub
+ '';
+ }
+ {
+ sources = { inherit x; a.file = "/fixtures/alt2"; };
+ check = ''
+ expect_eq msg "$(nix_eval_online ${flake "x"}#msg)" top:alt2,sub
+ '';
+ }
+ ];
+ }))
+ # What drawing inputs needs gets removed once a source draws no inputs
+ # anymore.
+ (modes "removal" ({ draw, mode, ... }: let
+ state = {
+ rewrite = "derived/x/result";
+ generated = "flakes/x/flake.lock";
+ }.${mode};
+ in {
+ phases = [
+ {
+ sources = alt // {
+ x.flake = { url = "/fixtures/top"; inherit fetchOnTarget; } // draw { dep.follows = "alt"; };
+ };
+ check = ''
+ test -e "$T"/.populate/${state} || fail "${state} missing"
+ '';
+ }
+ {
+ sources = alt // {
+ x.flake = { url = "/fixtures/top"; inherit fetchOnTarget; };
+ };
+ check = ''
+ test ! -e "$T"/.populate/${dirOf state} || fail "${dirOf state} remains"
+ '' + checkFlake { attr = "msg"; expected = "top:dep+inner,sub"; };
+ }
+ ];
+ }))
+ # Patches get applied in order to a copy, which becomes a store path and
+ # gets made again only when the patches change.
+ (s "patches" (let
+ x = patches: {
+ x.flake-input = {
+ url = "/fixtures/top"; name = "dep"; inherit fetchOnTarget patches;
+ };
+ };
+ both = x [ "/fixtures/patches/hello.patch" "/fixtures/patches/hello2.patch" ];
+ in {
+ phases = [
+ {
+ sources = both;
+ check = checkFlake { attr = "greeting"; expected = "hello2+inner"; };
+ # The key changes whenever the copy gets made again, which
+ # the store path doesn't show, being the same.
+ mutate = ''
+ readlink "$T"/x > "$T".first
+ stat -c %y "$T"/.populate/derived/x/key > "$T".key
+ '';
+ }
+ {
+ sources = both;
+ check = checkFlake { attr = "greeting"; expected = "hello2+inner"; } + ''
+ expect_eq "store path" "$(readlink "$T"/x)" "$(cat "$T".first)"
+ expect_eq "copy made at" "$(stat -c %y "$T"/.populate/derived/x/key)" "$(cat "$T".key)"
+ '';
+ }
+ {
+ sources = x [ "/fixtures/patches/hello.patch" ];
+ check = checkFlake { attr = "greeting"; expected = "hello+inner"; } + ''
+ test "$(readlink "$T"/x)" != "$(cat "$T".first)" || fail "store path unchanged"
+ expect_eq patches "$(ls "$T"/.populate/patches/x)" 0001-hello.patch
+ '';
+ }
+ {
+ sources = x [];
+ check = checkFlake { attr = "greeting"; expected = "dep+inner"; } + ''
+ test ! -e "$T"/.populate/patches || fail "patches remain"
+ test ! -e "$T"/.populate/derived || fail "derived remains"
+ '';
+ }
+ ];
+ }))
+ (s "patch-fails" {
+ phases = [{
+ sources.x.flake-input = {
+ url = "/fixtures/top"; name = "dep"; inherit fetchOnTarget;
+ patches = [ "/fixtures/patches/bad.patch" ];
+ };
+ expect.fail = "error: patch bad.patch does not apply to source x";
+ }];
+ })
+ # Patches apply in the subdirectory the target path shows.
+ (s "patch-subdir" {
+ phases = twice {
+ sources.x.flake-input = {
+ url = "/fixtures/top"; name = "sub"; inherit fetchOnTarget;
+ patches = [ "/fixtures/patches/sub.patch" ];
+ };
+ check = checkFlake { attr = "who"; expected = "patched"; subdir = true; };
+ };
+ })
+ # A copy with a rewritten lock file gets made again only when a source
+ # it draws inputs from changes.
+ (s "rewrite-cache" (let
+ x.flake-input = {
+ url = "/fixtures/top"; name = "dep"; inherit fetchOnTarget;
+ inputs.inner.follows = "a";
+ };
+ in {
+ phases = [
+ {
+ sources = { inherit x; a.file = "/fixtures/alt"; };
+ check = checkFlake { attr = "greeting"; expected = "dep+alt"; };
+ # The key changes whenever the copy gets made again, which
+ # the store path doesn't show, being the same.
+ mutate = ''
+ readlink "$T"/x > "$T".first
+ stat -c %y "$T"/.populate/derived/x/key > "$T".key
+ '';
+ }
+ {
+ sources = { inherit x; a.file = "/fixtures/alt"; };
+ check = checkFlake { attr = "greeting"; expected = "dep+alt"; } + ''
+ expect_eq "store path" "$(readlink "$T"/x)" "$(cat "$T".first)"
+ expect_eq "copy made at" "$(stat -c %y "$T"/.populate/derived/x/key)" "$(cat "$T".key)"
+ '';
+ }
+ {
+ sources = { inherit x; a.file = "/fixtures/alt2"; };
+ check = checkFlake { attr = "greeting"; expected = "dep+alt2"; } + ''
+ test "$(readlink "$T"/x)" != "$(cat "$T".first)" || fail "store path unchanged"
+ '';
+ }
+ ];
+ }))
+ # A materialized input is a writable copy, patched and with a rewritten
+ # lock file, which gets made again only when what it derives from
+ # changes, and becomes a link again when not materialized anymore.
+ (s "materialize" (let
+ x = patches: alt // {
+ x.flake-input = {
+ url = "/fixtures/top"; name = "dep"; inherit fetchOnTarget patches;
+ materialize = true;
+ inputs.inner.follows = "alt";
+ };
+ };
+ hello = [ "/fixtures/patches/hello.patch" ];
+ in {
+ phases = [
+ {
+ sources = x hello;
+ check = ''
+ test ! -L "$T"/x || fail "x is a link"
+ test -w "$T"/x/flake.nix || fail "flake.nix isn't writable"
+ gc
+ expect_eq greeting "$(nix_eval "$T"/x#greeting)" hello+alt
+ '';
+ mutate = ''
+ echo local > "$T"/x/local
+ '';
+ }
+ {
+ sources = x hello;
+ check = ''
+ expect_eq local "$(cat "$T"/x/local)" local
+ '';
+ }
+ {
+ sources = x (hello ++ [ "/fixtures/patches/hello2.patch" ]);
+ check = ''
+ test ! -e "$T"/x/local || fail "local remains"
+ gc
+ expect_eq greeting "$(nix_eval "$T"/x#greeting)" hello2+alt
+ '';
+ }
+ {
+ sources = alt // {
+ x.flake-input = { url = "/fixtures/top"; name = "dep"; inherit fetchOnTarget; };
+ };
+ check = checkFlake { attr = "greeting"; expected = "dep+inner"; } + ''
+ test ! -e "$T"/.populate/derived || fail "derived remains"
+ '';
+ }
+ ];
+ }))
+ # Roots follow the delivered input.
+ (s "retarget" {
+ phases = [
+ {
+ sources.x.flake-input = { url = "/fixtures/top"; name = "dep"; inherit fetchOnTarget; };
+ check = ''
+ expect_eq roots "$(ls "$T"/.populate/flake-inputs/x | wc -l)" 1
+ '';
+ }
+ {
+ sources.x.flake-input = { url = "/fixtures/top"; name = "sub"; inherit fetchOnTarget; };
+ check = ''
+ test ! -e "$T"/.populate/flake-inputs/x || fail "roots remain"
+ test ! -e "$T"/.populate/flake-inputs || fail "empty roots directory remains"
+ '' + checkFlake { attr = "who"; expected = "sub"; subdir = true; };
+ }
+ ];
+ })
+ # Pruning removes everything sources don't deliver anymore, including
+ # garbage collector roots, and keeps everything else.
+ (s "prune" (let
+ sources = {
+ a.file = "/fixtures/alt";
+ b.flake = { url = "/fixtures/top"; inherit fetchOnTarget; };
+ c.flake-input = { url = "/fixtures/top"; name = "dep"; inherit fetchOnTarget; };
+ d.symlink = "a";
+ };
+ in {
+ prepare = ''
+ mkdir -p "$T"
+ echo other > "$T"/other
+ '';
+ phases = [
+ {
+ inherit sources;
+ check = ''
+ test -e "$T"/.populate/flake-inputs/b || fail "roots of b missing"
+ test -e "$T"/.populate/flake-inputs/c || fail "roots of c missing"
+ '';
+ }
+ {
+ # Without pruning, nothing gets removed.
+ sources = { inherit (sources) a; };
+ check = ''
+ for name in b c d other .populate/flake-inputs/b .populate/flake-inputs/c; do
+ test -e "$T"/$name || fail "$name removed"
+ done
+ '';
+ # An upload an interrupted run left behind.
+ mutate = ''
+ mkdir -p "$T"/.populate/flake-uploads/c/tree
+ '';
+ }
+ {
+ # b changes its type, so its roots go, while b stays.
+ sources = { inherit (sources) a; b.symlink = "a"; };
+ prune = true;
+ output = map
+ (name: "pruning '${targetName targetKind "/var/krops/prune-${targetKind}/${name}"}'...")
+ [ "c" "d" "other" ];
+ check = ''
+ test -e "$T"/a/flake.nix || fail "a removed"
+ expect_eq b "$(readlink "$T"/b)" a
+ test -d "$T"/.populate || fail ".populate removed"
+ for name in c d other .populate/flake-inputs .populate/flake-uploads; do
+ test ! -e "$T"/$name || fail "$name not pruned"
+ done
+ if nix-store --gc --print-roots 2>/dev/null | grep -F "$T/"; then
+ fail "garbage collector roots remain"
+ fi
+ '';
+ }
+ ];
+ }))
+ (s "cycle" {
+ phases = [{
+ sources = {
+ a.file = { path = "/fixtures/alt"; inputs.foo.follows = "b"; };
+ b.file = { path = "/fixtures/alt"; inputs.foo.follows = "a"; };
+ };
+ expect = "eval-error";
+ refused = "krops: sources draw inputs from each other: a b";
+ control = {
+ a.file = { path = "/fixtures/alt"; inputs.foo.follows = "b"; };
+ b.file = "/fixtures/alt";
+ };
+ }];
+ })
+ (s "self" {
+ phases = [{
+ sources.a.file = { path = "/fixtures/alt"; inputs.foo.follows = "a"; };
+ expect = "eval-error";
+ refused = "krops: source a draws inputs from itself";
+ control = alt // {
+ a.file = { path = "/fixtures/alt"; inputs.foo.follows = "alt"; };
+ };
+ }];
+ })
+ (s "missing" {
+ phases = [{
+ sources.a.file = { path = "/fixtures/alt"; inputs.foo.follows = "nope"; };
+ expect = "eval-error";
+ refused = "krops: source a draws inputs from missing source nope";
+ control = {
+ a.file = { path = "/fixtures/alt"; inputs.foo.follows = "nope"; };
+ nope.file = "/fixtures/alt";
+ };
+ }];
+ })
+ (s "noflake" {
+ phases = [{
+ sources = alt // {
+ x.file = { path = "/fixtures/noflake"; inputs.foo.follows = "alt"; };
+ };
+ expect.fail = "error: source x draws flake inputs from other sources, but contains no flake.nix";
+ }];
+ })
+ (s "noinput" {
+ phases = [{
+ sources.x.flake-input = { url = "/fixtures/top"; name = "nope"; inherit fetchOnTarget; };
+ expect.fail = "error: flake /fixtures/top has no input nope";
+ }];
+ })
+ # rsync's log gives sizes in units of 1024, as %'''l asks for.
+ (s "rsync-units" {
+ prepareDeployer = ''
+ rm -fR /tmp/rsync-units
+ mkdir /tmp/rsync-units
+ head -c 2048 /dev/zero > /tmp/rsync-units/f
+ '';
+ phases = [{
+ sources.x.file = "/tmp/rsync-units";
+ output = [ "/2.00K" ];
+ }];
+ })
+ # A pipe command without #!, which a shell runs as a script.
+ (s "pipe-script" {
+ prepareDeployer = ''
+ printf 'echo piped\n' > /tmp/pipe-script
+ chmod +x /tmp/pipe-script
+ '';
+ phases = [{
+ sources.x.pipe = "/tmp/pipe-script";
+ check = ''
+ expect_eq x "$(cat "$T"/x)" piped
+ '';
+ }];
+ })
+ # A pass source gets decrypted again only when the last commit of its
+ # passwords changes, which .pass_info records.
+ (s "pass-changes" (let
+ sources.x.pass = { dir = "/tmp/pass-changes"; name = "x"; };
+ in {
+ prepareDeployer = ''
+ rm -fR /tmp/pass-changes
+ cp -R /fixtures/pass-git /tmp/pass-changes
+ '';
+ phases = [
+ {
+ inherit sources;
+ check = ''
+ expect_eq a "$(cat "$T"/x/a)" secret-a
+ '';
+ mutate = ''
+ echo local-change > "$T"/x/a
+ '';
+ }
+ {
+ inherit sources;
+ check = ''
+ expect_eq a "$(cat "$T"/x/a)" local-change
+ '';
+ mutateDeployer = ''
+ export GIT_AUTHOR_NAME=krops GIT_AUTHOR_EMAIL=spam@krebsco.de
+ export GIT_COMMITTER_NAME=krops GIT_COMMITTER_EMAIL=spam@krebsco.de
+ printf 'secret-b\n' |
+ PASSWORD_STORE_DIR=/tmp/pass-changes pass insert -m -f x/a >/dev/null
+ '';
+ }
+ {
+ inherit sources;
+ check = ''
+ expect_eq a "$(cat "$T"/x/a)" secret-b
+ '';
+ }
+ ];
+ }))
+ # An input locked to a path the target doesn't have.
+ (s "gone" {
+ phases = [{
+ sources.x.flake = { url = "/fixtures/usegone"; fetchOnTarget = true; };
+ expect.fail = "error: flake /fixtures/usegone: input gone refers to /fixtures/gone, which does not exist on the target";
+ }];
+ })
+ ]) targetKinds;
+
+ # Edge cases of the flake source types and of generated flakes: unusual lock
+ # files, references, and changes between deploys.
+ edge-cases = concatMap (targetKind: let
+ s = name: args: scenario ({
+ name = "${name}-${targetKind}";
+ inherit targetKind;
+ isolate = true;
+ } // args);
+ copying = targetKind != "sudo";
+ fetchOnTarget = !copying;
+ roots = n: ''
+ expect_eq roots "$(ls "$T"/.populate/flake-inputs/x 2>/dev/null | wc -l)" ${toString n}
+ '';
+ # A scenario for each way of drawing inputs, which f gets as mode, along
+ # with draw, which adds inputs drawn that way to a source's options, and
+ # flake, which gives the flake to evaluate for a source.
+ modes = name: f: map
+ (mode: s "${name}-${mode}" (f {
+ draw = drawInputs mode;
+ flake = n:
+ if mode == "generated" then ''"$T"/.populate/flakes/${n}'' else ''"$T"/${n}'';
+ inherit mode;
+ }))
+ [ "rewrite" "generated" ];
+ in lib.flatten [
+ # The registry entry kropsremote, which fixtures.sh adds on the deploying
+ # machine, points at ${url}/remote.tar.gz. The deploying machine has no
+ # global registry, which it couldn't download.
+ (s "indirect" {
+ phases = twice {
+ sources.x.flake = { url = "kropsremote"; inherit fetchOnTarget; };
+ check = checkFlake { attr = "msg"; expected = "remote:dep+inner"; };
+ };
+ })
+ (s "remote-subdir" {
+ phases = twice {
+ sources.x.flake = { url = "${url}/mono.tar.gz?dir=sub"; inherit fetchOnTarget; };
+ check = checkFlake { attr = "who"; expected = "sub"; subdir = true; };
+ };
+ })
+ (modes "subdir-inputs" ({ draw, mode, ... }: {
+ phases = twice {
+ sources = alt // {
+ x.flake = {
+ url = "git+file:/fixtures/monoin?dir=sub"; inherit fetchOnTarget;
+ } // draw { dep.follows = "alt"; };
+ };
+ check = checkFlake {
+ attr = "msg"; expected = "monoin:alt"; subdir = true;
+ generated = mode == "generated";
+ };
+ };
+ }))
+ (s "dirty" {
+ phases = twice {
+ sources.x.flake = { url = "/fixtures/dirty"; inherit fetchOnTarget; };
+ check = checkFlake { attr = "msg"; expected = "dirty:dep+inner,sub"; } + ''
+ test ! -e "$T"/x/untracked || fail "untracked file delivered"
+ '';
+ };
+ })
+ (s "unlocked" {
+ phases = [{
+ sources.x.flake = { url = "/fixtures/unlocked"; inherit fetchOnTarget; };
+ expect.fail = "requires lock file changes";
+ }];
+ })
+ (s "notdir" {
+ phases = [{
+ sources = {
+ x.file = { path = "/fixtures/filesrc"; inputs.dep.follows = "p"; };
+ p.pipe = "/fixtures/pipe";
+ };
+ expect.fail = "error: source p is not a directory";
+ # Locking the flake with an empty URL would lock the current
+ # directory, the login directory on a remote target.
+ unexpected = [ "while fetching the input" ];
+ }];
+ })
+ (s "noinput-nested" {
+ phases = [{
+ sources.x.flake-input = { url = "/fixtures/top"; name = "dep/nope"; inherit fetchOnTarget; };
+ expect.fail = "error: flake /fixtures/top has no input dep/nope";
+ }];
+ })
+ (s "shared" {
+ phases = twice {
+ sources = {
+ x.flake = { url = "/fixtures/top"; inherit fetchOnTarget; };
+ dep.flake-input = { url = "/fixtures/top"; name = "dep"; inherit fetchOnTarget; };
+ sub.flake-input = { url = "/fixtures/top"; name = "sub"; fetchOnTarget = true; };
+ };
+ check = checkFlake { attr = "msg"; expected = "top:dep+inner,sub"; } + ''
+ expect_eq dep "$(nix_eval "$T"/dep#greeting)" dep+inner
+ expect_eq sub "$(nix_eval "$T"/sub#who)" sub
+ '';
+ };
+ })
+ (s "follows-empty" {
+ phases = twice {
+ sources.x.flake = { url = "/fixtures/nofollow"; fetchOnTarget = true; };
+ check = checkFlake { attr = "msg"; expected = "nofollow"; };
+ };
+ })
+ (s "follows-cycle" {
+ phases = twice {
+ sources.x.flake = { url = "/fixtures/cyc"; fetchOnTarget = true; };
+ check = checkFlake { attr = "msg"; expected = "cyc:cyca,cycb"; };
+ };
+ })
+ # Names resolve through the parent's lock file, so dep/inner is the
+ # parent's inner, which dep's inner follows.
+ (s "follows-name" {
+ phases = twice {
+ sources.x.flake-input = { url = "/fixtures/follow"; name = "dep/inner"; inherit fetchOnTarget; };
+ check = checkFlake { attr = "who"; expected = "inner2"; };
+ };
+ })
+ # The delivered input comes with the inputs its own lock file pins, which
+ # evaluating it needs, not with the parent's inner, which it follows there.
+ (s "follows-closure" {
+ phases = twice {
+ sources.x.flake-input = { url = "/fixtures/follow"; name = "dep"; inherit fetchOnTarget; };
+ check = checkFlake { attr = "greeting"; expected = "dep+inner"; } + roots 1;
+ };
+ })
+ (s "relative-input" {
+ phases = twice {
+ sources.x.flake = { url = "/fixtures/rel"; fetchOnTarget = true; };
+ check = checkFlake { attr = "msg"; expected = "rel:lib"; };
+ };
+ })
+ # The target can't fetch an input of the flake. The deploying machine
+ # fetched it when locking, so its fetcher cache has to go.
+ (s "fetch-failure" {
+ prepareDeployer = ''
+ rm -fR /root/.cache/nix
+ '';
+ phases = [{
+ sources.x.flake = { url = "/fixtures/broken"; fetchOnTarget = true; };
+ expect.fail = "vanishing.tar.gz";
+ }];
+ })
+ # The delivered input itself is locked to a path the target lacks.
+ (s "gone-input" {
+ phases = [{
+ sources.x.flake-input = { url = "/fixtures/usegone"; name = "gone"; fetchOnTarget = true; };
+ expect.fail = "input gone refers to /fixtures/gone, which does not exist on the target";
+ }];
+ })
+ # An input that gets drawn from a source isn't needed on the target, with
+ # all it pins.
+ (modes "overridden-roots" ({ draw, mode, ... }: {
+ # What gets delivered doesn't depend on how inputs get drawn.
+ exhaustive = mode == "generated";
+ phases = twice {
+ sources = alt // {
+ x.flake-input = {
+ url = "/fixtures/top"; name = "dep"; inherit fetchOnTarget;
+ } // draw { inner.follows = "alt"; };
+ };
+ check = checkFlake {
+ attr = "greeting"; expected = "dep+alt";
+ generated = mode == "generated";
+ } + roots 0;
+ };
+ }))
+ # The input's own lock file pins something unreachable, which gets drawn
+ # from a source anyway.
+ (modes "own-lock-unreachable" ({ draw, mode, ... }: {
+ # What gets delivered doesn't depend on how inputs get drawn.
+ exhaustive = mode == "generated";
+ prepareDeployer = ''
+ rm -fR /root/.cache/nix
+ '';
+ phases = twice {
+ sources = alt // {
+ x.flake-input = {
+ url = "/fixtures/usepinned"; name = "dep"; inherit fetchOnTarget;
+ } // draw { inner.follows = "alt"; };
+ };
+ check = checkFlake {
+ attr = "greeting"; expected = "pinned+alt";
+ generated = mode == "generated";
+ };
+ };
+ }))
+ # The input's own lock file pins something unreachable, or lacks an input
+ # its flake.nix has. That fails, unless the input isn't required to come
+ # with its inputs.
+ (s "own-lock-broken" (let
+ x = url: requireInputs: {
+ x.flake-input = { inherit url; name = "dep"; inherit fetchOnTarget requireInputs; };
+ };
+ delivered = ''
+ test -e "$T"/x/flake.nix || fail "input not delivered"
+ '';
+ in {
+ prepareDeployer = ''
+ rm -fR /root/.cache/nix
+ '';
+ phases = [
+ {
+ sources = x "/fixtures/usepinned" true;
+ expect.fail = "error: source x: the inputs its own lock file pins can't be delivered. Set its requireInputs to false to deliver it anyway.";
+ }
+ {
+ sources = x "/fixtures/usepinned" false;
+ check = delivered + roots 0;
+ }
+ {
+ sources = x "/fixtures/usestale" true;
+ expect.fail = "error: source x: the inputs its own lock file pins can't be delivered. Set its requireInputs to false to deliver it anyway.";
+ }
+ {
+ sources = x "/fixtures/usestale" false;
+ check = delivered + roots 2;
+ }
+ {
+ sources = x "/fixtures/usestale" false;
+ check = delivered + roots 2;
+ }
+ ];
+ }))
+ # Redeploying an unchanged input while where it comes from is
+ # unreachable and the fetcher caches are gone: the store has it.
+ (s "offline-redeploy" (let
+ sources.x.flake-input = { url = "/fixtures/offline"; name = "gap"; inherit fetchOnTarget; };
+ check = checkFlake { attr = "who"; expected = "gap"; };
+ in {
+ prepareDeployer = ''
+ if test -e /fixtures/www/gap.tar.gz.off; then
+ mv /fixtures/www/gap.tar.gz.off /fixtures/www/gap.tar.gz
+ fi
+ '';
+ phases = [
+ {
+ inherit sources check;
+ mutate = ''
+ rm -fR /root/.cache/nix
+ '';
+ mutateDeployer = ''
+ rm -fR /root/.cache/nix
+ mv /fixtures/www/gap.tar.gz /fixtures/www/gap.tar.gz.off
+ '';
+ }
+ {
+ inherit sources check;
+ mutateDeployer = ''
+ mv /fixtures/www/gap.tar.gz.off /fixtures/www/gap.tar.gz
+ '';
+ }
+ ];
+ }))
+ (s "flake-to-file" {
+ phases = [
+ {
+ sources.x.flake = { url = "/fixtures/top"; inherit fetchOnTarget; };
+ check = roots 3;
+ mutate = ''
+ readlink "$T"/x > "$T"/old
+ '';
+ }
+ {
+ sources.x.file = "/fixtures/alt";
+ check = ''
+ test ! -L "$T"/x || fail "x is still a link to $(readlink "$T"/x)"
+ test -e "$T"/x/flake.nix || fail "flake.nix missing"
+ nix-store --verify-path "$(cat "$T"/old)" || fail "store path changed"
+ '' + roots 0;
+ }
+ ];
+ })
+ (s "symlink-to-flake" {
+ phases = [
+ {
+ sources = alt // { x.symlink = "alt"; };
+ check = ''
+ expect_eq link "$(readlink "$T"/x)" alt
+ '';
+ }
+ {
+ sources = alt // { x.flake = { url = "/fixtures/top"; inherit fetchOnTarget; }; };
+ check = checkFlake { attr = "msg"; expected = "top:dep+inner,sub"; } + ''
+ test -e "$T"/alt/flake.nix || fail "alt changed"
+ '';
+ }
+ ];
+ })
+ # The ssh master stops, after failing as well as after succeeding.
+ (s "ssh-master" (let
+ checkDeployer = ''
+ for i in 1 2 3 4 5 6 7 8 9 10; do
+ if ! pgrep -f 'ssh: .*/populate\..*\[mux\]' >/dev/null; then
+ exit 0
+ fi
+ sleep 0.5
+ done
+ fail "ssh master remains"
+ '';
+ in {
+ phases = [
+ {
+ sources.x.flake-input = { url = "/fixtures/top"; name = "nope"; inherit fetchOnTarget; };
+ expect.fail = "error: flake /fixtures/top has no input nope";
+ inherit checkDeployer;
+ }
+ {
+ sources = alt;
+ inherit checkDeployer;
+ }
+ ];
+ }))
+ # Pruning entries whose names a shell's read would strip or split.
+ (s "prune-names" {
+ prepare = ''
+ mkdir -p "$T/ lead" "$T/trail " "$T/$(printf 'new\nline')" "$T/axb" "$T/[b]"
+ '';
+ phases = [{
+ sources = alt;
+ prune = true;
+ check = ''
+ # In the C locale, ls sorts by bytes, so . comes first.
+ expect_eq entries "$(LC_ALL=C ls -A "$T" | tr '\n' ' ')" '.populate alt '
+ '';
+ }];
+ })
+ # Misspelt input names draw nothing, so they fail before anything
+ # changes: of the source's flake, and below an input drawn from a source.
+ (modes "inputs-typo" ({ draw, ... }: {
+ phases = [
+ {
+ sources = alt // {
+ x.flake-input = {
+ url = "/fixtures/top"; name = "dep"; inherit fetchOnTarget;
+ } // draw { innr.follows = "alt"; };
+ };
+ expect.fail = "error: source x draws input innr from source alt, but its flake has no input innr";
+ }
+ {
+ sources = {
+ x.flake = {
+ url = "/fixtures/top"; inherit fetchOnTarget;
+ } // draw { dep = { follows = "b"; inputs.qq.follows = "c2"; }; };
+ b.file = "/fixtures/two";
+ c2.file = "/fixtures/c2";
+ };
+ expect.fail = "error: source x draws input dep/qq from source c2, but the flake of source b has no input qq";
+ }
+ ];
+ }))
+ # A signal ending populate leaves neither the ssh master nor the
+ # temporary directory.
+ (s "interrupt" {
+ prepareDeployer = ''
+ find "''${TMPDIR:-/tmp}" -maxdepth 1 -name 'populate.*' -exec rm -fR {} +
+ '';
+ phases = [{
+ sources.x.pipe = "/fixtures/pipe-slow";
+ interrupt = 3;
+ checkDeployer = ''
+ for i in 1 2 3 4 5 6 7 8 9 10; do
+ if ! pgrep -f 'ssh: .*/populate\..*\[mux\]' >/dev/null; then
+ break
+ fi
+ sleep 0.5
+ done
+ ! pgrep -f 'ssh: .*/populate\..*\[mux\]' >/dev/null ||
+ fail "ssh master remains"
+ expect_eq "temporary directories" \
+ "$(find "''${TMPDIR:-/tmp}" -maxdepth 1 -name 'populate.*')" ""
+ '';
+ }];
+ })
+ # An input of an input drawn from a source that draws inputs itself
+ # gets drawn from another source.
+ (modes "override" ({ draw, flake, ... }: {
+ # Drawing inputs happens on the target alike for every kind of target.
+ exhaustive = targetKind != "local";
+ phases = twice {
+ sources = {
+ x.flake = {
+ url = "/fixtures/top"; inherit fetchOnTarget;
+ } // draw { dep = { follows = "b"; inputs.q.follows = "c2"; }; };
+ b.file = { path = "/fixtures/two"; } // draw { p.follows = "c1"; };
+ c1.file = "/fixtures/c1";
+ c2.file = "/fixtures/c2";
+ };
+ check = ''
+ expect_eq msg "$(nix_eval_online ${flake "x"}#msg)" top:two:c1+c2,sub
+ '';
+ };
+ }))
+ # The same one level deeper: the input gets drawn from a source below
+ # an input that the source in between draws from a source itself.
+ (modes "override-deep" ({ draw, flake, ... }: {
+ # Drawing inputs happens on the target alike for every kind of target.
+ exhaustive = targetKind != "local";
+ phases = twice {
+ sources = {
+ x.flake = {
+ url = "/fixtures/top"; inherit fetchOnTarget;
+ } // draw { dep = { follows = "b"; inputs.p.inputs.z.follows = "c2"; }; };
+ b.file = { path = "/fixtures/two"; } // draw { p.follows = "c3"; };
+ c2.file = "/fixtures/c2";
+ c3.file = "/fixtures/c3";
+ };
+ check = ''
+ expect_eq msg "$(nix_eval_online ${flake "x"}#msg)" 'top:two:c3(c2)+inner,sub'
+ '';
+ };
+ }))
+ # A source with a rewritten lock file draws an input from a source with
+ # a generated flake, and an input below it from another source.
+ (s "override-mixed" {
+ # Drawing inputs happens on the target alike for every kind of target.
+ exhaustive = targetKind != "local";
+ phases = twice {
+ sources = {
+ x.flake = {
+ url = "/fixtures/top"; inherit fetchOnTarget;
+ inputs.dep = { follows = "b"; inputs.q.follows = "c2"; };
+ };
+ b.file = {
+ path = "/fixtures/two";
+ rewriteLock = false;
+ inputs.p.follows = "c1";
+ };
+ c1.file = "/fixtures/c1";
+ c2.file = "/fixtures/c2";
+ };
+ check = ''
+ expect_eq msg "$(nix_eval_online "$T"/x#msg)" top:two:c1+c2,sub
+ '';
+ };
+ })
+ (modes "nonflake-input" ({ draw, flake, ... }: {
+ # Drawing inputs happens on the target alike for every kind of target.
+ exhaustive = targetKind != "local";
+ phases = twice {
+ sources = {
+ x.file = { path = "/fixtures/usedata"; } // draw { data.follows = "data"; };
+ data.file = "/fixtures/data";
+ };
+ check = ''
+ expect_eq msg "$(nix_eval ${flake "x"}#msg)" data:hello
+ '';
+ };
+ }))
+ # The target directory in a Git work tree.
+ (modes "gitdir" ({ draw, flake, ... }: {
+ prepare = ''
+ mkdir -p "$T"
+ git -C "$T" init -q
+ '';
+ phases = twice {
+ sources = alt // {
+ x.file = { path = "/fixtures/filesrc"; } // draw { dep.follows = "alt"; };
+ };
+ check = ''
+ # Not just the path, which Nix takes for the enclosing work tree.
+ expect_eq msg "$(nix_eval_online path:${flake "x"}#msg)" top:alt,sub
+ '';
+ };
+ }))
+ # Output of the same length, which rsync's quick check doesn't notice.
+ (s "pipe-change" {
+ prepareDeployer = ''
+ echo piped > /fixtures/pipe2-data
+ '';
+ phases = [
+ {
+ sources.x.pipe = "/fixtures/pipe2";
+ check = ''
+ expect_eq x "$(cat "$T"/x)" piped
+ '';
+ mutateDeployer = ''
+ echo PIPED > /fixtures/pipe2-data
+ '';
+ }
+ {
+ sources.x.pipe = "/fixtures/pipe2";
+ check = ''
+ expect_eq x "$(cat "$T"/x)" PIPED
+ '';
+ mutateDeployer = ''
+ echo piped > /fixtures/pipe2-data
+ '';
+ }
+ ];
+ })
+ # Source names that aren't input names, or that Nix has another use
+ # for, both of sources inputs get drawn from and of sources drawing them.
+ (modes "source-name" ({ draw, flake, ... }: let
+ x = from: { path = "/fixtures/filesrc"; } // draw { dep.follows = from; };
+ in {
+ # Drawing inputs happens on the target alike for every kind of target.
+ exhaustive = targetKind != "local";
+ phases = twice {
+ sources = {
+ "alt-1.0".file = "/fixtures/alt";
+ self.file = "/fixtures/alt";
+ x.file = x "alt-1.0";
+ y.file = x "self";
+ "a.b_c-d".file = x "self";
+ };
+ check = ''
+ expect_eq x "$(nix_eval_online ${flake "x"}#msg)" top:alt,sub
+ expect_eq y "$(nix_eval_online ${flake "y"}#msg)" top:alt,sub
+ expect_eq a.b_c-d "$(nix_eval_online ${flake "a.b_c-d"}#msg)" top:alt,sub
+ '';
+ };
+ }))
+ # An upload an interrupted run left behind.
+ (s "upload-leftover" {
+ phases = [
+ {
+ sources.x.flake = { url = "/fixtures/top"; fetchOnTarget = true; };
+ check = checkFlake { attr = "msg"; expected = "top:dep+inner,sub"; };
+ mutate = ''
+ mkdir -p "$T"/.populate/flake-uploads/x/tree
+ echo half > "$T"/.populate/flake-uploads/x/tree/half
+ '';
+ }
+ {
+ sources.x.flake = { url = "/fixtures/top"; fetchOnTarget = true; };
+ check = checkFlake { attr = "msg"; expected = "top:dep+inner,sub"; };
+ }
+ ];
+ })
+ ] ++ lib.optional (targetKind == "local")
+ # The target has the path the input is locked to.
+ (s "uselocal" {
+ phases = twice {
+ sources.x.flake = { url = "/fixtures/uselocal"; fetchOnTarget = true; };
+ check = checkFlake { attr = "msg"; expected = "uselocal:alt"; };
+ };
+ })
+ ) targetKinds;
+
+
+ # Populate replaces and removes things in the target directory. These
+ # scenarios check that it doesn't anywhere else: names and paths that lead
+ # out of the target directory don't evaluate, and symlinks in it get
+ # replaced or refused instead of followed. The canaries, which every phase
+ # of every scenario checks, tell if something outside changed; these
+ # scenarios point links at them.
+ safety = concatMap (targetKind: let
+ s = name: args: scenario ({
+ name = "${name}-${targetKind}";
+ inherit targetKind;
+ } // args);
+ copying = targetKind != "sudo";
+ fetchOnTarget = !copying;
+ outside = "/var/krops-outside";
+ hello = [ "/fixtures/patches/hello.patch" ];
+ # Sources that use every kind of state populate keeps: x a copy with
+ # patches, y garbage collector roots and, where the flake gets uploaded,
+ # an upload, and z a generated flake.
+ stateful = alt // {
+ x.flake-input = {
+ url = "/fixtures/top"; name = "dep"; inherit fetchOnTarget;
+ patches = hello;
+ inputs.inner.follows = "alt";
+ };
+ y.flake = { url = "/fixtures/top"; inherit fetchOnTarget; };
+ z.file = {
+ path = "/fixtures/filesrc";
+ rewriteLock = false;
+ inputs.dep.follows = "alt";
+ };
+ };
+ # A source deployed where a symlink to something outside is. The link
+ # gets replaced, never followed.
+ link = type: { sources, to, check, says ? true }: let
+ phase = {
+ inherit sources;
+ check = ''
+ case $(readlink "$T"/x || :) in
+ ${outside}/*) fail "x still links outside" ;;
+ esac
+ '' + check;
+ };
+ in s "link-${type}" {
+ isolate = true;
+ prepare = ''
+ mkdir -p "$T"
+ ln -s ${outside}/${to} "$T"/x
+ '';
+ phases = [
+ (phase // {
+ output = optional says
+ "replacing symlink '${targetName targetKind "/var/krops/link-${type}-${targetKind}/x"}', which points to '${outside}/${to}'...";
+ })
+ phase
+ ];
+ };
+ # Something of populate's state replaced by a symlink to outside between
+ # two deploys. The second one refuses to use it.
+ state = name: {
+ entry, backup ? false,
+ message ? "/${entry} is a symlink, but populate needs a directory of its own there",
+ }: s "state-${name}" {
+ isolate = true;
+ phases = [
+ {
+ sources = stateful;
+ inherit backup;
+ mutate = ''
+ rm -fR "$T"/${entry}
+ mkdir -p "$(dirname "$T"/${entry})"
+ ln -s ${outside}/dir "$T"/${entry}
+ '';
+ }
+ {
+ sources = stateful;
+ inherit backup;
+ expect.fail = message;
+ }
+ ];
+ };
+ in lib.flatten [
+ (link "file" {
+ sources.x.file = "/fixtures/alt";
+ to = "dir";
+ check = ''
+ test -e "$T"/x/flake.nix || fail "flake.nix missing"
+ '';
+ })
+ # What rsync logs about each item: what changed, and its size.
+ (s "rsync-log" {
+ phases = [{
+ sources.x.file = "/fixtures/alt";
+ output = [
+ "f+++++++++ flake.nix " "/53"
+ # Named like pruning names it.
+ "populating '${targetName targetKind "/var/krops/rsync-log-${targetKind}/x"}'..."
+ ];
+ }];
+ })
+ # The link points at a Git work tree, which resetting and cleaning would
+ # change.
+ (link "git" {
+ sources.x.git = { url = gitUrl; ref = gitHead; };
+ to = "repo";
+ check = ''
+ expect_eq HEAD "$(git -C "$T"/x rev-parse HEAD)" ${gitHead}
+ '';
+ })
+ (link "pass" {
+ sources.x.pass = { dir = "/fixtures/pass"; name = "x"; };
+ to = "dir";
+ check = ''
+ expect_eq a "$(cat "$T"/x/a)" secret-a
+ '';
+ })
+ (link "passage" {
+ sources.x.passage = {
+ dir = "/fixtures/passage/store/x";
+ identities_file = "/fixtures/passage/identities";
+ };
+ to = "dir";
+ check = ''
+ expect_eq a "$(cat "$T"/x/a)" secret-a
+ '';
+ })
+ (link "pipe" {
+ sources.x.pipe = "/fixtures/pipe";
+ to = "file";
+ check = ''
+ expect_eq x "$(cat "$T"/x)" piped
+ '';
+ })
+ (link "derivation" {
+ sources.x.derivation = ''
+ derivation {
+ name = "krops-test";
+ system = builtins.currentSystem;
+ builder = "/bin/sh";
+ args = [ "-c" "echo built > $out" ];
+ }
+ '';
+ to = "file";
+ check = ''
+ expect_eq x "$(cat "$T"/x)" built
+ '';
+ })
+ (link "symlink" {
+ sources = alt // { x.symlink = "alt"; };
+ to = "dir";
+ says = false;
+ check = ''
+ expect_eq link "$(readlink "$T"/x)" alt
+ '';
+ })
+ (link "flake" {
+ sources.x.flake = { url = "/fixtures/top"; inherit fetchOnTarget; };
+ to = "dir";
+ check = checkFlake { attr = "msg"; expected = "top:dep+inner,sub"; };
+ })
+ (link "flake-materialized" {
+ sources.x.flake = {
+ url = "/fixtures/top"; inherit fetchOnTarget;
+ materialize = true;
+ };
+ to = "dir";
+ check = checkFlake {
+ attr = "msg"; expected = "top:dep+inner,sub"; materialized = true;
+ };
+ })
+ (link "flake-input-patched" {
+ sources.x.flake-input = {
+ url = "/fixtures/top"; name = "dep"; inherit fetchOnTarget;
+ patches = hello;
+ };
+ to = "dir";
+ check = checkFlake { attr = "greeting"; expected = "hello+inner"; };
+ })
+ # Symlinks to outside in what a source replaces: rsync and Git remove
+ # them, and so does replacing a directory by a link.
+ (s "inner-links" (let
+ prepare = ''
+ for name in a b c; do
+ mkdir -p "$T"/$name
+ ln -s ${outside}/dir "$T"/$name/keep
+ ln -s ${outside}/dir "$T"/$name/stale
+ ln -s ${outside}/repo/.git "$T"/$name/.git
+ done
+ # A work tree whose repository is elsewhere.
+ mkdir -p "$T"/d
+ echo "gitdir: ${outside}/repo/.git" > "$T"/d/.git
+ '';
+ in {
+ isolate = true;
+ inherit prepare;
+ phases = twice {
+ sources = {
+ a.file = "/fixtures/filesrc";
+ b.git = { url = gitUrl; ref = gitHead; };
+ c.flake = { url = "/fixtures/top"; inherit fetchOnTarget; };
+ d.git = { url = gitUrl; ref = gitHead; };
+ };
+ check = ''
+ expect_eq keep "$(cat "$T"/a/keep)" keep
+ test ! -L "$T"/a/stale || fail "a/stale remains"
+ expect_eq HEAD "$(git -C "$T"/b rev-parse HEAD)" ${gitHead}
+ test ! -L "$T"/b/.git || fail "b/.git remains a link"
+ test ! -L "$T"/b/stale || fail "b/stale remains"
+ expect_eq HEAD "$(git -C "$T"/d rev-parse HEAD)" ${gitHead}
+ test -d "$T"/d/.git || fail "d/.git isn't a directory"
+ expect_eq "store directory" "$(dirname "$(readlink "$T"/c)")" /nix/store
+ '';
+ };
+ }))
+ (state "sentinel" {
+ entry = ".populate";
+ message = "error: sentinel file is a symlink";
+ })
+ (state "backup" { entry = ".populate/backup"; backup = true; })
+ (map
+ (entry: state (lib.replaceStrings ["/"] ["-"] entry) {
+ entry = ".populate/${entry}";
+ })
+ [
+ "derived" "flake-inputs" "flake-uploads" "flakes" "patches"
+ "derived/x" "patches/x" "flake-inputs/y" "flake-uploads/y" "flakes/z"
+ ])
+ # Symlinks to outside among what populate removes: leftovers of
+ # interrupted runs, state of sources that are gone, and, when pruning,
+ # entries of the target directory. They get removed as links. So do
+ # hard links to a read-only file outside, whose mode stays.
+ (s "leftovers" {
+ isolate = true;
+ prepare = ''
+ mkdir -p "$T"/hard
+ ln -s ${outside}/dir "$T"/other
+ ln -s ${outside}/repo "$T"/repo
+ ln ${outside}/dir/sub/file "$T"/hard/link
+ ln ${outside}/dir/sub/file "$T"/hardlink
+ '';
+ phases = [
+ {
+ sources = stateful;
+ mutate = ''
+ ln -s ${outside}/dir "$T"/.populate/derived/x/work
+ ln -s ${outside}/dir "$T"/.populate/derived/gone
+ ln -s ${outside}/dir "$T"/.populate/flakes/gone
+ ln -s ${outside}/dir "$T"/.populate/patches/gone
+ ln -s ${outside}/dir "$T"/.populate/flake-inputs/gone
+ mkdir -p "$T"/.populate/flake-uploads
+ ln -s ${outside}/dir "$T"/.populate/flake-uploads/gone
+ '';
+ }
+ {
+ sources = stateful;
+ check = ''
+ for name in derived/x/work derived/gone flakes/gone patches/gone; do
+ test ! -L "$T"/.populate/$name || fail "$name remains"
+ done
+ for name in other repo .populate/flake-inputs/gone .populate/flake-uploads/gone; do
+ test -L "$T"/$name || fail "$name removed without pruning"
+ done
+ '';
+ }
+ {
+ sources = stateful;
+ prune = true;
+ check = ''
+ for name in other repo hard hardlink .populate/flake-inputs/gone .populate/flake-uploads; do
+ test ! -L "$T"/$name && test ! -e "$T"/$name || fail "$name not pruned"
+ done
+ '';
+ }
+ ];
+ })
+ # A target path with characters that mean something to shells and in
+ # URLs.
+ (s "odd-target" (let
+ targetPath = "/var/krops/odd-target-${targetKind}/a b*c?d#e%f";
+ in {
+ isolate = true;
+ phases = twice {
+ sources = stateful // {
+ g.git = { url = gitUrl; ref = gitHead; };
+ p.pipe = "/fixtures/pipe";
+ };
+ inherit targetPath;
+ prune = true;
+ check = ''
+ D=${lib.escapeShellArg targetPath}
+ test -e "$D"/alt/flake.nix || fail "alt missing"
+ expect_eq HEAD "$(git -C "$D"/g rev-parse HEAD)" ${gitHead}
+ expect_eq p "$(cat "$D"/p)" piped
+ expect_eq entries "$(ls "$T")" 'a b*c?d#e%f'
+ gc
+ expect_eq greeting "$(nix_eval "$(readlink -f "$D"/x)"#greeting)" hello+alt
+ expect_eq msg "$(nix_eval "$(readlink -f "$D"/y)"#msg)" top:dep+inner,sub
+ '';
+ };
+ }))
+ # The target directory itself may be a symlink: what it points to is the
+ # target directory then.
+ (s "linked-target" {
+ isolate = true;
+ prepare = ''
+ rm -fR "$T"-real
+ mkdir "$T"-real
+ ln -s "$(basename "$T")"-real "$T"
+ '';
+ phases = twice {
+ sources = stateful;
+ check = ''
+ test -L "$T" || fail "the target directory isn't a link anymore"
+ test -e "$T"-real/alt/flake.nix || fail "alt missing"
+ gc
+ expect_eq greeting "$(nix_eval "$(readlink -f "$T"/x)"#greeting)" hello+alt
+ expect_eq msg "$(nix_eval "$(readlink -f "$T"/y)"#msg)" top:dep+inner,sub
+ expect_eq msg "$(nix_eval_online "path:$(readlink -f "$T"/.populate/flakes/z)#msg")" top:alt,sub
+ '';
+ };
+ })
+ # Git work trees whose .git directory names places elsewhere, or a
+ # command: a work tree elsewhere, a configuration file elsewhere, another
+ # repository, and a command run when Git checks for changes. Populate
+ # refuses them, telling why, and replaces them by fresh clones with
+ # reclone. A hook doesn't run.
+ (s "git-foreign" (let
+ elsewhere = "/var/krops-elsewhere";
+ git = name: reclone: {
+ ${name}.git = { url = gitUrl; ref = gitHead; inherit reclone; };
+ };
+ untouched = ''
+ cmp -s ${elsewhere}/config ${elsewhere}/config.orig ||
+ fail "what b/.git/config links to changed"
+ test ! -e ${elsewhere}/ran || fail "a command in a .git directory ran"
+ '';
+ refused = name: message: {
+ sources = git name false;
+ expect.fail = "error: populate doesn't use the Git work tree ${targetName targetKind "/var/krops/git-foreign-${targetKind}/${name}"}, because ${message}. Fix that, remove the work tree, or set the source's reclone to true to replace it by a fresh clone.";
+ };
+ in {
+ isolate = true;
+ prepare = ''
+ rm -fR ${elsewhere}
+ mkdir -p ${elsewhere} "$T"
+ for name in a b c d e; do
+ git clone -q ${gitUrl} "$T"/$name
+ git -C "$T"/$name checkout -q ${gitFirst}
+ done
+ git -C "$T"/a config core.worktree ${outside}/dir
+ mv "$T"/b/.git/config ${elsewhere}/config
+ ln -s ${elsewhere}/config "$T"/b/.git/config
+ cp -p ${elsewhere}/config ${elsewhere}/config.orig
+ printf '#! /bin/sh\ntouch ${elsewhere}/ran\n' > "$T"/c/.git/hooks/reference-transaction
+ chmod +x "$T"/c/.git/hooks/reference-transaction
+ echo ${outside}/repo/.git > "$T"/d/.git/commondir
+ git -C "$T"/e config core.fsmonitor "touch ${elsewhere}/ran"
+ '';
+ phases = [
+ (refused "a" "/var/krops/git-foreign-${targetKind}/a/.git/config sets core.worktree to ${outside}/dir")
+ (refused "b" "/var/krops/git-foreign-${targetKind}/b/.git contains symlinks: /var/krops/git-foreign-${targetKind}/b/.git/config")
+ {
+ sources = git "c" false;
+ check = untouched + ''
+ expect_eq "c HEAD" "$(git -C "$T"/c rev-parse HEAD)" ${gitHead}
+ '';
+ }
+ (refused "d" "/var/krops/git-foreign-${targetKind}/d/.git/commondir names another repository")
+ (refused "e" "/var/krops/git-foreign-${targetKind}/e/.git/config sets core.fsmonitor, which neither git clone nor populate does")
+ {
+ sources = git "a" true // git "b" true // git "c" true // git "d" true // git "e" true;
+ output = map
+ (name: "replacing the Git work tree '${targetName targetKind "/var/krops/git-foreign-${targetKind}/${name}"}', because")
+ [ "a" "b" "d" "e" ];
+ check = untouched + ''
+ for name in a b c d e; do
+ expect_eq "$name HEAD" "$(git -C "$T"/$name rev-parse HEAD)" ${gitHead}
+ test -e "$T"/$name/two || fail "$name/two missing"
+ done
+ '';
+ }
+ ];
+ }))
+ # A Git work tree whose submodule's .git, which Git uses as it finds it,
+ # isn't the gitfile git clone made, naming the repository in
+ # .git/modules: a repository of its own, whose configuration names a
+ # command, and a gitfile naming a repository elsewhere.
+ (s "git-submodule" (let
+ elsewhere = "/var/krops-elsewhere";
+ url = dirOf gitUrl + "/withsub.git";
+ git = reclone: { x.git = { inherit url reclone; ref = "master"; }; };
+ untouched = ''
+ test ! -e ${elsewhere}/ran || fail "a command in a .git directory ran"
+ expect_eq "HEAD elsewhere" "$(git -C ${elsewhere}/repo rev-parse HEAD)" ${gitFirst}
+ '';
+ in {
+ isolate = true;
+ prepare = ''
+ rm -fR ${elsewhere}
+ git clone -q ${gitUrl} ${elsewhere}/repo
+ git -C ${elsewhere}/repo checkout -q ${gitFirst}
+ '';
+ phases = [
+ {
+ sources = git false;
+ check = untouched + ''
+ expect_eq "sub HEAD" "$(git -C "$T"/x/sub rev-parse HEAD)" ${gitHead}
+ '';
+ mutate = ''
+ rm "$T"/x/sub/.git
+ cp -R "$T"/x/.git/modules/sub "$T"/x/sub/.git
+ git config --file "$T"/x/sub/.git/config --unset core.worktree
+ git config --file "$T"/x/sub/.git/config \
+ filter.x.smudge "touch ${elsewhere}/ran; cat"
+ echo '* filter=x' > "$T"/x/sub/.git/info/attributes
+ git -C "$T"/x/sub update-ref --no-deref HEAD ${gitFirst}
+ git -C "$T"/x/sub read-tree ${gitFirst}
+ rm "$T"/x/sub/two
+ '';
+ }
+ {
+ sources = git false;
+ expect.fail = "error: populate doesn't use the Git work tree";
+ check = untouched;
+ mutate = ''
+ rm -fR "$T"/x/sub
+ mkdir "$T"/x/sub
+ echo gitdir: ${elsewhere}/repo/.git > "$T"/x/sub/.git
+ '';
+ }
+ {
+ sources = git false;
+ expect.fail = "error: populate doesn't use the Git work tree";
+ check = untouched;
+ }
+ {
+ sources = git true;
+ check = untouched + ''
+ expect_eq "sub HEAD" "$(git -C "$T"/x/sub rev-parse HEAD)" ${gitHead}
+ '';
+ }
+ ];
+ }))
+ # A ref that git fetch would take for an option: --upload-pack runs a
+ # command where the repository is local.
+ (s "git-ref-option" (let
+ elsewhere = "/var/krops-elsewhere";
+ in {
+ isolate = true;
+ prepare = ''
+ rm -fR ${elsewhere}
+ git clone -q --bare ${gitUrl} ${elsewhere}/repo.git
+ '';
+ phases = [
+ {
+ sources.x.git = {
+ url = "${elsewhere}/repo.git";
+ ref = "--upload-pack=touch ${elsewhere}/ran; git-upload-pack";
+ };
+ expect.fail = "invalid refspec";
+ check = ''
+ test ! -e ${elsewhere}/ran || fail "the ref ran a command"
+ '';
+ }
+ ];
+ }))
+ # populate exits with the status of the command that failed, here git
+ # fetch's.
+ (s "git-status" {
+ phases = [{
+ sources.x.git = { url = gitUrl; ref = "nonexistent"; };
+ expect = { fail = "couldn't find remote ref"; status = 128; };
+ }];
+ })
+ # A .git that a materialized flake got, and one that a file source
+ # keeps, which Nix uses for the git+file: URL of the flake there: writing
+ # its lock file, Nix runs Git, with that repository's configuration.
+ (s "lock-foreign-git" (let
+ elsewhere = "/var/krops-elsewhere";
+ sources = alt // {
+ x.flake-input = {
+ url = "/fixtures/top"; name = "dep"; inherit fetchOnTarget;
+ materialize = true;
+ inputs.inner.follows = "alt";
+ };
+ y.file = {
+ path = "/fixtures/filesrc";
+ deleteExcluded = false;
+ filters = [ { type = "exclude"; pattern = "/.git"; } ];
+ inputs.dep.follows = "alt";
+ };
+ };
+ in {
+ isolate = true;
+ prepare = ''
+ rm -fR ${elsewhere}
+ mkdir -p ${elsewhere}
+ '';
+ phases = [
+ {
+ inherit sources;
+ # Repositories tracking only flake.nix, so the lock file changes.
+ mutate = ''
+ for name in x y; do
+ git -C "$T"/$name init -q
+ git -C "$T"/$name add flake.nix
+ git -C "$T"/$name -c user.name=x -c user.email=x@x commit -q -m x
+ git -C "$T"/$name config core.fsmonitor "touch ${elsewhere}/ran; false"
+ done
+ '';
+ }
+ {
+ inherit sources;
+ check = ''
+ test ! -e ${elsewhere}/ran || fail "a command in a .git directory ran"
+ '';
+ }
+ ];
+ }))
+ # A .git that a file source keeps, with configuration a git source would
+ # use, but a hook, which Nix's Git runs when it adds the lock file it
+ # writes: populate's Git runs none, so git sources keep hooks, but
+ # drawing inputs refers to the flake as path:.
+ (s "lock-hooks" (let
+ elsewhere = "/var/krops-elsewhere";
+ sources = alt // {
+ y.file = {
+ path = "/fixtures/filesrc";
+ deleteExcluded = false;
+ filters = [ { type = "exclude"; pattern = "/.git"; } ];
+ inputs.dep.follows = "alt";
+ };
+ };
+ in {
+ isolate = true;
+ prepare = ''
+ rm -fR ${elsewhere}
+ mkdir -p ${elsewhere}
+ '';
+ phases = [
+ {
+ inherit sources;
+ # A repository tracking only flake.nix, so Nix adds the lock file,
+ # which runs post-index-change.
+ mutate = ''
+ git -C "$T"/y init -q
+ git -C "$T"/y add flake.nix
+ git -C "$T"/y -c user.name=x -c user.email=x@x commit -q -m x
+ printf '#! /bin/sh\ntouch ${elsewhere}/ran\n' > "$T"/y/.git/hooks/post-index-change
+ chmod +x "$T"/y/.git/hooks/post-index-change
+ '';
+ }
+ {
+ inherit sources;
+ check = ''
+ test ! -e ${elsewhere}/ran || fail "a hook in a .git directory ran"
+ test -e "$T"/y/flake.lock || fail "y/flake.lock missing"
+ '';
+ }
+ ];
+ }))
+ # A file in populate's state replaced by a symlink to a file outside.
+ (s "state-key" (let
+ sources.x.flake-input = {
+ url = "/fixtures/top"; name = "dep"; inherit fetchOnTarget;
+ patches = hello;
+ };
+ in {
+ isolate = true;
+ phases = [
+ {
+ inherit sources;
+ mutate = ''
+ rm "$T"/.populate/derived/x/key
+ ln -s ${outside}/file "$T"/.populate/derived/x/key
+ '';
+ }
+ {
+ inherit sources;
+ check = checkFlake { attr = "greeting"; expected = "hello+inner"; };
+ }
+ ];
+ }))
+ # Stale state that contains a hard link to a read-only file outside,
+ # whose mode stays.
+ (s "state-hard-link" (let
+ sources.x.flake-input = {
+ url = "/fixtures/top"; name = "dep"; inherit fetchOnTarget;
+ patches = hello;
+ };
+ in {
+ isolate = true;
+ phases = [
+ {
+ inherit sources;
+ mutate = ''
+ mkdir -p "$T"/.populate/derived/gone "$T"/.populate/derived/x/work
+ ln ${outside}/dir/sub/file "$T"/.populate/derived/gone/file
+ ln ${outside}/dir/sub/file "$T"/.populate/derived/x/work/file
+ '';
+ }
+ {
+ inherit sources;
+ check = ''
+ test ! -e "$T"/.populate/derived/gone || fail "gone remains"
+ '';
+ }
+ ];
+ }))
+ # The source contains flake.lock.populate and a file like the temporary
+ # copy of the lock file, symlinks to a file outside.
+ (s "lock-temporary-link" {
+ isolate = true;
+ prepareDeployer = ''
+ rm -fR /var/krops-src/lock-temporary-link
+ mkdir -p /var/krops-src
+ cp -R /fixtures/filesrc /var/krops-src/lock-temporary-link
+ ln -s ${outside}/file /var/krops-src/lock-temporary-link/flake.lock.populate
+ ln -s ${outside}/file /var/krops-src/lock-temporary-link/.flake.lock.XXXXXX
+ '';
+ phases = twice {
+ sources = alt // {
+ x.file = {
+ path = "/var/krops-src/lock-temporary-link";
+ inputs.dep.follows = "alt";
+ };
+ };
+ check = ''
+ expect_eq msg "$(nix_eval_online path:"$T"/x#msg)" top:alt,sub
+ '';
+ };
+ })
+ # A file in the target directory is a hard link to a read-only file
+ # outside with the same contents, which rsync takes for unchanged, and
+ # would fix the mode and time of in place.
+ (s "rsync-hard-link" {
+ isolate = true;
+ prepareDeployer = ''
+ rm -fR /var/krops-src/rsync-hard-link
+ mkdir -p /var/krops-src/rsync-hard-link
+ echo outside > /var/krops-src/rsync-hard-link/file
+ '';
+ prepare = ''
+ mkdir -p "$T"/x
+ ln ${outside}/dir/sub/file "$T"/x/file
+ '';
+ phases = twice {
+ sources.x.file = {
+ path = "/var/krops-src/rsync-hard-link";
+ useChecksum = true;
+ };
+ check = ''
+ expect_eq file "$(cat "$T"/x/file)" outside
+ expect_eq links "$(stat -c %h "$T"/x/file)" 1
+ '';
+ };
+ })
+ (lib.optionals (targetKind == "local") [
+ # Rewriting a lock file changes no other hard link to it. The lock
+ # file is like the source's, with its modification time, so rsync
+ # leaves the link in place. (The source is on the deploying machine,
+ # so this only runs there.)
+ (s "lock-hard-link" {
+ prepare = ''
+ mkdir -p "$T"
+ cp -a /fixtures/filesrc "$T"/x
+ cp -p "$T"/x/flake.lock "$T".lock
+ ln -f "$T".lock "$T"/x/flake.lock
+ '';
+ phases = twice {
+ sources = alt // {
+ x.file = { path = "/fixtures/filesrc"; inputs.dep.follows = "alt"; };
+ };
+ check = ''
+ cmp -s "$T".lock /fixtures/filesrc/flake.lock ||
+ fail "the other hard link changed"
+ expect_eq msg "$(nix_eval_online "$T"/x#msg)" top:alt,sub
+ '';
+ };
+ })
+ # Names that lead out of the target directory, to the directory itself,
+ # or to populate's state.
+ (s "names" {
+ phases = map
+ (name: {
+ sources.${name}.file = "/fixtures/alt";
+ expect = "eval-error";
+ refused = "krops: source name";
+ control = { ok.file = "/fixtures/alt"; };
+ })
+ [ "" "." ".." ".populate" "a/b" "../x" "/etc" "a\nb" "a b" "a*b" "a?b" "a+b" "a=b" "-a" ];
+ })
+ # Target paths that aren't absolute, that are the root directory, or
+ # that resolve to another directory than they name.
+ (s "paths" {
+ phases = map
+ (targetPath: {
+ sources = alt;
+ inherit targetPath;
+ expect = "eval-error";
+ refused =
+ if targetPath == "" then "the target's path is empty"
+ else "krops: target path";
+ controlTargetPath = "/var/krops/paths-local";
+ })
+ [ "" "/" "//" "/." "relative" "/var/krops/.." "/var/krops/../.." "/var/krops/a\nb" ];
+ })
+ ])
+ ]) targetKinds;
+
+}
diff --git a/pkgs/populate/tests/smoke.nix b/pkgs/populate/tests/smoke.nix
new file mode 100644
index 0000000..45510a0
--- /dev/null
+++ b/pkgs/populate/tests/smoke.nix
@@ -0,0 +1,21 @@
+# Names of the scenarios of the smoke test: a few covering each source type
+# and each kind of target. krops' smoke test runs them too.
+[
+ "file-local-checksum0-deleteExcluded0-filters0-inputsnone"
+ "file-sudo-checksum0-deleteExcluded0-filters1-inputsnone"
+ "git-remote-cleanExclude0-fetchAlways0-inputsnone-shallow0"
+ "git-local-shallow1-refs"
+ "symlink-local-absolute0-inputs0"
+ "pass-remote-storepass"
+ "passage-sudo-agedefault-dirstore"
+ "pipe-local"
+ "derivation-remote"
+ "flake-local-fetchOnTarget0-inputsrewrite-kindgit-materialize0"
+ "flake-sudo-fetchOnTarget1-inputsnone-kindgit-materialize0"
+ "flake-input-dep-remote-fetchOnTarget0-inputsgenerated-materialize0"
+ "flake-input-sub-local-fetchOnTarget0-materialize0"
+ "state-sentinel-local"
+ "prune-remote"
+ "names-local"
+ "paths-local"
+]
diff --git a/pkgs/populate/tests/vm.nix b/pkgs/populate/tests/vm.nix
new file mode 100644
index 0000000..dc599dd
--- /dev/null
+++ b/pkgs/populate/tests/vm.nix
@@ -0,0 +1,203 @@
+# Runs scenarios in a NixOS VM test named name, with two machines: deployer,
+# which runs populate, and target, which it delivers to via ssh. Local
+# scenarios deliver to deployer itself. module is more configuration for
+# both, e.g. stand-ins for programs the scenarios run.
+{ pkgs, name, scenarios, gitHead, rsync ? pkgs.rsync, module ? {} }:
+
+let
+ inherit (pkgs) lib;
+
+ sshKeys = import (pkgs.path + "/nixos/tests/ssh-keys.nix") pkgs;
+
+ script = name: text: if text == null then null else pkgs.writeText name text;
+
+ # The scenarios as JSON, with scripts as files.
+ plan = pkgs.writeText "${name}.json" (builtins.toJSON (map
+ (scenario: scenario // {
+ prepare = pkgs.writeText "prepare" scenario.prepare;
+ canarySetup = pkgs.writeText "canary-setup" scenario.canarySetup;
+ canaryCheck = pkgs.writeText "canary-check" scenario.canaryCheck;
+ prepareDeployer = script "prepare-deployer" scenario.prepareDeployer;
+ phases = map
+ (phase: phase // {
+ check = pkgs.writeText "check" phase.check;
+ mutate = script "mutate" phase.mutate;
+ checkDeployer = script "check-deployer" phase.checkDeployer;
+ mutateDeployer = script "mutate-deployer" phase.mutateDeployer;
+ })
+ scenario.phases;
+ })
+ scenarios));
+
+ common = { pkgs, ... }: {
+ imports = [ module ];
+ # The plan refers to all scripts the scenarios run. Being part of the
+ # system, they survive the scenarios collecting garbage.
+ system.extraDependencies = [ plan "${./fixtures.sh}" ];
+ nix.settings.substituters = lib.mkForce [];
+ environment.systemPackages = [
+ pkgs.git
+ rsync
+ ];
+ # Only warnings and errors on the console, which ends up in the log.
+ services.journald.extraConfig = lib.mkAfter "MaxLevelConsole=warning";
+ boot.initrd.systemd.contents."/etc/systemd/journald.conf".text =
+ lib.mkAfter "MaxLevelConsole=warning";
+ boot.consoleLogLevel = lib.mkForce 4;
+ # Neither probing BIOS disk information, nor iPXE option ROMs, which
+ # network boot would need, nor systemd-oomd, which warns about missing
+ # swap.
+ boot.kernelParams = lib.optional pkgs.stdenv.hostPlatform.isx86 "edd=off";
+ virtualisation.qemu.options = [ "-global virtio-net-pci.romfile=" ];
+ systemd.oomd.enable = false;
+ virtualisation.memorySize = 2048;
+ virtualisation.diskSize = 4096;
+ };
+
+in pkgs.testers.runNixOSTest {
+ inherit name;
+
+ globalTimeout = 4 * 60 * 60;
+
+ nodes.deployer = { pkgs, ... }: {
+ imports = [ common ];
+ environment.systemPackages = [
+ pkgs.age pkgs.gnupg pkgs.pass pkgs.passage
+ ];
+ networking.firewall.allowedTCPPorts = [ 8000 9418 ];
+ # Resolving indirect references fails if the global registry can't be
+ # downloaded, which it can't in the VM. Disabling it needs flakes, and a
+ # deploying machine using indirect references has them anyway. The
+ # target doesn't, so its checks for flake support still mean something.
+ nix.settings.experimental-features = [ "nix-command" "flakes" ];
+ nix.settings.flake-registry = "";
+ programs.ssh.extraConfig = ''
+ StrictHostKeyChecking no
+ UserKnownHostsFile /dev/null
+ LogLevel ERROR
+ '';
+ systemd.services.fixtures-http = {
+ wantedBy = [ "multi-user.target" ];
+ preStart = "mkdir -p /fixtures/www";
+ serviceConfig.ExecStart =
+ "${pkgs.python3}/bin/python3 -m http.server 8000 --directory /fixtures/www";
+ };
+ systemd.services.fixtures-git = {
+ wantedBy = [ "multi-user.target" ];
+ preStart = "mkdir -p /fixtures/git";
+ serviceConfig.ExecStart =
+ "${pkgs.git}/bin/git daemon --reuseaddr --export-all --base-path=/fixtures/git";
+ };
+ };
+
+ nodes.target = { ... }: {
+ imports = [ common ];
+ services.openssh.enable = true;
+ users.users.root.openssh.authorizedKeys.keys = [ sshKeys.snakeOilPublicKey ];
+ users.users.deploy = {
+ isNormalUser = true;
+ extraGroups = [ "wheel" ];
+ openssh.authorizedKeys.keys = [ sshKeys.snakeOilPublicKey ];
+ };
+ security.sudo.wheelNeedsPassword = false;
+ };
+
+ testScript = /* python */ ''
+ import json
+
+ plan = json.load(open("${plan}"))
+ nodes = {"deployer": deployer, "target": target}
+
+
+ # Runs a command, logging its output only if it fails.
+ def run(node, command):
+ status, out = node.execute(f"{command} 2>&1")
+ if status != 0:
+ raise Exception(f"{command} fails:\n{out}")
+
+
+ start_all()
+ deployer.wait_for_unit("fixtures-http.service")
+ deployer.wait_for_unit("fixtures-git.service")
+ deployer.wait_for_open_port(8000)
+ deployer.wait_for_open_port(9418)
+ target.wait_for_unit("sshd.service")
+ target.wait_for_open_port(22)
+
+ with subtest("setup"):
+ run(deployer, "install -Dm600 ${sshKeys.snakeOilPrivateKey} /root/.ssh/id_ecdsa")
+ run(deployer, "ssh root@target true")
+ run(deployer, "ssh deploy@target sudo true")
+ run(deployer, "HOME=/root sh ${./fixtures.sh} http://deployer:8000 ${gitHead}")
+
+ failed = []
+ # The canaries get created at the start, after isolating, which removes
+ # some, and after a failure, which may have changed some.
+ canaries = False
+ for scenario in plan:
+ node = nodes[scenario["node"]]
+ try:
+ with subtest(scenario["name"]):
+ if scenario["isolate"]:
+ for machine in nodes.values():
+ run(machine, "rm -fR /var/krops && nix-store --gc >/dev/null")
+ if scenario["isolate"] or not canaries:
+ for machine in nodes.values():
+ run(machine, f"sh {scenario['canarySetup']}")
+ canaries = True
+ if scenario["prepareDeployer"] is not None:
+ run(deployer, f"sh {scenario['prepareDeployer']}")
+ run(node, f"sh {scenario['prepare']}")
+ for i, phase in enumerate(scenario["phases"]):
+ # A hang fails the scenario, not the whole test. The
+ # input is the console the test driver talks through,
+ # a terminal, which ssh -t would take over.
+ limit = phase["interrupt"] or 600
+ status, out = deployer.execute(
+ f"env HOME=/root {phase['env']} timeout {limit} {phase['populate']} </dev/null 2>&1",
+ timeout=900,
+ )
+ for unexpected in phase["unexpected"]:
+ if unexpected in out:
+ raise Exception(
+ f"populate {i} prints {unexpected!r}:\n{out}"
+ )
+ if phase["interrupt"] is not None:
+ if status != 124:
+ raise Exception(f"populate {i} ends on its own:\n{out}")
+ elif phase["expect"] is None:
+ if status != 0:
+ raise Exception(f"populate {i} fails:\n{out}")
+ for expected in phase["output"]:
+ if expected not in out:
+ raise Exception(
+ f"populate {i} doesn't print {expected!r}:\n{out}"
+ )
+ elif status == 0 or phase["expect"] not in out:
+ raise Exception(
+ f"populate {i} doesn't fail with {phase['expect']!r}:\n{out}"
+ )
+ elif phase["status"] is not None and status != phase["status"]:
+ raise Exception(
+ f"populate {i} exits with {status} instead of {phase['status']}:\n{out}"
+ )
+ run(node, f"KROPS_TEST_GC=1 sh {phase['check']}")
+ if phase["checkDeployer"] is not None:
+ run(deployer, f"sh {phase['checkDeployer']}")
+ if phase["mutate"] is not None:
+ run(node, f"sh {phase['mutate']}")
+ if phase["mutateDeployer"] is not None:
+ run(deployer, f"sh {phase['mutateDeployer']}")
+ # Each phase's check covers the canaries of its machine.
+ for machine in nodes.values():
+ if machine is not node:
+ run(machine, f"sh {scenario['canaryCheck']}")
+ except Exception as e:
+ print(f"scenario {scenario['name']} fails: {e}")
+ failed.append(scenario["name"])
+ canaries = False
+
+ if failed:
+ raise Exception(f"{len(failed)} scenarios fail: {' '.join(failed)}")
+ '';
+}