From 9f824e06ee72fcb2cdacf9b90885f5cd0606a180 Mon Sep 17 00:00:00 2001 From: tv Date: Thu, 12 Nov 2026 19:56:13 +0000 Subject: populate flake-input: fail on failures, reuse the store, leave out replaced inputs Delivering flakes and flake inputs gets more robust: - With fetchOnTarget, the inputs get collected before the pipeline that links them, so populate fails if the target can't fetch one, instead of linking the flake without its inputs and succeeding. - What the lock file pins gets taken from the store if it's there: the store path is the one of a directory named source with the locked hash, which nix-store --print-fixed-path computes. builtins.fetchTree contacts where the input comes from even then, so redeploying unchanged sources fails without that if a server is unreachable. - With fetchOnTarget, a delivered input that is itself locked to a local path the target lacks fails with populate's error naming the input, like its inputs do. - Inputs that get drawn from other sources are left out, with all they pin: Nix archives an empty flake in their place. So nothing gets fetched, copied, and rooted that no evaluation uses, and a pin that is unreachable doesn't matter if it gets replaced anyway. - flake-input sources get the option requireInputs. By default, populating fails if the inputs that the input's own lock file pins can't be delivered, e.g. because the lock file lacks an input its flake.nix has. With false, populate warns and delivers the input with what Nix locks now, or else without its inputs. A remote flake that the target fetches entirely on its own collects its inputs like everything else does. The Nix function finding its subdirectory moves next to the jq programs. New edge cases cover each of these. --- README.md | 15 +++- lib/types/populate.nix | 4 + pkgs/populate/default.nix | 188 ++++++++++++++++++++++++++++++++------------- pkgs/populate/flake-jq.nix | 19 ++++- tests/fixtures.sh | 102 ++++++++++++++++++++++++ tests/scenarios.nix | 118 ++++++++++++++++++++++++++++ 6 files changed, 387 insertions(+), 59 deletions(-) diff --git a/README.md b/README.md index ae7eca2..9f1b760 100644 --- a/README.md +++ b/README.md @@ -476,6 +476,15 @@ Supported attributes: e.g. `stockholm/nixpkgs`, and resolved as the flake's lock file resolves them, so an input that follows another one names the other one. +* `requireInputs` (optional) - + boolean that controls whether populating fails if the inputs that the + input's own lock file pins can't be delivered, e.g. because the lock file + pins something unreachable, or lacks an input its `flake.nix` has. With + `false`, populate prints a warning and delivers the input with the inputs + Nix locks when it adds what the lock file lacks, or else without its + inputs. Evaluating the input as a flake then needs network access. + Defaults to `true`. + * `fetchOnTarget` (optional) - boolean that controls whether the target fetches the input itself instead of receiving it from the deploying machine. The target then uses the @@ -669,8 +678,10 @@ there are any), all other sources as `path:`, with symlinks resolved. Lock files get rewritten or generated on every run, so they match the sources. Copies of `flake` and `flake-input` sources get made again only if the delivered store path, the patches, or the sources they draw inputs from -change. Generated flakes and copies get removed once their sources no longer -need them. This requires Nix with flake support (Nix 2.4 or later) on the +change. Inputs that get drawn from other sources don't get delivered with +a `flake` or `flake-input` source, and neither does what they pin. +Generated flakes and copies get removed once their sources no longer need +them. This requires Nix with flake support (Nix 2.4 or later) on the target. Populating fails if a source that draws inputs from other sources contains no `flake.nix`. Sources that inputs get drawn from don't need to be flakes, as long as the inputs drawing from them are declared with diff --git a/lib/types/populate.nix b/lib/types/populate.nix index a278ffd..1a18b62 100644 --- a/lib/types/populate.nix +++ b/lib/types/populate.nix @@ -233,6 +233,10 @@ name = lib.mkOption { type = lib.types.str; }; + requireInputs = lib.mkOption { + default = true; + type = lib.types.bool; + }; }; }; git = lib.types.submodule { diff --git a/pkgs/populate/default.nix b/pkgs/populate/default.nix index 002326a..2811e27 100644 --- a/pkgs/populate/default.nix +++ b/pkgs/populate/default.nix @@ -550,7 +550,8 @@ let pop.flake = target: source: pop.flake-input target (source // { name = null; }); pop.flake-input = target: source: let - inherit (import ./flake-jq.nix) selected selfIsRemote selfLocked; + inherit (import ./flake-jq.nix) + metadataSubdir selected selfIsRemote selfLocked; nix' = "${nix}/bin/nix --extra-experimental-features ${quote "nix-command flakes"}"; inputPath = if source.name == null then [] else splitString "/" source.name; fetchOnTarget = source.fetchOnTarget or false; @@ -608,31 +609,81 @@ let ${quote source.url} ${quote (toString source.name)} exit 1 ''; + # The inputs of the flake in this source that get drawn from other + # sources, as paths of input names. + overridden = map (x: x.path) (flakeFollows { type = "flake"; flake = source; }); + requireInputs = source.requireInputs or true; # Shell function printing the store path $1, followed by the inputs its - # flake in the subdirectory $2 pins in its own lock file, using the nix - # command nix. Without a lock file, locking would change the flake, so - # there are no such inputs. - closure = nix: /* sh */ '' + # flake in the subdirectory $2 pins in its own lock file, using the + # commands nix and mktemp. Without a lock file, locking would change the + # flake, so there are no such inputs. Inputs that get drawn from other + # sources are left out, with all they pin: Nix archives an empty flake + # in their place, so it doesn't fetch them. + closure = { nix, mktemp }: /* sh */ '' + # Prints the store paths in nix flake archive's output $1. + closure_paths() { + ARCHIVE=$1 OVERRIDDEN=${quote (toJSON overridden)} \ + ${nix} eval --impure --raw --expr ' + let + overridden = builtins.fromJSON (builtins.getEnv "OVERRIDDEN"); + # Inputs given by relative paths are part of their parent. + paths = prefix: node: builtins.concatMap + (name: let + input = node.inputs.''${name}; + path = prefix ++ [ name ]; + in + if builtins.elem path overridden then [] + else + (if input ? path then [ input.path ] else []) ++ + paths path input) + (builtins.attrNames (node.inputs or {})); + in + builtins.concatStringsSep "\n" + (paths [] (builtins.fromJSON (builtins.getEnv "ARCHIVE"))) + ' + echo + } closure() { printf '%s\n' "$1" closure_dir=$1''${2:+/$2} if test -e "$closure_dir"/flake.nix && test -e "$closure_dir"/flake.lock; then - ARCHIVE=$( - ${nix} flake archive --json --no-update-lock-file \ - "path:$1''${2:+?dir=$2}" - ) \ - ${nix} eval --impure --raw --expr ' - let - # Inputs given by relative paths are part of their parent. - paths = node: builtins.concatMap - (input: - (if input ? path then [ input.path ] else []) ++ paths input) - (builtins.attrValues (node.inputs or {})); - in - builtins.concatStringsSep "\n" - (paths (builtins.fromJSON (builtins.getEnv "ARCHIVE"))) - ' - echo + closure_flake=path:$1''${2:+?dir=$2} + set -- + ${optionalString (overridden != []) /* sh */ '' + closure_empty=$(${mktemp} -d) + echo '{ outputs = _: {}; }' > "$closure_empty"/flake.nix + ${concatMapStrings (path: /* sh */ '' + set -- "$@" --override-input ${quote (concatStringsSep "/" path)} "path:$closure_empty" + '') overridden} + ''} + if closure_archive=$( + ${nix} flake archive --json --no-update-lock-file "$@" "$closure_flake" + ); then + closure_paths "$closure_archive" + else + ${if requireInputs then /* sh */ '' + >&2 printf 'error: source %s: the inputs its own lock file pins can'"'"'t be delivered%s\n' \ + ${quote name} ${quote (optionalString (source.name != null) + ". Set its requireInputs to false to deliver it anyway.")} + exit 1 + '' else /* sh */ '' + # Without --no-update-lock-file, Nix adds what the lock file + # lacks. + if closure_archive=$( + ${nix} flake archive --json --no-write-lock-file "$@" "$closure_flake" + ); then + >&2 printf 'warning: source %s: its own lock file needs to be updated; delivering it with the inputs Nix locks now, which evaluating it may not use\n' \ + ${quote name} + closure_paths "$closure_archive" + else + >&2 printf 'warning: source %s: the inputs its own lock file pins can'"'"'t be delivered; delivering it without them\n' \ + ${quote name} + fi + ''} + fi + ${optionalString (overridden != []) /* sh */ '' + rm -R "$closure_empty" + ''} fi } ''; @@ -709,6 +760,7 @@ let ${requireFlakes "fetching flake inputs on the target"} read -r selected read -r self + read -r expected if test -n "$self" && ! nix-store --check-validity "$self" 2>/dev/null; then name=$(basename "$self" | cut -c34-) mv ${quote upload}/tree ${quote upload}/"$name" @@ -720,16 +772,27 @@ let exit 1 fi fi + # Fetching contacts where the input comes from even if the store has + # what the lock file pins. + if test -n "$expected" && ! nix-store --check-validity "$expected" 2>/dev/null; then + expected= + fi # The store path and the subdirectory, on separate lines. Inputs # referring to local paths can only be fetched if the target has them # as well. fetched=$( - FLAKE=${quote source.url} SELECTED=$selected SELF=$self \ + FLAKE=${quote source.url} INPUT=${quote (toString source.name)} \ + SELECTED=$selected SELF=$self EXPECTED=$expected \ nix eval --impure --raw --expr ' let selected = builtins.fromJSON (builtins.getEnv "SELECTED"); + expected = builtins.getEnv "EXPECTED"; + selectedPath = + if selected.self || expected != "" then null + else localPath selected.locked; path = if selected.self then builtins.getEnv "SELF" + else if expected != "" then expected else (builtins.fetchTree selected.locked).outPath; dir = path + (if selected.subdir == "" then "" else "/" + selected.subdir); lock = @@ -750,7 +813,9 @@ let (builtins.attrNames lock.nodes); key = builtins.head missing; in - if missing != [] then + if selectedPath != null && builtins.substring 0 1 selectedPath == "/" && !builtins.pathExists selectedPath then + throw "flake ''${builtins.getEnv "FLAKE"}: input ''${builtins.getEnv "INPUT"} refers to ''${selectedPath}, which does not exist on the target" + else if missing != [] then throw "flake ''${builtins.getEnv "FLAKE"}: input ''${key} refers to ''${localPath lock.nodes.''${key}.locked}, which does not exist on the target" else path + "\n" + selected.subdir @@ -758,8 +823,10 @@ let ) path=$(printf '%s\n' "$fetched" | sed -n 1p) subdir=$(printf '%s\n' "$fetched" | sed -n 2p) - ${closure "nix"} - { printf '%s\n' "$subdir"; closure "$path" "$subdir"; } | { + ${closure { nix = "nix"; mktemp = "mktemp"; }} + # Not in the pipeline, where failing wouldn't fail populate. + paths=$(closure "$path" "$subdir") + printf '%s\n%s\n' "$subdir" "$paths" | { ${link} } ''; @@ -768,33 +835,20 @@ let set -efu ${requireFlakes "fetching flakes on the target"} metadata=$(nix flake metadata --json --no-update-lock-file ${quote source.url}) - archive=$(nix flake archive --json --no-update-lock-file ${quote source.url}) - lines=$( - METADATA=$metadata ARCHIVE=$archive \ - nix eval --impure --raw --expr ' + # The store path and the subdirectory, on separate lines. + fetched=$( + METADATA=$metadata nix eval --impure --raw --expr ' let - # Inputs given by relative paths are part of their parent. - paths = node: - (if node ? path then [ node.path ] else []) ++ - builtins.concatMap paths (builtins.attrValues (node.inputs or {})); metadata = builtins.fromJSON (builtins.getEnv "METADATA"); - archive = builtins.fromJSON (builtins.getEnv "ARCHIVE"); - # The subdirectory, as attribute dir or as query parameter dir - # of the URL, depending on the version of Nix. - subdir = ref: - ref.dir or (let - match = builtins.match ".*[?&]dir=([^&]*).*" (ref.url or ""); - in - if match == null then "" else builtins.head match); - subdirs = builtins.filter (dir: dir != "") - (map subdir [ (metadata.locked or {}) (metadata.original or {}) ]); in - builtins.concatStringsSep "\n" - ([ (if subdirs == [] then "" else builtins.head subdirs) ] ++ - paths archive) + metadata.path + "\n" + (${metadataSubdir}) metadata ' ) - printf '%s\n' "$lines" | { + path=$(printf '%s\n' "$fetched" | sed -n 1p) + subdir=$(printf '%s\n' "$fetched" | sed -n 2p) + ${closure { nix = "nix"; mktemp = "mktemp"; }} + paths=$(closure "$path" "$subdir") + printf '%s\n%s\n' "$subdir" "$paths" | { ${link} } ''; @@ -855,22 +909,46 @@ let ''} fi + # The store path the lock file pins what's selected to, which is the + # one of a directory named source with the locked hash. A store that + # has it needs no fetching, which contacts where the input comes from + # even then. + expected= + hash=$(printf '%s\n' "$selected" | ${jq}/bin/jq -r '.locked.narHash // empty') + case $hash in + sha256-*) + expected=$( + ${nix}/bin/nix-store --print-fixed-path --recursive sha256 "$( + printf %s "''${hash#sha256-}" | + ${coreutils}/bin/base64 -d | + ${coreutils}/bin/od -An -v -tx1 | + ${coreutils}/bin/tr -d ' \n' + )" source + ) + ;; + esac + ${if fetchOnTarget then /* sh */ '' - printf '%s\n%s\n' "$selected" "$self" | { + printf '%s\n%s\n%s\n' "$selected" "$self" "$expected" | { ${runShell target fetchAndLink} } '' else /* sh */ '' path=$self if test -z "$path"; then - path=$( - SELECTED=$selected ${nix'} eval --impure --raw --expr ' - (builtins.fetchTree - (builtins.fromJSON (builtins.getEnv "SELECTED")).locked).outPath - ' - ) + if test -n "$expected" && + ${nix}/bin/nix-store --check-validity "$expected" 2>/dev/null; then + path=$expected + else + path=$( + SELECTED=$selected ${nix'} eval --impure --raw --expr ' + (builtins.fetchTree + (builtins.fromJSON (builtins.getEnv "SELECTED")).locked).outPath + ' + ) + fi fi subdir=$(printf '%s\n' "$selected" | ${jq}/bin/jq -r .subdir) - ${closure nix'} + ${closure { nix = nix'; mktemp = "${coreutils}/bin/mktemp"; }} paths=$(closure "$path" "$subdir") ${optionalString (!isLocalTarget target) /* sh */ '' diff --git a/pkgs/populate/flake-jq.nix b/pkgs/populate/flake-jq.nix index 41edcc5..63afa9d 100644 --- a/pkgs/populate/flake-jq.nix +++ b/pkgs/populate/flake-jq.nix @@ -1,6 +1,21 @@ -# jq programs populate uses for flake and flake-input sources, in a file of -# their own, so tests can run them. +# Programs populate uses for flake and flake-input sources, in a file of +# their own, so tests can run them: jq programs, which run on the deploying +# machine, and a Nix function for the target, which may lack jq. rec { + # Nix function returning the subdirectory containing the flake that nix + # flake metadata describes, like subdirOf does in jq. + metadataSubdir = /* nix */ '' + metadata: let + subdir = ref: + ref.dir or (let + match = builtins.match ".*[?&]dir=([^&]*).*" (ref.url or ""); + in + if match == null then "" else builtins.head match); + subdirs = builtins.filter (dir: dir != "") + (map subdir [ (metadata.locked or {}) (metadata.original or {}) ]); + in + if subdirs == [] then "" else builtins.head subdirs + ''; # jq definitions for subdirectories containing flakes, which flake # references and lock entries name either as attribute dir or as query # parameter dir of their URL, depending on the version of Nix. fetchTree diff --git a/tests/fixtures.sh b/tests/fixtures.sh index 3839c1d..d9d8b2b 100755 --- a/tests/fixtures.sh +++ b/tests/fixtures.sh @@ -359,3 +359,105 @@ cat > /fixtures/patches/bad.patch <<'PATCH' -nothing like this +anything PATCH + +# broken: a flake with an input that can't be fetched anymore. +mkdir /fixtures/www/src/vanishing +cat > /fixtures/www/src/vanishing/flake.nix <<'NIX' +{ outputs = _: { who = "vanishing"; }; } +NIX +tarball vanishing +mkdir /fixtures/broken +cat > /fixtures/broken/flake.nix < /fixtures/www/src/pinned/flake.nix < /fixtures/usepinned/flake.nix < /fixtures/www/src/stale/flake.nix < /fixtures/www/src/stale/flake.nix < /fixtures/usestale/flake.nix < /fixtures/www/src/gap/flake.nix <<'NIX' +{ outputs = _: { who = "gap"; }; } +NIX +tarball gap +mkdir /fixtures/offline +cat > /fixtures/offline/flake.nix < /fixtures/c3/flake.nix < /fixtures/filtered-parent/.rsync-filter +echo keep > /fixtures/filtered-parent/sub/keep +echo garbage > /fixtures/filtered-parent/sub/garbage +ln -s filtered-parent /fixtures/filtered-parent-link +chown -h nobody /fixtures/filtered-parent-link diff --git a/tests/scenarios.nix b/tests/scenarios.nix index af072b9..7743482 100644 --- a/tests/scenarios.nix +++ b/tests/scenarios.nix @@ -1104,6 +1104,124 @@ assert !(krops.isLocalTarget (targets.remote "/")); check = checkFlake { attr = "msg"; expected = "rel:lib"; }; }; }) + # The target can't fetch an input of the flake. The deploying machine + # fetched it when locking, so its fetcher cache has to go. + (s "fetch-failure" { + prepareDeployer = '' + rm -fR /root/.cache/nix + ''; + phases = [{ + sources.x.flake = { url = "/fixtures/broken"; fetchOnTarget = true; }; + expect.fail = "vanishing.tar.gz"; + }]; + }) + # The delivered input itself is locked to a path the target lacks. + (s "gone-input" { + phases = [{ + sources.x.flake-input = { url = "/fixtures/usegone"; name = "gone"; fetchOnTarget = true; }; + expect.fail = "input gone refers to /fixtures/gone, which does not exist on the target"; + }]; + }) + # An input that gets drawn from a source isn't needed on the target, with + # all it pins. + (modes "overridden-roots" ({ draw, mode, ... }: { + phases = twice { + sources = alt // { + x.flake-input = { + url = "/fixtures/top"; name = "dep"; inherit fetchOnTarget; + } // draw { inner.follows = "alt"; }; + }; + check = checkFlake { + attr = "greeting"; expected = "dep+alt"; + generated = mode == "generated"; + } + roots 0; + }; + })) + # The input's own lock file pins something unreachable, which gets drawn + # from a source anyway. + (modes "own-lock-unreachable" ({ draw, mode, ... }: { + prepareDeployer = '' + rm -fR /root/.cache/nix + ''; + phases = twice { + sources = alt // { + x.flake-input = { + url = "/fixtures/usepinned"; name = "dep"; inherit fetchOnTarget; + } // draw { inner.follows = "alt"; }; + }; + check = checkFlake { + attr = "greeting"; expected = "pinned+alt"; + generated = mode == "generated"; + }; + }; + })) + # The input's own lock file pins something unreachable, or lacks an input + # its flake.nix has. That fails, unless the input isn't required to come + # with its inputs. + (s "own-lock-broken" (let + x = url: requireInputs: { + x.flake-input = { inherit url; name = "dep"; inherit fetchOnTarget requireInputs; }; + }; + delivered = '' + test -e "$T"/x/flake.nix || fail "input not delivered" + ''; + in { + prepareDeployer = '' + rm -fR /root/.cache/nix + ''; + phases = [ + { + sources = x "/fixtures/usepinned" true; + expect.fail = "error: source x: the inputs its own lock file pins can't be delivered. Set its requireInputs to false to deliver it anyway."; + } + { + sources = x "/fixtures/usepinned" false; + check = delivered + roots 0; + } + { + sources = x "/fixtures/usestale" true; + expect.fail = "error: source x: the inputs its own lock file pins can't be delivered. Set its requireInputs to false to deliver it anyway."; + } + { + sources = x "/fixtures/usestale" false; + check = delivered + roots 2; + } + { + sources = x "/fixtures/usestale" false; + check = delivered + roots 2; + } + ]; + })) + # Redeploying an unchanged input while where it comes from is + # unreachable and the fetcher caches are gone: the store has it. + (s "offline-redeploy" (let + sources.x.flake-input = { url = "/fixtures/offline"; name = "gap"; inherit fetchOnTarget; }; + check = checkFlake { attr = "who"; expected = "gap"; }; + in { + prepareDeployer = '' + if test -e /fixtures/www/gap.tar.gz.off; then + mv /fixtures/www/gap.tar.gz.off /fixtures/www/gap.tar.gz + fi + ''; + phases = [ + { + inherit sources check; + mutate = '' + rm -fR /root/.cache/nix + ''; + mutateDeployer = '' + rm -fR /root/.cache/nix + mv /fixtures/www/gap.tar.gz /fixtures/www/gap.tar.gz.off + ''; + } + { + inherit sources check; + mutateDeployer = '' + mv /fixtures/www/gap.tar.gz.off /fixtures/www/gap.tar.gz + ''; + } + ]; + })) (s "flake-to-file" { phases = [ { -- cgit v1.3.1