From 0e0880fc606e9277c1a52e637cebe7dd3438410d Mon Sep 17 00:00:00 2001 From: tv Date: Fri, 13 Nov 2026 17:03:25 +0000 Subject: populate: deliver flake and flake-input sources natively in Rust src/flake_input.rs reads what nix flake metadata and nix flake archive print as JSON itself: it selects the flake or input, decides what the target fetches or gets copied, and computes the store paths to root, which are what selected.jq, selfIsRemote.jq, selfLocked.jq, and the Nix expressions of the shell implementation do there. On the target, fixed snippets in sh::flake run Nix, getting values as positional parameters, and send JSON or store paths back. The jq test's cases are unit tests of the Rust equivalents. --- pkgs/populate/default.nix | 2 +- pkgs/populate/docs/design.md | 9 +- pkgs/populate/rust.nix | 4 +- pkgs/populate/src/flake.rs | 633 ++----------------------------- pkgs/populate/src/flake_input.rs | 777 +++++++++++++++++++++++++++++++++++++++ pkgs/populate/src/main.rs | 1 + pkgs/populate/src/populate.rs | 7 +- pkgs/populate/src/run.rs | 23 ++ pkgs/populate/src/sh.rs | 199 +++++++++- 9 files changed, 1032 insertions(+), 623 deletions(-) create mode 100644 pkgs/populate/src/flake_input.rs (limited to 'pkgs') diff --git a/pkgs/populate/default.nix b/pkgs/populate/default.nix index fe8de4e..c259b15 100644 --- a/pkgs/populate/default.nix +++ b/pkgs/populate/default.nix @@ -17,7 +17,7 @@ if implementation == "shell" then } else if implementation == "rust" then import ./rust.nix { - inherit coreutils findutils git jq nix openssh pass passage rsync writers; + inherit coreutils findutils git nix openssh pass passage rsync writers; program = rustProgram; } else diff --git a/pkgs/populate/docs/design.md b/pkgs/populate/docs/design.md index fd64c2b..3340ccf 100644 --- a/pkgs/populate/docs/design.md +++ b/pkgs/populate/docs/design.md @@ -177,10 +177,11 @@ Populate runs rsync, Git, ssh, and Nix as programs, found in `PATH`. ## Status The program delivers all source types, draws inputs from sources, makes -backups, and prunes. The code for flakes, in `src/flake.rs`, builds the -same shell code that `shell.nix` generates, with every value quoted, so -both implementations behave alike; it moves to Rust piece by piece, with -the scenarios telling whether it still behaves alike. +backups, and prunes. Drawing inputs, in `src/flake.rs`, builds the same +shell code that `shell.nix` generates, with every value quoted; it moves +to Rust like delivering flake and flake-input sources did, in +`src/flake_input.rs`, which reads what Nix prints as JSON itself, and runs +fixed snippets on the target that send JSON or store paths back. The jq programs in `flake-jq.nix`, which the shell implementation and the jq test use, have equivalents in Rust, with the jq test's cases as unit diff --git a/pkgs/populate/rust.nix b/pkgs/populate/rust.nix index f6ece25..4582532 100644 --- a/pkgs/populate/rust.nix +++ b/pkgs/populate/rust.nix @@ -1,6 +1,6 @@ # Populate's arguments, checked and as JSON, for the program in this # directory, which delivers the sources. -{ coreutils, findutils, git, jq, nix, openssh, pass, passage, program, rsync +{ coreutils, findutils, git, nix, openssh, pass, passage, program, rsync , writers }: @@ -36,7 +36,7 @@ writers.writeDash "populate.${target.host}" /* sh */ '' echo + "$0" >&2 # The tools populate depends on, in front of others of the same name. PATH=${makeBinPath [ - rsync openssh git nix pass passage jq findutils coreutils + coreutils findutils git nix openssh pass passage rsync ]}''${PATH:+:$PATH} export PATH # On a file descriptor of its own, so the commands populate runs keep diff --git a/pkgs/populate/src/flake.rs b/pkgs/populate/src/flake.rs index 5538288..f03e443 100644 --- a/pkgs/populate/src/flake.rs +++ b/pkgs/populate/src/flake.rs @@ -1,4 +1,4 @@ -//! Flake and flake-input sources, and drawing inputs from sources. +//! Drawing inputs of flakes in sources from other sources. //! //! This builds the same shell code as shell.nix generates, with every value //! quoted, so both implementations behave alike while the code moves to @@ -10,12 +10,28 @@ use crate::spec::{Follows, Source}; use serde_json::{Map, Value}; use std::collections::{BTreeMap, BTreeSet}; -pub const LOCK_HAS_INPUT: &str = include_str!("flake/lock-has-input.nix"); -pub const METADATA_SUBDIR: &str = include_str!("flake/metadata-subdir.nix"); -const SUBDIRS: &str = include_str!("flake/subdirs.jq"); -const SELECTED: &str = include_str!("flake/selected.jq"); -const SELF_IS_REMOTE: &str = include_str!("flake/self-is-remote.jq"); -const SELF_LOCKED: &str = include_str!("flake/self-locked.jq"); +/// Nix function returning 1 if the lock file locks has an input at path, a +/// list of input names, resolving inputs that follow others, and else 0. +/// Like lockHasInput in flake-jq.nix, for the shell code drawing inputs. +pub const LOCK_HAS_INPUT: &str = r#"locks: path: let + nodes = locks.nodes; + input = key: name: (nodes.${key}.inputs or {}).${name} or null; + # The node an input refers to, following follows from the root. + node = ref: + if ref == null then null + else if builtins.isList ref then + builtins.foldl' + (key: name: if key == null then null else node (input key name)) + locks.root + ref + else ref; + has = key: path: + if path == [] then true + else let next = node (input key (builtins.head path)); in + next != null && has next (builtins.tail path); +in + if has locks.root path then "1" else "0" +"#; fn bool_setting(settings: &Map, key: &str, default: bool) -> bool { settings @@ -24,10 +40,6 @@ fn bool_setting(settings: &Map, key: &str, default: bool) -> bool .unwrap_or(default) } -fn str_setting<'a>(settings: &'a Map, key: &str) -> Option<&'a str> { - settings.get(key).and_then(Value::as_str) -} - fn patches(settings: &Map) -> Vec<&str> { settings .get("patches") @@ -57,70 +69,6 @@ pub fn derives_flake(source: &Source) -> bool { || (bool_setting(settings, "rewriteLock", true) && !source.follows().is_empty())) } -/// Shell code running body on the target: as it is for the local target, -/// and over ssh, via sudo if the target says so, for remote ones. The -/// definitions in sh::prelude come first. -pub fn run_shell(context: &Context, body: &str) -> String { - let command = format!("{}\n{body}", sh::prelude()); - if context.local { - format!("(\n{command}\n)") - } else if context.target.sudo { - format!( - "{} {} {}", - q(&context.ssh.to_string_lossy()), - q(&context.target.host), - q(&format!("sudo sh -c {}", q(&command))) - ) - } else { - format!( - "{} {} {}", - q(&context.ssh.to_string_lossy()), - q(&context.target.host), - q(&command) - ) - } -} - -/// Shell code transferring source, a shell word, to path on the target, as -/// rsync::transfer does without .rsync-filter files. -fn rsync_text(context: &Context, path: &str, source: &str, checksum: bool) -> String { - format!( - r#"{prepare} -source_path={source} -if test -d "$source_path"; then - source_path=$source_path/ -fi -rsync {checksum} {rsync_path} -e {ssh} --protect-args \ - --itemize-changes \ - --out-format={format} \ - --info=backup,del,copy,misc,mount,name,nonreg,remove,skip,symsafe \ - --omit-dir-times \ - --omit-link-times \ - -rlptD \ - --delete-excluded \ - "$source_path" {destination} >&2 -"#, - prepare = run_shell( - context, - &format!( - "set -- {} {}\n{}", - q(path), - q(&context.name(path)), - sh::break_hard_links() - ) - ), - checksum = if checksum { "--checksum" } else { "" }, - rsync_path = if context.target.sudo { - "--rsync-path='sudo rsync'" - } else { - "" - }, - ssh = q(&context.ssh.to_string_lossy()), - format = q("%i %n%L %''b/%''l"), - destination = q(&context.rsync_destination(path)), - ) -} - fn use_state(sentinel: &str, names: &[&str]) -> String { let mut call = format!("use_state {}", q(sentinel)); for name in names { @@ -130,541 +78,6 @@ fn use_state(sentinel: &str, names: &[&str]) -> String { call } -/// Delivers a flake or flake-input source named name to path. Returns the -/// shell code doing it on the deploying machine, in the source's working -/// directory. -pub fn flake_input(context: &Context, name: &str, path: &str, source: &Source) -> String { - let settings = &source.settings; - let url = str_setting(settings, "url").unwrap_or_default(); - let input = if source.kind == "flake" { - None - } else { - str_setting(settings, "name") - }; - let input_path: Vec<&str> = input.map(|i| i.split('/').collect()).unwrap_or_default(); - let fetch_on_target = bool_setting(settings, "fetchOnTarget", false); - let dir = path.rsplit_once('/').map(|(d, _)| d).unwrap_or(""); - let sentinel = format!("{dir}/.populate"); - let derived = derives_flake(&Source { - kind: "flake".into(), - settings: settings.clone(), - }); - let link_path = if derived { - format!("{sentinel}/derived/{name}/original") - } else { - path.to_string() - }; - let patches = patches(settings); - - // Uploads the patches to .populate/patches/, numbered in order. - let upload_patches = if patches.is_empty() { - String::new() - } else { - let mut text = String::from("rm -fR \"$PWD\"/patches\nmkdir \"$PWD\"/patches\n"); - for (i, patch) in patches.iter().enumerate() { - let base = patch.rsplit('/').next().unwrap_or(patch); - text += &format!( - "cp {} \"$PWD\"/patches/{}\n", - q(patch), - q(&format!("{:04}-{base}", i + 1)) - ); - } - text += &run_shell( - context, - &format!( - "set -efu\n{}\nmkdir -p {}\n", - use_state(&sentinel, &[name]), - q(&format!("{sentinel}/patches")) - ), - ); - text.push('\n'); - text += &rsync_text( - context, - &format!("{sentinel}/patches/{name}"), - "\"$PWD\"/patches", - true, - ); - text - }; - - let is_remote_ref = { - let scheme: String = url - .chars() - .take_while(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || *c == '+') - .collect(); - !scheme.is_empty() - && url[scheme.len()..].starts_with(':') - && ![ - "file", - "flake", - "git+file", - "hg+file", - "path", - "tarball+file", - ] - .contains(&scheme.as_str()) - }; - - let no_input = format!( - ">&2 printf 'error: flake %s has no input %s\\n' {} {}\nexit 1\n", - q(url), - q(input.unwrap_or("")) - ); - - let overridden: Vec> = Source { - kind: "flake".into(), - settings: settings.clone(), - } - .follows() - .into_iter() - .map(|f| f.path) - .collect(); - let require_inputs = bool_setting(settings, "requireInputs", true); - - // Shell function printing the store path $1, followed by the inputs its - // flake in the subdirectory $2 pins in its own lock file. Without a - // lock file, locking would change the flake, so there are no such - // inputs. Inputs that get drawn from other sources are left out, with - // all they pin: Nix archives an empty flake in their place, so it - // doesn't fetch them. - let closure = |nix: &str, mktemp: &str| -> String { - let overridden_json = serde_json::to_string(&overridden).expect("serializes"); - let mut overrides = String::new(); - if !overridden.is_empty() { - overrides += &format!( - "closure_empty=$({mktemp} -d)\necho '{{ outputs = _: {{}}; }}' > \"$closure_empty\"/flake.nix\n" - ); - for path in &overridden { - overrides += &format!( - "set -- \"$@\" --override-input {} \"path:$closure_empty\"\n", - q(&path.join("/")) - ); - } - } - let on_failure = if require_inputs { - format!( - ">&2 printf 'error: source %s: the inputs its own lock file pins can'\"'\"'t be delivered%s\\n' \\\n {} {}\nexit 1\n", - q(name), - q(if input.is_some() { - ". Set its requireInputs to false to deliver it anyway." - } else { - "" - }) - ) - } else { - format!( - r#"# Without --no-update-lock-file, Nix adds what the lock file -# lacks. -if closure_archive=$( - {nix} flake archive --json --no-write-lock-file "$@" "$closure_flake" -); then - >&2 printf 'warning: source %s: its own lock file needs to be updated; delivering it with the inputs Nix locks now, which evaluating it may not use\n' \ - {name} - closure_paths "$closure_archive" -else - >&2 printf 'warning: source %s: the inputs its own lock file pins can'"'"'t be delivered; delivering it without them\n' \ - {name} -fi -"#, - name = q(name) - ) - }; - format!( - r#" -# Prints the store paths in nix flake archive's output $1. -closure_paths() {{ - ARCHIVE=$1 OVERRIDDEN={overridden_json} \ - {nix} eval --impure --raw --expr ' - let - overridden = builtins.fromJSON (builtins.getEnv "OVERRIDDEN"); - # Inputs given by relative paths are part of their parent. - paths = prefix: node: builtins.concatMap - (name: let - input = node.inputs.${{name}}; - path = prefix ++ [ name ]; - in - if builtins.elem path overridden then [] - else - (if input ? path then [ input.path ] else []) ++ - paths path input) - (builtins.attrNames (node.inputs or {{}})); - in - builtins.concatStringsSep "\n" - (paths [] (builtins.fromJSON (builtins.getEnv "ARCHIVE"))) - ' - echo -}} -closure() {{ - printf '%s\n' "$1" - closure_dir=$1${{2:+/$2}} - if test -e "$closure_dir"/flake.nix && test -e "$closure_dir"/flake.lock; then - closure_flake=path:$1${{2:+?dir=$2}} - set -- - {overrides} - if closure_archive=$( - {nix} flake archive --json --no-update-lock-file "$@" "$closure_flake" - ); then - closure_paths "$closure_archive" - else - {on_failure} - fi - {cleanup} - fi -}} -"#, - overridden_json = q(&overridden_json), - cleanup = if overridden.is_empty() { - "" - } else { - "rm -R \"$closure_empty\"" - }, - ) - }; - - let inputs_roots = format!("{sentinel}/flake-inputs/{name}"); - let inputs_roots_parent = format!("{sentinel}/flake-inputs"); - let uploads = format!("{sentinel}/flake-uploads"); - let upload = format!("{uploads}/{name}"); - let remove_upload = format!( - r#"remove_tree {upload} -# Sources get populated one after another, so no other source uses the -# directory meanwhile. -if test -d {uploads} && test -z "$(ls -A {uploads})"; then - rmdir {uploads} -fi -"#, - upload = q(&upload), - uploads = q(&uploads) - ); - - // Reads the subdirectory containing the flake or input, which may be - // empty, followed by store paths from stdin. Links the first store - // path, or the subdirectory therein, at the link path, and registers it - // and the remaining ones as garbage collector roots. Roots get added - // before obsolete ones get removed, so the inputs stay rooted. - let link = format!( - r#"set -efu -read -r dir -read -r path -{use_state} -{mkdir_derived} -make_room_for_link {link_path} {link_name} -nix-store --realise "$path" --add-root {link_path} --indirect >/dev/null -# The garbage collector follows the link into the store path. -if test -n "$dir"; then - ln -fns "$path/$dir" {link_path} -fi - -# Left behind by an interrupted run, or not needed anymore. -{remove_upload} -mkdir -p {roots} -roots= -while read -r path; do - if test -z "$path"; then - continue - fi - root=$(basename "$path") - if test "$(readlink {roots}/"$root")" != "$path"; then - nix-store --realise "$path" \ - --add-root {roots}/"$root" --indirect \ - >/dev/null - fi - roots="$roots $root" -done -ls -A {roots} | while read -r root; do - case " $roots " in - *" $root "*) ;; - *) rm {roots}/"$root" ;; - esac -done -# An input without own inputs leaves the directory empty. -if test -z "$roots"; then - rmdir {roots} - if test -z "$(ls -A {roots_parent})"; then - rmdir {roots_parent} - fi -fi -"#, - use_state = use_state(&sentinel, &[name]), - mkdir_derived = if derived { - format!( - "mkdir -p {}", - q(link_path.rsplit_once('/').map(|(d, _)| d).unwrap_or("")) - ) - } else { - String::new() - }, - link_path = q(&link_path), - link_name = q(&context.name(&link_path)), - roots = q(&inputs_roots), - roots_parent = q(&inputs_roots_parent), - ); - - // Reads the selection, as selected produces it, and the store path of - // the flake itself from stdin, fetches what's selected, unless it's the - // flake itself, which gets added from the upload directory unless the - // store has it already, then fetches the inputs its own lock file - // pins, and links the results. - let fetch_and_link = format!( - r#"set -efu -require_flakes 'fetching flake inputs on the target' -read -r selected -read -r self -read -r expected -if test -n "$self" && ! nix-store --check-validity "$self" 2>/dev/null; then - name=$(basename "$self" | cut -c34-) - mv {upload}/tree {upload}/"$name" - added=$(nix-store --add-fixed --recursive sha256 {upload}/"$name") - {remove_upload} - if test "$added" != "$self"; then - >&2 printf 'error: flake %s arrived as %s instead of %s\n' \ - {url} "$added" "$self" - exit 1 - fi -fi -# Fetching contacts where the input comes from even if the store has what -# the lock file pins. -if test -n "$expected" && ! nix-store --check-validity "$expected" 2>/dev/null; then - expected= -fi -# The store path and the subdirectory, on separate lines. Inputs referring -# to local paths can only be fetched if the target has them as well. -fetched=$( - FLAKE={url} INPUT={input} \ - SELECTED=$selected SELF=$self EXPECTED=$expected \ - nix eval --impure --raw --expr ' - let - selected = builtins.fromJSON (builtins.getEnv "SELECTED"); - expected = builtins.getEnv "EXPECTED"; - selectedPath = - if selected.self || expected != "" then null - else localPath selected.locked; - path = - if selected.self then builtins.getEnv "SELF" - else if expected != "" then expected - else (builtins.fetchTree selected.locked).outPath; - dir = path + (if selected.subdir == "" then "" else "/" + selected.subdir); - lock = - if builtins.pathExists (dir + "/flake.lock") - then builtins.fromJSON (builtins.readFile (dir + "/flake.lock")) - else {{ nodes = {{}}; }}; - localPath = locked: - if (locked.type or null) == "path" then locked.path - else let - match = builtins.match "file:(//)?(.*)" (locked.url or ""); - in - if match == null then null else builtins.elemAt match 1; - missing = builtins.filter - (key: let - p = localPath (lock.nodes.${{key}}.locked or {{}}); - in - p != null && builtins.substring 0 1 p == "/" && !builtins.pathExists p) - (builtins.attrNames lock.nodes); - key = builtins.head missing; - in - if selectedPath != null && builtins.substring 0 1 selectedPath == "/" && !builtins.pathExists selectedPath then - throw "flake ${{builtins.getEnv "FLAKE"}}: input ${{builtins.getEnv "INPUT"}} refers to ${{selectedPath}}, which does not exist on the target" - else if missing != [] then - throw "flake ${{builtins.getEnv "FLAKE"}}: input ${{key}} refers to ${{localPath lock.nodes.${{key}}.locked}}, which does not exist on the target" - else - path + "\n" + selected.subdir - ' -) -path=$(printf '%s\n' "$fetched" | sed -n 1p) -subdir=$(printf '%s\n' "$fetched" | sed -n 2p) -{closure} -# Not in the pipeline, where failing wouldn't fail populate. -paths=$(closure "$path" "$subdir") -printf '%s\n%s\n' "$subdir" "$paths" | {{ - {link} -}} -"#, - upload = q(&upload), - url = q(url), - input = q(input.unwrap_or("")), - closure = closure("nix", "mktemp"), - ); - - // Fetches the flake and its inputs, and links the results. - let archive_and_link = format!( - r#"set -efu -require_flakes 'fetching flakes on the target' -metadata=$(nix flake metadata --json --no-update-lock-file {url}) -# The store path and the subdirectory, on separate lines. -fetched=$( - METADATA=$metadata nix eval --impure --raw --expr {expr} -) -path=$(printf '%s\n' "$fetched" | sed -n 1p) -subdir=$(printf '%s\n' "$fetched" | sed -n 2p) -{closure} -paths=$(closure "$path" "$subdir") -printf '%s\n%s\n' "$subdir" "$paths" | {{ - {link} -}} -"#, - url = q(url), - expr = q(&format!( - "let\n metadata = builtins.fromJSON (builtins.getEnv \"METADATA\");\nin\n metadata.path + \"\\n\" + ({METADATA_SUBDIR}) metadata\n" - )), - closure = closure("nix", "mktemp"), - ); - - let nix_ = "nix --extra-experimental-features 'nix-command flakes'"; - let selected_jq = format!("{SUBDIRS}{SELECTED}"); - let self_locked_jq = format!("{SUBDIRS}{SELF_LOCKED}"); - - if fetch_on_target && input.is_none() && is_remote_ref { - return format!( - "{upload_patches}\n{}\n", - run_shell(context, &archive_and_link) - ); - } - - let self_text = if fetch_on_target { - let upload_self = if context.local { - String::new() - } else { - format!( - r#"if ! printf '%s\n' "$self" | {{ - {check} -}}; then - {prepare} - # Without applying .rsync-filter files, which would change the flake. - {rsync} -fi -"#, - check = run_shell( - context, - "read -r self\nnix-store --check-validity \"$self\" 2>/dev/null\n" - ), - prepare = run_shell( - context, - &format!( - "set -efu\n{}\n{remove_upload}\nmkdir -p {}\n", - use_state(&sentinel, &[name]), - q(&upload) - ) - ), - rsync = rsync_text(context, &format!("{upload}/tree"), "\"$self\"", false), - ) - }; - format!( - r#"# The flake itself has no lock entry, so the target fetches it by its -# locked reference, if it can, or else receives it. -if jq -e {self_is_remote} "$cache" >/dev/null; then - selected=$( - jq -c --argjson selected "$selected" {self_locked} \ - "$cache" - ) -else - self=$(jq -r .path "$cache") - {upload_self} -fi -"#, - self_is_remote = q(SELF_IS_REMOTE), - self_locked = q(&self_locked_jq), - ) - } else { - "self=$(jq -r .path \"$cache\")\n".to_string() - }; - - let deliver = if fetch_on_target { - format!( - "printf '%s\\n%s\\n%s\\n' \"$selected\" \"$self\" \"$expected\" | {{\n {}\n}}\n", - run_shell(context, &fetch_and_link) - ) - } else { - let copy = if context.local { - String::new() - } else { - format!( - "mkdir -p \"$PWD\"/bin\nln -fns {} \"$PWD\"/bin/ssh\nPATH=$PWD/bin:$PATH \\\n{nix_} copy --to {} $paths >&2\n", - q(&context.ssh_nix.to_string_lossy()), - q(&format!("ssh://{}", context.target.host)) - ) - }; - format!( - r#"path=$self -if test -z "$path"; then - if test -n "$expected" && - nix-store --check-validity "$expected" 2>/dev/null; then - path=$expected - else - path=$( - SELECTED=$selected {nix_} eval --impure --raw --expr ' - (builtins.fetchTree - (builtins.fromJSON (builtins.getEnv "SELECTED")).locked).outPath - ' - ) - fi -fi -subdir=$(printf '%s\n' "$selected" | jq -r .subdir) -{closure} -paths=$(closure "$path" "$subdir") - -{copy} -printf '%s\n%s\n' "$subdir" "$paths" | {{ - {link} -}} -"#, - closure = closure(nix_, "mktemp"), - link = run_shell(context, &link), - ) - }; - - format!( - r#"set -efu -{upload_patches} - -cache=$TMPDIR/flake-metadata.$( - printf %s {url} | - sha256sum | - cut -c-32 -).json -if ! test -e "$cache"; then - {{ - {nix_} flake metadata --json --no-update-lock-file {url} - }} > "$cache" -fi - -if ! selected=$( - jq -ce --argjson input {input_path} {selected} \ - "$cache" 2>/dev/null -); then - {no_input} -fi - -self= -if printf '%s\n' "$selected" | jq -e .self >/dev/null; then - {self_text} -fi - -# The store path the lock file pins what's selected to, which is the one of -# a directory named source with the locked hash. A store that has it needs -# no fetching, which contacts where the input comes from even then. -expected= -hash=$(printf '%s\n' "$selected" | jq -r '.locked.narHash // empty') -case $hash in - sha256-*) - expected=$( - nix-store --print-fixed-path --recursive sha256 "$( - printf %s "${{hash#sha256-}}" | - base64 -d | - od -An -v -tx1 | - tr -d ' \n' - )" source - ) - ;; -esac - -{deliver} -"#, - url = q(url), - input_path = q(&serde_json::to_string(&input_path).expect("serializes")), - selected = q(&selected_jq), - ) -} - /// Draws inputs of flakes in sources from other sources, and derives what /// the target paths of flake and flake-input sources that get patched, get /// their lock files rewritten, or are materialized get. Sources get diff --git a/pkgs/populate/src/flake_input.rs b/pkgs/populate/src/flake_input.rs new file mode 100644 index 0000000..0084689 --- /dev/null +++ b/pkgs/populate/src/flake_input.rs @@ -0,0 +1,777 @@ +//! Flake and flake-input sources: delivering a flake, or one input of a +//! flake as its lock file pins it, as a link into the store, with garbage +//! collector roots for its own inputs. + +use crate::flake::derives_flake; +use crate::rsync::{self, Config}; +use crate::run::{self, Context, Failure, Result}; +use crate::sh; +use crate::spec::Source; +use serde_json::{Map, Value}; +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; + +/// What a flake-input source delivers: the flake itself, or an input +/// relative to it, which the target can't fetch on its own, or what +/// fetchTree takes as locked, and the subdirectory containing the flake. +#[derive(Clone, Debug, PartialEq)] +pub struct Selection { + pub is_self: bool, + pub locked: Option, + pub subdir: String, +} + +impl Selection { + fn to_json(&self) -> String { + serde_json::json!({ + "self": self.is_self, + "locked": self.locked, + "subdir": self.subdir, + }) + .to_string() + } +} + +/// The subdirectory a flake reference or lock entry names, as attribute +/// dir or as query parameter dir of its URL, depending on the version of +/// Nix. fetchTree supports neither. +fn subdir(reference: &Value) -> String { + if let Some(dir) = reference.get("dir").and_then(Value::as_str) { + return dir.to_string(); + } + let url = reference.get("url").and_then(Value::as_str).unwrap_or(""); + query_dir(url) + .map(|(_, dir, _)| dir.to_string()) + .unwrap_or_default() +} + +/// The first query parameter dir in url: where its separator is, its +/// value, and where it ends, including a following &. +fn query_dir(url: &str) -> Option<(usize, &str, usize)> { + let bytes = url.as_bytes(); + (0..bytes.len()).find_map(|i| { + if (bytes[i] == b'?' || bytes[i] == b'&') && url[i + 1..].starts_with("dir=") { + let start = i + 5; + let len = url[start..].find('&').unwrap_or(url.len() - start); + let end = start + len; + let end_with_amp = if url[end..].starts_with('&') { + end + 1 + } else { + end + }; + Some((i, &url[start..end], end_with_amp)) + } else { + None + } + }) +} + +/// The reference without its subdirectory, for fetchTree. +fn without_subdir(reference: &Value) -> Value { + let mut reference = reference.clone(); + if let Value::Object(object) = &mut reference { + object.remove("__final"); + object.remove("dir"); + if let Some(Value::String(url)) = object.get_mut("url") { + if let Some((i, _, end)) = query_dir(url) { + let separator = url[i..i + 1].to_string(); + url.replace_range(i..end, &separator); + } + if url.ends_with('?') || url.ends_with('&') { + url.pop(); + } + } + } + reference +} + +/// The subdirectory a node's locked or original reference names. +fn subdir_of(node: &Value) -> String { + ["locked", "original"] + .iter() + .filter_map(|key| node.get(key)) + .filter(|reference| !reference.is_null()) + .map(subdir) + .find(|dir| !dir.is_empty()) + .unwrap_or_default() +} + +/// Selects the flake or input in nix flake metadata's output by the path +/// of input names input, following the flake's lock file, follows +/// included. An empty follows path removes an input. Errors say what's +/// missing. +pub fn select(metadata: &Value, input: &[&str]) -> std::result::Result { + let locks = &metadata["locks"]; + let root = locks["root"].as_str().unwrap_or("root"); + let nodes = &locks["nodes"]; + fn reference<'a>( + nodes: &'a Value, + root: &'a str, + path: &[&str], + depth: usize, + ) -> std::result::Result { + if depth > 64 { + return Err("follows nest too deep".to_string()); + } + let mut key = root.to_string(); + for name in path { + let next = &nodes[&key]["inputs"][name]; + key = match next { + Value::String(next) => next.clone(), + Value::Array(follows) if !follows.is_empty() => { + let follows: Vec<&str> = follows.iter().filter_map(Value::as_str).collect(); + reference(nodes, root, &follows, depth + 1)? + } + _ => return Err(format!("no input {name}")), + }; + } + Ok(key) + } + fn join(a: &str, b: &str) -> String { + match (a.is_empty(), b.is_empty()) { + (true, _) => b.to_string(), + (_, true) => a.to_string(), + _ => format!("{a}/{b}"), + } + } + fn resolve( + metadata: &Value, + nodes: &Value, + root: &str, + key: &str, + depth: usize, + ) -> std::result::Result { + let node = &nodes[key]; + if depth > 64 { + return Err("parents nest too deep".to_string()); + } + if key == root { + return Ok(Selection { + is_self: true, + locked: None, + subdir: subdir_of(metadata), + }); + } + if let Some(parent) = node.get("parent").and_then(Value::as_array) { + let parent: Vec<&str> = parent.iter().filter_map(Value::as_str).collect(); + let base = resolve( + metadata, + nodes, + root, + &reference(nodes, root, &parent, 0)?, + depth + 1, + )?; + let path = node["locked"]["path"].as_str().unwrap_or(""); + let path = path.strip_prefix("./").unwrap_or(path); + let path = path.strip_suffix('/').unwrap_or(path); + let path = if path == "." { "" } else { path }; + let subdir = join(&join(&base.subdir, path), &subdir_of(node)); + return Ok(Selection { subdir, ..base }); + } + Ok(Selection { + is_self: false, + locked: Some(without_subdir(&node["locked"])), + subdir: subdir_of(node), + }) + } + let key = reference(nodes, root, input, 0)?; + resolve(metadata, nodes, root, &key, 0) +} + +/// Whether the target can fetch the flake itself, i.e. whether its locked +/// reference doesn't refer to the deploying machine. +pub fn self_is_remote(metadata: &Value) -> bool { + let locked = &metadata["locked"]; + locked["type"].as_str() != Some("path") + && !locked["url"].as_str().unwrap_or("").starts_with("file:") +} + +/// The store paths in nix flake archive's output, but not those of inputs +/// at the paths overridden, nor what's below them. Inputs given by +/// relative paths are part of their parent. +pub fn archived_paths(archive: &Value, overridden: &[Vec]) -> Vec { + fn walk(node: &Value, prefix: &[String], overridden: &[Vec], out: &mut Vec) { + let Some(inputs) = node.get("inputs").and_then(Value::as_object) else { + return; + }; + for (name, input) in inputs { + let mut path = prefix.to_vec(); + path.push(name.clone()); + if overridden.contains(&path) { + continue; + } + if let Some(store_path) = input.get("path").and_then(Value::as_str) { + out.push(store_path.to_string()); + } + walk(input, &path, overridden, out); + } + } + let mut out = Vec::new(); + walk(archive, &[], overridden, &mut out); + out +} + +/// The hexadecimal SHA-256 hash in a narHash like sha256-. +fn nar_hash_hex(hash: &str) -> Option { + let encoded = hash.strip_prefix("sha256-")?; + let alphabet = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + let mut bits = 0u32; + let mut count = 0; + let mut out = String::new(); + for byte in encoded.bytes().filter(|b| *b != b'=') { + let value = alphabet.iter().position(|a| *a == byte)? as u32; + bits = (bits << 6) | value; + count += 6; + if count >= 8 { + count -= 8; + out += &format!("{:02x}", (bits >> count) & 0xff); + } + } + Some(out) +} + +/// A Nix command with flakes enabled. +fn nix(args: &[&str]) -> Command { + let mut command = Command::new("nix"); + command + .args(["--extra-experimental-features", "nix-command flakes"]) + .args(args); + command +} + +/// Whether the store on the deploying machine has path. +fn valid_here(path: &str) -> bool { + Command::new("nix-store") + .args(["--check-validity", path]) + .stderr(Stdio::null()) + .status() + .is_ok_and(|status| status.success()) +} + +/// The shell code of snippet for the target, with the definitions it uses, +/// enabling flakes for what if given. +fn on_target(snippet: &str, flakes: Option<&str>) -> String { + let require = flakes + .map(|what| format!("require_flakes {}\n", run::quote(what))) + .unwrap_or_default(); + format!( + "{}{}{require}{snippet}", + sh::prelude(), + sh::flake::REMOVE_UPLOAD + ) +} + +fn setting<'a>(settings: &'a Map, key: &str) -> Option<&'a Value> { + settings.get(key) +} + +fn flag(settings: &Map, key: &str, default: bool) -> bool { + setting(settings, key) + .and_then(Value::as_bool) + .unwrap_or(default) +} + +/// Delivers the flake or flake-input source named name to path, preparing +/// what it needs on the deploying machine in workdir. +pub fn deliver( + context: &Context, + name: &str, + path: &str, + source: &Source, + workdir: &Path, +) -> Result<()> { + let settings = &source.settings; + let url = setting(settings, "url") + .and_then(Value::as_str) + .unwrap_or_default(); + let input = if source.kind == "flake" { + None + } else { + setting(settings, "name").and_then(Value::as_str) + }; + let input_path: Vec<&str> = input.map(|i| i.split('/').collect()).unwrap_or_default(); + let fetch_on_target = flag(settings, "fetchOnTarget", false); + let dir = path.rsplit_once('/').map(|(d, _)| d).unwrap_or(""); + let sentinel = format!("{dir}/.populate"); + let as_flake = Source { + kind: "flake".into(), + settings: settings.clone(), + }; + // A source that gets patched, gets its lock file rewritten, or is + // materialized gets linked in .populate/derived/ instead of at + // the target path, and the flakes step derives what the target path + // gets. + let derived = derives_flake(&as_flake); + let link_path = if derived { + format!("{sentinel}/derived/{name}/original") + } else { + path.to_string() + }; + let overridden: Vec> = as_flake.follows().into_iter().map(|f| f.path).collect(); + let require_inputs = flag(settings, "requireInputs", true); + let hint = if input.is_some() { + ". Set its requireInputs to false to deliver it anyway." + } else { + "" + }; + let failure = |message: String| Failure::Message(message); + + upload_patches(context, name, &sentinel, settings, workdir)?; + + // The store paths of the flake or input, followed by the inputs its + // flake in subdir pins in its own lock file, archived where script + // runs. + let closure = |store_path: &str, subdir: &str, on: &dyn Fn(&str, &[&str]) -> Result| { + let overrides: Vec = overridden.iter().map(|p| p.join("/")).collect(); + let mut args = vec![ + store_path, + subdir, + if require_inputs { "1" } else { "" }, + name, + hint, + ]; + args.extend(overrides.iter().map(String::as_str)); + let archive = on(sh::flake::ARCHIVE, &args)?; + let mut paths = vec![store_path.to_string()]; + if !archive.trim().is_empty() { + let archive: Value = serde_json::from_str(&archive).map_err(|e| { + failure(format!( + "error: can't read what nix flake archive prints: {e}" + )) + })?; + paths.extend(archived_paths(&archive, &overridden)); + } + Ok::<_, Failure>(paths) + }; + let link = |subdir: &str, paths: &[String]| -> Result<()> { + let link_name = context.name(&link_path); + let mut args = vec![ + sentinel.as_str(), + name, + &link_path, + &link_name, + if derived { "1" } else { "" }, + subdir, + ]; + args.extend(paths.iter().map(String::as_str)); + context.run(&on_target(sh::flake::LINK, None), &args) + }; + let on_target_output = |what: &'static str| { + move |snippet: &str, args: &[&str]| context.output(&on_target(snippet, Some(what)), args) + }; + + let is_remote_ref = { + let scheme: String = url + .chars() + .take_while(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || *c == '+') + .collect(); + !scheme.is_empty() + && url[scheme.len()..].starts_with(':') + && ![ + "file", + "flake", + "git+file", + "hg+file", + "path", + "tarball+file", + ] + .contains(&scheme.as_str()) + }; + + // The target fetches the flake itself on its own. + if fetch_on_target && input.is_none() && is_remote_ref { + let fetching = "fetching flakes on the target"; + let metadata = context.output( + &on_target( + "nix flake metadata --json --no-update-lock-file \"$1\"\n", + Some(fetching), + ), + &[url], + )?; + let metadata: Value = serde_json::from_str(&metadata) + .map_err(|e| failure(format!("error: can't read flake metadata of {url}: {e}")))?; + let store_path = metadata["path"].as_str().unwrap_or_default(); + let subdir = subdir_of(&metadata); + let paths = closure(store_path, &subdir, &on_target_output(fetching))?; + return link(&subdir, &paths); + } + + let metadata = context.flake_metadata(url, || { + run::output(&mut nix(&[ + "flake", + "metadata", + "--json", + "--no-update-lock-file", + url, + ])) + })?; + let no_input = || { + failure(format!( + "error: flake {url} has no input {}", + input.unwrap_or("") + )) + }; + let mut selected = select(&metadata, &input_path).map_err(|_| no_input())?; + + let mut self_path = String::new(); + if selected.is_self { + if fetch_on_target && self_is_remote(&metadata) { + // The flake itself has no lock entry, so the target fetches it + // by its locked reference. + selected = Selection { + is_self: false, + locked: Some(without_subdir(&metadata["locked"])), + ..selected + }; + } else { + self_path = metadata["path"].as_str().unwrap_or_default().to_string(); + if fetch_on_target && !context.local { + upload_self(context, name, &sentinel, &self_path)?; + } + } + } + + // The store path the lock file pins what's selected to, which is the + // one of a directory named source with the locked hash. A store that + // has it needs no fetching, which contacts where the input comes from + // even then. + let mut expected = String::new(); + let nar_hash = selected + .locked + .as_ref() + .and_then(|locked| locked.get("narHash")) + .and_then(Value::as_str) + .and_then(nar_hash_hex); + if let Some(hex) = nar_hash { + expected = run::output(Command::new("nix-store").args([ + "--print-fixed-path", + "--recursive", + "sha256", + &hex, + "source", + ]))? + .trim_end() + .to_string(); + } + + if fetch_on_target { + let fetching = "fetching flake inputs on the target"; + let fetched = context.output( + &on_target(sh::flake::FETCH, Some(fetching)), + &[ + &sentinel, + name, + url, + input.unwrap_or(""), + &selected.to_json(), + &self_path, + &expected, + ], + )?; + let (store_path, subdir) = fetched.split_once('\n').unwrap_or((&fetched, "")); + let paths = closure(store_path, subdir, &on_target_output(fetching))?; + return link(subdir, &paths); + } + + let store_path = if !self_path.is_empty() { + self_path + } else if !expected.is_empty() && valid_here(&expected) { + expected + } else { + let locked = selected.locked.clone().unwrap_or(Value::Null).to_string(); + run::output( + nix(&[ + "eval", + "--impure", + "--raw", + "--expr", + "(builtins.fetchTree (builtins.fromJSON (builtins.getEnv \"LOCKED\"))).outPath", + ]) + .env("LOCKED", locked), + )? + }; + let here = |snippet: &str, args: &[&str]| { + let mut command = run::local_shell(snippet, args, workdir); + command.env("NIX_CONFIG", nix_config()); + run::output(&mut command) + }; + let paths = closure(&store_path, &selected.subdir, &here)?; + if !context.local { + copy(context, workdir, &paths)?; + } + link(&selected.subdir, &paths) +} + +/// NIX_CONFIG with flakes enabled. +fn nix_config() -> String { + let config = std::env::var("NIX_CONFIG").unwrap_or_default(); + format!("{config}\nextra-experimental-features = nix-command flakes") +} + +/// Copies store paths to the target with nix copy, which runs ssh from +/// PATH, so that's the ssh for Nix's ssh:// store there. +fn copy(context: &Context, workdir: &Path, paths: &[String]) -> Result<()> { + let bin = workdir.join("bin"); + let link = bin.join("ssh"); + let prepare = || -> std::io::Result<()> { + fs::create_dir_all(&bin)?; + if fs::symlink_metadata(&link).is_ok() { + fs::remove_file(&link)?; + } + std::os::unix::fs::symlink(&context.ssh_nix, &link) + }; + prepare() + .map_err(|e| Failure::Message(format!("error: can't prepare {}: {e}", bin.display())))?; + let path = std::env::var_os("PATH").unwrap_or_default(); + let path = + std::env::join_paths(std::iter::once(bin.clone()).chain(std::env::split_paths(&path))) + .map_err(|e| Failure::Message(format!("error: can't extend PATH: {e}")))?; + let mut command = nix(&["copy", "--to", &format!("ssh://{}", context.target.host)]); + command.args(paths).env("PATH", path).stdout(run::stderr()); + run::status(&mut command) +} + +/// Uploads the source's patches to .populate/patches/, numbered in +/// order. +fn upload_patches( + context: &Context, + name: &str, + sentinel: &str, + settings: &Map, + workdir: &Path, +) -> Result<()> { + let patches: Vec<&str> = settings + .get("patches") + .and_then(Value::as_array) + .into_iter() + .flatten() + .filter_map(Value::as_str) + .collect(); + if patches.is_empty() { + return Ok(()); + } + let dir = workdir.join("patches"); + let prepare = || -> std::io::Result<()> { + if dir.exists() { + fs::remove_dir_all(&dir)?; + } + fs::create_dir(&dir)?; + for (i, patch) in patches.iter().enumerate() { + let base = Path::new(patch) + .file_name() + .unwrap_or_default() + .to_string_lossy(); + fs::copy(patch, dir.join(format!("{:04}-{base}", i + 1)))?; + } + Ok(()) + }; + prepare().map_err(|e| Failure::Message(format!("error: can't copy patches: {e}")))?; + context.run(&on_target(sh::flake::PREPARE, None), &[sentinel, name, "1"])?; + let config = Config { + filter_files: false, + use_checksum: true, + ..Config::default() + }; + rsync::transfer( + context, + &config, + &dir.to_string_lossy(), + &format!("{sentinel}/patches/{name}"), + ) +} + +/// Uploads the flake itself to a target that can't fetch it, unless the +/// target's store has it already. +fn upload_self(context: &Context, name: &str, sentinel: &str, self_path: &str) -> Result<()> { + if context.run(sh::flake::VALID, &[self_path]).is_ok() { + return Ok(()); + } + run::check_signal()?; + context.run(&on_target(sh::flake::PREPARE, None), &[sentinel, name, ""])?; + // Without applying .rsync-filter files, which would change the flake. + let config = Config { + filter_files: false, + ..Config::default() + }; + let upload: PathBuf = Path::new(sentinel) + .join("flake-uploads") + .join(name) + .join("tree"); + rsync::transfer(context, &config, self_path, &upload.to_string_lossy()) +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + /// A flake with an input dep, which has an input inner following the + /// flake's input inner, and an input sub, a flake in a subdirectory. + fn metadata(locked: Value, original: Value) -> Value { + json!({ + "locked": { "type": "path", "path": "/src/top", "narHash": "sha256-top" }, + "path": "/nix/store/top-source", + "locks": { + "root": "root", + "version": 7, + "nodes": { + "root": { "inputs": { "dep": "dep", "inner": "inner", "sub": "sub" } }, + "dep": { + "inputs": { "inner": [ "inner" ] }, + "locked": { "type": "tarball", "url": "http://x/dep.tar.gz", "narHash": "sha256-dep" } + }, + "inner": { + "locked": { "type": "tarball", "url": "http://x/inner.tar.gz", "narHash": "sha256-inner" } + }, + "sub": { "locked": locked, "original": original } + } + } + }) + } + + fn attribute() -> Value { + metadata( + json!({ "type": "tarball", "url": "http://x/mono.tar.gz", "dir": "sub", "narHash": "sha256-mono" }), + json!({ "type": "tarball", "url": "http://x/mono.tar.gz", "dir": "sub" }), + ) + } + + fn mono() -> Selection { + Selection { + is_self: false, + locked: Some( + json!({ "type": "tarball", "url": "http://x/mono.tar.gz", "narHash": "sha256-mono" }), + ), + subdir: "sub".into(), + } + } + + #[test] + fn selection() { + let url = metadata( + json!({ "type": "tarball", "url": "http://x/mono.tar.gz?dir=sub", "narHash": "sha256-mono" }), + json!({ "type": "tarball", "url": "http://x/mono.tar.gz?dir=sub" }), + ); + assert_eq!(select(&attribute(), &["sub"]), Ok(mono())); + assert_eq!(select(&url, &["sub"]), Ok(mono())); + assert_eq!( + select(&attribute(), &[]), + Ok(Selection { + is_self: true, + locked: None, + subdir: String::new() + }) + ); + assert_eq!( + select(&attribute(), &["dep", "inner"]).map(|s| s.locked), + Ok(Some( + json!({ "type": "tarball", "url": "http://x/inner.tar.gz", "narHash": "sha256-inner" }) + )) + ); + assert_eq!(select(&attribute(), &["nope"]), Err("no input nope".into())); + } + + #[test] + fn removed_and_cyclic_follows() { + let mut removed = attribute(); + removed["locks"]["nodes"]["dep"]["inputs"]["inner"] = json!([]); + assert_eq!( + select(&removed, &["dep", "inner"]), + Err("no input inner".into()) + ); + let mut cyclic = attribute(); + cyclic["locks"]["nodes"]["dep"]["inputs"]["inner"] = json!(["inner", "x"]); + cyclic["locks"]["nodes"]["inner"]["inputs"] = json!({ "x": ["dep", "inner"] }); + assert_eq!( + select(&cyclic, &["dep", "inner"]), + Err("follows nest too deep".into()) + ); + } + + #[test] + fn relative_path() { + let mut relative = attribute(); + relative["locked"]["dir"] = json!("sub"); + relative["locks"]["nodes"]["root"]["inputs"]["lib"] = json!("lib"); + relative["locks"]["nodes"]["lib"] = + json!({ "locked": { "type": "path", "path": "./lib" }, "parent": [] }); + assert_eq!( + select(&relative, &["lib"]), + Ok(Selection { + is_self: true, + locked: None, + subdir: "sub/lib".into() + }) + ); + } + + #[test] + fn remote() { + let remote = |locked: Value| self_is_remote(&json!({ "locked": locked })); + assert!(!remote(json!({ "type": "path", "path": "/src" }))); + assert!(!remote(json!({ "type": "git", "url": "file:///src" }))); + assert!(remote(json!({ "type": "git", "url": "https://x/y" }))); + assert!(remote( + json!({ "type": "github", "owner": "o", "repo": "r" }) + )); + } + + #[test] + fn locked_self() { + let locked = json!({ "__final": true, "type": "tarball", "url": "http://x/mono.tar.gz?dir=sub", "narHash": "sha256-mono" }); + assert_eq!(Some(without_subdir(&locked)), mono().locked); + let url = json!({ "url": "http://x/y?rev=1&dir=sub&z=2" }); + assert_eq!( + without_subdir(&url), + json!({ "url": "http://x/y?rev=1&z=2" }) + ); + } + + #[test] + fn subdirs() { + let of = |m: Value| subdir_of(&m); + assert_eq!( + of(json!({ + "locked": { "type": "tarball", "url": "http://x/mono.tar.gz?dir=sub&narHash=sha256-mono" }, + "original": { "type": "tarball", "url": "http://x/mono.tar.gz?dir=sub" } + })), + "sub" + ); + assert_eq!( + of(json!({ + "locked": { "type": "tarball", "url": "http://x/mono.tar.gz" }, + "original": { "type": "tarball", "url": "http://x/mono.tar.gz?rev=1&dir=sub" } + })), + "sub" + ); + assert_eq!( + of(json!({ "locked": { "type": "path", "path": "/src/top" } })), + "" + ); + } + + #[test] + fn archive() { + let archive = json!({ + "path": "/nix/store/top", + "inputs": { + "a": { "path": "/nix/store/a", "inputs": { "b": { "path": "/nix/store/b" } } }, + "c": { "path": "/nix/store/c", "inputs": { "d": { "path": "/nix/store/d" } } }, + "rel": { "inputs": {} } + } + }); + assert_eq!( + archived_paths(&archive, &[vec!["c".into()]]), + vec!["/nix/store/a", "/nix/store/b"] + ); + } + + #[test] + fn nar_hashes() { + assert_eq!( + nar_hash_hex("sha256-47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=").as_deref(), + Some("e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855") + ); + } +} diff --git a/pkgs/populate/src/main.rs b/pkgs/populate/src/main.rs index c1e389a..17d9678 100644 --- a/pkgs/populate/src/main.rs +++ b/pkgs/populate/src/main.rs @@ -3,6 +3,7 @@ mod check; mod flake; +mod flake_input; mod populate; mod rsync; mod run; diff --git a/pkgs/populate/src/populate.rs b/pkgs/populate/src/populate.rs index ef806b0..ad0ec35 100644 --- a/pkgs/populate/src/populate.rs +++ b/pkgs/populate/src/populate.rs @@ -1,6 +1,7 @@ //! Delivering the sources of a spec. use crate::flake; +use crate::flake_input; use crate::rsync::{self, Config}; use crate::run::{self, Context, Failure, Result}; use crate::sh; @@ -132,11 +133,7 @@ fn deliver( }; rsync::transfer(context, &config, &out.to_string_lossy(), path) } - "flake" | "flake-input" => run::status(&mut run::local_shell( - &flake::flake_input(context, name, path, source), - &[], - &workdir, - )), + "flake" | "flake-input" => flake_input::deliver(context, name, path, source, &workdir), "git" => { let flag = |key| match settings.get(key).and_then(Value::as_bool) { Some(true) => "1", diff --git a/pkgs/populate/src/run.rs b/pkgs/populate/src/run.rs index 7525cc0..528977c 100644 --- a/pkgs/populate/src/run.rs +++ b/pkgs/populate/src/run.rs @@ -102,6 +102,9 @@ pub struct Context { /// because Nix waits for its LocalCommand, which may not run over a /// multiplexed connection. pub ssh_nix: PathBuf, + /// What nix flake metadata prints for flakes, by reference, so it's + /// obtained only once, even if several sources use the same flake. + metadata: std::cell::RefCell>, } impl Context { @@ -127,6 +130,7 @@ impl Context { tmpdir, ssh, ssh_nix, + metadata: Default::default(), }; context.write_ssh(&context.ssh, &context.ssh_options(true))?; context.write_ssh(&context.ssh_nix, &context.ssh_options(false))?; @@ -231,6 +235,25 @@ impl Context { status(&mut command) } + /// What nix flake metadata prints for the flake url, as get gets it, + /// once. + pub fn flake_metadata( + &self, + url: &str, + get: impl FnOnce() -> Result, + ) -> Result { + if let Some(metadata) = self.metadata.borrow().get(url) { + return Ok(metadata.clone()); + } + let metadata: serde_json::Value = serde_json::from_str(&get()?).map_err(|e| { + Failure::Message(format!("error: can't read flake metadata of {url}: {e}")) + })?; + self.metadata + .borrow_mut() + .insert(url.to_string(), metadata.clone()); + Ok(metadata) + } + /// Runs shell code on the target, returning its output. pub fn output(&self, script: &str, args: &[&str]) -> Result { output(&mut self.shell(script, args)) diff --git a/pkgs/populate/src/sh.rs b/pkgs/populate/src/sh.rs index 5605699..1a2d2c7 100644 --- a/pkgs/populate/src/sh.rs +++ b/pkgs/populate/src/sh.rs @@ -471,7 +471,8 @@ while read -r age_path; do mkdir -p "$(dirname "$pass_out")" if ! passage show "$pass_name" > "$pass_out" 2>&1; then echo ++ secrets "$host" show "$pass_name" >&2 - jq -Rr 'select(test("^[^[]"))' "$pass_out" >&2 + # What passage says, without age's progress lines. + grep '^[^[]' "$pass_out" >&2 || : exit 1 fi touch -r "$age_path" "$pass_out" @@ -533,3 +534,199 @@ pub fn prelude() -> String { ] .concat() } + +/// Shell code for flake and flake-input sources on the target. Each gets +/// SENTINEL and NAME, the source's name, first, and the definitions of +/// prelude. +pub mod flake { + /// Defines remove_upload, which removes the directory receiving the + /// flake itself, and the directory of uploads once it's empty. + /// Sources get populated one after another, so no other source uses + /// that meanwhile. + pub const REMOVE_UPLOAD: &str = r#" +remove_upload() { + remove_tree "$sentinel"/flake-uploads/"$name" + if test -d "$sentinel"/flake-uploads && + test -z "$(ls -A "$sentinel"/flake-uploads)"; then + rmdir "$sentinel"/flake-uploads + fi +} +"#; + + /// prepare_upload SENTINEL NAME: makes the directory receiving the + /// flake itself, or the patches if PATCHES isn't empty. + pub const PREPARE: &str = r#" +sentinel=$1 name=$2 patches=$3 +use_state "$sentinel" "$name" +if test -n "$patches"; then + mkdir -p "$sentinel"/patches +else + remove_upload + mkdir -p "$sentinel"/flake-uploads/"$name" +fi +"#; + + /// valid PATH: succeeds if the store has PATH. + pub const VALID: &str = r#" +nix-store --check-validity "$1" 2>/dev/null +"#; + + /// archive PATH SUBDIR REQUIRE NAME HINT OVERRIDDEN...: prints what + /// nix flake archive prints for the flake in SUBDIR of the store path + /// PATH, with an empty flake in place of each input at an OVERRIDDEN + /// path, so Nix doesn't fetch them, or nothing if the flake has no + /// lock file, since locking would change it. If its inputs can't be + /// archived, it fails if REQUIRE isn't empty, saying so for the source + /// NAME, with HINT appended, or else archives what Nix locks now, or + /// prints nothing, with a warning. + pub const ARCHIVE: &str = r#" +path=$1 subdir=$2 require=$3 name=$4 hint=$5 +shift 5 +dir=$path${subdir:+/$subdir} +if test -e "$dir"/flake.nix && test -e "$dir"/flake.lock; then + flake=path:$path${subdir:+?dir=$subdir} + empty= + if test $# -gt 0; then + empty=$(mktemp -d) + echo '{ outputs = _: {}; }' > "$empty"/flake.nix + n=$# + while test "$n" -gt 0; do + set -- "$@" --override-input "$1" "path:$empty" + shift + n=$((n - 1)) + done + fi + if nix flake archive --json --no-update-lock-file "$@" "$flake"; then + : + elif test -n "$require"; then + >&2 printf 'error: source %s: the inputs its own lock file pins can'"'"'t be delivered%s\n' \ + "$name" "$hint" + exit 1 + # Without --no-update-lock-file, Nix adds what the lock file lacks. + elif nix flake archive --json --no-write-lock-file "$@" "$flake"; then + >&2 printf 'warning: source %s: its own lock file needs to be updated; delivering it with the inputs Nix locks now, which evaluating it may not use\n' \ + "$name" + else + >&2 printf 'warning: source %s: the inputs its own lock file pins can'"'"'t be delivered; delivering it without them\n' \ + "$name" + fi + if test -n "$empty"; then + rm -R "$empty" + fi +fi +"#; + + /// link SENTINEL NAME LINK_PATH LINK_NAME DERIVED SUBDIR PATH ROOT...: + /// links the store path PATH, or SUBDIR therein, at LINK_PATH, and + /// registers it and the store paths ROOT as garbage collector roots of + /// the source NAME. Roots get added before obsolete ones get removed, + /// so the inputs stay rooted. LINK_NAME is LINK_PATH's name in + /// messages, and DERIVED isn't empty if LINK_PATH is in the source's + /// derived state. + pub const LINK: &str = r#" +sentinel=$1 name=$2 link_path=$3 link_name=$4 derived=$5 subdir=$6 path=$7 +shift 7 +roots_dir=$sentinel/flake-inputs/$name +use_state "$sentinel" "$name" +if test -n "$derived"; then + mkdir -p "$(dirname "$link_path")" +fi +make_room_for_link "$link_path" "$link_name" +nix-store --realise "$path" --add-root "$link_path" --indirect >/dev/null +# The garbage collector follows the link into the store path. +if test -n "$subdir"; then + ln -fns "$path/$subdir" "$link_path" +fi +# Left behind by an interrupted run, or not needed anymore. +remove_upload +mkdir -p "$roots_dir" +roots= +for path; do + root=$(basename "$path") + if test "$(readlink "$roots_dir"/"$root")" != "$path"; then + nix-store --realise "$path" --add-root "$roots_dir"/"$root" --indirect >/dev/null + fi + roots="$roots $root" +done +ls -A "$roots_dir" | while read -r root; do + case " $roots " in + *" $root "*) ;; + *) rm "$roots_dir"/"$root" ;; + esac +done +# An input without own inputs leaves the directory empty. +if test -z "$roots"; then + rmdir "$roots_dir" + if test -z "$(ls -A "$sentinel"/flake-inputs)"; then + rmdir "$sentinel"/flake-inputs + fi +fi +"#; + + /// fetch SENTINEL NAME URL INPUT SELECTED SELF EXPECTED: prints the + /// store path of what the selection SELECTED, as JSON, names, and the + /// subdirectory therein, on separate lines. That's SELF, the flake + /// itself, which gets added from the upload directory unless the store + /// has it already, or EXPECTED, the store path the lock file pins it + /// to, if the store has it, or else what fetching it gets. Inputs + /// referring to local paths can only be fetched if the target has them + /// as well. URL and INPUT name the flake and the input in messages. + pub const FETCH: &str = r#" +sentinel=$1 name=$2 url=$3 input=$4 selected=$5 self=$6 expected=$7 +upload=$sentinel/flake-uploads/$name +if test -n "$self" && ! nix-store --check-validity "$self" 2>/dev/null; then + store_name=$(basename "$self" | cut -c34-) + mv "$upload"/tree "$upload"/"$store_name" + added=$(nix-store --add-fixed --recursive sha256 "$upload"/"$store_name") + remove_upload + if test "$added" != "$self"; then + >&2 printf 'error: flake %s arrived as %s instead of %s\n' \ + "$url" "$added" "$self" + exit 1 + fi +fi +# Fetching contacts where the input comes from even if the store has what +# the lock file pins. +if test -n "$expected" && ! nix-store --check-validity "$expected" 2>/dev/null; then + expected= +fi +FLAKE=$url INPUT=$input SELECTED=$selected SELF=$self EXPECTED=$expected \ +nix eval --impure --raw --expr ' + let + selected = builtins.fromJSON (builtins.getEnv "SELECTED"); + expected = builtins.getEnv "EXPECTED"; + selectedPath = + if selected.self || expected != "" then null + else localPath selected.locked; + path = + if selected.self then builtins.getEnv "SELF" + else if expected != "" then expected + else (builtins.fetchTree selected.locked).outPath; + dir = path + (if selected.subdir == "" then "" else "/" + selected.subdir); + lock = + if builtins.pathExists (dir + "/flake.lock") + then builtins.fromJSON (builtins.readFile (dir + "/flake.lock")) + else { nodes = {}; }; + localPath = locked: + if (locked.type or null) == "path" then locked.path + else let + match = builtins.match "file:(//)?(.*)" (locked.url or ""); + in + if match == null then null else builtins.elemAt match 1; + missing = builtins.filter + (key: let + p = localPath (lock.nodes.${key}.locked or {}); + in + p != null && builtins.substring 0 1 p == "/" && !builtins.pathExists p) + (builtins.attrNames lock.nodes); + key = builtins.head missing; + in + if selectedPath != null && builtins.substring 0 1 selectedPath == "/" && !builtins.pathExists selectedPath then + throw "flake ${builtins.getEnv "FLAKE"}: input ${builtins.getEnv "INPUT"} refers to ${selectedPath}, which does not exist on the target" + else if missing != [] then + throw "flake ${builtins.getEnv "FLAKE"}: input ${key} refers to ${localPath lock.nodes.${key}.locked}, which does not exist on the target" + else + path + "\n" + selected.subdir +' +"#; +} -- cgit v1.3.1