diff options
Diffstat (limited to 'pkgs/populate/src/sh.rs')
| -rw-r--r-- | pkgs/populate/src/sh.rs | 362 |
1 files changed, 242 insertions, 120 deletions
diff --git a/pkgs/populate/src/sh.rs b/pkgs/populate/src/sh.rs index 70efd3f..66ca04d 100644 --- a/pkgs/populate/src/sh.rs +++ b/pkgs/populate/src/sh.rs @@ -327,104 +327,125 @@ git_problems() ( /// PATH a Git work tree of the repository at URL, with REF checked out, /// and everything removed that Git doesn't track, except what matches an /// EXCLUDE pattern. SHALLOW, FETCH_ALWAYS, and RECLONE are flags, empty -/// for false. NAME is PATH's name in messages. +/// for false. NAME is PATH's name in messages. Prints checked if +/// git_problems found nothing wrong with the work tree that was there, +/// which then is what's there, and else nothing; what commands print goes +/// to standard error. pub fn git() -> String { LINKS.to_string() + GIT_PROBLEMS + r#" -path=$1 name=$2 url=$3 ref=$4 shallow=$5 fetch_always=$6 reclone=$7 -shift 7 -# Hooks and commands a repository's configuration names run with Git's -# commands, so populate's Git doesn't run any. The configuration gets -# checked as well, but Git also runs a hook it has itself. -git() { - command git -c core.hooksPath=/dev/null -c core.fsmonitor=false "$@" -} -# A target path that doesn't look like a Git work tree gets removed, e.g. -# when it had another type before. A work tree that a symlink points to, -# or whose .git is a symlink or a file naming a repository elsewhere, isn't -# populate's own: fetching, resetting, and cleaning would change what's -# elsewhere. -remove_link "$path" "$name" -if test -L "$path"/.git || ! test -d "$path"/.git; then - rm -fR "$path" -fi -# A .git directory can name places elsewhere too, which Git's commands -# change, or commands, which they run. Populate only uses a work tree -# whose .git directory has nothing like that, and else replaces it if -# reclone is set, or fails, telling why. -if test -e "$path"; then - if problems=$(git_problems "$path"); then :; else - exit 1 +# What commands print goes to standard error. None of them gets the +# standard output, which populate reads to its end, so nothing they leave +# running, e.g. git gc --auto, keeps populate waiting. +checked= +deliver_git() { + path=$1 name=$2 url=$3 ref=$4 shallow=$5 fetch_always=$6 reclone=$7 + shift 7 + # Hooks and commands a repository's configuration names run with Git's + # commands, so populate's Git doesn't run any. The configuration gets + # checked as well, but Git also runs a hook it has itself. + git() { + command git -c core.hooksPath=/dev/null -c core.fsmonitor=false "$@" + } + # A target path that doesn't look like a Git work tree gets removed, e.g. + # when it had another type before. A work tree that a symlink points to, + # or whose .git is a symlink or a file naming a repository elsewhere, isn't + # populate's own: fetching, resetting, and cleaning would change what's + # elsewhere. + remove_link "$path" "$name" + if test -L "$path"/.git || ! test -d "$path"/.git; then + rm -fR "$path" fi - if test -n "$problems"; then - if test -n "$reclone"; then - >&2 printf "replacing the Git work tree '%s', because %s...\n" \ - "$name" "$problems" - rm -fR "$path" - else - >&2 printf 'error: populate doesn'"'"'t use the Git work tree %s, because %s. Fix that, remove the work tree, or set the source'"'"'s reclone to true to replace it by a fresh clone.\n' \ - "$name" "$problems" + # A .git directory can name places elsewhere too, which Git's commands + # change, or commands, which they run. Populate only uses a work tree + # whose .git directory has nothing like that, and else replaces it if + # reclone is set, or fails, telling why. + if test -e "$path"; then + if problems=$(git_problems "$path"); then :; else exit 1 fi + if test -n "$problems"; then + if test -n "$reclone"; then + >&2 printf "replacing the Git work tree '%s', because %s...\n" \ + "$name" "$problems" + rm -fR "$path" + else + >&2 printf 'error: populate doesn'"'"'t use the Git work tree %s, because %s. Fix that, remove the work tree, or set the source'"'"'s reclone to true to replace it by a fresh clone.\n' \ + "$name" "$problems" + exit 1 + fi + else + checked=1 + fi fi -fi -if ! test -e "$path"; then - if test -n "$shallow"; then - git init "$path" - else - git clone --recurse-submodules -- "$url" "$path" + if ! test -e "$path"; then + if test -n "$shallow"; then + git init "$path" + else + git clone --recurse-submodules -- "$url" "$path" + fi fi -fi -cd "$path" -if ! origin=$(git config remote.origin.url); then - git remote add -- origin "$url" -elif test "$origin" != "$url"; then - git remote set-url -- origin "$url" -fi -# A commit, given by its hash or a prefix thereof, gets fetched only if the -# work tree lacks it, unless fetchAlways is set. After --, a ref can't be -# taken for an option. A branch or tag gets -# fetched on every run, so the work tree follows it: resetting to its name -# would reset to a local branch, which fetching doesn't move. -fetch() { - if test -n "$shallow"; then - git fetch --depth=1 origin -- "$1" - else - git fetch origin -- "$1" + cd "$path" + if ! origin=$(git config remote.origin.url); then + git remote add -- origin "$url" + elif test "$origin" != "$url"; then + git remote set-url -- origin "$url" fi -} -fetch_commit() { - if test -n "$shallow"; then - fetch "$hash" + # A commit, given by its hash or a prefix thereof, gets fetched only if the + # work tree lacks it, unless fetchAlways is set. After --, a ref can't be + # taken for an option. A branch or tag gets + # fetched on every run, so the work tree follows it: resetting to its name + # would reset to a local branch, which fetching doesn't move. + fetch() { + if test -n "$shallow"; then + git fetch --depth=1 origin -- "$1" + else + git fetch origin -- "$1" + fi + } + fetch_commit() { + if test -n "$shallow"; then + fetch "$hash" + else + git fetch origin + fi + } + if printf %s "$ref" | grep -Eqx '[0-9a-f]{4,40}'; then + hash=$ref + if ! test "$(git log --format=%H -1)" = "$hash"; then + if test -n "$fetch_always"; then + fetch_commit + elif ! git log -1 "$hash" >/dev/null 2>&1; then + fetch_commit + fi + fi else - git fetch origin + fetch "$ref" + hash=$(git rev-parse --verify 'FETCH_HEAD^{commit}') fi -} -if printf %s "$ref" | grep -Eqx '[0-9a-f]{4,40}'; then - hash=$ref - if ! test "$(git log --format=%H -1)" = "$hash"; then - if test -n "$fetch_always"; then - fetch_commit - elif ! git log -1 "$hash" >/dev/null 2>&1; then - fetch_commit - fi + # Also if HEAD is there already, unless no tracked file differs from it: + # rewriting a lock file in place, e.g., changes the work tree, and stages + # the lock file if Nix created it. + head=$(git rev-parse --verify --quiet HEAD) || head= + commit=$(git rev-parse --verify --quiet "$hash^{commit}") || commit= + changes=$(git status --porcelain --untracked-files=no) || changes=unknown + if test -z "$head" || test "$head" != "$commit" || test -n "$changes"; then + git reset --hard "$hash" fi -else - fetch "$ref" - hash=$(git rev-parse --verify 'FETCH_HEAD^{commit}') + git submodule update --init --recursive + n=$# + while test "$n" -gt 0; do + set -- "$@" -e "$1" + shift + n=$((n - 1)) + done + git clean -dfx "$@" +} +deliver_git "$@" >&2 +if test -n "$checked"; then + printf checked fi -# Also if HEAD is there already: rewriting a lock file in place, e.g., -# changes the work tree, and stages the lock file if Nix created it. -git reset --hard "$hash" >&2 -git submodule update --init --recursive -n=$# -while test "$n" -gt 0; do - set -- "$@" -e "$1" - shift - n=$((n - 1)) -done -git clean -dfx "$@" "# } @@ -544,6 +565,50 @@ require_flakes() { } "#; +/// Defines nix_looking COMMAND..., which runs COMMAND, a Nix command +/// looking at a flake without writing its lock file, and leaves out of +/// what it prints on standard error Nix's warning that it doesn't write +/// the lock file it changed, listing what changed. The shell code for +/// flakes overrides inputs drawn from sources when looking at flakes, +/// which changes the lock file on every run, without anything changing, +/// or says itself that a lock file needs to be updated. +pub const NIX_LOOKING: &str = r#" +nix_looking() { + nix_looking_status=$(mktemp) + { + { + if "$@" 2>&1 >&3 3>&-; then + nix_looking_code=0 + else + nix_looking_code=$? + fi + echo "$nix_looking_code" > "$nix_looking_status" + } | { + # The warning's lines after the first begin with a bullet or blanks. + nix_looking_skip= + while IFS= read -r nix_looking_line || test -n "$nix_looking_line"; do + case $nix_looking_line in + 'warning: not writing modified lock file of flake '*) + nix_looking_skip=1 + continue + ;; + '• '*|' '*) + if test -n "$nix_looking_skip"; then + continue + fi + ;; + esac + nix_looking_skip= + printf '%s\n' "$nix_looking_line" + done + } >&2 + } 3>&1 + read -r nix_looking_code < "$nix_looking_status" + rm -f "$nix_looking_status" + return "$nix_looking_code" +} +"#; + /// The definitions the shell code for flakes uses. pub fn prelude() -> String { [ @@ -553,6 +618,7 @@ pub fn prelude() -> String { REMOVE_OTHERS, URL_PATH, REQUIRE_FLAKES, + NIX_LOOKING, ] .concat() } @@ -600,7 +666,7 @@ nix-store --check-validity "$1" 2>/dev/null /// lock file, since locking would change it. If its inputs can't be /// archived, it fails if REQUIRE isn't empty, saying so for the source /// NAME, with HINT appended, or else archives what Nix locks now, or - /// prints nothing, with a warning. + /// prints nothing, with a warning. It needs NIX_LOOKING. pub const ARCHIVE: &str = r#" path=$1 subdir=$2 require=$3 name=$4 hint=$5 shift 5 @@ -618,14 +684,14 @@ if test -e "$dir"/flake.nix && test -e "$dir"/flake.lock; then n=$((n - 1)) done fi - if nix flake archive --json --no-update-lock-file "$@" "$flake"; then + if nix_looking nix flake archive --json --no-update-lock-file "$@" "$flake"; then : elif test -n "$require"; then >&2 printf 'error: source %s: the inputs its own lock file pins can'"'"'t be delivered%s\n' \ "$name" "$hint" exit 1 # Without --no-update-lock-file, Nix adds what the lock file lacks. - elif nix flake archive --json --no-write-lock-file "$@" "$flake"; then + elif nix_looking nix flake archive --json --no-write-lock-file "$@" "$flake"; then >&2 printf 'warning: source %s: its own lock file needs to be updated; delivering it with the inputs Nix locks now, which evaluating it may not use\n' \ "$name" else @@ -759,8 +825,9 @@ nix eval --impure --raw --expr ' /// the definitions of prelude and DEFINITIONS. Populate decides what to /// do, from what these send back. pub mod draw { - /// Defines own_lock DIR, apply_patches PATCH_DIR NAME, and write_key - /// KEY_FILE KEY. + /// Defines own_lock DIR, apply_patches PATCH_DIR NAME, write_key + /// KEY_FILE KEY, git_flake PATH CHECKED, and clean_git_rev PATH + /// CHECKED. /// /// Nix rewrites lock files in place, which also changes what else is a /// hard link to the lock file, e.g. in a backup, or what it is a @@ -774,6 +841,19 @@ pub mod draw { /// /// write_key writes KEY to a new file, which then replaces KEY_FILE, so /// nothing at its place gets written through, e.g. a symlink. + /// + /// git_flake PATH CHECKED succeeds if Nix gets the directory at PATH + /// as a git+file: flake: if it has a .git directory that git_problems + /// finds nothing wrong with, and that has no hooks. Nix runs Git in a + /// git+file: flake, with the repository's configuration and hooks, so + /// it only gets such ones. If CHECKED isn't empty, git_problems found + /// nothing wrong already, so that doesn't get checked again: it takes + /// looking through all of the work tree, e.g. all of nixpkgs. + /// + /// clean_git_rev PATH CHECKED prints the commit HEAD of the Git work + /// tree at PATH points to, if git_flake PATH CHECKED, and none of its + /// tracked files differ from that commit, which is all of it Nix uses + /// then. Git runs without hooks and file system monitors. pub const DEFINITIONS: &str = r#" own_lock() { if test -e "$1"/flake.lock; then @@ -796,6 +876,31 @@ write_key() { printf '%s\n' "$2" > "$key_file" mv -f "$key_file" "$1" } +git_flake() { + ! test -L "$1"/.git && test -d "$1"/.git || return 1 + if test -z "$2"; then + git_flake_problems=$(git_problems "$1" 2>/dev/null) || return 1 + test -z "$git_flake_problems" || return 1 + fi + if test -d "$1"/.git/hooks; then + test -z "$(find "$1"/.git/hooks -mindepth 1 ! -name '*.sample' | head -n 1)" + fi +} +clean_git_rev() ( + git_flake "$1" "$2" || exit 1 + cd "$1" + git() { + command git -c core.hooksPath=/dev/null -c core.fsmonitor=false "$@" + } + # Without writing the index: in a copy of a work tree, e.g. in a file + # source, that makes it differ from the original's, so the next run + # copies it again, unless the original's changed in the same second, + # keeping its size, which rsync doesn't notice. Then the work tree + # differs from its index, which Nix refuses to lock. + changes=$(git --no-optional-locks status --porcelain --untracked-files=no 2>/dev/null) || exit 1 + test -z "$changes" || exit 1 + git rev-parse --verify --quiet HEAD +) "#; /// use_state SENTINEL NAME... @@ -803,27 +908,19 @@ write_key() { use_state "$@" "#; - /// resolve PATH...: prints, for each PATH, what's there and where it - /// resolves to, as two fields, each ending in a null byte. The first - /// holds d if PATH is a directory, g if it has a .git directory that - /// git_problems finds nothing wrong with, and that has no hooks, s if - /// it has .gitmodules, and f if it has flake.nix. Nix runs Git in a - /// git+file: flake, with the repository's configuration and hooks, so - /// it only gets such ones. + /// resolve CHECKED PATH...: prints, for each pair of CHECKED and PATH, + /// what's at PATH and where it resolves to, as two fields, each ending + /// in a null byte. The first holds d if PATH is a directory, g if + /// git_flake PATH CHECKED, s if it has .gitmodules, and f if it has + /// flake.nix. pub const RESOLVE: &str = r#" -for path; do +while test $# -gt 0; do + checked=$1 path=$2 + shift 2 resolved=$(readlink -f "$path") || resolved= flags= if test -d "$path"; then flags=${flags}d; fi - if ! test -L "$path"/.git && test -d "$path"/.git; then - problems=$(git_problems "$path" 2>/dev/null) || problems=failed - if test -d "$path"/.git/hooks; then - hooks=$(find "$path"/.git/hooks -mindepth 1 ! -name '*.sample' | head -n 1) - else - hooks= - fi - if test -z "$problems" && test -z "$hooks"; then flags=${flags}g; fi - fi + if git_flake "$path" "$checked"; then flags=${flags}g; fi if test -e "$path"/.gitmodules; then flags=${flags}s; fi if test -e "$path"/flake.nix; then flags=${flags}f; fi printf '%s\0%s\0' "$flags" "$resolved" @@ -846,7 +943,7 @@ while test "$n" -gt 0; do shift n=$((n - 1)) done -if ! nix flake metadata --json --no-write-lock-file "$@" "$url"; then +if ! nix_looking nix flake metadata --json --no-write-lock-file "$@" "$url"; then rm -R "$empty" >&2 printf 'error: source %s draws inputs from other sources, but its flake can'"'"'t be locked\n' \ "$name" @@ -864,14 +961,17 @@ own_lock "$dir" nix flake lock "$@" "$url" >&2 "#; - /// derived_state DIR TARGET_PATH PATCH_DIR DRAWN...: removes what an - /// interrupted run left in the source's derived state DIR, and prints - /// what tells whether the copy there needs to be made again, as fields - /// ending in null bytes: where DIR/original resolves to, the key of the - /// copy, the store path DIR/result links to, if the store has it, link - /// or directory, if TARGET_PATH is one, a line of sha256sum for each - /// patch in PATCH_DIR, an empty field, and the hash of each DRAWN path, - /// or nothing if it resolves into the store. + /// derived_state DIR TARGET_PATH PATCH_DIR CHECKED DRAWN...: removes + /// what an interrupted run left in the source's derived state DIR, and + /// prints what tells whether the copy there needs to be made again, as + /// fields ending in null bytes: where DIR/original resolves to, the key + /// of the copy, the store path DIR/result links to, if the store has + /// it, link or directory, if TARGET_PATH is one, a line of sha256sum + /// for each patch in PATCH_DIR, an empty field, and what tells whether + /// the DRAWN path of each pair of CHECKED and DRAWN changed: nothing if + /// it resolves into the store, git and the commit, if clean_git_rev + /// DRAWN CHECKED gives one, or else its hash, which takes reading all + /// of it. pub const DERIVED_STATE: &str = r#" dir=$1 target_path=$2 patch_dir=$3 shift 3 @@ -901,11 +1001,18 @@ if test -n "$patch_dir"; then done fi printf '\0' -for drawn; do - drawn=$(readlink -f "$drawn") +while test $# -gt 0; do + checked=$1 drawn=$(readlink -f "$2") + shift 2 case $drawn in /nix/store/*) printf '\0' ;; - *) printf '%s\0' "$(nix-hash --type sha256 "$drawn")" ;; + *) + if rev=$(clean_git_rev "$drawn" "$checked") && test -n "$rev"; then + printf 'git %s\0' "$rev" + else + printf '%s\0' "$(nix-hash --type sha256 "$drawn")" + fi + ;; esac done "#; @@ -973,16 +1080,31 @@ ln -fns "$result${subdir:+/$subdir}" "$target_path" "#; /// generate DIR TEXT: makes DIR a flake, with TEXT as its flake.nix, - /// and locks it. + /// and locks it anew, so the lock file pins what the inputs are now. + /// What Nix prints about creating the lock file only gets printed if + /// it differs from the one there was, since that's on every run. pub const GENERATE: &str = r#" dir=$1 text=$2 mkdir -p "$(dirname "$dir")" +old_lock= +if ! test -L "$dir"/flake.lock && test -f "$dir"/flake.lock; then + old_lock=$(cat "$dir"/flake.lock) +fi rm -fR "$dir" mkdir "$dir" cd "$dir" printf %s "$text" > flake.nix +log=$(mktemp) # Not just ., which Nix takes for the enclosing Git work tree, if any. -nix flake lock "path:$(url_path "$(pwd -P)")" >&2 +if ! nix flake lock "path:$(url_path "$(pwd -P)")" >&2 2>"$log"; then + cat "$log" >&2 + rm "$log" + exit 1 +fi +if test "$(cat flake.lock)" != "$old_lock"; then + cat "$log" >&2 +fi +rm "$log" "#; /// clean_up SENTINEL NAMES NOT_FLAKES GENERATED DERIVED PATCHED: removes |
