aboutsummaryrefslogtreecommitdiffstats
path: root/pkgs/populate/src/sh.rs
diff options
context:
space:
mode:
Diffstat (limited to 'pkgs/populate/src/sh.rs')
-rw-r--r--pkgs/populate/src/sh.rs362
1 files changed, 242 insertions, 120 deletions
diff --git a/pkgs/populate/src/sh.rs b/pkgs/populate/src/sh.rs
index 70efd3f..66ca04d 100644
--- a/pkgs/populate/src/sh.rs
+++ b/pkgs/populate/src/sh.rs
@@ -327,104 +327,125 @@ git_problems() (
/// PATH a Git work tree of the repository at URL, with REF checked out,
/// and everything removed that Git doesn't track, except what matches an
/// EXCLUDE pattern. SHALLOW, FETCH_ALWAYS, and RECLONE are flags, empty
-/// for false. NAME is PATH's name in messages.
+/// for false. NAME is PATH's name in messages. Prints checked if
+/// git_problems found nothing wrong with the work tree that was there,
+/// which then is what's there, and else nothing; what commands print goes
+/// to standard error.
pub fn git() -> String {
LINKS.to_string()
+ GIT_PROBLEMS
+ r#"
-path=$1 name=$2 url=$3 ref=$4 shallow=$5 fetch_always=$6 reclone=$7
-shift 7
-# Hooks and commands a repository's configuration names run with Git's
-# commands, so populate's Git doesn't run any. The configuration gets
-# checked as well, but Git also runs a hook it has itself.
-git() {
- command git -c core.hooksPath=/dev/null -c core.fsmonitor=false "$@"
-}
-# A target path that doesn't look like a Git work tree gets removed, e.g.
-# when it had another type before. A work tree that a symlink points to,
-# or whose .git is a symlink or a file naming a repository elsewhere, isn't
-# populate's own: fetching, resetting, and cleaning would change what's
-# elsewhere.
-remove_link "$path" "$name"
-if test -L "$path"/.git || ! test -d "$path"/.git; then
- rm -fR "$path"
-fi
-# A .git directory can name places elsewhere too, which Git's commands
-# change, or commands, which they run. Populate only uses a work tree
-# whose .git directory has nothing like that, and else replaces it if
-# reclone is set, or fails, telling why.
-if test -e "$path"; then
- if problems=$(git_problems "$path"); then :; else
- exit 1
+# What commands print goes to standard error. None of them gets the
+# standard output, which populate reads to its end, so nothing they leave
+# running, e.g. git gc --auto, keeps populate waiting.
+checked=
+deliver_git() {
+ path=$1 name=$2 url=$3 ref=$4 shallow=$5 fetch_always=$6 reclone=$7
+ shift 7
+ # Hooks and commands a repository's configuration names run with Git's
+ # commands, so populate's Git doesn't run any. The configuration gets
+ # checked as well, but Git also runs a hook it has itself.
+ git() {
+ command git -c core.hooksPath=/dev/null -c core.fsmonitor=false "$@"
+ }
+ # A target path that doesn't look like a Git work tree gets removed, e.g.
+ # when it had another type before. A work tree that a symlink points to,
+ # or whose .git is a symlink or a file naming a repository elsewhere, isn't
+ # populate's own: fetching, resetting, and cleaning would change what's
+ # elsewhere.
+ remove_link "$path" "$name"
+ if test -L "$path"/.git || ! test -d "$path"/.git; then
+ rm -fR "$path"
fi
- if test -n "$problems"; then
- if test -n "$reclone"; then
- >&2 printf "replacing the Git work tree '%s', because %s...\n" \
- "$name" "$problems"
- rm -fR "$path"
- else
- >&2 printf 'error: populate doesn'"'"'t use the Git work tree %s, because %s. Fix that, remove the work tree, or set the source'"'"'s reclone to true to replace it by a fresh clone.\n' \
- "$name" "$problems"
+ # A .git directory can name places elsewhere too, which Git's commands
+ # change, or commands, which they run. Populate only uses a work tree
+ # whose .git directory has nothing like that, and else replaces it if
+ # reclone is set, or fails, telling why.
+ if test -e "$path"; then
+ if problems=$(git_problems "$path"); then :; else
exit 1
fi
+ if test -n "$problems"; then
+ if test -n "$reclone"; then
+ >&2 printf "replacing the Git work tree '%s', because %s...\n" \
+ "$name" "$problems"
+ rm -fR "$path"
+ else
+ >&2 printf 'error: populate doesn'"'"'t use the Git work tree %s, because %s. Fix that, remove the work tree, or set the source'"'"'s reclone to true to replace it by a fresh clone.\n' \
+ "$name" "$problems"
+ exit 1
+ fi
+ else
+ checked=1
+ fi
fi
-fi
-if ! test -e "$path"; then
- if test -n "$shallow"; then
- git init "$path"
- else
- git clone --recurse-submodules -- "$url" "$path"
+ if ! test -e "$path"; then
+ if test -n "$shallow"; then
+ git init "$path"
+ else
+ git clone --recurse-submodules -- "$url" "$path"
+ fi
fi
-fi
-cd "$path"
-if ! origin=$(git config remote.origin.url); then
- git remote add -- origin "$url"
-elif test "$origin" != "$url"; then
- git remote set-url -- origin "$url"
-fi
-# A commit, given by its hash or a prefix thereof, gets fetched only if the
-# work tree lacks it, unless fetchAlways is set. After --, a ref can't be
-# taken for an option. A branch or tag gets
-# fetched on every run, so the work tree follows it: resetting to its name
-# would reset to a local branch, which fetching doesn't move.
-fetch() {
- if test -n "$shallow"; then
- git fetch --depth=1 origin -- "$1"
- else
- git fetch origin -- "$1"
+ cd "$path"
+ if ! origin=$(git config remote.origin.url); then
+ git remote add -- origin "$url"
+ elif test "$origin" != "$url"; then
+ git remote set-url -- origin "$url"
fi
-}
-fetch_commit() {
- if test -n "$shallow"; then
- fetch "$hash"
+ # A commit, given by its hash or a prefix thereof, gets fetched only if the
+ # work tree lacks it, unless fetchAlways is set. After --, a ref can't be
+ # taken for an option. A branch or tag gets
+ # fetched on every run, so the work tree follows it: resetting to its name
+ # would reset to a local branch, which fetching doesn't move.
+ fetch() {
+ if test -n "$shallow"; then
+ git fetch --depth=1 origin -- "$1"
+ else
+ git fetch origin -- "$1"
+ fi
+ }
+ fetch_commit() {
+ if test -n "$shallow"; then
+ fetch "$hash"
+ else
+ git fetch origin
+ fi
+ }
+ if printf %s "$ref" | grep -Eqx '[0-9a-f]{4,40}'; then
+ hash=$ref
+ if ! test "$(git log --format=%H -1)" = "$hash"; then
+ if test -n "$fetch_always"; then
+ fetch_commit
+ elif ! git log -1 "$hash" >/dev/null 2>&1; then
+ fetch_commit
+ fi
+ fi
else
- git fetch origin
+ fetch "$ref"
+ hash=$(git rev-parse --verify 'FETCH_HEAD^{commit}')
fi
-}
-if printf %s "$ref" | grep -Eqx '[0-9a-f]{4,40}'; then
- hash=$ref
- if ! test "$(git log --format=%H -1)" = "$hash"; then
- if test -n "$fetch_always"; then
- fetch_commit
- elif ! git log -1 "$hash" >/dev/null 2>&1; then
- fetch_commit
- fi
+ # Also if HEAD is there already, unless no tracked file differs from it:
+ # rewriting a lock file in place, e.g., changes the work tree, and stages
+ # the lock file if Nix created it.
+ head=$(git rev-parse --verify --quiet HEAD) || head=
+ commit=$(git rev-parse --verify --quiet "$hash^{commit}") || commit=
+ changes=$(git status --porcelain --untracked-files=no) || changes=unknown
+ if test -z "$head" || test "$head" != "$commit" || test -n "$changes"; then
+ git reset --hard "$hash"
fi
-else
- fetch "$ref"
- hash=$(git rev-parse --verify 'FETCH_HEAD^{commit}')
+ git submodule update --init --recursive
+ n=$#
+ while test "$n" -gt 0; do
+ set -- "$@" -e "$1"
+ shift
+ n=$((n - 1))
+ done
+ git clean -dfx "$@"
+}
+deliver_git "$@" >&2
+if test -n "$checked"; then
+ printf checked
fi
-# Also if HEAD is there already: rewriting a lock file in place, e.g.,
-# changes the work tree, and stages the lock file if Nix created it.
-git reset --hard "$hash" >&2
-git submodule update --init --recursive
-n=$#
-while test "$n" -gt 0; do
- set -- "$@" -e "$1"
- shift
- n=$((n - 1))
-done
-git clean -dfx "$@"
"#
}
@@ -544,6 +565,50 @@ require_flakes() {
}
"#;
+/// Defines nix_looking COMMAND..., which runs COMMAND, a Nix command
+/// looking at a flake without writing its lock file, and leaves out of
+/// what it prints on standard error Nix's warning that it doesn't write
+/// the lock file it changed, listing what changed. The shell code for
+/// flakes overrides inputs drawn from sources when looking at flakes,
+/// which changes the lock file on every run, without anything changing,
+/// or says itself that a lock file needs to be updated.
+pub const NIX_LOOKING: &str = r#"
+nix_looking() {
+ nix_looking_status=$(mktemp)
+ {
+ {
+ if "$@" 2>&1 >&3 3>&-; then
+ nix_looking_code=0
+ else
+ nix_looking_code=$?
+ fi
+ echo "$nix_looking_code" > "$nix_looking_status"
+ } | {
+ # The warning's lines after the first begin with a bullet or blanks.
+ nix_looking_skip=
+ while IFS= read -r nix_looking_line || test -n "$nix_looking_line"; do
+ case $nix_looking_line in
+ 'warning: not writing modified lock file of flake '*)
+ nix_looking_skip=1
+ continue
+ ;;
+ '• '*|' '*)
+ if test -n "$nix_looking_skip"; then
+ continue
+ fi
+ ;;
+ esac
+ nix_looking_skip=
+ printf '%s\n' "$nix_looking_line"
+ done
+ } >&2
+ } 3>&1
+ read -r nix_looking_code < "$nix_looking_status"
+ rm -f "$nix_looking_status"
+ return "$nix_looking_code"
+}
+"#;
+
/// The definitions the shell code for flakes uses.
pub fn prelude() -> String {
[
@@ -553,6 +618,7 @@ pub fn prelude() -> String {
REMOVE_OTHERS,
URL_PATH,
REQUIRE_FLAKES,
+ NIX_LOOKING,
]
.concat()
}
@@ -600,7 +666,7 @@ nix-store --check-validity "$1" 2>/dev/null
/// lock file, since locking would change it. If its inputs can't be
/// archived, it fails if REQUIRE isn't empty, saying so for the source
/// NAME, with HINT appended, or else archives what Nix locks now, or
- /// prints nothing, with a warning.
+ /// prints nothing, with a warning. It needs NIX_LOOKING.
pub const ARCHIVE: &str = r#"
path=$1 subdir=$2 require=$3 name=$4 hint=$5
shift 5
@@ -618,14 +684,14 @@ if test -e "$dir"/flake.nix && test -e "$dir"/flake.lock; then
n=$((n - 1))
done
fi
- if nix flake archive --json --no-update-lock-file "$@" "$flake"; then
+ if nix_looking nix flake archive --json --no-update-lock-file "$@" "$flake"; then
:
elif test -n "$require"; then
>&2 printf 'error: source %s: the inputs its own lock file pins can'"'"'t be delivered%s\n' \
"$name" "$hint"
exit 1
# Without --no-update-lock-file, Nix adds what the lock file lacks.
- elif nix flake archive --json --no-write-lock-file "$@" "$flake"; then
+ elif nix_looking nix flake archive --json --no-write-lock-file "$@" "$flake"; then
>&2 printf 'warning: source %s: its own lock file needs to be updated; delivering it with the inputs Nix locks now, which evaluating it may not use\n' \
"$name"
else
@@ -759,8 +825,9 @@ nix eval --impure --raw --expr '
/// the definitions of prelude and DEFINITIONS. Populate decides what to
/// do, from what these send back.
pub mod draw {
- /// Defines own_lock DIR, apply_patches PATCH_DIR NAME, and write_key
- /// KEY_FILE KEY.
+ /// Defines own_lock DIR, apply_patches PATCH_DIR NAME, write_key
+ /// KEY_FILE KEY, git_flake PATH CHECKED, and clean_git_rev PATH
+ /// CHECKED.
///
/// Nix rewrites lock files in place, which also changes what else is a
/// hard link to the lock file, e.g. in a backup, or what it is a
@@ -774,6 +841,19 @@ pub mod draw {
///
/// write_key writes KEY to a new file, which then replaces KEY_FILE, so
/// nothing at its place gets written through, e.g. a symlink.
+ ///
+ /// git_flake PATH CHECKED succeeds if Nix gets the directory at PATH
+ /// as a git+file: flake: if it has a .git directory that git_problems
+ /// finds nothing wrong with, and that has no hooks. Nix runs Git in a
+ /// git+file: flake, with the repository's configuration and hooks, so
+ /// it only gets such ones. If CHECKED isn't empty, git_problems found
+ /// nothing wrong already, so that doesn't get checked again: it takes
+ /// looking through all of the work tree, e.g. all of nixpkgs.
+ ///
+ /// clean_git_rev PATH CHECKED prints the commit HEAD of the Git work
+ /// tree at PATH points to, if git_flake PATH CHECKED, and none of its
+ /// tracked files differ from that commit, which is all of it Nix uses
+ /// then. Git runs without hooks and file system monitors.
pub const DEFINITIONS: &str = r#"
own_lock() {
if test -e "$1"/flake.lock; then
@@ -796,6 +876,31 @@ write_key() {
printf '%s\n' "$2" > "$key_file"
mv -f "$key_file" "$1"
}
+git_flake() {
+ ! test -L "$1"/.git && test -d "$1"/.git || return 1
+ if test -z "$2"; then
+ git_flake_problems=$(git_problems "$1" 2>/dev/null) || return 1
+ test -z "$git_flake_problems" || return 1
+ fi
+ if test -d "$1"/.git/hooks; then
+ test -z "$(find "$1"/.git/hooks -mindepth 1 ! -name '*.sample' | head -n 1)"
+ fi
+}
+clean_git_rev() (
+ git_flake "$1" "$2" || exit 1
+ cd "$1"
+ git() {
+ command git -c core.hooksPath=/dev/null -c core.fsmonitor=false "$@"
+ }
+ # Without writing the index: in a copy of a work tree, e.g. in a file
+ # source, that makes it differ from the original's, so the next run
+ # copies it again, unless the original's changed in the same second,
+ # keeping its size, which rsync doesn't notice. Then the work tree
+ # differs from its index, which Nix refuses to lock.
+ changes=$(git --no-optional-locks status --porcelain --untracked-files=no 2>/dev/null) || exit 1
+ test -z "$changes" || exit 1
+ git rev-parse --verify --quiet HEAD
+)
"#;
/// use_state SENTINEL NAME...
@@ -803,27 +908,19 @@ write_key() {
use_state "$@"
"#;
- /// resolve PATH...: prints, for each PATH, what's there and where it
- /// resolves to, as two fields, each ending in a null byte. The first
- /// holds d if PATH is a directory, g if it has a .git directory that
- /// git_problems finds nothing wrong with, and that has no hooks, s if
- /// it has .gitmodules, and f if it has flake.nix. Nix runs Git in a
- /// git+file: flake, with the repository's configuration and hooks, so
- /// it only gets such ones.
+ /// resolve CHECKED PATH...: prints, for each pair of CHECKED and PATH,
+ /// what's at PATH and where it resolves to, as two fields, each ending
+ /// in a null byte. The first holds d if PATH is a directory, g if
+ /// git_flake PATH CHECKED, s if it has .gitmodules, and f if it has
+ /// flake.nix.
pub const RESOLVE: &str = r#"
-for path; do
+while test $# -gt 0; do
+ checked=$1 path=$2
+ shift 2
resolved=$(readlink -f "$path") || resolved=
flags=
if test -d "$path"; then flags=${flags}d; fi
- if ! test -L "$path"/.git && test -d "$path"/.git; then
- problems=$(git_problems "$path" 2>/dev/null) || problems=failed
- if test -d "$path"/.git/hooks; then
- hooks=$(find "$path"/.git/hooks -mindepth 1 ! -name '*.sample' | head -n 1)
- else
- hooks=
- fi
- if test -z "$problems" && test -z "$hooks"; then flags=${flags}g; fi
- fi
+ if git_flake "$path" "$checked"; then flags=${flags}g; fi
if test -e "$path"/.gitmodules; then flags=${flags}s; fi
if test -e "$path"/flake.nix; then flags=${flags}f; fi
printf '%s\0%s\0' "$flags" "$resolved"
@@ -846,7 +943,7 @@ while test "$n" -gt 0; do
shift
n=$((n - 1))
done
-if ! nix flake metadata --json --no-write-lock-file "$@" "$url"; then
+if ! nix_looking nix flake metadata --json --no-write-lock-file "$@" "$url"; then
rm -R "$empty"
>&2 printf 'error: source %s draws inputs from other sources, but its flake can'"'"'t be locked\n' \
"$name"
@@ -864,14 +961,17 @@ own_lock "$dir"
nix flake lock "$@" "$url" >&2
"#;
- /// derived_state DIR TARGET_PATH PATCH_DIR DRAWN...: removes what an
- /// interrupted run left in the source's derived state DIR, and prints
- /// what tells whether the copy there needs to be made again, as fields
- /// ending in null bytes: where DIR/original resolves to, the key of the
- /// copy, the store path DIR/result links to, if the store has it, link
- /// or directory, if TARGET_PATH is one, a line of sha256sum for each
- /// patch in PATCH_DIR, an empty field, and the hash of each DRAWN path,
- /// or nothing if it resolves into the store.
+ /// derived_state DIR TARGET_PATH PATCH_DIR CHECKED DRAWN...: removes
+ /// what an interrupted run left in the source's derived state DIR, and
+ /// prints what tells whether the copy there needs to be made again, as
+ /// fields ending in null bytes: where DIR/original resolves to, the key
+ /// of the copy, the store path DIR/result links to, if the store has
+ /// it, link or directory, if TARGET_PATH is one, a line of sha256sum
+ /// for each patch in PATCH_DIR, an empty field, and what tells whether
+ /// the DRAWN path of each pair of CHECKED and DRAWN changed: nothing if
+ /// it resolves into the store, git and the commit, if clean_git_rev
+ /// DRAWN CHECKED gives one, or else its hash, which takes reading all
+ /// of it.
pub const DERIVED_STATE: &str = r#"
dir=$1 target_path=$2 patch_dir=$3
shift 3
@@ -901,11 +1001,18 @@ if test -n "$patch_dir"; then
done
fi
printf '\0'
-for drawn; do
- drawn=$(readlink -f "$drawn")
+while test $# -gt 0; do
+ checked=$1 drawn=$(readlink -f "$2")
+ shift 2
case $drawn in
/nix/store/*) printf '\0' ;;
- *) printf '%s\0' "$(nix-hash --type sha256 "$drawn")" ;;
+ *)
+ if rev=$(clean_git_rev "$drawn" "$checked") && test -n "$rev"; then
+ printf 'git %s\0' "$rev"
+ else
+ printf '%s\0' "$(nix-hash --type sha256 "$drawn")"
+ fi
+ ;;
esac
done
"#;
@@ -973,16 +1080,31 @@ ln -fns "$result${subdir:+/$subdir}" "$target_path"
"#;
/// generate DIR TEXT: makes DIR a flake, with TEXT as its flake.nix,
- /// and locks it.
+ /// and locks it anew, so the lock file pins what the inputs are now.
+ /// What Nix prints about creating the lock file only gets printed if
+ /// it differs from the one there was, since that's on every run.
pub const GENERATE: &str = r#"
dir=$1 text=$2
mkdir -p "$(dirname "$dir")"
+old_lock=
+if ! test -L "$dir"/flake.lock && test -f "$dir"/flake.lock; then
+ old_lock=$(cat "$dir"/flake.lock)
+fi
rm -fR "$dir"
mkdir "$dir"
cd "$dir"
printf %s "$text" > flake.nix
+log=$(mktemp)
# Not just ., which Nix takes for the enclosing Git work tree, if any.
-nix flake lock "path:$(url_path "$(pwd -P)")" >&2
+if ! nix flake lock "path:$(url_path "$(pwd -P)")" >&2 2>"$log"; then
+ cat "$log" >&2
+ rm "$log"
+ exit 1
+fi
+if test "$(cat flake.lock)" != "$old_lock"; then
+ cat "$log" >&2
+fi
+rm "$log"
"#;
/// clean_up SENTINEL NAMES NOT_FLAKES GENERATED DERIVED PATCHED: removes