populate: never write through files or change hard links elsewhere

Populate writes some files in place, and makes files writable, which
reaches beyond the target directory where such a file is a symlink or a
hard link to one elsewhere:

- The key of a derived copy gets written to a new file, which then
  replaces it, instead of written through a symlink at its place.
- Lock files get copied to a new file before Nix rewrites them, instead
  of to flake.lock.populate, which a source may contain as a symlink.
- Before rsync syncs a source, each file with other hard links gets
  replaced by a copy of its own.  rsync fixes the mode and time of a file
  it takes for unchanged in place, which changes every hard link to it:
  one elsewhere, e.g. in /etc/nixos, or a backup's.
- Removing state makes only directories writable, which is what
  removing what's in them needs, instead of everything, including files
  that are hard links to ones elsewhere.
- The sentinel file only gets created if it's missing, instead of
  touched on every run.
- A link only counts as pointing into the store if its resolved path is
  in the store, so a link like /nix/store/../.. gets replaced.

Safety scenarios cover a symlink in place of a key, symlinks in a source
named like the lock file's copies, a hard link to a read-only file
outside in the target directory, and in stale state.

3 files changed, 144 insertions, 13 deletions

diff --git a/README.md b/README.md
index 969afcd..c917d3f 100644
--- a/README.md
+++ b/README.md
@@ -316,6 +316,12 @@ everything else.  It doesn't change or remove anything outside:
   fails, telling what's wrong, unless the source's `reclone` is set.  Git
   runs no hooks.
 
+* A file in the target directory that has other hard links, e.g. one in
+  `/etc/nixos`, gets replaced by a copy of its own before rsync syncs it, so
+  the other ones keep their mode and time.  Populate writes its own files,
+  and replaces lock files it rewrites, by renaming new files into place, and
+  only makes directories writable that it removes.
+
 * Populate keeps its state in the sentinel, `.populate`, which becomes a
   directory once there is state to keep.  Populating fails if the sentinel,
   or a directory populate needs therein, is a symlink.
diff --git a/pkgs/populate/default.nix b/pkgs/populate/default.nix
index 7130570..71844cb 100644
--- a/pkgs/populate/default.nix
+++ b/pkgs/populate/default.nix
@@ -65,12 +65,14 @@ let
 
   # Shell code that removes what's at path, a shell word, with everything
   # below it, also what isn't writable.  A symlink gets removed itself,
-  # instead of followed.
+  # instead of followed.  Only directories get made writable, which is what
+  # removing what's in them needs: a file may be a hard link to one
+  # elsewhere, which shares its mode.
   removeTree = path: /* sh */ ''
     if test -L ${path}; then
       rm ${path}
     elif test -e ${path}; then
-      chmod -R u+w ${path}
+      find ${path} -type d -exec chmod u+w {} +
       rm -fR ${path}
     fi
   '';
@@ -79,8 +81,7 @@ let
   # that aren't named names.  find, unlike a loop reading names, copes with
   # every name, e.g. ones that end in blanks or contain newlines.  Entries of
   # populate's own state get removed like removeTree does.  Entries of the
-  # target directory get removed without making them writable first, which
-  # would change files elsewhere that are hard links to files therein; for
+  # target directory get removed without making anything writable first; for
   # those, log is a shell word for the directory's name in the line that
   # gets printed for each entry.
   removeOthers = { dir, names, log ? null }: /* sh */ ''
@@ -101,7 +102,8 @@ let
               if test -L "$path"; then
                 rm "$path"
               else
-                chmod -R u+w "$path"
+                # Escaped, or the outer find takes the braces for its own.
+                find "$path" -type d -exec chmod u+w {\} +
                 rm -fR "$path"
               fi
             ''}
@@ -124,7 +126,8 @@ let
   # it points into the store, like the links populate puts there itself.
   removeForeignLink = path: /* sh */ ''
     if test -L ${quote path}; then
-      case $(readlink ${quote path}) in
+      # Resolved, so a link like /nix/store/../.. doesn't count.
+      case $(readlink -f ${quote path}) in
         /nix/store/*) ;;
         *)
           ${removeLink path}
@@ -217,7 +220,9 @@ let
     fi
     ${optionalString force /* sh */ ''
       mkdir -vp ${quote (dirOf sentinelFile)} >&2
-      touch ${quote sentinelFile}
+      if ! test -e ${quote sentinelFile}; then
+        touch ${quote sentinelFile}
+      fi
     ''}
     if ! test -e ${quote sentinelFile}; then
       >&2 printf 'error: missing sentinel file: %s\n' ${quote (
@@ -371,6 +376,13 @@ let
       removeWork = removeTree (quote "${dir}/work");
     in /* sh */ ''
       (
+        # Writes the key $1 to a new file, which then replaces the key, so
+        # nothing at its place gets written through, e.g. a symlink.
+        write_key() {
+          key_file=$(mktemp ${quote "${dir}/key"}.XXXXXX)
+          printf '%s\n' "$1" > "$key_file"
+          mv -f "$key_file" ${quote "${dir}/key"}
+        }
         original=$(readlink -f ${quote "${dir}/original"})
         path=$(printf '%s\n' "$original" | cut -d/ -f1-4)
         subdir=''${original#"$path"}
@@ -417,7 +429,7 @@ let
             ${removeForeignLink targetPath}
             rm -fR ${quote targetPath}
             mv ${quote "${dir}/work"} ${quote targetPath}
-            printf '%s\n' "$key" > ${quote "${dir}/key"}
+            write_key "$key"
           fi
           ${optionalString rewriteLock /* sh */ ''
             ${overrideArgs name}
@@ -450,7 +462,7 @@ let
             ${removeWork}
             nix-store --realise "$result" \
                 --add-root ${quote "${dir}/result"} --indirect >/dev/null
-            printf '%s\n' "$key" > ${quote "${dir}/key"}
+            write_key "$key"
           fi
           ${makeRoomForLink targetPath}
           # .populate/derived/<name>/result roots the store path.
@@ -530,10 +542,13 @@ let
         # a hard link to the lock file, e.g. in a backup, or what it is a
         # symlink to.  So the lock file of the flake in the directory $1
         # gets replaced by a copy first.
+        # The copy goes to a new file, so nothing that's there already
+        # gets written through.
         own_lock() {
           if test -e "$1"/flake.lock; then
-            cp -p "$1"/flake.lock "$1"/flake.lock.populate
-            mv -f "$1"/flake.lock.populate "$1"/flake.lock
+            own_lock_copy=$(mktemp "$1"/.flake.lock.XXXXXX)
+            cp -p "$1"/flake.lock "$own_lock_copy"
+            mv -f "$own_lock_copy" "$1"/flake.lock
           fi
         }
 
@@ -1259,8 +1274,22 @@ let
 
   rsync' = target: config: sourcePath: /* sh */ ''
     # rsync writes through a symlink at the target path, e.g. one a symlink or
-    # flake source left there, into what it points to.
-    ${runShell target (removeLink target.path)}
+    # flake source left there, into what it points to.  And rsync fixes the
+    # mode and time of a file it takes for unchanged in place, which changes
+    # every hard link to it, e.g. one elsewhere, or a backup's.  So each file
+    # with other hard links gets replaced by a copy of its own first.
+    ${runShell target /* sh */ ''
+      ${removeLink target.path}
+      if test -d ${quote target.path}; then
+        find ${quote target.path} -type f -links +1 -exec sh -c '
+          for file; do
+            copy=$(mktemp "$file.XXXXXX")
+            cp -p "$file" "$copy"
+            mv -f "$copy" "$file"
+          done
+        ' sh {} +
+      fi
+    ''}
     source_path=${sourcePath}
     if test -d "$source_path"; then
       source_path=$source_path/
diff --git a/tests/scenarios.nix b/tests/scenarios.nix
index 6e1f89b..89a3b09 100644
--- a/tests/scenarios.nix
+++ b/tests/scenarios.nix
@@ -2013,6 +2013,102 @@ assert !(krops.isLocalTarget (targets.remote "/"));
         }
       ];
     }))
+    # A file in populate's state replaced by a symlink to a file outside.
+    (s "state-key" (let
+      sources.x.flake-input = {
+        url = "/fixtures/top"; name = "dep"; inherit fetchOnTarget;
+        patches = hello;
+      };
+    in {
+      isolate = true;
+      phases = [
+        {
+          inherit sources;
+          mutate = ''
+            rm "$T"/.populate/derived/x/key
+            ln -s ${outside}/file "$T"/.populate/derived/x/key
+          '';
+        }
+        {
+          inherit sources;
+          check = checkFlake { attr = "greeting"; expected = "hello+inner"; };
+        }
+      ];
+    }))
+    # Stale state that contains a hard link to a read-only file outside,
+    # whose mode stays.
+    (s "state-hard-link" (let
+      sources.x.flake-input = {
+        url = "/fixtures/top"; name = "dep"; inherit fetchOnTarget;
+        patches = hello;
+      };
+    in {
+      isolate = true;
+      phases = [
+        {
+          inherit sources;
+          mutate = ''
+            mkdir -p "$T"/.populate/derived/gone "$T"/.populate/derived/x/work
+            ln ${outside}/dir/sub/file "$T"/.populate/derived/gone/file
+            ln ${outside}/dir/sub/file "$T"/.populate/derived/x/work/file
+          '';
+        }
+        {
+          inherit sources;
+          check = ''
+            test ! -e "$T"/.populate/derived/gone || fail "gone remains"
+          '';
+        }
+      ];
+    }))
+    # The source contains flake.lock.populate and a file like the temporary
+    # copy of the lock file, symlinks to a file outside.
+    (s "lock-temporary-link" {
+      isolate = true;
+      prepareDeployer = ''
+        rm -fR /var/krops-src/lock-temporary-link
+        mkdir -p /var/krops-src
+        cp -R /fixtures/filesrc /var/krops-src/lock-temporary-link
+        ln -s ${outside}/file /var/krops-src/lock-temporary-link/flake.lock.populate
+        ln -s ${outside}/file /var/krops-src/lock-temporary-link/.flake.lock.XXXXXX
+      '';
+      phases = twice {
+        sources = alt // {
+          x.file = {
+            path = "/var/krops-src/lock-temporary-link";
+            inputs.dep.follows = "alt";
+          };
+        };
+        check = ''
+          expect_eq msg "$(nix_eval_online path:"$T"/x#msg)" top:alt,sub
+        '';
+      };
+    })
+    # A file in the target directory is a hard link to a read-only file
+    # outside with the same contents, which rsync takes for unchanged, and
+    # would fix the mode and time of in place.
+    (s "rsync-hard-link" {
+      isolate = true;
+      prepareDeployer = ''
+        rm -fR /var/krops-src/rsync-hard-link
+        mkdir -p /var/krops-src/rsync-hard-link
+        echo outside > /var/krops-src/rsync-hard-link/file
+      '';
+      prepare = ''
+        mkdir -p "$T"/x
+        ln ${outside}/dir/sub/file "$T"/x/file
+      '';
+      phases = twice {
+        sources.x.file = {
+          path = "/var/krops-src/rsync-hard-link";
+          useChecksum = true;
+        };
+        check = ''
+          expect_eq file "$(cat "$T"/x/file)" outside
+          expect_eq links "$(stat -c %h "$T"/x/file)" 1
+        '';
+      };
+    })
     (lib.optionals (targetKind == "local") [
       # Rewriting a lock file changes no other hard link to it.  The lock
       # file is like the source's, with its modification time, so rsync