Merge legacy fetchGit-builtin with the generic fetchTree-function

The original idea was to implement a git-fetcher in Nix's core that
supports content hashes[1]. In NixOS#3549[2] it has been suggested to
actually use `fetchTree` for this since it's a fairly generic wrapper
over the new fetcher-API[3] and already supports content-hashes.

This patch implements a new git-fetcher based on `fetchTree` by
incorporating the following changes:

* Removed the original `fetchGit`-implementation and replaced it with an
  alias on the `fetchTree` implementation.

* Ensured that the `git`-fetcher from `libfetchers` always computes a
  content-hash and returns an "empty" revision on dirty trees (the
  latter one is needed to retain backwards-compatibility).

* The hash-mismatch error in the fetcher-API exits with code 102 as it
  usually happens whenever a hash-mismatch is detected by Nix.

* Removed the `flakes`-feature-flag: I didn't see a reason why this API
  is so tightly coupled to the flakes-API and at least `fetchGit` should
  remain usable without any feature-flags.

* It's only possible to specify a `narHash` for a `git`-tree if either a
  `ref` or a `rev` is given[4].

* It's now possible to specify an URL without a protocol. If it's missing,
  `file://` is automatically added as it was the case in the original
  `fetchGit`-implementation.

[1] NixOS#3216
[2] NixOS#3549 (comment)
[3] NixOS#3459
[4] NixOS#3216 (comment)

src/libexpr/primops/fetchTree.cc

@@ -36,6 +36,9 @@ void emitTreeAttrs(
         mkString(*state.allocAttr(v, state.symbols.create("shortRev")), rev->gitShortRev());
     }
 
+    if (input.getType() == "git")
+        mkBool(*state.allocAttr(v, state.symbols.create("submodules")), maybeGetBoolAttr(input.attrs, "submodules").value_or(false));
+
     if (auto revCount = input.getRevCount())
         mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *revCount);
 
@@ -48,10 +51,25 @@ void emitTreeAttrs(
     v.attrs->sort();
 }
 
-static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, Value & v)
+std::string fixURI(std::string uri, EvalState &state)
 {
-    settings.requireExperimentalFeature("flakes");
+    state.checkURI(uri);
+    return uri.find("://") != std::string::npos ? uri : "file://" + uri;
+}
 
+void addURI(EvalState &state, fetchers::Attrs &attrs, Symbol name, std::string v)
+{
+    string n(name);
+    attrs.emplace(name, n == "url" ? fixURI(v, state) : v);
+}
+
+static void fetchTree(
+    EvalState &state,
+    const Pos &pos,
+    Value **args,
+    Value &v,
+    const std::optional<std::string> type
+) {
     fetchers::Input input;
     PathSet context;
 
@@ -64,8 +82,15 @@ static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, V
 
         for (auto & attr : *args[0]->attrs) {
             state.forceValue(*attr.value);
-            if (attr.value->type == tString)
-                attrs.emplace(attr.name, attr.value->string.s);
+            if (attr.value->type == tPath || attr.value->type == tString)
+                addURI(
+                    state,
+                    attrs,
+                    attr.name,
+                    state.coerceToString(*attr.pos, *attr.value, context, false, false)
+                );
+            else if (attr.value->type == tString)
+                addURI(state, attrs, attr.name, attr.value->string.s);
             else if (attr.value->type == tBool)
                 attrs.emplace(attr.name, fetchers::Explicit<bool>{attr.value->boolean});
             else if (attr.value->type == tInt)
@@ -75,15 +100,28 @@ static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, V
                     attr.name, showType(*attr.value));
         }
 
+        if (type)
+            attrs.emplace("type", type.value());
+
         if (!attrs.count("type"))
             throw Error({
                 .hint = hintfmt("attribute 'type' is missing in call to 'fetchTree'"),
                 .errPos = pos
             });
 
         input = fetchers::Input::fromAttrs(std::move(attrs));
-    } else
-        input = fetchers::Input::fromURL(state.coerceToString(pos, *args[0], context, false, false));
+    } else {
+        auto url = fixURI(state.coerceToString(pos, *args[0], context, false, false), state);
+
+        if (type == "git") {
+            fetchers::Attrs attrs;
+            attrs.emplace("type", "git");
+            attrs.emplace("url", url);
+            input = fetchers::Input::fromAttrs(std::move(attrs));
+        } else {
+            input = fetchers::Input::fromURL(url);
+        }
+    }
 
     if (!evalSettings.pureEval && !input.isDirect())
         input = lookupInRegistries(state.store, input).first;
@@ -99,6 +137,11 @@ static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, V
     emitTreeAttrs(state, tree, input2, v);
 }
 
+static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, Value & v)
+{
+    fetchTree(state, pos, args, v, std::nullopt);
+}
+
 static RegisterPrimOp r("fetchTree", 1, prim_fetchTree);
 
 static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v,
@@ -178,7 +221,13 @@ static void prim_fetchTarball(EvalState & state, const Pos & pos, Value * * args
     fetch(state, pos, args, v, "fetchTarball", true, "source");
 }
 
+static void prim_fetchGit(EvalState &state, const Pos &pos, Value **args, Value &v)
+{
+    fetchTree(state, pos, args, v, "git");
+}
+
 static RegisterPrimOp r2("__fetchurl", 1, prim_fetchurl);
 static RegisterPrimOp r3("fetchTarball", 1, prim_fetchTarball);
+static RegisterPrimOp r4("fetchGit", 1, prim_fetchGit);
 
 }

src/libfetchers/fetchers.cc

@@ -134,7 +134,7 @@ std::pair<Tree, Input> Input::fetch(ref<Store> store) const
 
     if (auto prevNarHash = getNarHash()) {
         if (narHash != *prevNarHash)
-            throw Error("NAR hash mismatch in input '%s' (%s), expected '%s', got '%s'",
+            throw Error((unsigned int) 102, "NAR hash mismatch in input '%s' (%s), expected '%s', got '%s'",
                 to_string(), tree.actualPath, prevNarHash->to_string(SRI, true), narHash.to_string(SRI, true));
     }
 

src/libfetchers/git.cc

@@ -271,6 +271,10 @@ struct GitInputScheme : InputScheme
                     "lastModified",
                     haveCommits ? std::stoull(runProgram("git", true, { "-C", actualUrl, "log", "-1", "--format=%ct", "HEAD" })) : 0);
 
+                // Backward compatibility: set 'rev' to
+                // 0000000000000000000000000000000000000000 for a dirty tree.
+                input.attrs.insert_or_assign("rev", "0000000000000000000000000000000000000000");
+
                 return {
                     Tree(store->printStorePath(storePath), std::move(storePath)),
                     input

tests/fetchGit.sh

@@ -32,6 +32,8 @@ rev2=$(git -C $repo rev-parse HEAD)
 # Fetch a worktree
 unset _NIX_FORCE_HTTP
 path0=$(nix eval --impure --raw --expr "(builtins.fetchGit file://$TEST_ROOT/worktree).outPath")
+path0_=$(nix eval --impure --raw --expr "(builtins.fetchTree { type = \"git\"; url = file://$TEST_ROOT/worktree; }).outPath")
+[[ $path0 = $path0_ ]]
 export _NIX_FORCE_HTTP=1
 [[ $(tail -n 1 $path0/hello) = "hello" ]]
 
@@ -102,6 +104,12 @@ git -C $repo commit -m 'Bla3' -a
 path4=$(nix eval --impure --refresh --raw --expr "(builtins.fetchGit file://$repo).outPath")
 [[ $path2 = $path4 ]]
 
+nix eval --impure --raw --expr "(builtins.fetchGit { url = $repo; rev = \"$rev2\"; narHash = \"sha256-B5yIPHhEm0eysJKEsO7nqxprh9vcblFxpJG11gXJus1=\"; }).outPath" || status=$?
+[[ "$status" = "102" ]]
+
+path5=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = $repo; rev = \"$rev2\"; narHash = \"sha256-Hr8g6AqANb3xqX28eu1XnjK/3ab8Gv6TJSnkb1LezG9=\"; }).outPath")
+[[ $path = $path5 ]]
+
 # tarball-ttl should be ignored if we specify a rev
 echo delft > $repo/hello
 git -C $repo add hello

tests/tarball.sh

@@ -27,10 +27,13 @@ test_tarball() {
 
     nix-build -o $TEST_ROOT/result -E "import (fetchTarball file://$tarball)"
 
-    nix-build --experimental-features flakes -o $TEST_ROOT/result -E "import (fetchTree file://$tarball)"
-    nix-build --experimental-features flakes -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; })"
-    nix-build --experimental-features flakes -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"$hash\"; })"
-    nix-build --experimental-features flakes -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"sha256-xdKv2pq/IiwLSnBBJXW8hNowI4MrdZfW+SYqDQs7Tzc=\"; })" 2>&1 | grep 'NAR hash mismatch in input'
+    nix-build  -o $TEST_ROOT/result -E "import (fetchTree file://$tarball)"
+    nix-build  -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; })"
+    nix-build  -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"$hash\"; })"
+    nix-build  -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"sha256-xdKv2pq/IiwLSnBBJXW8hNowI4MrdZfW+SYqDQs7Tzc=\"; })" 2>&1 | grep 'NAR hash mismatch in input'
+
+    nix-instantiate --strict --eval -E "!((import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"$hash\"; })) ? submodules)" >&2
+    nix-instantiate --strict --eval -E "!((import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"$hash\"; })) ? submodules)" 2>&1 | grep 'true'
 
     nix-instantiate --eval -E '1 + 2' -I fnord=file://no-such-tarball.tar$ext
     nix-instantiate --eval -E 'with <fnord/xyzzy>; 1 + 2' -I fnord=file://no-such-tarball$ext