Basic store path provenance tracking

Nix historically has been bad at being able to answer the question
"where did this store path come from", i.e. to provide traceability
from a store path back to the Nix expression from which is was
built. Nix tracks the "deriver" of a store path (the .drv file that
built it) but that's pretty useless in practice, since it doesn't link
back to the Nix expressions.

So this PR adds a "provenance" field (a JSON object) to the ValidPaths
table and to .narinfo files that describes where the store path came
from and how it can be reproduced.

There are currently 3 types of provenance:

* "copied": Records that the store path was copied or substituted from
  another store (typically a binary cache). Its "from" field is the
  URL of the origin store. Its "provenance" field propagates the
  provenance of the store path on the origin store.

* "derivation": Records that the store path is the output of a .drv
  file. This is equivalent for the "deriver" field, but it has a
  nested "provenance" field that records how the .drv file was
  created.

* "flake": Records that the store path was created during the
  evaluation of a flake output.

Example:

  $ nix path-info --json /nix/store/xcqzb13bd60zmfw6wv0z4242b9mfw042-patchelf-0.18.0
  {
    "/nix/store/xcqzb13bd60zmfw6wv0z4242b9mfw042-patchelf-0.18.0": {
      "provenance": {
        "from": "https://cache.example.org",
        "provenance": {
          "drv": "rlabxgjx88bavjkc694v1bqbwslwivxs-patchelf-0.18.0.drv",
          "output": "out",
          "provenance": {
            "flake": {
              "lastModified": 1729856604,
              "narHash": "sha256-obmE2ZI9sTPXczzGMerwQX4SALF+ABL9J0oB371yvZE=",
              "owner": "NixOS",
              "repo": "patchelf",
              "rev": "689f19e499caee8e5c3d387008bbd4ed7f8dc3a9",
              "type": "github",
            },
            "output": "packages.x86_64-linux.default",
            "type": "flake"
          },
          "type": "derivation"
        },
        "type": "copied"
      },
      ...
    }
  }

This specifies that the store path was copied from the binary cache
https://cache.example.org and it's the "out" output of a store
derivation that was produced by evaluating the flake ouput
`packages.x86_64-linux.default` of some revision of the patchelf
GitHub repository.

src/libcmd/installable-flake.cc

@@ -17,6 +17,7 @@
 #include "url.hh"
 #include "registry.hh"
 #include "build-result.hh"
+#include "provenance.hh"
 
 #include <regex>
 #include <queue>
@@ -81,6 +82,14 @@ DerivedPathsWithInfo InstallableFlake::toDerivedPaths()
 
     auto attrPath = attr->getAttrPathStr();
 
+    auto lockedRef = getLockedFlake()->flake.lockedRef;
+
+    state->setRootProvenance(std::make_shared<Provenance>(
+            Provenance::ProvFlake {
+                .flake = std::make_shared<nlohmann::json>(fetchers::attrsToJSON(lockedRef.input.attrs)),
+                .flakeOutput = attrPath,
+            }));
+
     if (!attr->isDerivation()) {
 
         // FIXME: use eval cache?
@@ -147,7 +156,7 @@ DerivedPathsWithInfo InstallableFlake::toDerivedPaths()
             },
             ExtraPathInfoFlake::Flake {
                 .originalRef = flakeRef,
-                .lockedRef = getLockedFlake()->flake.lockedRef,
+                .lockedRef = lockedRef,
             }),
     }};
 }

src/libexpr/eval.cc

@@ -20,6 +20,7 @@
 #include "fetch-to-store.hh"
 #include "tarball.hh"
 #include "parser-tab.hh"
+#include "provenance.hh"
 
 #include <algorithm>
 #include <iostream>
@@ -2364,7 +2365,8 @@ StorePath EvalState::copyPathToStore(NixStringContext & context, const SourcePat
                 path.baseName(),
                 ContentAddressMethod::Raw::NixArchive,
                 nullptr,
-                repair);
+                repair,
+                getRootProvenance());
             allowPath(dstPath);
             srcToStore.lock()->try_emplace(path, dstPath);
             printMsg(lvlChatty, "copied source '%1%' -> '%2%'", path, store->printStorePath(dstPath));
@@ -3163,4 +3165,15 @@ std::ostream & operator << (std::ostream & str, const ExternalValueBase & v) {
 }
 
 
+std::shared_ptr<const Provenance> EvalState::getRootProvenance()
+{
+    return rootProvenance;
+}
+
+
+void EvalState::setRootProvenance(std::shared_ptr<const Provenance> provenance)
+{
+    rootProvenance = provenance;
+}
+
 }

src/libexpr/eval.hh

@@ -37,6 +37,7 @@ class StorePath;
 struct SingleDerivedPath;
 enum RepairFlag : bool;
 struct MemorySourceAccessor;
+struct Provenance;
 namespace eval_cache {
     class EvalCache;
 }
@@ -863,6 +864,15 @@ private:
 
     friend struct Value;
     friend class ListBuilder;
+
+    // FIXME: how to handle this in the multi-threaded evaluator?
+    std::shared_ptr<const Provenance> rootProvenance;
+
+public:
+
+    std::shared_ptr<const Provenance> getRootProvenance();
+
+    void setRootProvenance(std::shared_ptr<const Provenance> provenance);
 };
 
 struct DebugTraceStacker {

src/libexpr/primops.cc

@@ -1519,7 +1519,7 @@ static void derivationStrictInternal(
     }
 
     /* Write the resulting term into the Nix store directory. */
-    auto drvPath = writeDerivation(*state.store, drv, state.repair);
+    auto drvPath = writeDerivation(*state.store, drv, state.repair, false, state.getRootProvenance());
     auto drvPathS = state.store->printStorePath(drvPath);
 
     printMsg(lvlChatty, "instantiated '%1%' -> '%2%'", drvName, drvPathS);
@@ -2320,7 +2320,15 @@ static void prim_toFile(EvalState & state, const PosIdx pos, Value * * args, Val
         })
         : ({
             StringSource s { contents };
-            state.store->addToStoreFromDump(s, name, FileSerialisationMethod::Flat, ContentAddressMethod::Raw::Text, HashAlgorithm::SHA256, refs, state.repair);
+            state.store->addToStoreFromDump(
+                s,
+                name,
+                FileSerialisationMethod::Flat,
+                ContentAddressMethod::Raw::Text,
+                HashAlgorithm::SHA256,
+                refs,
+                state.repair,
+                state.getRootProvenance());
         });
 
     /* Note: we don't need to add `context' to the context of the
@@ -2480,7 +2488,8 @@ static void addPath(
                 name,
                 method,
                 filter.get(),
-                state.repair);
+                state.repair,
+                state.getRootProvenance());
             if (expectedHash && expectedStorePath != dstPath)
                 state.error<EvalError>(
                     "store path mismatch in (possibly filtered) path added from '%s'",

src/libfetchers/fetch-to-store.cc

@@ -11,7 +11,8 @@ StorePath fetchToStore(
     std::string_view name,
     ContentAddressMethod method,
     PathFilter * filter,
-    RepairFlag repair)
+    RepairFlag repair,
+    std::shared_ptr<const Provenance> provenance)
 {
     // FIXME: add an optimisation for the case where the accessor is
     // a `PosixSourceAccessor` pointing to a store path.
@@ -42,7 +43,7 @@ StorePath fetchToStore(
         ? store.computeStorePath(
             name, path, method, HashAlgorithm::SHA256, {}, filter2).first
         : store.addToStore(
-            name, path, method, HashAlgorithm::SHA256, {}, filter2, repair);
+            name, path, method, HashAlgorithm::SHA256, {}, filter2, repair, provenance);
 
     if (cacheKey && mode == FetchMode::Copy)
         fetchers::getCache()->upsert(*cacheKey, store, {}, storePath);

src/libfetchers/fetch-to-store.hh

@@ -20,6 +20,7 @@ StorePath fetchToStore(
     std::string_view name = "source",
     ContentAddressMethod method = ContentAddressMethod::Raw::NixArchive,
     PathFilter * filter = nullptr,
-    RepairFlag repair = NoRepair);
+    RepairFlag repair = NoRepair,
+    std::shared_ptr<const Provenance> provenance = nullptr);
 
 }

src/libstore-test-support/tests/protocol.hh

@@ -32,11 +32,13 @@ public:
         CharacterizationTest::readTest(testStem, [&](const auto & encoded) {
             T got = ({
                 StringSource from { encoded };
+                std::set<std::string> features;
                 Proto::template Serialise<T>::read(
                     *LibStoreTest::store,
                     typename Proto::ReadConn {
                         .from = from,
                         .version = version,
+                        .features = features,
                     });
             });
 
@@ -52,11 +54,13 @@ public:
     {
         CharacterizationTest::writeTest(testStem, [&]() {
             StringSink to;
+            std::set<std::string> features;
             Proto::template Serialise<T>::write(
                 *LibStoreTest::store,
                 typename Proto::WriteConn {
                     .to = to,
                     .version = version,
+                    .features = features,
                 },
                 decoded);
             return std::move(to.s);

src/libstore-tests/data/nar-info/impure.json

@@ -6,6 +6,7 @@
   "downloadSize": 4029176,
   "narHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=",
   "narSize": 34878,
+  "provenance": null,
   "references": [
     "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar",
     "/nix/store/n5wkd9frr45pa74if5gpz9j7mifg27fh-foo"

src/libstore-tests/data/path-info/empty_impure.json

@@ -3,6 +3,7 @@
   "deriver": null,
   "narHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=",
   "narSize": 0,
+  "provenance": null,
   "references": [],
   "registrationTime": null,
   "signatures": [],

src/libstore-tests/data/path-info/impure.json

@@ -3,6 +3,7 @@
   "deriver": "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv",
   "narHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=",
   "narSize": 34878,
+  "provenance": null,
   "references": [
     "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar",
     "/nix/store/n5wkd9frr45pa74if5gpz9j7mifg27fh-foo"

src/libstore/binary-cache-store.cc

@@ -302,7 +302,8 @@ StorePath BinaryCacheStore::addToStoreFromDump(
     ContentAddressMethod hashMethod,
     HashAlgorithm hashAlgo,
     const StorePathSet & references,
-    RepairFlag repair)
+    RepairFlag repair,
+    std::shared_ptr<const Provenance> provenance)
 {
     std::optional<Hash> caHash;
     std::string nar;
@@ -364,6 +365,7 @@ StorePath BinaryCacheStore::addToStoreFromDump(
             nar.first,
         };
         info.narSize = nar.second;
+        info.provenance = provenance;
         return info;
     })->path;
 }
@@ -448,7 +450,8 @@ StorePath BinaryCacheStore::addToStore(
     HashAlgorithm hashAlgo,
     const StorePathSet & references,
     PathFilter & filter,
-    RepairFlag repair)
+    RepairFlag repair,
+    std::shared_ptr<const Provenance> provenance)
 {
     /* FIXME: Make BinaryCacheStore::addToStoreCommon support
        non-recursive+sha256 so we can just use the default
@@ -474,6 +477,7 @@ StorePath BinaryCacheStore::addToStore(
             nar.first,
         };
         info.narSize = nar.second;
+        info.provenance = provenance;
         return info;
     })->path;
 }

src/libstore/binary-cache-store.hh

@@ -68,6 +68,9 @@ protected:
 
 public:
 
+    bool uriIsUsefulProvenance() override
+    { return true; }
+
     virtual bool fileExists(const std::string & path) = 0;
 
     virtual void upsertFile(const std::string & path,
@@ -129,7 +132,8 @@ public:
         ContentAddressMethod hashMethod,
         HashAlgorithm hashAlgo,
         const StorePathSet & references,
-        RepairFlag repair) override;
+        RepairFlag repair,
+        std::shared_ptr<const Provenance> provenance) override;
 
     StorePath addToStore(
         std::string_view name,
@@ -138,7 +142,8 @@ public:
         HashAlgorithm hashAlgo,
         const StorePathSet & references,
         PathFilter & filter,
-        RepairFlag repair) override;
+        RepairFlag repair,
+        std::shared_ptr<const Provenance> provenance) override;
 
     void registerDrvOutput(const Realisation & info) override;
 

src/libstore/build/derivation-goal.cc

@@ -172,7 +172,7 @@ Goal::Co DerivationGoal::loadDerivation()
        things being garbage collected while we're busy. */
     worker.evalStore.addTempRoot(drvPath);
 
-    /* Get the derivation. It is probably in the eval store, but it might be inthe main store:
+    /* Get the derivation. It is probably in the eval store, but it might be in the main store:
 
          - Resolved derivation are resolved against main store realisations, and so must be stored there.
 
@@ -181,6 +181,7 @@ Goal::Co DerivationGoal::loadDerivation()
     for (auto * drvStore : { &worker.evalStore, &worker.store }) {
         if (drvStore->isValidPath(drvPath)) {
             drv = std::make_unique<Derivation>(drvStore->readDerivation(drvPath));
+            drvProvenance = drvStore->queryPathInfo(drvPath)->provenance;
             break;
         }
     }

src/libstore/build/derivation-goal.hh

@@ -67,6 +67,9 @@ struct DerivationGoal : public Goal
     /** The path of the derivation. */
     StorePath drvPath;
 
+    /** The provenance of the derivation, if any. */
+    std::shared_ptr<const Provenance> drvProvenance;
+
     /**
      * The goal for the corresponding resolved derivation
      */

src/libstore/daemon.cc

@@ -15,13 +15,16 @@
 #include "derivations.hh"
 #include "args.hh"
 #include "git.hh"
+#include "provenance.hh"
 
 #ifndef _WIN32 // TODO need graceful async exit support on Windows?
 # include "monitor-fd.hh"
 #endif
 
 #include <sstream>
 
+#include <nlohmann/json.hpp>
+
 namespace nix::daemon {
 
 Sink & operator << (Sink & sink, const Logger::Fields & fields)
@@ -398,6 +401,11 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
             bool repairBool;
             conn.from >> repairBool;
             auto repair = RepairFlag{repairBool};
+            std::shared_ptr<const Provenance> provenance;
+            if (conn.features.contains(WorkerProto::featureProvidence))
+                provenance =
+                    nlohmann::json::parse(readString(conn.from))
+                    .template get<std::shared_ptr<const Provenance>>();
 
             logger->startWork();
             auto pathInfo = [&]() {
@@ -423,7 +431,15 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
                     assert(false);
                 }
                 // TODO these two steps are essentially RemoteStore::addCAToStore. Move it up to Store.
-                auto path = store->addToStoreFromDump(source, name, dumpMethod, contentAddressMethod, hashAlgo, refs, repair);
+                auto path = store->addToStoreFromDump(
+                    source,
+                    name,
+                    dumpMethod,
+                    contentAddressMethod,
+                    hashAlgo,
+                    refs,
+                    repair,
+                    provenance);
                 return store->queryPathInfo(path);
             }();
             logger->stopWork();

src/libstore/derivations.cc

@@ -135,8 +135,12 @@ bool BasicDerivation::isBuiltin() const
 }
 
 
-StorePath writeDerivation(Store & store,
-    const Derivation & drv, RepairFlag repair, bool readOnly)
+StorePath writeDerivation(
+    Store & store,
+    const Derivation & drv,
+    RepairFlag repair,
+    bool readOnly,
+    std::shared_ptr<const Provenance> provenance)
 {
     auto references = drv.inputSrcs;
     for (auto & i : drv.inputDrvs.map)
@@ -153,7 +157,15 @@ StorePath writeDerivation(Store & store,
         })
         : ({
             StringSource s { contents };
-            store.addToStoreFromDump(s, suffix, FileSerialisationMethod::Flat, ContentAddressMethod::Raw::Text, HashAlgorithm::SHA256, references, repair);
+            store.addToStoreFromDump(
+                s,
+                suffix,
+                FileSerialisationMethod::Flat,
+                ContentAddressMethod::Raw::Text,
+                HashAlgorithm::SHA256,
+                references,
+                repair,
+                provenance);
         });
 }
 

src/libstore/derivations.hh

@@ -16,6 +16,7 @@
 namespace nix {
 
 struct StoreDirConfig;
+struct Provenance;
 
 /* Abstract syntax of derivations. */
 
@@ -395,7 +396,8 @@ class Store;
 StorePath writeDerivation(Store & store,
     const Derivation & drv,
     RepairFlag repair = NoRepair,
-    bool readOnly = false);
+    bool readOnly = false,
+    std::shared_ptr<const Provenance> provenance = nullptr);
 
 /**
  * Read a derivation from a file.