Basic store path provenance tracking

src/libcmd/installable-flake.cc

@@ -17,6 +17,7 @@
 #include "url.hh"
 #include "registry.hh"
 #include "build-result.hh"
+#include "provenance.hh"
 
 #include <regex>
 #include <queue>
@@ -81,6 +82,17 @@ DerivedPathsWithInfo InstallableFlake::toDerivedPaths()
 
     auto attrPath = attr->getAttrPathStr();
 
+    auto lockedRef = getLockedFlake()->flake.lockedRef;
+
+    state->setRootProvenance(
+        {
+            {
+                Provenance::ProvFlake {
+                    .flakeRef = fmt("%s#%s", lockedRef, attrPath)
+                }
+            }
+        });
+
     if (!attr->isDerivation()) {
 
         // FIXME: use eval cache?
@@ -147,7 +159,7 @@ DerivedPathsWithInfo InstallableFlake::toDerivedPaths()
             },
             ExtraPathInfoFlake::Flake {
                 .originalRef = flakeRef,
-                .lockedRef = getLockedFlake()->flake.lockedRef,
+                .lockedRef = lockedRef,
             }),
     }};
 }

src/libexpr/eval.cc

@@ -20,6 +20,7 @@
 #include "fetch-to-store.hh"
 #include "tarball.hh"
 #include "parser-tab.hh"
+#include "provenance.hh"
 
 #include <algorithm>
 #include <iostream>
@@ -2371,7 +2372,8 @@ StorePath EvalState::copyPathToStore(NixStringContext & context, const SourcePat
                 path.baseName(),
                 ContentAddressMethod::Raw::NixArchive,
                 nullptr,
-                repair);
+                repair,
+                getRootProvenance());
             allowPath(dstPath);
             srcToStore.lock()->try_emplace(path, dstPath);
             printMsg(lvlChatty, "copied source '%1%' -> '%2%'", path, store->printStorePath(dstPath));
@@ -3170,4 +3172,17 @@ std::ostream & operator << (std::ostream & str, const ExternalValueBase & v) {
 }
 
 
+std::optional<std::reference_wrapper<Provenance>> EvalState::getRootProvenance()
+{
+    return rootProvenance
+        ? std::optional<std::reference_wrapper<Provenance>>(*rootProvenance)
+        : std::nullopt;
+}
+
+
+void EvalState::setRootProvenance(std::optional<Provenance> provenance)
+{
+    rootProvenance = provenance ? std::make_shared<Provenance>(std::move(*provenance)) : nullptr;
+}
+
 }

src/libexpr/eval.hh

@@ -37,6 +37,7 @@ class StorePath;
 struct SingleDerivedPath;
 enum RepairFlag : bool;
 struct MemorySourceAccessor;
+struct Provenance;
 namespace eval_cache {
     class EvalCache;
 }
@@ -863,6 +864,15 @@ private:
 
     friend struct Value;
     friend class ListBuilder;
+
+    // FIXME: how to handle this in the multi-threaded evaluator?
+    std::shared_ptr<Provenance> rootProvenance;
+
+public:
+
+    std::optional<std::reference_wrapper<Provenance>> getRootProvenance();
+
+    void setRootProvenance(std::optional<Provenance> provenance);
 };
 
 struct DebugTraceStacker {

src/libexpr/primops.cc

@@ -1509,7 +1509,7 @@ static void derivationStrictInternal(
     }
 
     /* Write the resulting term into the Nix store directory. */
-    auto drvPath = writeDerivation(*state.store, drv, state.repair);
+    auto drvPath = writeDerivation(*state.store, drv, state.repair, false, state.getRootProvenance());
     auto drvPathS = state.store->printStorePath(drvPath);
 
     printMsg(lvlChatty, "instantiated '%1%' -> '%2%'", drvName, drvPathS);
@@ -2310,7 +2310,15 @@ static void prim_toFile(EvalState & state, const PosIdx pos, Value * * args, Val
         })
         : ({
             StringSource s { contents };
-            state.store->addToStoreFromDump(s, name, FileSerialisationMethod::Flat, ContentAddressMethod::Raw::Text, HashAlgorithm::SHA256, refs, state.repair);
+            state.store->addToStoreFromDump(
+                s,
+                name,
+                FileSerialisationMethod::Flat,
+                ContentAddressMethod::Raw::Text,
+                HashAlgorithm::SHA256,
+                refs,
+                state.repair,
+                state.getRootProvenance());
         });
 
     /* Note: we don't need to add `context' to the context of the
@@ -2470,7 +2478,8 @@ static void addPath(
                 name,
                 method,
                 filter.get(),
-                state.repair);
+                state.repair,
+                state.getRootProvenance());
             if (expectedHash && expectedStorePath != dstPath)
                 state.error<EvalError>(
                     "store path mismatch in (possibly filtered) path added from '%s'",

src/libfetchers/fetch-to-store.cc

@@ -11,7 +11,8 @@ StorePath fetchToStore(
     std::string_view name,
     ContentAddressMethod method,
     PathFilter * filter,
-    RepairFlag repair)
+    RepairFlag repair,
+    std::optional<std::reference_wrapper<Provenance>> provenance)
 {
     // FIXME: add an optimisation for the case where the accessor is
     // a `PosixSourceAccessor` pointing to a store path.
@@ -42,7 +43,7 @@ StorePath fetchToStore(
         ? store.computeStorePath(
             name, path, method, HashAlgorithm::SHA256, {}, filter2).first
         : store.addToStore(
-            name, path, method, HashAlgorithm::SHA256, {}, filter2, repair);
+            name, path, method, HashAlgorithm::SHA256, {}, filter2, repair, provenance);
 
     if (cacheKey && mode == FetchMode::Copy)
         fetchers::getCache()->upsert(*cacheKey, store, {}, storePath);

src/libfetchers/fetch-to-store.hh

@@ -20,6 +20,7 @@ StorePath fetchToStore(
     std::string_view name = "source",
     ContentAddressMethod method = ContentAddressMethod::Raw::NixArchive,
     PathFilter * filter = nullptr,
-    RepairFlag repair = NoRepair);
+    RepairFlag repair = NoRepair,
+    std::optional<std::reference_wrapper<Provenance>> provenance = std::nullopt);
 
 }

src/libstore/binary-cache-store.cc

@@ -302,8 +302,12 @@ StorePath BinaryCacheStore::addToStoreFromDump(
     ContentAddressMethod hashMethod,
     HashAlgorithm hashAlgo,
     const StorePathSet & references,
-    RepairFlag repair)
+    RepairFlag repair,
+    std::optional<std::reference_wrapper<Provenance>> provenance)
 {
+    if (provenance)
+        throw UnimplementedError("BinaryCacheStore::addToStoreFromDump() with provenance");
+
     std::optional<Hash> caHash;
     std::string nar;
 
@@ -448,8 +452,12 @@ StorePath BinaryCacheStore::addToStore(
     HashAlgorithm hashAlgo,
     const StorePathSet & references,
     PathFilter & filter,
-    RepairFlag repair)
+    RepairFlag repair,
+    std::optional<std::reference_wrapper<Provenance>> provenance)
 {
+    if (provenance)
+        throw UnimplementedError("BinaryCacheStore::addToStore() with provenance");
+
     /* FIXME: Make BinaryCacheStore::addToStoreCommon support
        non-recursive+sha256 so we can just use the default
        implementation of this method in terms of addToStoreFromDump. */

src/libstore/binary-cache-store.hh

@@ -129,7 +129,8 @@ public:
         ContentAddressMethod hashMethod,
         HashAlgorithm hashAlgo,
         const StorePathSet & references,
-        RepairFlag repair) override;
+        RepairFlag repair,
+        std::optional<std::reference_wrapper<Provenance>> provenance = std::nullopt) override;
 
     StorePath addToStore(
         std::string_view name,
@@ -138,7 +139,8 @@ public:
         HashAlgorithm hashAlgo,
         const StorePathSet & references,
         PathFilter & filter,
-        RepairFlag repair) override;
+        RepairFlag repair,
+        std::optional<std::reference_wrapper<Provenance>> provenance) override;
 
     void registerDrvOutput(const Realisation & info) override;
 

src/libstore/build/derivation-goal.cc

@@ -172,7 +172,7 @@ Goal::Co DerivationGoal::loadDerivation()
        things being garbage collected while we're busy. */
     worker.evalStore.addTempRoot(drvPath);
 
-    /* Get the derivation. It is probably in the eval store, but it might be inthe main store:
+    /* Get the derivation. It is probably in the eval store, but it might be in the main store:
 
          - Resolved derivation are resolved against main store realisations, and so must be stored there.
 
@@ -181,6 +181,7 @@ Goal::Co DerivationGoal::loadDerivation()
     for (auto * drvStore : { &worker.evalStore, &worker.store }) {
         if (drvStore->isValidPath(drvPath)) {
             drv = std::make_unique<Derivation>(drvStore->readDerivation(drvPath));
+            drvProvenance = drvStore->queryPathInfo(drvPath)->provenance;
             break;
         }
     }

src/libstore/build/derivation-goal.hh

@@ -67,6 +67,9 @@ struct DerivationGoal : public Goal
     /** The path of the derivation. */
     StorePath drvPath;
 
+    /** The provenance of the derivation, if any. */
+    std::shared_ptr<const Provenance> drvProvenance;
+
     /**
      * The goal for the corresponding resolved derivation
      */

src/libstore/derivations.cc

@@ -135,8 +135,12 @@ bool BasicDerivation::isBuiltin() const
 }
 
 
-StorePath writeDerivation(Store & store,
-    const Derivation & drv, RepairFlag repair, bool readOnly)
+StorePath writeDerivation(
+    Store & store,
+    const Derivation & drv,
+    RepairFlag repair,
+    bool readOnly,
+    std::optional<std::reference_wrapper<Provenance>> provenance)
 {
     auto references = drv.inputSrcs;
     for (auto & i : drv.inputDrvs.map)
@@ -153,7 +157,15 @@ StorePath writeDerivation(Store & store,
         })
         : ({
             StringSource s { contents };
-            store.addToStoreFromDump(s, suffix, FileSerialisationMethod::Flat, ContentAddressMethod::Raw::Text, HashAlgorithm::SHA256, references, repair);
+            store.addToStoreFromDump(
+                s,
+                suffix,
+                FileSerialisationMethod::Flat,
+                ContentAddressMethod::Raw::Text,
+                HashAlgorithm::SHA256,
+                references,
+                repair,
+                provenance);
         });
 }
 

src/libstore/derivations.hh

@@ -16,6 +16,7 @@
 namespace nix {
 
 struct StoreDirConfig;
+struct Provenance;
 
 /* Abstract syntax of derivations. */
 
@@ -395,7 +396,8 @@ class Store;
 StorePath writeDerivation(Store & store,
     const Derivation & drv,
     RepairFlag repair = NoRepair,
-    bool readOnly = false);
+    bool readOnly = false,
+    std::optional<std::reference_wrapper<Provenance>> provenance = std::nullopt);
 
 /**
  * Read a derivation from a file.

src/libstore/dummy-store.cc

@@ -72,7 +72,8 @@ struct DummyStore : public virtual DummyStoreConfig, public virtual Store
         ContentAddressMethod hashMethod = FileIngestionMethod::NixArchive,
         HashAlgorithm hashAlgo = HashAlgorithm::SHA256,
         const StorePathSet & references = StorePathSet(),
-        RepairFlag repair = NoRepair) override
+        RepairFlag repair = NoRepair,
+        std::optional<std::reference_wrapper<Provenance>> provenance = std::nullopt) override
     { unsupported("addToStore"); }
 
     void narFromPath(const StorePath & path, Sink & sink) override

src/libstore/legacy-ssh-store.hh

@@ -75,7 +75,8 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor
         HashAlgorithm hashAlgo,
         const StorePathSet & references,
         PathFilter & filter,
-        RepairFlag repair) override
+        RepairFlag repair,
+        std::optional<std::reference_wrapper<Provenance>> provenance) override
     { unsupported("addToStore"); }
 
     virtual StorePath addToStoreFromDump(
@@ -85,7 +86,8 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor
         ContentAddressMethod hashMethod = FileIngestionMethod::NixArchive,
         HashAlgorithm hashAlgo = HashAlgorithm::SHA256,
         const StorePathSet & references = StorePathSet(),
-        RepairFlag repair = NoRepair) override
+        RepairFlag repair = NoRepair,
+        std::optional<std::reference_wrapper<Provenance>> provenance = std::nullopt) override
     { unsupported("addToStore"); }
 
 public:

src/libstore/local-store.cc

@@ -17,6 +17,7 @@
 #include "posix-source-accessor.hh"
 #include "keys.hh"
 #include "users.hh"
+#include "provenance.hh"
 
 #include <iostream>
 #include <algorithm>
@@ -335,6 +336,12 @@ LocalStore::LocalStore(
             txn.commit();
         }
 
+        if (curSchema < 11) {
+            SQLiteTxn txn(state->db);
+            state->db.exec("alter table ValidPaths add column provenance text");
+            txn.commit();
+        }
+
         writeFile(schemaPath, fmt("%1%", nixSchemaVersion), 0666, true);
 
         lockFile(globalLock.get(), ltRead, true);
@@ -352,13 +359,13 @@ LocalStore::LocalStore(
 
     /* Prepare SQL statements. */
     state->stmts->RegisterValidPath.create(state->db,
-        "insert into ValidPaths (path, hash, registrationTime, deriver, narSize, ultimate, sigs, ca) values (?, ?, ?, ?, ?, ?, ?, ?);");
+        "insert into ValidPaths (path, hash, registrationTime, deriver, narSize, ultimate, sigs, ca, provenance) values (?, ?, ?, ?, ?, ?, ?, ?, ?);");
     state->stmts->UpdatePathInfo.create(state->db,
         "update ValidPaths set narSize = ?, hash = ?, ultimate = ?, sigs = ?, ca = ? where path = ?;");
     state->stmts->AddReference.create(state->db,
         "insert or replace into Refs (referrer, reference) values (?, ?);");
     state->stmts->QueryPathInfo.create(state->db,
-        "select id, hash, registrationTime, deriver, narSize, ultimate, sigs, ca from ValidPaths where path = ?;");
+        "select id, hash, registrationTime, deriver, narSize, ultimate, sigs, ca, provenance from ValidPaths where path = ?;");
     state->stmts->QueryReferences.create(state->db,
         "select path from Refs join ValidPaths on reference = id where referrer = ?;");
     state->stmts->QueryReferrers.create(state->db,
@@ -680,6 +687,10 @@ uint64_t LocalStore::addValidPath(State & state,
         (info.ultimate ? 1 : 0, info.ultimate)
         (concatStringsSep(" ", info.sigs), !info.sigs.empty())
         (renderContentAddress(info.ca), (bool) info.ca)
+        (info.provenance
+            ? nlohmann::json(*info.provenance).dump()
+            : "",
+            (bool) info.provenance)
         .exec();
     uint64_t id = state.db.getLastInsertedRowId();
 
@@ -771,6 +782,10 @@ std::shared_ptr<const ValidPathInfo> LocalStore::queryPathInfoInternal(State & s
     while (useQueryReferences.next())
         info->references.insert(parseStorePath(useQueryReferences.getStr(0)));
 
+    auto prov = (const char *) sqlite3_column_text(state.stmts->QueryPathInfo, 8);
+    if (prov)
+        info->provenance = std::make_shared<const Provenance>(nlohmann::json::parse(prov).template get<Provenance>());
+
     return info;
 }
 
@@ -1163,7 +1178,8 @@ StorePath LocalStore::addToStoreFromDump(
     ContentAddressMethod hashMethod,
     HashAlgorithm hashAlgo,
     const StorePathSet & references,
-    RepairFlag repair)
+    RepairFlag repair,
+    std::optional<std::reference_wrapper<Provenance>> provenance)
 {
     /* For computing the store path. */
     auto hashSink = std::make_unique<HashSink>(hashAlgo);
@@ -1263,7 +1279,6 @@ StorePath LocalStore::addToStoreFromDump(
         PathLocks outputLock({realPath});
 
         if (repair || !isValidPath(dstPath)) {
-
             deletePath(realPath);
 
             autoGC();
@@ -1312,6 +1327,7 @@ StorePath LocalStore::addToStoreFromDump(
                 narHash.first
             };
             info.narSize = narHash.second;
+            info.provenance = provenance ? std::make_shared<const Provenance>(*provenance) : nullptr;
             registerValidPath(info);
         }