From 0956b7ea00fef3f1e11ce6d2a08ffc39771afe03 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 9 Oct 2024 19:58:54 +0200 Subject: [PATCH] Basic store path provenance tracking Nix historically has been bad at being able to answer the question "where did this store path come from", i.e. to provide traceability from a store path back to the Nix expression from which is was built. Nix tracks the "deriver" of a store path (the .drv file that built it) but that's pretty useless in practice, since it doesn't link back to the Nix expressions. So this PR adds a "provenance" field (a JSON object) to the ValidPaths table and to .narinfo files that describes where the store path came from and how it can be reproduced. There are currently 3 types of provenance: * "copied": Records that the store path was copied or substituted from another store (typically a binary cache). Its "from" field is the URL of the origin store. Its "provenance" field propagates the provenance of the store path on the origin store. * "derivation": Records that the store path is the output of a .drv file. This is equivalent for the "deriver" field, but it has a nested "provenance" field that records how the .drv file was created. * "flake": Records that the store path was created during the evaluation of a flake output. Example: $ nix path-info --json /nix/store/xcqzb13bd60zmfw6wv0z4242b9mfw042-patchelf-0.18.0 { "/nix/store/xcqzb13bd60zmfw6wv0z4242b9mfw042-patchelf-0.18.0": { "provenance": { "from": "https://cache.example.org", "provenance": { "drv": "rlabxgjx88bavjkc694v1bqbwslwivxs-patchelf-0.18.0.drv", "output": "out", "provenance": { "flake": { "lastModified": 1729856604, "narHash": "sha256-obmE2ZI9sTPXczzGMerwQX4SALF+ABL9J0oB371yvZE=", "owner": "NixOS", "repo": "patchelf", "rev": "689f19e499caee8e5c3d387008bbd4ed7f8dc3a9", "type": "github", }, "output": "packages.x86_64-linux.default", "type": "flake" }, "type": "derivation" }, "type": "copied" }, ... } } This specifies that the store path was copied from the binary cache https://cache.example.org and it's the "out" output of a store derivation that was produced by evaluating the flake ouput `packages.x86_64-linux.default` of some revision of the patchelf GitHub repository. --- src/libcmd/installable-flake.cc | 15 ++- src/libexpr/eval.cc | 17 +++- src/libexpr/eval.hh | 10 ++ src/libexpr/primops.cc | 15 ++- src/libfetchers/fetch-to-store.cc | 5 +- src/libfetchers/fetch-to-store.hh | 3 +- src/libstore-tests/data/nar-info/impure.json | 1 + .../data/path-info/empty_impure.json | 1 + src/libstore-tests/data/path-info/impure.json | 1 + src/libstore/binary-cache-store.cc | 12 ++- src/libstore/binary-cache-store.hh | 9 +- src/libstore/build/derivation-goal.cc | 3 +- src/libstore/build/derivation-goal.hh | 3 + src/libstore/derivations.cc | 18 +++- src/libstore/derivations.hh | 4 +- src/libstore/dummy-store.cc | 3 +- src/libstore/legacy-ssh-store.hh | 9 +- src/libstore/local-store.cc | 22 ++++- src/libstore/local-store.hh | 3 +- src/libstore/meson.build | 2 + src/libstore/nar-info.cc | 7 ++ src/libstore/path-info.cc | 6 ++ src/libstore/path-info.hh | 7 ++ src/libstore/provenance.cc | 88 +++++++++++++++++ src/libstore/provenance.hh | 95 +++++++++++++++++++ src/libstore/remote-store.cc | 6 +- src/libstore/remote-store.hh | 3 +- src/libstore/ssh-store.cc | 3 + src/libstore/store-api.cc | 44 ++++++--- src/libstore/store-api.hh | 15 ++- .../unix/build/local-derivation-goal.cc | 16 +++- 31 files changed, 402 insertions(+), 44 deletions(-) create mode 100644 src/libstore/provenance.cc create mode 100644 src/libstore/provenance.hh diff --git a/src/libcmd/installable-flake.cc b/src/libcmd/installable-flake.cc index 6c9ee674808..b7f69ce6744 100644 --- a/src/libcmd/installable-flake.cc +++ b/src/libcmd/installable-flake.cc @@ -17,6 +17,7 @@ #include "url.hh" #include "registry.hh" #include "build-result.hh" +#include "provenance.hh" #include #include @@ -81,6 +82,18 @@ DerivedPathsWithInfo InstallableFlake::toDerivedPaths() auto attrPath = attr->getAttrPathStr(); + auto lockedRef = getLockedFlake()->flake.lockedRef; + + state->setRootProvenance( + { + { + Provenance::ProvFlake { + .flake = std::make_shared(fetchers::attrsToJSON(lockedRef.input.attrs)), + .flakeOutput = attrPath, + } + } + }); + if (!attr->isDerivation()) { // FIXME: use eval cache? @@ -147,7 +160,7 @@ DerivedPathsWithInfo InstallableFlake::toDerivedPaths() }, ExtraPathInfoFlake::Flake { .originalRef = flakeRef, - .lockedRef = getLockedFlake()->flake.lockedRef, + .lockedRef = lockedRef, }), }}; } diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index f17753415fc..6311da8a81b 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -20,6 +20,7 @@ #include "fetch-to-store.hh" #include "tarball.hh" #include "parser-tab.hh" +#include "provenance.hh" #include #include @@ -2364,7 +2365,8 @@ StorePath EvalState::copyPathToStore(NixStringContext & context, const SourcePat path.baseName(), ContentAddressMethod::Raw::NixArchive, nullptr, - repair); + repair, + getRootProvenance()); allowPath(dstPath); srcToStore.lock()->try_emplace(path, dstPath); printMsg(lvlChatty, "copied source '%1%' -> '%2%'", path, store->printStorePath(dstPath)); @@ -3163,4 +3165,17 @@ std::ostream & operator << (std::ostream & str, const ExternalValueBase & v) { } +std::optional> EvalState::getRootProvenance() +{ + return rootProvenance + ? std::optional>(*rootProvenance) + : std::nullopt; +} + + +void EvalState::setRootProvenance(std::optional provenance) +{ + rootProvenance = provenance ? std::make_shared(std::move(*provenance)) : nullptr; +} + } diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index f7ed6be83af..ac9369b00c9 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -37,6 +37,7 @@ class StorePath; struct SingleDerivedPath; enum RepairFlag : bool; struct MemorySourceAccessor; +struct Provenance; namespace eval_cache { class EvalCache; } @@ -863,6 +864,15 @@ private: friend struct Value; friend class ListBuilder; + + // FIXME: how to handle this in the multi-threaded evaluator? + std::shared_ptr rootProvenance; + +public: + + std::optional> getRootProvenance(); + + void setRootProvenance(std::optional provenance); }; struct DebugTraceStacker { diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 203d109324f..cad502826f9 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1519,7 +1519,7 @@ static void derivationStrictInternal( } /* Write the resulting term into the Nix store directory. */ - auto drvPath = writeDerivation(*state.store, drv, state.repair); + auto drvPath = writeDerivation(*state.store, drv, state.repair, false, state.getRootProvenance()); auto drvPathS = state.store->printStorePath(drvPath); printMsg(lvlChatty, "instantiated '%1%' -> '%2%'", drvName, drvPathS); @@ -2320,7 +2320,15 @@ static void prim_toFile(EvalState & state, const PosIdx pos, Value * * args, Val }) : ({ StringSource s { contents }; - state.store->addToStoreFromDump(s, name, FileSerialisationMethod::Flat, ContentAddressMethod::Raw::Text, HashAlgorithm::SHA256, refs, state.repair); + state.store->addToStoreFromDump( + s, + name, + FileSerialisationMethod::Flat, + ContentAddressMethod::Raw::Text, + HashAlgorithm::SHA256, + refs, + state.repair, + state.getRootProvenance()); }); /* Note: we don't need to add `context' to the context of the @@ -2480,7 +2488,8 @@ static void addPath( name, method, filter.get(), - state.repair); + state.repair, + state.getRootProvenance()); if (expectedHash && expectedStorePath != dstPath) state.error( "store path mismatch in (possibly filtered) path added from '%s'", diff --git a/src/libfetchers/fetch-to-store.cc b/src/libfetchers/fetch-to-store.cc index 65aa72a6c36..87ac5e1adfc 100644 --- a/src/libfetchers/fetch-to-store.cc +++ b/src/libfetchers/fetch-to-store.cc @@ -11,7 +11,8 @@ StorePath fetchToStore( std::string_view name, ContentAddressMethod method, PathFilter * filter, - RepairFlag repair) + RepairFlag repair, + std::optional> provenance) { // FIXME: add an optimisation for the case where the accessor is // a `PosixSourceAccessor` pointing to a store path. @@ -42,7 +43,7 @@ StorePath fetchToStore( ? store.computeStorePath( name, path, method, HashAlgorithm::SHA256, {}, filter2).first : store.addToStore( - name, path, method, HashAlgorithm::SHA256, {}, filter2, repair); + name, path, method, HashAlgorithm::SHA256, {}, filter2, repair, provenance); if (cacheKey && mode == FetchMode::Copy) fetchers::getCache()->upsert(*cacheKey, store, {}, storePath); diff --git a/src/libfetchers/fetch-to-store.hh b/src/libfetchers/fetch-to-store.hh index c762629f3cb..fe5dd5c46d4 100644 --- a/src/libfetchers/fetch-to-store.hh +++ b/src/libfetchers/fetch-to-store.hh @@ -20,6 +20,7 @@ StorePath fetchToStore( std::string_view name = "source", ContentAddressMethod method = ContentAddressMethod::Raw::NixArchive, PathFilter * filter = nullptr, - RepairFlag repair = NoRepair); + RepairFlag repair = NoRepair, + std::optional> provenance = std::nullopt); } diff --git a/src/libstore-tests/data/nar-info/impure.json b/src/libstore-tests/data/nar-info/impure.json index bb9791a6ace..63f2dcfc4a4 100644 --- a/src/libstore-tests/data/nar-info/impure.json +++ b/src/libstore-tests/data/nar-info/impure.json @@ -6,6 +6,7 @@ "downloadSize": 4029176, "narHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=", "narSize": 34878, + "provenance": null, "references": [ "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", "/nix/store/n5wkd9frr45pa74if5gpz9j7mifg27fh-foo" diff --git a/src/libstore-tests/data/path-info/empty_impure.json b/src/libstore-tests/data/path-info/empty_impure.json index be982dcef85..fbaafd25d3b 100644 --- a/src/libstore-tests/data/path-info/empty_impure.json +++ b/src/libstore-tests/data/path-info/empty_impure.json @@ -3,6 +3,7 @@ "deriver": null, "narHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=", "narSize": 0, + "provenance": null, "references": [], "registrationTime": null, "signatures": [], diff --git a/src/libstore-tests/data/path-info/impure.json b/src/libstore-tests/data/path-info/impure.json index 0c452cc4930..ffc22406d43 100644 --- a/src/libstore-tests/data/path-info/impure.json +++ b/src/libstore-tests/data/path-info/impure.json @@ -3,6 +3,7 @@ "deriver": "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", "narHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=", "narSize": 34878, + "provenance": null, "references": [ "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", "/nix/store/n5wkd9frr45pa74if5gpz9j7mifg27fh-foo" diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index e8c8892b337..2c725eff4e8 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -302,8 +302,12 @@ StorePath BinaryCacheStore::addToStoreFromDump( ContentAddressMethod hashMethod, HashAlgorithm hashAlgo, const StorePathSet & references, - RepairFlag repair) + RepairFlag repair, + std::optional> provenance) { + if (provenance) + throw UnimplementedError("BinaryCacheStore::addToStoreFromDump() with provenance"); + std::optional caHash; std::string nar; @@ -448,8 +452,12 @@ StorePath BinaryCacheStore::addToStore( HashAlgorithm hashAlgo, const StorePathSet & references, PathFilter & filter, - RepairFlag repair) + RepairFlag repair, + std::optional> provenance) { + if (provenance) + throw UnimplementedError("BinaryCacheStore::addToStore() with provenance"); + /* FIXME: Make BinaryCacheStore::addToStoreCommon support non-recursive+sha256 so we can just use the default implementation of this method in terms of addToStoreFromDump. */ diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index 695bc925277..80274a62ad1 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -68,6 +68,9 @@ protected: public: + bool uriIsUsefulProvenance() override + { return true; } + virtual bool fileExists(const std::string & path) = 0; virtual void upsertFile(const std::string & path, @@ -129,7 +132,8 @@ public: ContentAddressMethod hashMethod, HashAlgorithm hashAlgo, const StorePathSet & references, - RepairFlag repair) override; + RepairFlag repair, + std::optional> provenance = std::nullopt) override; StorePath addToStore( std::string_view name, @@ -138,7 +142,8 @@ public: HashAlgorithm hashAlgo, const StorePathSet & references, PathFilter & filter, - RepairFlag repair) override; + RepairFlag repair, + std::optional> provenance) override; void registerDrvOutput(const Realisation & info) override; diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 34ed16a3819..8d9ba11aaa7 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -172,7 +172,7 @@ Goal::Co DerivationGoal::loadDerivation() things being garbage collected while we're busy. */ worker.evalStore.addTempRoot(drvPath); - /* Get the derivation. It is probably in the eval store, but it might be inthe main store: + /* Get the derivation. It is probably in the eval store, but it might be in the main store: - Resolved derivation are resolved against main store realisations, and so must be stored there. @@ -181,6 +181,7 @@ Goal::Co DerivationGoal::loadDerivation() for (auto * drvStore : { &worker.evalStore, &worker.store }) { if (drvStore->isValidPath(drvPath)) { drv = std::make_unique(drvStore->readDerivation(drvPath)); + drvProvenance = drvStore->queryPathInfo(drvPath)->provenance; break; } } diff --git a/src/libstore/build/derivation-goal.hh b/src/libstore/build/derivation-goal.hh index ad3d9ca2acf..d93fd004c94 100644 --- a/src/libstore/build/derivation-goal.hh +++ b/src/libstore/build/derivation-goal.hh @@ -67,6 +67,9 @@ struct DerivationGoal : public Goal /** The path of the derivation. */ StorePath drvPath; + /** The provenance of the derivation, if any. */ + std::shared_ptr drvProvenance; + /** * The goal for the corresponding resolved derivation */ diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 9b6f6785291..1cd4d15a518 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -135,8 +135,12 @@ bool BasicDerivation::isBuiltin() const } -StorePath writeDerivation(Store & store, - const Derivation & drv, RepairFlag repair, bool readOnly) +StorePath writeDerivation( + Store & store, + const Derivation & drv, + RepairFlag repair, + bool readOnly, + std::optional> provenance) { auto references = drv.inputSrcs; for (auto & i : drv.inputDrvs.map) @@ -153,7 +157,15 @@ StorePath writeDerivation(Store & store, }) : ({ StringSource s { contents }; - store.addToStoreFromDump(s, suffix, FileSerialisationMethod::Flat, ContentAddressMethod::Raw::Text, HashAlgorithm::SHA256, references, repair); + store.addToStoreFromDump( + s, + suffix, + FileSerialisationMethod::Flat, + ContentAddressMethod::Raw::Text, + HashAlgorithm::SHA256, + references, + repair, + provenance); }); } diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index 58e5328a5eb..11eba969fec 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -16,6 +16,7 @@ namespace nix { struct StoreDirConfig; +struct Provenance; /* Abstract syntax of derivations. */ @@ -395,7 +396,8 @@ class Store; StorePath writeDerivation(Store & store, const Derivation & drv, RepairFlag repair = NoRepair, - bool readOnly = false); + bool readOnly = false, + std::optional> provenance = std::nullopt); /** * Read a derivation from a file. diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index c1e871e9384..725f38004e0 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -72,7 +72,8 @@ struct DummyStore : public virtual DummyStoreConfig, public virtual Store ContentAddressMethod hashMethod = FileIngestionMethod::NixArchive, HashAlgorithm hashAlgo = HashAlgorithm::SHA256, const StorePathSet & references = StorePathSet(), - RepairFlag repair = NoRepair) override + RepairFlag repair = NoRepair, + std::optional> provenance = std::nullopt) override { unsupported("addToStore"); } void narFromPath(const StorePath & path, Sink & sink) override diff --git a/src/libstore/legacy-ssh-store.hh b/src/libstore/legacy-ssh-store.hh index b541455b4e5..0ea630296cf 100644 --- a/src/libstore/legacy-ssh-store.hh +++ b/src/libstore/legacy-ssh-store.hh @@ -57,6 +57,9 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor std::string getUri() override; + bool uriIsUsefulProvenance() override + { return true; } + void queryPathInfoUncached(const StorePath & path, Callback> callback) noexcept override; @@ -75,7 +78,8 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor HashAlgorithm hashAlgo, const StorePathSet & references, PathFilter & filter, - RepairFlag repair) override + RepairFlag repair, + std::optional> provenance) override { unsupported("addToStore"); } virtual StorePath addToStoreFromDump( @@ -85,7 +89,8 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor ContentAddressMethod hashMethod = FileIngestionMethod::NixArchive, HashAlgorithm hashAlgo = HashAlgorithm::SHA256, const StorePathSet & references = StorePathSet(), - RepairFlag repair = NoRepair) override + RepairFlag repair = NoRepair, + std::optional> provenance = std::nullopt) override { unsupported("addToStore"); } public: diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 6f9d792ad09..0f24eade582 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -17,6 +17,7 @@ #include "posix-source-accessor.hh" #include "keys.hh" #include "users.hh" +#include "provenance.hh" #include #include @@ -305,13 +306,13 @@ LocalStore::LocalStore( /* Prepare SQL statements. */ state->stmts->RegisterValidPath.create(state->db, - "insert into ValidPaths (path, hash, registrationTime, deriver, narSize, ultimate, sigs, ca) values (?, ?, ?, ?, ?, ?, ?, ?);"); + "insert into ValidPaths (path, hash, registrationTime, deriver, narSize, ultimate, sigs, ca, provenance) values (?, ?, ?, ?, ?, ?, ?, ?, ?);"); state->stmts->UpdatePathInfo.create(state->db, "update ValidPaths set narSize = ?, hash = ?, ultimate = ?, sigs = ?, ca = ? where path = ?;"); state->stmts->AddReference.create(state->db, "insert or replace into Refs (referrer, reference) values (?, ?);"); state->stmts->QueryPathInfo.create(state->db, - "select id, hash, registrationTime, deriver, narSize, ultimate, sigs, ca from ValidPaths where path = ?;"); + "select id, hash, registrationTime, deriver, narSize, ultimate, sigs, ca, provenance from ValidPaths where path = ?;"); state->stmts->QueryReferences.create(state->db, "select path from Refs join ValidPaths on reference = id where referrer = ?;"); state->stmts->QueryReferrers.create(state->db, @@ -562,6 +563,10 @@ void LocalStore::upgradeDBSchema(State & state) "20220326-ca-derivations", #include "ca-specific-schema.sql.gen.hh" ); + + doUpgrade( + "202410124-provenance", + "alter table ValidPaths add column provenance text"); } @@ -679,6 +684,10 @@ uint64_t LocalStore::addValidPath(State & state, (info.ultimate ? 1 : 0, info.ultimate) (concatStringsSep(" ", info.sigs), !info.sigs.empty()) (renderContentAddress(info.ca), (bool) info.ca) + (info.provenance + ? nlohmann::json(*info.provenance).dump() + : "", + (bool) info.provenance) .exec(); uint64_t id = state.db.getLastInsertedRowId(); @@ -770,6 +779,10 @@ std::shared_ptr LocalStore::queryPathInfoInternal(State & s while (useQueryReferences.next()) info->references.insert(parseStorePath(useQueryReferences.getStr(0))); + auto prov = (const char *) sqlite3_column_text(state.stmts->QueryPathInfo, 8); + if (prov) + info->provenance = std::make_shared(nlohmann::json::parse(prov).template get()); + return info; } @@ -1162,7 +1175,8 @@ StorePath LocalStore::addToStoreFromDump( ContentAddressMethod hashMethod, HashAlgorithm hashAlgo, const StorePathSet & references, - RepairFlag repair) + RepairFlag repair, + std::optional> provenance) { /* For computing the store path. */ auto hashSink = std::make_unique(hashAlgo); @@ -1262,7 +1276,6 @@ StorePath LocalStore::addToStoreFromDump( PathLocks outputLock({realPath}); if (repair || !isValidPath(dstPath)) { - deletePath(realPath); autoGC(); @@ -1311,6 +1324,7 @@ StorePath LocalStore::addToStoreFromDump( narHash.first }; info.narSize = narHash.second; + info.provenance = provenance ? std::make_shared(*provenance) : nullptr; registerValidPath(info); } diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 83154d65193..ba970a452cb 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -191,7 +191,8 @@ public: ContentAddressMethod hashMethod, HashAlgorithm hashAlgo, const StorePathSet & references, - RepairFlag repair) override; + RepairFlag repair, + std::optional> provenance = std::nullopt) override; void addTempRoot(const StorePath & path) override; diff --git a/src/libstore/meson.build b/src/libstore/meson.build index 6a6aabf97aa..6a192e90389 100644 --- a/src/libstore/meson.build +++ b/src/libstore/meson.build @@ -220,6 +220,7 @@ sources = files( 'pathlocks.cc', 'posix-fs-canonicalise.cc', 'profiles.cc', + 'provenance.cc', 'realisation.cc', 'remote-fs-accessor.cc', 'remote-store.cc', @@ -289,6 +290,7 @@ headers = [config_h] + files( 'pathlocks.hh', 'posix-fs-canonicalise.hh', 'profiles.hh', + 'provenance.hh', 'realisation.hh', 'remote-fs-accessor.hh', 'remote-store-connection.hh', diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index 8b255706072..b8b3b9af26d 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -3,6 +3,7 @@ #include "store-api.hh" #include "strings.hh" #include "json-utils.hh" +#include "provenance.hh" namespace nix { @@ -81,6 +82,9 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & // FIXME: allow blank ca or require skipping field? ca = ContentAddress::parseOpt(value); } + else if (name == "Provenance") { + provenance = std::make_shared(nlohmann::json::parse(value).template get()); + } pos = eol + 1; line += 1; @@ -124,6 +128,9 @@ std::string NarInfo::to_string(const Store & store) const if (ca) res += "CA: " + renderContentAddress(*ca) + "\n"; + if (provenance) + res += "Provenance: " + nlohmann::json(*provenance).dump() + "\n"; + return res; } diff --git a/src/libstore/path-info.cc b/src/libstore/path-info.cc index 6e87e60f446..f140c9bb57d 100644 --- a/src/libstore/path-info.cc +++ b/src/libstore/path-info.cc @@ -5,6 +5,7 @@ #include "json-utils.hh" #include "comparator.hh" #include "strings.hh" +#include "provenance.hh" namespace nix { @@ -177,6 +178,8 @@ nlohmann::json UnkeyedValidPathInfo::toJSON( auto & sigsObj = jsonObject["signatures"] = json::array(); for (auto & sig : sigs) sigsObj.push_back(sig); + + jsonObject["provenance"] = provenance ? nlohmann::json(*provenance) : nlohmann::json(); } return jsonObject; @@ -224,6 +227,9 @@ UnkeyedValidPathInfo UnkeyedValidPathInfo::fromJSON( if (json.contains("signatures")) res.sigs = getStringSet(valueAt(json, "signatures")); + auto prov = json.find("provenance"); + res.provenance = prov == json.end() || prov->second.is_null() ? nullptr : std::make_shared(prov->second.get()); + return res; } diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh index 71f1476a672..7f80ad91326 100644 --- a/src/libstore/path-info.hh +++ b/src/libstore/path-info.hh @@ -13,6 +13,7 @@ namespace nix { class Store; +struct Provenance; struct SubstitutablePathInfo @@ -101,6 +102,12 @@ struct UnkeyedValidPathInfo */ std::optional ca; + /** + * The provenance of this store path, i.e. a link back to the Nix + * expression used to create it. + */ + std::shared_ptr provenance; + UnkeyedValidPathInfo(const UnkeyedValidPathInfo & other) = default; UnkeyedValidPathInfo(Hash narHash) : narHash(narHash) { }; diff --git a/src/libstore/provenance.cc b/src/libstore/provenance.cc new file mode 100644 index 00000000000..5c2e252206d --- /dev/null +++ b/src/libstore/provenance.cc @@ -0,0 +1,88 @@ +#include "provenance.hh" +#include "util.hh" + +#include + +namespace nix { + +std::string Provenance::type() const +{ + return std::visit( + overloaded{ + [&](const Provenance::ProvDerivation & p) -> std::string { return "derivation"; }, + [&](const Provenance::ProvCopied & p) -> std::string { return "copied"; }, + [&](const Provenance::ProvFlake & p) -> std::string { return "flake"; }, + }, + raw); +} + +void to_json(nlohmann::json & j, const Provenance & p) +{ + std::visit( + overloaded{ + [&](const Provenance::ProvDerivation & p) { nlohmann::to_json(j, p); }, + [&](const Provenance::ProvCopied & p) { nlohmann::to_json(j, p); }, + [&](const Provenance::ProvFlake & p) { nlohmann::to_json(j, p); }, + }, + p.raw); + + j["type"] = p.type(); +} + +void to_json(nlohmann::json & j, const Provenance::ProvDerivation & p) +{ + j = nlohmann::json{ + {"drv", p.drvPath.to_string()}, + {"provenance", p.provenance ? nlohmann::json(*p.provenance) : nlohmann::json()}, + {"output", p.output}}; +} + +void to_json(nlohmann::json & j, const Provenance::ProvCopied & p) +{ + j = nlohmann::json{ + {"from", p.from}, + {"provenance", p.provenance ? nlohmann::json(*p.provenance) : nlohmann::json()}, + }; +} + +void to_json(nlohmann::json & j, const Provenance::ProvFlake & p) +{ + j = nlohmann::json{ + {"flake", *p.flake}, + {"output", p.flakeOutput}, + }; +} + +void from_json(const nlohmann::json & j, Provenance & p) +{ + auto type = j.at("type").get(); + + if (type == "flake") + p = {Provenance::ProvFlake{ + .flake = std::make_shared(j.at("flake")), // FIXME: validate + .flakeOutput = j.at("output").get(), + }}; + + else if (type == "derivation") { + auto prov = j.at("provenance"); + p = {Provenance::ProvDerivation{ + .drvPath = StorePath(j.at("drv").get()), + .output = j.at("output").get(), + .provenance = prov.is_null() ? nullptr : std::make_shared(prov.get()), + }}; + } + + else if (type == "copied") { + auto prov = j.at("provenance"); + p = {Provenance::ProvCopied{ + .from = j.at("from").get(), + .provenance = prov.is_null() ? nullptr : std::make_shared(prov.get()), + }}; + } + + else + // FIXME: pass this through as raw json? + throw Error("unsupported provenance type '%s'", type); +} + +} diff --git a/src/libstore/provenance.hh b/src/libstore/provenance.hh new file mode 100644 index 00000000000..61a7bce5922 --- /dev/null +++ b/src/libstore/provenance.hh @@ -0,0 +1,95 @@ +#pragma once + +#include "outputs-spec.hh" +#include "path.hh" + +#include "nlohmann/json_fwd.hpp" + +namespace nix { + +/** + * This struct describes the provenance of a store path, i.e. a link + * back to the source code from which the store path was originally + * built. + */ +struct Provenance +{ + /** + * Type that denotes a store path that was produced by a + * derivation. + */ + struct ProvDerivation + { + /** + * The derivation that built this path. + */ + StorePath drvPath; + + /** + * The output of the derivation that corresponds to this path. + */ + OutputName output; + + /** + * The nested provenance of the derivation. + */ + std::shared_ptr provenance; + + // FIXME: do we need anything extra for CA derivations? + }; + + /** + * Type that denotes a store path that was copied/substituted + * from another store. + */ + struct ProvCopied + { + /** + * Store URL (typically a binary cache) from which this store + * path was copied. + */ + std::string from; + + /** + * Provenance of the store path in the upstream store, if any. + */ + std::shared_ptr provenance; + }; + + /** + * Type that denotes a store path (typically a .drv file or + * derivation input source) that was produced by the evaluation of + * a flake. + */ + struct ProvFlake + { + std::shared_ptr flake; // FIXME: change to Attrs + std::string flakeOutput; + }; + + using Raw = std::variant; + + Raw raw; + + Provenance(Raw raw) + : raw(std::move(raw)) + { + } + + // FIXME: ugly, nlohmann::json wants a default constructor. + Provenance() + : raw(ProvFlake{}) + { + } + + std::string type() const; +}; + +void to_json(nlohmann::json & j, const Provenance & p); +void to_json(nlohmann::json & j, const Provenance::ProvDerivation & p); +void to_json(nlohmann::json & j, const Provenance::ProvCopied & p); +void to_json(nlohmann::json & j, const Provenance::ProvFlake & p); + +void from_json(const nlohmann::json & j, Provenance & p); + +} diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 69bbc64fca3..16c350b754e 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -463,8 +463,12 @@ StorePath RemoteStore::addToStoreFromDump( ContentAddressMethod hashMethod, HashAlgorithm hashAlgo, const StorePathSet & references, - RepairFlag repair) + RepairFlag repair, + std::optional> provenance) { + if (provenance) + throw UnimplementedError("RemoteStore::addToStoreFromDump() with provenance"); + FileSerialisationMethod fsm; switch (hashMethod.getFileIngestionMethod()) { case FileIngestionMethod::Flat: diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 4e18962683d..d0136bf8e47 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -91,7 +91,8 @@ public: ContentAddressMethod hashMethod = FileIngestionMethod::NixArchive, HashAlgorithm hashAlgo = HashAlgorithm::SHA256, const StorePathSet & references = StorePathSet(), - RepairFlag repair = NoRepair) override; + RepairFlag repair = NoRepair, + std::optional> provenance = std::nullopt) override; void addToStore(const ValidPathInfo & info, Source & nar, RepairFlag repair, CheckSigsFlag checkSigs) override; diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc index 954a9746774..b0a99ed5284 100644 --- a/src/libstore/ssh-store.cc +++ b/src/libstore/ssh-store.cc @@ -52,6 +52,9 @@ class SSHStore : public virtual SSHStoreConfig, public virtual RemoteStore return *uriSchemes().begin() + "://" + host; } + bool uriIsUsefulProvenance() override + { return true; } + // FIXME extend daemon protocol, move implementation to RemoteStore std::optional getBuildLogExact(const StorePath & path) override { unsupported("getBuildLogExact"); } diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 10577fa2a32..c20f32a76fe 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -18,6 +18,8 @@ #include "worker-protocol.hh" #include "signals.hh" #include "users.hh" +#include "provenance.hh" +#include "local-fs-store.hh" #include #include @@ -195,7 +197,8 @@ StorePath Store::addToStore( HashAlgorithm hashAlgo, const StorePathSet & references, PathFilter & filter, - RepairFlag repair) + RepairFlag repair, + std::optional> provenance) { FileSerialisationMethod fsm; switch (method.getFileIngestionMethod()) { @@ -213,7 +216,7 @@ StorePath Store::addToStore( std::optional storePath; auto sink = sourceToSink([&](Source & source) { LengthSource lengthSource(source); - storePath = addToStoreFromDump(lengthSource, name, fsm, method, hashAlgo, references, repair); + storePath = addToStoreFromDump(lengthSource, name, fsm, method, hashAlgo, references, repair, provenance); if (settings.warnLargePathThreshold && lengthSource.total >= settings.warnLargePathThreshold) warn("copied large path '%s' to the store (%s)", path, renderSize(lengthSource.total)); }); @@ -953,6 +956,25 @@ static std::string makeCopyPathMessage( } +/** + * Wrap upstream provenance in a "copied" provenance record to record + * where the path was copied from. But uninformative origins like + * LocalStore are omitted. + */ +static std::shared_ptr addCopiedProvenance( + std::shared_ptr prov, + Store & srcStore) +{ + if (!srcStore.uriIsUsefulProvenance()) + return prov; + return std::make_shared( + Provenance::ProvCopied { + .from = srcStore.getUri(), + .provenance = prov, + }); +} + + void copyStorePath( Store & srcStore, Store & dstStore, @@ -973,26 +995,23 @@ void copyStorePath( {storePathS, srcUri, dstUri}); PushActivity pact(act.id); - auto info = srcStore.queryPathInfo(storePath); + auto srcInfo = srcStore.queryPathInfo(storePath); + auto info = make_ref(*srcInfo); uint64_t total = 0; // recompute store path on the chance dstStore does it differently if (info->ca && info->references.empty()) { - auto info2 = make_ref(*info); - info2->path = dstStore.makeFixedOutputPathFromCA( + info->path = dstStore.makeFixedOutputPathFromCA( info->path.name(), info->contentAddressWithReferences().value()); if (dstStore.storeDir == srcStore.storeDir) - assert(info->path == info2->path); - info = info2; + assert(info->path == srcInfo->path); } - if (info->ultimate) { - auto info2 = make_ref(*info); - info2->ultimate = false; - info = info2; - } + info->ultimate = false; + + info->provenance = addCopiedProvenance(info->provenance, srcStore); auto source = sinkToSource([&](Sink & sink) { LambdaSink progressSink([&](std::string_view data) { @@ -1119,6 +1138,7 @@ std::map copyPaths( ValidPathInfo infoForDst = *info; infoForDst.path = storePathForDst; + infoForDst.provenance = addCopiedProvenance(info->provenance, srcStore); auto source = sinkToSource([&](Sink & sink) { // We can reasonably assume that the copy will happen whenever we diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 8288cfdf01f..21e9e2bedba 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -75,6 +75,8 @@ struct SourceAccessor; class NarInfoDiskCache; class Store; +struct Provenance; + typedef std::map OutputPathMap; @@ -222,6 +224,13 @@ public: */ virtual std::string getUri() = 0; + /** + * Whether, when copying *from* this store, a "copied" provenance + * record should be added. + */ + virtual bool uriIsUsefulProvenance() + { return false; } + /** * Follow symlinks until we end up with a path in the Nix store. */ @@ -445,7 +454,8 @@ public: HashAlgorithm hashAlgo = HashAlgorithm::SHA256, const StorePathSet & references = StorePathSet(), PathFilter & filter = defaultPathFilter, - RepairFlag repair = NoRepair); + RepairFlag repair = NoRepair, + std::optional> provenance = std::nullopt); /** * Copy the contents of a path to the store and register the @@ -484,7 +494,8 @@ public: ContentAddressMethod hashMethod = ContentAddressMethod::Raw::NixArchive, HashAlgorithm hashAlgo = HashAlgorithm::SHA256, const StorePathSet & references = StorePathSet(), - RepairFlag repair = NoRepair) = 0; + RepairFlag repair = NoRepair, + std::optional> provenance = std::nullopt) = 0; /** * Add a mapping indicating that `deriver!outputName` maps to the output path diff --git a/src/libstore/unix/build/local-derivation-goal.cc b/src/libstore/unix/build/local-derivation-goal.cc index b4685b3a77e..53d214a3c79 100644 --- a/src/libstore/unix/build/local-derivation-goal.cc +++ b/src/libstore/unix/build/local-derivation-goal.cc @@ -19,6 +19,7 @@ #include "unix-domain-socket.hh" #include "posix-fs-canonicalise.hh" #include "posix-source-accessor.hh" +#include "provenance.hh" #include #include @@ -1381,7 +1382,8 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual In HashAlgorithm hashAlgo, const StorePathSet & references, PathFilter & filter, - RepairFlag repair) override + RepairFlag repair, + std::optional> provenance) override { throw Error("addToStore"); } void addToStore(const ValidPathInfo & info, Source & narSource, @@ -1398,9 +1400,10 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual In ContentAddressMethod hashMethod, HashAlgorithm hashAlgo, const StorePathSet & references, - RepairFlag repair) override + RepairFlag repair, + std::optional> provenance) override { - auto path = next->addToStoreFromDump(dump, name, dumpMethod, hashMethod, hashAlgo, references, repair); + auto path = next->addToStoreFromDump(dump, name, dumpMethod, hashMethod, hashAlgo, references, repair, provenance); goal.addDependency(path); return path; } @@ -2787,6 +2790,13 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() newInfo.deriver = drvPath; newInfo.ultimate = true; + if (drvProvenance) + newInfo.provenance = std::make_shared( + Provenance::ProvDerivation { + .drvPath = drvPath, + .output = outputName, + .provenance = drvProvenance, + }); localStore.signPathInfo(newInfo); finish(newInfo.path);