diff --git a/src/libstore/build/create-derivation-and-realise-goal.cc b/src/libstore/build/create-derivation-and-realise-goal.cc new file mode 100644 index 000000000000..b01042f0032f --- /dev/null +++ b/src/libstore/build/create-derivation-and-realise-goal.cc @@ -0,0 +1,157 @@ +#include "create-derivation-and-realise-goal.hh" +#include "worker.hh" + +namespace nix { + +CreateDerivationAndRealiseGoal::CreateDerivationAndRealiseGoal(ref drvReq, + const OutputsSpec & wantedOutputs, Worker & worker, BuildMode buildMode) + : Goal(worker, DerivedPath::Built { .drvPath = drvReq, .outputs = wantedOutputs }) + , drvReq(drvReq) + , wantedOutputs(wantedOutputs) + , buildMode(buildMode) +{ + state = &CreateDerivationAndRealiseGoal::getDerivation; + name = fmt( + "outer obtaining drv from '%s' and then building outputs %s", + drvReq->to_string(worker.store), + std::visit(overloaded { + [&](const OutputsSpec::All) -> std::string { + return "* (all of them)"; + }, + [&](const OutputsSpec::Names os) { + return concatStringsSep(", ", quoteStrings(os)); + }, + }, wantedOutputs.raw)); + trace("created outer"); + + worker.updateProgress(); +} + + +CreateDerivationAndRealiseGoal::~CreateDerivationAndRealiseGoal() +{ +} + + +static StorePath pathPartOfReq(const SingleDerivedPath & req) +{ + return std::visit(overloaded { + [&](const SingleDerivedPath::Opaque & bo) { + return bo.path; + }, + [&](const SingleDerivedPath::Built & bfd) { + return pathPartOfReq(*bfd.drvPath); + }, + }, req.raw()); +} + + +std::string CreateDerivationAndRealiseGoal::key() +{ + /* Ensure that derivations get built in order of their name, + i.e. a derivation named "aardvark" always comes before "baboon". And + substitution goals and inner derivation goals always happen before + derivation goals (due to "b$"). */ + return "c$" + std::string(pathPartOfReq(*drvReq).name()) + "$" + drvReq->to_string(worker.store); +} + + +void CreateDerivationAndRealiseGoal::timedOut(Error && ex) +{ +} + + +void CreateDerivationAndRealiseGoal::work() +{ + (this->*state)(); +} + + +void CreateDerivationAndRealiseGoal::addWantedOutputs(const OutputsSpec & outputs) +{ + /* If we already want all outputs, there is nothing to do. */ + auto newWanted = wantedOutputs.union_(outputs); + bool needRestart = !newWanted.isSubsetOf(wantedOutputs); + wantedOutputs = newWanted; + + if (!needRestart) return; + + if (!optDrvPath) + // haven't started steps where the outputs matter yet + return; + worker.makeDerivationGoal(*optDrvPath, outputs, buildMode); +} + + +void CreateDerivationAndRealiseGoal::getDerivation() +{ + trace("outer init"); + + /* The first thing to do is to make sure that the derivation + exists. If it doesn't, it may be created through a + substitute. */ + if (auto optDrvPath = [this]() -> std::optional { + if (buildMode != bmNormal) return std::nullopt; + + auto drvPath = StorePath::dummy; + try { + drvPath = resolveDerivedPath(worker.store, *drvReq); + } catch (MissingRealisation) { + return std::nullopt; + } + return worker.evalStore.isValidPath(drvPath) || worker.store.isValidPath(drvPath) + ? std::optional { drvPath } + : std::nullopt; + }()) { + trace(fmt("already have drv '%s' for '%s', can go straight to building", + worker.store.printStorePath(*optDrvPath), + drvReq->to_string(worker.store))); + + loadAndBuildDerivation(); + } else { + trace("need to obtain drv we want to build"); + + addWaitee(worker.makeGoal(DerivedPath::fromSingle(*drvReq))); + + state = &CreateDerivationAndRealiseGoal::loadAndBuildDerivation; + if (waitees.empty()) work(); + } +} + + +void CreateDerivationAndRealiseGoal::loadAndBuildDerivation() +{ + trace("outer load and build derivation"); + + if (nrFailed != 0) { + amDone(ecFailed, Error("cannot build missing derivation '%s'", drvReq->to_string(worker.store))); + return; + } + + StorePath drvPath = resolveDerivedPath(worker.store, *drvReq); + /* Build this step! */ + concreteDrvGoal = worker.makeDerivationGoal(drvPath, wantedOutputs, buildMode); + addWaitee(upcast_goal(concreteDrvGoal)); + state = &CreateDerivationAndRealiseGoal::buildDone; + optDrvPath = std::move(drvPath); + if (waitees.empty()) work(); +} + + +void CreateDerivationAndRealiseGoal::buildDone() +{ + trace("outer build done"); + + buildResult = upcast_goal(concreteDrvGoal)->getBuildResult(DerivedPath::Built { + .drvPath = drvReq, + .outputs = wantedOutputs, + }); + + if (buildResult.success()) + amDone(ecSuccess); + else + amDone(ecFailed, Error("building '%s' failed", drvReq->to_string(worker.store))); +} + + +} diff --git a/src/libstore/build/create-derivation-and-realise-goal.hh b/src/libstore/build/create-derivation-and-realise-goal.hh new file mode 100644 index 000000000000..fee1195d081a --- /dev/null +++ b/src/libstore/build/create-derivation-and-realise-goal.hh @@ -0,0 +1,94 @@ +#pragma once + +#include "parsed-derivations.hh" +#include "lock.hh" +#include "store-api.hh" +#include "pathlocks.hh" +#include "goal.hh" + +namespace nix { + +struct DerivationGoal; + +/** + * This goal type is essentially the serial composition (like function + * composition) of a goal for getting a derivation, and then a + * `DerivationGoal` using the newly-obtained derivation. + * + * In the (currently experimental) general inductive case of derivations + * that are themselves build outputs, that first goal will be *another* + * `CreateDerivationAndRealiseGoal`. In the (much more common) base-case + * where the derivation has no provence and is just referred to by + * (content-addressed) store path, that first goal is a + * `SubstitutionGoal`. + * + * If we already have the derivation (e.g. if the evalutator has created + * the derivation locally and then instructured the store to build it), + * we can skip the first goal entirely as a small optimization. + */ +struct CreateDerivationAndRealiseGoal : public Goal +{ + /** + * How to obtain a store path of the derivation to build. + */ + ref drvReq; + + /** + * The path of the derivation, once obtained. + **/ + std::optional optDrvPath; + + /** + * The goal for the corresponding concrete derivation. + **/ + std::shared_ptr concreteDrvGoal; + + /** + * The specific outputs that we need to build. + */ + OutputsSpec wantedOutputs; + + typedef void (CreateDerivationAndRealiseGoal::*GoalState)(); + GoalState state; + + /** + * The final output paths of the build. + * + * - For input-addressed derivations, always the precomputed paths + * + * - For content-addressed derivations, calcuated from whatever the + * hash ends up being. (Note that fixed outputs derivations that + * produce the "wrong" output still install that data under its + * true content-address.) + */ + OutputPathMap finalOutputs; + + BuildMode buildMode; + + CreateDerivationAndRealiseGoal(ref drvReq, + const OutputsSpec & wantedOutputs, Worker & worker, + BuildMode buildMode = bmNormal); + virtual ~CreateDerivationAndRealiseGoal(); + + void timedOut(Error && ex) override; + + std::string key() override; + + void work() override; + + /** + * Add wanted outputs to an already existing derivation goal. + */ + void addWantedOutputs(const OutputsSpec & outputs); + + /** + * The states. + */ + void getDerivation(); + void loadAndBuildDerivation(); + void buildDone(); + + JobCategory jobCategory() override { return JobCategory::Administration; }; +}; + +} diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index ea56c028829c..ef32a7fc358d 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -71,7 +71,7 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, , wantedOutputs(wantedOutputs) , buildMode(buildMode) { - state = &DerivationGoal::getDerivation; + state = &DerivationGoal::loadDerivation; name = fmt( "building of '%s' from .drv file", DerivedPath::Built { makeConstantStorePathRef(drvPath), wantedOutputs }.to_string(worker.store)); @@ -164,24 +164,6 @@ void DerivationGoal::addWantedOutputs(const OutputsSpec & outputs) } -void DerivationGoal::getDerivation() -{ - trace("init"); - - /* The first thing to do is to make sure that the derivation - exists. If it doesn't, it may be created through a - substitute. */ - if (buildMode == bmNormal && worker.evalStore.isValidPath(drvPath)) { - loadDerivation(); - return; - } - - addWaitee(upcast_goal(worker.makePathSubstitutionGoal(drvPath))); - - state = &DerivationGoal::loadDerivation; -} - - void DerivationGoal::loadDerivation() { trace("loading derivation"); @@ -1485,6 +1467,11 @@ void DerivationGoal::done( amDone(buildResult.success() ? ecSuccess : ecFailed, std::move(ex)); } +} + +#include "create-derivation-and-realise-goal.hh" + +namespace nix { void DerivationGoal::waiteeDone(GoalPtr waitee, ExitCode result) { @@ -1493,8 +1480,11 @@ void DerivationGoal::waiteeDone(GoalPtr waitee, ExitCode result) if (!useDerivation) return; auto & fullDrv = *dynamic_cast(drv.get()); - auto * dg = dynamic_cast(&*waitee); - if (!dg) return; + auto * odg = dynamic_cast(&*waitee); + if (!odg) return; + auto * dg = &*odg->concreteDrvGoal; + // By the time we get here, the outer goals should have. + assert(dg); auto outputs = fullDrv.inputDrvs.find(dg->drvPath); if (outputs == fullDrv.inputDrvs.end()) return; diff --git a/src/libstore/build/derivation-goal.hh b/src/libstore/build/derivation-goal.hh index 9d6fe1c0fa76..28ce0405de62 100644 --- a/src/libstore/build/derivation-goal.hh +++ b/src/libstore/build/derivation-goal.hh @@ -50,6 +50,13 @@ struct InitialOutput { std::optional known; }; +/** + * A goal for building some or all of the outputs of a derivation. + * + * The derivation must already be present, either in the store in a drv + * or in memory. If the derivation itself needs to be gotten first, a + * `CreateDerivationAndRealiseGoal` goal must be used instead. + */ struct DerivationGoal : public Goal { /** @@ -66,8 +73,7 @@ struct DerivationGoal : public Goal std::shared_ptr resolvedDrvGoal; /** - * The specific outputs that we need to build. Empty means all of - * them. + * The specific outputs that we need to build. */ OutputsSpec wantedOutputs; @@ -229,7 +235,6 @@ struct DerivationGoal : public Goal /** * The states. */ - void getDerivation(); void loadDerivation(); void haveDerivation(); void outputsSubstitutionTried(); diff --git a/src/libstore/build/entry-points.cc b/src/libstore/build/entry-points.cc index f71fb35a611c..f0f0e5519358 100644 --- a/src/libstore/build/entry-points.cc +++ b/src/libstore/build/entry-points.cc @@ -1,5 +1,6 @@ #include "worker.hh" #include "substitution-goal.hh" +#include "create-derivation-and-realise-goal.hh" #include "derivation-goal.hh" #include "local-store.hh" @@ -15,7 +16,7 @@ void Store::buildPaths(const std::vector & reqs, BuildMode buildMod worker.run(goals); - StorePathSet failed; + StringSet failed; std::optional ex; for (auto & i : goals) { if (i->ex) { @@ -25,8 +26,10 @@ void Store::buildPaths(const std::vector & reqs, BuildMode buildMod ex = std::move(i->ex); } if (i->exitCode != Goal::ecSuccess) { - if (auto i2 = dynamic_cast(i.get())) failed.insert(i2->drvPath); - else if (auto i2 = dynamic_cast(i.get())) failed.insert(i2->storePath); + if (auto i2 = dynamic_cast(i.get())) + failed.insert(i2->drvReq->to_string(*this)); + else if (auto i2 = dynamic_cast(i.get())) + failed.insert(printStorePath(i2->storePath)); } } @@ -35,7 +38,7 @@ void Store::buildPaths(const std::vector & reqs, BuildMode buildMod throw std::move(*ex); } else if (!failed.empty()) { if (ex) logError(ex->info()); - throw Error(worker.failingExitStatus(), "build of %s failed", showPaths(failed)); + throw Error(worker.failingExitStatus(), "build of %s failed", concatStringsSep(", ", quoteStrings(failed))); } } diff --git a/src/libstore/build/goal.hh b/src/libstore/build/goal.hh index d3127caeaa54..91142df4d00a 100644 --- a/src/libstore/build/goal.hh +++ b/src/libstore/build/goal.hh @@ -41,8 +41,24 @@ typedef std::map WeakGoalMap; * of each category in parallel. */ enum struct JobCategory { + /** + * A build of a derivation; it will use CPU and disk resources. + */ Build, + /** + * A substitution an arbitrary store object; it will use network resources. + */ Substitution, + /** + * A goal that does no "real" work by itself, and just exists to depend on + * other goals which *do* do real work. These goals therefore are not + * limited. + * + * These goals cannot infinitely create themselves, so there is no risk of + * a "fork bomb" type situation (which would be a problem even though the + * goal do no real work) either. + */ + Administration, }; struct Goal : public std::enable_shared_from_this diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index b58fc5c1c0c3..2b993b5aaf24 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -2,6 +2,7 @@ #include "worker.hh" #include "substitution-goal.hh" #include "drv-output-substitution-goal.hh" +#include "create-derivation-and-realise-goal.hh" #include "local-derivation-goal.hh" #include "hook-instance.hh" @@ -41,6 +42,24 @@ Worker::~Worker() } +std::shared_ptr Worker::makeCreateDerivationAndRealiseGoal( + ref drvReq, + const OutputsSpec & wantedOutputs, + BuildMode buildMode) +{ + std::weak_ptr & goal_weak = outerDerivationGoals.ensureSlot(*drvReq).value; + std::shared_ptr goal = goal_weak.lock(); + if (!goal) { + goal = std::make_shared(drvReq, wantedOutputs, *this, buildMode); + goal_weak = goal; + wakeUp(goal); + } else { + goal->addWantedOutputs(wantedOutputs); + } + return goal; +} + + std::shared_ptr Worker::makeDerivationGoalCommon( const StorePath & drvPath, const OutputsSpec & wantedOutputs, @@ -111,10 +130,7 @@ GoalPtr Worker::makeGoal(const DerivedPath & req, BuildMode buildMode) { return std::visit(overloaded { [&](const DerivedPath::Built & bfd) -> GoalPtr { - if (auto bop = std::get_if(&*bfd.drvPath)) - return makeDerivationGoal(bop->path, bfd.outputs, buildMode); - else - throw UnimplementedError("Building dynamic derivations in one shot is not yet implemented."); + return makeCreateDerivationAndRealiseGoal(bfd.drvPath, bfd.outputs, buildMode); }, [&](const DerivedPath::Opaque & bo) -> GoalPtr { return makePathSubstitutionGoal(bo.path, buildMode == bmRepair ? Repair : NoRepair); @@ -123,24 +139,46 @@ GoalPtr Worker::makeGoal(const DerivedPath & req, BuildMode buildMode) } +template +static void cullMap(std::map & goalMap, F f) +{ + for (auto i = goalMap.begin(); i != goalMap.end();) + if (!f(i->second)) + i = goalMap.erase(i); + else ++i; +} + + template static void removeGoal(std::shared_ptr goal, std::map> & goalMap) { /* !!! inefficient */ - for (auto i = goalMap.begin(); - i != goalMap.end(); ) - if (i->second.lock() == goal) { - auto j = i; ++j; - goalMap.erase(i); - i = j; - } - else ++i; + cullMap(goalMap, [&](const std::weak_ptr & gp) -> bool { + return gp.lock() != goal; + }); +} + +template +static void removeGoal(std::shared_ptr goal, std::map>::ChildNode> & goalMap); + +template +static void removeGoal(std::shared_ptr goal, std::map>::ChildNode> & goalMap) +{ + /* !!! inefficient */ + cullMap(goalMap, [&](DerivedPathMap>::ChildNode & node) -> bool { + if (node.value.lock() == goal) + node.value.reset(); + removeGoal(goal, node.childMap); + return !node.value.expired() || !node.childMap.empty(); + }); } void Worker::removeGoal(GoalPtr goal) { - if (auto drvGoal = std::dynamic_pointer_cast(goal)) + if (auto drvGoal = std::dynamic_pointer_cast(goal)) + nix::removeGoal(drvGoal, outerDerivationGoals.map); + else if (auto drvGoal = std::dynamic_pointer_cast(goal)) nix::removeGoal(drvGoal, derivationGoals); else if (auto subGoal = std::dynamic_pointer_cast(goal)) nix::removeGoal(subGoal, substitutionGoals); @@ -198,8 +236,19 @@ void Worker::childStarted(GoalPtr goal, const std::set & fds, child.respectTimeouts = respectTimeouts; children.emplace_back(child); if (inBuildSlot) { - if (goal->jobCategory() == JobCategory::Substitution) nrSubstitutions++; - else nrLocalBuilds++; + switch (goal->jobCategory()) { + case JobCategory::Substitution: + nrSubstitutions++; + break; + case JobCategory::Build: + nrLocalBuilds++; + break; + case JobCategory::Administration: + /* Intentionally not limited, see docs */ + break; + default: + abort(); + } } } @@ -211,12 +260,20 @@ void Worker::childTerminated(Goal * goal, bool wakeSleepers) if (i == children.end()) return; if (i->inBuildSlot) { - if (goal->jobCategory() == JobCategory::Substitution) { + switch (goal->jobCategory()) { + case JobCategory::Substitution: assert(nrSubstitutions > 0); nrSubstitutions--; - } else { + break; + case JobCategory::Build: assert(nrLocalBuilds > 0); nrLocalBuilds--; + break; + case JobCategory::Administration: + /* Intentionally not limited, see docs */ + break; + default: + abort(); } } @@ -267,9 +324,9 @@ void Worker::run(const Goals & _topGoals) for (auto & i : _topGoals) { topGoals.insert(i); - if (auto goal = dynamic_cast(i.get())) { + if (auto goal = dynamic_cast(i.get())) { topPaths.push_back(DerivedPath::Built { - .drvPath = makeConstantStorePathRef(goal->drvPath), + .drvPath = goal->drvReq, .outputs = goal->wantedOutputs, }); } else if (auto goal = dynamic_cast(i.get())) { @@ -522,10 +579,18 @@ void Worker::markContentsGood(const StorePath & path) } -GoalPtr upcast_goal(std::shared_ptr subGoal) { +GoalPtr upcast_goal(std::shared_ptr subGoal) +{ + return subGoal; +} + +GoalPtr upcast_goal(std::shared_ptr subGoal) +{ return subGoal; } -GoalPtr upcast_goal(std::shared_ptr subGoal) { + +GoalPtr upcast_goal(std::shared_ptr subGoal) +{ return subGoal; } diff --git a/src/libstore/build/worker.hh b/src/libstore/build/worker.hh index 5abceca0d71a..85a321fe3126 100644 --- a/src/libstore/build/worker.hh +++ b/src/libstore/build/worker.hh @@ -4,6 +4,7 @@ #include "types.hh" #include "lock.hh" #include "store-api.hh" +#include "derived-path-map.hh" #include "goal.hh" #include "realisation.hh" @@ -13,6 +14,7 @@ namespace nix { /* Forward definition. */ +struct CreateDerivationAndRealiseGoal; struct DerivationGoal; struct PathSubstitutionGoal; class DrvOutputSubstitutionGoal; @@ -31,6 +33,7 @@ class DrvOutputSubstitutionGoal; */ GoalPtr upcast_goal(std::shared_ptr subGoal); GoalPtr upcast_goal(std::shared_ptr subGoal); +GoalPtr upcast_goal(std::shared_ptr subGoal); typedef std::chrono::time_point steady_time_point; @@ -102,6 +105,9 @@ private: * Maps used to prevent multiple instantiations of a goal for the * same derivation / path. */ + + DerivedPathMap> outerDerivationGoals; + std::map> derivationGoals; std::map> substitutionGoals; std::map> drvOutputSubstitutionGoals; @@ -189,6 +195,9 @@ public: * @ref DerivationGoal "derivation goal" */ private: + std::shared_ptr makeCreateDerivationAndRealiseGoal( + ref drvPath, + const OutputsSpec & wantedOutputs, BuildMode buildMode = bmNormal); std::shared_ptr makeDerivationGoalCommon( const StorePath & drvPath, const OutputsSpec & wantedOutputs, std::function()> mkDrvGoal); diff --git a/src/libstore/derived-path-map.cc b/src/libstore/derived-path-map.cc new file mode 100644 index 000000000000..501286b33700 --- /dev/null +++ b/src/libstore/derived-path-map.cc @@ -0,0 +1,33 @@ +#include "derived-path-map.hh" + +namespace nix { + +template +typename DerivedPathMap::ChildNode & DerivedPathMap::ensureSlot(const SingleDerivedPath & k) +{ + std::function initIter; + initIter = [&](const auto & k) -> auto & { + return std::visit(overloaded { + [&](const SingleDerivedPath::Opaque & bo) -> auto & { + // will not overrwrite if already there + return map[bo.path]; + }, + [&](const SingleDerivedPath::Built & bfd) -> auto & { + auto & n = initIter(*bfd.drvPath); + return n.childMap[bfd.output]; + }, + }, k.raw()); + }; + return initIter(k); +} + +} + +// instantiations + +#include "create-derivation-and-realise-goal.hh" +namespace nix { + +template struct DerivedPathMap>; + +} diff --git a/src/libstore/derived-path-map.hh b/src/libstore/derived-path-map.hh new file mode 100644 index 000000000000..da7f922c2944 --- /dev/null +++ b/src/libstore/derived-path-map.hh @@ -0,0 +1,66 @@ +#pragma once + +#include "types.hh" +#include "derived-path.hh" + +namespace nix { + +/** + * A simple Trie, of sorts. Conceptually a map of `SingleDerivedPath` to + * values. + * + * Concretely, an n-ary tree, as described below. A + * `SingleDerivedPath::Opaque` maps to the value of an immediate child + * of the root node. A `SingleDerivedPath::Built` maps to a deeper child + * node: the `SingleDerivedPath::Built::drvPath` is first mapped to a a + * child node (inductively), and then the + * `SingleDerivedPath::Built::output` is used to look up that child's + * child via its map. In this manner, every `SingleDerivedPath` is + * mapped to a child node. + */ +template +struct DerivedPathMap { + /** + * A child node (non-root node). + */ + struct ChildNode { + /** + * Value of this child node. + * + * `V` probably should be an optional type so not every interior + * node has to have a value. + */ + V value; + + /** + * The map type for the root node. + */ + using Map = std::map; + + /** + * The map of the root node. + */ + Map childMap; + }; + + /** + * The map type for the root node. + */ + using Map = std::map; + + /** + * The map of root node. + */ + Map map; + + /** + * Find the node for `k`, creating it if needed. + * + * The node is referred to as a "slot" on the assumption that `V` is + * some sort of optional type, so the given key can be set or unset + * by changing this node. + */ + ChildNode & ensureSlot(const SingleDerivedPath & k); +}; + +} diff --git a/tests/dyn-drv/build-built-drv.sh b/tests/dyn-drv/build-built-drv.sh index 647be9457167..94f3550bdc35 100644 --- a/tests/dyn-drv/build-built-drv.sh +++ b/tests/dyn-drv/build-built-drv.sh @@ -18,4 +18,6 @@ clearStore drvDep=$(nix-instantiate ./text-hashed-output.nix -A producingDrv) -expectStderr 1 nix build "${drvDep}^out^out" --no-link | grepQuiet "Building dynamic derivations in one shot is not yet implemented" +out2=$(nix build "${drvDep}^out^out" --no-link) + +test $out1 == $out2