-
-
Notifications
You must be signed in to change notification settings - Fork 1.5k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Adapt scheduler to work with dynamic derivations
To avoid dealing with an optional `drvPath` (because we might not know it yet) everywhere, make an `CreateDerivationAndRealiseGoal`. This goal just builds/substitutes the derivation file, and then kicks of a build for that obtained derivation; in other words it does the chaining of goals when the drv file is missing (as can already be the case) or computed (new case). This also means the `getDerivation` state can be removed from `DerivationGoal`, which makes the `BasicDerivation` / in memory case and `Derivation` / drv file file case closer together. The map type is factored out for clarity, and because we will soon hvae a second use for it (`Derivation` itself).
- Loading branch information
1 parent
fb7afa4
commit ccdca62
Showing
11 changed files
with
490 additions
and
50 deletions.
There are no files selected for viewing
157 changes: 157 additions & 0 deletions
157
src/libstore/build/create-derivation-and-realise-goal.cc
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,157 @@ | ||
| #include "create-derivation-and-realise-goal.hh" | ||
| #include "worker.hh" | ||
|
|
||
| namespace nix { | ||
|
|
||
| CreateDerivationAndRealiseGoal::CreateDerivationAndRealiseGoal(ref<SingleDerivedPath> drvReq, | ||
| const OutputsSpec & wantedOutputs, Worker & worker, BuildMode buildMode) | ||
| : Goal(worker, DerivedPath::Built { .drvPath = drvReq, .outputs = wantedOutputs }) | ||
| , drvReq(drvReq) | ||
| , wantedOutputs(wantedOutputs) | ||
| , buildMode(buildMode) | ||
| { | ||
| state = &CreateDerivationAndRealiseGoal::getDerivation; | ||
| name = fmt( | ||
| "outer obtaining drv from '%s' and then building outputs %s", | ||
| drvReq->to_string(worker.store), | ||
| std::visit(overloaded { | ||
| [&](const OutputsSpec::All) -> std::string { | ||
| return "* (all of them)"; | ||
| }, | ||
| [&](const OutputsSpec::Names os) { | ||
| return concatStringsSep(", ", quoteStrings(os)); | ||
| }, | ||
| }, wantedOutputs.raw)); | ||
| trace("created outer"); | ||
|
|
||
| worker.updateProgress(); | ||
| } | ||
|
|
||
|
|
||
| CreateDerivationAndRealiseGoal::~CreateDerivationAndRealiseGoal() | ||
| { | ||
| } | ||
|
|
||
|
|
||
| static StorePath pathPartOfReq(const SingleDerivedPath & req) | ||
| { | ||
| return std::visit(overloaded { | ||
| [&](const SingleDerivedPath::Opaque & bo) { | ||
| return bo.path; | ||
| }, | ||
| [&](const SingleDerivedPath::Built & bfd) { | ||
| return pathPartOfReq(*bfd.drvPath); | ||
| }, | ||
| }, req.raw()); | ||
| } | ||
|
|
||
|
|
||
| std::string CreateDerivationAndRealiseGoal::key() | ||
| { | ||
| /* Ensure that derivations get built in order of their name, | ||
| i.e. a derivation named "aardvark" always comes before "baboon". And | ||
| substitution goals and inner derivation goals always happen before | ||
| derivation goals (due to "b$"). */ | ||
| return "c$" + std::string(pathPartOfReq(*drvReq).name()) + "$" + drvReq->to_string(worker.store); | ||
| } | ||
|
|
||
|
|
||
| void CreateDerivationAndRealiseGoal::timedOut(Error && ex) | ||
| { | ||
| } | ||
|
|
||
|
|
||
| void CreateDerivationAndRealiseGoal::work() | ||
| { | ||
| (this->*state)(); | ||
| } | ||
|
|
||
|
|
||
| void CreateDerivationAndRealiseGoal::addWantedOutputs(const OutputsSpec & outputs) | ||
| { | ||
| /* If we already want all outputs, there is nothing to do. */ | ||
| auto newWanted = wantedOutputs.union_(outputs); | ||
| bool needRestart = !newWanted.isSubsetOf(wantedOutputs); | ||
| wantedOutputs = newWanted; | ||
|
|
||
| if (!needRestart) return; | ||
|
|
||
| if (!optDrvPath) | ||
| // haven't started steps where the outputs matter yet | ||
| return; | ||
| worker.makeDerivationGoal(*optDrvPath, outputs, buildMode); | ||
| } | ||
|
|
||
|
|
||
| void CreateDerivationAndRealiseGoal::getDerivation() | ||
| { | ||
| trace("outer init"); | ||
|
|
||
| /* The first thing to do is to make sure that the derivation | ||
| exists. If it doesn't, it may be created through a | ||
| substitute. */ | ||
| if (auto optDrvPath = [this]() -> std::optional<StorePath> { | ||
| if (buildMode != bmNormal) return std::nullopt; | ||
|
|
||
| auto drvPath = StorePath::dummy; | ||
| try { | ||
| drvPath = resolveDerivedPath(worker.store, *drvReq); | ||
| } catch (MissingRealisation) { | ||
| return std::nullopt; | ||
| } | ||
| return worker.evalStore.isValidPath(drvPath) || worker.store.isValidPath(drvPath) | ||
| ? std::optional { drvPath } | ||
| : std::nullopt; | ||
| }()) { | ||
| trace(fmt("already have drv '%s' for '%s', can go straight to building", | ||
| worker.store.printStorePath(*optDrvPath), | ||
| drvReq->to_string(worker.store))); | ||
|
|
||
| loadAndBuildDerivation(); | ||
| } else { | ||
| trace("need to obtain drv we want to build"); | ||
|
|
||
| addWaitee(worker.makeGoal(DerivedPath::fromSingle(*drvReq))); | ||
|
|
||
| state = &CreateDerivationAndRealiseGoal::loadAndBuildDerivation; | ||
| if (waitees.empty()) work(); | ||
| } | ||
| } | ||
|
|
||
|
|
||
| void CreateDerivationAndRealiseGoal::loadAndBuildDerivation() | ||
| { | ||
| trace("outer load and build derivation"); | ||
|
|
||
| if (nrFailed != 0) { | ||
| amDone(ecFailed, Error("cannot build missing derivation '%s'", drvReq->to_string(worker.store))); | ||
| return; | ||
| } | ||
|
|
||
| StorePath drvPath = resolveDerivedPath(worker.store, *drvReq); | ||
| /* Build this step! */ | ||
| concreteDrvGoal = worker.makeDerivationGoal(drvPath, wantedOutputs, buildMode); | ||
| addWaitee(upcast_goal(concreteDrvGoal)); | ||
| state = &CreateDerivationAndRealiseGoal::buildDone; | ||
| optDrvPath = std::move(drvPath); | ||
| if (waitees.empty()) work(); | ||
| } | ||
|
|
||
|
|
||
| void CreateDerivationAndRealiseGoal::buildDone() | ||
| { | ||
| trace("outer build done"); | ||
|
|
||
| buildResult = upcast_goal(concreteDrvGoal)->getBuildResult(DerivedPath::Built { | ||
| .drvPath = drvReq, | ||
| .outputs = wantedOutputs, | ||
| }); | ||
|
|
||
| if (buildResult.success()) | ||
| amDone(ecSuccess); | ||
| else | ||
| amDone(ecFailed, Error("building '%s' failed", drvReq->to_string(worker.store))); | ||
| } | ||
|
|
||
|
|
||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,94 @@ | ||
| #pragma once | ||
|
|
||
| #include "parsed-derivations.hh" | ||
| #include "lock.hh" | ||
| #include "store-api.hh" | ||
| #include "pathlocks.hh" | ||
| #include "goal.hh" | ||
|
|
||
| namespace nix { | ||
|
|
||
| struct DerivationGoal; | ||
|
|
||
| /** | ||
| * This goal type is essentially the serial composition (like function | ||
| * composition) of a goal for getting a derivation, and then a | ||
| * `DerivationGoal` using the newly-obtained derivation. | ||
| * | ||
| * In the (currently experimental) general inductive case of derivations | ||
| * that are themselves build outputs, that first goal will be *another* | ||
| * `CreateDerivationAndRealiseGoal`. In the (much more common) base-case | ||
| * where the derivation has no provence and is just referred to by | ||
| * (content-addressed) store path, that first goal is a | ||
| * `SubstitutionGoal`. | ||
| * | ||
| * If we already have the derivation (e.g. if the evalutator has created | ||
| * the derivation locally and then instructured the store to build it), | ||
| * we can skip the first goal entirely as a small optimization. | ||
| */ | ||
| struct CreateDerivationAndRealiseGoal : public Goal | ||
| { | ||
| /** | ||
| * How to obtain a store path of the derivation to build. | ||
| */ | ||
| ref<SingleDerivedPath> drvReq; | ||
|
|
||
| /** | ||
| * The path of the derivation, once obtained. | ||
| **/ | ||
| std::optional<StorePath> optDrvPath; | ||
|
|
||
| /** | ||
| * The goal for the corresponding concrete derivation. | ||
| **/ | ||
| std::shared_ptr<DerivationGoal> concreteDrvGoal; | ||
|
|
||
| /** | ||
| * The specific outputs that we need to build. | ||
| */ | ||
| OutputsSpec wantedOutputs; | ||
|
|
||
| typedef void (CreateDerivationAndRealiseGoal::*GoalState)(); | ||
| GoalState state; | ||
|
|
||
| /** | ||
| * The final output paths of the build. | ||
| * | ||
| * - For input-addressed derivations, always the precomputed paths | ||
| * | ||
| * - For content-addressed derivations, calcuated from whatever the | ||
| * hash ends up being. (Note that fixed outputs derivations that | ||
| * produce the "wrong" output still install that data under its | ||
| * true content-address.) | ||
| */ | ||
| OutputPathMap finalOutputs; | ||
|
|
||
| BuildMode buildMode; | ||
|
|
||
| CreateDerivationAndRealiseGoal(ref<SingleDerivedPath> drvReq, | ||
| const OutputsSpec & wantedOutputs, Worker & worker, | ||
| BuildMode buildMode = bmNormal); | ||
| virtual ~CreateDerivationAndRealiseGoal(); | ||
|
|
||
| void timedOut(Error && ex) override; | ||
|
|
||
| std::string key() override; | ||
|
|
||
| void work() override; | ||
|
|
||
| /** | ||
| * Add wanted outputs to an already existing derivation goal. | ||
| */ | ||
| void addWantedOutputs(const OutputsSpec & outputs); | ||
|
|
||
| /** | ||
| * The states. | ||
| */ | ||
| void getDerivation(); | ||
| void loadAndBuildDerivation(); | ||
| void buildDone(); | ||
|
|
||
| JobCategory jobCategory() override { return JobCategory::Administration; }; | ||
| }; | ||
|
|
||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.