-
-
Notifications
You must be signed in to change notification settings - Fork 1.5k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Adapt scheduler to work with dynamic derivations
To avoid dealing with an optional `drvPath` (because we might not know it yet) everywhere, make an `OuterDerivationGoal`. This goal just builds/substitutes the derivation file, and then kicks of a build for that obtained derivation; in other words it does the chaining of goals when the drv file is missing (as can already be the case) or computed (new case). This also means the `getDerivation` state can be removed from `DerivationGoal`, which makes the `BasicDerivation` / in memory case and `Derivation` / drv file file case closer together.
- Loading branch information
1 parent
bcf0b49
commit 4e3db84
Showing
9 changed files
with
336 additions
and
44 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,155 @@ | ||
| #include "outer-derivation-goal.hh" | ||
| #include "worker.hh" | ||
|
|
||
| namespace nix { | ||
|
|
||
| OuterDerivationGoal::OuterDerivationGoal(ref<SingleDerivedPath> drvReq, | ||
| const OutputsSpec & wantedOutputs, Worker & worker, BuildMode buildMode) | ||
| : Goal(worker, DerivedPath::Built { .drvPath = drvReq, .outputs = wantedOutputs }) | ||
| , drvReq(drvReq) | ||
| , wantedOutputs(wantedOutputs) | ||
| , buildMode(buildMode) | ||
| { | ||
| state = &OuterDerivationGoal::getDerivation; | ||
| name = fmt( | ||
| "outer obtaining drv from '%s' and then building outputs %s", | ||
| drvReq->to_string(worker.store), | ||
| std::visit(overloaded { | ||
| [&](const OutputsSpec::All) -> std::string { | ||
| return "* (all of them)"; | ||
| }, | ||
| [&](const OutputsSpec::Names os) { | ||
| return concatStringsSep(", ", quoteStrings(os)); | ||
| }, | ||
| }, wantedOutputs.raw())); | ||
| trace("created outer"); | ||
|
|
||
| worker.updateProgress(); | ||
| } | ||
|
|
||
|
|
||
| OuterDerivationGoal::~OuterDerivationGoal() | ||
| { | ||
| } | ||
|
|
||
|
|
||
| static StorePath pathPartOfReq(const SingleDerivedPath & req) | ||
| { | ||
| return std::visit(overloaded { | ||
| [&](const SingleDerivedPath::Opaque & bo) { | ||
| return bo.path; | ||
| }, | ||
| [&](const SingleDerivedPath::Built & bfd) { | ||
| return pathPartOfReq(*bfd.drvPath); | ||
| }, | ||
| }, req.raw()); | ||
| } | ||
|
|
||
|
|
||
| std::string OuterDerivationGoal::key() | ||
| { | ||
| /* Ensure that derivations get built in order of their name, | ||
| i.e. a derivation named "aardvark" always comes before "baboon". And | ||
| substitution goals and inner derivation goals always happen before | ||
| derivation goals (due to "b$"). */ | ||
| return "c$" + std::string(pathPartOfReq(*drvReq).name()) + "$" + drvReq->to_string(worker.store); | ||
| } | ||
|
|
||
|
|
||
| void OuterDerivationGoal::timedOut(Error && ex) | ||
| { | ||
| } | ||
|
|
||
|
|
||
| void OuterDerivationGoal::work() | ||
| { | ||
| (this->*state)(); | ||
| } | ||
|
|
||
|
|
||
| void OuterDerivationGoal::addWantedOutputs(const OutputsSpec & outputs) | ||
| { | ||
| /* If we already want all outputs, there is nothing to do. */ | ||
| auto newWanted = wantedOutputs.union_(outputs); | ||
| bool needRestart = !newWanted.isSubsetOf(wantedOutputs); | ||
| wantedOutputs = newWanted; | ||
|
|
||
| if (!needRestart) return; | ||
|
|
||
| if (!optDrvPath) | ||
| // haven't started steps where the outputs matter yet | ||
| return; | ||
| worker.makeDerivationGoal(*optDrvPath, outputs, buildMode); | ||
| } | ||
|
|
||
|
|
||
| void OuterDerivationGoal::getDerivation() | ||
| { | ||
| trace("outer init"); | ||
|
|
||
| /* The first thing to do is to make sure that the derivation | ||
| exists. If it doesn't, it may be created through a | ||
| substitute. */ | ||
| { | ||
| if (buildMode != bmNormal) goto load; | ||
| auto drvReq2 = tryResolveDerivedPath(worker.store, *drvReq); | ||
| auto drvPathP = std::get_if<DerivedPath::Opaque>(&drvReq2); | ||
| if (!drvPathP) goto load; | ||
| auto & drvPath = drvPathP->path; | ||
| if (!worker.evalStore.isValidPath(drvPath) && !worker.store.isValidPath(drvPath)) | ||
| goto load; | ||
|
|
||
| trace(fmt("already have drv '%s' for '%s', can go straight to building", | ||
| worker.store.printStorePath(drvPath), | ||
| drvReq->to_string(worker.store))); | ||
|
|
||
| loadAndBuildDerivation(); | ||
| return; | ||
| } | ||
|
|
||
| load: | ||
| trace("need to obtain drv we want to build"); | ||
|
|
||
| addWaitee(worker.makeGoal(DerivedPath::fromSingle(*drvReq))); | ||
|
|
||
| state = &OuterDerivationGoal::loadAndBuildDerivation; | ||
| if (waitees.empty()) work(); | ||
| } | ||
|
|
||
|
|
||
| void OuterDerivationGoal::loadAndBuildDerivation() | ||
| { | ||
| trace("outer load and build derivation"); | ||
|
|
||
| if (nrFailed != 0) { | ||
| amDone(ecFailed, Error("cannot build missing derivation '%s'", drvReq->to_string(worker.store))); | ||
| return; | ||
| } | ||
|
|
||
| StorePath drvPath = resolveDerivedPath(worker.store, *drvReq); | ||
| /* Build this step! */ | ||
| concreteDrvGoal = worker.makeDerivationGoal(drvPath, wantedOutputs, buildMode); | ||
| addWaitee(upcast_goal(concreteDrvGoal)); | ||
| state = &OuterDerivationGoal::buildDone; | ||
| optDrvPath = std::move(drvPath); | ||
| if (waitees.empty()) work(); | ||
| } | ||
|
|
||
|
|
||
| void OuterDerivationGoal::buildDone() | ||
| { | ||
| trace("outer build done"); | ||
|
|
||
| buildResult = upcast_goal(concreteDrvGoal)->getBuildResult(DerivedPath::Built { | ||
| .drvPath = drvReq, | ||
| .outputs = wantedOutputs, | ||
| }); | ||
|
|
||
| if (buildResult.success()) | ||
| amDone(ecSuccess); | ||
| else | ||
| amDone(ecFailed, Error("building '%s' failed", drvReq->to_string(worker.store))); | ||
| } | ||
|
|
||
|
|
||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,65 @@ | ||
| #pragma once | ||
|
|
||
| #include "parsed-derivations.hh" | ||
| #include "lock.hh" | ||
| #include "store-api.hh" | ||
| #include "pathlocks.hh" | ||
| #include "goal.hh" | ||
|
|
||
| namespace nix { | ||
|
|
||
| struct DerivationGoal; | ||
|
|
||
| struct OuterDerivationGoal : public Goal | ||
| { | ||
| /* How to obtain a store path of the derivation to build. */ | ||
| ref<SingleDerivedPath> drvReq; | ||
|
|
||
| /* The path of the derivation, once obtained. */ | ||
| std::optional<StorePath> optDrvPath; | ||
|
|
||
| /* The goal for the corresponding concrete derivation */ | ||
| std::shared_ptr<DerivationGoal> concreteDrvGoal; | ||
|
|
||
| /* The specific outputs that we need to build. Empty means all of | ||
| them. */ | ||
| OutputsSpec wantedOutputs; | ||
|
|
||
| typedef void (OuterDerivationGoal::*GoalState)(); | ||
| GoalState state; | ||
|
|
||
| /* The final output paths of the build. | ||
| - For input-addressed derivations, always the precomputed paths | ||
| - For content-addressed derivations, calcuated from whatever the hash | ||
| ends up being. (Note that fixed outputs derivations that produce the | ||
| "wrong" output still install that data under its true content-address.) | ||
| */ | ||
| OutputPathMap finalOutputs; | ||
|
|
||
| BuildMode buildMode; | ||
|
|
||
| OuterDerivationGoal(ref<SingleDerivedPath> drvReq, | ||
| const OutputsSpec & wantedOutputs, Worker & worker, | ||
| BuildMode buildMode = bmNormal); | ||
| virtual ~OuterDerivationGoal(); | ||
|
|
||
| void timedOut(Error && ex) override; | ||
|
|
||
| std::string key() override; | ||
|
|
||
| void work() override; | ||
|
|
||
| /* Add wanted outputs to an already existing derivation goal. */ | ||
| void addWantedOutputs(const OutputsSpec & outputs); | ||
|
|
||
| /* The states. */ | ||
| void getDerivation(); | ||
| void loadAndBuildDerivation(); | ||
| void buildDone(); | ||
|
|
||
| JobCategory jobCategory() override { return JobCategory::Build; }; | ||
| }; | ||
|
|
||
| } |
Oops, something went wrong.