Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add option disallow-copy-paths to track down unnecessary copying #11746

Draft
wants to merge 4 commits into
base: master
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions src/libexpr/eval-settings.hh
Original file line number Diff line number Diff line change
Expand Up @@ -247,6 +247,17 @@ struct EvalSettings : Config

This option can be enabled by setting `NIX_ABORT_ON_WARN=1` in the environment.
)"};

Setting<std::set<std::string>> disallowCopyPaths{this, {}, "disallow-copy-paths",
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

std::string?

R"(
A list of paths that are not allowed to be copied.

This is useful for finding expressions which copy sources, which can slow down evaluation.
You may find copied sources by running `nix` commands with increased verbosity, such as `nix build -vvvv 2>&1 | grep /nix/store`.
After identifying one more more paths, run `nix build --option disallow-copy-paths /nix/store/... --show-trace` to find the expression that copies the path, or add `--debugger`.

A filtering copy is always allowed, such as `builtins.filterSource` or `builtins.path { filter = ...; }`.
)"};
};

/**
Expand Down
23 changes: 21 additions & 2 deletions src/libexpr/eval.cc
Original file line number Diff line number Diff line change
Expand Up @@ -2358,7 +2358,6 @@ StorePath EvalState::copyPathToStore(NixStringContext & context, const SourcePat
? *dstPathCached
: [&]() {
auto dstPath = fetchToStore(
*store,
path.resolveSymlinks(),
settings.readOnlyMode ? FetchMode::DryRun : FetchMode::Copy,
path.baseName(),
Expand All @@ -2377,6 +2376,26 @@ StorePath EvalState::copyPathToStore(NixStringContext & context, const SourcePat
return dstPath;
}

void EvalState::checkDisallowCopyPath(const SourcePath & path) {
if (path.accessor == rootFS && settings.disallowCopyPaths.get().contains(path.path.abs())) {
error<EvalError>("not allowed to copy '%1%' due to option '%2%'", path.path.abs(), settings.disallowCopyPaths.name).debugThrow();
}
}

StorePath EvalState::fetchToStore(
const SourcePath & path,
FetchMode mode,
std::string_view name,
ContentAddressMethod method,
PathFilter * filter,
RepairFlag repair)
{
if (!filter)
checkDisallowCopyPath(path);
return ::nix::fetchToStore(*store, path, mode, name, method, filter, repair);
}



SourcePath EvalState::coerceToPath(const PosIdx pos, Value & v, NixStringContext & context, std::string_view errorCtx)
{
Expand Down Expand Up @@ -3055,7 +3074,7 @@ std::optional<std::string> EvalState::resolveLookupPathPath(const LookupPath::Pa
store,
fetchSettings,
EvalSettings::resolvePseudoUrl(value));
auto storePath = fetchToStore(*store, SourcePath(accessor), FetchMode::Copy);
auto storePath = fetchToStore(SourcePath(accessor), FetchMode::Copy);
return finish(store->toRealPath(storePath));
} catch (Error & e) {
logWarning({
Expand Down
12 changes: 12 additions & 0 deletions src/libexpr/eval.hh
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@

#include "attr-set.hh"
#include "eval-error.hh"
#include "fetch-to-store.hh"
#include "types.hh"
#include "value.hh"
#include "nixexpr.hh"
Expand Down Expand Up @@ -804,6 +805,17 @@ public:

DocComment getDocCommentForPos(PosIdx pos);

StorePath fetchToStore(
const SourcePath & path,
FetchMode mode,
std::string_view name = "source",
ContentAddressMethod method = ContentAddressMethod::Raw::NixArchive,
PathFilter * filter = nullptr,
RepairFlag repair = NoRepair);

/** Throws if path occurs in the disallow-copy-path option. */
void checkDisallowCopyPath(const SourcePath & path);

private:

/**
Expand Down
9 changes: 6 additions & 3 deletions src/libexpr/primops.cc
Original file line number Diff line number Diff line change
Expand Up @@ -2473,8 +2473,7 @@ static void addPath(
{}));

if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) {
auto dstPath = fetchToStore(
*state.store,
auto dstPath = state.fetchToStore(
path.resolveSymlinks(),
settings.readOnlyMode ? FetchMode::DryRun : FetchMode::Copy,
name,
Expand All @@ -2487,8 +2486,12 @@ static void addPath(
path
).atPos(pos).debugThrow();
state.allowAndSetStorePathString(dstPath, v);
} else
} else {
if (!filterFun)
state.checkDisallowCopyPath(path);

state.allowAndSetStorePathString(*expectedStorePath, v);
}
} catch (Error & e) {
e.addTrace(state.positions[pos], "while adding path '%s'", path);
throw;
Expand Down
50 changes: 50 additions & 0 deletions tests/functional/disallow-copy-paths.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
#!/usr/bin/env bash

source common.sh

clearStoreIfPossible

# shellcheck disable=SC2016
path="$(nix eval --raw --impure --expr '"${./disallow-copy-paths.sh}"')"

all_tests() {

# shellcheck disable=SC2016
expectStderr 1 nix-instantiate \
--disallow-copy-paths "$path" \
--expr --strict \
--argstr path "$path" \
'{ path }: "${/. + path}" + "bla bla"' \
"$@" \
| grepQuiet "error.*not allowed to copy.*$path.* due to option.*disallow-copy-paths"

# shellcheck disable=SC2016
expectStderr 1 nix-instantiate \
--disallow-copy-paths "$path" \
--expr --strict \
--argstr path "$path" \
"$@" \
'{ path }: builtins.path { path = /. + path; name = "source"; } + "bla bla"' \
| grepQuiet "error.*not allowed to copy.*$path.* due to option.*disallow-copy-paths"

# shellcheck disable=SC2016
expectStderr 1 nix-instantiate \
--disallow-copy-paths "$path" \
--expr --strict \
--argstr path "$path" \
"$@" \
'{ path }: builtins.path { path = path; name = "source"; } + "bla bla"' \
| grepQuiet "error.*not allowed to copy.*$path.* due to option.*disallow-copy-paths"

# shellcheck disable=SC2016
nix-instantiate \
--disallow-copy-paths "$path" \
--expr --eval --strict \
"$@" \
--argstr path "$path" \
'{ path }: builtins.path { path = path; name = "source"; filter = _: _: true; } + "bla bla"' \

}

all_tests
all_tests --readonly-mode
1 change: 1 addition & 0 deletions tests/functional/local.mk
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
nix_tests = \
test-infra.sh \
disallow-copy-paths.sh \
gc.sh \
nix-collect-garbage-d.sh \
remote-store.sh \
Expand Down
1 change: 1 addition & 0 deletions tests/functional/meson.build
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,7 @@ suites = [
'deps': [],
'tests': [
'test-infra.sh',
'disallow-copy-paths.sh',
'gc.sh',
'nix-collect-garbage-d.sh',
'remote-store.sh',
Expand Down
Loading