Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

dotnet: add override mechanism for nuget packages #339953

Merged
merged 19 commits into from
Sep 7, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
19 commits
Select commit Hold shift + click to select a range
13bd100
update-dotnet-lockfiles.nix: disable aliases
corngood Aug 26, 2024
9a14832
dotnet: fix fetch-deps in paket projects
corngood Sep 5, 2024
e530139
buildDotnetModule: fix tool manifest detection
corngood Sep 5, 2024
e496425
buildDotnetModule: set TMPDIR in fetch-deps
corngood Aug 24, 2024
04686b6
dotnet: add .nupkg.metadata to VMR packages
corngood Sep 5, 2024
e788823
mkNugetDeps: unpack to source/
corngood Sep 4, 2024
2d43ecc
buildDotnetModule: clean-up stdenv attributes
corngood Sep 4, 2024
d477621
mkNugetSource/Deps: move into dotnetCorePackages
corngood Sep 4, 2024
7ea78aa
dotnetCorePackages.fetchNupkg: split from mkNugetDeps
corngood Sep 4, 2024
32ccfdc
dotnetCorePackages.fetchNupkg: run missing hooks
corngood Sep 4, 2024
1f6cd35
dotnetCorePackages.fetchNupkg: add override mechanism
corngood Sep 4, 2024
9a0be2f
dotnetCorePackages.fetchNupkg: override skiasharp to find fontconfig
corngood Sep 4, 2024
5b31367
buildDotnetModule: move into dotnetCorePackages
corngood Sep 4, 2024
e5cb52b
buildDotnetGlobalTool: move into dotnetCorePackages
corngood Sep 6, 2024
2563bd7
dotnet-sdk-setup-hook: make shell vars local
corngood Sep 5, 2024
6c5a9e5
dotnet-sdk-setup-hook: use temporary symlink dir for NUGET_FALLBACK_P…
corngood Sep 5, 2024
cc9c59c
buildDotnetModule: use individual dependencies for nugetDeps
corngood Sep 4, 2024
e6c700e
dotnetCorePackages.fetchNupkg: override avalonia.x11 to find native libs
corngood Sep 4, 2024
14c908c
tests.dotnet: add test for supported nugetDeps values
corngood Sep 6, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion doc/languages-frameworks/dotnet.section.md
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,7 @@ The `dotnetCorePackages.sdk` contains both a runtime and the full sdk of a given
To package Dotnet applications, you can use `buildDotnetModule`. This has similar arguments to `stdenv.mkDerivation`, with the following additions:

* `projectFile` is used for specifying the dotnet project file, relative to the source root. These have `.sln` (entire solution) or `.csproj` (single project) file extensions. This can be a list of multiple projects as well. When omitted, will attempt to find and build the solution (`.sln`). If running into problems, make sure to set it to a file (or a list of files) with the `.csproj` extension - building applications as entire solutions is not fully supported by the .NET CLI.
* `nugetDeps` takes either a path to a `deps.nix` file, or a derivation. The `deps.nix` file can be generated using the script attached to `passthru.fetch-deps`. If the argument is a derivation, it will be used directly and assume it has the same output as `mkNugetDeps`.
* `nugetDeps` takes either a path to a `deps.nix` file, or a derivation. The `deps.nix` file can be generated using the script attached to `passthru.fetch-deps`. For compatibility, if the argument is a list of derivations, they will be added to `buildInputs`.
::: {.note}
For more detail about managing the `deps.nix` file, see [Generating and updating NuGet dependencies](#generating-and-updating-nuget-dependencies)
:::
Expand Down
2 changes: 1 addition & 1 deletion maintainers/scripts/update-dotnet-lockfiles.nix
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
update scripts.
*/
let
pkgs = import ../.. {};
pkgs = import ../.. { config.allowAliases = false; };

inherit (pkgs) lib;

Expand Down
7 changes: 0 additions & 7 deletions pkgs/applications/audio/openutau/default.nix
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,6 @@
, dotnetCorePackages
, dbus
, fontconfig
, libICE
, libSM
, libX11
, portaudio
}:

Expand All @@ -32,10 +29,6 @@ buildDotnetModule rec {

runtimeDeps = [
dbus
fontconfig
libICE
libSM
libX11
portaudio
];

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,22 +2,16 @@
, fetchFromGitHub
, buildDotnetModule
, dotnetCorePackages
, libX11
, libICE
, libSM
, fontconfig
, libsecret
, git
, git-credential-manager
, gnupg
, pass
, testers
, withGuiSupport ? true
, withLibsecretSupport ? true
, withGpgSupport ? true
}:

assert withLibsecretSupport -> withGuiSupport;
buildDotnetModule rec {
pname = "git-credential-manager";
version = "2.5.1";
Expand All @@ -36,9 +30,8 @@ buildDotnetModule rec {
dotnetInstallFlags = [ "--framework" "net8.0" ];
executables = [ "git-credential-manager" ];

runtimeDeps = [ fontconfig ]
++ lib.optionals withGuiSupport [ libX11 libICE libSM ]
corngood marked this conversation as resolved.
Show resolved Hide resolved
++ lib.optional withLibsecretSupport libsecret;
runtimeDeps =
lib.optional withLibsecretSupport libsecret;
makeWrapperArgs = [
"--prefix PATH : ${lib.makeBinPath ([ git ] ++ lib.optionals withGpgSupport [ gnupg pass ])}"
];
Expand Down
18 changes: 10 additions & 8 deletions pkgs/build-support/dotnet/build-dotnet-global-tool/default.nix
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
{ buildDotnetModule, emptyDirectory, mkNugetDeps, dotnet-sdk }:
{ buildDotnetModule, emptyDirectory, fetchNupkg, dotnet-sdk }:

{ pname
, version
Expand All @@ -23,13 +23,15 @@ buildDotnetModule (args // {

src = emptyDirectory;

nugetDeps = mkNugetDeps {
name = pname;
nugetDeps = { fetchNuGet }: [
(fetchNuGet { pname = nugetName; inherit version; sha256 = nugetSha256; hash = nugetHash; })
] ++ (nugetDeps fetchNuGet);
installable = true;
};
buildInputs = [
(fetchNupkg {
pname = nugetName;
inherit version;
sha256 = nugetSha256;
hash = nugetHash;
installable = true;
})
];

dotnetGlobalTool = true;

Expand Down
64 changes: 39 additions & 25 deletions pkgs/build-support/dotnet/build-dotnet-module/default.nix
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
writeShellScript,
makeWrapper,
dotnetCorePackages,
mkNugetDeps,
fetchNupkg,
nuget-to-nix,
cacert,
unzip,
Expand All @@ -18,10 +18,7 @@ let
transformArgs =
finalAttrs:
{
name ? "${args.pname}-${args.version}",
pname ? name,
enableParallelBuilding ? true,
doCheck ? false,
# Flags to pass to `makeWrapper`. This is done to avoid double wrapping.
makeWrapperArgs ? [ ],

Expand Down Expand Up @@ -115,22 +112,36 @@ let
_nugetDeps =
if (nugetDeps != null) then
if lib.isDerivation nugetDeps then
[ nugetDeps ]
else if lib.isList nugetDeps then
nugetDeps
else
mkNugetDeps {
inherit name;
sourceFile = nugetDeps;
}
assert (lib.isPath nugetDeps);
callPackage nugetDeps { fetchNuGet = fetchNupkg; }
else
throw "Defining the `nugetDeps` attribute is required, as to lock the NuGet dependencies. This file can be generated by running the `passthru.fetch-deps` script.";
[ ];

nugetDepsFile = _nugetDeps.sourceFile;
nugetDepsFile = if lib.isPath nugetDeps then nugetDeps else null;

inherit (dotnetCorePackages) systemToDotnetRid;
in
# Not all args need to be passed through to mkDerivation
# TODO: We should probably filter out even more attrs
removeAttrs args [ "nugetDeps" ]
removeAttrs args [
"nugetDeps"
"installPath"
"executables"
"projectFile"
"projectReferences"
"runtimeDeps"
"runtimeId"
"disabledTests"
"testProjectFile"
"buildType"
"selfContainedBuild"
"useDotnet"
"useAppHost"
]
Comment on lines +130 to +144
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

❤️

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is a breaking change unfortunately. roslyn-ls for example, was using $projectFile, and needed to be changed to $dotnetProjectFiles.

It's a little unintuitive that something like:

projectFile = "foo";
buildPhase = "echo $projectFile";

doesn't work. @tie you did #339335. What do you think?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can I ask, what's the motivation for prefixing dotnet-specific derivation args in the first place?

I struggle to see how name conflicts would happen, for instance.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

For buildDotnetModules I agree. I probably wouldn't have prefixed them.

For hooks like dotnet-sdk-setup-hook I'm trying to be careful about collisions, because I want that stuff to work well in a mixed build environment. Ideally I'd like it to be as easy to build a dotnet project with stdenv[NoCC] as with buildDotnetModule.

Copy link
Contributor

@MattSturgeon MattSturgeon Sep 6, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

For hooks like dotnet-sdk-setup-hook I'm trying to be careful about collisions, because I want that stuff to work well in a mixed build environment.

Makes sense. 👍

This is a breaking change unfortunately. roslyn-ls for example, was using $projectFile, and needed to be changed to $dotnetProjectFiles.

From my perspective, we have the verbose/prefixed names for anything handled by the dotnet hook, since that hook may be used with arbitrary stdenv derivations and therefore should avoid potential name collisions.

The arguments used by buildDotnetModule could therefore almost be thought of as aliases, since the verbosity is redundant in that context.

Looking at it from this perspective, I don't think it is such a bad thing for buildDotnetModule to pass all its arguments to stdenv.

On reflection I think this is the least surprising behaviour, even if it does feel a little messy internally; an end user (packager) would likely expect any argument they've passed to a builder function to be also made available in stdenv.

Ideally I'd like it to be as easy to build a dotnet project with stdenv[NoCC] as with buildDotnetModule.

That sounds great!

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

On reflection I think this is the least surprising behaviour, even if it does feel a little messy internally; an end user (packager) would likely expect any argument they've passed to a builder function to be also made available in stdenv.

One problem with that is sometimes, like with nugetDeps, there are negative consequences to including it.

Stripping them definitely feels wrong though. Like either it should be two separate attrset arguments, or one as an attribute in the other.

// {
dotnetInstallPath = installPath;
dotnetExecutables = executables;
Expand All @@ -145,9 +156,18 @@ let
dotnetRuntimeDeps = map lib.getLib runtimeDeps;
dotnetSelfContainedBuild = selfContainedBuild;
dotnetUseAppHost = useAppHost;
inherit useDotnetFromEnv;

inherit enableParallelBuilding;
inherit
enableParallelBuilding
dotnetRestoreFlags
dotnetBuildFlags
dotnetTestFlags
dotnetInstallFlags
dotnetPackFlags
dotnetFlags
packNupkg
useDotnetFromEnv
;

nativeBuildInputs = args.nativeBuildInputs or [ ] ++ [
dotnetConfigureHook
Expand All @@ -163,13 +183,7 @@ let
yq
];

buildInputs =
args.buildInputs or [ ]
++ [
dotnet-sdk.packages
_nugetDeps
]
++ projectReferences;
buildInputs = args.buildInputs or [ ] ++ [ dotnet-sdk.packages ] ++ _nugetDeps ++ projectReferences;

# Parse the version attr into a format acceptable for the Version msbuild property
# The actual version attr is saved in InformationalVersion, which accepts an arbitrary string
Expand Down Expand Up @@ -213,13 +227,13 @@ let
{
nugetDeps = _nugetDeps;
}
// lib.optionalAttrs (!lib.isDerivation nugetDeps) {
// lib.optionalAttrs (nugetDeps == null || lib.isPath nugetDeps) {
fetch-deps =
let
pkg = finalAttrs.finalPackage.overrideAttrs (
old:
{
buildInputs = lib.remove _nugetDeps old.buildInputs;
buildInputs = lib.subtractLists _nugetDeps old.buildInputs;
keepNugetConfig = true;
}
// lib.optionalAttrs (runtimeId == null) {
Expand All @@ -238,15 +252,15 @@ let
# Note that toString is necessary here as it results in the path at
# eval time (i.e. to the file in your local Nixpkgs checkout) rather
# than the Nix store path of the path after it's been imported.
if lib.isPath nugetDepsFile && !lib.hasPrefix "${builtins.storeDir}/" (toString nugetDepsFile) then
if lib.isPath nugetDeps && !lib.isStorePath nugetDepsFile then
toString nugetDepsFile
else
''$(mktemp -t "${pname}-deps-XXXXXX.nix")'';
''$(mktemp -t "${finalAttrs.pname ? finalAttrs.finalPackage.name}-deps-XXXXXX.nix")'';
nugetToNix = (nuget-to-nix.override { inherit dotnet-sdk; });
};

in
writeShellScript "${name}-fetch-deps" ''
writeShellScript "${finalAttrs.finalPackage.name}-fetch-deps" ''
NIX_BUILD_SHELL="${runtimeShell}" exec ${nix}/bin/nix-shell \
--pure --run 'source "${innerScript}"' "${drv}"
'';
Expand Down
2 changes: 2 additions & 0 deletions pkgs/build-support/dotnet/build-dotnet-module/fetch-deps.sh
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@ tmp=$(mktemp -d)
trap 'chmod -R +w "$tmp" && rm -fr "$tmp"' EXIT

HOME=$tmp/.home
export TMPDIR="$tmp/.tmp"
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It'd be nice to restore this after the script is over so people don't have to restart their shells

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's not really meant to be sourced in a user shell. It's only meant to be called via nix-shell in the package-fetch-deps script.

I actually would have preferred to make it all work in one file with nix-shell shebangs, but I couldn't find a way to pass the derivation path, etc.

mkdir "$HOME" "$TMPDIR"
cd "$tmp"

phases="
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ dotnetConfigureHook() {
fi
done

if [[ -f .config/dotnet-tools.json || -f .dotnet-tools.json ]]; then
if [[ -f .config/dotnet-tools.json || -f dotnet-tools.json ]]; then
dotnet tool restore
fi

Expand Down
80 changes: 80 additions & 0 deletions pkgs/build-support/dotnet/fetch-nupkg/default.nix
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
{
symlinkJoin,
fetchurl,
stdenvNoCC,
lib,
unzip,
patchNupkgs,
nugetPackageHook,
callPackage,
overrides ? callPackage ./overrides.nix { },
}:
{
pname,
version,
sha256 ? "",
hash ? "",
url ? "https://www.nuget.org/api/v2/package/${pname}/${version}",
installable ? false,
}:
let
package = stdenvNoCC.mkDerivation rec {
inherit pname version;

src = fetchurl {
name = "${pname}.${version}.nupkg";
# There is no need to verify whether both sha256 and hash are
# valid here, because nuget-to-nix does not generate a deps.nix
# containing both.
inherit
url
sha256
hash
version
;
};

nativeBuildInputs = [
unzip
patchNupkgs
nugetPackageHook
];

unpackPhase = ''
runHook preUnpack

unzip -nqd source $src
chmod -R +rw source
cd source

runHook postUnpack
'';

prePatch = ''
shopt -s nullglob
local dir
for dir in tools runtimes/*/native; do
[[ ! -d "$dir" ]] || chmod -R +x "$dir"
done
rm -rf .signature.p7s
'';

installPhase = ''
runHook preInstall

dir=$out/share/nuget/packages/${lib.toLower pname}/${lib.toLower version}
mkdir -p $dir
cp -r . $dir
echo {} > "$dir"/.nupkg.metadata

runHook postInstall
'';

preFixup = ''
patch-nupkgs $out/share/nuget/packages
'';

createInstallableNugetSource = installable;
};
in
overrides.${pname} or lib.id package
55 changes: 55 additions & 0 deletions pkgs/build-support/dotnet/fetch-nupkg/overrides.nix
corngood marked this conversation as resolved.
Show resolved Hide resolved
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
{
autoPatchelfHook,
dotnetCorePackages,
fontconfig,
lib,
libICE,
libSM,
libX11,
stdenv,
writeText,
}:
{
# e.g.
# "Package.Id" =
# package:
# package.overrideAttrs (old: {
# buildInputs = old.buildInputs or [ ] ++ [ hello ];
# });

"Avalonia.X11" =
package:
package.overrideAttrs (
old:
lib.optionalAttrs (!stdenv.isDarwin) {
setupHook = writeText "setupHook.sh" ''
prependToVar dotnetRuntimeDeps \
"${lib.getLib libICE}" \
"${lib.getLib libSM}" \
"${lib.getLib libX11}"
'';
}
);

"SkiaSharp.NativeAssets.Linux" =
package:
package.overrideAttrs (
old:
lib.optionalAttrs stdenv.isLinux {
nativeBuildInputs = old.nativeBuildInputs or [ ] ++ [ autoPatchelfHook ];

buildInputs = old.buildInputs or [ ] ++ [ fontconfig ];

preInstall =
old.preInstall or ""
+ ''
cd runtimes
for platform in *; do
[[ $platform == "${dotnetCorePackages.systemToDotnetRid stdenv.hostPlatform.system}" ]] ||
rm -r "$platform"
done
cd - >/dev/null
'';
}
);
}
Loading