From 7c251e2b5fda2b42c3ff2983fdcc9eac82f2ced6 Mon Sep 17 00:00:00 2001 From: K900 Date: Sun, 26 Jan 2025 09:43:44 +0300 Subject: [PATCH] Revert "pkgs/top-level: make package sets composable" --- lib/modules.nix | 2 - nixos/modules/misc/nixpkgs.nix | 15 +-- pkgs/test/top-level/stage.nix | 128 -------------------- pkgs/top-level/default.nix | 18 +-- pkgs/top-level/stage.nix | 206 ++++++++++++++++++++------------- 5 files changed, 139 insertions(+), 230 deletions(-) delete mode 100644 pkgs/test/top-level/stage.nix diff --git a/lib/modules.nix b/lib/modules.nix index 8ec850b59193f..79b8f25c2f434 100644 --- a/lib/modules.nix +++ b/lib/modules.nix @@ -844,8 +844,6 @@ let in warnDeprecation opt // { value = addErrorContext "while evaluating the option `${showOption loc}':" value; - # raw value before "apply" above - rawValue = addErrorContext "while evaluating the option `${showOption loc}':" res.mergedValue; inherit (res.defsFinal') highestPrio; definitions = map (def: def.value) res.defsFinal; files = map (def: def.file) res.defsFinal; diff --git a/nixos/modules/misc/nixpkgs.nix b/nixos/modules/misc/nixpkgs.nix index f99f75af7659b..e1c705eef3b5a 100644 --- a/nixos/modules/misc/nixpkgs.nix +++ b/nixos/modules/misc/nixpkgs.nix @@ -70,24 +70,19 @@ let ++ lib.optional (opt.localSystem.highestPrio < (lib.mkOptionDefault { }).priority) opt.localSystem ++ lib.optional (opt.crossSystem.highestPrio < (lib.mkOptionDefault { }).priority) opt.crossSystem; - # pkgs/top-level/default.nix takes great strides to pass the *original* localSystem/crossSystem args - # on to nixpkgsFun to create package sets like pkgsStatic, pkgsMusl. This is to be able to infer default - # values again. Since cfg.xxxPlatform and cfg.xxxSystem are elaborated via apply, those can't be passed - # directly. Instead we use the rawValue before the apply/elaboration step, via opt.xxx.rawValue. defaultPkgs = if opt.hostPlatform.isDefined then let - # This compares elaborated systems on purpose, **not** using rawValue. isCross = cfg.buildPlatform != cfg.hostPlatform; systemArgs = if isCross then { - localSystem = opt.buildPlatform.rawValue; - crossSystem = opt.hostPlatform.rawValue; + localSystem = cfg.buildPlatform; + crossSystem = cfg.hostPlatform; } else { - localSystem = opt.hostPlatform.rawValue; + localSystem = cfg.hostPlatform; }; in import ../../.. ( @@ -101,9 +96,9 @@ let inherit (cfg) config overlays + localSystem + crossSystem ; - localSystem = opt.localSystem.rawValue; - crossSystem = opt.crossSystem.rawValue; }; finalPkgs = if opt.pkgs.isDefined then cfg.pkgs.appendOverlays cfg.overlays else defaultPkgs; diff --git a/pkgs/test/top-level/stage.nix b/pkgs/test/top-level/stage.nix deleted file mode 100644 index 449a36a425abe..0000000000000 --- a/pkgs/test/top-level/stage.nix +++ /dev/null @@ -1,128 +0,0 @@ -# run like this: -# nix-build pkgs/test/top-level/stage.nix -{ - localSystem ? { - system = builtins.currentSystem; - }, -}: - -with import ../../top-level { inherit localSystem; }; - -let - # To silence platform specific evaluation errors - discardEvaluationErrors = e: (builtins.tryEval e).success -> e; - - # Basic test for idempotency of the package set, i.e: - # Applying the same package set twice should work and - # not change anything. - isIdempotent = set: discardEvaluationErrors (pkgs.${set}.stdenv == pkgs.${set}.${set}.stdenv); - - # Some package sets should be noops in certain circumstances. - # This is very similar to the idempotency test, but not going - # via the super' overlay. - isNoop = - parent: child: - discardEvaluationErrors ( - (lib.getAttrFromPath parent pkgs).stdenv == (lib.getAttrFromPath parent pkgs).${child}.stdenv - ); - - allMuslExamples = builtins.attrNames ( - lib.filterAttrs (_: system: lib.hasSuffix "-musl" system.config) lib.systems.examples - ); - - allLLVMExamples = builtins.attrNames ( - lib.filterAttrs (_: system: system.useLLVM or false) lib.systems.examples - ); - - # A package set should only change specific configuration, but needs - # to keep all other configuration from previous layers in place. - # Each package set has one or more key characteristics for which we - # test here. Those should be kept, even when applying the "set" package - # set. - isComposable = - set: - ( - # Can't compose two different libcs... - builtins.elem set [ "pkgsLLVMLibc" ] - || discardEvaluationErrors ( - pkgsCross.mingwW64.${set}.stdenv.hostPlatform.config == "x86_64-w64-mingw32" - ) - ) - && ( - # Can't compose two different libcs... - builtins.elem set [ "pkgsLLVMLibc" ] - || discardEvaluationErrors (pkgsCross.mingwW64.${set}.stdenv.hostPlatform.libc == "msvcrt") - ) - && discardEvaluationErrors (pkgsCross.ppc64-musl.${set}.stdenv.hostPlatform.gcc.abi == "elfv2") - && discardEvaluationErrors ( - builtins.elem "trivialautovarinit" pkgs.pkgsExtraHardening.${set}.stdenv.cc.defaultHardeningFlags - ) - && discardEvaluationErrors (pkgs.pkgsLLVM.${set}.stdenv.hostPlatform.useLLVM) - && ( - # Can't compose two different libcs... - builtins.elem set [ - "pkgsMusl" - "pkgsStatic" - ] - || discardEvaluationErrors (pkgs.pkgsLLVMLibc.${set}.stdenv.hostPlatform.isLLVMLibc) - ) - && discardEvaluationErrors (pkgs.pkgsArocc.${set}.stdenv.hostPlatform.useArocc) - && discardEvaluationErrors (pkgs.pkgsZig.${set}.stdenv.hostPlatform.useZig) - && discardEvaluationErrors (pkgs.pkgsLinux.${set}.stdenv.buildPlatform.isLinux) - && ( - # Can't compose two different libcs... - builtins.elem set [ "pkgsLLVMLibc" ] - || discardEvaluationErrors (pkgs.pkgsMusl.${set}.stdenv.hostPlatform.isMusl) - ) - && discardEvaluationErrors (pkgs.pkgsStatic.${set}.stdenv.hostPlatform.isStatic) - && discardEvaluationErrors (pkgs.pkgsi686Linux.${set}.stdenv.hostPlatform.isx86_32) - && discardEvaluationErrors (pkgs.pkgsx86_64Darwin.${set}.stdenv.hostPlatform.isx86_64); -in - -# Appends same defaultHardeningFlags again on each .pkgsExtraHardening - thus not idempotent. -# assert isIdempotent "pkgsExtraHardening"; -# TODO: Remove the isDarwin condition, which currently results in infinite recursion. -# Also see https://github.com/NixOS/nixpkgs/pull/330567#discussion_r1894653309 -assert (stdenv.hostPlatform.isDarwin || isIdempotent "pkgsLLVM"); -# TODO: This currently results in infinite recursion, even on Linux -# assert isIdempotent "pkgsLLVMLibc"; -assert isIdempotent "pkgsArocc"; -assert isIdempotent "pkgsZig"; -assert isIdempotent "pkgsLinux"; -assert isIdempotent "pkgsMusl"; -assert isIdempotent "pkgsStatic"; -assert isIdempotent "pkgsi686Linux"; -assert isIdempotent "pkgsx86_64Darwin"; - -assert isNoop [ "pkgsStatic" ] "pkgsMusl"; -assert lib.all (sys: isNoop [ "pkgsCross" sys ] "pkgsMusl") allMuslExamples; -assert lib.all (sys: isNoop [ "pkgsCross" sys ] "pkgsLLVM") allLLVMExamples; - -assert isComposable "pkgsExtraHardening"; -assert isComposable "pkgsLLVM"; -# TODO: Results in infinite recursion -# assert isComposable "pkgsLLVMLibc"; -assert isComposable "pkgsArocc"; -# TODO: unexpected argument 'bintools' - uncomment once https://github.com/NixOS/nixpkgs/pull/331011 is done -# assert isComposable "pkgsZig"; -assert isComposable "pkgsMusl"; -assert isComposable "pkgsStatic"; -assert isComposable "pkgsi686Linux"; - -# Special cases regarding buildPlatform vs hostPlatform -assert discardEvaluationErrors (pkgsCross.gnu64.pkgsMusl.stdenv.hostPlatform.isMusl); -assert discardEvaluationErrors (pkgsCross.gnu64.pkgsi686Linux.stdenv.hostPlatform.isx86_32); -assert discardEvaluationErrors (pkgsCross.mingwW64.pkgsLinux.stdenv.hostPlatform.isLinux); -assert discardEvaluationErrors ( - pkgsCross.aarch64-darwin.pkgsx86_64Darwin.stdenv.hostPlatform.isx86_64 -); - -# pkgsCross should keep upper cross settings -assert discardEvaluationErrors ( - with pkgsStatic.pkgsCross.gnu64.stdenv.hostPlatform; isGnu && isStatic -); -assert discardEvaluationErrors ( - with pkgsLLVM.pkgsCross.musl64.stdenv.hostPlatform; isMusl && useLLVM -); - -emptyFile diff --git a/pkgs/top-level/default.nix b/pkgs/top-level/default.nix index 6787793dbdfc7..fd443ac773ffb 100644 --- a/pkgs/top-level/default.nix +++ b/pkgs/top-level/default.nix @@ -122,18 +122,22 @@ in let config = lib.showWarnings configEval.config.warnings configEval.config; # A few packages make a new package set to draw their dependencies from. - # Rather than give `all-packages.nix` all the arguments to this function, - # even ones that don't concern it, we give it this function to "re-call" - # nixpkgs, inheriting whatever arguments it doesn't explicitly provide. This - # way, `all-packages.nix` doesn't know more than it needs to. + # (Currently to get a cross tool chain, or forced-i686 package.) Rather than + # give `all-packages.nix` all the arguments to this function, even ones that + # don't concern it, we give it this function to "re-call" nixpkgs, inheriting + # whatever arguments it doesn't explicitly provide. This way, + # `all-packages.nix` doesn't know more than it needs too. # # It's OK that `args` doesn't include default arguments from this file: # they'll be deterministically inferred. In fact we must *not* include them, # because it's important that if some parameter which affects the default is # substituted with a different argument, the default is re-inferred. # - # To put this in concrete terms, we want the provided non-native `localSystem` - # and `crossSystem` arguments to affect the stdenv chosen. + # To put this in concrete terms, this function is basically just used today to + # use package for a different platform for the current platform (namely cross + # compiling toolchains and 32-bit packages on x86_64). In both those cases we + # want the provided non-native `localSystem` argument to affect the stdenv + # chosen. # # NB!!! This thing gets its `config` argument from `args`, i.e. it's actually # `config0`. It is important to keep it to `config0` format (as opposed to the @@ -142,7 +146,7 @@ in let # via `evalModules` is not idempotent. In other words, if you add `config` to # `newArgs`, expect strange very hard to debug errors! (Yes, I'm speaking from # experience here.) - nixpkgsFun = f0: import ./. (args // f0 args); + nixpkgsFun = newArgs: import ./. (args // newArgs); # Partially apply some arguments for building bootstraping stage pkgs # sets. Only apply arguments which no stdenv would want to override. diff --git a/pkgs/top-level/stage.nix b/pkgs/top-level/stage.nix index 49034a7384f4f..64bc2308f6da7 100644 --- a/pkgs/top-level/stage.nix +++ b/pkgs/top-level/stage.nix @@ -180,111 +180,168 @@ let ((config.packageOverrides or (super: {})) super); # Convenience attributes for instantitating package sets. Each of - # these will instantiate a new version of allPackages. - otherPackageSets = let - mkPkgs = name: fn: nixpkgsFun (prevArgs: let nixpkgsArgs = fn prevArgs; in nixpkgsArgs // { - overlays = [ - (self': super': { - "${name}" = super'; - }) - ] ++ nixpkgsArgs.overlays or [] ++ prevArgs.overlays or []; - }); - # This is always cross. - mkCrossPkgs = name: crossAttrs: mkPkgs name (prevArgs: { - crossSystem = - (lib.systems.systemToAttrs (lib.defaultTo prevArgs.localSystem prevArgs.crossSystem or null)) // crossAttrs; - }); - # This is only cross when we are already cross, otherwise local. - # For the case of "native cross", i.e. pkgsCross.gnu64 on a x86_64-linux system, we need to adjust **both** - # localSystem **and** crossSystem, otherwise they're out of sync. - mkHybridPkgs = name: hybridAttrs: mkPkgs name (prevArgs: let - newSystem = (lib.systems.systemToAttrs (lib.defaultTo prevArgs.localSystem prevArgs.crossSystem or null)) // hybridAttrs; - in { crossSystem = newSystem; } - // lib.optionalAttrs (stdenv.hostPlatform == stdenv.buildPlatform) { localSystem = newSystem; } - ); - in self: super: { + # these will instantiate a new version of allPackages. Currently the + # following package sets are provided: + # + # - pkgsCross. where system is a member of lib.systems.examples + # - pkgsMusl + # - pkgsi686Linux + otherPackageSets = self: super: { # This maps each entry in lib.systems.examples to its own package # set. Each of these will contain all packages cross compiled for # that target system. For instance, pkgsCross.raspberryPi.hello, # will refer to the "hello" package built for the ARM6-based # Raspberry Pi. pkgsCross = lib.mapAttrs (n: crossSystem: - nixpkgsFun (prevArgs: { crossSystem = (lib.systems.systemToAttrs (lib.defaultTo { } prevArgs.crossSystem or null)) // crossSystem; })) + nixpkgsFun { inherit crossSystem; }) lib.systems.examples; - # Bootstrap a cross stdenv using the LLVM toolchain. - # This is currently not possible when compiling natively. - pkgsLLVM = mkCrossPkgs "pkgsLLVM" { - useLLVM = true; - linker = "lld"; + pkgsLLVM = nixpkgsFun { + overlays = [ + (self': super': { + pkgsLLVM = super'; + }) + ] ++ overlays; + # Bootstrap a cross stdenv using the LLVM toolchain. + # This is currently not possible when compiling natively, + # so we don't need to check hostPlatform != buildPlatform. + crossSystem = stdenv.hostPlatform // { + useLLVM = true; + linker = "lld"; + }; }; - # Bootstrap a cross stdenv using LLVM libc. - # This is currently not possible when compiling natively. - pkgsLLVMLibc = mkCrossPkgs "pkgsLLVMLibc" { - config = lib.systems.parse.tripleFromSystem (makeLLVMParsedPlatform stdenv.hostPlatform.parsed); - libc = "llvm"; + pkgsLLVMLibc = nixpkgsFun { + overlays = [ (self': super': { + pkgsLLVMLibc = super'; + })] ++ overlays; + # Bootstrap a cross stdenv using LLVM libc. + # This is currently not possible when compiling natively, + # so we don't need to check hostPlatform != buildPlatform. + crossSystem = stdenv.hostPlatform // { + config = lib.systems.parse.tripleFromSystem (makeLLVMParsedPlatform stdenv.hostPlatform.parsed); + libc = "llvm"; + }; }; - # Bootstrap a cross stdenv using the Aro C compiler. - # This is currently not possible when compiling natively. - pkgsArocc = mkCrossPkgs "pkgsArocc" { - useArocc = true; - linker = "lld"; + pkgsArocc = nixpkgsFun { + overlays = [ + (self': super': { + pkgsArocc = super'; + }) + ] ++ overlays; + # Bootstrap a cross stdenv using the Aro C compiler. + # This is currently not possible when compiling natively, + # so we don't need to check hostPlatform != buildPlatform. + crossSystem = stdenv.hostPlatform // { + useArocc = true; + linker = "lld"; + }; }; - # Bootstrap a cross stdenv using the Zig toolchain. - # This is currently not possible when compiling natively. - pkgsZig = mkCrossPkgs "pkgsZig" { - useZig = true; - linker = "lld"; + pkgsZig = nixpkgsFun { + overlays = [ + (self': super': { + pkgsZig = super'; + }) + ] ++ overlays; + # Bootstrap a cross stdenv using the Zig toolchain. + # This is currently not possible when compiling natively, + # so we don't need to check hostPlatform != buildPlatform. + crossSystem = stdenv.hostPlatform // { + useZig = true; + linker = "lld"; + }; }; # All packages built with the Musl libc. This will override the # default GNU libc on Linux systems. Non-Linux systems are not # supported. 32-bit is also not supported. - pkgsMusl = if stdenv.hostPlatform.isLinux && stdenv.buildPlatform.is64bit then mkHybridPkgs "pkgsMusl" { - config = lib.systems.parse.tripleFromSystem (makeMuslParsedPlatform stdenv.hostPlatform.parsed); + pkgsMusl = if stdenv.hostPlatform.isLinux && stdenv.buildPlatform.is64bit then nixpkgsFun { + overlays = [ (self': super': { + pkgsMusl = super'; + })] ++ overlays; + ${if stdenv.hostPlatform == stdenv.buildPlatform + then "localSystem" else "crossSystem"} = { + config = lib.systems.parse.tripleFromSystem (makeMuslParsedPlatform stdenv.hostPlatform.parsed); + }; } else throw "Musl libc only supports 64-bit Linux systems."; # All packages built for i686 Linux. # Used by wine, firefox with debugging version of Flash, ... - pkgsi686Linux = if stdenv.hostPlatform.isLinux && stdenv.hostPlatform.isx86 then mkHybridPkgs "pkgsi686Linux" { - config = lib.systems.parse.tripleFromSystem (stdenv.hostPlatform.parsed // { - cpu = lib.systems.parse.cpuTypes.i686; - }); + pkgsi686Linux = if stdenv.hostPlatform.isLinux && stdenv.hostPlatform.isx86 then nixpkgsFun { + overlays = [ (self': super': { + pkgsi686Linux = super'; + })] ++ overlays; + ${if stdenv.hostPlatform == stdenv.buildPlatform + then "localSystem" else "crossSystem"} = { + config = lib.systems.parse.tripleFromSystem (stdenv.hostPlatform.parsed // { + cpu = lib.systems.parse.cpuTypes.i686; + }); + }; } else throw "i686 Linux package set can only be used with the x86 family."; # x86_64-darwin packages for aarch64-darwin users to use with Rosetta for incompatible packages - pkgsx86_64Darwin = if stdenv.hostPlatform.isDarwin then mkHybridPkgs "pkgsx86_64Darwin" { - config = lib.systems.parse.tripleFromSystem (stdenv.hostPlatform.parsed // { - cpu = lib.systems.parse.cpuTypes.x86_64; - }); + pkgsx86_64Darwin = if stdenv.hostPlatform.isDarwin then nixpkgsFun { + overlays = [ (self': super': { + pkgsx86_64Darwin = super'; + })] ++ overlays; + localSystem = { + config = lib.systems.parse.tripleFromSystem (stdenv.hostPlatform.parsed // { + cpu = lib.systems.parse.cpuTypes.x86_64; + }); + }; } else throw "x86_64 Darwin package set can only be used on Darwin systems."; # If already linux: the same package set unaltered - # Otherwise, return a linux package set for the current cpu architecture string. + # Otherwise, return a natively built linux package set for the current cpu architecture string. # (ABI and other details will be set to the default for the cpu/os pair) pkgsLinux = if stdenv.hostPlatform.isLinux then self - else mkHybridPkgs "pkgsLinux" { - config = lib.systems.parse.tripleFromSystem (lib.systems.elaborate "${stdenv.hostPlatform.parsed.cpu.name}-linux").parsed; + else nixpkgsFun { + localSystem = lib.systems.elaborate "${stdenv.hostPlatform.parsed.cpu.name}-linux"; }; + # Extend the package set with zero or more overlays. This preserves + # preexisting overlays. Prefer to initialize with the right overlays + # in one go when calling Nixpkgs, for performance and simplicity. + appendOverlays = extraOverlays: + if extraOverlays == [] + then self + else nixpkgsFun { overlays = args.overlays ++ extraOverlays; }; + + # NOTE: each call to extend causes a full nixpkgs rebuild, adding ~130MB + # of allocations. DO NOT USE THIS IN NIXPKGS. + # + # Extend the package set with a single overlay. This preserves + # preexisting overlays. Prefer to initialize with the right overlays + # in one go when calling Nixpkgs, for performance and simplicity. + # Prefer appendOverlays if used repeatedly. + extend = f: self.appendOverlays [f]; + # Fully static packages. # Currently uses Musl on Linux (couldn’t get static glibc to work). - pkgsStatic = mkCrossPkgs "pkgsStatic" ({ - isStatic = true; - } // lib.optionalAttrs stdenv.hostPlatform.isLinux { - config = lib.systems.parse.tripleFromSystem (makeMuslParsedPlatform stdenv.hostPlatform.parsed); - } // lib.optionalAttrs (stdenv.hostPlatform.system == "powerpc64-linux") { - gcc = { abi = "elfv2"; } // stdenv.hostPlatform.gcc or {}; + pkgsStatic = nixpkgsFun ({ + overlays = [ (self': super': { + pkgsStatic = super'; + })] ++ overlays; + crossSystem = { + isStatic = true; + config = lib.systems.parse.tripleFromSystem ( + if stdenv.hostPlatform.isLinux + then makeMuslParsedPlatform stdenv.hostPlatform.parsed + else stdenv.hostPlatform.parsed + ); + gcc = lib.optionalAttrs (stdenv.hostPlatform.system == "powerpc64-linux") { abi = "elfv2"; } // + stdenv.hostPlatform.gcc or {}; + }; }); - pkgsExtraHardening = mkPkgs "pkgsExtraHardening" (_: { + pkgsExtraHardening = nixpkgsFun { overlays = [ (self': super': { + pkgsExtraHardening = super'; stdenv = super'.withDefaultHardeningFlags ( super'.stdenv.cc.defaultHardeningFlags ++ [ "shadowstack" @@ -303,25 +360,8 @@ let pcre-cpp = super'.pcre-cpp.override { enableJit = false; }; pcre16 = super'.pcre16.override { enableJit = false; }; }) - ]; - }); - - # Extend the package set with zero or more overlays. This preserves - # preexisting overlays. Prefer to initialize with the right overlays - # in one go when calling Nixpkgs, for performance and simplicity. - appendOverlays = extraOverlays: - if extraOverlays == [] - then self - else nixpkgsFun (prevArgs: { overlays = prevArgs.overlays ++ extraOverlays; }); - - # NOTE: each call to extend causes a full nixpkgs rebuild, adding ~130MB - # of allocations. DO NOT USE THIS IN NIXPKGS. - # - # Extend the package set with a single overlay. This preserves - # preexisting overlays. Prefer to initialize with the right overlays - # in one go when calling Nixpkgs, for performance and simplicity. - # Prefer appendOverlays if used repeatedly. - extend = f: self.appendOverlays [f]; + ] ++ overlays; + }; }; # The complete chain of package set builders, applied from top to bottom.