From 67f4f9f21db916894020072b13ff7d61558fc73d Mon Sep 17 00:00:00 2001 From: savil <676452+savil@users.noreply.github.com> Date: Thu, 1 Feb 2024 11:15:33 -0800 Subject: [PATCH] [nix profile] Changes to support format changes from nix 2.20 (#1770) ## Summary The latest nix version (2.20) changed how the nix profile output is represented: From the [release notes](https://nixos.org/manual/nix/stable/release-notes/rl-2.20): > nix profile now allows referring to elements by human-readable names https://github.com/NixOS/nix/pull/8678 > [nix profile](https://nixos.org/manual/nix/stable/command-ref/new-cli/nix3-profile) now uses names to refer to installed packages when running [list](https://nixos.org/manual/nix/stable/command-ref/new-cli/nix3-profile-list), [remove](https://nixos.org/manual/nix/stable/command-ref/new-cli/nix3-profile-remove) or [upgrade](https://nixos.org/manual/nix/stable/command-ref/new-cli/nix3-profile-upgrade) as opposed to indices. Profile element names are generated when a package is installed and remain the same until the package is removed. > Warning: The manifest.nix file used to record the contents of profiles has changed. Nix will automatically upgrade profiles to the new version when you modify the profile. After that, the profile can no longer be used by older versions of Nix. and for `nix search`: > Disallow empty search regex in nix search [#9481](https://github.com/NixOS/nix/pull/9481) > [nix search](https://nixos.org/manual/nix/stable/command-ref/new-cli/nix3-search) now requires a search regex to be passed. To show all packages, use ^. TODOs: - [x] update `nix.readManifest` to handle the new format - [x] `nix search` requires a regex to be passed - [x] manually test on nix < 2.20 on devbox.sh to verify the older nix still works Fixes #1767 ## How was it tested? CICD should pass Installed nix 2.20.1 locally and am using Devbox with it to add, remove packages and run scripts and shell. verified flake updating works: 1. `examples/flakes/remote`. Did `devbox shell`, dropped the `v0.43.1` from process-compose flake, did `devbox update`, and verified that `process-compose` now had the latest version (IIRC `0.80+`) 2. `examples/flakes/php`. Did `devbox shell`, edited the flake to drop `ds` and did `devbox update`. Verified no `ds` in `php -m | grep ds` --- .github/workflows/cli-tests.yaml | 5 +- internal/nix/nixprofile/item.go | 21 +++++++-- internal/nix/nixprofile/profile.go | 61 +++++++++++++++++-------- internal/nix/nixprofile/profile_test.go | 15 +++--- internal/nix/nixprofile/upgrade.go | 6 +-- internal/nix/profiles.go | 37 +++++++++++++-- internal/nix/search.go | 9 +++- internal/nix/upgrade.go | 5 +- 8 files changed, 118 insertions(+), 41 deletions(-) diff --git a/.github/workflows/cli-tests.yaml b/.github/workflows/cli-tests.yaml index d0a306fecfe..c2efdf61737 100644 --- a/.github/workflows/cli-tests.yaml +++ b/.github/workflows/cli-tests.yaml @@ -129,8 +129,9 @@ jobs: run-project-tests: ["project-tests", "project-tests-off"] # Run tests on: # 1. the oldest supported nix version (which is 2.9.0? But determinate-systems installer has 2.12.0) - # 2. latest nix version - nix-version: ["2.12.0", "2.19.2"] + # 2. nix 2.19.2: version before nix profile changes + # 2. latest nix version (note, 2.20.1 introduced a new profile change) + nix-version: ["2.12.0", "2.19.2", "2.20.1"] exclude: - is-main: "not-main" os: "${{ inputs.run-mac-tests && 'dummy' || 'macos-latest' }}" diff --git a/internal/nix/nixprofile/item.go b/internal/nix/nixprofile/item.go index af14ad0b4f0..adab885ae75 100644 --- a/internal/nix/nixprofile/item.go +++ b/internal/nix/nixprofile/item.go @@ -16,6 +16,11 @@ type NixProfileListItem struct { // invocations of nix profile remove and nix profile upgrade. index int + // name of the package + // nix 2.20 introduced a new format for the output of nix profile list, which includes the package name. + // This field is used instead of index for `list`, `remove` and `upgrade` subcommands of `nix profile`. + name string + // The original ("unlocked") flake reference and output attribute path used at installation time. // NOTE that this will be empty if the package was added to the nix profile via store path. unlockedReference string @@ -74,10 +79,10 @@ func (i *NixProfileListItem) addedByStorePath() bool { return i.unlockedReference == "" } -// String serializes the NixProfileListItem back into the format printed by `nix profile list` +// String serializes the NixProfileListItem for debuggability func (i *NixProfileListItem) String() string { - return fmt.Sprintf("{%d %s %s %s}", - i.index, + return fmt.Sprintf("{nameOrIndex:%s unlockedRef:%s lockedRef:%s, nixStorePaths:%s}", + i.NameOrIndex(), i.unlockedReference, i.lockedReference, i.nixStorePaths, @@ -87,3 +92,13 @@ func (i *NixProfileListItem) String() string { func (i *NixProfileListItem) StorePaths() []string { return i.nixStorePaths } + +// NameOrIndex is a helper method to get the name of the package if it exists, or the index if it doesn't. +// `nix profile` subcommands `list`, `remove`, and `upgrade` use either name (nix >= 2.20) or index (nix < 2.20) +// to identify the package. +func (i *NixProfileListItem) NameOrIndex() string { + if i.name != "" { + return i.name + } + return fmt.Sprintf("%d", i.index) +} diff --git a/internal/nix/nixprofile/profile.go b/internal/nix/nixprofile/profile.go index d293667c586..eeb0e710fc9 100644 --- a/internal/nix/nixprofile/profile.go +++ b/internal/nix/nixprofile/profile.go @@ -42,17 +42,38 @@ func ProfileListItems( URL string `json:"url"` } type ProfileListOutput struct { - Elements []ProfileListElement `json:"elements"` - Version int `json:"version"` + Elements map[string]ProfileListElement `json:"elements"` + Version int `json:"version"` } + // Modern nix profiles: nix >= 2.20 var structOutput ProfileListOutput - if err := json.Unmarshal([]byte(output), &structOutput); err != nil { - return nil, err + if err := json.Unmarshal([]byte(output), &structOutput); err == nil { + items := []*NixProfileListItem{} + for name, element := range structOutput.Elements { + items = append(items, &NixProfileListItem{ + name: name, + unlockedReference: lo.Ternary(element.OriginalURL != "", element.OriginalURL+"#"+element.AttrPath, ""), + lockedReference: lo.Ternary(element.URL != "", element.URL+"#"+element.AttrPath, ""), + nixStorePaths: element.StorePaths, + }) + } + return items, nil } + // Fall back to trying format for nix < version 2.20 + // ProfileListOutputJSONLegacy is for parsing `nix profile list --json` in nix < version 2.20 + // that relied on index instead of name for each package installed. + type ProfileListOutputJSONLegacy struct { + Elements []ProfileListElement `json:"elements"` + Version int `json:"version"` + } + var structOutput2 ProfileListOutputJSONLegacy + if err := json.Unmarshal([]byte(output), &structOutput2); err != nil { + return nil, err + } items := []*NixProfileListItem{} - for index, element := range structOutput.Elements { + for index, element := range structOutput2.Elements { items = append(items, &NixProfileListItem{ index: index, unlockedReference: lo.Ternary(element.OriginalURL != "", element.OriginalURL+"#"+element.AttrPath, ""), @@ -60,6 +81,7 @@ func ProfileListItems( nixStorePaths: element.StorePaths, }) } + return items, nil } @@ -88,7 +110,7 @@ func profileListLegacy( if line == "" { continue } - item, err := parseNixProfileListItem(line) + item, err := parseNixProfileListItemLegacy(line) if err != nil { return nil, err } @@ -98,7 +120,7 @@ func profileListLegacy( return items, nil } -type ProfileListIndexArgs struct { +type ProfileListNameOrIndexArgs struct { // For performance, you can reuse the same list in multiple operations if you // are confident index has not changed. Items []*NixProfileListItem @@ -108,21 +130,21 @@ type ProfileListIndexArgs struct { ProfileDir string } -// ProfileListIndex returns the index of args.Package in the nix profile specified by args.ProfileDir, -// or -1 if it's not found. Callers can pass in args.Items to avoid having to call `nix-profile list` again. -func ProfileListIndex(args *ProfileListIndexArgs) (int, error) { +// ProfileListNameOrIndex returns the name or index of args.Package in the nix profile specified by args.ProfileDir, +// or nix.ErrPackageNotFound if it's not found. Callers can pass in args.Items to avoid having to call `nix-profile list` again. +func ProfileListNameOrIndex(args *ProfileListNameOrIndexArgs) (string, error) { var err error items := args.Items if items == nil { items, err = ProfileListItems(args.Writer, args.ProfileDir) if err != nil { - return -1, err + return "", err } } inCache, err := args.Package.IsInBinaryCache() if err != nil { - return -1, err + return "", err } if !inCache && args.Package.IsDevboxPackage { @@ -131,28 +153,29 @@ func ProfileListIndex(args *ProfileListIndexArgs) (int, error) { // of an existing profile item. ref, err := args.Package.NormalizedDevboxPackageReference() if err != nil { - return -1, errors.Wrapf(err, "failed to get installable for %s", args.Package.String()) + return "", errors.Wrapf(err, "failed to get installable for %s", args.Package.String()) } for _, item := range items { if ref == item.unlockedReference { - return item.index, nil + return item.NameOrIndex(), nil } } - return -1, errors.Wrap(nix.ErrPackageNotFound, args.Package.String()) + return "", errors.Wrap(nix.ErrPackageNotFound, args.Package.String()) } for _, item := range items { if item.Matches(args.Package, args.Lockfile) { - return item.index, nil + return item.NameOrIndex(), nil } } - return -1, errors.Wrap(nix.ErrPackageNotFound, args.Package.String()) + return "", errors.Wrap(nix.ErrPackageNotFound, args.Package.String()) } -// parseNixProfileListItem reads each line of output (from `nix profile list`) and converts +// parseNixProfileListItemLegacy reads each line of output (from `nix profile list`) and converts // into a golang struct. Refer to NixProfileListItem struct definition for explanation of each field. -func parseNixProfileListItem(line string) (*NixProfileListItem, error) { +// NOTE: this API is for legacy nix. Newer nix versions use --json output. +func parseNixProfileListItemLegacy(line string) (*NixProfileListItem, error) { scanner := bufio.NewScanner(strings.NewReader(line)) scanner.Split(bufio.ScanWords) diff --git a/internal/nix/nixprofile/profile_test.go b/internal/nix/nixprofile/profile_test.go index 426db621b1f..2fdb1c471c1 100644 --- a/internal/nix/nixprofile/profile_test.go +++ b/internal/nix/nixprofile/profile_test.go @@ -15,7 +15,10 @@ type expectedTestData struct { packageName string } -func TestNixProfileListItem(t *testing.T) { +// TestNixProfileListItemLegacy tests the parsing of legacy nix profile list items. +// It only applies to much older nix versions. Newer nix versions rely on the --json output +// instead parsing the legacy output. +func TestNixProfileListItemLegacy(t *testing.T) { testCases := map[string]struct { line string expected expectedTestData @@ -49,10 +52,10 @@ func TestNixProfileListItem(t *testing.T) { ), expected: expectedTestData{ item: &NixProfileListItem{ - 2, - "github:NixOS/nixpkgs/52e3e80afff4b16ccb7c52e9f0f5220552f03d04#legacyPackages.x86_64-darwin.python39Packages.numpy", - "github:NixOS/nixpkgs/52e3e80afff4b16ccb7c52e9f0f5220552f03d04#legacyPackages.x86_64-darwin.python39Packages.numpy", - []string{"/nix/store/qly36iy1p4q1h5p4rcbvsn3ll0zsd9pd-python3.9-numpy-1.23.3"}, + index: 2, + unlockedReference: "github:NixOS/nixpkgs/52e3e80afff4b16ccb7c52e9f0f5220552f03d04#legacyPackages.x86_64-darwin.python39Packages.numpy", + lockedReference: "github:NixOS/nixpkgs/52e3e80afff4b16ccb7c52e9f0f5220552f03d04#legacyPackages.x86_64-darwin.python39Packages.numpy", + nixStorePaths: []string{"/nix/store/qly36iy1p4q1h5p4rcbvsn3ll0zsd9pd-python3.9-numpy-1.23.3"}, }, attrPath: "legacyPackages.x86_64-darwin.python39Packages.numpy", packageName: "python39Packages.numpy", @@ -68,7 +71,7 @@ func TestNixProfileListItem(t *testing.T) { } func testItem(t *testing.T, line string, expected expectedTestData) { - item, err := parseNixProfileListItem(line) + item, err := parseNixProfileListItemLegacy(line) if err != nil { t.Fatalf("unexpected error %v", err) } diff --git a/internal/nix/nixprofile/upgrade.go b/internal/nix/nixprofile/upgrade.go index 2200e56b911..e905cb67f06 100644 --- a/internal/nix/nixprofile/upgrade.go +++ b/internal/nix/nixprofile/upgrade.go @@ -12,8 +12,8 @@ import ( ) func ProfileUpgrade(ProfileDir string, pkg *devpkg.Package, lock *lock.File) error { - idx, err := ProfileListIndex( - &ProfileListIndexArgs{ + nameOrIndex, err := ProfileListNameOrIndex( + &ProfileListNameOrIndexArgs{ Lockfile: lock, Writer: os.Stderr, Package: pkg, @@ -24,5 +24,5 @@ func ProfileUpgrade(ProfileDir string, pkg *devpkg.Package, lock *lock.File) err return err } - return nix.ProfileUpgrade(ProfileDir, idx) + return nix.ProfileUpgrade(ProfileDir, nameOrIndex) } diff --git a/internal/nix/profiles.go b/internal/nix/profiles.go index 5ca87e3d5f6..2589305c071 100644 --- a/internal/nix/profiles.go +++ b/internal/nix/profiles.go @@ -94,8 +94,8 @@ func ProfileRemove(profilePath string, indexes ...string) error { type manifest struct { Elements []struct { - Priority int `json:"priority"` - } `json:"elements"` + Priority int + } } func readManifest(profilePath string) (manifest, error) { @@ -107,8 +107,37 @@ func readManifest(profilePath string) (manifest, error) { return manifest{}, err } - var m manifest - return m, json.Unmarshal(data, &m) + type manifestModern struct { + Elements map[string]struct { + Priority int `json:"priority"` + } `json:"elements"` + } + var modernMani manifestModern + if err := json.Unmarshal(data, &modernMani); err == nil { + // Convert to the result format + result := manifest{} + for _, e := range modernMani.Elements { + result.Elements = append(result.Elements, struct{ Priority int }{e.Priority}) + } + return result, nil + } + + type manifestLegacy struct { + Elements []struct { + Priority int `json:"priority"` + } `json:"elements"` + } + var legacyMani manifestLegacy + if err := json.Unmarshal(data, &legacyMani); err != nil { + return manifest{}, err + } + + // Convert to the result format + result := manifest{} + for _, e := range legacyMani.Elements { + result.Elements = append(result.Elements, struct{ Priority int }{e.Priority}) + } + return result, nil } const DefaultPriority = 5 diff --git a/internal/nix/search.go b/internal/nix/search.go index a8ea01f0463..b7d1c33e53f 100644 --- a/internal/nix/search.go +++ b/internal/nix/search.go @@ -98,7 +98,8 @@ func searchSystem(url, system string) (map[string]*Info, error) { _ = EnsureNixpkgsPrefetched(writer, hash) } - cmd := exec.Command("nix", "search", "--json", url) + // The `^` is added to indicate we want to show all packages + cmd := exec.Command("nix", "search", url, "^" /*regex*/, "--json") cmd.Args = append(cmd.Args, ExperimentalFlags()...) if system != "" { cmd.Args = append(cmd.Args, "--system", system) @@ -106,7 +107,13 @@ func searchSystem(url, system string) (map[string]*Info, error) { debug.Log("running command: %s\n", cmd) out, err := cmd.Output() if err != nil { + if exitErr := (&exec.ExitError{}); errors.As(err, &exitErr) { + err = fmt.Errorf("nix search exit code: %d, stderr: %s, original error: %w", exitErr.ExitCode(), exitErr.Stderr, err) + } + // for now, assume all errors are invalid packages. + // TODO: check the error string for "did not find attribute" and + // return ErrPackageNotFound only for that case. return nil, fmt.Errorf("error searching for pkg %s: %w", url, err) } return parseSearchResults(out), nil diff --git a/internal/nix/upgrade.go b/internal/nix/upgrade.go index 1f303ce2801..f09a8ca7810 100644 --- a/internal/nix/upgrade.go +++ b/internal/nix/upgrade.go @@ -4,7 +4,6 @@ package nix import ( - "fmt" "os" "os/exec" @@ -13,11 +12,11 @@ import ( "go.jetpack.io/devbox/internal/vercheck" ) -func ProfileUpgrade(ProfileDir string, idx int) error { +func ProfileUpgrade(ProfileDir, indexOrName string) error { cmd := command( "profile", "upgrade", "--profile", ProfileDir, - fmt.Sprintf("%d", idx), + indexOrName, ) out, err := cmd.CombinedOutput() if err != nil {