From b017a603f0884b7e28726cf1e4e006035cb8c274 Mon Sep 17 00:00:00 2001 From: Carolyn Van Slyck Date: Mon, 6 Feb 2023 08:51:27 -0600 Subject: [PATCH] Use same canonical json library as cnab-go The cnab-go library that we use to work with bundles switched to a canonical json library with support for numeric types a while back and I forgot to update cnab-to-oci at the same time. This updates cnab-to-oci to use the same library, github.com/cyberphone/json-canonicalization, and when possible just use the helper methods defined on cnab-go's Bundle type instead of doing the marshaling ourselves. Signed-off-by: Carolyn Van Slyck --- cmd/cnab-to-oci/fixup.go | 9 +++++---- cmd/cnab-to-oci/pull.go | 9 +++++++-- converter/types.go | 7 ++++--- go.mod | 5 ++--- go.sum | 9 ++++----- remotes/pull_test.go | 2 +- remotes/push_test.go | 22 +++++++++++----------- tests/helpers.go | 14 +++++++++++++- 8 files changed, 47 insertions(+), 30 deletions(-) diff --git a/cmd/cnab-to-oci/fixup.go b/cmd/cnab-to-oci/fixup.go index 12c5a2e..eedecd2 100644 --- a/cmd/cnab-to-oci/fixup.go +++ b/cmd/cnab-to-oci/fixup.go @@ -10,7 +10,6 @@ import ( containerdRemotes "github.com/containerd/containerd/remotes" "github.com/docker/cli/cli/config" "github.com/docker/distribution/reference" - "github.com/docker/go/canonical/json" "github.com/spf13/cobra" ) @@ -44,14 +43,16 @@ func fixupCmd() *cobra.Command { } func runFixup(opts fixupOptions) error { - var b bundle.Bundle bundleJSON, err := os.ReadFile(opts.input) if err != nil { return err } - if err := json.Unmarshal(bundleJSON, &b); err != nil { + + b, err := bundle.Unmarshal(bundleJSON) + if err != nil { return err } + ref, err := reference.ParseNormalizedNamed(opts.targetRef) if err != nil { return err @@ -63,7 +64,7 @@ func runFixup(opts fixupOptions) error { if opts.autoUpdateBundle { fixupOptions = append(fixupOptions, remotes.WithAutoBundleUpdate()) } - relocationMap, err := remotes.FixupBundle(context.Background(), &b, ref, createResolver(opts.insecureRegistries), fixupOptions...) + relocationMap, err := remotes.FixupBundle(context.Background(), b, ref, createResolver(opts.insecureRegistries), fixupOptions...) if err != nil { return err } diff --git a/cmd/cnab-to-oci/pull.go b/cmd/cnab-to-oci/pull.go index a8c3480..d3df1fc 100644 --- a/cmd/cnab-to-oci/pull.go +++ b/cmd/cnab-to-oci/pull.go @@ -2,12 +2,13 @@ package main import ( "context" + "encoding/json" "fmt" "os" "github.com/cnabio/cnab-to-oci/remotes" + "github.com/cyberphone/json-canonicalization/go/src/webpki.org/jsoncanonicalizer" "github.com/docker/distribution/reference" - "github.com/docker/go/canonical/json" "github.com/spf13/cobra" ) @@ -53,7 +54,11 @@ func runPull(opts pullOptions) error { } func writeOutput(file string, data interface{}) error { - bytes, err := json.MarshalCanonical(data) + plainJSON, err := json.Marshal(data) + if err != nil { + return err + } + bytes, err := jsoncanonicalizer.Transform(plainJSON) if err != nil { return err } diff --git a/converter/types.go b/converter/types.go index 1338359..e6a834e 100644 --- a/converter/types.go +++ b/converter/types.go @@ -1,11 +1,12 @@ package converter import ( + "encoding/json" + "github.com/cnabio/cnab-go/bundle" "github.com/docker/distribution" "github.com/docker/distribution/manifest/schema2" - "github.com/docker/go/canonical/json" - digest "github.com/opencontainers/go-digest" + "github.com/opencontainers/go-digest" ocischema "github.com/opencontainers/image-spec/specs-go" ocischemav1 "github.com/opencontainers/image-spec/specs-go/v1" ) @@ -26,7 +27,7 @@ type PreparedBundleConfig struct { // PrepareForPush serializes a bundle config, generates its image manifest, and its manifest descriptor func PrepareForPush(b *bundle.Bundle) (*PreparedBundleConfig, error) { - blob, err := json.MarshalCanonical(b) + blob, err := b.Marshal() if err != nil { return nil, err } diff --git a/go.mod b/go.mod index 31eed0d..bbf5a0e 100644 --- a/go.mod +++ b/go.mod @@ -5,10 +5,10 @@ go 1.19 require ( github.com/cnabio/cnab-go v0.25.1 github.com/containerd/containerd v1.6.18 + github.com/cyberphone/json-canonicalization v0.0.0-20220623050100-57a0ce2678a7 github.com/docker/cli v23.0.1+incompatible github.com/docker/distribution v2.8.1+incompatible github.com/docker/docker v23.0.1+incompatible - github.com/docker/go v1.5.1-1 github.com/hashicorp/go-multierror v1.1.1 github.com/opencontainers/go-digest v1.0.0 github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 @@ -24,7 +24,6 @@ require ( github.com/Microsoft/go-winio v0.5.2 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect - github.com/cyberphone/json-canonicalization v0.0.0-20210303052042-6bc126869bf4 // indirect github.com/docker/docker-credential-helpers v0.6.3 // indirect github.com/docker/go-connections v0.4.0 // indirect github.com/docker/go-metrics v0.0.1 // indirect @@ -49,7 +48,7 @@ require ( github.com/qri-io/jsonschema v0.2.2-0.20210831022256-780655b2ba0e // indirect github.com/sergi/go-diff v1.1.0 // indirect github.com/spf13/pflag v1.0.5 // indirect - github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect + github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect github.com/xeipuuv/gojsonschema v1.2.0 // indirect golang.org/x/net v0.7.0 // indirect diff --git a/go.sum b/go.sum index 2492c7f..89b88a8 100644 --- a/go.sum +++ b/go.sum @@ -85,8 +85,8 @@ github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSV github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creack/pty v1.1.11 h1:07n33Z8lZxZ2qwegKbObQohDhXDQxiMMz1NOUGYlesw= github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/cyberphone/json-canonicalization v0.0.0-20210303052042-6bc126869bf4 h1:7AjYfmq7AmviXsuZjV5DcE7PuhJ4dWMi8gLllpLVDQY= -github.com/cyberphone/json-canonicalization v0.0.0-20210303052042-6bc126869bf4/go.mod h1:uzvlm1mxhHkdfqitSA92i7Se+S9ksOn3a3qmv/kyOCw= +github.com/cyberphone/json-canonicalization v0.0.0-20220623050100-57a0ce2678a7 h1:vU+EP9ZuFUCYE0NYLwTSob+3LNEJATzNfP/DC7SWGWI= +github.com/cyberphone/json-canonicalization v0.0.0-20220623050100-57a0ce2678a7/go.mod h1:uzvlm1mxhHkdfqitSA92i7Se+S9ksOn3a3qmv/kyOCw= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -98,8 +98,6 @@ github.com/docker/docker v23.0.1+incompatible h1:vjgvJZxprTTE1A37nm+CLNAdwu6xZek github.com/docker/docker v23.0.1+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/docker-credential-helpers v0.6.3 h1:zI2p9+1NQYdnG6sMU26EX4aVGlqbInSQxQXLvzJ4RPQ= github.com/docker/docker-credential-helpers v0.6.3/go.mod h1:WRaJzqw3CTB9bk10avuGsjVBZsD05qeibJ1/TYlvc0Y= -github.com/docker/go v1.5.1-1 h1:hr4w35acWBPhGBXlzPoHpmZ/ygPjnmFVxGxxGnMyP7k= -github.com/docker/go v1.5.1-1/go.mod h1:CADgU4DSXK5QUlFslkQu2yW2TKzFZcXq/leZfM0UH5Q= github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= github.com/docker/go-metrics v0.0.1 h1:AgB/0SvBxihN0X8OR4SjsblXkbMvalQ8cjmtKQ2rQV8= @@ -355,8 +353,9 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= -github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= +github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo= +github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74= diff --git a/remotes/pull_test.go b/remotes/pull_test.go index 2f93a43..4741590 100644 --- a/remotes/pull_test.go +++ b/remotes/pull_test.go @@ -97,7 +97,7 @@ func ExamplePull() { fmt.Printf("\n") fmt.Println(string(buf)) // Output: - //{"actions":{"action-1":{"modifies":true}},"credentials":{"cred-1":{"env":"env-var","path":"/some/path"}},"custom":{"my-key":"my-value"},"definitions":{"output1Type":{"type":"string"},"param1Type":{"default":"hello","enum":["value1",true,1],"type":["string","boolean","number"]}},"description":"description","images":{"another-image":{"contentDigest":"sha256:d59a1aa7866258751a261bae525a1842c7ff0662d4f34a355d5f36826abc0342","description":"","image":"my.registry/namespace/another-image","imageType":"oci","mediaType":"application/vnd.oci.image.manifest.v1+json","size":507},"image-1":{"contentDigest":"sha256:d59a1aa7866258751a261bae525a1842c7ff0662d4f34a355d5f36826abc0341","description":"","image":"my.registry/namespace/image-1","imageType":"oci","mediaType":"application/vnd.oci.image.manifest.v1+json","size":507}},"invocationImages":[{"contentDigest":"sha256:d59a1aa7866258751a261bae525a1842c7ff0662d4f34a355d5f36826abc0343","image":"my.registry/namespace/my-app-invoc","imageType":"docker","mediaType":"application/vnd.docker.distribution.manifest.v2+json","size":506}],"keywords":["keyword1","keyword2"],"maintainers":[{"email":"docker@docker.com","name":"docker","url":"docker.com"}],"name":"my-app","outputs":{"output1":{"applyTo":["install"],"definition":"output1Type","description":"magic","path":"/cnab/app/outputs/magic"}},"parameters":{"param1":{"definition":"param1Type","destination":{"env":"env_var","path":"/some/path"}}},"schemaVersion":"v1.0.0","version":"0.1.0"} + //{"actions":{"action-1":{"modifies":true}},"credentials":{"cred-1":{"env":"env-var","path":"/some/path"}},"custom":{"my-key":"my-value"},"definitions":{"numberType":{"default":0.5,"type":"number"},"output1Type":{"type":"string"},"param1Type":{"default":"hello","enum":["value1",true,1],"type":["string","boolean","number"]}},"description":"description","images":{"another-image":{"contentDigest":"sha256:d59a1aa7866258751a261bae525a1842c7ff0662d4f34a355d5f36826abc0342","description":"","image":"my.registry/namespace/another-image","imageType":"oci","mediaType":"application/vnd.oci.image.manifest.v1+json","size":507},"image-1":{"contentDigest":"sha256:d59a1aa7866258751a261bae525a1842c7ff0662d4f34a355d5f36826abc0341","description":"","image":"my.registry/namespace/image-1","imageType":"oci","mediaType":"application/vnd.oci.image.manifest.v1+json","size":507}},"invocationImages":[{"contentDigest":"sha256:d59a1aa7866258751a261bae525a1842c7ff0662d4f34a355d5f36826abc0343","image":"my.registry/namespace/my-app-invoc","imageType":"docker","mediaType":"application/vnd.docker.distribution.manifest.v2+json","size":506}],"keywords":["keyword1","keyword2"],"maintainers":[{"email":"docker@docker.com","name":"docker","url":"docker.com"}],"name":"my-app","outputs":{"output1":{"applyTo":["install"],"definition":"output1Type","description":"magic","path":"/cnab/app/outputs/magic"}},"parameters":{"param1":{"definition":"param1Type","destination":{"env":"env_var","path":"/some/path"}},"param2":{"definition":"numberType","destination":{"env":"PARM2"}}},"schemaVersion":"v1.0.0","version":"0.1.0"} //{"my.registry/namespace/image-1":"my.registry/namespace/my-app@sha256:d59a1aa7866258751a261bae525a1842c7ff0662d4f34a355d5f36826abc0341","my.registry/namespace/my-app-invoc":"my.registry/namespace/my-app@sha256:d59a1aa7866258751a261bae525a1842c7ff0662d4f34a355d5f36826abc0341"} } diff --git a/remotes/push_test.go b/remotes/push_test.go index 70b32fa..67c93fb 100644 --- a/remotes/push_test.go +++ b/remotes/push_test.go @@ -2,6 +2,7 @@ package remotes import ( "context" + "encoding/json" "errors" "fmt" "strings" @@ -11,7 +12,6 @@ import ( "github.com/cnabio/cnab-to-oci/converter" "github.com/cnabio/cnab-to-oci/tests" "github.com/docker/distribution/reference" - "github.com/docker/go/canonical/json" ocischemav1 "github.com/opencontainers/image-spec/specs-go/v1" "gotest.tools/v3/assert" ) @@ -22,7 +22,7 @@ const ( "manifests": [ { "mediaType":"application/vnd.oci.image.manifest.v1+json", - "digest":"sha256:91ada58e30bc0aea920e69a9356085534ce31c50700e650c96b432fc3a5d1661", + "digest":"sha256:122a5dc186ec285488de9d25e99c96a11d3f7ff71c6e05a06c98e8627472a920", "size":189, "annotations":{ "io.cnab.manifest.type":"config" @@ -70,14 +70,14 @@ const ( "mediaType": "application/vnd.docker.distribution.manifest.v2+json", "config": { "mediaType": "application/vnd.docker.container.image.v1+json", - "size": 1484, - "digest": "sha256:e978bf2a3c57b25fc3bb7f1adccb6fa0f0284d38163d5faffce7bf5fca3f39b3" + "size": 1596, + "digest": "sha256:dbe3480b9cb300f389e8d02e4a682f9107772468feb6845f912dc8deed6d76fd" }, "layers": [ { "mediaType": "application/vnd.docker.container.image.v1+json", - "size": 1484, - "digest": "sha256:e978bf2a3c57b25fc3bb7f1adccb6fa0f0284d38163d5faffce7bf5fca3f39b3" + "size": 1596, + "digest": "sha256:dbe3480b9cb300f389e8d02e4a682f9107772468feb6845f912dc8deed6d76fd" } ] }` @@ -87,14 +87,14 @@ func TestPush(t *testing.T) { pusher := &mockPusher{} resolver := &mockResolver{pusher: pusher} b := tests.MakeTestBundle() - expectedBundleConfig, err := json.MarshalCanonical(b) - assert.NilError(t, err) + expectedBundleConfig, err := b.Marshal() + assert.NilError(t, err, "marshaling to canonical json failed") ref, err := reference.ParseNamed("my.registry/namespace/my-app:my-tag") - assert.NilError(t, err) + assert.NilError(t, err, "parsing the OCI reference failed") // push the bundle descriptor, err := Push(context.Background(), b, tests.MakeRelocationMap(), ref, resolver, true) - assert.NilError(t, err) + assert.NilError(t, err, "push failed") assert.Equal(t, tests.BundleDigest, descriptor.Digest) assert.Equal(t, len(resolver.pushedReferences), 3) assert.Equal(t, len(pusher.pushedDescriptors), 3) @@ -167,7 +167,7 @@ func ExamplePush() { // Output: // { // "mediaType": "application/vnd.oci.image.index.v1+json", - // "digest": "sha256:4cfae04045c6f0fd14330ab86dea9694fb19ce9602ba2da0af8c826fc0053631", + // "digest": "sha256:00043ca96c3c9470fc0f647c67613ddf500941556d1ecc14d75bc9b2834f66c3", // "size": 1360 // } } diff --git a/tests/helpers.go b/tests/helpers.go index a7a638a..673adc3 100644 --- a/tests/helpers.go +++ b/tests/helpers.go @@ -10,7 +10,7 @@ import ( ocischemav1 "github.com/opencontainers/image-spec/specs-go/v1" ) -const BundleDigest digest.Digest = "sha256:4cfae04045c6f0fd14330ab86dea9694fb19ce9602ba2da0af8c826fc0053631" +const BundleDigest digest.Digest = "sha256:00043ca96c3c9470fc0f647c67613ddf500941556d1ecc14d75bc9b2834f66c3" // MakeTestBundle creates a simple bundle for tests func MakeTestBundle() *bundle.Bundle { @@ -79,6 +79,12 @@ func MakeTestBundle() *bundle.Bundle { "output1Type": { Type: "string", }, + // Include a number as a regression test for marshaling with the proper canonical json library + // We should always be marshaling with a library that supports numeric types + "numberType": { + Type: "number", + Default: 0.5, + }, }, Parameters: map[string]bundle.Parameter{ "param1": { @@ -88,6 +94,12 @@ func MakeTestBundle() *bundle.Bundle { Path: "/some/path", }, }, + "param2": { + Definition: "numberType", + Destination: &bundle.Location{ + EnvironmentVariable: "PARM2", + }, + }, }, Outputs: map[string]bundle.Output{ "output1": {