diff --git a/cmd/soci/commands/create.go b/cmd/soci/commands/create.go index f9eb609bf..de563846b 100644 --- a/cmd/soci/commands/create.go +++ b/cmd/soci/commands/create.go @@ -86,10 +86,11 @@ var CreateCommand = cli.Command{ return err } - ctx, blobStore, err := store.NewContentStore(ctx, store.WithType(store.ContentStoreType(cliContext.GlobalString("content-store"))), store.WithNamespace(cliContext.GlobalString("namespace"))) + ctx, blobStore, done, err := store.NewContentStore(ctx, store.WithType(store.ContentStoreType(cliContext.GlobalString("content-store"))), store.WithNamespace(cliContext.GlobalString("namespace"))) if err != nil { return err } + defer done(ctx) ps, err := internal.GetPlatforms(ctx, cliContext, srcImg, cs) if err != nil { diff --git a/cmd/soci/commands/index/info.go b/cmd/soci/commands/index/info.go index 19c68ab02..f3c693a49 100644 --- a/cmd/soci/commands/index/info.go +++ b/cmd/soci/commands/index/info.go @@ -53,10 +53,13 @@ var infoCommand = cli.Command{ } ctx, cancel := context.WithTimeout(context.Background(), cliContext.GlobalDuration("timeout")) defer cancel() - ctx, store, err := store.NewContentStore(ctx, store.WithType(store.ContentStoreType(cliContext.GlobalString("content-store"))), store.WithNamespace(cliContext.GlobalString("namespace"))) + + ctx, store, done, err := store.NewContentStore(ctx, store.WithType(store.ContentStoreType(cliContext.GlobalString("content-store"))), store.WithNamespace(cliContext.GlobalString("namespace"))) if err != nil { return err } + defer done(ctx) + reader, err := store.Fetch(ctx, v1.Descriptor{Digest: digest}) if err != nil { return err diff --git a/cmd/soci/commands/index/rm.go b/cmd/soci/commands/index/rm.go index d44abd886..3bce3ed1c 100644 --- a/cmd/soci/commands/index/rm.go +++ b/cmd/soci/commands/index/rm.go @@ -47,10 +47,11 @@ var rmCommand = cli.Command{ } ctx := context.Background() - ctx, contentStore, err := store.NewContentStore(ctx, store.WithType(store.ContentStoreType(cliContext.GlobalString("content-store"))), store.WithNamespace(cliContext.GlobalString("namespace"))) + ctx, contentStore, done, err := store.NewContentStore(ctx, store.WithType(store.ContentStoreType(cliContext.GlobalString("content-store"))), store.WithNamespace(cliContext.GlobalString("namespace"))) if err != nil { return fmt.Errorf("cannot create local content store: %w", err) } + defer done(ctx) db, err := soci.NewDB(soci.ArtifactsDbPath()) if err != nil { diff --git a/cmd/soci/commands/push.go b/cmd/soci/commands/push.go index 06295f63b..2740bf927 100644 --- a/cmd/soci/commands/push.go +++ b/cmd/soci/commands/push.go @@ -132,10 +132,11 @@ if they are available in the snapshotter's local content store. }, nil } - ctx, src, err := store.NewContentStore(ctx, store.WithType(store.ContentStoreType(cliContext.GlobalString("content-store"))), store.WithNamespace(cliContext.GlobalString("namespace"))) + ctx, src, done, err := store.NewContentStore(ctx, store.WithType(store.ContentStoreType(cliContext.GlobalString("content-store"))), store.WithNamespace(cliContext.GlobalString("namespace"))) if err != nil { return fmt.Errorf("cannot create local content store: %w", err) } + defer done(ctx) dst.Client = authClient dst.PlainHTTP = cliContext.Bool("plain-http") diff --git a/cmd/soci/commands/rebuild_db.go b/cmd/soci/commands/rebuild_db.go index c588d3c88..092e5d36d 100644 --- a/cmd/soci/commands/rebuild_db.go +++ b/cmd/soci/commands/rebuild_db.go @@ -39,10 +39,11 @@ var RebuildDBCommand = cli.Command{ if err != nil { return err } - ctx, blobStore, err := store.NewContentStore(ctx, store.WithType(store.ContentStoreType(cliContext.GlobalString("content-store"))), store.WithNamespace(cliContext.GlobalString("namespace"))) + ctx, blobStore, done, err := store.NewContentStore(ctx, store.WithType(store.ContentStoreType(cliContext.GlobalString("content-store"))), store.WithNamespace(cliContext.GlobalString("namespace"))) if err != nil { return err } + defer done(ctx) contentStorePath, err := store.GetContentStorePath(store.ContentStoreType(cliContext.GlobalString("content-store"))) if err != nil { diff --git a/cmd/soci/commands/ztoc/get-file.go b/cmd/soci/commands/ztoc/get-file.go index 25927e1df..d04b4bf14 100644 --- a/cmd/soci/commands/ztoc/get-file.go +++ b/cmd/soci/commands/ztoc/get-file.go @@ -87,10 +87,11 @@ var getFileCommand = cli.Command{ } func getZtoc(ctx context.Context, cliContext *cli.Context, d digest.Digest) (*ztoc.Ztoc, error) { - ctx, blobStore, err := store.NewContentStore(ctx, store.WithType(store.ContentStoreType(cliContext.GlobalString("content-store"))), store.WithNamespace(cliContext.GlobalString("namespace"))) + ctx, blobStore, done, err := store.NewContentStore(ctx, store.WithType(store.ContentStoreType(cliContext.GlobalString("content-store"))), store.WithNamespace(cliContext.GlobalString("namespace"))) if err != nil { return nil, err } + defer done(ctx) reader, err := blobStore.Fetch(ctx, v1.Descriptor{Digest: d}) if err != nil { diff --git a/cmd/soci/commands/ztoc/info.go b/cmd/soci/commands/ztoc/info.go index 28d62fd84..1aca14993 100644 --- a/cmd/soci/commands/ztoc/info.go +++ b/cmd/soci/commands/ztoc/info.go @@ -72,10 +72,13 @@ var infoCommand = cli.Command{ } ctx, cancel := context.WithTimeout(context.Background(), cliContext.GlobalDuration("timeout")) defer cancel() - ctx, store, err := store.NewContentStore(ctx, store.WithType(store.ContentStoreType(cliContext.GlobalString("content-store"))), store.WithNamespace(cliContext.GlobalString("namespace"))) + + ctx, store, done, err := store.NewContentStore(ctx, store.WithType(store.ContentStoreType(cliContext.GlobalString("content-store"))), store.WithNamespace(cliContext.GlobalString("namespace"))) if err != nil { return err } + defer done(ctx) + reader, err := store.Fetch(ctx, v1.Descriptor{Digest: digest}) if err != nil { return err diff --git a/fs/fs.go b/fs/fs.go index 98e458b7a..ad4bd7cb8 100644 --- a/fs/fs.go +++ b/fs/fs.go @@ -141,10 +141,12 @@ func NewFilesystem(ctx context.Context, root string, cfg config.FSConfig, opts . return docker.ConfigureDefaultRegistries(docker.WithPlainHTTP(docker.MatchLocalhost))(refspec.Hostname()) }) } - ctx, store, err := store.NewContentStore(ctx, store.WithType(store.ContentStoreType(cfg.ContentStoreConfig.Type)), store.WithNamespace(cfg.ContentStoreConfig.Namespace)) + ctx, store, done, err := store.NewContentStore(ctx, store.WithType(store.ContentStoreType(cfg.ContentStoreConfig.Type)), store.WithNamespace(cfg.ContentStoreConfig.Namespace)) if err != nil { return nil, fmt.Errorf("cannot create content store: %w", err) } + // FIXME: This runs far too soon. Writing index.json for the soci content store should happen after each change to the store and/or when the snapshotter shuts down. + defer done(ctx) var bgFetcher *bf.BackgroundFetcher diff --git a/soci/store/store.go b/soci/store/store.go index 1405aa9c0..c514d78cb 100644 --- a/soci/store/store.go +++ b/soci/store/store.go @@ -136,12 +136,12 @@ type CleanupFunc func(context.Context) error func nopCleanup(context.Context) error { return nil } -func NewContentStore(ctx context.Context, opts ...Option) (context.Context, Store, error) { +func NewContentStore(ctx context.Context, opts ...Option) (context.Context, Store, CleanupFunc, error) { storeConfig := NewStoreConfig(opts...) contentStoreType, err := CanonicalizeContentStoreType(ContentStoreType(storeConfig.Type)) if err != nil { - return ctx, nil, err + return ctx, nil, nopCleanup, err } switch contentStoreType { case ContainerdContentStoreType: @@ -149,7 +149,7 @@ func NewContentStore(ctx context.Context, opts ...Option) (context.Context, Stor case SociContentStoreType: return NewSociStore(ctx) } - return ctx, nil, errors.New("unexpectedly reached end of NewContentStore") + return ctx, nil, nopCleanup, errors.New("unexpectedly reached end of NewContentStore") } // SociStore wraps oci.Store and adds or stubs additional functionality of the Store interface. @@ -161,9 +161,18 @@ type SociStore struct { var _ Store = (*SociStore)(nil) // NewSociStore creates a sociStore. -func NewSociStore(ctx context.Context) (context.Context, *SociStore, error) { +func NewSociStore(ctx context.Context) (context.Context, *SociStore, CleanupFunc, error) { store, err := oci.New(DefaultSociContentStorePath) - return ctx, &SociStore{store}, err + return ctx, &SociStore{store}, func(_ context.Context) error { return store.SaveIndex() }, err +} + +// Push adds a new content item to the sociStore then tags it for indexing +func (s *SociStore) Push(ctx context.Context, expected ocispec.Descriptor, reader io.Reader) error { + if err := s.Store.Push(ctx, expected, reader); err != nil { + return err + } + + return s.Tag(ctx, expected, expected.Digest.String()) } // Label is a no-op for sociStore until sociStore and ArtifactsDb are better integrated. @@ -181,6 +190,11 @@ func (s *SociStore) BatchOpen(ctx context.Context) (context.Context, CleanupFunc return ctx, nopCleanup, nil } +// Cleanup on a sociStore will update the content/index.json file +func (s *SociStore) Cleanup(ctx context.Context) error { + return s.SaveIndex() +} + type ContainerdStore struct { config.ContentStoreConfig client *containerd.Client @@ -189,10 +203,10 @@ type ContainerdStore struct { // assert that ContainerdStore implements Store var _ Store = (*ContainerdStore)(nil) -func NewContainerdStore(ctx context.Context, storeConfig config.ContentStoreConfig) (context.Context, *ContainerdStore, error) { +func NewContainerdStore(ctx context.Context, storeConfig config.ContentStoreConfig) (context.Context, *ContainerdStore, CleanupFunc, error) { client, err := containerd.New(config.DefaultImageServiceAddress) if err != nil { - return ctx, nil, fmt.Errorf("could not connect to containerd socket for content store access: %w", err) + return ctx, nil, nopCleanup, fmt.Errorf("could not connect to containerd socket for content store access: %w", err) } ctx = namespaces.WithNamespace(ctx, storeConfig.Namespace) @@ -203,7 +217,7 @@ func NewContainerdStore(ctx context.Context, storeConfig config.ContentStoreConf containerdStore.ContentStoreConfig = storeConfig - return ctx, &containerdStore, nil + return ctx, &containerdStore, nopCleanup, nil } // Exists returns true iff the described content exists. @@ -333,3 +347,8 @@ func (s *ContainerdStore) BatchOpen(ctx context.Context) (context.Context, Clean } return ctx, leaseDone, nil } + +// Cleanup is a no-op for ContainerdStore +func (s *ContainerdStore) Cleanup(ctx context.Context) error { + return nil +}