From 59d7778fc06510ac38b9a5d871770aeaac4131c6 Mon Sep 17 00:00:00 2001 From: fredbi Date: Wed, 11 Dec 2019 08:27:10 +0100 Subject: [PATCH] doc: tidy up a little our generated godoc (#330) Signed-off-by: Frederic BIDON --- docs/concepts.md | 2 +- pkg/auth/google/doc.go | 5 +++++ pkg/auth/status/status.go | 2 +- pkg/cafs/doc.go | 2 ++ pkg/config/{Processor.go => processor.go} | 5 +++-- pkg/convert/doc.go | 4 ++++ pkg/core/status/status.go | 1 + pkg/dlogger/logger.go | 7 ++++++- pkg/model/bundle.go | 1 + pkg/model/context.go | 6 ++++-- pkg/model/doc.go | 21 +++++++++++++++++++++ pkg/storage/doc.go | 7 ++++++- pkg/storage/gcs/store.go | 1 + pkg/storage/localfs/store.go | 1 + pkg/storage/status/status.go | 12 ++++++++++-- pkg/storage/sthree/store.go | 1 + pkg/storage/store.go | 11 +++++++++-- pkg/wal/wal.go | 7 +++++++ pkg/web/routes.go | 3 +++ 19 files changed, 87 insertions(+), 12 deletions(-) create mode 100644 pkg/auth/google/doc.go create mode 100644 pkg/cafs/doc.go rename pkg/config/{Processor.go => processor.go} (79%) create mode 100644 pkg/convert/doc.go create mode 100644 pkg/model/doc.go diff --git a/docs/concepts.md b/docs/concepts.md index 398ed968..7616b7de 100644 --- a/docs/concepts.md +++ b/docs/concepts.md @@ -13,7 +13,7 @@ A bundle is a point in time read-only view of a repo, composed of individual fil See [usage](usage/datamon_bundle.md) -Bundle metadata can be _downloaded_ (e.g. contributors), bundle blob content can be _mounted_ on a host mount path. +Bundle metadata can be _downloaded_ (e.g. to see contributors), bundle blob content can be _mounted_ on a host mount path. A simple read-only mount provide a file system view of the bundle content. diff --git a/pkg/auth/google/doc.go b/pkg/auth/google/doc.go new file mode 100644 index 00000000..84df1589 --- /dev/null +++ b/pkg/auth/google/doc.go @@ -0,0 +1,5 @@ +// Package google supports authentication via Google Auth API. +// +// Datamon auth requires the email scope to identify data contributors. +// Allowing the profile scope is useful to be get the actual name of a contributor. +package google diff --git a/pkg/auth/status/status.go b/pkg/auth/status/status.go index bd13e328..6a55758f 100644 --- a/pkg/auth/status/status.go +++ b/pkg/auth/status/status.go @@ -1,4 +1,4 @@ -// Package status declares error constants returned by the various +// Package status declares error constants returned by // implementations of the Authable interface. // // NOTE: such constants are located in a separate package to avoid diff --git a/pkg/cafs/doc.go b/pkg/cafs/doc.go new file mode 100644 index 00000000..a3e7032b --- /dev/null +++ b/pkg/cafs/doc.go @@ -0,0 +1,2 @@ +// Package cafs provides a file system view of a repository. +package cafs diff --git a/pkg/config/Processor.go b/pkg/config/processor.go similarity index 79% rename from pkg/config/Processor.go rename to pkg/config/processor.go index b1bdc977..4994a54a 100644 --- a/pkg/config/Processor.go +++ b/pkg/config/processor.go @@ -1,6 +1,7 @@ +// Package config is currently unused. package config -// Spec for what to process +// Processor provides a specification for what to process type Processor struct { Name string `json:"name" yaml:"name"` Branch string `json:"branch" yaml:"branch"` @@ -16,7 +17,7 @@ type Resources struct { Mem ResourceLimit `json:"mem,omitempty" yaml:"mem,omitempty"` } -// Resource limits +// ResourceLimit applied to a resource used by the processor (e.g. Min/Max) type ResourceLimit struct { Min string `json:"min,omitempty" yaml:"min,omitempty"` Max string `json:"max,omitempty" yaml:"max,omitempty"` diff --git a/pkg/convert/doc.go b/pkg/convert/doc.go new file mode 100644 index 00000000..216c1aba --- /dev/null +++ b/pkg/convert/doc.go @@ -0,0 +1,4 @@ +// Package convert provides some helpers for fast binary conversion of common go types. +// +// Conversion operations are essentially unsafe and avoid the use of memcpy(). +package convert diff --git a/pkg/core/status/status.go b/pkg/core/status/status.go index 228604ed..aa0702eb 100644 --- a/pkg/core/status/status.go +++ b/pkg/core/status/status.go @@ -1,3 +1,4 @@ +// Package status exports errors produced by the core package. package status import ( diff --git a/pkg/dlogger/logger.go b/pkg/dlogger/logger.go index c583b97c..dbe73d51 100644 --- a/pkg/dlogger/logger.go +++ b/pkg/dlogger/logger.go @@ -1,3 +1,4 @@ +// Package dlogger exposes a simple zap logger, with log levels package dlogger import ( @@ -6,10 +7,14 @@ import ( ) const ( - LogLevelInfo = "info" + // LogLevelInfo sets the log level to info + LogLevelInfo = "info" + + // LogLevelDebug sets the log level to debug LogLevelDebug = "debug" ) +// GetLogger returns a zap logger with the specified level func GetLogger(logLevel string) (*zap.Logger, error) { zapConfig := zap.NewProductionConfig() var lvl zapcore.Level diff --git a/pkg/model/bundle.go b/pkg/model/bundle.go index 7f16c17a..5a817552 100644 --- a/pkg/model/bundle.go +++ b/pkg/model/bundle.go @@ -63,6 +63,7 @@ type BundleEntry struct { const ( // ConsumableStorePathTypeDescriptor defines consumable store metadata of type "descriptor" ConsumableStorePathTypeDescriptor byte = iota + // ConsumableStorePathTypeFileList defines consumable store metadata of type "file list" ConsumableStorePathTypeFileList ) diff --git a/pkg/model/context.go b/pkg/model/context.go index a4ba48c5..82132e0d 100644 --- a/pkg/model/context.go +++ b/pkg/model/context.go @@ -11,8 +11,10 @@ import ( "gopkg.in/yaml.v2" ) -// ContextVersion indicates the version of the context model -const ContextVersion = 1.0 +const ( + // ContextVersion indicates the version of the context model + ContextVersion = 1.0 +) // Context defines the details for a datamon context. type Context struct { diff --git a/pkg/model/doc.go b/pkg/model/doc.go new file mode 100644 index 00000000..b3fda0ec --- /dev/null +++ b/pkg/model/doc.go @@ -0,0 +1,21 @@ +// Package model exposes the base objects manipulated by datamon. +// +// The object model for datamon is composed of: +// +// Repos: +// A datamon repository is analogous to a git repo. A repo is a dataset that has a unified lifecycle. +// A particular version of the files in a repo is called a bundle. +// +// Bundles: +// A bundle is a point in time read-only view of a repo, composed of individual files. +// This is analogous to a commit in git. +// +// Labels: +// A name given to a bundle, analogous to tags in git. Examples: Latest, production. +// +// Contexts: +// A context provides a way to define multiple instances of datamon. Example: development, production +// +// WAL: +// A write-ahead log tracks all changes to a repo. +package model diff --git a/pkg/storage/doc.go b/pkg/storage/doc.go index 999e6bf0..5d011658 100644 --- a/pkg/storage/doc.go +++ b/pkg/storage/doc.go @@ -1,4 +1,9 @@ // Copyright © 2018 One Concern -// Package storage provides interface to handle objects storeed on some backend storage, e.g. gcs, S3. +// Package storage provides interface to handle backend storage objects. +// +// This package supports the following backends: +// - GCS (Google) +// - S3 (AWS) +// - local file system package storage diff --git a/pkg/storage/gcs/store.go b/pkg/storage/gcs/store.go index 1cd0d8e7..48ec82c9 100644 --- a/pkg/storage/gcs/store.go +++ b/pkg/storage/gcs/store.go @@ -1,5 +1,6 @@ // Copyright © 2018 One Concern +// Package gcs implements datamon Store for Google GCS package gcs import ( diff --git a/pkg/storage/localfs/store.go b/pkg/storage/localfs/store.go index 1cc7ddad..75596727 100644 --- a/pkg/storage/localfs/store.go +++ b/pkg/storage/localfs/store.go @@ -1,5 +1,6 @@ // Copyright © 2018 One Concern +// Package localfs implements datamon Store for a local file system package localfs import ( diff --git a/pkg/storage/status/status.go b/pkg/storage/status/status.go index aafe34a4..1e78c859 100644 --- a/pkg/storage/status/status.go +++ b/pkg/storage/status/status.go @@ -1,6 +1,6 @@ // Copyright © 2018 One Concern -// Package status declares error constants returned by the various +// Package status declares error constants returned by // implementations of the Store interface. // // NOTE: such constants are located in a separate package to avoid @@ -11,27 +11,35 @@ package status import "github.com/oneconcern/datamon/pkg/errors" var ( - // Sentinel errors returned by implementations of interfaces defined by storage + // Sentinel errors returned by implementations of the interface defined by storage // ErrNotExists indicates that the fetched object does not exist on storage ErrNotExists = errors.New("object doesn't exist") // ErrNotFound indicates that the backend API call did not find the target resource ErrNotFound = errors.New("not found") + // ErrUnauthorized indicates that you don't provided correct credentials to the API ErrUnauthorized = errors.New("unauthorized") + // ErrForbidden indicates that the backend API forbids access to the target resource ErrForbidden = errors.New("forbidden") + // ErrNotSupported indicates that the backend API does not support this call ErrNotSupported = errors.New("not supported") + // ErrExists indicates that the resource already exists and cannot be overridden ErrExists = errors.New("exists already") + // ErrObjectTooBig indicates that the object is too big and cannot be handled by datamon ErrObjectTooBig = errors.New("object too big to be read into memory") + // ErrInvalidResource indicates that the storage resource has an invalid name ErrInvalidResource = errors.New("invalid storage resource name") + // ErrStorageAPI indicates any other storage AI error ErrStorageAPI = errors.New("storage API error") + // ErrNotImplemented tells that this feature has not been implemented yet ErrNotImplemented = errors.New("not implemented") ) diff --git a/pkg/storage/sthree/store.go b/pkg/storage/sthree/store.go index 1093d7c4..4d176654 100644 --- a/pkg/storage/sthree/store.go +++ b/pkg/storage/sthree/store.go @@ -1,3 +1,4 @@ +// Package sthree implements datamon Store for AWS S3 package sthree import ( diff --git a/pkg/storage/store.go b/pkg/storage/store.go index ef2bf7bc..6336bb9a 100644 --- a/pkg/storage/store.go +++ b/pkg/storage/store.go @@ -26,7 +26,7 @@ type Attributes struct { Owner string } -// Store implementations know how to write entries to a K/V model.Store. +// Store implementations know how to fetch and write entries from a and a K/V store. // // Typically this is something file system-like. Examples are S3, local FS, NFS, ... // Implementations of this interface are assumed to be fairly simple. @@ -39,15 +39,22 @@ type Store interface { Touch(context.Context, string) error Put(context.Context, string, io.Reader, NewKey) error Delete(context.Context, string) error - Keys(context.Context) ([]string, error) Clear(context.Context) error + + // Keys returns all keys known to the store. + // Depending on the implementation, some limit may exist on the maximum number of such returned keys + Keys(context.Context) ([]string, error) + + // KeyPrefix provides a paginated key iterator using "pageToken" as the next starting point KeysPrefix(ctx context.Context, pageToken string, prefix string, delimiter string, count int) ([]string, string, error) } +// StoreCRC knows how to update an object with a computed CRC checksum type StoreCRC interface { PutCRC(context.Context, string, io.Reader, bool, uint32) error } +// PipeIO copies data from a reader to a writer using io.Pipe func PipeIO(writer io.Writer, reader io.Reader) (n int64, err error) { pr, pw := io.Pipe() errC := make(chan error, 1) diff --git a/pkg/wal/wal.go b/pkg/wal/wal.go index 9f3c08ba..8718ecbb 100644 --- a/pkg/wal/wal.go +++ b/pkg/wal/wal.go @@ -1,3 +1,7 @@ +// Package wal provides a write-ahead log. +// +// The WAL keeps track of all changes to a repo, that is, +// which contributor did change what and when. package wal import ( @@ -22,6 +26,7 @@ const ( maxConcurrency = 1024 ) +// WAL describes a write-ahead log type WAL struct { mutableStore storage.Store // Location for updating token generator object tokenGeneratorPath string // Path to the token generator object in the mutable store @@ -31,6 +36,7 @@ type WAL struct { l *zap.Logger // Logging } +// Options to the write-ahead log type Options func(w *WAL) func MaxConcurrency(c int) Options { @@ -38,6 +44,7 @@ func MaxConcurrency(c int) Options { w.maxConcurrency = c } } + func TokenGeneratorPath(path string) Options { return func(w *WAL) { w.tokenGeneratorPath = path diff --git a/pkg/web/routes.go b/pkg/web/routes.go index 66f91799..c04d4074 100644 --- a/pkg/web/routes.go +++ b/pkg/web/routes.go @@ -1,3 +1,4 @@ +// Package web builds a web server to navigate through datamon repos. package web import ( @@ -105,11 +106,13 @@ type ServerParams struct { Credential string } +// Server describe a web server with templates type Server struct { tmpl appTemplates params ServerParams } +// NewServer builds a new web server with page handlers func NewServer(params ServerParams) (*Server, error) { tmpl, err := loadTemplates() if err != nil {