feat: #164 WIP implement initially upload with some basic validation

server-go/Makefile

@@ -25,6 +25,9 @@ test_list_auths:
 test_list_auths_other_user:
 	curl -s -X GET -H 'Authorization: Bearer ${TOKEN}' -H 'Content-Type: application/json' 'http://localhost:8080/api/stable/auth/token?userName=some-user'
 
+test_upload_by_form:
+	curl -s -X POST -H 'Authorization: Bearer ${TOKEN}' -F "file=@./storage/.test_data/test.gpg" 'http://localhost:8080/api/stable/repository/collection/iwa-ait/version'
+
 postgres:
 	docker run -d \
         --name br_postgres \
@@ -44,4 +47,9 @@ postgres_refresh:
 minio:
 	docker run -d \
 		--name br_minio \
-		minio
+		-p 9000:9000 \
+	    -p 9001:9001 \
+	    -v /tmp/br_minio:/data \
+	    -e "MINIO_ROOT_USER=AKIAIOSFODNN7EXAMPLE" \
+	    -e "MINIO_ROOT_PASSWORD=wJaFuCKtnFEMI/CApItaliSM/bPxRfiCYEXAMPLEKEY" \
+		quay.io/minio/minio:RELEASE.2022-02-16T00-35-27Z server /data

server-go/README.md

@@ -30,7 +30,7 @@ Security
 - All JWT's can be revoked anytime. There is a list of generated tokens stored in configuration (only sha256 shortcuts)
 - Passwords are encoded with `argon2di` (winner of the 2015 Password Hashing Competition, recommended by OWASP)
 - All objects are managed by RBAC (Role Based Access Control) and ACL (Access Control Lists)
-- Server works on 1001, [non-root container](https://kubesec.io/basics/containers-securitycontext-runasnonroot-true/)
+- Server works on `uid=1001`, [non-root container](https://kubesec.io/basics/containers-securitycontext-runasnonroot-true/)
 - There is a separate [ServiceAccount](https://kubesec.io/basics/service-accounts/) using namespace-scoped roles
 - We use [distroless](https://github.com/GoogleContainerTools/distroless) images
 - By default, we set [requests and limits](https://kubesec.io/basics/containers-resources-limits-memory/) for `kind: Pod` in Kubernetes

server-go/collections/entity.go

@@ -108,5 +108,5 @@ func (c Collection) CanUploadToMe(user *users.User) bool {
 }
 
 func (c *Collection) GenerateNextVersionFilename(version int) string {
-	return strings.Replace(c.Spec.FilenameTemplate, "${version}", string(rune(version)), 1)
+	return strings.Replace(c.Spec.FilenameTemplate, "${version}", fmt.Sprintf("%v", version), 1)
 }

server-go/db/main.go

@@ -3,6 +3,7 @@ package db
 import (
 	"fmt"
 	"github.com/riotkit-org/backup-repository/security"
+	"github.com/riotkit-org/backup-repository/storage"
 	"github.com/sirupsen/logrus"
 	"gorm.io/driver/postgres"
 	"gorm.io/gorm"
@@ -20,6 +21,10 @@ func InitializeDatabase(db *gorm.DB) bool {
 		logrus.Errorf("Cannot initialize GrantedAccess model: %v", err)
 		return false
 	}
+	if err := storage.InitializeModel(db); err != nil {
+		logrus.Errorf("Cannot initialize UploadedVersion model: %v", err)
+		return false
+	}
 
 	return true
 }

server-go/docs/examples/user.yaml

@@ -16,5 +16,5 @@ spec:
         - 1.2.3.4
     roles:
         - collectionManager
-        - userManager
+        - usersManager
         - systemAdmin

server-go/go.mod

@@ -19,6 +19,18 @@ require (
 )
 
 require (
+	cloud.google.com/go v0.94.0 // indirect
+	cloud.google.com/go/storage v1.16.1 // indirect
+	github.com/aws/aws-sdk-go v1.40.34 // indirect
+	github.com/aws/aws-sdk-go-v2 v1.9.0 // indirect
+	github.com/aws/aws-sdk-go-v2/config v1.7.0 // indirect
+	github.com/aws/aws-sdk-go-v2/credentials v1.4.0 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.5.0 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.2.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.3.0 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.4.0 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.7.0 // indirect
+	github.com/aws/smithy-go v1.8.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/evanphx/json-patch v4.12.0+incompatible // indirect
 	github.com/gin-contrib/sse v0.1.0 // indirect
@@ -33,6 +45,7 @@ require (
 	github.com/google/go-cmp v0.5.6 // indirect
 	github.com/google/gofuzz v1.1.0 // indirect
 	github.com/google/uuid v1.3.0 // indirect
+	github.com/google/wire v0.5.0 // indirect
 	github.com/googleapis/gax-go/v2 v2.1.0 // indirect
 	github.com/googleapis/gnostic v0.5.5 // indirect
 	github.com/imdario/mergo v0.3.12 // indirect
@@ -46,6 +59,7 @@ require (
 	github.com/jackc/pgx/v4 v4.14.0 // indirect
 	github.com/jinzhu/inflection v1.0.0 // indirect
 	github.com/jinzhu/now v1.1.4 // indirect
+	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/juju/ratelimit v1.0.1 // indirect
 	github.com/julianshen/gin-limiter v0.0.0-20161123033831-fc39b5e90fe7 // indirect

server-go/http/collection.go

@@ -2,26 +2,27 @@ package http
 
 import (
 	"errors"
+	"fmt"
 	"github.com/gin-gonic/gin"
 	"github.com/riotkit-org/backup-repository/core"
 	"github.com/riotkit-org/backup-repository/security"
+	"io"
 	"time"
 )
 
 func addUploadRoute(r *gin.RouterGroup, ctx *core.ApplicationContainer) {
 	r.POST("/repository/collection/:collectionId/version", func(c *gin.Context) {
 		// todo: check if rotation strategy allows uploading
 		// todo: deactivate token if temporary token is used
-		// todo: handle upload
 		// todo: check uploaded file size, respect quotas and additional space
 		// todo: check if there are gpg header and footer
 		// todo: handle upload interruptions
 
 		ctxUser, _ := GetContextUser(ctx, c)
 
-		// Check if Colection exists
-		collection, err := ctx.Collections.GetCollectionById(c.Param("collectionId"))
-		if err != nil {
+		// Check if Collection exists
+		collection, findError := ctx.Collections.GetCollectionById(c.Param("collectionId"))
+		if findError != nil {
 			NotFoundResponse(c, errors.New("cannot find specified collection"))
 			return
 		}
@@ -40,10 +41,52 @@ func addUploadRoute(r *gin.RouterGroup, ctx *core.ApplicationContainer) {
 			return
 		}
 
-		// todo: support Url encoded and raw body
-		// c.Request.Body
-		// ctx.Storage
+		// Increment a version, generate target file path name that will be used on storage
+		sessionId := GetCurrentSessionId(c)
+		version, factoryError := ctx.Storage.CreateNewVersionFromCollection(collection, ctxUser.Metadata.Name, sessionId, 0)
+		if factoryError != nil {
+			ServerErrorResponse(c, errors.New(fmt.Sprintf("cannot increment version. %v", factoryError)))
+			return
+		}
+
+		var stream io.ReadCloser
+
+		// Support form data
+		if c.ContentType() == "application/x-www-form-urlencoded" || c.ContentType() == "multipart/form-data" {
+			var openErr error
+			fh, ffErr := c.FormFile("file")
+			if ffErr != nil {
+				ServerErrorResponse(c, errors.New(fmt.Sprintf("cannot read file from multipart/urlencoded form: %v", ffErr)))
+				return
+			}
+			stream, openErr = fh.Open()
+			if openErr != nil {
+				ServerErrorResponse(c, errors.New(fmt.Sprintf("cannot open file from multipart/urlencoded form: %v", openErr)))
+			}
+
+		} else {
+			// Support RAW sent data via body
+			stream = c.Request.Body
+		}
+
+		// Upload a file from selected source, then handle errors - delete file from storage if not uploaded successfully
+		wroteLen, uploadError := ctx.Storage.UploadFile(stream, &version)
+		if uploadError != nil {
+			// todo: make sure the uploaded file will be deleted
+
+			ServerErrorResponse(c, errors.New(fmt.Sprintf("cannot upload version. %v", uploadError)))
+			return
+		}
+
+		// Set a valid filesize that is known after receiving the file
+		version.Filesize = wroteLen
+
+		// Append version to the registry
+		//ctx.Storage.SubmitVersion(version)
 
-		println(collection)
+		// todo: add UploadedVersion to database
+		OKResponse(c, gin.H{
+			"version": version,
+		})
 	})
 }

server-go/main.go

@@ -26,7 +26,8 @@ type options struct {
 	DbPort               int    `long:"db-port" description:"Database name inside a database" default:"5432"`
 	JwtSecretKey         string `long:"jwt-secret-key" short:"s" description:"Secret used for generating JSON Web Tokens for authentication"`
 	Level                string `long:"log-level" description:"Log level" default:"debug"`
-	StorageDriverUrl     string `long:"--storage-url" description:"Storage driver url compatible with GO Cloud (https://gocloud.dev/howto/blob/)"`
+	StorageDriverUrl     string `long:"storage-url" description:"Storage driver url compatible with GO Cloud (https://gocloud.dev/howto/blob/)"`
+	IsGCS                bool   `long:"use-google-cloud" description:"If using Google Cloud Storage, then in --storage-url just type bucket name"`
 }
 
 func main() {
@@ -70,8 +71,8 @@ func main() {
 	usersService := users.NewUsersService(configProvider)
 	gaService := security.NewService(dbDriver)
 	collectionsService := collections.NewService(configProvider)
-	storageService, storageError := storage.NewService(opts.StorageDriverUrl)
-	if err != nil {
+	storageService, storageError := storage.NewService(dbDriver, opts.StorageDriverUrl, opts.IsGCS)
+	if storageError != nil {
 		log.Errorln("Cannot initialize storage driver")
 		log.Fatal(storageError)
 	}

server-go/storage/.test_data/test.gpg

@@ -0,0 +1,41 @@
+-----BEGIN PGP MESSAGE-----
+YnVpbGQ6CglnbyBidWlsZCAtdGFncz1ub21zZ3BhY2sgLgoKdGVzdF9sb2dpbjoKCWN1cmwgLXMg
+LVggUE9TVCAtZCAneyJ1c2VybmFtZSI6ImFkbWluIiwicGFzc3dvcmQiOiJhZG1pbiJ9JyAtSCAn
+Q29udGVudC1UeXBlOiBhcHBsaWNhdGlvbi9qc29uJyAnaHR0cDovL2xvY2FsaG9zdDo4MDgwL2Fw
+aS9zdGFibGUvYXV0aC9sb2dpbicKCnRlc3RfbG9naW5fc29tZV91c2VyOgoJY3VybCAtcyAtWCBQ
+T1NUIC1kICd7InVzZXJuYW1lIjoic29tZS11c2VyIiwicGFzc3dvcmQiOiJhZG1pbiJ9JyAtSCAn
+Q29udGVudC1UeXBlOiBhcHBsaWNhdGlvbi9qc29uJyAnaHR0cDovL2xvY2FsaG9zdDo4MDgwL2Fw
+aS9zdGFibGUvYXV0aC9sb2dpbicKCnRlc3RfbG9va3VwOgoJY3VybCAtcyAtWCBHRVQgLUggJ0F1
+dGhvcml6YXRpb246IEJlYXJlciAke1RPS0VOfScgLUggJ0NvbnRlbnQtVHlwZTogYXBwbGljYXRp
+b24vanNvbicgJ2h0dHA6Ly9sb2NhbGhvc3Q6ODA4MC9hcGkvc3RhYmxlL2F1dGgvdXNlci9zb21l
+LXVzZXInCgp0ZXN0X3dob2FtaToKCWN1cmwgLXMgLVggR0VUIC1IICdBdXRob3JpemF0aW9uOiBC
+ZWFyZXIgJHtUT0tFTn0nIC1IICdDb250ZW50LVR5cGU6IGFwcGxpY2F0aW9uL2pzb24nICdodHRw
+Oi8vbG9jYWxob3N0OjgwODAvYXBpL3N0YWJsZS9hdXRoL3dob2FtaScKCnRlc3RfbG9nb3V0OgoJ
+Y3VybCAtcyAtWCBERUxFVEUgLUggJ0F1dGhvcml6YXRpb246IEJlYXJlciAke1RPS0VOfScgLUgg
+J0NvbnRlbnQtVHlwZTogYXBwbGljYXRpb24vanNvbicgJ2h0dHA6Ly9sb2NhbGhvc3Q6ODA4MC9h
+cGkvc3RhYmxlL2F1dGgvbG9nb3V0JwoKdGVzdF9sb2dvdXRfb3RoZXJfdXNlcjoKCWN1cmwgLXMg
+LVggREVMRVRFIC1IICdBdXRob3JpemF0aW9uOiBCZWFyZXIgJHtUT0tFTn0nIC1IICdDb250ZW50
+LVR5cGU6IGFwcGxpY2F0aW9uL2pzb24nICdodHRwOi8vbG9jYWxob3N0OjgwODAvYXBpL3N0YWJs
+ZS9hdXRoL2xvZ291dD9zZXNzaW9uSWQ9JHtPVEhFUl9VU0VSX1NFU1NJT05fSUR9JwoKdGVzdF9s
+aXN0X2F1dGhzOgoJY3VybCAtcyAtWCBHRVQgLUggJ0F1dGhvcml6YXRpb246IEJlYXJlciAke1RP
+S0VOfScgLUggJ0NvbnRlbnQtVHlwZTogYXBwbGljYXRpb24vanNvbicgJ2h0dHA6Ly9sb2NhbGhv
+c3Q6ODA4MC9hcGkvc3RhYmxlL2F1dGgvdG9rZW4nCgp0ZXN0X2xpc3RfYXV0aHNfb3RoZXJfdXNl
+cjoKCWN1cmwgLXMgLVggR0VUIC1IICdBdXRob3JpemF0aW9uOiBCZWFyZXIgJHtUT0tFTn0nIC1I
+ICdDb250ZW50LVR5cGU6IGFwcGxpY2F0aW9uL2pzb24nICdodHRwOi8vbG9jYWxob3N0OjgwODAv
+YXBpL3N0YWJsZS9hdXRoL3Rva2VuP3VzZXJOYW1lPXNvbWUtdXNlcicKCnRlc3RfdXBsb2FkX2J5
+X2Zvcm06CgljdXJsIC1zIC1YIFBPU1QgLUggJ0F1dGhvcml6YXRpb246IEJlYXJlciAke1RPS0VO
+fScgLUYgImZpbGU9QE1ha2VmaWxlIiAnaHR0cDovL2xvY2FsaG9zdDo4MDgwL2FwaS9zdGFibGUv
+cmVwb3NpdG9yeS9jb2xsZWN0aW9uL2l3YS1haXQvdmVyc2lvbicKCnBvc3RncmVzOgoJZG9ja2Vy
+IHJ1biAtZCBcCiAgICAgICAgLS1uYW1lIGJyX3Bvc3RncmVzIFwKICAgICAgICAtZSBQT1NUR1JF
+U19QQVNTV09SRD1wb3N0Z3JlcyBcCiAgICAgICAgLWUgUE9TVEdSRVNfVVNFUj1wb3N0Z3JlcyBc
+CiAgICAgICAgLWUgUE9TVEdSRVNfREI9cG9zdGdyZXMgXAogICAgICAgIC1lIFBHREFUQT0vdmFy
+L2xpYi9wb3N0Z3Jlc3FsL2RhdGEvcGdkYXRhIFwKICAgICAgICAtdiAvdG1wL2JyX3Bvc3RncmVz
+Oi92YXIvbGliL3Bvc3RncmVzcWwvZGF0YSBcCiAgICAgICAgLXAgNTQzMjo1NDMyIFwKICAgICAg
+ICBwb3N0Z3JlczoxNC4xLWFscGluZQoKcG9zdGdyZXNfcmVmcmVzaDoKCWRvY2tlciBybSAtZiBi
+cl9wb3N0Z3JlcyB8fCB0cnVlCglzdWRvIHJtIC1yZiAvdG1wL2JyX3Bvc3RncmVzCgltYWtlIHBv
+c3RncmVzCgptaW5pbzoKCWRvY2tlciBydW4gLWQgXAoJCS0tbmFtZSBicl9taW5pbyBcCgkJLXAg
+OTAwMDo5MDAwIFwKCSAgICAtcCA5MDAxOjkwMDEgXAoJICAgIC12IC90bXAvYnJfbWluaW86L2Rh
+dGEgXAoJICAgIC1lICJNSU5JT19ST09UX1VTRVI9QUtJQUlPU0ZPRE5ON0VYQU1QTEUiIFwKCSAg
+ICAtZSAiTUlOSU9fUk9PVF9QQVNTV09SRD13SmFGdUNLdG5GRU1JL0NBcEl0YWxpU00vYlB4UmZp
+Q1lFWEFNUExFS0VZIiBcCgkJcXVheS5pby9taW5pby9taW5pbzpSRUxFQVNFLjIwMjItMDItMTZU
+-----END PGP MESSAGE-----

server-go/storage/entity.go

@@ -1,8 +1,6 @@
 package storage
 
 import (
-	"github.com/google/uuid"
-	"github.com/riotkit-org/backup-repository/collections"
 	"gorm.io/gorm"
 	"time"
 )
@@ -25,20 +23,3 @@ type UploadedVersion struct {
 func (u *UploadedVersion) GetTargetPath() string {
 	return u.CollectionId + "/" + u.Filename
 }
-
-func CreateNewVersionFromCollection(c collections.Collection, svc Service, uploader string, uploaderSessionId string, filesize int) UploadedVersion {
-	nextVersion := svc.FindNextVersionForCollectionId(c.Metadata.Name)
-
-	return UploadedVersion{
-		Id:                  uuid.New().String(),
-		CollectionId:        c.Metadata.Name,
-		VersionNumber:       nextVersion,
-		Filename:            c.GenerateNextVersionFilename(nextVersion),
-		Filesize:            filesize,
-		UploadedBySessionId: uploaderSessionId,
-		Uploader:            uploader,
-		CreatedAt:           time.Time{},
-		UpdatedAt:           time.Time{},
-		DeletedAt:           gorm.DeletedAt{},
-	}
-}

server-go/storage/repository.go

@@ -1,8 +1,22 @@
 package storage
 
+import (
+	"gorm.io/gorm"
+)
+
 type VersionsRepository struct {
+	db *gorm.DB
 }
 
-func (r VersionsRepository) findLastHighestVersionNumber(name string) interface{} {
+func (vr VersionsRepository) findLastHighestVersionNumber(collectionId string) (int, error) {
+	maxNum := 0
+	err := vr.db.Model(&UploadedVersion{}).Select("uploaded_versions.version_number").Where("uploaded_versions.collection_id = ?", collectionId).Order("uploaded_versions.version_number DESC").Limit(1).Find(&maxNum).Error
+	if err != nil {
+		return 0, err
+	}
+	return maxNum, nil
+}
 
+func InitializeModel(db *gorm.DB) error {
+	return db.AutoMigrate(&UploadedVersion{})
 }

server-go/storage/service.go

@@ -2,26 +2,78 @@ package storage
 
 import (
 	"context"
+	"errors"
+	"fmt"
+	"github.com/google/uuid"
+	"github.com/riotkit-org/backup-repository/collections"
 	"github.com/sirupsen/logrus"
 	"gocloud.dev/blob"
+	_ "gocloud.dev/blob/fileblob"
+	"gocloud.dev/blob/gcsblob"
+	_ "gocloud.dev/blob/s3blob"
+	"gocloud.dev/gcp"
+	"gorm.io/gorm"
+	"time"
 )
 
 type Service struct {
 	storage    *blob.Bucket
 	repository *VersionsRepository
 }
 
-func (s *Service) FindNextVersionForCollectionId(name string) int {
-	lastHigherVersion := s.repository.findLastHighestVersionNumber(name)
-	return lastHigherVersion + 1
+func (s *Service) FindNextVersionForCollectionId(name string) (int, error) {
+	lastHigherVersion, err := s.repository.findLastHighestVersionNumber(name)
+	if err != nil {
+		logrus.Errorf("[FindNextVersionForCollectionId] Attempted to find next version number for collectionId=%v. Got error: %v", name, err)
+		return 0, err
+	}
+	return lastHigherVersion + 1, nil
 }
 
-func NewService(driverUrl string) (Service, error) {
+func (s *Service) CreateNewVersionFromCollection(c *collections.Collection, uploader string, uploaderSessionId string, filesize int) (UploadedVersion, error) {
+	nextVersion, err := s.FindNextVersionForCollectionId(c.Metadata.Name)
+	if err != nil {
+		return UploadedVersion{}, err
+	}
+	return UploadedVersion{
+		Id:                  uuid.New().String(),
+		CollectionId:        c.Metadata.Name,
+		VersionNumber:       nextVersion,
+		Filename:            c.GenerateNextVersionFilename(nextVersion),
+		Filesize:            filesize,
+		UploadedBySessionId: uploaderSessionId,
+		Uploader:            uploader,
+		CreatedAt:           time.Time{},
+		UpdatedAt:           time.Time{},
+		DeletedAt:           gorm.DeletedAt{},
+	}, nil
+}
+
+func NewService(db *gorm.DB, driverUrl string, isUsingGCS bool) (Service, error) {
+	repository := VersionsRepository{db: db}
+
+	// Google Cloud requires extra support
+	if isUsingGCS {
+		gcsCredentials, err := gcp.DefaultCredentials(context.Background())
+		if err != nil {
+			return Service{}, errors.New(fmt.Sprintf("cannot grab credentials for Google Cloud Storage: %v", err))
+		}
+		client, loginErr := gcp.NewHTTPClient(gcp.DefaultTransport(), gcp.CredentialsTokenSource(gcsCredentials))
+		if loginErr != nil {
+			return Service{}, errors.New(fmt.Sprintf("cannot login to Google Cloud Storage: %v", loginErr))
+		}
+		driver, openErr := gcsblob.OpenBucket(context.Background(), client, driverUrl, nil)
+		if openErr != nil {
+			return Service{}, errors.New(fmt.Sprintf("cannot open Google Cloud Storage bucket: %v", openErr))
+		}
+		return Service{storage: driver, repository: &repository}, nil
+	}
+
 	driver, err := blob.OpenBucket(context.Background(), driverUrl)
 	if err != nil {
 		logrus.Errorf("Cannot construct storage driver: %v", err)
 		return Service{}, err
 	}
 
-	return Service{storage: driver}, nil
+	return Service{storage: driver, repository: &repository}, nil
 }