From 9d3fac3cce3d03d983e75ececfa3c53f7a2ec7ac Mon Sep 17 00:00:00 2001 From: blackandred Date: Fri, 18 Feb 2022 22:41:52 +0100 Subject: [PATCH] feat: #164 implemented working rotation --- server-go/Makefile | 3 ++- server-go/docs/examples/collection.yaml | 2 +- server-go/http/collection.go | 14 ++++++------ server-go/storage/repository.go | 9 ++++++++ server-go/storage/rotation.go | 4 ++-- server-go/storage/service.go | 30 +++++++++++++++++++++++++ 6 files changed, 51 insertions(+), 11 deletions(-) diff --git a/server-go/Makefile b/server-go/Makefile index a04f6391..9aec1966 100644 --- a/server-go/Makefile +++ b/server-go/Makefile @@ -3,6 +3,7 @@ build: test_login: curl -s -X POST -d '{"username":"admin","password":"admin"}' -H 'Content-Type: application/json' 'http://localhost:8080/api/stable/auth/login' + @echo "Now do export TOKEN=..." test_login_some_user: curl -s -X POST -d '{"username":"some-user","password":"admin"}' -H 'Content-Type: application/json' 'http://localhost:8080/api/stable/auth/login' @@ -52,4 +53,4 @@ minio: -v /tmp/br_minio:/data \ -e "MINIO_ROOT_USER=AKIAIOSFODNN7EXAMPLE" \ -e "MINIO_ROOT_PASSWORD=wJaFuCKtnFEMI/CApItaliSM/bPxRfiCYEXAMPLEKEY" \ - quay.io/minio/minio:RELEASE.2022-02-16T00-35-27Z server /data + quay.io/minio/minio:RELEASE.2022-02-16T00-35-27Z server /data --console-address 0.0.0.0:9001 diff --git a/server-go/docs/examples/collection.yaml b/server-go/docs/examples/collection.yaml index 933a3594..42a05e5e 100644 --- a/server-go/docs/examples/collection.yaml +++ b/server-go/docs/examples/collection.yaml @@ -16,7 +16,7 @@ spec: duration: 1h # fifo, fifo-plus-older - strategyName: fifo-plus-older + strategyName: fifo strategySpec: keepLastOlderNotMoreThan: 5d maxOlderCopies: 2 diff --git a/server-go/http/collection.go b/server-go/http/collection.go index 1800773d..d2228c42 100644 --- a/server-go/http/collection.go +++ b/server-go/http/collection.go @@ -13,7 +13,6 @@ import ( func addUploadRoute(r *gin.RouterGroup, ctx *core.ApplicationContainer) { r.POST("/repository/collection/:collectionId/version", func(c *gin.Context) { - // todo: check if rotation strategy allows uploading // todo: deactivate token if temporary token is used // todo: check uploaded file size, respect quotas and additional space // todo: handle upload interruptions @@ -55,6 +54,7 @@ func addUploadRoute(r *gin.RouterGroup, ctx *core.ApplicationContainer) { if strategyFactorialError != nil { logrus.Errorf(fmt.Sprintf("Cannot create collection strategy for collectionId=%v, error: %v", collection.Metadata.Name, strategyFactorialError)) ServerErrorResponse(c, errors.New("internal error while trying to create rotation strategy. Check server logs")) + return } if err := rotationStrategyCase.CanUpload(version); err != nil { UnauthorizedResponse(c, errors.New(fmt.Sprintf("backup collection strategy declined a possibility to upload, %v", err))) @@ -84,7 +84,7 @@ func addUploadRoute(r *gin.RouterGroup, ctx *core.ApplicationContainer) { // Upload a file from selected source, then handle errors - delete file from storage if not uploaded successfully wroteLen, uploadError := ctx.Storage.UploadFile(stream, &version) if uploadError != nil { - // todo: make sure the uploaded file will be deleted + _ = ctx.Storage.Delete(&version) ServerErrorResponse(c, errors.New(fmt.Sprintf("cannot upload version. %v", uploadError))) return @@ -94,12 +94,12 @@ func addUploadRoute(r *gin.RouterGroup, ctx *core.ApplicationContainer) { version.Filesize = wroteLen // Append version to the registry - // todo - //ctx.Storage.SubmitVersion(version) - //ctx.Storage.CleanUpOlderVersions(rotationStrategyCase.GetVersionsThatShouldBeDeletedIfThisVersionUploaded(version)) + if err := ctx.Storage.RegisterVersion(&version); err != nil { + _ = ctx.Storage.Delete(&version) + } + ctx.Storage.CleanUpOlderVersions(rotationStrategyCase.GetVersionsThatShouldBeDeletedIfThisVersionUploaded(version)) + logrus.Infof("Uploaded v%v for collectionId=%v, size=%v", version.VersionNumber, version.CollectionId, version.Filesize) - // todo: Rotate collection - // todo: add UploadedVersion to database OKResponse(c, gin.H{ "version": version, }) diff --git a/server-go/storage/repository.go b/server-go/storage/repository.go index 291d140a..b6859f39 100644 --- a/server-go/storage/repository.go +++ b/server-go/storage/repository.go @@ -27,6 +27,15 @@ func (vr VersionsRepository) findAllVersionsForCollectionId(collectionId string) return foundVersions, nil } +func (vr VersionsRepository) delete(version *UploadedVersion) (error, bool) { + var result bool + return vr.db.Model(&UploadedVersion{}).Where("uploaded_versions.id = ?", version.Id).Delete(&result).Error, result +} + +func (vr VersionsRepository) create(version *UploadedVersion) error { + return vr.db.Create(version).Error +} + func InitializeModel(db *gorm.DB) error { return db.AutoMigrate(&UploadedVersion{}) } diff --git a/server-go/storage/rotation.go b/server-go/storage/rotation.go index 6c718260..a873f501 100644 --- a/server-go/storage/rotation.go +++ b/server-go/storage/rotation.go @@ -34,8 +34,8 @@ func (frs *FifoRotationStrategy) GetVersionsThatShouldBeDeletedIfThisVersionUplo } // order by version number DESCENDING - sort.SliceStable(&existingVersions, func(i, j int) bool { - return existingVersions[i].VersionNumber > existingVersions[j].VersionNumber + sort.SliceStable(existingVersions, func(i, j int) bool { + return existingVersions[i].VersionNumber < existingVersions[j].VersionNumber }) // oldest element diff --git a/server-go/storage/service.go b/server-go/storage/service.go index da1366af..b3defd78 100644 --- a/server-go/storage/service.go +++ b/server-go/storage/service.go @@ -62,6 +62,36 @@ func (s *Service) CreateRotationStrategyCase(collection *collections.Collection) return &FifoRotationStrategy{}, errors.New(fmt.Sprintf("collection configuration error: unrecognized backup strategy type '%v'", collection.Spec.StrategyName)) } +func (s *Service) CleanUpOlderVersions(versions []UploadedVersion) bool { + logrus.Infof("Cleaning up older versions") + isError := false + + for _, version := range versions { + if err := s.Delete(&version); err != nil { + logrus.Errorf("Consistency error! Cannot delete version id=%v, of collectionId=%v, error: %v", version.Id, version.CollectionId, err) + isError = true + } else { + logrus.Debugf("Deleted version: id=%v, version=%v, of collectionId=%v", version.Id, version.VersionNumber, version.CollectionId) + } + } + + return isError +} + +func (s *Service) Delete(version *UploadedVersion) error { + if err := s.storage.Delete(context.TODO(), version.GetTargetPath()); err != nil { + return errors.New(fmt.Sprintf("cannot delete from storage at path '%v', error: %v", version.GetTargetPath(), err)) + } + if err, _ := s.repository.delete(version); err != nil { + return errors.New(fmt.Sprintf("cannot delete version from database - id=%v, version=%v, error=%v", version.Id, version.VersionNumber, err)) + } + return nil +} + +func (s *Service) RegisterVersion(version *UploadedVersion) error { + return s.repository.create(version) +} + // NewService is a factory method that knows how to construct a Storage provider, distincting multiple types of providers func NewService(db *gorm.DB, driverUrl string, isUsingGCS bool) (Service, error) { repository := VersionsRepository{db: db}