-
-
Notifications
You must be signed in to change notification settings - Fork 831
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[server] Add support for file-data (#2662)
## Description ## Tests
- Loading branch information
Showing
23 changed files
with
1,589 additions
and
48 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,122 @@ | ||
| package filedata | ||
|
|
||
| import ( | ||
| "fmt" | ||
| "github.com/ente-io/museum/ente" | ||
| ) | ||
|
|
||
| type Entity struct { | ||
| FileID int64 `json:"fileID"` | ||
| Type ente.ObjectType `json:"type"` | ||
| EncryptedData string `json:"encryptedData"` | ||
| DecryptionHeader string `json:"decryptionHeader"` | ||
| } | ||
|
|
||
| // GetFilesData should only be used for getting the preview video playlist and derived metadata. | ||
| type GetFilesData struct { | ||
| FileIDs []int64 `json:"fileIDs" binding:"required"` | ||
| Type ente.ObjectType `json:"type" binding:"required"` | ||
| } | ||
|
|
||
| func (g *GetFilesData) Validate() error { | ||
| if g.Type != ente.PreviewVideo && g.Type != ente.MlData { | ||
| return ente.NewBadRequestWithMessage(fmt.Sprintf("unsupported object type %s", g.Type)) | ||
| } | ||
| if len(g.FileIDs) == 0 { | ||
| return ente.NewBadRequestWithMessage("fileIDs are required") | ||
| } | ||
| if len(g.FileIDs) > 200 { | ||
| return ente.NewBadRequestWithMessage("fileIDs should be less than or equal to 200") | ||
| } | ||
| return nil | ||
| } | ||
|
|
||
| type GetFileData struct { | ||
| FileID int64 `form:"fileID" binding:"required"` | ||
| Type ente.ObjectType `form:"type" binding:"required"` | ||
| } | ||
|
|
||
| func (g *GetFileData) Validate() error { | ||
| if g.Type != ente.PreviewVideo && g.Type != ente.MlData { | ||
| return ente.NewBadRequestWithMessage(fmt.Sprintf("unsupported object type %s", g.Type)) | ||
| } | ||
| return nil | ||
| } | ||
|
|
||
| type GetFilesDataResponse struct { | ||
| Data []Entity `json:"data"` | ||
| PendingIndexFileIDs []int64 `json:"pendingIndexFileIDs"` | ||
| ErrFileIDs []int64 `json:"errFileIDs"` | ||
| } | ||
|
|
||
| // S3FileMetadata stuck represents the metadata that is stored in the S3 bucket for non-file type metadata | ||
| // that is stored in the S3 bucket. | ||
| type S3FileMetadata struct { | ||
| Version int `json:"v"` | ||
| EncryptedData string `json:"encryptedData"` | ||
| DecryptionHeader string `json:"header"` | ||
| Client string `json:"client"` | ||
| } | ||
|
|
||
| type GetPreviewURLRequest struct { | ||
| FileID int64 `form:"fileID" binding:"required"` | ||
| Type ente.ObjectType `form:"type" binding:"required"` | ||
| } | ||
|
|
||
| func (g *GetPreviewURLRequest) Validate() error { | ||
| if g.Type != ente.PreviewVideo && g.Type != ente.PreviewImage { | ||
| return ente.NewBadRequestWithMessage(fmt.Sprintf("unsupported object type %s", g.Type)) | ||
| } | ||
| return nil | ||
| } | ||
|
|
||
| type PreviewUploadUrlRequest struct { | ||
| FileID int64 `form:"fileID" binding:"required"` | ||
| Type ente.ObjectType `form:"type" binding:"required"` | ||
| } | ||
|
|
||
| func (g *PreviewUploadUrlRequest) Validate() error { | ||
| if g.Type != ente.PreviewVideo && g.Type != ente.PreviewImage { | ||
| return ente.NewBadRequestWithMessage(fmt.Sprintf("unsupported object type %s", g.Type)) | ||
| } | ||
| return nil | ||
| } | ||
|
|
||
| // Row represents the data that is stored in the file_data table. | ||
| type Row struct { | ||
| FileID int64 | ||
| UserID int64 | ||
| Type ente.ObjectType | ||
| // If a file type has multiple objects, then the size is the sum of all the objects. | ||
| Size int64 | ||
| LatestBucket string | ||
| ReplicatedBuckets []string | ||
| DeleteFromBuckets []string | ||
| InflightReplicas []string | ||
| PendingSync bool | ||
| IsDeleted bool | ||
| SyncLockedTill int64 | ||
| CreatedAt int64 | ||
| UpdatedAt int64 | ||
| } | ||
|
|
||
| // S3FileMetadataObjectKey returns the object key for the metadata stored in the S3 bucket. | ||
| func (r *Row) S3FileMetadataObjectKey() string { | ||
| if r.Type == ente.MlData { | ||
| return derivedMetaPath(r.FileID, r.UserID) | ||
| } | ||
| if r.Type == ente.PreviewVideo { | ||
| return previewVideoPlaylist(r.FileID, r.UserID) | ||
| } | ||
| panic(fmt.Sprintf("S3FileMetadata should not be written for %s type", r.Type)) | ||
| } | ||
|
|
||
| // GetS3FileObjectKey returns the object key for the file data stored in the S3 bucket. | ||
| func (r *Row) GetS3FileObjectKey() string { | ||
| if r.Type == ente.PreviewVideo { | ||
| return previewVideoPath(r.FileID, r.UserID) | ||
| } else if r.Type == ente.PreviewImage { | ||
| return previewImagePath(r.FileID, r.UserID) | ||
| } | ||
| panic(fmt.Sprintf("unsupported object type %s", r.Type)) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,53 @@ | ||
| package filedata | ||
|
|
||
| import ( | ||
| "fmt" | ||
| "github.com/ente-io/museum/ente" | ||
| ) | ||
|
|
||
| // BasePrefix returns the base prefix for all objects related to a file. To check if the file data is deleted, | ||
| // ensure that there's no file in the S3 bucket with this prefix. | ||
| func BasePrefix(fileID int64, ownerID int64) string { | ||
| return fmt.Sprintf("%d/file-data/%d/", ownerID, fileID) | ||
| } | ||
|
|
||
| func AllObjects(fileID int64, ownerID int64, oType ente.ObjectType) []string { | ||
| switch oType { | ||
| case ente.PreviewVideo: | ||
| return []string{previewVideoPath(fileID, ownerID), previewVideoPlaylist(fileID, ownerID)} | ||
| case ente.MlData: | ||
| return []string{derivedMetaPath(fileID, ownerID)} | ||
| case ente.PreviewImage: | ||
| return []string{previewImagePath(fileID, ownerID)} | ||
| default: | ||
| // throw panic saying current object type is not supported | ||
| panic(fmt.Sprintf("object type %s is not supported", oType)) | ||
| } | ||
| } | ||
|
|
||
| func PreviewUrl(fileID int64, ownerID int64, oType ente.ObjectType) string { | ||
| switch oType { | ||
| case ente.PreviewVideo: | ||
| return previewVideoPath(fileID, ownerID) | ||
| case ente.PreviewImage: | ||
| return previewImagePath(fileID, ownerID) | ||
| default: | ||
| panic(fmt.Sprintf("object type %s is not supported", oType)) | ||
| } | ||
| } | ||
|
|
||
| func previewVideoPath(fileID int64, ownerID int64) string { | ||
| return fmt.Sprintf("%s%s", BasePrefix(fileID, ownerID), string(ente.PreviewVideo)) | ||
| } | ||
|
|
||
| func previewVideoPlaylist(fileID int64, ownerID int64) string { | ||
| return fmt.Sprintf("%s%s", previewVideoPath(fileID, ownerID), "_playlist.m3u8") | ||
| } | ||
|
|
||
| func previewImagePath(fileID int64, ownerID int64) string { | ||
| return fmt.Sprintf("%s%s", BasePrefix(fileID, ownerID), string(ente.PreviewImage)) | ||
| } | ||
|
|
||
| func derivedMetaPath(fileID int64, ownerID int64) string { | ||
| return fmt.Sprintf("%s%s", BasePrefix(fileID, ownerID), string(ente.MlData)) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,66 @@ | ||
| package filedata | ||
|
|
||
| import ( | ||
| "fmt" | ||
| "github.com/ente-io/museum/ente" | ||
| ) | ||
|
|
||
| type PutFileDataRequest struct { | ||
| FileID int64 `json:"fileID" binding:"required"` | ||
| Type ente.ObjectType `json:"type" binding:"required"` | ||
| EncryptedData *string `json:"encryptedData,omitempty"` | ||
| DecryptionHeader *string `json:"decryptionHeader,omitempty"` | ||
| // ObjectKey is the key of the object in the S3 bucket. This is needed while putting the object in the S3 bucket. | ||
| ObjectKey *string `json:"objectKey,omitempty"` | ||
| // size of the object that is being uploaded. This helps in checking the size of the object that is being uploaded. | ||
| ObjectSize *int64 `json:"objectSize,omitempty"` | ||
| Version *int `json:"version,omitempty"` | ||
| } | ||
|
|
||
| func (r PutFileDataRequest) isEncDataPresent() bool { | ||
| return r.EncryptedData != nil && r.DecryptionHeader != nil && *r.EncryptedData != "" && *r.DecryptionHeader != "" | ||
| } | ||
|
|
||
| func (r PutFileDataRequest) isObjectDataPresent() bool { | ||
| return r.ObjectKey != nil && *r.ObjectKey != "" && r.ObjectSize != nil && *r.ObjectSize > 0 | ||
| } | ||
|
|
||
| func (r PutFileDataRequest) Validate() error { | ||
| switch r.Type { | ||
| case ente.PreviewVideo: | ||
| if !r.isEncDataPresent() || !r.isObjectDataPresent() { | ||
| return ente.NewBadRequestWithMessage("object and metadata are required") | ||
| } | ||
| case ente.PreviewImage: | ||
| if !r.isObjectDataPresent() || r.isEncDataPresent() { | ||
| return ente.NewBadRequestWithMessage("object (only) data is required for preview image") | ||
| } | ||
| case ente.MlData: | ||
| if !r.isEncDataPresent() || r.isObjectDataPresent() { | ||
| return ente.NewBadRequestWithMessage("encryptedData and decryptionHeader (only) are required for derived meta") | ||
| } | ||
| default: | ||
| return ente.NewBadRequestWithMessage(fmt.Sprintf("invalid object type %s", r.Type)) | ||
| } | ||
| return nil | ||
| } | ||
|
|
||
| func (r PutFileDataRequest) S3FileMetadataObjectKey(ownerID int64) string { | ||
| if r.Type == ente.MlData { | ||
| return derivedMetaPath(r.FileID, ownerID) | ||
| } | ||
| if r.Type == ente.PreviewVideo { | ||
| return previewVideoPlaylist(r.FileID, ownerID) | ||
| } | ||
| panic(fmt.Sprintf("S3FileMetadata should not be written for %s type", r.Type)) | ||
| } | ||
|
|
||
| func (r PutFileDataRequest) S3FileObjectKey(ownerID int64) string { | ||
| if r.Type == ente.PreviewVideo { | ||
| return previewVideoPath(r.FileID, ownerID) | ||
| } | ||
| if r.Type == ente.PreviewImage { | ||
| return previewImagePath(r.FileID, ownerID) | ||
| } | ||
| panic(fmt.Sprintf("S3FileObjectKey should not be written for %s type", r.Type)) | ||
| } |
Oops, something went wrong.