From c2e53c6ec9c0803342a4fa1ac6d1cbf6c34bc2f7 Mon Sep 17 00:00:00 2001
From: Neeraj Gupta <254676+ua741@users.noreply.github.com>
Date: Wed, 4 Sep 2024 14:39:36 +0530
Subject: [PATCH 1/7] [server] Clean up

---
 server/ente/filedata/filedata.go             |  1 +
 server/ente/filedata/putfiledata.go          | 30 ++------------------
 server/pkg/api/file_data.go                  |  2 +-
 server/pkg/controller/filedata/controller.go | 15 ++--------
 server/pkg/controller/filedata/delete.go     |  4 +++
 5 files changed, 11 insertions(+), 41 deletions(-)

diff --git a/server/ente/filedata/filedata.go b/server/ente/filedata/filedata.go
index 3db26fefb0..24457364cd 100644
--- a/server/ente/filedata/filedata.go
+++ b/server/ente/filedata/filedata.go
@@ -61,6 +61,7 @@ type S3FileMetadata struct {
 type GetPreviewURLRequest struct {
 	FileID int64           `form:"fileID" binding:"required"`
 	Type   ente.ObjectType `form:"type" binding:"required"`
+	Suffix *string         `form:"suffix"`
 }
 
 func (g *GetPreviewURLRequest) Validate() error {
diff --git a/server/ente/filedata/putfiledata.go b/server/ente/filedata/putfiledata.go
index 58394989da..dcd2a5d67b 100644
--- a/server/ente/filedata/putfiledata.go
+++ b/server/ente/filedata/putfiledata.go
@@ -10,33 +10,17 @@ type PutFileDataRequest struct {
 	Type             ente.ObjectType `json:"type" binding:"required"`
 	EncryptedData    *string         `json:"encryptedData,omitempty"`
 	DecryptionHeader *string         `json:"decryptionHeader,omitempty"`
-	// ObjectKey is the key of the object in the S3 bucket. This is needed while putting the object in the S3 bucket.
-	ObjectKey *string `json:"objectKey,omitempty"`
-	// size of the object that is being uploaded. This helps in checking the size of the object that is being uploaded.
-	ObjectSize *int64 `json:"objectSize,omitempty"`
-	Version    *int   `json:"version,omitempty"`
+	Version          *int            `json:"version,omitempty"`
 }
 
 func (r PutFileDataRequest) isEncDataPresent() bool {
 	return r.EncryptedData != nil && r.DecryptionHeader != nil && *r.EncryptedData != "" && *r.DecryptionHeader != ""
 }
 
-func (r PutFileDataRequest) isObjectDataPresent() bool {
-	return r.ObjectKey != nil && *r.ObjectKey != "" && r.ObjectSize != nil && *r.ObjectSize > 0
-}
-
 func (r PutFileDataRequest) Validate() error {
 	switch r.Type {
-	case ente.PreviewVideo:
-		if !r.isEncDataPresent() || !r.isObjectDataPresent() {
-			return ente.NewBadRequestWithMessage("object and metadata are required")
-		}
-	case ente.PreviewImage:
-		if !r.isObjectDataPresent() || r.isEncDataPresent() {
-			return ente.NewBadRequestWithMessage("object (only) data is required for preview image")
-		}
 	case ente.MlData:
-		if !r.isEncDataPresent() || r.isObjectDataPresent() {
+		if !r.isEncDataPresent() {
 			return ente.NewBadRequestWithMessage("encryptedData and decryptionHeader (only) are required for derived meta")
 		}
 	default:
@@ -49,18 +33,10 @@ func (r PutFileDataRequest) S3FileMetadataObjectKey(ownerID int64) string {
 	if r.Type == ente.MlData {
 		return derivedMetaPath(r.FileID, ownerID)
 	}
-	if r.Type == ente.PreviewVideo {
-		return previewVideoPlaylist(r.FileID, ownerID)
-	}
+
 	panic(fmt.Sprintf("S3FileMetadata should not be written for %s type", r.Type))
 }
 
 func (r PutFileDataRequest) S3FileObjectKey(ownerID int64) string {
-	if r.Type == ente.PreviewVideo {
-		return previewVideoPath(r.FileID, ownerID)
-	}
-	if r.Type == ente.PreviewImage {
-		return previewImagePath(r.FileID, ownerID)
-	}
 	panic(fmt.Sprintf("S3FileObjectKey should not be written for %s type", r.Type))
 }
diff --git a/server/pkg/api/file_data.go b/server/pkg/api/file_data.go
index 36c863e65c..94532ed4bb 100644
--- a/server/pkg/api/file_data.go
+++ b/server/pkg/api/file_data.go
@@ -25,7 +25,7 @@ func (h *FileHandler) PutFileData(ctx *gin.Context) {
 		version := 1
 		reqInt.Version = &version
 	}
-	err := h.FileDataCtrl.InsertOrUpdate(ctx, &req)
+	err := h.FileDataCtrl.InsertOrUpdateMetadata(ctx, &req)
 	if err != nil {
 		handler.Error(ctx, err)
 
diff --git a/server/pkg/controller/filedata/controller.go b/server/pkg/controller/filedata/controller.go
index 1c8465d870..251aeb8b34 100644
--- a/server/pkg/controller/filedata/controller.go
+++ b/server/pkg/controller/filedata/controller.go
@@ -19,7 +19,6 @@ import (
 	"github.com/ente-io/stacktrace"
 	"github.com/gin-gonic/gin"
 	log "github.com/sirupsen/logrus"
-	"strings"
 	"sync"
 	gTime "time"
 )
@@ -75,7 +74,7 @@ func New(repo *fileDataRepo.Repository,
 	}
 }
 
-func (c *Controller) InsertOrUpdate(ctx *gin.Context, req *fileData.PutFileDataRequest) error {
+func (c *Controller) InsertOrUpdateMetadata(ctx *gin.Context, req *fileData.PutFileDataRequest) error {
 	if err := req.Validate(); err != nil {
 		return stacktrace.Propagate(err, "validation failed")
 	}
@@ -84,21 +83,11 @@ func (c *Controller) InsertOrUpdate(ctx *gin.Context, req *fileData.PutFileDataR
 	if err != nil {
 		return stacktrace.Propagate(err, "")
 	}
-	if req.Type != ente.MlData && req.Type != ente.PreviewVideo {
+	if req.Type != ente.MlData {
 		return stacktrace.Propagate(ente.NewBadRequestWithMessage("unsupported object type "+string(req.Type)), "")
 	}
 	fileOwnerID := userID
 	bucketID := c.S3Config.GetBucketID(req.Type)
-	if req.Type == ente.PreviewVideo {
-		fileObjectKey := req.S3FileObjectKey(fileOwnerID)
-		if !strings.Contains(*req.ObjectKey, fileObjectKey) {
-			return stacktrace.Propagate(ente.NewBadRequestWithMessage("objectKey should contain the file object key"), "")
-		}
-		err = c.copyObject(*req.ObjectKey, fileObjectKey, bucketID)
-		if err != nil {
-			return err
-		}
-	}
 	objectKey := req.S3FileMetadataObjectKey(fileOwnerID)
 	obj := fileData.S3FileMetadata{
 		Version:          *req.Version,
diff --git a/server/pkg/controller/filedata/delete.go b/server/pkg/controller/filedata/delete.go
index c46eba9d66..735c1da10c 100644
--- a/server/pkg/controller/filedata/delete.go
+++ b/server/pkg/controller/filedata/delete.go
@@ -5,6 +5,7 @@ import (
 	"database/sql"
 	"errors"
 	"fmt"
+	"github.com/ente-io/museum/ente"
 	"github.com/ente-io/museum/ente/filedata"
 	fileDataRepo "github.com/ente-io/museum/pkg/repo/filedata"
 	enteTime "github.com/ente-io/museum/pkg/utils/time"
@@ -79,6 +80,9 @@ func (c *Controller) deleteFileRow(fileDataRow filedata.Row) error {
 		panic(fmt.Sprintf("file %d does not belong to user %d", fileID, ownerID))
 	}
 	ctxLogger := log.WithField("file_id", fileDataRow.DeleteFromBuckets).WithField("type", fileDataRow.Type).WithField("user_id", fileDataRow.UserID)
+	if fileDataRow.Type != ente.MlData {
+		panic(fmt.Sprintf("unsupported object type for filedata deletion %s", fileDataRow.Type))
+	}
 	objectKeys := filedata.AllObjects(fileID, ownerID, fileDataRow.Type)
 	bucketColumnMap, err := getMapOfBucketItToColumn(fileDataRow)
 	if err != nil {

From 41c242a0ee698a038224ee6bc8a42b0fbcb4abe3 Mon Sep 17 00:00:00 2001
From: Neeraj Gupta <254676+ua741@users.noreply.github.com>
Date: Wed, 4 Sep 2024 16:05:50 +0530
Subject: [PATCH 2/7] [server] Ignore __debug_bin

---
 server/.gitignore             | 1 +
 server/ente/filedata/video.go | 1 +
 2 files changed, 2 insertions(+)
 create mode 100644 server/ente/filedata/video.go

diff --git a/server/.gitignore b/server/.gitignore
index 3c7fd79017..088f4ae0bc 100644
--- a/server/.gitignore
+++ b/server/.gitignore
@@ -8,3 +8,4 @@ tmp/**
 museum.yaml
 bin/**
 data/
+__debug_bin*
\ No newline at end of file
diff --git a/server/ente/filedata/video.go b/server/ente/filedata/video.go
new file mode 100644
index 0000000000..41fd5c6b9e
--- /dev/null
+++ b/server/ente/filedata/video.go
@@ -0,0 +1 @@
+package filedata

From aa482ea227e3ccbf1a7823b4f4fe2c1e3847be66 Mon Sep 17 00:00:00 2001
From: Neeraj Gupta <254676+ua741@users.noreply.github.com>
Date: Fri, 6 Sep 2024 12:01:16 +0530
Subject: [PATCH 3/7] [server] Return both objectID and url for previewUrl

---
 server/ente/base/id.go                        |  8 +++++
 server/ente/filedata/filedata.go              | 20 ++++++++-----
 server/ente/filedata/path.go                  | 30 ++++++++-----------
 server/pkg/api/file_data.go                   |  6 ++--
 .../pkg/controller/filedata/preview_files.go  | 11 ++++---
 5 files changed, 42 insertions(+), 33 deletions(-)

diff --git a/server/ente/base/id.go b/server/ente/base/id.go
index 559bc41542..6d23ce9ee9 100644
--- a/server/ente/base/id.go
+++ b/server/ente/base/id.go
@@ -30,6 +30,14 @@ func NewID(prefix string) (*string, error) {
 	return &result, nil
 }
 
+func MustNewID(prefix string) string {
+	id, err := NewID(prefix)
+	if err != nil {
+		panic(err)
+	}
+	return *id
+}
+
 func ServerReqID() string {
 	// Generate a nanoid with a custom alphabet and length of 22
 	id, err := NewID("ser")
diff --git a/server/ente/filedata/filedata.go b/server/ente/filedata/filedata.go
index 24457364cd..76d81cdf5d 100644
--- a/server/ente/filedata/filedata.go
+++ b/server/ente/filedata/filedata.go
@@ -2,6 +2,7 @@ package filedata
 
 import (
 	"fmt"
+
 	"github.com/ente-io/museum/ente"
 )
 
@@ -76,6 +77,11 @@ type PreviewUploadUrlRequest struct {
 	Type   ente.ObjectType `form:"type" binding:"required"`
 }
 
+type PreviewUploadUrl struct {
+	Id  string `json:"id" binding:"required"`
+	Url string `json:"url" binding:"required"`
+}
+
 func (g *PreviewUploadUrlRequest) Validate() error {
 	if g.Type != ente.PreviewVideo && g.Type != ente.PreviewImage {
 		return ente.NewBadRequestWithMessage(fmt.Sprintf("unsupported object type %s", g.Type))
@@ -106,18 +112,16 @@ func (r *Row) S3FileMetadataObjectKey() string {
 	if r.Type == ente.MlData {
 		return derivedMetaPath(r.FileID, r.UserID)
 	}
-	if r.Type == ente.PreviewVideo {
-		return previewVideoPlaylist(r.FileID, r.UserID)
-	}
+
 	panic(fmt.Sprintf("S3FileMetadata should not be written for %s type", r.Type))
 }
 
 // GetS3FileObjectKey returns the object key for the file data stored in the S3 bucket.
 func (r *Row) GetS3FileObjectKey() string {
-	if r.Type == ente.PreviewVideo {
-		return previewVideoPath(r.FileID, r.UserID)
-	} else if r.Type == ente.PreviewImage {
-		return previewImagePath(r.FileID, r.UserID)
-	}
+	//if r.Type == ente.PreviewVideo {
+	//	return previewVideoPath(r.FileID, r.UserID)
+	//} else if r.Type == ente.PreviewImage {
+	//	return previewImagePath(r.FileID, r.UserID)
+	//}
 	panic(fmt.Sprintf("unsupported object type %s", r.Type))
 }
diff --git a/server/ente/filedata/path.go b/server/ente/filedata/path.go
index d0ea908800..b6ca1fe4cc 100644
--- a/server/ente/filedata/path.go
+++ b/server/ente/filedata/path.go
@@ -2,7 +2,9 @@ package filedata
 
 import (
 	"fmt"
+
 	"github.com/ente-io/museum/ente"
+	"github.com/ente-io/museum/ente/base"
 )
 
 // BasePrefix returns the base prefix for all objects related to a file. To check if the file data is deleted,
@@ -13,39 +15,33 @@ func BasePrefix(fileID int64, ownerID int64) string {
 
 func AllObjects(fileID int64, ownerID int64, oType ente.ObjectType) []string {
 	switch oType {
-	case ente.PreviewVideo:
-		return []string{previewVideoPath(fileID, ownerID), previewVideoPlaylist(fileID, ownerID)}
 	case ente.MlData:
 		return []string{derivedMetaPath(fileID, ownerID)}
-	case ente.PreviewImage:
-		return []string{previewImagePath(fileID, ownerID)}
+
 	default:
 		// throw panic saying current object type is not supported
 		panic(fmt.Sprintf("object type %s is not supported", oType))
 	}
 }
 
-func PreviewUrl(fileID int64, ownerID int64, oType ente.ObjectType) string {
+func CompleteObjectKey(fileID int64, ownerID int64, oType ente.ObjectType, id string) string {
 	switch oType {
 	case ente.PreviewVideo:
-		return previewVideoPath(fileID, ownerID)
 	case ente.PreviewImage:
-		return previewImagePath(fileID, ownerID)
+		return fmt.Sprintf("%s%s/%s", BasePrefix(fileID, ownerID), string(oType), id)
 	default:
 		panic(fmt.Sprintf("object type %s is not supported", oType))
 	}
+	panic(fmt.Sprintf("object type %s is not supported", oType))
 }
 
-func previewVideoPath(fileID int64, ownerID int64) string {
-	return fmt.Sprintf("%s%s", BasePrefix(fileID, ownerID), string(ente.PreviewVideo))
-}
-
-func previewVideoPlaylist(fileID int64, ownerID int64) string {
-	return fmt.Sprintf("%s%s", previewVideoPath(fileID, ownerID), "_playlist.m3u8")
-}
-
-func previewImagePath(fileID int64, ownerID int64) string {
-	return fmt.Sprintf("%s%s", BasePrefix(fileID, ownerID), string(ente.PreviewImage))
+func NewUploadID(oType ente.ObjectType) string {
+	if oType == ente.PreviewVideo {
+		return base.MustNewID("pv")
+	} else if oType == ente.PreviewImage {
+		return base.MustNewID("pi")
+	}
+	panic(fmt.Sprintf("object type %s is not supported", oType))
 }
 
 func derivedMetaPath(fileID int64, ownerID int64) string {
diff --git a/server/pkg/api/file_data.go b/server/pkg/api/file_data.go
index 94532ed4bb..810998dc80 100644
--- a/server/pkg/api/file_data.go
+++ b/server/pkg/api/file_data.go
@@ -70,14 +70,12 @@ func (h *FileHandler) GetPreviewUploadURL(c *gin.Context) {
 		handler.Error(c, stacktrace.Propagate(ente.ErrBadRequest, fmt.Sprintf("Request binding failed %s", err)))
 		return
 	}
-	url, err := h.FileDataCtrl.PreviewUploadURL(c, request)
+	resp, err := h.FileDataCtrl.PreviewUploadURL(c, request)
 	if err != nil {
 		handler.Error(c, stacktrace.Propagate(err, ""))
 		return
 	}
-	c.JSON(http.StatusOK, gin.H{
-		"url": url,
-	})
+	c.JSON(http.StatusOK, resp)
 }
 
 func (h *FileHandler) GetPreviewURL(c *gin.Context) {
diff --git a/server/pkg/controller/filedata/preview_files.go b/server/pkg/controller/filedata/preview_files.go
index 4d6b0113ba..e5785e70f6 100644
--- a/server/pkg/controller/filedata/preview_files.go
+++ b/server/pkg/controller/filedata/preview_files.go
@@ -1,7 +1,6 @@
 package filedata
 
 import (
-	"fmt"
 	"github.com/ente-io/museum/ente"
 	"github.com/ente-io/museum/ente/filedata"
 	"github.com/ente-io/museum/pkg/utils/auth"
@@ -31,7 +30,7 @@ func (c *Controller) GetPreviewUrl(ctx *gin.Context, request filedata.GetPreview
 	return &enteUrl.URL, nil
 }
 
-func (c *Controller) PreviewUploadURL(ctx *gin.Context, request filedata.PreviewUploadUrlRequest) (*string, error) {
+func (c *Controller) PreviewUploadURL(ctx *gin.Context, request filedata.PreviewUploadUrlRequest) (*filedata.PreviewUploadUrl, error) {
 	if err := request.Validate(); err != nil {
 		return nil, err
 	}
@@ -43,12 +42,16 @@ func (c *Controller) PreviewUploadURL(ctx *gin.Context, request filedata.Preview
 	if err != nil {
 		return nil, stacktrace.Propagate(err, "")
 	}
+	id := filedata.NewUploadID(request.Type)
 	// note: instead of the final url, give a temp url for upload purpose.
-	uploadUrl := fmt.Sprintf("%s_temp_upload", filedata.PreviewUrl(request.FileID, fileOwnerID, request.Type))
+	uploadUrl := filedata.CompleteObjectKey(request.FileID, fileOwnerID, request.Type, id)
 	bucketID := c.S3Config.GetBucketID(request.Type)
 	enteUrl, err := c.getUploadURL(bucketID, uploadUrl)
 	if err != nil {
 		return nil, stacktrace.Propagate(err, "")
 	}
-	return &enteUrl.URL, nil
+	return &filedata.PreviewUploadUrl{
+		Id:  id,
+		Url: enteUrl.URL,
+	}, nil
 }

From 1fabaf9aaad272175d034560ddadbcaa8f694016 Mon Sep 17 00:00:00 2001
From: Neeraj Gupta <254676+ua741@users.noreply.github.com>
Date: Fri, 6 Sep 2024 12:02:29 +0530
Subject: [PATCH 4/7] [server] Request model for putting video preview

---
 server/ente/filedata/video.go           | 26 +++++++++++++++++++++++++
 server/pkg/controller/filedata/video.go |  1 +
 2 files changed, 27 insertions(+)
 create mode 100644 server/pkg/controller/filedata/video.go

diff --git a/server/ente/filedata/video.go b/server/ente/filedata/video.go
index 41fd5c6b9e..d5c876bec6 100644
--- a/server/ente/filedata/video.go
+++ b/server/ente/filedata/video.go
@@ -1 +1,27 @@
 package filedata
+
+import "github.com/ente-io/museum/ente"
+
+type PutVidRequest struct {
+	FileID                int64           `json:"fileID" binding:"required"`
+	Type                  ente.ObjectType `json:"type" binding:"required"`
+	ObjectID              string          `json:"objectID" binding:"required"`
+	ObjectSize            int64           `json:"objectSize" binding:"required"`
+	PlayListEncryptedData string          `json:"playListEncryptedData,omitempty"`
+	PlayListDecryptionKey string          `json:"playListDecryptionHeader,omitempty"`
+}
+
+func (r PutVidRequest) Validate() error {
+	switch r.Type {
+	case ente.PreviewVideo:
+		if r.PlayListEncryptedData == "" || r.PlayListDecryptionKey == "" {
+			return ente.NewBadRequestWithMessage("playListEncryptedData and playListDecryptionHeader are required for preview video")
+		}
+		if r.ObjectSize <= 0 {
+			return ente.NewBadRequestWithMessage("objectSize should be greater than 0")
+		}
+	default:
+		return ente.NewBadRequestWithMessage("invalid object type")
+	}
+	return nil
+}
diff --git a/server/pkg/controller/filedata/video.go b/server/pkg/controller/filedata/video.go
new file mode 100644
index 0000000000..41fd5c6b9e
--- /dev/null
+++ b/server/pkg/controller/filedata/video.go
@@ -0,0 +1 @@
+package filedata

From d4a68069baa8cc55f5d2f0e8fe215d541ccafd84 Mon Sep 17 00:00:00 2001
From: Neeraj Gupta <254676+ua741@users.noreply.github.com>
Date: Wed, 25 Sep 2024 15:40:52 +0530
Subject: [PATCH 5/7] [server] Add columns to store preview objects

---
 server/migrations/92_file_data_preview.down.sql | 3 +++
 server/migrations/92_file_data_preview.up.sql   | 7 +++++++
 2 files changed, 10 insertions(+)
 create mode 100644 server/migrations/92_file_data_preview.down.sql
 create mode 100644 server/migrations/92_file_data_preview.up.sql

diff --git a/server/migrations/92_file_data_preview.down.sql b/server/migrations/92_file_data_preview.down.sql
new file mode 100644
index 0000000000..98b1223b83
--- /dev/null
+++ b/server/migrations/92_file_data_preview.down.sql
@@ -0,0 +1,3 @@
+ALTER TABLE file_data
+    DROP COLUMN obj_id,
+    DROP COLUMN obj_nonce;
\ No newline at end of file
diff --git a/server/migrations/92_file_data_preview.up.sql b/server/migrations/92_file_data_preview.up.sql
new file mode 100644
index 0000000000..8e26d28a54
--- /dev/null
+++ b/server/migrations/92_file_data_preview.up.sql
@@ -0,0 +1,7 @@
+ALTER TYPE OBJECT_TYPE ADD VALUE 'vid_preview';
+ALTER TYPE OBJECT_TYPE ADD VALUE 'img_preview';
+ALTER TABLE file_data
+    ADD COLUMN obj_id TEXT,
+    ADD COLUMN obj_nonce TEXT;
+
+

From be615197fd5a7018e72ca32486571e2d0714507b Mon Sep 17 00:00:00 2001
From: Neeraj Gupta <254676+ua741@users.noreply.github.com>
Date: Wed, 25 Sep 2024 15:50:33 +0530
Subject: [PATCH 6/7] [server] Fix error in getting preview url

---
 server/ente/filedata/path.go | 1 +
 server/pkg/api/file_data.go  | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/server/ente/filedata/path.go b/server/ente/filedata/path.go
index b6ca1fe4cc..4d9e2ddff8 100644
--- a/server/ente/filedata/path.go
+++ b/server/ente/filedata/path.go
@@ -27,6 +27,7 @@ func AllObjects(fileID int64, ownerID int64, oType ente.ObjectType) []string {
 func CompleteObjectKey(fileID int64, ownerID int64, oType ente.ObjectType, id string) string {
 	switch oType {
 	case ente.PreviewVideo:
+		return fmt.Sprintf("%s%s/%s", BasePrefix(fileID, ownerID), string(oType), id)
 	case ente.PreviewImage:
 		return fmt.Sprintf("%s%s/%s", BasePrefix(fileID, ownerID), string(oType), id)
 	default:
diff --git a/server/pkg/api/file_data.go b/server/pkg/api/file_data.go
index 810998dc80..746dc9fc41 100644
--- a/server/pkg/api/file_data.go
+++ b/server/pkg/api/file_data.go
@@ -66,7 +66,7 @@ func (h *FileHandler) GetFileData(ctx *gin.Context) {
 
 func (h *FileHandler) GetPreviewUploadURL(c *gin.Context) {
 	var request fileData.PreviewUploadUrlRequest
-	if err := c.ShouldBindJSON(&request); err != nil {
+	if err := c.ShouldBindQuery(&request); err != nil {
 		handler.Error(c, stacktrace.Propagate(ente.ErrBadRequest, fmt.Sprintf("Request binding failed %s", err)))
 		return
 	}

From b8f1bce341a99c5f9fca5dd3d2f21c84c77264ab Mon Sep 17 00:00:00 2001
From: Neeraj Gupta <254676+ua741@users.noreply.github.com>
Date: Mon, 30 Sep 2024 14:58:10 +0530
Subject: [PATCH 7/7] [server] Initial support for storing video preview data

---
 server/cmd/museum/main.go                     |  2 +-
 server/ente/filedata/filedata.go              | 15 ++---
 server/ente/filedata/path.go                  | 18 +++--
 server/ente/filedata/putfiledata.go           | 12 ----
 server/ente/filedata/video.go                 | 31 ++++-----
 server/pkg/controller/filedata/controller.go  |  2 +-
 .../pkg/controller/filedata/preview_files.go  |  2 +-
 server/pkg/controller/filedata/replicate.go   |  5 ++
 server/pkg/controller/filedata/s3.go          | 20 ++++++
 server/pkg/controller/filedata/video.go       | 66 +++++++++++++++++++
 server/pkg/repo/filedata/repository.go        | 50 ++++++++++++--
 server/pkg/repo/object_cleanup.go             | 17 +++++
 12 files changed, 191 insertions(+), 49 deletions(-)

diff --git a/server/cmd/museum/main.go b/server/cmd/museum/main.go
index c4a17cb74f..5d444a6dd9 100644
--- a/server/cmd/museum/main.go
+++ b/server/cmd/museum/main.go
@@ -164,7 +164,7 @@ func main() {
 	fileRepo := &repo.FileRepository{DB: db, S3Config: s3Config, QueueRepo: queueRepo,
 		ObjectRepo: objectRepo, ObjectCleanupRepo: objectCleanupRepo,
 		ObjectCopiesRepo: objectCopiesRepo, UsageRepo: usageRepo}
-	fileDataRepo := &fileDataRepo.Repository{DB: db}
+	fileDataRepo := &fileDataRepo.Repository{DB: db, ObjectCleanupRepo: objectCleanupRepo}
 	familyRepo := &repo.FamilyRepository{DB: db}
 	trashRepo := &repo.TrashRepository{DB: db, ObjectRepo: objectRepo, FileRepo: fileRepo, QueueRepo: queueRepo}
 	publicCollectionRepo := repo.NewPublicCollectionRepository(db, viper.GetString("apps.public-albums"))
diff --git a/server/ente/filedata/filedata.go b/server/ente/filedata/filedata.go
index 76d81cdf5d..c186ee6d16 100644
--- a/server/ente/filedata/filedata.go
+++ b/server/ente/filedata/filedata.go
@@ -97,6 +97,8 @@ type Row struct {
 	// If a file type has multiple objects, then the size is the sum of all the objects.
 	Size              int64
 	LatestBucket      string
+	ObjectID          *string
+	ObjectNonce       *string
 	ReplicatedBuckets []string
 	DeleteFromBuckets []string
 	InflightReplicas  []string
@@ -109,19 +111,16 @@ type Row struct {
 
 // S3FileMetadataObjectKey returns the object key for the metadata stored in the S3 bucket.
 func (r *Row) S3FileMetadataObjectKey() string {
-	if r.Type == ente.MlData {
-		return derivedMetaPath(r.FileID, r.UserID)
+	if r.Type == ente.MlData || r.Type == ente.PreviewVideo {
+		return ObjectMedata(r.FileID, r.UserID, r.Type, r.ObjectID)
 	}
-
 	panic(fmt.Sprintf("S3FileMetadata should not be written for %s type", r.Type))
 }
 
 // GetS3FileObjectKey returns the object key for the file data stored in the S3 bucket.
 func (r *Row) GetS3FileObjectKey() string {
-	//if r.Type == ente.PreviewVideo {
-	//	return previewVideoPath(r.FileID, r.UserID)
-	//} else if r.Type == ente.PreviewImage {
-	//	return previewImagePath(r.FileID, r.UserID)
-	//}
+	if r.Type == ente.PreviewVideo || r.Type == ente.PreviewImage {
+		return ObjectKey(r.FileID, r.UserID, r.Type, r.ObjectID)
+	}
 	panic(fmt.Sprintf("unsupported object type %s", r.Type))
 }
diff --git a/server/ente/filedata/path.go b/server/ente/filedata/path.go
index 4d9e2ddff8..4a19462b1c 100644
--- a/server/ente/filedata/path.go
+++ b/server/ente/filedata/path.go
@@ -24,16 +24,26 @@ func AllObjects(fileID int64, ownerID int64, oType ente.ObjectType) []string {
 	}
 }
 
-func CompleteObjectKey(fileID int64, ownerID int64, oType ente.ObjectType, id string) string {
+func ObjectKey(fileID int64, ownerID int64, oType ente.ObjectType, id *string) string {
 	switch oType {
 	case ente.PreviewVideo:
-		return fmt.Sprintf("%s%s/%s", BasePrefix(fileID, ownerID), string(oType), id)
+		return fmt.Sprintf("%s%s/%s", BasePrefix(fileID, ownerID), string(oType), *id)
 	case ente.PreviewImage:
-		return fmt.Sprintf("%s%s/%s", BasePrefix(fileID, ownerID), string(oType), id)
+		return fmt.Sprintf("%s%s/%s", BasePrefix(fileID, ownerID), string(oType), *id)
 	default:
 		panic(fmt.Sprintf("object type %s is not supported", oType))
 	}
-	panic(fmt.Sprintf("object type %s is not supported", oType))
+}
+
+func ObjectMedata(fileID int64, ownerID int64, oType ente.ObjectType, id *string) string {
+	switch oType {
+	case ente.PreviewVideo:
+		return fmt.Sprintf("%s_playlist", ObjectKey(fileID, ownerID, oType, id))
+	case ente.MlData:
+		return fmt.Sprintf("%s%s", BasePrefix(fileID, ownerID), string(oType))
+	default:
+		panic(fmt.Sprintf("ObjectMetadata not supported for type %s", string(oType)))
+	}
 }
 
 func NewUploadID(oType ente.ObjectType) string {
diff --git a/server/ente/filedata/putfiledata.go b/server/ente/filedata/putfiledata.go
index dcd2a5d67b..a4b4f7bf5b 100644
--- a/server/ente/filedata/putfiledata.go
+++ b/server/ente/filedata/putfiledata.go
@@ -28,15 +28,3 @@ func (r PutFileDataRequest) Validate() error {
 	}
 	return nil
 }
-
-func (r PutFileDataRequest) S3FileMetadataObjectKey(ownerID int64) string {
-	if r.Type == ente.MlData {
-		return derivedMetaPath(r.FileID, ownerID)
-	}
-
-	panic(fmt.Sprintf("S3FileMetadata should not be written for %s type", r.Type))
-}
-
-func (r PutFileDataRequest) S3FileObjectKey(ownerID int64) string {
-	panic(fmt.Sprintf("S3FileObjectKey should not be written for %s type", r.Type))
-}
diff --git a/server/ente/filedata/video.go b/server/ente/filedata/video.go
index d5c876bec6..6068a3494b 100644
--- a/server/ente/filedata/video.go
+++ b/server/ente/filedata/video.go
@@ -2,26 +2,21 @@ package filedata
 
 import "github.com/ente-io/museum/ente"
 
-type PutVidRequest struct {
-	FileID                int64           `json:"fileID" binding:"required"`
-	Type                  ente.ObjectType `json:"type" binding:"required"`
-	ObjectID              string          `json:"objectID" binding:"required"`
-	ObjectSize            int64           `json:"objectSize" binding:"required"`
-	PlayListEncryptedData string          `json:"playListEncryptedData,omitempty"`
-	PlayListDecryptionKey string          `json:"playListDecryptionHeader,omitempty"`
+type VidPreviewRequest struct {
+	FileID        int64  `json:"fileID" binding:"required"`
+	ObjectID      string `json:"objectID" binding:"required"`
+	ObjectNonce   string `json:"objectNonce" binding:"required"`
+	ObjectSize    int64  `json:"objectSize" binding:"required"`
+	Playlist      string `json:"playlist" binding:"required"`
+	PlayListNonce string `json:"playListNonce" binding:"required"`
 }
 
-func (r PutVidRequest) Validate() error {
-	switch r.Type {
-	case ente.PreviewVideo:
-		if r.PlayListEncryptedData == "" || r.PlayListDecryptionKey == "" {
-			return ente.NewBadRequestWithMessage("playListEncryptedData and playListDecryptionHeader are required for preview video")
-		}
-		if r.ObjectSize <= 0 {
-			return ente.NewBadRequestWithMessage("objectSize should be greater than 0")
-		}
-	default:
-		return ente.NewBadRequestWithMessage("invalid object type")
+func (r VidPreviewRequest) Validate() error {
+	if r.Playlist == "" || r.PlayListNonce == "" {
+		return ente.NewBadRequestWithMessage("playlist and playListNonce are required for preview video")
+	}
+	if r.ObjectNonce == "" {
+		return ente.NewBadRequestWithMessage("objectNonce is required for preview video")
 	}
 	return nil
 }
diff --git a/server/pkg/controller/filedata/controller.go b/server/pkg/controller/filedata/controller.go
index 251aeb8b34..15b88fa4a5 100644
--- a/server/pkg/controller/filedata/controller.go
+++ b/server/pkg/controller/filedata/controller.go
@@ -88,7 +88,7 @@ func (c *Controller) InsertOrUpdateMetadata(ctx *gin.Context, req *fileData.PutF
 	}
 	fileOwnerID := userID
 	bucketID := c.S3Config.GetBucketID(req.Type)
-	objectKey := req.S3FileMetadataObjectKey(fileOwnerID)
+	objectKey := fileData.ObjectMedata(req.FileID, fileOwnerID, req.Type, nil)
 	obj := fileData.S3FileMetadata{
 		Version:          *req.Version,
 		EncryptedData:    *req.EncryptedData,
diff --git a/server/pkg/controller/filedata/preview_files.go b/server/pkg/controller/filedata/preview_files.go
index e5785e70f6..9572b23541 100644
--- a/server/pkg/controller/filedata/preview_files.go
+++ b/server/pkg/controller/filedata/preview_files.go
@@ -44,7 +44,7 @@ func (c *Controller) PreviewUploadURL(ctx *gin.Context, request filedata.Preview
 	}
 	id := filedata.NewUploadID(request.Type)
 	// note: instead of the final url, give a temp url for upload purpose.
-	uploadUrl := filedata.CompleteObjectKey(request.FileID, fileOwnerID, request.Type, id)
+	uploadUrl := filedata.ObjectKey(request.FileID, fileOwnerID, request.Type, &id)
 	bucketID := c.S3Config.GetBucketID(request.Type)
 	enteUrl, err := c.getUploadURL(bucketID, uploadUrl)
 	if err != nil {
diff --git a/server/pkg/controller/filedata/replicate.go b/server/pkg/controller/filedata/replicate.go
index fa34aa2e9b..e52f460215 100644
--- a/server/pkg/controller/filedata/replicate.go
+++ b/server/pkg/controller/filedata/replicate.go
@@ -5,6 +5,7 @@ import (
 	"database/sql"
 	"errors"
 	"fmt"
+	"github.com/ente-io/museum/ente"
 	"github.com/ente-io/museum/ente/filedata"
 	fileDataRepo "github.com/ente-io/museum/pkg/repo/filedata"
 	enteTime "github.com/ente-io/museum/pkg/utils/time"
@@ -67,6 +68,10 @@ func (c *Controller) tryReplicate() error {
 		}
 		return err
 	}
+	if row.Type == ente.PreviewVideo {
+		log.Infof("Skipping replication for preview video %d", row.FileID)
+		return nil
+	}
 	err = c.replicateRowData(ctx, *row)
 	if err != nil {
 		log.Errorf("Could not delete file data: %s", err)
diff --git a/server/pkg/controller/filedata/s3.go b/server/pkg/controller/filedata/s3.go
index e6f1200bf8..307894e2c1 100644
--- a/server/pkg/controller/filedata/s3.go
+++ b/server/pkg/controller/filedata/s3.go
@@ -92,6 +92,26 @@ func (c *Controller) uploadObject(obj fileData.S3FileMetadata, objectKey string,
 	return int64(len(embeddingObj)), nil
 }
 
+func (c *Controller) verifySize(bucketID string, objectKey string, expectedSize int64) error {
+	s3Client := c.S3Config.GetS3Client(bucketID)
+	bucket := c.S3Config.GetBucket(bucketID)
+	res, err := s3Client.HeadObject(&s3.HeadObjectInput{
+		Bucket: bucket,
+		Key:    &objectKey,
+	})
+	if err != nil {
+		return stacktrace.Propagate(err, "Fetching object info from bucket %s failed", *bucket)
+	}
+
+	if *res.ContentLength != expectedSize {
+		err = fmt.Errorf("size of the uploaded file (%d) does not match the expected size (%d) in bucket %s",
+			*res.ContentLength, expectedSize, *bucket)
+		//c.notifyDiscord(fmt.Sprint(err))
+		return stacktrace.Propagate(err, "")
+	}
+	return nil
+}
+
 // copyObject copies the object from srcObjectKey to destObjectKey in the same bucket and returns the object size
 func (c *Controller) copyObject(srcObjectKey string, destObjectKey string, bucketID string) error {
 	bucket := c.S3Config.GetBucket(bucketID)
diff --git a/server/pkg/controller/filedata/video.go b/server/pkg/controller/filedata/video.go
index 41fd5c6b9e..9dbd221c32 100644
--- a/server/pkg/controller/filedata/video.go
+++ b/server/pkg/controller/filedata/video.go
@@ -1 +1,67 @@
 package filedata
+
+import (
+	"context"
+	"github.com/ente-io/museum/ente"
+	"github.com/ente-io/museum/ente/filedata"
+	"github.com/ente-io/museum/pkg/utils/auth"
+	"github.com/ente-io/museum/pkg/utils/network"
+	"github.com/ente-io/stacktrace"
+	"github.com/gin-gonic/gin"
+	log "github.com/sirupsen/logrus"
+)
+
+func (c *Controller) InsertVideoPreview(ctx *gin.Context, req filedata.VidPreviewRequest) error {
+	if err := req.Validate(); err != nil {
+		return stacktrace.Propagate(err, "validation failed")
+	}
+	userID := auth.GetUserID(ctx.Request.Header)
+	err := c._validatePermission(ctx, req.FileID, userID)
+	if err != nil {
+		return stacktrace.Propagate(err, "")
+	}
+	fileOwnerID := userID
+
+	bucketID := c.S3Config.GetBucketID(ente.PreviewVideo)
+	fileObjectKey := filedata.ObjectKey(req.FileID, fileOwnerID, ente.PreviewVideo, &req.ObjectID)
+	objectKey := filedata.ObjectMedata(req.FileID, fileOwnerID, ente.PreviewVideo, &req.ObjectID)
+
+	if sizeErr := c.verifySize(bucketID, fileObjectKey, req.ObjectSize); sizeErr != nil {
+		return stacktrace.Propagate(sizeErr, "failed to validate size")
+	}
+	// Start a goroutine to handle the upload and insert operations
+	go func() {
+		obj := filedata.S3FileMetadata{
+			Version:          1,
+			EncryptedData:    req.Playlist,
+			DecryptionHeader: req.PlayListNonce,
+			Client:           network.GetClientInfo(ctx),
+		}
+		logger := log.
+			WithField("objectKey", objectKey).
+			WithField("fileID", req.FileID).
+			WithField("type", ente.PreviewVideo)
+		size, uploadErr := c.uploadObject(obj, objectKey, bucketID)
+		if uploadErr != nil {
+			logger.WithError(uploadErr).Error("upload failed")
+			return
+		}
+		row := filedata.Row{
+			FileID:       req.FileID,
+			Type:         ente.PreviewVideo,
+			UserID:       fileOwnerID,
+			Size:         size + req.ObjectSize,
+			LatestBucket: bucketID,
+			ObjectID:     &req.ObjectID,
+			ObjectNonce:  &req.ObjectNonce,
+		}
+
+		dbInsertErr := c.Repo.InsertOrUpdatePreviewData(context.Background(), row)
+		if dbInsertErr != nil {
+			logger.WithError(dbInsertErr).Error("insert or update failed")
+			return
+		}
+	}()
+	return nil
+
+}
diff --git a/server/pkg/repo/filedata/repository.go b/server/pkg/repo/filedata/repository.go
index ae024e5d2b..69ff74e7f7 100644
--- a/server/pkg/repo/filedata/repository.go
+++ b/server/pkg/repo/filedata/repository.go
@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"github.com/ente-io/museum/ente"
 	"github.com/ente-io/museum/ente/filedata"
+	"github.com/ente-io/museum/pkg/repo"
 	"github.com/ente-io/museum/pkg/utils/array"
 	"github.com/ente-io/stacktrace"
 	"github.com/lib/pq"
@@ -14,7 +15,8 @@ import (
 
 // Repository defines the methods for inserting, updating, and retrieving file data.
 type Repository struct {
-	DB *sql.DB
+	DB                *sql.DB
+	ObjectCleanupRepo *repo.ObjectCleanupRepository
 }
 
 const (
@@ -54,8 +56,48 @@ func (r *Repository) InsertOrUpdate(ctx context.Context, data filedata.Row) erro
 	return nil
 }
 
+func (r *Repository) InsertOrUpdatePreviewData(ctx context.Context, data filedata.Row) error {
+	tx, err := r.DB.BeginTx(ctx, nil)
+	query := `
+        INSERT INTO file_data 
+            (file_id, user_id, data_type, size, latest_bucket, obj_id, obj_nonce ) 
+        VALUES 
+            ($1, $2, $3, $4, $5, $6, $7)
+        ON CONFLICT (file_id, data_type)
+        DO UPDATE SET 
+            size = EXCLUDED.size,
+            delete_from_buckets = array(
+                SELECT DISTINCT elem FROM unnest(
+                    array_append(
+                        array_cat(array_cat(file_data.replicated_buckets, file_data.delete_from_buckets), file_data.inflight_rep_buckets),
+                        CASE WHEN file_data.latest_bucket != EXCLUDED.latest_bucket THEN file_data.latest_bucket  END
+                    )
+                ) AS elem
+                WHERE elem IS NOT NULL AND elem != EXCLUDED.latest_bucket
+            ),
+            replicated_buckets = ARRAY[]::s3region[],
+            pending_sync = true,
+            latest_bucket = EXCLUDED.latest_bucket,
+            obj_id = EXCLUDED.obj_id,
+            obj_nonce = excluded.obj_nonce,
+            updated_at = now_utc_micro_seconds()
+        WHERE file_data.is_deleted = false`
+	_, err = tx.ExecContext(ctx, query,
+		data.FileID, data.UserID, string(data.Type), data.Size, data.LatestBucket, *data.ObjectID, *data.ObjectNonce)
+	if err != nil {
+		tx.Rollback()
+		return stacktrace.Propagate(err, "failed to insert file data")
+	}
+	err = r.ObjectCleanupRepo.RemoveTempObjectFromDC(ctx, tx, "", data.LatestBucket)
+	if err != nil {
+		tx.Rollback()
+		return stacktrace.Propagate(err, "failed to remove object from tempObjects")
+	}
+	return tx.Commit()
+}
+
 func (r *Repository) GetFilesData(ctx context.Context, oType ente.ObjectType, fileIDs []int64) ([]filedata.Row, error) {
-	rows, err := r.DB.QueryContext(ctx, `SELECT file_id, user_id, data_type, size, latest_bucket, replicated_buckets, delete_from_buckets, inflight_rep_buckets, pending_sync, is_deleted, sync_locked_till, created_at, updated_at
+	rows, err := r.DB.QueryContext(ctx, `SELECT file_id, user_id, data_type, size, latest_bucket, replicated_buckets, delete_from_buckets, inflight_rep_buckets, pending_sync, is_deleted, sync_locked_till, created_at, updated_at, obj_id, obj_nonce
 										FROM file_data
 										WHERE data_type = $1 AND file_id = ANY($2)`, string(oType), pq.Array(fileIDs))
 	if err != nil {
@@ -65,7 +107,7 @@ func (r *Repository) GetFilesData(ctx context.Context, oType ente.ObjectType, fi
 }
 
 func (r *Repository) GetFileData(ctx context.Context, fileIDs int64) ([]filedata.Row, error) {
-	rows, err := r.DB.QueryContext(ctx, `SELECT file_id, user_id, data_type, size, latest_bucket, replicated_buckets, delete_from_buckets,inflight_rep_buckets, pending_sync, is_deleted, sync_locked_till, created_at, updated_at
+	rows, err := r.DB.QueryContext(ctx, `SELECT file_id, user_id, data_type, size, latest_bucket, replicated_buckets, delete_from_buckets,inflight_rep_buckets, pending_sync, is_deleted, sync_locked_till, created_at, updated_at, obj_id, obj_nonce
 										FROM file_data
 										WHERE file_id = $1`, fileIDs)
 	if err != nil {
@@ -251,7 +293,7 @@ func convertRowsToFilesData(rows *sql.Rows) ([]filedata.Row, error) {
 	var filesData []filedata.Row
 	for rows.Next() {
 		var fileData filedata.Row
-		err := rows.Scan(&fileData.FileID, &fileData.UserID, &fileData.Type, &fileData.Size, &fileData.LatestBucket, pq.Array(&fileData.ReplicatedBuckets), pq.Array(&fileData.DeleteFromBuckets), pq.Array(&fileData.InflightReplicas), &fileData.PendingSync, &fileData.IsDeleted, &fileData.SyncLockedTill, &fileData.CreatedAt, &fileData.UpdatedAt)
+		err := rows.Scan(&fileData.FileID, &fileData.UserID, &fileData.Type, &fileData.Size, &fileData.LatestBucket, pq.Array(&fileData.ReplicatedBuckets), pq.Array(&fileData.DeleteFromBuckets), pq.Array(&fileData.InflightReplicas), &fileData.PendingSync, &fileData.IsDeleted, &fileData.SyncLockedTill, &fileData.CreatedAt, &fileData.UpdatedAt, &fileData.ObjectID, &fileData.ObjectNonce)
 		if err != nil {
 			return nil, stacktrace.Propagate(err, "")
 		}
diff --git a/server/pkg/repo/object_cleanup.go b/server/pkg/repo/object_cleanup.go
index d892b7157f..e568007963 100644
--- a/server/pkg/repo/object_cleanup.go
+++ b/server/pkg/repo/object_cleanup.go
@@ -4,6 +4,7 @@ import (
 	"context"
 	"database/sql"
 	"errors"
+	"fmt"
 
 	"github.com/ente-io/stacktrace"
 	log "github.com/sirupsen/logrus"
@@ -40,6 +41,22 @@ func (repo *ObjectCleanupRepository) RemoveTempObjectKey(ctx context.Context, tx
 	return stacktrace.Propagate(err, "")
 }
 
+// RemoveTempObjectFromDC will also return how many rows were affected
+func (repo *ObjectCleanupRepository) RemoveTempObjectFromDC(ctx context.Context, tx *sql.Tx, objectKey string, dc string) error {
+	res, err := tx.ExecContext(ctx, `DELETE FROM temp_objects WHERE object_key = $1 and bucket_id = $2`, objectKey, dc)
+	if err != nil {
+		return stacktrace.Propagate(err, "")
+	}
+	rowsAffected, err := res.RowsAffected()
+	if err != nil {
+		return stacktrace.Propagate(err, "")
+	}
+	if rowsAffected != 1 {
+		return stacktrace.Propagate(fmt.Errorf("only one row should be affected not %d", rowsAffected), "")
+	}
+	return nil
+}
+
 // GetExpiredObjects returns the list of object keys that have expired
 func (repo *ObjectCleanupRepository) GetAndLockExpiredObjects() (*sql.Tx, []ente.TempObject, error) {
 	tx, err := repo.DB.Begin()