[server] API to get file data status (#3868)

## Description
Note:
This API won't really return status/diff for deleted files. The clients
will primarily use this data to identify for which all files we already
have preview generated or it's ML inference is done.
This doesn't simulate perfect diff behaviour as we won't maintain a
tombstone entries for the deleted API.
## Tests

server/cmd/museum/main.go

@@ -424,6 +424,7 @@ func main() {
 	privateAPI.GET("/files/preview/v2/:fileID", fileHandler.GetThumbnail)
 
 	privateAPI.PUT("/files/data", fileHandler.PutFileData)
+	privateAPI.POST("/files/data/status-diff", fileHandler.FileDataStatusDiff)
 	privateAPI.POST("/files/data/fetch", fileHandler.GetFilesData)
 	privateAPI.GET("/files/data/fetch", fileHandler.GetFileData)
 	privateAPI.GET("/files/data/preview-upload-url", fileHandler.GetPreviewUploadURL)

server/ente/filedata/filedata.go

@@ -12,6 +12,19 @@ type Entity struct {
 	DecryptionHeader string          `json:"decryptionHeader"`
 }
 
+type IndexDiffRequest struct {
+	LastUpdated int64 `form:"lastUpdated" binding:"required"`
+}
+
+type IndexStatus struct {
+	FileID    int64           `json:"fileID" binding:"required"`
+	UserID    int64           `json:"userID" binding:"required"`
+	Type      ente.ObjectType `json:"type" binding:"required"`
+	IsDeleted bool            `json:"isDeleted" binding:"required"`
+	Size      int64           `json:"size"  binding:"required"`
+	UpdatedAt int64           `json:"updatedAt"  binding:"required"`
+}
+
 // GetFilesData should only be used for getting the preview video playlist and derived metadata.
 type GetFilesData struct {
 	FileIDs []int64         `json:"fileIDs" binding:"required"`

server/pkg/api/file_data.go

@@ -48,6 +48,22 @@ func (h *FileHandler) GetFilesData(ctx *gin.Context) {
 	ctx.JSON(http.StatusOK, resp)
 }
 
+// FileDataStatusDiff API won't really return status/diff for deleted files. The clients will primarily use this data to identify for which all files we already have preview generated or it's ML inference is done.
+// This doesn't simulate perfect diff behaviour as we won't maintain a tombstone entries for the deleted API.
+func (h *FileHandler) FileDataStatusDiff(ctx *gin.Context) {
+	var req fileData.IndexDiffRequest
+	if err := ctx.ShouldBindJSON(&req); err != nil {
+		ctx.JSON(http.StatusBadRequest, ente.NewBadRequestWithMessage(err.Error()))
+		return
+	}
+	resp, err := h.FileDataCtrl.FileDataStatusDiff(ctx, req)
+	if err != nil {
+		handler.Error(ctx, err)
+		return
+	}
+	ctx.JSON(http.StatusOK, resp)
+}
+
 func (h *FileHandler) GetFileData(ctx *gin.Context) {
 	var req fileData.GetFileData
 	if err := ctx.ShouldBindJSON(&req); err != nil {

server/pkg/controller/filedata/controller.go

@@ -313,3 +313,8 @@ func (c *Controller) _validatePermission(ctx *gin.Context, fileID int64, actorID
 	}
 	return nil
 }
+
+func (c *Controller) FileDataStatusDiff(ctx *gin.Context, req fileData.IndexDiffRequest) ([]fileData.IndexStatus, error) {
+	userID := auth.GetUserID(ctx.Request.Header)
+	return c.Repo.GetIndexStatusForUser(ctx, userID, req.LastUpdated, 5000)
+}

server/pkg/repo/filedata/repository.go

@@ -124,6 +124,26 @@ func (r *Repository) RemoveBucket(row filedata.Row, bucketID string, columnName
 	return nil
 }
 
+func (r *Repository) GetIndexStatusForUser(ctx context.Context, userID int64, lastUpdatedAt int64, limit int64) ([]filedata.IndexStatus, error) {
+	rows, err := r.DB.QueryContext(ctx, `SELECT file_id, user_id, data_type, size, is_deleted, updated_at
+										FROM file_data
+										WHERE user_id = $1 AND updated_at > $2 ORDER BY updated_at  
+										LIMIT $3`, userID, lastUpdatedAt, limit)
+	if err != nil {
+		return nil, stacktrace.Propagate(err, "")
+	}
+	var indexStatuses []filedata.IndexStatus
+	for rows.Next() {
+		var indexStatus filedata.IndexStatus
+		scanErr := rows.Scan(&indexStatus.FileID, &indexStatus.UserID, &indexStatus.Type, &indexStatus.Size, &indexStatus.IsDeleted, &indexStatus.UpdatedAt)
+		if scanErr != nil {
+			return nil, stacktrace.Propagate(scanErr, "")
+		}
+		indexStatuses = append(indexStatuses, indexStatus)
+	}
+	return indexStatuses, nil
+}
+
 func (r *Repository) MoveBetweenBuckets(row filedata.Row, bucketID string, sourceColumn string, destColumn string) error {
 	query := fmt.Sprintf(`
   UPDATE file_data