[server] API to get file data status

server/cmd/museum/main.go

@@ -424,6 +424,7 @@ func main() {
 	privateAPI.GET("/files/preview/v2/:fileID", fileHandler.GetThumbnail)
 
 	privateAPI.PUT("/files/data", fileHandler.PutFileData)
+	privateAPI.POST("/files/data/status-diff", fileHandler.FileDataStatusDiff)
 	privateAPI.POST("/files/data/fetch", fileHandler.GetFilesData)
 	privateAPI.GET("/files/data/fetch", fileHandler.GetFileData)
 	privateAPI.GET("/files/data/preview-upload-url", fileHandler.GetPreviewUploadURL)

server/ente/filedata/filedata.go

@@ -12,6 +12,19 @@ type Entity struct {
 	DecryptionHeader string          `json:"decryptionHeader"`
 }
 
+type IndexDiffRequest struct {
+	LastUpdated int64 `form:"lastUpdated" binding:"required"`
+}
+
+type IndexStatus struct {
+	FileID    int64           `json:"fileID" binding:"required"`
+	UserID    int64           `json:"userID" binding:"required"`
+	Type      ente.ObjectType `json:"type" binding:"required"`
+	IsDeleted bool            `json:"isDeleted" binding:"required"`
+	Size      int64           `json:"size"  binding:"required"`
+	UpdatedAt int64           `json:"updatedAt"  binding:"required"`
+}
+
 // GetFilesData should only be used for getting the preview video playlist and derived metadata.
 type GetFilesData struct {
 	FileIDs []int64         `json:"fileIDs" binding:"required"`

server/pkg/api/file_data.go

@@ -48,6 +48,22 @@ func (h *FileHandler) GetFilesData(ctx *gin.Context) {
 	ctx.JSON(http.StatusOK, resp)
 }
 
+// FileDataStatusDiff API won't really return status/diff for deleted files. The clients will primarily use this data to identify for which all files we already have preview generated or it's ML inference is done.
+// This doesn't simulate perfect diff behaviour as we won't maintain a tombstone entries for the deleted API.
+func (h *FileHandler) FileDataStatusDiff(ctx *gin.Context) {
+	var req fileData.IndexDiffRequest
+	if err := ctx.ShouldBindJSON(&req); err != nil {
+		ctx.JSON(http.StatusBadRequest, ente.NewBadRequestWithMessage(err.Error()))
+		return
+	}
+	resp, err := h.FileDataCtrl.FileDataStatusDiff(ctx, req)
+	if err != nil {
+		handler.Error(ctx, err)
+		return
+	}
+	ctx.JSON(http.StatusOK, resp)
+}
+
 func (h *FileHandler) GetFileData(ctx *gin.Context) {
 	var req fileData.GetFileData
 	if err := ctx.ShouldBindJSON(&req); err != nil {

server/pkg/controller/filedata/controller.go

@@ -313,3 +313,8 @@ func (c *Controller) _validatePermission(ctx *gin.Context, fileID int64, actorID
 	}
 	return nil
 }
+
+func (c *Controller) FileDataStatusDiff(ctx *gin.Context, req fileData.IndexDiffRequest) ([]fileData.IndexStatus, error) {
+	userID := auth.GetUserID(ctx.Request.Header)
+	return c.Repo.GetIndexStatusForUser(ctx, userID, req.LastUpdated, 5000)
+}

server/pkg/repo/filedata/repository.go

@@ -124,6 +124,26 @@ func (r *Repository) RemoveBucket(row filedata.Row, bucketID string, columnName
 	return nil
 }
 
+func (r *Repository) GetIndexStatusForUser(ctx context.Context, userID int64, lastUpdatedAt int64, limit int64) ([]filedata.IndexStatus, error) {
+	rows, err := r.DB.QueryContext(ctx, `SELECT file_id, user_id, data_type, size, is_deleted, updated_at
+										FROM file_data
+										WHERE user_id = $1 AND updated_at > $2 ORDER BY updated_at  
+										LIMIT $3`, userID, lastUpdatedAt, limit)
+	if err != nil {
+		return nil, stacktrace.Propagate(err, "")
+	}
+	var indexStatuses []filedata.IndexStatus
+	for rows.Next() {
+		var indexStatus filedata.IndexStatus
+		scanErr := rows.Scan(&indexStatus.FileID, &indexStatus.UserID, &indexStatus.Type, &indexStatus.Size, &indexStatus.IsDeleted, &indexStatus.UpdatedAt)
+		if scanErr != nil {
+			return nil, stacktrace.Propagate(scanErr, "")
+		}
+		indexStatuses = append(indexStatuses, indexStatus)
+	}
+	return indexStatuses, nil
+}
+
 func (r *Repository) MoveBetweenBuckets(row filedata.Row, bucketID string, sourceColumn string, destColumn string) error {
 	query := fmt.Sprintf(`
   UPDATE file_data