From 7d8a7257b40acf1c801b09553d70e0ffbd11e0ee Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Thu, 1 Aug 2024 17:46:05 +0530 Subject: [PATCH 1/3] [cli] Add list of filters --- cli/pkg/model/export/filter.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 cli/pkg/model/export/filter.go diff --git a/cli/pkg/model/export/filter.go b/cli/pkg/model/export/filter.go new file mode 100644 index 0000000000..742d5fef38 --- /dev/null +++ b/cli/pkg/model/export/filter.go @@ -0,0 +1,14 @@ +package export + +type Filters struct { + // When true, none of the shared albums are exported + ExcludeShared bool + // When true, none of the shared files are exported + ExcludeSharedFiles bool + // When true, hidden albums are not exported + ExcludeHidden bool + // when album name is provided, only files in those albums are exported + Albums []string + // when email is provided, only files shared with that email are exported + Emails []string +} From 0526c63681e86ba20391126a7341290b7cac8707 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Thu, 1 Aug 2024 17:52:58 +0530 Subject: [PATCH 2/3] [cli] Extend export command to pass filters --- cli/cmd/export.go | 29 +++++++++++++++++++++++++++-- cli/pkg/sync.go | 3 ++- 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/cli/cmd/export.go b/cli/cmd/export.go index 57548c4fe5..fa253cce64 100644 --- a/cli/cmd/export.go +++ b/cli/cmd/export.go @@ -1,19 +1,44 @@ package cmd import ( + "github.com/ente-io/cli/pkg/model/export" "github.com/spf13/cobra" ) -// versionCmd represents the version command +// exportCmd represents the export command var exportCmd = &cobra.Command{ Use: "export", Short: "Starts the export process", Long: ``, Run: func(cmd *cobra.Command, args []string) { - ctrl.Export() + // Retrieve flag values + shared, _ := cmd.Flags().GetBool("shared") + sharedFiles, _ := cmd.Flags().GetBool("shared-files") + hidden, _ := cmd.Flags().GetBool("hidden") + albums, _ := cmd.Flags().GetStringSlice("albums") + emails, _ := cmd.Flags().GetStringSlice("emails") + + // Create Filters struct with flag values + filters := export.Filters{ + ExcludeShared: !shared, + ExcludeSharedFiles: !sharedFiles, + ExcludeHidden: !hidden, + Albums: albums, + Emails: emails, + } + + // Call the Export function with the filters + ctrl.Export(filters) }, } func init() { rootCmd.AddCommand(exportCmd) + + // Add flags for Filters struct fields with default value true + exportCmd.Flags().Bool("shared", true, "Include shared albums in export") + exportCmd.Flags().Bool("shared-files", true, "Include shared files in export") + exportCmd.Flags().Bool("hidden", true, "Include hidden albums in export") + exportCmd.Flags().StringSlice("albums", []string{}, "Comma-separated list of album names to export") + exportCmd.Flags().StringSlice("emails", []string{}, "Comma-separated list of emails to export files shared with") } diff --git a/cli/pkg/sync.go b/cli/pkg/sync.go index 6aee7cbb75..d6cbf06bf5 100644 --- a/cli/pkg/sync.go +++ b/cli/pkg/sync.go @@ -7,12 +7,13 @@ import ( "github.com/ente-io/cli/internal" "github.com/ente-io/cli/internal/api" "github.com/ente-io/cli/pkg/model" + "github.com/ente-io/cli/pkg/model/export" bolt "go.etcd.io/bbolt" "log" "time" ) -func (c *ClICtrl) Export() error { +func (c *ClICtrl) Export(filters export.Filters) error { accounts, err := c.GetAccounts(context.Background()) if err != nil { return err From 395f0384a063dbb9693b70ba04c79d6f07994803 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Thu, 1 Aug 2024 19:13:03 +0530 Subject: [PATCH 3/3] [cli] Add option to exlude shared or hidden folders --- cli/cmd/export.go | 21 ++++------- cli/pkg/admin_actions.go | 2 +- cli/pkg/model/constants.go | 6 +++ cli/pkg/model/filter.go | 66 +++++++++++++++++++++++++++++++++ cli/pkg/model/remote.go | 7 ++++ cli/pkg/remote_sync.go | 5 +++ cli/pkg/remote_to_disk_album.go | 5 ++- cli/pkg/remote_to_disk_file.go | 14 +++++++ cli/pkg/sync.go | 19 +++++++--- 9 files changed, 124 insertions(+), 21 deletions(-) create mode 100644 cli/pkg/model/filter.go diff --git a/cli/cmd/export.go b/cli/cmd/export.go index fa253cce64..0979bbeb5e 100644 --- a/cli/cmd/export.go +++ b/cli/cmd/export.go @@ -1,7 +1,7 @@ package cmd import ( - "github.com/ente-io/cli/pkg/model/export" + "github.com/ente-io/cli/pkg/model" "github.com/spf13/cobra" ) @@ -13,20 +13,16 @@ var exportCmd = &cobra.Command{ Run: func(cmd *cobra.Command, args []string) { // Retrieve flag values shared, _ := cmd.Flags().GetBool("shared") - sharedFiles, _ := cmd.Flags().GetBool("shared-files") hidden, _ := cmd.Flags().GetBool("hidden") albums, _ := cmd.Flags().GetStringSlice("albums") emails, _ := cmd.Flags().GetStringSlice("emails") - // Create Filters struct with flag values - filters := export.Filters{ - ExcludeShared: !shared, - ExcludeSharedFiles: !sharedFiles, - ExcludeHidden: !hidden, - Albums: albums, - Emails: emails, + filters := model.Filter{ + ExcludeShared: !shared, + ExcludeHidden: !hidden, + Albums: albums, + Emails: emails, } - // Call the Export function with the filters ctrl.Export(filters) }, @@ -36,9 +32,8 @@ func init() { rootCmd.AddCommand(exportCmd) // Add flags for Filters struct fields with default value true - exportCmd.Flags().Bool("shared", true, "Include shared albums in export") - exportCmd.Flags().Bool("shared-files", true, "Include shared files in export") - exportCmd.Flags().Bool("hidden", true, "Include hidden albums in export") + exportCmd.Flags().Bool("shared", true, "to exclude shared albums, pass --shared=false") + exportCmd.Flags().Bool("hidden", true, "to exclude hidden albums, pass --hidden=false") exportCmd.Flags().StringSlice("albums", []string{}, "Comma-separated list of album names to export") exportCmd.Flags().StringSlice("emails", []string{}, "Comma-separated list of emails to export files shared with") } diff --git a/cli/pkg/admin_actions.go b/cli/pkg/admin_actions.go index 44af3c27a9..1335ffceb1 100644 --- a/cli/pkg/admin_actions.go +++ b/cli/pkg/admin_actions.go @@ -186,7 +186,7 @@ func (c *ClICtrl) buildAdminContext(ctx context.Context, adminEmail string) (con if err != nil { return nil, err } - accountCtx := c.buildRequestContext(ctx, *acc) + accountCtx := c.buildRequestContext(ctx, *acc, model.Filter{}) c.Client.AddToken(acc.AccountKey(), secretInfo.TokenStr()) return accountCtx, nil } diff --git a/cli/pkg/model/constants.go b/cli/pkg/model/constants.go index 2a9f4cafa1..6ffef41dde 100644 --- a/cli/pkg/model/constants.go +++ b/cli/pkg/model/constants.go @@ -13,3 +13,9 @@ const ( CollectionsSyncKey = "lastCollectionSync" CollectionsFileSyncKeyFmt = "collectionFilesSync-%d" ) + +type ContextKey string + +const ( + FilterKey ContextKey = "export_filter" +) diff --git a/cli/pkg/model/filter.go b/cli/pkg/model/filter.go new file mode 100644 index 0000000000..6f083c3f32 --- /dev/null +++ b/cli/pkg/model/filter.go @@ -0,0 +1,66 @@ +package model + +import ( + "log" + "strings" +) + +type Filter struct { + // When true, none of the shared albums are exported + ExcludeShared bool + // When true, none of the shared files are exported + ExcludeSharedFiles bool + // When true, hidden albums are not exported + ExcludeHidden bool + // when album name is provided, only files in those albums are exported + Albums []string + // when email is provided, only files shared with that email are exported + Emails []string +} + +func (f Filter) SkipAccount(email string) bool { + if len(f.Emails) == 0 { + return false + } + for _, e := range f.Emails { + if strings.ToLower(e) == strings.ToLower(strings.TrimSpace(email)) { + return false + } + } + return true +} + +func (f Filter) SkipAlbum(album RemoteAlbum, shouldLog bool) bool { + if f.excludeByName(album) { + if shouldLog { + log.Printf("Skipping album %s as it's not part of album to export", album.AlbumName) + } + return true + } + if f.ExcludeShared && album.IsShared { + if shouldLog { + log.Printf("Skipping album %s as it's shared", album.AlbumName) + } + return true + } + if f.ExcludeHidden && album.IsHidden() { + if shouldLog { + log.Printf("Skipping album %s as it's hidden", album.AlbumName) + } + return true + } + return false +} + +// excludeByName returns true if albums list is not empty and album name is not in the list +func (f Filter) excludeByName(album RemoteAlbum) bool { + if len(f.Albums) > 0 { + for _, a := range f.Albums { + if strings.ToLower(a) == strings.ToLower(strings.TrimSpace(album.AlbumName)) { + return false + } + } + return true + } + return false +} diff --git a/cli/pkg/model/remote.go b/cli/pkg/model/remote.go index d8d6f3fcee..0f8db91fa5 100644 --- a/cli/pkg/model/remote.go +++ b/cli/pkg/model/remote.go @@ -47,6 +47,13 @@ type RemoteAlbum struct { LastUpdatedAt int64 `json:"lastUpdatedAt"` } +func (r *RemoteAlbum) IsHidden() bool { + if value, ok := r.PrivateMeta["visibility"]; ok { + return int64(value.(float64)) == int64(2) + } + return false +} + type AlbumFileEntry struct { FileID int64 `json:"fileID"` AlbumID int64 `json:"albumID"` diff --git a/cli/pkg/remote_sync.go b/cli/pkg/remote_sync.go index 5ca149d719..8ffb6ce41d 100644 --- a/cli/pkg/remote_sync.go +++ b/cli/pkg/remote_sync.go @@ -50,13 +50,18 @@ func (c *ClICtrl) fetchRemoteCollections(ctx context.Context) error { func (c *ClICtrl) fetchRemoteFiles(ctx context.Context) error { albums, err := c.getRemoteAlbums(ctx) + filter := ctx.Value(model.FilterKey).(model.Filter) if err != nil { return err } + for _, album := range albums { if album.IsDeleted { continue } + if filter.SkipAlbum(album, true) { + continue + } lastSyncTime, lastSyncTimeErr := c.GetInt64ConfigValue(ctx, fmt.Sprintf(model.CollectionsFileSyncKeyFmt, album.ID)) if lastSyncTimeErr != nil { diff --git a/cli/pkg/remote_to_disk_album.go b/cli/pkg/remote_to_disk_album.go index 7b0b64f972..dbdb065bec 100644 --- a/cli/pkg/remote_to_disk_album.go +++ b/cli/pkg/remote_to_disk_album.go @@ -24,8 +24,11 @@ func (c *ClICtrl) createLocalFolderForRemoteAlbums(ctx context.Context, account if err != nil { return err } - + filter := ctx.Value(model.FilterKey).(model.Filter) for _, album := range albums { + if filter.SkipAlbum(album, false) { + continue + } if album.IsDeleted { if meta, ok := albumIDToMetaMap[album.ID]; ok { log.Printf("Deleting album %s as it is deleted", meta.AlbumName) diff --git a/cli/pkg/remote_to_disk_file.go b/cli/pkg/remote_to_disk_file.go index 91a105032c..87f5362c6b 100644 --- a/cli/pkg/remote_to_disk_file.go +++ b/cli/pkg/remote_to_disk_file.go @@ -24,6 +24,17 @@ func (c *ClICtrl) syncFiles(ctx context.Context, account model.Account) error { if err != nil { return err } + albumsToSkip := make(map[int64]bool) + filter := ctx.Value(model.FilterKey).(model.Filter) + remoteAlbums, readAlbumErr := c.getRemoteAlbums(ctx) + if readAlbumErr != nil { + return readAlbumErr + } + for _, album := range remoteAlbums { + if !album.IsDeleted && filter.SkipAlbum(album, false) { + albumsToSkip[album.ID] = true + } + } entries, err := c.getRemoteAlbumEntries(ctx) if err != nil { return err @@ -36,6 +47,9 @@ func (c *ClICtrl) syncFiles(ctx context.Context, account model.Account) error { if albumFileEntry.SyncedLocally { continue } + if _, ok := albumsToSkip[albumFileEntry.AlbumID]; ok { + continue + } albumInfo, ok := albumIDToMetaMap[albumFileEntry.AlbumID] if !ok { log.Printf("Album %d not found in local metadata", albumFileEntry.AlbumID) diff --git a/cli/pkg/sync.go b/cli/pkg/sync.go index d6cbf06bf5..af8cc9f649 100644 --- a/cli/pkg/sync.go +++ b/cli/pkg/sync.go @@ -7,13 +7,12 @@ import ( "github.com/ente-io/cli/internal" "github.com/ente-io/cli/internal/api" "github.com/ente-io/cli/pkg/model" - "github.com/ente-io/cli/pkg/model/export" bolt "go.etcd.io/bbolt" "log" "time" ) -func (c *ClICtrl) Export(filters export.Filters) error { +func (c *ClICtrl) Export(filter model.Filter) error { accounts, err := c.GetAccounts(context.Background()) if err != nil { return err @@ -22,8 +21,13 @@ func (c *ClICtrl) Export(filters export.Filters) error { fmt.Printf("No accounts to sync\n Add account using `account add` cmd\n") return nil } + for _, account := range accounts { log.SetPrefix(fmt.Sprintf("[%s-%s] ", account.App, account.Email)) + if filter.SkipAccount(account.Email) { + log.Printf("Skip account %s: account is excluded by filter", account.Email) + continue + } if account.ExportDir == "" { log.Printf("Skip account %s: no export directory configured", account.Email) continue @@ -40,7 +44,7 @@ func (c *ClICtrl) Export(filters export.Filters) error { log.Println("start sync") retryCount := 0 for { - err = c.SyncAccount(account) + err = c.SyncAccount(account, filter) if err != nil { if model.ShouldRetrySync(err) && retryCount < 20 { retryCount = retryCount + 1 @@ -61,12 +65,12 @@ func (c *ClICtrl) Export(filters export.Filters) error { return nil } -func (c *ClICtrl) SyncAccount(account model.Account) error { +func (c *ClICtrl) SyncAccount(account model.Account, filters model.Filter) error { secretInfo, err := c.KeyHolder.LoadSecrets(account) if err != nil { return err } - ctx := c.buildRequestContext(context.Background(), account) + ctx := c.buildRequestContext(context.Background(), account, filters) err = createDataBuckets(c.DB, account) if err != nil { return err @@ -95,10 +99,13 @@ func (c *ClICtrl) SyncAccount(account model.Account) error { return nil } -func (c *ClICtrl) buildRequestContext(ctx context.Context, account model.Account) context.Context { +func (c *ClICtrl) buildRequestContext(ctx context.Context, + account model.Account, + filter model.Filter) context.Context { ctx = context.WithValue(ctx, "app", string(account.App)) ctx = context.WithValue(ctx, "account_key", account.AccountKey()) ctx = context.WithValue(ctx, "user_id", account.UserID) + ctx = context.WithValue(ctx, model.FilterKey, filter) return ctx }