From 8aafeccd1d3e793b620fe36addd7fdfd20f93dc7 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Mon, 25 Sep 2023 11:53:12 +0530 Subject: [PATCH] Add support for persisting remote files metadata --- internal/api/collection.go | 21 ++++++++++- internal/api/files.go | 1 + pkg/collections.go | 72 ++++++++++++++++++++++++++++++++++++++ pkg/model/constants.go | 6 ++-- pkg/remote_sync.go | 4 +++ pkg/store.go | 43 +++++++++++++++-------- 6 files changed, 129 insertions(+), 18 deletions(-) create mode 100644 internal/api/files.go diff --git a/internal/api/collection.go b/internal/api/collection.go index 41e387be5..6af702381 100644 --- a/internal/api/collection.go +++ b/internal/api/collection.go @@ -20,6 +20,25 @@ func (c *Client) GetCollections(ctx context.Context, sinceTime int64) ([]Collect Message: r.String(), } } - return res.Collections, err } + +func (c *Client) GetFiles(ctx context.Context, collectionID, sinceTime int64) ([]File, bool, error) { + var res struct { + Files []File `json:"diff"` + HasMore bool `json:"hasMore"` + } + r, err := c.restClient.R(). + SetContext(ctx). + SetQueryParam("sinceTime", strconv.FormatInt(sinceTime, 10)). + SetQueryParam("collectionID", strconv.FormatInt(collectionID, 10)). + SetResult(&res). + Get("/collections/v2/diff") + if r.IsError() { + return nil, false, &ApiError{ + StatusCode: r.StatusCode(), + Message: r.String(), + } + } + return res.Files, res.HasMore, err +} diff --git a/internal/api/files.go b/internal/api/files.go new file mode 100644 index 000000000..778f64ec1 --- /dev/null +++ b/internal/api/files.go @@ -0,0 +1 @@ +package api diff --git a/pkg/collections.go b/pkg/collections.go index 246f9eb37..943d7e084 100644 --- a/pkg/collections.go +++ b/pkg/collections.go @@ -5,7 +5,9 @@ import ( "cli-go/pkg/model" "cli-go/utils/encoding" "context" + "encoding/json" "fmt" + "log" "strconv" ) @@ -45,3 +47,73 @@ func (c *ClICtrl) fetchRemoteCollections(ctx context.Context, info model.Account } return nil } + +func (c *ClICtrl) fetchRemoteFiles(ctx context.Context, info model.Account) error { + albums, err := c.getRemoteAlbums(ctx) + if err != nil { + return err + } + for _, album := range albums { + if album.IsDeleted { + log.Printf("Skipping album %s as it is deleted", album.AlbumName) + continue + } + lastSyncTime, lastSyncTimeErr := c.GetInt64ConfigValue(ctx, fmt.Sprintf(model.CollectionsFileSyncKeyFmt, album.ID)) + if lastSyncTimeErr != nil { + return lastSyncTimeErr + } + isFirstSync := lastSyncTime == 0 + for { + if lastSyncTime == album.LastUpdatedAt { + break + } + files, hasMore, err := c.Client.GetFiles(ctx, album.ID, lastSyncTime) + if err != nil { + return err + } + maxUpdated := lastSyncTime + for _, file := range files { + if file.UpdationTime > maxUpdated { + maxUpdated = file.UpdationTime + } + if isFirstSync && file.IsDeleted { + // on first sync, no need to sync delete markers + continue + } + fileJson := encoding.MustMarshalJSON(file) + putErr := c.PutValue(ctx, model.RemoteFiles, []byte(strconv.FormatInt(file.ID, 10)), fileJson) + if putErr != nil { + return putErr + } + } + if !hasMore { + maxUpdated = album.LastUpdatedAt + } + if maxUpdated > lastSyncTime || !hasMore { + err = c.PutConfigValue(ctx, fmt.Sprintf(model.CollectionsFileSyncKeyFmt, album.ID), []byte(strconv.FormatInt(maxUpdated, 10))) + if err != nil { + return fmt.Errorf("failed to update last sync time: %s", err) + } else { + lastSyncTime = maxUpdated + } + } + } + } + return nil +} +func (c *ClICtrl) getRemoteAlbums(ctx context.Context) ([]model.Album, error) { + albums := make([]model.Album, 0) + albumBytes, err := c.GetAllValues(ctx, model.RemoteAlbums) + if err != nil { + return nil, err + } + for _, albumJson := range albumBytes { + album := model.Album{} + err = json.Unmarshal(albumJson, &album) + if err != nil { + return nil, err + } + albums = append(albums, album) + } + return albums, nil +} diff --git a/pkg/model/constants.go b/pkg/model/constants.go index 65db2b450..a891a964b 100644 --- a/pkg/model/constants.go +++ b/pkg/model/constants.go @@ -3,9 +3,9 @@ package model type PhotosStore string const ( - KVConfig PhotosStore = "kvConfig" - RemoteAlbums PhotosStore = "remoteAlbums" - RemoteFiles PhotosStore = "remoteFiles" + KVConfig PhotosStore = "akvConfig" + RemoteAlbums PhotosStore = "aremoteAlbums" + RemoteFiles PhotosStore = "aremoteFiles" ) const ( diff --git a/pkg/remote_sync.go b/pkg/remote_sync.go index c3e0831cb..c31a44c46 100644 --- a/pkg/remote_sync.go +++ b/pkg/remote_sync.go @@ -24,6 +24,10 @@ func (c *ClICtrl) SyncAccount(account model.Account) error { if err != nil { log.Printf("Error fetching collections: %s", err) } + err = c.fetchRemoteFiles(ctx, account) + if err != nil { + log.Printf("Error fetching files: %s", err) + } return nil } diff --git a/pkg/store.go b/pkg/store.go index 56adf5a87..6ed09697b 100644 --- a/pkg/store.go +++ b/pkg/store.go @@ -19,19 +19,6 @@ func GetDB(path string) (*bolt.DB, error) { return db, err } -func (c *ClICtrl) GetConfigValue(ctx context.Context, key string) ([]byte, error) { - var value []byte - err := c.DB.View(func(tx *bolt.Tx) error { - kvBucket, err := getAccountStore(ctx, tx, model.KVConfig) - if err != nil { - return err - } - value = kvBucket.Get([]byte(key)) - return nil - }) - return value, err -} - func (c *ClICtrl) GetInt64ConfigValue(ctx context.Context, key string) (int64, error) { value, err := c.GetConfigValue(ctx, key) if err != nil { @@ -47,6 +34,35 @@ func (c *ClICtrl) GetInt64ConfigValue(ctx context.Context, key string) (int64, e return result, nil } +func (c *ClICtrl) GetConfigValue(ctx context.Context, key string) ([]byte, error) { + var value []byte + err := c.DB.View(func(tx *bolt.Tx) error { + kvBucket, err := getAccountStore(ctx, tx, model.KVConfig) + if err != nil { + return err + } + value = kvBucket.Get([]byte(key)) + return nil + }) + return value, err +} + +func (c *ClICtrl) GetAllValues(ctx context.Context, store model.PhotosStore) ([][]byte, error) { + result := make([][]byte, 0) + err := c.DB.View(func(tx *bolt.Tx) error { + kvBucket, err := getAccountStore(ctx, tx, store) + if err != nil { + return err + } + kvBucket.ForEach(func(k, v []byte) error { + result = append(result, v) + return nil + }) + return nil + }) + return result, err +} + func (c *ClICtrl) PutConfigValue(ctx context.Context, key string, value []byte) error { return c.DB.Update(func(tx *bolt.Tx) error { kvBucket, err := getAccountStore(ctx, tx, model.KVConfig) @@ -65,7 +81,6 @@ func (c *ClICtrl) PutValue(ctx context.Context, store model.PhotosStore, key []b return kvBucket.Put(key, value) }) } - func getAccountStore(ctx context.Context, tx *bolt.Tx, storeType model.PhotosStore) (*bolt.Bucket, error) { accountId := ctx.Value("account_id").(string) accountBucket := tx.Bucket([]byte(accountId))