beenull/ente
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Add support for persisting remote files metadata

Browse source
This commit is contained in:
Neeraj Gupta 2023-09-25 11:53:12 +05:30
parent cab929da2b
commit 8aafeccd1d
6 changed files with 129 additions and 18 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

21
internal/api/collection.go
View file

@ -20,6 +20,25 @@ func (c *Client) GetCollections(ctx context.Context, sinceTime int64) ([]Collect
Message: r.String(),
}
}
return res.Collections, err
}
func (c *Client) GetFiles(ctx context.Context, collectionID, sinceTime int64) ([]File, bool, error) {
var res struct {
Files []File `json:"diff"`
HasMore bool `json:"hasMore"`
}
r, err := c.restClient.R().
SetContext(ctx).
SetQueryParam("sinceTime", strconv.FormatInt(sinceTime, 10)).
SetQueryParam("collectionID", strconv.FormatInt(collectionID, 10)).
SetResult(&res).
Get("/collections/v2/diff")
if r.IsError() {
return nil, false, &ApiError{
StatusCode: r.StatusCode(),
Message: r.String(),
}
}
return res.Files, res.HasMore, err
}

1
internal/api/files.go Normal file
View file

@ -0,0 +1 @@
package api

72
pkg/collections.go
View file

@ -5,7 +5,9 @@ import (
"cli-go/pkg/model"
"cli-go/utils/encoding"
"context"
"encoding/json"
"fmt"
"log"
"strconv"
)
@ -45,3 +47,73 @@ func (c *ClICtrl) fetchRemoteCollections(ctx context.Context, info model.Account
}
return nil
}
func (c *ClICtrl) fetchRemoteFiles(ctx context.Context, info model.Account) error {
albums, err := c.getRemoteAlbums(ctx)
if err != nil {
return err
}
for _, album := range albums {
if album.IsDeleted {
log.Printf("Skipping album %s as it is deleted", album.AlbumName)
continue
}
lastSyncTime, lastSyncTimeErr := c.GetInt64ConfigValue(ctx, fmt.Sprintf(model.CollectionsFileSyncKeyFmt, album.ID))
if lastSyncTimeErr != nil {
return lastSyncTimeErr
}
isFirstSync := lastSyncTime == 0
for {
if lastSyncTime == album.LastUpdatedAt {
break
}
files, hasMore, err := c.Client.GetFiles(ctx, album.ID, lastSyncTime)
if err != nil {
return err
}
maxUpdated := lastSyncTime
for _, file := range files {
if file.UpdationTime > maxUpdated {
maxUpdated = file.UpdationTime
}
if isFirstSync && file.IsDeleted {
// on first sync, no need to sync delete markers
continue
}
fileJson := encoding.MustMarshalJSON(file)
putErr := c.PutValue(ctx, model.RemoteFiles, []byte(strconv.FormatInt(file.ID, 10)), fileJson)
if putErr != nil {
return putErr
}
}
if !hasMore {
maxUpdated = album.LastUpdatedAt
}
if maxUpdated > lastSyncTime || !hasMore {
err = c.PutConfigValue(ctx, fmt.Sprintf(model.CollectionsFileSyncKeyFmt, album.ID), []byte(strconv.FormatInt(maxUpdated, 10)))
if err != nil {
return fmt.Errorf("failed to update last sync time: %s", err)
} else {
lastSyncTime = maxUpdated
}
}
}
}
return nil
}
func (c *ClICtrl) getRemoteAlbums(ctx context.Context) ([]model.Album, error) {
albums := make([]model.Album, 0)
albumBytes, err := c.GetAllValues(ctx, model.RemoteAlbums)
if err != nil {
return nil, err
}
for _, albumJson := range albumBytes {
album := model.Album{}
err = json.Unmarshal(albumJson, &album)
if err != nil {
return nil, err
}
albums = append(albums, album)
}
return albums, nil
}

6
pkg/model/constants.go
View file

@ -3,9 +3,9 @@ package model
type PhotosStore string
const (
KVConfig PhotosStore = "kvConfig"
RemoteAlbums PhotosStore = "remoteAlbums"
RemoteFiles PhotosStore = "remoteFiles"
KVConfig PhotosStore = "akvConfig"
RemoteAlbums PhotosStore = "aremoteAlbums"
RemoteFiles PhotosStore = "aremoteFiles"
)
const (

4
pkg/remote_sync.go
View file

@ -24,6 +24,10 @@ func (c *ClICtrl) SyncAccount(account model.Account) error {
if err != nil {
log.Printf("Error fetching collections: %s", err)
}
err = c.fetchRemoteFiles(ctx, account)
if err != nil {
log.Printf("Error fetching files: %s", err)
}
return nil
}

43
pkg/store.go
View file

@ -19,19 +19,6 @@ func GetDB(path string) (*bolt.DB, error) {
return db, err
}
func (c *ClICtrl) GetConfigValue(ctx context.Context, key string) ([]byte, error) {
var value []byte
err := c.DB.View(func(tx *bolt.Tx) error {
kvBucket, err := getAccountStore(ctx, tx, model.KVConfig)
if err != nil {
return err
}
value = kvBucket.Get([]byte(key))
return nil
})
return value, err
}
func (c *ClICtrl) GetInt64ConfigValue(ctx context.Context, key string) (int64, error) {
value, err := c.GetConfigValue(ctx, key)
if err != nil {
@ -47,6 +34,35 @@ func (c *ClICtrl) GetInt64ConfigValue(ctx context.Context, key string) (int64, e
return result, nil
}
func (c *ClICtrl) GetConfigValue(ctx context.Context, key string) ([]byte, error) {
var value []byte
err := c.DB.View(func(tx *bolt.Tx) error {
kvBucket, err := getAccountStore(ctx, tx, model.KVConfig)
if err != nil {
return err
}
value = kvBucket.Get([]byte(key))
return nil
})
return value, err
}
func (c *ClICtrl) GetAllValues(ctx context.Context, store model.PhotosStore) ([][]byte, error) {
result := make([][]byte, 0)
err := c.DB.View(func(tx *bolt.Tx) error {
kvBucket, err := getAccountStore(ctx, tx, store)
if err != nil {
return err
}
kvBucket.ForEach(func(k, v []byte) error {
result = append(result, v)
return nil
})
return nil
})
return result, err
}
func (c *ClICtrl) PutConfigValue(ctx context.Context, key string, value []byte) error {
return c.DB.Update(func(tx *bolt.Tx) error {
kvBucket, err := getAccountStore(ctx, tx, model.KVConfig)
@ -65,7 +81,6 @@ func (c *ClICtrl) PutValue(ctx context.Context, store model.PhotosStore, key []b
return kvBucket.Put(key, value)
})
}
func getAccountStore(ctx context.Context, tx *bolt.Tx, storeType model.PhotosStore) (*bolt.Bucket, error) {
accountId := ctx.Value("account_id").(string)
accountBucket := tx.Bucket([]byte(accountId))