276 lines
8.6 KiB
Go
276 lines
8.6 KiB
Go
package pkg
|
|
|
|
import (
|
|
"archive/zip"
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"github.com/ente-io/cli/pkg/mapper"
|
|
"github.com/ente-io/cli/pkg/model"
|
|
"github.com/ente-io/cli/pkg/model/export"
|
|
"github.com/ente-io/cli/utils"
|
|
"log"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
func (c *ClICtrl) syncFiles(ctx context.Context, account model.Account) error {
|
|
log.Printf("Starting file download")
|
|
exportRoot := account.ExportDir
|
|
_, albumIDToMetaMap, err := readFolderMetadata(exportRoot)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
entries, err := c.getRemoteAlbumEntries(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
log.Println("total entries", len(entries))
|
|
model.SortAlbumFileEntry(entries)
|
|
defer utils.TimeTrack(time.Now(), "process_files")
|
|
var albumDiskInfo *albumDiskInfo
|
|
for i, albumFileEntry := range entries {
|
|
if albumFileEntry.SyncedLocally {
|
|
continue
|
|
}
|
|
albumInfo, ok := albumIDToMetaMap[albumFileEntry.AlbumID]
|
|
if !ok {
|
|
log.Printf("Album %d not found in local metadata", albumFileEntry.AlbumID)
|
|
continue
|
|
}
|
|
if albumInfo.IsDeleted {
|
|
putErr := c.DeleteAlbumEntry(ctx, albumFileEntry)
|
|
if putErr != nil {
|
|
return putErr
|
|
}
|
|
continue
|
|
}
|
|
|
|
if albumDiskInfo == nil || albumDiskInfo.AlbumMeta.ID != albumInfo.ID {
|
|
albumDiskInfo, err = readFilesMetadata(exportRoot, albumInfo)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
fileBytes, err := c.GetValue(ctx, model.RemoteFiles, []byte(fmt.Sprintf("%d", albumFileEntry.FileID)))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if fileBytes != nil {
|
|
var existingEntry *model.RemoteFile
|
|
err = json.Unmarshal(fileBytes, &existingEntry)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
log.Printf("[%d/%d] Sync %s for album %s", i, len(entries), existingEntry.GetTitle(), albumInfo.AlbumName)
|
|
err = c.downloadEntry(ctx, albumDiskInfo, *existingEntry, albumFileEntry)
|
|
if err != nil {
|
|
if errors.Is(err, model.ErrDecryption) {
|
|
continue
|
|
} else if existingEntry.IsLivePhoto() && errors.Is(err, zip.ErrFormat) {
|
|
log.Printf(fmt.Sprintf("err processing live photo %s (%d), %s", existingEntry.GetTitle(), existingEntry.ID, err.Error()))
|
|
continue
|
|
} else if existingEntry.IsLivePhoto() && errors.Is(err, model.ErrLiveZip) {
|
|
continue
|
|
} else {
|
|
return err
|
|
}
|
|
}
|
|
} else {
|
|
// file metadata is missing in the localDB
|
|
if albumFileEntry.IsDeleted {
|
|
delErr := c.DeleteAlbumEntry(ctx, albumFileEntry)
|
|
if delErr != nil {
|
|
log.Fatalf("Error deleting album entry %d (deleted: %v) %v", albumFileEntry.FileID, albumFileEntry.IsDeleted, delErr)
|
|
}
|
|
} else {
|
|
log.Fatalf("Failed to find entry in db for file %d (deleted: %v)", albumFileEntry.FileID, albumFileEntry.IsDeleted)
|
|
}
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (c *ClICtrl) downloadEntry(ctx context.Context,
|
|
diskInfo *albumDiskInfo,
|
|
file model.RemoteFile,
|
|
albumEntry *model.AlbumFileEntry,
|
|
) error {
|
|
if !diskInfo.AlbumMeta.IsDeleted && albumEntry.IsDeleted {
|
|
albumEntry.IsDeleted = true
|
|
diskFileMeta := diskInfo.GetDiskFileMetadata(file)
|
|
if diskFileMeta != nil {
|
|
removeErr := removeDiskFile(diskFileMeta, diskInfo)
|
|
if removeErr != nil {
|
|
return removeErr
|
|
}
|
|
}
|
|
delErr := c.DeleteAlbumEntry(ctx, albumEntry)
|
|
if delErr != nil {
|
|
return delErr
|
|
}
|
|
return nil
|
|
}
|
|
diskFileMeta := diskInfo.GetDiskFileMetadata(file)
|
|
if diskFileMeta != nil {
|
|
removeErr := removeDiskFile(diskFileMeta, diskInfo)
|
|
if removeErr != nil {
|
|
return removeErr
|
|
}
|
|
}
|
|
if !diskInfo.IsFilePresent(file) {
|
|
decrypt, err := c.downloadAndDecrypt(ctx, file, c.KeyHolder.DeviceKey)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
fileDiskMetadata := mapper.MapRemoteFileToDiskMetadata(file)
|
|
// Get the extension
|
|
extension := filepath.Ext(fileDiskMetadata.Title)
|
|
baseFileName := strings.TrimSuffix(filepath.Clean(filepath.Base(fileDiskMetadata.Title)), extension)
|
|
diskMetaFileName := diskInfo.GenerateUniqueMetaFileName(baseFileName, extension)
|
|
if file.IsLivePhoto() {
|
|
imagePath, videoPath, err := UnpackLive(*decrypt)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if imagePath == "" && videoPath == "" {
|
|
log.Printf("imagePath %s, videoPath %s", imagePath, videoPath)
|
|
return model.ErrLiveZip
|
|
}
|
|
if imagePath != "" {
|
|
imageExtn := filepath.Ext(imagePath)
|
|
imageFileName := diskInfo.GenerateUniqueFileName(baseFileName, imageExtn)
|
|
imageFilePath := filepath.Join(diskInfo.ExportRoot, diskInfo.AlbumMeta.FolderName, imageFileName)
|
|
moveErr := Move(imagePath, imageFilePath)
|
|
if moveErr != nil {
|
|
return moveErr
|
|
}
|
|
fileDiskMetadata.AddFileName(imageFileName)
|
|
}
|
|
if videoPath == "" {
|
|
videoExtn := filepath.Ext(videoPath)
|
|
videoFileName := diskInfo.GenerateUniqueFileName(baseFileName, videoExtn)
|
|
videoFilePath := filepath.Join(diskInfo.ExportRoot, diskInfo.AlbumMeta.FolderName, videoFileName)
|
|
// move the decrypt file to filePath
|
|
moveErr := Move(videoPath, videoFilePath)
|
|
if moveErr != nil {
|
|
return moveErr
|
|
}
|
|
fileDiskMetadata.AddFileName(videoFileName)
|
|
}
|
|
} else {
|
|
fileName := diskInfo.GenerateUniqueFileName(baseFileName, extension)
|
|
filePath := filepath.Join(diskInfo.ExportRoot, diskInfo.AlbumMeta.FolderName, fileName)
|
|
// move the decrypt file to filePath
|
|
err = Move(*decrypt, filePath)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
fileDiskMetadata.AddFileName(fileName)
|
|
}
|
|
|
|
fileDiskMetadata.MetaFileName = diskMetaFileName
|
|
err = diskInfo.AddEntry(fileDiskMetadata)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
err = writeJSONToFile(filepath.Join(diskInfo.ExportRoot, diskInfo.AlbumMeta.FolderName, ".meta", diskMetaFileName), fileDiskMetadata)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
albumEntry.SyncedLocally = true
|
|
putErr := c.UpsertAlbumEntry(ctx, albumEntry)
|
|
if putErr != nil {
|
|
return putErr
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func removeDiskFile(diskFileMeta *export.DiskFileMetadata, diskInfo *albumDiskInfo) error {
|
|
// remove the file from disk
|
|
log.Printf("Removing file %s from disk", diskFileMeta.MetaFileName)
|
|
err := os.Remove(filepath.Join(diskInfo.ExportRoot, diskInfo.AlbumMeta.FolderName, ".meta", diskFileMeta.MetaFileName))
|
|
if err != nil && !os.IsNotExist(err) {
|
|
return err
|
|
}
|
|
for _, fileName := range diskFileMeta.Info.FileNames {
|
|
err = os.Remove(filepath.Join(diskInfo.ExportRoot, diskInfo.AlbumMeta.FolderName, fileName))
|
|
if err != nil && !os.IsNotExist(err) {
|
|
return err
|
|
}
|
|
}
|
|
return diskInfo.RemoveEntry(diskFileMeta)
|
|
}
|
|
|
|
// readFolderMetadata reads the metadata of the files in the given path
|
|
// For disk export, a particular albums files are stored in a folder named after the album.
|
|
// Inside the folder, the files are stored at top level and its metadata is stored in a .meta folder
|
|
func readFilesMetadata(home string, albumMeta *export.AlbumMetadata) (*albumDiskInfo, error) {
|
|
albumMetadataFolder := filepath.Join(home, albumMeta.FolderName, albumMetaFolder)
|
|
albumPath := filepath.Join(home, albumMeta.FolderName)
|
|
// verify the both the album folder and the .meta folder exist
|
|
if _, err := os.Stat(albumMetadataFolder); err != nil {
|
|
return nil, err
|
|
}
|
|
if _, err := os.Stat(albumPath); err != nil {
|
|
return nil, err
|
|
}
|
|
result := make(map[string]*export.DiskFileMetadata)
|
|
//fileNameToFileName := make(map[string]*export.DiskFileMetadata)
|
|
fileIdToMetadata := make(map[int64]*export.DiskFileMetadata)
|
|
claimedFileName := make(map[string]bool)
|
|
// Read the top-level directories in the given path
|
|
albumFileEntries, err := os.ReadDir(albumPath)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
for _, entry := range albumFileEntries {
|
|
if !entry.IsDir() {
|
|
claimedFileName[strings.ToLower(entry.Name())] = true
|
|
}
|
|
}
|
|
metaEntries, err := os.ReadDir(albumMetadataFolder)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
for _, entry := range metaEntries {
|
|
if !entry.IsDir() {
|
|
fileName := entry.Name()
|
|
if fileName == albumMetaFile {
|
|
continue
|
|
}
|
|
if !strings.HasSuffix(fileName, ".json") {
|
|
log.Printf("Skipping file %s as it is not a JSON file", fileName)
|
|
continue
|
|
}
|
|
fileMetadataPath := filepath.Join(albumMetadataFolder, fileName)
|
|
// Initialize as nil, will remain nil if JSON file is not found or not readable
|
|
result[strings.ToLower(fileName)] = nil
|
|
// Read the JSON file if it exists
|
|
var metaData export.DiskFileMetadata
|
|
metaDataBytes, err := os.ReadFile(fileMetadataPath)
|
|
if err != nil {
|
|
continue // Skip this entry if reading fails
|
|
}
|
|
if err := json.Unmarshal(metaDataBytes, &metaData); err == nil {
|
|
metaData.MetaFileName = fileName
|
|
result[strings.ToLower(fileName)] = &metaData
|
|
fileIdToMetadata[metaData.Info.ID] = &metaData
|
|
}
|
|
}
|
|
}
|
|
return &albumDiskInfo{
|
|
ExportRoot: home,
|
|
AlbumMeta: albumMeta,
|
|
FileNames: &claimedFileName,
|
|
MetaFileNameToDiskFileMap: &result,
|
|
FileIdToDiskFileMap: &fileIdToMetadata,
|
|
}, nil
|
|
}
|