WIP 1
This commit is contained in:
parent
2051ccee46
commit
52c35108ca
|
@ -27,8 +27,3 @@ export const fsIsDir = async (dirPath: string) => {
|
||||||
const stat = await fs.stat(dirPath);
|
const stat = await fs.stat(dirPath);
|
||||||
return stat.isDirectory();
|
return stat.isDirectory();
|
||||||
};
|
};
|
||||||
|
|
||||||
export const fsLsFiles = async (dirPath: string) =>
|
|
||||||
(await fs.readdir(dirPath, { withFileTypes: true }))
|
|
||||||
.filter((e) => e.isFile())
|
|
||||||
.map((e) => e.name);
|
|
||||||
|
|
|
@ -20,7 +20,7 @@ import {
|
||||||
import {
|
import {
|
||||||
fsExists,
|
fsExists,
|
||||||
fsIsDir,
|
fsIsDir,
|
||||||
fsLsFiles,
|
fsListFiles,
|
||||||
fsMkdirIfNeeded,
|
fsMkdirIfNeeded,
|
||||||
fsReadTextFile,
|
fsReadTextFile,
|
||||||
fsRename,
|
fsRename,
|
||||||
|
@ -135,7 +135,7 @@ export const attachIPCHandlers = () => {
|
||||||
|
|
||||||
ipcMain.handle("fsIsDir", (_, dirPath: string) => fsIsDir(dirPath));
|
ipcMain.handle("fsIsDir", (_, dirPath: string) => fsIsDir(dirPath));
|
||||||
|
|
||||||
ipcMain.handle("fsLsFiles", (_, dirPath: string) => fsLsFiles(dirPath));
|
ipcMain.handle("fsListFiles", (_, dirPath: string) => fsListFiles(dirPath));
|
||||||
|
|
||||||
// - Conversion
|
// - Conversion
|
||||||
|
|
||||||
|
|
|
@ -2,6 +2,32 @@ import type { FSWatcher } from "chokidar";
|
||||||
import ElectronLog from "electron-log";
|
import ElectronLog from "electron-log";
|
||||||
import { FolderWatch, WatchStoreType } from "../../types/ipc";
|
import { FolderWatch, WatchStoreType } from "../../types/ipc";
|
||||||
import { watchStore } from "../stores/watch.store";
|
import { watchStore } from "../stores/watch.store";
|
||||||
|
import path from "node:path";
|
||||||
|
import fs from "node:fs/promises";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Return the paths of all the files under the given directory (recursive).
|
||||||
|
*
|
||||||
|
* This function walks the directory tree starting at {@link dirPath}, and
|
||||||
|
* returns a list of the absolute paths of all the files that exist therein. It
|
||||||
|
* will recursively traverse into nested directories, and return the absolute
|
||||||
|
* paths of the files there too.
|
||||||
|
*
|
||||||
|
* The returned paths are guaranteed to use POSIX separators ('/').
|
||||||
|
*/
|
||||||
|
export const findFiles = async (dirPath: string) => {
|
||||||
|
const items = await fs.readdir(dirPath, { withFileTypes: true })
|
||||||
|
let paths: string[] = [];
|
||||||
|
for (const item of items) {
|
||||||
|
const itemPath = path.posix.join(dirPath, item.name);
|
||||||
|
if (item.isFile()) {
|
||||||
|
paths.push(itemPath)
|
||||||
|
} else if (item.isDirectory()) {
|
||||||
|
paths = [...paths, ...await findFiles(itemPath)]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return paths
|
||||||
|
}
|
||||||
|
|
||||||
export const addWatchMapping = async (
|
export const addWatchMapping = async (
|
||||||
watcher: FSWatcher,
|
watcher: FSWatcher,
|
||||||
|
|
|
@ -121,8 +121,8 @@ const fsWriteFile = (path: string, contents: string): Promise<void> =>
|
||||||
const fsIsDir = (dirPath: string): Promise<boolean> =>
|
const fsIsDir = (dirPath: string): Promise<boolean> =>
|
||||||
ipcRenderer.invoke("fsIsDir", dirPath);
|
ipcRenderer.invoke("fsIsDir", dirPath);
|
||||||
|
|
||||||
const fsLsFiles = (dirPath: string): Promise<boolean> =>
|
const fsListFiles = (dirPath: string): Promise<string[]> =>
|
||||||
ipcRenderer.invoke("fsLsFiles", dirPath);
|
ipcRenderer.invoke("fsListFiles", dirPath);
|
||||||
|
|
||||||
// - AUDIT below this
|
// - AUDIT below this
|
||||||
|
|
||||||
|
@ -325,7 +325,7 @@ contextBridge.exposeInMainWorld("electron", {
|
||||||
readTextFile: fsReadTextFile,
|
readTextFile: fsReadTextFile,
|
||||||
writeFile: fsWriteFile,
|
writeFile: fsWriteFile,
|
||||||
isDir: fsIsDir,
|
isDir: fsIsDir,
|
||||||
lsFiles: fsLsFiles,
|
listFiles: fsListFiles,
|
||||||
},
|
},
|
||||||
|
|
||||||
// - Conversion
|
// - Conversion
|
||||||
|
|
|
@ -12,19 +12,43 @@ import uploadManager from "services/upload/uploadManager";
|
||||||
import { Collection } from "types/collection";
|
import { Collection } from "types/collection";
|
||||||
import { EncryptedEnteFile } from "types/file";
|
import { EncryptedEnteFile } from "types/file";
|
||||||
import { ElectronFile, FileWithCollection } from "types/upload";
|
import { ElectronFile, FileWithCollection } from "types/upload";
|
||||||
import {
|
import { WatchMapping, WatchMappingSyncedFile } from "types/watchFolder";
|
||||||
EventQueueItem,
|
|
||||||
WatchMapping,
|
|
||||||
WatchMappingSyncedFile,
|
|
||||||
} from "types/watchFolder";
|
|
||||||
import { groupFilesBasedOnCollectionID } from "utils/file";
|
import { groupFilesBasedOnCollectionID } from "utils/file";
|
||||||
import { isSystemFile } from "utils/upload";
|
import { isSystemFile } from "utils/upload";
|
||||||
import { removeFromCollection } from "./collectionService";
|
import { removeFromCollection } from "./collectionService";
|
||||||
import { getLocalFiles } from "./fileService";
|
import { getLocalFiles } from "./fileService";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A file system event encapsulates a change that has occurred on disk that
|
||||||
|
* needs us to take some action within Ente to synchronize with the user's
|
||||||
|
* (Ente) albums.
|
||||||
|
*
|
||||||
|
* Events get added in two ways:
|
||||||
|
*
|
||||||
|
* - When the app starts, it reads the current state of files on disk and
|
||||||
|
* compares that with its last known state to determine what all events it
|
||||||
|
* missed. This is easier than it sounds as we have only two events, add and
|
||||||
|
* remove.
|
||||||
|
*
|
||||||
|
* - When the app is running, it gets live notifications from our file system
|
||||||
|
* watcher (from the Node.js layer) about changes that have happened on disk,
|
||||||
|
* which the app then enqueues onto the event queue if they pertain to the
|
||||||
|
* files we're interested in.
|
||||||
|
*/
|
||||||
|
interface FSEvent {
|
||||||
|
/** The action to take */
|
||||||
|
action: "upload" | "trash";
|
||||||
|
/** The path of the root folder corresponding to the {@link FolderWatch}. */
|
||||||
|
folderPath: string;
|
||||||
|
/** If applicable, the name of the (Ente) collection the file belongs to. */
|
||||||
|
collectionName?: string;
|
||||||
|
/** The absolute path to the file under consideration. */
|
||||||
|
filePath: string;
|
||||||
|
}
|
||||||
|
|
||||||
class WatchFolderService {
|
class WatchFolderService {
|
||||||
private eventQueue: EventQueueItem[] = [];
|
private eventQueue: FSEvent[] = [];
|
||||||
private currentEvent: EventQueueItem;
|
private currentEvent: FSEvent;
|
||||||
private currentlySyncedMapping: WatchMapping;
|
private currentlySyncedMapping: WatchMapping;
|
||||||
private trashingDirQueue: string[] = [];
|
private trashingDirQueue: string[] = [];
|
||||||
private isEventRunning: boolean = false;
|
private isEventRunning: boolean = false;
|
||||||
|
@ -94,6 +118,7 @@ class WatchFolderService {
|
||||||
|
|
||||||
pushEvent(event: EventQueueItem) {
|
pushEvent(event: EventQueueItem) {
|
||||||
this.eventQueue.push(event);
|
this.eventQueue.push(event);
|
||||||
|
log.info("FS event", event);
|
||||||
this.debouncedRunNextEvent();
|
this.debouncedRunNextEvent();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -566,55 +591,41 @@ const getParentFolderName = (filePath: string) => {
|
||||||
};
|
};
|
||||||
|
|
||||||
async function diskFileAddedCallback(file: ElectronFile) {
|
async function diskFileAddedCallback(file: ElectronFile) {
|
||||||
try {
|
const collectionNameAndFolderPath =
|
||||||
const collectionNameAndFolderPath =
|
await watchFolderService.getCollectionNameAndFolderPath(file.path);
|
||||||
await watchFolderService.getCollectionNameAndFolderPath(file.path);
|
|
||||||
|
|
||||||
if (!collectionNameAndFolderPath) {
|
if (!collectionNameAndFolderPath) {
|
||||||
return;
|
return;
|
||||||
}
|
|
||||||
|
|
||||||
const { collectionName, folderPath } = collectionNameAndFolderPath;
|
|
||||||
|
|
||||||
const event: EventQueueItem = {
|
|
||||||
type: "upload",
|
|
||||||
collectionName,
|
|
||||||
folderPath,
|
|
||||||
files: [file],
|
|
||||||
};
|
|
||||||
watchFolderService.pushEvent(event);
|
|
||||||
log.info(
|
|
||||||
`added (upload) to event queue, collectionName:${event.collectionName} folderPath:${event.folderPath}, filesCount: ${event.files.length}`,
|
|
||||||
);
|
|
||||||
} catch (e) {
|
|
||||||
log.error("error while calling diskFileAddedCallback", e);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const { collectionName, folderPath } = collectionNameAndFolderPath;
|
||||||
|
|
||||||
|
const event: EventQueueItem = {
|
||||||
|
type: "upload",
|
||||||
|
collectionName,
|
||||||
|
folderPath,
|
||||||
|
path: file.path,
|
||||||
|
};
|
||||||
|
watchFolderService.pushEvent(event);
|
||||||
}
|
}
|
||||||
|
|
||||||
async function diskFileRemovedCallback(filePath: string) {
|
async function diskFileRemovedCallback(filePath: string) {
|
||||||
try {
|
const collectionNameAndFolderPath =
|
||||||
const collectionNameAndFolderPath =
|
await watchFolderService.getCollectionNameAndFolderPath(filePath);
|
||||||
await watchFolderService.getCollectionNameAndFolderPath(filePath);
|
|
||||||
|
|
||||||
if (!collectionNameAndFolderPath) {
|
if (!collectionNameAndFolderPath) {
|
||||||
return;
|
return;
|
||||||
}
|
|
||||||
|
|
||||||
const { collectionName, folderPath } = collectionNameAndFolderPath;
|
|
||||||
|
|
||||||
const event: EventQueueItem = {
|
|
||||||
type: "trash",
|
|
||||||
collectionName,
|
|
||||||
folderPath,
|
|
||||||
paths: [filePath],
|
|
||||||
};
|
|
||||||
watchFolderService.pushEvent(event);
|
|
||||||
log.info(
|
|
||||||
`added (trash) to event queue collectionName:${event.collectionName} folderPath:${event.folderPath} , pathsCount: ${event.paths.length}`,
|
|
||||||
);
|
|
||||||
} catch (e) {
|
|
||||||
log.error("error while calling diskFileRemovedCallback", e);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const { collectionName, folderPath } = collectionNameAndFolderPath;
|
||||||
|
|
||||||
|
const event: EventQueueItem = {
|
||||||
|
type: "trash",
|
||||||
|
collectionName,
|
||||||
|
folderPath,
|
||||||
|
path: filePath,
|
||||||
|
};
|
||||||
|
watchFolderService.pushEvent(event);
|
||||||
}
|
}
|
||||||
|
|
||||||
async function diskFolderRemovedCallback(folderPath: string) {
|
async function diskFolderRemovedCallback(folderPath: string) {
|
||||||
|
@ -682,34 +693,51 @@ const syncWithDisk = async (
|
||||||
const events: EventQueueItem[] = [];
|
const events: EventQueueItem[] = [];
|
||||||
|
|
||||||
for (const mapping of activeMappings) {
|
for (const mapping of activeMappings) {
|
||||||
const files = await electron.getDirFiles(mapping.folderPath);
|
const folderPath = mapping.folderPath;
|
||||||
|
|
||||||
const filesToUpload = getValidFilesToUpload(files, mapping);
|
const paths = (await electron.fs.listFiles(folderPath))
|
||||||
|
// Filter out hidden files (files whose names begins with a dot)
|
||||||
|
.filter((n) => !n.startsWith("."))
|
||||||
|
// Prepend folderPath to get the full path
|
||||||
|
.map((f) => `${folderPath}/${f}`);
|
||||||
|
|
||||||
for (const file of filesToUpload)
|
// Files that are on disk but not yet synced.
|
||||||
|
const pathsToUpload = paths.filter(
|
||||||
|
(path) => !isSyncedOrIgnoredPath(path, mapping),
|
||||||
|
);
|
||||||
|
|
||||||
|
for (const path of pathsToUpload)
|
||||||
events.push({
|
events.push({
|
||||||
type: "upload",
|
type: "upload",
|
||||||
collectionName: getCollectionNameForMapping(mapping, file.path),
|
collectionName: getCollectionNameForMapping(mapping, path),
|
||||||
folderPath: mapping.folderPath,
|
folderPath,
|
||||||
files: [file],
|
filePath: path,
|
||||||
});
|
});
|
||||||
|
|
||||||
const filesToRemove = mapping.syncedFiles.filter((file) => {
|
// Synced files that are no longer on disk
|
||||||
return !files.find((f) => f.path === file.path);
|
const pathsToRemove = mapping.syncedFiles.filter(
|
||||||
});
|
(file) => !paths.includes(file.path),
|
||||||
|
);
|
||||||
|
|
||||||
for (const file of filesToRemove)
|
for (const path of pathsToRemove)
|
||||||
events.push({
|
events.push({
|
||||||
type: "trash",
|
type: "trash",
|
||||||
collectionName: getCollectionNameForMapping(mapping, file.path),
|
collectionName: getCollectionNameForMapping(mapping, path),
|
||||||
folderPath: mapping.folderPath,
|
folderPath: mapping.folderPath,
|
||||||
paths: [file.path],
|
filePath: path,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
return { events, nonExistentFolderPaths };
|
return { events, nonExistentFolderPaths };
|
||||||
};
|
};
|
||||||
|
|
||||||
|
function isSyncedOrIgnoredPath(path: string, mapping: WatchMapping) {
|
||||||
|
return (
|
||||||
|
mapping.ignoredFiles.includes(path) ||
|
||||||
|
mapping.syncedFiles.find((f) => f.path === path)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
const getCollectionNameForMapping = (
|
const getCollectionNameForMapping = (
|
||||||
mapping: WatchMapping,
|
mapping: WatchMapping,
|
||||||
filePath: string,
|
filePath: string,
|
||||||
|
|
|
@ -1,5 +1,4 @@
|
||||||
import { UPLOAD_STRATEGY } from "constants/upload";
|
import { UPLOAD_STRATEGY } from "constants/upload";
|
||||||
import { ElectronFile } from "types/upload";
|
|
||||||
|
|
||||||
export interface WatchMappingSyncedFile {
|
export interface WatchMappingSyncedFile {
|
||||||
path: string;
|
path: string;
|
||||||
|
@ -14,11 +13,3 @@ export interface WatchMapping {
|
||||||
syncedFiles: WatchMappingSyncedFile[];
|
syncedFiles: WatchMappingSyncedFile[];
|
||||||
ignoredFiles: string[];
|
ignoredFiles: string[];
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface EventQueueItem {
|
|
||||||
type: "upload" | "trash";
|
|
||||||
folderPath: string;
|
|
||||||
collectionName?: string;
|
|
||||||
paths?: string[];
|
|
||||||
files?: ElectronFile[];
|
|
||||||
}
|
|
||||||
|
|
|
@ -207,7 +207,7 @@ export interface Electron {
|
||||||
isDir: (dirPath: string) => Promise<boolean>;
|
isDir: (dirPath: string) => Promise<boolean>;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Return a list of the names of the files in the given directory.
|
* Return a list of the file names of the files in the given directory.
|
||||||
*
|
*
|
||||||
* Note:
|
* Note:
|
||||||
*
|
*
|
||||||
|
@ -216,7 +216,7 @@ export interface Electron {
|
||||||
*
|
*
|
||||||
* - It will return only the names of files, not directories.
|
* - It will return only the names of files, not directories.
|
||||||
*/
|
*/
|
||||||
lsFiles: (dirPath: string) => Promise<string>;
|
listFiles: (dirPath: string) => Promise<string[]>;
|
||||||
};
|
};
|
||||||
|
|
||||||
/*
|
/*
|
||||||
|
|
Loading…
Reference in a new issue