beenull/ente
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Merge pull request #9 from ente-io/file-loading-single-update

Browse source 
File loading single update
This commit is contained in:
Pushkar Anand 2021-02-07 21:15:07 +05:30 committed by GitHub
parent 9f6388b8ef 45fd07bc5f
commit 36c84807f1
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 116 additions and 59 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
package.json
View file

@ -24,6 +24,7 @@
"react-dom": "16.13.1",
"react-dropzone": "^11.2.4",
"react-photoswipe": "^1.3.0",
"react-top-loading-bar": "^2.0.1",
"react-virtualized-auto-sizer": "^1.0.2",
"react-window": "^1.8.6",
"react-window-infinite-loader": "^1.0.5",

44
src/pages/gallery/index.tsx
View file

@ -7,19 +7,27 @@ import {
getFile,
getPreview,
fetchData,
localFiles,
} from 'services/fileService';
import { getData, LS_KEYS } from 'utils/storage/localStorage';
import PreviewCard from './components/PreviewCard';
import { getActualKey } from 'utils/common/key';
import { getActualKey, getToken } from 'utils/common/key';
import styled from 'styled-components';
import PhotoSwipe from 'components/PhotoSwipe/PhotoSwipe';
import { Options } from 'photoswipe';
import AutoSizer from 'react-virtualized-auto-sizer';
import { VariableSizeList as List } from 'react-window';
import LoadingBar from 'react-top-loading-bar'
import Collections from './components/Collections';
import SadFace from 'components/SadFace';
import Upload from './components/Upload';
import { collection, fetchCollections, collectionLatestFile, getCollectionLatestFile, getFavItemIds } from 'services/collectionService';
import {
collection,
fetchUpdatedCollections,
collectionLatestFile,
getCollectionLatestFile,
getFavItemIds,
getLocalCollections
} from 'services/collectionService';
import constants from 'utils/strings/constants';
enum ITEM_TYPE {
@ -106,7 +114,7 @@ export default function Gallery(props) {
});
const fetching: { [k: number]: boolean } = {};
const [progress, setProgress] = useState(0)
useEffect(() => {
const key = getKey(SESSION_KEYS.ENCRYPTION_KEY);
@ -115,24 +123,35 @@ export default function Gallery(props) {
}
const main = async () => {
setLoading(true);
await syncWithRemote();
const data = await localFiles();
const collections = await getLocalCollections();
setData(data);
setCollections(collections);
setLoading(false);
setProgress(80);
await syncWithRemote();
setProgress(100);
};
main();
props.setUploadButtonView(true);
}, []);
const syncWithRemote = async () => {
const token = getData(LS_KEYS.USER).token;
const token = getToken();
const encryptionKey = await getActualKey();
const collections = await fetchCollections(token, encryptionKey);
const data = await fetchData(token, collections);
const updatedCollections = await fetchUpdatedCollections(token, encryptionKey);
const data = await fetchData(token, updatedCollections);
const collections = await getLocalCollections();
const collectionLatestFile = await getCollectionLatestFile(collections, data);
const favItemIds = await getFavItemIds(data);
setCollections(collections);
setData(data);
if (updatedCollections.length > 0) {
setCollections(collections);
setData(data);
}
setCollectionLatestFile(collectionLatestFile);
setFavItemIds(favItemIds);
props.setUploadButtonView(true);
}
if (!data || loading) {
return (
@ -288,6 +307,11 @@ export default function Gallery(props) {
return (
<>
<LoadingBar
color='#f11946'
progress={progress}
onLoaderFinished={() => setProgress(0)}
/>
<Collections
collections={collections}
selected={router.query.collection?.toString()}

28
src/services/collectionService.ts
View file

@ -97,15 +97,33 @@ const getCollections = async (
return await Promise.all(promises);
}
catch (e) {
console.log("getCollections falied- " + e);
console.log("getCollections failed- " + e);
}
};
export const fetchCollections = async (token: string, key: string) => {
const collections = await getCollections(token, '0', key);
const favCollection = collections.filter(collection => collection.type === CollectionType.favorites);
await localForage.setItem('fav-collection', favCollection);
export const getLocalCollections = async (): Promise<collection[]> => {
const collections = await localForage.getItem('collections') as collection[] ?? [];
return collections;
}
export const fetchUpdatedCollections = async (token: string, key: string) => {
const collectionUpdateTime = await localForage.getItem('collection-update-time') as string;
const updatedCollections = await getCollections(token, collectionUpdateTime ?? '0', key) || [];
const favCollection = await localForage.getItem('fav-collection') as collection[] ?? updatedCollections.filter(collection => collection.type === CollectionType.favorites);
const localCollections = await getLocalCollections();
const allCollectionsInstances = [...localCollections, ...updatedCollections];
var latestCollectionsInstances = new Map<string, collection>();
allCollectionsInstances.forEach((collection) => {
if (!latestCollectionsInstances.has(collection.id) || latestCollectionsInstances.get(collection.id).updationTime < collection.updationTime) {
latestCollectionsInstances.set(collection.id, collection);
}
});
let collections = [];
for (const [_, collection] of latestCollectionsInstances) {
collections.push(collection);
}
await localForage.setItem('fav-collection', favCollection);
await localForage.setItem('collections', collections);
return updatedCollections;
};
export const getCollectionLatestFile = (

102
src/services/fileService.ts
View file

@ -66,13 +66,23 @@ export const fetchData = async (token, collections) => {
);
}
export const localFiles = async () => {
let files: Array<file> = (await localForage.getItem<file[]>('files')) || [];
return files;
}
export const fetchFiles = async (
token: string,
collections: collection[]
) => {
let files: Array<file> = (await localForage.getItem<file[]>('files')) || [];
const fetchedFiles = await getFiles(collections, null, "100", token);
let files = await localFiles();
const collectionUpdationTime = new Map<string, string>();
let fetchedFiles = [];
for (let collection of collections) {
const files = await getFiles(collection, null, 100, token);
fetchedFiles.push(...files);
collectionUpdationTime.set(collection.id, files.length > 0 ? files.slice(-1)[0].updationTime.toString() : "0");
}
files.push(...fetchedFiles);
var latestFiles = new Map<string, file>();
files.forEach((file) => {
@ -82,7 +92,7 @@ export const fetchFiles = async (
}
});
files = [];
for (const [_, file] of latestFiles.entries()) {
for (const [_, file] of latestFiles) {
if (!file.isDeleted)
files.push(file);
}
@ -90,53 +100,57 @@ export const fetchFiles = async (
(a, b) => b.metadata.creationTime - a.metadata.creationTime
);
await localForage.setItem('files', files);
for (let [collectionID, updationTime] of collectionUpdationTime) {
await localForage.setItem(`${collectionID}-time`, updationTime);
}
let updationTime = await localForage.getItem('collection-update-time') as number;
for (let collection of collections) {
updationTime = Math.max(updationTime, collection.updationTime);
}
await localForage.setItem('collection-update-time', updationTime);
return files;
};
export const getFiles = async (collections: collection[], sinceTime: string, limit: string, token: string): Promise<file[]> => {
export const getFiles = async (collection: collection, sinceTime: string, limit: number, token: string): Promise<file[]> => {
try {
const worker = await new CryptoWorker();
let promises: Promise<file>[] = [];
for (const index in collections) {
const collection = collections[index];
if (collection.isDeleted) {
// TODO: Remove files in this collection from localForage and cache
continue;
}
let time =
sinceTime || (await localForage.getItem<string>(`${collection.id}-time`)) || "0";
let resp;
do {
resp = await HTTPService.get(`${ENDPOINT}/collections/diff`, {
collectionID: collection.id,
sinceTime: time,
limit,
},
{
'X-Auth-Token': token
});
promises.push(...resp.data.diff.map(
async (file: file) => {
if (!file.isDeleted) {
file.key = await worker.decryptB64(
file.encryptedKey,
file.keyDecryptionNonce,
collection.key
);
file.metadata = await worker.decryptMetadata(file);
}
return file;
}
));
if (resp.data.diff.length) {
time = resp.data.diff.slice(-1)[0].updationTime.toString();
}
} while (resp.data.diff.length);
await localForage.setItem(`${collection.id}-time`, time);
if (collection.isDeleted) {
// TODO: Remove files in this collection from localForage and cache
return;
}
return Promise.all(promises);
let time =
sinceTime || (await localForage.getItem<string>(`${collection.id}-time`)) || "0";
let resp;
do {
resp = await HTTPService.get(`${ENDPOINT}/collections/diff`, {
collectionID: collection.id,
sinceTime: time,
limit: limit.toString(),
},
{
'X-Auth-Token': token
});
promises.push(...resp.data.diff.map(
async (file: file) => {
if (!file.isDeleted) {
file.key = await worker.decryptB64(
file.encryptedKey,
file.keyDecryptionNonce,
collection.key
);
file.metadata = await worker.decryptMetadata(file);
}
return file;
}
));
if (resp.data.diff.length) {
time = resp.data.diff.slice(-1)[0].updationTime.toString();
}
} while (resp.data.diff.length === limit);
return await Promise.all(promises);
} catch (e) {
console.log("Get files failed" + e);
}