beenull/ente
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

has collection identifier with metadat

Browse source
This commit is contained in:
Abhinav-grd 2021-08-11 12:50:26 +05:30
parent 9aef824603
commit ba065405eb
7 changed files with 199 additions and 179 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

271
src/components/pages/gallery/UploadProgress.tsx
View file

@ -1,11 +1,12 @@
import ExpandLess from 'components/icons/ExpandLess';
import ExpandMore from 'components/icons/ExpandMore';
import React, { useState } from 'react';
import {
Button, Modal, ProgressBar,
} from 'react-bootstrap';
import { Button, Modal, ProgressBar } from 'react-bootstrap';
import { FileRejection } from 'react-dropzone';
import { FileUploadResults, UPLOAD_STAGES } from 'services/upload/uploadService';
import {
FileUploadResults,
UPLOAD_STAGES,
} from 'services/upload/uploadManager';
import styled from 'styled-components';
import { DESKTOP_APP_DOWNLOAD_URL } from 'utils/common';
import constants from 'utils/strings/constants';
@ -19,77 +20,85 @@ interface Props {
retryFailed;
fileProgress: Map<string, number>;
show;
fileRejections:FileRejection[]
uploadResult:Map<string, number>;
fileRejections: FileRejection[];
uploadResult: Map<string, number>;
}
interface FileProgresses{
fileName:string;
progress:number;
interface FileProgresses {
fileName: string;
progress: number;
}
const Content =styled.div<{collapsed:boolean, sm?:boolean, height?:number}>`
const Content = styled.div<{
collapsed: boolean;
sm?: boolean;
height?: number;
}>`
overflow: hidden;
height:${(props)=> props.collapsed?'0px':props.height+'px'};
transition:${(props)=> 'height '+0.001*props.height+'s ease-out'};
height: ${(props) => (props.collapsed ? '0px' : props.height + 'px')};
transition: ${(props) => 'height ' + 0.001 * props.height + 's ease-out'};
margin-bottom: 20px;
& >p {
padding-left:35px;
margin:0;
& > p {
padding-left: 35px;
margin: 0;
}
`;
const FileList =styled.ul`
padding-left:50px;
margin-top:5px;
const FileList = styled.ul`
padding-left: 50px;
margin-top: 5px;
& > li {
padding-left:10px;
margin-bottom:10px;
color:#ccc;
padding-left: 10px;
margin-bottom: 10px;
color: #ccc;
}
`;
const SectionTitle =styled.div`
display:flex;
justify-content:space-between;
padding:0 20px;
color:#eee;
font-size:20px;
cursor:pointer;
const SectionTitle = styled.div`
display: flex;
justify-content: space-between;
padding: 0 20px;
color: #eee;
font-size: 20px;
cursor: pointer;
`;
interface ResultSectionProps{
interface ResultSectionProps {
fileUploadResultMap: Map<FileUploadResults, string[]>;
fileUploadResult:FileUploadResults;
fileUploadResult: FileUploadResults;
sectionTitle;
sectionInfo;
infoHeight:number;
infoHeight: number;
}
const ResultSection =(props:ResultSectionProps)=>{
const [listView, setListView]=useState(false);
const fileList=props.fileUploadResultMap?.get(props.fileUploadResult);
const ResultSection = (props: ResultSectionProps) => {
const [listView, setListView] = useState(false);
const fileList = props.fileUploadResultMap?.get(props.fileUploadResult);
if (!fileList?.length) {
return <></>;
}
return (<>
<SectionTitle onClick={()=>setListView(!listView)} > {props.sectionTitle} {listView?<ExpandLess/>:<ExpandMore/>}</SectionTitle>
<Content collapsed={!listView} height={fileList.length *33 + props.infoHeight}>
<p>{props.sectionInfo}</p>
<FileList>
{fileList.map((fileName) => (
<li key={fileName}>
{fileName}
</li>
))}
</FileList>
</Content>
</>);
return (
<>
<SectionTitle onClick={() => setListView(!listView)}>
{' '}
{props.sectionTitle}{' '}
{listView ? <ExpandLess /> : <ExpandMore />}
</SectionTitle>
<Content
collapsed={!listView}
height={fileList.length * 33 + props.infoHeight}>
<p>{props.sectionInfo}</p>
<FileList>
{fileList.map((fileName) => (
<li key={fileName}>{fileName}</li>
))}
</FileList>
</Content>
</>
);
};
export default function UploadProgress(props: Props) {
const fileProgressStatuses = [] as FileProgresses[];
const fileUploadResultMap = new Map<FileUploadResults, string[]>();
let filesNotUploaded=false;
let filesNotUploaded = false;
if (props.fileProgress) {
for (const [fileName, progress] of props.fileProgress) {
@ -101,10 +110,10 @@ export default function UploadProgress(props: Props) {
if (!fileUploadResultMap.has(progress)) {
fileUploadResultMap.set(progress, []);
}
if (progress<0) {
filesNotUploaded=true;
if (progress < 0) {
filesNotUploaded = true;
}
const fileList= fileUploadResultMap.get(progress);
const fileList = fileUploadResultMap.get(progress);
fileUploadResultMap.set(progress, [...fileList, fileName]);
}
}
@ -113,90 +122,120 @@ export default function UploadProgress(props: Props) {
<Modal
show={props.show}
onHide={
props.uploadStage !== UPLOAD_STAGES.FINISH ?
() => null :
props.closeModal
props.uploadStage !== UPLOAD_STAGES.FINISH
? () => null
: props.closeModal
}
aria-labelledby="contained-modal-title-vcenter"
centered
backdrop={
fileProgressStatuses?.length !== 0 ? 'static' : 'true'
}
>
backdrop={fileProgressStatuses?.length !== 0 ? 'static' : 'true'}>
<Modal.Header
style={{ display: 'flex',
style={{
display: 'flex',
justifyContent: 'center',
textAlign: 'center',
borderBottom: 'none',
paddingTop: '30px',
paddingBottom: '0px' }}
closeButton={props.uploadStage === UPLOAD_STAGES.FINISH}
>
paddingBottom: '0px',
}}
closeButton={props.uploadStage === UPLOAD_STAGES.FINISH}>
<h4 style={{ width: '100%' }}>
{props.uploadStage === UPLOAD_STAGES.UPLOADING ?
constants.UPLOAD[props.uploadStage](
props.fileCounter,
) :
constants.UPLOAD[props.uploadStage]}
{props.uploadStage === UPLOAD_STAGES.UPLOADING
? constants.UPLOAD[props.uploadStage](props.fileCounter)
: constants.UPLOAD[props.uploadStage]}
</h4>
</Modal.Header>
<Modal.Body>
{(props.uploadStage === UPLOAD_STAGES.READING_GOOGLE_METADATA_FILES ||
props.uploadStage === UPLOAD_STAGES.UPLOADING) &&
(
< ProgressBar
now={props.now}
animated
variant="upload-progress-bar"
/>
)
}
{fileProgressStatuses.length>0 &&
<FileList>
{fileProgressStatuses.map(({ fileName, progress }) => (
<li key={fileName} style={{ marginTop: '12px' }}>
{props.uploadStage===UPLOAD_STAGES.FINISH ?
fileName :
constants.FILE_UPLOAD_PROGRESS(
fileName,
progress,
)
}
</li>
))}
</FileList>}
{(props.uploadStage ===
UPLOAD_STAGES.READING_GOOGLE_METADATA_FILES ||
props.uploadStage === UPLOAD_STAGES.UPLOADING) && (
<ProgressBar
now={props.now}
animated
variant="upload-progress-bar"
/>
)}
{fileProgressStatuses.length > 0 && (
<FileList>
{fileProgressStatuses.map(({ fileName, progress }) => (
<li key={fileName} style={{ marginTop: '12px' }}>
{props.uploadStage === UPLOAD_STAGES.FINISH
? fileName
: constants.FILE_UPLOAD_PROGRESS(
fileName,
progress,
)}
</li>
))}
</FileList>
)}
<ResultSection fileUploadResultMap={fileUploadResultMap} fileUploadResult={FileUploadResults.UPLOADED} sectionTitle={constants.SUCCESSFUL_UPLOADS} sectionInfo={constants.SUCCESS_INFO} infoHeight={32}/>
<ResultSection
fileUploadResultMap={fileUploadResultMap}
fileUploadResult={FileUploadResults.UPLOADED}
sectionTitle={constants.SUCCESSFUL_UPLOADS}
sectionInfo={constants.SUCCESS_INFO}
infoHeight={32}
/>
{props.uploadStage === UPLOAD_STAGES.FINISH && filesNotUploaded && (
{props.uploadStage === UPLOAD_STAGES.FINISH &&
filesNotUploaded && (
<AlertBanner variant="warning">
{constants.FILE_NOT_UPLOADED_LIST}
</AlertBanner>
)}
<ResultSection fileUploadResultMap={fileUploadResultMap} fileUploadResult={FileUploadResults.BLOCKED} sectionTitle={constants.BLOCKED_UPLOADS} sectionInfo={constants.ETAGS_BLOCKED(DESKTOP_APP_DOWNLOAD_URL)} infoHeight={140}/>
<ResultSection fileUploadResultMap={fileUploadResultMap} fileUploadResult={FileUploadResults.FAILED} sectionTitle={constants.FAILED_UPLOADS} sectionInfo={constants.FAILED_INFO} infoHeight={48}/>
<ResultSection fileUploadResultMap={fileUploadResultMap} fileUploadResult={FileUploadResults.SKIPPED} sectionTitle={constants.SKIPPED_FILES} sectionInfo={constants.SKIPPED_INFO} infoHeight={32}/>
<ResultSection fileUploadResultMap={fileUploadResultMap} fileUploadResult={FileUploadResults.UNSUPPORTED} sectionTitle={constants.UNSUPPORTED_FILES} sectionInfo={constants.UNSUPPORTED_INFO} infoHeight={32}/>
<ResultSection
fileUploadResultMap={fileUploadResultMap}
fileUploadResult={FileUploadResults.BLOCKED}
sectionTitle={constants.BLOCKED_UPLOADS}
sectionInfo={constants.ETAGS_BLOCKED(
DESKTOP_APP_DOWNLOAD_URL,
)}
infoHeight={140}
/>
<ResultSection
fileUploadResultMap={fileUploadResultMap}
fileUploadResult={FileUploadResults.FAILED}
sectionTitle={constants.FAILED_UPLOADS}
sectionInfo={constants.FAILED_INFO}
infoHeight={48}
/>
<ResultSection
fileUploadResultMap={fileUploadResultMap}
fileUploadResult={FileUploadResults.SKIPPED}
sectionTitle={constants.SKIPPED_FILES}
sectionInfo={constants.SKIPPED_INFO}
infoHeight={32}
/>
<ResultSection
fileUploadResultMap={fileUploadResultMap}
fileUploadResult={FileUploadResults.UNSUPPORTED}
sectionTitle={constants.UNSUPPORTED_FILES}
sectionInfo={constants.UNSUPPORTED_INFO}
infoHeight={32}
/>
{props.uploadStage === UPLOAD_STAGES.FINISH && (
<Modal.Footer style={{ border: 'none' }}>
{props.uploadStage===UPLOAD_STAGES.FINISH && ((fileUploadResultMap?.get(FileUploadResults.FAILED)?.length>0 || fileUploadResultMap?.get(FileUploadResults.BLOCKED)?.length>0)? (
<Button
variant="outline-success"
style={{ width: '100%' }}
onClick={props.retryFailed}
>
{constants.RETRY_FAILED}
</Button>
) : ( <Button
variant="outline-secondary"
style={{ width: '100%' }}
onClick={props.closeModal}
>
{constants.CLOSE}
</Button>
))}
{props.uploadStage === UPLOAD_STAGES.FINISH &&
(fileUploadResultMap?.get(FileUploadResults.FAILED)
?.length > 0 ||
fileUploadResultMap?.get(FileUploadResults.BLOCKED)
?.length > 0 ? (
<Button
variant="outline-success"
style={{ width: '100%' }}
onClick={props.retryFailed}>
{constants.RETRY_FAILED}
</Button>
) : (
<Button
variant="outline-secondary"
style={{ width: '100%' }}
onClick={props.closeModal}>
{constants.CLOSE}
</Button>
))}
</Modal.Footer>
)}
</Modal.Body>

3
src/services/upload/metadataService.ts
View file

@ -46,6 +46,9 @@ export async function extractMetatdata(
return extractedMetadata;
}
export const getMetadataKey = (fileWithCollection: number, title: string) =>
`${fileWithCollection}_${title}`;
export async function parseMetadataJSON(receivedFile: globalThis.File) {
try {
const metadataJSON: object = await new Promise((resolve, reject) => {

38
src/services/upload/readFileService.ts
View file

@ -8,14 +8,13 @@ const TYPE_HEIC = 'HEIC';
export const TYPE_IMAGE = 'image';
const EDITED_FILE_SUFFIX = '-edited';
export async function getFileData(reader:FileReader, file:globalThis.File) {
return file.size > MIN_STREAM_FILE_SIZE ?
getFileStream(reader, file) :
await getUint8ArrayView(reader, file);
export async function getFileData(reader: FileReader, file: globalThis.File) {
return file.size > MIN_STREAM_FILE_SIZE
? getFileStream(reader, file)
: await getUint8ArrayView(reader, file);
}
export function getFileType(receivedFile:globalThis.File) {
export function getFileType(receivedFile: globalThis.File) {
let fileType: FILE_TYPE;
switch (receivedFile.type.split('/')[0]) {
case TYPE_IMAGE:
@ -37,18 +36,14 @@ export function getFileType(receivedFile:globalThis.File) {
return fileType;
}
export function getFileOriginalName(file: globalThis.File) {
let originalName: string = null;
export function getFileOriginalName(file:globalThis.File) {
let originalName:string=null;
const isEditedFile=file.name.endsWith(EDITED_FILE_SUFFIX);
const isEditedFile = file.name.endsWith(EDITED_FILE_SUFFIX);
if (isEditedFile) {
originalName = file.name.slice(
0,
-1 * EDITED_FILE_SUFFIX.length,
);
originalName = file.name.slice(0, -1 * EDITED_FILE_SUFFIX.length);
} else {
originalName=file.name;
originalName = file.name;
}
return originalName;
}
@ -70,7 +65,10 @@ function getFileStream(reader: FileReader, file: globalThis.File) {
};
}
async function* fileChunkReaderMaker(reader:FileReader, file:globalThis.File) {
async function* fileChunkReaderMaker(
reader: FileReader,
file: globalThis.File,
) {
let offset = 0;
while (offset < file.size) {
const blob = file.slice(offset, ENCRYPTION_CHUNK_SIZE + offset);
@ -92,9 +90,9 @@ export async function getUint8ArrayView(
reader.onload = () => {
// Do whatever you want with the file contents
const result =
typeof reader.result === 'string' ?
new TextEncoder().encode(reader.result) :
new Uint8Array(reader.result);
typeof reader.result === 'string'
? new TextEncoder().encode(reader.result)
: new Uint8Array(reader.result);
resolve(result);
};
reader.readAsArrayBuffer(file);
@ -104,5 +102,3 @@ export async function getUint8ArrayView(
throw e;
}
}

16
src/services/upload/s3Service.ts
View file

@ -1,17 +1,19 @@
import {
CHUNKS_COMBINED_FOR_A_UPLOAD_PART,
DataStream,
MultipartUploadURLs,
RANDOM_PERCENTAGE_PROGRESS_FOR_PUT,
} from './uploadService';
import { CHUNKS_COMBINED_FOR_A_UPLOAD_PART, DataStream } from './uploadService';
import NetworkClient from './networkClient';
import * as convert from 'xml-js';
import UIService from './uiService';
import UIService, { RANDOM_PERCENTAGE_PROGRESS_FOR_PUT } from './uiService';
interface PartEtag {
PartNumber: number;
Etag: string;
}
export interface MultipartUploadURLs {
objectKey: string;
partURLs: string[];
completeURL: string;
}
export function calculatePartCount(chunkCount: number) {
const partCount = Math.ceil(chunkCount / CHUNKS_COMBINED_FOR_A_UPLOAD_PART);
return partCount;

44
src/services/upload/uploadService.ts
View file

@ -3,7 +3,11 @@ import { Collection } from '../collectionService';
import { FILE_TYPE } from 'pages/gallery';
import { logError } from 'utils/sentry';
import NetworkClient from './networkClient';
import { extractMetatdata, ParsedMetaDataJSON } from './metadataService';
import {
extractMetatdata,
getMetadataKey,
ParsedMetaDataJSON,
} from './metadataService';
import { generateThumbnail } from './thumbnailService';
import {
getFileType,
@ -15,30 +19,18 @@ import { ENCRYPTION_CHUNK_SIZE } from 'types';
import { uploadStreamUsingMultipart } from './s3Service';
import UIService from './uiService';
import { parseError } from 'utils/common/errorUtil';
import { FileWithCollection, MetadataMap } from './uploadManager';
export const RANDOM_PERCENTAGE_PROGRESS_FOR_PUT = () => 90 + 10 * Math.random();
export const MIN_STREAM_FILE_SIZE = 20 * 1024 * 1024;
export const CHUNKS_COMBINED_FOR_A_UPLOAD_PART = Math.floor(
MIN_STREAM_FILE_SIZE / ENCRYPTION_CHUNK_SIZE,
);
export enum FileUploadResults {
FAILED = -1,
SKIPPED = -2,
UNSUPPORTED = -3,
BLOCKED = -4,
UPLOADED = 100,
}
export interface UploadURL {
url: string;
objectKey: string;
}
export interface FileWithCollection {
file: globalThis.File;
collection: Collection;
}
export interface DataStream {
stream: ReadableStream<Uint8Array>;
chunkCount: number;
@ -57,12 +49,6 @@ export interface B64EncryptionResult {
nonce: string;
}
export interface MultipartUploadURLs {
objectKey: string;
partURLs: string[];
completeURL: string;
}
export interface MetadataObject {
title: string;
creationTime: number;
@ -91,21 +77,12 @@ export interface ProcessedFile {
}
export interface BackupedFile extends Omit<ProcessedFile, 'filename'> {}
export type MetadataMap = Map<string, ParsedMetaDataJSON>;
export interface UploadFile extends BackupedFile {
collectionID: number;
encryptedKey: string;
keyDecryptionNonce: string;
}
export enum UPLOAD_STAGES {
START,
READING_GOOGLE_METADATA_FILES,
UPLOADING,
FINISH,
}
class UploadService {
private uploadURLs: UploadURL[] = [];
private metadataMap: Map<string, ParsedMetaDataJSON>;
@ -117,9 +94,10 @@ class UploadService {
await this.preFetchUploadURLs();
}
async readFile(reader: FileReader, receivedFile: globalThis.File) {
async readFile(reader: FileReader, fileWithCollection: FileWithCollection) {
try {
const fileType = getFileType(receivedFile);
const { file: receivedFile, collectionID } = fileWithCollection;
const fileType = getFileType(fileWithCollection.file);
const { thumbnail, hasStaticThumbnail } = await generateThumbnail(
reader,
@ -128,7 +106,9 @@ class UploadService {
);
const originalName = getFileOriginalName(receivedFile);
const googleMetadata = this.metadataMap.get(originalName);
const googleMetadata = this.metadataMap.get(
getMetadataKey(collectionID, originalName),
);
const extractedMetadata: MetadataObject = await extractMetatdata(
reader,
receivedFile,

2
src/services/upload/uploader.ts
View file

@ -34,7 +34,7 @@ export default async function uploader(
let file: FileInMemory = null;
let encryptedFile: EncryptedFile = null;
try {
file = await UploadService.readFile(reader, rawFile);
file = await UploadService.readFile(reader, fileWithCollection);
if (
fileAlreadyInCollection(

4
src/utils/upload/index.ts
View file

@ -37,7 +37,7 @@ export function areFilesSame(
export function segregateFiles(
filesWithCollectionToUpload: FileWithCollection[],
) {
const metadataFiles: globalThis.File[] = [];
const metadataFiles: FileWithCollection[] = [];
const mediaFiles: FileWithCollection[] = [];
filesWithCollectionToUpload.forEach((fileWithCollection) => {
const file = fileWithCollection.file;
@ -46,7 +46,7 @@ export function segregateFiles(
return;
}
if (file.name.slice(-4) === TYPE_JSON) {
metadataFiles.push(fileWithCollection.file);
metadataFiles.push(fileWithCollection);
} else {
mediaFiles.push(fileWithCollection);
}