Show people face chips in info panel of image

src/components/PhotoSwipe/PhotoSwipe.tsx

@@ -36,6 +36,9 @@ import DatePicker from 'react-datepicker';
 import 'react-datepicker/dist/react-datepicker.css';
 import CloseIcon from 'components/icons/CloseIcon';
 import TickIcon from 'components/icons/TickIcon';
+import TFJSImage from 'components/TFJSImage';
+import { Person } from 'utils/machineLearning/types';
+import { getPeopleList } from 'utils/machineLearning';
 
 interface Iprops {
     isOpen: boolean;
@@ -67,6 +70,24 @@ const Pre = styled.pre`
     padding: 7px 15px;
 `;
 
+const FaceChipContainer = styled.div`
+    display: flex;
+    flex-wrap: wrap;
+    justify-content: center;
+    align-items: center;
+    margin-top: 10px;
+    margin-bottom: 10px;
+`;
+
+const FaceChip = styled.div`
+    width: 112px;
+    height: 112px;
+    margin-right: 10px;
+    border-radius: 50%;
+    overflow: hidden;
+    position: relative;
+`;
+
 const renderInfoItem = (label: string, value: string | JSX.Element) => (
     <Row>
         <Label width="30%">{label}</Label>
@@ -228,6 +249,35 @@ function ExifData(props: { exif: any }) {
     );
 }
 
+function PeopleList(props: { file }) {
+    const [peopleList, setPeopleList] = useState<Array<Person>>([]);
+
+    const updateFaceImages = async () => {
+        const peopleList = await getPeopleList(props.file);
+        setPeopleList(peopleList);
+    };
+
+    useEffect(() => {
+        // TODO: handle multiple async updates
+        updateFaceImages();
+    }, [props.file]);
+
+    return (
+        <>
+            <div>
+                <Legend>{constants.PEOPLE}</Legend>
+            </div>
+            <FaceChipContainer>
+                {peopleList.map((person, index) => (
+                    <FaceChip key={index}>
+                        <TFJSImage faceImage={person.faceImage}></TFJSImage>
+                    </FaceChip>
+                ))}
+            </FaceChipContainer>
+        </>
+    );
+}
+
 function InfoModal({
     showInfo,
     handleCloseInfo,
@@ -274,6 +324,7 @@ function InfoModal({
                             {constants.SHOW_MAP}
                         </a>
                     )}
+                <PeopleList file={items[photoSwipe?.getCurrentIndex()]} />
                 {exif && (
                     <>
                         <ExifData exif={exif} />

src/services/machineLearning/machineLearningService.ts

@@ -1,5 +1,4 @@
 import { File, getLocalFiles } from 'services/fileService';
-import DownloadManager from 'services/downloadManager';
 
 import * as tf from '@tensorflow/tfjs-core';
 import '@tensorflow/tfjs-backend-webgl';
@@ -21,13 +20,16 @@ import {
     Person,
 } from 'utils/machineLearning/types';
 
-import * as jpeg from 'jpeg-js';
 import ClusteringService from './clusteringService';
 
 import { toTSNE } from 'utils/machineLearning/visualization';
 import { mlFilesStore, mlPeopleStore } from 'utils/storage/localForage';
 import ArcfaceAlignmentService from './arcfaceAlignmentService';
-import { getAllFacesFromMap } from 'utils/machineLearning';
+import {
+    getAllFacesFromMap,
+    getFaceImage,
+    getThumbnailTFImage,
+} from 'utils/machineLearning';
 
 class MachineLearningService {
     private faceDetectionService: FaceDetectionService;
@@ -156,19 +158,9 @@ class MachineLearningService {
             fileId: file.id,
             mlVersion: syncContext.config.mlVersion,
         };
-        const fileUrl = await DownloadManager.getPreview(
-            file,
-            syncContext.token
-        );
-        console.log('[MLService] Got thumbnail: ', file.id.toString(), fileUrl);
-
-        const thumbFile = await fetch(fileUrl);
-        const arrayBuffer = await thumbFile.arrayBuffer();
-        const decodedImg = await jpeg.decode(arrayBuffer);
-        console.log('[MLService] decodedImg: ', decodedImg);
 
         // console.log('1 TF Memory stats: ', tf.memory());
-        const tfImage = tf.browser.fromPixels(decodedImg);
+        const tfImage = await getThumbnailTFImage(file, syncContext.token);
         // console.log('2 TF Memory stats: ', tf.memory());
         const detectedFaces = await this.faceDetectionService.detectFaces(
             tfImage
@@ -322,9 +314,18 @@ class MachineLearningService {
             const faces = cluster.faces
                 .map((f) => allFaces[f])
                 .filter((f) => f);
+
+            const faceImageTensor = await getFaceImage(
+                faces[0],
+                syncContext.token
+            );
+            const faceImage = await faceImageTensor.array();
+            tf.dispose(faceImageTensor);
+
             const person: Person = {
                 id: index,
                 files: faces.map((f) => f.fileId),
+                faceImage: faceImage,
             };
 
             await mlPeopleStore.setItem(person.id.toString(), person);

src/utils/machineLearning/index.ts

@@ -1,7 +1,12 @@
 import * as tf from '@tensorflow/tfjs-core';
 import { DataType } from '@tensorflow/tfjs-core';
+import DownloadManager from 'services/downloadManager';
+import * as jpeg from 'jpeg-js';
+import { File, getLocalFiles } from 'services/fileService';
 import { Box, Point } from '../../../thirdparty/face-api/classes';
-import { Face, MLSyncConfig } from './types';
+import { Face, MlFileData, MLSyncConfig, Person } from './types';
+import { extractFaceImage } from './faceAlign';
+import { mlFilesStore, mlPeopleStore } from 'utils/storage/localForage';
 
 export function f32Average(descriptors: Float32Array[]) {
     if (descriptors.length < 1) {
@@ -136,6 +141,65 @@ export function getAllFacesFromMap(allFacesMap: Map<number, Array<Face>>) {
     return allFaces;
 }
 
+export async function getFaceImage(
+    face: Face,
+    token: string,
+    file?: File,
+    faceSize?: number
+): Promise<tf.Tensor3D> {
+    if (!file) {
+        const localFiles = await getLocalFiles();
+        file = localFiles.find((f) => f.id === face.fileId);
+    }
+
+    const tfImage = await getThumbnailTFImage(file, token);
+
+    return tf.tidy(() => {
+        const tf4DF32Image = toTensor4D(tfImage, 'float32');
+        const faceImage = extractFaceImage(tf4DF32Image, face, faceSize || 112);
+        const normalizedImage = tf.sub(tf.div(faceImage, 127.5), 1.0);
+        tf.dispose(tfImage);
+
+        return normalizedImage.squeeze([0]) as tf.Tensor3D;
+    });
+}
+
+export async function getThumbnailTFImage(file: File, token: string) {
+    const fileUrl = await DownloadManager.getPreview(file, token);
+    console.log('[MLService] Got thumbnail: ', file.id.toString(), fileUrl);
+
+    const thumbFile = await fetch(fileUrl);
+    const arrayBuffer = await thumbFile.arrayBuffer();
+    const decodedImg = await jpeg.decode(arrayBuffer);
+    // console.log('[MLService] decodedImg: ', decodedImg);
+
+    return tf.browser.fromPixels(decodedImg);
+}
+
+export async function getPeopleList(file: File): Promise<Array<Person>> {
+    const mlFileData: MlFileData = await mlFilesStore.getItem(
+        file.id.toString()
+    );
+    if (!mlFileData.faces || mlFileData.faces.length < 1) {
+        return [];
+    }
+
+    const peopleIds = mlFileData.faces
+        .map((f) => f.personId)
+        .filter((pid) => pid >= 0);
+    if (!peopleIds || peopleIds.length < 1) {
+        return [];
+    }
+    // console.log("peopleIds: ", peopleIds);
+    const peoplePromises = peopleIds.map(
+        (p) => mlPeopleStore.getItem(p.toString()) as Promise<Person>
+    );
+    const peopleList = await Promise.all(peoplePromises);
+    // console.log("peopleList: ", peopleList);
+
+    return peopleList;
+}
+
 export const DEFAULT_ML_SYNC_CONFIG: MLSyncConfig = {
     syncIntervalSec: 5, // 300
     batchSize: 5, // 200

src/utils/machineLearning/types.ts

@@ -132,6 +132,7 @@ export interface Person {
     id: number;
     name?: string;
     files: Array<number>;
+    faceImage: FaceImage;
 }
 
 export interface MlFileData {

src/utils/strings/englishConstants.tsx

@@ -427,6 +427,7 @@ const englishConstants = {
         'this video cannot be played on your browser',
     METADATA: 'metadata',
     INFO: 'information',
+    PEOPLE: 'people',
     FILE_ID: 'file id',
     FILE_NAME: 'file name',
     CREATION_TIME: 'creation time',