From f1185fcadef13892554bb958ff90060035d9dd8c Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 4 Sep 2024 10:33:16 +0530 Subject: [PATCH 1/9] Add low quality face condition --- web/packages/new/photos/services/ml/cluster.ts | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/web/packages/new/photos/services/ml/cluster.ts b/web/packages/new/photos/services/ml/cluster.ts index 1aa665666a..fe77f7d6e8 100644 --- a/web/packages/new/photos/services/ml/cluster.ts +++ b/web/packages/new/photos/services/ml/cluster.ts @@ -474,3 +474,12 @@ const clusterBatchLinear = ( return state; }; + +/** + * Return true if the given face is above the minimum inclusion thresholds, but + * is otherwise heuristically determined to be possibly spurious face detection. + * + * We apply a higher threshold when clustering such faces. + */ +const isBadFace = (face: Face) => + face.blur < 50 || (face.blur < 200 && face.blur < 0.85) || false; //face.isSideways, From 6a9fdd6c77b003ac86fad01d26c5b1a86bc6ddae Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 4 Sep 2024 10:36:27 +0530 Subject: [PATCH 2/9] sw --- web/packages/new/photos/services/ml/cluster.ts | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/web/packages/new/photos/services/ml/cluster.ts b/web/packages/new/photos/services/ml/cluster.ts index fe77f7d6e8..581859b73a 100644 --- a/web/packages/new/photos/services/ml/cluster.ts +++ b/web/packages/new/photos/services/ml/cluster.ts @@ -3,7 +3,7 @@ import { newNonSecureID } from "@/base/id-worker"; import log from "@/base/log"; import { ensure } from "@/utils/ensure"; import type { EnteFile } from "../../types/file"; -import type { Face, FaceIndex } from "./face"; +import { faceDirection, type Face, type FaceIndex } from "./face"; import { dotProduct } from "./math"; /** @@ -482,4 +482,9 @@ const clusterBatchLinear = ( * We apply a higher threshold when clustering such faces. */ const isBadFace = (face: Face) => - face.blur < 50 || (face.blur < 200 && face.blur < 0.85) || false; //face.isSideways, + face.blur < 50 || + (face.blur < 200 && face.blur < 0.85) || + isSidewaysFace(face); + +const isSidewaysFace = (face: Face) => + faceDirection(face.detection) != "straight"; From d91cd53375d308853733534a9c4bec8073f3c614 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 4 Sep 2024 10:37:37 +0530 Subject: [PATCH 3/9] Ren --- web/packages/new/photos/services/ml/cluster.ts | 7 ++++--- web/packages/new/photos/services/ml/index.ts | 4 ++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/web/packages/new/photos/services/ml/cluster.ts b/web/packages/new/photos/services/ml/cluster.ts index 581859b73a..99ab3d7ee4 100644 --- a/web/packages/new/photos/services/ml/cluster.ts +++ b/web/packages/new/photos/services/ml/cluster.ts @@ -130,7 +130,8 @@ export interface ClusteringProgress { export type OnClusteringProgress = (progress: ClusteringProgress) => void; -export type FaceF32 = Omit & { +/** A {@link Face} annotated with data needed during clustering. */ +export type ClusterFace = Omit & { embedding: Float32Array; }; @@ -140,7 +141,7 @@ export interface ClusterPreview { } export interface ClusterPreviewFace { - face: FaceF32; + face: ClusterFace; cosineSimilarity: number; wasMerged: boolean; } @@ -403,7 +404,7 @@ interface ClusteringState { } const clusterBatchLinear = ( - faces: FaceF32[], + faces: ClusterFace[], oldState: ClusteringState, joinThreshold: number, earlyExitThreshold: number, diff --git a/web/packages/new/photos/services/ml/index.ts b/web/packages/new/photos/services/ml/index.ts index 276db1083c..b3ce72ae0d 100644 --- a/web/packages/new/photos/services/ml/index.ts +++ b/web/packages/new/photos/services/ml/index.ts @@ -20,10 +20,10 @@ import { getRemoteFlag, updateRemoteFlag } from "../remote-store"; import type { SearchPerson } from "../search/types"; import type { UploadItem } from "../upload/types"; import { + type ClusterFace, type ClusteringOpts, type ClusterPreviewFace, type FaceCluster, - type FaceF32, type OnClusteringProgress, } from "./cluster"; import { regenerateFaceCrops } from "./crop"; @@ -366,7 +366,7 @@ export interface ClusterDebugPageContents { clusters: FaceCluster[]; clusterPreviewsWithFile: ClusterPreviewWithFile[]; unclusteredFacesWithFile: { - face: FaceF32; + face: ClusterFace; enteFile: EnteFile; }[]; } From f990863bb2f287fa3f11b8001b836684f015d900 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 4 Sep 2024 10:44:32 +0530 Subject: [PATCH 4/9] Retain that info --- .../new/photos/services/ml/cluster.ts | 60 +++++++++---------- 1 file changed, 29 insertions(+), 31 deletions(-) diff --git a/web/packages/new/photos/services/ml/cluster.ts b/web/packages/new/photos/services/ml/cluster.ts index 99ab3d7ee4..24de764f03 100644 --- a/web/packages/new/photos/services/ml/cluster.ts +++ b/web/packages/new/photos/services/ml/cluster.ts @@ -133,6 +133,7 @@ export type OnClusteringProgress = (progress: ClusteringProgress) => void; /** A {@link Face} annotated with data needed during clustering. */ export type ClusterFace = Omit & { embedding: Float32Array; + isBadFace: boolean; }; export interface ClusterPreview { @@ -149,25 +150,17 @@ export interface ClusterPreviewFace { /** * Cluster faces into groups. * - * [Note: Face clustering algorithm] - * * A cgroup (cluster group) consists of clusters, each of which itself is a set * of faces. * * cgroup << cluster << face * - * The clusters are generated locally by clients using the following algorithm: - * - * 1. clusters = [] initially, or fetched from remote. - * - * 2. For each face, find its nearest neighbour in the embedding space. - * - * 3. If no such neighbour is found within our threshold, create a new cluster. - * - * 4. Otherwise assign this face to the same cluster as its nearest neighbour. + * This function generates clusters locally using a batched form of linear + * clustering, with a bit of lookback (and a dollop of heuristics) to get the + * clusters to merge across batches. * - * This user can then tweak the output of the algorithm by performing the - * following actions to the list of clusters that they can see: + * This user can later tweak these clusters by performing the following actions + * to the list of clusters that they can see: * * - They can provide a name for a cluster ("name a person"). This upgrades a * cluster into a "cgroup", which is an entity that gets synced via remote @@ -365,20 +358,39 @@ export const clusterFaces = ( }; /** - * A generator function that returns a stream of {faceID, embedding} values, + * A generator function that returns a stream of {@link ClusterFace}s by * flattening all the the faces present in the given {@link faceIndices}. * - * It also converts the embeddings to Float32Arrays to speed up the dot product - * calculations that will happen during clustering. + * During this, it also converts the embeddings to Float32Arrays to speed up the + * dot product calculations that will happen during clustering and attaches + * other information that the clustering algorithm needs. */ function* enumerateFaces(faceIndices: FaceIndex[]) { for (const fi of faceIndices) { for (const f of fi.faces) { - yield { ...f, embedding: new Float32Array(f.embedding) }; + yield { + ...f, + embedding: new Float32Array(f.embedding), + isBadFace: isBadFace(f), + }; } } } +/** + * Return true if the given face is above the minimum inclusion thresholds, but + * is otherwise heuristically determined to be possibly spurious face detection. + * + * We apply a higher threshold when clustering such faces. + */ +const isBadFace = (face: Face) => + face.blur < 50 || + (face.blur < 200 && face.blur < 0.85) || + isSidewaysFace(face); + +const isSidewaysFace = (face: Face) => + faceDirection(face.detection) != "straight"; + /** Generate a new cluster ID. */ const newClusterID = () => newNonSecureID("cluster_"); @@ -475,17 +487,3 @@ const clusterBatchLinear = ( return state; }; - -/** - * Return true if the given face is above the minimum inclusion thresholds, but - * is otherwise heuristically determined to be possibly spurious face detection. - * - * We apply a higher threshold when clustering such faces. - */ -const isBadFace = (face: Face) => - face.blur < 50 || - (face.blur < 200 && face.blur < 0.85) || - isSidewaysFace(face); - -const isSidewaysFace = (face: Face) => - faceDirection(face.detection) != "straight"; From 263f94418d7c02c5749af496cbdd1f13ab6a5e3c Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 4 Sep 2024 10:55:23 +0530 Subject: [PATCH 5/9] Filter during enumeration --- .../new/photos/services/ml/cluster.ts | 39 ++++++++++++------- 1 file changed, 25 insertions(+), 14 deletions(-) diff --git a/web/packages/new/photos/services/ml/cluster.ts b/web/packages/new/photos/services/ml/cluster.ts index 24de764f03..b67fe6a678 100644 --- a/web/packages/new/photos/services/ml/cluster.ts +++ b/web/packages/new/photos/services/ml/cluster.ts @@ -199,11 +199,8 @@ export const clusterFaces = ( const localFileByID = new Map(localFiles.map((f) => [f.id, f])); - // A flattened array of faces. - const allFaces = [...enumerateFaces(faceIndexes)]; - const filteredFaces = allFaces - .filter((f) => f.blur > minBlur) - .filter((f) => f.score > minScore); + // A flattened array of filtered and annotated faces. + const filteredFaces = [...enumerateFaces(faceIndexes, minBlur, minScore)]; const fileForFaceID = new Map( filteredFaces.map(({ faceID }) => [ @@ -329,7 +326,9 @@ export const clusterFaces = ( }); } - const totalFaceCount = allFaces.length; + // TODO-Cluster the total face count is only needed during debugging + let totalFaceCount = 0; + for (const fi of faceIndexes) totalFaceCount += fi.faces.length; const filteredFaceCount = faces.length; const clusteredFaceCount = clusterIDForFaceID.size; const unclusteredFaceCount = filteredFaceCount - clusteredFaceCount; @@ -358,25 +357,37 @@ export const clusterFaces = ( }; /** - * A generator function that returns a stream of {@link ClusterFace}s by - * flattening all the the faces present in the given {@link faceIndices}. + * A generator function that returns a stream of eligible {@link ClusterFace}s + * by flattening all the the faces present in the given {@link faceIndices}. * * During this, it also converts the embeddings to Float32Arrays to speed up the * dot product calculations that will happen during clustering and attaches * other information that the clustering algorithm needs. */ -function* enumerateFaces(faceIndices: FaceIndex[]) { +function* enumerateFaces( + faceIndices: FaceIndex[], + minBlur: number, + minScore: number, +) { for (const fi of faceIndices) { for (const f of fi.faces) { - yield { - ...f, - embedding: new Float32Array(f.embedding), - isBadFace: isBadFace(f), - }; + if (shouldIncludeFace(f, minBlur, minScore)) { + yield { + ...f, + embedding: new Float32Array(f.embedding), + isBadFace: isBadFace(f), + }; + } } } } +/** + * Return true if the given face is above the minimum inclusion thresholds. + */ +const shouldIncludeFace = (face: Face, minBlur: number, minScore: number) => + face.blur > minBlur && face.score > minScore; + /** * Return true if the given face is above the minimum inclusion thresholds, but * is otherwise heuristically determined to be possibly spurious face detection. From 46d7d4e587ccbea36bdfe253aa310b72c89f2fe6 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 4 Sep 2024 11:00:35 +0530 Subject: [PATCH 6/9] UI 1 --- web/apps/photos/src/pages/cluster-debug.tsx | 15 ++++++++------- web/packages/new/photos/services/ml/cluster.ts | 1 + 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/web/apps/photos/src/pages/cluster-debug.tsx b/web/apps/photos/src/pages/cluster-debug.tsx index 3b9a7ad230..c9dcbe38f4 100644 --- a/web/apps/photos/src/pages/cluster-debug.tsx +++ b/web/apps/photos/src/pages/cluster-debug.tsx @@ -6,9 +6,9 @@ import { type ClusterDebugPageContents, } from "@/new/photos/services/ml"; import { + type ClusterFace, type ClusteringOpts, type ClusteringProgress, - type FaceF32, type OnClusteringProgress, } from "@/new/photos/services/ml/cluster"; import { faceDirection } from "@/new/photos/services/ml/face"; @@ -72,6 +72,7 @@ export default function ClusterDebug() { earlyExitThreshold: 0.9, batchSize: 10000, offsetIncrement: 7500, + filterBadFaces: true, }, onSubmit: (values) => cluster( @@ -83,6 +84,7 @@ export default function ClusterDebug() { earlyExitThreshold: toFloat(values.earlyExitThreshold), batchSize: toFloat(values.batchSize), offsetIncrement: toFloat(values.offsetIncrement), + filterBadFaces: values.filterBadFaces, }, (progress: ClusteringProgress) => onProgressRef.current?.(progress), @@ -454,8 +456,7 @@ const ClusterResHeader: React.FC = ({ clusterRes }) => { blur - score - cosineSimilarity - direction. - Faces added to the cluster as a result of next batch merging are - outlined. + Bad faces are outlined. ); @@ -494,15 +495,15 @@ interface FaceItemProps { } interface FaceWithFile { - face: FaceF32; + face: ClusterFace; enteFile: EnteFile; cosineSimilarity?: number; wasMerged?: boolean; } const FaceItem: React.FC = ({ faceWithFile }) => { - const { face, enteFile, cosineSimilarity, wasMerged } = faceWithFile; - const { faceID } = face; + const { face, enteFile, cosineSimilarity } = faceWithFile; + const { faceID, isBadFace } = face; const [objectURL, setObjectURL] = useState(); @@ -526,7 +527,7 @@ const FaceItem: React.FC = ({ faceWithFile }) => { return ( diff --git a/web/packages/new/photos/services/ml/cluster.ts b/web/packages/new/photos/services/ml/cluster.ts index b67fe6a678..cf635c75b5 100644 --- a/web/packages/new/photos/services/ml/cluster.ts +++ b/web/packages/new/photos/services/ml/cluster.ts @@ -121,6 +121,7 @@ export interface ClusteringOpts { earlyExitThreshold: number; batchSize: number; offsetIncrement: number; + filterBadFaces: boolean; } export interface ClusteringProgress { From 816b26475a33d53412de7b848b91b91b5e8382ae Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 4 Sep 2024 11:06:20 +0530 Subject: [PATCH 7/9] UI 2 --- web/apps/photos/src/pages/cluster-debug.tsx | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/web/apps/photos/src/pages/cluster-debug.tsx b/web/apps/photos/src/pages/cluster-debug.tsx index c9dcbe38f4..75722d4a87 100644 --- a/web/apps/photos/src/pages/cluster-debug.tsx +++ b/web/apps/photos/src/pages/cluster-debug.tsx @@ -22,6 +22,8 @@ import BackButton from "@mui/icons-material/ArrowBackOutlined"; import { Box, Button, + Checkbox, + FormControlLabel, IconButton, LinearProgress, Stack, @@ -229,7 +231,17 @@ const MemoizedForm = memo( onChange={handleChange} /> - + + + } + label="filterBadFaces" + /> - + ), From 5395ca5caf21215d82913bd8557ac885b2416374 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 4 Sep 2024 12:18:55 +0530 Subject: [PATCH 8/9] UI 3 --- web/apps/photos/src/pages/cluster-debug.tsx | 15 ++++++++++----- web/packages/new/photos/services/ml/cluster.ts | 2 +- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/web/apps/photos/src/pages/cluster-debug.tsx b/web/apps/photos/src/pages/cluster-debug.tsx index 75722d4a87..e5f8148fbc 100644 --- a/web/apps/photos/src/pages/cluster-debug.tsx +++ b/web/apps/photos/src/pages/cluster-debug.tsx @@ -74,7 +74,7 @@ export default function ClusterDebug() { earlyExitThreshold: 0.9, batchSize: 10000, offsetIncrement: 7500, - filterBadFaces: true, + badFaceHeuristics: true, }, onSubmit: (values) => cluster( @@ -86,7 +86,7 @@ export default function ClusterDebug() { earlyExitThreshold: toFloat(values.earlyExitThreshold), batchSize: toFloat(values.batchSize), offsetIncrement: toFloat(values.offsetIncrement), - filterBadFaces: values.filterBadFaces, + badFaceHeuristics: values.badFaceHeuristics, }, (progress: ClusteringProgress) => onProgressRef.current?.(progress), @@ -235,12 +235,17 @@ const MemoizedForm = memo( } - label="filterBadFaces" + label={ + + Bad face heuristics + + } />