Merge pull request #575 from LibrePhotos/feat/improved-faces

Frontend: Reintroduce Face Classification

package.json

@@ -52,6 +52,7 @@
     "@tiptap/pm": "^2.3.2",
     "@tiptap/react": "^2.3.2",
     "@tiptap/suggestion": "^2.3.2",
+    "@use-gesture/react": "^10.3.1",
     "@visx/gradient": "3.3.0",
     "@visx/group": "3.3.0",
     "@visx/hierarchy": "3.3.0",
@@ -84,7 +85,6 @@
     "react-dom": "18.3.1",
     "react-dropzone": "14.2.3",
     "react-i18next": "13.5.0",
-    "react-image-lightbox": "npm:librephotos-react-image-lightbox@^1.0.0",
     "react-leaflet": "^1.9.1",
     "react-leaflet-markercluster": "^1.1.8",
     "react-player": "2.16.0",

src/actions/photosActions.types.ts

@@ -42,7 +42,14 @@ export const PhotoHashSchema = z.object({
   video: z.boolean(),
 });
 
-export const PeopleSchema = z.object({ name: z.string(), face_url: z.string(), face_id: z.number() });
+export const PeopleSchema = z.object({
+  name: z.string(),
+  type: z.string(),
+  probability: z.number(),
+  location: z.object({ top: z.number(), bottom: z.number(), left: z.number(), right: z.number() }),
+  face_url: z.string(),
+  face_id: z.number(),
+});
 
 export const PhotoSchema = z.object({
   camera: z.string().nullable(),

src/api_client/albums/date.ts

@@ -43,7 +43,6 @@ export const dateAlbumsApi = api
         transformResponse: response => {
           const { results } = FetchDateAlbumsListResponseSchema.parse(response);
           addTempElementsToGroups(results);
-
           return results;
         },
       }),

src/api_client/albums/people.ts

@@ -15,19 +15,20 @@ export const PersonResponseSchema = z.object({
   cover_photo: z.string().optional(),
 });
 
-export const PeopleSchema = z
-  .object({
-    key: z.string(),
-    value: z.string(),
-    text: z.string(),
-    video: z.boolean(),
-    face_count: z.number(),
-    face_photo_url: z.string(),
-    face_url: z.string(),
-  })
-  .array();
+export const PersonSchema = z.object({
+  id: z.string(),
+  name: z.string(),
+  video: z.boolean(),
+  face_count: z.number(),
+  face_photo_url: z.string(),
+  face_url: z.string(),
+});
+
+export const PeopleSchema = PersonSchema.array();
+
+export type Person = z.infer<typeof PersonSchema>;
 
-type People = z.infer<typeof PeopleSchema>;
+export type People = z.infer<typeof PeopleSchema>;
 
 const PeopleResponseSchema = z.object({
   count: z.number(),
@@ -50,15 +51,14 @@ export const peopleAlbumsApi = api
         query: () => "persons/?page_size=1000",
         transformResponse: response => {
           const people = PeopleResponseSchema.parse(response).results.map(item => ({
-            key: item.id.toString(),
-            value: item.name,
-            text: item.name,
+            id: item.id.toString(),
+            name: item.name ?? "",
             video: !!item.video,
             face_count: item.face_count,
             face_photo_url: item.face_photo_url ?? "",
             face_url: item.face_url ?? "",
           }));
-          return _.orderBy(people, ["text", "face_count"], ["asc", "desc"]);
+          return _.orderBy(people, ["name", "face_count"], ["asc", "desc"]);
         },
       }),
       [Endpoints.renamePersonAlbum]: builder.mutation<void, { id: string; personName: string; newPersonName: string }>({

src/api_client/api.ts

@@ -4,6 +4,7 @@ import { createApi, fetchBaseQuery } from "@reduxjs/toolkit/query/react";
 import { Cookies } from "react-cookie";
 
 import type { IGenerateEventAlbumsTitlesResponse } from "../actions/utilActions.types";
+import { notification } from "../service/notifications";
 import type {
   IApiDeleteUserPost,
   IApiLoginPost,
@@ -12,18 +13,21 @@ import type {
   UserSignupResponse,
 } from "../store/auth/auth.zod";
 import { ApiLoginResponseSchema, UserSignupResponseSchema } from "../store/auth/auth.zod";
-import type {
-  IClusterFacesResponse,
-  IDeleteFacesRequest,
-  IDeleteFacesResponse,
-  IIncompletePersonFaceListRequest,
-  IIncompletePersonFaceListResponse,
-  IPersonFaceListRequest,
-  IPersonFaceListResponse,
-  IScanFacesResponse,
-  ISetFacesLabelRequest,
-  ISetFacesLabelResponse,
-  ITrainFacesResponse,
+import {
+  ClusterFacesResponse,
+  CompletePersonFace,
+  CompletePersonFaceList,
+  DeleteFacesRequest,
+  DeleteFacesResponse,
+  IncompletePersonFaceListRequest,
+  IncompletePersonFaceListResponse,
+  PersonFaceList,
+  PersonFaceListRequest,
+  PersonFaceListResponse,
+  ScanFacesResponse,
+  SetFacesLabelRequest,
+  SetFacesLabelResponse,
+  TrainFacesResponse,
 } from "../store/faces/facesActions.types";
 import type { IUploadOptions, IUploadResponse } from "../store/upload/upload.zod";
 import { UploadExistResponse, UploadResponse } from "../store/upload/upload.zod";
@@ -205,24 +209,128 @@ export const api = createApi({
         method: "GET",
       }),
     }),
-    [Endpoints.incompleteFaces]: builder.query<IIncompletePersonFaceListResponse, IIncompletePersonFaceListRequest>({
-      query: ({ inferred = false }) => ({
-        url: `faces/incomplete/?inferred=${inferred}`,
+    [Endpoints.incompleteFaces]: builder.query<CompletePersonFaceList, IncompletePersonFaceListRequest>({
+      query: ({ inferred = false, method = "clustering", orderBy = "confidence", minConfidence }) => ({
+        url: `faces/incomplete/?inferred=${inferred}${inferred ? `&analysis_method=${method}&order_by=${orderBy}` : ""}${minConfidence ? `&min_confidence=${minConfidence}` : ""}`,
       }),
-      providesTags: ["Faces"],
+      transformResponse: response => {
+        const payload = IncompletePersonFaceListResponse.parse(response);
+        const newFacesList: CompletePersonFaceList = payload.map(person => {
+          const completePersonFace: CompletePersonFace = { ...person, faces: [] };
+          for (let i = 0; i < person.face_count; i += 1) {
+            completePersonFace.faces.push({
+              id: i,
+              image: null,
+              face_url: null,
+              photo: "",
+              person_label_probability: 1,
+              person: person.id,
+              isTemp: true,
+            });
+          }
+          return completePersonFace;
+        });
+        return newFacesList;
+      },
+      providesTags: (result, error, { inferred, method, orderBy }) =>
+        result ? result.map(({ id }) => ({ type: "Faces", id })) : ["Faces"],
     }),
-    [Endpoints.fetchFaces]: builder.query<IPersonFaceListResponse, IPersonFaceListRequest>({
-      query: ({ person, page = 0, inferred = false, orderBy = "confidence" }) => ({
-        url: `faces/?person=${person}&page=${page}&inferred=${inferred}&order_by=${orderBy}`,
+    [Endpoints.fetchFaces]: builder.query<PersonFaceList, PersonFaceListRequest>({
+      query: ({ person, page = 0, inferred = false, orderBy = "confidence", method, minConfidence }) => ({
+        url: `faces/?person=${person}&page=${page}&inferred=${inferred}&order_by=${orderBy}${method ? `&analysis_method=${method}` : ""}${minConfidence ? `&min_confidence=${minConfidence}` : ""}`,
       }),
-      providesTags: ["Faces"],
+      transformResponse: (response: any) => {
+        const parsedResponse = PersonFaceListResponse.parse(response);
+        return parsedResponse.results;
+      },
+      async onQueryStarted(options, { dispatch, queryFulfilled }) {
+        const { data } = await queryFulfilled;
+        dispatch(
+          api.util.updateQueryData(
+            Endpoints.incompleteFaces,
+            {
+              method: options.method,
+              orderBy: options.orderBy,
+              inferred: options.inferred,
+              minConfidence: options.minConfidence,
+            },
+            draft => {
+              const indexToReplace = draft.findIndex(group => group.id === options.person);
+              const groupToChange = draft[indexToReplace];
+              if (!groupToChange) return;
+
+              const { faces } = groupToChange;
+              groupToChange.faces = faces
+                .slice(0, (options.page - 1) * 100)
+                .concat(data)
+                .concat(faces.slice(options.page * 100));
+
+              // eslint-disable-next-line no-param-reassign
+              draft[indexToReplace] = groupToChange;
+            }
+          )
+        );
+      },
+      providesTags: (result, error, { person }) => [{ type: "Faces", id: person }],
     }),
-    [Endpoints.clusterFaces]: builder.query<IClusterFacesResponse, void>({
+    [Endpoints.deleteFaces]: builder.mutation<DeleteFacesResponse, DeleteFacesRequest>({
+      query: ({ faceIds }) => ({
+        url: "/deletefaces",
+        method: "POST",
+        body: { face_ids: faceIds },
+      }),
+      transformResponse: response => {
+        const payload = DeleteFacesResponse.parse(response);
+        return payload;
+      },
+      async onQueryStarted({ faceIds }, { dispatch, queryFulfilled, getState }) {
+        const { activeTab, analysisMethod, orderBy } = getState().face;
+        const incompleteFacesArgs = { inferred: activeTab !== "labeled", method: analysisMethod, orderBy: orderBy };
+
+        const patchIncompleteFaces = dispatch(
+          api.util.updateQueryData(Endpoints.incompleteFaces, incompleteFacesArgs, draft => {
+            draft.forEach(personGroup => {
+              personGroup.faces = personGroup.faces.filter(face => !faceIds.includes(face.id));
+            });
+            draft.forEach(personGroup => {
+              personGroup.face_count = personGroup.faces.length;
+            });
+
+            draft = draft.filter(personGroup => personGroup.faces.length > 0);
+          })
+        );
+
+        try {
+          await queryFulfilled;
+        } catch {
+          patchIncompleteFaces.undo();
+        }
+      },
+    }),
+    [Endpoints.setFacesPersonLabel]: builder.mutation<SetFacesLabelResponse, SetFacesLabelRequest>({
+      query: ({ faceIds, personName }) => ({
+        url: "/labelfaces",
+        method: "POST",
+        body: { person_name: personName, face_ids: faceIds },
+      }),
+      transformResponse: response => {
+        const payload = SetFacesLabelResponse.parse(response);
+        notification.addFacesToPerson(payload.results[0].person_name, payload.results.length);
+        return payload;
+      },
+      // To-Do: Handle optimistic updates by updating the cache. The issue is that there are multiple caches that need to be updated, where we need to remove the faces from the incomplete faces cache and add them to the labeled faces cache.
+      // This is surprisingly complex to do with the current API, so we will just invalidate the cache for now.
+      // To-Do: Invalidating faces is also broken, because we do not know, which faces have which person ids, we need to invalidate.
+      // Need to restructure, by providing the full face object when queried, so we can invalidate the cache properly.
+      invalidatesTags: ["Faces", "PeopleAlbums"],
+    }),
+
+    [Endpoints.clusterFaces]: builder.query<ClusterFacesResponse, void>({
       query: () => ({
         url: "/clusterfaces",
       }),
     }),
-    [Endpoints.rescanFaces]: builder.query<IScanFacesResponse, void>({
+    [Endpoints.rescanFaces]: builder.query<ScanFacesResponse, void>({
       query: () => ({
         url: "/scanfaces",
       }),
@@ -232,26 +340,12 @@ export const api = createApi({
         url: "/autoalbumtitlegen",
       }),
     }),
-    [Endpoints.trainFaces]: builder.mutation<ITrainFacesResponse, void>({
+    [Endpoints.trainFaces]: builder.mutation<TrainFacesResponse, void>({
       query: () => ({
         url: "/trainfaces",
         method: "POST",
       }),
     }),
-    [Endpoints.deleteFaces]: builder.mutation<IDeleteFacesResponse, IDeleteFacesRequest>({
-      query: ({ faceIds }) => ({
-        url: "/deletefaces",
-        method: "POST",
-        body: { face_ids: faceIds },
-      }),
-    }),
-    [Endpoints.setFacesPersonLabel]: builder.mutation<ISetFacesLabelResponse, ISetFacesLabelRequest>({
-      query: ({ faceIds, personName }) => ({
-        url: "/labelfaces",
-        method: "POST",
-        body: { person_name: personName, face_ids: faceIds },
-      }),
-    }),
     [Endpoints.fetchServerStats]: builder.query<ServerStatsResponseType, void>({
       query: () => ({
         url: `serverstats`,
@@ -274,6 +368,7 @@ export const {
   useFetchUserListQuery,
   useFetchPredefinedRulesQuery,
   useFetchIncompleteFacesQuery,
+  useFetchFacesQuery,
   useLoginMutation,
   useSignUpMutation,
   useLogoutMutation,

src/components/CustomSearch.tsx

@@ -14,7 +14,7 @@ import type { KeyboardEvent, ReactNode } from "react";
 import { useTranslation } from "react-i18next";
 import { push } from "redux-first-history";
 
-import { useFetchPeopleAlbumsQuery } from "../api_client/albums/people";
+import { Person, useFetchPeopleAlbumsQuery } from "../api_client/albums/people";
 import { useFetchPlacesAlbumsQuery } from "../api_client/albums/places";
 import { useFetchThingsAlbumsQuery } from "../api_client/albums/things";
 import { useFetchUserAlbumsQuery } from "../api_client/albums/user";
@@ -52,12 +52,12 @@ function toUserAlbumSuggestion(item: any) {
   return { value: item.title, icon: <Album />, type: SuggestionType.USER_ALBUM, id: item.id };
 }
 
-function toPeopleSuggestion(item: any) {
+function toPeopleSuggestion(item: Person) {
   return {
-    value: item.value,
-    icon: <Avatar src={item.face_url} alt={item.value} size="xl" />,
+    value: item.name,
+    icon: <Avatar src={item.face_url} alt={item.name} size="xl" />,
     type: SuggestionType.PEOPLE,
-    id: item.key,
+    id: item.id,
   };
 }
 
@@ -130,7 +130,7 @@ export function CustomSearch() {
           .slice(0, 2)
           .map(toUserAlbumSuggestion),
         ...people
-          .filter((item: any) => fuzzyMatch(query, item.value))
+          .filter((item: Person) => fuzzyMatch(query, item.name))
           .slice(0, 2)
           .map(toPeopleSuggestion),
       ]);