Created the database and enhanced the camera functionality.

backend/app/config/settings.py

@@ -1,6 +1,7 @@
 DEFAULT_OBJ_DETECTION_MODEL = "app/models/yolov8n.onnx"
 DEFAULT_FACE_DETECTION_MODEL = "app/models/yolov8n-face.onnx"
 DEFAULT_FACENET_MODEL = "app/models/facenet.onnx"
+DEFAULT_NER_MODEL = "app/models/bert-base-NER.onnx"
 # MEDIUM_OBJ_DETECTION_MODEL = "app/models/yolov8m.onnx" # not supported
 
 TEST_INPUT_PATH = "tests/inputs"
@@ -13,3 +14,5 @@
 MAPPINGS_DATABASE_PATH = "app/database/yolo_mapping.db"
 FACES_DATABASE_PATH = "app/database/faces.db"
 CLUSTERS_DATABASE_PATH = "app/database/clusters.db"
+NER_DATABASE_PATH = "app/database/ner.db"
+DEFAULT_FRONTAL_MODEL = "app/ner/haarcascade_frontalface_default.xml"

backend/app/database/ner.py

@@ -0,0 +1,102 @@
+import sqlite3
+import json
+from app.ner.test import ner_marking
+from app.ner.camera import scanned_embeddings
+from app.config.settings import NER_DATABASE_PATH,FACES_DATABASE_PATH,IMAGES_DATABASE_PATH
+import numpy as np
+
+
+def ner_table():
+    conn = sqlite3.connect(NER_DATABASE_PATH)
+    cursor = conn.cursor()
+    cursor.execute(
+        """
+        CREATE TABLE IF NOT EXISTS ner_mapping (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            name STRING,
+            embeddings TEXT
+        )
+        """
+    )  
+    conn.commit()
+    conn.close()
+
+def add_nerdata(text):
+    conn = sqlite3.connect(NER_DATABASE_PATH)
+    cursor = conn.cursor()
+    text_to_display = ner_marking(text)
+    for name in text_to_display:
+        print(name)
+        cursor.execute("SELECT id FROM ner_mapping WHERE name = ?", (name,))
+        result = cursor.fetchone()
+        if result is None:
+            embedding_json = json.dumps(scanned_embeddings(name).tolist())
+            cursor.execute(
+                "INSERT INTO ner_mapping (name, embeddings) VALUES (?, ?)",
+                (name, embedding_json)
+            )
+    conn.commit()
+    conn.close()
+
+def extract_ner(text):
+    conn = sqlite3.connect(NER_DATABASE_PATH)
+    cursor = conn.cursor()
+
+    text_to_display = ner_marking(text)  
+    store = []
+
+    for name in text_to_display:
+        cursor.execute("SELECT embeddings FROM ner_mapping WHERE name = ?", (name,))
+        result = cursor.fetchone()  
+        if result:
+            embeddings = json.loads(result[0])  
+            store.append(embeddings)
+        else:
+            print(f"No embeddings found for name: {name}")
+
+    conn.close()
+    return store
+
+def compare_ner(store):
+    conn = sqlite3.connect(FACES_DATABASE_PATH)
+    cursor = conn.cursor()
+
+    conn2 = sqlite3.connect(IMAGES_DATABASE_PATH)
+    cursor2 = conn2.cursor()
+
+    cursor.execute("SELECT id, image_id, embeddings FROM faces")
+    rows = cursor.fetchall()
+
+    pathways = []
+    for row in rows:
+
+        python_list = json.loads(row[2])
+
+        if len(store) == len(python_list):
+            python_list = np.array(python_list)
+            store = np.array(store)
+            concatenated_embedding1 = np.concatenate(python_list)
+            concatenated_embedding2 = np.concatenate(store)
+
+            cosine_similarity = np.dot(concatenated_embedding1, concatenated_embedding2) / (np.linalg.norm(concatenated_embedding1) * np.linalg.norm(concatenated_embedding2))
+            print(cosine_similarity)
+
+            if 0.5 <= cosine_similarity <= 1:
+
+                cursor2.execute('''
+                SELECT path
+                FROM image_id_mapping 
+                WHERE id = ?
+            ''', (row[1],))  
+
+                related_data = cursor2.fetchone() 
+                pathways.append(related_data[0])
+
+    conn.close()
+    conn2.close()            
+
+    return pathways
+
+
+
+

backend/app/ner/camera.py

@@ -0,0 +1,128 @@
+import numpy as np
+import cv2
+import numpy as np
+import onnxruntime
+from app.config.settings import DEFAULT_FRONTAL_MODEL,DEFAULT_FACENET_MODEL
+import cv2
+import asyncio
+from numpy.linalg import norm
+import time
+
+def preprocess_face_for_onnx(face_image):
+    """
+    Preprocess the face image before passing it to the ONNX model.
+
+    Args:
+        face_image (numpy.ndarray): A cropped image containing a single face.
+
+    Returns:
+        numpy.ndarray: Preprocessed image ready for the ONNX model.
+    """
+
+    resized_face = cv2.resize(face_image, (160, 160))
+
+    rgb_face = resized_face[:, :, ::-1]  
+
+    normalized_face = rgb_face.astype(np.float32) / 255.0
+
+
+
+    preprocessed_face = np.expand_dims(normalized_face, axis=0)  
+
+    preprocessed_face = np.transpose(preprocessed_face, (0, 3, 1, 2)) 
+
+    return preprocessed_face   
+
+session = onnxruntime.InferenceSession(
+    DEFAULT_FACENET_MODEL, providers=["CPUExecutionProvider"]
+)
+
+input_tensor_name = session.get_inputs()[0].name
+output_tensor_name = session.get_outputs()[0].name   
+
+def normalize_embedding(embedding):
+    return embedding / np.linalg.norm(embedding)
+
+def brightness(img):
+    if len(img.shape) == 3:
+        # Colored RGB or BGR (*Do Not* use HSV images with this function)
+        # create brightness with euclidean norm
+        return np.average(norm(img, axis=2)) / np.sqrt(3)
+    else:
+        # Grayscale
+        return np.average(img)
+
+def get_face_embeddings(image):
+    result = session.run([output_tensor_name], {input_tensor_name: image})[0]
+    embedding = result[0]
+    return normalize_embedding(embedding) 
+
+
+def scanned_embeddings(name): 
+
+    face_cascade = cv2.CascadeClassifier(DEFAULT_FRONTAL_MODEL)
+    if face_cascade.empty():
+        raise FileNotFoundError("Failed to load Haar cascade file. Check the file path.")
+
+    cap = cv2.VideoCapture(0)
+
+    def scanning(names):
+        while True:
+            ret, frame = cap.read()
+            if not ret:
+                break
+
+            gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
+            faces = face_cascade.detectMultiScale(gray, scaleFactor=1.1, minNeighbors=5, minSize=(30, 30))
+
+            for (x, y, w, h) in faces:
+                face = frame[y:y + h, x:x + w]
+                face_gray = cv2.cvtColor(face, cv2.COLOR_BGR2GRAY)
+                mean_brightness = brightness(face_gray)
+
+                # Add messages for brightness issues
+                if mean_brightness < 60:
+                    cv2.putText(frame, f"Brightness too low! ({int(mean_brightness)})", (10, 30), 
+                                cv2.FONT_HERSHEY_SIMPLEX, 0.9, (255, 255, 0), 2, cv2.LINE_AA)
+                if mean_brightness > 150:
+                    cv2.putText(frame, f"Brightness too high! ({int(mean_brightness)})", (10, 30), 
+                                cv2.FONT_HERSHEY_SIMPLEX, 0.9, (255, 255, 0), 2, cv2.LINE_AA)
+
+                cv2.rectangle(frame, (x, y), (x + w, y + h), (0, 0, 255), 2)
+                cv2.putText(frame, f"Enter to Capture", (50, 50), cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 0, 0), 2, cv2.LINE_AA)
+                cv2.putText(frame, names, (x, y - 10), cv2.FONT_HERSHEY_SIMPLEX, 0.9, (255, 255, 255), 2, cv2.LINE_AA)
+
+            cv2.imshow("Face Embedding Scanner", frame)
+
+            if cv2.waitKey(1) & 0xFF == 13:  
+                try:
+                    time.sleep(2)
+                    if len(faces) == 0:
+                        print("No face detected. Please try again.")
+                        return
+
+                    face = frame[y:y + h, x:x + w]
+                    face_gray = cv2.cvtColor(face, cv2.COLOR_BGR2GRAY)
+                    mean_brightness = np.mean(face_gray)
+
+                    if mean_brightness < 50 or mean_brightness > 200:
+                        print("Brightness is not optimal. Please adjust the lighting.")
+                        return
+
+                    face = face.astype(np.float32)
+                    preprocessed_face = preprocess_face_for_onnx(face)
+                    embedding_vector = get_face_embeddings(preprocessed_face) 
+                    print(f"Captured embedding for {names}: {embedding_vector}")
+                    return embedding_vector
+
+                except Exception as e:
+                    print(f"Error capturing embedding: {e}")
+                    return
+
+    embedding_vector = scanning(name)
+    time.sleep(2)  
+
+    cap.release()
+    cv2.destroyAllWindows()
+
+    return embedding_vector

backend/app/ner/ner_onnx.py

@@ -3,7 +3,8 @@
 
 # Pre-trained model name
 model_name = "dslim/bert-base-NER"
-model_path = r'C:\Users\sanid\Downloads\gsoc_@pictopy\PictoPy\backend\app\models\bert-base-NER.onnx'
+# should always be in models directory
+model_path = r'C:\Users\sanid\Downloads\gsoc\PictoPy\backend\app\models\bert-base-NER.onnx'
 
 # Load the pre-trained model for token classification
 model = AutoModelForTokenClassification.from_pretrained(model_name)

backend/app/ner/test.py

@@ -3,12 +3,12 @@
 from transformers import AutoTokenizer, AutoConfig
 import cv2
 import asyncio
+from app.config.settings import DEFAULT_NER_MODEL
 import time
 
 # Run the ner_onnx.py to create the onnx model in the models folder
 def ner_marking(text1):
-    # change the path is required
-    model_path = r'C:\Users\sanid\Downloads\gsoc_@pictopy\PictoPy\backend\app\models\bert-base-NER.onnx'
+    model_path = DEFAULT_NER_MODEL
     session = onnxruntime.InferenceSession(model_path)
 
     tokenizer = AutoTokenizer.from_pretrained("dslim/bert-base-NER")
@@ -131,7 +131,7 @@ def preprocess_face_for_onnx(face_image):
 
 #change the path if required
 session = onnxruntime.InferenceSession(
-    r'C:\Users\sanid\Downloads\gsoc_@pictopy\PictoPy\backend\app\models\facenet.onnx', providers=["CPUExecutionProvider"]
+    r'C:\Users\sanid\Downloads\gsoc\PictoPy\backend\app\models\facenet.onnx', providers=["CPUExecutionProvider"]
 )
 
 input_tensor_name = session.get_inputs()[0].name
@@ -150,7 +150,7 @@ def scanned_embeddings(name):
     print(f"Text to Display: {text_to_display}")
 
     #change the path if required
-    face_cascade = cv2.CascadeClassifier(r'C:\Users\sanid\Downloads\gsoc_@pictopy\PictoPy\backend\app\ner\haarcascade_frontalface_default.xml')
+    face_cascade = cv2.CascadeClassifier(r'C:\Users\sanid\Downloads\gsoc\PictoPy\backend\app\ner\haarcascade_frontalface_default.xml')
     if face_cascade.empty():
         raise FileNotFoundError("Failed to load Haar cascade file. Check the file path.")
 

backend/app/routes/ner.py

@@ -0,0 +1,30 @@
+from app.database.ner import add_nerdata, extract_ner, compare_ner
+from fastapi import APIRouter
+from fastapi.responses import JSONResponse
+from pydantic import BaseModel
+
+router = APIRouter()
+
+class QueryPayload(BaseModel):
+    query: str
+
+@router.post("/api", tags=["ner"])
+async def get_images(payload: QueryPayload):
+    print("hello")
+    try:
+        text = payload.query
+
+        add_nerdata(text)
+        it = extract_ner(text)
+        pythonic_comeback = compare_ner(it)
+
+
+        return pythonic_comeback
+
+    except Exception as e:
+        return JSONResponse(status_code=500, content={"error": str(e)})
+
+
+
+
+

backend/main.py

@@ -17,6 +17,8 @@
 from app.routes.facetagging import router as tagging_router
 import multiprocessing
 from app.custom_logging import CustomizeLogger
+from app.database.ner import (ner_table)
+from app.routes.ner import router as ner_router
 
 
 @asynccontextmanager
@@ -28,6 +30,7 @@ async def lifespan(app: FastAPI):
     create_albums_table()
     cleanup_face_embeddings()
     init_face_cluster()
+    ner_table()
     yield
     face_cluster = get_face_cluster()
     if face_cluster:
@@ -56,6 +59,7 @@ async def root():
 app.include_router(images_router, prefix="/images", tags=["Images"])
 app.include_router(albums_router, prefix="/albums", tags=["Albums"])
 app.include_router(tagging_router, prefix="/tag", tags=["Tagging"])
+app.include_router(ner_router, prefix="/ner", tags=["ner"])
 
 
 # Runs when we use this command: python3 main.py (As in production)