Skip to content
This repository has been archived by the owner on Jul 7, 2024. It is now read-only.

Commit

Permalink
Merge branch 'main' into rework-api-call
Browse files Browse the repository at this point in the history
  • Loading branch information
@PatNei
PatNei authored Feb 12, 2024
2 parents 6cc29a2 + a930aee commit d8bb5b5
Show file tree
Hide file tree
Showing 13 changed files with 5,803 additions and 3 deletions.
4 changes: 3 additions & 1 deletion .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -159,4 +159,6 @@ cython_debug/
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
#.idea/

*/node_modules
*/node_modules

.devbox
203 changes: 201 additions & 2 deletions backend/poetry.lock
View file

Large diffs are not rendered by default.

2 changes: 2 additions & 0 deletions backend/pyproject.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@ seaborn = "^0.13.2"
matplotlib = "^3.8.2"
fastapi = "^0.109.2"
uvicorn = "^0.27.0.post1"
opencv-python = "^4.9.0.80"
mediapipe = "^0.10.9"


[build-system]
Expand Down
96 changes: 96 additions & 0 deletions backend/sign/app.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,96 @@
import homemade.imageloader as imageloader
import homemade.mediapiper as mediapiper
import homemade.drawer as drawer
import homemade.model as model
import homemade.sussyproc as sussyproc
import copy
import os
from collections import deque
from joblib import dump, load
from sklearn.linear_model import SGDClassifier
import cv2 as cv

data_base_path = 'data/archive/asl_alphabet_train/'
train_path = 'sign/model/alphabet/train.csv'
model_path = 'sign/model/alphabet/model.joblib'
training_amount = 250

piper = mediapiper.MediaPiper()

if(not os.path.isfile(model_path)):
print("Creating a model")

l = imageloader.load_images_from_directory(data_base_path, amount = training_amount)
processed = piper.process_images_for_training_data(l)
processed.save_processed_image_to_csv(train_path)

data = model.load_training_data(train_path)
data.train_test_split()

#labels_train_A = (data.labels_train == 'A')
#a_or_not_a_classifier = model.SignClassifier(
# SGDClassifier,
# data.landmarks_train,
# labels_train_A)
#classifier = a_or_not_a_classifier
#expected = data.labels_test[0] == 'A'

mutli_classifier = model.SignClassifier(SGDClassifier,
data.landmarks_train,
data.labels_train,
)
classifier = mutli_classifier
expected = data.labels_test[0]

predicted = classifier.predict( [data.landmarks_test[0]] )
print("Trained a model: ", "predicted: " + str(predicted), "should return " + str( [expected] ))
dump(mutli_classifier, model_path)
print("Dumped model to: " + model_path)
else:
classifier = load(model_path)
print("loaded model from: " + model_path)

cap = cv.VideoCapture(0)
cap.set(cv.CAP_PROP_FRAME_WIDTH, 500)
cap.set(cv.CAP_PROP_FRAME_HEIGHT, 500)

while True:
# Process Key (ESC: end) #################################################
key = cv.waitKey(10)
if key == 27: # ESC
break

ret, image = cap.read()
if not ret:
break
image = cv.flip(image, 1) # Mirror display
debug_image = copy.deepcopy(image)

# Detection implementation #############################################################
image = cv.cvtColor(image, cv.COLOR_BGR2RGB)

image.flags.writeable = False
(landmarks, raw_landmarks) = piper.process_image_for_prediction(image)
image.flags.writeable = True

if landmarks is not None and len(landmarks) > 0:
res = classifier.predict( landmarks )

for raw_hand_landmark in raw_landmarks.multi_hand_landmarks:
landmark_list = sussyproc.calc_landmark_list(debug_image, raw_hand_landmark)

brect = drawer.calc_bounding_rect(debug_image, raw_hand_landmark)

debug_image = drawer.draw_bounding_rect(True, debug_image, brect)
debug_image = drawer.draw_landmarks(debug_image, landmark_list)
debug_image = drawer.draw_info_text(
debug_image,
brect,
str(res),
)

cv.imshow('Hand Gesture Recognition', debug_image)


cap.release()
cv.destroyAllWindows()
236 changes: 236 additions & 0 deletions backend/sign/homemade/drawer.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,236 @@
import cv2 as cv
import numpy as np

def draw_info_text(image, brect, hand_sign_text):
cv.rectangle(image, (brect[0], brect[1]), (brect[2], brect[1] - 22),
(0, 0, 0), -1)

info_text = "" #handedness.classification[0].label[0:]
if hand_sign_text != "":
info_text = info_text + ':' + hand_sign_text
cv.putText(image, info_text, (brect[0] + 5, brect[1] - 4),
cv.FONT_HERSHEY_SIMPLEX, 0.6, (255, 255, 255), 1, cv.LINE_AA)

# if finger_gesture_text != "":
# cv.putText(image, "Finger Gesture:" + finger_gesture_text, (10, 60),
# cv.FONT_HERSHEY_SIMPLEX, 1.0, (0, 0, 0), 4, cv.LINE_AA)
# cv.putText(image, "Finger Gesture:" + finger_gesture_text, (10, 60),
# cv.FONT_HERSHEY_SIMPLEX, 1.0, (255, 255, 255), 2,
# cv.LINE_AA)

return image


def calc_bounding_rect(image, landmarks):
image_width, image_height = image.shape[1], image.shape[0]

landmark_array = np.empty((0, 2), int)

for _, landmark in enumerate(landmarks.landmark):
landmark_x = min(int(landmark.x * image_width), image_width - 1)
landmark_y = min(int(landmark.y * image_height), image_height - 1)

landmark_point = [np.array((landmark_x, landmark_y))]

landmark_array = np.append(landmark_array, landmark_point, axis=0)

x, y, w, h = cv.boundingRect(landmark_array)

return [x, y, x + w, y + h]


def draw_landmarks(image, landmark_point):
if len(landmark_point) > 0:
# Thumb
cv.line(image, tuple(landmark_point[2]), tuple(landmark_point[3]),
(0, 0, 0), 6)
cv.line(image, tuple(landmark_point[2]), tuple(landmark_point[3]),
(255, 255, 255), 2)
cv.line(image, tuple(landmark_point[3]), tuple(landmark_point[4]),
(0, 0, 0), 6)
cv.line(image, tuple(landmark_point[3]), tuple(landmark_point[4]),
(255, 255, 255), 2)

# Index finger
cv.line(image, tuple(landmark_point[5]), tuple(landmark_point[6]),
(0, 0, 0), 6)
cv.line(image, tuple(landmark_point[5]), tuple(landmark_point[6]),
(255, 255, 255), 2)
cv.line(image, tuple(landmark_point[6]), tuple(landmark_point[7]),
(0, 0, 0), 6)
cv.line(image, tuple(landmark_point[6]), tuple(landmark_point[7]),
(255, 255, 255), 2)
cv.line(image, tuple(landmark_point[7]), tuple(landmark_point[8]),
(0, 0, 0), 6)
cv.line(image, tuple(landmark_point[7]), tuple(landmark_point[8]),
(255, 255, 255), 2)

# Middle finger
cv.line(image, tuple(landmark_point[9]), tuple(landmark_point[10]),
(0, 0, 0), 6)
cv.line(image, tuple(landmark_point[9]), tuple(landmark_point[10]),
(255, 255, 255), 2)
cv.line(image, tuple(landmark_point[10]), tuple(landmark_point[11]),
(0, 0, 0), 6)
cv.line(image, tuple(landmark_point[10]), tuple(landmark_point[11]),
(255, 255, 255), 2)
cv.line(image, tuple(landmark_point[11]), tuple(landmark_point[12]),
(0, 0, 0), 6)
cv.line(image, tuple(landmark_point[11]), tuple(landmark_point[12]),
(255, 255, 255), 2)

# Ring finger
cv.line(image, tuple(landmark_point[13]), tuple(landmark_point[14]),
(0, 0, 0), 6)
cv.line(image, tuple(landmark_point[13]), tuple(landmark_point[14]),
(255, 255, 255), 2)
cv.line(image, tuple(landmark_point[14]), tuple(landmark_point[15]),
(0, 0, 0), 6)
cv.line(image, tuple(landmark_point[14]), tuple(landmark_point[15]),
(255, 255, 255), 2)
cv.line(image, tuple(landmark_point[15]), tuple(landmark_point[16]),
(0, 0, 0), 6)
cv.line(image, tuple(landmark_point[15]), tuple(landmark_point[16]),
(255, 255, 255), 2)

# Little finger
cv.line(image, tuple(landmark_point[17]), tuple(landmark_point[18]),
(0, 0, 0), 6)
cv.line(image, tuple(landmark_point[17]), tuple(landmark_point[18]),
(255, 255, 255), 2)
cv.line(image, tuple(landmark_point[18]), tuple(landmark_point[19]),
(0, 0, 0), 6)
cv.line(image, tuple(landmark_point[18]), tuple(landmark_point[19]),
(255, 255, 255), 2)
cv.line(image, tuple(landmark_point[19]), tuple(landmark_point[20]),
(0, 0, 0), 6)
cv.line(image, tuple(landmark_point[19]), tuple(landmark_point[20]),
(255, 255, 255), 2)

# Palm
cv.line(image, tuple(landmark_point[0]), tuple(landmark_point[1]),
(0, 0, 0), 6)
cv.line(image, tuple(landmark_point[0]), tuple(landmark_point[1]),
(255, 255, 255), 2)
cv.line(image, tuple(landmark_point[1]), tuple(landmark_point[2]),
(0, 0, 0), 6)
cv.line(image, tuple(landmark_point[1]), tuple(landmark_point[2]),
(255, 255, 255), 2)
cv.line(image, tuple(landmark_point[2]), tuple(landmark_point[5]),
(0, 0, 0), 6)
cv.line(image, tuple(landmark_point[2]), tuple(landmark_point[5]),
(255, 255, 255), 2)
cv.line(image, tuple(landmark_point[5]), tuple(landmark_point[9]),
(0, 0, 0), 6)
cv.line(image, tuple(landmark_point[5]), tuple(landmark_point[9]),
(255, 255, 255), 2)
cv.line(image, tuple(landmark_point[9]), tuple(landmark_point[13]),
(0, 0, 0), 6)
cv.line(image, tuple(landmark_point[9]), tuple(landmark_point[13]),
(255, 255, 255), 2)
cv.line(image, tuple(landmark_point[13]), tuple(landmark_point[17]),
(0, 0, 0), 6)
cv.line(image, tuple(landmark_point[13]), tuple(landmark_point[17]),
(255, 255, 255), 2)
cv.line(image, tuple(landmark_point[17]), tuple(landmark_point[0]),
(0, 0, 0), 6)
cv.line(image, tuple(landmark_point[17]), tuple(landmark_point[0]),
(255, 255, 255), 2)

# Key Points
for index, landmark in enumerate(landmark_point):
if index == 0: # 手首1
cv.circle(image, (landmark[0], landmark[1]), 5, (255, 255, 255),
-1)
cv.circle(image, (landmark[0], landmark[1]), 5, (0, 0, 0), 1)
if index == 1: # 手首2
cv.circle(image, (landmark[0], landmark[1]), 5, (255, 255, 255),
-1)
cv.circle(image, (landmark[0], landmark[1]), 5, (0, 0, 0), 1)
if index == 2: # 親指：付け根
cv.circle(image, (landmark[0], landmark[1]), 5, (255, 255, 255),
-1)
cv.circle(image, (landmark[0], landmark[1]), 5, (0, 0, 0), 1)
if index == 3: # 親指：第1関節
cv.circle(image, (landmark[0], landmark[1]), 5, (255, 255, 255),
-1)
cv.circle(image, (landmark[0], landmark[1]), 5, (0, 0, 0), 1)
if index == 4: # 親指：指先
cv.circle(image, (landmark[0], landmark[1]), 8, (255, 255, 255),
-1)
cv.circle(image, (landmark[0], landmark[1]), 8, (0, 0, 0), 1)
if index == 5: # 人差指：付け根
cv.circle(image, (landmark[0], landmark[1]), 5, (255, 255, 255),
-1)
cv.circle(image, (landmark[0], landmark[1]), 5, (0, 0, 0), 1)
if index == 6: # 人差指：第2関節
cv.circle(image, (landmark[0], landmark[1]), 5, (255, 255, 255),
-1)
cv.circle(image, (landmark[0], landmark[1]), 5, (0, 0, 0), 1)
if index == 7: # 人差指：第1関節
cv.circle(image, (landmark[0], landmark[1]), 5, (255, 255, 255),
-1)
cv.circle(image, (landmark[0], landmark[1]), 5, (0, 0, 0), 1)
if index == 8: # 人差指：指先
cv.circle(image, (landmark[0], landmark[1]), 8, (255, 255, 255),
-1)
cv.circle(image, (landmark[0], landmark[1]), 8, (0, 0, 0), 1)
if index == 9: # 中指：付け根
cv.circle(image, (landmark[0], landmark[1]), 5, (255, 255, 255),
-1)
cv.circle(image, (landmark[0], landmark[1]), 5, (0, 0, 0), 1)
if index == 10: # 中指：第2関節
cv.circle(image, (landmark[0], landmark[1]), 5, (255, 255, 255),
-1)
cv.circle(image, (landmark[0], landmark[1]), 5, (0, 0, 0), 1)
if index == 11: # 中指：第1関節
cv.circle(image, (landmark[0], landmark[1]), 5, (255, 255, 255),
-1)
cv.circle(image, (landmark[0], landmark[1]), 5, (0, 0, 0), 1)
if index == 12: # 中指：指先
cv.circle(image, (landmark[0], landmark[1]), 8, (255, 255, 255),
-1)
cv.circle(image, (landmark[0], landmark[1]), 8, (0, 0, 0), 1)
if index == 13: # 薬指：付け根
cv.circle(image, (landmark[0], landmark[1]), 5, (255, 255, 255),
-1)
cv.circle(image, (landmark[0], landmark[1]), 5, (0, 0, 0), 1)
if index == 14: # 薬指：第2関節
cv.circle(image, (landmark[0], landmark[1]), 5, (255, 255, 255),
-1)
cv.circle(image, (landmark[0], landmark[1]), 5, (0, 0, 0), 1)
if index == 15: # 薬指：第1関節
cv.circle(image, (landmark[0], landmark[1]), 5, (255, 255, 255),
-1)
cv.circle(image, (landmark[0], landmark[1]), 5, (0, 0, 0), 1)
if index == 16: # 薬指：指先
cv.circle(image, (landmark[0], landmark[1]), 8, (255, 255, 255),
-1)
cv.circle(image, (landmark[0], landmark[1]), 8, (0, 0, 0), 1)
if index == 17: # 小指：付け根
cv.circle(image, (landmark[0], landmark[1]), 5, (255, 255, 255),
-1)
cv.circle(image, (landmark[0], landmark[1]), 5, (0, 0, 0), 1)
if index == 18: # 小指：第2関節
cv.circle(image, (landmark[0], landmark[1]), 5, (255, 255, 255),
-1)
cv.circle(image, (landmark[0], landmark[1]), 5, (0, 0, 0), 1)
if index == 19: # 小指：第1関節
cv.circle(image, (landmark[0], landmark[1]), 5, (255, 255, 255),
-1)
cv.circle(image, (landmark[0], landmark[1]), 5, (0, 0, 0), 1)
if index == 20: # 小指：指先
cv.circle(image, (landmark[0], landmark[1]), 8, (255, 255, 255),
-1)
cv.circle(image, (landmark[0], landmark[1]), 8, (0, 0, 0), 1)

return image


def draw_bounding_rect(use_brect, image, brect):
if use_brect:
# Outer rectangle
cv.rectangle(image, (brect[0], brect[1]), (brect[2], brect[3]),
(0, 0, 0), 1)

return image
34 changes: 34 additions & 0 deletions backend/sign/homemade/imageloader.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
from PIL import Image
import os

import numpy as np

class LabelledImage:
def __init__(self, label, data) -> None:
self.data = data
self.label = label

def __str__(self):
return "<< " + self.label + ": \n\t" + str(self.data)[:50] + " >>\n"

def load_images_from_directory(base_path, amount = 50) -> list[LabelledImage]:
lablledImages = []

for subdir,_,files in os.walk(base_path):
if subdir == base_path:
continue

label = subdir.replace(base_path, '')
data = []

for idx, file_name in enumerate(files):
if idx >= amount:
continue

imgPath = subdir + '/' + file_name
img = np.asarray(Image.open(imgPath))
data.append(img)

lablledImages.append( LabelledImage(label, data) )

return lablledImages
Loading

0 comments on commit d8bb5b5

Please sign in to comment.