Output Confusion Matrix

confusion_matrix.py

@@ -0,0 +1,6 @@
+import numpy as np
+from sklearn.metrics import confusion_matrix
+def compute_confusion_matrix(y_actual, y_predicted):
+    cm = confusion_matrix(y_actual, y_predicted, labels=[0, 1])
+    tn, fp, fn, tp = cm.ravel()
+    return {"TP": tp, "TN": tn, "FP": fp, "FN": fn}

fewshotmodel.py

@@ -7,6 +7,7 @@
 from torch.utils.data import DataLoader, Dataset
 import numpy as np
 import json
+from confusion_matrix import compute_confusion_matrix
 
 # Model and hyperparameters
 input_dim = 17  # number of features for the gesture data
@@ -131,8 +132,14 @@ def load_data(path):
         test_auc_roc = roc_auc_score(test_labels, test_scores)
         test_precision = precision_score(test_labels, test_preds)
         test_recall = recall_score(test_labels, test_preds)
-        print(f"Epoch [{epoch+1}/{num_epochs}], Test Accuracy: {test_accuracy:.4f}, AUC-ROC: {test_auc_roc:.4f}, Precision: {test_precision:.4f}, Recall: {test_recall:.4f}")
-
+        cm = compute_confusion_matrix(test_labels, test_preds)
+        print(
+            "Epoch [{} / {}], Test Accuracy: {:.4f}, AUC-ROC: {:.4f}, Precision: {:.4f}, "
+            "Recall: {:.4f}, Confusion_Matrix: {}".format(
+                epoch + 1, num_epochs, test_accuracy, test_auc_roc, test_precision, test_recall,
+                ", ".join(f"{key}: {value}" for key, value in cm.items())
+            )
+        )
 # Extract the model's state dictionary, convert to JSON serializable format
 state_dict = model.state_dict()
 serializable_state_dict = {key: value.tolist() for key, value in state_dict.items()}

fewshottripletloss.py

@@ -7,6 +7,7 @@
 from torch.utils.data import DataLoader, Dataset
 import numpy as np
 import json
+from confusion_matrix import compute_confusion_matrix
 
 # Model and hyperparameters
 input_dim = 17  # number of features for the gesture data
@@ -143,7 +144,13 @@ def load_data(path):
         test_auc_roc = roc_auc_score(test_labels, test_scores)
         test_precision = precision_score(test_labels, test_preds)
         test_recall = recall_score(test_labels, test_preds)
-        print(f"Epoch [{epoch+1}/{num_epochs}], Test Accuracy: {test_accuracy:.4f}, AUC-ROC: {test_auc_roc:.4f}, Precision: {test_precision:.4f}, Recall: {test_recall:.4f}")
+        cm = compute_confusion_matrix(test_labels, test_preds)
+        print(
+            "Epoch [{} / {}], Test Accuracy: {:.4f}, AUC-ROC: {:.4f}, Precision: {:.4f}, Recall: {:.4f}, Confusion_Matrix: {}".format(
+                epoch + 1, num_epochs, test_accuracy, test_auc_roc, test_precision, test_recall,
+                ", ".join(f"{key}: {value}" for key, value in cm.items())
+            )
+        )  
 
 # Extract the model's state dictionary, convert to JSON serializable format
 state_dict = model.state_dict()

model.py

@@ -13,6 +13,7 @@
 import json
 from pprint import pprint
 from sklearn.metrics import roc_auc_score, precision_recall_curve, auc
+from confusion_matrix import compute_confusion_matrix
 
 output_dim = 1  # binary classification for thumbs up or down
 input_dim = 17  # 17 features
@@ -121,16 +122,19 @@ def main():
                 auc_roc = roc_auc_score(all_labels, all_probs)
                 precision, recall, _ = precision_recall_curve(all_labels, all_probs)
                 auc_pr = auc(recall, precision)
+                binary_preds = [1 if prob > detect_threshold else 0 for prob in all_probs] 
+                cm = compute_confusion_matrix(all_labels, binary_preds)
 
                 # Example: Log metrics to the database
                 model_type = "binary classification"  # Adjust based on your specific model type
                 utils.log_training_metrics(auc_pr, auc_roc, loss.item(), model_type)
 
                 print(
-                    "Iteration: {}. Loss: {}. Accuracy: {}. AUC-ROC: {:.4f}. AUC-PR: {:.4f}".format(
-                        iter, loss.item(), accuracy, auc_roc, auc_pr
-                    )
+                    "Iteration: {}. Loss: {}. Accuracy: {}. AUC-ROC: {:.4f}. AUC-PR: {:.4f}. Confusion_Matrix: {}".format(
+                    iter, loss.item(), accuracy, auc_roc, auc_pr, 
+                    ", ".join(f"{key}: {value}" for key, value in cm.items())
                 )
+)
 
     # Extract the model's state dictionary, convert to JSON serializable format
     state_dict = model.state_dict()

requirements.txt

@@ -9,4 +9,5 @@ torchtext
 torchvision
 onnxscript
 onnxruntime==1.19.2
+scikit-learn==1.5.2
 flask