Changed testing

backend/app/use_cases/generate.py

@@ -6,7 +6,7 @@
 project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
 sys.path.append(project_root)
 
-from backend.ml_model.use_cases.model import model
+from backend.ml_model.use_cases.model import ModelTrainer
 
 from backend.app.use_cases.DatabaseRepositoryInterface import (
     DatabaseRepositoryInterface,
@@ -39,7 +39,8 @@ def execute(
         self.file_repo.update_comparison_csv(demographics, choices, time)
         self.db_repo.update_db_for_user(demographics, choices, time)
         print("GENERATE:", demographics, choices, time)
-        output = model()
+        model_executer = ModelTrainer()
+        output = model_executer.train_and_evaluate()
 
         if output is None:
             output = []

backend/ml_model/repository/model_saver.py

@@ -1,22 +1,72 @@
 import os
 import pickle
-
 import pandas as pd
 from sklearn.model_selection import GridSearchCV
 
 
-def save_model(best_clf: GridSearchCV, x_test: pd.DataFrame, y_test: pd.Series) -> None:
+class ModelSaver:
     """
-    Saves the model as a pkl file
+            Saves the trained model and its evaluation score as a pickle (.pkl) file.
+
+            Parameters:
+            -----------
+            best_clf : GridSearchCV
+                The trained model object, which is an instance of GridSearchCV containing the best estimator after hyperparameter tuning.
+
+            x_test : pd.DataFrame
+                The test dataset features used for evaluating the model.
+
+            y_test : pd.Series
+                The actual labels corresponding to the test dataset.
+
+            Returns:
+            --------
+            None
+                The function saves the model and its score to a file named `model_with_score.pkl` in the parent directory of the current script location.
     """
-    # Overall model score
-    score = best_clf.score(x_test, y_test)
+    def __init__(self, best_clf: GridSearchCV, x_test: pd.DataFrame, y_test: pd.Series):
+        """
+        Initializes the ModelSaver class with model, test features, and test labels.
+
+        Parameters:
+        -----------
+        best_clf : GridSearchCV
+            The trained model object, which is an instance of GridSearchCV containing the best estimator after hyperparameter tuning.
+
+        x_test : pd.DataFrame
+            The test dataset features used for evaluating the model.
+
+        y_test : pd.Series
+            The actual labels corresponding to the test dataset.
+        """
+        self.best_clf = best_clf
+        self.x_test = x_test
+        self.y_test = y_test
+
+    def save_model(self) -> None:
+        """
+        Saves the trained model and its evaluation score as a pickle (.pkl) file.
+
+        Returns:
+        --------
+        None
+            The function saves the model and its score to a file named `model_with_score.pkl` in the parent directory of the current script location.
+
+        Notes:
+        ------
+        - The `score` is calculated using the `score` method of the `best_clf` object, which typically represents accuracy for classification models.
+        - The resulting pickle file contains a dictionary with two keys:
+            - "model": the `best_clf` object.
+            - "score": the evaluation score of the model on the test data.
+        """
+        # Overall model score
+        score = self.best_clf.score(self.x_test, self.y_test)
 
-    curr_dir = os.path.dirname(__file__)
-    model_path = os.path.join(curr_dir, "../model_with_score.pkl")
+        curr_dir = os.path.dirname(__file__)
+        model_path = os.path.join(curr_dir, "../model_with_score.pkl")
 
-    # Save the model and its score
-    with open(model_path, "wb") as f:
-        pickle.dump({"model": best_clf, "score": score}, f)
+        # Save the model and its score
+        with open(model_path, "wb") as f:
+            pickle.dump({"model": self.best_clf, "score": score}, f)
 
-    return
+        return

backend/ml_model/repository/multiple_models.py

@@ -12,95 +12,124 @@
 from backend.ml_model.repository.data_preprocessing_multiple_models import DataProcessorMultiple
 from backend.ml_model.repository.fairness import FairnessEvaluator
 from backend.ml_model.repository.file_reader_multiple_models import FileReaderMultiple
-from backend.ml_model.repository.safe_train_grid import safe_train_test_split
+from backend.ml_model.repository.safe_split import SafeSplitter
 
 current_dir = os.path.dirname(os.path.abspath(__file__))
 csv_file_path = os.path.join(current_dir, "../../../database/output.csv")
 
 
-def evaluate_multiple_models(model_files):
-    file_reader = FileReaderMultiple(csv_file_path)
-    df_dropped, inputs, target = file_reader.read_file()
-
-    data_processor = DataProcessorMultiple(inputs)
-    inputs_encoded = data_processor.encode_categorical_columns()
-    inputs_n = data_processor.drop_categorical_columns()
-
-    split_data = safe_train_test_split(inputs_n, target)
-    if split_data is None:
-        return {}
-
-    x_train, x_test, y_train, y_test = split_data
-
-    # Debug: Print columns of x_test
-    print(f"Columns in x_test: {x_test.columns}")
-
-    # Update sensitive features list to match the actual column names in x_test
-    sensitive_features_list = ["gender_N", "age_groups_N", "race_N", "state_N"]
-
-    # Dictionary to store results
-    results = {}
-
-    for model_file in model_files:
-        try:
-            # Load the model
-            with open(model_file, "rb") as f:
-                model_dict = pickle.load(f)
-                model = model_dict["model"]
-                print(f"Loaded object type: {type(model)}")
-                print(f"Loaded object: {model}")
-
-            # Make predictions
-            y_pred = model.predict(x_test)
-
-            # Debug: Check predictions
-            print(f"True Labels (y_test): {y_test.head()}")
-            print(f"Predicted Labels (y_pred): {y_pred[:10]}")
-
-            # Dictionary and list to store fairness results for the model
-            model_results = {}
-            fairness_values = []
-
-            for sensitive_feature in sensitive_features_list:
-                # Select the sensitive feature column for the evaluation
-                if sensitive_feature in x_test.columns:
-                    sensitive_col = x_test[sensitive_feature]
-                    print(
-                        f"Sensitive Feature: {sensitive_feature}, "
-                        f"Column Data: {sensitive_col.head()}"
-                    )
-
-                    # Evaluate fairness for this specific sensitive feature
-                    fairness_evaluator = FairnessEvaluator(
-                        y_test, y_pred, sensitive_col
-                    )
-                    metric_frame = fairness_evaluator.evaluate_fairness()
-
-                    average = sum(metric_frame.by_group["accuracy"]) / len(
-                        metric_frame.by_group["accuracy"]
+class MultiModelEvaluator:
+    """
+    A class for evaluating multiple machine learning models on a dataset.
+
+    This class handles data preprocessing, model loading, and fairness evaluation
+    based on specified sensitive features, producing metrics for each model.
+    """
+    def __init__(self, model_files: list):
+        """
+        Initializes the ModelEvaluator with the file path to the CSV and model files.
+
+        Parameters:
+        -----------
+
+        model_files : list
+            List of file paths to the model pickle files to evaluate.
+        """
+        self.model_files = model_files
+
+    def evaluate_models(self) -> dict:
+        """
+        Evaluates multiple models for fairness and performance metrics.
+
+        Returns:
+        --------
+        dict
+            A dictionary containing evaluation results for each model file.
+        """
+        # Read and preprocess the data
+        file_reader = FileReaderMultiple(csv_file_path)
+        df_dropped, inputs, target = file_reader.read_file()
+
+        data_processor = DataProcessorMultiple(inputs)
+        inputs_encoded = data_processor.encode_categorical_columns()
+        inputs_n = data_processor.drop_categorical_columns()
+
+        data_splitter = SafeSplitter()
+        split_data = data_splitter.train_test_split(inputs_n, target)
+        if split_data is None:
+            return {}
+
+        x_train, x_test, y_train, y_test = split_data
+
+        # Debug: Print columns of x_test
+        print(f"Columns in x_test: {x_test.columns}")
+
+        # Update sensitive features list to match the actual column names in x_test
+        sensitive_features_list = ["gender_N", "age_groups_N", "race_N", "state_N"]
+
+        # Dictionary to store results
+        results = {}
+
+        for model_file in self.model_files:
+            try:
+                # Load the model
+                with open(model_file, "rb") as f:
+                    model_dict = pickle.load(f)
+                    model = model_dict["model"]
+                    print(f"Loaded object type: {type(model)}")
+                    print(f"Loaded object: {model}")
+
+                # Make predictions
+                y_pred = model.predict(x_test)
+
+                # Debug: Check predictions
+                print(f"True Labels (y_test): {y_test.head()}")
+                print(f"Predicted Labels (y_pred): {y_pred[:10]}")
+
+                # Dictionary and list to store fairness results for the model
+                model_results = {}
+                fairness_values = []
+
+                for sensitive_feature in sensitive_features_list:
+                    # Select the sensitive feature column for the evaluation
+                    if sensitive_feature in x_test.columns:
+                        sensitive_col = x_test[sensitive_feature]
+                        print(
+                            f"Sensitive Feature: {sensitive_feature}, "
+                            f"Column Data: {sensitive_col.head()}"
+                        )
+
+                        # Evaluate fairness for this specific sensitive feature
+                        fairness_evaluator = FairnessEvaluator(
+                            y_test, y_pred, sensitive_col
+                        )
+                        metric_frame = fairness_evaluator.evaluate_fairness()
+
+                        average = sum(metric_frame.by_group["accuracy"]) / len(
+                            metric_frame.by_group["accuracy"]
+                        )
+                        rounded = round(average, 3)
+
+                        fairness_values.append(average)
+
+                        # Store the metrics for this demographic
+                        demo_name = sensitive_feature.replace("_N", "")
+                        model_results[demo_name] = rounded
+                    else:
+                        print(
+                            f"Sensitive feature '{sensitive_feature}' "
+                            f"not found in x_test."
+                        )
+
+                if fairness_values:
+                    model_results["variance"] = round(pd.Series(fairness_values).var(), 7)
+                    model_results["mean"] = round(
+                        sum(fairness_values) / len(fairness_values), 3
                     )
-                    rounded = round(average, 3)
-
-                    fairness_values.append(average)
-
-                    # Store the metrics for this demographic
-                    demo_name = sensitive_feature.replace("_N", "")
-                    model_results[demo_name] = rounded
-                else:
-                    print(
-                        f"Sensitive feature '{sensitive_feature}' "
-                        f"not found in x_test."
-                    )
-
-            if fairness_values:
-                model_results["variance"] = round(pd.Series(fairness_values).var(), 7)
-                model_results["mean"] = round(
-                    sum(fairness_values) / len(fairness_values), 3
-                )
-            # Store results for the model
-            results[model_file] = model_results
+                # Store results for the model
+                results[model_file] = model_results
 
-        except Exception as e:
-            results[model_file] = f"Error: {e}"
+            except Exception as e:
+                results[model_file] = f"Error: {e}"
 
-    return results
+        return results

backend/ml_model/repository/safe_grid_search.py

@@ -0,0 +1,59 @@
+from sklearn.model_selection import GridSearchCV
+from sklearn.tree import DecisionTreeClassifier
+import pandas as pd
+
+
+class SafeGridSearch:
+    """
+    A utility class for safely performing grid search for hyperparameter tuning.
+
+    This class ensures that grid search is executed while handling cases
+    where there are insufficient samples for cross-validation.
+    """
+
+    def __init__(self, classifier=DecisionTreeClassifier(), param_grid=None):
+        """
+        Initializes the SafeGridSearch with a classifier and a parameter grid.
+
+        Parameters:
+        -----------
+        classifier : estimator object, optional (default=DecisionTreeClassifier())
+            The base classifier to use for grid search.
+
+        param_grid : dict, optional
+            The parameter grid to use for tuning hyperparameters. If None,
+            a default parameter grid for DecisionTreeClassifier is used.
+        """
+        self.classifier = classifier
+        self.param_grid = param_grid or {
+            "criterion": ["gini", "entropy"],
+            "max_depth": [None] + list(range(1, 11)),
+            "min_samples_split": [2, 5, 10],
+        }
+
+    def perform_search(self, x_train: pd.DataFrame, y_train: pd.Series):
+        """
+        Performs a safe grid search for hyperparameter tuning.
+
+        Parameters:
+        -----------
+        x_train : pd.DataFrame
+            The training feature set.
+
+        y_train : pd.Series
+            The training labels.
+
+        Returns:
+        --------
+        estimator or None
+            Returns the best estimator if grid search is successful.
+            Returns None if there are insufficient samples for cross-validation.
+        """
+        try:
+            grid_search = GridSearchCV(self.classifier, self.param_grid, cv=5, scoring="accuracy")
+            grid_search.fit(x_train, y_train)
+            return grid_search.best_estimator_
+        except ValueError as e:
+            if "Cannot have number of splits n_splits" in str(e):
+                print("Not enough samples for cross-validation. Returning None.")
+                return None