Zero-shot_classification.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "d7e57db0",
   "metadata": {},
   "source": [
    "Import necessary libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "93d00638",
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import pipeline\n",
    "import pandas as pd\n",
    "import re\n",
    "from sklearn.metrics import classification_report, confusion_matrix, ConfusionMatrixDisplay, accuracy_score, precision_score, recall_score, f1_score"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6c54f9d9",
   "metadata": {},
   "source": [
    "Upload zero-shot model. You can download it here https://huggingface.co/MoritzLaurer/mDeBERTa-v3-base-xnli-multilingual-nli-2mil7."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "15e24389",
   "metadata": {},
   "outputs": [],
   "source": [
    "classifier = pipeline(\"zero-shot-classification\", model=\"MoritzLaurer/mDeBERTa-v3-base-mnli-xnli\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "19166ea3",
   "metadata": {},
   "source": [
    "Upload test dataset.We explained how to generate in dataset_generation file, here you can find a link to this file https://github.com/preste-ai/rnd-nlp-cot-chatgptdatagen/blob/main/dataset_generation.ipynb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "14f5130c",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = pd.read_csv(\"Your test set\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "30899cb9",
   "metadata": {},
   "source": [
    "Run zero-shot model on our data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "33676edd",
   "metadata": {},
   "outputs": [],
   "source": [
    "candidate_labels = [\"réservation_de_restaurant\", \"réserver_un_vol\", \"annuler_la_réservation\", \"état_du_vol\", \"réserver_un_hôtel\", \"suggestion_de_voyage\", \"visa_international\", \"transfert\", \"location_de_voiture\", \"bagages_perdus\"]\n",
    "def run_zero_shot(dataset, candidate_labels):\n",
    "    data = dataset[\"text-fr\"].tolist()\n",
    "    pred_dict = []\n",
    "    for sent in data:\n",
    "        sequence_to_classify = sent\n",
    "        output = classifier(sequence_to_classify, candidate_labels, multi_label=False)\n",
    "        pred_dict.append(output)\n",
    "    print(pred_dict)\n",
    "    pred_labels = []\n",
    "    for pred in pred_dict:\n",
    "        pred_labels.append(pred[\"labels\"][0])\n",
    "    dataset['Predictions'] = pred_labels\n",
    "    dataset['Predictions_encoding'] = dataset['Predictions'].replace({\"réservation_de_restaurant\" : 0 , \"bagages_perdus\" : 1,\n",
    "                                     \"annuler_la_réservation\" : 2, \"suggestion_de_voyage\" : 3, \n",
    "                                     \"visa_international\" : 4, \"état_du_vol\" : 5, \"transfert\" : 6,\n",
    "                                     \"réserver_un_vol\" : 7, \"location_de_voiture\" : 8, \"réserver_un_hôtel\" : 9\n",
    "                                     })\n",
    "    return dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "1e59d64b",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = run_zero_shot(dataset, candidate_labels)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6fba0d77",
   "metadata": {},
   "source": [
    "Get metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "fbd66098",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_metrics(y_pred, y_test): \n",
    "    test_report = classification_report(y_test, y_pred, output_dict=True)\n",
    "    \n",
    "    accuracy = accuracy_score(y_test, y_pred)\n",
    "    precision = precision_score(y_test, y_pred, average='weighted')\n",
    "    recall = recall_score(y_test, y_pred, average='weighted')\n",
    "    f1 = f1_score(y_test, y_pred, average='weighted')\n",
    "\n",
    "    return test_report, accuracy, precision, recall, f1\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "8f14b511",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/preste-dakam/Documents/Projects_Preste/MadMonkeys/Mad_Monkeys/lib/python3.10/site-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/preste-dakam/Documents/Projects_Preste/MadMonkeys/Mad_Monkeys/lib/python3.10/site-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/preste-dakam/Documents/Projects_Preste/MadMonkeys/Mad_Monkeys/lib/python3.10/site-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/preste-dakam/Documents/Projects_Preste/MadMonkeys/Mad_Monkeys/lib/python3.10/site-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/preste-dakam/Documents/Projects_Preste/MadMonkeys/Mad_Monkeys/lib/python3.10/site-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/preste-dakam/Documents/Projects_Preste/MadMonkeys/Mad_Monkeys/lib/python3.10/site-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/preste-dakam/Documents/Projects_Preste/MadMonkeys/Mad_Monkeys/lib/python3.10/site-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/preste-dakam/Documents/Projects_Preste/MadMonkeys/Mad_Monkeys/lib/python3.10/site-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Recall is ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    }
   ],
   "source": [
    "test_report, accuracy, precision, recall, f1 = get_metrics(dataset[\"Predictions_encoding\"], dataset[\"labels\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "1075d1e2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy:  0.7\n",
      "Precision:  0.7\n",
      "Recall:  0.7\n",
      "F1:  0.7\n"
     ]
    }
   ],
   "source": [
    "print(\"Accuracy: \", accuracy)\n",
    "print(\"Precision: \", precision)\n",
    "print(\"Recall: \", recall)\n",
    "print(\"F1: \", f1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4d9f0c56",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}