notebooks/models/Evaluate flair models.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Evaluate Flair models for person names, orgs and locations using the Presidio Evaluator framework\n",
    "\n",
    "Data = `generated_test_November 12 2019`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [],
   "source": [
    "from presidio_evaluator.data_generator import read_synth_dataset\n",
    "%reload_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Select data for evaluation:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [],
   "source": [
    "synth_samples = read_synth_dataset(\"../../data/synth_dataset.txt\")\n",
    "print(len(synth_samples))\n",
    "DATASET = synth_samples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [],
   "source": [
    "from collections import Counter\n",
    "entity_counter = Counter()\n",
    "for sample in DATASET:\n",
    "    for tag in sample.tags:\n",
    "        entity_counter[tag]+=1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [],
   "source": [
    "entity_counter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [],
   "source": [
    "DATASET[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [],
   "source": [
    "#max length sentence\n",
    "max([len(sample.tokens) for sample in DATASET])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Select models for evaluation:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [],
   "source": [
    "flair_ner = 'ner'\n",
    "flair_ner_fast = 'ner-fast'\n",
    "flair_ontonotes = 'ner-ontonotes-fast'\n",
    "flair_bert_embeddings = '../../models/presidio-ner/flair-bert-embeddings.pt'\n",
    "glove_flair_embeddings = '../../models/presidio-ner/flair-embeddings.pt'\n",
    "models = [flair_bert_embeddings, glove_flair_embeddings, flair_ner,flair_ner_fast,flair_ontonotes]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "is_executing": true
    }
   },
   "outputs": [],
   "source": [
    "from presidio_evaluator.flair_evaluator import FlairEvaluator\n",
    "\n",
    "for model in models:\n",
    "    print(\"-----------------------------------\")\n",
    "    print(\"Evaluating model {}\".format(model))\n",
    "    flair_evaluator = FlairEvaluator(model_path=model)\n",
    "    evaluation_results = flair_evaluator.evaluate_all(DATASET)\n",
    "    scores = flair_evaluator.calculate_score(evaluation_results)\n",
    "    \n",
    "     \n",
    "    print(\"Confusion matrix:\")\n",
    "    print(scores.results)\n",
    "\n",
    "    print(\"Precision and recall\")\n",
    "    scores.print()\n",
    "    errors = scores.model_errors\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Custom evaluation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### False positives"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1. Most false positive tokens:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [],
   "source": [
    "errors = scores.model_errors\n",
    "\n",
    "from presidio_evaluator import ModelEvaluator\n",
    "ModelEvaluator.most_common_fp_tokens(errors)#[model_error for model_error in errors if model_error.error_type =='FP']\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [],
   "source": [
    "fps_df = ModelEvaluator.get_fps_dataframe(errors,entity=['PERSON'])\n",
    "fps_df[['full_text','token','prediction']]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "2. False negative examples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [],
   "source": [
    "ModelEvaluator.most_common_fn_tokens(errors,n=50, entity=['PERSON'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "More FN analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [],
   "source": [
    "fns_df = ModelEvaluator.get_fns_dataframe(errors,entity=['PERSON'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "is_executing": false,
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "fns_df[['full_text','token','annotation','prediction']]"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  },
  "pycharm": {
   "stem_cell": {
    "cell_type": "raw",
    "source": [],
    "metadata": {
     "collapsed": false
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}