docs/notebooks/example_pipeline.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Read data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "sentences = pd.read_csv(\"demo_file.csv\") # dataset already split into sentences, not included in repository"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Correct spelling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from psynlp.spelling import SpellChecker"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "spacy_model_name = \"default_model\"\n",
    "\n",
    "sc = SpellChecker(spacy_model_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "sentences['text_corrected'] = sentences['text'].apply(lambda x : sc.correct(x))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Find entities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "entity_phrases = {} \n",
    "\n",
    "# some examples\n",
    "entity_phrases['abdominaal'] = ['buikpijn', 'diarree', 'buikklachten', 'buikpijnklachten', 'buikkrampen', 'diaree', 'maagpijn', 'misselijkheid', 'maagklachten', 'misselijkheidsklachten']\n",
    "\n",
    "entity_phrases['angst'] = ['angstig', 'bang', 'angstig', 'anstig', 'angstiger', 'paniekerig', 'paniek', 'angstklachten']\n",
    "\n",
    "entity_phrases['spanning'] = ['gespannen', 'onrustig', 'prikkelbaar', 'geprikkeld', 'geagiteerd', 'geladen', 'geaggiteerd']\n",
    "\n",
    "entity_phrases['depressie'] = ['depressie', 'depressies', 'stemmingsklachten', 'somberheidsklachten', 'stemmingsstoornis', 'depressiviteit', 'depressieve', 'stemmingsproblematiek', 'stemmingsprobleem', 'stemmingsproblemen', 'stemmings', 'stemmingstoornis']\n",
    "\n",
    "entity_phrases['vermoeid'] = ['vermoeid', 'versufd', 'apatisch', 'energieloos', 'initiatiefloos', 'lamlendig', 'futloos', 'lusteloos', 'apathisch', 'inactief', 'duf', 'suf', 'suffig', 'versuft']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "from psynlp.entity import BasicEntityMatcher"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "bem = BasicEntityMatcher(entity_phrases, spacy_model_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "sentences['entities'] = sentences['text_corrected'].apply(lambda x : bem.extract_entities(x))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Determine context"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "from psynlp.context import ContextMatcher"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "cm = ContextMatcher(spacy_model_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sentences.apply(lambda x : cm.match_context(x['text_corrected'], x['entities']), axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Process output into variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total number of entities matched = 212\n"
     ]
    }
   ],
   "source": [
    "num_entities = np.sum(sentences['entities'].apply(lambda x : len(x)))\n",
    "\n",
    "print(\"Total number of entities matched = {}\".format(num_entities))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total Number of mentions of historical depression = 1\n"
     ]
    }
   ],
   "source": [
    "## Example: historische depressie\n",
    "\n",
    "def has_depression_historical(entities):\n",
    "    \n",
    "    # Filter rule == depressie\n",
    "    entities = [e for e in entities if e.rule == 'depressie']\n",
    "    \n",
    "    # Filter HISTORICAL in context\n",
    "    entities = [e for e in entities if \"HISTORICAL\" in e.context]\n",
    "    \n",
    "    return len(entities) > 0\n",
    "\n",
    "sentences['historical_depression'] = sentences['entities'].apply(lambda x : has_depression_historical(x))\n",
    "\n",
    "print(\"Total Number of mentions of historical depression = {}\".format(sentences['historical_depression'].sum()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total number of relevant entities = 167\n"
     ]
    }
   ],
   "source": [
    "## Example: count only relevant entities\n",
    "\n",
    "def count_relevant_entities(entities):\n",
    "    \n",
    "    entities = [e for e in entities if \"PATIENT\" in e.context]\n",
    "    entities = [e for e in entities if \"AFFIRMED\" in e.context]\n",
    "    entities = [e for e in entities if \"PLAUSIBLE\" in e.context]\n",
    "    entities = [e for e in entities if \"CURRENT\" in e.context]\n",
    "    \n",
    "    return len(entities)   \n",
    "\n",
    "sentences['num_relevant_entities'] = sentences['entities'].apply(lambda x : count_relevant_entities(x))\n",
    "\n",
    "print(\"Total number of relevant entities = {}\".format(sentences['num_relevant_entities'].sum()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Further aggregate on text/patient level\n",
    "texts = sentences.groupby(\"text_hash\").agg({'historical_depression' : min, # any mention \n",
    "                                            'num_relevant_entities' : sum  # count all mentions\n",
    "                                           }).reset_index()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}