conll_to_dataset.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Convert the datasets from conll format to HuggingFace DatasetDict "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import os\n",
    "from datasets import Dataset, DatasetDict\n",
    "from pathlib import Path\n",
    "\n",
    "\n",
    "def parse_conll(raw:str, sep=\"\\t\"):\n",
    "    \"\"\"Parses the norec-fine conll files with tab separator and sentence id\"\"\"\n",
    "    doc_parsed = [] # One dict per sentence. meta, tokens and tags\n",
    "    for sent in raw.strip().split(\"\\n\\n\"):\n",
    "        meta = \"\"\n",
    "        tokens, tags = [], []\n",
    "        for line in sent.split(\"\\n\"):\n",
    "            if line.startswith(\"#\") and \"=\" in line:\n",
    "                meta = line.split(\"=\")[-1]\n",
    "            else:\n",
    "                elems = line.strip().split(sep)\n",
    "                assert len(elems) == 2\n",
    "                tokens.append(elems[0])\n",
    "                tags.append(elems[1])\n",
    "        assert len(meta) > 0\n",
    "        doc_parsed.append({\"idx\": meta, \"tokens\":tokens, \"tsa_tags\":tags})\n",
    "    return doc_parsed\n",
    "\n",
    "\n",
    "conll_folders = [\"tsa_conll\", \n",
    "                \"tsa-conll-intensity\"\n",
    "                ] \n",
    "assert all( [\"conll\" in s for s in conll_folders]) # If you remove this, change rule for naming arrow folder.\n",
    "assert all([Path(s).is_dir() for s in conll_folders ]), \"Not all source folders exist\"\n",
    "\n",
    "splits = {\"train\": \"train\", \"dev\": \"validation\", \"test\": \"test\"} # \"validation\" for HF naming convention\n",
    "for c_folder in conll_folders:\n",
    "    arrow_folder = c_folder.replace(\"conll\", \"arrow\")\n",
    "    d_sets = {}\n",
    "    for split in splits:\n",
    "        conll_txt = Path(c_folder, split+\".conll\").read_text()\n",
    "        print(\"\\n\",c_folder, split, len(conll_txt.split(\"\\n\\n\")))\n",
    "        sents = parse_conll(conll_txt)\n",
    "        # for sent in sents:\n",
    "            # sent[\"labels\"] = [label_mapping[tag] for tag in sent[\"tsa_tags\"]]\n",
    "        d_sets[splits[split]] = Dataset.from_pandas(pd.DataFrame(sents))\n",
    "        print(d_sets[splits[split]][102])\n",
    "\n",
    "        DatasetDict(d_sets).save_to_disk(arrow_folder)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "d_sets\n",
    "labels = [l for s in d_sets[\"test\"]['tsa_tags'] for l in s]\n",
    "set(labels)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "phdev",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.18"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}