pdf2json.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: tika in c:\\programdata\\anaconda3\\lib\\site-packages (1.19)\n",
      "Requirement already satisfied: requests in c:\\programdata\\anaconda3\\lib\\site-packages (from tika) (2.21.0)\n",
      "Requirement already satisfied: setuptools in c:\\programdata\\anaconda3\\lib\\site-packages (from tika) (40.8.0)\n",
      "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in c:\\programdata\\anaconda3\\lib\\site-packages (from requests->tika) (3.0.4)\n",
      "Requirement already satisfied: urllib3<1.25,>=1.21.1 in c:\\programdata\\anaconda3\\lib\\site-packages (from requests->tika) (1.24.1)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in c:\\programdata\\anaconda3\\lib\\site-packages (from requests->tika) (2019.3.9)\n",
      "Requirement already satisfied: idna<2.9,>=2.5 in c:\\programdata\\anaconda3\\lib\\site-packages (from requests->tika) (2.8)\n"
     ]
    }
   ],
   "source": [
    "import re\n",
    "import json\n",
    "import sys\n",
    "!{sys.executable} -m pip install tika\n",
    "from tika import parser\n",
    "from io import StringIO\n",
    "from itertools import islice "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "file = 'C://Users//gshar//Downloads//AyuSandhan//PDF//API1.pdf'\n",
    "# Parse data from file\n",
    "parsedPDF = parser.from_file(file)\n",
    "# Get files text content\n",
    "text = parsedPDF['content']\n",
    "\n",
    "text = text.replace('Ë','A')\n",
    "text = text.replace('¢','i')\n",
    "text = text.replace('á','S')\n",
    "text = text.replace('Ì','I')\n",
    "text = text.replace('â','S')\n",
    "text = text.replace('Û','N')\n",
    "text = text.replace('Ú','D')\n",
    "text = text.replace('Í','U')\n",
    "text = text.replace('Ø','N')\n",
    "text = text.replace('×','N')\n",
    "text = text.replace('ê','M')\n",
    "text = text.replace('Ù','T')\n",
    "\n",
    "text = text.replace('¿','s')\n",
    "text = text.replace('¡','a')\n",
    "text = text.replace('¶','t')\n",
    "text = text.replace('£','u')\n",
    "text = text.replace('À','s')\n",
    "text = text.replace('¤','r')\n",
    "text = text.replace('¸','n')\n",
    "text = text.replace('·','d')\n",
    "text = text.replace('´','n')\n",
    "text = text.replace('(',' (')\n",
    "text = text.replace('®','e')\n",
    "text = text.replace('°','o')\n",
    "text = text.replace('AGNlMANTHA','AGNIMANTHA')\n",
    "text = text.replace('KARAViRA','KARAVIRA')\n",
    "text = text.replace('ń́́','n')\n",
    "text = text.replace('¾','s')\n",
    "text = text.replace('¨','r')\n",
    "text = text.replace('¦','u')\n",
    "text = text.replace('º','s')\n",
    "text = text.replace('µ','micrometer')\n",
    "text = text.replace('Æ','m')\n",
    "text = text.replace('α','alpha (greek)')\n",
    "text = text.replace('Î','R')\n",
    "text = text.replace('¥','r')\n",
    "text = text.replace('β','beta (greek)')\n",
    "text = text.replace('˚','degree')\n",
    "\n",
    "#print(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "sep = \"Whole Plant Wh. Pl.\"\n",
    "txt2 = text.split(sep, 1)[1]\n",
    "re.sub('\\s\\s',' ','txt2')\n",
    "txt3 = ' '.join(txt2.split('  '))\n",
    "\n",
    "txt3 = re.sub(r\"\\n\\d{1,3}\\n\",\"\", txt3)\n",
    "#print(repr(txt3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "with StringIO(txt3) as f:   # replace StringIO(s) by open(filename) for real files on disk\n",
    "    key = []\n",
    "    temp = []\n",
    "    for line in f:\n",
    "        if line.strip() != '':\n",
    "            #print(line)\n",
    "            #print('----------------------------------------')\n",
    "            if line.strip()[0].isupper() and line.strip()[1].isupper() and line.strip()[2].isupper() and line.strip()[3].isupper():\n",
    "                #print(line)\n",
    "                list2= line.split(\" \")\n",
    "                temp = []\n",
    "                #print(list2)\n",
    "                for i in list2:\n",
    "                    #print(i,'--',len(i))\n",
    "                    if i.isupper() and len(i)>2 and i[2]!=';': \n",
    "                        #print(i)\n",
    "                        temp.append(i)\n",
    "                key.append(' '.join(temp))\n",
    "            #elif line == \"T.L.C. -\\n\" or line == \"T.L.C. - \\n\" or line == \"T.L.C. \\n\" or line == \"T.L.C.  \\n\" or line == \" T.L.C. \\n\":\n",
    "                #print(\"true\")\n",
    "                #key.append(line)\n",
    "\n",
    "#for i in key:\n",
    "#    print(repr(i))\n",
    "#    print('~~~~~~~~~~~~~')\n",
    "\n",
    "#key.remove('')\n",
    "key = list(filter(None, key))\n",
    "for i in range(len(key)):\n",
    "    key[i] = key[i].split('(')[0] \n",
    "    key[i] = key[i].split(')')[0] \n",
    "    key[i] = key[i].strip()\n",
    "    key[i]= key[i].replace('SYNONYMS-','SYNONYMS')\n",
    "    #key[i]= key[i].replace('T.L.C.','\\nT.L.C.')\n",
    "split_str = '|'.join(key)\n",
    "\n",
    "val = re.split(split_str,txt3)\n",
    "val = list(filter(None, val))\n",
    "#val = list(filter(\" \", val))\n",
    "val = val[1:]\n",
    "#dictionary = dict(zip(key,val[1:]))\n",
    "while \" \\n\\n\" in val: val.remove(\" \\n\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "name_list = []\n",
    "for i in key:\n",
    "    if i!='SYNONYMS' and i!='DESCRIPTION' and i!='IDENTITY, PURITY AND STRENGTH' and i!= 'IDENTITY, PURITY AND STRENGTH-' and i!='CONSTITUENTS' and i!='PROPERTIES AND ACTION' and i!='IMPORTANT FORMULATIONS' and i!='THERAPEUTIC USES' and i!='DOSE' and i!='ASSAY' and i!='IDENTIFICATION TEST'and i!='T.L.C. -':\n",
    "        name_list.append(i)\n",
    "        \n",
    "smaller_keys = [[]]\n",
    "for x in key:\n",
    "    if x in name_list:\n",
    "        a, b = x.split(x)\n",
    "        if a:  # remove the check if you also want the empty elements\n",
    "            smaller_keys[-1].append(a)\n",
    "        smaller_keys.append([])\n",
    "        if b:  # remove the check if you also want the empty elements\n",
    "            smaller_keys[-1].append(b)\n",
    "    else:\n",
    "        smaller_keys[-1].append(x)\n",
    "smaller_keys.remove([])\n",
    "\n",
    "for i in range(len(smaller_keys)):\n",
    "    smaller_keys[i].insert(0,'ABOUT')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "length_to_split = []\n",
    "for i in smaller_keys:\n",
    "    length_to_split.append(len(i))\n",
    "\n",
    "Input = iter(val) \n",
    "smaller_vals = [list(islice(Input, elem)) \n",
    "          for elem in length_to_split] \n",
    "#print(smaller_vals)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "nkey = []\n",
    "nkey = re.findall(r\"\\n\\s?\\d{1,3}\\.\\s[A-Z][a-z]+[^\\n]+\",txt3)\n",
    "\n",
    "\n",
    "for i in range(len(smaller_vals)):\n",
    "    for j in range(len(smaller_vals[i])):\n",
    "#        smaller_vals[i][j] = smaller_vals[i][j].replace('\\n','')\n",
    "        smaller_vals[i][j] = smaller_vals[i][j].replace('Rishi\\nHighlight','')\n",
    "        smaller_vals[i][j] = smaller_vals[i][j].strip()\n",
    "#    for k in range(len(smaller_keys[i])):\n",
    "#        smaller_keys[i][k] = smaller_keys[i][k].replace('\\n','')\n",
    "\n",
    "        \n",
    "name_list = list(dict.fromkeys(nkey))    #filter out unique\n",
    "for i in range(len(name_list)):\n",
    "    name_list[i]=name_list[i][5:].strip()\n",
    "data_list = []\n",
    "\n",
    "#for i in range(len(smaller_keys)):\n",
    "#    for j in range(len(smaller_keys[i])):\n",
    "#        smaller_keys[i][j] = smaller_keys[i][j].replace('\\n','')\n",
    "#    for k in range(len(smaller_vals[i])):\n",
    "#        smaller_vals[i][j] = smaller_vals[i][j].replace('\\n','')\n",
    "        \n",
    "for i in range(len(smaller_keys)):\n",
    "    smaller_keys[i].insert(0,'NAME')\n",
    "    smaller_vals[i].insert(0,name_list[i])\n",
    "    smaller_keys[i].insert(9,'T.L.C.')\n",
    "    smaller_vals[i].insert(9,None)\n",
    "    data_list.append(dict(zip(smaller_keys[i],smaller_vals[i])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "counter = 0\n",
    "for i in range(len(data_list)):\n",
    "    \n",
    "    synn = data_list[i]['SYNONYMS']\n",
    "    \n",
    "    synn = synn.replace('Eng \\n','Hing \\n')\n",
    "\n",
    "    synn = synn.replace('Assamese','@Assamese')\n",
    "    synn = synn.replace('Bengali','@Bengali')\n",
    "    synn = synn.replace('English','@English')\n",
    "    synn = synn.replace('Gujrati','@Gujrati')\n",
    "    synn = synn.replace('Hindi    :','@Hindi    :')\n",
    "    synn = synn.replace('Kannada','@Kannada ')\n",
    "    synn = synn.replace('Kashmiri','@Kashmiri')\n",
    "    synn = synn.replace('Malayalam','@Malayalam')\n",
    "    synn = synn.replace('Marathi','@Marathi')\n",
    "    synn = synn.replace('Oriya','@Oriya')\n",
    "    synn = synn.replace('Punjabi','@Punjabi')\n",
    "    synn = synn.replace('Tamil','@Tamil')\n",
    "    synn = synn.replace('Telugu','@Telugu')\n",
    "    synn = synn.replace('Urdu','@Urdu')\n",
    "    \n",
    "    synn = synn.replace('Sansk. ','Sanskrit')\n",
    "    synn = synn.replace('Assam. ','@Assamese')\n",
    "    synn = synn.replace('Beng. ','@Bengali')\n",
    "    synn = synn.replace('Eng. ','@English')\n",
    "    synn = synn.replace('Guj. ','@Gujrati')\n",
    "    synn = synn.replace('Hindi.: ','@Hindi: ')\n",
    "    synn = synn.replace('Kan. ','@Kannada ')\n",
    "    synn = synn.replace('Kash. ','@Kashmiri ')\n",
    "    synn = synn.replace('Mal. ','@Malayalam ')\n",
    "    synn = synn.replace('Mar. ','@Marathi ')\n",
    "    synn = synn.replace('Ori. ','@Oriya ')\n",
    "    synn = synn.replace('Punj. ','@Punjabi ')\n",
    "    synn = synn.replace('Tam. ','@Tamil ')\n",
    "    synn = synn.replace('Tel. ','@Telugu ')\n",
    "    synn = synn.replace('Urdu. ','@Urdu ')\n",
    "    \n",
    "    \n",
    "    synn = synn.replace('Sansk ','Sanskrit ')\n",
    "    synn = synn.replace('Assam ','@Assamese ')\n",
    "    synn = synn.replace('Beng ','@Bengali ')\n",
    "    synn = synn.replace('Eng ','@English ')\n",
    "    synn = synn.replace('Guj ','@Gujrati ')\n",
    "    synn = synn.replace('Hindi: ','@Hindi: ')\n",
    "    synn = synn.replace('Kan ','@Kannada ')\n",
    "    synn = synn.replace('Kash ','@Kashmiri ')\n",
    "    synn = synn.replace('Mal ','@Malayalam ')\n",
    "    synn = synn.replace('Mar ','@Marathi ')\n",
    "    synn = synn.replace('Ori ','@Oriya ')\n",
    "    synn = synn.replace('Punj ','@Punjabi ')\n",
    "    synn = synn.replace('Tam ','@Tamil ')\n",
    "    synn = synn.replace('Tel ','@Telugu ')\n",
    "    synn = synn.replace('Urdu ','@Urdu ')\n",
    "\n",
    "    \n",
    "    synn_list = synn.split('@')\n",
    "    synn_list = list(filter(lambda a: a != '', synn_list))\n",
    "    synn_list = list(filter(lambda a: a != ' ', synn_list))\n",
    "    n_list = []\n",
    "    \n",
    "    for j in synn_list:\n",
    "        n_list.append(j.split(':'))\n",
    "    counter = counter+1\n",
    "    #print(counter)\n",
    "    for k in n_list:\n",
    "        for w in range(len(k)):\n",
    "            k[w] = k[w].replace('\\n',\"\")\n",
    "            k[w] = k[w].strip()        \n",
    "        #print(k)\n",
    "        k[1] = k[1].split(',')\n",
    "    l1 = []\n",
    "    l2 = []\n",
    "    for m in n_list:\n",
    "        l1.append(m[0])\n",
    "        l2.append(list(m[1]))\n",
    "    data_list[i]['SYNONYMS'] = dict(zip(l1,l2))\n",
    "    \n",
    "#################################################################  \n",
    "\n",
    "    panda = data_list[i]['PROPERTIES AND ACTION']\n",
    "    #print(panda)\n",
    "    panda = panda.replace('Guna ','@Guna')\n",
    "    panda = panda.replace('Virya ','@Sita')\n",
    "    panda = panda.replace('Vipaka ','@Vipaka')\n",
    "    panda = panda.replace('Karma ','@Karma')\n",
    "    panda = panda.replace('-\\n','')\n",
    "    panda = panda.replace('- \\n','')\n",
    "    panda = panda.replace('–\\n','')\n",
    "    \n",
    "    panda_list = panda.split('@')\n",
    "    panda_list = list(filter(lambda a: a != '', panda_list))\n",
    "    panda_list = list(filter(lambda a: a != ' ', panda_list))\n",
    "\n",
    "    n_list2 = []\n",
    "    \n",
    "    for p in panda_list:\n",
    "        n_list2.append(p.split(':'))\n",
    "\n",
    "    for q in n_list2:\n",
    "        q[0] = q[0].replace('\\n',\"\")\n",
    "        q[1] = q[1].strip()\n",
    "        q[1] = q[1].split(',')\n",
    "        for s in range(len(q[1])):\n",
    "            q[1][s] = q[1][s].replace('\\n',\"\")\n",
    "            q[1][s] = q[1][s].strip()\n",
    "        #print(q)   \n",
    "    l3 = []\n",
    "    l4 = []\n",
    "    for r in n_list2:\n",
    "        l3.append(r[0].strip())\n",
    "        l4.append(list(r[1]))\n",
    "    data_list[i]['PROPERTIES AND ACTION'] = dict(zip(l3,l4))\n",
    "    \n",
    "###############################################################################\n",
    "    if 'IMPORTANT FORMULATIONS' in data_list[i]:\n",
    "        impfor = data_list[i]['IMPORTANT FORMULATIONS']\n",
    "        impfor = impfor.replace('-','')\n",
    "        impfor = impfor.replace('.','')\n",
    "        impfor = impfor.strip()\n",
    "        impfor = impfor.split(',')\n",
    "        for k in range(len(impfor)):\n",
    "            impfor[k] = impfor[k].replace(\"\\n\",\"\")\n",
    "            impfor[k] = impfor[k].strip()\n",
    "        #print(impfor)\n",
    "        data_list[i]['IMPORTANT FORMULATIONS'] = impfor\n",
    "    \n",
    "###############################################################################    \n",
    "    thuse = data_list[i]['THERAPEUTIC USES']\n",
    "    thuse = thuse.replace('-','')\n",
    "    thuse = thuse.replace('.','')\n",
    "    thuse = thuse.strip()\n",
    "    thuse = thuse.split(',')\n",
    "    for t in range(len(thuse)):\n",
    "        thuse[t] = thuse[t].replace(\"\\n\",\"\")\n",
    "        thuse[t] = thuse[t].strip()\n",
    "    data_list[i]['THERAPEUTIC USES'] = thuse\n",
    "    \n",
    "############################################################################### \n",
    "    if \"IDENTITY, PURITY AND STRENGTH\" in data_list[i]:\n",
    "        \n",
    "        qq = data_list[i]['IDENTITY, PURITY AND STRENGTH'].split(\" \")\n",
    "        for ll in range(len(qq)):\n",
    "            if qq[ll]==\"\\nT.L.C.\":\n",
    "                data_list[i]['T.L.C.'] = \" \".join(qq[ll+1:len(qq)])\n",
    "                break\n",
    "        \n",
    "        \n",
    "        ips = data_list[i]['IDENTITY, PURITY AND STRENGTH'].split('\\n')\n",
    "        ips2 = []\n",
    "        \n",
    "        for p in range(len(ips)):\n",
    "            ips[p] = ips[p].split(',')\n",
    "            ips[p] = ips[p][0]\n",
    "            ips[p] = ips[p].split(' N')\n",
    "            if len(ips[p])>1:\n",
    "                ips2.append(ips[p])\n",
    "        for r in range(len(ips2)):\n",
    "            ips2[r][1] = 'N'+ips2[r][1]\n",
    "            ips2[r][1] = ips2[r][1].strip()\n",
    "            ips2[r][0] = ips2[r][0].strip()\n",
    "        l5 = []\n",
    "        l6 = []\n",
    "        for q in ips2:\n",
    "            l5.append(q[0])\n",
    "            l6.append(q[1])\n",
    "        data_list[i]['IDENTITY, PURITY AND STRENGTH'] = dict(zip(l5,l6))\n",
    "    \n",
    "############################################################################### \n",
    "    if \"IDENTITY, PURITY AND STRENGTH-\" in data_list[i]:\n",
    "        ips2 = data_list[i]['IDENTITY, PURITY AND STRENGTH-']\n",
    "        ips2 = ips2.replace(\"\\n\",\"\")\n",
    "        ips2 = ips2.replace(\"  \",\" \")\n",
    "        data_list[i][\"IDENTITY, PURITY AND STRENGTH-\"] = ips2.strip()\n",
    "\n",
    "#######################################################################################\n",
    "    abt = data_list[i][\"ABOUT\"]\n",
    "    abt = abt.replace(\"\\n\",\"\")\n",
    "    abt = abt.replace(\"  \",\" \")\n",
    "    data_list[i][\"ABOUT\"] = abt.strip()\n",
    "    \n",
    "#######################################################################################\n",
    "    if \"CONSTITUENTS\" in data_list[i]:\n",
    "        constit = data_list[i][\"CONSTITUENTS\"]\n",
    "        constit = constit.replace(\"\\n\",\"\")\n",
    "        constit = constit.replace(\" and \", \",\")\n",
    "        constit = constit.replace(\"  \",\" \")\n",
    "        constit = constit.replace(\"- \",\"\")\n",
    "        constit = constit.replace(\".\",\"\")\n",
    "        \n",
    "        constit_list = constit.split(',')\n",
    "        for z in range(len(constit_list)):\n",
    "            constit_list[z] = constit_list[z].strip()\n",
    "        data_list[i][\"CONSTITUENTS\"] = constit_list\n",
    "    \n",
    "    \n",
    "#######################################################################################\n",
    "\n",
    "    if \"ASSAY\" in data_list[i]:\n",
    "        asy = data_list[i][\"ASSAY\"]\n",
    "        asy = asy.replace(\"\\n\",\"\")\n",
    "        asy = asy.replace(\"  \",\" \")\n",
    "        data_list[i][\"ASSAY\"] = asy.strip()\n",
    "#####################################################################################################\n",
    "    \n",
    "    if \"DOSE\" in data_list[i]:\n",
    "        dse = data_list[i][\"DOSE\"]\n",
    "        dse = re.sub(r\"\\n\\d{1,3}\\.\",\"qwerty\", dse)\n",
    "        dse = dse.split(\"qwerty\")[0]\n",
    "        dse = dse.replace(\"- \",\"\")\n",
    "        dse = dse.replace(\"\\n\",\" \")\n",
    "        dse = dse.strip()\n",
    "        data_list[i][\"DOSE\"] = dse\n",
    "    \n",
    "###########################################################################################\n",
    "\n",
    "    desc = data_list[i][\"DESCRIPTION\"]\n",
    "    desc = desc.replace('a) Macroscopic\\n','')\n",
    "    desc_list = desc.split('\\nb) Microscopic')\n",
    "    for z in range(len(desc_list)):\n",
    "        desc_list[z] = desc_list[z].replace('\\n','')\n",
    "        desc_list[z] = desc_list[z].strip()\n",
    "    if len(desc_list)==2:\n",
    "        l7 = ['Macroscopic', 'Microscopic']\n",
    "    else:\n",
    "        l7 = ['Macroscopic']\n",
    "        \n",
    "    data_list[i][\"DESCRIPTION\"] = dict(zip(l7,desc_list))\n",
    "   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "dd = json.dumps(data_list)\n",
    "loaded_dd = json.loads(dd)\n",
    "with open('api1.json', 'w') as fp:\n",
    "    json.dump(loaded_dd, fp)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}