Summarization Compression.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "conda_tensorflow_p36"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from ste import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "JSON not found. Loading .ZIP\n"
     ]
    }
   ],
   "source": [
    "data = getJsonData(\"code_summarization/abstract_code_dict.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "File larger than 100MB, Zipped for git.\n",
      "Previous size: 188MB Current size: 20MB.\n"
     ]
    }
   ],
   "source": [
    "dump_dict(data, \"code_summarization/abstract_code_dict4.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "678033"
      ]
     },
     "execution_count": 89,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "F:\\GDrive RA\\RA\\Code Translation\\Sub-Tree-Encoding\n"
     ]
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# add class\n",
    "# parse\n",
    "# build dictionary\n",
    "# Build Numto compress\n",
    "# compress\n",
    "# tokenize\n",
    "# remove class tokens"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Building Parse Tree"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "data "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['code_summarization/dev/code.original',\n",
       " 'code_summarization/test/code.original',\n",
       " 'code_summarization/train/code.original']"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Use glob to find all files\n",
    "# Or just list the 6 directories manually\n",
    "paths = [\n",
    "    'code_summarization/dev/code.original',\n",
    "    'code_summarization/test/code.original',\n",
    "    'code_summarization/train/code.original'\n",
    "]\n",
    "paths"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "=====[ code_summarization/dev/code.original ]=====\n",
      "[ 0 ] 0:00:01.312469\n",
      "[ 500 ] 0:02:41.982281\n",
      "[ 1000 ] 0:22:05.500365\n",
      "[ 1500 ] 0:24:43.864442\n",
      "[ 2000 ] 0:27:27.896460\n",
      "[ 2500 ] 0:30:01.367123\n",
      "[ 3000 ] 0:32:34.534325\n",
      "[ 3500 ] 0:35:05.583873\n",
      "[ 4000 ] 0:37:38.314485\n",
      "[ 4500 ] 0:40:23.299780\n",
      "[ 5000 ] 0:43:02.799612\n",
      "[ 5500 ] 0:45:48.744744\n",
      "[ 6000 ] 0:48:26.169399\n",
      "[ 6500 ] 0:51:10.574182\n",
      "[ 7000 ] 0:54:10.393760\n",
      "[ 7500 ] 0:57:05.023494\n",
      "[ 8000 ] 1:00:00.804039\n",
      "[ 8500 ] 1:02:43.917919\n",
      "Dictionary building complete in -  1:04:14.831206\n",
      "Error Count 0\n",
      "=====[ code_summarization/test/code.original ]=====\n",
      "[ 0 ] 1:04:16.304373\n",
      "[ 500 ] 1:06:47.155864\n",
      "[ 1000 ] 1:09:41.348292\n",
      "[ 1500 ] 1:12:33.855576\n",
      "[ 2000 ] 1:15:19.036959\n",
      "[ 2500 ] 1:18:03.088125\n",
      "[ 3000 ] 1:20:49.363653\n",
      "[ 3500 ] 1:23:34.984649\n",
      "[ 4000 ] 1:26:15.692592\n",
      "[ 4500 ] 1:28:57.781763\n",
      "[ 5000 ] 1:31:40.595355\n",
      "[ 5500 ] 1:34:15.282945\n",
      "[ 6000 ] 1:36:46.284227\n",
      "[ 6500 ] 1:39:17.940677\n",
      "[ 7000 ] 1:41:51.258433\n",
      "[ 7500 ] 1:44:22.820012\n",
      "[ 8000 ] 1:46:53.144747\n",
      "[ 8500 ] 1:49:24.495547\n",
      "Dictionary building complete in -  1:50:54.197764\n",
      "Error Count 0\n",
      "=====[ code_summarization/train/code.original ]=====\n",
      "[ 0 ] 1:50:54.535266\n",
      "[ 500 ] 1:53:39.098343\n",
      "[ 1000 ] 1:56:12.538997\n",
      "[ 1500 ] 1:58:48.409316\n",
      "[ 2000 ] 2:01:29.528035\n",
      "[ 2500 ] 2:04:00.483893\n",
      "[ 3000 ] 2:06:30.733494\n",
      "[ 3500 ] 2:09:00.534850\n",
      "[ 4000 ] 2:11:32.830651\n",
      "[ 4500 ] 2:14:05.551695\n",
      "[ 5000 ] 2:16:34.775156\n",
      "[ 5500 ] 2:19:06.158230\n",
      "[ 6000 ] 2:21:36.928970\n",
      "[ 6500 ] 2:24:05.844920\n",
      "[ 7000 ] 2:26:36.023642\n",
      "[ 7500 ] 2:29:05.767316\n",
      "[ 8000 ] 2:31:36.772362\n",
      "[ 8500 ] 2:34:08.364148\n",
      "[ 9000 ] 2:36:37.536330\n",
      "[ 9500 ] 2:39:08.031316\n",
      "[ 10000 ] 2:41:38.476527\n",
      "[ 10500 ] 2:44:06.976236\n",
      "[ 11000 ] 2:46:38.380015\n",
      "[ 11500 ] 2:49:08.331512\n",
      "[ 12000 ] 2:51:40.208145\n",
      "[ 12500 ] 2:54:15.660673\n",
      "[ 13000 ] 2:56:46.852844\n",
      "[ 13500 ] 2:59:25.078387\n",
      "[ 14000 ] 3:01:58.471886\n",
      "[ 14500 ] 3:04:30.102408\n",
      "[ 15000 ] 3:06:59.845767\n",
      "[ 15500 ] 3:09:30.687157\n",
      "[ 16000 ] 3:12:04.417616\n",
      "[ 16500 ] 3:14:41.665695\n",
      "[ 17000 ] 3:17:36.860477\n",
      "[ 17500 ] 3:20:30.733076\n",
      "[ 18000 ] 3:23:23.481695\n",
      "[ 18500 ] 3:26:14.938813\n",
      "[ 19000 ] 3:29:13.463435\n",
      "[ 19500 ] 3:32:13.056661\n",
      "[ 20000 ] 3:34:52.056543\n",
      "[ 20500 ] 3:37:27.248544\n",
      "[ 21000 ] 3:40:00.687049\n",
      "[ 21500 ] 3:42:33.083497\n",
      "[ 22000 ] 3:45:06.173374\n",
      "[ 22500 ] 3:47:37.342848\n",
      "[ 23000 ] 3:50:12.671410\n",
      "[ 23500 ] 3:52:48.910791\n",
      "[ 24000 ] 3:55:40.560442\n",
      "[ 24500 ] 3:58:15.638296\n",
      "[ 25000 ] 4:00:55.913944\n",
      "[ 25500 ] 4:03:34.562782\n",
      "[ 26000 ] 4:06:18.751146\n",
      "[ 26500 ] 4:08:53.735274\n",
      "[ 27000 ] 4:11:40.751798\n",
      "[ 27500 ] 4:14:20.234738\n",
      "[ 28000 ] 4:16:56.086516\n",
      "[ 28500 ] 4:19:30.713017\n",
      "[ 29000 ] 4:22:06.033658\n",
      "[ 29500 ] 4:24:41.046110\n",
      "[ 30000 ] 4:27:14.959174\n",
      "[ 30500 ] 4:29:53.147670\n",
      "[ 31000 ] 4:32:31.070360\n",
      "[ 31500 ] 4:35:03.129687\n",
      "[ 32000 ] 4:37:38.990574\n",
      "[ 32500 ] 4:40:14.863273\n",
      "[ 33000 ] 4:42:48.162779\n",
      "[ 33500 ] 4:45:21.099724\n",
      "[ 34000 ] 4:47:57.450233\n",
      "[ 34500 ] 4:50:31.749146\n",
      "[ 35000 ] 4:53:14.309409\n",
      "[ 35500 ] 4:55:58.004493\n",
      "[ 36000 ] 4:58:32.961805\n",
      "[ 36500 ] 5:01:30.383021\n",
      "[ 37000 ] 5:04:29.759218\n",
      "[ 37500 ] 5:07:22.165955\n",
      "[ 38000 ] 5:09:58.098722\n",
      "[ 38500 ] 5:12:31.628364\n",
      "[ 39000 ] 5:15:03.352609\n",
      "[ 39500 ] 5:17:36.610988\n",
      "[ 40000 ] 5:20:14.316662\n",
      "[ 40500 ] 5:22:47.249305\n",
      "[ 41000 ] 5:25:18.534342\n",
      "[ 41500 ] 5:27:48.141575\n",
      "[ 42000 ] 5:30:20.857998\n",
      "[ 42500 ] 5:32:51.601196\n",
      "[ 43000 ] 5:35:22.432460\n",
      "[ 43500 ] 5:38:21.752960\n",
      "[ 44000 ] 5:40:54.665771\n",
      "[ 44500 ] 5:43:28.856593\n",
      "[ 45000 ] 5:46:01.467310\n",
      "[ 45500 ] 5:48:34.136500\n",
      "[ 46000 ] 5:51:09.360579\n",
      "[ 46500 ] 5:53:49.968984\n",
      "[ 47000 ] 5:56:26.318463\n",
      "[ 47500 ] 5:58:59.604495\n",
      "[ 48000 ] 6:01:40.250471\n",
      "[ 48500 ] 6:04:32.650636\n",
      "[ 49000 ] 6:07:15.317615\n",
      "[ 49500 ] 6:34:49.786941\n",
      "[ 50000 ] 6:37:42.512520\n",
      "[ 50500 ] 6:40:20.208495\n",
      "[ 51000 ] 6:43:01.129118\n",
      "[ 51500 ] 6:45:46.885603\n",
      "[ 52000 ] 6:48:29.459586\n",
      "[ 52500 ] 6:51:14.058739\n",
      "[ 53000 ] 6:53:53.709198\n",
      "[ 53500 ] 6:56:32.154278\n",
      "[ 54000 ] 7:05:31.340035\n",
      "[ 54500 ] 7:10:56.853612\n",
      "[ 55000 ] 7:13:37.426823\n",
      "[ 55500 ] 7:16:15.502345\n",
      "[ 56000 ] 7:18:50.970118\n",
      "[ 56500 ] 7:21:22.243531\n",
      "[ 57000 ] 7:23:53.447894\n",
      "[ 57500 ] 7:26:23.701323\n",
      "[ 58000 ] 7:28:55.791131\n",
      "[ 58500 ] 7:31:29.552979\n",
      "[ 59000 ] 7:34:02.036529\n",
      "[ 59500 ] 7:36:33.902278\n",
      "[ 60000 ] 7:39:05.156297\n",
      "[ 60500 ] 7:41:37.391433\n",
      "[ 61000 ] 7:44:07.623768\n",
      "[ 61500 ] 7:46:36.804062\n",
      "[ 62000 ] 7:49:08.195370\n",
      "[ 62500 ] 7:51:39.320892\n",
      "[ 63000 ] 7:54:12.248795\n",
      "[ 63500 ] 7:56:53.084204\n",
      "[ 64000 ] 7:59:25.221603\n",
      "[ 64500 ] 8:01:58.922694\n",
      "[ 65000 ] 8:04:29.967759\n",
      "[ 65500 ] 8:07:02.793499\n",
      "[ 66000 ] 8:09:36.112826\n",
      "[ 66500 ] 8:12:09.351331\n",
      "[ 67000 ] 8:14:39.928150\n",
      "[ 67500 ] 8:23:00.544744\n",
      "[ 68000 ] 8:26:15.697033\n",
      "[ 68500 ] 8:29:04.768638\n",
      "[ 69000 ] 8:31:52.482740\n",
      "[ 69500 ] 8:34:39.379890\n",
      "Dictionary building complete in -  8:42:46.285094\n",
      "Error Count 0\n"
     ]
    }
   ],
   "source": [
    "import datetime\n",
    "\n",
    "#path = 'code_summarization/train/code.original'\n",
    "for path in paths:\n",
    "    start = datetime.datetime.now()\n",
    "    print(\"=====[\", path,\"]=====\")\n",
    "    i = 0\n",
    "    error_count = 0\n",
    "    parse_trees = []\n",
    "    codes = []\n",
    "    with open(path) as data:\n",
    "        for raw_code in data:\n",
    "            try:\n",
    "                end = datetime.datetime.now()\n",
    "                if i%500 == 0: print(\"[\", i, \"]\", end-start)\n",
    "                raw_code = raw_code.replace(\"STRING\", \"\\\"STRING\\\"\").replace(\"_NUM\", \"1\").replace(\"_BOOL\", \"true\")\n",
    "                tree = parse(raw_code)\n",
    "                code = tree2code(tree, -1)[0]\n",
    "            except Exception as e:\n",
    "                print(e)\n",
    "                print(\"Line:\",i,\"is corrupted. Cannot build dictionary.\")\n",
    "                error_count += 1\n",
    "                tree = \"{}\"\n",
    "                code = \"\"\n",
    "            parse_trees.append(tree)\n",
    "            codes.append({\"code\": code})\n",
    "            i+=1\n",
    "\n",
    "    dump_dict(parse_trees, path.replace(\"original\", \"tree\"))\n",
    "    dump_dict(codes, path.replace(\"original\", \"noliteral\"))\n",
    "    end = datetime.datetime.now()\n",
    "    print(\"Dictionary building complete in - \",end-start)\n",
    "    print(\"Error Count\", error_count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}