eacl-demo.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# Numerical Fact Checking System. Univeristy of Sheffield\n",
    "\n",
    "Pre-requisites:\n",
    " * Gradle\n",
    " * Java jdk8\n",
    " * Python 3\n",
    "  * numpy\n",
    "  * jnius\n",
    "  * fuzzywuzzy\n",
    "  * sklearn\n",
    "  * urllib3\n",
    " "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Configuration\n",
    "This defines the colleciton of tables that is used to populate the knowledge base"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "world = \"herox\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Common setup\n",
    "\n",
    "Import required dependencies and download/install Stanford CoreNLP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import sys\n",
    "import os\n",
    "import re\n",
    "\n",
    "#Set path manually to incldue sources location\n",
    "if 'src/' not in sys.path:\n",
    "    sys.path.append('src/')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "If the following step fails. Run `gradlew writeClasspath` on the terminal in this folder. Then try again"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading classpath\n",
      "Done\n"
     ]
    }
   ],
   "source": [
    "#Load Java classpath for stanford corenlp using gradle. this will also install it if missing\n",
    "from subprocess import run,PIPE\n",
    "if 'CLASSPATH' not in os.environ:\n",
    "    if not (os.path.exists('build') and os.path.exists('build/classpath.txt')):\n",
    "        print(\"Generating classpath\")\n",
    "        r=run([\"./gradlew\", \"writeClasspath\"],stdout=PIPE, stderr=PIPE, universal_newlines=True)\n",
    "        print(r.stdout)\n",
    "        print(r.stderr)\n",
    "              \n",
    "    print(\"Loading classpath\")\n",
    "    os.environ['CLASSPATH'] = open('build/classpath.txt','r').read()\n",
    "    print(\"Done\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Feature Generation\n",
    "\n",
    "For each of the downloaded web pages. Parse the page and identify matches between the values in our tables and the data given in the web page. This only needs to be run once and will rememeber if it has been run before"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/james/anaconda/lib/python3.5/site-packages/fuzzywuzzy/fuzz.py:35: UserWarning: Using slow pure-python SequenceMatcher. Install python-Levenshtein to remove this warning\n",
      "  warnings.warn('Using slow pure-python SequenceMatcher. Install python-Levenshtein to remove this warning')\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done 1 out of 28377\n",
      "Search for \"Uzbekistan\" UZB  1960\n",
      "Query already executed\n",
      "Done URL 1 out of 31\n",
      " \n",
      "Looking in document for values similar to 58.88553659\n",
      "https://www.bing.com/cr?IG=E11C3CC6BEE6457D9B9B886407C50D19&CID=1F113AB2565A6C9E3ED930E457BD6D96&rd=1&h=Efbiv6l4Y1Wuqhtd0jnW2MZ94dV9LN0ZfrGB4th8sdo&v=1&r=https%3a%2f%2fweatherspark.com%2fhistory%2f33885%2f1960%2fTashkent-Tashkent-Province-Uzbekistan&p=DevEx,5085.1\n",
      "Done URL 2 out of 31\n",
      " \n",
      "Looking in document for values similar to 58.88553659\n",
      "https://www.bing.com/cr?IG=E11C3CC6BEE6457D9B9B886407C50D19&CID=1F113AB2565A6C9E3ED930E457BD6D96&rd=1&h=uz0sTYz94IlloU3bdWGIeuVc23AxziL-LA9pyEvXNNI&v=1&r=https%3a%2f%2fweatherspark.com%2fhistory%2f33883%2f1960%2fTermez-Surkhandarya-Province-Uzbekistan&p=DevEx,5096.1\n",
      "Done URL 3 out of 31\n",
      " \n",
      "Looking in document for values similar to 58.88553659\n",
      "http://www.bing.com/cr?IG=E11C3CC6BEE6457D9B9B886407C50D19&CID=1F113AB2565A6C9E3ED930E457BD6D96&rd=1&h=WZasDwW2hOJb_eALluSyV7VGbTJoHXE5Or7er9PGj7M&v=1&r=http%3a%2f%2fwww.microsoft.com%2fen-gb%2fstore%2fmusic%2falbum%2fvarious-artists%2fmusic-of-uzbekistan-from-78-rpm-recordings-1932-19%2f8d6kgwzxncdf&p=DevEx,5110.1\n",
      "No meaningful text in this document\n",
      "Done URL 4 out of 31\n",
      " \n",
      "Looking in document for values similar to 58.88553659\n",
      "https://www.bing.com/cr?IG=E11C3CC6BEE6457D9B9B886407C50D19&CID=1F113AB2565A6C9E3ED930E457BD6D96&rd=1&h=EKbWXZRAo2aPfXJ3wV88YrwVCHUghd-ucBMP0aV0rmo&v=1&r=https%3a%2f%2fen.wikipedia.org%2fwiki%2fAral_sea&p=DevEx,5130.1\n",
      "Done URL 5 out of 31\n",
      " \n",
      "Looking in document for values similar to 58.88553659\n",
      "http://www.bing.com/cr?IG=E11C3CC6BEE6457D9B9B886407C50D19&CID=1F113AB2565A6C9E3ED930E457BD6D96&rd=1&h=Ea_qu2DKZSAjbddEwvSbR4B4NwIoAPlGiTP9f_e7f3Y&v=1&r=http%3a%2f%2fwww.tradingeconomics.com%2fuzbekistan%2fpopulation&p=DevEx,5144.1\n",
      "No meaningful text in this document\n",
      "Done URL 6 out of 31\n",
      " \n",
      "Looking in document for values similar to 58.88553659\n",
      "https://www.bing.com/cr?IG=E11C3CC6BEE6457D9B9B886407C50D19&CID=1F113AB2565A6C9E3ED930E457BD6D96&rd=1&h=13jpEWb7c2J7-t3cLJIhmKm1kHrzRBxfh2IuIbMKnXo&v=1&r=https%3a%2f%2fen.wikipedia.org%2fwiki%2fCategory%3a1960s_in_Uzbekistan&p=DevEx,5155.1\n",
      "No meaningful text in this document\n",
      "Done URL 7 out of 31\n",
      " \n",
      "Looking in document for values similar to 58.88553659\n",
      "http://www.bing.com/cr?IG=E11C3CC6BEE6457D9B9B886407C50D19&CID=1F113AB2565A6C9E3ED930E457BD6D96&rd=1&h=xcG9ahHfjbQeBzjvU-Y2Dd8IpQQ2Yddq1bDHlNFSmRI&v=1&r=http%3a%2f%2fpopulationpyramid.net%2fuzbekistan%2f1960%2f&p=DevEx,5168.1\n",
      "No meaningful text in this document\n",
      "Done URL 8 out of 31\n",
      " \n",
      "Looking in document for values similar to 58.88553659\n",
      "http://www.bing.com/cr?IG=E11C3CC6BEE6457D9B9B886407C50D19&CID=1F113AB2565A6C9E3ED930E457BD6D96&rd=1&h=B3M1tBfT96YfiaCoEb3wVOJHZM1W91VJcTrAQ9bdy_0&v=1&r=http%3a%2f%2fslideplayer.com%2fslide%2f10383208%2f&p=DevEx,5179.1\n",
      "Done URL 9 out of 31\n",
      " \n",
      "Looking in document for values similar to 58.88553659\n",
      "http://www.bing.com/cr?IG=E11C3CC6BEE6457D9B9B886407C50D19&CID=1F113AB2565A6C9E3ED930E457BD6D96&rd=1&h=XhtGXqQVKWIFwhxj3lRjAy0v3V4QLHm_dxLRl5xcFzQ&v=1&r=http%3a%2f%2fatlas.media.mit.edu%2fen%2fprofile%2fcountry%2fuzb%2f&p=DevEx,5193.1\n",
      "Done URL 10 out of 31\n",
      " \n",
      "Looking in document for values similar to 58.88553659\n",
      "https://www.bing.com/cr?IG=E11C3CC6BEE6457D9B9B886407C50D19&CID=1F113AB2565A6C9E3ED930E457BD6D96&rd=1&h=WIi7x9hB8pfAFbSicPMeu3eut0BEESaJBM6Jrf7tHLg&v=1&r=https%3a%2f%2fwww.jw.org%2fen%2fnews%2flegal%2fby-region%2fuzbekistan%2fHistoric-Milestone-in-Uzbekistan-1960s%2f&p=DevEx,5205.1\n",
      "No meaningful text in this document\n",
      "Done URL 11 out of 31\n",
      " \n",
      "Looking in document for values similar to 58.88553659\n",
      "http://www.bing.com/cr?IG=E11C3CC6BEE6457D9B9B886407C50D19&CID=1F113AB2565A6C9E3ED930E457BD6D96&rd=1&h=JAPstTUFjPqLaYaxbiujZxdwlIRCEl-QOK6xOx9Fz-c&v=1&r=http%3a%2f%2fwww.uzbek-craft.com%2fen%2f16-suzani&p=DevEx,5218.1\n",
      "No meaningful text in this document\n",
      "Done URL 12 out of 31\n",
      " \n",
      "Looking in document for values similar to 58.88553659\n",
      "http://www.bing.com/cr?IG=E11C3CC6BEE6457D9B9B886407C50D19&CID=1F113AB2565A6C9E3ED930E457BD6D96&rd=1&h=wC_Ufd2BcAFgh4eudzvQ1edcxB-9VeKxD4tCP-dj0Mw&v=1&r=http%3a%2f%2fwww.the-saleroom.com%2fen-gb%2fauction-catalogues%2fwannenes%2fcatalogue-id-wannen10005%2flot-76810c8d-974b-4b77-8f07-a55600e4259c&p=DevEx,5230.1\n",
      "No meaningful text in this document\n",
      "Done URL 13 out of 31\n",
      " \n",
      "Looking in document for values similar to 58.88553659\n",
      "http://www.bing.com/cr?IG=E11C3CC6BEE6457D9B9B886407C50D19&CID=1F113AB2565A6C9E3ED930E457BD6D96&rd=1&h=mo4kuQ9Wf7bQOcW0s8iBTC9ZmPkkLjJJ-_f_l4-aQIw&v=1&r=http%3a%2f%2fwww.susanmeller.com%2fstore%2fproducts%2fnapramach-bag-face-atsbnb-118-uzbekistan-c-1960s-80s%2f&p=DevEx,5241.1\n",
      "Done URL 14 out of 31\n",
      " \n",
      "Looking in document for values similar to 58.88553659\n",
      "http://www.bing.com/cr?IG=E11C3CC6BEE6457D9B9B886407C50D19&CID=1F113AB2565A6C9E3ED930E457BD6D96&rd=1&h=TH9182lgBh37F5qgXoDtwGRyYBY17cIxbSW3wi1hqyg&v=1&r=http%3a%2f%2fwww.columbia.edu%2f%7etmt2120%2fintroduction.htm&p=DevEx,5254.1\n",
      "Done URL 15 out of 31\n",
      " \n",
      "Looking in document for values similar to 58.88553659\n",
      "http://www.bing.com/cr?IG=E11C3CC6BEE6457D9B9B886407C50D19&CID=1F113AB2565A6C9E3ED930E457BD6D96&rd=1&h=uhiHhY15b0EgSOovm_XaHFiJMYS5DKdNVSf6jnk-eQs&v=1&r=http%3a%2f%2fwww.digplanet.com%2fwiki%2fCategory%3a1960_in_Uzbekistan&p=DevEx,5265.1\n",
      "No meaningful text in this document\n",
      "Done URL 16 out of 31\n",
      " \n",
      "Looking in document for values similar to 58.88553659\n",
      "http://www.bing.com/cr?IG=E11C3CC6BEE6457D9B9B886407C50D19&CID=1F113AB2565A6C9E3ED930E457BD6D96&rd=1&h=Z98v55ucUSayJTmxXYCl1MMwBEibKXVbzy7PpKWMEiM&v=1&r=http%3a%2f%2fwww.amusingplanet.com%2f2012%2f06%2fdisappearing-aral-lake-and-ship.html&p=DevEx,5279.1\n",
      "Done URL 17 out of 31\n",
      " \n",
      "Looking in document for values similar to 58.88553659\n",
      "http://www.bing.com/cr?IG=E11C3CC6BEE6457D9B9B886407C50D19&CID=1F113AB2565A6C9E3ED930E457BD6D96&rd=1&h=OuXaKKcqAP2oTAjIiRjBsQLx-MN4xvMXBWgH1UCX1Dw&v=1&r=http%3a%2f%2fwww.ebay.com%2fsch%2fi.html%3f_nkw%3duzbek%2bjewelry&p=DevEx,5294.1\n",
      "No meaningful text in this document\n",
      "Done URL 18 out of 31\n",
      " \n",
      "Looking in document for values similar to 58.88553659\n",
      "http://www.bing.com/cr?IG=E11C3CC6BEE6457D9B9B886407C50D19&CID=1F113AB2565A6C9E3ED930E457BD6D96&rd=1&h=tGx-5j0t4tGoDKrGg4Jr2Wd7ZoIA3e0zJlEhs8YZICE&v=1&r=http%3a%2f%2fwww.worldcat.org%2ftitle%2fuzbekistan-a-soviet-state%2foclc%2f4929412&p=DevEx,5306.1\n",
      "No meaningful text in this document\n",
      "Done URL 19 out of 31\n",
      " \n",
      "Looking in document for values similar to 58.88553659\n",
      "https://www.bing.com/cr?IG=E11C3CC6BEE6457D9B9B886407C50D19&CID=1F113AB2565A6C9E3ED930E457BD6D96&rd=1&h=oK-fLRmdGLzSOnKDCpZpDlYiQvqZMx2mlU7FiVB8VJE&v=1&r=https%3a%2f%2fearthobservatory.nasa.gov%2fFeatures%2fWorldOfChange%2faral_sea.php&p=DevEx,5318.1\n",
      "Done URL 20 out of 31\n",
      " \n",
      "Looking in document for values similar to 58.88553659\n",
      "http://www.bing.com/cr?IG=E11C3CC6BEE6457D9B9B886407C50D19&CID=1F113AB2565A6C9E3ED930E457BD6D96&rd=1&h=ScxuDw0ya-jYyF8CTtE0q-oRrHW73FwAftO5TNDj7dQ&v=1&r=http%3a%2f%2fwww.susanmeller.com%2fstore%2fproducts%2fsegusha-atbbnb-102-uzbekistan-c-1960s%2f&p=DevEx,5330.1\n",
      "Done URL 21 out of 31\n",
      " \n",
      "Looking in document for values similar to 58.88553659\n",
      "http://www.bing.com/cr?IG=E11C3CC6BEE6457D9B9B886407C50D19&CID=1F113AB2565A6C9E3ED930E457BD6D96&rd=1&h=4v2BqaNKpfhu1bgQ_LJuZbD8p4uRguKD5NjB0aC9YhA&v=1&r=http%3a%2f%2fschools-wikipedia.org%2fwp%2fa%2fAral_Sea.htm&p=DevEx,5343.1\n",
      "Done URL 22 out of 31\n",
      " \n",
      "Looking in document for values similar to 58.88553659\n",
      "http://www.bing.com/cr?IG=E11C3CC6BEE6457D9B9B886407C50D19&CID=1F113AB2565A6C9E3ED930E457BD6D96&rd=1&h=h-HEttqZNhhuMsD8qxivifoD-Petmh_s_XMmc6Qmv40&v=1&r=http%3a%2f%2fwww.rferl.org%2fa%2fuzbekistan-film-industry-quantity-versus-quality%2f24745666.html&p=DevEx,5356.1\n",
      "Done URL 23 out of 31\n",
      " \n",
      "Looking in document for values similar to 58.88553659\n",
      "https://www.bing.com/cr?IG=E11C3CC6BEE6457D9B9B886407C50D19&CID=1F113AB2565A6C9E3ED930E457BD6D96&rd=1&h=35EtM7chptV7icnCbVm7I8ExrYK-oP19a23xHU4f5Gk&v=1&r=https%3a%2f%2fwww.liveauctioneers.com%2fitem%2f42819402_decoro-lakai-uzbekistan-asia-centrale-1960-circa&p=DevEx,5368.1\n",
      "No meaningful text in this document\n",
      "Done URL 24 out of 31\n",
      " \n",
      "Looking in document for values similar to 58.88553659\n",
      "https://www.bing.com/cr?IG=E11C3CC6BEE6457D9B9B886407C50D19&CID=1F113AB2565A6C9E3ED930E457BD6D96&rd=1&h=TdE5TGZWX_QHEHCj2dn7u1vn7NwJOr9P9fGw6qJiwoc&v=1&r=https%3a%2f%2fwww.olympic.org%2fuzbekistan&p=DevEx,5380.1\n",
      "No meaningful text in this document\n",
      "Done URL 25 out of 31\n",
      " \n",
      "Looking in document for values similar to 58.88553659\n",
      "https://www.bing.com/cr?IG=E11C3CC6BEE6457D9B9B886407C50D19&CID=1F113AB2565A6C9E3ED930E457BD6D96&rd=1&h=n7Z1mepZFf2KcFMjQZ66kzSZ2ehtXXznvoLPwh_xPl4&v=1&r=https%3a%2f%2fknoema.com%2fatlas%2fUzbekistan%2fBirth-rate&p=DevEx,5396.1\n",
      "Done URL 26 out of 31\n",
      " \n",
      "Looking in document for values similar to 58.88553659\n",
      "http://www.bing.com/cr?IG=E11C3CC6BEE6457D9B9B886407C50D19&CID=1F113AB2565A6C9E3ED930E457BD6D96&rd=1&h=V0vQ1yRcFXrUFxTWXzXyZEXzO8yJzlV9iJhFaoEMCEc&v=1&r=http%3a%2f%2fwww.livescience.com%2f48034-shrinking-aral-sea.html&p=DevEx,5410.1\n",
      "Done URL 27 out of 31\n",
      " \n",
      "Looking in document for values similar to 58.88553659\n",
      "https://www.bing.com/cr?IG=E11C3CC6BEE6457D9B9B886407C50D19&CID=1F113AB2565A6C9E3ED930E457BD6D96&rd=1&h=xdLjpzJyF-wsmhi6v4eAt_CAq2ks2FJeGE9Kt8VnE6Y&v=1&r=https%3a%2f%2fknoema.com%2fWBPEP2014%2fpopulation-estimates-and-projections-1960-2050%3fcountry%3d1002060-uzbekistan&p=DevEx,5425.1\n",
      "Done URL 28 out of 31\n",
      " \n",
      "Looking in document for values similar to 58.88553659\n",
      "http://www.bing.com/cr?IG=E11C3CC6BEE6457D9B9B886407C50D19&CID=1F113AB2565A6C9E3ED930E457BD6D96&rd=1&h=OkGg_Wb0BrqQdL73bdaybzLGHHS67sPhaFLuIXMydfQ&v=1&r=http%3a%2f%2fwww.worldcat.org%2ftitle%2fuzbekistan-another-big-leap-forward%2foclc%2f8156322&p=DevEx,5437.1\n",
      "No meaningful text in this document\n",
      "Done URL 29 out of 31\n",
      " \n",
      "Looking in document for values similar to 58.88553659\n",
      "https://www.bing.com/cr?IG=E11C3CC6BEE6457D9B9B886407C50D19&CID=1F113AB2565A6C9E3ED930E457BD6D96&rd=1&h=GATrTCJ5JzJ1vr2o6dcha3vMxlc_MJRRv7NNEYnWN8M&v=1&r=https%3a%2f%2fwww.the-saleroom.com%2fde-de%2fauction-catalogues%2fwannenes%2fcatalogue-id-wannen10005%2flot-76810c8d-974b-4b77-8f07-a55600e4259c&p=DevEx,5448.1\n",
      "No meaningful text in this document\n",
      "Done URL 30 out of 31\n",
      " \n",
      "Looking in document for values similar to 58.88553659\n",
      "https://www.bing.com/cr?IG=E11C3CC6BEE6457D9B9B886407C50D19&CID=1F113AB2565A6C9E3ED930E457BD6D96&rd=1&h=5RcRJiXkhlB_2FBwyefIb4_SVOazXbQi8CIqHEw8xjM&v=1&r=https%3a%2f%2fwww.pinterest.com%2fpin%2f533606255834797863%2f&p=DevEx,5460.1\n",
      "No meaningful text in this document\n",
      "Done URL 31 out of 31\n",
      " \n",
      "Looking in document for values similar to 58.88553659\n",
      "http://www.bing.com/cr?IG=E11C3CC6BEE6457D9B9B886407C50D19&CID=1F113AB2565A6C9E3ED930E457BD6D96&rd=1&h=llXKIN5N6BOzowTjLf91J3G6lmn1IeYMXXlmj9Eus_E&v=1&r=http%3a%2f%2fu16201083.onlinehome-server.com%2fcurrency-converter%2farmenian-dram-amd_uzs-uzbekistan-sum.htm%2f1960&p=DevEx,5471.1\n",
      "Done 2 out of 28377\n",
      "Search for \"Estonia\" EST  1968\n",
      "Query already executed\n",
      "Done URL 1 out of 32\n",
      " \n",
      "Looking in document for values similar to 69.88292683\n",
      "https://www.bing.com/cr?IG=F0FF4FA097964A958355FFEE4F4BCC67&CID=2FC4F4355AFD66FC1B8CFE635B1A6766&rd=1&h=BEShXvdjX3XnDCs5zoFb64l-y5GuSpTMaopvLNVIwrU&v=1&r=https%3a%2f%2fen.wikipedia.org%2fwiki%2fEstonia&p=DevEx,5104.1\n",
      "Done URL 2 out of 32\n",
      " \n",
      "Looking in document for values similar to 69.88292683\n",
      "https://www.bing.com/cr?IG=F0FF4FA097964A958355FFEE4F4BCC67&CID=2FC4F4355AFD66FC1B8CFE635B1A6766&rd=1&h=vtC2dNkwMi9kqdu8ShhbvZXEyUewukvm_R88K54eqck&v=1&r=https%3a%2f%2fen.wikipedia.org%2fwiki%2fTallinn&p=DevEx,5127.1\n",
      "Done URL 3 out of 32\n",
      " \n",
      "Looking in document for values similar to 69.88292683\n",
      "http://www.bing.com/cr?IG=F0FF4FA097964A958355FFEE4F4BCC67&CID=2FC4F4355AFD66FC1B8CFE635B1A6766&rd=1&h=uIQt_n7oEzL2pPxgsZnV5ppIv1odlT1gR_TEP3jELkY&v=1&r=http%3a%2f%2fdtc.jrc.ec.europa.eu%2fdtc_nation_codes.php&p=DevEx,5143.1\n",
      "Done URL 4 out of 32\n",
      " \n",
      "Looking in document for values similar to 69.88292683\n",
      "http://www.bing.com/cr?IG=F0FF4FA097964A958355FFEE4F4BCC67&CID=2FC4F4355AFD66FC1B8CFE635B1A6766&rd=1&h=Ll_tISaObFiIaMxMK07CkETde9Pjy2qmFUgKWaj0fcc&v=1&r=http%3a%2f%2fwww.ebay.co.uk%2fsch%2fi.html%3f_nkw%3dEST&p=DevEx,5157.1\n",
      "No meaningful text in this document\n",
      "Done URL 5 out of 32\n",
      " \n",
      "Looking in document for values similar to 69.88292683\n",
      "https://www.bing.com/cr?IG=F0FF4FA097964A958355FFEE4F4BCC67&CID=2FC4F4355AFD66FC1B8CFE635B1A6766&rd=1&h=fE_FSyAK_ANU0MJEqPNfbnhfDll2PRunFqw1GDqIq2k&v=1&r=https%3a%2f%2flink.springer.com%2farticle%2f10.1007%2fBF00810133&p=DevEx,5168.1\n",
      "Done URL 6 out of 32\n",
      " \n",
      "Looking in document for values similar to 69.88292683\n",
      "https://www.bing.com/cr?IG=F0FF4FA097964A958355FFEE4F4BCC67&CID=2FC4F4355AFD66FC1B8CFE635B1A6766&rd=1&h=q9FCPxIQ360sgr1Is_zv_S4ZvCKjF9BJt_3XS-9d6sk&v=1&r=https%3a%2f%2fwww.teetkask.com%2f&p=DevEx,5180.1\n",
      "Done URL 7 out of 32\n",
      " \n",
      "Looking in document for values similar to 69.88292683\n",
      "http://www.bing.com/cr?IG=F0FF4FA097964A958355FFEE4F4BCC67&CID=2FC4F4355AFD66FC1B8CFE635B1A6766&rd=1&h=qBEEj253yd09bxzfSGkm58A-SH1k6S0iHTkcePsrQ-4&v=1&r=http%3a%2f%2festarecovery.com%2f&p=DevEx,5192.1\n",
      "No meaningful text in this document\n",
      "Done URL 8 out of 32\n",
      " \n",
      "Looking in document for values similar to 69.88292683\n",
      "http://www.bing.com/cr?IG=F0FF4FA097964A958355FFEE4F4BCC67&CID=2FC4F4355AFD66FC1B8CFE635B1A6766&rd=1&h=pkklVaFeg9ICYHCfkSntC-vu0HPA2ts_GP-32jxsFu4&v=1&r=http%3a%2f%2fearth-info.nga.mil%2fGandG%2fcoordsys%2fonlinedatum%2fDatumTable.html&p=DevEx,5204.1\n",
      "No meaningful text in this document\n",
      "Done URL 9 out of 32\n",
      " \n",
      "Looking in document for values similar to 69.88292683\n",
      "https://www.bing.com/cr?IG=F0FF4FA097964A958355FFEE4F4BCC67&CID=2FC4F4355AFD66FC1B8CFE635B1A6766&rd=1&h=Je0TWO00ivNtI6aePVCatWXh9nWS0V4N01lzFg4aDrc&v=1&r=https%3a%2f%2fwww.esta-registration.co.uk%2festa-application.html&p=DevEx,5216.1\n",
      "No meaningful text in this document\n",
      "Done URL 10 out of 32\n",
      " \n",
      "Looking in document for values similar to 69.88292683\n",
      "http://www.bing.com/cr?IG=F0FF4FA097964A958355FFEE4F4BCC67&CID=2FC4F4355AFD66FC1B8CFE635B1A6766&rd=1&h=HTjD_2MWsWGAGvdGH9CTRj_tC-vrHxUSIlO0xLzqFTs&v=1&r=http%3a%2f%2fwww.esta-express-usa.com%2f&p=DevEx,5228.1\n",
      "No meaningful text in this document\n",
      "Done URL 11 out of 32\n",
      " \n",
      "Looking in document for values similar to 69.88292683\n",
      "https://www.bing.com/cr?IG=F0FF4FA097964A958355FFEE4F4BCC67&CID=2FC4F4355AFD66FC1B8CFE635B1A6766&rd=1&h=tzfjLzjsPCIo06r0Zvf0u1i2VUPgRNOZRS8gPlJqm6k&v=1&r=https%3a%2f%2fwww.olympic.org%2festonia&p=DevEx,5240.1\n",
      "No meaningful text in this document\n",
      "Done URL 12 out of 32\n",
      " \n",
      "Looking in document for values similar to 69.88292683\n",
      "http://www.bing.com/cr?IG=F0FF4FA097964A958355FFEE4F4BCC67&CID=2FC4F4355AFD66FC1B8CFE635B1A6766&rd=1&h=n_IjC-GqozrtaLCqlDSAIsVSUVWWfsSuWhQg78C0FH0&v=1&r=http%3a%2f%2fwww.olimpbase.org%2fteams%2fest_tea.html&p=DevEx,5253.1\n",
      "No meaningful text in this document\n",
      "Done URL 13 out of 32\n",
      " \n",
      "Looking in document for values similar to 69.88292683\n",
      "http://www.bing.com/cr?IG=F0FF4FA097964A958355FFEE4F4BCC67&CID=2FC4F4355AFD66FC1B8CFE635B1A6766&rd=1&h=9V04w72fyEjMkw-MHx6OrTGxy3H0QcGyRGCQfnSW-FA&v=1&r=http%3a%2f%2ftartutriatlon.ee%2fen%2fevent-info%2f&p=DevEx,5265.1\n",
      "Done URL 14 out of 32\n",
      " \n",
      "Looking in document for values similar to 69.88292683\n",
      "http://www.bing.com/cr?IG=F0FF4FA097964A958355FFEE4F4BCC67&CID=2FC4F4355AFD66FC1B8CFE635B1A6766&rd=1&h=S8WoV9CBKojx1tTEgpqVWvFRPRF06USL3a-IJPjKcPE&v=1&r=http%3a%2f%2fwww.youtube.com%2fwatch%3fv%3dgZnoq3BBRE4&p=DevEx,5292.1\n",
      "No meaningful text in this document\n",
      "Done URL 15 out of 32\n",
      " \n",
      "Looking in document for values similar to 69.88292683\n",
      "http://www.bing.com/cr?IG=F0FF4FA097964A958355FFEE4F4BCC67&CID=2FC4F4355AFD66FC1B8CFE635B1A6766&rd=1&h=C5yQNQKNl5ab5KmmPylJWmaoZpx7wXxRDF4Upa9vkOQ&v=1&r=http%3a%2f%2ffie.org%2fresults-statistic%2fresult&p=DevEx,5305.1\n",
      "No meaningful text in this document\n",
      "Done URL 16 out of 32\n",
      " \n",
      "Looking in document for values similar to 69.88292683\n",
      "http://www.bing.com/cr?IG=F0FF4FA097964A958355FFEE4F4BCC67&CID=2FC4F4355AFD66FC1B8CFE635B1A6766&rd=1&h=fFNgW01m9_6CHWcolD_a_g62EWfJYm0DFCrUpuIY13Q&v=1&r=http%3a%2f%2fwww.crwflags.com%2ffotw%2fflags%2fee-histo.html&p=DevEx,5319.1\n",
      "Done URL 17 out of 32\n",
      " \n",
      "Looking in document for values similar to 69.88292683\n",
      "http://www.bing.com/cr?IG=F0FF4FA097964A958355FFEE4F4BCC67&CID=2FC4F4355AFD66FC1B8CFE635B1A6766&rd=1&h=UFAW6Pok86YyO7Zbf3_q23DFCAc4hpAlZ1d4eSUVO0M&v=1&r=http%3a%2f%2fwww.ebay.com%2fitm%2fNWT-Men-039-s-Size-Medium-MLB-Oakland-Athletics-Est-1968-Shirt-%2f172596459513&p=DevEx,5338.1\n",
      "No meaningful text in this document\n",
      "Done URL 18 out of 32\n",
      " \n",
      "Looking in document for values similar to 69.88292683\n",
      "http://www.bing.com/cr?IG=F0FF4FA097964A958355FFEE4F4BCC67&CID=2FC4F4355AFD66FC1B8CFE635B1A6766&rd=1&h=MLEzMhTu56CBxOzSxGSIj0PphU0uo4ovaCqEE5wGS-I&v=1&r=http%3a%2f%2ffie.org%2fresults-statistic%2franking&p=DevEx,5351.1\n",
      "No meaningful text in this document\n",
      "Done URL 19 out of 32\n",
      " \n",
      "Looking in document for values similar to 69.88292683\n",
      "http://www.bing.com/cr?IG=F0FF4FA097964A958355FFEE4F4BCC67&CID=2FC4F4355AFD66FC1B8CFE635B1A6766&rd=1&h=V2K6Sj8-8m7JqsYsthiTW3WQHUpS4nPYzHRI4FhDU5k&v=1&r=http%3a%2f%2fwww.filateelia.ee%2fefur%2farticles%2fefur50.html&p=DevEx,5364.1\n",
      "Done URL 20 out of 32\n",
      " \n",
      "Looking in document for values similar to 69.88292683\n",
      "http://www.bing.com/cr?IG=F0FF4FA097964A958355FFEE4F4BCC67&CID=2FC4F4355AFD66FC1B8CFE635B1A6766&rd=1&h=pdv6-WqS7W3TNICue5rjH-wnjDP6bMHdKXOFZCOlTb4&v=1&r=http%3a%2f%2fwww.ebay.co.uk%2fsch%2fi.html%3f_nkw%3dbeatles%2bwhite%2balbum&p=DevEx,5380.1\n",
      "No meaningful text in this document\n",
      "Done URL 21 out of 32\n",
      " \n",
      "Looking in document for values similar to 69.88292683\n",
      "http://www.bing.com/cr?IG=F0FF4FA097964A958355FFEE4F4BCC67&CID=2FC4F4355AFD66FC1B8CFE635B1A6766&rd=1&h=Q0di1VWzPf26PUOjeWKdwx0EmyySmMeHtIwQ8Wglwrg&v=1&r=http%3a%2f%2fwww.youtube.com%2fuser%2feurovision&p=DevEx,5396.1\n",
      "Done URL 22 out of 32\n",
      " \n",
      "Looking in document for values similar to 69.88292683\n",
      "http://www.bing.com/cr?IG=F0FF4FA097964A958355FFEE4F4BCC67&CID=2FC4F4355AFD66FC1B8CFE635B1A6766&rd=1&h=SYCIWykmBgwMC0MZ8HsUYE-_4j97-1Vab0TN-fxPKtY&v=1&r=http%3a%2f%2fwww.sports-reference.com%2folympics%2fathletes%2fno%2fvoldemar-noormagi-1.html&p=DevEx,5413.1\n",
      "Done URL 23 out of 32\n",
      " \n",
      "Looking in document for values similar to 69.88292683\n",
      "https://www.bing.com/cr?IG=F0FF4FA097964A958355FFEE4F4BCC67&CID=2FC4F4355AFD66FC1B8CFE635B1A6766&rd=1&h=YEbciqrsTTnpZT39hXm7-RiXGSvU9krBn30rWByRTQ0&v=1&r=https%3a%2f%2festatour.com%2festa%2festa_search.php&p=DevEx,5426.1\n",
      "No meaningful text in this document\n",
      "Done URL 24 out of 32\n",
      " \n",
      "Looking in document for values similar to 69.88292683\n",
      "http://www.bing.com/cr?IG=F0FF4FA097964A958355FFEE4F4BCC67&CID=2FC4F4355AFD66FC1B8CFE635B1A6766&rd=1&h=ZkJ1kCUgtbR6g3afAaFNWiKtCTFyYJUo2DQVw9ZwLFo&v=1&r=http%3a%2f%2fwww.fifa.com%2fmm%2fdocument%2ffifafacts%2forganisation%2f52%2f00%2f10%2ffs-120_01a_mas.pdf&p=DevEx,5437.1\n",
      "No meaningful text in this document\n",
      "Done URL 25 out of 32\n",
      " \n",
      "Looking in document for values similar to 69.88292683\n",
      "http://www.bing.com/cr?IG=F0FF4FA097964A958355FFEE4F4BCC67&CID=2FC4F4355AFD66FC1B8CFE635B1A6766&rd=1&h=Ii7wmI6LnUiMidoSH_MdhK07263mMuVA_JgNXBPVrkc&v=1&r=http%3a%2f%2fwww.sports-reference.com%2folympics%2fathletes%2fvo%2fanzela-voronova-1.html&p=DevEx,5453.1\n",
      "No meaningful text in this document\n",
      "Done URL 26 out of 32\n",
      " \n",
      "Looking in document for values similar to 69.88292683\n",
      "https://www.bing.com/cr?IG=F0FF4FA097964A958355FFEE4F4BCC67&CID=2FC4F4355AFD66FC1B8CFE635B1A6766&rd=1&h=OnVUtYYtPNuolW1sOVNtApZO14LDRB8YX-U5AJvP6z0&v=1&r=https%3a%2f%2fgl.wikipedia.org%2fwiki%2fEstonia&p=DevEx,5472.1\n",
      "No meaningful text in this document\n",
      "Done URL 27 out of 32\n",
      " \n",
      "Looking in document for values similar to 69.88292683\n",
      "https://www.bing.com/cr?IG=F0FF4FA097964A958355FFEE4F4BCC67&CID=2FC4F4355AFD66FC1B8CFE635B1A6766&rd=1&h=YrJnqU1WJ3gw3QW7-6CXYCVht7Ci3D-KYrXnyUOC82Y&v=1&r=https%3a%2f%2ftreaties.un.org%2fPages%2fViewDetailsIII.aspx%3fsrc%3dIND%26mtdsg_no%3dXI-B-19%26chapter%3d11%26Temp%3dmtdsg3%26clang%3d_en&p=DevEx,5483.1\n",
      "Done URL 28 out of 32\n",
      " \n",
      "Looking in document for values similar to 69.88292683\n",
      "http://www.bing.com/cr?IG=F0FF4FA097964A958355FFEE4F4BCC67&CID=2FC4F4355AFD66FC1B8CFE635B1A6766&rd=1&h=we9WgO-mJlkr_yV0fafRDYSsoX9qjGHEyleACiWncL8&v=1&r=http%3a%2f%2fratings.fide.com%2ftopfed.phtml%3fina%3d1%26country%3dEST&p=DevEx,5495.1\n",
      "No meaningful text in this document\n",
      "Done URL 29 out of 32\n",
      " \n",
      "Looking in document for values similar to 69.88292683\n",
      "http://www.bing.com/cr?IG=F0FF4FA097964A958355FFEE4F4BCC67&CID=2FC4F4355AFD66FC1B8CFE635B1A6766&rd=1&h=Zga-LkjGoZ7Q53m-HxeahMdfn-nVIYb0aRPiBWwMang&v=1&r=http%3a%2f%2fapply-esta.info%2fapplyesta.php&p=DevEx,5509.1\n",
      "Done URL 30 out of 32\n",
      " \n",
      "Looking in document for values similar to 69.88292683\n",
      "https://www.bing.com/cr?IG=F0FF4FA097964A958355FFEE4F4BCC67&CID=2FC4F4355AFD66FC1B8CFE635B1A6766&rd=1&h=p6SZ7gwlz44n9K385tKyx5tJqLrKZ8NjHZ_EKcTuKg4&v=1&r=https%3a%2f%2ftwitter.com%2fCanadaEstonia&p=DevEx,5529.1\n",
      "Done URL 31 out of 32\n",
      " \n",
      "Looking in document for values similar to 69.88292683\n",
      "http://www.bing.com/cr?IG=F0FF4FA097964A958355FFEE4F4BCC67&CID=2FC4F4355AFD66FC1B8CFE635B1A6766&rd=1&h=sjnnjjtyJcoA-kzHz1aWqasqknDEx9Rv4bs6nPhzM54&v=1&r=http%3a%2f%2fwww.fei.org%2fevents&p=DevEx,5541.1\n",
      "No meaningful text in this document\n",
      "Done URL 32 out of 32\n",
      " \n",
      "Looking in document for values similar to 69.88292683\n",
      "http://www.bing.com/cr?IG=F0FF4FA097964A958355FFEE4F4BCC67&CID=2FC4F4355AFD66FC1B8CFE635B1A6766&rd=1&h=s4roReeHjPthM_a99aPixBoWBjGDi_Vg_3rZIvlLesU&v=1&r=http%3a%2f%2fwww.ebay.ca%2fsch%2fComics-%2f63%2fi.html%3f_nkw%3dinvincible%2biron%2bman%2b1968&p=DevEx,5555.1\n",
      "No meaningful text in this document\n",
      "Done 3 out of 28377\n",
      "Search for \"Malaysia\" Life expectancy at birth  1985\n",
      "Query already executed\n",
      "Done URL 1 out of 32\n",
      " \n",
      "Looking in document for values similar to 69.5007561\n",
      "http://www.bing.com/cr?IG=30CD706EAC0F4F0DA3AEF69555E1BCF9&CID=2F6AF5D6774565193D03FF8076A2647B&rd=1&h=tXJlvFIouwvsnmsbe4JIjIi9V5fzvf2DM6s7UPbveso&v=1&r=http%3a%2f%2fdata.worldbank.org%2findicator%2fSP.DYN.LE00.FE.IN&p=DevEx,5086.1\n",
      "No meaningful text in this document\n",
      "Done URL 2 out of 32\n",
      " \n",
      "Looking in document for values similar to 69.5007561\n",
      "http://www.bing.com/cr?IG=30CD706EAC0F4F0DA3AEF69555E1BCF9&CID=2F6AF5D6774565193D03FF8076A2647B&rd=1&h=fBEQT4E1esfQT6HCvPsopRg4uvou_GlQw3SSPC7Wqrc&v=1&r=http%3a%2f%2fdata.worldbank.org%2findicator%2fSP.DYN.LE00.IN&p=DevEx,5099.1\n",
      "No meaningful text in this document\n",
      "Done URL 3 out of 32\n",
      " \n",
      "Looking in document for values similar to 69.5007561\n",
      "http://www.bing.com/cr?IG=30CD706EAC0F4F0DA3AEF69555E1BCF9&CID=2F6AF5D6774565193D03FF8076A2647B&rd=1&h=vPNHlHB8Ya1RhlKPw1_A6sdTagzdHlz_Sgy2iXwFE6o&v=1&r=http%3a%2f%2fcountryeconomy.com%2fdemography%2flife-expectancy%2fmalaysia&p=DevEx,5115.1\n",
      "No meaningful text in this document\n",
      "Done URL 4 out of 32\n",
      " \n",
      "Looking in document for values similar to 69.5007561\n",
      "http://www.bing.com/cr?IG=30CD706EAC0F4F0DA3AEF69555E1BCF9&CID=2F6AF5D6774565193D03FF8076A2647B&rd=1&h=wTe2w-49zWJVopxvL3FA66F66RAhMgjdxgjZK0sBMpE&v=1&r=http%3a%2f%2fwww.indexmundi.com%2ffacts%2fmalaysia%2flife-expectancy-at-birth&p=DevEx,5134.1\n",
      "Done URL 5 out of 32\n",
      " \n",
      "Looking in document for values similar to 69.5007561\n",
      "http://www.bing.com/cr?IG=30CD706EAC0F4F0DA3AEF69555E1BCF9&CID=2F6AF5D6774565193D03FF8076A2647B&rd=1&h=oHREML1kGTtIJndZR4HbyegF17wzfggtkjlke9w0WZ4&v=1&r=http%3a%2f%2fwww.nationmaster.com%2fcountry-info%2fstats%2fHealth%2fLife-expectancy-at-birth%252C-total%2fYears&p=DevEx,5152.1\n",
      "24 candidate matches\n",
      "[\"International Statistics at NationMaster.com '' , Derived from male and female life expectancy at birth from sources such as : -LRB- 1 -RRB- United Nations Population Division .\", 'World Population Prospects , -LRB- 2 -RRB- United Nations Statistical Division .', 'Population and Vital Statistics Report -LRB- various years -RRB- , -LRB- 3 -RRB- Census reports and other statistical publications from national statistical offices , -LRB- 4 -RRB- Eurostat : Demographic Statistics , -LRB- 5 -RRB- Secretariat of the Pacific Community : Statistics and Demography Programme , and -LRB- 6 -RRB- U.S. Census Bureau : International Database .', 'International Statistics at NationMaster.com , Derived from male and female life expectancy at birth from sources such as : -LRB- 1 -RRB- United Nations Population Division .', 'World Population Prospects , -LRB- 2 -RRB- United Nations Statistical Division .', 'Population and Vital Statistics Report -LRB- various years -RRB- , -LRB- 3 -RRB- Census reports and other statistical publications from national statistical offices , -LRB- 4 -RRB- Eurostat : Demographic Statistics , -LRB- 5 -RRB- Secretariat of the Pacific Community : Statistics and Demography Programme , and -LRB- 6 -RRB- U.S. Census Bureau : International Database .', 'International Statistics at NationMaster.com , Derived from male and female life expectancy at birth from sources such as : -LRB- 1 -RRB- United Nations Population Division .', 'World Population Prospects , -LRB- 2 -RRB- United Nations Statistical Division .', 'Population and Vital Statistics Report -LRB- various years -RRB- , -LRB- 3 -RRB- Census reports and other statistical publications from national statistical offices , -LRB- 4 -RRB- Eurostat : Demographic Statistics , -LRB- 5 -RRB- Secretariat of the Pacific Community : Statistics and Demography Programme , and -LRB- 6 -RRB- U.S. Census Bureau : International Database .', \"International Statistics at NationMaster.com '' , Derived from male and female life expectancy at birth from sources such as : -LRB- 1 -RRB- United Nations Population Division .\", 'World Population Prospects , -LRB- 2 -RRB- United Nations Statistical Division .', 'Population and Vital Statistics Report -LRB- various years -RRB- , -LRB- 3 -RRB- Census reports and other statistical publications from national statistical offices , -LRB- 4 -RRB- Eurostat : Demographic Statistics , -LRB- 5 -RRB- Secretariat of the Pacific Community : Statistics and Demography Programme , and -LRB- 6 -RRB- U.S. Census Bureau : International Database .', \"International Statistics at NationMaster.com '' , Derived from male and female life expectancy at birth from sources such as : -LRB- 1 -RRB- United Nations Population Division .\", 'World Population Prospects , -LRB- 2 -RRB- United Nations Statistical Division .', 'Population and Vital Statistics Report -LRB- various years -RRB- , -LRB- 3 -RRB- Census reports and other statistical publications from national statistical offices , -LRB- 4 -RRB- Eurostat : Demographic Statistics , -LRB- 5 -RRB- Secretariat of the Pacific Community : Statistics and Demography Programme , and -LRB- 6 -RRB- U.S. Census Bureau : International Database .', 'International Statistics at NationMaster.com , Derived from male and female life expectancy at birth from sources such as : -LRB- 1 -RRB- United Nations Population Division .', 'World Population Prospects , -LRB- 2 -RRB- United Nations Statistical Division .', 'Population and Vital Statistics Report -LRB- various years -RRB- , -LRB- 3 -RRB- Census reports and other statistical publications from national statistical offices , -LRB- 4 -RRB- Eurostat : Demographic Statistics , -LRB- 5 -RRB- Secretariat of the Pacific Community : Statistics and Demography Programme , and -LRB- 6 -RRB- U.S. Census Bureau : International Database .', \"International Statistics at NationMaster.com '' , Derived from male and female life expectancy at birth from sources such as : -LRB- 1 -RRB- United Nations Population Division .\", 'World Population Prospects , -LRB- 2 -RRB- United Nations Statistical Division .', 'Population and Vital Statistics Report -LRB- various years -RRB- , -LRB- 3 -RRB- Census reports and other statistical publications from national statistical offices , -LRB- 4 -RRB- Eurostat : Demographic Statistics , -LRB- 5 -RRB- Secretariat of the Pacific Community : Statistics and Demography Programme , and -LRB- 6 -RRB- U.S. Census Bureau : International Database .', \"International Statistics at NationMaster.com '' , Derived from male and female life expectancy at birth from sources such as : -LRB- 1 -RRB- United Nations Population Division .\", 'World Population Prospects , -LRB- 2 -RRB- United Nations Statistical Division .', 'Population and Vital Statistics Report -LRB- various years -RRB- , -LRB- 3 -RRB- Census reports and other statistical publications from national statistical offices , -LRB- 4 -RRB- Eurostat : Demographic Statistics , -LRB- 5 -RRB- Secretariat of the Pacific Community : Statistics and Demography Programme , and -LRB- 6 -RRB- U.S. Census Bureau : International Database .']\n",
      "About to parse\n"
     ]
    }
   ],
   "source": [
    "from run.ds_generate_positive_features_for_query import precompute_features\n",
    "precompute_features(world)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Fact Checking\n",
    "\n",
    "### Training\n",
    "Load Modules for fact checking, generate the features and train our classifier from our training data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from classifier.Classifier import Classifier\n",
    "from classifier.LogisticRegressionClassifier import LogisticRegressionClassifier\n",
    "from classifier.features.generate_features import FeatureGenerator, num, is_num\n",
    "from distant_supervision.utterance_detection import f_threshold_match\n",
    "from factchecking.question import Question\n",
    "from tabular.filtering import load_collection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "fg = FeatureGenerator()\n",
    "Xs,ys = fg.generate_training(world)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "class LogisticRegressionClassifier(Classifier):\n",
    "    def train(self, Xs, ys):\n",
    "        print(\"Training classifier 3\")\n",
    "        self.lr = LogisticRegression(penalty='l1', C=0.78)\n",
    "        self.lr.fit(Xs, ys)\n",
    "        print(\"Trained\")\n",
    "\n",
    "    def predict(self, q_features):\n",
    "        ys = (self.lr.predict(q_features), self.lr.predict_proba(q_features))\n",
    "        return ys\n",
    "\n",
    "\n",
    "classifier = LogisticRegressionClassifier()\n",
    "classifier.train(Xs,ys)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "### Runtime\n",
    "\n",
    "Load the source data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "tables = load_collection(\"herox\")\n",
    "print(tables.files)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Define the fact checking function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def fact_check(q):\n",
    "    question = Question(text=q, type=\"NUM\")\n",
    "    question.parse()\n",
    "    tuples,q_features = fg.generate_test(tables,question)\n",
    "    q_match = False\n",
    "    \n",
    "  \n",
    "    \n",
    "    if len(tuples)>0:\n",
    "  \n",
    "    \n",
    "        \n",
    "        \n",
    "        q_predicted = classifier.predict(q_features)\n",
    "\n",
    "        for i in range(len(tuples)):\n",
    "            tuple = tuples[i]\n",
    "            \n",
    "            skip = False\n",
    "            if 'date' in tuple[1].keys() and len(question.dates):\n",
    "                for date in question.dates:\n",
    "                    dstrs = set()\n",
    "                    for d in question.dates:\n",
    "                        dstrs.add(str(d))\n",
    "                    if not len(set(tuple[1]['date']).intersection(dstrs)):\n",
    "                        skip = True\n",
    "                        \n",
    "            if skip:\n",
    "                continue\n",
    "    \n",
    "\n",
    "            if is_num(tuple[1]['value']):\n",
    "                prediction = q_predicted[0][i]\n",
    "                features = q_features[i]\n",
    "\n",
    "                \n",
    "             \n",
    "                if prediction == 1:\n",
    "                    print(str(tuple) + \"\\t\\t\" + (\"Possible Match\" if prediction else \"No match\"))\n",
    "                    for number in question.numbers:\n",
    "                        value = num(tuple[1]['value'])\n",
    "\n",
    "                        if value is None:\n",
    "                            continue\n",
    "\n",
    "                        if f_threshold_match(number, value, 0.05):\n",
    "                            print(str(tuple) + \"\\t\\t\" + \"Threshold Match to 5%\")\n",
    "                            q_match = True\n",
    "\n",
    "                    for number in question.dates:\n",
    "                        value = num(tuple[1]['value'])\n",
    "                        if number == value:\n",
    "                            print(str(tuple) + \"\\t\\t\" + \"Exact Match\")\n",
    "                            q_match = True\n",
    "        print(question.text)\n",
    "        print(q_match)\n",
    "\n",
    "    else:\n",
    "        print(question.text)\n",
    "        print(\"No supporting information can be found in the knowledge base\")\n",
    "    print(\"\\n\\n\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Fact checking"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "fact_check(\"23,000 sheep Germany 2014\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import csv\n",
    "base = \"/Users/james/Dropbox/Fact Checking/james\"\n",
    "rels = []\n",
    "from distant_supervision.normalisation import normalise,normalise_keep_nos\n",
    "from collections import defaultdict\n",
    "import re \n",
    "import pickle\n",
    "import ast\n",
    "\n",
    "from fuzzywuzzy import process\n",
    "\n",
    "\n",
    "def normalise(text):\n",
    "    text = text.replace(\"lrb\",\"\")\n",
    "    text = text.replace(\"lsb\", \"\")\n",
    "    text = text.replace(\"rrb\", \"\")\n",
    "    text = text.replace(\"rsb\", \"\")\n",
    "    \n",
    "    text = text.replace(\"-lrb-\",\"\")\n",
    "    text = text.replace(\"-lsb-\", \"\")\n",
    "    text = text.replace(\"-rrb-\", \"\")\n",
    "    text = text.replace(\"-rsb-\", \"\")\n",
    "    \n",
    "    text = text.replace(\"-LRB-\",\"\")\n",
    "    text = text.replace(\"-LSB-\", \"\")\n",
    "    text = text.replace(\"-RRB-\", \"\")\n",
    "    text = text.replace(\"-LSB-\", \"\")\n",
    "\n",
    "    text = re.sub(r'[^\\w]', ' ', text)\n",
    "    text = re.sub(r'[0-9]','D', text.lower())\n",
    "    return text\n",
    "\n",
    "def fact_check_and_test(q, rel):\n",
    "    question = q # Question(text=q, type=\"NUM\")\n",
    "    tuples, q_features = fg.generate_test(tables, question)\n",
    "    q_match = False\n",
    "\n",
    "    matches = dict()\n",
    "    \n",
    "    p_match = 0.0\n",
    "    found_match = False \n",
    "    total_geq = 0\n",
    "    total_gt = 0\n",
    "    total_match= 0\n",
    "    \n",
    "    entities = set()\n",
    "    if len(tuples) > 0:\n",
    "        for i in range(len(tuples)):\n",
    "            \n",
    "            tuple = tuples[i]\n",
    "            skip = False\n",
    "            if 'date' in tuple[1].keys() and len(question.dates):\n",
    "                for date in question.dates:\n",
    "                    dstrs = set()\n",
    "                    for d in question.dates:\n",
    "                        dstrs.add(str(d))\n",
    "                    if not len(set(tuple[1]['date']).intersection(dstrs)):\n",
    "                        skip = True\n",
    "            \n",
    "            if skip or not is_num(tuple[1]['value']):\n",
    "                continue\n",
    "            \n",
    "            entities.add(tuple[1]['entity'])\n",
    "            matches[tuple[1]['entity'] + \"-----\" + tuple[1]['relation']] = (tuple,q_features[i])\n",
    "\n",
    "    if len(matches.keys()) > 0:\n",
    "        for i in matches.keys():\n",
    "            \n",
    "            tuple = matches[i][0]\n",
    "          \n",
    "            features = matches[i][1]\n",
    "            q_predicted = classifier.predict([features])\n",
    "\n",
    "            skip = False\n",
    "            if 'date' in tuple[1].keys() and len(question.dates):\n",
    "                for date in question.dates:\n",
    "                    dstrs = set()\n",
    "                    for d in question.dates:\n",
    "                        dstrs.add(str(d))\n",
    "                    if not len(set(tuple[1]['date']).intersection(dstrs)):\n",
    "                        skip = True\n",
    "\n",
    "            if skip or not is_num(tuple[1]['value']):\n",
    "                continue\n",
    "\n",
    "            prediction = q_predicted[0][0]\n",
    "            \n",
    "            if prediction == 1:\n",
    "                \n",
    "                if (tuple[1]['relation'] == rel):\n",
    "                    p_match = q_predicted[1][0][1]\n",
    "                    found_match = True\n",
    "            \n",
    "    else:\n",
    "        return (-1,0,0)\n",
    "\n",
    "    \n",
    "    if found_match:\n",
    "        for i in matches.keys():\n",
    "            \n",
    "            tuple = matches[i][0]\n",
    "            features = matches[i][1]\n",
    "            q_predicted = classifier.predict([features])\n",
    "\n",
    "            \n",
    "            skip = False\n",
    "            if 'date' in tuple[1].keys() and len(question.dates):\n",
    "                for date in question.dates:\n",
    "                    dstrs = set()\n",
    "                    for d in question.dates:\n",
    "                        dstrs.add(str(d))\n",
    "                    if not len(set(tuple[1]['date']).intersection(dstrs)):\n",
    "                        skip = True\n",
    "\n",
    "            if skip or not is_num(tuple[1]['value']):\n",
    "                continue\n",
    "\n",
    "            prediction = q_predicted[0][0]\n",
    "           \n",
    "            if prediction == 1:\n",
    "                if not (tuple[1]['relation'] == rel):\n",
    "                    if q_predicted[1][0][1] > p_match:\n",
    "                        total_gt += 1\n",
    "                    if q_predicted[1][0][1] >= p_match:\n",
    "                        total_geq += 1\n",
    "                    total_match += 1\n",
    "                    \n",
    "        \n",
    "    if found_match:\n",
    "        print(\"matched - \")\n",
    "        print(total_gt)\n",
    "        print(total_geq)\n",
    "        print(total_match)\n",
    "        return (1,total_gt,total_match,total_geq)\n",
    "    \n",
    "    rs = set()\n",
    "    for tuple in tuples:\n",
    "        if not is_num(tuple[1]['value']):\n",
    "            pass\n",
    "        rs.add(tuple[1]['relation'])\n",
    "\n",
    "    if rel not in rs:\n",
    "        return (-1,0,0,0)\n",
    "    return (0,0,0,0)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "for filename in os.listdir(base):\n",
    "    if filename.endswith(\".tsv\"):\n",
    "        with open(base+\"/\"+filename,encoding = \"ISO-8859-1\") as tsv:\n",
    "            for line in tsv.readlines():\n",
    "                row = line.split(\"\\t\")\n",
    "                if(len(row) == 12) and len(row[5].strip())>0:\n",
    "                    if(row[0].lower().strip()=='y') or (row[1].lower().strip()=='y') :\n",
    "                        rels.append({\"claim\":row[2],\"relation\":row[5],\"entity\":row[3],\"num\":row[9],\"parsed\":row[8]})\n",
    "                elif len(row) == 11:\n",
    "                    if(row[0].lower().strip()=='y') and len(row[4].strip())>0:\n",
    "                        rels.append({\"claim\":row[1],\"relation\":row[4],\"entity\":row[2],\"num\":row[8],\"parsed\":row[7]})\n",
    "           \n",
    "                 \n",
    "    \n",
    "\n",
    "\n",
    "property_names = dict()\n",
    "\n",
    "property_names['fertility_rate'] = \"Fertility rate, total (births per woman)\"\n",
    "property_names['gdp_growth_rate'] = \"GDP growth (annual %)\"\n",
    "property_names['gdp_nominal'] = \"GDP (current US$)\"\n",
    "property_names['gdp_nominal_per_capita'] = \"GDP per capita (current US$)\"\n",
    "property_names['gni_per_capita_in_ppp_dollars'] = \"GNI per capita, PPP (current international $)\"\n",
    "property_names['life_expectancy'] = \"Life expectancy at birth, total (years)\"\n",
    "property_names['cpi_inflation_rate'] = \"Inflation, consumer prices (annual %)\"\n",
    "property_names['consumer_price_index'] = \"Consumer price index (2010 = 100)\"\n",
    "property_names['diesel_price_liter'] = \"Pump price for diesel fuel (US$ per liter)\"\n",
    "property_names['gni_in_ppp_dollars'] = \"GNI (current US$)\"\n",
    "property_names['population_growth_rate'] = \"Population growth (annual %)\"\n",
    "property_names['population'] = \"Population, total\"\n",
    "property_names['prevalence_of_undernourisment'] = \"Prevalence of undernourishment (% of population)\"\n",
    "property_names['renewable_freshwater_per_capita'] = \"Renewable internal freshwater resources per capita (cubic meters)\"\n",
    "property_names['health_expenditure_as_percent_of_gdp'] = \"Health expenditure, total (% of GDP)\"\n",
    "property_names['internet_users_percent_population'] = \"Internet users (per 100 people)\"\n",
    "\n",
    "tested = defaultdict(int)\n",
    "results = defaultdict(int)\n",
    "pr = defaultdict(int)\n",
    "\n",
    "num_better = defaultdict(int)\n",
    "num_total = defaultdict(int)\n",
    "num_better_or_equal = defaultdict(int)\n",
    "\n",
    "print(len(rels))\n",
    "\n",
    "\n",
    "claim_loc = re.compile(r'<location[^>]*>([^<]+)</location>')\n",
    "claim_num = re.compile(r'<number[^>]*>([^<]+)</number>')\n",
    "\n",
    "\n",
    "class NewQuestion():\n",
    "    def __init__(self,text,entity,number):\n",
    "        self.text = text\n",
    "        self.nes = {entity}\n",
    "        self.numbers = {num(number)}\n",
    "        self.dates = set()\n",
    "        self.nps = set()\n",
    "    def parse(self):\n",
    "        pass\n",
    "        \n",
    "qs = []\n",
    "        \n",
    "for rel in rels:\n",
    "    if len(claim_loc.findall(rel['claim'])) > 0:\n",
    "        rel['num'] = claim_num.findall(rel['claim'])[0]\n",
    "\n",
    "    start_claim_idx = rel['claim'].index(rel['entity'])\n",
    "    end_claim_idx = start_claim_idx + len(rel['entity'])\n",
    "\n",
    "\n",
    "    start_num_idx = rel['claim'].index(rel['num'])\n",
    "    end_num_idx = start_num_idx + len(rel['num'])\n",
    "\n",
    "    span = \"\"\n",
    "    if end_claim_idx < start_num_idx:\n",
    "        span = (rel['claim'][end_claim_idx:start_num_idx])\n",
    "    else:\n",
    "        span =(rel['claim'][start_num_idx:end_claim_idx])\n",
    "\n",
    "    span = re.sub('<[^<]+?>', '', span)\n",
    "    #print(normalise(span).split())\n",
    "    \n",
    "    spanwords = span.split()\n",
    "    \n",
    "    if(rel['parsed'][0]==\"\\\"\"):\n",
    "        rel['parsed'] = rel['parsed'][1:-1]\n",
    "        \n",
    "    dep_parse = ast.literal_eval(rel['parsed'])\n",
    "    \n",
    "    tokens = []\n",
    "    for token in dep_parse:\n",
    "        \n",
    "        for w in (token.replace(\"*extend*\",\"\").split(\"+\")):\n",
    "            we = w.split(\"~\")[0].replace(\"\\\"\",\"\")\n",
    "            \n",
    "            if \",\" in we:\n",
    "                for t in we.split(\",\"):\n",
    "                    if not(t  == \"NUMBER_SLOT\" or t == \"LOCATION_SLOT\"):\n",
    "                        tokens.append(t)\n",
    "            elif not(we  == \"NUMBER_SLOT\" or we == \"LOCATION_SLOT\"):\n",
    "                tokens.append(we)\n",
    "    tokens = \" \".join(tokens).replace(\"DATE\",\"1000\").replace(\"PERCENT\",\"10\").replace(\"MISC\",\"$\")\n",
    "    tokens += \" \"\n",
    "    tokens += \" \".join(spanwords).replace(\"DATE\",\"1000\").replace(\"PERCENT\",\"10\").replace(\"MISC\",\"$\")\n",
    "    \n",
    "    q = NewQuestion(rel['claim'],rel['entity'],rel['num'])\n",
    "    words = normalise_keep_nos(q.text).split()\n",
    "\n",
    "    qs.append((q,rel))\n",
    "\n",
    "    \n",
    "done= 0\n",
    "for item in qs:\n",
    "    done += 1\n",
    "    print(rel['claim'])\n",
    "    print(rel['relation'])\n",
    "    \n",
    "    rel = item[1]\n",
    "    q = item[0]\n",
    "    result = fact_check_and_test(q, property_names[rel['relation']])\n",
    "    \n",
    "    if result[0] == 1:\n",
    "        results[rel['relation']] += 1\n",
    "        if result[1] == 0:\n",
    "            pr[rel['relation']] += result[3]\n",
    "        num_better[rel['relation']] += result[1]\n",
    "        num_better_or_equal[rel['relation']] += result[3]    \n",
    "        num_total[rel['relation']] += result[2]\n",
    "\n",
    "    if result[0] != -1:\n",
    "        tested[rel['relation']] += 1\n",
    "\n",
    "    print(result)\n",
    "    print(\"done\" + str(done) )\n",
    "    print(\"\")\n",
    "    \n",
    "    if done%5000 == 0:\n",
    "        for key in tested.keys():\n",
    "            print(key + \" \" + str(results[key]) + \" \" + str(num_better[key]) + \" \" + str(num_better_or_equal[key]) + \" \" + str(num_total[key]) + \" \" + str(pr[key]) + \" \" + str(tested[key]) + \" \" + str(results[key] / tested[key]))\n",
    "\n",
    "print(\"Done\")\n",
    "for key in tested.keys():\n",
    "    print(key + \" \" + str(results[key]) + \" \" + str(num_better[key]) + \" \" + str(num_better_or_equal[key]) + \" \" + str(num_total[key]) + \" \" + str(pr[key]) + \" \" + str(tested[key]) + \" \" + str(results[key] / tested[key]))\n"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [conda root]",
   "language": "python",
   "name": "conda-root-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}