1.retrieval_from_chatnoir.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "import xml.etree.ElementTree as ET\n",
    "import requests\n",
    "from string import punctuation \n",
    "import json\n",
    "import pickle\n",
    "from nltk.tokenize import word_tokenize\n",
    "from nltk.stem import WordNetLemmatizer \n",
    "from nltk.corpus import stopwords\n",
    "import nltk\n",
    "from nltk import pos_tag\n",
    "from nltk.corpus import wordnet\n",
    "import re\n",
    "import pandas as pd\n",
    "nltk.download('stopwords')\n",
    "nltk.download('punkt')\n",
    "import sys\n",
    "import os\n",
    "from zipfile import ZipFile\n",
    "import pytrec_eval\n",
    "from boilerpy3 import extractors\n",
    "from urllib.error import HTTPError"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Convert XML-file with topics to json, save number of results for ClueWeb12"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "#parse the XML-file with queries\n",
    "\n",
    "mytree = ET.parse('data/topics-task-2.xml')\n",
    "myroot = mytree.getroot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "#preprocess the queries\n",
    "\n",
    "q = []\n",
    "topics = []\n",
    "for item in myroot:\n",
    "    d = {}\n",
    "    for x in item:\n",
    "        d[x.tag] = x.text.strip('\\n')\n",
    "        #print(d)\n",
    "        \n",
    "        if x.tag == \"title\":\n",
    "            #print(x.text)\n",
    "            \n",
    "            #clean up punctuation:\n",
    "            no_punctuation = \"\"\n",
    "            for char in x.text:\n",
    "                if char not in punctuation:\n",
    "                    no_punctuation = no_punctuation + char\n",
    "                    \n",
    "            #set up the query with an operator AND\n",
    "            #print(no_punctuation)\n",
    "            s = ' AND '.join(no_punctuation.split())\n",
    "            #print(s)\n",
    "            \n",
    "            #append the query to an array\n",
    "            q.append(s)\n",
    "    topics.append(d)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "50"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(topics)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "#save topics as json\n",
    "with open('data/topics.json', 'w') as file:\n",
    "    json.dump(topics, file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "What AND is AND the AND difference AND between AND sex AND and AND love\n",
      "{'query_time': 11422, 'total_results': 194416, 'indices': ['cw12']}\n",
      "Which AND is AND better AND a AND laptop AND or AND a AND desktop\n",
      "{'query_time': 8621, 'total_results': 117594, 'indices': ['cw12']}\n",
      "Which AND is AND better AND Canon AND or AND Nikon\n",
      "{'query_time': 2724, 'total_results': 17321, 'indices': ['cw12']}\n",
      "What AND are AND the AND best AND dish AND detergents\n",
      "{'query_time': 2182, 'total_results': 6321, 'indices': ['cw12']}\n",
      "What AND are AND the AND best AND cities AND to AND live AND in\n",
      "{'query_time': 10167, 'total_results': 2602861, 'indices': ['cw12']}\n",
      "What AND is AND the AND longest AND river AND in AND the AND US\n",
      "{'query_time': 3627, 'total_results': 81792, 'indices': ['cw12']}\n",
      "Which AND is AND healthiest AND coffee AND green AND tea AND or AND black AND tea AND and AND why\n",
      "{'query_time': 4528, 'total_results': 1340, 'indices': ['cw12']}\n",
      "What AND are AND the AND advantages AND and AND disadvantages AND of AND PHP AND over AND Python AND and AND vice AND versa\n",
      "{'query_time': 7118, 'total_results': 210, 'indices': ['cw12']}\n",
      "Why AND is AND Linux AND better AND than AND Windows\n",
      "{'query_time': 6010, 'total_results': 147688, 'indices': ['cw12']}\n",
      "How AND to AND sleep AND better\n",
      "{'query_time': 7684, 'total_results': 1033081, 'indices': ['cw12']}\n",
      "Should AND I AND buy AND an AND LCD AND TV AND or AND a AND plasma AND TV\n",
      "{'query_time': 8818, 'total_results': 8937, 'indices': ['cw12']}\n",
      "Train AND or AND plane AND Which AND is AND the AND better AND choice\n",
      "{'query_time': 5406, 'total_results': 43117, 'indices': ['cw12']}\n",
      "What AND is AND the AND highest AND mountain AND on AND Earth\n",
      "{'query_time': 4513, 'total_results': 110731, 'indices': ['cw12']}\n",
      "Should AND one AND prefer AND Chinese AND medicine AND or AND Western AND medicine\n",
      "{'query_time': 2229, 'total_results': 11217, 'indices': ['cw12']}\n",
      "What AND are AND the AND best AND washing AND machine AND brands\n",
      "{'query_time': 2737, 'total_results': 38399, 'indices': ['cw12']}\n",
      "Should AND I AND buy AND or AND rent\n",
      "{'query_time': 2438, 'total_results': 270060, 'indices': ['cw12']}\n",
      "Do AND you AND prefer AND cats AND or AND dogs AND and AND why\n",
      "{'query_time': 6854, 'total_results': 50288, 'indices': ['cw12']}\n",
      "What AND is AND the AND better AND way AND to AND grill AND outdoors AND gas AND or AND charcoal\n",
      "{'query_time': 2225, 'total_results': 1723, 'indices': ['cw12']}\n",
      "Which AND is AND better AND MAC AND or AND PC\n",
      "{'query_time': 2227, 'total_results': 184891, 'indices': ['cw12']}\n",
      "What AND is AND better AND to AND use AND a AND brush AND or AND a AND sponge\n",
      "{'query_time': 1749, 'total_results': 7719, 'indices': ['cw12']}\n",
      "Which AND is AND better AND Linux AND or AND Microsoft\n",
      "{'query_time': 1910, 'total_results': 151323, 'indices': ['cw12']}\n",
      "Which AND is AND better AND Pepsi AND or AND Coke\n",
      "{'query_time': 1340, 'total_results': 12481, 'indices': ['cw12']}\n",
      "What AND is AND better AND Google AND search AND or AND Yahoo AND search\n",
      "{'query_time': 3016, 'total_results': 328290, 'indices': ['cw12']}\n",
      "Which AND one AND is AND better AND Netflix AND or AND Blockbuster\n",
      "{'query_time': 1164, 'total_results': 3445, 'indices': ['cw12']}\n",
      "Which AND browser AND is AND better AND Internet AND Explorer AND or AND Firefox\n",
      "{'query_time': 3382, 'total_results': 86478, 'indices': ['cw12']}\n",
      "Which AND is AND a AND better AND vehicle AND BMW AND or AND Audi\n",
      "{'query_time': 1416, 'total_results': 17155, 'indices': ['cw12']}\n",
      "Which AND one AND is AND better AND an AND electric AND stove AND or AND a AND gas AND stove\n",
      "{'query_time': 4541, 'total_results': 8840, 'indices': ['cw12']}\n",
      "What AND planes AND are AND best AND Boeing AND or AND Airbus\n",
      "{'query_time': 854, 'total_results': 4558, 'indices': ['cw12']}\n",
      "Which AND is AND better AND Disneyland AND or AND Disney AND World\n",
      "{'query_time': 2954, 'total_results': 8035, 'indices': ['cw12']}\n",
      "Should AND I AND buy AND an AND Xbox AND or AND a AND PlayStation\n",
      "{'query_time': 1552, 'total_results': 20266, 'indices': ['cw12']}\n",
      "Which AND has AND more AND caffeine AND coffee AND or AND tea\n",
      "{'query_time': 1713, 'total_results': 26915, 'indices': ['cw12']}\n",
      "Which AND is AND better AND LED AND or AND LCD AND Reception AND Displays\n",
      "{'query_time': 1478, 'total_results': 2042, 'indices': ['cw12']}\n",
      "What AND is AND better AND ASP AND or AND PHP\n",
      "{'query_time': 832, 'total_results': 25652, 'indices': ['cw12']}\n",
      "What AND is AND better AND for AND the AND environment AND a AND real AND or AND a AND fake AND Christmas AND tree\n",
      "{'query_time': 4224, 'total_results': 5042, 'indices': ['cw12']}\n",
      "Do AND you AND prefer AND tampons AND or AND pads\n",
      "{'query_time': 2825, 'total_results': 554, 'indices': ['cw12']}\n",
      "What AND IDE AND is AND better AND for AND Java AND NetBeans AND or AND Eclipse\n",
      "{'query_time': 1582, 'total_results': 11735, 'indices': ['cw12']}\n",
      "Is AND OpenGL AND better AND than AND Direct3D AND in AND terms AND of AND portability AND to AND different AND platforms\n",
      "{'query_time': 2326, 'total_results': 34, 'indices': ['cw12']}\n",
      "What AND are AND the AND differences AND between AND MySQL AND and AND PostgreSQL AND in AND performance\n",
      "{'query_time': 3766, 'total_results': 4572, 'indices': ['cw12']}\n",
      "Is AND Java AND code AND more AND readable AND than AND code AND written AND in AND Scala\n",
      "{'query_time': 2159, 'total_results': 1330, 'indices': ['cw12']}\n",
      "Which AND operating AND system AND has AND better AND performance AND Windows AND 7 AND or AND Windows AND 8\n",
      "{'query_time': 3559, 'total_results': 89976, 'indices': ['cw12']}\n",
      "Which AND smartphone AND has AND a AND better AND battery AND life AND Xperia AND or AND iPhone\n",
      "{'query_time': 2054, 'total_results': 1903, 'indices': ['cw12']}\n",
      "Which AND four AND wheel AND truck AND is AND better AND Ford AND or AND Toyota\n",
      "{'query_time': 1216, 'total_results': 5792, 'indices': ['cw12']}\n",
      "Should AND I AND prefer AND a AND Leica AND camera AND over AND Nikon AND for AND portrait AND photographs\n",
      "{'query_time': 2993, 'total_results': 172, 'indices': ['cw12']}\n",
      "Which AND company AND has AND a AND larger AND capitalization AND Apple AND or AND Microsoft\n",
      "{'query_time': 2367, 'total_results': 1101, 'indices': ['cw12']}\n",
      "Which AND laptop AND has AND a AND better AND durability AND HP AND or AND Dell\n",
      "{'query_time': 1092, 'total_results': 366, 'indices': ['cw12']}\n",
      "Which AND beverage AND has AND more AND calories AND per AND glass AND beer AND or AND cider\n",
      "{'query_time': 2168, 'total_results': 280, 'indices': ['cw12']}\n",
      "Is AND admission AND rate AND in AND Stanford AND higher AND than AND that AND of AND MIT\n",
      "{'query_time': 2829, 'total_results': 2342, 'indices': ['cw12']}\n",
      "Is AND pasta AND healthier AND than AND pizza\n",
      "{'query_time': 948, 'total_results': 10405, 'indices': ['cw12']}\n",
      "Which AND city AND is AND more AND expensive AND to AND live AND in AND San AND Francisco AND or AND New AND York\n",
      "{'query_time': 9531, 'total_results': 34834, 'indices': ['cw12']}\n",
      "Whose AND salary AND is AND higher AND basketball AND or AND soccer AND players\n",
      "{'query_time': 4527, 'total_results': 1259, 'indices': ['cw12']}\n"
     ]
    }
   ],
   "source": [
    "results = []\n",
    "chatnoir = \"https://www.chatnoir.eu/api/v1/_search\"\n",
    "attr = {\"apikey\": \"7dd15626-53aa-46c6-bd34-b2feaa2d9d81\",\n",
    "        \"query\": \"hello world\",\n",
    "        \"index\": \"cw12\",\n",
    "        \"pretty\": True\n",
    "}\n",
    "\n",
    "for x in q:\n",
    "    attr[\"query\"] = x\n",
    "    #somehow index doesn't work correctly when passed as an array (it only searches the 1st index of the array), so search\n",
    "    #in each index separately and sum up the results\n",
    "    response = requests.post(chatnoir, data = attr)\n",
    "    print(x)\n",
    "    print(response.json()[\"meta\"])\n",
    "    res = response.json()[\"meta\"][\"total_results\"]\n",
    "    \n",
    "    results.append(res)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "#save results as txt\n",
    "\n",
    "with open(\"data/results.txt\", \"wb\") as fp:   #Pickling\n",
    "    pickle.dump(results, fp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[194416, 117594, 17321, 6321, 2602861, 81792, 1340, 210, 147688, 1033081, 8937, 43117, 110731, 11217, 38399, 270060, 50288, 1723, 184891, 7719, 151323, 12481, 328290, 3445, 86478, 17155, 8840, 4558, 8035, 20266, 26915, 2042, 25652, 5042, 554, 11735, 34, 4572, 1330, 89976, 1903, 5792, 172, 1101, 366, 280, 2342, 10405, 34834, 1259]\n"
     ]
    }
   ],
   "source": [
    "#test if results were saved correctly\n",
    "\n",
    "with open(\"data/results.txt\", \"rb\") as fp:   # Unpickling\n",
    "    res = pickle.load(fp)\n",
    "    \n",
    "print(res)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Retrieve documents from ChatNoir"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#open the file with topics\n",
    "\n",
    "f = open(\"data/topics.json\", encoding='utf8')\n",
    "topics = json.load(f)\n",
    "\n",
    "#open the file with amount of results for each topic\n",
    "with open(\"data/results.txt\", \"rb\") as fp:   # Unpickling\n",
    "    res = pickle.load(fp)\n",
    "    \n",
    "print(res)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#preprocess the queries\n",
    "\n",
    "for i in range(len(topics)):\n",
    "    topics[i]['title'] = topics[i]['title'].replace(' ', ' AND ')\n",
    "    topics[i]['title'] = re.sub('[?:,]', '', topics[i]['title'])\n",
    "    topics[i]['results'] = res[i]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "attr = {\"apikey\": \"7dd15626-53aa-46c6-bd34-b2feaa2d9d81\",\n",
    "        \"query\": \"\",\n",
    "        \"index\": \"cw12\",\n",
    "        \"size\": 10\n",
    "       }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "extractor = extractors.ArticleExtractor()\n",
    "\n",
    "\n",
    "def topics_iter(q):\n",
    "    docs = []\n",
    "    attr[\"query\"] = q['title']\n",
    "    #attr['size'] = q['results']\n",
    "    if q['results']<=10000:\n",
    "        num_of_res = q['results']\n",
    "    else:\n",
    "        num_of_res = 10000\n",
    "    count = 0\n",
    "    print(attr)\n",
    "    print(num_of_res)\n",
    "    url = \"https://www.chatnoir.eu/api/v1/_search?\"\n",
    "    while count < num_of_res:\n",
    "        attr[\"from\"] = count\n",
    "        while True:\n",
    "            try:\n",
    "                r = requests.post(url, json = attr)\n",
    "                res = r.json()\n",
    "                print(count)\n",
    "                #print(res)\n",
    "                res_len = len(res['results'])\n",
    "                print(res_len)\n",
    "            except KeyError:\n",
    "                continue\n",
    "            break\n",
    "        \n",
    "    \n",
    "        for i in range(res_len):\n",
    "            \n",
    "            doc_url = \"https://www.chatnoir.eu/cache?uuid=\"+res['results'][i]['uuid']+\"&index=cw12&raw\"\n",
    "            #print(doc_url)\n",
    "            try:\n",
    "                doc = extractor.get_doc_from_url(doc_url)\n",
    "                content = doc.content\n",
    "                title = doc.title\n",
    "                res['results'][i]['document'] = content\n",
    "            except HTTPError:\n",
    "                print(\"HTTPError\")\n",
    "                continue\n",
    "        docs.append(res['results'])\n",
    "        count+=10\n",
    "        \n",
    "    return docs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for q in topics[40:41]:\n",
    "    q['documents'] = topics_iter(q)\n",
    "    with open(\"data/docs/docs_for_topic_{}.txt\".format(q['number']), \"w\") as f:\n",
    "        json.dump(q, f)\n",
    "        \n",
    "    f.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Lemmatize topics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "f = open('data/topics.json')\n",
    "topics = json.load(f)\n",
    "f.close()\n",
    "lis =[]\n",
    "for i in range(len(topics)):\n",
    "    x=(topics[i]['title'])\n",
    "    lis.append(re.sub('[?:,]', '', x))\n",
    "converted_list = [x.lower() for x in lis]\n",
    "#print (\"Topics: \", converted_list)\n",
    "#print(\"\\n\")\n",
    "\n",
    "tokenized_sents = [word_tokenize(i) for i in converted_list]\n",
    "#for i in tokenized_sents:\n",
    "    #print (i)\n",
    "lis3 =[]\n",
    "\n",
    "for i in tokenized_sents:\n",
    "    tokens_without_sw = [word for word in i if not word in stopwords.words()]\n",
    "    lemmatizer = WordNetLemmatizer()\n",
    "    lemmatized_output_0 = ([lemmatizer.lemmatize(w,pos=\"n\") for w in tokens_without_sw])\n",
    "    lemmatized_output_1 = ' '.join(([lemmatizer.lemmatize(w,pos=\"v\") for w in lemmatized_output_0]))\n",
    "    lis3.append(lemmatized_output_1)   \n",
    "#print(\"Lemmatized verbs and nouns: \\n\", lis3)\n",
    "with open('data/topics_lemmatized.txt', 'wb') as fp:\n",
    "    pickle.dump(lis3, fp)\n",
    "fp.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create and save bulk data for the index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_dir = 'data/docs'\n",
    "output_dir = 'results'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "def strip_punct(s):\n",
    "    s = re.sub('[^A-Za-z0-9]', ' ', s)\n",
    "    s = s.lower()\n",
    "    return \" \".join(s.split())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "def return_arguments(doc_raw, model):\n",
    "    url = 'https://demo.webis.de/targer-api/classify'+model\n",
    "    #print(url)\n",
    "    #print(doc_raw)\n",
    "    res = requests.post(url, json=doc_raw)\n",
    "    if res.status_code == 200:\n",
    "#print(res)\n",
    "        res = res.json()\n",
    "        args = []\n",
    "        for x in res:\n",
    "            arg = []\n",
    "            for el in x:\n",
    "                if el['label']!='O' and float(el['prob'])>0.99:\n",
    "                    arg.append(el['token'])\n",
    "            s = ' '.join(arg)\n",
    "            args.append(s)\n",
    "        arguments = ' '.join(args)\n",
    "        arguments = word_tokenize(strip_punct(arguments).lower())\n",
    "        tokens_without_sw = [word for word in arguments if not word in stopwords.words()]\n",
    "        lemmatizer = WordNetLemmatizer()\n",
    "        lemmatized_output_0 = ([lemmatizer.lemmatize(w,pos=\"n\") for w in tokens_without_sw])\n",
    "        arguments = ' '.join(([lemmatizer.lemmatize(w,pos=\"v\") for w in lemmatized_output_0]))\n",
    "    else:\n",
    "        #print('EMPTY')\n",
    "        #print(res)\n",
    "        arguments = ''\n",
    "    return arguments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "docs_for_topic_31.txt\n",
      "which and has and more and caffeine and coffee and or and tea\n",
      "1\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "docs_for_topic_27.txt\n",
      "which and one and is and better and an and electric and stove and or and a and gas and stove\n",
      "2\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "docs_for_topic_9.txt\n",
      "why and is and linux and better and than and windows\n",
      "3\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "docs_for_topic_44.txt\n",
      "which and company and has and a and larger and capitalization and apple and or and microsoft\n",
      "4\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "docs_for_topic_34.txt\n",
      "what and is and better and for and the and environment and a and real and or and a and fake and christmas and tree\n",
      "5\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "docs_for_topic_50.txt\n",
      "whose and salary and is and higher and basketball and or and soccer and players\n",
      "6\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "docs_for_topic_22.txt\n",
      "which and is and better and pepsi and or and coke\n",
      "7\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "docs_for_topic_7.txt\n",
      "which and is and healthiest and coffee and green and tea and or and black and tea and and and why\n",
      "8\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "docs_for_topic_29.txt\n",
      "which and is and better and disneyland and or and disney and world\n",
      "9\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "docs_for_topic_45.txt\n",
      "which and laptop and has and a and better and durability and hp and or and dell\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "docs_for_topic_23.txt\n",
      "what and is and better and google and search and or and yahoo and search\n",
      "11\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "docs_for_topic_18.txt\n",
      "what and is and the and better and way and to and grill and outdoors and gas and or and charcoal\n",
      "12\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "docs_for_topic_47.txt\n",
      "is and admission and rate and in and stanford and higher and than and that and of and mit\n",
      "13\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "docs_for_topic_13.txt\n",
      "what and is and the and highest and mountain and on and earth\n",
      "14\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "docs_for_topic_43.txt\n",
      "should and i and prefer and a and leica and camera and over and nikon and for and portrait and photographs\n",
      "15\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "docs_for_topic_37.txt\n",
      "is and opengl and better and than and direct3d and in and terms and of and portability and to and different and platforms\n",
      "16\n",
      "10\n",
      "10\n",
      "10\n",
      "4\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "docs_for_topic_25.txt\n",
      "which and browser and is and better and internet and explorer and or and firefox\n",
      "17\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "docs_for_topic_3.txt\n",
      "which and is and better and canon and or and nikon\n",
      "18\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "docs_for_topic_12.txt\n",
      "train and or and plane and which and is and the and better and choice\n",
      "19\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "docs_for_topic_41.txt\n",
      "which and smartphone and has and a and better and battery and life and xperia and or and iphone\n",
      "20\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "docs_for_topic_2.txt\n",
      "which and is and better and a and laptop and or and a and desktop\n",
      "21\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "docs_for_topic_35.txt\n",
      "do and you and prefer and tampons and or and pads\n",
      "22\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "docs_for_topic_14_1.txt\n",
      "should and one and prefer and chinese and medicine and or and western and medicine\n",
      "23\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "docs_for_topic_38.txt\n",
      "what and are and the and differences and between and mysql and and and postgresql and in and performance\n",
      "24\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "docs_for_topic_6.txt\n",
      "what and is and the and longest and river and in and the and u s\n",
      "25\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "docs_for_topic_10.txt\n",
      "how and to and sleep and better\n",
      "26\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "docs_for_topic_20.txt\n",
      "what and is and better and to and use and a and brush and or and a and sponge\n",
      "27\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "docs_for_topic_32.txt\n",
      "which and is and better and led and or and lcd and reception and displays\n",
      "28\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "docs_for_topic_24.txt\n",
      "which and one and is and better and netflix and or and blockbuster\n",
      "29\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "docs_for_topic_28.txt\n",
      "what and planes and are and best and boeing and or and airbus\n",
      "30\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "docs_for_topic_4.txt\n",
      "what and are and the and best and dish and detergents\n",
      "31\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "docs_for_topic_40.txt\n",
      "which and operating and system and has and better and performance and windows and 7 and or and windows and 8\n",
      "32\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "docs_for_topic_16.txt\n",
      "should and i and buy and or and rent\n",
      "33\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "docs_for_topic_48.txt\n",
      "is and pasta and healthier and than and pizza\n",
      "34\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "docs_for_topic_14_2.txt\n",
      "should and one and prefer and chinese and medicine and or and western and medicine\n",
      "35\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "docs_for_topic_17.txt\n",
      "do and you and prefer and cats and or and dogs and and and why\n",
      "36\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "docs_for_topic_8.txt\n",
      "what and are and the and advantages and and and disadvantages and of and php and over and python and and and vice and versa\n",
      "37\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "docs_for_topic_1.txt\n",
      "what and is and the and difference and between and sex and and and love\n",
      "38\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "docs_for_topic_33.txt\n",
      "what and is and better and asp and or and php\n",
      "39\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "docs_for_topic_15.txt\n",
      "what and are and the and best and washing and machine and brands\n",
      "40\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "docs_for_topic_46.txt\n",
      "which and beverage and has and more and calories and per and glass and beer and or and cider\n",
      "41\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "docs_for_topic_19.txt\n",
      "which and is and better and mac and or and pc\n",
      "42\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "docs_for_topic_11.txt\n",
      "should and i and buy and an and lcd and tv and or and a and plasma and tv\n",
      "43\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "docs_for_topic_21.txt\n",
      "which and is and better and linux and or and microsoft\n",
      "44\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "docs_for_topic_42.txt\n",
      "which and four and wheel and truck and is and better and ford and or and toyota\n",
      "45\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "docs_for_topic_26.txt\n",
      "which and is and a and better and vehicle and bmw and or and audi\n",
      "46\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "docs_for_topic_39.txt\n",
      "is and java and code and more and readable and than and code and written and in and scala\n",
      "47\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "docs_for_topic_49.txt\n",
      "which and city and is and more and expensive and to and live and in and san and francisco and or and new and york\n",
      "48\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "docs_for_topic_30.txt\n",
      "should and i and buy and an and xbox and or and a and playstation\n",
      "49\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "docs_for_topic_5.txt\n",
      "what and are and the and best and cities and to and live and in\n",
      "50\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "docs_for_topic_36.txt\n",
      "what and ide and is and better and for and java and netbeans and or and eclipse\n",
      "51\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n",
      "10\n"
     ]
    }
   ],
   "source": [
    "count = 0\n",
    "c=0\n",
    "t = 1\n",
    "url = 'https://demo.webis.de/targer-api/classifyCombo'\n",
    "headers = {'accept': 'application/json', 'Content-Type': 'text/plain'}\n",
    "bulk_data = []\n",
    "#extract docs from zip-files\n",
    "for filename in os.listdir(input_dir):\n",
    "    if filename.endswith(\".zip\"):\n",
    "        with ZipFile((input_dir+\"/{}\").format(filename), 'r') as zip:\n",
    "            zip.extractall(input_dir)\n",
    "            \n",
    "for filename in os.listdir(input_dir):\n",
    "    if filename.endswith(\".txt\"):\n",
    "        name = re.split('_|\\.', filename)\n",
    "        num = next(obj for obj in name if obj.isdigit())\n",
    "        with open((input_dir+\"/{}\").format(filename), \"r\") as f:\n",
    "            topic = json.load(f)\n",
    "            topic['title'] = strip_punct(topic['title']).lower()\n",
    "            print(filename)\n",
    "            print(topic['title'])\n",
    "            print(t)\n",
    "            t+=1\n",
    "            for n in topic[\"documents\"]:\n",
    "                print(len(n))\n",
    "                #print(count)\n",
    "                if count<10:\n",
    "                    count+=1\n",
    "                    for doc in n:\n",
    "                        try:\n",
    "                            doc_raw = doc['document'].rstrip('\\n')\n",
    "                            doc_raw = doc_raw.rstrip('\\\\n')\n",
    "                            \n",
    "                            doc['lem'] = word_tokenize(strip_punct(doc_raw).lower())\n",
    "                            tokens_without_sw = [word for word in doc['lem'] if not word in stopwords.words()]\n",
    "                            lemmatizer = WordNetLemmatizer()\n",
    "                            lemmatized_output_0 = ([lemmatizer.lemmatize(w,pos=\"n\") for w in tokens_without_sw])\n",
    "                            doc['lem'] = ' '.join(([lemmatizer.lemmatize(w,pos=\"v\") for w in lemmatized_output_0]))\n",
    "                            doc['lem'] = doc['lem'].rstrip()\n",
    "                            \n",
    "                            title_lem = strip_punct(doc['title'].lower().rstrip())\n",
    "                            title_lem = word_tokenize(title_lem)\n",
    "                            tokens_without_sw = [word for word in title_lem if not word in stopwords.words()]\n",
    "                            lemmatizer = WordNetLemmatizer()\n",
    "                            lemmatized_output_0 = ([lemmatizer.lemmatize(w,pos=\"n\") for w in tokens_without_sw])\n",
    "                            title_lem = ' '.join(([lemmatizer.lemmatize(w,pos=\"v\") for w in lemmatized_output_0]))\n",
    "                            \n",
    "                            #doc_raw = doc['document'].rstrip('\\n')\n",
    "                            #doc_raw = doc_raw.rstrip('\\\\n')\n",
    "                            \n",
    "                            \n",
    "                            combo = return_arguments(doc_raw, 'Combo')\n",
    "                            \n",
    "                            #es = return_arguments(doc_raw, 'ES')\n",
    "                            \n",
    "                            #es_dep = return_arguments(doc_raw, 'ES_dep')\n",
    "                            \n",
    "                            #ibm = return_arguments(doc_raw, 'IBM')\n",
    "                            \n",
    "                            #new_pe = return_arguments(doc_raw, 'NewPE')\n",
    "                            \n",
    "                            #new_wd = return_arguments(doc_raw, 'NewWD')\n",
    "                            \n",
    "                            #wd = return_arguments(doc_raw, 'WD')\n",
    "                            \n",
    "                            #wd_dep = return_arguments(doc_raw, 'WD_dep')\n",
    "                        \n",
    "                            \n",
    "                            \n",
    "                            \n",
    "                            b = {\n",
    "                                    'query': topic['title'],\n",
    "                                    'title': doc['title'],\n",
    "                                    'title_lem': title_lem,\n",
    "                                    'num': num,\n",
    "                                    'uuid': doc['uuid'],\n",
    "                                    'score': doc['score'],\n",
    "                                    'document': doc['document'],\n",
    "                                    'lem': doc['lem'],\n",
    "                                    'args': combo\n",
    "                                    #'es': es,\n",
    "                                    #'es_dep': es_dep,\n",
    "                                    #'ibm': ibm,\n",
    "                                    #'new_pe': new_pe,\n",
    "                                    #'new_wd': new_wd,\n",
    "                                    #'wd': wd,\n",
    "                                    #'wd_dep': wd_dep\n",
    "                                }\n",
    "                            #es.index(index='test_index',doc_type='doc',id=doc['trec_id'],body=b)\n",
    "                            templ = {'index': {'_index': 'test_index', \n",
    "                                           '_type': 'doc', \n",
    "                                           '_id': doc['trec_id']}}\n",
    "                            bulk_data.append(templ)\n",
    "                            bulk_data.append(b)\n",
    "\n",
    "                            c+=1\n",
    "                        except KeyError:\n",
    "                            pass\n",
    "                else:\n",
    "                    break\n",
    "            count=0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"data/bulk_data.json\", \"w\") as f:\n",
    "    json.dump(bulk_data,f)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}