Trip-advisor_scrape_(airlines).ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-07-08T07:34:42.152772Z",
     "start_time": "2020-07-08T07:34:42.149819Z"
    }
   },
   "outputs": [],
   "source": [
    "#if geckodriver in local directory, this will add to path temporarily\n",
    "import os\n",
    "path = os.getcwd()\n",
    "os.environ[\"PATH\"] += os.pathsep + path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-07-08T07:33:17.968466Z",
     "start_time": "2020-07-08T07:33:17.892695Z"
    }
   },
   "outputs": [],
   "source": [
    "from selenium import webdriver\n",
    "from bs4 import BeautifulSoup\n",
    "from tqdm import trange\n",
    "import pickle\n",
    "import time\n",
    "import sys"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-07-08T07:33:21.374604Z",
     "start_time": "2020-07-08T07:33:21.372122Z"
    }
   },
   "outputs": [],
   "source": [
    "main_url = 'https://www.tripadvisor.com/Airlines'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-07-08T07:33:24.502354Z",
     "start_time": "2020-07-08T07:33:24.492375Z"
    }
   },
   "outputs": [],
   "source": [
    "def get_links_to_scrape(main_url, start=0, num_pages=10):\n",
    "    '''\n",
    "    navigates to airline page, extracts html links to airlines for future scrape.\n",
    "    '''\n",
    "    driver = webdriver.Firefox()\n",
    "    driver.get(main_url)\n",
    "    driver.implicitly_wait(100)\n",
    "\n",
    "    for n in trange(start):\n",
    "        button = driver.find_element_by_class_name(\"nav.next\")\n",
    "        button.click()\n",
    "        time.sleep(2)\n",
    "\n",
    "    url_list = []\n",
    "    for n in trange(num_pages):\n",
    "        urls = driver.find_elements_by_class_name('detailsLink')\n",
    "        for url in urls[:10]:\n",
    "            href = url.get_attribute('href')\n",
    "            url_list.append(href)\n",
    "\n",
    "        button = driver.find_element_by_class_name(\"nav.next\")\n",
    "        button.click()\n",
    "        time.sleep(2)\n",
    "\n",
    "    driver.quit()\n",
    "    new_airlines = '\\n'.join(url_list)\n",
    "    with open('airlines.txt', 'a') as outfile:\n",
    "        outfile.write(new_airlines)\n",
    "\n",
    "    return url_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-07-08T15:45:04.494689Z",
     "start_time": "2020-07-08T15:45:04.485258Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#list of airlines (also output to screen)\n",
    "get_links_to_scrape(main_url, start=0, num_pages=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-07-08T07:35:46.825652Z",
     "start_time": "2020-07-08T07:35:46.821367Z"
    }
   },
   "outputs": [],
   "source": [
    "def start_scrape(url, start=0):\n",
    "    '''\n",
    "    starts selenium web driver\n",
    "    param: url - the page url of the page you wish to scrape\n",
    "    param = int: if scrape crashes - change parameter (start) to page number where crash happened to restart from\n",
    "    that point\n",
    "\n",
    "    '''\n",
    "    driver = webdriver.Firefox()\n",
    "    driver.get(url)\n",
    "    driver.implicitly_wait(100)\n",
    "\n",
    "    for n in trange(start):\n",
    "        button = driver.find_element_by_link_text(\"Next\")\n",
    "        button.click()\n",
    "    return driver"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-07-08T07:59:27.017102Z",
     "start_time": "2020-07-08T07:59:27.007150Z"
    }
   },
   "outputs": [],
   "source": [
    "def trip_advisor_scrape(driver, url, start=0, num_reviews=1500):\n",
    "    basename = url.split(\"Reviews-\")[-1]\n",
    "    soup_list = []\n",
    "    for n in trange(num_reviews):\n",
    "        try:\n",
    "            # expand the 'read more' element\n",
    "            elements = driver.find_element_by_class_name('_36B4Vw6t')\n",
    "            elements.click()\n",
    "        except:\n",
    "            pass\n",
    "        soup = BeautifulSoup(driver.page_source, 'html.parser')\n",
    "# this class identifier may be changed in future\n",
    "        review_soup = soup.find_all('div', attrs={'class': 'oETBfkHU'})\n",
    "        soup_list.append(str(review_soup))\n",
    "        try:\n",
    "            button = driver.find_element_by_link_text(\"Next\")\n",
    "            button.click()\n",
    "            time.sleep(1)\n",
    "        except:\n",
    "            sys.setrecursionlimit(10000)\n",
    "            pickle.dump(soup_list, open(f'{basename}_{start}-{start + n}.pkl', 'wb'))\n",
    "            break\n",
    "\n",
    "    print('finished at: ', n+1)\n",
    "    sys.setrecursionlimit(10000)\n",
    "    #this will dump to local directory\n",
    "    pickle.dump(soup_list, open(f'{basename}_{start}-{start + n}.pkl', 'wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-07-08T15:44:56.656392Z",
     "start_time": "2020-07-08T15:44:56.639736Z"
    }
   },
   "outputs": [],
   "source": [
    "# uncomment one of below for either mass scrape or select airlines\n",
    "# urls = ['https://www.tripadvisor.com/Airline_Review-d8728984-Reviews-Adria-Airways-No-Longer-Operating']\n",
    "# urls = get_links_to_scrape(main_url)\n",
    "\n",
    "for url in urls:\n",
    "    start = 0\n",
    "    num_reviews = 15\n",
    "    driver = start_scrape(url, start=int(start))\n",
    "    trip_advisor_scrape(driver, url, start=int(start), num_reviews=int(num_reviews))\n",
    "    driver.quit()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}