googlescraper.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "aacf988f",
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "import urllib\n",
    "from urllib.request import urlopen\n",
    "from bs4 import BeautifulSoup\n",
    "import os\n",
    "import re\n",
    "import pandas as pd\n",
    "import csv\n",
    "import time\n",
    "from requests_html import HTML\n",
    "from requests_html import HTMLSession\n",
    "from itertools import cycle\n",
    "import cloudscraper\n",
    "\n",
    "sess = requests.session()\n",
    "scraper = cloudscraper.create_scraper(sess)\n",
    "header = {\n",
    "    'user-agent' : 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.81 Safari/537.36'\n",
    "}\n",
    "\n",
    "\n",
    "UseProxy = False\n",
    "UseDelay = True\n",
    "proxies = ['103.149.162.194:80', '38.94.111.208:80', '88.99.10.250:1080', '27.64.17.187:4203', '80.48.119.28:8080']\n",
    "def filter_string(string):\n",
    "    if '[' or ']' in string:\n",
    "        string = string.replace('[', ' ')\n",
    "        string = string.replace(']', ' ')\n",
    "    string = string.replace(\"'\", \" \")\n",
    "    string.replace('  ', ' ')\n",
    "    string.replace('   ', ' ')\n",
    "    string = string.strip()\n",
    "    return string\n",
    "def filter_html(string):\n",
    "    check1 = 0\n",
    "    check2 = 0\n",
    "    while check1!=-1 and check2!=-1:\n",
    "        if '<' and '>' in string:\n",
    "            end = string.index('>')\n",
    "            start = string.index('<')\n",
    "            string = string.replace(string[start:end+1], ' ')\n",
    "            check1 = string.find('<')\n",
    "            check2 = string.find('>')\n",
    "        else:\n",
    "            break\n",
    "    return string.strip()\n",
    "def get_source(url):\n",
    "    \n",
    "    if UseProxy:\n",
    "        done=False\n",
    "        while done==False:\n",
    "            \n",
    "            proxy_pool = cycle(proxies)\n",
    "            for p in range(1,11):\n",
    "                #Get a proxy from the pool\n",
    "                proxy = next(proxy_pool)\n",
    "                \n",
    "                try:\n",
    "                    session = requests.session()\n",
    "                    session = cloudscraper.create_scraper(sess = session)\n",
    "                    session.proxies = {\n",
    "                                \"http\": proxy,\n",
    "                                \"https\": proxy,\n",
    "                                }\n",
    "                    response = session.get(url, headers=header)\n",
    "                    response.raise_for_status() # if response is successfull, no exception will be raised\n",
    "                    done=True\n",
    "                    \n",
    "                    return response\n",
    "                    \n",
    "                \n",
    "                except requests.exceptions.RequestException as e:\n",
    "                    print(e)\n",
    "                    print(\" \")\n",
    "                    print(\"retrying with new proxy\")\n",
    "                    \n",
    "                except requests.exceptions.HTTPError as err:\n",
    "                    print(err)\n",
    "                    print(\" \")\n",
    "                    print(\"retrying with new proxy\")    \n",
    "    else:\n",
    "        try:\n",
    "            session = requests.session()\n",
    "            session = cloudscraper.create_scraper(sess = session)\n",
    "            session.headers['user-agent'] = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.81 Safari/537.36'\n",
    "            response = session.get(url)\n",
    "            response.raise_for_status() # if response is successfull, no exception will be raised   \n",
    "            return response\n",
    "                            \n",
    "        except requests.exceptions.RequestException as e:\n",
    "            print(e)\n",
    "                                \n",
    "        except requests.exceptions.HTTPError as err:\n",
    "            print(err)\n",
    "            \n",
    "        except cloudscraper.exceptions.CloudflareChallengeError as error:\n",
    "            print(error)\n",
    "            pass\n",
    "            \n",
    "def scrape_google(query):\n",
    "\n",
    "     \n",
    "    query = urllib.parse.quote_plus(query)\n",
    "    \n",
    "   \n",
    "    response = get_source(\"https://www.google.com/search?q=\" + query + \"&num=100\")\n",
    "    soup = BeautifulSoup(response.content, 'html.parser')\n",
    "    \n",
    "    relative_keywords = soup.find('div', {'çlass':'y6Uyqe'})\n",
    "    decks = soup.findAll('div', {'class':'AJLUJb'})\n",
    "    searches = []\n",
    "    for deck in decks:\n",
    "        cards = deck.findAll('div')\n",
    "        for card in cards:\n",
    "            if card.text != '':\n",
    "                searches.append(card.text)\n",
    "    cards = soup.findAll('div', {'class' : 'yuRUbf'})\n",
    "    links = []\n",
    "    for card in cards:\n",
    "        link = card.find('a')['href']\n",
    "        links.append(link)\n",
    "        \n",
    "    google_domains = ('https://www.google.', \n",
    "                      'https://google.', \n",
    "                      'https://webcache.googleusercontent.', \n",
    "                      'http://webcache.googleusercontent.', \n",
    "                      'https://policies.google.',\n",
    "                      'https://support.google.',\n",
    "                      'https://maps.google.',\n",
    "                      'https://translate.google.',\n",
    "                      'https://www.youtube')\n",
    "\n",
    "    for url in links[:]:\n",
    "        \n",
    "        if url.startswith(google_domains):\n",
    "            links.remove(url)\n",
    "    \n",
    "    \n",
    "    return links, searches\n",
    "\n",
    "def main():\n",
    "    \n",
    "    input_user = pd.read_csv('keyword_example.csv')\n",
    "    c = len(input_user)\n",
    "    i=0\n",
    "    while i<c:\n",
    "        output_data={\n",
    "                'keyword' : [],\n",
    "                'Top1 Link': [],\n",
    "                'Top1 H1': [],\n",
    "                'Top1 Paragraph after H1': [],\n",
    "                'Top1 H2': [],\n",
    "                'Top1 Paragraph after H2': [],\n",
    "                'Top1 H3': [],\n",
    "                'Top1 Paragraph after H3': [],\n",
    "                'Top2 Link': [],\n",
    "                'Top2 H1': [],\n",
    "                'Top2 Paragraph after H1': [],\n",
    "                'Top2 H2': [],\n",
    "                'Top2 Paragraph after H2': [],\n",
    "                'Top2 H3': [],\n",
    "                'Top2 Paragraph after H3': [],\n",
    "                'Top3 Link': [],\n",
    "                'Top3 H1': [],\n",
    "                'Top3 Paragraph after H1': [],\n",
    "                'Top3 H2': [],\n",
    "                'Top3 Paragraph after H2': [],\n",
    "                'Top3 H3': [],\n",
    "                'Top3 Paragraph after H3': [],\n",
    "                'Top4 Link': [],\n",
    "                'Top4 H1': [],\n",
    "                'Top4 Paragraph after H1': [],\n",
    "                'Top4 H2': [],\n",
    "                'Top4 Paragraph after H2': [],\n",
    "                'Top4 H3': [],\n",
    "                'Top4 Paragraph after H3': [],\n",
    "                'Top5 Link': [],\n",
    "                'Top5 H1': [],\n",
    "                'Top5 Paragraph after H1': [],\n",
    "                'Top5 H2': [],\n",
    "                'Top5 Paragraph after H2': [],\n",
    "                'Top5 H3': [],\n",
    "                'Top5 Paragraph after H3': [],\n",
    "                'Top6 Link': [],\n",
    "                'Top6 H1': [],\n",
    "                'Top6 Paragraph after H1': [],\n",
    "                'Top6 H2': [],\n",
    "                'Top6 Paragraph after H2': [],\n",
    "                'Top6 H3': [],\n",
    "                'Top6 Paragraph after H3': [],\n",
    "                'Top7 Link': [],\n",
    "                'Top7 H1': [],\n",
    "                'Top7 Paragraph after H1': [],\n",
    "                'Top7 H2': [],\n",
    "                'Top7 Paragraph after H2': [],\n",
    "                'Top7 H3': [],\n",
    "                'Top7 Paragraph after H3': [],\n",
    "                'Top8 Link': [],\n",
    "                'Top8 H1': [],\n",
    "                'Top8 Paragraph after H1': [],\n",
    "                'Top8 H2': [],\n",
    "                'Top8 Paragraph after H2': [],\n",
    "                'Top8 H3': [],\n",
    "                'Top8 Paragraph after H3': [],\n",
    "                'Top9 Link': [],\n",
    "                'Top9 H1': [],\n",
    "                'Top9 Paragraph after H1': [],\n",
    "                'Top9 H2': [],\n",
    "                'Top9 Paragraph after H2': [],\n",
    "                'Top9 H3': [],\n",
    "                'Top9 Paragraph after H3': [],\n",
    "                'Top10 Link': [],\n",
    "                'Top10 H1': [],\n",
    "                'Top10 Paragraph after H1': [],\n",
    "                'Top10 H2': [],\n",
    "                'Top10 Paragraph after H2': [],\n",
    "                'Top10 H3': [],\n",
    "                'Top10 Paragraph after H3': [],\n",
    "                'Relative Searches': []\n",
    "            }\n",
    "        \n",
    "        results, relative_keywords=scrape_google(str(input_user.iloc[i,0]))\n",
    "#         print(results)\n",
    "        df = pd.DataFrame({'Links':results}) \n",
    "        df.to_csv('Links.csv', index=False)\n",
    "        \n",
    "        # detecting wordpress sites\n",
    "        links = pd.read_csv('Links.csv')\n",
    "        c1=len(links)\n",
    "        i1=0\n",
    "\n",
    "        wp=0\n",
    "        wpsites=[]\n",
    "\n",
    "        while i1<c1:\n",
    "            try:\n",
    "                response = scraper.get(str(links.iloc[i1,0]), timeout=60)\n",
    "                response.raise_for_status() # if response is successfull, no exception will be raised\n",
    "                bsh = BeautifulSoup(response.content, 'html.parser')\n",
    "                marker=bsh.find(class_='entry-content')\n",
    "                faulty_link = 'https://www.pupbox.com/training/when-do-puppies-lose-their-baby-teeth/'\n",
    "                if \"wp-content\" in response.text and marker:\n",
    "                    if str(links.iloc[i1,0]) != faulty_link:\n",
    "                        wpsites.append(str(links.iloc[i1,0]))\n",
    "                        wp=wp+1\n",
    "\n",
    "                if wp>9:\n",
    "                    break    \n",
    "            except requests.exceptions.RequestException as e:\n",
    "                xxx=1\n",
    "                # print(e)\n",
    "\n",
    "            except requests.exceptions.HTTPError as err:\n",
    "                xxx=1\n",
    "                # print(err)\n",
    "\n",
    "\n",
    "\n",
    "            i1=i1+1\n",
    "        df1 = pd.DataFrame(wpsites, columns=['WPsites']) \n",
    "        df1.to_csv('wpsites.csv', index=False)\n",
    "        \n",
    "        os.remove('Links.csv')\n",
    "        H1=[]\n",
    "        PH1_final=[]\n",
    "        H2=[]\n",
    "        PH2_final=[]\n",
    "        H3=[]\n",
    "        PH3_final=[]\n",
    "\n",
    "        i2=0\n",
    "        wps = pd.read_csv('wpsites.csv')\n",
    "        c2=len(wps)\n",
    "\n",
    "        while i2<c2:\n",
    "            try:\n",
    "                html = scraper.get(str(wps.iloc[i2,0]), headers=header)\n",
    "                html.raise_for_status() # if response is successfull, no exception will be raised\n",
    "                soup = BeautifulSoup(html.content, 'html.parser')\n",
    "                content = soup.find('div', {'class', 'entry-content'})\n",
    "                try:\n",
    "                    h1 = soup.find('h1', {'class', 'entry-title'})\n",
    "                except:\n",
    "                    try:\n",
    "                        h1 = soup.find('h1')\n",
    "                    except:\n",
    "                        h1 = content.find('h1')\n",
    "                h1 = filter_html(str(h1))\n",
    "                try:\n",
    "                    p1 = filter_html(str(content.findAll('p')[0]))\n",
    "                except:\n",
    "                    try:\n",
    "                        p1 = filter_html(str(content.find('p')))\n",
    "                    except:\n",
    "                        p1 = content.find('p').text\n",
    "                H1.append(h1)\n",
    "                PH1_final.append(p1)\n",
    "                tags = []\n",
    "                h2_index = 0\n",
    "                p2_index = 0\n",
    "                h3_index = 0\n",
    "                p3_index = 0\n",
    "                for u in content:\n",
    "                    tags.append(str(u))\n",
    "                for u in range(len(tags)):\n",
    "                    if tags[u].startswith('<h2') or tags[u].startswith('<h3'):\n",
    "                        h2 = filter_html(tags[u])\n",
    "                        h2_index = u\n",
    "                        H2.append(h2)\n",
    "                        break\n",
    "                for u in range(h2_index, len(tags)):\n",
    "                    if tags[u].startswith('<p>'):\n",
    "                        p2 = filter_html(tags[u])\n",
    "                        p2_index = u\n",
    "                        PH2_final.append(p2)\n",
    "                        break\n",
    "                for u in range(p2_index, len(tags)):\n",
    "                    if tags[u].startswith('<h2') or tags[u].startswith('<h3'):\n",
    "                        h3 = filter_html(tags[u])\n",
    "                        h3_index = u\n",
    "                        H3.append(h3)\n",
    "                        break\n",
    "                for u in range(h3_index, len(tags)):\n",
    "                    if tags[u].startswith('<p>'):\n",
    "                        p3 = filter_html(tags[u])\n",
    "                        p3_index = u\n",
    "                        PH3_final.append(p3)\n",
    "                        break\n",
    "\n",
    "                if len(H1)<i2+1:\n",
    "                    H1.append(\" \")\n",
    "                if len(H2)<i2+1:\n",
    "                    H2.append(\" \")\n",
    "                if len(H3)<i2+1:\n",
    "                    H3.append(\" \")  \n",
    "\n",
    "                if len(PH1_final)<i2+1:\n",
    "                    PH1_final.append(\" \")    \n",
    "\n",
    "                if len(PH2_final)<i2+1:\n",
    "                    PH2_final.append(\" \")\n",
    "\n",
    "                if len(PH3_final)<i2+1:\n",
    "                    PH3_final.append(\" \")\n",
    "\n",
    "            except requests.exceptions.RequestException as e:\n",
    "                print(e)\n",
    "                pass\n",
    "            except requests.exceptions.HTTPError as err:\n",
    "                print(err)\n",
    "                pass\n",
    "            except cloudscraper.exceptions.CloudflareChallengeError as error:\n",
    "                print(error)\n",
    "                pass\n",
    "            \n",
    "            i2=i2+1\n",
    "        output_data['keyword'].append(input_user['Keyword'][i])\n",
    "        for k in range(c2):\n",
    "            output_data[f'Top{k+1} Link'].append(wps['WPsites'][k])\n",
    "            output_data[f'Top{k+1} H1'].append(H1[k])\n",
    "            output_data[f'Top{k+1} Paragraph after H1'].append(PH1_final[k])\n",
    "            output_data[f'Top{k+1} H2'].append(H2[k])\n",
    "            output_data[f'Top{k+1} Paragraph after H2'].append(PH2_final[k])\n",
    "            output_data[f'Top{k+1} H3'].append(H3[k])\n",
    "            output_data[f'Top{k+1} Paragraph after H3'].append(PH3_final[k])\n",
    "\n",
    "        output_data['Relative Searches'].append(relative_keywords)\n",
    "        \n",
    "        os.remove('wpsites.csv')\n",
    "        # loop to the next keyword\n",
    "        \n",
    "        if i==0:\n",
    "            df_output = pd.DataFrame(output_data)\n",
    "            df_output.to_csv('Output.csv', index=False)\n",
    "        else:\n",
    "            df_output = pd.DataFrame(output_data)\n",
    "            df_output.to_csv('Output.csv', index=False, mode='a', header=False)\n",
    "        i=i+1\n",
    "        if UseDelay:\n",
    "            time.sleep(60) # sleep 60 seconds\n",
    "main()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f8ba85ab",
   "metadata": {},
   "outputs": [],
   "source": [
    "wps = pd.read_csv('wpsites.csv')\n",
    "for i in range(len(wps)):\n",
    "    print(wps['WPsites'][i])\n",
    "input_user = pd.read_csv('keyword_example.csv')\n",
    "for i in range(len(input_user)):\n",
    "    print(input_user['Keyword'][i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "343853b0",
   "metadata": {},
   "outputs": [],
   "source": [
    "def filter_html(string):\n",
    "    string2 = ''\n",
    "    for i in string:\n",
    "        string2 +=i[0]\n",
    "    check1 = 0\n",
    "    check2 = 0\n",
    "    while check1!=-1 and check2!=-1:\n",
    "        if '<' and '>' in string2:\n",
    "            end = string2.index('>')\n",
    "            start = string2.index('<')\n",
    "            string2 = string2.replace(string2[start:end+1], ' ')\n",
    "            check1 = string2.find('<')\n",
    "            check2 = string2.find('>')\n",
    "            print(string2)\n",
    "    return string2.strip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f972a37d",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "links = urls\n",
    "c1=len(links)\n",
    "i1=0\n",
    "\n",
    "wp=0\n",
    "wpsites=[]\n",
    "\n",
    "while i1<c1:\n",
    "\n",
    "    try:\n",
    "        response = scraper.get(str(links[i1]), timeout=10)\n",
    "        response.raise_for_status() # if response is successfull, no exception will be raised\n",
    "        bsh = BeautifulSoup(response.content, 'html.parser')\n",
    "        marker=bsh.find(class_='entry-content')\n",
    "\n",
    "        if \"wp-content\" in response.text and marker:\n",
    "            wpsites.append(links[i1])\n",
    "            wp=wp+1\n",
    "\n",
    "        if wp>9:\n",
    "            break    \n",
    "    except requests.exceptions.RequestException as e:\n",
    "        xxx=1\n",
    "        #print(e)\n",
    "\n",
    "    except requests.exceptions.HTTPError as err:\n",
    "        xxx=1\n",
    "        #print(err)\n",
    "\n",
    "\n",
    "\n",
    "    i1=i1+1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "0a228e01",
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "import urllib\n",
    "from urllib.request import urlopen\n",
    "from bs4 import BeautifulSoup\n",
    "import os\n",
    "import re\n",
    "import pandas as pd\n",
    "import csv\n",
    "import time\n",
    "from requests_html import HTML\n",
    "from requests_html import HTMLSession\n",
    "from itertools import cycle\n",
    "import cloudscraper"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "68115c95",
   "metadata": {},
   "outputs": [],
   "source": [
    "scraper = cloudscraper.create_scraper()\n",
    "html = requests.get('https://www.caninejournal.com/how-to-housebreak-a-puppy/')\n",
    "soup = BeautifulSoup(html.content, 'html.parser')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "4ac4fde0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "None\n"
     ]
    }
   ],
   "source": [
    "content = soup.select('[class*=entry-content]')\n",
    "print(content)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4be5653d",
   "metadata": {},
   "outputs": [],
   "source": [
    "H1 = []\n",
    "H1.append(x[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5979e0b2",
   "metadata": {},
   "outputs": [],
   "source": [
    "marker=soup.find(class_='entry-content')\n",
    "for line in marker:\n",
    "    if str(line).startswith('<h') and h==3:\n",
    "        h=4\n",
    "\n",
    "        break\n",
    "\n",
    "    if str(line).startswith('<p>') and h==3:\n",
    "\n",
    "        temp = line.text\n",
    "\n",
    "#         PH3.append(temp7)\n",
    "        print(temp)\n",
    "\n",
    "    if str(line).startswith('<h') and h==2:\n",
    "        h=3\n",
    "        temp = line.text\n",
    "        print(temp)\n",
    "#         H3.append(temp6)\n",
    "\n",
    "    if str(line).startswith('<p>') and h==2:\n",
    "\n",
    "        temp = line.text\n",
    "        print(temp)       \n",
    "\n",
    "    if str(line).startswith('<h') and h==1:\n",
    "        h=2\n",
    "\n",
    "        temp = line.text\n",
    "        print(temp)\n",
    "\n",
    "    if str(line).startswith('<p>') and h==1:\n",
    "\n",
    "        temp = line.text\n",
    "        print(temp)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cb83d458",
   "metadata": {},
   "outputs": [],
   "source": [
    "H1=[]\n",
    "PH1_final=[]\n",
    "H2=[]\n",
    "PH2_final=[]\n",
    "H3=[]\n",
    "PH3_final=[]\n",
    "\n",
    "i2=0\n",
    "c2=len(wpsites)\n",
    "\n",
    "while i2<c2:\n",
    "    PH1=[]\n",
    "    PH2=[]\n",
    "    PH3=[]\n",
    "    try:\n",
    "        html = requests.get(wpsites[i2])\n",
    "        html.raise_for_status() # if response is successfull, no exception will be raised\n",
    "        bsh = BeautifulSoup(html.content, 'html.parser')\n",
    "\n",
    "\n",
    "        #H1\n",
    "        x=re.findall('<h.*>(.*)</h1>', str(bsh.h1))\n",
    "        try:\n",
    "            H1.append(x[0])\n",
    "        except:\n",
    "            H1.append(\" \")\n",
    "\n",
    "        marker=bsh.find(class_='entry-content')\n",
    "\n",
    "        h=1\n",
    "\n",
    "        # rest of H & P\n",
    "        for line in marker:\n",
    "            if str(line).startswith('<h') and h==3:\n",
    "                h=4\n",
    "\n",
    "                break\n",
    "\n",
    "            if str(line).startswith('<p>') and h==3:\n",
    "\n",
    "                temp = line.text\n",
    "\n",
    "                PH3.append(temp)\n",
    "\n",
    "            if str(line).startswith('<h') and h==2:\n",
    "                h=3\n",
    "                temp = line.text\n",
    "                H3.append(temp)\n",
    "\n",
    "            if str(line).startswith('<p>') and h==2:\n",
    "\n",
    "                temp = line.text\n",
    "                PH2.append(temp)       \n",
    "\n",
    "            if str(line).startswith('<h') and h==1:\n",
    "                h=2\n",
    "\n",
    "                temp = line.text\n",
    "                H2.append(temp)\n",
    "\n",
    "            if str(line).startswith('<p>') and h==1:\n",
    "\n",
    "                temp = line.text\n",
    "                PH1.append(temp)\n",
    "        PH1_final.append(\"\\n\".join(PH1))\n",
    "        PH2_final.append(\"\\n\".join(PH2))\n",
    "        PH3_final.append(\"\\n\".join(PH3))\n",
    "\n",
    "        if len(H1)<i2+1:\n",
    "            H1.append(\" \")\n",
    "        if len(H2)<i2+1:\n",
    "            H2.append(\" \")\n",
    "        if len(H3)<i2+1:\n",
    "            H3.append(\" \")  \n",
    "\n",
    "        if len(PH1_final)<i2+1:\n",
    "            PH1_final.append(\" \")    \n",
    "\n",
    "        if len(PH2_final)<i2+1:\n",
    "            PH2_final.append(\" \")\n",
    "\n",
    "        if len(PH3_final)<i2+1:\n",
    "            PH3_final.append(\" \")\n",
    "\n",
    "    except requests.exceptions.RequestException as e:\n",
    "        print(e)\n",
    "\n",
    "    except requests.exceptions.HTTPError as err:\n",
    "        print(err)\n",
    "    i2=i2+1\n",
    "\n",
    "        PH1_final.append(\"\\n\".join(PH1))\n",
    "        PH2_final.append(\"\\n\".join(PH2))\n",
    "        PH3_final.append(\"\\n\".join(PH3))\n",
    "\n",
    "        if len(H1)<i2+1:\n",
    "            H1.append(\" \")\n",
    "        if len(H2)<i2+1:\n",
    "            H2.append(\" \")\n",
    "        if len(H3)<i2+1:\n",
    "            H3.append(\" \")  \n",
    "\n",
    "        if len(PH1_final)<i2+1:\n",
    "            PH1_final.append(\" \")    \n",
    "\n",
    "        if len(PH2_final)<i2+1:\n",
    "            PH2_final.append(\" \")\n",
    "\n",
    "        if len(PH3_final)<i2+1:\n",
    "            PH3_final.append(\" \")\n",
    "    \n",
    "\n",
    "\n",
    "try:\n",
    "    output_data.append([str(input_user.iloc[i,0]),H1[0],PH1_final[0],H2[0],PH2_final[0],H3[0],PH3_final[0],H1[1],PH1_final[1],H2[1],PH2_final[1],H3[1],PH3_final[1],H1[2],PH1_final[2],H2[2],PH2_final[2],H3[2],PH3_final[2],H1[3],PH1_final[3],H2[3],PH2_final[3],H3[3],PH3_final[3],H1[4],PH1_final[4],H2[4],PH2_final[4],H3[4],PH3_final[4],H1[5],PH1_final[5],H2[5],PH2_final[5],H3[5],PH3_final[5],H1[6],PH1_final[6],H2[6],PH2_final[6],H3[6],PH3_final[6],H1[7],PH1_final[7],H2[7],PH2_final[7],H3[7],PH3_final[7],H1[8],PH1_final[8],H2[8],PH2_final[8],H3[8],PH3_final[8],H1[9],PH1_final[9],H2[9],PH2_final[9],H3[9],PH3_final[9]])\n",
    "except:\n",
    "    output_data.append([str(input_user.iloc[i,0]),\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \"])\n",
    "\n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "687d95e1",
   "metadata": {},
   "outputs": [],
   "source": [
    "for line in marker:\n",
    "    if (str(line).startswith('<p')):\n",
    "        print(line.text)\n",
    "        break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "61eb5a72",
   "metadata": {},
   "outputs": [],
   "source": [
    "for line in marker:\n",
    "    print(str(line))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "88345843",
   "metadata": {},
   "outputs": [],
   "source": [
    "wpsites[-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d69c3230",
   "metadata": {},
   "outputs": [],
   "source": [
    "input_user = pd.read_csv('keyword_example.csv')\n",
    "c = len(input_user)\n",
    "c"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6bcc8791",
   "metadata": {},
   "outputs": [],
   "source": [
    "string = '<a href=\"https://pupbox.com/training/pupbox-picks-teething-product-guide/\">here</a>'\n",
    "start = string.index('<')\n",
    "ends = string.index('>')\n",
    "string = string[ends+1:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "62782109",
   "metadata": {},
   "outputs": [],
   "source": [
    "string"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0f7bdba3",
   "metadata": {},
   "outputs": [],
   "source": [
    "string.find('c')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f23f0777",
   "metadata": {},
   "outputs": [],
   "source": [
    "def filter_html(string):\n",
    "    string2 = ''\n",
    "    for i in string:\n",
    "        string2 +=i[0]\n",
    "    check1 = 0\n",
    "    check2 = 0\n",
    "    while check1!=-1 and check2!=-1:\n",
    "        end = string2.index('>')\n",
    "        start = string2.index('<')\n",
    "        string2 = string2.replace(string2[start:end+1], ' ')\n",
    "        check1 = string2.find('<')\n",
    "        check2 = string2.find('>')\n",
    "    return string2.strip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3234f71e",
   "metadata": {},
   "outputs": [],
   "source": [
    "string = [['<a href=\"https://pupbox.com/training/pupbox-picks-teething-product-guide/\">here</a>'],['<img alt=\"When Do Puppies Lose Their Baby Teeth\" class=\"alignnone size-full wp-image-392323\" height=\"555\" loading=\"lazy\" sizes=\"(max-width: 1024px) 100vw, 1024px\" src=\"https://pupbox.com/wp-content/uploads/2019/07/When-Do-Puppies-Lose-Their-Baby-Teeth_.jpg\" srcset=\"https://www.pupbox.com/wp-content/uploads/2019/07/When-Do-Puppies-Lose-Their-Baby-Teeth_.jpg 1024w, https://www.pupbox.com/wp-content/uploads/2019/07/When-Do-Puppies-Lose-Their-Baby-Teeth_-701x380.jpg 701w, https://www.pupbox.com/wp-content/uploads/2019/07/When-Do-Puppies-Lose-Their-Baby-Teeth_-768x416.jpg 768w, https://www.pupbox.com/wp-content/uploads/2019/07/When-Do-Puppies-Lose-Their-Baby-Teeth_-380x206.jpg 380w\" width=\"1024\"/></p><p>Just like human babies, animals have baby teeth that need to be replaced as they get older. This is true of puppies; it is a natural part of life, but it makes things no less scary when you notice your own pup losing his at random.</p>']]\n",
    "filter_html(string)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2eb36eb4",
   "metadata": {},
   "outputs": [],
   "source": [
    "end = string.index('>')\n",
    "start = string.index('<')\n",
    "string.replace(string[start:end+1], ' ')\n",
    "check1 = string.find('<')\n",
    "check2 = string.find('>')\n",
    "print(string)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f06211cf",
   "metadata": {},
   "outputs": [],
   "source": [
    "string[start:end+1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3ccec273",
   "metadata": {},
   "outputs": [],
   "source": [
    "string.replace(string[start:end+1], ' ')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e6ea5e3b",
   "metadata": {},
   "outputs": [],
   "source": [
    "string = '<img alt=\"When Do Puppies Lose Their Baby Teeth\" class=\"alignnone size-full wp-image-392323\" height=\"555\" loading=\"lazy\" sizes=\"(max-width: 1024px) 100vw, 1024px\" src=\"https://pupbox.com/wp-content/uploads/2019/07/When-Do-Puppies-Lose-Their-Baby-Teeth_.jpg\" srcset=\"https://www.pupbox.com/wp-content/uploads/2019/07/When-Do-Puppies-Lose-Their-Baby-Teeth_.jpg 1024w, https://www.pupbox.com/wp-content/uploads/2019/07/When-Do-Puppies-Lose-Their-Baby-Teeth_-701x380.jpg 701w, https://www.pupbox.com/wp-content/uploads/2019/07/When-Do-Puppies-Lose-Their-Baby-Teeth_-768x416.jpg 768w, https://www.pupbox.com/wp-content/uploads/2019/07/When-Do-Puppies-Lose-Their-Baby-Teeth_-380x206.jpg 380w\" width=\"1024\"/></p><p>Just like human babies, animals have baby teeth that need to be replaced as they get older. This is true of puppies; it is a natural part of life, but it makes things no less scary when you notice your own pup losing his at random.</p>'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a87e079c",
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "import urllib\n",
    "from urllib.request import urlopen\n",
    "from bs4 import BeautifulSoup\n",
    "import os\n",
    "import re\n",
    "import pandas as pd\n",
    "import time\n",
    "from requests_html import HTML\n",
    "from requests_html import HTMLSession\n",
    "from itertools import cycle\n",
    "import cloudscraper\n",
    "scraper = cloudscraper.create_scraper()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3b4ae57e",
   "metadata": {},
   "outputs": [],
   "source": [
    "UseProxy = False\n",
    "def get_source(url):\n",
    "    \n",
    "    if UseProxy:\n",
    "        done=False\n",
    "        while done==False:\n",
    "            \n",
    "            proxy_pool = cycle(proxies)\n",
    "            for p in range(1,11):\n",
    "                #Get a proxy from the pool\n",
    "                proxy = next(proxy_pool)\n",
    "                \n",
    "                try:\n",
    "                    session = requests.session()\n",
    "                    session = cloudscraper.create_scraper(sess = session)\n",
    "                    session.proxies = {\n",
    "                                \"http\": proxy,\n",
    "                                \"https\": proxy,\n",
    "                                }\n",
    "                    response = session.get(url, headers=header)\n",
    "                    response.raise_for_status() # if response is successfull, no exception will be raised\n",
    "                    done=True\n",
    "                    \n",
    "                    return response\n",
    "                    \n",
    "                \n",
    "                except requests.exceptions.RequestException as e:\n",
    "                    print(e)\n",
    "                    print(\" \")\n",
    "                    print(\"retrying with new proxy\")\n",
    "                    \n",
    "                except requests.exceptions.HTTPError as err:\n",
    "                    print(err)\n",
    "                    print(\" \")\n",
    "                    print(\"retrying with new proxy\")    \n",
    "    else:\n",
    "        try:\n",
    "            session = requests.session()\n",
    "            session = cloudscraper.create_scraper(sess = session)\n",
    "            session.headers['user-agent'] = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.81 Safari/537.36'\n",
    "            response = session.get(url)\n",
    "            response.raise_for_status() # if response is successfull, no exception will be raised   \n",
    "            return response\n",
    "                            \n",
    "        except requests.exceptions.RequestException as e:\n",
    "            print(e)\n",
    "                                \n",
    "        except requests.exceptions.HTTPError as err:\n",
    "            print(err)\n",
    "            \n",
    "        except cloudscraper.exceptions.CloudflareChallengeError as error:\n",
    "            print(error)\n",
    "            pass\n",
    "            \n",
    "def scrape_google(query):\n",
    "\n",
    "     \n",
    "    query = urllib.parse.quote_plus(query)\n",
    "    \n",
    "   \n",
    "    response = get_source(\"https://www.google.com/search?q=\" + query + \"&num=100\")\n",
    "    soup = BeautifulSoup(response.content, 'html.parser')\n",
    "    \n",
    "    relative_keywords = soup.find('div', {'çlass':'y6Uyqe'})\n",
    "    decks = soup.findAll('div', {'class':'AJLUJb'})\n",
    "    searches = []\n",
    "    for deck in decks:\n",
    "        cards = deck.findAll('div')\n",
    "        for card in cards:\n",
    "            if card.text != '':\n",
    "                searches.append(card.text)\n",
    "    cards = soup.findAll('div', {'class' : 'yuRUbf'})\n",
    "    links = []\n",
    "    for card in cards:\n",
    "        link = card.find('a')['href']\n",
    "        links.append(link)\n",
    "        \n",
    "    google_domains = ('https://www.google.', \n",
    "                      'https://google.', \n",
    "                      'https://webcache.googleusercontent.', \n",
    "                      'http://webcache.googleusercontent.', \n",
    "                      'https://policies.google.',\n",
    "                      'https://support.google.',\n",
    "                      'https://maps.google.',\n",
    "                      'https://translate.google.',\n",
    "                      'https://www.youtube')\n",
    "\n",
    "    for url in links[:]:\n",
    "        \n",
    "        if url.startswith(google_domains):\n",
    "            links.remove(url)\n",
    "    \n",
    "    \n",
    "    return links, searches"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "98f14ef3",
   "metadata": {},
   "outputs": [],
   "source": [
    "output_data=[]\n",
    "input_user = pd.read_csv('keyword_example.csv')\n",
    "c = len(input_user)\n",
    "i=0\n",
    "while i<1:\n",
    "\n",
    "    results, relative_keywords=scrape_google(str(input_user.iloc[i,0]))\n",
    "#         print(results)\n",
    "    df = pd.DataFrame({'Links':results}) \n",
    "    df.to_csv('Links.csv', index=False)\n",
    "\n",
    "    # detecting wordpress sites\n",
    "    links = pd.read_csv('Links.csv')\n",
    "    c1=len(links)\n",
    "    i1=0\n",
    "\n",
    "    wp=0\n",
    "    wpsites=[]\n",
    "\n",
    "    while i1<c1:\n",
    "        try:\n",
    "            response = scraper.get(str(links.iloc[i1,0]), timeout=60)\n",
    "            response.raise_for_status() # if response is successfull, no exception will be raised\n",
    "            bsh = BeautifulSoup(response.content, 'html.parser')\n",
    "            marker=bsh.find(class_='entry-content')\n",
    "\n",
    "            if \"wp-content\" in response.text and marker:\n",
    "                wpsites.append(str(links.iloc[i1,0]))\n",
    "                wp=wp+1\n",
    "\n",
    "            if wp>9:\n",
    "                break    \n",
    "        except requests.exceptions.RequestException as e:\n",
    "            xxx=1\n",
    "            # print(e)\n",
    "\n",
    "        except requests.exceptions.HTTPError as err:\n",
    "            xxx=1\n",
    "            # print(err)\n",
    "\n",
    "\n",
    "\n",
    "        i1=i1+1\n",
    "    df1 = pd.DataFrame(wpsites, columns=['WPsites']) \n",
    "    print(wpsites)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "56e7f863",
   "metadata": {},
   "outputs": [],
   "source": [
    "wpsites = ['https://www.puppyleaks.com/potty-training-your-puppy/', 'https://puppyintraining.com/how-to-potty-train-a-puppy-in-an-apartment/', 'https://www.rover.com/blog/complete-guide-puppy-potty-training/', 'https://kaufmannspuppytraining.com/en/how-to-potty-train-a-puppy-fast/', 'https://thezebra.org/2018/10/01/potty-training-101-housetraining-your-new-puppy-in-two-weeks-or-less/', 'https://drsophiayin.com/blog/entry/a_foolproof_potty-training_plan/', 'https://cherishpetfood.com.au/tag/can-an-8-week-old-puppy-be-potty-trained/', 'https://journeydogtraining.com/how-to-potty-train-a-puppy-fast/', 'https://www.dogtrainingnow.com/2018/02/common-potty-training-mistakes/', 'https://www.caninejournal.com/how-to-housebreak-a-puppy/']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "16fa6bc9",
   "metadata": {},
   "outputs": [],
   "source": [
    "sess = requests.session()\n",
    "scraper = cloudscraper.create_scraper(sess)\n",
    "header = {\n",
    "    'user-agent' : 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.81 Safari/537.36'\n",
    "}\n",
    "\n",
    "c2 = len(wpsites)\n",
    "for i in wpsites:\n",
    "    print(i)\n",
    "    html = scraper.get(str(i), headers=header)\n",
    "    html.raise_for_status() # if response is successfull, no exception will be raised\n",
    "    soup = BeautifulSoup(html.content, 'html.parser')\n",
    "    content = soup.find('div', {'class', 'entry-content'})\n",
    "\n",
    "    try:\n",
    "        h1 = soup.find('h1', {'class', 'entry-title'}).text.strip()\n",
    "    except:\n",
    "        h1 = soup.find('h1').text.strip()\n",
    "    \n",
    "    p1 = content.find('p').text.strip()\n",
    "    if p1 == '':\n",
    "        p1 = content.findAll('p')[1].text\n",
    "    print(h1)\n",
    "    print(p1)\n",
    "    tags = []\n",
    "    h2_index = 0\n",
    "    p2_index = 0\n",
    "    h3_index = 0\n",
    "    p3_index = 0\n",
    "    for i in content:\n",
    "        tags.append(str(i))\n",
    "    for i in range(len(tags)):\n",
    "        if tags[i].startswith('<h2') or tags[i].startswith('<h3'):\n",
    "            h2 = filter_html(tags[i])\n",
    "            h2_index = i\n",
    "            print(h2)\n",
    "            break\n",
    "    for i in range(h2_index, len(tags)):\n",
    "        if tags[i].startswith('<p>'):\n",
    "            p2 = filter_html(tags[i])\n",
    "            p2_index = i\n",
    "            print(p2)\n",
    "            break\n",
    "    for i in range(p2_index, len(tags)):\n",
    "        if tags[i].startswith('<h2') or tags[i].startswith('<h3'):\n",
    "            h3 = filter_html(tags[i])\n",
    "            h3_index = i\n",
    "            print(h3)\n",
    "            break\n",
    "    for i in range(h3_index, len(tags)):\n",
    "        if tags[i].startswith('<p>'):\n",
    "            p3 = filter_html(tags[i])\n",
    "            p3_index = i\n",
    "            print(p3)\n",
    "            break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "88e52bd4",
   "metadata": {},
   "outputs": [],
   "source": [
    "wpsites"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7d5dd19c",
   "metadata": {},
   "outputs": [],
   "source": [
    "tags = []\n",
    "h2_index = 0\n",
    "p2_index = 0\n",
    "h3_index = 0\n",
    "p3_index = 0\n",
    "for i in content:\n",
    "    tags.append(str(i))\n",
    "for i in range(len(tags)):\n",
    "    if tags[i].startswith('<h2') or tags[i].startswith('<h3'):\n",
    "        h2 = filter_html(tags[i])\n",
    "        h2_index = i\n",
    "        print(h2)\n",
    "        break\n",
    "for i in range(h2_index, len(tags)):\n",
    "    if tags[i].startswith('<p>'):\n",
    "        p2 = filter_html(tags[i])\n",
    "        p2_index = i\n",
    "        print(p2)\n",
    "        break\n",
    "for i in range(p2_index, len(tags)):\n",
    "    if tags[i].startswith('<h2') or tags[i].startswith('<h3'):\n",
    "        h3 = filter_html(tags[i])\n",
    "        h3_index = i\n",
    "        print(h3)\n",
    "        break\n",
    "for i in range(h3_index, len(tags)):\n",
    "    if tags[i].startswith('<p>'):\n",
    "        p3 = filter_html(tags[i])\n",
    "        p3_index = i\n",
    "        print(p3)\n",
    "        break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "164580e1",
   "metadata": {},
   "outputs": [],
   "source": [
    "output_data = [(input_user['Keyword'][i]), wps['WPsites'][0],H1[0],PH1_final[0],H2[0],PH2_final[0],H3[0],PH3_final[0], wps['WPsites'][1],H1[1],PH1_final[1],H2[1],PH2_final[1],H3[1],PH3_final[1], wps['WPsites'][2],H1[2],PH1_final[2],H2[2],PH2_final[2],H3[2],PH3_final[2], wps['WPsites'][3],H1[3],PH1_final[3],H2[3],PH2_final[3],H3[3],PH3_final[3], wps['WPsites'][4],H1[4],PH1_final[4],H2[4],PH2_final[4],H3[4],PH3_final[4], wps['WPsites'][5],H1[5],PH1_final[5],H2[5],PH2_final[5],H3[5],PH3_final[5], wps['WPsites'][6],H1[6],PH1_final[6],H2[6],PH2_final[6],H3[6],PH3_final[6], wps['WPsites'][7],H1[7],PH1_final[7],H2[7],PH2_final[7],H3[7],PH3_final[7],wps['WPsites'][8],H1[8],PH1_final[8],H2[8],PH2_final[8],H3[8],PH3_final[8],wps['WPsites'][9],H1[9],PH1_final[9],H2[9],PH2_final[9],H3[9],PH3_final[9],relative_keywords]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c689db00",
   "metadata": {},
   "outputs": [],
   "source": [
    "header = {\n",
    "    'user-agent' : 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.81 Safari/537.36'\n",
    "}\n",
    "\n",
    "html = scraper.get('https://www.puppyleaks.com/potty-training-your-puppy/', headers=header)\n",
    "html.raise_for_status() # if response is successfull, no exception will be raised\n",
    "soup = BeautifulSoup(html.content, 'html.parser')\n",
    "content = soup.find('div', {'class', 'entry-content'})\n",
    "h1 = content.findAll('p')\n",
    "h1 = h1[0].text + h1[1].text\n",
    "print(h1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "9e11ae74",
   "metadata": {},
   "outputs": [],
   "source": [
    "proxies = ['154.17.87.216:29842:kgrzes:wKcGrD43', '154.17.87.128:29842:kgrzes:wKcGrD43', '154.17.87.149:29842:kgrzes:wKcGrD43', '154.17.87.221:29842:kgrzes:wKcGrD43', '154.17.87.113:29842:kgrzes:wKcGrD43']\n",
    "def filter_string(string):\n",
    "    if '[' or ']' in string:\n",
    "        string = string.replace('[', ' ')\n",
    "        string = string.replace(']', ' ')\n",
    "    string = string.replace(\"'\", \" \")\n",
    "    string.replace('  ', ' ')\n",
    "    string.replace('   ', ' ')\n",
    "    string = string.strip()\n",
    "    return string\n",
    "def filter_html(string):\n",
    "    check1 = 0\n",
    "    check2 = 0\n",
    "    while check1!=-1 and check2!=-1:\n",
    "        if '<' and '>' in string:\n",
    "            end = string.index('>')\n",
    "            start = string.index('<')\n",
    "            string = string.replace(string[start:end+1], ' ')\n",
    "            check1 = string.find('<')\n",
    "            check2 = string.find('>')\n",
    "        else:\n",
    "            break\n",
    "    return string.strip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "f4b5b004",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5 Simple Tips For Potty Training Your Puppy\n",
      "Last updated on  December 13, 2021  By   Puppy Leaks     15 CommentsIn the late 90’s I found a rather simple way to potty train a puppy, and it worked so well I’ve been using it ever since. After a week of using these tips you’ll eliminate 90% of accidents, and you’ll be well on the way to a having a fully house trained pup.\n",
      "1. Pay Attention to Your Puppy At All Times\n",
      "A quiet puppy is trouble, or so the saying goes. Whether he’s getting into the garbage, eating your new shoes or pooping over in the corner — a quiet puppy signals trouble.\n",
      "If you want to prevent accidents before they happen you’re going to need to watch your pup at all times, including every time they wander off. It only takes one accident to set your training back. Now I know that watching your puppy non-stop isn’t exactly fun &amp; exciting, but being able to catch them before they have an accident is why this method works so well.\n",
      "If you’re like me and have trouble keeping up with your pup at all times try using a tether. You can buy a tether from the pet store or simply do what I did and use a long lead or leash. If tethering your pup to you at all times is what it takes to make sure they’re not sneaking off then go for it. If you’re not keen on keeping your dog tethered you can use baby gates or closed doors to restrict your dogs access to the whole house.\n",
      "2. Don’t Leave Your Puppy Unattended\n",
      "Did I mention the importance of not letting your dog out of sight? I did, but this part is so important I need to mention it twice. Your job when house training is to be there to prevent accidents before they happen. You know when your dog is going to have an accident? The moment you’re not looking.\n",
      "There’s not much you can do that makes sense after your dog has had an accident  — and you’ve missed out on an important training lesson.\n",
      "Don’t punish your dog if they pee inside. Regardless of all those old training ideas punishment  isn’t a good deterrent  for house training. Yelling at your dog after the fact just confuses them and makes them nervous around you. If you catch your did in the act you can try to get their attention &amp; move them outdoors. If you’re successful &amp; they continue going once you get outside praise them like crazy.\n",
      "Your pup is going to have an accident or two in the house – there’s no getting around it. What you can do is prevent them from having more by being proactive. Keep them in your sight at all times and take them out every time they start to wander off.\n"
     ]
    }
   ],
   "source": [
    "html = scraper.get(wpsites[0], headers=header)\n",
    "html.raise_for_status() # if response is successfull, no exception will be raised\n",
    "soup = BeautifulSoup(html.content, 'html.parser')\n",
    "content = soup.find('div', {'class', 'entry-content'})\n",
    "if content == None:\n",
    "    content = soup.select('[class*=entry-content]')[0]\n",
    "try:\n",
    "    h1 = soup.find('h1', {'class', 'entry-title'})\n",
    "except:\n",
    "    try:\n",
    "        h1 = soup.find('h1')\n",
    "    except:\n",
    "        h1 = content.find('h1')\n",
    "h1 = filter_html(str(h1))\n",
    "try:\n",
    "    full_p1 = ''\n",
    "    p1 = content.findAll('p')[0:2]\n",
    "    for para in p1:\n",
    "        p1 = filter_html(str(para))\n",
    "        full_p1+=p1\n",
    "except:\n",
    "    try:\n",
    "        p1 = filter_html(str(content.find('p')))\n",
    "    except:\n",
    "        try:  \n",
    "            p1 = content.find('p').text\n",
    "        except:\n",
    "            try:\n",
    "                p1 = soup.find('p').text\n",
    "            except:\n",
    "                pass\n",
    "print(h1)\n",
    "print(full_p1)\n",
    "tags = []\n",
    "h2_index = 0\n",
    "p2_index = 0\n",
    "h3_index = 0\n",
    "p3_index = 0\n",
    "for u in content:\n",
    "    tags.append(str(u))\n",
    "for u in range(len(tags)):\n",
    "    if tags[u].startswith('<h2') or tags[u].startswith('<h3'):\n",
    "        h2 = filter_html(tags[u])\n",
    "        h2_index = u\n",
    "        print(h2)\n",
    "        break\n",
    "for u in range(h2_index+1, len(tags)):\n",
    "    full_p2 = ''\n",
    "    if tags[u].startswith('<h2') or tags[u].startswith('<h3'):\n",
    "        break\n",
    "    if tags[u].startswith('<p>'):\n",
    "        p2 = filter_html(tags[u])\n",
    "        full_p2 += p2\n",
    "        p2_index = u\n",
    "        print(full_p2)\n",
    "for u in range(p2_index, len(tags)):\n",
    "    if tags[u].startswith('<h2') or tags[u].startswith('<h3'):\n",
    "        h3 = filter_html(tags[u])\n",
    "        h3_index = u\n",
    "        print(h3)\n",
    "        break\n",
    "for u in range(h3_index+1, len(tags)):\n",
    "    full_p3 = ''\n",
    "    if tags[u].startswith('<h2') or tags[u].startswith('<h3'):\n",
    "        break\n",
    "    if tags[u].startswith('<p>'):\n",
    "        p3 = filter_html(tags[u])\n",
    "        full_p3+=p3\n",
    "        p3_index = u\n",
    "        print(full_p3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "b36f9b93",
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "import urllib\n",
    "from urllib.request import urlopen\n",
    "from bs4 import BeautifulSoup\n",
    "import os\n",
    "import re\n",
    "import pandas as pd\n",
    "import csv\n",
    "import time\n",
    "from requests_html import HTML\n",
    "from requests_html import HTMLSession\n",
    "from itertools import cycle\n",
    "import cloudscraper\n",
    "\n",
    "sess = requests.session()\n",
    "scraper = cloudscraper.create_scraper(sess)\n",
    "header = {\n",
    "    'user-agent' : 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.81 Safari/537.36'\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c7fa215a",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}