-
Notifications
You must be signed in to change notification settings - Fork 1
/
utils.py
30 lines (24 loc) · 840 Bytes
/
utils.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
import undetected_chromedriver.v2 as uc
import base64
from time import sleep
import os
import asyncio
import requests
import json
import sys
def scrapeWebsite(url):
options = uc.ChromeOptions()
options.binary_location = os.environ.get("GOOGLE_CHROME_BIN")
options.add_argument("--headless")
options.add_argument("--disable-dev-shm-usage")
options.add_argument("--no-sandbox")
driver = uc.Chrome(executable_path=os.environ.get("CHROMEDRIVER_PATH"), keep_alive=True, version_main=90, patcher_force_close=True, options=options, enable_cdp_events=True)
driver.get(url)
print("website loaded successfully")
try:
html = driver.page_source
driver.quit()
return html
except:
driver.quit()
return "No website found with this URL."