Merge pull request #10 from beudbeud/fix/request_fail

fix: replace request by subprocess lynx
beudbeud · Jan 14, 2024 · 8a09a21 · 8a09a21
2 parents b27a4f8 + bea8697
commit 8a09a21
Show file tree

Hide file tree

Showing 4 changed files with 7 additions and 6 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -2,7 +2,7 @@ FROM alpine:3.17
 
 LABEL maintainer="[email protected]"
 
-RUN apk update && apk add build-base python3 py3-pip
+RUN apk update && apk add build-base python3 py3-pip lynx
 
 COPY . /src
 

diff --git a/discoger/_info.py b/discoger/_info.py
@@ -1,4 +1,4 @@
-__version__ = "2.1.5"
+__version__ = "2.1.6"
 
 __title__ = "discoger"
 __author__ = "Beudbeud"

diff --git a/discoger/scrap.py b/discoger/scrap.py
@@ -1,14 +1,14 @@
 from bs4 import BeautifulSoup
-import requests
 import re
 import logging
+import delegator
 
 
 class DiscogsScraper:
     def __init__(self, url, d):
-        headers = {"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1)"}
-        req = requests.get(url, headers=headers)
-        soup = BeautifulSoup(req.text, "html.parser")
+        cmd = "lynx -source -accept_all_cookies %s" % url
+        req = delegator.run(cmd)
+        soup = BeautifulSoup(req.out, "html.parser")
         self.soup = soup
         self.release_id = re.findall(r"\d+", soup.select("a.release-page")[0]["href"])[
             0

diff --git a/requirements.txt b/requirements.txt
@@ -17,3 +17,4 @@ six==1.16.0
 soupsieve==2.4.1
 urllib3==2.0.2
 yamldb==0.1.3
+delegator.py==0.1.1