Fix category parsing due to artvee.com HTML changes

zduclos · Jul 3, 2022 · a4fb8ec · a4fb8ec
1 parent b07fc1e
commit a4fb8ec
Show file tree

Hide file tree

Showing 2 changed files with 6 additions and 9 deletions.
diff --git a/artvee_scraper/__init__.py b/artvee_scraper/__init__.py
@@ -1 +1 @@
-__version__ = "2.0.0"
+__version__ = "2.0.1"
diff --git a/artvee_scraper/scraper.py b/artvee_scraper/scraper.py
@@ -76,7 +76,8 @@ def start(self):
                 logger.info("Processing %s (%d/%d)",
                             category, page, page_count)
                 page_url = f"https://www.artvee.com/c/{category}/page/{page}/?per_page={ArtveeScraper._ITEMS_PER_PAGE}"
-                artwork_list = ArtveeScraper._scrape_artwork_data(page_url)
+                artwork_list = ArtveeScraper._scrape_artwork_data(
+                    page_url, category.value.capitalize())
 
                 results = self.workers.map(self._worker_task, artwork_list)
 
@@ -179,7 +180,7 @@ def _num_pages_for_category(category: CategoryType) -> int:
         return 0
 
     @staticmethod
-    def _scrape_artwork_data(page_url: str) -> List[Artwork]:
+    def _scrape_artwork_data(page_url: str, category: str) -> List[Artwork]:
         scraped_artwork = []
 
         try:
@@ -195,7 +196,7 @@ def _scrape_artwork_data(page_url: str) -> List[Artwork]:
                 )
 
                 for meta in all_metadata_html:
-                    if artwork := ArtveeScraper._parse_metadata_html(meta):
+                    if artwork := ArtveeScraper._parse_metadata_html(meta, category):
                         scraped_artwork.append(artwork)
 
             else:
@@ -214,15 +215,11 @@ def _scrape_artwork_data(page_url: str) -> List[Artwork]:
         return scraped_artwork
 
     @staticmethod
-    def _parse_metadata_html(metadata_html: Tag) -> Optional[Artwork]:
+    def _parse_metadata_html(metadata_html: Tag, category: str) -> Optional[Artwork]:
         try:
             img_details = metadata_html.find("h3", {"class": "product-title"})
-
             url = img_details.a.get("href")
             title = img_details.get_text(strip=True)
-            category = metadata_html.find(
-                "div", {"class": "woodmart-product-cats"}
-            ).get_text(strip=True)
 
             artwork = Artwork(url, title, category)