parser.py

import re
import router
import numpy as np
from bs4 import BeautifulSoup
from operator import itemgetter


class Parser:
    """Get bestsellers data from Amazon departments and filter it."""
    PERCENTILE = 95

    def get_department_urls(self):
        """Get links of all first level departments."""
        soup = BeautifulSoup(router.do_get(
            "https://www.amazon.com/Best-Sellers/zgbs/").text, "html.parser")

        return [a.attrs["href"] for a in soup.find_all(
            "a",
            href=re.compile("https://www.amazon.com/Best-Sellers-"))]

    def get_department_bestsellers(self, url):
        """Parse name, rating, reviews and price from department url"""
        # print(url)
        dep_bestsellers = []
        b_soup = BeautifulSoup(router.do_get(url).text, "html.parser")
        # Find first bestseller data
        bestsellers = b_soup.find_all("div", class_="zg_itemImmersion")
        for b in bestsellers:
            name = b.find("a").text.strip()
            rating = float(b.find("a", href=re.compile(
                "/product-reviews")).text.split()[0])
            reviews = int(b.find("a", class_="a-size-small").text.
                          replace(",", ""))
            price = b.find("span", class_="p13n-sc-price").text[1:]

            if reviews > 300:
                b = {"name": name, "rating": rating, "reviews": reviews,
                     "price": float(price)}
                # Track the most reviewed
                if reviews > self.max_reviews:
                    self.top = b
                    self.max_reviews = reviews
                dep_bestsellers.append(b)

        return dep_bestsellers

    def get_percentile(self, list, p):
        a = np.array(list)
        return np.percentile(a, 80)

    def get_bestsellers(self, urls):
        """Create list from each department"""
        self.max_reviews = 0
        bestsellers = []
        for u in urls:
            try:
                b = self.get_department_bestsellers(u)
                bestsellers += b
            except Exception as e:
                print("Skipping {}".format(u))
                print(str(e))
        return bestsellers

    def filter_data(self, dict, key, value):
        return list(filter(lambda b: b['price'] < value, dict))

    def get_data(self):
        dep_urls = self.get_department_urls()
        bestsellers = self.get_bestsellers(dep_urls)

        # Remove duplicates
        bestsellers = [dict(t) for t in set([tuple(d.items())
                                             for d in bestsellers])]
        print("Found {} bestsellers".format(len(bestsellers)))

        sorted_b = sorted(bestsellers, key=itemgetter('price', 'reviews'))
        p = self.get_percentile([b['price']
                                 for b in sorted_b], self.PERCENTILE)
        print("Percentile value is: {}".format(p))

        # Filter by percentile
        filtered_b = self.filter_data(dict=sorted_b, key='price', value=p)
        print("Filtered to {} by {} percentile".format(
            len(filtered_b), self.PERCENTILE))

        return filtered_b