Merge pull request #5 from michadenheijer/Article-Search-Fix

Fix Article Search, use exponential backdown, update to version 0.3

.travis.yml

@@ -5,6 +5,7 @@ python:
   - "3.6"
   - "3.7"
   - "3.8"
+  - "3.9-dev"
   - "pypy3"
 
 install:

pynytimes/__init__.py

@@ -2,4 +2,4 @@
 from .api import NYTAPI
 
 __all__ = ["NYTAPI"]
-__version__ = "0.2"
+__version__ = "0.3"

pynytimes/api.py

@@ -6,6 +6,9 @@
 
 import requests
 
+from requests.adapters import HTTPAdapter
+from requests.packages.urllib3.util.retry import Retry
+
 BASE_TOP_STORIES = "https://api.nytimes.com/svc/topstories/v2/"
 BASE_MOST_POPULAR = "https://api.nytimes.com/svc/mostpopular/v2/"
 BASE_BOOKS = "https://api.nytimes.com/svc/books/v3/"
@@ -114,7 +117,7 @@ def movie_reviews(session, key, keyword, options, max_results):
         for i in range(math.ceil(max_results/20)):
             offset = i*20
             params["offset"] = str(offset)
-            res = session.get(url, params=params)
+            res = session.get(url, params=params, timeout=(4, 10))
             res.raise_for_status()
             res_parsed = res.json()
             results += res_parsed.get("results")
@@ -157,32 +160,23 @@ def archive_metadata(session, key, date):
     @staticmethod
     def article_search(session, key, options, results):
         """Get articles from search"""
-        start = datetime.datetime.now()
-
         params = options
         params["api-key"] = key
 
-        rate_limit = options.get("rate_limit", True)
-
         url = BASE_ARTICLE_SEARCH
 
         result = []
         for i in range(math.ceil(results/10)):
             params["page"] = str(i)
 
             res = session.get(url, params=params)
+            res.raise_for_status()
+
             result += res.json().get("response").get("docs")
 
             if res.json().get("response").get("meta").get("hits") <= i*10:
                 break
 
-            if (i + 1) % 10 == 0 and rate_limit:
-                now = datetime.datetime.now()
-                time_spend = now - start
-                time_sleep = math.ceil(60 - time_spend.total_seconds())
-                time.sleep(time_sleep)
-                start = datetime.datetime.now()
-
 
         return result
 
@@ -193,7 +187,15 @@ def __init__(self, key=None):
         self.key = key
         self.session = requests.Session()
 
-        self.session.headers.update({"User-Agent": "pynytimes/0.2"})
+        retry_strategy = Retry(
+            total = 10,
+            backoff_factor = 1,
+            status_forcelist = [429, 500, 502, 503, 504]
+        )
+
+        self.session.mount("https://", HTTPAdapter(max_retries = retry_strategy))
+
+        self.session.headers.update({"User-Agent": "pynytimes/0.3"})
 
         if self.key is None:
             raise Exception("No API key")
@@ -379,12 +381,12 @@ def article_search(self, query=None, dates=None, options=None, results=None):
         if results is None:
             results = 10
 
-        if results > 100:
+        if results >= 100:
             warnings.warn(
                 "Asking for a lot of results, because of rate limits it can take a while."
             )
 
-        if results > 2010:
+        if results >= 2010:
             results = 2010
             warnings.warn(
                 "Asking for more results then the API can provide, loading maximum results."

setup.py

@@ -5,7 +5,7 @@
 
 setuptools.setup(
     name="pynytimes",
-    version="0.2.1",
+    version="0.3",
     description="A Python wrapper for (most) New York Times APIs",
     long_description=long_description,
     long_description_content_type="text/markdown",
@@ -16,7 +16,7 @@
     include_package_data = True,
     url="https://github.com/michadenheijer/pynytimes",
     license="MIT",
-    install_requires = ["requests==2.22.0"],
+    install_requires = ["requests==2.23.0"],
     classifiers = [
         "License :: OSI Approved :: MIT License",
         "Programming Language :: Python :: 3 :: Only"

tests.py

@@ -1,53 +1,53 @@
 from pynytimes import NYTAPI
 
 import datetime
-import random
-import time
 import os
 
-random_wait = random.randint(0, 60)
-time.sleep(random_wait)
-
 begin = datetime.datetime.now()
 
 API_KEY = os.environ["NewYorkTimesAPIKey"]
 nyt = NYTAPI(API_KEY)
 
 nyt.top_stories(section="science")
+
 nyt.most_viewed(days=30)
-time.sleep(5)
+
 nyt.most_shared(
     days = 30,
     method = "email"
 )
+
 nyt.book_reviews(
     author = "Michelle Obama"
 )
-time.sleep(5)
+
 nyt.best_sellers_lists()
+
 nyt.best_sellers_list(
     date = datetime.datetime(2019, 1, 1),
     name = "hardcover-fiction"
 )
-time.sleep(5)
+
 nyt.movie_reviews(
     keyword = "FBI",
     options = {
         "order": "by-opening-date"
     }
 )
+
 nyt.article_metadata(
     url = "https://www.nytimes.com/2019/10/20/world/middleeast/erdogan-turkey-nuclear-weapons-trump.html"
 )
-time.sleep(5)
+
 nyt.tag_query(
     "Pentagon",
     max_results = 20
 )
+
 nyt.archive_metadata(
     date = datetime.datetime(2019, 1, 1)
 )
-time.sleep(5)
+
 nyt.article_search(
     query = "Trump",
     results = 20,
@@ -61,4 +61,4 @@
 )
 
 end = datetime.datetime.now()
-print(end - begin)
+print(end - begin)