diff --git a/.travis.yml b/.travis.yml index 066530e..f9097c8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,6 +5,7 @@ python: - "3.6" - "3.7" - "3.8" + - "3.9-dev" - "pypy3" install: diff --git a/pynytimes/__init__.py b/pynytimes/__init__.py index 7daf5ed..41ba438 100644 --- a/pynytimes/__init__.py +++ b/pynytimes/__init__.py @@ -2,4 +2,4 @@ from .api import NYTAPI __all__ = ["NYTAPI"] -__version__ = "0.2" +__version__ = "0.3" diff --git a/pynytimes/api.py b/pynytimes/api.py index 8753817..f4a16c8 100644 --- a/pynytimes/api.py +++ b/pynytimes/api.py @@ -6,6 +6,9 @@ import requests +from requests.adapters import HTTPAdapter +from requests.packages.urllib3.util.retry import Retry + BASE_TOP_STORIES = "https://api.nytimes.com/svc/topstories/v2/" BASE_MOST_POPULAR = "https://api.nytimes.com/svc/mostpopular/v2/" BASE_BOOKS = "https://api.nytimes.com/svc/books/v3/" @@ -114,7 +117,7 @@ def movie_reviews(session, key, keyword, options, max_results): for i in range(math.ceil(max_results/20)): offset = i*20 params["offset"] = str(offset) - res = session.get(url, params=params) + res = session.get(url, params=params, timeout=(4, 10)) res.raise_for_status() res_parsed = res.json() results += res_parsed.get("results") @@ -157,13 +160,9 @@ def archive_metadata(session, key, date): @staticmethod def article_search(session, key, options, results): """Get articles from search""" - start = datetime.datetime.now() - params = options params["api-key"] = key - rate_limit = options.get("rate_limit", True) - url = BASE_ARTICLE_SEARCH result = [] @@ -171,18 +170,13 @@ def article_search(session, key, options, results): params["page"] = str(i) res = session.get(url, params=params) + res.raise_for_status() + result += res.json().get("response").get("docs") if res.json().get("response").get("meta").get("hits") <= i*10: break - if (i + 1) % 10 == 0 and rate_limit: - now = datetime.datetime.now() - time_spend = now - start - time_sleep = math.ceil(60 - time_spend.total_seconds()) - time.sleep(time_sleep) - start = datetime.datetime.now() - return result @@ -193,7 +187,15 @@ def __init__(self, key=None): self.key = key self.session = requests.Session() - self.session.headers.update({"User-Agent": "pynytimes/0.2"}) + retry_strategy = Retry( + total = 10, + backoff_factor = 1, + status_forcelist = [429, 500, 502, 503, 504] + ) + + self.session.mount("https://", HTTPAdapter(max_retries = retry_strategy)) + + self.session.headers.update({"User-Agent": "pynytimes/0.3"}) if self.key is None: raise Exception("No API key") @@ -379,12 +381,12 @@ def article_search(self, query=None, dates=None, options=None, results=None): if results is None: results = 10 - if results > 100: + if results >= 100: warnings.warn( "Asking for a lot of results, because of rate limits it can take a while." ) - if results > 2010: + if results >= 2010: results = 2010 warnings.warn( "Asking for more results then the API can provide, loading maximum results." diff --git a/setup.py b/setup.py index c624c03..2da4b18 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setuptools.setup( name="pynytimes", - version="0.2.1", + version="0.3", description="A Python wrapper for (most) New York Times APIs", long_description=long_description, long_description_content_type="text/markdown", @@ -16,7 +16,7 @@ include_package_data = True, url="https://github.com/michadenheijer/pynytimes", license="MIT", - install_requires = ["requests==2.22.0"], + install_requires = ["requests==2.23.0"], classifiers = [ "License :: OSI Approved :: MIT License", "Programming Language :: Python :: 3 :: Only" diff --git a/tests.py b/tests.py index 1b9168a..288a24c 100644 --- a/tests.py +++ b/tests.py @@ -1,53 +1,53 @@ from pynytimes import NYTAPI import datetime -import random -import time import os -random_wait = random.randint(0, 60) -time.sleep(random_wait) - begin = datetime.datetime.now() API_KEY = os.environ["NewYorkTimesAPIKey"] nyt = NYTAPI(API_KEY) nyt.top_stories(section="science") + nyt.most_viewed(days=30) -time.sleep(5) + nyt.most_shared( days = 30, method = "email" ) + nyt.book_reviews( author = "Michelle Obama" ) -time.sleep(5) + nyt.best_sellers_lists() + nyt.best_sellers_list( date = datetime.datetime(2019, 1, 1), name = "hardcover-fiction" ) -time.sleep(5) + nyt.movie_reviews( keyword = "FBI", options = { "order": "by-opening-date" } ) + nyt.article_metadata( url = "https://www.nytimes.com/2019/10/20/world/middleeast/erdogan-turkey-nuclear-weapons-trump.html" ) -time.sleep(5) + nyt.tag_query( "Pentagon", max_results = 20 ) + nyt.archive_metadata( date = datetime.datetime(2019, 1, 1) ) -time.sleep(5) + nyt.article_search( query = "Trump", results = 20, @@ -61,4 +61,4 @@ ) end = datetime.datetime.now() -print(end - begin) +print(end - begin) \ No newline at end of file