diff --git a/fimfarchive/fetchers/fimfiction2.py b/fimfarchive/fetchers/fimfiction2.py index a86a65d..a0e1968 100644 --- a/fimfarchive/fetchers/fimfiction2.py +++ b/fimfarchive/fetchers/fimfiction2.py @@ -26,6 +26,7 @@ from abc import ABC, abstractmethod from collections import OrderedDict, defaultdict from copy import deepcopy +from time import sleep from typing import Any, Dict, Iterator, Optional, Set from urllib.parse import urlencode @@ -36,6 +37,7 @@ from fimfarchive import __version__ as version from fimfarchive.flavors import DataFormat, MetaFormat, MetaPurity, StorySource +from fimfarchive.utils import tqdm from fimfarchive.exceptions import ( FimfarchiveError, @@ -302,9 +304,16 @@ def get_meta(self, key: int) -> ResourceObject: return response.resource def get_data(self, key: int) -> Iterator[ResourceObject]: - path = f'stories/{key}/chapters' - response = self.get(key, path, DATA_PARAMS) - return response.resources + list_path = f'stories/{key}/chapters' + list_params = deepcopy(DATA_PARAMS) + list_params['fields[chapter]'].remove('content_html') + list_response = self.get(key, list_path, list_params) + + for resource in tqdm(list_response.resources): + path = f"chapters/{resource.id}" + response = self.get(key, path, DATA_PARAMS) + yield response.resource + sleep(5) class BulkRequester(Requester): @@ -685,7 +694,7 @@ class Fimfiction2Fetcher(Fetcher): Fetcher for Fimfiction APIv2. """ prefetch_meta = True - prefetch_data = False + prefetch_data = True flavors = frozenset(( StorySource.FIMFICTION, @@ -703,12 +712,14 @@ def __init__(self, token: str, bulk_meta=False, bulk_data=False) -> None: bulk_meta: Toggles bulk fetching of story meta. bulk_data: Toggles bulk fetching of story data. """ + assert not bulk_meta + assert not bulk_data client = ApiClient(token) self.extract_meta = MetaDocumentifier() self.extract_chapter = Documentifier() self.verify_meta = BetaFormatVerifier.from_meta_params() self.verify_chapter = BetaFormatVerifier.from_data_params() - self.requester = RoutedRequester(client, bulk_meta, bulk_data) + self.requester = SingleRequester(client) def fetch_meta(self, key: int) -> Dict[str, Any]: resource = self.requester.get_meta(int(key))