Skip to content

Commit

Permalink
Adding a type of caching mechanism for the metadata/user information.
Browse files Browse the repository at this point in the history
The current logic would work just fine if looking up for a singular user, however for the multiple posts via normal filtering would cause it to either:

1. Error out due to lack of service and/or creator id.

2. Produce no metadata

This builds a local cache during the process so it should only make a call for the user info once during the process.
  • Loading branch information
BishopRed90 committed Jan 15, 2025
1 parent 00cdd3e commit 1b54629
Showing 1 changed file with 14 additions and 13 deletions.
27 changes: 14 additions & 13 deletions gallery_dl/extractor/kemonoparty.py
Original file line number Diff line number Diff line change
Expand Up @@ -55,28 +55,20 @@ def _init(self):
sort_keys=True, separators=(",", ":")).encode

def items(self):
service = self.groups[2]
creator_id = self.groups[3]
_creator_info: dict = {}

find_hash = re.compile(HASH_PATTERN).match
generators = self._build_file_generators(self.config("files"))
announcements = True if self.config("announcements") else None
comments = True if self.config("comments") else False
duplicates = True if self.config("duplicates") else False
dms = True if self.config("dms") else None
profile = username = None
max_posts = self.config("max-posts")

# prevent files from being sent with gzip compression
headers = {"Accept-Encoding": "identity"}

if self.config("metadata") and service and creator_id:
#TODO - not sure if we should make this call for every post for a
# /posts all as it may be expensive
profile = self.api.creator_profile(service, creator_id)
username = profile["name"]

posts = self.posts()
max_posts = self.config("max-posts")
if max_posts:
posts = itertools.islice(posts, max_posts)
if self.revisions:
Expand All @@ -88,10 +80,19 @@ def items(self):
post["_http_headers"] = headers
post["date"] = self._parse_datetime(
post.get("published") or post.get("added") or "")
service = post["service"]
creator_id = post["user"]

if self.config("metadata"):
if _creator_info.get(f"{service}_{creator_id}") is None:
_creator = self.api.creator_profile(service, creator_id)
_creator_info[f"{service}_{creator_id}"] = _creator

post["user_profile"] = _creator_info[f"{service}_{creator_id}"]
post["username"] = _creator_info[f"{service}_{creator_id}"]['name']
else:
post["user_profile"] = post["username"] = post["user"]

if profile is not None:
post["username"] = username
post["user_profile"] = profile
if comments:
try:
post["comments"] = self.api.creator_post_comments(
Expand Down

0 comments on commit 1b54629

Please sign in to comment.