[kemonoparty] various updates and fixes (#4676, #4681)

- fix pagination - fix 'date' metadata - fix discord channel API endpoint
mikf · Oct 19, 2023 · c9a976d · c9a976d
1 parent 761fa68
commit c9a976d
Show file tree

Hide file tree

Showing 2 changed files with 14 additions and 25 deletions.
diff --git a/gallery_dl/extractor/kemonoparty.py b/gallery_dl/extractor/kemonoparty.py
@@ -70,8 +70,7 @@ def items(self):
                 self.root, post["service"], post["user"], post["id"])
             post["_http_headers"] = headers
             post["date"] = text.parse_datetime(
-                post["published"] or post["added"],
-                "%a, %d %b %Y %H:%M:%S %Z")
+                post["published"] or post["added"], "%Y-%m-%dT%H:%M:%S")
             if username:
                 post["username"] = username
             if comments:
@@ -208,7 +207,7 @@ def _extract_dms(self, post):
 
     @memcache(keyarg=1)
     def _discord_channels(self, server):
-        url = "{}/api/v1/discord/channels/lookup/{}".format(
+        url = "{}/api/v1/discord/channel/lookup/{}".format(
             self.root, server)
         return self.request(url).json()
 
@@ -241,10 +240,9 @@ def posts(self):
             posts = self.request(url, params=params).json()
             yield from posts
 
-            cnt = len(posts)
-            if cnt < 25:
-                return
-            params["o"] += cnt
+            if len(posts) < 50:
+                break
+            params["o"] += 50
 
 
 class KemonopartyPostExtractor(KemonopartyExtractor):
@@ -262,8 +260,7 @@ def __init__(self, match):
         self.user_url = "{}/{}/user/{}".format(self.root, service, user_id)
 
     def posts(self):
-        posts = self.request(self.api_url).json()
-        return (posts[0],) if len(posts) > 1 else posts
+        return (self.request(self.api_url).json(),)
 
 
 class KemonopartyDiscordExtractor(KemonopartyExtractor):
@@ -325,7 +322,7 @@ def items(self):
 
             post["channel_name"] = self.channel_name
             post["date"] = text.parse_datetime(
-                post["published"], "%a, %d %b %Y %H:%M:%S %Z")
+                post["published"], "%Y-%m-%dT%H:%M:%S.%f")
             post["count"] = len(files)
             yield Message.Directory, post
 
@@ -348,15 +345,7 @@ def posts(self):
         url = "{}/api/v1/discord/channel/{}".format(
             self.root, self.channel_id)
         params = {"skip": 0}
-
-        while True:
-            posts = self.request(url, params=params).json()
-            yield from posts
-
-            cnt = len(posts)
-            if cnt < 25:
-                break
-            params["skip"] += cnt
+        return self.request(url, params=params).json()
 
 
 class KemonopartyDiscordServerExtractor(KemonopartyExtractor):

diff --git a/test/results/kemonoparty.py b/test/results/kemonoparty.py
@@ -12,17 +12,16 @@
     "#url"     : "https://kemono.party/fanbox/user/6993449",
     "#category": ("", "kemonoparty", "fanbox"),
     "#class"   : kemonoparty.KemonopartyUserExtractor,
-    "#range"   : "1-25",
-    "#count"   : 25,
+    "#count"   : 847,
 },
 
 {
     "#url"     : "https://kemono.party/patreon/user/881792?o=150",
     "#comment" : "'max-posts' option, 'o' query parameter (#1674)",
     "#category": ("", "kemonoparty", "patreon"),
     "#class"   : kemonoparty.KemonopartyUserExtractor,
-    "#options" : {"max-posts": 25},
-    "#count"   : "< 100",
+    "#options" : {"max-posts": 100},
+    "#count"   : range(200, 300),
 },
 
 {
@@ -44,7 +43,7 @@
     "#pattern"     : r"https://kemono.party/data/21/0f/210f35388e28bbcf756db18dd516e2d82ce75[0-9a-f]+\.jpg",
     "#sha1_content": "900949cefc97ab8dc1979cc3664785aac5ba70dd",
 
-    "added"      : "Wed, 06 May 2020 20:28:02 GMT",
+    "added"      : "2020-05-06T20:28:02.302000",
     "content"    : str,
     "count"      : 1,
     "date"       : "dt:2019-08-11 02:09:04",
@@ -55,7 +54,7 @@
     "hash"       : "210f35388e28bbcf756db18dd516e2d82ce758e0d32881eeee76d43e1716d382",
     "id"         : "506575",
     "num"        : 1,
-    "published"  : "Sun, 11 Aug 2019 02:09:04 GMT",
+    "published"  : "2019-08-11T02:09:04",
     "service"    : "fanbox",
     "shared_file": False,
     "subcategory": "fanbox",
@@ -183,6 +182,7 @@
 
     "channel"     : "608504710906904576",
     "channel_name": "finish-work",
+    "date"        : "type:datetime",
 },
 
 {