From 482f3ec3d3220436036fa4988a74d4666d381d28 Mon Sep 17 00:00:00 2001 From: Dominik Prange Date: Sat, 4 Jan 2025 19:49:48 +0100 Subject: [PATCH 1/6] [boosty] added new direct message extractor --- docs/supportedsites.md | 3 +- gallery_dl/extractor/boosty.py | 128 +++++++++++++++++++++++++++++++++ 2 files changed, 130 insertions(+), 1 deletion(-) diff --git a/docs/supportedsites.md b/docs/supportedsites.md index ea18f1f06c..19f0a49c6c 100644 --- a/docs/supportedsites.md +++ b/docs/supportedsites.md @@ -1,6 +1,7 @@ # Supported Sites + Consider all listed sites to potentially be NSFW. @@ -148,7 +149,7 @@ Consider all listed sites to potentially be NSFW. - + diff --git a/gallery_dl/extractor/boosty.py b/gallery_dl/extractor/boosty.py index c28fad97a2..ca8d9fd7d0 100644 --- a/gallery_dl/extractor/boosty.py +++ b/gallery_dl/extractor/boosty.py @@ -219,6 +219,105 @@ def items(self): yield Message.Queue, url, user +class BoostyDirectMessageExtractor(BoostyExtractor): + """Extractor for boosty.to direct messages""" + subcategory = "direct-messages" + directory_fmt = "{category}", "{user[blogUrl]} ({user[id]})", "direct-messages" + pattern = BASE_PATTERN + r"/app/messages\?dialogId=(\d+)" + example = "https://boosty.to/app/messages?dialogId=123456" + + def items(self): + """Yield direct messages from a given dialog ID.""" + dialog_id = self.groups[0] + response = self.api.dialog(dialog_id) + sign = response.get("signedQuery", "") + messages = response.get("messages", {}).get("data", []) + chatmate = response.get("chatmate", {}) + user = chatmate.get("url", "") + + if self._user is None: + self._user = self.api.user(user) + + for message in messages: + files = self._process_dialog(message, sign=sign) + + if files: + data = { + "post": message, + "user": self._user, + "count": len(files), + } + + yield Message.Directory, data + + for data["num"], file in enumerate(files, 1): + data["file"] = file + url = file["url"] + yield Message.Url, url, text.nameext_from_url(url, data) + + for message in self.api.dialog_messages(dialog_id=dialog_id, offset=messages[0]["id"]): + files = self._process_dialog(message, sign=sign) + if files: + data = { + "post": message, + "user": self._user, + "count": len(files), + } + yield Message.Directory, data + + for data["num"], file in enumerate(files, 1): + data["file"] = file + url = file["url"] + yield Message.Url, url, text.nameext_from_url(url, data) + + def _process_dialog(self, post, sign=None): + """Process a post and return the list of files.""" + files = [] + + for block in post.get("data", []): + post["signedQuery"] = sign + type = block["type"] + if type == "text": + continue + + if type == "image": + files.append(self._update_url(post, block)) + + elif type == "ok_video": + if not self.videos: + self.log.debug("%s: Skipping video %s", post["id"], block["id"]) + continue + fmts = { + fmt["type"]: fmt["url"] + for fmt in block.get("playerUrls", []) + if fmt["url"] + } + formats = [ + fmts[fmt] + for fmt in self.videos + if fmt in fmts + ] + if formats: + formats = iter(formats) + block["url"] = next(formats) + block["_fallback"] = formats + files.append(block) + else: + self.log.warning("%s: Found no suitable video format for %s", post["id"], block["id"]) + + elif type == "link": + url = block["url"] + files.append({"url": url}) + + elif type == "audio_file": + files.append(self._update_url(post, block)) + + elif type == "file": + files.append(self._update_url(post, block)) + + return files + + class BoostyAPI(): """Interface for the Boosty API""" root = "https://api.boosty.to" @@ -367,3 +466,32 @@ def _pagination_users(self, endpoint, params): if offset > data["total"]: return params["offset"] = offset + + def dialog(self, dialog_id): + endpoint = f"/v1/dialog/{dialog_id}" + return self._call(endpoint) + + def dialog_messages(self, dialog_id, limit=300, offset=None): + endpoint = f"/v1/dialog/{dialog_id}/message/" + params = { + "limit": str(limit), + "reverse": "true", + "offset": str(offset) if offset else None, + } + return self._pagination_dialog(endpoint, params) + + def _pagination_dialog(self, endpoint, params): + all_messages = [] + + while True: + data = self._call(endpoint, params) + messages = data.get("messages", {}).get("data", []) + all_messages.extend(messages) + extra = data.get("extra", {}) + offset = extra.get("offset") + yield from data["data"] + if not offset: + break + if extra.get("isLast"): + break + params["offset"] = offset From cd98c86c98f1750182ad80789148dbc326839f61 Mon Sep 17 00:00:00 2001 From: 5n0wstorm Date: Sat, 4 Jan 2025 19:54:24 +0100 Subject: [PATCH 2/6] formatting --- docs/supportedsites.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/supportedsites.md b/docs/supportedsites.md index 19f0a49c6c..0b7c9304e5 100644 --- a/docs/supportedsites.md +++ b/docs/supportedsites.md @@ -1,7 +1,6 @@ # Supported Sites - Consider all listed sites to potentially be NSFW.
Boosty https://www.boosty.to/Subscriptions Feed, Followed Users, Media Files, Posts, User ProfilesSubscriptions Feed, Followed Users, Media Files, Posts, User Profiles, Direct Messages Cookies
From a0a2a3151ae4f892243e17749175c108eabf7b10 Mon Sep 17 00:00:00 2001 From: 5n0wstorm Date: Sat, 4 Jan 2025 20:14:19 +0100 Subject: [PATCH 3/6] fixed linting formatting errors --- gallery_dl/extractor/boosty.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/gallery_dl/extractor/boosty.py b/gallery_dl/extractor/boosty.py index ca8d9fd7d0..b193760ac8 100644 --- a/gallery_dl/extractor/boosty.py +++ b/gallery_dl/extractor/boosty.py @@ -222,7 +222,8 @@ def items(self): class BoostyDirectMessageExtractor(BoostyExtractor): """Extractor for boosty.to direct messages""" subcategory = "direct-messages" - directory_fmt = "{category}", "{user[blogUrl]} ({user[id]})", "direct-messages" + directory_fmt = ("{category}", "{user[blogUrl]} ({user[id]})", + "direct-messages") pattern = BASE_PATTERN + r"/app/messages\?dialogId=(\d+)" example = "https://boosty.to/app/messages?dialogId=123456" @@ -255,7 +256,8 @@ def items(self): url = file["url"] yield Message.Url, url, text.nameext_from_url(url, data) - for message in self.api.dialog_messages(dialog_id=dialog_id, offset=messages[0]["id"]): + for message in self.api.dialog_messages(dialog_id=dialog_id, + offset=messages[0]["id"]): files = self._process_dialog(message, sign=sign) if files: data = { @@ -285,7 +287,9 @@ def _process_dialog(self, post, sign=None): elif type == "ok_video": if not self.videos: - self.log.debug("%s: Skipping video %s", post["id"], block["id"]) + self.log.debug( + "%s: Skipping video %s", post["id"], + block["id"]) continue fmts = { fmt["type"]: fmt["url"] @@ -303,7 +307,9 @@ def _process_dialog(self, post, sign=None): block["_fallback"] = formats files.append(block) else: - self.log.warning("%s: Found no suitable video format for %s", post["id"], block["id"]) + self.log.warning( + "%s: Found no suitable video format for %s", + post["id"], block["id"]) elif type == "link": url = block["url"] From 4771f4cbce57648878a2b89c6c727456a77d0dba Mon Sep 17 00:00:00 2001 From: 5n0wstorm Date: Sat, 4 Jan 2025 20:20:51 +0100 Subject: [PATCH 4/6] fixed E999 SyntaxError: invalid syntax --- gallery_dl/extractor/boosty.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gallery_dl/extractor/boosty.py b/gallery_dl/extractor/boosty.py index b193760ac8..e7be057175 100644 --- a/gallery_dl/extractor/boosty.py +++ b/gallery_dl/extractor/boosty.py @@ -474,11 +474,11 @@ def _pagination_users(self, endpoint, params): params["offset"] = offset def dialog(self, dialog_id): - endpoint = f"/v1/dialog/{dialog_id}" + endpoint = "/v1/dialog/{}".format(dialog_id) return self._call(endpoint) def dialog_messages(self, dialog_id, limit=300, offset=None): - endpoint = f"/v1/dialog/{dialog_id}/message/" + endpoint = "/v1/dialog/{}/message/".format(dialog_id) params = { "limit": str(limit), "reverse": "true", From 861a5bb91303b6551dc2568ded1e59a5a251b735 Mon Sep 17 00:00:00 2001 From: 5n0wstorm Date: Sat, 4 Jan 2025 20:22:14 +0100 Subject: [PATCH 5/6] fixed class naming --- gallery_dl/extractor/boosty.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gallery_dl/extractor/boosty.py b/gallery_dl/extractor/boosty.py index e7be057175..1c4bd83881 100644 --- a/gallery_dl/extractor/boosty.py +++ b/gallery_dl/extractor/boosty.py @@ -219,7 +219,7 @@ def items(self): yield Message.Queue, url, user -class BoostyDirectMessageExtractor(BoostyExtractor): +class BoostyDirectMessagesExtractor(BoostyExtractor): """Extractor for boosty.to direct messages""" subcategory = "direct-messages" directory_fmt = ("{category}", "{user[blogUrl]} ({user[id]})", From 1439b2c021f3d43f7b9de43046c01d02ee60349f Mon Sep 17 00:00:00 2001 From: 5n0wstorm Date: Sun, 5 Jan 2025 10:05:39 +0100 Subject: [PATCH 6/6] fixed mandatory extractor.boosty.metadata as true requirement --- gallery_dl/extractor/boosty.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/gallery_dl/extractor/boosty.py b/gallery_dl/extractor/boosty.py index 1c4bd83881..deace80f11 100644 --- a/gallery_dl/extractor/boosty.py +++ b/gallery_dl/extractor/boosty.py @@ -234,10 +234,7 @@ def items(self): sign = response.get("signedQuery", "") messages = response.get("messages", {}).get("data", []) chatmate = response.get("chatmate", {}) - user = chatmate.get("url", "") - - if self._user is None: - self._user = self.api.user(user) + user = self.api.user(chatmate.get("url", "")) for message in messages: files = self._process_dialog(message, sign=sign) @@ -245,10 +242,9 @@ def items(self): if files: data = { "post": message, - "user": self._user, + "user": user, "count": len(files), } - yield Message.Directory, data for data["num"], file in enumerate(files, 1): @@ -262,7 +258,7 @@ def items(self): if files: data = { "post": message, - "user": self._user, + "user": user, "count": len(files), } yield Message.Directory, data