From dd8b6bf6e3e91026459fd7ecebfb11951dfb9805 Mon Sep 17 00:00:00 2001 From: Alessandro Maggio Date: Wed, 29 Sep 2021 15:56:46 +0200 Subject: [PATCH 1/4] Attempt to use grapql instead of helix request for get followers and streamer_id --- TwitchChannelPointsMiner/classes/Twitch.py | 45 ++++++++++++------- .../classes/TwitchLogin.py | 22 ++++++--- TwitchChannelPointsMiner/constants.py | 30 ++++++++++++- 3 files changed, 72 insertions(+), 25 deletions(-) diff --git a/TwitchChannelPointsMiner/classes/Twitch.py b/TwitchChannelPointsMiner/classes/Twitch.py index 23679fb3..cd6a9471 100644 --- a/TwitchChannelPointsMiner/classes/Twitch.py +++ b/TwitchChannelPointsMiner/classes/Twitch.py @@ -23,7 +23,7 @@ ) from TwitchChannelPointsMiner.classes.Settings import Priority, Settings from TwitchChannelPointsMiner.classes.TwitchLogin import TwitchLogin -from TwitchChannelPointsMiner.constants import API, CLIENT_ID, GQLOperations +from TwitchChannelPointsMiner.constants import CLIENT_ID, GQLOperations from TwitchChannelPointsMiner.utils import ( _millify, create_chunks, @@ -143,16 +143,20 @@ def check_streamer_online(self, streamer): streamer.set_offline() def get_channel_id(self, streamer_username): - json_response = self.__do_helix_request(f"/users?login={streamer_username}") - if "data" not in json_response: + # json_response = self.__do_helix_request(f"/users?login={streamer_username}") + json_data = copy.deepcopy(GQLOperations.ReportMenuItem) + json_data["variables"] = {"channelLogin": streamer_username} + json_response = self.post_gql_request(json_data) + if ( + "data" not in json_response + or "user" not in json_response["data"] + or json_response["data"]["user"] is None + ): raise StreamerDoesNotExistException else: - data = json_response["data"] - if len(data) >= 1: - return data[0]["id"] - else: - raise StreamerDoesNotExistException + return json_response["data"]["user"]["id"] + """ def get_followers(self, first=100): followers = [] pagination = {} @@ -170,6 +174,23 @@ def get_followers(self, first=100): break return followers + """ + + def get_followers(self): + json_data = copy.deepcopy(GQLOperations.PersonalSections) + json_response = self.post_gql_request(json_data) + try: + if ( + "data" in json_response + and "personalSections" in json_response["data"] + and json_response["data"]["personalSections"] != [] + ): + return [ + fw["user"]["login"] + for fw in json_response["data"]["personalSections"][0]["items"] + ] + except KeyError: + return [] def update_raid(self, streamer, raid): if streamer.raid != raid: @@ -211,14 +232,6 @@ def __check_connection_handler(self, chunk_size): ) self.__chuncked_sleep(random_sleep * 60, chunk_size=chunk_size) - def __do_helix_request(self, query, response_as_json=True): - url = f"{API}/helix/{query.strip('/')}" - response = self.twitch_login.session.get(url) - logger.debug( - f"Query: {query}, Status code: {response.status_code}, Content: {response.json()}" - ) - return response.json() if response_as_json is True else response - def post_gql_request(self, json_data): try: response = requests.post( diff --git a/TwitchChannelPointsMiner/classes/TwitchLogin.py b/TwitchChannelPointsMiner/classes/TwitchLogin.py index 95f46a6d..3c2464a8 100644 --- a/TwitchChannelPointsMiner/classes/TwitchLogin.py +++ b/TwitchChannelPointsMiner/classes/TwitchLogin.py @@ -2,6 +2,7 @@ # Original Copyright (c) 2020 Rodney # The MIT License (MIT) +import copy import getpass import logging import os @@ -14,6 +15,7 @@ BadCredentialsException, WrongCookiesException, ) +from TwitchChannelPointsMiner.constants import GQLOperations logger = logging.getLogger(__name__) @@ -177,13 +179,19 @@ def check_login(self): if self.token is None: return False - response = self.session.get( - f"https://api.twitch.tv/helix/users?login={self.username}" - ) - response = response.json() - if "data" in response: - self.login_check_result = True - self.user_id = response["data"][0]["id"] + json_data = copy.deepcopy(GQLOperations.ReportMenuItem) + json_data["variables"] = {"channelLogin": self.usrername} + response = self.session.post(GQLOperations.url, json=json_data) + + if response.status_code == 200: + json_response = response.json() + if ( + "data" in json_response + and "user" in json_response["data"] + and json_response["data"]["user"]["id"] is not None + ): + self.user_id = json_response["data"]["user"]["id"] + self.login_check_result = True return self.login_check_result diff --git a/TwitchChannelPointsMiner/constants.py b/TwitchChannelPointsMiner/constants.py index 6127c7ff..5189f4ae 100644 --- a/TwitchChannelPointsMiner/constants.py +++ b/TwitchChannelPointsMiner/constants.py @@ -1,6 +1,5 @@ # Twitch endpoints URL = "https://www.twitch.tv" -API = "https://api.twitch.tv" IRC = "irc.chat.twitch.tv" IRC_PORT = 6667 WEBSOCKET = "wss://pubsub-edge.twitch.tv/v1" @@ -118,7 +117,7 @@ class GQLOperations: "extensions": { "persistedQuery": { "version": 1, - "sha256Hash": "14b5e8a50777165cfc3971e1d93b4758613fe1c817d5542c398dce70b7a45c05", # "7da6078b1bfa2f0a4dd061cb47bdcd1ffddf31cccadd966ec192e4cd06666e2b", + "sha256Hash": "14b5e8a50777165cfc3971e1d93b4758613fe1c817d5542c398dce70b7a45c05", } }, } @@ -131,3 +130,30 @@ class GQLOperations: } }, } + ReportMenuItem = { # Use for replace https://api.twitch.tv/helix/users?login={self.username} + "operationName": "ReportMenuItem", + "extensions": { + "persistedQuery": { + "version": 1, + "sha256Hash": "8f3628981255345ca5e5453dfd844efffb01d6413a9931498836e6268692a30c", + } + }, + } + PersonalSections = { + "operationName": "PersonalSections", + "variables": { + "input": { + "sectionInputs": ["FOLLOWED_SECTION"], + "recommendationContext": {"platform": "web"}, + }, + "channelLogin": None, + "withChannelUser": False, + "creatorAnniversariesExperimentEnabled": False, + }, + "extensions": { + "persistedQuery": { + "version": 1, + "sha256Hash": "9fbdfb00156f754c26bde81eb47436dee146655c92682328457037da1a48ed39", + } + }, + } From b64fb60ccaacb7162c10d0f618d6379ef7d5a8f3 Mon Sep 17 00:00:00 2001 From: Alessandro Maggio Date: Wed, 29 Sep 2021 19:25:49 +0200 Subject: [PATCH 2/4] typo on usrername --- TwitchChannelPointsMiner/classes/TwitchLogin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TwitchChannelPointsMiner/classes/TwitchLogin.py b/TwitchChannelPointsMiner/classes/TwitchLogin.py index 3c2464a8..180be287 100644 --- a/TwitchChannelPointsMiner/classes/TwitchLogin.py +++ b/TwitchChannelPointsMiner/classes/TwitchLogin.py @@ -180,7 +180,7 @@ def check_login(self): return False json_data = copy.deepcopy(GQLOperations.ReportMenuItem) - json_data["variables"] = {"channelLogin": self.usrername} + json_data["variables"] = {"channelLogin": self.username} response = self.session.post(GQLOperations.url, json=json_data) if response.status_code == 200: From 5f6c75560e131f17c17f304444c62060e5acf982 Mon Sep 17 00:00:00 2001 From: Alessandro Maggio Date: Wed, 29 Sep 2021 22:54:43 +0200 Subject: [PATCH 3/4] Check if the fw['user'] is None #289 --- TwitchChannelPointsMiner/classes/Twitch.py | 22 +--------------------- 1 file changed, 1 insertion(+), 21 deletions(-) diff --git a/TwitchChannelPointsMiner/classes/Twitch.py b/TwitchChannelPointsMiner/classes/Twitch.py index cd6a9471..e082262a 100644 --- a/TwitchChannelPointsMiner/classes/Twitch.py +++ b/TwitchChannelPointsMiner/classes/Twitch.py @@ -143,7 +143,6 @@ def check_streamer_online(self, streamer): streamer.set_offline() def get_channel_id(self, streamer_username): - # json_response = self.__do_helix_request(f"/users?login={streamer_username}") json_data = copy.deepcopy(GQLOperations.ReportMenuItem) json_data["variables"] = {"channelLogin": streamer_username} json_response = self.post_gql_request(json_data) @@ -156,26 +155,6 @@ def get_channel_id(self, streamer_username): else: return json_response["data"]["user"]["id"] - """ - def get_followers(self, first=100): - followers = [] - pagination = {} - while 1: - query = f"/users/follows?from_id={self.twitch_login.get_user_id()}&first={first}" - if pagination != {}: - query += f"&after={pagination['cursor']}" - - json_response = self.__do_helix_request(query) - pagination = json_response["pagination"] - followers += [fw["to_login"].lower() for fw in json_response["data"]] - time.sleep(random.uniform(0.3, 0.7)) - - if pagination == {}: - break - - return followers - """ - def get_followers(self): json_data = copy.deepcopy(GQLOperations.PersonalSections) json_response = self.post_gql_request(json_data) @@ -188,6 +167,7 @@ def get_followers(self): return [ fw["user"]["login"] for fw in json_response["data"]["personalSections"][0]["items"] + if fw["user"] is not None ] except KeyError: return [] From 3c3101516ffa29d31a648ec9ae64fe5ff387e2ac Mon Sep 17 00:00:00 2001 From: Alessandro Maggio Date: Thu, 30 Sep 2021 15:53:48 +0200 Subject: [PATCH 4/4] The user_id can't be None, double check before return If It's None please do a gql request --- .../TwitchChannelPointsMiner.py | 5 ++- TwitchChannelPointsMiner/classes/Twitch.py | 6 +-- .../classes/TwitchLogin.py | 41 ++++++++++++------- .../classes/TwitchWebSocket.py | 1 - .../classes/WebSocketsPool.py | 1 - 5 files changed, 32 insertions(+), 22 deletions(-) diff --git a/TwitchChannelPointsMiner/TwitchChannelPointsMiner.py b/TwitchChannelPointsMiner/TwitchChannelPointsMiner.py index 7b44c892..f32be2bd 100644 --- a/TwitchChannelPointsMiner/TwitchChannelPointsMiner.py +++ b/TwitchChannelPointsMiner/TwitchChannelPointsMiner.py @@ -240,10 +240,11 @@ def run(self, streamers: list = [], blacklist: list = [], followers=False): ) # Subscribe to community-points-user. Get update for points spent or gains + user_id = self.twitch.twitch_login.get_user_id() self.ws_pool.submit( PubsubTopic( "community-points-user-v1", - user_id=self.twitch.twitch_login.get_user_id(), + user_id=user_id, ) ) @@ -252,7 +253,7 @@ def run(self, streamers: list = [], blacklist: list = [], followers=False): self.ws_pool.submit( PubsubTopic( "predictions-user-v1", - user_id=self.twitch.twitch_login.get_user_id(), + user_id=user_id, ) ) diff --git a/TwitchChannelPointsMiner/classes/Twitch.py b/TwitchChannelPointsMiner/classes/Twitch.py index e082262a..9dc0ac94 100644 --- a/TwitchChannelPointsMiner/classes/Twitch.py +++ b/TwitchChannelPointsMiner/classes/Twitch.py @@ -355,14 +355,12 @@ def send_minute_watched_events(self, streamers, priority, chunk_size=3): drop.has_preconditions_met is not False and drop.is_printable is True ): - # print("=" * 125) logger.info( f"{streamers[index]} is streaming {streamers[index].stream}" ) logger.info(f"Campaign: {campaign}") logger.info(f"Drop: {drop}") logger.info(f"{drop.progress_bar()}") - # print("=" * 125) except requests.exceptions.ConnectionError as e: logger.error(f"Error while trying to send minute watched: {e}") @@ -534,7 +532,9 @@ def __get_campaigns_details(self, campaigns): } response = self.post_gql_request(json_data) - result += list(map(lambda x: x["data"]["user"]["dropCampaign"], response)) + for r in response: + if r["data"]["user"] is not None: + result.append(r["data"]["user"]["dropCampaign"]) return result def __sync_campaigns(self, campaigns): diff --git a/TwitchChannelPointsMiner/classes/TwitchLogin.py b/TwitchChannelPointsMiner/classes/TwitchLogin.py index 180be287..d257a856 100644 --- a/TwitchChannelPointsMiner/classes/TwitchLogin.py +++ b/TwitchChannelPointsMiner/classes/TwitchLogin.py @@ -179,20 +179,7 @@ def check_login(self): if self.token is None: return False - json_data = copy.deepcopy(GQLOperations.ReportMenuItem) - json_data["variables"] = {"channelLogin": self.username} - response = self.session.post(GQLOperations.url, json=json_data) - - if response.status_code == 200: - json_response = response.json() - if ( - "data" in json_response - and "user" in json_response["data"] - and json_response["data"]["user"]["id"] is not None - ): - self.user_id = json_response["data"]["user"]["id"] - self.login_check_result = True - + self.login_check_result = self.__set_user_id() return self.login_check_result def save_cookies(self, cookies_file): @@ -211,6 +198,7 @@ def get_cookie_value(self, key): if cookie["name"] == key: if cookie["value"] is not None: return cookie["value"] + return None def load_cookies(self, cookies_file): if os.path.isfile(cookies_file): @@ -219,7 +207,30 @@ def load_cookies(self, cookies_file): raise WrongCookiesException("There must be a cookies file!") def get_user_id(self): - return int(self.get_cookie_value("persistent").split("%")[0]) + persistent = self.get_cookie_value("persistent") + user_id = ( + int(persistent.split("%")[0]) if persistent is not None else self.user_id + ) + if user_id is None: + if self.__set_user_id() is True: + return self.user_id + return user_id + + def __set_user_id(self): + json_data = copy.deepcopy(GQLOperations.ReportMenuItem) + json_data["variables"] = {"channelLogin": self.username} + response = self.session.post(GQLOperations.url, json=json_data) + + if response.status_code == 200: + json_response = response.json() + if ( + "data" in json_response + and "user" in json_response["data"] + and json_response["data"]["user"]["id"] is not None + ): + self.user_id = json_response["data"]["user"]["id"] + return True + return False def get_auth_token(self): return self.get_cookie_value("auth-token") diff --git a/TwitchChannelPointsMiner/classes/TwitchWebSocket.py b/TwitchChannelPointsMiner/classes/TwitchWebSocket.py index 2e85bc5f..2b3dc334 100644 --- a/TwitchChannelPointsMiner/classes/TwitchWebSocket.py +++ b/TwitchChannelPointsMiner/classes/TwitchWebSocket.py @@ -43,7 +43,6 @@ def listen(self, topic, auth_token=None): data = {"topics": [str(topic)]} if topic.is_user_topic() and auth_token is not None: data["auth_token"] = auth_token - nonce = create_nonce() self.send({"type": "LISTEN", "nonce": nonce, "data": data}) diff --git a/TwitchChannelPointsMiner/classes/WebSocketsPool.py b/TwitchChannelPointsMiner/classes/WebSocketsPool.py index 71fa0fbc..8fecf6b2 100644 --- a/TwitchChannelPointsMiner/classes/WebSocketsPool.py +++ b/TwitchChannelPointsMiner/classes/WebSocketsPool.py @@ -142,7 +142,6 @@ def handle_reconnection(ws): # Why not create a new ws on the same array index? Let's try. self = ws.parent_pool self.ws[ws.index] = self.__new(ws.index) # Create a new connection. - # self.ws[ws.index].topics = ws.topics self.__start(ws.index) # Start a new thread. time.sleep(30)