-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathcollector.py
185 lines (146 loc) · 6.51 KB
/
collector.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
import time
from datetime import datetime
from threading import Thread
from typing import Iterable
from database import DatabaseReader, DatabaseSaver
from custom_logger import Logger
from utils import request_api
logger = Logger('gwaff.collect')
MAX_RETRIES: int = 5 # How many times to attempt to collect and save data
WAIT_SUCCESS: int = 60 # How many minutes to wait after a success
WAIT_FAIL: int = 30 # How many minutes to wait after a failure
MIN_SEPARATION: int = 30 # Do not store new data if the last collection was less than this many minutes ago
COLLECTION_SMALL: int = 2 # Collect data from up to this page every collection event
COLLECTION_LARGE: int = 6 # Collect data from up to this page every second collection event
COLLECTION_LARGEST: int = 10 # Update names up to this page when updating names
SERVER_ID = "377946908783673344"
API_URL = f"https://joegaming.duckdns.org/polaris/api/leaderboard/{SERVER_ID}"
def record_data(pages: Iterable[int] = range(1, COLLECTION_LARGE),
min_time: int = MIN_SEPARATION) -> bool:
"""
Record the current XP data and ensure records are separated by at least min_time minutes.
Args:
pages (Iterable[int]): The pages to collect data from. Defaults to range(1, COLLECTION_LARGE).
min_time (int): Minimum time in minutes between data collections. Defaults to MIN_SEPARATION.
Returns:
bool: True if data was successfully gathered, False otherwise.
"""
logger.info("Starting data collection")
dbr = DatabaseReader()
lasttime = dbr.get_last_timestamp()
now = datetime.now()
difference = now - lasttime
if difference.total_seconds() < min_time * 60:
logger.info(
f"Too soon - {int(difference.total_seconds() / 60)}/{min_time} minutes required")
return False
dbi = DatabaseSaver()
success, failure = (0, 0)
for page in pages:
data = request_api(API_URL, page=page)
if not data:
logger.error("Skipping page after max retries")
continue
leaderboard = data.get('leaderboard', [])
for member in leaderboard:
if 'missing' not in member and member.get('color', '#000000') != '#000000':
count = 0
while count < MAX_RETRIES:
try:
# Extract member data
member_id = int(member.get('id'))
xp = int(member.get('xp'))
name = (member.get('nickname')
or member.get('displayName')
or member.get('username'))
colour = member.get('color')
avatar = member.get('avatar')
if any(x is None for x in (member_id, xp)):
logger.warning(f"Skipping record with missing data")
failure += 1
break
# Update profile and record
dbi.update_profile(member_id, name, colour, avatar)
dbi.insert_record(member_id, now, xp)
success += 1
break # Exit retry loop on success
except Exception as e:
logger.warning(f"Failed to save record (attempt {count + 1}): {str(e)}")
if count < MAX_RETRIES:
count += 1
else:
logger.error("Skipping record after max retries")
failure += 1
break
logger.debug(f"Page {page} collected")
dbi.commit()
if success > failure:
logger.info("Successfully saved the latest data!")
return True
else:
logger.error("Considerable record save failures!")
return False
def update_profiles(pages: Iterable[int] = range(1, COLLECTION_LARGEST)) -> None:
logger.info("Starting profile collection")
dbi = DatabaseSaver()
success, failure = (0, 0)
for page in pages:
data = request_api(API_URL, page=page)
if not data:
logger.error("Skipping page after max retries")
continue
leaderboard = data.get('leaderboard', [])
for member in leaderboard:
if 'missing' not in member and member.get('color', '#000000') != '#000000':
count = 0
while count < MAX_RETRIES:
try:
# Extract member data
member_id = int(member.get('id'))
name = (member.get('nickname')
or member.get('displayName')
or member.get('username'))
colour = member.get('color')
avatar = member.get('avatar')
if member_id is None:
logger.warning(f"Skipping profile with missing data")
failure += 1
break
# Update profile
dbi.update_profile(member_id, name, colour, avatar)
success += 1
break # Exit retry loop on success
except Exception as e:
logger.warning(f"Failed to save record (attempt {count + 1}): {str(e)}")
if count < MAX_RETRIES:
count += 1
else:
logger.error("Skipping record after max retries")
failure += 1
break
logger.debug(f"Page {page} updated")
dbi.commit()
def run() -> None:
"""
Periodically collects data.
"""
while True:
success = record_data(pages=range(1, COLLECTION_LARGE + 1))
wait = WAIT_SUCCESS if success else WAIT_FAIL
for i in range(wait // 10):
logger.debug(f"Slept {i * 10}/{wait} minutes")
time.sleep(10 * 60)
success = record_data(pages=range(1, COLLECTION_SMALL + 1))
wait = WAIT_SUCCESS if success else WAIT_FAIL
for i in range(wait // 10):
logger.debug(f"Slept {i * 10}/{wait} minutes")
time.sleep(10 * 60)
def collect() -> None:
"""
Creates a thread to periodically collect data.
"""
t = Thread(target=run)
t.start()
if __name__ == '__main__':
collect()
logger.info("Collection started")