diff --git a/code/kmd_ntx_api/models.py b/code/kmd_ntx_api/models.py index bac75eb..f44084a 100644 --- a/code/kmd_ntx_api/models.py +++ b/code/kmd_ntx_api/models.py @@ -185,6 +185,37 @@ class Meta: ] +class mined_archive(models.Model): + block_height = models.PositiveIntegerField(default=0) + block_time = models.PositiveIntegerField(default=0) + block_datetime = models.DateTimeField() + value = models.DecimalField(max_digits=18, decimal_places=8) + address = models.CharField(max_length=34) + name = models.CharField(max_length=34) + txid = models.CharField(max_length=64) + diff = models.FloatField(default=0) + season = models.CharField(max_length=34) + usd_price = models.DecimalField(max_digits=18, decimal_places=8, default=0) + btc_price = models.DecimalField(max_digits=18, decimal_places=8, default=0) + category = models.CharField(max_length=34 ,default="") + + class Meta: + db_table = 'mined_archive' + indexes = [ + models.Index(fields=['name']), + models.Index(fields=['category']), + models.Index(fields=['season']), + models.Index(fields=['-block_height']), + models.Index(fields=['-block_time']) + ] + constraints = [ + models.UniqueConstraint( + fields=['block_height'], + name='unique_block_archive' + ) + ] + + class mined_count_daily(models.Model): mined_date = models.DateField() notary = models.CharField(max_length=64) @@ -360,6 +391,47 @@ class Meta: ] +class notarised_archive(models.Model): + txid = models.CharField(max_length=64) + coin = models.CharField(max_length=32) + block_hash = models.CharField(max_length=64) + block_time = models.PositiveIntegerField(default=0) + block_datetime = models.DateTimeField() + block_height = models.PositiveIntegerField(default=0) + notaries = ArrayField(models.CharField(max_length=34),size=13) + notary_addresses = ArrayField(models.CharField(max_length=34),size=13, default=list) + ac_ntx_blockhash = models.CharField(max_length=64) + ac_ntx_height = models.PositiveIntegerField(default=0) + opret = models.CharField(max_length=2048) + season = models.CharField(max_length=32) + server = models.CharField(max_length=32, default='') + epoch = models.CharField(max_length=32, default='') + scored = models.BooleanField(default=True) + score_value = models.DecimalField(max_digits=18, decimal_places=8, default=0) + + class Meta: + db_table = 'notarised_archive' + indexes = [ + models.Index(fields=['txid']), + models.Index(fields=['-block_time']), + models.Index(fields=['-block_height']), + models.Index(fields=['season']), + models.Index(fields=['server']), + models.Index(fields=['epoch']), + models.Index(fields=['coin']), + models.Index(fields=['coin', '-block_height']), + models.Index(fields=['season', 'server']), + models.Index(fields=['season', 'server', 'coin']), + models.Index(fields=['season', 'server', 'epoch']), + models.Index(fields=['season', 'server', 'epoch', 'coin']), + ] + constraints = [ + models.UniqueConstraint(fields=['txid'], + name='unique_txid_archive') + ] + + + class notarised_coin_daily(models.Model): notarised_date = models.DateField() season = models.CharField(max_length=24) diff --git a/code/requirements/base.txt b/code/requirements/base.txt index d51742f..26b36d5 100644 --- a/code/requirements/base.txt +++ b/code/requirements/base.txt @@ -23,3 +23,4 @@ pytz==2022.7.1 python-dateutil==2.8.2 eth_keys==0.4.0 pycryptodome==3.18.0 +pymemcache==4.0.0 diff --git a/code/scripts/archive_season_data.py b/code/scripts/archive_season_data.py new file mode 100755 index 0000000..ea5da1e --- /dev/null +++ b/code/scripts/archive_season_data.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 +from lib_const import * +from decorators import * +from lib_dpow_const import SEASONS_INFO +from lib_query_ntx import select_from_notarised_tbl_where +from lib_update_ntx import delete_from_notarised_tbl_where, update_ntx_row +import lib_helper as helper +import lib_ntx + + +def archive_past_seasons(current_seasons=["Season_7"]): + for season in SEASONS_INFO: + print(season) + if season not in current_seasons: + print(season) + data = select_from_notarised_tbl_where(season=season) + for i in data: + txid = i[1] + coin = i[2] + block_hash = i[3] + block_time = i[4] + block_datetime = i[5] + block_height = i[6] + notaries = i[7] + ac_ntx_blockhash = i[8] + ac_ntx_height = i[9] + opret = i[10] + season = i[11] + notary_addresses = i[12] + scored = i[13] + server = i[14] + score_value = i[15] + epoch = i[16] + + row_data = (coin, block_height, + block_time, block_datetime, block_hash, + notaries, notary_addresses, + ac_ntx_blockhash, ac_ntx_height, txid, + opret, season, server, scored, + score_value, epoch) + update_ntx_row(row_data, table='notarised_archive', unique='unique_txid_archive') + delete_from_notarised_tbl_where(season=season) + + + + +if __name__ == '__main__': + current_seasons = ["Season_7"] + archive_past_seasons(current_seasons) diff --git a/code/scripts/cron_update_ntx_tables.py b/code/scripts/cron_update_ntx_tables.py index 5b6e1a3..3f8c067 100755 --- a/code/scripts/cron_update_ntx_tables.py +++ b/code/scripts/cron_update_ntx_tables.py @@ -22,7 +22,6 @@ def update_ntx_tables(seasons, rescan=False): notarised_table = lib_ntx.notarised(season, rescan) if CLEAN_UP: - ntx_season_tables.clean_up() notarised_table.clean_up() rescan = True @@ -32,6 +31,8 @@ def update_ntx_tables(seasons, rescan=False): last_ntx_tables.update_coin_table() last_ntx_tables.update_notary_table() ntx_season_tables = lib_ntx.ntx_season_stats(season) + if CLEAN_UP: + ntx_season_tables.clean_up() ntx_season_tables.update_ntx_season_stats_tables() ntx_daily_tables = lib_ntx.ntx_daily_stats(season, rescan) ntx_daily_tables.update_daily_ntx_tables() diff --git a/code/scripts/lib_dpow_const.py b/code/scripts/lib_dpow_const.py index fce7aab..1d18897 100644 --- a/code/scripts/lib_dpow_const.py +++ b/code/scripts/lib_dpow_const.py @@ -527,8 +527,8 @@ def get_season_server_coins(season, server): "start_block": 3484958, "end_block": 4484958, "start_time": 1688132253, - "end_time": 1988132253, - "post_season_end_time": 1988132253, + "end_time": 1717199999, + "post_season_end_time": 1725148799, "servers": {} }, "VOTE2022_Testnet": { diff --git a/code/scripts/lib_update_ntx.py b/code/scripts/lib_update_ntx.py index 793a057..8a1e6e6 100644 --- a/code/scripts/lib_update_ntx.py +++ b/code/scripts/lib_update_ntx.py @@ -6,15 +6,15 @@ from lib_filter import get_notarised_conditions_filter -def update_ntx_row(row_data): +def update_ntx_row(row_data, table='notarised', unique='unique_txid'): CONN = connect_db() CURSOR = CONN.cursor() - sql = f"INSERT INTO notarised (coin, block_height, \ + sql = f"INSERT INTO {table} (coin, block_height, \ block_time, block_datetime, block_hash, \ notaries, notary_addresses, ac_ntx_blockhash, ac_ntx_height, \ txid, opret, season, server, scored, score_value, epoch) \ VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) \ - ON CONFLICT ON CONSTRAINT unique_txid DO UPDATE SET \ + ON CONFLICT ON CONSTRAINT {unique} DO UPDATE SET \ season='{row_data[11]}', server='{row_data[12]}', scored='{row_data[13]}', \ notaries=ARRAY{row_data[5]}, notary_addresses=ARRAY{row_data[6]}, \ score_value={row_data[14]}, epoch='{row_data[15]}';" @@ -29,6 +29,7 @@ def update_ntx_row(row_data): + def update_server_notarised_tbl(old_server, server): CONN = connect_db() CURSOR = CONN.cursor() @@ -157,7 +158,7 @@ def delete_from_notarised_tbl_where( coin=coin, include_coins=include_coins, exclude_coins=exclude_coins ) - CURSOR.execute() + CURSOR.execute(sql) CONN.commit() @@ -200,7 +201,6 @@ def update_server_ntx_season_row(row_data): CONN.commit() - def update_daily_notarised_coin_row(row_data): CONN = connect_db() CURSOR = CONN.cursor() diff --git a/code/scripts/rescan_season_ntx.py b/code/scripts/rescan_season_ntx.py new file mode 100755 index 0000000..b138b66 --- /dev/null +++ b/code/scripts/rescan_season_ntx.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python3 +from decorators import print_runtime +import lib_ntx + +''' +Run after season ends to ensure full ntx coverage +and epoch scores alignment. Runtime 4-5hrs. +''' + +@print_runtime +def rescan_season(season="Season_7"): + notarised_table = lib_ntx.notarised(season, rescan=True) + # Revalidates existing data, run after epoch changes to correct the scores. + notarised_table.clean_up() + # Scan all season blocks for missing txids + notarised_table.update_table() + + ntx_season_tables = lib_ntx.ntx_season_stats(season) + # Deletes all season ntx aggregate data for a clean slate + ntx_season_tables.clean_up() + # Recalculates season ntx aggregate data + ntx_season_tables.update_ntx_season_stats_tables() + + ntx_daily_tables = lib_ntx.ntx_daily_stats(season, rescan=True) + # Recalculate all daily ntx stats data + ntx_daily_tables.update_daily_ntx_tables() + +if __name__ == "__main__": + rescan_season() diff --git a/docker-compose.yml b/docker-compose.yml index 7a730b6..62b1490 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -112,12 +112,28 @@ services: - mm2 - db - kmd + - memcached logging: driver: "json-file" options: max-size: "20m" max-file: "10" + memcached: + container_name: memcached + image: memcached:latest + ports: + - "127.0.0.1:11211:11211" + restart: always + depends_on: + - pgsqldb + command: ["memcached", "-I", "40m", "-m", "100m"] + ulimits: + nproc: 65535 + nofile: + soft: 65535 + hard: 65535 + networks: default: ipam: