end of s8 scan

code/kmd_ntx_api/models.py

@@ -185,6 +185,37 @@ class Meta:
         ]
 
 
+class mined_archive(models.Model):
+    block_height = models.PositiveIntegerField(default=0)
+    block_time = models.PositiveIntegerField(default=0)
+    block_datetime = models.DateTimeField()
+    value = models.DecimalField(max_digits=18, decimal_places=8)
+    address = models.CharField(max_length=34)
+    name = models.CharField(max_length=34)
+    txid = models.CharField(max_length=64)
+    diff = models.FloatField(default=0)
+    season = models.CharField(max_length=34)
+    usd_price = models.DecimalField(max_digits=18, decimal_places=8, default=0)
+    btc_price = models.DecimalField(max_digits=18, decimal_places=8, default=0)
+    category = models.CharField(max_length=34 ,default="")
+
+    class Meta:
+        db_table = 'mined_archive'
+        indexes = [
+            models.Index(fields=['name']),
+            models.Index(fields=['category']),
+            models.Index(fields=['season']),
+            models.Index(fields=['-block_height']),
+            models.Index(fields=['-block_time'])
+        ]
+        constraints = [
+            models.UniqueConstraint(
+                fields=['block_height'], 
+                name='unique_block_archive'
+            )
+        ]
+
+
 class mined_count_daily(models.Model):
     mined_date = models.DateField()
     notary = models.CharField(max_length=64)
@@ -360,6 +391,47 @@ class Meta:
         ]
 
 
+class notarised_archive(models.Model):
+    txid = models.CharField(max_length=64)
+    coin = models.CharField(max_length=32)
+    block_hash = models.CharField(max_length=64)
+    block_time = models.PositiveIntegerField(default=0)
+    block_datetime = models.DateTimeField()
+    block_height = models.PositiveIntegerField(default=0)
+    notaries = ArrayField(models.CharField(max_length=34),size=13)
+    notary_addresses = ArrayField(models.CharField(max_length=34),size=13, default=list)
+    ac_ntx_blockhash = models.CharField(max_length=64)
+    ac_ntx_height = models.PositiveIntegerField(default=0)
+    opret = models.CharField(max_length=2048)
+    season = models.CharField(max_length=32)
+    server = models.CharField(max_length=32, default='')
+    epoch = models.CharField(max_length=32, default='')
+    scored = models.BooleanField(default=True)
+    score_value = models.DecimalField(max_digits=18, decimal_places=8, default=0)
+
+    class Meta:
+        db_table = 'notarised_archive'
+        indexes = [
+            models.Index(fields=['txid']),
+            models.Index(fields=['-block_time']),
+            models.Index(fields=['-block_height']),
+            models.Index(fields=['season']),
+            models.Index(fields=['server']),
+            models.Index(fields=['epoch']),
+            models.Index(fields=['coin']),
+            models.Index(fields=['coin', '-block_height']),
+            models.Index(fields=['season', 'server']),
+            models.Index(fields=['season', 'server', 'coin']),
+            models.Index(fields=['season', 'server', 'epoch']),
+            models.Index(fields=['season', 'server', 'epoch', 'coin']),
+        ]
+        constraints = [
+            models.UniqueConstraint(fields=['txid'],
+                                 name='unique_txid_archive')
+        ]
+
+
+
 class notarised_coin_daily(models.Model):
     notarised_date = models.DateField()
     season = models.CharField(max_length=24)

code/requirements/base.txt

@@ -23,3 +23,4 @@ pytz==2022.7.1
 python-dateutil==2.8.2
 eth_keys==0.4.0
 pycryptodome==3.18.0
+pymemcache==4.0.0

code/scripts/archive_season_data.py

@@ -0,0 +1,49 @@
+#!/usr/bin/env python3
+from lib_const import *
+from decorators import *
+from lib_dpow_const import SEASONS_INFO
+from lib_query_ntx import select_from_notarised_tbl_where
+from lib_update_ntx import delete_from_notarised_tbl_where, update_ntx_row
+import lib_helper as helper
+import lib_ntx
+
+
+def archive_past_seasons(current_seasons=["Season_7"]):
+    for season in SEASONS_INFO:
+        print(season)
+        if season not in current_seasons:
+            print(season)
+            data = select_from_notarised_tbl_where(season=season)
+            for i in data:
+                txid = i[1]
+                coin = i[2]
+                block_hash = i[3]
+                block_time = i[4]
+                block_datetime = i[5]
+                block_height = i[6]
+                notaries = i[7]                
+                ac_ntx_blockhash = i[8]
+                ac_ntx_height = i[9]
+                opret = i[10]
+                season = i[11]
+                notary_addresses = i[12]
+                scored = i[13]
+                server = i[14]
+                score_value = i[15]                
+                epoch = i[16]
+
+                row_data = (coin, block_height, 
+                    block_time, block_datetime, block_hash, 
+                    notaries, notary_addresses,
+                    ac_ntx_blockhash, ac_ntx_height, txid,
+                    opret, season, server, scored,
+                    score_value, epoch)
+                update_ntx_row(row_data, table='notarised_archive', unique='unique_txid_archive')
+            delete_from_notarised_tbl_where(season=season)
+
+
+
+
+if __name__ == '__main__':
+    current_seasons = ["Season_7"]
+    archive_past_seasons(current_seasons)

code/scripts/cron_update_ntx_tables.py

@@ -22,7 +22,6 @@ def update_ntx_tables(seasons, rescan=False):
 
         notarised_table = lib_ntx.notarised(season, rescan)
         if CLEAN_UP:
-            ntx_season_tables.clean_up()
             notarised_table.clean_up()
             rescan = True
 
@@ -32,6 +31,8 @@ def update_ntx_tables(seasons, rescan=False):
         last_ntx_tables.update_coin_table()
         last_ntx_tables.update_notary_table()
         ntx_season_tables = lib_ntx.ntx_season_stats(season)
+        if CLEAN_UP:
+            ntx_season_tables.clean_up()
         ntx_season_tables.update_ntx_season_stats_tables()
         ntx_daily_tables = lib_ntx.ntx_daily_stats(season, rescan)
         ntx_daily_tables.update_daily_ntx_tables()

code/scripts/lib_dpow_const.py

@@ -527,8 +527,8 @@ def get_season_server_coins(season, server):
         "start_block": 3484958,
         "end_block": 4484958,
         "start_time": 1688132253,
-        "end_time": 1988132253,
-        "post_season_end_time": 1988132253,
+        "end_time": 1717199999,
+        "post_season_end_time": 1725148799,
         "servers": {}
     },
     "VOTE2022_Testnet": {

code/scripts/lib_update_ntx.py

@@ -6,15 +6,15 @@
 from lib_filter import get_notarised_conditions_filter
 
 
-def update_ntx_row(row_data):
+def update_ntx_row(row_data, table='notarised', unique='unique_txid'):
     CONN = connect_db()
     CURSOR = CONN.cursor()
-    sql = f"INSERT INTO notarised (coin, block_height, \
+    sql = f"INSERT INTO {table} (coin, block_height, \
                                 block_time, block_datetime, block_hash, \
                                 notaries, notary_addresses, ac_ntx_blockhash, ac_ntx_height, \
                                 txid, opret, season, server, scored, score_value, epoch) \
             VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) \
-                ON CONFLICT ON CONSTRAINT unique_txid DO UPDATE SET \
+                ON CONFLICT ON CONSTRAINT {unique} DO UPDATE SET \
                 season='{row_data[11]}', server='{row_data[12]}', scored='{row_data[13]}', \
                 notaries=ARRAY{row_data[5]}, notary_addresses=ARRAY{row_data[6]}, \
                 score_value={row_data[14]}, epoch='{row_data[15]}';"
@@ -29,6 +29,7 @@ def update_ntx_row(row_data):
 
 
 
+
 def update_server_notarised_tbl(old_server, server):
     CONN = connect_db()
     CURSOR = CONN.cursor()
@@ -157,7 +158,7 @@ def delete_from_notarised_tbl_where(
         coin=coin, include_coins=include_coins,
         exclude_coins=exclude_coins
     )
-    CURSOR.execute()
+    CURSOR.execute(sql)
     CONN.commit()
 
 
@@ -200,7 +201,6 @@ def update_server_ntx_season_row(row_data):
     CONN.commit()
 
 
-
 def update_daily_notarised_coin_row(row_data):
     CONN = connect_db()
     CURSOR = CONN.cursor()

code/scripts/rescan_season_ntx.py

@@ -0,0 +1,29 @@
+#!/usr/bin/env python3
+from decorators import print_runtime
+import lib_ntx
+
+'''
+Run after season ends to ensure full ntx coverage
+and epoch scores alignment. Runtime 4-5hrs.
+'''
+
+@print_runtime
+def rescan_season(season="Season_7"):
+    notarised_table = lib_ntx.notarised(season, rescan=True)
+    # Revalidates existing data, run after epoch changes to correct the scores.
+    notarised_table.clean_up()
+    # Scan all season blocks for missing txids
+    notarised_table.update_table()
+
+    ntx_season_tables = lib_ntx.ntx_season_stats(season)
+    # Deletes all season ntx aggregate data for a clean slate
+    ntx_season_tables.clean_up()
+    # Recalculates season ntx aggregate data
+    ntx_season_tables.update_ntx_season_stats_tables()
+
+    ntx_daily_tables = lib_ntx.ntx_daily_stats(season, rescan=True)
+    # Recalculate all daily ntx stats data
+    ntx_daily_tables.update_daily_ntx_tables()
+
+if __name__ == "__main__":
+    rescan_season()

docker-compose.yml

@@ -112,12 +112,28 @@ services:
       - mm2
       - db
       - kmd
+      - memcached
     logging:
       driver: "json-file"
       options:
         max-size: "20m"
         max-file: "10"
 
+  memcached:
+    container_name: memcached
+    image: memcached:latest
+    ports:
+      - "127.0.0.1:11211:11211"
+    restart: always
+    depends_on:
+      - pgsqldb
+    command: ["memcached", "-I", "40m", "-m", "100m"]
+    ulimits:
+      nproc: 65535
+      nofile:
+        soft: 65535
+        hard: 65535
+
 networks:
   default:
     ipam: