feat: update to more general layout

.github/workflows/push-main.yml

@@ -23,126 +23,100 @@ jobs:
         run: make ps
 
       - name: Run tests with coverage
-        run: make test-coverage
-
-#      - uses: codecov/codecov-action@v2
-#        with:
-#          token: ${{ secrets.CODECOV_TOKEN }}
-#          files: ./coverage.xml
-#          fail_ci_if_error: true
-#
-#  docker:
-#    runs-on: ubuntu-latest
-#    needs: test
-#    outputs:
-#      tag: ${{ steps.source.outputs.TAG }}
-#
-#    steps:
-#      - name: Checkout
-#        uses: actions/checkout@v2
-#
-#      - id: last
-#        uses: pozetroninc/github-action-get-latest-release@master
-#        with:
-#          repository: ${{ github.repository }}
-#
-#      - name: Tag name
-#        id: source
-#        run: |
-#          echo ::set-output name=TAG::${{ steps.last.outputs.release }}-${{ github.run_number }}
-#
-#      - name: Login to DockerHub
-#        uses: docker/login-action@v1
-#        with:
-#          username: ${{ secrets.DOCKERHUB_USERNAME }}
-#          password: ${{ secrets.DOCKERHUB_TOKEN }}
-#
-#      - name: Build and push API
-#        uses: docker/build-push-action@v2
-#        with:
-#          context: .
-#          file: ./Dockerfile
-#          target: prod
-#          push: true
-#          build-args: |
-#            SERVICE_NAME=api
-#          tags: balancednetwork/icon-governance-api:latest, balancednetwork/icon-governance-api:${{ steps.source.outputs.TAG }}
-#
-#      - name: Build and push worker
-#        uses: docker/build-push-action@v2
-#        with:
-#          context: .
-#          file: ./Dockerfile
-#          target: prod
-#          push: true
-#          build-args: |
-#            SERVICE_NAME=worker
-#          tags: balancednetwork/icon-governance-worker:latest, balancednetwork/icon-governance-worker:${{ steps.source.outputs.TAG }}
-#
-#  push_refs:
-#    runs-on: ubuntu-latest
-#    needs: docker
-#    strategy:
-#      max-parallel: 1
-#      matrix:
-#        include:
-#          - cluster: prod-sng
-#            network_name: mainnet
-#            network_version: v2
-#          - cluster: prod-sng
-#            network_name: sejong
-#            network_version: v2
-#          - cluster: prod-sng
-#            network_name: lisbon
-#            network_version: v2
-#          - cluster: prod-sng
-#            network_name: berlin
-#            network_version: v2
-#          - cluster: prod-ams
-#            network_name: mainnet
-#            network_version: v2
-#          - cluster: prod-ams
-#            network_name: sejong
-#            network_version: v2
-#          - cluster: prod-ams
-#            network_name: lisbon
-#            network_version: v2
-#          - cluster: prod-ams
-#            network_name: berlin
-#            network_version: v2
-#
-#    steps:
-#      - name: Checkout charts repo
-#        uses: actions/checkout@v2
-#        with:
-#          repository: sudoblockio/icon-charts
-#          ref: main
-#          path: charts
-#          token: ${{ secrets.ICON_CHARTS_PAT }}
-#
-#      - name: Update API ${{ matrix.cluster }}/${{ matrix.network_name }}-${{ matrix.network_version }} deployment values file
-#        uses: fjogeleit/yaml-update-action@master
-#        with:
-#          workDir: charts
-#          repository: sudoblockio/icon-charts
-#          valueFile: 'deployments/${{ matrix.cluster }}/${{ matrix.network_name }}-${{ matrix.network_version }}/governance/tags.yaml'
-#          propertyPath: 'api.image.tag'
-#          value: ${{needs.docker.outputs.tag}}
-#          branch: main
-#          createPR: 'false'
-#          updateFile: true
-#          commitChange: false
-#
-#      - name: Update worker ${{ matrix.cluster }}/${{ matrix.network_name }}-${{ matrix.network_version }} deployment values file
-#        uses: fjogeleit/yaml-update-action@master
-#        with:
-#          workDir: charts
-#          repository: sudoblockio/icon-charts
-#          valueFile: 'deployments/${{ matrix.cluster }}/${{ matrix.network_name }}-${{ matrix.network_version }}/governance/tags.yaml'
-#          propertyPath: 'worker.image.tag'
-#          value: ${{needs.docker.outputs.tag}}
-#          branch: main
-#          createPR: 'false'
-#          updateFile: true
-#          message: "governance ${{ matrix.cluster }}/${{ matrix.network_name }}-${{ matrix.network_version }} deployment image version to ${{needs.docker.outputs.tag}}"
-#          token: '${{ secrets.ICON_CHARTS_PAT }}'
+        run: make test
+
+  docker:
+    runs-on: ubuntu-latest
+    needs: test
+    outputs:
+      tag: ${{ steps.source.outputs.TAG }}
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - id: last
+        uses: pozetroninc/github-action-get-latest-release@master
+        with:
+          repository: ${{ github.repository }}
+
+      - name: Tag name
+        id: source
+        run: |
+          echo ::set-output name=TAG::${{ steps.last.outputs.release }}-${{ github.run_number }}
+
+      - name: Login to DockerHub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Build and push API
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          file: ./Dockerfile
+          target: prod
+          push: true
+          build-args: |
+            SERVICE_NAME=api
+          tags: balancednetwork/balanced-backend-api:latest, balancednetwork/balanced-backend-api:${{ steps.source.outputs.TAG }}
+
+      - name: Build and push worker
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          file: ./Dockerfile
+          target: prod
+          push: true
+          build-args: |
+            SERVICE_NAME=worker
+          tags: balancednetwork/balanced-backend-worker:latest, balancednetwork/balanced-backend-worker:${{ steps.source.outputs.TAG }}
+
+  push_refs:
+    runs-on: ubuntu-latest
+    needs: docker
+    strategy:
+      max-parallel: 1
+      matrix:
+        include:
+          - cluster: prod-sng
+            network_name: mainnet
+          - cluster: prod-ams
+            network_name: mainnet
+
+    steps:
+      - name: Checkout charts repo
+        uses: actions/checkout@v2
+        with:
+          repository: balancednetwork/balanced-community-infra
+          ref: main
+          path: charts
+          token: ${{ secrets.INFRA_DEPLOYMENT_PAT }}
+
+      - name: Update API ${{ matrix.cluster }}/${{ matrix.network_name }} deployment values file
+        uses: fjogeleit/yaml-update-action@master
+        with:
+          workDir: charts
+          repository: balancednetwork/balanced-community-infra
+          valueFile: 'deployments/${{ matrix.cluster }}/${{ matrix.network_name }}/backend-v2/tags.yaml'
+          propertyPath: 'api.image.tag'
+          value: ${{needs.docker.outputs.tag}}
+          branch: main
+          createPR: 'false'
+          updateFile: true
+          commitChange: false
+
+      - name: Update worker ${{ matrix.cluster }}/${{ matrix.network_name }} deployment values file
+        uses: fjogeleit/yaml-update-action@master
+        with:
+          workDir: charts
+          repository: balancednetwork/balanced-community-infra
+          valueFile: 'deployments/${{ matrix.cluster }}/${{ matrix.network_name }}/backend-v2/tags.yaml'
+          propertyPath: 'worker.image.tag'
+          value: ${{needs.docker.outputs.tag}}
+          branch: main
+          createPR: 'false'
+          updateFile: true
+          message: "backend-v2 ${{ matrix.cluster }}/${{ matrix.network_name }} deployment image version to ${{needs.docker.outputs.tag}}"
+          token: '${{ secrets.INFRA_DEPLOYMENT_PAT }}'

.gitignore

@@ -6,6 +6,7 @@ __pycache__/
 *$py.class
 
 __pytest*
+.pytest_cache
 
 # C extensions
 *.so

Makefile

@@ -16,10 +16,6 @@ test-unit:  ## Run unit tests
 test-integration:  ## Run integration tests - Need DB compose up
 	python3 -m pytest tests/integration
 
-test-coverage:  ## Run unit tests - Need DB compose up
-	PYTHONPATH=$PYTHONPATH:`pwd` pytest --cov=balanced_backend --cov-report xml tests/integration
-	PYTHONPATH=$PYTHONPATH:`pwd` pytest --cov=balanced_backend --cov-append --cov-report xml tests/unit
-
 up:  ## Bring everything up as containers
 	docker-compose -f docker-compose.db.yml -f docker-compose.yml up -d
 

NOTES.md

@@ -54,3 +54,20 @@ TTD:
     - https://balanced.sudoblock.io/api/v1/stats/token-stats
   - Stats 
     - https://balanced.sudoblock.io/api/v1/dex/stats/10
+
+
+## Additional Improvments 
+
+#### Add skip to Tx feature
+
+The above routines work well if you have very active addresses where the method changes each day but if we are going to index a large number of addresses (ie, every address in ICON) then we should think about a few optimizations to speed up sync time.  
+
+Further, if we want to be able to get more fine grained charts (ie every hour?), then the num data points will be much higher. 
+
+We could instead put something in that is some callable to determine when the next time to run the method. 
+
+Would work by supplying a callable that would inform the rpc getter when to call the next get. 
+
+Would be a different endpoint and produce a sparse time series 
+
+Endpoint could fill in the gaps but would be better if the FE did that 

balanced_backend/alembic/env.py

@@ -8,8 +8,7 @@
 
 from balanced_backend.db import ASYNC_SQLALCHEMY_DATABASE_URL, SQLALCHEMY_DATABASE_URL
 
-from balanced_backend.models.loans_chart import LoansChart
-from balanced_backend.models.bnusd_supply_chart import BnusdSupplyChart
+from balanced_backend.models.historical import DailyHistorical
 
 # Other versions imported each object
 config = context.config

balanced_backend/alembic/versions/134216866d62_init.py

@@ -0,0 +1,45 @@
+"""init
+
+Revision ID: 134216866d62
+Revises: 
+Create Date: 2022-06-20 05:00:18.204096
+
+"""
+from alembic import op
+import sqlalchemy as sa
+import sqlmodel
+
+
+# revision identifiers, used by Alembic.
+revision = '134216866d62'
+down_revision = None
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('daily_historical',
+    sa.Column('date', sa.DateTime(timezone=True), nullable=True),
+    sa.Column('timestamp', sa.Integer(), nullable=True),
+    sa.Column('update_interval', sa.Integer(), nullable=True),
+    sa.Column('days_since_launch', sa.Integer(), nullable=True),
+    sa.Column('address', sqlmodel.sql.sqltypes.AutoString(), nullable=True),
+    sa.Column('contract_name', sqlmodel.sql.sqltypes.AutoString(), nullable=True),
+    sa.Column('method', sqlmodel.sql.sqltypes.AutoString(), nullable=True),
+    sa.Column('value', sa.Float(), nullable=True),
+    sa.PrimaryKeyConstraint('timestamp', 'address', 'method')
+    )
+    op.create_index(op.f('ix_daily_historical_contract_name'), 'daily_historical', ['contract_name'], unique=False)
+    op.create_index(op.f('ix_daily_historical_days_since_launch'), 'daily_historical', ['days_since_launch'], unique=False)
+    op.create_index(op.f('ix_daily_historical_update_interval'), 'daily_historical', ['update_interval'], unique=False)
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index(op.f('ix_daily_historical_update_interval'), table_name='daily_historical')
+    op.drop_index(op.f('ix_daily_historical_days_since_launch'), table_name='daily_historical')
+    op.drop_index(op.f('ix_daily_historical_contract_name'), table_name='daily_historical')
+    op.drop_table('daily_historical')
+    # ### end Alembic commands ###