From c4d27fa21f5111fb05dd4afe5cab17e520c2bedf Mon Sep 17 00:00:00 2001 From: Earle Lowe Date: Fri, 3 Nov 2023 10:29:35 -0700 Subject: [PATCH 01/11] Many mypy update --- app/api/dependencies.py | 19 +++++++++++++------ app/api/v1/activities.py | 15 ++++++--------- app/api/v1/core.py | 4 +++- app/api/v1/cron.py | 28 +++++++++++++--------------- app/api/v1/keys.py | 8 ++++---- app/api/v1/tokens.py | 20 ++++++++++---------- app/api/v1/transactions.py | 10 +++++----- app/core/chialisp/gateway.py | 3 ++- app/core/types.py | 8 +++++--- app/crud/chia.py | 13 ++++++++----- app/crud/db.py | 21 +++++++++++++-------- app/db/session.py | 15 +++++---------- app/errors.py | 8 ++++---- app/main.py | 2 +- app/models/activity.py | 2 +- app/models/state.py | 2 +- app/schemas/activity.py | 8 +++++--- app/utils.py | 11 ----------- pytest.ini | 2 +- 19 files changed, 100 insertions(+), 99 deletions(-) diff --git a/app/api/dependencies.py b/app/api/dependencies.py index 2e6a06d..3b993fc 100644 --- a/app/api/dependencies.py +++ b/app/api/dependencies.py @@ -1,12 +1,14 @@ import enum +from contextlib import asynccontextmanager from pathlib import Path -from typing import Iterator +from typing import AsyncGenerator, AsyncIterator from chia.rpc.full_node_rpc_client import FullNodeRpcClient from chia.rpc.rpc_client import RpcClient from chia.rpc.wallet_rpc_client import WalletRpcClient from chia.util.config import load_config from chia.util.default_root import DEFAULT_ROOT_PATH +from chia.util.ints import uint16 from sqlalchemy.orm import Session from app.config import settings @@ -14,7 +16,8 @@ from app.logger import logger -async def get_db_session() -> Session: +@asynccontextmanager +async def get_db_session() -> AsyncIterator[Session]: SessionLocal = await get_session_local_cls() db = SessionLocal() @@ -37,17 +40,20 @@ async def _get_rpc_client( self_hostname: str, rpc_port: int, root_path: Path = DEFAULT_ROOT_PATH, -) -> Iterator[RpcClient]: +) -> AsyncGenerator[RpcClient, None]: rpc_client_cls = { NodeType.FULL_NODE: FullNodeRpcClient, NodeType.WALLET: WalletRpcClient, }.get(node_type) + if rpc_client_cls is None: + raise ValueError(f"Invalid node_type: {node_type}") + config = load_config(root_path, "config.yaml") client = await rpc_client_cls.create( self_hostname=self_hostname, - port=rpc_port, + port=uint16(rpc_port), root_path=root_path, net_config=config, ) @@ -61,7 +67,7 @@ async def _get_rpc_client( await client.await_closed() -async def get_wallet_rpc_client() -> Iterator[WalletRpcClient]: +async def get_wallet_rpc_client() -> AsyncIterator[WalletRpcClient]: async for _ in _get_rpc_client( node_type=NodeType.WALLET, self_hostname=settings.CHIA_HOSTNAME, @@ -71,7 +77,8 @@ async def get_wallet_rpc_client() -> Iterator[WalletRpcClient]: yield _ -async def get_full_node_rpc_client() -> Iterator[FullNodeRpcClient]: +@asynccontextmanager +async def get_full_node_rpc_client() -> AsyncIterator[FullNodeRpcClient]: async for _ in _get_rpc_client( node_type=NodeType.FULL_NODE, self_hostname=settings.CHIA_HOSTNAME, diff --git a/app/api/v1/activities.py b/app/api/v1/activities.py index 4028b29..f0497e2 100644 --- a/app/api/v1/activities.py +++ b/app/api/v1/activities.py @@ -1,7 +1,8 @@ -from typing import Dict, List, Optional -from pydantic import ValidationError +from typing import Any, Dict, List, Optional + from fastapi import APIRouter, Depends from fastapi.encoders import jsonable_encoder +from pydantic import ValidationError from sqlalchemy.orm import Session from app import crud, models, schemas @@ -11,7 +12,7 @@ from app.errors import ErrorCode from app.logger import logger from app.utils import disallow -import pprint + router = APIRouter() @@ -26,7 +27,7 @@ async def get_activity( limit: int = 10, sort: str = "desc", db: Session = Depends(deps.get_db_session), -): +) -> schemas.ActivitiesResponse: """Get activity. This endpoint is to be called by the explorer. @@ -34,7 +35,7 @@ async def get_activity( db_crud = crud.DBCrud(db=db) - activity_filters = {"or": [], "and": []} + activity_filters: Dict[str, Any] = {"or": [], "and": []} cw_filters = {} match search_by: case schemas.ActivitySearchBy.ONCHAIN_METADATA: @@ -96,12 +97,8 @@ async def get_activity( ) return schemas.ActivitiesResponse() - pp = pprint.PrettyPrinter(indent=4) - - pp.pprint(f"Got {len(activities)} activities from activities table.") activities_with_cw: List[schemas.ActivityWithCW] = [] for activity in activities: - pp.pprint(f"Checking activity: {activity}") unit = units.get(activity.asset_id) if unit is None: continue diff --git a/app/api/v1/core.py b/app/api/v1/core.py index 0e3aa9e..83bf574 100644 --- a/app/api/v1/core.py +++ b/app/api/v1/core.py @@ -1,3 +1,5 @@ +from typing import Dict + from fastapi import APIRouter from app.api.v1 import activities, cron, keys, tokens, transactions @@ -9,7 +11,7 @@ @router.get("/info") -async def get_info(): +async def get_info() -> Dict[str, str]: return { "blockchain_name": "Chia Network", "blockchain_name_short": "chia", diff --git a/app/api/v1/cron.py b/app/api/v1/cron.py index 36c2a1a..26bb5bb 100644 --- a/app/api/v1/cron.py +++ b/app/api/v1/cron.py @@ -1,5 +1,5 @@ import asyncio -from typing import Any, Dict, List, Optional +from typing import List from blspy import G1Element from chia.consensus.block_record import BlockRecord @@ -18,7 +18,7 @@ from app.errors import ErrorCode from app.logger import logger from app.models import State -from app.utils import as_async_contextmanager, disallow +from app.utils import disallow router = APIRouter() errorcode = ErrorCode() @@ -27,7 +27,7 @@ @router.on_event("startup") @disallow([ExecutionMode.REGISTRY, ExecutionMode.CLIENT]) -async def init_db(): +async def init_db() -> None: Engine = await get_engine_cls() if not database_exists(Engine.url): @@ -38,7 +38,7 @@ async def init_db(): Base.metadata.create_all(Engine) - async with as_async_contextmanager(deps.get_db_session) as db: + async with deps.get_db_session() as db: state = State(id=1, current_height=settings.BLOCK_START, peak_height=None) db_state = [jsonable_encoder(state)] @@ -70,11 +70,9 @@ async def _scan_token_activity( logger.info(f"Scanning blocks {start_height} - {end_height} for activity") - climate_units: Dict[ - str, Any - ] = climate_warehouse.combine_climate_units_and_metadata(search={}) + climate_units = climate_warehouse.combine_climate_units_and_metadata(search={}) for unit in climate_units: - token: Optional[Dict] = unit.get("token") + token = unit.get("token") # is None or empty if not token: @@ -112,8 +110,8 @@ async def scan_token_activity() -> None: async with ( lock, - as_async_contextmanager(deps.get_db_session) as db, - as_async_contextmanager(deps.get_full_node_rpc_client) as full_node_client, + deps.get_db_session() as db, + deps.get_full_node_rpc_client() as full_node_client, ): db_crud = crud.DBCrud(db=db) climate_warehouse = crud.ClimateWareHouseCrud( @@ -145,9 +143,9 @@ async def scan_token_activity() -> None: async def _scan_blockchain_state( db_crud: crud.DBCrud, full_node_client: FullNodeRpcClient, -): - state: Dict = await full_node_client.get_blockchain_state() - peak: Dict = state.get("peak") +) -> None: + state = await full_node_client.get_blockchain_state() + peak = state.get("peak") if peak is None: logger.warning("Full node is not synced") @@ -162,8 +160,8 @@ async def _scan_blockchain_state( @disallow([ExecutionMode.REGISTRY, ExecutionMode.CLIENT]) async def scan_blockchain_state() -> None: async with ( - as_async_contextmanager(deps.get_db_session) as db, - as_async_contextmanager(deps.get_full_node_rpc_client) as full_node_client, + deps.get_db_session() as db, + deps.get_full_node_rpc_client() as full_node_client, ): db_crud = crud.DBCrud(db=db) diff --git a/app/api/v1/keys.py b/app/api/v1/keys.py index ad28984..2da2667 100644 --- a/app/api/v1/keys.py +++ b/app/api/v1/keys.py @@ -1,4 +1,4 @@ -from typing import Dict, Optional +from typing import Optional from blspy import G1Element, PrivateKey from chia.consensus.coinbase import create_puzzlehash_for_pk @@ -31,10 +31,10 @@ async def get_key( derivation_index: int = 0, prefix: str = "bls1238", wallet_rpc_client: WalletRpcClient = Depends(deps.get_wallet_rpc_client), -): +) -> schemas.Key: fingerprint: int = await wallet_rpc_client.get_logged_in_fingerprint() - result: Dict = await wallet_rpc_client.get_private_key(fingerprint) + result = await wallet_rpc_client.get_private_key(fingerprint) secret_key = PrivateKey.from_bytes(hexstr_to_bytes(result["sk"])) @@ -62,7 +62,7 @@ async def get_key( ) async def parse_key( address: str, -): +) -> Optional[schemas.Key]: try: puzzle_hash: bytes = decode_puzzle_hash(address) except ValueError: diff --git a/app/api/v1/tokens.py b/app/api/v1/tokens.py index f37734b..3a18519 100644 --- a/app/api/v1/tokens.py +++ b/app/api/v1/tokens.py @@ -29,7 +29,7 @@ async def create_tokenization_tx( request: schemas.TokenizationTxRequest, wallet_rpc_client: WalletRpcClient = Depends(deps.get_wallet_rpc_client), -): +) -> schemas.TokenizationTxResponse: """Create and send tokenization tx. This endpoint is to be called by the registry. @@ -53,7 +53,7 @@ async def create_tokenization_tx( root_secret_key=climate_secret_key, wallet_client=wallet_rpc_client, ) - result: Dict = await wallet.send_tokenization_transaction( + result = await wallet.send_tokenization_transaction( to_puzzle_hash=payment.to_puzzle_hash, amount=payment.amount, fee=payment.fee, @@ -116,7 +116,7 @@ async def create_detokenization_tx( asset_id: str, request: schemas.DetokenizationTxRequest, wallet_rpc_client: WalletRpcClient = Depends(deps.get_wallet_rpc_client), -): +) -> schemas.DetokenizationTxResponse: """Sign and send detokenization tx. This endpoint is to be called by the registry. @@ -140,7 +140,7 @@ async def create_detokenization_tx( root_secret_key=climate_secret_key, wallet_client=wallet_rpc_client, ) - result: Dict = await wallet.sign_and_send_detokenization_request(content=content) + result = await wallet.sign_and_send_detokenization_request(content=content) (transaction_record, *_) = result["transaction_records"] return schemas.DetokenizationTxResponse( @@ -160,7 +160,7 @@ async def create_detokenization_file( asset_id: str, request: schemas.DetokenizationFileRequest, wallet_rpc_client: WalletRpcClient = Depends(deps.get_wallet_rpc_client), -): +) -> schemas.DetokenizationFileResponse: """Create detokenization file. This endpoint is to be called by the client. @@ -207,7 +207,7 @@ async def create_detokenization_file( if cat_wallet_info is None: raise ValueError(f"Asset id {asset_id} not found in wallet!") - result: Dict = await wallet.create_detokenization_request( + result = await wallet.create_detokenization_request( amount=payment.amount, fee=payment.fee, wallet_id=cat_wallet_info.id, @@ -231,13 +231,13 @@ async def create_detokenization_file( @disallow([ExecutionMode.EXPLORER, ExecutionMode.CLIENT]) async def parse_detokenization_file( content: str, -): +) -> schemas.DetokenizationFileParseResponse: """Parse detokenization file. This endpoint is to be called by the registry. """ - result: Dict = await ClimateWallet.parse_detokenization_request(content=content) + result = await ClimateWallet.parse_detokenization_request(content=content) mode: GatewayMode = result["mode"] gateway_coin_spend: CoinSpend = result["gateway_coin_spend"] spend_bundle: SpendBundle = result["spend_bundle"] @@ -269,7 +269,7 @@ async def create_permissionless_retirement_tx( asset_id: str, request: schemas.PermissionlessRetirementTxRequest, wallet_rpc_client: WalletRpcClient = Depends(deps.get_wallet_rpc_client), -): +) -> schemas.PermissionlessRetirementTxResponse: """Create and send permissionless retirement tx. This endpoint is to be called by the client. @@ -325,7 +325,7 @@ async def create_permissionless_retirement_tx( # Log the JSON-formatted string logger.warning(log_data_json) - result: Dict = await wallet.send_permissionless_retirement_transaction( + result = await wallet.send_permissionless_retirement_transaction( amount=payment.amount, fee=payment.fee, beneficiary_name=payment.beneficiary_name.encode(), diff --git a/app/api/v1/transactions.py b/app/api/v1/transactions.py index ecae8c8..07c9a9d 100644 --- a/app/api/v1/transactions.py +++ b/app/api/v1/transactions.py @@ -1,4 +1,4 @@ -from typing import Dict, List, Optional +from typing import List, Optional from chia.rpc.wallet_rpc_client import WalletRpcClient from chia.types.blockchain_format.coin import Coin @@ -33,7 +33,7 @@ async def get_transaction( transaction_id: str, wallet_rpc_client: WalletRpcClient = Depends(deps.get_wallet_rpc_client), -): +) -> schemas.Transaction: """Get transaction by id. This endpoint is to be called by the registry or the client. @@ -62,7 +62,7 @@ async def get_transactions( reverse: bool = False, to_address: Optional[str] = None, wallet_rpc_client: WalletRpcClient = Depends(deps.get_wallet_rpc_client), -): +) -> schemas.Transactions: """Get transactions. This endpoint is to be called by the client. @@ -119,7 +119,7 @@ async def get_transactions( ) gateway_cat_puzzle_hash: bytes32 = gateway_cat_puzzle.get_tree_hash() - transactions: List[Dict] = [] + transactions = [] for transaction_record in transaction_records: if transaction_record.to_puzzle_hash != gateway_puzzle_hash: continue @@ -143,7 +143,7 @@ async def get_transactions( tail_spend: CoinSpend (mode, tail_spend) = parse_gateway_spend(coin_spend=coin_spend, is_cat=True) - transaction: Dict = transaction_record.to_json_dict() + transaction = transaction_record.to_json_dict() transaction["type"] = CLIMATE_WALLET_INDEX + mode.to_int() transaction["type_name"] = mode.name diff --git a/app/core/chialisp/gateway.py b/app/core/chialisp/gateway.py index 70da44e..bbac3b9 100644 --- a/app/core/chialisp/gateway.py +++ b/app/core/chialisp/gateway.py @@ -26,7 +26,8 @@ def create_gateway_puzzle() -> Program: def create_gateway_solution( conditions_program: Program, ) -> Program: - return Program.to([conditions_program]) + ret: Program = Program.to([conditions_program]) + return ret def create_gateway_announcement( diff --git a/app/core/types.py b/app/core/types.py index 981ff4f..9df43c2 100644 --- a/app/core/types.py +++ b/app/core/types.py @@ -33,14 +33,15 @@ class ClimateTokenIndex(object): sequence_num: int = 0 def name(self) -> bytes32: - return Program.to( + to_hash: Program = Program.to( [ self.org_uid, self.warehouse_project_id, self.vintage_year, self.sequence_num, ] - ).get_tree_hash() + ) + return to_hash.get_tree_hash() @dataclasses.dataclass(frozen=True) @@ -71,7 +72,8 @@ def to_program(self) -> Program: if self.fee: conditions.append([ConditionOpcode.RESERVE_FEE, self.fee]) - return Program.to(conditions) + ret: Program = Program.to(conditions) + return ret @property def additions(self) -> List[Dict[str, Any]]: diff --git a/app/crud/chia.py b/app/crud/chia.py index af2e2d9..6c7728a 100644 --- a/app/crud/chia.py +++ b/app/crud/chia.py @@ -33,7 +33,7 @@ def _headers(self) -> Dict[str, str]: return headers - def get_climate_units(self, search: Dict[str, Any]) -> List[Dict]: + def get_climate_units(self, search: Dict[str, Any]) -> Any: try: params = urlencode(search) url = urlparse(self.url + "/v1/units") @@ -51,7 +51,7 @@ def get_climate_units(self, search: Dict[str, Any]) -> List[Dict]: logger.error("Call Climate API Timeout, ErrorMessage: " + str(e)) raise error_code.internal_server_error("Call Climate API Timeout") - def get_climate_projects(self) -> List[Dict]: + def get_climate_projects(self) -> Any: try: url = urlparse(self.url + "/v1/projects") @@ -68,7 +68,7 @@ def get_climate_projects(self) -> List[Dict]: logger.error("Call Climate API Timeout, ErrorMessage: " + str(e)) raise error_code.internal_server_error("Call Climate API Timeout") - def get_climate_organizations(self) -> Dict[str, Dict]: + def get_climate_organizations(self) -> Any: try: url = urlparse(self.url + "/v1/organizations") @@ -85,7 +85,7 @@ def get_climate_organizations(self) -> Dict[str, Dict]: logger.error("Call Climate API Timeout, ErrorMessage: " + str(e)) raise error_code.internal_server_error("Call Climate API Timeout") - def get_climate_organizations_metadata(self, org_uid: str) -> Dict[str, Any]: + def get_climate_organizations_metadata(self, org_uid: str) -> Any: try: condition = {"orgUid": org_uid} @@ -162,7 +162,10 @@ def combine_climate_units_and_metadata( ) continue - org_metadata: Dict[str, str] = metadata_by_id.get(unit_org_uid) + org_metadata = metadata_by_id.get(unit_org_uid) + if org_metadata is None: + continue + metadata = dict() # some versions perpended "meta_" to the key, so check both ways if marketplace_id in org_metadata: diff --git a/app/crud/db.py b/app/crud/db.py index 9985011..99324f9 100644 --- a/app/crud/db.py +++ b/app/crud/db.py @@ -1,5 +1,5 @@ import dataclasses -from typing import Any, AnyStr, List, Optional +from typing import Any, AnyStr, List, Optional, Tuple from fastapi.encoders import jsonable_encoder from sqlalchemy import and_, desc, insert, or_, update @@ -17,7 +17,7 @@ class DBCrudBase(object): db: Session - def create_object(self, model: Base): + def create_object(self, model: Base) -> Base: self.db.add(model) self.db.commit() self.db.refresh(model) @@ -57,7 +57,7 @@ def update_db(self, table: Any, stmt: Any) -> bool: logger.error(f"Update DB Failure:{e}") raise errorcode.internal_server_error(message="Update DB Failure") - def select_first_db(self, model: Any, order_by: Any): + def select_first_db(self, model: Any, order_by: Any) -> Any: try: return self.db.query(model).order_by(desc(order_by)).first() except Exception as e: @@ -66,7 +66,7 @@ def select_first_db(self, model: Any, order_by: Any): def select_activity_with_pagination( self, model: Any, filters: Any, order_by: Any, limit: int, page: int - ): + ) -> Tuple[Any, int]: try: query = self.db.query(model).filter( or_(*filters["or"]), and_(*filters["and"]) @@ -88,7 +88,10 @@ def select_activity_with_pagination( @dataclasses.dataclass class DBCrud(DBCrudBase): def create_activity(self, activity: schemas.Activity) -> models.Activity: - return self.create_object(models.Activity(**jsonable_encoder(activity))) + new_activity: models.Activity = self.create_object( + models.Activity(**jsonable_encoder(activity)) + ) + return new_activity def batch_insert_ignore_activity( self, @@ -108,7 +111,7 @@ def update_block_state( self, peak_height: Optional[int] = None, current_height: Optional[int] = None, - ): + ) -> bool: state = models.State() if peak_height is not None: state.peak_height = peak_height @@ -122,13 +125,15 @@ def update_block_state( ) def select_block_state_first(self) -> models.State: - return self.select_first_db( + selected_state: models.State = self.select_first_db( model=models.State, order_by=models.State.id, ) + return selected_state def select_activity_first(self) -> models.Activity: - return self.select_first_db( + selected_activity: models.Activity = self.select_first_db( model=models.Activity, order_by=models.Activity.created_at, ) + return selected_activity diff --git a/app/db/session.py b/app/db/session.py index 3d88691..ba1b420 100644 --- a/app/db/session.py +++ b/app/db/session.py @@ -1,31 +1,26 @@ -from typing import Type - from sqlalchemy import create_engine, engine from sqlalchemy.orm import Session, sessionmaker from app import crud from app.api import dependencies as deps from app.config import settings -from app.utils import as_async_contextmanager -async def get_engine_cls() -> Type[engine.Engine]: - async with as_async_contextmanager( - deps.get_full_node_rpc_client - ) as full_node_client: +async def get_engine_cls() -> engine.Engine: + async with deps.get_full_node_rpc_client() as full_node_client: blockchain_crud = crud.BlockChainCrud(full_node_client) challenge: str = await blockchain_crud.get_challenge() db_url: str = "sqlite:///" + str(settings.DB_PATH).replace("CHALLENGE", challenge) - Engine = create_engine( + Engine: engine.Engine = create_engine( db_url, connect_args={"check_same_thread": False, "timeout": 15} ) return Engine -async def get_session_local_cls() -> Type[Session]: +async def get_session_local_cls() -> Session: Engine = await get_engine_cls() - SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=Engine) + SessionLocal: Session = sessionmaker(autocommit=False, autoflush=False, bind=Engine) return SessionLocal diff --git a/app/errors.py b/app/errors.py index 755f499..99b6280 100644 --- a/app/errors.py +++ b/app/errors.py @@ -2,8 +2,8 @@ class ErrorCode: - def internal_server_error(self, message: str) -> None: - raise HTTPException(status_code=500, detail=message) + def internal_server_error(self, message: str) -> HTTPException: + return HTTPException(status_code=500, detail=message) - def bad_request_error(self, message: str) -> None: - raise HTTPException(status_code=400, detail=message) + def bad_request_error(self, message: str) -> HTTPException: + return HTTPException(status_code=400, detail=message) diff --git a/app/main.py b/app/main.py index 7a0d841..6e9bfda 100644 --- a/app/main.py +++ b/app/main.py @@ -37,7 +37,7 @@ async def exception_handler(request: Request, e: Exception) -> Response: if __name__ == "__main__": logger.info(f"Using settings {settings.dict()}") - wait_until_dir_exists(settings.CHIA_ROOT) + wait_until_dir_exists(str(settings.CHIA_ROOT)) server_host = "" diff --git a/app/models/activity.py b/app/models/activity.py index 8435385..9f0d39b 100644 --- a/app/models/activity.py +++ b/app/models/activity.py @@ -12,7 +12,7 @@ from app.db.base import Base -class Activity(Base): +class Activity(Base): # type: ignore[misc] __tablename__ = "activity" id = Column(Integer, primary_key=True, index=True) diff --git a/app/models/state.py b/app/models/state.py index 3fb594d..dfedbea 100644 --- a/app/models/state.py +++ b/app/models/state.py @@ -5,7 +5,7 @@ from app.db.base import Base -class State(Base): +class State(Base): # type: ignore[misc] __tablename__ = "state" id = Column(Integer, primary_key=True, index=True) diff --git a/app/schemas/activity.py b/app/schemas/activity.py index 7aff0cd..c0baad4 100644 --- a/app/schemas/activity.py +++ b/app/schemas/activity.py @@ -1,5 +1,5 @@ import enum -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List, Optional, Union from pydantic import Field, validator @@ -26,12 +26,14 @@ class ActivityBase(BaseModel): timestamp: int @validator("mode") - def mode_from_str(cls, v): + def mode_from_str(cls, v: Union[str, GatewayMode]) -> GatewayMode: + if isinstance(v, GatewayMode): + return v for mode in GatewayMode: if (v == mode.name) or (v == mode.value): return mode - return v + raise ValueError(f"Invalid mode {v}") class Activity(ActivityBase): diff --git a/app/utils.py b/app/utils.py index 0ebb130..eee0774 100644 --- a/app/utils.py +++ b/app/utils.py @@ -1,25 +1,14 @@ import functools -import inspect import os import time -from contextlib import asynccontextmanager, contextmanager from typing import Callable, List from fastapi import status -from fastapi.concurrency import contextmanager_in_threadpool from app.config import ExecutionMode, settings from app.logger import logger -def as_async_contextmanager(func: Callable, *args, **kwargs): - if inspect.isasyncgenfunction(func): - return asynccontextmanager(func)(*args, **kwargs) - - elif inspect.isgeneratorfunction(func): - return contextmanager_in_threadpool(contextmanager(func)(*args, **kwargs)) - - def disallow(modes: List[ExecutionMode]): def _disallow(f: Callable): if settings.MODE in modes: diff --git a/pytest.ini b/pytest.ini index 4ab425c..b66fbeb 100644 --- a/pytest.ini +++ b/pytest.ini @@ -18,4 +18,4 @@ testpaths = tests filterwarnings = ignore::sqlalchemy.exc.MovedIn20Warning ignore:ssl_context is deprecated:DeprecationWarning - ignore:pkg_resources is deprecated as an API:DeprecationWarning \ No newline at end of file + ignore:pkg_resources is deprecated as an API:DeprecationWarning From 57135762fb3b89ecc7dd4c581091ea67e0e8fcc6 Mon Sep 17 00:00:00 2001 From: Earle Lowe Date: Fri, 3 Nov 2023 14:08:17 -0700 Subject: [PATCH 02/11] isort, black, flake8, mypy updates --- .isort.cfg | 5 ++ app/api/dependencies.py | 2 + app/api/v1/__init__.py | 2 + app/api/v1/activities.py | 6 +- app/api/v1/core.py | 2 + app/api/v1/cron.py | 6 +- app/api/v1/keys.py | 11 ++-- app/api/v1/tokens.py | 30 +++------- app/api/v1/transactions.py | 19 ++----- app/config.py | 25 ++++---- app/core/chialisp/gateway.py | 2 + app/core/chialisp/load_clvm.py | 2 + app/core/chialisp/tail.py | 2 + app/core/climate_wallet/wallet.py | 76 +++++++------------------ app/core/climate_wallet/wallet_utils.py | 33 +++-------- app/core/derive_keys.py | 2 + app/core/types.py | 14 ++--- app/core/utils.py | 18 ++---- app/crud/__init__.py | 2 + app/crud/chia.py | 38 ++++--------- app/crud/db.py | 23 ++------ app/db/base.py | 2 + app/db/session.py | 6 +- app/errors.py | 2 + app/logger.py | 2 + app/main.py | 6 +- app/models/__init__.py | 2 + app/models/activity.py | 13 +---- app/models/state.py | 2 + app/schemas/__init__.py | 16 +++--- app/schemas/activity.py | 2 + app/schemas/core.py | 2 + app/schemas/key.py | 2 + app/schemas/metadata.py | 2 + app/schemas/payment.py | 6 +- app/schemas/state.py | 2 + app/schemas/token.py | 13 ++--- app/schemas/transaction.py | 2 + app/schemas/types.py | 2 + app/utils.py | 28 +++++---- mypy.ini | 6 ++ pyproject.toml | 11 ++++ tests/conftest.py | 2 + tests/test_activities_api.py | 42 ++++---------- tests/test_cat_lifecycle.py | 29 +++------- tests/test_cat_workflow.py | 44 ++++---------- tests/test_crud_chia.py | 40 ++++++------- tests/test_crud_core.py | 2 + tests/test_disallow.py | 23 ++++++++ 49 files changed, 273 insertions(+), 358 deletions(-) create mode 100644 .isort.cfg create mode 100644 tests/test_disallow.py diff --git a/.isort.cfg b/.isort.cfg new file mode 100644 index 0000000..7557202 --- /dev/null +++ b/.isort.cfg @@ -0,0 +1,5 @@ +[settings] +line_length = 120 +profile=black +skip_gitignore=true +add_imports=from __future__ import annotations diff --git a/app/api/dependencies.py b/app/api/dependencies.py index 3b993fc..6751d06 100644 --- a/app/api/dependencies.py +++ b/app/api/dependencies.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import enum from contextlib import asynccontextmanager from pathlib import Path diff --git a/app/api/v1/__init__.py b/app/api/v1/__init__.py index f513463..8a001cc 100644 --- a/app/api/v1/__init__.py +++ b/app/api/v1/__init__.py @@ -1 +1,3 @@ +from __future__ import annotations + from app.api.v1.core import router diff --git a/app/api/v1/activities.py b/app/api/v1/activities.py index f0497e2..847e9e4 100644 --- a/app/api/v1/activities.py +++ b/app/api/v1/activities.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from typing import Any, Dict, List, Optional from fastapi import APIRouter, Depends @@ -92,9 +94,7 @@ async def get_activity( limit=limit, ) if len(activities) == 0: - logger.warning( - f"No data to get from activities. filters:{activity_filters} page:{page} limit:{limit}" - ) + logger.warning(f"No data to get from activities. filters:{activity_filters} page:{page} limit:{limit}") return schemas.ActivitiesResponse() activities_with_cw: List[schemas.ActivityWithCW] = [] diff --git a/app/api/v1/core.py b/app/api/v1/core.py index 83bf574..0933bd9 100644 --- a/app/api/v1/core.py +++ b/app/api/v1/core.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from typing import Dict from fastapi import APIRouter diff --git a/app/api/v1/cron.py b/app/api/v1/cron.py index 26bb5bb..dd073f0 100644 --- a/app/api/v1/cron.py +++ b/app/api/v1/cron.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import asyncio from typing import List @@ -114,9 +116,7 @@ async def scan_token_activity() -> None: deps.get_full_node_rpc_client() as full_node_client, ): db_crud = crud.DBCrud(db=db) - climate_warehouse = crud.ClimateWareHouseCrud( - url=settings.CADT_API_SERVER_HOST, api_key=settings.CADT_API_KEY - ) + climate_warehouse = crud.ClimateWareHouseCrud(url=settings.CADT_API_SERVER_HOST, api_key=settings.CADT_API_KEY) blockchain = crud.BlockChainCrud(full_node_client=full_node_client) try: diff --git a/app/api/v1/keys.py b/app/api/v1/keys.py index 2da2667..04b0e0b 100644 --- a/app/api/v1/keys.py +++ b/app/api/v1/keys.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from typing import Optional from blspy import G1Element, PrivateKey @@ -7,10 +9,7 @@ from chia.util.bech32m import decode_puzzle_hash, encode_puzzle_hash from chia.util.byte_types import hexstr_to_bytes from chia.util.ints import uint32 -from chia.wallet.derive_keys import ( - master_sk_to_wallet_sk, - master_sk_to_wallet_sk_unhardened, -) +from chia.wallet.derive_keys import master_sk_to_wallet_sk, master_sk_to_wallet_sk_unhardened from fastapi import APIRouter, Depends from app import schemas @@ -42,9 +41,7 @@ async def get_key( if hardened: wallet_secret_key = master_sk_to_wallet_sk(secret_key, uint32(derivation_index)) else: - wallet_secret_key = master_sk_to_wallet_sk_unhardened( - secret_key, uint32(derivation_index) - ) + wallet_secret_key = master_sk_to_wallet_sk_unhardened(secret_key, uint32(derivation_index)) wallet_public_key: G1Element = wallet_secret_key.get_g1() puzzle_hash: bytes32 = create_puzzlehash_for_pk(wallet_public_key) diff --git a/app/api/v1/tokens.py b/app/api/v1/tokens.py index 3a18519..1b0265d 100644 --- a/app/api/v1/tokens.py +++ b/app/api/v1/tokens.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import json from typing import Any, Dict, Tuple @@ -35,9 +37,7 @@ async def create_tokenization_tx( This endpoint is to be called by the registry. """ - climate_secret_key = await utils.get_climate_secret_key( - wallet_client=wallet_rpc_client - ) + climate_secret_key = await utils.get_climate_secret_key(wallet_client=wallet_rpc_client) token = request.token payment = request.payment @@ -90,9 +90,7 @@ async def create_tokenization_tx( signature=bytes(signature), ) elif mode == GatewayMode.PERMISSIONLESS_RETIREMENT: - token_obj[ - "permissionless_retirement" - ] = schemas.PermissionlessRetirementTailMetadata( + token_obj["permissionless_retirement"] = schemas.PermissionlessRetirementTailMetadata( mod_hash=mod_hash, signature=bytes(signature), ) @@ -101,9 +99,7 @@ async def create_tokenization_tx( return schemas.TokenizationTxResponse( token=token_on_chain, token_hexstr=token_on_chain.hexstr(), - tx=schemas.Transaction( - id=transaction_record.name, record=transaction_record.to_json_dict() - ), + tx=schemas.Transaction(id=transaction_record.name, record=transaction_record.to_json_dict()), ) @@ -122,9 +118,7 @@ async def create_detokenization_tx( This endpoint is to be called by the registry. """ - climate_secret_key = await utils.get_climate_secret_key( - wallet_client=wallet_rpc_client - ) + climate_secret_key = await utils.get_climate_secret_key(wallet_client=wallet_rpc_client) token: schemas.Token = request.token content: str = request.content @@ -145,9 +139,7 @@ async def create_detokenization_tx( return schemas.DetokenizationTxResponse( token=token, - tx=schemas.Transaction( - id=transaction_record.name, record=transaction_record.to_json_dict() - ), + tx=schemas.Transaction(id=transaction_record.name, record=transaction_record.to_json_dict()), ) @@ -218,9 +210,7 @@ async def create_detokenization_file( return schemas.DetokenizationFileResponse( token=token, content=content, - tx=schemas.Transaction( - id=transaction_record.name, record=transaction_record.to_json_dict() - ), + tx=schemas.Transaction(id=transaction_record.name, record=transaction_record.to_json_dict()), ) @@ -338,7 +328,5 @@ async def create_permissionless_retirement_tx( return schemas.PermissionlessRetirementTxResponse( token=token, - tx=schemas.Transaction( - id=transaction_record.name, record=transaction_record.to_json_dict() - ), + tx=schemas.Transaction(id=transaction_record.name, record=transaction_record.to_json_dict()), ) diff --git a/app/api/v1/transactions.py b/app/api/v1/transactions.py index 07c9a9d..a5aee32 100644 --- a/app/api/v1/transactions.py +++ b/app/api/v1/transactions.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from typing import List, Optional from chia.rpc.wallet_rpc_client import WalletRpcClient @@ -68,9 +70,7 @@ async def get_transactions( This endpoint is to be called by the client. """ - transaction_records: List[ - TransactionRecord - ] = await wallet_rpc_client.get_transactions( + transaction_records: List[TransactionRecord] = await wallet_rpc_client.get_transactions( wallet_id=wallet_id, start=start, end=end, @@ -82,9 +82,7 @@ async def get_transactions( wallet_objs: List[ChiaJsonObject] = await wallet_rpc_client.get_wallets( wallet_type=WalletType.CAT, ) - wallet_infos: List[WalletInfo] = [ - WalletInfo.from_json_dict(wallet_obj) for wallet_obj in wallet_objs - ] + wallet_infos: List[WalletInfo] = [WalletInfo.from_json_dict(wallet_obj) for wallet_obj in wallet_objs] wallet_info: Optional[WalletInfo] cat_info: Optional[CATInfo] = None @@ -103,10 +101,7 @@ async def get_transactions( sort_key=sort_key, reverse=reverse, to_address=to_address, - transactions=[ - transaction_record.to_json_dict() - for transaction_record in transaction_records - ], + transactions=[transaction_record.to_json_dict() for transaction_record in transaction_records], ) if cat_info is None: raise ValueError(f"Wallet {wallet_id} is not a CAT wallet") @@ -135,9 +130,7 @@ async def get_transactions( break else: - raise ValueError( - f"No coin with puzzle hash {gateway_cat_puzzle_hash.hex()}" - ) + raise ValueError(f"No coin with puzzle hash {gateway_cat_puzzle_hash.hex()}") mode: GatewayMode tail_spend: CoinSpend diff --git a/app/config.py b/app/config.py index dc1394e..acb02c5 100644 --- a/app/config.py +++ b/app/config.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import enum import shutil import sys @@ -56,8 +58,10 @@ class Settings(BaseSettings): CLIMATE_TOKEN_REGISTRY_PORT: Optional[int] = None DEV_PORT: Optional[int] = None + _instance: Optional[Settings] = None + @classmethod - def get_instance(cls) -> "Settings": + def get_instance(cls) -> Settings: if cls._instance is None: cls._instance = get_settings() return cls._instance @@ -65,17 +69,11 @@ def get_instance(cls) -> "Settings": @root_validator def configure_port(cls, values: Dict[str, Any]) -> Dict[str, Any]: if values["MODE"] == ExecutionMode.REGISTRY: - values["SERVER_PORT"] = values.get( - "CLIMATE_TOKEN_REGISTRY_PORT", ServerPort.CLIMATE_TOKEN_REGISTRY.value - ) + values["SERVER_PORT"] = values.get("CLIMATE_TOKEN_REGISTRY_PORT", ServerPort.CLIMATE_TOKEN_REGISTRY.value) elif values["MODE"] == ExecutionMode.CLIENT: - values["SERVER_PORT"] = values.get( - "CLIMATE_TOKEN_CLIENT_PORT", ServerPort.CLIMATE_TOKEN_CLIENT.value - ) + values["SERVER_PORT"] = values.get("CLIMATE_TOKEN_CLIENT_PORT", ServerPort.CLIMATE_TOKEN_CLIENT.value) elif values["MODE"] == ExecutionMode.EXPLORER: - values["SERVER_PORT"] = values.get( - "CLIMATE_EXPLORER_PORT", ServerPort.CLIMATE_EXPLORER.value - ) + values["SERVER_PORT"] = values.get("CLIMATE_EXPLORER_PORT", ServerPort.CLIMATE_EXPLORER.value) elif values["MODE"] == ExecutionMode.DEV: values["SERVER_PORT"] = values.get("DEV_PORT", ServerPort.DEV.value) else: @@ -102,13 +100,14 @@ def get_settings() -> Settings: default_env_file: Path default_config_file: Path if in_pyinstaller: - default_env_file = Path(sys._MEIPASS) / ".env" - default_config_file = Path(sys._MEIPASS) / "config.yaml" + base_path = getattr(sys, "_MEIPASS") + default_env_file = Path(base_path).joinpath(".env") + default_config_file = Path(base_path).joinpath("config.yaml") else: default_env_file = Path(".env") default_config_file = Path("config.yaml") - default_settings = Settings(_env_file=default_env_file) + default_settings = Settings(_env_file=default_env_file) # type: ignore[call-arg] config_file: Path = default_settings.CONFIG_PATH if not config_file.is_file(): diff --git a/app/core/chialisp/gateway.py b/app/core/chialisp/gateway.py index bbac3b9..65d4e6c 100644 --- a/app/core/chialisp/gateway.py +++ b/app/core/chialisp/gateway.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from typing import Optional, Tuple from chia.types.announcement import Announcement diff --git a/app/core/chialisp/load_clvm.py b/app/core/chialisp/load_clvm.py index 753f330..27d4cfd 100644 --- a/app/core/chialisp/load_clvm.py +++ b/app/core/chialisp/load_clvm.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import functools from chia.wallet.puzzles.load_clvm import load_clvm diff --git a/app/core/chialisp/tail.py b/app/core/chialisp/tail.py index 24bd643..d3a3e05 100644 --- a/app/core/chialisp/tail.py +++ b/app/core/chialisp/tail.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from typing import Optional from blspy import G1Element diff --git a/app/core/climate_wallet/wallet.py b/app/core/climate_wallet/wallet.py index 69d994e..6534e2d 100644 --- a/app/core/climate_wallet/wallet.py +++ b/app/core/climate_wallet/wallet.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import dataclasses import time from typing import Any, Dict, Iterator, List, Optional, Tuple, Union @@ -30,23 +32,10 @@ from app.core.chialisp.gateway import create_gateway_puzzle, parse_gateway_spend from app.core.chialisp.tail import create_delegated_puzzle, create_tail_program -from app.core.climate_wallet.wallet_utils import ( - create_gateway_request_and_spend, - create_gateway_signature, -) +from app.core.climate_wallet.wallet_utils import create_gateway_request_and_spend, create_gateway_signature from app.core.derive_keys import root_sk_to_gateway_sk -from app.core.types import ( - CLIMATE_WALLET_INDEX, - ClimateTokenIndex, - GatewayMode, - TransactionRequest, -) -from app.core.utils import ( - get_constants, - get_created_signed_transactions, - get_first_puzzle_hash, - get_wallet_info_by_id, -) +from app.core.types import CLIMATE_WALLET_INDEX, ClimateTokenIndex, GatewayMode, TransactionRequest +from app.core.utils import get_constants, get_created_signed_transactions, get_first_puzzle_hash, get_wallet_info_by_id from app.logger import logger @@ -73,12 +62,8 @@ def tail_program_hash(self) -> bytes32: @dataclasses.dataclass class ClimateWallet(ClimateWalletBase): - mode_to_public_key: Optional[Dict[GatewayMode, G1Element]] = dataclasses.field( - default=None, kw_only=True - ) - mode_to_secret_key: Optional[Dict[GatewayMode, PrivateKey]] = dataclasses.field( - default=None, kw_only=True - ) + mode_to_public_key: Optional[Dict[GatewayMode, G1Element]] = dataclasses.field(default=None, kw_only=True) + mode_to_secret_key: Optional[Dict[GatewayMode, PrivateKey]] = dataclasses.field(default=None, kw_only=True) mode_to_message_and_signature: Dict[GatewayMode, Tuple[bytes, G2Element]] wallet_client: WalletRpcClient @@ -93,9 +78,7 @@ async def create( ) -> "ClimateWallet": root_public_key: G1Element = root_secret_key.get_g1() token_index_hash: bytes32 = token_index.name() - tail_program = create_tail_program( - public_key=root_public_key, index=Program.to(token_index_hash) - ) + tail_program = create_tail_program(public_key=root_public_key, index=Program.to(token_index_hash)) tail_program_hash: bytes32 = tail_program.get_tree_hash() logger.info("Creating climate wallet for") @@ -118,9 +101,7 @@ async def create( public_key=public_key, ) delegated_puzzle_hash: bytes32 = delegated_puzzle.get_tree_hash() - message: bytes32 = Program.to( - [token_index_hash, delegated_puzzle] - ).get_tree_hash() + message: bytes32 = Program.to([token_index_hash, delegated_puzzle]).get_tree_hash() signature: G2Element = AugSchemeMPL.sign(root_secret_key, message) mode_to_public_key[mode] = secret_key.get_g1() @@ -327,9 +308,7 @@ async def _create_client_transaction( wallet_client=self.wallet_client, ) if len(transaction_records) != 1: - raise ValueError( - f"Transaction record has unexpected length {len(transaction_records)}!" - ) + raise ValueError(f"Transaction record has unexpected length {len(transaction_records)}!") transaction_record = transaction_records[0] if transaction_record.spend_bundle is None: @@ -343,9 +322,7 @@ async def _create_client_transaction( key_value_pairs: Optional[List[Tuple[str, Union[str, int]]]] = None if gateway_key_values: - key_value_pairs = [ - (key, value) for (key, value) in gateway_key_values.items() - ] + key_value_pairs = [(key, value) for (key, value) in gateway_key_values.items()] return await self._create_transaction( mode=mode, @@ -368,9 +345,7 @@ async def send_tokenization_transaction( wallet_id: int = 1, ) -> Dict[str, Any]: self.check_user(is_registry=True) - await self.check_wallet( - wallet_id=wallet_id, wallet_type=WalletType.STANDARD_WALLET - ) + await self.check_wallet(wallet_id=wallet_id, wallet_type=WalletType.STANDARD_WALLET) mode = GatewayMode.TOKENIZATION if self.mode_to_secret_key is None: @@ -437,8 +412,7 @@ async def create_detokenization_request( content: str = bech32_encode("detok", convertbits(bytes(spend_bundle), 8, 5)) transaction_records = [ - dataclasses.replace(transaction_record, spend_bundle=None) - for transaction_record in transaction_records + dataclasses.replace(transaction_record, spend_bundle=None) for transaction_record in transaction_records ] await self.wallet_client.push_transactions(txs=transaction_records) @@ -478,9 +452,7 @@ async def parse_detokenization_request( solution: Program = coin_spend.solution.to_program() coin: Coin = coin_spend.coin - puzzle_args: Optional[Iterator[Program]] = match_cat_puzzle( - uncurry_puzzle(puzzle) - ) + puzzle_args: Optional[Iterator[Program]] = match_cat_puzzle(uncurry_puzzle(puzzle)) # gateway spend is a CAT if puzzle_args is None: @@ -517,9 +489,7 @@ async def parse_detokenization_request( puzzle = coin_spend.puzzle_reveal.to_program() puzzle_args = match_cat_puzzle(uncurry_puzzle(puzzle)) if puzzle_args is None: - raise ValueError( - "Did not match CAT - invalid detokenization request" - ) + raise ValueError("Did not match CAT - invalid detokenization request") (_, _, inner_puzzle) = puzzle_args inner_puzzle_hash = inner_puzzle.get_tree_hash() @@ -587,9 +557,7 @@ async def sign_and_send_detokenization_request( ] ) if gateway_coin_spend is None: - raise ValueError( - "Invalid detokenization request: Could not find gateway coin spend!" - ) + raise ValueError("Invalid detokenization request: Could not find gateway coin spend!") transaction_record = TransactionRecord( confirmed_at_height=uint32(0), @@ -634,9 +602,7 @@ async def send_permissionless_retirement_transaction( if beneficiary_puzzle_hash is None: # no beneficiary supplied at all if beneficiary_address is None: - beneficiary_puzzle_hash = await get_first_puzzle_hash( - self.wallet_client - ) + beneficiary_puzzle_hash = await get_first_puzzle_hash(self.wallet_client) result = await self._create_client_transaction( mode=mode, @@ -680,9 +646,7 @@ async def get_activities( ) gateway_cat_puzzle_hash: bytes32 = gateway_cat_puzzle.get_tree_hash() - coin_records: List[ - CoinRecord - ] = await self.full_node_client.get_coin_records_by_puzzle_hash( + coin_records: List[CoinRecord] = await self.full_node_client.get_coin_records_by_puzzle_hash( puzzle_hash=gateway_cat_puzzle_hash, start_height=start_height, end_height=end_height, @@ -712,9 +676,7 @@ async def get_activities( metadata: Dict[bytes, bytes] = {} for key_value_pair in key_value_pairs.as_iter(): if (not key_value_pair.listp()) or (key_value_pair.at("r").listp()): - logger.warning( - f"Coin {coin.name()} has incorrect metadata structure" - ) + logger.warning(f"Coin {coin.name()} has incorrect metadata structure") continue key = key_value_pair.at("f").as_atom() diff --git a/app/core/climate_wallet/wallet_utils.py b/app/core/climate_wallet/wallet_utils.py index cacb693..dade485 100644 --- a/app/core/climate_wallet/wallet_utils.py +++ b/app/core/climate_wallet/wallet_utils.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from typing import Any, Dict, List, Optional, Tuple from blspy import AugSchemeMPL, G1Element, G2Element, PrivateKey @@ -6,10 +8,7 @@ from chia.types.blockchain_format.sized_bytes import bytes32 from chia.types.coin_spend import CoinSpend from chia.types.condition_opcodes import ConditionOpcode -from chia.util.condition_tools import ( - conditions_dict_for_solution, - pkm_pairs_for_conditions_dict, -) +from chia.util.condition_tools import conditions_dict_for_solution, pkm_pairs_for_conditions_dict from chia.util.ints import uint64 from chia.wallet.cat_wallet.cat_utils import ( CAT_MOD, @@ -20,11 +19,7 @@ from chia.wallet.lineage_proof import LineageProof from chia.wallet.payment import Payment -from app.core.chialisp.gateway import ( - create_gateway_announcement, - create_gateway_puzzle, - create_gateway_solution, -) +from app.core.chialisp.gateway import create_gateway_announcement, create_gateway_puzzle, create_gateway_solution from app.core.chialisp.load_clvm import load_clvm_locally from app.core.chialisp.tail import create_delegated_puzzle from app.core.types import GatewayMode, TransactionRequest @@ -103,9 +98,7 @@ def create_gateway_request_and_spend( extra_delta: int = 0 conditions = [] - conditions.append( - [ConditionOpcode.CREATE_COIN, None, -113, tail_program, tail_solution] - ) + conditions.append([ConditionOpcode.CREATE_COIN, None, -113, tail_program, tail_solution]) if to_puzzle_hash is None: if mode in [GatewayMode.TOKENIZATION]: @@ -114,9 +107,7 @@ def create_gateway_request_and_spend( extra_delta = -amount else: - conditions.append( - [ConditionOpcode.CREATE_COIN, to_puzzle_hash, amount, [to_puzzle_hash]] - ) + conditions.append([ConditionOpcode.CREATE_COIN, to_puzzle_hash, amount, [to_puzzle_hash]]) conditions_program = Program.to(conditions) gateway_announcement = create_gateway_announcement( @@ -141,9 +132,7 @@ def create_gateway_request_and_spend( inner_puzzle=gateway_puzzle, inner_solution=gateway_solution, ) - unsigned_spend_bundle = unsigned_spend_bundle_for_spendable_cats( - CAT_MOD, [spendable_cat] - ) + unsigned_spend_bundle = unsigned_spend_bundle_for_spendable_cats(CAT_MOD, [spendable_cat]) unsigned_coin_spend: CoinSpend = unsigned_spend_bundle.coin_spends[0] return (transaction_request, unsigned_coin_spend) @@ -153,9 +142,7 @@ def create_gateway_signature( coin_spend: CoinSpend, agg_sig_additional_data: bytes, public_key_to_secret_key: Optional[Dict[bytes, PrivateKey]] = None, - public_key_message_to_signature: Optional[ - Dict[Tuple[bytes, bytes], G2Element] - ] = None, + public_key_message_to_signature: Optional[Dict[Tuple[bytes, bytes], G2Element]] = None, allow_missing: bool = False, ) -> G2Element: if public_key_to_secret_key is None: @@ -189,9 +176,7 @@ def create_gateway_signature( if allow_missing: continue else: - raise ValueError( - f"Cannot sign for key {public_key.hex()} and message {message.hex()}" - ) + raise ValueError(f"Cannot sign for key {public_key.hex()} and message {message.hex()}") signatures.append(signature) diff --git a/app/core/derive_keys.py b/app/core/derive_keys.py index 110c44b..d1f81ca 100644 --- a/app/core/derive_keys.py +++ b/app/core/derive_keys.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from blspy import PrivateKey from chia.wallet.derive_keys import _derive_path_unhardened diff --git a/app/core/types.py b/app/core/types.py index 9df43c2..31076d9 100644 --- a/app/core/types.py +++ b/app/core/types.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import dataclasses import enum from typing import Any, Dict, List, Optional @@ -55,19 +57,13 @@ class TransactionRequest(object): def to_program(self) -> Program: conditions = [] for payment in self.payments: - conditions.append( - [ConditionOpcode.CREATE_COIN] + payment.as_condition_args() - ) + conditions.append([ConditionOpcode.CREATE_COIN] + payment.as_condition_args()) for announcement in self.coin_announcements: - conditions.append( - [ConditionOpcode.ASSERT_COIN_ANNOUNCEMENT, announcement.name()] - ) + conditions.append([ConditionOpcode.ASSERT_COIN_ANNOUNCEMENT, announcement.name()]) for announcement in self.puzzle_announcements: - conditions.append( - [ConditionOpcode.ASSERT_PUZZLE_ANNOUNCEMENT, announcement.name()] - ) + conditions.append([ConditionOpcode.ASSERT_PUZZLE_ANNOUNCEMENT, announcement.name()]) if self.fee: conditions.append([ConditionOpcode.RESERVE_FEE, self.fee]) diff --git a/app/core/utils.py b/app/core/utils.py index 604550f..da787a4 100644 --- a/app/core/utils.py +++ b/app/core/utils.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from typing import Any, Dict, List, Optional from blspy import G1Element, PrivateKey @@ -53,9 +55,7 @@ async def get_cat_wallet_info_by_asset_id( wallet_client: WalletRpcClient, ) -> Optional[WalletInfo]: wallet_objs: List[Dict[str, Any]] = await wallet_client.get_wallets() - wallet_infos: List[WalletInfo] = [ - WalletInfo.from_json_dict(wallet_obj) for wallet_obj in wallet_objs - ] + wallet_infos: List[WalletInfo] = [WalletInfo.from_json_dict(wallet_obj) for wallet_obj in wallet_objs] wallet_info: WalletInfo for wallet_info in wallet_infos: @@ -76,9 +76,7 @@ async def get_wallet_info_by_id( wallet_client: WalletRpcClient, ) -> Optional[WalletInfo]: wallet_objs: List[Dict[str, Any]] = await wallet_client.get_wallets() - wallet_infos: List[WalletInfo] = [ - WalletInfo.from_json_dict(wallet_obj) for wallet_obj in wallet_objs - ] + wallet_infos: List[WalletInfo] = [WalletInfo.from_json_dict(wallet_obj) for wallet_obj in wallet_objs] wallet_info: WalletInfo for wallet_info in wallet_infos: @@ -96,14 +94,10 @@ async def get_first_puzzle_hash( fingerprint: int = await wallet_client.get_logged_in_fingerprint() result = await wallet_client.get_private_key(fingerprint=fingerprint) master_secret_key: PrivateKey = PrivateKey.from_bytes(bytes.fromhex(result["sk"])) - wallet_secret_key: PrivateKey = master_sk_to_wallet_sk_unhardened( - master_secret_key, uint32(0) - ) + wallet_secret_key: PrivateKey = master_sk_to_wallet_sk_unhardened(master_secret_key, uint32(0)) wallet_public_key: G1Element = wallet_secret_key.get_g1() - first_puzzle_hash: bytes32 = puzzle_for_pk( - public_key=wallet_public_key - ).get_tree_hash() + first_puzzle_hash: bytes32 = puzzle_for_pk(public_key=wallet_public_key).get_tree_hash() logger.info(f"First puzzle hash = {first_puzzle_hash.hex()}") diff --git a/app/crud/__init__.py b/app/crud/__init__.py index 5d9a157..ed0ca48 100644 --- a/app/crud/__init__.py +++ b/app/crud/__init__.py @@ -1,2 +1,4 @@ +from __future__ import annotations + from app.crud.chia import BlockChainCrud, ClimateWareHouseCrud # noqa from app.crud.db import DBCrud, DBCrudBase # noqa diff --git a/app/crud/chia.py b/app/crud/chia.py index 6c7728a..68e910a 100644 --- a/app/crud/chia.py +++ b/app/crud/chia.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import dataclasses import json from typing import Any, Dict, List, Optional @@ -41,9 +43,7 @@ def get_climate_units(self, search: Dict[str, Any]) -> Any: r = requests.get(url.geturl(), params=params, headers=self._headers()) if r.status_code != requests.codes.ok: logger.error(f"Request Url: {r.url} Error Message: {r.text}") - raise error_code.internal_server_error( - message="Call Climate API Failure" - ) + raise error_code.internal_server_error(message="Call Climate API Failure") return r.json() @@ -58,9 +58,7 @@ def get_climate_projects(self) -> Any: r = requests.get(url.geturl(), headers=self._headers()) if r.status_code != requests.codes.ok: logger.error(f"Request Url: {r.url} Error Message: {r.text}") - raise error_code.internal_server_error( - message="Call Climate API Failure" - ) + raise error_code.internal_server_error(message="Call Climate API Failure") return r.json() @@ -75,9 +73,7 @@ def get_climate_organizations(self) -> Any: r = requests.get(url.geturl(), headers=self._headers()) if r.status_code != requests.codes.ok: logger.error(f"Request Url: {r.url} Error Message: {r.text}") - raise error_code.internal_server_error( - message="Call Climate API Failure" - ) + raise error_code.internal_server_error(message="Call Climate API Failure") return r.json() @@ -95,9 +91,7 @@ def get_climate_organizations_metadata(self, org_uid: str) -> Any: r = requests.get(url.geturl(), params=params, headers=self._headers()) if r.status_code != requests.codes.ok: logger.error(f"Request Url: {r.url} Error Message: {r.text}") - raise error_code.internal_server_error( - message="Call Climate API Failure" - ) + raise error_code.internal_server_error(message="Call Climate API Failure") return r.json() @@ -105,15 +99,11 @@ def get_climate_organizations_metadata(self, org_uid: str) -> Any: logger.error("Call Climate API Timeout, ErrorMessage: " + str(e)) raise error_code.internal_server_error("Call Climate API Timeout") - def combine_climate_units_and_metadata( - self, search: Dict[str, Any] - ) -> List[Dict[str, Any]]: + def combine_climate_units_and_metadata(self, search: Dict[str, Any]) -> List[Dict[str, Any]]: # units: [unit] units = self.get_climate_units(search) if len(units) == 0: - logger.warning( - f"Search climate warehouse units by search is empty. search:{search}" - ) + logger.warning(f"Search climate warehouse units by search is empty. search:{search}") return [] projects = self.get_climate_projects() @@ -141,25 +131,19 @@ def combine_climate_units_and_metadata( unit_org_uid = unit.get("orgUid") if unit_org_uid is None: - logger.warning( - f"Can not get climate warehouse orgUid in unit. unit:{unit}" - ) + logger.warning(f"Can not get climate warehouse orgUid in unit. unit:{unit}") continue org = organization_by_id.get(unit_org_uid) if org is None: - logger.warning( - f"Can not get organization by org_uid. org_uid:{unit_org_uid}" - ) + logger.warning(f"Can not get organization by org_uid. org_uid:{unit_org_uid}") continue try: warehouse_project_id = unit["issuance"]["warehouseProjectId"] project = project_by_id[warehouse_project_id] except KeyError: - logger.warning( - f"Can not get project by warehouse_project_id: {warehouse_project_id}" - ) + logger.warning(f"Can not get project by warehouse_project_id: {warehouse_project_id}") continue org_metadata = metadata_by_id.get(unit_org_uid) diff --git a/app/crud/db.py b/app/crud/db.py index 99324f9..46c3f00 100644 --- a/app/crud/db.py +++ b/app/crud/db.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import dataclasses from typing import Any, AnyStr, List, Optional, Tuple @@ -25,11 +27,7 @@ def create_object(self, model: Base) -> Base: def batch_insert_ignore_db(self, table: AnyStr, models: List[Any]) -> bool: try: - s = ( - insert(Base.metadata.tables[table]) - .prefix_with("OR IGNORE") - .values(models) - ) + s = insert(Base.metadata.tables[table]).prefix_with("OR IGNORE").values(models) self.db.execute(s) self.db.commit() return True @@ -68,16 +66,9 @@ def select_activity_with_pagination( self, model: Any, filters: Any, order_by: Any, limit: int, page: int ) -> Tuple[Any, int]: try: - query = self.db.query(model).filter( - or_(*filters["or"]), and_(*filters["and"]) - ) + query = self.db.query(model).filter(or_(*filters["or"]), and_(*filters["and"])) return ( - ( - query.order_by(*order_by) - .limit(limit) - .offset((page - 1) * limit) - .all() - ), + (query.order_by(*order_by).limit(limit).offset((page - 1) * limit).all()), query.count(), ) except Exception as e: @@ -88,9 +79,7 @@ def select_activity_with_pagination( @dataclasses.dataclass class DBCrud(DBCrudBase): def create_activity(self, activity: schemas.Activity) -> models.Activity: - new_activity: models.Activity = self.create_object( - models.Activity(**jsonable_encoder(activity)) - ) + new_activity: models.Activity = self.create_object(models.Activity(**jsonable_encoder(activity))) return new_activity def batch_insert_ignore_activity( diff --git a/app/db/base.py b/app/db/base.py index 860e542..982cb54 100644 --- a/app/db/base.py +++ b/app/db/base.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from sqlalchemy.ext.declarative import declarative_base Base = declarative_base() diff --git a/app/db/session.py b/app/db/session.py index ba1b420..23de893 100644 --- a/app/db/session.py +++ b/app/db/session.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from sqlalchemy import create_engine, engine from sqlalchemy.orm import Session, sessionmaker @@ -12,9 +14,7 @@ async def get_engine_cls() -> engine.Engine: challenge: str = await blockchain_crud.get_challenge() db_url: str = "sqlite:///" + str(settings.DB_PATH).replace("CHALLENGE", challenge) - Engine: engine.Engine = create_engine( - db_url, connect_args={"check_same_thread": False, "timeout": 15} - ) + Engine: engine.Engine = create_engine(db_url, connect_args={"check_same_thread": False, "timeout": 15}) return Engine diff --git a/app/errors.py b/app/errors.py index 99b6280..dc92d8c 100644 --- a/app/errors.py +++ b/app/errors.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from fastapi import HTTPException diff --git a/app/logger.py b/app/logger.py index 9890462..5caddf3 100644 --- a/app/logger.py +++ b/app/logger.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import logging import uvicorn diff --git a/app/main.py b/app/main.py index 6e9bfda..e61b17b 100644 --- a/app/main.py +++ b/app/main.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import sys import traceback @@ -65,7 +67,5 @@ async def exception_handler(request: Request, e: Exception) -> Response: log_config=log_config, ) else: - print( - f"Climate Token Driver can only run on localhost in {settings.MODE.name} mode." - ) + print(f"Climate Token Driver can only run on localhost in {settings.MODE.name} mode.") sys.exit(1) diff --git a/app/models/__init__.py b/app/models/__init__.py index 3b46fa3..fdf4d8a 100644 --- a/app/models/__init__.py +++ b/app/models/__init__.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from app.models.activity import Activity # noqa from app.models.state import State # noqa diff --git a/app/models/activity.py b/app/models/activity.py index 9f0d39b..918d69f 100644 --- a/app/models/activity.py +++ b/app/models/activity.py @@ -1,13 +1,6 @@ -from sqlalchemy import ( - JSON, - BigInteger, - Column, - DateTime, - Integer, - String, - UniqueConstraint, - func, -) +from __future__ import annotations + +from sqlalchemy import JSON, BigInteger, Column, DateTime, Integer, String, UniqueConstraint, func from app.db.base import Base diff --git a/app/models/state.py b/app/models/state.py index dfedbea..aadc788 100644 --- a/app/models/state.py +++ b/app/models/state.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from datetime import datetime from sqlalchemy import BigInteger, Column, DateTime, Integer, func diff --git a/app/schemas/__init__.py b/app/schemas/__init__.py index f898efe..8d74d16 100644 --- a/app/schemas/__init__.py +++ b/app/schemas/__init__.py @@ -1,25 +1,27 @@ -from app.schemas.activity import ( # noqa +from __future__ import annotations + +from app.schemas.activity import ( # noqa: F401 ActivitiesResponse, Activity, ActivityBase, ActivitySearchBy, ActivityWithCW, ) -from app.schemas.key import Key # noqa -from app.schemas.metadata import ( # noqa +from app.schemas.key import Key # noqa: F401 +from app.schemas.metadata import ( # noqa: F401 DetokenizationTailMetadata, PermissionlessRetirementTailMetadata, TailMetadataBase, TokenizationTailMetadata, ) -from app.schemas.payment import ( # noqa +from app.schemas.payment import ( # noqa: F401 PaymentBase, PaymentWithPayee, PaymentWithPayer, RetirementPaymentWithPayer, ) -from app.schemas.state import State # noqa -from app.schemas.token import ( # noqa +from app.schemas.state import State # noqa: F401 +from app.schemas.token import ( # noqa: F401 DetokenizationFileParseResponse, DetokenizationFileRequest, DetokenizationFileResponse, @@ -34,4 +36,4 @@ TokenOnChainBase, TokenOnChainSimple, ) -from app.schemas.transaction import Transaction, Transactions # noqa +from app.schemas.transaction import Transaction, Transactions # noqa: F401 diff --git a/app/schemas/activity.py b/app/schemas/activity.py index c0baad4..40d10b1 100644 --- a/app/schemas/activity.py +++ b/app/schemas/activity.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import enum from typing import Any, Dict, List, Optional, Union diff --git a/app/schemas/core.py b/app/schemas/core.py index fabe9b7..30d79f8 100644 --- a/app/schemas/core.py +++ b/app/schemas/core.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from typing import Any, Dict, get_type_hints from chia.util.byte_types import hexstr_to_bytes diff --git a/app/schemas/key.py b/app/schemas/key.py index 46a3a82..d081582 100644 --- a/app/schemas/key.py +++ b/app/schemas/key.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from app.schemas.core import BaseModel diff --git a/app/schemas/metadata.py b/app/schemas/metadata.py index 9e935c1..025de8d 100644 --- a/app/schemas/metadata.py +++ b/app/schemas/metadata.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from app.schemas.core import BaseModel diff --git a/app/schemas/payment.py b/app/schemas/payment.py index 0ff2e95..818e556 100644 --- a/app/schemas/payment.py +++ b/app/schemas/payment.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from typing import Optional from chia.types.blockchain_format.sized_bytes import bytes32 @@ -14,13 +16,15 @@ class PaymentBase(BaseModel): class PaymentWithPayee(PaymentBase): - to_address: str = Field( + to_address: Optional[str] = Field( default=None, example="txch1clzn09v7lapulm7j8mwx9jaqh35uh7jzjeukpv7pj50tv80zze4s5060sx", ) @property def to_puzzle_hash(self) -> bytes32: + if self.to_address is None: + raise ValueError("to_address is not set") return decode_puzzle_hash(self.to_address) diff --git a/app/schemas/state.py b/app/schemas/state.py index a64a170..54b5f62 100644 --- a/app/schemas/state.py +++ b/app/schemas/state.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from typing import Optional from pydantic import BaseModel diff --git a/app/schemas/token.py b/app/schemas/token.py index 4be33ac..fcc28f7 100644 --- a/app/schemas/token.py +++ b/app/schemas/token.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from chia.util.byte_types import hexstr_to_bytes from pydantic import Field @@ -8,20 +10,13 @@ PermissionlessRetirementTailMetadata, TokenizationTailMetadata, ) -from app.schemas.payment import ( - PaymentBase, - PaymentWithPayee, - PaymentWithPayer, - RetirementPaymentWithPayer, -) +from app.schemas.payment import PaymentBase, PaymentWithPayee, PaymentWithPayer, RetirementPaymentWithPayer from app.schemas.transaction import Transaction from app.schemas.types import ChiaJsonObject class Token(BaseModel): - org_uid: str = Field( - example="3e70df56ff67a6806df6ad101c159363845550d1f9afd81e3e0d5a5ab51af867" - ) + org_uid: str = Field(example="3e70df56ff67a6806df6ad101c159363845550d1f9afd81e3e0d5a5ab51af867") warehouse_project_id: str = Field(example="GS1") vintage_year: int = Field(example=2099) sequence_num: int = Field(example=0) diff --git a/app/schemas/transaction.py b/app/schemas/transaction.py index 45c3e17..3c8341b 100644 --- a/app/schemas/transaction.py +++ b/app/schemas/transaction.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from typing import List, Optional from app.schemas.core import BaseModel diff --git a/app/schemas/types.py b/app/schemas/types.py index 0b74e57..4fab45b 100644 --- a/app/schemas/types.py +++ b/app/schemas/types.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from typing import Any, Dict ChiaJsonObject = Dict[str, Any] diff --git a/app/utils.py b/app/utils.py index eee0774..c69f452 100644 --- a/app/utils.py +++ b/app/utils.py @@ -1,30 +1,34 @@ -import functools +from __future__ import annotations + import os import time -from typing import Callable, List +from typing import Any, Callable, Concatenate, Coroutine, List, ParamSpec, TypeVar from fastapi import status from app.config import ExecutionMode, settings from app.logger import logger +P = ParamSpec("P") +R = TypeVar("R") + -def disallow(modes: List[ExecutionMode]): - def _disallow(f: Callable): +def disallow( + modes: List[ExecutionMode], +) -> Callable[[Callable[Concatenate[P], Coroutine[Any, Any, R]]], Callable[Concatenate[P], Coroutine[Any, Any, R]],]: + def decorator( + f: Callable[Concatenate[P], Coroutine[Any, Any, R]] + ) -> Callable[Concatenate[P], Coroutine[Any, Any, R]]: if settings.MODE in modes: - async def _f(*args, **kargs): + async def not_allowed(*args: P.args, **kwargs: P.kwargs) -> Any: return status.HTTP_405_METHOD_NOT_ALLOWED - else: - - @functools.wraps(f) - async def _f(*args, **kargs): - return await f(*args, **kargs) + return not_allowed - return _f + return f - return _disallow + return decorator def wait_until_dir_exists(path: str, interval: int = 1) -> None: diff --git a/mypy.ini b/mypy.ini index 1d8cc76..017dd6e 100644 --- a/mypy.ini +++ b/mypy.ini @@ -17,3 +17,9 @@ no_implicit_reexport = False strict_equality = True warn_redundant_casts = True enable_incomplete_feature = Unpack +plugins = pydantic.mypy + +[pydantic-mypy] +init_forbid_extra = True +init_typed = True +warn_required_dynamic_aliases = True diff --git a/pyproject.toml b/pyproject.toml index a54487d..e16ba58 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,3 +46,14 @@ changelog_start_rev = "1.0.3" [build-system] requires = ["poetry-core>=1.0.0"] build-backend = "poetry.core.masonry.api" + +[tool.black] +line-length = 120 +target-version = ['py38', 'py39', 'py310', 'py311'] +include = ''' +^/( + [^/]*.py + | (app|tests)/.*\.pyi? +)$ +''' +exclude = '' diff --git a/tests/conftest.py b/tests/conftest.py index 2578e28..23240a1 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import pytest import pytest_asyncio from chia.clvm.spend_sim import SimClient, SpendSim diff --git a/tests/test_activities_api.py b/tests/test_activities_api.py index 0a05171..f71fec6 100644 --- a/tests/test_activities_api.py +++ b/tests/test_activities_api.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from unittest import mock from urllib.parse import urlencode @@ -19,9 +21,7 @@ def test_activities_with_search_by_then_error( self, fastapi_client: TestClient, monkeypatch: pytest.MonkeyPatch ) -> None: with anyio.from_thread.start_blocking_portal() as portal, monkeypatch.context() as m: - fastapi_client.portal = ( - portal # workaround anyio 4.0.0 incompat with TextClient - ) + fastapi_client.portal = portal # workaround anyio 4.0.0 incompat with TextClient m.setattr(crud.BlockChainCrud, "get_challenge", mock_get_challenge) test_request = {"search_by": "error", "search": ""} @@ -39,9 +39,7 @@ def test_activities_with_empty_climate_warehouse_then_success( mock_climate_warehouse_data.return_value = [] with anyio.from_thread.start_blocking_portal() as portal, monkeypatch.context() as m: - fastapi_client.portal = ( - portal # workaround anyio 4.0.0 incompat with TextClient - ) + fastapi_client.portal = portal # workaround anyio 4.0.0 incompat with TextClient m.setattr( crud.ClimateWareHouseCrud, "combine_climate_units_and_metadata", @@ -64,9 +62,7 @@ def test_activities_with_empty_db_then_success( mock_db_data.return_value = ([], 0) with anyio.from_thread.start_blocking_portal() as portal, monkeypatch.context() as m: - fastapi_client.portal = ( - portal # workaround anyio 4.0.0 incompat with TextClient - ) + fastapi_client.portal = portal # workaround anyio 4.0.0 incompat with TextClient m.setattr(crud.BlockChainCrud, "get_challenge", mock_get_challenge) m.setattr(crud.DBCrud, "select_activity_with_pagination", mock_db_data) @@ -76,9 +72,7 @@ def test_activities_with_empty_db_then_success( assert response.status_code == fastapi.status.HTTP_200_OK assert response.json() == test_response - def test_activities_then_success( - self, fastapi_client: TestClient, monkeypatch: pytest.MonkeyPatch - ) -> None: + def test_activities_then_success(self, fastapi_client: TestClient, monkeypatch: pytest.MonkeyPatch) -> None: test_activity_data = models.activity.Activity( org_uid="cf7af8da584b6c115ba8247c5cdd05506c3b3c5c632ed975cc2b16262493e2bd", amount=10000, @@ -103,9 +97,7 @@ def test_activities_then_success( activities=[ schemas.activity.ActivityWithCW( **jsonable_encoder(test_activity_data), - cw_org={ - "orgUid": "cf7af8da584b6c115ba8247c5cdd05506c3b3c5c632ed975cc2b16262493e2bd" - }, + cw_org={"orgUid": "cf7af8da584b6c115ba8247c5cdd05506c3b3c5c632ed975cc2b16262493e2bd"}, cw_project={ "orgUid": "cf7af8da584b6c115ba8247c5cdd05506c3b3c5c632ed975cc2b16262493e2bd", "warehouseProjectId": "c9b98579-debb-49f3-b417-0adbae4ed5c7", @@ -234,9 +226,7 @@ def test_activities_then_success( "createdAt": "2022-10-24T06:25:13.440Z", "updatedAt": "2022-10-24T06:25:13.440Z", }, - "organization": { - "orgUid": "cf7af8da584b6c115ba8247c5cdd05506c3b3c5c632ed975cc2b16262493e2bd" - }, + "organization": {"orgUid": "cf7af8da584b6c115ba8247c5cdd05506c3b3c5c632ed975cc2b16262493e2bd"}, "token": { "org_uid": "cf7af8da584b6c115ba8247c5cdd05506c3b3c5c632ed975cc2b16262493e2bd", "warehouse_project_id": "c9b98579-debb-49f3-b417-0adbae4ed5c7", @@ -266,9 +256,7 @@ def test_activities_then_success( } ] with anyio.from_thread.start_blocking_portal() as portal, monkeypatch.context() as m: - fastapi_client.portal = ( - portal # workaround anyio 4.0.0 incompat with TextClient - ) + fastapi_client.portal = portal # workaround anyio 4.0.0 incompat with TextClient m.setattr(crud.BlockChainCrud, "get_challenge", mock_get_challenge) m.setattr(crud.DBCrud, "select_activity_with_pagination", mock_db_data) m.setattr( @@ -345,9 +333,7 @@ def test_activities_with_mode_search_search_by_then_success( "bp": "0xe122763ec4076d3fa356fbff8bb63d1f9d78b52c3c577a01140cd4559ee32966", }, **jsonable_encoder(test_activity_data), - cw_org={ - "orgUid": "cf7af8da584b6c115ba8247c5cdd05506c3b3c5c632ed975cc2b16262493e2bd" - }, + cw_org={"orgUid": "cf7af8da584b6c115ba8247c5cdd05506c3b3c5c632ed975cc2b16262493e2bd"}, cw_project={ "orgUid": "cf7af8da584b6c115ba8247c5cdd05506c3b3c5c632ed975cc2b16262493e2bd", "warehouseProjectId": "c9b98579-debb-49f3-b417-0adbae4ed5c7", @@ -448,9 +434,7 @@ def test_activities_with_mode_search_search_by_then_success( "createdAt": "2022-10-24T06:25:13.440Z", "updatedAt": "2022-10-24T06:25:13.440Z", }, - "organization": { - "orgUid": "cf7af8da584b6c115ba8247c5cdd05506c3b3c5c632ed975cc2b16262493e2bd" - }, + "organization": {"orgUid": "cf7af8da584b6c115ba8247c5cdd05506c3b3c5c632ed975cc2b16262493e2bd"}, "token": { "org_uid": "cf7af8da584b6c115ba8247c5cdd05506c3b3c5c632ed975cc2b16262493e2bd", "warehouse_project_id": "c9b98579-debb-49f3-b417-0adbae4ed5c7", @@ -481,9 +465,7 @@ def test_activities_with_mode_search_search_by_then_success( ] with anyio.from_thread.start_blocking_portal() as portal, monkeypatch.context() as m: - fastapi_client.portal = ( - portal # workaround anyio 4.0.0 incompat with TextClient - ) + fastapi_client.portal = portal # workaround anyio 4.0.0 incompat with TextClient m.setattr(crud.BlockChainCrud, "get_challenge", mock_get_challenge) m.setattr(crud.DBCrud, "select_activity_with_pagination", mock_db_data) m.setattr( diff --git a/tests/test_cat_lifecycle.py b/tests/test_cat_lifecycle.py index 6d8cf66..dbd8428 100644 --- a/tests/test_cat_lifecycle.py +++ b/tests/test_cat_lifecycle.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import logging import secrets from typing import Dict @@ -12,20 +14,13 @@ from chia.types.mempool_inclusion_status import MempoolInclusionStatus from chia.types.spend_bundle import SpendBundle from chia.util.ints import uint64 -from chia.wallet.cat_wallet.cat_utils import ( - CAT_MOD, - SpendableCAT, - unsigned_spend_bundle_for_spendable_cats, -) +from chia.wallet.cat_wallet.cat_utils import CAT_MOD, SpendableCAT, unsigned_spend_bundle_for_spendable_cats from chia.wallet.lineage_proof import LineageProof from chia.wallet.payment import Payment from app.core.chialisp.gateway import create_gateway_puzzle from app.core.chialisp.tail import create_tail_program -from app.core.climate_wallet.wallet_utils import ( - create_gateway_request_and_spend, - create_gateway_signature, -) +from app.core.climate_wallet.wallet_utils import create_gateway_request_and_spend, create_gateway_signature from app.core.types import GatewayMode, TransactionRequest logger = logging.getLogger(__name__) @@ -89,9 +84,7 @@ async def test_cat_lifecycle( await node.farm_block(xch_puzzle_hash) - xch_coin: Coin = ( - await client.get_coin_records_by_puzzle_hash(xch_puzzle_hash) - )[0].coin + xch_coin: Coin = (await client.get_coin_records_by_puzzle_hash(xch_puzzle_hash))[0].coin # mint @@ -176,9 +169,7 @@ async def test_cat_lifecycle( inner_puzzle=ACS_MOD, inner_solution=transaction_request.to_program(), ) - spend_bundle = unsigned_spend_bundle_for_spendable_cats( - CAT_MOD, [spendable_cat] - ) + spend_bundle = unsigned_spend_bundle_for_spendable_cats(CAT_MOD, [spendable_cat]) ( cat_coin_for_detokenization, @@ -232,9 +223,7 @@ async def test_cat_lifecycle( inner_puzzle=ACS_MOD, inner_solution=transaction_request.to_program(), ) - cat_spend_bundle = unsigned_spend_bundle_for_spendable_cats( - CAT_MOD, [spendable_cat] - ) + cat_spend_bundle = unsigned_spend_bundle_for_spendable_cats(CAT_MOD, [spendable_cat]) spend_bundle = SpendBundle.aggregate( [ @@ -290,9 +279,7 @@ async def test_cat_lifecycle( inner_puzzle=ACS_MOD, inner_solution=transaction_request.to_program(), ) - cat_spend_bundle = unsigned_spend_bundle_for_spendable_cats( - CAT_MOD, [spendable_cat] - ) + cat_spend_bundle = unsigned_spend_bundle_for_spendable_cats(CAT_MOD, [spendable_cat]) spend_bundle = SpendBundle.aggregate( [ diff --git a/tests/test_cat_workflow.py b/tests/test_cat_workflow.py index 93e12f6..3dd1b61 100644 --- a/tests/test_cat_workflow.py +++ b/tests/test_cat_workflow.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import logging from typing import List @@ -15,11 +17,7 @@ from app.core.derive_keys import master_sk_to_root_sk from app.core.types import ClimateTokenIndex, GatewayMode from tests.wallet.rpc.test_wallet_rpc import wallet_rpc_environment # noqa: F401 -from tests.wallet.rpc.test_wallet_rpc import ( - WalletRpcTestEnvironment, - farm_transaction, - generate_funds, -) +from tests.wallet.rpc.test_wallet_rpc import WalletRpcTestEnvironment, farm_transaction, generate_funds logger = logging.getLogger(__name__) @@ -30,13 +28,9 @@ async def check_transactions( transaction_records: List[TransactionRecord], ) -> None: for transaction_record in transaction_records: - tx = await wallet_client.get_transaction( - wallet_id=wallet_id, transaction_id=transaction_record.name - ) + tx = await wallet_client.get_transaction(wallet_id=wallet_id, transaction_id=transaction_record.name) - assert ( - tx.confirmed_at_height != 0 - ), f"Transaction {transaction_record.name.hex()} not found!" + assert tx.confirmed_at_height != 0, f"Transaction {transaction_record.name.hex()} not found!" async def check_balance( @@ -45,9 +39,7 @@ async def check_balance( amount: int, ) -> None: result = await wallet_client.get_wallet_balance(wallet_id=wallet_id) - assert ( - result["confirmed_wallet_balance"] == amount - ), "Target wallet CAT amount does not match!" + assert result["confirmed_wallet_balance"] == amount, "Target wallet CAT amount does not match!" async def get_confirmed_balance(client: WalletRpcClient, wallet_id: int) -> int: @@ -86,9 +78,7 @@ async def test_cat_tokenization_workflow( fingerprint: int = await wallet_client_1.get_logged_in_fingerprint() result = await wallet_client_1.get_private_key(fingerprint=fingerprint) - master_secret_key: PrivateKey = PrivateKey.from_bytes( - bytes.fromhex(result["sk"]) - ) + master_secret_key: PrivateKey = PrivateKey.from_bytes(bytes.fromhex(result["sk"])) root_secret_key: PrivateKey = master_sk_to_root_sk(master_secret_key) token_index = ClimateTokenIndex( @@ -131,9 +121,7 @@ async def test_cat_tokenization_workflow( assert result["success"] cat_wallet_id: int = result["wallet_id"] - await time_out_assert( - 60, get_confirmed_balance, amount, wallet_client_2, cat_wallet_id - ) + await time_out_assert(60, get_confirmed_balance, amount, wallet_client_2, cat_wallet_id) @pytest.mark.asyncio async def test_cat_detokenization_workflow( @@ -155,9 +143,7 @@ async def test_cat_detokenization_workflow( fingerprint: int = await wallet_client_1.get_logged_in_fingerprint() result = await wallet_client_1.get_private_key(fingerprint=fingerprint) - master_secret_key: PrivateKey = PrivateKey.from_bytes( - bytes.fromhex(result["sk"]) - ) + master_secret_key: PrivateKey = PrivateKey.from_bytes(bytes.fromhex(result["sk"])) root_secret_key: PrivateKey = master_sk_to_root_sk(master_secret_key) # block: initial fund deposits @@ -229,9 +215,7 @@ async def test_cat_detokenization_workflow( await farm_transaction(full_node_api, wallet_node_1, spend_bundle) await full_node_api.wait_for_wallet_synced(env.wallet_2.node, timeout=60) await check_transactions(wallet_client_2, cat_wallet_id, transaction_records) - await time_out_assert( - 60, get_confirmed_balance, 0, wallet_client_2, cat_wallet_id - ) + await time_out_assert(60, get_confirmed_balance, 0, wallet_client_2, cat_wallet_id) @pytest.mark.asyncio async def test_cat_permissionless_retirement_workflow( @@ -312,9 +296,7 @@ async def test_cat_permissionless_retirement_workflow( await full_node_api.process_all_wallet_transactions( wallet=env.wallet_2.node.wallet_state_manager.main_wallet, timeout=120 ) - await time_out_assert( - 60, get_confirmed_balance, 0, wallet_client_2, cat_wallet_id - ) + await time_out_assert(60, get_confirmed_balance, 0, wallet_client_2, cat_wallet_id) # block: # - observer: observe retirement activity @@ -324,9 +306,7 @@ async def test_cat_permissionless_retirement_workflow( root_public_key=climate_wallet_1.root_public_key, full_node_client=full_node_client, ) - activities = await climate_observer.get_activities( - mode=GatewayMode.PERMISSIONLESS_RETIREMENT - ) + activities = await climate_observer.get_activities(mode=GatewayMode.PERMISSIONLESS_RETIREMENT) assert activities[0]["metadata"]["bn"] == beneficiary_name.decode() assert activities[0]["metadata"]["ba"] == test_address.decode() diff --git a/tests/test_crud_chia.py b/tests/test_crud_chia.py index 25150ff..7a2c7a7 100644 --- a/tests/test_crud_chia.py +++ b/tests/test_crud_chia.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from typing import Any, Dict, List from unittest import mock @@ -7,17 +9,15 @@ class TestClimateWareHouseCrud: - def test_combine_climate_units_and_metadata_empty_units_then_success( - self, monkeypatch: pytest.MonkeyPatch - ) -> None: + def test_combine_climate_units_and_metadata_empty_units_then_success(self, monkeypatch: pytest.MonkeyPatch) -> None: mock_units = mock.MagicMock() mock_units.return_value = [] monkeypatch.setattr(crud.ClimateWareHouseCrud, "get_climate_units", mock_units) - response = crud.ClimateWareHouseCrud( - url=mock.MagicMock(), api_key=None - ).combine_climate_units_and_metadata(search={}) + response = crud.ClimateWareHouseCrud(url=mock.MagicMock(), api_key=None).combine_climate_units_and_metadata( + search={} + ) assert len(response) == 0 @@ -71,20 +71,16 @@ def test_combine_climate_units_and_metadata_empty_projects_then_success( ] mock_projects.return_value = [] - monkeypatch.setattr( - crud.ClimateWareHouseCrud, "get_climate_projects", mock_projects - ) + monkeypatch.setattr(crud.ClimateWareHouseCrud, "get_climate_projects", mock_projects) monkeypatch.setattr(crud.ClimateWareHouseCrud, "get_climate_units", mock_units) - response = crud.ClimateWareHouseCrud( - url=mock.MagicMock(), api_key=None - ).combine_climate_units_and_metadata(search={}) + response = crud.ClimateWareHouseCrud(url=mock.MagicMock(), api_key=None).combine_climate_units_and_metadata( + search={} + ) assert len(response) == 0 - def test_combine_climate_units_and_metadata_empty_orgs_then_success( - self, monkeypatch: pytest.MonkeyPatch - ) -> None: + def test_combine_climate_units_and_metadata_empty_orgs_then_success(self, monkeypatch: pytest.MonkeyPatch) -> None: test_response: List[Dict[str, Any]] = [ { "correspondingAdjustmentDeclaration": "Committed", @@ -275,22 +271,18 @@ def test_combine_climate_units_and_metadata_empty_orgs_then_success( "0x8df0a9aa3739e24467b8a6409b49efe355dd4999a51215aed1f944314af07c60": '{"org_uid": "cf7af8da584b6c115ba8247c5cdd05506c3b3c5c632ed975cc2b16262493e2bd","warehouse_project_id": "c9b98579-debb-49f3-b417-0adbae4ed5c7", "vintage_year": 2099,"sequence_num": 0,"index": "0x8b0aa9633464b5437f4b980b864a3ab5dda49e6a754ef2b1cde6d30fb28a9330","public_key": "0x9650dc15356ba1fe3a48e50daa55ac3dfde5323226922c9bf09aae1bd9612105f323e573cfa0778c681467a0c62bc315", "asset_id": "0x8df0a9aa3739e24467b8a6409b49efe355dd4999a51215aed1f944314af07c60","tokenization": { "mod_hash": "0xbe97af91e9833541c4c5dd0ab08bad1b0653cccd96e56ae43b7314469e458f5b","public_key": "0x8cba9cb11eed6e2a04843d94c9cabecc3f8eb3118f3a4c1dd5260684f462a8c886db5963f2dcac03f54a745a42777e7c"}, "detokenization": {"mod_hash": "0xed13201cb8b52b4c7ef851e220a3d2bddd57120e6e6afde2aabe3fcc400765ea", "public_key": "0xb431835fe9fa64e9bea1bbab1d4bffd15d17d997f3754b2f97c8db43ea173a8b9fa79ac3a7d58c80111fbfdd4e485f0d", "signature": "0xa627c8779c2d8096444d44879294c7d963180c166564e9c9569c23c3a744af514aae03aeaa5e2d5fd12d0c008c1630410e9d4516b58863658f7ac5b35d09d8810fb28ed43b3f6243c645f0bd934b434aac87cd5718dafd87b51d8bf9c821ba24"},"permissionless_retirement": {"mod_hash": "0x36ab0a0666149598070b7c40ab10c3aaff51384d4ad4544a1c301636e917c039","signature": "0xaa1f6b71999333761fbd9eb914ce5ab1c3acb83e7fa7eb5b59c226f20b644c835f8238edbe3ddfeed1a916f0307fe1200174a211b8169ace5afcd9162f88b46565f3ffbbf6dfdf8d154e6337e30829c23ab3f6796d9a319bf0d9168685541d62"}}' } - monkeypatch.setattr( - crud.ClimateWareHouseCrud, "get_climate_projects", mock_projects - ) + monkeypatch.setattr(crud.ClimateWareHouseCrud, "get_climate_projects", mock_projects) monkeypatch.setattr(crud.ClimateWareHouseCrud, "get_climate_units", mock_units) - monkeypatch.setattr( - crud.ClimateWareHouseCrud, "get_climate_organizations", mock_orgs - ) + monkeypatch.setattr(crud.ClimateWareHouseCrud, "get_climate_organizations", mock_orgs) monkeypatch.setattr( crud.ClimateWareHouseCrud, "get_climate_organizations_metadata", mock_org_metadata, ) - response = crud.ClimateWareHouseCrud( - url=mock.MagicMock(), api_key=None - ).combine_climate_units_and_metadata(search={}) + response = crud.ClimateWareHouseCrud(url=mock.MagicMock(), api_key=None).combine_climate_units_and_metadata( + search={} + ) print(f"EMLEML: {response}") assert response == test_response diff --git a/tests/test_crud_core.py b/tests/test_crud_core.py index 6d62095..268f04e 100644 --- a/tests/test_crud_core.py +++ b/tests/test_crud_core.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from unittest import mock from app.crud.db import DBCrud diff --git a/tests/test_disallow.py b/tests/test_disallow.py new file mode 100644 index 0000000..6bc59f6 --- /dev/null +++ b/tests/test_disallow.py @@ -0,0 +1,23 @@ +from __future__ import annotations + +import pytest +from fastapi import status + +from app.config import ExecutionMode, settings +from app.utils import disallow + + +@pytest.mark.asyncio +async def test_disallow() -> None: + settings.MODE = ExecutionMode.DEV + + @disallow([ExecutionMode.DEV]) + async def disallow_dev() -> int: + return 5 + + @disallow([ExecutionMode.REGISTRY]) + async def allow_dev() -> int: + return 5 + + assert await disallow_dev() == status.HTTP_405_METHOD_NOT_ALLOWED + assert await allow_dev() == 5 From aeff2b8a18ead5b31738d7bd348a70d8f455d273 Mon Sep 17 00:00:00 2001 From: Earle Lowe Date: Mon, 6 Nov 2023 13:07:01 -0800 Subject: [PATCH 03/11] fix some odd incompatibilities --- .isort.cfg | 1 - app/api/v1/activities.py | 2 -- app/api/v1/keys.py | 2 -- app/api/v1/tokens.py | 2 -- app/api/v1/transactions.py | 2 -- app/utils.py | 48 +++++++++++++++++++++++++++----------- 6 files changed, 35 insertions(+), 22 deletions(-) diff --git a/.isort.cfg b/.isort.cfg index 7557202..f255c33 100644 --- a/.isort.cfg +++ b/.isort.cfg @@ -2,4 +2,3 @@ line_length = 120 profile=black skip_gitignore=true -add_imports=from __future__ import annotations diff --git a/app/api/v1/activities.py b/app/api/v1/activities.py index 847e9e4..308797a 100644 --- a/app/api/v1/activities.py +++ b/app/api/v1/activities.py @@ -1,5 +1,3 @@ -from __future__ import annotations - from typing import Any, Dict, List, Optional from fastapi import APIRouter, Depends diff --git a/app/api/v1/keys.py b/app/api/v1/keys.py index 04b0e0b..9668e95 100644 --- a/app/api/v1/keys.py +++ b/app/api/v1/keys.py @@ -1,5 +1,3 @@ -from __future__ import annotations - from typing import Optional from blspy import G1Element, PrivateKey diff --git a/app/api/v1/tokens.py b/app/api/v1/tokens.py index 1b0265d..b5d572a 100644 --- a/app/api/v1/tokens.py +++ b/app/api/v1/tokens.py @@ -1,5 +1,3 @@ -from __future__ import annotations - import json from typing import Any, Dict, Tuple diff --git a/app/api/v1/transactions.py b/app/api/v1/transactions.py index a5aee32..30df6b7 100644 --- a/app/api/v1/transactions.py +++ b/app/api/v1/transactions.py @@ -1,5 +1,3 @@ -from __future__ import annotations - from typing import List, Optional from chia.rpc.wallet_rpc_client import WalletRpcClient diff --git a/app/utils.py b/app/utils.py index c69f452..2218da7 100644 --- a/app/utils.py +++ b/app/utils.py @@ -1,34 +1,56 @@ from __future__ import annotations +import functools import os import time -from typing import Any, Callable, Concatenate, Coroutine, List, ParamSpec, TypeVar +from typing import Callable, List from fastapi import status from app.config import ExecutionMode, settings from app.logger import logger -P = ParamSpec("P") -R = TypeVar("R") +# from typing import Any, Callable, Concatenate, Coroutine, List, ParamSpec, TypeVar -def disallow( - modes: List[ExecutionMode], -) -> Callable[[Callable[Concatenate[P], Coroutine[Any, Any, R]]], Callable[Concatenate[P], Coroutine[Any, Any, R]],]: - def decorator( - f: Callable[Concatenate[P], Coroutine[Any, Any, R]] - ) -> Callable[Concatenate[P], Coroutine[Any, Any, R]]: +# P = ParamSpec("P") +# R = TypeVar("R") + + +# def disallow( +# modes: List[ExecutionMode], +# ) -> Callable[[Callable[Concatenate[P], Coroutine[Any, Any, R]]], Callable[Concatenate[P], Coroutine[Any, Any, R]],]: +# def decorator( +# f: Callable[Concatenate[P], Coroutine[Any, Any, R]] +# ) -> Callable[Concatenate[P], Coroutine[Any, Any, R]]: +# if settings.MODE in modes: + +# async def not_allowed(*args: P.args, **kwargs: P.kwargs) -> Any: +# return status.HTTP_405_METHOD_NOT_ALLOWED + +# return not_allowed + +# return f + +# return decorator + + +def disallow(modes: List[ExecutionMode]): + def _disallow(f: Callable): if settings.MODE in modes: - async def not_allowed(*args: P.args, **kwargs: P.kwargs) -> Any: + async def _f(*args, **kargs): return status.HTTP_405_METHOD_NOT_ALLOWED - return not_allowed + else: + + @functools.wraps(f) + async def _f(*args, **kargs): + return await f(*args, **kargs) - return f + return _f - return decorator + return _disallow def wait_until_dir_exists(path: str, interval: int = 1) -> None: From f3832052994add050956987fc6250498af0d3ef5 Mon Sep 17 00:00:00 2001 From: Michael Taylor Date: Tue, 7 Nov 2023 13:27:55 -0500 Subject: [PATCH 04/11] feat: optimized sub-optimal scan_token_activity cron --- app/api/v1/cron.py | 71 ++++++++++++++++++++++++++++++---------------- app/config.py | 1 + config.yaml | 1 + 3 files changed, 48 insertions(+), 25 deletions(-) diff --git a/app/api/v1/cron.py b/app/api/v1/cron.py index dd073f0..91123d7 100644 --- a/app/api/v1/cron.py +++ b/app/api/v1/cron.py @@ -1,6 +1,7 @@ from __future__ import annotations import asyncio +import json from typing import List from blspy import G1Element @@ -72,37 +73,57 @@ async def _scan_token_activity( logger.info(f"Scanning blocks {start_height} - {end_height} for activity") - climate_units = climate_warehouse.combine_climate_units_and_metadata(search={}) - for unit in climate_units: - token = unit.get("token") - - # is None or empty - if not token: - logger.warning(f"Can not get token in climate warehouse unit. unit:{unit}") - continue - - public_key = G1Element.from_bytes(hexstr_to_bytes(token["public_key"])) - - activities: List[schemas.Activity] = await blockchain.get_activities( - org_uid=token["org_uid"], - warehouse_project_id=token["warehouse_project_id"], - vintage_year=token["vintage_year"], - sequence_num=token["sequence_num"], - public_key=public_key, - start_height=start_height, - end_height=end_height, - peak_height=state.peak_height, - ) - - if len(activities) == 0: + # Check if SCAN_ALL_ORGANIZATIONS is defined and True, otherwise treat as False + scan_all = getattr(settings, 'SCAN_ALL_ORGANIZATIONS', False) + + all_organizations = climate_warehouse.get_climate_organizations() + if not scan_all: + # Convert to a list of organizations where `isHome` is True + climate_organizations = [org for org in all_organizations.values() if org.get('isHome', False)] + else: + # Convert to a list of all organizations + climate_organizations = list(all_organizations.values()) + + for org_uid, org_name in climate_organizations.items(): + org_metadata = climate_warehouse.get_climate_organizations_metadata(org_uid) + if not org_metadata: + logger.warning(f"Cannot get metadata in CADT organization: {org_name}") continue - db_crud.batch_insert_ignore_activity(activities) + for key, value_str in org_metadata.items(): + try: + tokenization_dict = json.loads(value_str) + required_fields = ['org_uid', 'warehouse_project_id', 'vintage_year', 'sequence_num', 'public_key', 'index'] + optional_fields = ['permissionless_retirement', 'detokenization'] + + if not all(field in tokenization_dict for field in required_fields) or not any(field in tokenization_dict for field in optional_fields): + # not a tokenization record + continue + + public_key = G1Element.from_bytes(hexstr_to_bytes(tokenization_dict["public_key"])) + activities = await blockchain.get_activities( + org_uid=tokenization_dict["org_uid"], + warehouse_project_id=tokenization_dict["warehouse_project_id"], + vintage_year=tokenization_dict["vintage_year"], + sequence_num=tokenization_dict["sequence_num"], + public_key=public_key, + start_height=state.current_height, + end_height=end_height, + peak_height=state.peak_height + ) + + if activities: + db_crud.batch_insert_ignore_activity(activities) + logger.info(f"Activities for {org_name} and asset id: {key} added to the database.") + + except json.JSONDecodeError as e: + logger.error(f"Failed to parse JSON for key {key} in organization {org_name}: {str(e)}") + except Exception as e: + logger.error(f"An error occurred for organization {org_name} under key {key}: {str(e)}") db_crud.update_block_state(current_height=target_start_height) return True - @router.on_event("startup") @repeat_every(seconds=60, logger=logger) @disallow([ExecutionMode.REGISTRY, ExecutionMode.CLIENT]) diff --git a/app/config.py b/app/config.py index acb02c5..d05866d 100644 --- a/app/config.py +++ b/app/config.py @@ -57,6 +57,7 @@ class Settings(BaseSettings): CLIMATE_TOKEN_CLIENT_PORT: Optional[int] = None CLIMATE_TOKEN_REGISTRY_PORT: Optional[int] = None DEV_PORT: Optional[int] = None + SCAN_ALL_ORGANIZATIONS: Optional[bool] = False _instance: Optional[Settings] = None diff --git a/config.yaml b/config.yaml index de415d2..22e2ddc 100644 --- a/config.yaml +++ b/config.yaml @@ -14,3 +14,4 @@ CLIMATE_TOKEN_REGISTRY_PORT: 31312 CLIMATE_EXPLORER_PORT: 31313 CLIMATE_TOKEN_CLIENT_PORT: 31314 DEV_PORT: 31999 +SCAN_ALL_ORGANIZATIONS: false From 0c125b36fb4f467cd3d59332365db507a20f7396 Mon Sep 17 00:00:00 2001 From: Michael Taylor Date: Wed, 8 Nov 2023 14:07:03 -0500 Subject: [PATCH 05/11] fix org loop --- app/api/v1/cron.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/app/api/v1/cron.py b/app/api/v1/cron.py index 91123d7..ca22a81 100644 --- a/app/api/v1/cron.py +++ b/app/api/v1/cron.py @@ -84,7 +84,10 @@ async def _scan_token_activity( # Convert to a list of all organizations climate_organizations = list(all_organizations.values()) - for org_uid, org_name in climate_organizations.items(): + for org in climate_organizations: + org_uid = org.orgUid; + org_name = org.name; + org_metadata = climate_warehouse.get_climate_organizations_metadata(org_uid) if not org_metadata: logger.warning(f"Cannot get metadata in CADT organization: {org_name}") From 2cd1fc6c5d0348c836a99000494296cd5f5a7621 Mon Sep 17 00:00:00 2001 From: Michael Taylor Date: Wed, 8 Nov 2023 14:11:01 -0500 Subject: [PATCH 06/11] fix org loop --- app/api/v1/cron.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/api/v1/cron.py b/app/api/v1/cron.py index ca22a81..e695460 100644 --- a/app/api/v1/cron.py +++ b/app/api/v1/cron.py @@ -85,8 +85,8 @@ async def _scan_token_activity( climate_organizations = list(all_organizations.values()) for org in climate_organizations: - org_uid = org.orgUid; - org_name = org.name; + org_uid = org["orgUid"] + org_name = org["name"] org_metadata = climate_warehouse.get_climate_organizations_metadata(org_uid) if not org_metadata: From e5f276ddeebb0136a8d94e95e9d592a3ce079be3 Mon Sep 17 00:00:00 2001 From: Earle Lowe Date: Wed, 8 Nov 2023 11:49:00 -0800 Subject: [PATCH 07/11] style: linting update --- app/api/v1/cron.py | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/app/api/v1/cron.py b/app/api/v1/cron.py index e695460..c393532 100644 --- a/app/api/v1/cron.py +++ b/app/api/v1/cron.py @@ -74,12 +74,12 @@ async def _scan_token_activity( logger.info(f"Scanning blocks {start_height} - {end_height} for activity") # Check if SCAN_ALL_ORGANIZATIONS is defined and True, otherwise treat as False - scan_all = getattr(settings, 'SCAN_ALL_ORGANIZATIONS', False) + scan_all = getattr(settings, "SCAN_ALL_ORGANIZATIONS", False) all_organizations = climate_warehouse.get_climate_organizations() if not scan_all: # Convert to a list of organizations where `isHome` is True - climate_organizations = [org for org in all_organizations.values() if org.get('isHome', False)] + climate_organizations = [org for org in all_organizations.values() if org.get("isHome", False)] else: # Convert to a list of all organizations climate_organizations = list(all_organizations.values()) @@ -96,10 +96,19 @@ async def _scan_token_activity( for key, value_str in org_metadata.items(): try: tokenization_dict = json.loads(value_str) - required_fields = ['org_uid', 'warehouse_project_id', 'vintage_year', 'sequence_num', 'public_key', 'index'] - optional_fields = ['permissionless_retirement', 'detokenization'] - - if not all(field in tokenization_dict for field in required_fields) or not any(field in tokenization_dict for field in optional_fields): + required_fields = [ + "org_uid", + "warehouse_project_id", + "vintage_year", + "sequence_num", + "public_key", + "index", + ] + optional_fields = ["permissionless_retirement", "detokenization"] + + if not all(field in tokenization_dict for field in required_fields) or not any( + field in tokenization_dict for field in optional_fields + ): # not a tokenization record continue @@ -112,9 +121,9 @@ async def _scan_token_activity( public_key=public_key, start_height=state.current_height, end_height=end_height, - peak_height=state.peak_height + peak_height=state.peak_height, ) - + if activities: db_crud.batch_insert_ignore_activity(activities) logger.info(f"Activities for {org_name} and asset id: {key} added to the database.") @@ -127,6 +136,7 @@ async def _scan_token_activity( db_crud.update_block_state(current_height=target_start_height) return True + @router.on_event("startup") @repeat_every(seconds=60, logger=logger) @disallow([ExecutionMode.REGISTRY, ExecutionMode.CLIENT]) From e69d540b8b6d32a31dda2a9f5ebd4e6f30d1d0a6 Mon Sep 17 00:00:00 2001 From: Michael Taylor Date: Wed, 8 Nov 2023 15:21:11 -0500 Subject: [PATCH 08/11] fix: activities scan --- app/api/v1/cron.py | 36 ++++++++++++++++++++++++------------ 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/app/api/v1/cron.py b/app/api/v1/cron.py index e695460..68e0ffa 100644 --- a/app/api/v1/cron.py +++ b/app/api/v1/cron.py @@ -74,12 +74,12 @@ async def _scan_token_activity( logger.info(f"Scanning blocks {start_height} - {end_height} for activity") # Check if SCAN_ALL_ORGANIZATIONS is defined and True, otherwise treat as False - scan_all = getattr(settings, 'SCAN_ALL_ORGANIZATIONS', False) + scan_all = getattr(settings, "SCAN_ALL_ORGANIZATIONS", False) all_organizations = climate_warehouse.get_climate_organizations() if not scan_all: # Convert to a list of organizations where `isHome` is True - climate_organizations = [org for org in all_organizations.values() if org.get('isHome', False)] + climate_organizations = [org for org in all_organizations.values() if org.get("isHome", False)] else: # Convert to a list of all organizations climate_organizations = list(all_organizations.values()) @@ -96,15 +96,24 @@ async def _scan_token_activity( for key, value_str in org_metadata.items(): try: tokenization_dict = json.loads(value_str) - required_fields = ['org_uid', 'warehouse_project_id', 'vintage_year', 'sequence_num', 'public_key', 'index'] - optional_fields = ['permissionless_retirement', 'detokenization'] - - if not all(field in tokenization_dict for field in required_fields) or not any(field in tokenization_dict for field in optional_fields): + required_fields = [ + "org_uid", + "warehouse_project_id", + "vintage_year", + "sequence_num", + "public_key", + "index", + ] + optional_fields = ["permissionless_retirement", "detokenization"] + + if not all(field in tokenization_dict for field in required_fields) or not any( + field in tokenization_dict for field in optional_fields + ): # not a tokenization record continue public_key = G1Element.from_bytes(hexstr_to_bytes(tokenization_dict["public_key"])) - activities = await blockchain.get_activities( + activities: List[schemas.Activity] = await blockchain.get_activities( org_uid=tokenization_dict["org_uid"], warehouse_project_id=tokenization_dict["warehouse_project_id"], vintage_year=tokenization_dict["vintage_year"], @@ -112,12 +121,14 @@ async def _scan_token_activity( public_key=public_key, start_height=state.current_height, end_height=end_height, - peak_height=state.peak_height + peak_height=state.peak_height, ) - - if activities: - db_crud.batch_insert_ignore_activity(activities) - logger.info(f"Activities for {org_name} and asset id: {key} added to the database.") + + if len(activities) == 0: + continue + + db_crud.batch_insert_ignore_activity(activities) + logger.info(f"Activities for {org_name} and asset id: {key} added to the database.") except json.JSONDecodeError as e: logger.error(f"Failed to parse JSON for key {key} in organization {org_name}: {str(e)}") @@ -127,6 +138,7 @@ async def _scan_token_activity( db_crud.update_block_state(current_height=target_start_height) return True + @router.on_event("startup") @repeat_every(seconds=60, logger=logger) @disallow([ExecutionMode.REGISTRY, ExecutionMode.CLIENT]) From 81e5e0b4c4f83822d7398809b9499b8285fbc244 Mon Sep 17 00:00:00 2001 From: Earle Lowe Date: Wed, 8 Nov 2023 14:16:34 -0800 Subject: [PATCH 09/11] fix: issues with reading from DBs due to bad context manager code --- app/api/dependencies.py | 9 ++++++--- app/api/v1/cron.py | 6 +++--- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/app/api/dependencies.py b/app/api/dependencies.py index 6751d06..d5d00fd 100644 --- a/app/api/dependencies.py +++ b/app/api/dependencies.py @@ -1,7 +1,7 @@ from __future__ import annotations import enum -from contextlib import asynccontextmanager +from contextlib import asynccontextmanager, AbstractAsyncContextManager from pathlib import Path from typing import AsyncGenerator, AsyncIterator @@ -18,8 +18,11 @@ from app.logger import logger -@asynccontextmanager -async def get_db_session() -> AsyncIterator[Session]: +def get_db_session_context() -> AbstractAsyncContextManager[Session]: + return asynccontextmanager(get_db_session)() + + +async def get_db_session() -> Session: SessionLocal = await get_session_local_cls() db = SessionLocal() diff --git a/app/api/v1/cron.py b/app/api/v1/cron.py index dd073f0..9a14479 100644 --- a/app/api/v1/cron.py +++ b/app/api/v1/cron.py @@ -40,7 +40,7 @@ async def init_db() -> None: Base.metadata.create_all(Engine) - async with deps.get_db_session() as db: + async with deps.get_db_session_context() as db: state = State(id=1, current_height=settings.BLOCK_START, peak_height=None) db_state = [jsonable_encoder(state)] @@ -112,7 +112,7 @@ async def scan_token_activity() -> None: async with ( lock, - deps.get_db_session() as db, + deps.get_db_session_context() as db, deps.get_full_node_rpc_client() as full_node_client, ): db_crud = crud.DBCrud(db=db) @@ -160,7 +160,7 @@ async def _scan_blockchain_state( @disallow([ExecutionMode.REGISTRY, ExecutionMode.CLIENT]) async def scan_blockchain_state() -> None: async with ( - deps.get_db_session() as db, + deps.get_db_session_context() as db, deps.get_full_node_rpc_client() as full_node_client, ): db_crud = crud.DBCrud(db=db) From aceb5496233921824c32150d2f71be05206b13d7 Mon Sep 17 00:00:00 2001 From: Earle Lowe Date: Mon, 13 Nov 2023 11:30:06 -0800 Subject: [PATCH 10/11] fix: issues with token pydantic classes and forward refs --- app/api/dependencies.py | 2 +- app/schemas/token.py | 18 +++++++++-------- tests/test_crud_chia.py | 43 ++++++++++++++++++++++++++++++++++++++++- 3 files changed, 53 insertions(+), 10 deletions(-) diff --git a/app/api/dependencies.py b/app/api/dependencies.py index d5d00fd..85ba7f7 100644 --- a/app/api/dependencies.py +++ b/app/api/dependencies.py @@ -1,7 +1,7 @@ from __future__ import annotations import enum -from contextlib import asynccontextmanager, AbstractAsyncContextManager +from contextlib import AbstractAsyncContextManager, asynccontextmanager from pathlib import Path from typing import AsyncGenerator, AsyncIterator diff --git a/app/schemas/token.py b/app/schemas/token.py index fcc28f7..e037fad 100644 --- a/app/schemas/token.py +++ b/app/schemas/token.py @@ -74,11 +74,12 @@ class DetokenizationFileParseResponse(BaseModel): gateway_coin_spend: ChiaJsonObject -class DetokenizationFileRequest(BaseModel): - class _TokenOnChain(TokenOnChainBase): - detokenization: DetokenizationTailMetadata +class _TokenOnChain_Detokenize(TokenOnChainBase): + detokenization: DetokenizationTailMetadata - token: _TokenOnChain + +class DetokenizationFileRequest(BaseModel): + token: _TokenOnChain_Detokenize payment: PaymentBase @@ -88,11 +89,12 @@ class DetokenizationFileResponse(BaseModel): tx: Transaction -class PermissionlessRetirementTxRequest(BaseModel): - class _TokenOnChain(TokenOnChainBase): - permissionless_retirement: PermissionlessRetirementTailMetadata +class _TokenOnChain_Permissionless(TokenOnChainBase): + permissionless_retirement: PermissionlessRetirementTailMetadata - token: _TokenOnChain + +class PermissionlessRetirementTxRequest(BaseModel): + token: _TokenOnChain_Permissionless payment: RetirementPaymentWithPayer diff --git a/tests/test_crud_chia.py b/tests/test_crud_chia.py index 7a2c7a7..1937af3 100644 --- a/tests/test_crud_chia.py +++ b/tests/test_crud_chia.py @@ -5,7 +5,7 @@ import pytest -from app import crud +from app import crud, schemas class TestClimateWareHouseCrud: @@ -286,3 +286,44 @@ def test_combine_climate_units_and_metadata_empty_orgs_then_success(self, monkey print(f"EMLEML: {response}") assert response == test_response + + def test_PermissionlessRetirementTxRequest(self) -> None: + test_data = { + "token": { + "org_uid": "cf7af8da584b6c115ba8247c5cdd05506c3b3c5c632ed975cc2b16262493e2bd", + "warehouse_project_id": "c9b98579-debb-49f3-b417-0adbae4ed5c7", + "vintage_year": "2099", + "sequence_num": 0, + "asset_id": "0x8df0a9aa3739e24467b8a6409b49efe355dd4999a51215aed1f944314af07c60", + "index": "0x8b0aa9633464b5437f4b980b864a3ab5dda49e6a754ef2b1cde6d30fb28a9330", + "public_key": "0x9650dc15356ba1fe3a48e50daa55ac3dfde5323226922c9bf09aae1bd9612105f323e573cfa0778c681467a0c62bc315", + "permissionless_retirement": { + "signature": "0xaa1f6b71999333761fbd9eb914ce5ab1c3acb83e7fa7eb5b59c226f20b644c835f8238edbe3ddfeed1a916f0307fe1200174a211b8169ace5afcd9162f88b46565f3ffbbf6dfdf8d154e6337e30829c23ab3f6796d9a319bf0d9168685541d62", + "mod_hash": "0x36ab0a0666149598070b7c40ab10c3aaff51384d4ad4544a1c301636e917c039", + }, + }, + "payment": {"amount": 5, "fee": 5, "beneficiary_name": "fred", "beneficiary_address": "bar"}, + } + + schemas.PermissionlessRetirementTxRequest.parse_obj(test_data) + + def test_DetokenizationFileRequest(self) -> None: + test_data = { + "token": { + "org_uid": "cf7af8da584b6c115ba8247c5cdd05506c3b3c5c632ed975cc2b16262493e2bd", + "warehouse_project_id": "c9b98579-debb-49f3-b417-0adbae4ed5c7", + "vintage_year": "2099", + "sequence_num": 0, + "asset_id": "0x8df0a9aa3739e24467b8a6409b49efe355dd4999a51215aed1f944314af07c60", + "index": "0x8b0aa9633464b5437f4b980b864a3ab5dda49e6a754ef2b1cde6d30fb28a9330", + "public_key": "0x9650dc15356ba1fe3a48e50daa55ac3dfde5323226922c9bf09aae1bd9612105f323e573cfa0778c681467a0c62bc315", + "detokenization": { + "signature": "0xa627c8779c2d8096444d44879294c7d963180c166564e9c9569c23c3a744af514aae03aeaa5e2d5fd12d0c008c1630410e9d4516b58863658f7ac5b35d09d8810fb28ed43b3f6243c645f0bd934b434aac87cd5718dafd87b51d8bf9c821ba24", + "mod_hash": "0xed13201cb8b52b4c7ef851e220a3d2bddd57120e6e6afde2aabe3fcc400765ea", + "public_key": "0xb431835fe9fa64e9bea1bbab1d4bffd15d17d997f3754b2f97c8db43ea173a8b9fa79ac3a7d58c80111fbfdd4e485f0d", + }, + }, + "payment": {"amount": 5, "fee": 5, "beneficiary_name": "fred", "beneficiary_address": "bar"}, + } + + schemas.DetokenizationFileRequest.parse_obj(test_data) From f540df31c2a5dc739e4e8585ddf46a6bf7a27c20 Mon Sep 17 00:00:00 2001 From: Zachary Brown Date: Mon, 13 Nov 2023 13:25:13 -0800 Subject: [PATCH 11/11] chore: bump version --- README.md | 1 + pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index cd2ce2a..74976b7 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,6 @@ # Climate Token Driver Suite +![Minimum Chia Version](https://raw.githubusercontent.com/Chia-Network/core-registry-api/main/minimumChiaVersion.svg) ![Tested Up to Chia Version](https://raw.githubusercontent.com/Chia-Network/core-registry-api/main/testedChiaVersion.svg) This application can run in 4 modes, each providing a separate application with a distinct use case: diff --git a/pyproject.toml b/pyproject.toml index e16ba58..b754df6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "Chia Climate Token Driver" -version = "1.0.35" +version = "1.0.36" description = "https://github.com/Chia-Network/climate-token-driver" authors = ["Harry Hsu ", "Chia Network Inc "]