From c98019f9962cf4f4c74329827e18cb46c126a8f8 Mon Sep 17 00:00:00 2001 From: Leonard G Date: Tue, 6 Oct 2020 10:56:50 +0800 Subject: [PATCH] Added list storage deal api. (#39) * Added list storage deal api. * Added more code set up file. * Added github workflow. * Removed redundant installation. * Added back pipenv installation. * Try. * Fixed integration test failure. * Fix. * Improved error handling. * Fixed bug. * Fix. * avoid remove. * remove. * fix missing import. * Increase wait time. * decrease wait time a bit. * Added more wait time. * Changed wait time to 20 seconds. * change to inequality. * WIP: add list deal info example * feat: list storage/retrieval deals * fix: reformat * feat(test): add retrieval deals test * feat(test): add retrieval deals test Co-authored-by: Peter Van Garderen Co-authored-by: apogiatzis Closes #37 --- .github/workflows/tests.yml | 4 +- Pipfile | 7 +- Pipfile.lock | 140 +++++++++++++++++++++++++++++-- examples/ffs_config.py | 1 - examples/ffs_deals.py | 49 +++++++++++ examples/ffs_logs.py | 3 - pygate_grpc/buildinfo.py | 7 +- pygate_grpc/client.py | 2 +- pygate_grpc/errors.py | 10 +-- pygate_grpc/exceptions.py | 2 +- pygate_grpc/faults.py | 3 +- pygate_grpc/ffs.py | 58 ++++++++++--- pygate_grpc/health.py | 4 +- pygate_grpc/net.py | 2 +- pygate_grpc/wallet.py | 2 +- setup.cfg | 15 +++- setup.py | 9 +- tests/integration/conftest.py | 29 ++++--- tests/integration/test_faults.py | 4 +- tests/integration/test_ffs.py | 41 +++++++-- tests/integration/test_health.py | 4 +- tests/integration/test_net.py | 10 +-- tests/integration/test_wallet.py | 4 +- 23 files changed, 322 insertions(+), 88 deletions(-) create mode 100644 examples/ffs_deals.py diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index d787a19..c8b1562 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -21,9 +21,9 @@ jobs: uses: actions/setup-python@v2 with: python-version: '3.7' - - name: Install dependencies + - name: Install CI dependencies run: | - python -m pip install --upgrade pip + python -m pip install --upgrade pip pip install pipenv pipenv install --dev - name: Run Integration Tests diff --git a/Pipfile b/Pipfile index c9fa6d4..53f9c18 100644 --- a/Pipfile +++ b/Pipfile @@ -17,6 +17,10 @@ pytest-docker = "*" grpcio-tools = "*" secretstorage = {markers = "sys_platform == 'linux'"} keyring = "==19.1.0" +flake8 = "*" +isort = "*" +pylint = "*" +mypy = "*" [requires] python_version = "3.7" @@ -24,7 +28,8 @@ python_version = "3.7" [scripts] build = "python setup.py sdist bdist_wheel" publish = "twine upload dist/*" -format = "bash -c \"python -m black $(git ls-files '*.py')\"" +format = "bash -c \"python -m isort setup.py pygate_grpc tests && python -m black $(git ls-files '*.py')\"" +lint = "bash -c \"python -m flake8\"" integration-test = "python -m pytest tests/integration/" test-ffs = "python -m pytest tests/integration/test_ffs.py" diff --git a/Pipfile.lock b/Pipfile.lock index d47b4e0..1421021 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "56a3a6f22e0914e07470d9d094724d1f08c52652c0d10178cf268291da423cbf" + "sha256": "f20e5d999342212cfb0f34940f3b95287841c254c17ce54ce9969fa8e214c767" }, "pipfile-spec": 6, "requires": { @@ -116,6 +116,13 @@ ], "version": "==1.4.4" }, + "astroid": { + "hashes": [ + "sha256:2f4078c2a41bf377eea06d71c9d2ba4eb8f6b1af2135bec27bbbb7d8f12bb703", + "sha256:bc58d83eb610252fd8de6363e39d4f1d0619c894b0ed24603b881c02e64c7386" + ], + "version": "==2.4.2" + }, "attrs": { "hashes": [ "sha256:26b54ddbbb9ee1d34d5d3668dd37d6cf74990ab23c828c2888dccdceee395594", @@ -325,6 +332,14 @@ ], "version": "==0.3" }, + "flake8": { + "hashes": [ + "sha256:749dbbd6bfd0cf1318af27bf97a14e28e5ff548ef8e5b1566ccfb25a11e7c839", + "sha256:aadae8761ec651813c24be05c6f7b4680857ef6afaae4651a4eccaef97ce6c3b" + ], + "index": "pypi", + "version": "==3.8.4" + }, "gitdb": { "hashes": [ "sha256:91f36bfb1ab7949b3b40e23736db18231bf7593edada2ba5c3a174a7b23657ac", @@ -452,6 +467,14 @@ ], "version": "==1.0.1" }, + "isort": { + "hashes": [ + "sha256:36f0c6659b9000597e92618d05b72d4181104cf59472b1c6a039e3783f930c95", + "sha256:ba040c24d20aa302f78f4747df549573ae1eaf8e1084269199154da9c483f07f" + ], + "index": "pypi", + "version": "==5.5.4" + }, "jeepney": { "hashes": [ "sha256:3479b861cc2b6407de5188695fa1a8d57e5072d7059322469b62628869b8e36e", @@ -474,6 +497,66 @@ "index": "pypi", "version": "==19.1.0" }, + "lazy-object-proxy": { + "hashes": [ + "sha256:0c4b206227a8097f05c4dbdd323c50edf81f15db3b8dc064d08c62d37e1a504d", + "sha256:194d092e6f246b906e8f70884e620e459fc54db3259e60cf69a4d66c3fda3449", + "sha256:1be7e4c9f96948003609aa6c974ae59830a6baecc5376c25c92d7d697e684c08", + "sha256:4677f594e474c91da97f489fea5b7daa17b5517190899cf213697e48d3902f5a", + "sha256:48dab84ebd4831077b150572aec802f303117c8cc5c871e182447281ebf3ac50", + "sha256:5541cada25cd173702dbd99f8e22434105456314462326f06dba3e180f203dfd", + "sha256:59f79fef100b09564bc2df42ea2d8d21a64fdcda64979c0fa3db7bdaabaf6239", + "sha256:8d859b89baf8ef7f8bc6b00aa20316483d67f0b1cbf422f5b4dc56701c8f2ffb", + "sha256:9254f4358b9b541e3441b007a0ea0764b9d056afdeafc1a5569eee1cc6c1b9ea", + "sha256:9651375199045a358eb6741df3e02a651e0330be090b3bc79f6d0de31a80ec3e", + "sha256:97bb5884f6f1cdce0099f86b907aa41c970c3c672ac8b9c8352789e103cf3156", + "sha256:9b15f3f4c0f35727d3a0fba4b770b3c4ebbb1fa907dbcc046a1d2799f3edd142", + "sha256:a2238e9d1bb71a56cd710611a1614d1194dc10a175c1e08d75e1a7bcc250d442", + "sha256:a6ae12d08c0bf9909ce12385803a543bfe99b95fe01e752536a60af2b7797c62", + "sha256:ca0a928a3ddbc5725be2dd1cf895ec0a254798915fb3a36af0964a0a4149e3db", + "sha256:cb2c7c57005a6804ab66f106ceb8482da55f5314b7fcb06551db1edae4ad1531", + "sha256:d74bb8693bf9cf75ac3b47a54d716bbb1a92648d5f781fc799347cfc95952383", + "sha256:d945239a5639b3ff35b70a88c5f2f491913eb94871780ebfabb2568bd58afc5a", + "sha256:eba7011090323c1dadf18b3b689845fd96a61ba0a1dfbd7f24b921398affc357", + "sha256:efa1909120ce98bbb3777e8b6f92237f5d5c8ea6758efea36a473e1d38f7d3e4", + "sha256:f3900e8a5de27447acbf900b4750b0ddfd7ec1ea7fbaf11dfa911141bc522af0" + ], + "version": "==1.4.3" + }, + "mccabe": { + "hashes": [ + "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42", + "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f" + ], + "version": "==0.6.1" + }, + "mypy": { + "hashes": [ + "sha256:2c6cde8aa3426c1682d35190b59b71f661237d74b053822ea3d748e2c9578a7c", + "sha256:3fdda71c067d3ddfb21da4b80e2686b71e9e5c72cca65fa216d207a358827f86", + "sha256:5dd13ff1f2a97f94540fd37a49e5d255950ebcdf446fb597463a40d0df3fac8b", + "sha256:6731603dfe0ce4352c555c6284c6db0dc935b685e9ce2e4cf220abe1e14386fd", + "sha256:6bb93479caa6619d21d6e7160c552c1193f6952f0668cdda2f851156e85186fc", + "sha256:81c7908b94239c4010e16642c9102bfc958ab14e36048fa77d0be3289dda76ea", + "sha256:9c7a9a7ceb2871ba4bac1cf7217a7dd9ccd44c27c2950edbc6dc08530f32ad4e", + "sha256:a4a2cbcfc4cbf45cd126f531dedda8485671545b43107ded25ce952aac6fb308", + "sha256:b7fbfabdbcc78c4f6fc4712544b9b0d6bf171069c6e0e3cb82440dd10ced3406", + "sha256:c05b9e4fb1d8a41d41dec8786c94f3b95d3c5f528298d769eb8e73d293abc48d", + "sha256:d7df6eddb6054d21ca4d3c6249cae5578cb4602951fd2b6ee2f5510ffb098707", + "sha256:e0b61738ab504e656d1fe4ff0c0601387a5489ca122d55390ade31f9ca0e252d", + "sha256:eff7d4a85e9eea55afa34888dfeaccde99e7520b51f867ac28a48492c0b1130c", + "sha256:f05644db6779387ccdb468cc47a44b4356fc2ffa9287135d05b70a98dc83b89a" + ], + "index": "pypi", + "version": "==0.782" + }, + "mypy-extensions": { + "hashes": [ + "sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d", + "sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8" + ], + "version": "==0.4.3" + }, "orderedmultidict": { "hashes": [ "sha256:04070bbb5e87291cc9bfa51df413677faf2141c73c61d2a5f7b26bea3cd882ad", @@ -585,6 +668,13 @@ ], "version": "==1.9.0" }, + "pycodestyle": { + "hashes": [ + "sha256:2295e7b2f6b5bd100585ebcb1f616591b652db8a741695b3d8f5d28bdc934367", + "sha256:c58a7d2815e0e8d7972bf1803331fb0152f867bd89adf8a01dfd55085434192e" + ], + "version": "==2.6.0" + }, "pycparser": { "hashes": [ "sha256:2d475327684562c3a96cc71adf7dc8c4f0565175cf86b6d7a404ff4c771f15f0", @@ -592,6 +682,13 @@ ], "version": "==2.20" }, + "pyflakes": { + "hashes": [ + "sha256:0d94e0e05a19e57a99444b6ddcf9a6eb2e5c68d3ca1e98e90707af8152c90a92", + "sha256:35b2d75ee967ea93b55750aa9edbbf72813e06a66ba54438df2cfac9e3c27fc8" + ], + "version": "==2.2.0" + }, "pygments": { "hashes": [ "sha256:307543fe65c0947b126e83dd5a61bd8acbd84abec11f43caebaf5534cbc17998", @@ -599,6 +696,14 @@ ], "version": "==2.7.1" }, + "pylint": { + "hashes": [ + "sha256:bb4a908c9dadbc3aac18860550e870f58e1a02c9f2c204fdf5693d73be061210", + "sha256:bfe68f020f8a0fece830a22dd4d5dddb4ecc6137db04face4c3420a46a52239f" + ], + "index": "pypi", + "version": "==2.6.0" + }, "pynacl": { "hashes": [ "sha256:06cbb4d9b2c4bd3c8dc0d267416aaed79906e7b33f114ddbf0911969794b1cc4", @@ -637,11 +742,11 @@ }, "pytest": { "hashes": [ - "sha256:1cd09785c0a50f9af72220dd12aa78cfa49cbffc356c61eab009ca189e018a33", - "sha256:d010e24666435b39a4cf48740b039885642b6c273a3f77be3e7e03554d2806b7" + "sha256:7a8190790c17d79a11f847fba0b004ee9a8122582ebff4729a082c109e81a4c9", + "sha256:8f593023c1a0f916110285b6efd7f99db07d59546e3d8c36fc60e2ab05d3be92" ], "index": "pypi", - "version": "==6.1.0" + "version": "==6.1.1" }, "pytest-docker": { "hashes": [ @@ -692,12 +797,17 @@ "hashes": [ "sha256:088afc8c63e7bd187a3c70a94b9e50ab3f17e1d3f52a32750b5b77dbe99ef5ef", "sha256:1fe0a41437bbd06063aa184c34804efa886bcc128222e9916310c92cd54c3b4c", + "sha256:3d20024a70b97b4f9546696cbf2fd30bae5f42229fbddf8661261b1eaff0deb7", "sha256:41bb65f54bba392643557e617316d0d899ed5b4946dccee1cb6696152b29844b", "sha256:4318d56bccfe7d43e5addb272406ade7a2274da4b70eb15922a071c58ab0108c", "sha256:4707f3695b34335afdfb09be3802c87fa0bc27030471dbc082f815f23688bc63", + "sha256:49f23ebd5ac073765ecbcf046edc10d63dcab2f4ae2bce160982cb30df0c0302", "sha256:5533a959a1748a5c042a6da71fe9267a908e21eded7a4f373efd23a2cbdb0ecc", + "sha256:5d892a4f1c999834eaa3c32bc9e8b976c5825116cde553928c4c8e7e48ebda67", "sha256:5f18875ac23d9aa2f060838e8b79093e8bb2313dbaaa9f54c6d8e52a5df097be", "sha256:60b0e9e6dc45683e569ec37c55ac20c582973841927a85f2d8a7d20ee80216ab", + "sha256:816064fc915796ea1f26966163f6845de5af78923dfcecf6551e095f00983650", + "sha256:84cada8effefe9a9f53f9b0d2ba9b7b6f5edf8d2155f9fdbe34616e06ececf81", "sha256:84e9407db1b2eb368b7ecc283121b5e592c9aaedbe8c78b1a2f1102eb2e21d19", "sha256:8d69cef61fa50c8133382e61fd97439de1ae623fe943578e477e76a9d9471637", "sha256:9a02d0ae31d35e1ec12a4ea4d4cca990800f66a917d0fb997b20fbc13f5321fc", @@ -705,6 +815,7 @@ "sha256:a6f32aea4260dfe0e55dc9733ea162ea38f0ea86aa7d0f77b15beac5bf7b369d", "sha256:ae91972f8ac958039920ef6e8769277c084971a142ce2b660691793ae44aae6b", "sha256:c570f6fa14b9c4c8a4924aaad354652366577b4f98213cf76305067144f7b100", + "sha256:c9443124c67b1515e4fe0bb0aa18df640965e1030f468a2a5dc2589b26d130ad", "sha256:d23a18037313714fb3bb5a94434d3151ee4300bae631894b1ac08111abeaa4a3", "sha256:eaf548d117b6737df379fdd53bdde4f08870e66d7ea653e230477f071f861121", "sha256:ebbe29186a3d9b0c591e71b7393f1ae08c83cb2d8e517d2a822b8f7ec99dfd8b", @@ -826,8 +937,17 @@ "sha256:fc0fea399acb12edbf8a628ba8d2312f583bdbdb3335635db062fa98cf71fca4", "sha256:fe460b922ec15dd205595c9b5b99e2f056fd98ae8f9f56b888e7a17dc2b757e7" ], + "markers": "implementation_name == 'cpython' and python_version < '3.8'", "version": "==1.4.1" }, + "typing-extensions": { + "hashes": [ + "sha256:7cb407020f00f7bfc3cb3e7881628838e69d8f3fcab2f64742a5e76b2f841918", + "sha256:99d4073b617d30288f569d3f13d2bd7548c3a7e4c8de87db09a9d29bb3a4a60c", + "sha256:dafc7639cde7f1b6e1acc0f457842a83e722ccca8eef5270af2d74792619a89f" + ], + "version": "==3.7.4.3" + }, "urllib3": { "hashes": [ "sha256:91056c15fa70756691db97756772bb1eb9678fa585d9184f24534b100dc60f4a", @@ -864,13 +984,19 @@ "index": "pypi", "version": "==0.35.1" }, + "wrapt": { + "hashes": [ + "sha256:b62ffa81fb85f4332a4f609cab4ac40709470da05643a082ec1eb88e6d9b97d7" + ], + "version": "==1.12.1" + }, "zipp": { "hashes": [ - "sha256:43f4fa8d8bb313e65d8323a3952ef8756bf40f9a5c3ea7334be23ee4ec8278b6", - "sha256:b52f22895f4cfce194bc8172f3819ee8de7540aa6d873535a8668b730b8b411f" + "sha256:64ad89efee774d1897a58607895d80789c59778ea02185dd846ac38394a8642b", + "sha256:eed8ec0b8d1416b2ca33516a37a08892442f3954dee131e92cfd92d8fe3e7066" ], "markers": "python_version < '3.8'", - "version": "==3.2.0" + "version": "==3.3.0" } } } diff --git a/examples/ffs_config.py b/examples/ffs_config.py index bdc0dfe..af01aa3 100644 --- a/examples/ffs_config.py +++ b/examples/ffs_config.py @@ -1,4 +1,3 @@ -import json from pygate_grpc.client import PowerGateClient client = PowerGateClient("127.0.0.1:5002", False) diff --git a/examples/ffs_deals.py b/examples/ffs_deals.py new file mode 100644 index 0000000..e6478c2 --- /dev/null +++ b/examples/ffs_deals.py @@ -0,0 +1,49 @@ +import time + +from io import BytesIO + +from pygate_grpc.client import PowerGateClient +from pygate_grpc.ffs import bytes_to_chunks + +if __name__ == "__main__": + + hostName = "127.0.0.1:5002" + + # Create client + client = PowerGateClient(hostName) + + # Create FFS + ffs = client.ffs.create() + print("FFS created:") + print(ffs) + + test_file = BytesIO(b"These are the contents of a test file") + stage_requests_iter = bytes_to_chunks(test_file) + + print("Pushing file to FFS...") + stage_res = client.ffs.stage(stage_requests_iter, token=ffs.token) + push_res = client.ffs.push(stage_res.cid, token=ffs.token) + + # Check that CID is pinned to FFS + check = client.ffs.info(stage_res.cid, ffs.token) + print("Checking FFS pins...") + print(check) + + # Wait some time so that we can get some deals + time.sleep(5) + + # Check information about the storage deal + storage_deals = client.ffs.list_storage_deal_records( + include_pending=True, include_final=True, token=ffs.token + ) + print("Storage deals: ") + for record in storage_deals.records: + print(record) + + # Check information about the retrieval deals + retrieval_deals = client.ffs.list_retrieval_deal_records( + include_pending=True, include_final=True, token=ffs.token + ) + print("Retrieval deals: ") + for record in retrieval_deals.records: + print(record) diff --git a/examples/ffs_logs.py b/examples/ffs_logs.py index 3bbf923..2a3f5f0 100644 --- a/examples/ffs_logs.py +++ b/examples/ffs_logs.py @@ -1,11 +1,8 @@ -import time - from io import BytesIO from pygate_grpc.client import PowerGateClient from pygate_grpc.ffs import bytes_to_chunks from pygate_grpc.exceptions import GRPCTimeoutException -from pygate_grpc.errors import error_handler client = PowerGateClient("127.0.0.1:5002", False) diff --git a/pygate_grpc/buildinfo.py b/pygate_grpc/buildinfo.py index 34d663d..a0a3640 100644 --- a/pygate_grpc/buildinfo.py +++ b/pygate_grpc/buildinfo.py @@ -1,12 +1,9 @@ -import grpc - -from proto import buildinfo_rpc_pb2 -from proto import buildinfo_rpc_pb2_grpc +from proto import buildinfo_rpc_pb2, buildinfo_rpc_pb2_grpc from pygate_grpc.errors import ErrorHandlerMeta class BuildinfoClient(object, metaclass=ErrorHandlerMeta): - ## THIS USED AN OUTDATED PROTO SPECIFICATION IT NEEDS RE DEVELOPMENT + # THIS USED AN OUTDATED PROTO SPECIFICATION IT NEEDS RE DEVELOPMENT def __init__(self, channel): self.client = buildinfo_rpc_pb2_grpc.RPCServiceStub(channel) diff --git a/pygate_grpc/client.py b/pygate_grpc/client.py index deade43..a3b54fc 100644 --- a/pygate_grpc/client.py +++ b/pygate_grpc/client.py @@ -1,6 +1,6 @@ import grpc -from pygate_grpc import health, faults, buildinfo, ffs, wallet, net +from pygate_grpc import buildinfo, faults, ffs, health, net, wallet from pygate_grpc.errors import ErrorHandlerMeta diff --git a/pygate_grpc/errors.py b/pygate_grpc/errors.py index 272de99..d0c2c82 100644 --- a/pygate_grpc/errors.py +++ b/pygate_grpc/errors.py @@ -1,13 +1,9 @@ -import grpc import logging - from functools import wraps -from pygate_grpc.exceptions import ( - GRPCNotAvailableException, - GRPCTimeoutException, - PyGateGenericException, -) +import grpc + +from pygate_grpc.exceptions import GRPCNotAvailableException, GRPCTimeoutException logger = logging.getLogger(__name__) diff --git a/pygate_grpc/exceptions.py b/pygate_grpc/exceptions.py index b1bdeb2..41655e6 100644 --- a/pygate_grpc/exceptions.py +++ b/pygate_grpc/exceptions.py @@ -1,4 +1,4 @@ -from grpc._channel import _MultiThreadedRendezvous, _InactiveRpcError +from grpc._channel import _MultiThreadedRendezvous class GRPCNotAvailableException(Exception): diff --git a/pygate_grpc/faults.py b/pygate_grpc/faults.py index 90a1995..67de82b 100644 --- a/pygate_grpc/faults.py +++ b/pygate_grpc/faults.py @@ -1,5 +1,4 @@ -from proto import faults_rpc_pb2 -from proto import faults_rpc_pb2_grpc +from proto import faults_rpc_pb2, faults_rpc_pb2_grpc from pygate_grpc.errors import ErrorHandlerMeta diff --git a/pygate_grpc/ffs.py b/pygate_grpc/ffs.py index 685e36b..9212751 100644 --- a/pygate_grpc/ffs.py +++ b/pygate_grpc/ffs.py @@ -1,12 +1,10 @@ -from time import time -import grpc +from typing import Iterable, List, Tuple -from typing import Iterable, Tuple -from proto import ffs_rpc_pb2 -from proto import ffs_rpc_pb2_grpc +from deprecated import deprecated from google.protobuf.json_format import Parse + +from proto import ffs_rpc_pb2, ffs_rpc_pb2_grpc from pygate_grpc.errors import ErrorHandlerMeta, future_error_handler -from deprecated import deprecated TOKEN_KEY = "x-ffs-token" CHUNK_SIZE = 1024 * 1024 # 1MB @@ -140,7 +138,9 @@ def get_storage_job(self, jid: str, token: str = None): req = ffs_rpc_pb2.GetStorageJobRequest(jid=jid) return self.client.GetStorageJob(req, metadata=self._get_meta_data(token)) - def push(self, cid, token: str = None): + def push( + self, cid, token: str = None, + ): req = ffs_rpc_pb2.PushStorageConfigRequest(cid=cid) return self.client.PushStorageConfig(req, metadata=self._get_meta_data(token)) @@ -166,11 +166,47 @@ def redeem_pay_channel( req = ffs_rpc_pb2.CreateRequest(kwargs) return self.client.CreatePayChannel(req, metadata=self._get_meta_data(token)) - def list_storage_deal_records(self, token: str = None): - pass + def list_storage_deal_records( + self, + include_final=True, + include_pending=False, + from_addrs: List[str] = None, + data_cids: List[str] = None, + ascending: bool = False, + token: str = None, + ): + deal_config = ffs_rpc_pb2.ListDealRecordsConfig( + from_addrs=from_addrs, + data_cids=data_cids, + include_pending=include_pending, + include_final=include_final, + ascending=ascending, + ) + req = ffs_rpc_pb2.ListStorageDealRecordsRequest(config=deal_config) + return self.client.ListStorageDealRecords( + req, metadata=self._get_meta_data(token) + ) - def list_retrieval_deal_records(self, token: str = None): - pass + def list_retrieval_deal_records( + self, + include_final=True, + include_pending=False, + from_addrs: List[str] = None, + data_cids: List[str] = None, + ascending: bool = False, + token: str = None, + ): + deal_config = ffs_rpc_pb2.ListDealRecordsConfig( + from_addrs=from_addrs, + data_cids=data_cids, + include_pending=include_pending, + include_final=include_final, + ascending=ascending, + ) + req = ffs_rpc_pb2.ListRetrievalDealRecordsRequest(config=deal_config) + return self.client.ListRetrievalDealRecords( + req, metadata=self._get_meta_data(token) + ) # The metadata is set in here https://github.com/textileio/js-powergate-client/blob # /9d1ad04a7e1f2a6e18cc5627751f9cbddaf6fe05/src/util/grpc-helpers.ts#L7 Note that you can't have capital letter in diff --git a/pygate_grpc/health.py b/pygate_grpc/health.py index 8c81e6b..2629685 100644 --- a/pygate_grpc/health.py +++ b/pygate_grpc/health.py @@ -1,8 +1,6 @@ -import grpc import logging -from proto import health_rpc_pb2 -from proto import health_rpc_pb2_grpc +from proto import health_rpc_pb2, health_rpc_pb2_grpc from pygate_grpc.errors import ErrorHandlerMeta logger = logging.getLogger(__name__) diff --git a/pygate_grpc/net.py b/pygate_grpc/net.py index f573362..87d05be 100644 --- a/pygate_grpc/net.py +++ b/pygate_grpc/net.py @@ -1,5 +1,5 @@ -import grpc import logging + import proto.net_rpc_pb2 as net_rpc_pb2 import proto.net_rpc_pb2_grpc as net_rpc_pb2_grpc diff --git a/pygate_grpc/wallet.py b/pygate_grpc/wallet.py index 2e395dc..c52cba7 100644 --- a/pygate_grpc/wallet.py +++ b/pygate_grpc/wallet.py @@ -1,5 +1,5 @@ -import grpc import logging + import proto.wallet_rpc_pb2 as wallet_rpc_pb2 import proto.wallet_rpc_pb2_grpc as wallet_rpc_pb2_grpc diff --git a/setup.cfg b/setup.cfg index f493806..5b56f69 100644 --- a/setup.cfg +++ b/setup.cfg @@ -4,4 +4,17 @@ description-file = README.md [tool:pytest] addopts = -W ignore testpaths = - tests \ No newline at end of file + tests + +[isort] +multi_line_output = 3 +include_trailing_comma = True +force_grid_wrap = 0 +use_parentheses = True +ensure_newline_before_comments = True +line_length = 120 + +[flake8] +max-line-length = 120 +extend-ignore = E203, W503 +exclude = proto,.git \ No newline at end of file diff --git a/setup.py b/setup.py index 541b806..8843acc 100644 --- a/setup.py +++ b/setup.py @@ -1,8 +1,8 @@ -from setuptools import setup, find_packages - # read the contents of your README file from os import path +from setuptools import setup + this_directory = path.abspath(path.dirname(__file__)) with open(path.join(this_directory, "README.md"), encoding="utf-8") as f: long_description = f.read() @@ -10,7 +10,7 @@ setup( install_requires=[ "deprecated==1.2.10", - "grpcio==1.30.0", + "grpcio==1.32.0", "protobuf==3.12.4", "six==1.15.0", "wrapt==1.12.1", @@ -27,7 +27,8 @@ long_description_content_type="text/markdown", zip_safe=False, classifiers=[ - "Development Status :: 3 - Alpha", # Chose either "3 - Alpha", "4 - Beta" or "5 - Production/Stable" as the current state of your package + # Chose either "3 - Alpha", "4 - Beta" or "5 - Production/Stable" as the current state of your package + "Development Status :: 3 - Alpha", "Intended Audience :: Developers", # Define that your audience are developers "Intended Audience :: Information Technology", "Topic :: Software Development :: Build Tools", diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 35b09f0..27fcf17 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -1,15 +1,15 @@ -import os -import pytest -import docker import logging +import os import shutil - +import subprocess +from logging.config import fileConfig from time import sleep, time -from subprocess import Popen, DEVNULL + +import docker +import pytest from git import Repo -from logging.config import fileConfig + from pygate_grpc.client import PowerGateClient -from pygate_grpc.health import HealthClient fileConfig("logging.ini") @@ -27,7 +27,7 @@ def is_docker_running(): is_running = True try: client.info() - except Exception as e: + except Exception: is_running = False finally: client.close() @@ -37,16 +37,15 @@ def is_docker_running(): def is_docker_compose_installed(): """Checks if docker composed is installed in the system""" logger.debug("Checking if docker-compose is installed...") - proc = Popen(["docker-compose", "help"], stdout=DEVNULL, stderr=DEVNULL) - streamdata = proc.communicate()[0] - return proc.returncode == 0 + res = subprocess.run(["docker-compose", "--version"]) + return res.returncode == 0 def clone_powergate_repo(): """Clones official Powergate repo """ repo_url = "https://github.com/textileio/powergate" logger.debug(f"Cloning powergate repo from {repo_url}") - repo = Repo.clone_from(repo_url, REPO_LOCAL_PATH, branch="master") + Repo.clone_from(repo_url, REPO_LOCAL_PATH, branch="master") @pytest.fixture(scope="session") @@ -83,7 +82,7 @@ def pytest_unconfigure(config): """Runs before test process exits. Cleans up any artifacts from configure""" try: shutil.rmtree(REPO_LOCAL_PATH) - except OSError as e: + except OSError: logger.warning( "Couldn't delete powergate repository. Maybe it wasn't cloned in the first place" ) @@ -94,7 +93,7 @@ def localnet(docker_services): """Starts a cli container to interact with localnet""" client = docker.from_env() container = client.containers.run( - "pygate/powergate-cli:v0.0.1-beta.13", + "pygate/powergate-cli:v0.7.0", network_mode="host", auto_remove=True, detach=True, @@ -112,7 +111,7 @@ def localnet(docker_services): result = container.exec_run("pow health") if result.exit_code > 0: continue - except docker.errors.ContainerError as e: + except docker.errors.ContainerError: continue break diff --git a/tests/integration/test_faults.py b/tests/integration/test_faults.py index 1dc648f..782a212 100644 --- a/tests/integration/test_faults.py +++ b/tests/integration/test_faults.py @@ -1,9 +1,7 @@ import logging -import grpc -import pytest -from pygate_grpc.client import PowerGateClient from proto.faults_rpc_pb2 import GetResponse, Index +from pygate_grpc.client import PowerGateClient logger = logging.getLogger(__name__) diff --git a/tests/integration/test_ffs.py b/tests/integration/test_ffs.py index 6f3707e..f8d6378 100644 --- a/tests/integration/test_ffs.py +++ b/tests/integration/test_ffs.py @@ -1,10 +1,11 @@ import logging -from pygate_grpc.exceptions import GRPCTimeoutException -import pytest import time -from proto.ffs_rpc_pb2 import CreateResponse, StageRequest, AddrInfo +import pytest + +from proto.ffs_rpc_pb2 import AddrInfo, CreateResponse, StageRequest from pygate_grpc.client import PowerGateClient +from pygate_grpc.exceptions import GRPCTimeoutException logger = logging.getLogger(__name__) @@ -102,11 +103,11 @@ def test_send_fil(pygate_client: PowerGateClient, ffs_instance: CreateResponse): assert before_receiver_fil.balance < after_receiver_fil.balance -def test_ffs_logs(pygate_client: PowerGateClient, ffs_instance): +def test_ffs_logs(pygate_client: PowerGateClient): ffs = pygate_client.ffs.create() stage_res = pygate_client.ffs.stage(chunks(), ffs.token) - push_res = pygate_client.ffs.push(stage_res.cid, ffs.token) + pygate_client.ffs.push(stage_res.cid, ffs.token) logs_res = pygate_client.ffs.logs(stage_res.cid, ffs.token, history=True, timeout=5) logs = [] @@ -119,6 +120,36 @@ def test_ffs_logs(pygate_client: PowerGateClient, ffs_instance): assert len(logs) > 0 +def test_storage_deals(pygate_client: PowerGateClient): + ffs = pygate_client.ffs.create() + + stage_res = pygate_client.ffs.stage(chunks(), ffs.token) + pygate_client.ffs.push(stage_res.cid, ffs.token) + + time.sleep(3) + + storage_deals = pygate_client.ffs.list_storage_deal_records( + include_pending=True, include_final=True, token=ffs.token + ) + + assert len(storage_deals.records) > 0 + + +def test_retrieval_deals(pygate_client: PowerGateClient): + ffs = pygate_client.ffs.create() + + stage_res = pygate_client.ffs.stage(chunks(), ffs.token) + pygate_client.ffs.push(stage_res.cid, ffs.token) + + time.sleep(3) + + retrieval_deals = pygate_client.ffs.list_retrieval_deal_records( + include_pending=True, include_final=True, token=ffs.token + ) + + assert len(retrieval_deals.records) == 0 + + def chunks(): for _ in range(1): yield StageRequest(chunk=bytes("test_content", "ASCII")) diff --git a/tests/integration/test_health.py b/tests/integration/test_health.py index 492c7ba..3e9ed23 100644 --- a/tests/integration/test_health.py +++ b/tests/integration/test_health.py @@ -1,9 +1,7 @@ import logging -import grpc -import pytest +from proto.health_rpc_pb2 import STATUS_OK, CheckResponse from pygate_grpc.client import PowerGateClient -from proto.health_rpc_pb2 import CheckResponse, STATUS_OK logger = logging.getLogger(__name__) diff --git a/tests/integration/test_net.py b/tests/integration/test_net.py index d4e7293..2631810 100644 --- a/tests/integration/test_net.py +++ b/tests/integration/test_net.py @@ -1,15 +1,7 @@ import logging +from proto.net_rpc_pb2 import FindPeerResponse, ListenAddrResponse, PeersResponse from pygate_grpc.client import PowerGateClient -from proto.net_rpc_pb2 import ( - ListenAddrResponse, - PeersResponse, - FindPeerResponse, - ConnectPeerResponse, - DisconnectPeerResponse, - ConnectPeerResponse, -) -import time logger = logging.getLogger(__name__) diff --git a/tests/integration/test_wallet.py b/tests/integration/test_wallet.py index 8b727ef..c24654b 100644 --- a/tests/integration/test_wallet.py +++ b/tests/integration/test_wallet.py @@ -1,8 +1,8 @@ import logging +import time +from proto.wallet_rpc_pb2 import BalanceResponse, ListResponse from pygate_grpc.client import PowerGateClient -from proto.wallet_rpc_pb2 import ListResponse, BalanceResponse -import time logger = logging.getLogger(__name__)