feat(api): compatibility with powergate v2 (#55)

* feat(api): compatibility with powergate v2

* fix(format): codefactor fix

* fix: bump docker container version

* fix:debug actions

* fix(test): add ipfs healthcheck

* feat(test):added resource constraints

* fix: add delay

* fix(format): codefactor fixes

Pipfile

@@ -33,6 +33,6 @@ lint = "bash -c \"python -m flake8\""
 integration-test = "python -m pytest tests/integration/"
 
 [packages]
-grpc-powergate-client = "==1.1.2"
+grpc-powergate-client = "==2.1.0"
 mypy-extensions = "*"
 deprecated = "*"

examples/config.py

@@ -33,4 +33,12 @@
 staged_file = client.data.stage_bytes(test_bytes, user.token)
 print("StagedFile: ", staged_file)
 
-stage = client.config.apply(staged_file.cid, token=user.token, config=config)
+print("Disabling from hot and cold storage...")
+config["cold"]["enabled"] = False
+config["hot"]["enabled"] = False
+stage = client.config.apply(
+    staged_file.cid, token=user.token, config=config, no_exec=False, import_deal_ids=[]
+)
+
+print("Removing...")
+client.config.remove(staged_file.cid, token=user.token)

examples/deals.py

@@ -21,12 +21,12 @@
     apply_res = client.config.apply(stage_res.cid, token=user.token)
 
     # Check that cid is in the process of being stored by Powegate
-    check = client.data.cid_info([stage_res.cid], user.token)
+    check = client.data.cid_info(stage_res.cid, user.token)
     print("Checking cid storage...")
     print(check)
 
     # Wait some time so that we can get some deals
-    time.sleep(60)
+    time.sleep(10)
 
     # Check information about the storage deal
     storage_deals = client.deals.storage_deal_records(

examples/storage_info.py

@@ -0,0 +1,33 @@
+import os
+from pathlib import Path
+from pygate_grpc.client import PowerGateClient
+
+
+if __name__ == "__main__":
+
+    hostName = "127.0.0.1:5002"
+
+    # Create client
+    c = PowerGateClient(hostName, False)
+
+    # Create user
+    user = c.admin.users.create()
+    print("User created:")
+    print(user)
+
+    # Stage file
+    print("Staging 'testfile.txt' to IPFS storage...")
+    path = Path(os.path.abspath(__file__))
+    staged_file = c.data.stage_file(path.parent / "testfile.txt", user.token)
+    print("IPFS CID: " + staged_file.cid)
+
+    # Apply the default storage config to the given file
+    print("Applying Filecoin storage config to CID...")
+    job = c.config.apply(staged_file.cid, override=False, token=user.token)
+
+    # Report back the Job ID for the successful Filecoin storage job
+    print("File successfully added to Filecoin storage.")
+    print("Job ID: " + job.jobId)
+
+    storage_infos = c.storage_info.list(cids=[staged_file.cid], token=user.token)
+    print(storage_infos)

examples/storage_jobs.py

@@ -0,0 +1,34 @@
+import os
+from pathlib import Path
+from pygate_grpc.client import PowerGateClient
+
+
+if __name__ == "__main__":
+
+    hostName = "127.0.0.1:5002"
+
+    # Create client
+    c = PowerGateClient(hostName, False)
+
+    # Create user
+    user = c.admin.users.create()
+    print("User created:")
+    print(user)
+
+    # Stage file
+    print("Staging 'testfile.txt' to IPFS storage...")
+    path = Path(os.path.abspath(__file__))
+    staged_file = c.data.stage_file(path.parent / "testfile.txt", user.token)
+    print("IPFS CID: " + staged_file.cid)
+
+    # Apply the default storage config to the given file
+    print("Applying Filecoin storage config to CID...")
+    job = c.config.apply(staged_file.cid, override=False, token=user.token)
+
+    # Report back the Job ID for the successful Filecoin storage job
+    print("File successfully added to Filecoin storage.")
+    print("Job ID: " + job.jobId)
+
+    storage_job = c.storage_jobs.storage_job(job.jobId, token=user.token)
+    storage_config = c.storage_jobs.storage_config_for_job(job.jobId, token=user.token)
+    jobs_list = c.storage_jobs.list(cid_filter=staged_file.cid, token=user.token)

examples/store_retrieve.py

@@ -1,5 +1,6 @@
 import os
 from pathlib import Path
+
 from pygate_grpc.client import PowerGateClient
 
 
@@ -54,7 +55,7 @@
     c.config.apply(staged_file.cid, override=True, config=new_config, token=user.token)
 
     # Check that CID is stored
-    check = c.data.cid_info([staged_file.cid], user.token)
+    check = c.data.cid_info(staged_file.cid, user.token)
     print("Checking CID storage...")
     print(check)
 
@@ -66,3 +67,6 @@
     with open(path.parent / "testfile_copy.txt", "wb") as f:
         f.write(file_bytes)
     print("Saved as 'testfile_copy.txt'")
+
+    cid_summary = c.data.cid_summary(cids=[staged_file.cid], token=user.token)
+    print("CID summary: ", cid_summary)

pygate_grpc/client.py

@@ -9,6 +9,7 @@
 from pygate_grpc.deals import DealsClient
 from pygate_grpc.decorators import unmarshal_with
 from pygate_grpc.errors import ErrorHandlerMeta
+from pygate_grpc.storage_info import StorageInfoClient
 from pygate_grpc.storage_jobs import StorageJobsClient
 from pygate_grpc.types import BuildInfo
 from pygate_grpc.wallet import WalletClient
@@ -34,6 +35,7 @@ def __init__(self, host_name, is_secure=False):
         self.deals = DealsClient(channel, self.get_metadata)
         self.config = ConfigClient(channel, self.get_metadata)
         self.storage_jobs = StorageJobsClient(channel, self.get_metadata)
+        self.storage_info = StorageInfoClient(channel, self.get_metadata)
         self.wallet = WalletClient(channel, self.get_metadata)
 
     def set_token(self, token: str):
@@ -61,8 +63,8 @@ def get_metadata(
         token_data: Tuple[Tuple[str, str]] = ()
         admin_token_data: Tuple[Tuple[str, str]] = ()
 
-        final_token = token if token else self.token
-        final_admin_token = admin_token if admin_token else self.admin_token
+        final_token = token or self.token
+        final_admin_token = admin_token or self.admin_token
 
         if final_token is not None:
             token_data = ((TOKEN_KEY, final_token),)

pygate_grpc/config.py

@@ -22,7 +22,7 @@ def default(self, token: str = None) -> dict:
 
     # Currently you need to pass in the user_pb2.DefaultConfig. However, this is not a good design.
     def set_default(self, config: str, token: str = None):
-        if type(config) == dict:
+        if isinstance(config, dict):
             config = json.dumps(config)
 
         config = Parse(config, user_pb2.StorageConfig())
@@ -33,9 +33,15 @@ def set_default(self, config: str, token: str = None):
 
     @unmarshal_with(Job)
     def apply(
-        self, cid, token: str = None, override: bool = False, config: str = None,
+        self,
+        cid,
+        token: str = None,
+        override: bool = False,
+        config: str = None,
+        import_deal_ids=[],
+        no_exec=False,
     ) -> Job:
-        if type(config) == dict:
+        if isinstance(config, dict):
             config = json.dumps(config)
 
         if config:
@@ -47,6 +53,8 @@ def apply(
             has_override_config=override,
             config=config,
             has_config=config is not None,
+            import_deal_ids=import_deal_ids,
+            no_exec=no_exec,
         )
         return self.client.ApplyStorageConfig(req, metadata=self.get_metadata(token))
 

pygate_grpc/data.py

@@ -6,7 +6,7 @@
 
 from pygate_grpc.decorators import unmarshal_with
 from pygate_grpc.errors import ErrorHandlerMeta, future_error_handler
-from pygate_grpc.types import CidInfo, StagedFile
+from pygate_grpc.types import CidInfo, CidSummary, StagedFile
 
 CHUNK_SIZE = 1024 * 1024  # 1MB
 
@@ -101,7 +101,14 @@ def watch_logs(
             req, metadata=self.get_metadata(token), timeout=timeout
         )
 
-    @unmarshal_with(CidInfo, many=True)
-    def cid_info(self, cids: List[str], token: str = None) -> List[CidInfo]:
-        req = user_pb2.CidInfoRequest(cids=cids)
-        return self.client.CidInfo(req, metadata=self.get_metadata(token)).cid_infos
+    @unmarshal_with(CidInfo)
+    def cid_info(self, cid: str, token: str = None) -> CidInfo:
+        req = user_pb2.CidInfoRequest(cid=cid)
+        return self.client.CidInfo(req, metadata=self.get_metadata(token)).cid_info
+
+    @unmarshal_with(CidSummary, many=True)
+    def cid_summary(self, cids: List[str], token: str = None) -> CidSummary:
+        req = user_pb2.CidSummaryRequest(cids=cids)
+        return self.client.CidSummary(
+            req, metadata=self.get_metadata(token)
+        ).cid_summary

pygate_grpc/decorators.py

@@ -15,18 +15,16 @@ def wrap(original_func):
         def wrapper(*args, **kwargs):
             if cls is None:
                 if many:
-
                     return [
                         MessageToDict(obj) for obj in original_func(*args, **kwargs)
                     ]
                 return MessageToDict(original_func(*args, **kwargs))
-            else:
-                if many:
-                    return [
-                        cls(**MessageToDict(obj))
-                        for obj in original_func(*args, **kwargs)
-                    ]
-                return cls(**MessageToDict(original_func(*args, **kwargs)))
+            # cls is not None
+            if many:
+                return [
+                    cls(**MessageToDict(obj)) for obj in original_func(*args, **kwargs)
+                ]
+            return cls(**MessageToDict(original_func(*args, **kwargs)))
 
         return wrapper
 

pygate_grpc/storage_info.py

@@ -0,0 +1,27 @@
+from typing import List
+
+from powergate.user.v1 import user_pb2, user_pb2_grpc
+
+from pygate_grpc.decorators import unmarshal_with
+from pygate_grpc.errors import ErrorHandlerMeta
+from pygate_grpc.types import StorageInfo
+
+
+class StorageInfoClient(object, metaclass=ErrorHandlerMeta):
+    def __init__(self, channel, get_metadata):
+        self.client = user_pb2_grpc.UserServiceStub(channel)
+        self.get_metadata = get_metadata
+
+    @unmarshal_with(StorageInfo)
+    def get(self, cid: str, token: str = None) -> StorageInfo:
+        req = user_pb2.StorageInfoRequest(cid=cid)
+        return self.client.StorageInfo(
+            req, metadata=self.get_metadata(token)
+        ).storage_info
+
+    @unmarshal_with(StorageInfo, many=True)
+    def list(self, cids: List[str], token: str = None) -> List[StorageInfo]:
+        req = user_pb2.ListStorageInfoRequest(cids=cids)
+        return self.client.ListStorageInfo(
+            req, metadata=self.get_metadata(token)
+        ).storage_info

pygate_grpc/storage_jobs.py

@@ -18,9 +18,21 @@ def storage_config_for_job(self, job_id: str, token: str = None):
         req = user_pb2.StorageConfigForJobRequest(job_id=job_id)
         return self.client.StorageConfigForJob(req, metadata=self.get_metadata(token))
 
-    def queued(self, cids: List[str], token: str = None):
-        req = user_pb2.QueuedStorageJobsRequest(cids=cids)
-        return self.client.QueuedStorageJobs(req, metadata=self.get_metadata(token))
+    def list(
+        self,
+        cid_filter: str,
+        limit: int = None,
+        ascending: bool = False,
+        next_page_token=None,
+        token: str = None,
+    ):
+        req = user_pb2.ListStorageJobsRequest(
+            cid_filter=cid_filter,
+            limit=limit,
+            ascending=ascending,
+            next_page_token=next_page_token,
+        )
+        return self.client.ListStorageJobs(req, metadata=self.get_metadata(token))
 
     def executing(self, cids: List[str], token: str = None):
         req = user_pb2.ExecutingStorageJobsRequest(cids=cids)

pygate_grpc/types.py

@@ -1,9 +1,13 @@
 from collections import namedtuple
 
 User = namedtuple("User", ["id", "token"])
+
 StagedFile = namedtuple("StagedFile", ["cid"])
+
 Address = namedtuple("Address", ["name", "address", "type", "balance"])
+
 Job = namedtuple("Job", ["jobId"])
+
 CidInfo = namedtuple(
     "CidInfo",
     [
@@ -12,11 +16,16 @@
         "executingStorageJob",
         "queuedStorageJobs",
         "currentStorageInfo",
-        "latestFinalStorageJob",
-        "latestSuccessfulStorageJob",
     ],
-    defaults=(None,) * 7,
+    defaults=(None,) * 5,
 )
+
+CidSummary = namedtuple(
+    "CidSummary", ["cid", "stored", "queuedJobs", "executingJob"], defaults=(None,) * 4,
+)
+
+StorageInfo = namedtuple("StorageInfo", ["job_id", "cid", "created", "hot", "cold"])
+
 BuildInfo = namedtuple(
     "BuildInfo",
     ["gitCommit", "gitBranch", "gitState", "gitSummary", "buildDate", "version"],

setup.py

@@ -1,19 +1,19 @@
 # read the contents of your README file
 from os import path
 
-from setuptools import setup,find_packages
+from setuptools import find_packages, setup
 
 this_directory = path.abspath(path.dirname(__file__))
 with open(path.join(this_directory, "README.md"), encoding="utf-8") as f:
     long_description = f.read()
 
 setup(
     install_requires=[
-        "deprecated==1.2.10",
-        "grpc-powergate-client==1.1.2",
-        "grpcio==1.33.2",
+        "deprecated==1.2.11",
+        "grpc-powergate-client==2.1.0",
+        "grpcio==1.35.0",
         "mypy-extensions==0.4.3",
-        "protobuf==3.14.0",
+        "protobuf==3.15.1",
         "six==1.15.0",
         "wrapt==1.12.1",
     ],