From 3fc4fd9e4282fff584bece64bd98ba5c6e13ee5f Mon Sep 17 00:00:00 2001 From: dapineyro Date: Wed, 15 Jan 2025 15:34:11 +0100 Subject: [PATCH 1/6] updates get_workflow_list --- cloudos/clos.py | 53 +++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 47 insertions(+), 6 deletions(-) diff --git a/cloudos/clos.py b/cloudos/clos.py index bb768e2..402a59f 100644 --- a/cloudos/clos.py +++ b/cloudos/clos.py @@ -388,17 +388,30 @@ def get_curated_workflow_list(self, workspace_id, get_all=True, page=1, verify=T else: return content['pipelines'] - def get_workflow_list(self, workspace_id, verify=True): + def get_workflow_list(self, workspace_id, verify=True, get_all=True, + page=1, page_size=10, max_page_size=1000, + archived_status=False): """Get all the workflows from a CloudOS workspace. Parameters ---------- workspace_id : string The CloudOS workspace id from to collect the workflows. - verify: [bool|string] + verify : [bool|string] Whether to use SSL verification or not. Alternatively, if a string is passed, it will be interpreted as the path to the SSL certificate file. + get_all : bool + Whether to get all available curated workflows or just the + indicated page. + page : int + The page number to retrieve, from the paginated response. + page_size : int + The number of workflows by page. From 1 to 1000. + max_page_size : int + Max page size defined by the API server. It is currently 1000. + archived_status : bool + Whether to retrieve archived workflows or not. Returns ------- @@ -409,12 +422,40 @@ def get_workflow_list(self, workspace_id, verify=True): "Content-type": "application/json", "apikey": self.apikey } - r = retry_requests_get("{}/api/v1/workflows?teamId={}".format(self.cloudos_url, - workspace_id), - headers=headers, verify=verify) + r = retry_requests_get( + "{}/api/v3/workflows?teamId={}&pageSize={}&page={}&archied.status={}".format( + self.cloudos_url, workspace_id, page_size, page, archived_status), + headers=headers, verify=verify) if r.status_code >= 400: raise BadRequestException(r) - return json.loads(r.content) + content = json.loads(r.content) + if get_all: + total_workflows = content['paginationMetadata']['Pagination-Count'] + if total_workflows <= max_page_size: + r = retry_requests_get( + "{}/api/v3/workflows?teamId={}&pageSize={}&page={}&archied.status={}".format( + self.cloudos_url, workspace_id, total_workflows, 1, archived_status), + headers=headers, verify=verify) + if r.status_code >= 400: + raise BadRequestException(r) + return json.loads(r.content['workflows']) + else: + n_pages = (total_workflows // max_page_size) + int((total_workflows % max_page_size) > 0) + for p in range(n_pages): + p += 1 + r = retry_requests_get( + "{}/api/v3/workflows?teamId={}&pageSize={}&page={}&archied.status={}".format( + self.cloudos_url, workspace_id, max_page_size, p, archived_status), + headers=headers, verify=verify) + if r.status_code >= 400: + raise BadRequestException(r) + if p == 1: + all_content = json.loads(r.content['workflows']) + else: + all_content += json.loads(r.content['workflows']) + return all_content + else: + return content['workflows'] @staticmethod def process_workflow_list(r, all_fields=False): From ceddd176b40ac57339ed03b2d60d0770e916bc9d Mon Sep 17 00:00:00 2001 From: dapineyro Date: Wed, 15 Jan 2025 18:10:46 +0100 Subject: [PATCH 2/6] updates --- cloudos/_version.py | 2 +- cloudos/clos.py | 79 +++++++++++++++++++-------------------------- cloudos/jobs/job.py | 13 ++------ 3 files changed, 36 insertions(+), 58 deletions(-) diff --git a/cloudos/_version.py b/cloudos/_version.py index 6684d2b..754dd42 100644 --- a/cloudos/_version.py +++ b/cloudos/_version.py @@ -1 +1 @@ -__version__ = '2.14.0' +__version__ = '2.15.0' diff --git a/cloudos/clos.py b/cloudos/clos.py index 402a59f..fbd3f6a 100644 --- a/cloudos/clos.py +++ b/cloudos/clos.py @@ -340,53 +340,30 @@ def get_curated_workflow_list(self, workspace_id, get_all=True, page=1, verify=T r : list A list of dicts, each corresponding to a workflow. """ - data = {"search": "", - "page": page, - "filters": [ - [ - { - "isPredefined": True, - "isCurated": True, - "isFeatured": False, - "isModule": False - }, - { - "isPredefined": True, - "isCurated": False, - "isFeatured": False, - "isModule": False - }, - { - "isPredefined": True, - "isCurated": True, - "isFeatured": True, - "isModule": False - } - ] - ] - } headers = { "Content-type": "application/json", "apikey": self.apikey } - r = retry_requests_post("{}/api/v1/workflows/getByType?teamId={}".format(self.cloudos_url, - workspace_id), - json=data, headers=headers, verify=verify) + r = retry_requests_get( + "{}/api/v3/workflows?search=&groups[]=curated&page={}&teamId={}".format( + self.cloudos_url, page, workspace_id), + headers=headers, verify=verify) + print(page) if r.status_code >= 400: raise BadRequestException(r) content = json.loads(r.content) if get_all: - workflows_collected = len(content['pipelines']) - workflows_to_get = content['total'] + workflows_collected = len(content['workflows']) + workflows_to_get = content['paginationMetadata']['Pagination-Count'] if workflows_to_get <= workflows_collected or workflows_collected == 0: - return content['pipelines'] + return content['workflows'] if workflows_to_get > workflows_collected: - return content['pipelines'] + self.get_curated_workflow_list(workspace_id, + return content['workflows'] + self.get_curated_workflow_list(workspace_id, get_all=True, page=page+1, verify=verify) else: - return content['pipelines'] + return content['workflows'] def get_workflow_list(self, workspace_id, verify=True, get_all=True, page=1, page_size=10, max_page_size=1000, @@ -422,8 +399,9 @@ def get_workflow_list(self, workspace_id, verify=True, get_all=True, "Content-type": "application/json", "apikey": self.apikey } + archived_status = str(archived_status).lower() r = retry_requests_get( - "{}/api/v3/workflows?teamId={}&pageSize={}&page={}&archied.status={}".format( + "{}/api/v3/workflows?teamId={}&pageSize={}&page={}&archived.status={}".format( self.cloudos_url, workspace_id, page_size, page, archived_status), headers=headers, verify=verify) if r.status_code >= 400: @@ -433,26 +411,26 @@ def get_workflow_list(self, workspace_id, verify=True, get_all=True, total_workflows = content['paginationMetadata']['Pagination-Count'] if total_workflows <= max_page_size: r = retry_requests_get( - "{}/api/v3/workflows?teamId={}&pageSize={}&page={}&archied.status={}".format( + "{}/api/v3/workflows?teamId={}&pageSize={}&page={}&archived.status={}".format( self.cloudos_url, workspace_id, total_workflows, 1, archived_status), headers=headers, verify=verify) if r.status_code >= 400: raise BadRequestException(r) - return json.loads(r.content['workflows']) + return json.loads(r.content)['workflows'] else: n_pages = (total_workflows // max_page_size) + int((total_workflows % max_page_size) > 0) for p in range(n_pages): p += 1 r = retry_requests_get( - "{}/api/v3/workflows?teamId={}&pageSize={}&page={}&archied.status={}".format( + "{}/api/v3/workflows?teamId={}&pageSize={}&page={}&archived.status={}".format( self.cloudos_url, workspace_id, max_page_size, p, archived_status), headers=headers, verify=verify) if r.status_code >= 400: raise BadRequestException(r) if p == 1: - all_content = json.loads(r.content['workflows']) + all_content = json.loads(r.content)['workflows'] else: - all_content += json.loads(r.content['workflows']) + all_content += json.loads(r.content)['workflows'] return all_content else: return content['workflows'] @@ -476,11 +454,10 @@ def process_workflow_list(r, all_fields=False): """ COLUMNS = ['_id', 'name', - 'isModule', 'archived.status', 'mainFile', 'workflowType', - 'parameters', + 'group', 'repository.name', 'repository.platform', 'repository.url', @@ -552,14 +529,24 @@ def is_module(self, workflow_name, workspace_id, verify=True): """ my_workflows_r = self.get_workflow_list(workspace_id, verify=verify) my_workflows = self.process_workflow_list(my_workflows_r) - is_module = my_workflows.loc[ + group = my_workflows.loc[ (my_workflows['name'] == workflow_name) & (my_workflows['archived.status'] == False), - 'isModule'] - if len(is_module) == 0: + 'group'] + if len(group) == 0: raise ValueError(f'No workflow found with name: {workflow_name}') - if len(is_module) > 1: + if len(group) > 1: raise ValueError(f'More than one workflow found with name: {workflow_name}') - return is_module.values[0] + module_groups = ['system-tools', + 'data-factory-data-connection-etl', + 'data-factory', + 'data-factory-omics-etl', + 'drug-discovery', + 'data-factory-omics-insights' + ] + if group.values[0] in module_groups: + return True + else: + return False def get_project_list(self, workspace_id, verify=True): """Get all the project from a CloudOS workspace. diff --git a/cloudos/jobs/job.py b/cloudos/jobs/job.py index 54e70a8..668c3bb 100644 --- a/cloudos/jobs/job.py +++ b/cloudos/jobs/job.py @@ -143,18 +143,8 @@ def fetch_cloudos_id(self, if resource not in allowed_resources: raise ValueError('Your specified resource is not supported. ' + f'Use one of the following: {allowed_resources}') - headers = { - "Content-type": "application/json", - "apikey": apikey - } - r = retry_requests_get("{}/api/v1/{}?teamId={}".format(cloudos_url, - resource, - workspace_id), - headers=headers, verify=verify) - if r.status_code >= 400: - raise BadRequestException(r) - content = json.loads(r.content) if resource == 'workflows': + content = self.get_workflow_list(workflow_id, verify=verify) for element in content: if (element["name"] == name and element["repository"]["platform"] == repository_platform and @@ -167,6 +157,7 @@ def fetch_cloudos_id(self, elif "importsFile" in element.keys() and element["importsFile"] == importsfile: return element["_id"] elif resource == 'projects': + content = self.get_project_list(workspace_id, verify=verify) # New API projects endpoint spec if type(content) is dict: for element in content["projects"]: From d3173fd8729e970ad156403772c001da4e90da80 Mon Sep 17 00:00:00 2001 From: dapineyro Date: Wed, 15 Jan 2025 21:01:13 +0100 Subject: [PATCH 3/6] fix pytest --- cloudos/jobs/job.py | 5 ++-- tests/test_clos/test_detect_workflow.py | 12 +++++++--- .../test_get_curated_workflow_list.py | 23 +++++++++++++----- tests/test_clos/test_get_workflow_list.py | 21 +++++++++++----- tests/test_clos/test_is_module.py | 12 +++++++--- tests/test_clos/test_process_workflow_list.py | 2 +- ...process_workflow_list_initial_request.json | 24 ++++++------------- .../process_workflow_list_results.csv | 2 +- .../process_workflow_list_results_FULL.csv | 8 +++---- tests/test_data/workflows.json | 12 ++++------ .../workflows/curated_workflows.json | 2 +- tests/test_data/workflows/workflows.json | 2 +- tests/test_jobs/test_project_id.py | 21 +++++++++++----- tests/test_jobs/test_send_job.py | 13 ++++++++-- tests/test_jobs/test_workflow_id.py | 13 ++++++++-- 15 files changed, 110 insertions(+), 62 deletions(-) diff --git a/cloudos/jobs/job.py b/cloudos/jobs/job.py index 668c3bb..cddff04 100644 --- a/cloudos/jobs/job.py +++ b/cloudos/jobs/job.py @@ -144,7 +144,7 @@ def fetch_cloudos_id(self, raise ValueError('Your specified resource is not supported. ' + f'Use one of the following: {allowed_resources}') if resource == 'workflows': - content = self.get_workflow_list(workflow_id, verify=verify) + content = self.get_workflow_list(workspace_id, verify=verify) for element in content: if (element["name"] == name and element["repository"]["platform"] == repository_platform and @@ -157,7 +157,8 @@ def fetch_cloudos_id(self, elif "importsFile" in element.keys() and element["importsFile"] == importsfile: return element["_id"] elif resource == 'projects': - content = self.get_project_list(workspace_id, verify=verify) + r = self.get_project_list(workspace_id, verify=verify) + content = json.loads(r.content) # New API projects endpoint spec if type(content) is dict: for element in content["projects"]: diff --git a/tests/test_clos/test_detect_workflow.py b/tests/test_clos/test_detect_workflow.py index bda50a7..8a92f74 100644 --- a/tests/test_clos/test_detect_workflow.py +++ b/tests/test_clos/test_detect_workflow.py @@ -9,6 +9,9 @@ APIKEY = 'vnoiweur89u2ongs' CLOUDOS_URL = 'http://cloudos.lifebit.ai' WORKSPACE_ID = 'lv89ufc838sdig' +PAGE_SIZE = 10 +PAGE = 1 +ARCHIVED_STATUS = "false" @mock.patch('cloudos.clos', mock.MagicMock()) @@ -19,17 +22,20 @@ def test_detect_workflow(): API request is mocked and replicated with json files """ json_data = load_json_file(INPUT) - params = {"teamId": WORKSPACE_ID} + params = {"teamId": WORKSPACE_ID, + "pageSize": PAGE_SIZE, + "page": PAGE, + "archived.status": ARCHIVED_STATUS} header = { "Accept": "application/json, text/plain, */*", "Content-Type": "application/json;charset=UTF-8", "apikey": APIKEY } - search_str = f"teamId={WORKSPACE_ID}" + search_str = f"teamId={WORKSPACE_ID}&pageSize={PAGE_SIZE}&page={PAGE}&archived.status={ARCHIVED_STATUS}" # mock GET method with the .json responses.add( responses.GET, - url=f"{CLOUDOS_URL}/api/v1/workflows?{search_str}", + url=f"{CLOUDOS_URL}/api/v3/workflows?{search_str}", body=json_data, headers=header, match=[matchers.query_param_matcher(params)], diff --git a/tests/test_clos/test_get_curated_workflow_list.py b/tests/test_clos/test_get_curated_workflow_list.py index b7bf245..7185f97 100644 --- a/tests/test_clos/test_get_curated_workflow_list.py +++ b/tests/test_clos/test_get_curated_workflow_list.py @@ -2,6 +2,7 @@ import json import pytest import responses +from responses import matchers from cloudos.clos import Cloudos from cloudos.utils.errors import BadRequestException from tests.functions_for_pytest import load_json_file @@ -10,6 +11,7 @@ APIKEY = 'vnoiweur89u2ongs' CLOUDOS_URL = 'http://cloudos.lifebit.ai' WORKSPACE_ID = 'lv89ufc838sdig' +PAGE = 1 @mock.patch('cloudos.clos', mock.MagicMock()) @@ -20,12 +22,21 @@ def test_get_curated_workflow_list_correct_response(): API request is mocked and replicated with json files """ create_json = load_json_file(OUTPUT) - search_str = f"teamId={WORKSPACE_ID}" + params = {"teamId": WORKSPACE_ID, + "groups[]": "curated", + "page": PAGE} + header = { + "Accept": "application/json, text/plain, */*", + "Content-Type": "application/json;charset=UTF-8", + "apikey": APIKEY} + search_str = f"search=&groups[]=curated&page={PAGE}&teamId={WORKSPACE_ID}" # mock GET method with the .json responses.add( - responses.POST, - url=f"{CLOUDOS_URL}/api/v1/workflows/getByType?{search_str}", + responses.GET, + url=f"{CLOUDOS_URL}/api/v3/workflows?{search_str}", body=create_json, + headers=header, + match=[matchers.query_param_matcher(params)], status=200) # start cloudOS service clos = Cloudos(apikey=APIKEY, cromwell_token=None, cloudos_url=CLOUDOS_URL) @@ -46,11 +57,11 @@ def test_get_curated_workflow_list_incorrect_response(): error_message = {"statusCode": 400, "code": "BadRequest", "message": "Bad Request.", "time": "2022-11-23_17:31:07"} error_json = json.dumps(error_message) - search_str = f"teamId={WORKSPACE_ID}" + search_str = f"search=&groups[]=curated&page={PAGE}&teamId={WORKSPACE_ID}" # mock GET method with the .json responses.add( - responses.POST, - url=f"{CLOUDOS_URL}/api/v1/workflows/getByType?{search_str}", + responses.GET, + url=f"{CLOUDOS_URL}/api/v3/workflows?{search_str}", body=error_json, status=400) # raise 400 error diff --git a/tests/test_clos/test_get_workflow_list.py b/tests/test_clos/test_get_workflow_list.py index 5b7e29d..e72be3d 100644 --- a/tests/test_clos/test_get_workflow_list.py +++ b/tests/test_clos/test_get_workflow_list.py @@ -11,6 +11,9 @@ APIKEY = 'vnoiweur89u2ongs' CLOUDOS_URL = 'http://cloudos.lifebit.ai' WORKSPACE_ID = 'lv89ufc838sdig' +PAGE_SIZE = 10 +PAGE = 1 +ARCHIVED_STATUS = "false" @mock.patch('cloudos.clos', mock.MagicMock()) @@ -21,17 +24,20 @@ def test_get_workflow_list_correct_response(): API request is mocked and replicated with json files """ create_json = load_json_file(INPUT) - params = {"teamId": WORKSPACE_ID} + params = {"teamId": WORKSPACE_ID, + "pageSize": PAGE_SIZE, + "page": PAGE, + "archived.status": ARCHIVED_STATUS} header = { "Accept": "application/json, text/plain, */*", "Content-Type": "application/json;charset=UTF-8", "apikey": APIKEY } - search_str = f"teamId={WORKSPACE_ID}" + search_str = f"teamId={WORKSPACE_ID}&pageSize={PAGE_SIZE}&page={PAGE}&archived.status={ARCHIVED_STATUS}" # mock GET method with the .json responses.add( responses.GET, - url=f"{CLOUDOS_URL}/api/v1/workflows?{search_str}", + url=f"{CLOUDOS_URL}/api/v3/workflows?{search_str}", body=create_json, headers=header, match=[matchers.query_param_matcher(params)], @@ -55,17 +61,20 @@ def test_get_workflow_list_incorrect_response(): error_message = {"statusCode": 400, "code": "BadRequest", "message": "Bad Request.", "time": "2022-11-23_17:31:07"} error_json = json.dumps(error_message) - params = {"teamId": WORKSPACE_ID} + params = {"teamId": WORKSPACE_ID, + "pageSize": PAGE_SIZE, + "page": PAGE, + "archived.status": ARCHIVED_STATUS} header = { "Accept": "application/json, text/plain, */*", "Content-Type": "application/json;charset=UTF-8", "apikey": APIKEY } - search_str = f"teamId={WORKSPACE_ID}" + search_str = f"teamId={WORKSPACE_ID}&pageSize={PAGE_SIZE}&page={PAGE}&archived.status={ARCHIVED_STATUS}" # mock GET method with the .json responses.add( responses.GET, - url=f"{CLOUDOS_URL}/api/v1/workflows?{search_str}", + url=f"{CLOUDOS_URL}/api/v3/workflows?{search_str}", body=error_json, headers=header, match=[matchers.query_param_matcher(params)], diff --git a/tests/test_clos/test_is_module.py b/tests/test_clos/test_is_module.py index 6a2148d..847c1b4 100644 --- a/tests/test_clos/test_is_module.py +++ b/tests/test_clos/test_is_module.py @@ -9,6 +9,9 @@ APIKEY = 'vnoiweur89u2ongs' CLOUDOS_URL = 'http://cloudos.lifebit.ai' WORKSPACE_ID = 'lv89ufc838sdig' +PAGE_SIZE = 10 +PAGE = 1 +ARCHIVED_STATUS = "false" @mock.patch('cloudos.clos', mock.MagicMock()) @@ -19,17 +22,20 @@ def test_is_module(): API request is mocked and replicated with json files """ json_data = load_json_file(INPUT) - params = {"teamId": WORKSPACE_ID} + params = {"teamId": WORKSPACE_ID, + "pageSize": PAGE_SIZE, + "page": PAGE, + "archived.status": ARCHIVED_STATUS} header = { "Accept": "application/json, text/plain, */*", "Content-Type": "application/json;charset=UTF-8", "apikey": APIKEY } - search_str = f"teamId={WORKSPACE_ID}" + search_str = f"teamId={WORKSPACE_ID}&pageSize={PAGE_SIZE}&page={PAGE}&archived.status={ARCHIVED_STATUS}" # mock GET method with the .json responses.add( responses.GET, - url=f"{CLOUDOS_URL}/api/v1/workflows?{search_str}", + url=f"{CLOUDOS_URL}/api/v3/workflows?{search_str}", body=json_data, headers=header, match=[matchers.query_param_matcher(params)], diff --git a/tests/test_clos/test_process_workflow_list.py b/tests/test_clos/test_process_workflow_list.py index e998641..1a0c8f6 100644 --- a/tests/test_clos/test_process_workflow_list.py +++ b/tests/test_clos/test_process_workflow_list.py @@ -21,7 +21,7 @@ def fixture_mocked_requests_get(): mock.get(f"http://test_cloud_os/api/v1/jobs?teamId={test_workspace_id}", json=data_d) r_get = requests.get(f"http://test_cloud_os/api/v1/jobs?teamId={test_workspace_id}") - return json.loads(r_get.content) + return json.loads(r_get.content)['workflows'] def test_process_workflow_list_all_fields_false(mocked_requests_get): diff --git a/tests/test_data/process_workflow_list_initial_request.json b/tests/test_data/process_workflow_list_initial_request.json index ea2bc61..3ba9c7b 100644 --- a/tests/test_data/process_workflow_list_initial_request.json +++ b/tests/test_data/process_workflow_list_initial_request.json @@ -1,4 +1,4 @@ -[ +{"workflows": [ { "_id": "XXX", "owner": { @@ -77,11 +77,7 @@ "dockerRegistriesCredentials": [] }, "name": "picard", - "isPredefined": false, - "isCurated": false, - "isFeatured": false, - "isModule": false, - "isPublic": false, + "group": "user", "priceAmount": 0, "priceUnit": "PER_SAMPLE", "tags": [], @@ -177,11 +173,7 @@ }, "name": "nf-core-deepvariant", "defaultContainer": null, - "isPredefined": false, - "isCurated": false, - "isFeatured": false, - "isModule": false, - "isPublic": false, + "group": "user", "priceAmount": 0, "priceUnit": "PER_SAMPLE", "tags": [], @@ -289,11 +281,7 @@ "dockerRegistriesCredentials": [] }, "name": "multiqc", - "isPredefined": false, - "isCurated": false, - "isFeatured": false, - "isModule": true, - "isPublic": false, + "group": "system-tools", "priceAmount": 0, "priceUnit": "PER_SAMPLE", "tags": [], @@ -309,4 +297,6 @@ "workflowType": "docker", "containerName": "ewels/multiqc" } -] +], +"paginationMetadata": {"Pagination-Count": 10} +} diff --git a/tests/test_data/process_workflow_list_results.csv b/tests/test_data/process_workflow_list_results.csv index 0c83ea5..c71b80c 100644 --- a/tests/test_data/process_workflow_list_results.csv +++ b/tests/test_data/process_workflow_list_results.csv @@ -1,4 +1,4 @@ -_id,name,isModule,archived.status,mainFile,workflowType,repository.name,repository.platform,repository.url,repository.isPrivate +_id,name,group,archived.status,mainFile,workflowType,repository.name,repository.platform,repository.url,repository.isPrivate XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX diff --git a/tests/test_data/process_workflow_list_results_FULL.csv b/tests/test_data/process_workflow_list_results_FULL.csv index 109cca8..d009ce0 100644 --- a/tests/test_data/process_workflow_list_results_FULL.csv +++ b/tests/test_data/process_workflow_list_results_FULL.csv @@ -1,4 +1,4 @@ -_id,name,isPredefined,isCurated,isFeatured,isModule,isPublic,priceAmount,priceUnit,tags,overview,createdAt,updatedAt,imageUrl,parametersInfo,workflowType,containerName,defaultCommand,owner.id,owner.name,owner.surname,owner.email,owner.picture,owner.completedMilestones.hasDoneTutorial,owner.completedMilestones.hasSeenWelcomeMessage,owner.isPredefined,owner.isSuspended,owner.favoritedPublicBuckets,owner.favoritedPublicAzureContainers,owner.adGroups,owner.isApproved,owner.isReviewed,owner.isVerified,owner.isAllowedToCreateWorkspace,owner.totalCreditsInCents,owner.dockerRegistriesCredentials,archived.status,archived.archivalTimestamp,defaultContainer,mainFile,processes,repository.owner.login,repository.owner.id,repository.platform,repository.repositoryId,repository.name,repository.isPrivate,repository.url,repository.commit,repository.branch -XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX -XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX -XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX +_id,name,group,priceAmount,priceUnit,tags,overview,createdAt,updatedAt,imageUrl,parametersInfo,workflowType,containerName,defaultCommand,owner.id,owner.name,owner.surname,owner.email,owner.picture,owner.completedMilestones.hasDoneTutorial,owner.completedMilestones.hasSeenWelcomeMessage,owner.isPredefined,owner.isSuspended,owner.favoritedPublicBuckets,owner.favoritedPublicAzureContainers,owner.adGroups,owner.isApproved,owner.isReviewed,owner.isVerified,owner.isAllowedToCreateWorkspace,owner.totalCreditsInCents,owner.dockerRegistriesCredentials,archived.status,archived.archivalTimestamp,defaultContainer,mainFile,processes,repository.owner.login,repository.owner.id,repository.platform,repository.repositoryId,repository.name,repository.isPrivate,repository.url,repository.commit,repository.branch +XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX +XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX +XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX,XXX diff --git a/tests/test_data/workflows.json b/tests/test_data/workflows.json index 1d485d4..5625782 100644 --- a/tests/test_data/workflows.json +++ b/tests/test_data/workflows.json @@ -1,4 +1,4 @@ -[ +{"workflows": [ { "_id": "111XXX111", "owner": { @@ -78,11 +78,7 @@ }, "name": "nf-core-deepvariant", "defaultContainer": null, - "isPredefined": false, - "isCurated": false, - "isFeatured": false, - "isModule": false, - "isPublic": false, + "group": "user", "priceAmount": 0, "priceUnit": "PER_SAMPLE", "tags": [], @@ -112,4 +108,6 @@ "mainFile": "main.nf", "processes": [] } -] \ No newline at end of file +], +"paginationMetadata": {"Pagination-Count": 10} +} diff --git a/tests/test_data/workflows/curated_workflows.json b/tests/test_data/workflows/curated_workflows.json index 83caa38..ba06229 100644 --- a/tests/test_data/workflows/curated_workflows.json +++ b/tests/test_data/workflows/curated_workflows.json @@ -1 +1 @@ -{"pipelines":[{"_id":"xxxxxxxxxxxxxx","owner":{"id":"xxxxxxxxxxxxxxxxxxxxxxx"},"name":"test","instanceType":"c3.xlarge","docsLink":"https://xxx","description":"test","category":"none","isPredefined":true,"isCurated":true,"isFeatured":false,"isModule":false,"isPublic":false,"priceAmount":0,"priceUnit":"PER_SAMPLE","tags":[],"overview":"","createdAt":"2023-04-18T16:10:42.534Z","updatedAt":"2023-04-18T16:10:44.791Z","imageUrl":null,"parametersInfo":[],"parametersProvider":"s3","archived":{"status":false,"archivalTimestamp":null},"workflowType":"docker","containerName":"lifebitai/test","defaultCommand":"test","parameters":[{"dataItem":{"kind":"File","item":{"parent":{"kind":"Workflow","id":"c2bd1dc7bec3a73a95ba678f"}}}}]}],"total":1,"categories":[]} +{"workflows":[{"_id":"xxxxxxxxxxxxxx","owner":{"id":"xxxxxxxxxxxxxxxxxxxxxxx"},"name":"test","instanceType":"c3.xlarge","docsLink":"https://xxx","description":"test","category":"none","group":"curated","priceAmount":0,"priceUnit":"PER_SAMPLE","tags":[],"overview":"","createdAt":"2023-04-18T16:10:42.534Z","updatedAt":"2023-04-18T16:10:44.791Z","imageUrl":null,"parametersInfo":[],"parametersProvider":"s3","archived":{"status":false,"archivalTimestamp":null},"workflowType":"docker","containerName":"lifebitai/test","defaultCommand":"test","parameters":[{"dataItem":{"kind":"File","item":{"parent":{"kind":"Workflow","id":"c2bd1dc7bec3a73a95ba678f"}}}}]}],"total":1,"categories":[], "paginationMetadata": {"Pagination-Count": 1}} diff --git a/tests/test_data/workflows/workflows.json b/tests/test_data/workflows/workflows.json index 48c9d28..663b7da 100644 --- a/tests/test_data/workflows/workflows.json +++ b/tests/test_data/workflows/workflows.json @@ -1 +1 @@ -[{"_id": "5c86a754b37c6700b253042e", "owner": {"id": "5b59e0cf7cbd27320976585f", "name": "Test", "surname": "Testy", "email": "test@test.ml.ai", "picture": "", "completedMilestones": {"hasDoneTutorial": true, "hasSeenWelcomeMessage": true}, "isPredefined": false, "isSuspended": false, "favoritedPublicBuckets": [{"_id": "5e9f271e5d1cd501034447ff", "name": "lifebit-sars-cov-2"}, {"_id": "5e9c5af25d1cd5010340f2bc", "name": "lifebit-sars-cov-2"}, {"_id": "5e9c5af25d1cd5010340f2bb", "name": "lifebit-featured-datasets"}, {"_id": "5e9c5af25d1cd5010340f2ba", "name": "my-first-deploit-bucket"}, {"_id": "5e9c5af25d1cd5010340f2b9", "name": "ngi-igenomes"}, {"_id": "5e9c5af25d1cd5010340f2b8", "name": "lifebit-public"}, {"_id": "5e9c5af25d1cd5010340f2b7", "name": "lifebit-featured-datasets"}, {"_id": "5e9c5af25d1cd5010340f2b6", "name": "lifebit-featured-datasets"}, {"_id": "5e9c5af25d1cd5010340f2b5", "name": "lifebit-featured-datasets"}, {"_id": "5e9c5af25d1cd5010340f2b4", "name": "rosalind-test-data"}, {"_id": "5e9c5af25d1cd5010340f2b3", "name": "lifebit-featured-datasets"}, {"_id": "5e9c5af25d1cd5010340f2b2", "name": "lifebit-featured-datasets"}, {"_id": "5e9c5af25d1cd5010340f2b1", "name": "deepvariant-data"}], "favoritedPublicAzureContainers": [], "adGroups": [], "isApproved": true, "isReviewed": true, "isVerified": true, "isAllowedToCreateWorkspace": true, "totalCreditsInCents": 0, "dockerRegistriesCredentials": []}, "name": "picard", "isPredefined": false, "isCurated": false, "isFeatured": false, "isModule": false, "isPublic": false, "priceAmount": 0, "priceUnit": "PER_SAMPLE", "tags": [], "overview": "", "createdAt": "2019-03-11T18:22:12.365Z", "updatedAt": "2019-03-11T18:22:12.365Z", "imageUrl": null, "parametersInfo": [], "archived": {"status": false, "archivalTimestamp": null}, "workflowType": "docker", "containerName": "lifebitai/picard", "defaultCommand": "picard"}] \ No newline at end of file +{"workflows": [{"_id": "5c86a754b37c6700b253042e", "owner": {"id": "5b59e0cf7cbd27320976585f", "name": "Test", "surname": "Testy", "email": "test@test.ml.ai", "picture": "", "completedMilestones": {"hasDoneTutorial": true, "hasSeenWelcomeMessage": true}, "isPredefined": false, "isSuspended": false, "favoritedPublicBuckets": [{"_id": "5e9f271e5d1cd501034447ff", "name": "lifebit-sars-cov-2"}, {"_id": "5e9c5af25d1cd5010340f2bc", "name": "lifebit-sars-cov-2"}, {"_id": "5e9c5af25d1cd5010340f2bb", "name": "lifebit-featured-datasets"}, {"_id": "5e9c5af25d1cd5010340f2ba", "name": "my-first-deploit-bucket"}, {"_id": "5e9c5af25d1cd5010340f2b9", "name": "ngi-igenomes"}, {"_id": "5e9c5af25d1cd5010340f2b8", "name": "lifebit-public"}, {"_id": "5e9c5af25d1cd5010340f2b7", "name": "lifebit-featured-datasets"}, {"_id": "5e9c5af25d1cd5010340f2b6", "name": "lifebit-featured-datasets"}, {"_id": "5e9c5af25d1cd5010340f2b5", "name": "lifebit-featured-datasets"}, {"_id": "5e9c5af25d1cd5010340f2b4", "name": "rosalind-test-data"}, {"_id": "5e9c5af25d1cd5010340f2b3", "name": "lifebit-featured-datasets"}, {"_id": "5e9c5af25d1cd5010340f2b2", "name": "lifebit-featured-datasets"}, {"_id": "5e9c5af25d1cd5010340f2b1", "name": "deepvariant-data"}], "favoritedPublicAzureContainers": [], "adGroups": [], "isApproved": true, "isReviewed": true, "isVerified": true, "isAllowedToCreateWorkspace": true, "totalCreditsInCents": 0, "dockerRegistriesCredentials": []}, "name": "picard", "isPredefined": false, "isCurated": false, "isFeatured": false, "isModule": false, "isPublic": false, "priceAmount": 0, "priceUnit": "PER_SAMPLE", "tags": [], "overview": "", "createdAt": "2019-03-11T18:22:12.365Z", "updatedAt": "2019-03-11T18:22:12.365Z", "imageUrl": null, "parametersInfo": [], "archived": {"status": false, "archivalTimestamp": null}, "workflowType": "docker", "containerName": "lifebitai/picard", "defaultCommand": "picard"}], "paginationMetadata": {"Pagination-Count": 10}} diff --git a/tests/test_jobs/test_project_id.py b/tests/test_jobs/test_project_id.py index a3358c7..5019e9c 100644 --- a/tests/test_jobs/test_project_id.py +++ b/tests/test_jobs/test_project_id.py @@ -12,6 +12,9 @@ WORKSPACE_ID = 'lv89ufc838sdig' PROJECT_NAME = "lifebit-testing" WORKFLOW_NAME = "nf-core-deepvariant" +PAGE_SIZE = 10 +PAGE = 1 +ARCHIVED_STATUS = "false" @mock.patch('cloudos.clos', mock.MagicMock()) @@ -23,27 +26,33 @@ def test_project_id(): """ create_json_project = load_json_file(INPUT_PROJECT) create_json_workflow = load_json_file(INPUT_WORKFLOW) - params = {"teamId": WORKSPACE_ID} + params_projects = {"teamId": WORKSPACE_ID} + params_workflows = { + "teamId": WORKSPACE_ID, + "pageSize": PAGE_SIZE, + "page": PAGE, + "archived.status": ARCHIVED_STATUS} header = { "Accept": "application/json, text/plain, */*", "Content-Type": "application/json;charset=UTF-8", "apikey": APIKEY } - search_str = f"teamId={WORKSPACE_ID}" + search_str_projects = f"teamId={WORKSPACE_ID}" + search_str_workflows = f"teamId={WORKSPACE_ID}&pageSize={PAGE_SIZE}&page={PAGE}&archived.status={ARCHIVED_STATUS}" # mock GET method with the .json responses.add( responses.GET, - url=f"{CLOUDOS_URL}/api/v1/projects?{search_str}", + url=f"{CLOUDOS_URL}/api/v1/projects?{search_str_projects}", body=create_json_project, headers=header, - match=[matchers.query_param_matcher(params)], + match=[matchers.query_param_matcher(params_projects)], status=200) responses.add( responses.GET, - url=f"{CLOUDOS_URL}/api/v1/workflows?{search_str}", + url=f"{CLOUDOS_URL}/api/v3/workflows?{search_str_workflows}", body=create_json_workflow, headers=header, - match=[matchers.query_param_matcher(params)], + match=[matchers.query_param_matcher(params_workflows)], status=201) # start cloudOS service job = Job(apikey=APIKEY, diff --git a/tests/test_jobs/test_send_job.py b/tests/test_jobs/test_send_job.py index b2df515..6a87c23 100644 --- a/tests/test_jobs/test_send_job.py +++ b/tests/test_jobs/test_send_job.py @@ -15,6 +15,9 @@ WORKFLOW_NAME = "nf-core-deepvariant" INPUT_PROJECT = "tests/test_data/projects.json" INPUT_WORKFLOW = "tests/test_data/workflows.json" +PAGE_SIZE = 10 +PAGE = 1 +ARCHIVED_STATUS = "false" param_dict = { "config": "cloudos/examples/rnatoy.config" @@ -32,11 +35,17 @@ def test_send_job(): create_json_workflow = load_json_file(INPUT_WORKFLOW) create_json = load_json_file(INPUT) params_job = {"teamId": WORKSPACE_ID} + params_workflows = { + "teamId": WORKSPACE_ID, + "pageSize": PAGE_SIZE, + "page": PAGE, + "archived.status": ARCHIVED_STATUS} header = { "Content-type": "application/json", "apikey": APIKEY } search_str = f"teamId={WORKSPACE_ID}" + search_str_workflows = f"teamId={WORKSPACE_ID}&pageSize={PAGE_SIZE}&page={PAGE}&archived.status={ARCHIVED_STATUS}" # mock GET method with the .json responses.add( responses.POST, @@ -54,10 +63,10 @@ def test_send_job(): status=200) responses.add( responses.GET, - url=f"{CLOUDOS_URL}/api/v1/workflows?{search_str}", + url=f"{CLOUDOS_URL}/api/v3/workflows?{search_str_workflows}", body=create_json_workflow, headers=header, - match=[matchers.query_param_matcher(params_job)], + match=[matchers.query_param_matcher(params_workflows)], status=200) # start cloudOS service job = Job(apikey=APIKEY, diff --git a/tests/test_jobs/test_workflow_id.py b/tests/test_jobs/test_workflow_id.py index 6be5783..1acfb25 100644 --- a/tests/test_jobs/test_workflow_id.py +++ b/tests/test_jobs/test_workflow_id.py @@ -12,6 +12,9 @@ WORKSPACE_ID = 'lv89ufc838sdig' PROJECT_NAME = "lifebit-testing" WORKFLOW_NAME = "nf-core-deepvariant" +PAGE_SIZE = 10 +PAGE = 1 +ARCHIVED_STATUS = "false" @mock.patch('cloudos.clos', mock.MagicMock()) @@ -24,12 +27,18 @@ def test_workflow_id(): create_json_project = load_json_file(INPUT_PROJECT) create_json_workflow = load_json_file(INPUT_WORKFLOW) params = {"teamId": WORKSPACE_ID} + params_workflows = { + "teamId": WORKSPACE_ID, + "pageSize": PAGE_SIZE, + "page": PAGE, + "archived.status": ARCHIVED_STATUS} header = { "Accept": "application/json, text/plain, */*", "Content-Type": "application/json;charset=UTF-8", "apikey": APIKEY } search_str = f"teamId={WORKSPACE_ID}" + search_str_workflows = f"teamId={WORKSPACE_ID}&pageSize={PAGE_SIZE}&page={PAGE}&archived.status={ARCHIVED_STATUS}" # mock GET method with the .json responses.add( responses.GET, @@ -40,10 +49,10 @@ def test_workflow_id(): status=200) responses.add( responses.GET, - url=f"{CLOUDOS_URL}/api/v1/workflows?{search_str}", + url=f"{CLOUDOS_URL}/api/v3/workflows?{search_str_workflows}", body=create_json_workflow, headers=header, - match=[matchers.query_param_matcher(params)], + match=[matchers.query_param_matcher(params_workflows)], status=201) # start cloudOS service job = Job(apikey=APIKEY, From 3f68d0feaa2f984128eff16c37f1d6bbfdca4b2e Mon Sep 17 00:00:00 2001 From: dapineyro Date: Wed, 15 Jan 2025 21:05:38 +0100 Subject: [PATCH 4/6] remove a print --- cloudos/clos.py | 1 - 1 file changed, 1 deletion(-) diff --git a/cloudos/clos.py b/cloudos/clos.py index fbd3f6a..a79fe89 100644 --- a/cloudos/clos.py +++ b/cloudos/clos.py @@ -348,7 +348,6 @@ def get_curated_workflow_list(self, workspace_id, get_all=True, page=1, verify=T "{}/api/v3/workflows?search=&groups[]=curated&page={}&teamId={}".format( self.cloudos_url, page, workspace_id), headers=headers, verify=verify) - print(page) if r.status_code >= 400: raise BadRequestException(r) content = json.loads(r.content) From 43f8b1c9836a3f160611baa814f2362db2e7800b Mon Sep 17 00:00:00 2001 From: dapineyro Date: Thu, 16 Jan 2025 12:55:31 +0100 Subject: [PATCH 5/6] changelog --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 16fec70..aeb6559 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ ## lifebit-ai/cloudos-cli: changelog +## v2.15.0 (2025-01-16) + +### Feature + +- Updates GET workflows endpoint to v3. + ## v2.14.0 (2024-12-18) - Adds the new `--accelerate-file-staging` parameter to job submission to add support for AWS S3 mountpoint for quicker file staging. From 354bf28d328a104ffaf84f46f26192f383958c4a Mon Sep 17 00:00:00 2001 From: dapineyro Date: Thu, 16 Jan 2025 17:55:13 +0100 Subject: [PATCH 6/6] review suggestions --- cloudos/clos.py | 5 +++-- tests/test_clos/test_get_curated_workflow_list.py | 6 +++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/cloudos/clos.py b/cloudos/clos.py index a79fe89..d39794a 100644 --- a/cloudos/clos.py +++ b/cloudos/clos.py @@ -345,7 +345,7 @@ def get_curated_workflow_list(self, workspace_id, get_all=True, page=1, verify=T "apikey": self.apikey } r = retry_requests_get( - "{}/api/v3/workflows?search=&groups[]=curated&page={}&teamId={}".format( + "{}/api/v3/workflows?search=&groups[]=curated&groups[]=featured&groups[]=predefined&page={}&teamId={}".format( self.cloudos_url, page, workspace_id), headers=headers, verify=verify) if r.status_code >= 400: @@ -540,7 +540,8 @@ def is_module(self, workflow_name, workspace_id, verify=True): 'data-factory', 'data-factory-omics-etl', 'drug-discovery', - 'data-factory-omics-insights' + 'data-factory-omics-insights', + 'intermediate' ] if group.values[0] in module_groups: return True diff --git a/tests/test_clos/test_get_curated_workflow_list.py b/tests/test_clos/test_get_curated_workflow_list.py index 7185f97..c579c64 100644 --- a/tests/test_clos/test_get_curated_workflow_list.py +++ b/tests/test_clos/test_get_curated_workflow_list.py @@ -23,13 +23,13 @@ def test_get_curated_workflow_list_correct_response(): """ create_json = load_json_file(OUTPUT) params = {"teamId": WORKSPACE_ID, - "groups[]": "curated", + "groups[]": ["curated", "featured", "predefined"], "page": PAGE} header = { "Accept": "application/json, text/plain, */*", "Content-Type": "application/json;charset=UTF-8", "apikey": APIKEY} - search_str = f"search=&groups[]=curated&page={PAGE}&teamId={WORKSPACE_ID}" + search_str = f"search=&groups[]=curated&groups[]=featured&groups[]=predefined&page={PAGE}&teamId={WORKSPACE_ID}" # mock GET method with the .json responses.add( responses.GET, @@ -57,7 +57,7 @@ def test_get_curated_workflow_list_incorrect_response(): error_message = {"statusCode": 400, "code": "BadRequest", "message": "Bad Request.", "time": "2022-11-23_17:31:07"} error_json = json.dumps(error_message) - search_str = f"search=&groups[]=curated&page={PAGE}&teamId={WORKSPACE_ID}" + search_str = f"search=&groups[]=curated&groups[]=featured&groups[]=predefined&page={PAGE}&teamId={WORKSPACE_ID}" # mock GET method with the .json responses.add( responses.GET,