From a6a29980e1e5ad8683014a1413f368934d028216 Mon Sep 17 00:00:00 2001 From: Vinoth Date: Tue, 18 Jul 2023 17:48:25 +0530 Subject: [PATCH 1/7] 684207 -US Implementation Upload annotation file in IGT Cloud --- .../client/services/entities_service.py | 7 +++++ src/igtcloud/client/tools/cli.py | 15 ++++++++-- src/igtcloud/client/tools/upload_project.py | 30 ++++++++++++++++++- 3 files changed, 48 insertions(+), 4 deletions(-) diff --git a/src/igtcloud/client/services/entities_service.py b/src/igtcloud/client/services/entities_service.py index 52614c1..2c9c347 100644 --- a/src/igtcloud/client/services/entities_service.py +++ b/src/igtcloud/client/services/entities_service.py @@ -232,6 +232,13 @@ def get_study_files(service: EntitiesService, study: RootStudy) -> FilesCollecti return data_store['files'] +def upload_annotation_files(s3_key: str = None, annotation_filename: str = None, s3_path: str = None) -> bool: + files_collection_wrapper_obj = FilesCollectionWrapper(s3_prefix=s3_path, f_auth=lambda action, prefix: s3_creds(action, prefix), f_extend=lambda x: x) + annotation_data = FilesCollectionWrapper.upload(files_collection_wrapper_obj, filename=annotation_filename, key=s3_key) + return annotation_data + + + def get_dicom_files(service: EntitiesService, study: RootStudy) -> FilesCollectionWrapper: data_store = study.get('_data_store') initial_value = data_store.get('dicom', None) diff --git a/src/igtcloud/client/tools/cli.py b/src/igtcloud/client/tools/cli.py index 38b9298..4c296a6 100644 --- a/src/igtcloud/client/tools/cli.py +++ b/src/igtcloud/client/tools/cli.py @@ -170,13 +170,22 @@ def _get_domain(domain, environment): type=click.Choice(['flat', 'hierarchical'], case_sensitive=False), help='Folder structure of the data to be uploaded.' ) +@click.option( + '--category', + default=['annotation'], + type=click.Choice(['annotation'], case_sensitive=False), + help='Uploads annotation file' +) def upload(local_folder, project, institute, environment, domain, user, submit, debug, concurrent_studies, - concurrent_files, folder_structure): + concurrent_files, folder_structure, category): """Upload data to Philips Interventional Cloud. \b This tool will upload all files in LOCAL_FOLDER to project PROJECT. - the folder structure should be LOCAL_FOLDER / INSTITUTE / / / / < study_id > / annotations / 0 / < sequence_id > / < image > / < user_id > / """ if debug: logging.getLogger('igtcloud.client').setLevel(logging.DEBUG) @@ -192,7 +201,7 @@ def upload(local_folder, project, institute, environment, domain, user, submit, with smart_auth(domain, username=user) as auth: set_auth(auth) logger.info(f"Using url: {auth.domain}") - upload_project(local_folder, project, institute, submit, concurrent_studies, concurrent_files, folder_structure) + upload_project(local_folder, project, institute, submit, concurrent_studies, concurrent_files, folder_structure, category) @click.command(short_help="Login to Philips Interventional Cloud") diff --git a/src/igtcloud/client/tools/upload_project.py b/src/igtcloud/client/tools/upload_project.py index 73134bd..e264866 100644 --- a/src/igtcloud/client/tools/upload_project.py +++ b/src/igtcloud/client/tools/upload_project.py @@ -15,13 +15,15 @@ from igtcloud.client.services.entities.model.patient import Patient from igtcloud.client.services.entities.model.root_study import RootStudy from igtcloud.client.services.entities.model_utils import validate_and_convert_types +from igtcloud.client.services.entities_service import upload_annotation_files from igtcloud.client.tools.common import find_project_and_institutes, study_key logger = logging.getLogger(__name__) def upload_project(local_folder: str, project_name: str, institute_name: str = None, submit: bool = False, - max_workers_studies: int = None, max_workers_files: int = None, folder_structure: str = None): + max_workers_studies: int = None, max_workers_files: int = None, folder_structure: str = None, + category: str = None): project, institutes = find_project_and_institutes(project_name, institute_name) if not project and not institutes: @@ -32,6 +34,32 @@ def upload_project(local_folder: str, project_name: str, institute_name: str = N if submit: _password = getpass("For electronic record state it is required to reenter the password") + if category.lower() == "annotation": + # Annotation file upload + logger.info("Annotation File Uploading...") + if os.path.isdir(local_folder): + for root, dirs, local_files in os.walk(local_folder): + key = root + annotation_file = local_files + else: + logger.error("Not a valid Directory") + key = key + "/" + str(annotation_file[0]) + logger.debug("Upload annotation path" + key) + try: + f = open(os.path.abspath(key), "r") + json.loads(f.read()) + except ValueError as e: + logger.error("Annotation File JSON is not valid : %s" % e) + return + + is_uploaded = upload_annotation_files(s3_key=key, annotation_filename=key, s3_path=institutes[0].s3_prefix) + if is_uploaded: + logger.info("Annotation File uploaded successfully") + else: + logger.info("Annotation File failed to upload") + + return + if project: # Project level file upload when there is a "files" folder in the root directory files_folder = os.path.join(local_folder, 'files') From 3b3fe4fe541051bdd0622f3baaafba3b6993dfa5 Mon Sep 17 00:00:00 2001 From: Vinoth Date: Wed, 19 Jul 2023 23:24:01 +0530 Subject: [PATCH 2/7] Resolving review comments from Robin --- src/igtcloud/client/tools/cli.py | 6 +- src/igtcloud/client/tools/upload_project.py | 70 +++++++++++++-------- 2 files changed, 48 insertions(+), 28 deletions(-) diff --git a/src/igtcloud/client/tools/cli.py b/src/igtcloud/client/tools/cli.py index 4c296a6..02e40c5 100644 --- a/src/igtcloud/client/tools/cli.py +++ b/src/igtcloud/client/tools/cli.py @@ -172,9 +172,9 @@ def _get_domain(domain, environment): ) @click.option( '--category', - default=['annotation'], - type=click.Choice(['annotation'], case_sensitive=False), - help='Uploads annotation file' + default=['annotations'], + type=click.Choice(['annotations'], case_sensitive=False), + help='Uploads annotation files' ) def upload(local_folder, project, institute, environment, domain, user, submit, debug, concurrent_studies, concurrent_files, folder_structure, category): diff --git a/src/igtcloud/client/tools/upload_project.py b/src/igtcloud/client/tools/upload_project.py index e264866..b827203 100644 --- a/src/igtcloud/client/tools/upload_project.py +++ b/src/igtcloud/client/tools/upload_project.py @@ -34,31 +34,8 @@ def upload_project(local_folder: str, project_name: str, institute_name: str = N if submit: _password = getpass("For electronic record state it is required to reenter the password") - if category.lower() == "annotation": - # Annotation file upload - logger.info("Annotation File Uploading...") - if os.path.isdir(local_folder): - for root, dirs, local_files in os.walk(local_folder): - key = root - annotation_file = local_files - else: - logger.error("Not a valid Directory") - key = key + "/" + str(annotation_file[0]) - logger.debug("Upload annotation path" + key) - try: - f = open(os.path.abspath(key), "r") - json.loads(f.read()) - except ValueError as e: - logger.error("Annotation File JSON is not valid : %s" % e) - return - - is_uploaded = upload_annotation_files(s3_key=key, annotation_filename=key, s3_path=institutes[0].s3_prefix) - if is_uploaded: - logger.info("Annotation File uploaded successfully") - else: - logger.info("Annotation File failed to upload") - - return + if category.lower() == "annotations": + return upload_annotation_files_private_method(institutes, local_folder) if project: # Project level file upload when there is a "files" folder in the root directory @@ -111,6 +88,49 @@ def upload_project(local_folder: str, project_name: str, institute_name: str = N f"files_skipped: {len(files_skipped)}") +def upload_annotation_files_private_method(institutes, local_folder): + # Annotation file upload + logger.info("Annotation File Uploading...") + annotation_files = [] + if os.path.isdir(local_folder): + for root, dirs, local_files in os.walk(local_folder): + local_path = root + annotation_files.extend(local_files) + else: + logger.error("Not a valid Directory") + local_path = local_path.replace("/name", "") + initial_path = local_path.split('/') + annotation_paths = local_path.split('/0') + annotation_path = annotation_paths[1] + annotation_path = "annotations/0" + annotation_path + if "---" in initial_path[2]: + study_id_human_readable = initial_path[2].replace("---", "/") + else: + study_id_human_readable = initial_path[2] + "/" + initial_path[3] + for institute in institutes: + for studies in institute.studies: + if studies.study_id_human_readable == study_id_human_readable: + s3_prefix_for_annotation_file = studies.s3_prefix + break + index = 0 + for annotation_file in annotation_files: + if str(annotation_files[index]).endswith(".json"): + try: + file_path = local_path + "/" + str(annotation_files[index]) + with open(os.path.abspath(file_path), "r") as file: + json.loads(file.read()) + except ValueError as e: + logger.error("Annotation File JSON is not valid : %s" % e) + return + is_uploaded = upload_annotation_files(s3_key=annotation_path+"/"+str(annotation_files[index]), annotation_filename=file_path, s3_path=s3_prefix_for_annotation_file) + index += 1 + if is_uploaded: + logger.info("Annotation File uploaded successfully") + else: + logger.info("Annotation File failed to upload") + return + + def upload_study(study_type: str, study_folder: str, patient_name: str, institute_id: str, studies: CollectionWrapper[RootStudy], _submit_password: str = None, max_workers_files: int = None) -> Tuple[RootStudy, List[str], List[str]]: From 1cf76399026b55522c31225a956b6e1f61cff88d Mon Sep 17 00:00:00 2001 From: Vinoth Date: Mon, 24 Jul 2023 23:52:02 +0530 Subject: [PATCH 3/7] Multithreading Implementation for Annotation file upload --- .../client/services/entities_service.py | 10 +-- src/igtcloud/client/tools/cli.py | 2 +- src/igtcloud/client/tools/upload_project.py | 76 +++++++++++++------ 3 files changed, 56 insertions(+), 32 deletions(-) diff --git a/src/igtcloud/client/services/entities_service.py b/src/igtcloud/client/services/entities_service.py index 2c9c347..464b27d 100644 --- a/src/igtcloud/client/services/entities_service.py +++ b/src/igtcloud/client/services/entities_service.py @@ -232,13 +232,6 @@ def get_study_files(service: EntitiesService, study: RootStudy) -> FilesCollecti return data_store['files'] -def upload_annotation_files(s3_key: str = None, annotation_filename: str = None, s3_path: str = None) -> bool: - files_collection_wrapper_obj = FilesCollectionWrapper(s3_prefix=s3_path, f_auth=lambda action, prefix: s3_creds(action, prefix), f_extend=lambda x: x) - annotation_data = FilesCollectionWrapper.upload(files_collection_wrapper_obj, filename=annotation_filename, key=s3_key) - return annotation_data - - - def get_dicom_files(service: EntitiesService, study: RootStudy) -> FilesCollectionWrapper: data_store = study.get('_data_store') initial_value = data_store.get('dicom', None) @@ -257,7 +250,8 @@ def get_annotation_files(service: EntitiesService, study: RootStudy) -> FilesCol data_store['annotations'] = FilesCollectionWrapper(s3_prefix=study.s3_prefix + 'annotations/', category='annotations', f_auth=lambda action, prefix: s3_creds(action, prefix), - f_init=lambda: get_aux_files(service, study, 'annotations')) + f_init=lambda: get_aux_files(service, study, 'annotations'), + f_extend=lambda x: x) return data_store['annotations'] diff --git a/src/igtcloud/client/tools/cli.py b/src/igtcloud/client/tools/cli.py index 02e40c5..538ed63 100644 --- a/src/igtcloud/client/tools/cli.py +++ b/src/igtcloud/client/tools/cli.py @@ -185,7 +185,7 @@ def upload(local_folder, project, institute, environment, domain, user, submit, the folder structure should be LOCAL_FOLDER / INSTITUTE / / / < study_id > / annotations / 0 / < sequence_id > / < image > / < user_id > / """ + < patient_id > / < study_id > / Annotation Folder upload structure """ if debug: logging.getLogger('igtcloud.client').setLevel(logging.DEBUG) diff --git a/src/igtcloud/client/tools/upload_project.py b/src/igtcloud/client/tools/upload_project.py index b827203..421d992 100644 --- a/src/igtcloud/client/tools/upload_project.py +++ b/src/igtcloud/client/tools/upload_project.py @@ -15,7 +15,6 @@ from igtcloud.client.services.entities.model.patient import Patient from igtcloud.client.services.entities.model.root_study import RootStudy from igtcloud.client.services.entities.model_utils import validate_and_convert_types -from igtcloud.client.services.entities_service import upload_annotation_files from igtcloud.client.tools.common import find_project_and_institutes, study_key logger = logging.getLogger(__name__) @@ -35,7 +34,7 @@ def upload_project(local_folder: str, project_name: str, institute_name: str = N _password = getpass("For electronic record state it is required to reenter the password") if category.lower() == "annotations": - return upload_annotation_files_private_method(institutes, local_folder) + return upload_annotation_files_private_method(institutes, local_folder, max_workers_files) if project: # Project level file upload when there is a "files" folder in the root directory @@ -88,7 +87,7 @@ def upload_project(local_folder: str, project_name: str, institute_name: str = N f"files_skipped: {len(files_skipped)}") -def upload_annotation_files_private_method(institutes, local_folder): +def upload_annotation_files_private_method(institutes, local_folder, max_workers_files): # Annotation file upload logger.info("Annotation File Uploading...") annotation_files = [] @@ -98,11 +97,10 @@ def upload_annotation_files_private_method(institutes, local_folder): annotation_files.extend(local_files) else: logger.error("Not a valid Directory") - local_path = local_path.replace("/name", "") - initial_path = local_path.split('/') - annotation_paths = local_path.split('/0') + initial_path = local_path.split("/") + annotation_paths = local_path.split("/0") if "/0" in local_path else local_path.split("/files") annotation_path = annotation_paths[1] - annotation_path = "annotations/0" + annotation_path + annotation_path = "0" + annotation_path if "/0" in local_path else "files" + annotation_path if "---" in initial_path[2]: study_id_human_readable = initial_path[2].replace("---", "/") else: @@ -111,23 +109,55 @@ def upload_annotation_files_private_method(institutes, local_folder): for studies in institute.studies: if studies.study_id_human_readable == study_id_human_readable: s3_prefix_for_annotation_file = studies.s3_prefix + hospital_id = studies.hospital_id + study_id = studies.study_database_id break - index = 0 - for annotation_file in annotation_files: - if str(annotation_files[index]).endswith(".json"): - try: - file_path = local_path + "/" + str(annotation_files[index]) - with open(os.path.abspath(file_path), "r") as file: - json.loads(file.read()) - except ValueError as e: - logger.error("Annotation File JSON is not valid : %s" % e) - return - is_uploaded = upload_annotation_files(s3_key=annotation_path+"/"+str(annotation_files[index]), annotation_filename=file_path, s3_path=s3_prefix_for_annotation_file) - index += 1 - if is_uploaded: - logger.info("Annotation File uploaded successfully") - else: - logger.info("Annotation File failed to upload") + + studies_id = entities_service.get_study(hospital_id=hospital_id, study_id=study_id) + + files_uploaded = list() + files_skipped = list() + + with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers_files or 4) as executor: + fs = dict() + with tqdm(total=0, leave=False, desc=f" Annotation file Upload", unit='B', unit_scale=True, + unit_divisor=1024) as pbar: + def callback(x): + pbar.update(x) + + for annotation_file in annotation_files: + if annotation_file.endswith(".json"): + try: + local_path_json = local_path.replace("/name", "") + annotation_path_json = annotation_path.replace("/name", "") + file_path = local_path_json + "/" + annotation_file + size = os.path.getsize(file_path) + pbar.total += size + with open(os.path.abspath(file_path), "r") as file: + json.loads(file.read()) + except ValueError as e: + logger.error("Annotation File JSON is not valid : %s" % e) + return + fs[executor.submit(studies_id.annotations.upload, file_path, + annotation_path_json + "/" + annotation_file, callback=callback)] = ( + s3_prefix_for_annotation_file, size) + else: + file_path = local_path + "/" + annotation_file + size = os.path.getsize(file_path) + pbar.total += size + fs[executor.submit(studies_id.annotations.upload, file_path, + annotation_path + "/" + annotation_file, callback=callback)] = ( + s3_prefix_for_annotation_file, size) + + for f in concurrent.futures.as_completed(fs.keys()): + file, size = fs.pop(f) + if f.result(): + files_uploaded.append(file) + else: + files_skipped.append(file) + logger.info(f"files_uploaded: {len(files_uploaded)}, " + f"files_skipped: {len(files_skipped)}") + logger.info("Annotation File uploaded successfully") return From 840dd0dc3c623251dadc3af01f01a951ccdec710 Mon Sep 17 00:00:00 2001 From: Vinoth Date: Tue, 25 Jul 2023 00:02:09 +0530 Subject: [PATCH 4/7] Multithreading Implementation for Annotation file upload --- src/igtcloud/client/tools/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/igtcloud/client/tools/cli.py b/src/igtcloud/client/tools/cli.py index 538ed63..8718a45 100644 --- a/src/igtcloud/client/tools/cli.py +++ b/src/igtcloud/client/tools/cli.py @@ -185,7 +185,7 @@ def upload(local_folder, project, institute, environment, domain, user, submit, the folder structure should be LOCAL_FOLDER / INSTITUTE / / / < study_id > / Annotation Folder upload structure """ + LOCAL_FOLDER / INSTITUTE / < patient_id > / < study_id > / Annotation Folder upload structure """ if debug: logging.getLogger('igtcloud.client').setLevel(logging.DEBUG) From 81db9035d06d15e229255ad30695b9bf44700a98 Mon Sep 17 00:00:00 2001 From: Vinoth Date: Tue, 25 Jul 2023 16:35:32 +0530 Subject: [PATCH 5/7] None check for annotation file upload --- src/igtcloud/client/tools/cli.py | 2 +- src/igtcloud/client/tools/upload_project.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/igtcloud/client/tools/cli.py b/src/igtcloud/client/tools/cli.py index 8718a45..25a1e11 100644 --- a/src/igtcloud/client/tools/cli.py +++ b/src/igtcloud/client/tools/cli.py @@ -172,7 +172,7 @@ def _get_domain(domain, environment): ) @click.option( '--category', - default=['annotations'], + default=None, type=click.Choice(['annotations'], case_sensitive=False), help='Uploads annotation files' ) diff --git a/src/igtcloud/client/tools/upload_project.py b/src/igtcloud/client/tools/upload_project.py index 421d992..11dfa87 100644 --- a/src/igtcloud/client/tools/upload_project.py +++ b/src/igtcloud/client/tools/upload_project.py @@ -33,7 +33,7 @@ def upload_project(local_folder: str, project_name: str, institute_name: str = N if submit: _password = getpass("For electronic record state it is required to reenter the password") - if category.lower() == "annotations": + if category is not None and category.lower() == "annotations": return upload_annotation_files_private_method(institutes, local_folder, max_workers_files) if project: From 836cd67a7fb53df0843fab8471ef10baf8c48f6d Mon Sep 17 00:00:00 2001 From: Vinoth Date: Wed, 26 Jul 2023 12:50:19 +0530 Subject: [PATCH 6/7] Remove extra logging and naming conventions --- src/igtcloud/client/tools/upload_project.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/src/igtcloud/client/tools/upload_project.py b/src/igtcloud/client/tools/upload_project.py index 11dfa87..0469d39 100644 --- a/src/igtcloud/client/tools/upload_project.py +++ b/src/igtcloud/client/tools/upload_project.py @@ -34,7 +34,7 @@ def upload_project(local_folder: str, project_name: str, institute_name: str = N _password = getpass("For electronic record state it is required to reenter the password") if category is not None and category.lower() == "annotations": - return upload_annotation_files_private_method(institutes, local_folder, max_workers_files) + return upload_annotation_files(institutes, local_folder, max_workers_files) if project: # Project level file upload when there is a "files" folder in the root directory @@ -87,9 +87,8 @@ def upload_project(local_folder: str, project_name: str, institute_name: str = N f"files_skipped: {len(files_skipped)}") -def upload_annotation_files_private_method(institutes, local_folder, max_workers_files): +def upload_annotation_files(institutes, local_folder, max_workers_files): # Annotation file upload - logger.info("Annotation File Uploading...") annotation_files = [] if os.path.isdir(local_folder): for root, dirs, local_files in os.walk(local_folder): @@ -113,14 +112,14 @@ def upload_annotation_files_private_method(institutes, local_folder, max_workers study_id = studies.study_database_id break - studies_id = entities_service.get_study(hospital_id=hospital_id, study_id=study_id) + study = entities_service.get_study(hospital_id=hospital_id, study_id=study_id) files_uploaded = list() files_skipped = list() with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers_files or 4) as executor: fs = dict() - with tqdm(total=0, leave=False, desc=f" Annotation file Upload", unit='B', unit_scale=True, + with tqdm(total=0, leave=False, desc=f"Annotation file Upload", unit='B', unit_scale=True, unit_divisor=1024) as pbar: def callback(x): pbar.update(x) @@ -138,14 +137,14 @@ def callback(x): except ValueError as e: logger.error("Annotation File JSON is not valid : %s" % e) return - fs[executor.submit(studies_id.annotations.upload, file_path, + fs[executor.submit(study.annotations.upload, file_path, annotation_path_json + "/" + annotation_file, callback=callback)] = ( s3_prefix_for_annotation_file, size) else: file_path = local_path + "/" + annotation_file size = os.path.getsize(file_path) pbar.total += size - fs[executor.submit(studies_id.annotations.upload, file_path, + fs[executor.submit(study.annotations.upload, file_path, annotation_path + "/" + annotation_file, callback=callback)] = ( s3_prefix_for_annotation_file, size) @@ -157,7 +156,6 @@ def callback(x): files_skipped.append(file) logger.info(f"files_uploaded: {len(files_uploaded)}, " f"files_skipped: {len(files_skipped)}") - logger.info("Annotation File uploaded successfully") return From 87d07bbdc33c5ece2345a10d229c909e2330e198 Mon Sep 17 00:00:00 2001 From: Vinoth Date: Wed, 26 Jul 2023 15:06:25 +0530 Subject: [PATCH 7/7] Align with both Windows and Linux --- src/igtcloud/client/tools/upload_project.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/src/igtcloud/client/tools/upload_project.py b/src/igtcloud/client/tools/upload_project.py index 0469d39..f48c0cb 100644 --- a/src/igtcloud/client/tools/upload_project.py +++ b/src/igtcloud/client/tools/upload_project.py @@ -96,14 +96,15 @@ def upload_annotation_files(institutes, local_folder, max_workers_files): annotation_files.extend(local_files) else: logger.error("Not a valid Directory") - initial_path = local_path.split("/") - annotation_paths = local_path.split("/0") if "/0" in local_path else local_path.split("/files") - annotation_path = annotation_paths[1] - annotation_path = "0" + annotation_path if "/0" in local_path else "files" + annotation_path + initial_path = local_path.split(os.sep) if "---" in initial_path[2]: study_id_human_readable = initial_path[2].replace("---", "/") + annotation_paths = local_path.split(initial_path[2] + os.sep) + annotation_path = annotation_paths[1] else: study_id_human_readable = initial_path[2] + "/" + initial_path[3] + annotation_paths = local_path.split(initial_path[3] + os.sep) + annotation_path = annotation_paths[1] for institute in institutes: for studies in institute.studies: if studies.study_id_human_readable == study_id_human_readable: @@ -127,9 +128,9 @@ def callback(x): for annotation_file in annotation_files: if annotation_file.endswith(".json"): try: - local_path_json = local_path.replace("/name", "") - annotation_path_json = annotation_path.replace("/name", "") - file_path = local_path_json + "/" + annotation_file + local_path_json = local_path.replace(os.sep + "name", "") + annotation_path_json = annotation_path.replace(os.sep + "name", "") + file_path = os.path.join(local_path_json, annotation_file) size = os.path.getsize(file_path) pbar.total += size with open(os.path.abspath(file_path), "r") as file: @@ -141,7 +142,7 @@ def callback(x): annotation_path_json + "/" + annotation_file, callback=callback)] = ( s3_prefix_for_annotation_file, size) else: - file_path = local_path + "/" + annotation_file + file_path = os.path.join(local_path, annotation_file) size = os.path.getsize(file_path) pbar.total += size fs[executor.submit(study.annotations.upload, file_path, @@ -155,7 +156,7 @@ def callback(x): else: files_skipped.append(file) logger.info(f"files_uploaded: {len(files_uploaded)}, " - f"files_skipped: {len(files_skipped)}") + f"files_skipped: {len(files_skipped)}") return