From c61dbb49e4b941d46421f02026d346541ccc192d Mon Sep 17 00:00:00 2001 From: svarona Date: Wed, 27 Sep 2023 12:43:51 +0200 Subject: [PATCH 01/10] Fixed some variables strings --- bu_isciii/scratch.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/bu_isciii/scratch.py b/bu_isciii/scratch.py index 04bf2f5d..8ef0d18e 100644 --- a/bu_isciii/scratch.py +++ b/bu_isciii/scratch.py @@ -54,7 +54,7 @@ def __init__( self.direction = direction # Load conf - conf_api = bu_isciii.config_json.ConfigJson().get_configuration("api_settings") + conf_api = bu_isciii.config_json.ConfigJson().get_configuration("xtutatis_api_settings") # Obtain info from iskylims api rest_api = bu_isciii.drylab_api.RestServiceApi( conf_api["server"], @@ -65,11 +65,10 @@ def __init__( self.rsync_command = self.conf["command"] self.resolution_info = rest_api.get_request( - request_info="resolutionFullData", safe=False, resolution=self.resolution_id + request_info="service-data", safe=False, resolution=self.resolution_id ) - - self.service_folder = self.resolution_info["resolutions"][ - "resolutionFullNumber" + self.service_folder = self.resolution_info["resolutions"][0][ + "resolution_full_number" ] self.scratch_path = os.path.join(self.tmp_dir, self.service_folder) self.out_file = os.path.join( From 1e1a9c0e6b2e5d2ef2d842b4f5cbcd6583c5f2fa Mon Sep 17 00:00:00 2001 From: svarona Date: Wed, 27 Sep 2023 12:46:03 +0200 Subject: [PATCH 02/10] Fixed for new rsync command format --- bu_isciii/scratch.py | 61 ++++++++++++++++++++++++++++++++++---------- 1 file changed, 48 insertions(+), 13 deletions(-) diff --git a/bu_isciii/scratch.py b/bu_isciii/scratch.py index 8ef0d18e..aeb1c5b2 100644 --- a/bu_isciii/scratch.py +++ b/bu_isciii/scratch.py @@ -3,9 +3,10 @@ # Generic imports import sys import os -import subprocess import logging import re +import sysrsync +from sysrsync.exceptions import RsyncError import rich import shutil @@ -62,7 +63,6 @@ def __init__( api_token, ) self.conf = bu_isciii.config_json.ConfigJson().get_configuration("scratch_copy") - self.rsync_command = self.conf["command"] self.resolution_info = rest_api.get_request( request_info="service-data", safe=False, resolution=self.resolution_id @@ -101,9 +101,26 @@ def copy_scratch(self): stderr.print("[blue]I will copy the service from %s" % self.full_path) stderr.print("[blue]to %s" % self.scratch_path) if self.service_folder in self.full_path: - rsync_command = self.rsync_command + self.full_path + " " + self.tmp_dir + protocol = self.conf["protocol"] try: - subprocess.run(rsync_command, shell=True, check=True) + if protocol == "rsync": + sysrsync.run( + source=self.full_path, + destination=self.tmp_dir, + options=self.conf["options"], + exclusions=self.conf["exclusions"], + sync_source_contents=False, + ) + stderr.print( + "[green] Data copied to the sftp folder successfully", + highlight=False, + ) + else: + stderr.print( + "[ref] This protocol is not allowd for the moment", + highlight=False, + ) + sys.exit() f = open(self.out_file, "w") f.write("Temporal directory: " + self.scratch_path + "\n") f.write("Origin service directory: " + self.full_path + "\n") @@ -113,13 +130,12 @@ def copy_scratch(self): % self.scratch_path, highlight=False, ) - except subprocess.CalledProcessError as e: + except RsyncError as e: + stderr.print(e) stderr.print( "[red]ERROR: Copy of the directory %s failed" % self.full_path, highlight=False, ) - log.exception("Unable to create pdf.", exc_info=e) - sys.exit() else: log.error( f"Directory path not the same as service resolution. Skip folder copy '{self.full_path}'" @@ -143,12 +159,31 @@ def revert_copy_scratch(self): dest_dir = os.path.normpath("/".join(dest_folder.split("/")[:-1])) stderr.print("[blue]to %s" % dest_folder) if self.service_folder in dest_folder: - rsync_command = self.rsync_command + self.scratch_path + " " + dest_dir - subprocess.run(rsync_command, shell=True, check=True) - stderr.print( - "[green]Successfully copyed the directory to %s" % dest_folder, - highlight=False, - ) + try: + if self.conf["protocol"] == "rsync": + sysrsync.run( + source=self.scratch_path, + destination=dest_dir, + options=self.conf["options"], + exclusions=self.conf["exclusions"], + sync_source_contents=False, + ) + stderr.print( + "[green]Successfully copyed the directory to %s" % dest_folder, + highlight=False, + ) + else: + stderr.print( + "[ref] This protocol is not allowd for the moment", + highlight=False, + ) + sys.exit() + except RsyncError as e: + stderr.print(e) + stderr.print( + "[red]ERROR: Copy of the directory %s failed" % self.scratch_path, + highlight=False, + ) else: log.error( f"Directory path not the same as service resolution. Skip folder copy '{dest_folder}'" From 450ddb26a6c9c2a349ad09a9d160e25fd8a2d7eb Mon Sep 17 00:00:00 2001 From: svarona Date: Wed, 27 Sep 2023 14:37:10 +0200 Subject: [PATCH 03/10] Fixed variables to fit in DB --- bu_isciii/clean.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/bu_isciii/clean.py b/bu_isciii/clean.py index 4684ac56..4fd58f88 100644 --- a/bu_isciii/clean.py +++ b/bu_isciii/clean.py @@ -58,13 +58,13 @@ def __init__( conf_api["server"], conf_api["api_url"], api_token ) self.resolution_info = rest_api.get_request( - request_info="serviceFullData", safe=False, resolution=self.resolution_id + request_info="service-data", safe=False, resolution=self.resolution_id ) self.service_folder = self.resolution_info["resolutions"][0][ - "resolutionFullNumber" + "resolution_full_number" ] self.services_requested = self.resolution_info["resolutions"][0][ - "availableServices" + "available_services" ] self.service_samples = self.resolution_info["samples"] @@ -86,7 +86,9 @@ def __init__( ) sys.exit() else: - self.path = bu_isciii.utils.get_service_paths(self.resolution_info) + self.path = bu_isciii.utils.get_service_paths( + "services_and_colaborations", self.resolution_info, "non_archived_path" + ) self.full_path = os.path.join(self.path, self.service_folder) @@ -323,7 +325,7 @@ def purge_files(self): files_to_delete = [] for sample_info in self.service_samples: for file in self.delete_files: - file_to_delete = file.replace("sample_name", sample_info["sampleName"]) + file_to_delete = file.replace("sample_name", sample_info["sample_name"]) files_to_delete.append(file_to_delete) path_content = self.scan_dirs(to_find=files_to_delete) for file in path_content: From 2b70e67eab6ace8748b57bb2a56202b1a48c945e Mon Sep 17 00:00:00 2001 From: svarona Date: Wed, 27 Sep 2023 14:38:20 +0200 Subject: [PATCH 04/10] Clean some functions and code --- bu_isciii/clean.py | 27 ++------------------------- 1 file changed, 2 insertions(+), 25 deletions(-) diff --git a/bu_isciii/clean.py b/bu_isciii/clean.py index 4fd58f88..7c154ca0 100644 --- a/bu_isciii/clean.py +++ b/bu_isciii/clean.py @@ -31,17 +31,6 @@ def __init__( option=None, api_token=None, ): - """ - Description: - Class to perform the cleaning. - - Usage: - - Attributes: - - Methods: - - """ # access the api with the resolution name to obtain the data # ask away if no input given if resolution_id is None: @@ -299,18 +288,6 @@ def rename(self, to_find, add, verbose=True): print(f"Renamed {directory_to_rename} to {newpath}.") return - def rename_nocopy(self, verbose=True): - """ - Description: - - Usage: - - Params: - - """ - self.rename(to_find=self.nocopy, add="_NC", verbose=verbose) - return - def purge_files(self): """ Description: @@ -444,7 +421,7 @@ def full_clean(self): """ self.delete_rename() - self.rename_nocopy() + self.rename(to_find=self.nocopy, add="_NC", verbose=True) def handle_clean(self): """ @@ -457,7 +434,7 @@ def handle_clean(self): if self.option == "full_clean": self.full_clean() if self.option == "rename_nocopy": - self.rename_nocopy() + self.rename(to_find=self.nocopy, add="_NC", verbose=True) if self.option == "clean": self.delete_rename() if self.option == "revert_renaming": From 0e1ae7c0b077aecb7ae5de486549d48d267be50b Mon Sep 17 00:00:00 2001 From: svarona Date: Wed, 27 Sep 2023 16:03:25 +0200 Subject: [PATCH 05/10] Fixed database variables --- bu_isciii/copy_sftp.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/bu_isciii/copy_sftp.py b/bu_isciii/copy_sftp.py index 94493be6..44bf782f 100644 --- a/bu_isciii/copy_sftp.py +++ b/bu_isciii/copy_sftp.py @@ -62,7 +62,7 @@ def __init__( ) self.resolution_info = rest_api.get_request( - request_info="serviceFullData", safe=False, resolution=self.resolution_id + request_info="service-data", safe=False, resolution=self.resolution_id ) if sftp_folder is None: self.sftp_folder = bu_isciii.utils.get_sftp_folder(self.resolution_info)[0] @@ -70,17 +70,14 @@ def __init__( self.sftp_folder = sftp_folder self.service_folder = self.resolution_info["resolutions"][0][ - "resolutionFullNumber" + "resolution_full_number" ] self.services_requested = self.resolution_info["resolutions"][0][ - "availableServices" + "available_services" ] self.sftp_options = bu_isciii.config_json.ConfigJson().get_find( "sftp_copy", "options" ) - self.sftp_exclusions = bu_isciii.config_json.ConfigJson().get_find( - "sftp_copy", "exclusions" - ) self.services_to_copy = bu_isciii.utils.get_service_ids(self.services_requested) self.last_folders = self.get_last_folders( From c0e0fb779057e9cf3d88bbffaefd1010819eec44 Mon Sep 17 00:00:00 2001 From: svarona Date: Wed, 27 Sep 2023 16:03:52 +0200 Subject: [PATCH 06/10] Cleaning and fixes --- bu_isciii/copy_sftp.py | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/bu_isciii/copy_sftp.py b/bu_isciii/copy_sftp.py index 44bf782f..ee21fbd4 100644 --- a/bu_isciii/copy_sftp.py +++ b/bu_isciii/copy_sftp.py @@ -33,18 +33,6 @@ def __init__( sftp_folder=None, api_token=None, ): - """ - Description: - Class to perform the copy of the service to sftp folfer. - - Usage: - - Attributes: - - Methods: - - """ - if resolution_id is None: self.resolution_id = bu_isciii.utils.prompt_resolution_id() else: @@ -99,7 +87,9 @@ def __init__( ) sys.exit() else: - self.path = bu_isciii.utils.get_service_paths(self.resolution_info) + self.path = bu_isciii.utils.get_service_paths( + "services_and_colaborations", self.resolution_info, "non_archived_path" + ) self.full_path = os.path.join(self.path, self.service_folder) From f40e0e3df1df1c6c8b2d9e87b60276c8679c5da8 Mon Sep 17 00:00:00 2001 From: svarona Date: Wed, 27 Sep 2023 16:04:40 +0200 Subject: [PATCH 07/10] Fixed new rsync command --- bu_isciii/copy_sftp.py | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/bu_isciii/copy_sftp.py b/bu_isciii/copy_sftp.py index ee21fbd4..6d163216 100644 --- a/bu_isciii/copy_sftp.py +++ b/bu_isciii/copy_sftp.py @@ -134,17 +134,24 @@ def copy_sftp(self): ) self.sftp_options.append(log_command) try: - sysrsync.run( - source=self.full_path, - destination=self.sftp_folder, - options=self.sftp_options, - exclusions=self.sftp_exclusions, - sync_source_contents=False, - ) - stderr.print( - "[green] Data copied to the sftp folder successfully", - highlight=False, - ) + if self.conf["protocol"] == "rsync": + sysrsync.run( + source=self.full_path, + destination=self.sftp_folder, + options=self.sftp_options, + exclusions=self.conf["exclusions"], + sync_source_contents=False, + ) + stderr.print( + "[green] Data copied to the sftp folder successfully", + highlight=False, + ) + else: + stderr.print( + "[ref] This protocol is not allowd for the moment", + highlight=False, + ) + sys.exit() except RsyncError as e: stderr.print(e) stderr.print( From c8b8fc9b5a62150c581d9cad303c348a1e1a88ec Mon Sep 17 00:00:00 2001 From: svarona Date: Wed, 27 Sep 2023 16:05:01 +0200 Subject: [PATCH 08/10] Removed duplicated entry --- bu_isciii/templates/sftp_user.json | 1 - 1 file changed, 1 deletion(-) diff --git a/bu_isciii/templates/sftp_user.json b/bu_isciii/templates/sftp_user.json index 8b2df2c9..05736dba 100644 --- a/bu_isciii/templates/sftp_user.json +++ b/bu_isciii/templates/sftp_user.json @@ -2,7 +2,6 @@ "imago": ["rnaseqrsv"], "miglesias": ["Labvirusres"], "bmartinezd": ["SpainUDP", "SpainUDP_ext","GeneticDiagnosis","GeneticDiagnosis_ext"], - "sresino": ["Labvirushep"], "a.vazquez": ["Labarbovirus"], "aavellon": ["Labvirushep"], "adelavieja": ["LabEndocrineTumors"], From 6aaa0089a40beab8b29ef337dde4654a6946856b Mon Sep 17 00:00:00 2001 From: svarona Date: Wed, 27 Sep 2023 16:06:52 +0200 Subject: [PATCH 09/10] Fixed black --- bu_isciii/scratch.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/bu_isciii/scratch.py b/bu_isciii/scratch.py index aeb1c5b2..c0ea9a17 100644 --- a/bu_isciii/scratch.py +++ b/bu_isciii/scratch.py @@ -55,7 +55,9 @@ def __init__( self.direction = direction # Load conf - conf_api = bu_isciii.config_json.ConfigJson().get_configuration("xtutatis_api_settings") + conf_api = bu_isciii.config_json.ConfigJson().get_configuration( + "xtutatis_api_settings" + ) # Obtain info from iskylims api rest_api = bu_isciii.drylab_api.RestServiceApi( conf_api["server"], @@ -169,7 +171,8 @@ def revert_copy_scratch(self): sync_source_contents=False, ) stderr.print( - "[green]Successfully copyed the directory to %s" % dest_folder, + "[green]Successfully copyed the directory to %s" + % dest_folder, highlight=False, ) else: @@ -181,7 +184,8 @@ def revert_copy_scratch(self): except RsyncError as e: stderr.print(e) stderr.print( - "[red]ERROR: Copy of the directory %s failed" % self.scratch_path, + "[red]ERROR: Copy of the directory %s failed" + % self.scratch_path, highlight=False, ) else: From 436868ecb62b8ea249fad939a5778247914eacfc Mon Sep 17 00:00:00 2001 From: svarona Date: Wed, 27 Sep 2023 17:40:58 +0200 Subject: [PATCH 10/10] Resolved suggestions --- bu_isciii/copy_sftp.py | 2 +- bu_isciii/scratch.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/bu_isciii/copy_sftp.py b/bu_isciii/copy_sftp.py index 6d163216..1989ad1d 100644 --- a/bu_isciii/copy_sftp.py +++ b/bu_isciii/copy_sftp.py @@ -148,7 +148,7 @@ def copy_sftp(self): ) else: stderr.print( - "[ref] This protocol is not allowd for the moment", + "[ref] This protocol is not allowed at the moment", highlight=False, ) sys.exit() diff --git a/bu_isciii/scratch.py b/bu_isciii/scratch.py index c0ea9a17..d4fc8342 100644 --- a/bu_isciii/scratch.py +++ b/bu_isciii/scratch.py @@ -119,7 +119,7 @@ def copy_scratch(self): ) else: stderr.print( - "[ref] This protocol is not allowd for the moment", + "[ref] This protocol is not allowed at the moment", highlight=False, ) sys.exit() @@ -177,7 +177,7 @@ def revert_copy_scratch(self): ) else: stderr.print( - "[ref] This protocol is not allowd for the moment", + "[ref] This protocol is not allowed at the moment", highlight=False, ) sys.exit()