From 6397a27e546df312ae49ca2bde8684e4030e46cb Mon Sep 17 00:00:00 2001 From: Sakib Rahman Date: Wed, 15 Jan 2025 12:36:21 -0500 Subject: [PATCH] Accept lists as argument for rucio register script (#50) * Accept lists as argument for rucio register script * Add some debugging statements * Remove call to did_name before it's defined and unnecessary debug statements * Make sure file paths and did names list have same length * Fix typo in upload item list --- scripts/register_to_rucio.py | 29 +++++++++++++++++++++-------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/scripts/register_to_rucio.py b/scripts/register_to_rucio.py index 195ca48..bf2d594 100644 --- a/scripts/register_to_rucio.py +++ b/scripts/register_to_rucio.py @@ -7,30 +7,43 @@ import logging parser = argparse.ArgumentParser(prog='Register to RUCIO', description='Registers files to RUCIO') -parser.add_argument("-f", dest="file_path", action="store", required=True, help="Enter the local file path") -parser.add_argument("-d", dest="did_name", action="store", required=True, help="Enter the data identifier for rucio catalogue") +parser.add_argument("-f", dest="file_paths", action="store", nargs='+', required=True, help="Enter the local file path") +parser.add_argument("-d", dest="did_names", action="store", nargs='+', required=True, help="Enter the data identifier for rucio catalogue") parser.add_argument("-s", dest="scope", action="store", required=True, help="Enter the scope") parser.add_argument("-r", dest="rse", action="store", required=True, help="Enter the rucio storage element. EIC-XRD is for storing production outputs.") args=parser.parse_args() -file_path = args.file_path -did_name = args.did_name -parent_directory = os.path.dirname(did_name) +file_paths = args.file_paths +did_names = args.did_names scope= args.scope rse= args.rse -uploads_items = [{ +# Validation to ensure file_paths and did_names have the same length +if len(file_paths) != len(did_names): + raise ValueError("The number of file paths must match the number of did names.") + +upload_items = [] # List to hold the upload items + +# Loop through the file paths and did names (assuming did_names length matches file_paths length) +for file_path, did_name in zip(file_paths, did_names): + parent_directory = os.path.dirname(did_name) # Get the parent directory from did_name + + # Create a new dictionary for each file and did_name + upload_item = { 'path': file_path, 'rse': rse, 'did_scope': scope, 'did_name': did_name, 'dataset_scope': scope, 'dataset_name': parent_directory -}] + } + + # Append the new item to the upload_items list + upload_items.append(upload_item) logger = logging.getLogger('upload_client') logger.addHandler(logging.StreamHandler()) logger.setLevel(logging.INFO) upload_client=UploadClient(logger=logger) -upload_client.upload(uploads_items) +upload_client.upload(upload_items)