Skip to content

Commit

Permalink
Accept lists as argument for rucio register script (#50)
Browse files Browse the repository at this point in the history
* Accept lists as argument for rucio register script

* Add some debugging statements

* Remove call to did_name before it's defined and unnecessary debug statements

* Make sure file paths and did names list have same length

* Fix typo in upload item list
  • Loading branch information
rahmans1 authored Jan 15, 2025
1 parent 7597b77 commit 6397a27
Showing 1 changed file with 21 additions and 8 deletions.
29 changes: 21 additions & 8 deletions scripts/register_to_rucio.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,30 +7,43 @@
import logging

parser = argparse.ArgumentParser(prog='Register to RUCIO', description='Registers files to RUCIO')
parser.add_argument("-f", dest="file_path", action="store", required=True, help="Enter the local file path")
parser.add_argument("-d", dest="did_name", action="store", required=True, help="Enter the data identifier for rucio catalogue")
parser.add_argument("-f", dest="file_paths", action="store", nargs='+', required=True, help="Enter the local file path")
parser.add_argument("-d", dest="did_names", action="store", nargs='+', required=True, help="Enter the data identifier for rucio catalogue")
parser.add_argument("-s", dest="scope", action="store", required=True, help="Enter the scope")
parser.add_argument("-r", dest="rse", action="store", required=True, help="Enter the rucio storage element. EIC-XRD is for storing production outputs.")

args=parser.parse_args()

file_path = args.file_path
did_name = args.did_name
parent_directory = os.path.dirname(did_name)
file_paths = args.file_paths
did_names = args.did_names
scope= args.scope
rse= args.rse

uploads_items = [{
# Validation to ensure file_paths and did_names have the same length
if len(file_paths) != len(did_names):
raise ValueError("The number of file paths must match the number of did names.")

upload_items = [] # List to hold the upload items

# Loop through the file paths and did names (assuming did_names length matches file_paths length)
for file_path, did_name in zip(file_paths, did_names):
parent_directory = os.path.dirname(did_name) # Get the parent directory from did_name

# Create a new dictionary for each file and did_name
upload_item = {
'path': file_path,
'rse': rse,
'did_scope': scope,
'did_name': did_name,
'dataset_scope': scope,
'dataset_name': parent_directory
}]
}

# Append the new item to the upload_items list
upload_items.append(upload_item)

logger = logging.getLogger('upload_client')
logger.addHandler(logging.StreamHandler())
logger.setLevel(logging.INFO)
upload_client=UploadClient(logger=logger)
upload_client.upload(uploads_items)
upload_client.upload(upload_items)

0 comments on commit 6397a27

Please sign in to comment.