Updated training and data preparation configs

Signed-off-by: AWS ParallelCluster user <[email protected]>
aws-samples · Jun 11, 2024 · d0227b2 · d0227b2
1 parent a9b5181
commit d0227b2
Show file tree

Hide file tree

Showing 4 changed files with 383,091 additions and 1 deletion.
diff --git a/3.test_cases/9.nemo-multimodal/nemo_configs/1.config.yaml b/3.test_cases/9.nemo-multimodal/nemo_configs/1.config.yaml
@@ -27,7 +27,7 @@ wandb_api_key_file: null  # File where the w&B api key is stored. Key must be on
 env_vars:
   NCCL_DEBUG: INFO # Logging level for NCCL. Set to "INFO" for debug information
   TRANSFORMER_OFFLINE: 1
-  FI_EFA_USE_DEVICE_RDMA: 1
+  #FI_EFA_USE_DEVICE_RDMA: 1
   FI_PROVIDER: efa
   NCCL_LAUNCH_MODE: parallel
   FI_EFA_FORK_SAFE: 1

diff --git a/3.test_cases/9.nemo-multimodal/nemo_configs/3.download_multimodal.yaml b/3.test_cases/9.nemo-multimodal/nemo_configs/3.download_multimodal.yaml
@@ -7,7 +7,11 @@ run:
 
 dataset_repo_id: ${HUGGINGFACE_DATASET_REPO_ID}  # huggingface dataset repo id, in the format of {user_or_company}/{dataset_name}
 #  See https://huggingface.co/datasets?task_categories=task_categories:text-to-image&sort=downloads
+<<<<<<< HEAD:3.test_cases/9.nemo-multimodal/nemo_configs/3.download_multimodal.yaml
 dataset_output_root: ${DATASET_PATH}
+=======
+dataset_output_root: /fsx/laion-art-data
+>>>>>>> adea2bd (Updated training and data preparation configs):3.test_cases/8.nemo-multimodal/download_multimodal.yaml
 
 download_parquet:
   enable: True