Skip to content

Commit

Permalink
1. add args in ray_single_node_image_loader_microbenchmark.py
Browse files Browse the repository at this point in the history
2. avoid all possible copies in multiple http get
  • Loading branch information
lucyge2022 committed May 8, 2024
1 parent c2595ef commit 731c635
Show file tree
Hide file tree
Showing 4 changed files with 132 additions and 207 deletions.
10 changes: 10 additions & 0 deletions benchmark/ray_single_node_image_loader_microbenchmark.py
Original file line number Diff line number Diff line change
Expand Up @@ -439,6 +439,8 @@ def setup_alluxio(args):
alluxio_options[
"alluxio.worker.page.store.page.size"
] = args.alluxio_page_size
if args.use_alluxiocommon:
alluxio_options["alluxio.common.extension.enable"] = "True"
if args.alluxio_cluster_name:
alluxio_options["alluxio.cluster.name"] = args.alluxio_cluster_name
if alluxio_options:
Expand Down Expand Up @@ -523,6 +525,12 @@ def setup_alluxio(args):
default=False,
help="Whether to use Alluxio instead of original ufs filesystem for data loading.",
)
parser.add_argument(
"--use-alluxiocommon",
action="store_true",
default=False,
help="Whether to use AlluxioCommon native extension lib.",
)
parser.add_argument(
"--alluxio-etcd-hosts",
default=None,
Expand Down Expand Up @@ -720,6 +728,7 @@ def setup_alluxio(args):

if args.parquet_data_root is not None:
# HuggingFace Dataset, reading from parquet.
"""
hf_dataset = build_hf_dataloader(
args.parquet_data_root,
args.batch_size,
Expand All @@ -735,6 +744,7 @@ def setup_alluxio(args):
metrics,
args.output_file,
)
"""

# Ray Data, reading from parquet.
ray_dataset = ray.data.read_parquet(
Expand Down
Loading

0 comments on commit 731c635

Please sign in to comment.