diff --git a/executor/app/envs/prod/Dockerfile b/executor/app/envs/prod/Dockerfile index d4b35491a..7c82e5084 100644 --- a/executor/app/envs/prod/Dockerfile +++ b/executor/app/envs/prod/Dockerfile @@ -41,6 +41,9 @@ RUN apt-get update \ WORKDIR /root/src/ ENV PYTHONUNBUFFERED=1 ENV PATH="/root/src/.venv/bin:$PATH" + +# Enable hf_transfer download acceleration package +# https://huggingface.co/docs/huggingface_hub/package_reference/environment_variables#hfhubenablehftransfer ENV HF_HUB_ENABLE_HF_TRANSFER=1 COPY --from=base-image /root/src/ /root/src/ diff --git a/executor/app/src/compute_horde_executor/executor/management/commands/run_executor.py b/executor/app/src/compute_horde_executor/executor/management/commands/run_executor.py index 012a07f39..02a57341c 100644 --- a/executor/app/src/compute_horde_executor/executor/management/commands/run_executor.py +++ b/executor/app/src/compute_horde_executor/executor/management/commands/run_executor.py @@ -3,7 +3,6 @@ import csv import io import logging -import os import pathlib import random import re @@ -390,9 +389,6 @@ def download_from_huggingface( self, relative_path: pathlib.Path, repo_id: str, revision: str | None ): try: - # enable hf_transfer download acceleration package - # https://huggingface.co/docs/huggingface_hub/package_reference/environment_variables#hfhubenablehftransfer - os.environ["HF_HUB_ENABLE_HF_TRANSFER"] = "1" snapshot_download( repo_id=repo_id, revision=revision,