Merge pull request #380 from backend-developers-ltd/hf-transfer

Enable hf_transfer env var only in Dockerfile
backend-developers-ltd · Jan 24, 2025 · a279983 · a279983
2 parents dbaea2d + b0f3b46
commit a279983
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 4 deletions.
diff --git a/executor/app/envs/prod/Dockerfile b/executor/app/envs/prod/Dockerfile
@@ -41,6 +41,9 @@ RUN apt-get update \
 WORKDIR /root/src/
 ENV PYTHONUNBUFFERED=1
 ENV PATH="/root/src/.venv/bin:$PATH"
+
+# Enable hf_transfer download acceleration package
+# https://huggingface.co/docs/huggingface_hub/package_reference/environment_variables#hfhubenablehftransfer
 ENV HF_HUB_ENABLE_HF_TRANSFER=1
 
 COPY --from=base-image /root/src/ /root/src/

diff --git a/executor/app/src/compute_horde_executor/executor/management/commands/run_executor.py b/executor/app/src/compute_horde_executor/executor/management/commands/run_executor.py
@@ -3,7 +3,6 @@
 import csv
 import io
 import logging
-import os
 import pathlib
 import random
 import re
@@ -390,9 +389,6 @@ def download_from_huggingface(
         self, relative_path: pathlib.Path, repo_id: str, revision: str | None
     ):
         try:
-            # enable hf_transfer download acceleration package
-            # https://huggingface.co/docs/huggingface_hub/package_reference/environment_variables#hfhubenablehftransfer
-            os.environ["HF_HUB_ENABLE_HF_TRANSFER"] = "1"
             snapshot_download(
                 repo_id=repo_id,
                 revision=revision,