diff --git a/.github/workflows/build_pr_documentation.yml b/.github/workflows/build_pr_documentation.yml index 9d6b2d4005..cfc901a9e4 100644 --- a/.github/workflows/build_pr_documentation.yml +++ b/.github/workflows/build_pr_documentation.yml @@ -51,7 +51,7 @@ jobs: echo ${{ env.COMMIT_SHA }} > ./commit_sha echo ${{ env.PR_NUMBER }} > ./pr_number - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 with: name: doc-build-artifact path: optimum-habana/habana-doc-build/ diff --git a/.github/workflows/fast_tests.yml b/.github/workflows/fast_tests.yml index e2d58fbe25..cdd7d1dbf5 100644 --- a/.github/workflows/fast_tests.yml +++ b/.github/workflows/fast_tests.yml @@ -21,7 +21,7 @@ jobs: uses: actions/checkout@v2 - name: Pull image run: | - docker pull vault.habana.ai/gaudi-docker/1.18.0/ubuntu22.04/habanalabs/pytorch-installer-2.4.0:latest + docker pull vault.habana.ai/gaudi-docker/1.19.0/ubuntu22.04/habanalabs/pytorch-installer-2.5.1:latest - name: Run tests run: | docker run \ @@ -36,7 +36,7 @@ jobs: --cap-add=sys_nice \ --net=host \ --ipc=host \ - vault.habana.ai/gaudi-docker/1.18.0/ubuntu22.04/habanalabs/pytorch-installer-2.4.0:latest \ + vault.habana.ai/gaudi-docker/1.19.0/ubuntu22.04/habanalabs/pytorch-installer-2.5.1:latest \ /bin/bash tests/ci/fast_tests.sh diffusers: name: Run tests for optimum.habana.diffusers @@ -46,7 +46,7 @@ jobs: uses: actions/checkout@v2 - name: Pull image run: | - docker pull vault.habana.ai/gaudi-docker/1.18.0/ubuntu22.04/habanalabs/pytorch-installer-2.4.0:latest + docker pull vault.habana.ai/gaudi-docker/1.19.0/ubuntu22.04/habanalabs/pytorch-installer-2.5.1:latest - name: Run tests run: | docker run \ @@ -61,5 +61,5 @@ jobs: --cap-add=sys_nice \ --net=host \ --ipc=host \ - vault.habana.ai/gaudi-docker/1.18.0/ubuntu22.04/habanalabs/pytorch-installer-2.4.0:latest \ + vault.habana.ai/gaudi-docker/1.19.0/ubuntu22.04/habanalabs/pytorch-installer-2.5.1:latest \ /bin/bash tests/ci/fast_tests_diffusers.sh diff --git a/.github/workflows/slow_tests.yml b/.github/workflows/slow_tests.yml index b969273a3c..d0fcb85051 100644 --- a/.github/workflows/slow_tests.yml +++ b/.github/workflows/slow_tests.yml @@ -19,7 +19,7 @@ jobs: uses: actions/checkout@v2 - name: Pull image run: | - docker pull vault.habana.ai/gaudi-docker/1.18.0/ubuntu22.04/habanalabs/pytorch-installer-2.4.0:latest + docker pull vault.habana.ai/gaudi-docker/1.19.0/ubuntu22.04/habanalabs/pytorch-installer-2.5.1:latest - name: Run tests run: | docker run \ @@ -31,7 +31,7 @@ jobs: --cap-add=sys_nice \ --net=host \ --ipc=host \ - vault.habana.ai/gaudi-docker/1.18.0/ubuntu22.04/habanalabs/pytorch-installer-2.4.0:latest \ + vault.habana.ai/gaudi-docker/1.19.0/ubuntu22.04/habanalabs/pytorch-installer-2.5.1:latest \ /bin/bash tests/ci/example_diff_tests.sh stable-diffusion: name: Test Stable Diffusion @@ -45,7 +45,7 @@ jobs: uses: actions/checkout@v2 - name: Pull image run: | - docker pull vault.habana.ai/gaudi-docker/1.18.0/ubuntu22.04/habanalabs/pytorch-installer-2.4.0:latest + docker pull vault.habana.ai/gaudi-docker/1.19.0/ubuntu22.04/habanalabs/pytorch-installer-2.5.1:latest - name: Run tests run: | docker run \ @@ -57,7 +57,7 @@ jobs: --cap-add=sys_nice \ --net=host \ --ipc=host \ - vault.habana.ai/gaudi-docker/1.18.0/ubuntu22.04/habanalabs/pytorch-installer-2.4.0:latest \ + vault.habana.ai/gaudi-docker/1.19.0/ubuntu22.04/habanalabs/pytorch-installer-2.5.1:latest \ /bin/bash tests/ci/slow_tests_diffusers.sh deepspeed: name: Test DeepSpeed models @@ -72,7 +72,7 @@ jobs: uses: actions/checkout@v2 - name: Pull image run: | - docker pull vault.habana.ai/gaudi-docker/1.18.0/ubuntu22.04/habanalabs/pytorch-installer-2.4.0:latest + docker pull vault.habana.ai/gaudi-docker/1.19.0/ubuntu22.04/habanalabs/pytorch-installer-2.5.1:latest - name: Run tests run: | docker run \ @@ -84,7 +84,7 @@ jobs: --cap-add=sys_nice \ --net=host \ --ipc=host \ - vault.habana.ai/gaudi-docker/1.18.0/ubuntu22.04/habanalabs/pytorch-installer-2.4.0:latest \ + vault.habana.ai/gaudi-docker/1.19.0/ubuntu22.04/habanalabs/pytorch-installer-2.5.1:latest \ /bin/bash tests/ci/slow_tests_deepspeed.sh multi-card: name: Test multi-card models @@ -99,7 +99,7 @@ jobs: uses: actions/checkout@v2 - name: Pull image run: | - docker pull vault.habana.ai/gaudi-docker/1.18.0/ubuntu22.04/habanalabs/pytorch-installer-2.4.0:latest + docker pull vault.habana.ai/gaudi-docker/1.19.0/ubuntu22.04/habanalabs/pytorch-installer-2.5.1:latest - name: Run tests run: | docker run \ @@ -111,7 +111,7 @@ jobs: --cap-add=sys_nice \ --net=host \ --ipc=host \ - vault.habana.ai/gaudi-docker/1.18.0/ubuntu22.04/habanalabs/pytorch-installer-2.4.0:latest \ + vault.habana.ai/gaudi-docker/1.19.0/ubuntu22.04/habanalabs/pytorch-installer-2.5.1:latest \ /bin/bash tests/ci/slow_tests_8x.sh single-card: name: Test single-card models @@ -127,7 +127,7 @@ jobs: uses: actions/checkout@v2 - name: Pull image run: | - docker pull vault.habana.ai/gaudi-docker/1.18.0/ubuntu22.04/habanalabs/pytorch-installer-2.4.0:latest + docker pull vault.habana.ai/gaudi-docker/1.19.0/ubuntu22.04/habanalabs/pytorch-installer-2.5.1:latest - name: Run tests run: | docker run \ @@ -139,7 +139,7 @@ jobs: --cap-add=sys_nice \ --net=host \ --ipc=host \ - vault.habana.ai/gaudi-docker/1.18.0/ubuntu22.04/habanalabs/pytorch-installer-2.4.0:latest \ + vault.habana.ai/gaudi-docker/1.19.0/ubuntu22.04/habanalabs/pytorch-installer-2.5.1:latest \ /bin/bash tests/ci/slow_tests_1x.sh albert-xxl-single-card: name: Test single-card ALBERT XXL @@ -158,7 +158,7 @@ jobs: - name: Pull image if: github.event.schedule == '0 21 * * 6' run: | - docker pull vault.habana.ai/gaudi-docker/1.18.0/ubuntu22.04/habanalabs/pytorch-installer-2.4.0:latest + docker pull vault.habana.ai/gaudi-docker/1.19.0/ubuntu22.04/habanalabs/pytorch-installer-2.5.1:latest - name: Run test if: github.event.schedule == '0 21 * * 6' run: | @@ -171,7 +171,7 @@ jobs: --cap-add=sys_nice \ --net=host \ --ipc=host \ - vault.habana.ai/gaudi-docker/1.18.0/ubuntu22.04/habanalabs/pytorch-installer-2.4.0:latest \ + vault.habana.ai/gaudi-docker/1.19.0/ubuntu22.04/habanalabs/pytorch-installer-2.5.1:latest \ /bin/bash tests/ci/albert_xxl_1x.sh - name: Warning if: github.event.schedule != '0 21 * * 6' @@ -192,7 +192,7 @@ jobs: uses: actions/checkout@v2 - name: Pull image run: | - docker pull vault.habana.ai/gaudi-docker/1.18.0/ubuntu22.04/habanalabs/pytorch-installer-2.4.0:latest + docker pull vault.habana.ai/gaudi-docker/1.19.0/ubuntu22.04/habanalabs/pytorch-installer-2.5.1:latest - name: Run tests run: | docker run \ @@ -204,7 +204,7 @@ jobs: --cap-add=sys_nice \ --net=host \ --ipc=host \ - vault.habana.ai/gaudi-docker/1.18.0/ubuntu22.04/habanalabs/pytorch-installer-2.4.0:latest \ + vault.habana.ai/gaudi-docker/1.19.0/ubuntu22.04/habanalabs/pytorch-installer-2.5.1:latest \ make slow_tests_text_generation_example TOKEN=${{ secrets.TEXT_GENERATION_CI_HUB_TOKEN }} trl: name: Test TRL integration @@ -223,7 +223,7 @@ jobs: uses: actions/checkout@v2 - name: Pull image run: | - docker pull vault.habana.ai/gaudi-docker/1.18.0/ubuntu22.04/habanalabs/pytorch-installer-2.4.0:latest + docker pull vault.habana.ai/gaudi-docker/1.19.0/ubuntu22.04/habanalabs/pytorch-installer-2.5.1:latest - name: Run tests run: | docker run \ @@ -235,7 +235,7 @@ jobs: --cap-add=sys_nice \ --net=host \ --ipc=host \ - vault.habana.ai/gaudi-docker/1.18.0/ubuntu22.04/habanalabs/pytorch-installer-2.4.0:latest \ + vault.habana.ai/gaudi-docker/1.19.0/ubuntu22.04/habanalabs/pytorch-installer-2.5.1:latest \ /bin/bash tests/ci/slow_tests_trl.sh sentence-transformers: name: Test Sentence Transformers integration @@ -263,7 +263,7 @@ jobs: path: sentence-transformers - name: Pull image run: | - docker pull vault.habana.ai/gaudi-docker/1.18.0/ubuntu22.04/habanalabs/pytorch-installer-2.4.0:latest + docker pull vault.habana.ai/gaudi-docker/1.19.0/ubuntu22.04/habanalabs/pytorch-installer-2.5.1:latest - name: Run tests run: | docker run \ @@ -275,5 +275,5 @@ jobs: --cap-add=sys_nice \ --net=host \ --ipc=host \ - vault.habana.ai/gaudi-docker/1.18.0/ubuntu22.04/habanalabs/pytorch-installer-2.4.0:latest \ + vault.habana.ai/gaudi-docker/1.19.0/ubuntu22.04/habanalabs/pytorch-installer-2.5.1:latest \ /bin/bash optimum-habana/tests/ci/sentence_transformers.sh diff --git a/.github/workflows/slow_tests_gaudi2.yml b/.github/workflows/slow_tests_gaudi2.yml index 2e561a2765..86b50d6e2c 100644 --- a/.github/workflows/slow_tests_gaudi2.yml +++ b/.github/workflows/slow_tests_gaudi2.yml @@ -17,7 +17,7 @@ jobs: uses: actions/checkout@v2 - name: Pull image run: | - docker pull vault.habana.ai/gaudi-docker/1.18.0/ubuntu22.04/habanalabs/pytorch-installer-2.4.0:latest + docker pull vault.habana.ai/gaudi-docker/1.19.0/ubuntu22.04/habanalabs/pytorch-installer-2.5.1:latest - name: Run tests run: | docker run \ @@ -30,7 +30,7 @@ jobs: --cap-add=sys_nice \ --net=host \ --ipc=host \ - vault.habana.ai/gaudi-docker/1.18.0/ubuntu22.04/habanalabs/pytorch-installer-2.4.0:latest \ + vault.habana.ai/gaudi-docker/1.19.0/ubuntu22.04/habanalabs/pytorch-installer-2.5.1:latest \ /bin/bash tests/ci/example_diff_tests.sh stable-diffusion: name: Test Stable Diffusion @@ -43,7 +43,7 @@ jobs: uses: actions/checkout@v2 - name: Pull image run: | - docker pull vault.habana.ai/gaudi-docker/1.18.0/ubuntu22.04/habanalabs/pytorch-installer-2.4.0:latest + docker pull vault.habana.ai/gaudi-docker/1.19.0/ubuntu22.04/habanalabs/pytorch-installer-2.5.1:latest - name: Run tests run: | docker run \ @@ -59,8 +59,8 @@ jobs: --cap-add=sys_nice \ --net=host \ --ipc=host \ - vault.habana.ai/gaudi-docker/1.18.0/ubuntu22.04/habanalabs/pytorch-installer-2.4.0:latest \ - /bin/bash tests/ci/slow_tests_diffusers.sh + vault.habana.ai/gaudi-docker/1.19.0/ubuntu22.04/habanalabs/pytorch-installer-2.5.1:latest \ + /bin/bash tests/ci/slow_tests_diffusers.sh ${{ secrets.TEXT_GENERATION_CI_HUB_TOKEN }} deepspeed: name: Test DeepSpeed models if: ${{ !cancelled() && (success() || failure()) }} @@ -72,7 +72,7 @@ jobs: uses: actions/checkout@v2 - name: Pull image run: | - docker pull vault.habana.ai/gaudi-docker/1.18.0/ubuntu22.04/habanalabs/pytorch-installer-2.4.0:latest + docker pull vault.habana.ai/gaudi-docker/1.19.0/ubuntu22.04/habanalabs/pytorch-installer-2.5.1:latest - name: Run tests run: | docker run \ @@ -88,7 +88,7 @@ jobs: --cap-add=sys_nice \ --net=host \ --ipc=host \ - vault.habana.ai/gaudi-docker/1.18.0/ubuntu22.04/habanalabs/pytorch-installer-2.4.0:latest \ + vault.habana.ai/gaudi-docker/1.19.0/ubuntu22.04/habanalabs/pytorch-installer-2.5.1:latest \ /bin/bash tests/ci/slow_tests_deepspeed.sh ${{ secrets.TEXT_GENERATION_CI_HUB_TOKEN }} fsdp: name: Test FSDP models @@ -101,7 +101,7 @@ jobs: uses: actions/checkout@v2 - name: Pull image run: | - docker pull vault.habana.ai/gaudi-docker/1.18.0/ubuntu22.04/habanalabs/pytorch-installer-2.4.0:latest + docker pull vault.habana.ai/gaudi-docker/1.19.0/ubuntu22.04/habanalabs/pytorch-installer-2.5.1:latest - name: Run tests run: | docker run \ @@ -117,7 +117,7 @@ jobs: --cap-add=sys_nice \ --net=host \ --ipc=host \ - vault.habana.ai/gaudi-docker/1.18.0/ubuntu22.04/habanalabs/pytorch-installer-2.4.0:latest \ + vault.habana.ai/gaudi-docker/1.19.0/ubuntu22.04/habanalabs/pytorch-installer-2.5.1:latest \ make slow_tests_fsdp TOKEN=${{ secrets.TEXT_GENERATION_CI_HUB_TOKEN }} multi-card: name: Test multi-card models @@ -130,7 +130,7 @@ jobs: uses: actions/checkout@v2 - name: Pull image run: | - docker pull vault.habana.ai/gaudi-docker/1.18.0/ubuntu22.04/habanalabs/pytorch-installer-2.4.0:latest + docker pull vault.habana.ai/gaudi-docker/1.19.0/ubuntu22.04/habanalabs/pytorch-installer-2.5.1:latest - name: Run tests run: | docker run \ @@ -146,7 +146,7 @@ jobs: --cap-add=sys_nice \ --net=host \ --ipc=host \ - vault.habana.ai/gaudi-docker/1.18.0/ubuntu22.04/habanalabs/pytorch-installer-2.4.0:latest \ + vault.habana.ai/gaudi-docker/1.19.0/ubuntu22.04/habanalabs/pytorch-installer-2.5.1:latest \ /bin/bash tests/ci/slow_tests_8x.sh ${{ secrets.TEXT_GENERATION_CI_HUB_TOKEN }} single-card: name: Test single-card models @@ -160,7 +160,7 @@ jobs: uses: actions/checkout@v2 - name: Pull image run: | - docker pull vault.habana.ai/gaudi-docker/1.18.0/ubuntu22.04/habanalabs/pytorch-installer-2.4.0:latest + docker pull vault.habana.ai/gaudi-docker/1.19.0/ubuntu22.04/habanalabs/pytorch-installer-2.5.1:latest - name: Run tests run: | docker run \ @@ -177,7 +177,7 @@ jobs: --cap-add=sys_nice \ --net=host \ --ipc=host \ - vault.habana.ai/gaudi-docker/1.18.0/ubuntu22.04/habanalabs/pytorch-installer-2.4.0:latest \ + vault.habana.ai/gaudi-docker/1.19.0/ubuntu22.04/habanalabs/pytorch-installer-2.5.1:latest \ /bin/bash tests/ci/slow_tests_1x.sh text-generation: name: Test text-generation example @@ -192,7 +192,7 @@ jobs: uses: actions/checkout@v2 - name: Pull image run: | - docker pull vault.habana.ai/gaudi-docker/1.18.0/ubuntu22.04/habanalabs/pytorch-installer-2.4.0:latest + docker pull vault.habana.ai/gaudi-docker/1.19.0/ubuntu22.04/habanalabs/pytorch-installer-2.5.1:latest - name: Run tests run: | docker run \ @@ -208,7 +208,7 @@ jobs: --cap-add=sys_nice \ --net=host \ --ipc=host \ - vault.habana.ai/gaudi-docker/1.18.0/ubuntu22.04/habanalabs/pytorch-installer-2.4.0:latest \ + vault.habana.ai/gaudi-docker/1.19.0/ubuntu22.04/habanalabs/pytorch-installer-2.5.1:latest \ make slow_tests_text_generation_example TOKEN=${{ secrets.TEXT_GENERATION_CI_HUB_TOKEN }} trl: name: Test TRL integration @@ -221,7 +221,7 @@ jobs: uses: actions/checkout@v2 - name: Pull image run: | - docker pull vault.habana.ai/gaudi-docker/1.18.0/ubuntu22.04/habanalabs/pytorch-installer-2.4.0:latest + docker pull vault.habana.ai/gaudi-docker/1.19.0/ubuntu22.04/habanalabs/pytorch-installer-2.5.1:latest - name: Run tests run: | docker run \ @@ -237,7 +237,7 @@ jobs: --cap-add=sys_nice \ --net=host \ --ipc=host \ - vault.habana.ai/gaudi-docker/1.18.0/ubuntu22.04/habanalabs/pytorch-installer-2.4.0:latest \ + vault.habana.ai/gaudi-docker/1.19.0/ubuntu22.04/habanalabs/pytorch-installer-2.5.1:latest \ /bin/bash tests/ci/slow_tests_trl.sh sentence-transformers: name: Test Sentence Transformers integration @@ -258,7 +258,7 @@ jobs: path: sentence-transformers - name: Pull image run: | - docker pull vault.habana.ai/gaudi-docker/1.18.0/ubuntu22.04/habanalabs/pytorch-installer-2.4.0:latest + docker pull vault.habana.ai/gaudi-docker/1.19.0/ubuntu22.04/habanalabs/pytorch-installer-2.5.1:latest - name: Run tests run: | docker run \ @@ -274,5 +274,5 @@ jobs: --cap-add=sys_nice \ --net=host \ --ipc=host \ - vault.habana.ai/gaudi-docker/1.18.0/ubuntu22.04/habanalabs/pytorch-installer-2.4.0:latest \ + vault.habana.ai/gaudi-docker/1.19.0/ubuntu22.04/habanalabs/pytorch-installer-2.5.1:latest \ /bin/bash optimum-habana/tests/ci/sentence_transformers.sh diff --git a/Makefile b/Makefile index e6989aa1b0..34fd13bd07 100644 --- a/Makefile +++ b/Makefile @@ -93,12 +93,12 @@ slow_tests_8x: test_installs # Run DeepSpeed non-regression tests slow_tests_deepspeed: test_installs - python -m pip install git+https://github.com/HabanaAI/DeepSpeed.git@1.18.0 + python -m pip install git+https://github.com/HabanaAI/DeepSpeed.git@1.19.0 python -m pytest tests/test_examples.py -v -s -k "deepspeed" slow_tests_diffusers: test_installs python -m pip install -r examples/stable-diffusion/requirements.txt - python -m pytest tests/test_diffusers.py -v -s -k "test_textual_inversion" + python -m pytest tests/test_diffusers.py -v -s -k "textual_inversion" python -m pip install peft==0.7.0 python -m pytest tests/test_diffusers.py -v -s -k "test_train_text_to_image_" python -m pytest tests/test_diffusers.py -v -s -k "test_train_controlnet" @@ -107,8 +107,9 @@ slow_tests_diffusers: test_installs # Run text-generation non-regression tests slow_tests_text_generation_example: test_installs + python -m pip install triton==3.1.0 autoawq BUILD_CUDA_EXT=0 python -m pip install -vvv --no-build-isolation git+https://github.com/HabanaAI/AutoGPTQ.git - python -m pip install git+https://github.com/HabanaAI/DeepSpeed.git@1.18.0 + python -m pip install git+https://github.com/HabanaAI/DeepSpeed.git@1.19.0 python -m pytest tests/test_text_generation_example.py tests/test_encoder_decoder.py -v -s --token $(TOKEN) # Run image-to-text non-regression tests @@ -120,6 +121,11 @@ slow_tests_openclip_vqa_example: test_installs python -m pip install -r examples/visual-question-answering/openclip_requirements.txt python -m pytest tests/test_openclip_vqa.py +# Run video comprehension tests +slow_tests_video_llava_example: test_installs + python -m pip install -r examples/video-comprehension/requirements.txt + python -m pytest tests/test_video_llava.py + slow_tests_fsdp: test_installs python -m pytest tests/test_fsdp_examples.py -v -s --token $(TOKEN) diff --git a/README.md b/README.md index 429caebffd..e363edb26e 100644 --- a/README.md +++ b/README.md @@ -25,24 +25,30 @@ limitations under the License. # Optimum for IntelĀ® GaudiĀ® Accelerators -Optimum for Intel Gaudi - a.k.a. `optimum-habana` - is the interface between the Transformers and Diffusers libraries and [Intel Gaudi AI Accelerators (HPU)](https://docs.habana.ai/en/latest/index.html). -It provides a set of tools enabling easy model loading, training and inference on single- and multi-HPU settings for different downstream tasks. -The list of officially validated models and tasks is available [here](https://github.com/huggingface/optimum-habana#validated-models). Users can try other of the thousands of Hugging Face models on Intel Gaudi accelerators and tasks with only few changes. +Optimum for Intel Gaudi - a.k.a. `optimum-habana` - is the interface between the Transformers and Diffusers libraries and +[Intel Gaudi AI Accelerators (HPU)](https://docs.habana.ai/en/latest/index.html). It provides a set of tools enabling easy +model loading, training and inference on single- and multi-HPU settings for different downstream tasks. The list of officially +validated models and tasks is available [here](https://github.com/huggingface/optimum-habana#validated-models). Users can +try other of the thousands of Hugging Face models on Intel Gaudi accelerators and tasks with only few changes. ## What are Intel Gaudi AI Accelerators (HPUs)? HPUs offer fast model training and inference as well as a great price-performance ratio. -Check out [this blog post about BLOOM inference](https://huggingface.co/blog/habana-gaudi-2-bloom) and [this post benchmarking Intel Gaudi 2 and NVIDIA A100 GPUs for BridgeTower training](https://huggingface.co/blog/bridgetower) for concrete examples. +Check out [this blog post about BLOOM inference](https://huggingface.co/blog/habana-gaudi-2-bloom) and +[this post benchmarking Intel Gaudi 2 and NVIDIA A100 GPUs for BridgeTower training](https://huggingface.co/blog/bridgetower) +for concrete examples. ## Gaudi Setup Please refer to the Intel Gaudi AI Accelerator official [installation guide](https://docs.habana.ai/en/latest/Installation_Guide/index.html). -> Tests should be run in a Docker container based on Intel Gaudi Docker images. -> -> The current version has been validated for SynapseAI 1.18. +> [!NOTE] +> Tests should be run in a Docker container based on Intel Gaudi's official images. Instructions to +> obtain the latest containers from the Intel Gaudi Vault are available +> [here](https://docs.habana.ai/en/latest/Installation_Guide/Additional_Installation/Docker_Installation.html#use-intel-gaudi-containers). +> The current Optimum for Intel Gaudi has been validated with Intel Gaudi v1.19 stack. ## Install the library and get example scripts @@ -50,18 +56,18 @@ Please refer to the Intel Gaudi AI Accelerator official [installation guide](htt ### Option 1: Use the latest stable release To install the latest stable release of this package ->```bash ->pip install --upgrade-strategy eager optimum[habana] ->``` +```bash +pip install --upgrade-strategy eager optimum[habana] +``` The `--upgrade-strategy eager` option is needed to ensure `optimum-habana` is upgraded to the latest stable release. To use the example associated with the latest stable release, run: -> ``` -> git clone https://github.com/huggingface/optimum-habana -> cd optimum-habana && git checkout v1.14.0 -> ``` -> with `v1.14.0` the version number of this release. +```bash +git clone https://github.com/huggingface/optimum-habana +cd optimum-habana && git checkout v1.15.0 +``` +with `v1.15.0` being the latest Optimum for Intel Gaudi release version. ### Option 2: Use the latest main branch under development @@ -74,7 +80,8 @@ git clone https://github.com/huggingface/optimum-habana ### Option 3: Use the `transformers_future` branch to have the latest changes from Transformers -The `transformers_future` branch is regularly updated with the latest changes from the main branches of Optimum Habana and Transformers. This enables you to try out new Transformers features that have not been merged into the main branch yet. +The `transformers_future` branch is regularly updated with the latest changes from the main branches of Optimum for Intel Gaudi +and Transformers. This enables you to try out new Transformers features that have not been merged into the main branch yet. > [!WARNING] > The `transformers_future` branch may have some regressions or bugs and may be less stable than the main branch. @@ -84,34 +91,40 @@ pip install git+https://github.com/huggingface/optimum-habana.git@transformers_f git clone -b transformers_future https://github.com/huggingface/optimum-habana ``` -## Install dependencies +## Install Dependencies To use DeepSpeed on HPUs, you also need to run the following command: ->```bash ->pip install git+https://github.com/HabanaAI/DeepSpeed.git@1.18.0 ->``` +```bash +pip install git+https://github.com/HabanaAI/DeepSpeed.git@1.19.0 +``` To install the requirements for every example: ->```bash ->cd ->pip install -r requirements.txt ->``` - +```bash +cd +pip install -r requirements.txt +``` ## How to use it? -### Quick Start +Optimum for Intel Gaudi was designed with one goal in mind: **to make training and inference straightforward for Transformers +and Diffusers users, while fully leveraging the power of Intel Gaudi AI Accelerators**. -Optimum for Intel Gaudi was designed with one goal in mind: **to make training and inference straightforward for Transformers and Diffusers users, while fully leveraging the power of Intel Gaudi AI Accelerators**. - -#### Transformers Interface +### Transformers Interface There are two main classes one needs to know: -- [GaudiTrainer](https://huggingface.co/docs/optimum/habana/package_reference/trainer): the trainer class that takes care of compiling and distributing the model to run on HPUs, and performing training and evaluation. -- [GaudiConfig](https://huggingface.co/docs/optimum/habana/package_reference/gaudi_config): the class that enables to configure Habana Mixed Precision and to decide whether optimized operators and optimizers should be used or not. -The [GaudiTrainer](https://huggingface.co/docs/optimum/habana/package_reference/trainer) is very similar to the [Transformers Trainer](https://huggingface.co/docs/transformers/main_classes/trainer), and adapting a script using the Trainer to make it work with Intel Gaudi accelerators will mostly consist in simply swapping the `Trainer` class for the `GaudiTrainer` one. -That's how most of the [example scripts](https://github.com/huggingface/optimum-habana/tree/main/examples) were adapted from their [original counterparts](https://github.com/huggingface/transformers/tree/main/examples/pytorch). +- [GaudiTrainer](https://huggingface.co/docs/optimum/habana/package_reference/trainer): the trainer class that takes care of + compiling and distributing the model to run on HPUs, and performing training and evaluation. + +- [GaudiConfig](https://huggingface.co/docs/optimum/habana/package_reference/gaudi_config): the class that enables to configure + Gaudi Mixed Precision and to decide whether optimized operators and optimizers should be used or not. + +The [GaudiTrainer](https://huggingface.co/docs/optimum/habana/package_reference/trainer) is very similar to the +[Transformers Trainer](https://huggingface.co/docs/transformers/main_classes/trainer), and adapting a script using the Trainer to +make it work with Intel Gaudi accelerators will mostly consist in simply swapping the `Trainer` class for the `GaudiTrainer` one. + +That's how most of the [example scripts](https://github.com/huggingface/optimum-habana/tree/main/examples) were adapted from their +[original counterparts](https://github.com/huggingface/transformers/tree/main/examples/pytorch). Here is an example: ```diff @@ -141,12 +154,17 @@ Here is an example: ) ``` -where `gaudi_config_name` is the name of a model from the [Hub](https://huggingface.co/Habana) (Intel Gaudi configurations are stored in model repositories) or a path to a local Intel Gaudi configuration file (you can see [here](https://huggingface.co/docs/optimum/habana/package_reference/gaudi_config) how to write your own). +where `gaudi_config_name` is the name of a model from the [Hub](https://huggingface.co/Habana) (Intel Gaudi configurations +are stored in model repositories) or a path to a local Intel Gaudi configuration file (you can see +[here](https://huggingface.co/docs/optimum/habana/package_reference/gaudi_config) how to write your own). -#### Diffusers Interface +### Diffusers Interface -You can generate images from prompts using Stable Diffusion on Intel Gaudi using the [`GaudiStableDiffusionPipeline`](https://huggingface.co/docs/optimum/habana/package_reference/stable_diffusion_pipeline) class and the [`GaudiDDIMScheduler`] which have been both optimized for HPUs. Here is how to use them and the differences with the Diffusers library: +You can generate images from prompts using Stable Diffusion on Intel Gaudi using the +[`GaudiStableDiffusionPipeline`](https://huggingface.co/docs/optimum/habana/package_reference/stable_diffusion_pipeline) class and the +[`GaudiDDIMScheduler`](https://huggingface.co/docs/optimum/habana/package_reference/stable_diffusion_pipeline#optimum.habana.diffusers.GaudiDDIMScheduler) +class which have been both optimized for HPUs. Here is how to use them and the differences with the Diffusers library: ```diff - from diffusers import DDIMScheduler, StableDiffusionPipeline @@ -167,7 +185,7 @@ model_name = "CompVis/stable-diffusion-v1-4" + gaudi_config="Habana/stable-diffusion", ) -outputs = generator( +outputs = pipeline( ["An image of a squirrel in Picasso style"], num_images_per_prompt=16, + batch_size=4, @@ -175,6 +193,27 @@ outputs = generator( ``` +## Important Note on Pytorch 2.5 Performance Degradation + +With the upgrade to PyTorch 2.5, users may experience some performance degradation due to changes in the handling of FP16/BF16 inputs. +The note from PyTorch 2.5 states: + +"A naive SDPA math backend, when using FP16/BF16 inputs, can accumulate significant numerical errors due to the usage of low-precision +intermediate buffers. To mitigate this issue, the default behavior now involves upcasting FP16/BF16 inputs to FP32. Computations are performed +in FP32/TF32, and the final FP32 results are then downcasted back to FP16/BF16. This will improve numerical accuracy of the final output for +the math backend with FP16/BF16 inputs, but increases memory usages and may cause the performance regressions in the math backend as computations +shift from FP16/BF16 BMM to FP32/TF32 BMM/Matmul." + +For scenarios where reduced-precision reductions are preferred for speed, they can be enabled with the following setting: +```python +torch.backends.cuda.allow_fp16_bf16_reduction_math_sdp(True) +``` +Additionally, the next release of Optimum Habana will include a Gaudi-specific safe_softmax implementation that will also improve performance. + +More info: +- https://pytorch.org/docs/stable/notes/numerical_accuracy.html + + ### Documentation Check out [the documentation of Optimum for Intel Gaudi](https://huggingface.co/docs/optimum/habana/index) for more advanced usage. @@ -184,108 +223,106 @@ Check out [the documentation of Optimum for Intel Gaudi](https://huggingface.co/ The following model architectures, tasks and device distributions have been validated for Optimum for Intel Gaudi: +> [!NOTE] > In the tables below, :heavy_check_mark: means single-card, multi-card and DeepSpeed have all been validated. -- Transformers: -
+### Transformers: -| Architecture | Training | Inference |
Tasks
| -|--------------|:--------:|:---------:|:-----------------------| +| Architecture | Training | Inference | Tasks | +|:-------------|:--------:|:---------:|:------| | BERT | :heavy_check_mark: | :heavy_check_mark: |
  • [text classification](https://github.com/huggingface/optimum-habana/tree/main/examples/text-classification)
  • [question answering](https://github.com/huggingface/optimum-habana/tree/main/examples/question-answering)
  • [language modeling](https://github.com/huggingface/optimum-habana/tree/main/examples/language-modeling)
  • [text feature extraction](https://github.com/huggingface/optimum-habana/tree/main/examples/text-feature-extraction)
    • | | RoBERTa | :heavy_check_mark: | :heavy_check_mark: |
  • [question answering](https://github.com/huggingface/optimum-habana/tree/main/examples/question-answering)
  • [language modeling](https://github.com/huggingface/optimum-habana/tree/main/examples/language-modeling)
    • | | ALBERT | :heavy_check_mark: | :heavy_check_mark: |
  • [question answering](https://github.com/huggingface/optimum-habana/tree/main/examples/question-answering)
  • [language modeling](https://github.com/huggingface/optimum-habana/tree/main/examples/language-modeling)
    • | | DistilBERT |:heavy_check_mark: | :heavy_check_mark: |
  • [question answering](https://github.com/huggingface/optimum-habana/tree/main/examples/question-answering)
  • [language modeling](https://github.com/huggingface/optimum-habana/tree/main/examples/language-modeling)
    • | | GPT2 | :heavy_check_mark: | :heavy_check_mark: |
  • [language modeling](https://github.com/huggingface/optimum-habana/tree/main/examples/language-modeling)
  • [text generation](https://github.com/huggingface/optimum-habana/tree/main/examples/text-generation)
    • | -| BLOOM(Z) | |
  • DeepSpeed
    • |
  • [text generation](https://github.com/huggingface/optimum-habana/tree/main/examples/text-generation)
    • | -| StarCoder / StarCoder2 | :heavy_check_mark: |
  • Single card
    • |
  • [language modeling](https://github.com/huggingface/optimum-habana/tree/main/examples/language-modeling)
  • [text generation](https://github.com/huggingface/optimum-habana/tree/main/examples/text-generation)
    • | -| GPT-J |
  • DeepSpeed
    • |
  • Single card
  • DeepSpeed
    • |
  • [language modeling](https://github.com/huggingface/optimum-habana/tree/main/examples/language-modeling)
  • [text generation](https://github.com/huggingface/optimum-habana/tree/main/examples/text-generation)
    • | -| GPT-Neo | |
  • Single card
    • |
  • [text generation](https://github.com/huggingface/optimum-habana/tree/main/examples/text-generation)
    • | -| GPT-NeoX |
  • DeepSpeed
    • |
  • DeepSpeed
    • |
  • [language modeling](https://github.com/huggingface/optimum-habana/tree/main/examples/language-modeling)
  • [text generation](https://github.com/huggingface/optimum-habana/tree/main/examples/text-generation)
    • | -| OPT | |
  • DeepSpeed
    • |
  • [text generation](https://github.com/huggingface/optimum-habana/tree/main/examples/text-generation)
    • | +| BLOOM(Z) | |
  • DeepSpeed
    • |
  • [text generation](https://github.com/huggingface/optimum-habana/tree/main/examples/text-generation)
    • | +| StarCoder / StarCoder2 | :heavy_check_mark: |
  • Single-card
    • |
  • [language modeling](https://github.com/huggingface/optimum-habana/tree/main/examples/language-modeling)
  • [text generation](https://github.com/huggingface/optimum-habana/tree/main/examples/text-generation)
    • | +| GPT-J |
  • DeepSpeed
    • |
  • Single card
  • DeepSpeed
    • |
  • [language modeling](https://github.com/huggingface/optimum-habana/tree/main/examples/language-modeling)
  • [text generation](https://github.com/huggingface/optimum-habana/tree/main/examples/text-generation)
    • | +| GPT-Neo | |
  • Single card
    • |
  • [text generation](https://github.com/huggingface/optimum-habana/tree/main/examples/text-generation)
    • | +| GPT-NeoX |
  • DeepSpeed
    • |
  • DeepSpeed
    • |
  • [language modeling](https://github.com/huggingface/optimum-habana/tree/main/examples/language-modeling)
  • [text generation](https://github.com/huggingface/optimum-habana/tree/main/examples/text-generation)
    • | +| OPT | |
  • DeepSpeed
    • |
  • [text generation](https://github.com/huggingface/optimum-habana/tree/main/examples/text-generation)
    • | | Llama 2 / CodeLlama / Llama 3 / Llama Guard / Granite | :heavy_check_mark: | :heavy_check_mark: |
  • [language modeling](https://github.com/huggingface/optimum-habana/tree/main/examples/language-modeling)
  • [text generation](https://github.com/huggingface/optimum-habana/tree/main/examples/text-generation)
  • [question answering](https://github.com/huggingface/optimum-habana/tree/main/examples/question-answering)
  • [text classification](https://github.com/huggingface/optimum-habana/tree/main/examples/text-classification) (Llama Guard)
    • | -| StableLM | |
  • Single card
    • |
  • [text generation](https://github.com/huggingface/optimum-habana/tree/main/examples/text-generation)
    • | -| Falcon |
  • LoRA
    • | :heavy_check_mark: |
  • [language modeling](https://github.com/huggingface/optimum-habana/tree/main/examples/language-modeling)
  • [text generation](https://github.com/huggingface/optimum-habana/tree/main/examples/text-generation)
    • | -| CodeGen | |
  • Single card
    • |
  • [text generation](https://github.com/huggingface/optimum-habana/tree/main/examples/text-generation)
    • | -| MPT | |
  • Single card
    • |
  • [text generation](https://github.com/huggingface/optimum-habana/tree/main/examples/text-generation)
    • | -| Mistral | |
  • Single card
    • |
  • [text generation](https://github.com/huggingface/optimum-habana/tree/main/examples/text-generation)
    • | -| Phi | :heavy_check_mark: |
  • Single card
    • |
  • [language modeling](https://github.com/huggingface/optimum-habana/tree/main/examples/language-modeling)
  • [text generation](https://github.com/huggingface/optimum-habana/tree/main/examples/text-generation)
    • | -| Mixtral | |
  • Single card
    • |
  • [text generation](https://github.com/huggingface/optimum-habana/tree/main/examples/text-generation)
    • | -| Persimmon | |
  • Single card
    • |
  • [text generation](https://github.com/huggingface/optimum-habana/tree/main/examples/text-generation)
    • | -| Qwen2 |
  • Single card
    • |
  • Single card
    • |
  • [language modeling](https://github.com/huggingface/optimum-habana/tree/main/examples/language-modeling)
  • [text generation](https://github.com/huggingface/optimum-habana/tree/main/examples/text-generation)
    • | -| Qwen2-MoE | |
  • Single card
    • |
  • [text generation](https://github.com/huggingface/optimum-habana/tree/main/examples/text-generation)
    • | -| Gemma | :heavy_check_mark: |
  • Single card
    • |
  • [language modeling](https://github.com/huggingface/optimum-habana/tree/main/examples/language-modeling)
  • [text generation](https://github.com/huggingface/optimum-habana/tree/main/examples/text-generation)
    • | +| StableLM | |
  • Single card
    • |
  • [text generation](https://github.com/huggingface/optimum-habana/tree/main/examples/text-generation)
    • | +| Falcon |
  • LoRA
    • | :heavy_check_mark: |
  • [language modeling](https://github.com/huggingface/optimum-habana/tree/main/examples/language-modeling)
  • [text generation](https://github.com/huggingface/optimum-habana/tree/main/examples/text-generation)
    • | +| CodeGen | |
  • Single card
    • |
  • [text generation](https://github.com/huggingface/optimum-habana/tree/main/examples/text-generation)
    • | +| MPT | |
  • Single card
    • |
  • [text generation](https://github.com/huggingface/optimum-habana/tree/main/examples/text-generation)
    • | +| Mistral | |
  • Single card
    • |
  • [text generation](https://github.com/huggingface/optimum-habana/tree/main/examples/text-generation)
    • | +| Phi | :heavy_check_mark: |
  • Single card
    • |
  • [language modeling](https://github.com/huggingface/optimum-habana/tree/main/examples/language-modeling)
  • [text generation](https://github.com/huggingface/optimum-habana/tree/main/examples/text-generation)
    • | +| Mixtral | |
  • Single card
    • |
  • [text generation](https://github.com/huggingface/optimum-habana/tree/main/examples/text-generation)
    • | +| Persimmon | |
  • Single card
    • |
  • [text generation](https://github.com/huggingface/optimum-habana/tree/main/examples/text-generation)
    • | +| Qwen2 |
  • Single card
    • |
  • Single card
    • |
  • [language modeling](https://github.com/huggingface/optimum-habana/tree/main/examples/language-modeling)
  • [text generation](https://github.com/huggingface/optimum-habana/tree/main/examples/text-generation)
    • | +| Qwen2-MoE | |
  • Single card
    • |
  • [text generation](https://github.com/huggingface/optimum-habana/tree/main/examples/text-generation)
    • | +| Gemma | :heavy_check_mark: |
  • Single card
    • |
  • [language modeling](https://github.com/huggingface/optimum-habana/tree/main/examples/language-modeling)
  • [text generation](https://github.com/huggingface/optimum-habana/tree/main/examples/text-generation)
    • | | Gemma2 | | :heavy_check_mark: |
  • [text generation](https://github.com/huggingface/optimum-habana/tree/main/examples/text-generation)
    • | -| XGLM | |
  • Single card
    • |
  • [text generation](https://github.com/huggingface/optimum-habana/tree/main/examples/text-generation)
    • | -| Cohere | |
  • Single card
    • |
  • [text generation](https://github.com/huggingface/optimum-habana/tree/main/examples/text-generation)
    • | +| XGLM | |
  • Single card
    • |
  • [text generation](https://github.com/huggingface/optimum-habana/tree/main/examples/text-generation)
    • | +| Cohere | |
  • Single card
    • |
  • [text generation](https://github.com/huggingface/optimum-habana/tree/main/examples/text-generation)
    • | | T5 / Flan T5 | :heavy_check_mark: | :heavy_check_mark: |
  • [summarization](https://github.com/huggingface/optimum-habana/tree/main/examples/summarization)
  • [translation](https://github.com/huggingface/optimum-habana/tree/main/examples/translation)
  • [question answering](https://github.com/huggingface/optimum-habana/tree/main/examples/question-answering#fine-tuning-t5-on-squad20)
    • | -| BART | |
  • Single card
    • |
  • [summarization](https://github.com/huggingface/optimum-habana/tree/main/examples/summarization)
  • [translation](https://github.com/huggingface/optimum-habana/tree/main/examples/translation)
  • [question answering](https://github.com/huggingface/optimum-habana/tree/main/examples/question-answering#fine-tuning-t5-on-squad20)
    • | +| BART | |
  • Single card
    • |
  • [summarization](https://github.com/huggingface/optimum-habana/tree/main/examples/summarization)
  • [translation](https://github.com/huggingface/optimum-habana/tree/main/examples/translation)
  • [question answering](https://github.com/huggingface/optimum-habana/tree/main/examples/question-answering#fine-tuning-t5-on-squad20)
    • | | ViT | :heavy_check_mark: | :heavy_check_mark: |
  • [image classification](https://github.com/huggingface/optimum-habana/tree/main/examples/image-classification)
    • | | Swin | :heavy_check_mark: | :heavy_check_mark: |
  • [image classification](https://github.com/huggingface/optimum-habana/tree/main/examples/image-classification)
    • | | Wav2Vec2 | :heavy_check_mark: | :heavy_check_mark: |
  • [audio classification](https://github.com/huggingface/optimum-habana/tree/main/examples/audio-classification)
  • [speech recognition](https://github.com/huggingface/optimum-habana/tree/main/examples/speech-recognition)
    • | | Whisper | :heavy_check_mark: | :heavy_check_mark: |
  • [speech recognition](https://github.com/huggingface/optimum-habana/tree/main/examples/speech-recognition)
    • | -| SpeechT5 | |
  • Single card
    • |
  • [text to speech](https://github.com/huggingface/optimum-habana/tree/main/examples/text-to-speech)
    • | +| SpeechT5 | |
  • Single card
    • |
  • [text to speech](https://github.com/huggingface/optimum-habana/tree/main/examples/text-to-speech)
    • | | CLIP | :heavy_check_mark: | :heavy_check_mark: |
  • [contrastive image-text training](https://github.com/huggingface/optimum-habana/tree/main/examples/contrastive-image-text)
    • | | BridgeTower | :heavy_check_mark: | :heavy_check_mark: |
  • [contrastive image-text training](https://github.com/huggingface/optimum-habana/tree/main/examples/contrastive-image-text)
    • | -| ESMFold | |
  • Single card
    • |
  • [protein folding](https://github.com/huggingface/optimum-habana/tree/main/examples/protein-folding)
    • | -| Blip | |
  • Single card
    • |
  • [visual question answering](https://github.com/huggingface/optimum-habana/tree/main/examples/visual-question-answering)
  • [image to text](https://github.com/huggingface/optimum-habana/tree/main/examples/image-to-text)
    • | -| OWLViT | |
  • Single card
    • |
  • [zero shot object detection](https://github.com/huggingface/optimum-habana/tree/main/examples/zero-shot-object-detection)
    • | -| ClipSeg | |
  • Single card
    • |
  • [object segmentation](https://github.com/huggingface/optimum-habana/tree/main/examples/object-segementation)
    • | -| Llava / Llava-next | |
  • Single card
    • |
  • [image to text](https://github.com/huggingface/optimum-habana/tree/main/examples/image-to-text)
    • | -| idefics2 |
  • LoRA
    • |
  • Single card
    • |
  • [image to text](https://github.com/huggingface/optimum-habana/tree/main/examples/image-to-text)
    • | -| Paligemma | |
  • Single card
    • |
  • [image to text](https://github.com/huggingface/optimum-habana/tree/main/examples/image-to-text)
    • | -| Segment Anything Model | |
  • Single card
    • |
  • [object segmentation](https://github.com/huggingface/optimum-habana/tree/main/examples/object-segementation)
    • | -| VideoMAE | |
  • Single card
    • |
  • [Video classification](https://github.com/huggingface/optimum-habana/tree/main/examples/video-classification)
    • | -| TableTransformer | |
  • Single card
    • |
  • [table object detection](https://github.com/huggingface/optimum-habana/tree/main/examples/table-detection)
    • | -| DETR | |
  • Single card
    • |
  • [object detection](https://github.com/huggingface/optimum-habana/tree/main/examples/object-detection)
    • | -| Mllama |
  • LoRA
    • | :heavy_check_mark: |
  • [image to text](https://github.com/huggingface/optimum-habana/tree/main/examples/image-to-text)
    • | -| MiniCPM3 | |
  • Single card
    • |
  • [text generation](https://github.com/huggingface/optimum-habana/tree/main/examples/text-generation)
    • | -| Baichuan2 |
  • DeepSpeed
    • |
  • Single card
    • |
  • [language modeling](https://github.com/huggingface/optimum-habana/tree/main/examples/language-modeling)
  • [text generation](https://github.com/huggingface/optimum-habana/tree/main/examples/text-generation)
    • | -| DeepSeek-V2 | | :heavy_check_mark: |
  • [text generation](https://github.com/huggingface/optimum-habana/tree/main/examples/text-generation)
    • | -| ChatGLM |
  • DeepSpeed
    • |
  • Single card
    • |
  • [language modeling](https://github.com/huggingface/optimum-habana/tree/main/examples/language-modeling)
  • [text generation](https://github.com/huggingface/optimum-habana/tree/main/examples/text-generation)
    • | -
    - -- Diffusers: - -
    - -| Architecture | Training | Inference | Tasks | -|------------------|:--------:|:--------------------:|:------| -| Stable Diffusion |
  • [textual inversion](https://github.com/huggingface/optimum-habana/tree/main/examples/stable-diffusion/training#textual-inversion)
  • [ControlNet](https://github.com/huggingface/optimum-habana/tree/main/examples/stable-diffusion/training#controlnet-training)
    • |
  • Single card
    • |
  • [text-to-image generation](https://github.com/huggingface/optimum-habana/tree/main/examples/stable-diffusion)
    • | -| Stable Diffusion XL |
  • [fine-tuning](https://github.com/huggingface/optimum-habana/tree/main/examples/stable-diffusion/training#fine-tuning-for-stable-diffusion-xl)
    • |
  • Single card
    • |
  • [text-to-image generation](https://github.com/huggingface/optimum-habana/tree/main/examples/stable-diffusion)
    • | -| Stable Diffusion Depth2img | |
  • Single card
    • |
  • [depth-to-image generation](https://github.com/huggingface/optimum-habana/tree/main/examples/stable-diffusion)
    • | -| LDM3D | |
  • Single card
    • |
  • [text-to-image generation](https://github.com/huggingface/optimum-habana/tree/main/examples/stable-diffusion)
    • | -| FLUX.1 |
  • [fine-tuning](https://github.com/huggingface/optimum-habana/tree/main/examples/stable-diffusion/training#dreambooth-lora-fine-tuning-with-flux1-dev)
    • |
  • Single card
    • |
  • [text-to-image generation](https://github.com/huggingface/optimum-habana/tree/main/examples/stable-diffusion)
    • | -| Text to Video | |
  • Single card
    • |
  • [text-to-video generation](https://github.com/huggingface/optimum-habana/tree/main/examples/text-to-video)
    • | +| ESMFold | |
  • Single card
    • |
  • [protein folding](https://github.com/huggingface/optimum-habana/tree/main/examples/protein-folding)
    • | +| Blip | |
  • Single card
    • |
  • [visual question answering](https://github.com/huggingface/optimum-habana/tree/main/examples/visual-question-answering)
  • [image to text](https://github.com/huggingface/optimum-habana/tree/main/examples/image-to-text)
    • | +| OWLViT | |
  • Single card
    • |
  • [zero shot object detection](https://github.com/huggingface/optimum-habana/tree/main/examples/zero-shot-object-detection)
    • | +| ClipSeg | |
  • Single card
    • |
  • [object segmentation](https://github.com/huggingface/optimum-habana/tree/main/examples/object-segementation)
    • | +| Llava / Llava-next | |
  • Single card
    • |
  • [image to text](https://github.com/huggingface/optimum-habana/tree/main/examples/image-to-text)
    • | +| idefics2 |
  • LoRA
    • |
  • Single card
    • |
  • [image to text](https://github.com/huggingface/optimum-habana/tree/main/examples/image-to-text)
    • | +| Paligemma | |
  • Single card
    • |
  • [image to text](https://github.com/huggingface/optimum-habana/tree/main/examples/image-to-text)
    • | +| Segment Anything Model | |
  • Single card
    • |
  • [object segmentation](https://github.com/huggingface/optimum-habana/tree/main/examples/object-segementation)
    • | +| VideoMAE | |
  • Single card
    • |
  • [Video classification](https://github.com/huggingface/optimum-habana/tree/main/examples/video-classification)
    • | +| TableTransformer | |
  • Single card
    • |
  • [table object detection](https://github.com/huggingface/optimum-habana/tree/main/examples/table-detection)
    • | +| DETR | |
  • Single card
    • |
  • [object detection](https://github.com/huggingface/optimum-habana/tree/main/examples/object-detection)
    • | +| Mllama |
  • LoRA
    • | :heavy_check_mark: |
  • [image to text](https://github.com/huggingface/optimum-habana/tree/main/examples/image-to-text)
    • | +| MiniCPM3 | |
  • Single card
    • |
  • [text generation](https://github.com/huggingface/optimum-habana/tree/main/examples/text-generation)
    • | +| Baichuan2 |
  • DeepSpeed
    • |
  • Single card
    • |
  • [language modeling](https://github.com/huggingface/optimum-habana/tree/main/examples/language-modeling)
  • [text generation](https://github.com/huggingface/optimum-habana/tree/main/examples/text-generation)
    • | +| DeepSeek-V2 | :heavy_check_mark: | :heavy_check_mark: |
  • [text generation](https://github.com/huggingface/optimum-habana/tree/main/examples/text-generation)
    • | +| ChatGLM |
  • DeepSpeed
    • |
  • Single card
    • |
  • [language modeling](https://github.com/huggingface/optimum-habana/tree/main/examples/language-modeling)
  • [text generation](https://github.com/huggingface/optimum-habana/tree/main/examples/text-generation)
    • | +| Qwen2-VL | |
  • Single card
    • |
  • [image to text](https://github.com/huggingface/optimum-habana/tree/main/examples/image-to-text)
    • | +| VideoLLaVA | |
  • Single card
    • |
  • [Video comprehension](https://github.com/huggingface/optimum-habana/tree/main/examples/video-comprehension)
    • |
    -- PyTorch Image Models/TIMM: -
    +### Diffusers: | Architecture | Training | Inference | Tasks | -|---------------------|:--------:|:---------:|:------| -| FastViT | |
  • Single card
    • |
  • [image classification](https://github.com/huggingface/optimum-habana/tree/main/examples/image-classification)
    • | - -
    +|:--------------------|:--------:|:---------:|:------| +| Stable Diffusion | :heavy_check_mark: | :heavy_check_mark: |
  • [text-to-image generation](https://github.com/huggingface/optimum-habana/tree/main/examples/stable-diffusion#stable-diffusion)
  • [image-to-image generation](https://github.com/huggingface/optimum-habana/tree/main/examples/stable-diffusion#stable-diffusion-based-image-to-image)
    • | +| Stable Diffusion XL | :heavy_check_mark: | :heavy_check_mark: |
  • [text-to-image generation](https://github.com/huggingface/optimum-habana/tree/main/examples/stable-diffusion#stable-diffusion-xl-sdxl)
  • [image-to-image generation](https://github.com/huggingface/optimum-habana/tree/main/examples/stable-diffusion#stable-diffusion-xl-refiner)
    • | +| Stable Diffusion Depth2img | |
  • Single card
    • |
  • [depth-to-image generation](https://github.com/huggingface/optimum-habana/tree/main/examples/stable-diffusion#depth-to-image-generation)
    • | +| Stable Diffusion 3 | |
  • Single card
    • |
  • [text-to-image generation](https://github.com/huggingface/optimum-habana/tree/main/examples/stable-diffusion#stable-diffusion-3-sd3)
    • | +| LDM3D | |
  • Single card
    • |
  • [text-to-image generation](https://github.com/huggingface/optimum-habana/tree/main/examples/stable-diffusion#latent-diffusion-model-for-3d-ldm3d)
    • | +| FLUX.1 |
  • LoRA
    • |
  • Single card
    • |
  • [text-to-image generation](https://github.com/huggingface/optimum-habana/tree/main/examples/stable-diffusion#flux1)
  • [image-to-image generation](https://github.com/huggingface/optimum-habana/tree/main/examples/stable-diffusion#flux1-image-to-image)
    • | +| Text to Video | |
  • Single card
    • |
  • [text-to-video generation](https://github.com/huggingface/optimum-habana/tree/main/examples/stable-diffusion#text-to-video-generation)
    • | +| Image to Video | |
  • Single card
    • |
  • [image-to-video generation](https://github.com/huggingface/optimum-habana/tree/main/examples/stable-diffusion#image-to-video-generation)
    • | + +### PyTorch Image Models/TIMM: -- TRL: +| Architecture | Training | Inference | Tasks | +|:--------------------|:--------:|:---------:|:------| +| FastViT | |
  • Single card
    • |
  • [image classification](https://github.com/huggingface/optimum-habana/tree/main/examples/image-classification)
    • | -
    +### TRL: | Architecture | Training | Inference | Tasks | -|------------------|:--------:|:--------------------:|:-----------------------------------------------------------------------------------------------| -| Llama 2 | :heavy_check_mark: | |
  • [DPO Pipeline](https://github.com/huggingface/optimum-habana/tree/main/examples/trl)
    • | -| Llama 2 | :heavy_check_mark: | |
  • [PPO Pipeline](https://github.com/huggingface/optimum-habana/tree/main/examples/trl)
    • | -| Stable Diffusion | :heavy_check_mark: | |
  • [DDPO Pipeline](https://github.com/huggingface/optimum-habana/tree/main/examples/trl)
    • | - -
    +|:-----------------|:--------:|:--------------------:|:-----------------------------------------------------------------------------------------------| +| Llama 2 | :heavy_check_mark: | |
  • [DPO Pipeline](https://github.com/huggingface/optimum-habana/tree/main/examples/trl#dpo-pipeline)
    • | +| Llama 2 | :heavy_check_mark: | |
  • [PPO Pipeline](https://github.com/huggingface/optimum-habana/tree/main/examples/trl#ppo-pipeline)
    • | +| Stable Diffusion | :heavy_check_mark: | |
  • [DDPO Pipeline](https://github.com/huggingface/optimum-habana/tree/main/examples/trl#ddpo-pipeline)
    • | -Other models and tasks supported by the Transformers and Diffusers libraries may also work. You can refer to this [section](https://github.com/huggingface/optimum-habana#how-to-use-it) for using them with Optimum for Intel Gaudi. In addition, [this page](https://github.com/huggingface/optimum-habana/tree/main/examples) explains how to modify any [example](https://github.com/huggingface/transformers/tree/main/examples/pytorch) from the Transformers library to make it work with Optimum for Intel Gaudi. +Other models and tasks supported by the Transformers and Diffusers libraries may also work. You can refer to this [section](https://github.com/huggingface/optimum-habana#how-to-use-it) +for using them with Optimum for Intel Gaudi. In addition, [this page](https://github.com/huggingface/optimum-habana/tree/main/examples) explains how to modify any +[example](https://github.com/huggingface/transformers/tree/main/examples/pytorch) from the Transformers library to make it work with Optimum for Intel Gaudi. If you find any issues while using those, please open an issue or a pull request. -After training your model, feel free to submit it to the Intel [leaderboard](https://huggingface.co/spaces/Intel/powered_by_intel_llm_leaderboard) which is designed to evaluate, score, and rank open-source LLMs that have been pre-trained or fine-tuned on Intel Hardwares. Models submitted to the leaderboard will be evaluated on the Intel Developer Cloud. The evaluation platform consists of Gaudi Accelerators and Xeon CPUs running benchmarks from the Eleuther AI Language Model Evaluation Harness. +After training your model, feel free to submit it to the Intel [leaderboard](https://huggingface.co/spaces/Intel/powered_by_intel_llm_leaderboard) which is designed +to evaluate, score, and rank open-source LLMs that have been pre-trained or fine-tuned on Intel Hardwares. Models submitted to the leaderboard will be evaluated on +the Intel Developer Cloud. The evaluation platform consists of Gaudi Accelerators and Xeon CPUs running benchmarks from the Eleuther AI Language Model Evaluation Harness. ## Development diff --git a/conftest.py b/conftest.py index 5775644c48..71cb6bb7ca 100644 --- a/conftest.py +++ b/conftest.py @@ -1,88 +1,3 @@ -# coding=utf-8 -# Copyright 2020 The HuggingFace Team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# tests directory-specific settings - this file is run automatically -# by pytest before any tests are run -import doctest -import sys -import warnings -from os.path import abspath, dirname, join - -import _pytest -import pytest -from transformers.testing_utils import HfDoctestModule, HfDocTestParser - - -NOT_DEVICE_TESTS = { - "test_tokenization", - "test_processor", - "test_processing", - "test_beam_constraints", - "test_configuration_utils", - "test_data_collator", - "test_trainer_callback", - "test_trainer_utils", - "test_feature_extraction", - "test_image_processing", - "test_image_processor", - "test_image_transforms", - "test_optimization", - "test_retrieval", - "test_config", - "test_from_pretrained_no_checkpoint", - "test_keep_in_fp32_modules", - "test_gradient_checkpointing_backward_compatibility", - "test_gradient_checkpointing_enable_disable", - "test_save_load_fast_init_from_base", - "test_fast_init_context_manager", - "test_fast_init_tied_embeddings", - "test_save_load_fast_init_to_base", - "test_torch_save_load", - "test_initialization", - "test_forward_signature", - "test_model_get_set_embeddings", - "test_model_main_input_name", - "test_correct_missing_keys", - "test_tie_model_weights", - "test_can_use_safetensors", - "test_load_save_without_tied_weights", - "test_tied_weights_keys", - "test_model_weights_reload_no_missing_tied_weights", - "test_pt_tf_model_equivalence", - "test_mismatched_shapes_have_properly_initialized_weights", - "test_matched_shapes_have_loaded_weights_when_some_mismatched_shapes_exist", - "test_model_is_small", - "test_tf_from_pt_safetensors", - "test_flax_from_pt_safetensors", - "ModelTest::test_pipeline_", # None of the pipeline tests from PipelineTesterMixin (of which XxxModelTest inherits from) are running on device - "ModelTester::test_pipeline_", - "/repo_utils/", - "/utils/", - "/agents/", -} - -# allow having multiple repository checkouts and not needing to remember to rerun -# `pip install -e '.[dev]'` when switching between checkouts and running tests. -git_repo_path = abspath(join(dirname(__file__), "src")) -sys.path.insert(1, git_repo_path) - -# silence FutureWarning warnings in tests since often we can't act on them until -# they become normal warnings - i.e. the tests still need to test the current functionality -warnings.simplefilter(action="ignore", category=FutureWarning) - - class Secret: """ Taken from: https://stackoverflow.com/a/67393351 @@ -98,47 +13,9 @@ def __str___(self): return "*******" -def pytest_configure(config): - config.addinivalue_line( - "markers", "is_pt_tf_cross_test: mark test to run only when PT and TF interactions are tested" - ) - config.addinivalue_line( - "markers", "is_pt_flax_cross_test: mark test to run only when PT and FLAX interactions are tested" - ) - config.addinivalue_line("markers", "is_pipeline_test: mark test to run only when pipelines are tested") - config.addinivalue_line("markers", "is_staging_test: mark test to run only in the staging environment") - config.addinivalue_line("markers", "accelerate_tests: mark test that require accelerate") - config.addinivalue_line("markers", "agent_tests: mark the agent tests that are run on their specific schedule") - config.addinivalue_line("markers", "not_device_test: mark the tests always running on cpu") - - -def pytest_collection_modifyitems(items): - for item in items: - if any(test_name in item.nodeid for test_name in NOT_DEVICE_TESTS): - item.add_marker(pytest.mark.not_device_test) - - def pytest_addoption(parser): parser.addoption("--token", action="store", default=None) - from transformers.testing_utils import pytest_addoption_shared - - pytest_addoption_shared(parser) - - -def pytest_terminal_summary(terminalreporter): - from transformers.testing_utils import pytest_terminal_summary_main - - make_reports = terminalreporter.config.getoption("--make-reports") - if make_reports: - pytest_terminal_summary_main(terminalreporter, id=make_reports) - - -def pytest_sessionfinish(session, exitstatus): - # If no tests are collected, pytest exists with code 5, which makes the CI fail. - if exitstatus == 5: - session.exitstatus = 0 - def pytest_generate_tests(metafunc): # This is called for every test. Only get/set command line arguments @@ -146,21 +23,3 @@ def pytest_generate_tests(metafunc): option_value = Secret(metafunc.config.option.token) if "token" in metafunc.fixturenames: metafunc.parametrize("token", [option_value]) - - -# Doctest custom flag to ignore output. -IGNORE_RESULT = doctest.register_optionflag("IGNORE_RESULT") - -OutputChecker = doctest.OutputChecker - - -class CustomOutputChecker(OutputChecker): - def check_output(self, want, got, optionflags): - if IGNORE_RESULT & optionflags: - return True - return OutputChecker.check_output(self, want, got, optionflags) - - -doctest.OutputChecker = CustomOutputChecker -_pytest.doctest.DoctestModule = HfDoctestModule -doctest.DocTestParser = HfDocTestParser diff --git a/docs/Dockerfile b/docs/Dockerfile index 6dd8d3a29f..060b7413dc 100644 --- a/docs/Dockerfile +++ b/docs/Dockerfile @@ -1,4 +1,4 @@ -FROM vault.habana.ai/gaudi-docker/1.18.0/ubuntu22.04/habanalabs/pytorch-installer-2.4.0:latest +FROM vault.habana.ai/gaudi-docker/1.19.0/ubuntu22.04/habanalabs/pytorch-installer-2.5.1:latest ARG commit_sha ARG clone_url diff --git a/docs/source/index.mdx b/docs/source/index.mdx index 51d6dadf0f..2b8cdf06ef 100644 --- a/docs/source/index.mdx +++ b/docs/source/index.mdx @@ -105,10 +105,12 @@ In the tables below, āœ… means single-card, multi-card and DeepSpeed have all be | TableTransformer | |
  • Single card
    • |
  • [table object detection](https://github.com/huggingface/optimum-habana/tree/main/examples/table-detection)
    • | | DETR | |
  • Single card
    • |
  • [object detection](https://github.com/huggingface/optimum-habana/tree/main/examples/object-detection)
    • | | Mllama |
  • LoRA
    • |āœ… |
  • [image to text](https://github.com/huggingface/optimum-habana/tree/main/examples/image-to-text)
    • | +| Video-LLaVA | |
  • Single card
    • |
  • [video comprehension](https://github.com/huggingface/optimum-habana/tree/main/examples/video-comprehension)
    • | | MiniCPM3 | |
  • Single card
    • |
  • [text generation](https://github.com/huggingface/optimum-habana/tree/main/examples/text-generation)
    • | | Baichuan2 |
  • DeepSpeed
    • |
  • Single card
    • |
  • [language modeling](https://github.com/huggingface/optimum-habana/tree/main/examples/language-modeling)
  • [text generation](https://github.com/huggingface/optimum-habana/tree/main/examples/text-generation)
    • | -| DeepSeek-V2 | | āœ… |
  • [text generation](https://github.com/huggingface/optimum-habana/tree/main/examples/text-generation)
    • | +| DeepSeek-V2 | āœ… | āœ… |
  • [text generation](https://github.com/huggingface/optimum-habana/tree/main/examples/text-generation)
    • | | ChatGLM |
  • DeepSpeed
    • |
  • Single card
    • |
  • [language modeling](https://github.com/huggingface/optimum-habana/tree/main/examples/language-modeling)
  • [text generation](https://github.com/huggingface/optimum-habana/tree/main/examples/text-generation)
    • | +| Qwen2-VL | |
  • Single card
    • |
  • [image to text](https://github.com/huggingface/optimum-habana/tree/main/examples/image-to-text)
    • | - Diffusers @@ -119,7 +121,7 @@ In the tables below, āœ… means single-card, multi-card and DeepSpeed have all be | Stable Diffusion Depth2img | |
  • Single card
    • |
  • [depth-to-image generation](https://github.com/huggingface/optimum-habana/tree/main/examples/stable-diffusion)
    • | | LDM3D | |
  • Single card
    • |
  • [text-to-image generation](https://github.com/huggingface/optimum-habana/tree/main/examples/stable-diffusion)
    • | | FLUX.1 |
  • [fine-tuning](https://github.com/huggingface/optimum-habana/tree/main/examples/stable-diffusion/training#dreambooth-lora-fine-tuning-with-flux1-dev)
    • |
  • Single card
    • |
  • [text-to-image generation](https://github.com/huggingface/optimum-habana/tree/main/examples/stable-diffusion)
    • | -| Text to Video | |
  • Single card
    • |
  • [text-to-video generation](https://github.com/huggingface/optimum-habana/tree/main/examples/text-to-video)
    • | +| Text to Video | |
  • Single card
    • |
  • [text-to-video generation](https://github.com/huggingface/optimum-habana/tree/main/examples/stable-diffusion#text-to-video-generation)
    • | - PyTorch Image Models/TIMM: diff --git a/docs/source/installation.mdx b/docs/source/installation.mdx index 2b6e8a0a5c..fa54c4446e 100644 --- a/docs/source/installation.mdx +++ b/docs/source/installation.mdx @@ -24,7 +24,7 @@ python -m pip install --upgrade-strategy eager optimum[habana] To use MicrosoftĀ® DeepSpeed with Intel Gaudi devices, you also need to run the following command: ```bash -python -m pip install git+https://github.com/HabanaAI/DeepSpeed.git@1.18.0 +python -m pip install git+https://github.com/HabanaAI/DeepSpeed.git@1.19.0 ``` To ensure that you are installing the correct Intel Gaudi Software, please run the `hl-smi` command to confirm the software version diff --git a/docs/source/quickstart.mdx b/docs/source/quickstart.mdx index ec79ac05f9..c882de2629 100644 --- a/docs/source/quickstart.mdx +++ b/docs/source/quickstart.mdx @@ -32,12 +32,12 @@ platform for deep learning and follow the steps to start and connect to the node ## Docker Setup Now that you have access to the node, you will use the latest Intel Gaudi AI Accelerator docker image by executing the docker run command which will -automatically download and run the docker. At the time of writing this guide, latest Gaudi docker version was 1.18.0: +automatically download and run the docker. At the time of writing this guide, latest Gaudi docker version was 1.19.0: ```bash -release=1.18.0 +release=1.19.0 os=ubuntu22.04 -torch=2.4.0 +torch=2.5.1 docker_image=vault.habana.ai/gaudi-docker/$release/$os/habanalabs/pytorch-installer-$torch:latest ``` @@ -65,11 +65,11 @@ docker run -itd \ ## Optimum for Intel Gaudi Setup Check latest release of Optimum for Intel Gaudi [here](https://github.com/huggingface/optimum-habana/releases). -At the time of writing this guide, latest Optimum for Intel Gaudi release version was v1.14.0, which is paired with Intel Gaudi Software release -version 1.18.0. Install Optimum for Intel Gaudi as follows: +At the time of writing this guide, latest Optimum for Intel Gaudi release version was v1.15.0, which is paired with Intel Gaudi Software release +version 1.19.0. Install Optimum for Intel Gaudi as follows: ```bash -git clone -b v1.14.0 https://github.com/huggingface/optimum-habana +git clone -b v1.15.0 https://github.com/huggingface/optimum-habana pip install ./optimum-habana ``` @@ -115,7 +115,7 @@ MicrosoftĀ® DeepSpeed. Gaudi-specific fork of the library is maintained by Intel To install the library compatible with the same Gaudi software release stack, use: ```bash -pip install git+https://github.com/HabanaAI/DeepSpeed.git@1.18.0 +pip install git+https://github.com/HabanaAI/DeepSpeed.git@1.19.0 ``` With DeepSpeed successfully installed we can now run a distributed GPT-2 inference on an 8 HPU system as follows: @@ -135,7 +135,7 @@ run_generation.py \ šŸ¤— Optimum for Intel Gaudi contains a number of examples demonstrating single and multi Gaudi device training/fine-tuning. -For example, a number of language models can be trained with the scripts provided +For example, a number of language models can be trained with the scripts provided [language modeling examples section](https://github.com/huggingface/optimum-habana/tree/main/examples/language-modeling). As an illustration, let us run GPT-2 single and multi card training examples on Gaudi. @@ -239,7 +239,7 @@ outputs = pipeline( ) ``` -In addition, sample scripts for fine-tuning diffusion models are given in +In addition, sample scripts for fine-tuning diffusion models are given in [Stable Diffusion training section](https://github.com/huggingface/optimum-habana/tree/main/examples/stable-diffusion/training). A more comprehensive list of examples in Optimum for Intel Gaudi is given next. diff --git a/docs/source/usage_guides/deepspeed.mdx b/docs/source/usage_guides/deepspeed.mdx index 833358d9c4..f6617e92ce 100644 --- a/docs/source/usage_guides/deepspeed.mdx +++ b/docs/source/usage_guides/deepspeed.mdx @@ -32,7 +32,7 @@ You can find more information about DeepSpeed Gaudi integration [here](https://d To use DeepSpeed on Gaudi, you need to install Optimum for Intel Gaudi and [DeepSpeed fork for Intel Gaudi](https://github.com/HabanaAI/DeepSpeed) with: ```bash pip install optimum[habana] -pip install git+https://github.com/HabanaAI/DeepSpeed.git@1.18.0 +pip install git+https://github.com/HabanaAI/DeepSpeed.git@1.19.0 ``` @@ -79,7 +79,7 @@ It is strongly advised to read [this section](https://huggingface.co/docs/transf -Other examples of configurations for HPUs are proposed [here](https://github.com/HabanaAI/Model-References/tree/1.18.0/PyTorch/nlp/DeepSpeedExamples/deepspeed-bert/scripts) by Intel. +Other examples of configurations for HPUs are proposed [here](https://github.com/HabanaAI/Model-References/tree/1.19.0/PyTorch/nlp/DeepSpeedExamples/deepspeed-bert/scripts) by Intel. The [Transformers documentation](https://huggingface.co/docs/transformers/main_classes/deepspeed#configuration) explains how to write a configuration from scratch very well. A more complete description of all configuration possibilities is available [here](https://www.deepspeed.ai/docs/config-json/). diff --git a/examples/audio-classification/README.md b/examples/audio-classification/README.md index aaa45425cc..64f5e0daba 100644 --- a/examples/audio-classification/README.md +++ b/examples/audio-classification/README.md @@ -56,6 +56,7 @@ python run_audio_classification.py \ --use_hpu_graphs_for_inference \ --gaudi_config_name Habana/wav2vec2 \ --throughput_warmup_steps 3 \ + --sdp_on_bf16 \ --bf16 \ --trust_remote_code True ``` @@ -93,6 +94,7 @@ PT_HPU_LAZY_MODE=0 python ../gaudi_spawn.py \ --use_lazy_mode False\ --gaudi_config_name Habana/wav2vec2 \ --throughput_warmup_steps 3 \ + --sdp_on_bf16 \ --bf16 \ --trust_remote_code True \ --torch_compile \ @@ -105,52 +107,6 @@ On 8 HPUs, this script should run in ~12 minutes and yield an accuracy of **80.4 > If you get an error reporting unused parameters in the model, you can specify `--ddp_find_unused_parameters True`. Using this parameter might affect the training speed. - -## DeepSpeed - -> You need to install DeepSpeed with: -> ```bash -> pip install git+https://github.com/HabanaAI/DeepSpeed.git@1.18.0 -> ``` - -DeepSpeed can be used with almost the same command as for a multi-card run: -- `use_mpi` should be replaced by `use_deepspeed`, -- an additional `--deepspeed path_to_my_deepspeed config` argument should be provided, for instance `--deepspeed ../../tests/configs/deepspeed_zero_2.json`. - -For example: -```bash -PT_HPU_LAZY_MODE=0 python ../gaudi_spawn.py \ - --world_size 8 --use_deepspeed run_audio_classification.py \ - --model_name_or_path facebook/wav2vec2-base \ - --dataset_name common_language \ - --audio_column_name audio \ - --label_column_name language \ - --output_dir /tmp/wav2vec2-base-lang-id \ - --overwrite_output_dir \ - --remove_unused_columns False \ - --do_train \ - --do_eval \ - --learning_rate 3e-4 \ - --max_length_seconds 8 \ - --attention_mask False \ - --warmup_ratio 0.1 \ - --num_train_epochs 10 \ - --per_device_train_batch_size 16 \ - --per_device_eval_batch_size 32 \ - --seed 0 \ - --use_habana \ - --use_lazy_mode False\ - --gaudi_config_name Habana/wav2vec2 \ - --throughput_warmup_steps 3 \ - --deepspeed ../../tests/configs/deepspeed_zero_2.json \ - --trust_remote_code True -``` - -[The documentation](https://huggingface.co/docs/optimum/habana/usage_guides/deepspeed) provides more information about how to use DeepSpeed within Optimum Habana. - -> If your model classification head dimensions do not fit the number of labels in the dataset, you can specify `--ignore_mismatched_sizes` to adapt it. - - ## Inference To run only inference, you can start from the commands above and you just have to remove the training-only arguments such as `--do_train`, `--per_device_train_batch_size`, `--num_train_epochs`, etc... @@ -164,6 +120,7 @@ python run_audio_classification.py \ --output_dir /tmp/wav2vec2-base-ft-keyword-spotting \ --overwrite_output_dir \ --remove_unused_columns False \ + --bf16 \ --do_eval \ --max_length_seconds 1 \ --attention_mask False \ @@ -172,11 +129,9 @@ python run_audio_classification.py \ --use_habana \ --use_lazy_mode \ --use_hpu_graphs_for_inference \ + --throughput_warmup_steps 3 \ --gaudi_config_name Habana/wav2vec2 \ - --bf16 \ - --trust_remote_code True\ - --torch_compile \ - --torch_compile_backend hpu_backend + --trust_remote_code ``` diff --git a/examples/audio-classification/requirements.txt b/examples/audio-classification/requirements.txt index 720a5a4abc..bae36f7451 100644 --- a/examples/audio-classification/requirements.txt +++ b/examples/audio-classification/requirements.txt @@ -1,3 +1,4 @@ datasets>=1.14.0 evaluate +numba==0.60.0 librosa diff --git a/examples/audio-classification/run_audio_classification.py b/examples/audio-classification/run_audio_classification.py index 9a23428866..c93e88def0 100644 --- a/examples/audio-classification/run_audio_classification.py +++ b/examples/audio-classification/run_audio_classification.py @@ -47,7 +47,7 @@ def check_optimum_habana_min_version(*a, **b): # Will error if the minimal version of Transformers and Optimum Habana are not installed. Remove at your own risks. check_min_version("4.45.0") -check_optimum_habana_min_version("1.14.0.dev0") +check_optimum_habana_min_version("1.16.0.dev0") require_version("datasets>=1.14.0", "To fix: pip install -r examples/pytorch/audio-classification/requirements.txt") @@ -177,6 +177,31 @@ class ModelArguments: default=False, metadata={"help": "Will enable to load a pretrained model whose head dimensions are different."}, ) + use_flash_attention: bool = field( + default=False, metadata={"help": "Whether to use Habana flash attention for fine-tuning"} + ) + flash_attention_recompute: bool = field( + default=False, + metadata={ + "help": "Whether to enable recompute in Habana flash attention for fine-tuning." + " It is applicable only when use_flash_attention is True." + }, + ) + flash_attention_fast_softmax: bool = field( + default=False, + metadata={ + "help": "Whether to use fast softmax for Habana flash attention." + " It is applicable only when use_flash_attention is True." + }, + ) + + def __post_init__(self): + if self.flash_attention_recompute: + assert self.use_flash_attention, "flash_attention_recompute is set, but use_flash_attention is not" + os.environ["FLASH_ATTENTION_RECOMPUTE"] = "1" + if self.flash_attention_fast_softmax: + assert self.use_flash_attention, "flash_attention_fast_softmax is set, but use_flash_attention is not" + os.environ["FLASH_ATTENTION_FAST_SOFTMAX"] = "1" def main(): @@ -364,6 +389,7 @@ def compute_metrics(eval_pred): revision=model_args.model_revision, token=model_args.token, trust_remote_code=model_args.trust_remote_code, + attn_implementation="sdpa" if model_args.use_flash_attention else "eager", ) model = AutoModelForAudioClassification.from_pretrained( model_args.model_name_or_path, diff --git a/examples/contrastive-image-text/README.md b/examples/contrastive-image-text/README.md index 7a095bc9ca..d21eece8bf 100644 --- a/examples/contrastive-image-text/README.md +++ b/examples/contrastive-image-text/README.md @@ -115,6 +115,7 @@ PT_HPU_LAZY_MODE=0 python run_clip.py \ --gaudi_config_name Habana/clip \ --throughput_warmup_steps 3 \ --dataloader_num_workers 16 \ + --sdp_on_bf16 \ --bf16 \ --trust_remote_code \ --torch_compile_backend=hpu_backend \ @@ -127,32 +128,34 @@ PT_HPU_LAZY_MODE=0 python run_clip.py \ Run the following command for distributed training: ```bash -PT_HPU_LAZY_MODE=0 \ -python ../gaudi_spawn.py --world_size 8 --use_mpi run_clip.py \ - --output_dir ./clip-roberta-finetuned \ - --model_name_or_path ./clip-roberta \ +PT_HPU_LAZY_MODE=0 PT_ENABLE_INT64_SUPPORT=1 \ +python3 ../gaudi_spawn.py --world_size 8 --use_mpi run_clip.py \ + --output_dir=/tmp/clip_roberta \ + --model_name_or_path=./clip-roberta \ --data_dir $PWD/data \ --dataset_name ydshieh/coco_dataset_script \ - --dataset_config_name=2017 \ + --dataset_config_name 2017 \ --image_column image_path \ --caption_column caption \ --remove_unused_columns=False \ - --do_train --do_eval \ - --per_device_train_batch_size="512" \ + --do_train --do_eval \ + --mediapipe_dataloader \ + --per_device_train_batch_size="64" \ --per_device_eval_batch_size="64" \ --learning_rate="5e-5" --warmup_steps="0" --weight_decay 0.1 \ --overwrite_output_dir \ - --save_strategy epoch \ --use_habana \ - --gaudi_config_name Habana/clip \ - --throughput_warmup_steps 3 \ - --dataloader_num_workers 16 \ - --mediapipe_dataloader \ - --bf16 \ - --distribution_strategy fast_ddp \ - --trust_remote_code \ + --use_lazy_mode=False \ + --gaudi_config_name="Habana/clip" \ + --throughput_warmup_steps=3 \ + --save_strategy="no" \ + --dataloader_num_workers=2 \ + --use_hpu_graphs \ + --max_steps=100 \ --torch_compile_backend=hpu_backend \ - --torch_compile + --torch_compile \ + --logging_nan_inf_filter \ + --trust_remote_code ``` > `--mediapipe_dataloader` only works on Gaudi2. @@ -160,54 +163,8 @@ python ../gaudi_spawn.py --world_size 8 --use_mpi run_clip.py \ ### DeepSpeed -Run the following command for training with DeepSpeed: - -```bash -PT_HPU_LAZY_MODE=0 \ -python ../gaudi_spawn.py --world_size 8 --use_deepspeed run_clip.py \ - --output_dir ./clip-roberta-finetuned \ - --model_name_or_path ./clip-roberta \ - --data_dir $PWD/data \ - --dataset_name ydshieh/coco_dataset_script \ - --dataset_config_name=2017 \ - --image_column image_path \ - --caption_column caption \ - --remove_unused_columns=False \ - --do_train --do_eval \ - --per_device_train_batch_size="512" \ - --per_device_eval_batch_size="64" \ - --learning_rate="5e-5" --warmup_steps="0" --weight_decay 0.1 \ - --overwrite_output_dir \ - --save_strategy epoch \ - --use_habana \ - --gaudi_config_name Habana/clip \ - --throughput_warmup_steps 3 \ - --deepspeed path_to_my_deepspeed_config \ - --trust_remote_code \ - --torch_compile_backend=hpu_backend \ - --torch_compile -``` - -You can look at the [documentation](https://huggingface.co/docs/optimum/habana/usage_guides/deepspeed) for more information about how to use DeepSpeed in Optimum Habana. -Here is a DeepSpeed configuration you can use to train your models on Gaudi: -```json -{ - "steps_per_print": 64, - "train_batch_size": "auto", - "train_micro_batch_size_per_gpu": "auto", - "gradient_accumulation_steps": "auto", - "bf16": { - "enabled": true - }, - "gradient_clipping": 1.0, - "zero_optimization": { - "stage": 2, - "overlap_comm": false, - "reduce_scatter": false, - "contiguous_gradients": false - } -} -``` +You can check the [DeepSpeed](https://github.com/huggingface/optimum-habana/tree/main/examples#deepspeed) section in Optimum Habana examples for how to run DeepSpeed. +You can also look at the [documentation](https://huggingface.co/docs/optimum/habana/usage_guides/deepspeed) for more information about how to use DeepSpeed in Optimum Habana. ## BridgeTower @@ -265,6 +222,7 @@ python run_clip.py \ --use_hpu_graphs_for_inference \ --gaudi_config_name Habana/clip \ --bf16 \ + --sdp_on_bf16 \ --mediapipe_dataloader \ --trust_remote_code ``` diff --git a/examples/contrastive-image-text/run_bridgetower.py b/examples/contrastive-image-text/run_bridgetower.py index b54ca8e7c0..5964b2cdcc 100644 --- a/examples/contrastive-image-text/run_bridgetower.py +++ b/examples/contrastive-image-text/run_bridgetower.py @@ -59,7 +59,7 @@ def check_optimum_habana_min_version(*a, **b): # Will error if the minimal version of Transformers and Optimum Habana are not installed. Remove at your own risks. check_min_version("4.45.0") -check_optimum_habana_min_version("1.14.0.dev0") +check_optimum_habana_min_version("1.16.0.dev0") require_version("datasets>=1.8.0", "To fix: pip install -r examples/pytorch/contrastive-image-text/requirements.txt") diff --git a/examples/contrastive-image-text/run_clip.py b/examples/contrastive-image-text/run_clip.py index 6a8ca235e1..fc3bb4886e 100644 --- a/examples/contrastive-image-text/run_clip.py +++ b/examples/contrastive-image-text/run_clip.py @@ -62,7 +62,7 @@ def check_optimum_habana_min_version(*a, **b): # Will error if the minimal version of Transformers and Optimum Habana are not installed. Remove at your own risks. check_min_version("4.45.0") -check_optimum_habana_min_version("1.14.0.dev0") +check_optimum_habana_min_version("1.16.0.dev0") require_version("datasets>=1.8.0", "To fix: pip install -r examples/pytorch/contrastive-image-text/requirements.txt") diff --git a/examples/gaudi_spawn.py b/examples/gaudi_spawn.py index 0f76dcd379..f282809a31 100644 --- a/examples/gaudi_spawn.py +++ b/examples/gaudi_spawn.py @@ -84,7 +84,7 @@ def main(): if not is_deepspeed_available(): raise ImportError( "--use_deepspeed requires deepspeed: `pip install" - " git+https://github.com/HabanaAI/DeepSpeed.git@1.18.0`." + " git+https://github.com/HabanaAI/DeepSpeed.git@1.19.0`." ) # Patch sys.argv diff --git a/examples/image-classification/README.md b/examples/image-classification/README.md index 08c4d67123..01b19b25ba 100644 --- a/examples/image-classification/README.md +++ b/examples/image-classification/README.md @@ -57,6 +57,7 @@ PT_HPU_LAZY_MODE=0 python run_image_classification.py \ --gaudi_config_name Habana/vit \ --throughput_warmup_steps 6 \ --dataloader_num_workers 1 \ + --sdp_on_bf16 \ --bf16 ``` @@ -107,6 +108,7 @@ PT_HPU_LAZY_MODE=0 python run_image_classification.py \ --gaudi_config_name Habana/vit \ --throughput_warmup_steps 3 \ --dataloader_num_workers 1 \ + --sdp_on_bf16 \ --bf16 ``` @@ -211,6 +213,7 @@ PT_HPU_LAZY_MODE=0 python ../gaudi_spawn.py \ --gaudi_config_name Habana/vit \ --throughput_warmup_steps 8 \ --dataloader_num_workers 1 \ + --sdp_on_bf16 \ --bf16 ``` @@ -298,6 +301,7 @@ python run_image_classification.py \ --use_hpu_graphs_for_inference \ --gaudi_config_name Habana/vit \ --dataloader_num_workers 1 \ + --sdp_on_bf16 \ --bf16 ``` diff --git a/examples/image-classification/run_image_classification.py b/examples/image-classification/run_image_classification.py index b2694665a3..bc45087f9e 100644 --- a/examples/image-classification/run_image_classification.py +++ b/examples/image-classification/run_image_classification.py @@ -64,7 +64,7 @@ def check_optimum_habana_min_version(*a, **b): # Will error if the minimal version of Transformers and Optimum Habana are not installed. Remove at your own risks. check_min_version("4.45.0") -check_optimum_habana_min_version("1.14.0.dev0") +check_optimum_habana_min_version("1.16.0.dev0") require_version("datasets>=2.14.0", "To fix: pip install -r examples/pytorch/image-classification/requirements.txt") diff --git a/examples/image-to-text/README.md b/examples/image-to-text/README.md index 51f4a5dda2..7a8ad04664 100644 --- a/examples/image-to-text/README.md +++ b/examples/image-to-text/README.md @@ -17,102 +17,12 @@ limitations under the License. # Image to Text Examples This directory contains a script that showcases how to perform image to text generation on IntelĀ® GaudiĀ® AI Accelerators. -## Single-HPU inference +Habana FusedSDPA is a fused and optimized implementation of torch.nn.functional.scaled_dot_product_attention() for Gaudi. For more details, refer to [Gaudi online documentation](https://docs.habana.ai/en/latest/PyTorch/Model_Optimization_PyTorch/Optimization_in_PyTorch_Models.html?highlight=fusedsdpa#using-fused-scaled-dot-product-attention-fusedsdpa). We optimized many models with FusedSDPA implementation as in [optimum/habana/transformers/models](https://github.com/huggingface/optimum-habana/tree/main/optimum/habana/transformers/models). If a model is not optimized with FusedSDPA, it uses [SDPA implementation](https://pytorch.org/docs/stable/generated/torch.nn.functional.scaled_dot_product_attention.html). -Models that have been validated: - - [nlpconnect/vit-gpt2-image-captioning](https://huggingface.co/nlpconnect/vit-gpt2-image-captioning) - - [Salesforce/blip-image-captioning-large](https://huggingface.co/Salesforce/blip-image-captioning-large) - - [Salesforce/blip-image-captioning-base](https://huggingface.co/Salesforce/blip-image-captioning-base) - - [llava-hf/llava-1.5-7b-hf](https://huggingface.co/llava-hf/llava-1.5-7b-hf) - - [llava-hf/llava-1.5-13b-hf](https://huggingface.co/llava-hf/llava-1.5-13b-hf) - - [llava-hf/llava-v1.6-mistral-7b-hf](https://huggingface.co/llava-hf/llava-v1.6-mistral-7b-hf) - - [llava-hf/llava-v1.6-vicuna-7b-hf](https://huggingface.co/llava-hf/llava-v1.6-vicuna-7b-hf) - - [llava-hf/llava-v1.6-vicuna-13b-hf](https://huggingface.co/llava-hf/llava-v1.6-vicuna-13b-hf) - - [llava-hf/llava-v1.6-34b-hf](https://huggingface.co/llava-hf/llava-v1.6-34b-hf) - - [llava-hf/llama3-llava-next-8b-hf](https://huggingface.co/llava-hf/llama3-llava-next-8b-hf) - - [HuggingFaceM4/idefics2-8b](https://huggingface.co/HuggingFaceM4/idefics2-8b) - - [meta-llama/Llama-3.2-11B-Vision-Instruct](https://huggingface.co/meta-llama/Llama-3.2-11B-Vision-Instruct) - - [meta-llama/Llama-3.2-90B-Vision-Instruct](https://huggingface.co/meta-llama/Llama-3.2-90B-Vision-Instruct) - - [tiiuae/falcon-11B-vlm](https://huggingface.co/tiiuae/falcon-11B-vlm) - - [google/paligemma-3b-mix-224](https://huggingface.co/google/paligemma-3b-mix-224) +## Inference with mixed-precision (BF16) -### Inference with BF16 - -To run Salesforce/blip-image-captioning-large inference, use the following command: -```bash -python3 run_pipeline.py \ - --model_name_or_path Salesforce/blip-image-captioning-large \ - --image_path "https://ankur3107.github.io/assets/images/image-captioning-example.png" \ - --use_hpu_graphs \ - --bf16 -``` - -To run Llava-1.5-7b inference, use the following command: -```bash -python3 run_pipeline.py \ - --model_name_or_path llava-hf/llava-1.5-7b-hf \ - --use_hpu_graphs \ - --bf16 -``` - -To run Llava-1.5-13b inference, use the following command: -```bash -python3 run_pipeline.py \ - --model_name_or_path llava-hf/llava-1.5-13b-hf \ - --use_hpu_graphs \ - --bf16 -``` - -To run Llava-v1.6-mistral-7b inference, use the following command: -```bash -python3 run_pipeline.py \ - --model_name_or_path llava-hf/llava-v1.6-mistral-7b-hf \ - --use_hpu_graphs \ - --bf16 -``` - -To run Llava-v1.6-vicuna-13b inference, use the following command: -```bash -python3 run_pipeline.py \ - --model_name_or_path llava-hf/llava-v1.6-vicuna-13b-hf \ - --use_hpu_graphs \ - --bf16 -``` - -To run Llava-hf/llava-v1.6-34b-hf inference, use the following command: -```bash -python3 run_pipeline.py \ - --model_name_or_path llava-hf/llava-v1.6-34b-hf \ - --use_hpu_graphs \ - --bf16 -``` - -To run google/paligemma-3b-mix-224 inference, use the following command: -```bash -python3 run_pipeline.py \ - --model_name_or_path google/paligemma-3b-mix-224 \ - --use_hpu_graphs \ - --bf16 -``` - -To run Llava-hf/llama3-llava-next-8b-hf inference, use the following command: -```bash -python3 run_pipeline.py \ - --model_name_or_path llava-hf/llama3-llava-next-8b-hf \ - --use_hpu_graphs \ - --bf16 -``` - -To run idefics2 inference, use the following command: - -```bash -python3 run_pipeline.py \ - --model_name_or_path HuggingFaceM4/idefics2-8b \ - --use_hpu_graphs \ - --bf16 -``` - -To run mllama inference using reduced precision in the SDPA, use the following command: +### Single card inference with BF16 +To run Llama inference with SDPA, use the following command: ```bash python3 run_pipeline.py \ @@ -121,87 +31,53 @@ python3 run_pipeline.py \ --bf16 \ --sdp_on_bf16 ``` +> SDPA may introduce [reduced precison](https://pytorch.org/docs/stable/notes/numerical_accuracy.html#reduced-precision-reduction-for-fp16-and-bf16-in-scaled-dot-product-attention-sdpa) -### Inference with FP8 -Inference for Llava-1.5-7b, Llava-1.5-13b, Llava-v1.6-mistral-7b and Llava-v1.6-vicuna-13b in FP8 precision are enabled using [Intel Neural Compressor (INC)](https://docs.habana.ai/en/latest/PyTorch/Inference_on_PyTorch/Inference_Using_FP8.html), which provides model measurement and quantization capabilities in PyTorch. -More information on enabling FP8 in SynapseAI is available here: -https://docs.habana.ai/en/latest/PyTorch/Inference_on_PyTorch/Inference_Using_FP8.html +### Multi-cards inference with BF16 -Here is an example to measure the tensor quantization statistics on Llava-1.5-7b: -```bash -QUANT_CONFIG=./quantization_config/maxabs_measure.json python run_pipeline.py \ - --model_name_or_path llava-hf/llava-1.5-7b-hf \ - --image_path "https://llava-vl.github.io/static/images/view.jpg" \ - --use_hpu_graphs \ - --bf16 -``` - -Here is an example to quantize the model based on previous measurements for Llava-1.5-7b: +Use the following commands to run Llama-3.2-90B-Vision-Instruct BF16 inference with FusedSDPA on 8 HPUs: ```bash -QUANT_CONFIG=./quantization_config/maxabs_quant_scale_format_const.json python run_pipeline.py \ - --model_name_or_path llava-hf/llava-1.5-7b-hf \ +PT_HPU_ENABLE_LAZY_COLLECTIVES=true python ../gaudi_spawn.py --use_deepspeed --world_size 8 run_pipeline.py \ + --model_name_or_path meta-llama/Llama-3.2-90B-Vision-Instruct \ --image_path "https://llava-vl.github.io/static/images/view.jpg" \ --use_hpu_graphs \ - --bf16 + --bf16 \ + --use_flash_attention \ + --flash_attention_recompute ``` +## Inference with FP8 -Here is an example to measure the tensor quantization statistics on Llava-v1.6-mistral-7b: -```bash -QUANT_CONFIG=./quantization_config/maxabs_measure.json python run_pipeline.py \ - --model_name_or_path llava-hf/llava-v1.6-mistral-7b-hf \ - --image_path "https://llava-vl.github.io/static/images/view.jpg" \ - --use_hpu_graphs \ - --bf16 -``` - -Here is an example to quantize the model based on previous measurements for Llava-v1.6-mistral-7b: -```bash -QUANT_CONFIG=./quantization_config/maxabs_quant_scale_format_const.json python run_pipeline.py \ - --model_name_or_path llava-hf/llava-v1.6-mistral-7b-hf \ - --image_path "https://llava-vl.github.io/static/images/view.jpg" \ - --use_hpu_graphs \ - --bf16 -``` +Inference with FP8 precision is enabled using [Intel Neural Compressor (INC)](https://docs.habana.ai/en/latest/PyTorch/Inference_on_PyTorch/Quantization/index.html?highlight=inc), which provides model measurement and quantization capabilities in PyTorch. +More information on enabling FP8 in SynapseAI is available here: +[Run Inference Using FP8](https://docs.habana.ai/en/latest/PyTorch/Inference_on_PyTorch/Quantization/Inference_Using_FP8.html?highlight=fp8) -Here is an example to measure the tensor quantization statistics on Llava-v1.6-vicuna-13b: +### Single card inference with FP8 +Here is an example to measure the tensor quantization statistics on Llava-v1.6-vicuna-13b with SDPA: ```bash QUANT_CONFIG=./quantization_config/maxabs_measure.json python run_pipeline.py \ --model_name_or_path llava-hf/llava-v1.6-vicuna-13b-hf \ --image_path "https://llava-vl.github.io/static/images/view.jpg" \ --use_hpu_graphs \ - --bf16 + --bf16 \ + --sdp_on_bf16 ``` -Here is an example to quantize the model based on previous measurements for Llava-v1.6-vicuna-13b: +Here is an example to quantize the model based on previous measurements for Llava-v1.6-vicuna-13b with SDPA: ```bash QUANT_CONFIG=./quantization_config/maxabs_quant_scale_format_const.json python run_pipeline.py \ --model_name_or_path llava-hf/llava-v1.6-vicuna-13b-hf \ --image_path "https://llava-vl.github.io/static/images/view.jpg" \ --use_hpu_graphs \ - --bf16 -``` - -### Inference with FusedSDPA - -Habana FusedSDPA is a fused and optimized implementation of torch.nn.functional.scaled_dot_product_attention() for Gaudi. For more details, refer to [Gaudi online documentation](https://docs.habana.ai/en/latest/PyTorch/Model_Optimization_PyTorch/Optimization_in_PyTorch_Models.html?highlight=fusedsdpa#using-fused-scaled-dot-product-attention-fusedsdpa). - -Use the following command to run Llava-1.5-7b BF16 inference with FusedSDPA -```bash -python3 run_pipeline.py \ - --model_name_or_path llava-hf/llava-1.5-7b-hf \ - --image_path "https://llava-vl.github.io/static/images/view.jpg" \ - --use_hpu_graphs \ --bf16 \ - --use_flash_attention \ - --flash_attention_recompute + --sdp_on_bf16 ``` - -Use the following command to run Llava-v1.6-mistral-7b BF16 inference with FusedSDPA +### Multi-cards inference with FP8 +Here is an example of measuring the tensor quantization statistics on Llava-v1.6-mistral-7b with FusedSDPA on 8 HPUs: ```bash -python3 run_pipeline.py \ +QUANT_CONFIG=./quantization_config/maxabs_measure.json python ../gaudi_spawn.py --use_deepspeed --world_size 8 run_pipeline.py \ --model_name_or_path llava-hf/llava-v1.6-mistral-7b-hf \ --image_path "https://llava-vl.github.io/static/images/view.jpg" \ --use_hpu_graphs \ @@ -210,12 +86,9 @@ python3 run_pipeline.py \ --flash_attention_recompute ``` - -Use the following commands to run Llava-v1.6-mistral-7b FP8 inference with FusedSDPA - -Here is an example of measuring the tensor quantization statistics on Llava-v1.6-mistral-7b: +Here is an example of quantizing the model based on previous measurements for Llava-v1.6-mistral-7b with FusedSDPA on 8 HPUs: ```bash -QUANT_CONFIG=./quantization_config/maxabs_measure.json python run_pipeline.py \ +QUANT_CONFIG=./quantization_config/maxabs_quant_scale_format_const.json python ../gaudi_spawn.py --use_deepspeed --world_size 8 run_pipeline.py \ --model_name_or_path llava-hf/llava-v1.6-mistral-7b-hf \ --image_path "https://llava-vl.github.io/static/images/view.jpg" \ --use_hpu_graphs \ @@ -224,88 +97,8 @@ QUANT_CONFIG=./quantization_config/maxabs_measure.json python run_pipeline.py \ --flash_attention_recompute ``` -Here is an example of quantizing the model based on previous measurements for Llava-v1.6-mistral-7b: -```bash -QUANT_CONFIG=./quantization_config/maxabs_quant_scale_format_const.json python run_pipeline.py \ - --model_name_or_path llava-hf/llava-v1.6-mistral-7b-hf \ - --image_path "https://llava-vl.github.io/static/images/view.jpg" \ - --use_hpu_graphs \ - --bf16 \ - --use_flash_attention \ - --flash_attention_recompute -``` ## LORA Finetune -To run LoRA finetuning, you can use `run_image2text_lora_finetune.py`. -Here are single-/multi-device command examples for HuggingFaceM4/idefics2-8b. - -```bash -python3 run_image2text_lora_finetune.py \ - --model_name_or_path HuggingFaceM4/idefics2-8b \ - --dataset_name nielsr/docvqa_1200_examples \ - --bf16 True \ - --output_dir ./model_lora_llama \ - --num_train_epochs 1 \ - --per_device_train_batch_size 2 \ - --per_device_eval_batch_size 2 \ - --gradient_accumulation_steps 8 \ - --weight_decay 0.01 \ - --logging_steps 25 \ - --eval_strategy "no" \ - --save_strategy "no" \ - --learning_rate 5e-5 \ - --warmup_steps 50 \ - --lr_scheduler_type "constant" \ - --input_column_names 'image' 'query' \ - --output_column_names 'answers' \ - --remove_unused_columns False \ - --do_train \ - --do_eval \ - --use_habana \ - --use_lazy_mode \ - --lora_rank=8 \ - --lora_alpha=8 \ - --lora_dropout=0.1 \ - --max_seq_length=512 \ - --use_hpu_graphs_for_inference \ - --low_cpu_mem_usage True \ - --lora_target_modules '.*(text_model|modality_projection|perceiver_resampler).*(down_proj|gate_proj|up_proj|k_proj|q_proj|v_proj|o_proj).*$' -``` - -```bash -python3 ../gaudi_spawn.py \ - --world_size 8 --use_mpi run_image2text_lora_finetune.py \ - --model_name_or_path HuggingFaceM4/idefics2-8b \ - --dataset_name nielsr/docvqa_1200_examples \ - --bf16 True \ - --output_dir ./model_lora_llama \ - --num_train_epochs 1 \ - --per_device_train_batch_size 2 \ - --per_device_eval_batch_size 2 \ - --gradient_accumulation_steps 8 \ - --weight_decay 0.01 \ - --logging_steps 25 \ - --eval_strategy "no" \ - --save_strategy "no" \ - --learning_rate 5e-5 \ - --warmup_steps 50 \ - --lr_scheduler_type "constant" \ - --input_column_names 'image' 'query' \ - --output_column_names 'answers' \ - --remove_unused_columns False \ - --do_train \ - --do_eval \ - --use_habana \ - --use_lazy_mode \ - --lora_rank=8 \ - --lora_alpha=8 \ - --lora_dropout=0.1 \ - --max_seq_length=512 \ - --use_hpu_graphs_for_inference \ - --low_cpu_mem_usage True \ - --lora_target_modules '".*(text_model|modality_projection|perceiver_resampler).*(down_proj|gate_proj|up_proj|k_proj|q_proj|v_proj|o_proj).*$"' -``` - Here are single-/multi-device command examples for meta-llama/Llama-3.2-11B-Vision-Instruct. ```bash @@ -375,54 +168,6 @@ python3 ../gaudi_spawn.py \ --lora_target_modules '".*(language_model).*(down_proj|gate_proj|up_proj|k_proj|q_proj|v_proj|o_proj).*$"' ``` -## Multi-HPU inference - -### BF16 Inference with FusedSDPA on 8 HPUs - -Use the following commands to run Llava-v1.6-mistral-7b BF16 inference with FusedSDPA on 8 HPUs: -```bash -python ../gaudi_spawn.py --use_deepspeed --world_size 8 run_pipeline.py \ - --model_name_or_path llava-hf/llava-v1.6-mistral-7b-hf \ - --image_path "https://llava-vl.github.io/static/images/view.jpg" \ - --use_hpu_graphs \ - --bf16 \ - --use_flash_attention \ - --flash_attention_recompute -``` - -Use the following commands to run Llama-3.2-90B-Vision-Instruct BF16 inference with FusedSDPA on 8 HPUs: -```bash -PT_HPU_ENABLE_LAZY_COLLECTIVES=true python ../gaudi_spawn.py --use_deepspeed --world_size 8 run_pipeline.py \ - --model_name_or_path meta-llama/Llama-3.2-90B-Vision-Instruct \ - --image_path "https://llava-vl.github.io/static/images/view.jpg" \ - --use_hpu_graphs \ - --bf16 \ - --use_flash_attention \ - --flash_attention_recompute -``` - - -### FP8 Inference with FusedSDPA on 8 HPUs - -Use the following commands to run Llava-v1.6-mistral-7b FP8 inference with FusedSDPA on 8 HPUs. -Here is an example of measuring the tensor quantization statistics on Llava-v1.6-mistral-7b on 8 HPUs: -```bash -QUANT_CONFIG=./quantization_config/maxabs_measure.json python ../gaudi_spawn.py --use_deepspeed --world_size 8 run_pipeline.py \ - --model_name_or_path llava-hf/llava-v1.6-mistral-7b-hf \ - --image_path "https://llava-vl.github.io/static/images/view.jpg" \ - --use_hpu_graphs \ - --bf16 \ - --use_flash_attention \ - --flash_attention_recompute -``` - -Here is an example of quantizing the model based on previous measurements for Llava-v1.6-mistral-7b on 8 HPUs: -```bash -QUANT_CONFIG=./quantization_config/maxabs_quant_scale_format_const.json python ../gaudi_spawn.py --use_deepspeed --world_size 8 run_pipeline.py \ - --model_name_or_path llava-hf/llava-v1.6-mistral-7b-hf \ - --image_path "https://llava-vl.github.io/static/images/view.jpg" \ - --use_hpu_graphs \ - --bf16 \ - --use_flash_attention \ - --flash_attention_recompute -``` +> For different models, please adjust training parameters and `lora_target_modules`. Such as replace `lora_target_modules` +> with below for HuggingFaceM4/idefics2-8b. +> '".*(text_model|modality_projection|perceiver_resampler).*(down_proj|gate_proj|up_proj|k_proj|q_proj|v_proj|o_proj).*$"' diff --git a/examples/image-to-text/run_pipeline.py b/examples/image-to-text/run_pipeline.py index 44eb8d575a..de849e3469 100644 --- a/examples/image-to-text/run_pipeline.py +++ b/examples/image-to-text/run_pipeline.py @@ -25,6 +25,10 @@ import torch from transformers import AutoConfig, AutoModelForVision2Seq, AutoProcessor, pipeline +from optimum.habana.utils import ( + set_seed, +) + logging.basicConfig( format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", @@ -179,6 +183,23 @@ def main(): action="store_true", help="Allow PyTorch to use reduced precision in the SDPA math backend", ) + parser.add_argument( + "--max_input_tokens", + type=int, + default=None, + help="If > 0 then pad the input sequences to this specified length of tokens. will not apply truncate to avoid deleting the image tag", + ) + parser.add_argument( + "--do_sample", + action="store_true", + help="Whether to use sampling for generation.", + ) + parser.add_argument( + "--seed", + default=27, + type=int, + help="Seed to use for random generation. Useful to reproduce your runs with `--do_sample`.", + ) args = parser.parse_args() @@ -192,14 +213,18 @@ def main(): os.environ.setdefault("EXPERIMENTAL_WEIGHT_SHARING", "FALSE") if args.world_size > 0: os.environ.setdefault("PT_HPU_ENABLE_LAZY_COLLECTIVES", "true") + os.environ.setdefault("DEEPSPEED_USE_HABANA_FRAMEWORKS_DETERMINISTIC_API", "1") from optimum.habana.transformers.modeling_utils import adapt_transformers_to_gaudi adapt_transformers_to_gaudi() + set_seed(args.seed) + config = AutoConfig.from_pretrained(args.model_name_or_path) model_type = config.model_type - if args.image_path is None and model_type in ["llava", "idefics2", "mllama"]: + + if args.image_path is None and model_type in ["llava", "idefics2", "mllama", "qwen2_vl"]: args.image_path = ["https://llava-vl.github.io/static/images/view.jpg"] elif args.image_path is None and model_type == "paligemma": args.image_path = [ @@ -210,8 +235,8 @@ def main(): "https://github.com/haotian-liu/LLaVA/blob/1a91fc274d7c35a9b50b3cb29c4247ae5837ce39/images/llava_v1_5_radar.jpg?raw=true" ] - if model_type in ["llava", "idefics2", "llava_next", "mllama", "paligemma"]: - processor = AutoProcessor.from_pretrained(args.model_name_or_path) + if model_type in ["llava", "idefics2", "llava_next", "mllama", "paligemma", "qwen2_vl"]: + processor = AutoProcessor.from_pretrained(args.model_name_or_path, padding_side="left") if args.prompt is None: if processor.chat_template is not None: conversation = [ @@ -289,6 +314,9 @@ def main(): generator = pipeline( "image-to-text", model=args.model_name_or_path, + config=args.model_name_or_path, + tokenizer=args.model_name_or_path, + image_processor=args.model_name_or_path, torch_dtype=model_dtype, device="hpu", ) @@ -308,6 +336,7 @@ def main(): "use_flash_attention": args.use_flash_attention, "flash_attention_recompute": args.flash_attention_recompute, "limit_hpu_graphs": args.limit_hpu_graphs, + "do_sample": args.do_sample, } if args.sdp_on_bf16: @@ -316,17 +345,27 @@ def main(): if args.use_kv_cache: generate_kwargs["use_cache"] = args.use_kv_cache + if model_type == "qwen2_vl": + generate_kwargs["use_cache"] = True + generate_kwargs["cache_implementation"] = "static" + if args.quant_config: generator.model = setup_quantization(generator.model, args) htcore.hpu_initialize(generator.model) # delete once pipeline integrate AutoProcessor as preprocess engine - if model_type in ["idefics2", "mllama", "paligemma"]: + # could use "image-text-to-text" pipeline in transformers 4.47 + + if model_type in ["idefics2", "mllama", "paligemma", "qwen2_vl", "llava", "llava_next"]: from transformers.image_utils import load_image def preprocess(self, image, prompt=None, timeout=None): + kwargs = {} + if args.max_input_tokens is not None and args.max_input_tokens > 0: + kwargs["max_length"] = args.max_input_tokens + kwargs["padding"] = "max_length" image = load_image(image, timeout=timeout) - model_inputs = processor(images=image, text=prompt, return_tensors=self.framework) + model_inputs = processor(images=image, text=prompt, return_tensors=self.framework, **kwargs) return model_inputs generator.__class__.preprocess = preprocess @@ -355,7 +394,7 @@ def preprocess(self, image, prompt=None, timeout=None): throughput = total_new_tokens_generated / duration logger.info(f"result = {result}") logger.info( - f"time = {(end-start) * 1000 / args.n_iterations }ms, Throughput (including tokenization) = {throughput} tokens/second" + f"time = {(end - start) * 1000 / args.n_iterations}ms, Throughput (including tokenization) = {throughput} tokens/second" ) # Store results if necessary diff --git a/examples/kubernetes/Chart.yaml b/examples/kubernetes/Chart.yaml index dc0400ccb0..d1c1778076 100644 --- a/examples/kubernetes/Chart.yaml +++ b/examples/kubernetes/Chart.yaml @@ -3,7 +3,7 @@ name: optimum-habana-example-chart description: This Helm chart deploys example jobs using Optimum for IntelĀ® GaudiĀ® Accelerators to a Kubernetes cluster. # Compatible Kubernetes versions -kubeVersion: 1.27-1.29 +kubeVersion: 1.27 - 1.29 # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. diff --git a/examples/kubernetes/Dockerfile b/examples/kubernetes/Dockerfile index 08f2937fca..7ebfd93894 100644 --- a/examples/kubernetes/Dockerfile +++ b/examples/kubernetes/Dockerfile @@ -1,7 +1,7 @@ -ARG GAUDI_SW_VER=1.18.0 +ARG GAUDI_SW_VER=1.19.0 ARG OS=ubuntu22.04 -ARG TORCH_VER=2.4.0 -ARG OPTIMUM_HABANA_VER=1.14.0 +ARG TORCH_VER=2.5.1 +ARG OPTIMUM_HABANA_VER=1.15.0 FROM vault.habana.ai/gaudi-docker/${GAUDI_SW_VER}/${OS}/habanalabs/pytorch-installer-${TORCH_VER}:latest AS optimum-habana diff --git a/examples/kubernetes/README.md b/examples/kubernetes/README.md index 2ba6b017f1..06f4f01d09 100644 --- a/examples/kubernetes/README.md +++ b/examples/kubernetes/README.md @@ -43,12 +43,12 @@ Use the the following commands to build the containers: ```bash # Specify the Gaudi SW version, OS, and PyTorch version which will be used for the base container -export GAUDI_SW_VER=1.18.0 +export GAUDI_SW_VER=1.19.0 export OS=ubuntu22.04 -export TORCH_VER=2.4.0 +export TORCH_VER=2.5.1 # Specify the version of optimum-habana to install in the container -export OPTIMUM_HABANA_VER=1.14.0 +export OPTIMUM_HABANA_VER=1.15.0 git clone https://github.com/huggingface/optimum-habana.git diff --git a/examples/kubernetes/README.md.gotmpl b/examples/kubernetes/README.md.gotmpl index 52a2c4fbab..431f8ad611 100644 --- a/examples/kubernetes/README.md.gotmpl +++ b/examples/kubernetes/README.md.gotmpl @@ -43,12 +43,12 @@ Use the the following commands to build the containers: ```bash # Specify the Gaudi SW version, OS, and PyTorch version which will be used for the base container -export GAUDI_SW_VER=1.18.0 +export GAUDI_SW_VER=1.19.0 export OS=ubuntu22.04 -export TORCH_VER=2.4.0 +export TORCH_VER=2.5.1 # Specify the version of optimum-habana to install in the container -export OPTIMUM_HABANA_VER=1.14.0 +export OPTIMUM_HABANA_VER=1.15.0 git clone https://github.com/huggingface/optimum-habana.git diff --git a/examples/kubernetes/docker-compose.yaml b/examples/kubernetes/docker-compose.yaml index 214707eccb..6bdea75bbd 100644 --- a/examples/kubernetes/docker-compose.yaml +++ b/examples/kubernetes/docker-compose.yaml @@ -5,30 +5,30 @@ services: http_proxy: ${http_proxy:-""} https_proxy: ${https_proxy:-""} no_proxy: ${no_proxy:-""} - GAUDI_SW_VER: ${GAUDI_SW_VER:-1.18.0} + GAUDI_SW_VER: ${GAUDI_SW_VER:-1.19.0} OS: ${OS:-ubuntu22.04} - OPTIMUM_HABANA_VER: ${OPTIMUM_HABANA_VER:-1.14.0} - TORCH_VER: ${TORCH_VER:-2.4.0} + OPTIMUM_HABANA_VER: ${OPTIMUM_HABANA_VER:-1.15.0} + TORCH_VER: ${TORCH_VER:-2.5.1} REGISTRY: ${REGISTRY} REPO: ${REPO} context: . labels: - org.opencontainers.base.name: "vault.habana.ai/gaudi-docker/${GAUDI_SW_VER:-1.18.0}/${OS:-ubuntu22.04}/habanalabs/pytorch-installer-${TORCH_VER:-2.3.1}:latest" + org.opencontainers.base.name: "vault.habana.ai/gaudi-docker/${GAUDI_SW_VER:-1.19.0}/${OS:-ubuntu22.04}/habanalabs/pytorch-installer-${TORCH_VER:-2.5.1}:latest" org.opencontainers.image.title: "Optimum for IntelĀ® GaudiĀ® Accelerators" - org.opencontainers.image.version: gaudi-${GAUDI_SW_VER:-1.18.0}-optimum-habana-${OPTIMUM_HABANA_VER:-1.13.0} + org.opencontainers.image.version: gaudi-${GAUDI_SW_VER:-1.19.0}-optimum-habana-${OPTIMUM_HABANA_VER:-1.15.0} command: > sh -c "python -c 'from optimum import habana; print(\"optimum-habana:\", habana.__version__)'" - image: ${REGISTRY}/${REPO}:b-${GITHUB_RUN_NUMBER:-0}-gaudi-${GAUDI_SW_VER:-1.18.0}-optimum-habana-${OPTIMUM_HABANA_VER:-1.13.0} + image: ${REGISTRY}/${REPO}:b-${GITHUB_RUN_NUMBER:-0}-gaudi-${GAUDI_SW_VER:-1.19.0}-optimum-habana-${OPTIMUM_HABANA_VER:-1.15.0} pull_policy: always optimum-habana-examples: build: labels: - org.opencontainers.base.name: "${REGISTRY}/${REPO}:gaudi-${GAUDI_SW_VER:-1.18.0}-optimum-habana-${OPTIMUM_HABANA_VER:-1.13.0}" + org.opencontainers.base.name: "${REGISTRY}/${REPO}:gaudi-${GAUDI_SW_VER:-1.19.0}-optimum-habana-${OPTIMUM_HABANA_VER:-1.15.0}" org.opencontainers.image.title: "Optimum for IntelĀ® GaudiĀ® Accelerators Examples" - org.opencontainers.image.version: gaudi-${GAUDI_SW_VER:-1.18.0}-optimum-habana-examples-${OPTIMUM_HABANA_VER:-1.13.0} + org.opencontainers.image.version: gaudi-${GAUDI_SW_VER:-1.19.0}-optimum-habana-examples-${OPTIMUM_HABANA_VER:-1.15.0} target: optimum-habana-examples command: > sh -c "python -c 'from optimum import habana; print(\"optimum-habana:\", habana.__version__)'" extends: optimum-habana - image: ${REGISTRY}/${REPO}:b-${GITHUB_RUN_NUMBER:-0}-gaudi-${GAUDI_SW_VER:-1.18.0}-optimum-habana-examples-${OPTIMUM_HABANA_VER:-1.13.0} + image: ${REGISTRY}/${REPO}:b-${GITHUB_RUN_NUMBER:-0}-gaudi-${GAUDI_SW_VER:-1.19.0}-optimum-habana-examples-${OPTIMUM_HABANA_VER:-1.15.0} diff --git a/examples/language-modeling/README.md b/examples/language-modeling/README.md index 9ef27f9e73..5cce1528dc 100644 --- a/examples/language-modeling/README.md +++ b/examples/language-modeling/README.md @@ -131,60 +131,6 @@ python ../gaudi_spawn.py \ This example has been validated with the following DeepSpeed ZeRO-2 config: https://github.com/huggingface/optimum-habana/blob/main/tests/configs/deepspeed_zero_2.json -### Multi-card Training with Deepspeed (chatglm3-6b) -```bash -python ../gaudi_spawn.py \ - --world_size 8 --use_deepspeed run_clm.py \ - --config_name THUDM/chatglm3-6b \ - --tokenizer_name THUDM/chatglm3-6b \ - --dataset_name wikitext \ - --dataset_config_name wikitext-2-raw-v1 \ - --per_device_train_batch_size 6 \ - --per_device_eval_batch_size 4 \ - --do_train \ - --do_eval \ - --deepspeed llama2_ds_zero3_config.json \ - --output_dir /tmp/test-clm \ - --gaudi_config_name Habana/gpt2 \ - --use_habana \ - --use_lazy_mode \ - --throughput_warmup_steps 3 \ - --bf16 \ - --block_size 1024 \ - --use_cache False \ - --overwrite_output_dir \ - --logging_first_step True \ - --logging_steps 20 -``` - -### Multi-card Training with Deepspeed (Baichuan2-13B-Chat) -```bash -python ../gaudi_spawn.py \ - --world_size 8 --use_deepspeed run_clm.py \ - --config_name baichuan-inc/Baichuan2-13B-Chat \ - --tokenizer_name baichuan-inc/Baichuan2-13B-Chat \ - --dataset_name wikitext \ - --num_train_epochs 30 \ - --dataset_config_name wikitext-2-raw-v1 \ - --per_device_train_batch_size 2 \ - --per_device_eval_batch_size 2 \ - --do_train \ - --do_eval \ - --deepspeed llama2_ds_zero3_config.json \ - --output_dir /tmp/test-clm \ - --gaudi_config_name Habana/gpt2 \ - --use_habana \ - --use_lazy_mode \ - --throughput_warmup_steps 3 \ - --bf16 \ - --block_size 1024 \ - --use_cache False \ - --overwrite_output_dir \ - --logging_first_step True \ - --logging_steps 20 -``` - - ## Multi-Node Training with Deepspeed (GPT-NeoX) The following command triggers the fine-tuning of [GPT-NeoX-20B](https://huggingface.co/EleutherAI/gpt-neox-20b) on WikiText-2 with Deepspeed ZeRO-2. @@ -226,10 +172,11 @@ Following the RoBERTa paper, we use dynamic masking rather than static masking. converge slightly slower (over-fitting takes more epochs). -### Single-card Training +### Multi-card Training ```bash -python run_mlm.py \ +python ../gaudi_spawn.py \ + --world_size 8 --use_mpi run_mlm.py \ --model_name_or_path roberta-base \ --dataset_name wikitext \ --dataset_config_name wikitext-2-raw-v1 \ @@ -246,54 +193,12 @@ python run_mlm.py \ --bf16 ``` -To run on your own training and validation files, use the following command: - -```bash -python run_mlm.py \ - --model_name_or_path roberta-base \ - --train_file path_to_train_file \ - --validation_file path_to_validation_file \ - --per_device_train_batch_size 8 \ - --per_device_eval_batch_size 8 \ - --do_train \ - --do_eval \ - --output_dir /tmp/test-mlm \ - --use_habana \ - --use_lazy_mode \ - --use_hpu_graphs_for_inference \ - --gaudi_config_name Habana/roberta-base \ - --throughput_warmup_steps 3 \ - --bf16 -``` - If your dataset is organized with one sample per line, you can use the `--line_by_line` flag (otherwise the script concatenates all texts and then splits them into blocks of the same length). **Note:** On HPU, you should use the flag `--pad_to_max_length` in conjunction with the `--line_by_line` flag to make sure all your batches have the same length. -### Multi-card Training - -```bash -python ../gaudi_spawn.py \ - --world_size 8 --use_mpi run_mlm.py \ - --model_name_or_path roberta-base \ - --dataset_name wikitext \ - --dataset_config_name wikitext-2-raw-v1 \ - --per_device_train_batch_size 8 \ - --per_device_eval_batch_size 8 \ - --do_train \ - --do_eval \ - --output_dir /tmp/test-mlm \ - --use_habana \ - --use_lazy_mode \ - --use_hpu_graphs_for_inference \ - --gaudi_config_name Habana/roberta-base \ - --throughput_warmup_steps 3 \ - --bf16 -``` - - ### Training in torch.compile mode RoBERTa-Large model training in [torch.compile](pytorch.org/tutorials/intermediate/torch_compile_tutorial.html) mode is enabled by applying the following changes to your command, a) Set the following environment variables `PT_HPU_LAZY_MODE=0` and `PT_ENABLE_INT64_SUPPORT=1`. @@ -324,78 +229,6 @@ python run_clm.py \ --bf16 ``` - -## Using DeepSpeed - -Multi-card examples can be simply adapted to be run with DeepSpeed. Here is the CLM example with GPT2-XL: - -```bash -python ../gaudi_spawn.py \ - --world_size 8 --use_deepspeed run_clm.py \ - --model_name_or_path gpt2-xl \ - --dataset_name wikitext \ - --dataset_config_name wikitext-2-raw-v1 \ - --per_device_train_batch_size 16 \ - --per_device_eval_batch_size 4 \ - --do_train \ - --do_eval \ - --learning_rate 4e-4 \ - --output_dir /tmp/test-clm \ - --gaudi_config_name Habana/gpt2 \ - --use_habana \ - --use_lazy_mode \ - --use_hpu_graphs_for_inference \ - --gradient_checkpointing \ - --use_cache False \ - --throughput_warmup_steps 3 \ - --deepspeed path_to_my_deepspeed_config -``` - -You can look at the [documentation](https://huggingface.co/docs/optimum/habana/usage_guides/deepspeed) for more information about how to use DeepSpeed in Optimum Habana. -Here is a DeepSpeed configuration you can use to train your models on Gaudi: -```json -{ - "steps_per_print": 64, - "train_batch_size": "auto", - "train_micro_batch_size_per_gpu": "auto", - "gradient_accumulation_steps": "auto", - "bf16": { - "enabled": true - }, - "gradient_clipping": 1.0, - "zero_optimization": { - "stage": 2, - "overlap_comm": false, - "reduce_scatter": false, - "contiguous_gradients": false - } -} -``` - -Here is another example with Bloom-7B1: - -```bash -DEEPSPEED_HPU_ZERO3_SYNC_MARK_STEP_REQUIRED=1 PT_HPU_MAX_COMPOUND_OP_SYNC=1 PT_HPU_MAX_COMPOUND_OP_SIZE=1 python ../gaudi_spawn.py \ - --world_size 8 --use_deepspeed run_clm.py \ - --model_name_or_path bigscience/bloom-7b1 \ - --dataset_name wikitext \ - --dataset_config_name wikitext-2-raw-v1 \ - --per_device_train_batch_size 8 \ - --do_train \ - --output_dir /tmp/test-clm \ - --gaudi_config_name Habana/roberta-base \ - --use_habana \ - --use_lazy_mode \ - --gradient_checkpointing \ - --use_cache False \ - --throughput_warmup_steps 3 \ - --save_strategy "no" \ - --learning_rate 1e-04 \ - --deepspeed path_to_my_deepspeed_config -``` -[This](https://github.com/huggingface/optimum-habana/blob/main/tests/configs/deepspeed_zero_3_gaudi1.json) is a DeepSpeed configuration you can use to train this model on Gaudi1. - - ## Inference To run only inference, you can start from the commands above and you just have to remove the training-only arguments such as `--do_train`, `--per_device_train_batch_size`, `--num_train_epochs`, etc... @@ -456,141 +289,6 @@ python3 run_lora_clm.py \ --validation_split_percentage 4 \ --adam_epsilon 1e-08 ``` -- Single-card finetuning of Falcon-40B: -```bash -PT_HPU_AUTOCAST_LOWER_PRECISION_OPS_LIST=ops_bf16.txt python3 run_lora_clm.py \ - --model_name_or_path tiiuae/falcon-40b \ - --dataset_name timdettmers/openassistant-guanaco \ - --bf16 True \ - --output_dir ./model_lora_falcon \ - --num_train_epochs 3 \ - --per_device_train_batch_size 1 \ - --per_device_eval_batch_size 1 \ - --gradient_accumulation_steps 16 \ - --eval_strategy "no" \ - --save_strategy "no" \ - --learning_rate 3e-4 \ - --max_grad_norm 0.3 \ - --warmup_ratio 0.03 \ - --lr_scheduler_type "constant" \ - --logging_steps 1 \ - --do_train \ - --use_habana \ - --use_lazy_mode \ - --pipelining_fwd_bwd \ - --throughput_warmup_steps 3 \ - --lora_rank=64 \ - --lora_alpha=16 \ - --lora_dropout=0.1 \ - --lora_target_modules "query_key_value" "dense" "dense_h_to_4h" "dense_4h_to_h" \ - --dataset_concatenation \ - --max_seq_length 256 \ - --low_cpu_mem_usage True \ - --adam_epsilon 1e-08 \ - --do_eval \ - --validation_split_percentage 5 -``` - -- Multi-card finetuning of Llama1-7B: -```bash -python ../gaudi_spawn.py \ - --world_size 8 --use_mpi run_lora_clm.py \ - --model_name_or_path huggyllama/llama-7b \ - --dataset_name tatsu-lab/alpaca \ - --bf16 True \ - --output_dir ./model_lora_llama_ddp \ - --num_train_epochs 3 \ - --per_device_train_batch_size 8 \ - --gradient_accumulation_steps 2 \ - --eval_strategy "no" \ - --save_strategy "no" \ - --learning_rate 3e-4 \ - --warmup_ratio 0.03 \ - --lr_scheduler_type "constant" \ - --max_grad_norm 0.3 \ - --logging_steps 1 \ - --do_train \ - --do_eval \ - --use_habana \ - --use_lazy_mode \ - --throughput_warmup_steps 3 \ - --lora_rank=8 \ - --lora_alpha=16 \ - --lora_dropout=0.05 \ - --lora_target_modules "q_proj" "v_proj" \ - --dataset_concatenation \ - --max_seq_length 512 \ - --ddp_bucket_cap_mb 50 \ - --adam_epsilon 1e-08 \ - --validation_split_percentage 4 \ - --low_cpu_mem_usage True -``` - -- Multi-card finetuning of Llama2-7B with FP8: -```bash -PT_HPU_AUTOCAST_LOWER_PRECISION_OPS_LIST=ops_bf16.txt python ../gaudi_spawn.py \ - --world_size 8 --use_mpi run_lora_clm.py \ - --model_name_or_path meta-llama/Llama-2-7b-hf \ - --dataset_name tatsu-lab/alpaca \ - --bf16 True \ - --output_dir ./model_lora_llama \ - --num_train_epochs 3 \ - --per_device_train_batch_size 16 \ - --gradient_accumulation_steps 1 \ - --eval_strategy "no" \ - --save_strategy "no" \ - --learning_rate 3e-4 \ - --warmup_ratio 0.03 \ - --lr_scheduler_type "constant" \ - --max_grad_norm 0.3 \ - --logging_steps 20 \ - --do_train \ - --do_eval \ - --use_habana \ - --use_lazy_mode \ - --throughput_warmup_steps 18 \ - --lora_rank=8 \ - --lora_alpha=16 \ - --lora_dropout=0.05 \ - --lora_target_modules "q_proj" "v_proj" \ - --dataset_concatenation \ - --max_seq_length 512 \ - --ddp_bucket_cap_mb 50 \ - --adam_epsilon 1e-08 \ - --validation_split_percentage 10 \ - --low_cpu_mem_usage True \ - --pipelining_fwd_bwd \ - --fp8 True -``` - -- Multi-card finetuning of codegen-16B-mono: -```bash -python ../gaudi_spawn.py \ - --world_size 8 --use_mpi run_lora_clm.py \ - --model_name_or_path Salesforce/codegen-16B-mono \ - --dataset_name b-mc2/sql-create-context \ - --sql_prompt \ - --bf16 True \ - --output_dir ./finetuned-models/codegen-finetune-on-sql-create-context-hpu8-lora8-bs4 \ - --num_train_epochs 5 \ - --per_device_train_batch_size 4 \ - --per_device_eval_batch_size 4 \ - --eval_strategy "no" \ - --save_strategy "no" \ - --learning_rate 1e-4 \ - --logging_steps 1 \ - --dataset_concatenation \ - --do_train \ - --use_habana \ - --use_lazy_mode \ - --throughput_warmup_steps 3 \ - --use_hpu_graphs_for_inference \ - --lora_target_modules "qkv_proj" \ - --lora_rank 8 \ - --do_eval \ - --validation_split_percentage 10 \ - --use_cache False -``` - Multi-card finetuning of gemma2 using chat template: ```bash @@ -740,43 +438,6 @@ python3 ../gaudi_spawn.py --world_size 8 --use_mpi run_lora_clm.py \ --flash_attention_causal_mask True ``` -- Multi-card finetuning of Falcon-180B: - - Falcon-180B example command saves only the LoRA parameters at end - - For inference we need to merge the pretrained model and LoRA weights -```bash -PT_HPU_AUTOCAST_LOWER_PRECISION_OPS_LIST=ops_bf16.txt python3 ../gaudi_spawn.py \ - --world_size 8 --use_deepspeed run_lora_clm.py \ - --model_name_or_path tiiuae/falcon-180B \ - --dataset_name timdettmers/openassistant-guanaco \ - --bf16 True \ - --output_dir ./model_lora_falcon_ddp \ - --num_train_epochs 3 \ - --per_device_train_batch_size 1 \ - --per_device_eval_batch_size 1 \ - --gradient_accumulation_steps 16 \ - --eval_strategy "no" \ - --save_strategy "no" \ - --learning_rate 4e-4 \ - --max_grad_norm 0.3 \ - --warmup_ratio 0.03 \ - --lr_scheduler_type "constant" \ - --logging_steps 1 \ - --do_train \ - --use_habana \ - --use_lazy_mode \ - --pipelining_fwd_bwd \ - --throughput_warmup_steps 3 \ - --lora_rank=64 \ - --lora_alpha=16 \ - --lora_dropout=0.1 \ - --lora_target_modules "query_key_value" "dense" "dense_h_to_4h" "dense_4h_to_h" \ - --dataset_concatenation \ - --max_seq_length 256 \ - --adam_epsilon 1e-08 \ - --do_eval \ - --validation_split_percentage 5 \ - --deepspeed ds_falcon_180b_z3.json -``` Default `peft_type` is `lora`, you could enable adalora or ia3 using `--peft_type adalora` or `--peft_type ia3`, or enable llama-adapter for llama model using `--peft_type llama-adapter`, or enable ln-tuning using `--peft_type ln_tuning`, or enable vera using `--peft_type vera`. #### Custom Files @@ -824,7 +485,7 @@ The format of the text files (with extensions .text or .txt) is expected to be ### Prompt/Prefix/P-tuning To run prompt tuning finetuning, you can use `run_prompt_tuning_clm.py`. -Here are single-/multi-device command examples for Llama2-7B: +Here are single-card command examples for Llama2-7B: - single-card finetuning of meta-llama/Llama-2-7b-hf with dataset "ought/raft" and config "twitter_complaints": ```bash python3 run_prompt_tuning_clm.py \ @@ -844,25 +505,6 @@ python3 run_prompt_tuning_clm.py \ --use_lazy_mode ``` -- multi-card finetuning of meta-llama/Llama-2-7b-hf with dataset "ought/raft" and config "twitter_complaints": -```bash -python3 ../gaudi_spawn.py \ - --world_size 8 --use_mpi run_prompt_tuning_clm.py \ - --model_name_or_path meta-llama/Llama-2-7b-hf \ - --output_dir prompt_tuning_out \ - --bf16 True \ - --report_to=none \ - --per_device_train_batch_size 1 \ - --per_device_eval_batch_size 1 \ - --gradient_accumulation_steps 1 \ - --low_cpu_mem_usage True \ - --logging_steps 1 \ - --do_train \ - --num_train_epochs 50 \ - --do_eval \ - --use_habana \ - --use_lazy_mode -``` Default `peft_type` is `prompt_tuning`, you could enable prefix-tuning or p-tuning using `--peft_type prefix_tuning` or `--peft_type p_tuning`. Use the prompt finetuned model for text-generation: diff --git a/examples/language-modeling/llama3_ds_zero1_config.json b/examples/language-modeling/llama3_ds_zero1_config.json new file mode 100755 index 0000000000..b04ef0f0a9 --- /dev/null +++ b/examples/language-modeling/llama3_ds_zero1_config.json @@ -0,0 +1,13 @@ +{ + "steps_per_print": 64, + "train_batch_size": "auto", + "train_micro_batch_size_per_gpu": "auto", + "gradient_accumulation_steps": "auto", + "bf16": { + "enabled": true + }, + "gradient_clipping": 1.0, + "zero_optimization": { + "stage": 1 + } +} diff --git a/examples/language-modeling/run_clm.py b/examples/language-modeling/run_clm.py index b97b634941..93d85ba54b 100644 --- a/examples/language-modeling/run_clm.py +++ b/examples/language-modeling/run_clm.py @@ -63,7 +63,7 @@ def check_optimum_habana_min_version(*a, **b): # Will error if the minimal version of Transformers and Optimum Habana are not installed. Remove at your own risks. check_min_version("4.45.0") -check_optimum_habana_min_version("1.14.0.dev0") +check_optimum_habana_min_version("1.16.0.dev0") require_version("datasets>=2.14.0", "To fix: pip install -r examples/pytorch/language-modeling/requirements.txt") @@ -156,6 +156,32 @@ class ModelArguments: ) }, ) + attn_softmax_bf16: bool = field( + default=False, + metadata={"help": ("Whether to run attention softmax layer in bf16 precision for fine-tuning.")}, + ) + use_flash_attention: bool = field( + default=False, + metadata={"help": ("Whether to use Habana flash attention for fine-tuning.")}, + ) + flash_attention_recompute: bool = field( + default=False, + metadata={ + "help": ( + "Whether to enable recompute in Habana flash attention for fine-tuning." + " It is applicable only when use_flash_attention is True." + ) + }, + ) + flash_attention_causal_mask: bool = field( + default=False, + metadata={ + "help": ( + "Whether to enable causal mask in Habana flash attention for fine-tuning." + " It is applicable only when use_flash_attention is True." + ) + }, + ) low_cpu_mem_usage: bool = field( default=False, metadata={ @@ -472,7 +498,7 @@ def main(): else: model = AutoModelForCausalLM.from_config(config, trust_remote_code=model_args.trust_remote_code) n_params = sum({p.data_ptr(): p.numel() for p in model.parameters()}.values()) - logger.info(f"Training new model from scratch - Total size={n_params/2**20:.2f}M params") + logger.info(f"Training new model from scratch - Total size={n_params / 2**20:.2f}M params") # We resize the embeddings only when necessary to avoid index errors. If you are creating a model from scratch # on a small vocab and want a smaller embedding size, remove this test. @@ -482,6 +508,14 @@ def main(): if len(tokenizer) > embedding_size: model.resize_token_embeddings(len(tokenizer)) + # We need to add these fused kernels config + if model_args.attn_softmax_bf16: + model.generation_config.attn_softmax_bf16 = True + if model_args.use_flash_attention: + model.generation_config.use_flash_attention = True + model.generation_config.flash_attention_recompute = model_args.flash_attention_recompute + model.generation_config.flash_attention_causal_mask = model_args.flash_attention_causal_mask + # Preprocessing the datasets. # First we tokenize all the texts. if training_args.do_train: diff --git a/examples/language-modeling/run_lora_clm.py b/examples/language-modeling/run_lora_clm.py index 4782ed58ae..3ff7fbfd3a 100644 --- a/examples/language-modeling/run_lora_clm.py +++ b/examples/language-modeling/run_lora_clm.py @@ -70,7 +70,7 @@ def check_optimum_habana_min_version(*a, **b): logger = logging.getLogger(__name__) # Will error if the minimal version of Optimum Habana is not installed. Remove at your own risks. -check_optimum_habana_min_version("1.14.0.dev0") +check_optimum_habana_min_version("1.16.0.dev0") @dataclass diff --git a/examples/language-modeling/run_mlm.py b/examples/language-modeling/run_mlm.py index 30315bfc84..2de43c910b 100644 --- a/examples/language-modeling/run_mlm.py +++ b/examples/language-modeling/run_mlm.py @@ -62,7 +62,7 @@ def check_optimum_habana_min_version(*a, **b): # Will error if the minimal version of Transformers and Optimum Habana are not installed. Remove at your own risks. check_min_version("4.45.0") -check_optimum_habana_min_version("1.14.0.dev0") +check_optimum_habana_min_version("1.16.0.dev0") require_version("datasets>=2.14.0", "To fix: pip install -r examples/pytorch/language-modeling/requirements.txt") diff --git a/examples/language-modeling/run_multitask_prompt_tuning.py b/examples/language-modeling/run_multitask_prompt_tuning.py index 1d81bcc496..9f955db44e 100644 --- a/examples/language-modeling/run_multitask_prompt_tuning.py +++ b/examples/language-modeling/run_multitask_prompt_tuning.py @@ -61,7 +61,7 @@ def check_optimum_habana_min_version(*a, **b): # Will error if the minimal version of Transformers and Optimum Habana are not installed. Remove at your own risk. check_min_version("4.45.0") -check_optimum_habana_min_version("1.14.0.dev0") +check_optimum_habana_min_version("1.16.0.dev0") require_version("datasets>=1.8.0", "To fix: pip install -r examples/pytorch/language-modeling/requirements.txt") diff --git a/examples/language-modeling/run_prompt_tuning_clm.py b/examples/language-modeling/run_prompt_tuning_clm.py index e263c0c1b6..44ea542d14 100644 --- a/examples/language-modeling/run_prompt_tuning_clm.py +++ b/examples/language-modeling/run_prompt_tuning_clm.py @@ -63,7 +63,7 @@ def check_optimum_habana_min_version(*a, **b): # Will error if the minimal version of Transformers and Optimum Habana are not installed. Remove at your own risks. check_min_version("4.45.0") -check_optimum_habana_min_version("1.14.0.dev0") +check_optimum_habana_min_version("1.16.0.dev0") require_version("datasets>=1.8.0", "To fix: pip install -r examples/pytorch/language-modeling/requirements.txt") diff --git a/examples/multi-node-training/EFA/Dockerfile b/examples/multi-node-training/EFA/Dockerfile index a527f99603..bc6f827164 100644 --- a/examples/multi-node-training/EFA/Dockerfile +++ b/examples/multi-node-training/EFA/Dockerfile @@ -1,4 +1,4 @@ -FROM vault.habana.ai/gaudi-docker/1.18.0/ubuntu22.04/habanalabs/pytorch-installer-2.4.0:latest +FROM vault.habana.ai/gaudi-docker/1.19.0/ubuntu22.04/habanalabs/pytorch-installer-2.5.1:latest # Installs pdsh and upgrade pip RUN apt-get update && apt-get install -y pdsh && \ @@ -14,11 +14,12 @@ RUN git clone "https://github.com/HabanaAI/hccl_ofi_wrapper.git" "${OFI_WRAPPER_ RUN sed -i 's/#Port 22/Port 3022/g' /etc/ssh/sshd_config && \ sed -i 's/# Port 22/ Port 3022/g' /etc/ssh/ssh_config && \ sed -i 's/#PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config && \ + /usr/bin/ssh-keygen -A && \ service ssh restart # Installs Optimum Habana and Habana's fork of DeepSpeed RUN pip install optimum[habana] && \ - pip install git+https://github.com/HabanaAI/DeepSpeed.git@1.18.0 + pip install git+https://github.com/HabanaAI/DeepSpeed.git@1.19.0 CMD ssh-keygen -t rsa -b 4096 -N '' -f ~/.ssh/id_rsa && \ chmod 600 ~/.ssh/id_rsa && \ diff --git a/examples/multi-node-training/GaudiNIC/Dockerfile b/examples/multi-node-training/GaudiNIC/Dockerfile index b3763c4277..5375a6fcc7 100644 --- a/examples/multi-node-training/GaudiNIC/Dockerfile +++ b/examples/multi-node-training/GaudiNIC/Dockerfile @@ -1,4 +1,4 @@ -FROM vault.habana.ai/gaudi-docker/1.18.0/ubuntu22.04/habanalabs/pytorch-installer-2.4.0:latest +FROM vault.habana.ai/gaudi-docker/1.19.0/ubuntu22.04/habanalabs/pytorch-installer-2.5.1:latest # Installs pdsh and upgrade pip RUN apt-get update && apt-get install -y pdsh && \ @@ -8,11 +8,12 @@ RUN apt-get update && apt-get install -y pdsh && \ RUN sed -i 's/#Port 22/Port 3022/g' /etc/ssh/sshd_config && \ sed -i 's/# Port 22/ Port 3022/g' /etc/ssh/ssh_config && \ sed -i 's/#PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config && \ + /usr/bin/ssh-keygen -A && \ service ssh restart # Installs Optimum Habana and Habana's fork of DeepSpeed RUN pip install optimum[habana] && \ - pip install git+https://github.com/HabanaAI/DeepSpeed.git@1.18.0 + pip install git+https://github.com/HabanaAI/DeepSpeed.git@1.19.0 CMD ssh-keygen -t rsa -b 4096 -N '' -f ~/.ssh/id_rsa && \ chmod 600 ~/.ssh/id_rsa && \ diff --git a/examples/object-detection/README.md b/examples/object-detection/README.md index aa82013326..0ce639dc9b 100644 --- a/examples/object-detection/README.md +++ b/examples/object-detection/README.md @@ -28,7 +28,3 @@ python3 run_example.py \ --bf16 \ --print_result ``` - -Models that have been validated: - - [facebook/detr-resnet-101](https://huggingface.co/facebook/detr-resnet-101) - - [facebook/detr-resnet-50](https://huggingface.co/facebook/detr-resnet-50) \ No newline at end of file diff --git a/examples/object-segementation/README.md b/examples/object-segementation/README.md index 936180e4f2..2b8728eb56 100644 --- a/examples/object-segementation/README.md +++ b/examples/object-segementation/README.md @@ -30,8 +30,6 @@ python3 run_example.py \ --bf16 \ --print_result ``` -Models that have been validated: - - [clipseg-rd64-refined ](https://huggingface.co/CIDAS/clipseg-rd64-refined) ### Segment Anything Model @@ -45,7 +43,4 @@ python3 run_example_sam.py \ --use_hpu_graphs \ --bf16 \ --print_result -``` -Models that have been validated: - - [facebook/sam-vit-base](https://huggingface.co/facebook/sam-vit-base) - - [facebook/sam-vit-huge](https://huggingface.co/facebook/sam-vit-huge) \ No newline at end of file +``` \ No newline at end of file diff --git a/examples/protein-folding/run_esmfold.py b/examples/protein-folding/run_esmfold.py index 6941e6e5c1..230d1c61e8 100644 --- a/examples/protein-folding/run_esmfold.py +++ b/examples/protein-folding/run_esmfold.py @@ -40,7 +40,7 @@ def check_optimum_habana_min_version(*a, **b): # Will error if the minimal version of Optimum Habana is not installed. Remove at your own risks. -check_optimum_habana_min_version("1.14.0.dev0") +check_optimum_habana_min_version("1.16.0.dev0") def convert_outputs_to_pdb(outputs): diff --git a/examples/protein-folding/run_sequence_classification.py b/examples/protein-folding/run_sequence_classification.py index dde75a2564..fa35d8b803 100644 --- a/examples/protein-folding/run_sequence_classification.py +++ b/examples/protein-folding/run_sequence_classification.py @@ -41,7 +41,7 @@ def check_optimum_habana_min_version(*a, **b): # Will error if the minimal version of Optimum Habana is not installed. Remove at your own risks. -check_optimum_habana_min_version("1.14.0.dev0") +check_optimum_habana_min_version("1.16.0.dev0") logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) diff --git a/examples/protein-folding/run_zero_shot_eval.py b/examples/protein-folding/run_zero_shot_eval.py index 3b475883e8..7da135f080 100644 --- a/examples/protein-folding/run_zero_shot_eval.py +++ b/examples/protein-folding/run_zero_shot_eval.py @@ -36,7 +36,7 @@ def check_optimum_habana_min_version(*a, **b): # Will error if the minimal version of Optimum Habana is not installed. Remove at your own risks. -check_optimum_habana_min_version("1.14.0.dev0") +check_optimum_habana_min_version("1.16.0.dev0") logging.basicConfig( diff --git a/examples/pytorch-image-models/README.md b/examples/pytorch-image-models/README.md index 8567a77fe6..731e61d612 100644 --- a/examples/pytorch-image-models/README.md +++ b/examples/pytorch-image-models/README.md @@ -16,20 +16,7 @@ limitations under the License. # pyTorch-IMage-Models (TIMM) Examples with HPUs -This directory contains the scripts that showcases how to inference/fine-tune the TIMM models on intel's HPUs with the lazy/graph modes. We support the trainging for single/multiple HPU cards both two. Currently we support several most downloadable models from Hugging Face as below list. - -- [timm/resnet50.a1_in1k](https://huggingface.co/timm/resnet50.a1_in1k) -- [timm/resnet18.a1_in1k](https://huggingface.co/timm/resnet18.a1_in1k) -- [timm/resnet18.fb_swsl_ig1b_ft_in1k](https://huggingface.co/timm/resnet18.fb_swsl_ig1b_ft_in1k) -- [timm/wide_resnet50_2.racm_in1k](https://huggingface.co/timm/wide_resnet50_2.racm_in1k) -- [timm/efficientnet_b3.ra2_in1k](https://huggingface.co/timm/efficientnet_b3.ra2_in1k) -- [timm/efficientnet_lite0.ra_in1k](https://huggingface.co/timm/efficientnet_lite0.ra_in1k) -- [timm/efficientnet_b0.ra_in1k](https://huggingface.co/timm/efficientnet_b0.ra_in1k) -- [timm/nf_regnet_b1.ra2_in1k](https://huggingface.co/timm/nf_regnet_b1.ra2_in1k) -- [timm/mobilenetv3_large_100.ra_in1k](https://huggingface.co/timm/mobilenetv3_large_100.ra_in1k) -- [timm/tf_mobilenetv3_large_minimal_100.in1k](https://huggingface.co/timm/tf_mobilenetv3_large_minimal_100.in1k) -- [timm/vit_base_patch16_224.augreg2_in21k_ft_in1k](https://huggingface.co/timm/vit_base_patch16_224.augreg2_in21k_ft_in1k) -- [timm/vgg19.tv_in1k](https://huggingface.co/timm/vgg19.tv_in1k) +This directory contains the scripts that showcase how to inference/fine-tune the TIMM models on Intel's HPUs with the lazy/graph modes. Training is supported for single/multiple HPU cards. Currently we can support first 10 most downloadable models from [Hugging Face timm link](https://huggingface.co/timm). In our example below for inference/training we will use [timm/resnet50.a1_in1k](https://huggingface.co/timm/resnet50.a1_in1k) as our testing model and same usage for other models. ## Requirements @@ -46,72 +33,30 @@ pip install . Here we show how to fine-tune the [imagenette2-320 dataset](https://huggingface.co/datasets/johnowhitaker/imagenette2-320) and model with [timm/resnet50.a1_in1k](https://huggingface.co/timm/resnet50.a1_in1k) from Hugging Face. -### Training with HPU lazy mode - -```bash -python train_hpu_lazy.py \ - --data-dir ./ \ - --dataset hfds/johnowhitaker/imagenette2-320 \ - --device 'hpu' \ - --model resnet50.a1_in1k \ - --train-split train \ - --val-split train \ - --dataset-download -``` - -python train_hpu_lazy.py --data-dir='./' --dataset hfds/johnowhitaker/imagenette2-320 --device='hpu' --model resnet50.a1_in1k ### Training with HPU graph mode ```bash python train_hpu_graph.py \ --data-dir ./ \ - --dataset hfds/johnowhitaker/imagenette2-320 \ + --dataset hfds/johnowhitaker/imagenette2-320 \ --device 'hpu' \ --model resnet50.a1_in1k \ --train-split train \ --val-split train \ - --dataset-download + --dataset-download ``` -Here the results for lazy mode is shown below for example: - -```bash -Train: 0 [ 0/73 ( 1%)] Loss: 6.86 (6.86) Time: 9.575s, 13.37/s (9.575s, 13.37/s) LR: 1.000e-05 Data: 0.844 (0.844) -Train: 0 [ 50/73 ( 70%)] Loss: 6.77 (6.83) Time: 0.320s, 400.32/s (0.470s, 272.39/s) LR: 1.000e-05 Data: 0.217 (0.047) -Test: [ 0/30] Time: 6.593 (6.593) Loss: 6.723 ( 6.723) Acc@1: 0.000 ( 0.000) Acc@5: 0.000 ( 0.000) -Test: [ 30/30] Time: 3.856 (0.732) Loss: 6.615 ( 6.691) Acc@1: 0.000 ( 0.076) Acc@5: 1.176 ( 3.287) - -Train: 1 [ 0/73 ( 1%)] Loss: 6.69 (6.69) Time: 0.796s, 160.74/s (0.796s, 160.74/s) LR: 1.001e-02 Data: 0.685 (0.685) -Train: 1 [ 50/73 ( 70%)] Loss: 3.23 (3.76) Time: 0.160s, 798.85/s (0.148s, 863.22/s) LR: 1.001e-02 Data: 0.053 (0.051) -Test: [ 0/30] Time: 0.663 (0.663) Loss: 1.926 ( 1.926) Acc@1: 46.094 ( 46.094) Acc@5: 85.938 ( 85.938) -Test: [ 30/30] Time: 0.022 (0.126) Loss: 1.462 ( 1.867) Acc@1: 63.529 ( 39.261) Acc@5: 83.529 ( 85.096) - -``` - - ## Multi-HPU training Here we show how to fine-tune the [imagenette2-320 dataset](https://huggingface.co/datasets/johnowhitaker/imagenette2-320) and model with [timm/resnet50.a1_in1k](https://huggingface.co/timm/resnet50.a1_in1k) from Hugging Face. -### Training with HPU lazy mode -```bash -torchrun --nnodes 1 --nproc_per_node 2 \ - train_hpu_lazy.py \ - --data-dir ./ \ - --dataset hfds/johnowhitaker/imagenette2-320 \ - --device 'hpu' \ - --model resnet50.a1_in1k \ - --train-split train \ - --val-split train \ - --dataset-download -``` ### Training with HPU graph mode ```bash torchrun --nnodes 1 --nproc_per_node 2 \ train_hpu_graph.py \ --data-dir ./ \ - --dataset hfds/johnowhitaker/imagenette2-320 \ + --dataset hfds/johnowhitaker/imagenette2-320 \ --device 'hpu' \ --model resnet50.a1_in1k \ --train-split train \ @@ -119,20 +64,6 @@ torchrun --nnodes 1 --nproc_per_node 2 \ --dataset-download ``` -Here the results for lazy mode is shown below for example: - -```bash -Train: 0 [ 0/36 ( 3%)] Loss: 6.88 (6.88) Time: 10.036s, 25.51/s (10.036s, 25.51/s) LR: 1.000e-05 Data: 0.762 (0.762) -Test: [ 0/15] Time: 7.796 (7.796) Loss: 6.915 ( 6.915) Acc@1: 0.000 ( 0.000) Acc@5: 0.000 ( 0.000) -Test: [ 15/15] Time: 1.915 (1.263) Loss: 6.847 ( 6.818) Acc@1: 0.000 ( 0.000) Acc@5: 0.000 ( 0.688) - -Train: 1 [ 0/36 ( 3%)] Loss: 6.84 (6.84) Time: 6.687s, 38.28/s (6.687s, 38.28/s) LR: 2.001e-02 Data: 0.701 (0.701) -Test: [ 0/15] Time: 1.315 (1.315) Loss: 2.463 ( 2.463) Acc@1: 14.062 ( 14.062) Acc@5: 48.828 ( 48.828) -Test: [ 15/15] Time: 0.020 (0.180) Loss: 1.812 ( 1.982) Acc@1: 52.326 ( 32.934) Acc@5: 66.279 ( 75.064) - -``` - - ## Single-HPU inference @@ -142,22 +73,13 @@ Here we show how to fine-tune the [imagenette2-320 dataset](https://huggingface. ```bash python inference.py \ --data-dir='./' \ - --dataset hfds/johnowhitaker/imagenette2-320 \ + --dataset hfds/johnowhitaker/imagenette2-320 \ --device='hpu' \ --model resnet50.a1_in1k \ --split train \ --graph_mode ``` -### HPU with lazy mode -```bash -python inference.py \ - --data-dir='./' \ - --dataset hfds/johnowhitaker/imagenette2-320 \ - --device='hpu' \ - --model resnet50.a1_in1k \ - --split train -``` diff --git a/examples/pytorch-image-models/train_hpu_graph.py b/examples/pytorch-image-models/train_hpu_graph.py index 767142e469..c9d0974258 100755 --- a/examples/pytorch-image-models/train_hpu_graph.py +++ b/examples/pytorch-image-models/train_hpu_graph.py @@ -136,6 +136,12 @@ metavar="PATH", help="Load this checkpoint into model after initialization (default: none)", ) +group.add_argument( + "--save_checkpoint", + action="store_true", + default=False, + help="saving checkpoint for each epoch", +) group.add_argument( "--resume", default="", @@ -635,10 +641,6 @@ def _parse_args(): return args, args_text -def setup(): - dist.init_process_group(backend="hccl") - - def cleanup(): dist.destroy_process_group() @@ -663,8 +665,6 @@ def main(): device = torch.device("hpu") if args.distributed: - setup() - _logger.info( "Training in distributed mode with multiple processes, 1 device per process." f"Process {args.rank}, total {args.world_size}, device {args.device}." @@ -1054,17 +1054,18 @@ def main(): ] ) output_dir = utils.get_outdir(args.output if args.output else "./output/train", exp_name) - saver = utils.CheckpointSaver( - model=model, - optimizer=optimizer, - args=args, - model_ema=model_ema, - amp_scaler=loss_scaler, - checkpoint_dir=output_dir, - recovery_dir=output_dir, - decreasing=decreasing_metric, - max_history=args.checkpoint_hist, - ) + if args.save_checkpoint: + saver = utils.CheckpointSaver( + model=model, + optimizer=optimizer, + args=args, + model_ema=model_ema, + amp_scaler=loss_scaler, + checkpoint_dir=output_dir, + recovery_dir=output_dir, + decreasing=decreasing_metric, + max_history=args.checkpoint_hist, + ) with open(os.path.join(output_dir, "args.yaml"), "w") as f: f.write(args_text) @@ -1098,7 +1099,7 @@ def main(): if utils.is_primary(args): _logger.info( - f'Scheduled epochs: {num_epochs}. LR stepped per {"epoch" if lr_scheduler.t_in_epochs else "update"}.' + f"Scheduled epochs: {num_epochs}. LR stepped per {'epoch' if lr_scheduler.t_in_epochs else 'update'}." ) results = [] @@ -1330,7 +1331,7 @@ def _backward(_loss): if utils.is_primary(args): _logger.info( f"Train: {epoch} [{update_idx:>4d}/{updates_per_epoch} " - f"({100. * (update_idx + 1) / updates_per_epoch:>3.0f}%)] " + f"({100.0 * (update_idx + 1) / updates_per_epoch:>3.0f}%)] " f"Loss: {losses_m.val:#.3g} ({losses_m.avg:#.3g}) " f"Time: {update_time_m.val:.3f}s, {update_sample_count / update_time_m.val:>7.2f}/s " f"({update_time_m.avg:.3f}s, {update_sample_count / update_time_m.avg:>7.2f}/s) " diff --git a/examples/pytorch-image-models/train_hpu_lazy.py b/examples/pytorch-image-models/train_hpu_lazy.py index 834f9ce043..17f1dac0d9 100755 --- a/examples/pytorch-image-models/train_hpu_lazy.py +++ b/examples/pytorch-image-models/train_hpu_lazy.py @@ -138,6 +138,12 @@ metavar="PATH", help="Load this checkpoint into model after initialization (default: none)", ) +group.add_argument( + "--save_checkpoint", + action="store_true", + default=False, + help="saving checkpoint for each epoch", +) group.add_argument( "--resume", default="", @@ -637,10 +643,6 @@ def _parse_args(): return args, args_text -def setup(): - dist.init_process_group(backend="hccl") - - def cleanup(): dist.destroy_process_group() @@ -665,8 +667,6 @@ def main(): device = torch.device("hpu") if args.distributed: - setup() - _logger.info( "Training in distributed mode with multiple processes, 1 device per process." f"Process {args.rank}, total {args.world_size}, device {args.device}." @@ -1053,17 +1053,18 @@ def main(): ] ) output_dir = utils.get_outdir(args.output if args.output else "./output/train", exp_name) - saver = utils.CheckpointSaver( - model=model, - optimizer=optimizer, - args=args, - model_ema=model_ema, - amp_scaler=loss_scaler, - checkpoint_dir=output_dir, - recovery_dir=output_dir, - decreasing=decreasing_metric, - max_history=args.checkpoint_hist, - ) + if args.save_checkpoint: + saver = utils.CheckpointSaver( + model=model, + optimizer=optimizer, + args=args, + model_ema=model_ema, + amp_scaler=loss_scaler, + checkpoint_dir=output_dir, + recovery_dir=output_dir, + decreasing=decreasing_metric, + max_history=args.checkpoint_hist, + ) with open(os.path.join(output_dir, "args.yaml"), "w") as f: f.write(args_text) @@ -1097,7 +1098,7 @@ def main(): if utils.is_primary(args): _logger.info( - f'Scheduled epochs: {num_epochs}. LR stepped per {"epoch" if lr_scheduler.t_in_epochs else "update"}.' + f"Scheduled epochs: {num_epochs}. LR stepped per {'epoch' if lr_scheduler.t_in_epochs else 'update'}." ) results = [] @@ -1331,7 +1332,7 @@ def _backward(_loss): if utils.is_primary(args): _logger.info( f"Train: {epoch} [{update_idx:>4d}/{updates_per_epoch} " - f"({100. * (update_idx + 1) / updates_per_epoch:>3.0f}%)] " + f"({100.0 * (update_idx + 1) / updates_per_epoch:>3.0f}%)] " f"Loss: {losses_m.val:#.3g} ({losses_m.avg:#.3g}) " f"Time: {update_time_m.val:.3f}s, {update_sample_count / update_time_m.val:>7.2f}/s " f"({update_time_m.avg:.3f}s, {update_sample_count / update_time_m.avg:>7.2f}/s) " diff --git a/examples/question-answering/README.md b/examples/question-answering/README.md index bf6cd04aec..d7a83ea5c8 100755 --- a/examples/question-answering/README.md +++ b/examples/question-answering/README.md @@ -33,171 +33,6 @@ First, you should install the requirements: pip install -r requirements.txt ``` -## Fine-tuning BERT on SQuAD1.1 - -For the following cases, an example of a Gaudi configuration file is given -[here](https://github.com/huggingface/optimum-habana#how-to-use-it). - - -### Single-card Training - -This example code fine-tunes BERT on the SQuAD1.1 dataset. - -```bash -python run_qa.py \ - --model_name_or_path bert-large-uncased-whole-word-masking \ - --gaudi_config_name Habana/bert-large-uncased-whole-word-masking \ - --dataset_name squad \ - --do_train \ - --do_eval \ - --per_device_train_batch_size 32 \ - --per_device_eval_batch_size 8 \ - --learning_rate 3e-5 \ - --num_train_epochs 2 \ - --max_seq_length 384 \ - --doc_stride 128 \ - --output_dir /tmp/squad/ \ - --use_habana \ - --use_lazy_mode \ - --use_hpu_graphs_for_inference \ - --throughput_warmup_steps 3 \ - --bf16 \ - --sdp_on_bf16 -``` - -For torch.compile mode, -```bash -PT_HPU_LAZY_MODE=0 PT_ENABLE_INT64_SUPPORT=1 python run_qa.py \ - --model_name_or_path bert-large-uncased-whole-word-masking \ - --gaudi_config_name Habana/bert-large-uncased-whole-word-masking \ - --dataset_name squad \ - --do_train \ - --do_eval \ - --per_device_train_batch_size 32 \ - --per_device_eval_batch_size 8 \ - --learning_rate 3e-5 \ - --num_train_epochs 2 \ - --max_seq_length 384 \ - --doc_stride 128 \ - --output_dir /tmp/squad/ \ - --use_habana \ - --torch_compile_backend hpu_backend \ - --torch_compile \ - --use_lazy_mode false \ - --throughput_warmup_steps 3 \ - --bf16 \ - --sdp_on_bf16 -``` - -### Multi-card Training - -Here is how you would fine-tune the BERT large model (with whole word masking) on the SQuAD dataset using the `run_qa` script, with 8 HPUs: - -```bash -python ../gaudi_spawn.py \ - --world_size 8 --use_mpi run_qa.py \ - --model_name_or_path bert-large-uncased-whole-word-masking \ - --gaudi_config_name Habana/bert-large-uncased-whole-word-masking \ - --dataset_name squad \ - --do_train \ - --do_eval \ - --per_device_train_batch_size 32 \ - --per_device_eval_batch_size 8 \ - --learning_rate 3e-5 \ - --num_train_epochs 2 \ - --max_seq_length 384 \ - --doc_stride 128 \ - --output_dir /tmp/squad_output/ \ - --use_habana \ - --use_lazy_mode \ - --use_hpu_graphs_for_inference \ - --throughput_warmup_steps 3 \ - --bf16 \ - --sdp_on_bf16 -``` - -For torch.compile mode, -```bash -PT_HPU_LAZY_MODE=0 PT_ENABLE_INT64_SUPPORT=1 python ../gaudi_spawn.py \ - --world_size 8 --use_mpi run_qa.py \ - --model_name_or_path bert-large-uncased-whole-word-masking \ - --gaudi_config_name Habana/bert-large-uncased-whole-word-masking \ - --dataset_name squad \ - --do_train \ - --do_eval \ - --per_device_train_batch_size 32 \ - --per_device_eval_batch_size 8 \ - --learning_rate 3e-5 \ - --num_train_epochs 2 \ - --max_seq_length 384 \ - --doc_stride 128 \ - --output_dir /tmp/squad_output/ \ - --use_habana \ - --torch_compile_backend hpu_backend \ - --torch_compile \ - --use_lazy_mode false \ - --throughput_warmup_steps 3 \ - --bf16 \ - --sdp_on_bf16 -``` - - -### Using DeepSpeed - -Similarly to multi-card training, here is how you would fine-tune the BERT large model (with whole word masking) on the SQuAD dataset using DeepSpeed with 8 HPUs: - -```bash -python ../gaudi_spawn.py \ - --world_size 8 --use_deepspeed run_qa.py \ - --model_name_or_path bert-large-uncased-whole-word-masking \ - --gaudi_config_name Habana/bert-large-uncased-whole-word-masking \ - --dataset_name squad \ - --do_train \ - --do_eval \ - --per_device_train_batch_size 32 \ - --per_device_eval_batch_size 8 \ - --learning_rate 3e-5 \ - --num_train_epochs 2 \ - --max_seq_length 384 \ - --doc_stride 128 \ - --output_dir /tmp/squad_output/ \ - --use_habana \ - --use_lazy_mode \ - --use_hpu_graphs_for_inference \ - --throughput_warmup_steps 3 \ - --deepspeed path_to_my_deepspeed_config \ - --sdp_on_bf16 -``` - -You can look at the [documentation](https://huggingface.co/docs/optimum/habana/usage_guides/deepspeed) for more information about how to use DeepSpeed in Optimum Habana. -Here is a DeepSpeed configuration you can use to train your models on Gaudi: -```json -{ - "steps_per_print": 64, - "train_batch_size": "auto", - "train_micro_batch_size_per_gpu": "auto", - "gradient_accumulation_steps": "auto", - "bf16": { - "enabled": true - }, - "gradient_clipping": 1.0, - "zero_optimization": { - "stage": 2, - "overlap_comm": false, - "reduce_scatter": false, - "contiguous_gradients": false - } -} -``` - - -### Training in torch.compile mode - -Albert XXL model training in [torch.compile](pytorch.org/tutorials/intermediate/torch_compile_tutorial.html) mode is enabled by applying the following changes to your command, \ -a) Set the following environment variables `PT_HPU_LAZY_MODE=0` and `PT_ENABLE_INT64_SUPPORT=1`. \ -b) Run the above commands with `--model_name_or_path albert-xxlarge-v1`, `--use_lazy_mode False` and add `--torch_compile`, `--torch_compile_backend hpu_backend` and remove `--use_hpu_graphs_for_inference` flags. - - ## Fine-tuning Llama on SQuAD1.1 > [!NOTE] @@ -207,7 +42,7 @@ Here is a command you can run to train a Llama model for question answering: ```bash python ../gaudi_spawn.py \ --world_size 8 --use_deepspeed run_qa.py \ - --model_name_or_path FlagAlpha/Llama2-Chinese-13b-Chat \ + --model_name_or_path meta-llama/Llama-2-7b-chat-hf \ --gaudi_config_name Habana/bert-large-uncased-whole-word-masking \ --dataset_name squad \ --do_train \ @@ -232,76 +67,3 @@ python ../gaudi_spawn.py \ ## Inference To run only inference, you can start from the commands above and you just have to remove the training-only arguments such as `--do_train`, `--per_device_train_batch_size`, `--num_train_epochs`, etc... - -For instance, you can run inference with BERT on SQuAD on 1 Gaudi card with the following command: -```bash -python run_qa.py \ - --model_name_or_path bert-large-uncased-whole-word-masking \ - --gaudi_config_name Habana/bert-large-uncased-whole-word-masking \ - --dataset_name squad \ - --do_eval \ - --per_device_eval_batch_size 8 \ - --max_seq_length 384 \ - --doc_stride 128 \ - --output_dir /tmp/squad/ \ - --use_habana \ - --use_lazy_mode \ - --use_hpu_graphs_for_inference \ - --bf16 \ - --sdp_on_bf16 -``` - - -## Recommended Hyperparameters for Mixed Precision - -| | learning_rate | num_train_epochs | per_device_train_batch_size | per_device_eval_batch_size | -|----------------------------|:----:|:--:|:-:|:-:| -| BERT base | 3e-5 | 2 | 24 | 8 | -| BERT large | 3e-5 | 2 | 24 | 8 | -| RoBERTa base | 3e-5 | 2 | 12 | 8 | -| RoBERTa large | 3e-5 | 2 | 12 | 8 | -| ALBERT large (single-card) | 5e-5 | 2 | 32 | 4 | -| ALBERT large (multi-card) | 6e-5 | 2 | 32 | 4 | -| ALBERT XXL (single-card) | 5e-6 | 2 | 16 | 2 | -| ALBERT XXL (multi-card) | 5e-5 | 2 | 16 | 2 | -| DistilBERT | 5e-5 | 3 | 8 | 8 | -| meta-llama/Llama-2-13b-chat-hf (multi-card) | 3e-5 | 2 | 8 | 8 | -| FlagAlpha/Llama2-Chinese-13b-Chat (multi-card) | 3e-5 | 2 | 8 | 8 | - - -## Fine-tuning T5 on SQuAD2.0 - -The [`run_seq2seq_qa.py`](https://github.com/huggingface/optimum-habana/blob/main/examples/question-answering/run_seq2seq_qa.py) script is meant for encoder-decoder (also called seq2seq) Transformer models, such as T5 or BART. These models are generative, rather than discriminative. This means that they learn to generate the correct answer, rather than predicting the start and end position of the tokens of the answer. - -The following command fine-tunes T5 on the SQuAD2.0 dataset: - -```bash -python run_seq2seq_qa.py \ - --model_name_or_path t5-small \ - --gaudi_config_name Habana/t5 \ - --dataset_name squad_v2 \ - --version_2_with_negative \ - --context_column context \ - --question_column question \ - --answer_column answers \ - --do_train \ - --do_eval \ - --per_device_train_batch_size 16 \ - --per_device_eval_batch_size 33 \ - --learning_rate 3e-5 \ - --num_train_epochs 2 \ - --max_seq_length 384 \ - --doc_stride 128 \ - --output_dir /tmp/seq2seq_squad/ \ - --predict_with_generate \ - --use_habana \ - --use_lazy_mode \ - --use_hpu_graphs_for_inference \ - --ignore_pad_token_for_loss False \ - --pad_to_max_length \ - --save_strategy epoch \ - --throughput_warmup_steps 3 \ - --bf16 -``` - -For multi-card and DeepSpeed runs, you can use `python ../gaudi_spawn.py --world_size 8 --use_mpi` and `python ../gaudi_spawn.py --world_size 8 --use_deepspeed` as shown in the previous sections. diff --git a/examples/question-answering/run_qa.py b/examples/question-answering/run_qa.py index d22949c076..5ad77be381 100644 --- a/examples/question-answering/run_qa.py +++ b/examples/question-answering/run_qa.py @@ -61,7 +61,7 @@ def check_optimum_habana_min_version(*a, **b): # Will error if the minimal version of Transformers and Optimum Habana are not installed. Remove at your own risks. check_min_version("4.45.0") -check_optimum_habana_min_version("1.14.0.dev0") +check_optimum_habana_min_version("1.16.0.dev0") require_version("datasets>=1.8.0", "To fix: pip install -r examples/pytorch/question-answering/requirements.txt") diff --git a/examples/question-answering/run_seq2seq_qa.py b/examples/question-answering/run_seq2seq_qa.py index 1f045552bd..aaadbee417 100644 --- a/examples/question-answering/run_seq2seq_qa.py +++ b/examples/question-answering/run_seq2seq_qa.py @@ -57,7 +57,7 @@ def check_optimum_habana_min_version(*a, **b): # Will error if the minimal version of Transformers and Optimum Habana are not installed. Remove at your own risks. check_min_version("4.45.0") -check_optimum_habana_min_version("1.14.0.dev0") +check_optimum_habana_min_version("1.16.0.dev0") require_version("datasets>=1.8.0", "To fix: pip install -r examples/pytorch/question-answering/requirements.txt") diff --git a/examples/speech-recognition/README.md b/examples/speech-recognition/README.md index 4d5eb69b91..fe80cf775f 100644 --- a/examples/speech-recognition/README.md +++ b/examples/speech-recognition/README.md @@ -85,6 +85,7 @@ python run_speech_recognition_ctc.py \ --use_lazy_mode \ --gaudi_config_name="Habana/wav2vec2" \ --throughput_warmup_steps="3" \ + --sdp_on_bf16 \ --bf16 \ --use_hpu_graphs_for_training \ --use_hpu_graphs_for_inference \ @@ -128,6 +129,7 @@ python ../gaudi_spawn.py \ --gaudi_config_name Habana/wav2vec2 \ --throughput_warmup_steps 3 \ --bf16 \ + --sdp_on_bf16 \ --use_hpu_graphs_for_training \ --use_hpu_graphs_for_inference \ --sdp_on_bf16 @@ -143,7 +145,7 @@ On 8 HPUs, this script should run in *ca.* 49 minutes and yield a CTC loss of ** > You need to install DeepSpeed with: > ```bash -> pip install git+https://github.com/HabanaAI/DeepSpeed.git@1.18.0 +> pip install git+https://github.com/HabanaAI/DeepSpeed.git@1.19.0 > ``` DeepSpeed can be used with almost the same command as for a multi-card run: @@ -210,6 +212,7 @@ python run_speech_recognition_ctc.py \ --use_habana \ --use_lazy_mode \ --gaudi_config_name="Habana/wav2vec2" \ + --sdp_on_bf16 \ --bf16 \ --use_hpu_graphs_for_inference \ --sdp_on_bf16 @@ -250,6 +253,7 @@ python run_speech_recognition_seq2seq.py \ --max_duration_in_seconds="30" \ --text_column_name="sentence" \ --freeze_feature_encoder="False" \ + --sdp_on_bf16 \ --bf16 \ --overwrite_output_dir \ --do_train \ @@ -259,7 +263,8 @@ python run_speech_recognition_seq2seq.py \ --use_hpu_graphs_for_inference \ --label_features_max_length 128 \ --dataloader_num_workers 8 \ - --throughput_warmup_steps 3 + --throughput_warmup_steps 3 \ + --sdp_on_bf16 ``` If training on a different language, you should be sure to change the `language` argument. The `language` and `task` arguments should be omitted for English speech recognition. @@ -289,6 +294,7 @@ python ../gaudi_spawn.py \ --max_duration_in_seconds="30" \ --text_column_name="sentence" \ --freeze_feature_encoder="False" \ + --sdp_on_bf16 \ --bf16 \ --overwrite_output_dir \ --do_train \ @@ -322,6 +328,7 @@ python run_speech_recognition_seq2seq.py \ --max_duration_in_seconds="30" \ --text_column_name="sentence" \ --freeze_feature_encoder="False" \ + --sdp_on_bf16 \ --bf16 \ --overwrite_output_dir \ --do_eval \ @@ -329,5 +336,6 @@ python run_speech_recognition_seq2seq.py \ --use_habana \ --use_hpu_graphs_for_inference \ --label_features_max_length 128 \ - --dataloader_num_workers 8 + --dataloader_num_workers 8 \ + --sdp_on_bf16 ``` diff --git a/examples/speech-recognition/requirements.txt b/examples/speech-recognition/requirements.txt index 6bdf66fe01..b7c33c8ba1 100644 --- a/examples/speech-recognition/requirements.txt +++ b/examples/speech-recognition/requirements.txt @@ -1,4 +1,5 @@ -datasets >= 1.18.0 +datasets >= 1.18.0, <= 2.19.2 +numba==0.60.0 librosa jiwer evaluate diff --git a/examples/speech-recognition/run_speech_recognition_ctc.py b/examples/speech-recognition/run_speech_recognition_ctc.py index 83865556d1..f5da991dbf 100644 --- a/examples/speech-recognition/run_speech_recognition_ctc.py +++ b/examples/speech-recognition/run_speech_recognition_ctc.py @@ -60,7 +60,7 @@ def check_optimum_habana_min_version(*a, **b): # Will error if the minimal version of Transformers and Optimum Habana are not installed. Remove at your own risks. check_min_version("4.45.0") -check_optimum_habana_min_version("1.14.0.dev0") +check_optimum_habana_min_version("1.16.0.dev0") require_version("datasets>=1.18.0", "To fix: pip install -r examples/pytorch/speech-recognition/requirements.txt") @@ -504,7 +504,7 @@ def main(): # E.g. characters, such as `,` and `.` do not really have an acoustic characteristic # that could be easily picked up by the model chars_to_ignore_regex = ( - f'[{"".join(data_args.chars_to_ignore).replace(" ", "")}]' if data_args.chars_to_ignore is not None else None + f"[{''.join(data_args.chars_to_ignore).replace(' ', '')}]" if data_args.chars_to_ignore is not None else None ) text_column_name = data_args.text_column_name diff --git a/examples/speech-recognition/run_speech_recognition_seq2seq.py b/examples/speech-recognition/run_speech_recognition_seq2seq.py index ff9702e80c..db25b852eb 100755 --- a/examples/speech-recognition/run_speech_recognition_seq2seq.py +++ b/examples/speech-recognition/run_speech_recognition_seq2seq.py @@ -56,7 +56,7 @@ def check_optimum_habana_min_version(*a, **b): # Will error if the minimal version of Transformers is not installed. Remove at your own risks. check_min_version("4.45.0") -check_optimum_habana_min_version("1.14.0.dev0") +check_optimum_habana_min_version("1.16.0.dev0") require_version("datasets>=1.18.0", "To fix: pip install -r examples/pytorch/speech-recognition/requirements.txt") diff --git a/examples/stable-diffusion/README.md b/examples/stable-diffusion/README.md index f4df474f09..98e818a5c6 100644 --- a/examples/stable-diffusion/README.md +++ b/examples/stable-diffusion/README.md @@ -16,10 +16,10 @@ limitations under the License. # Stable Diffusion Examples -This directory contains a script that showcases how to perform text-to-image generation using Stable Diffusion on IntelĀ® GaudiĀ® AI Accelerators. - -Stable Diffusion was proposed in [Stable Diffusion Announcement](https://stability.ai/blog/stable-diffusion-announcement) by Patrick Esser and Robin Rombach and the Stability AI team. +This directory contains sample scripts demonstrating how to perform diffusion-based generative tasks on IntelĀ® GaudiĀ® AI Accelerators. +Stable Diffusion was introduced in [Stable Diffusion Announcement](https://stability.ai/blog/stable-diffusion-announcement) by Patrick Esser, +Robin Rombach and the Stability AI team. ## Requirements @@ -28,11 +28,11 @@ First, you should install the requirements: pip install -r requirements.txt ``` -## Text-to-image Generation +## Text-to-Image Generation -### Single Prompt +### Stable Diffusion -Here is how to generate images with one prompt: +Here's how to generate images using the Stable Diffusion 1.4 model with a single prompt: ```bash python text_to_image_generation.py \ @@ -44,16 +44,16 @@ python text_to_image_generation.py \ --use_habana \ --use_hpu_graphs \ --gaudi_config Habana/stable-diffusion \ + --sdp_on_bf16 \ --bf16 ``` +> [!NOTE] > HPU graphs are recommended when generating images by batches to get the fastest possible generations. > The first batch of images entails a performance penalty. All subsequent batches will be generated much faster. > You can enable this mode with `--use_hpu_graphs`. -### Multiple Prompts - -Here is how to generate images with several prompts: +To generate images with multiple prompts, simply include two prompts in your input as shown below: ```bash python text_to_image_generation.py \ @@ -65,12 +65,11 @@ python text_to_image_generation.py \ --use_habana \ --use_hpu_graphs \ --gaudi_config Habana/stable-diffusion \ + --sdp_on_bf16 \ --bf16 ``` -### Distributed inference with multiple HPUs - -Here is how to generate images with two prompts on two HPUs: +Distributed inference with multiple HPUs is also supported. Below is an example demonstrating how to generate images with two prompts on two HPUs: ```bash python ../gaudi_spawn.py \ @@ -83,17 +82,23 @@ python ../gaudi_spawn.py \ --use_habana \ --use_hpu_graphs \ --gaudi_config Habana/stable-diffusion \ + --sdp_on_bf16 \ --bf16 \ --distributed ``` +> [!NOTE] > HPU graphs are recommended when generating images by batches to get the fastest possible generations. > The first batch of images entails a performance penalty. All subsequent batches will be generated much faster. > You can enable this mode with `--use_hpu_graphs`. +You can run other older Stable Diffusion models in a similar manner. For example, to generate images with Stable Diffusion 1.5, use the option: +`--model_name_or_path stable-diffusion-v1-5/stable-diffusion-v1-5`. Examples showcasing Stable Diffusion 2 are provided next. + ### Stable Diffusion 2 -[Stable Diffusion 2](https://huggingface.co/docs/diffusers/main/en/api/pipelines/stable_diffusion_2) can also be used to generate images with this script. Here is an example for a single prompt: +[Stable Diffusion 2](https://huggingface.co/docs/diffusers/main/en/api/pipelines/stable_diffusion_2) can also be used +to generate images with this script. Here is an example demonstrating image generation with a single prompt: ```bash python text_to_image_generation.py \ @@ -107,20 +112,22 @@ python text_to_image_generation.py \ --use_habana \ --use_hpu_graphs \ --gaudi_config Habana/stable-diffusion-2 \ + --sdp_on_bf16 \ --bf16 ``` +> [!NOTE] > There are two different checkpoints for Stable Diffusion 2: -> > - use [stabilityai/stable-diffusion-2-1](https://huggingface.co/stabilityai/stable-diffusion-2-1) for generating 768x768 images > - use [stabilityai/stable-diffusion-2-1-base](https://huggingface.co/stabilityai/stable-diffusion-2-1-base) for generating 512x512 images ### Latent Diffusion Model for 3D (LDM3D) -[LDM3D](https://arxiv.org/abs/2305.10853) generates both image and depth map data from a given text prompt, allowing users to generate RGBD images from text prompts. +[LDM3D](https://arxiv.org/abs/2305.10853) generates both image and depth map data from a given text prompt, allowing users +to generate RGBD images from text prompts. -[Original checkpoint](https://huggingface.co/Intel/ldm3d) and [latest checkpoint](https://huggingface.co/Intel/ldm3d-4c) are open source. -A [demo](https://huggingface.co/spaces/Intel/ldm3d) is also available. Here is how to run this model: +[Original checkpoint](https://huggingface.co/Intel/ldm3d) and [latest checkpoint](https://huggingface.co/Intel/ldm3d-4c) +are open source. A [demo](https://huggingface.co/spaces/Intel/ldm3d) is also available. Here is how to run this model: ```bash python text_to_image_generation.py \ @@ -134,15 +141,13 @@ python text_to_image_generation.py \ --use_habana \ --use_hpu_graphs \ --gaudi_config Habana/stable-diffusion-2 \ - --ldm3d \ - --bf16 + --ldm3d ``` Here is how to generate images and depth maps with two prompts on two HPUs: ```bash -python ../gaudi_spawn.py \ - --world_size 2 text_to_image_generation.py \ +python ../gaudi_spawn.py --world_size 2 text_to_image_generation.py \ --model_name_or_path "Intel/ldm3d-4c" \ --prompts "An image of a squirrel in Picasso style" "A shiny flying horse taking off" \ --num_images_per_prompt 10 \ @@ -157,15 +162,16 @@ python ../gaudi_spawn.py \ --distributed ``` +> [!NOTE] > There are three different checkpoints for LDM3D: -> > - use [original checkpoint](https://huggingface.co/Intel/ldm3d) to generate outputs from the paper > - use [the latest checkpoint](https://huggingface.co/Intel/ldm3d-4c) for generating improved results > - use [the pano checkpoint](https://huggingface.co/Intel/ldm3d-pano) to generate panoramic view ### Stable Diffusion XL (SDXL) -Stable Diffusion XL was proposed in [SDXL: Improving Latent Diffusion Models for High-Resolution Image Synthesis](https://arxiv.org/pdf/2307.01952.pdf) by the Stability AI team. +Stable Diffusion XL was proposed in [SDXL: Improving Latent Diffusion Models for High-Resolution Image Synthesis](https://arxiv.org/pdf/2307.01952.pdf) +by the Stability AI team. Here is how to generate SDXL images with a single prompt: @@ -175,37 +181,22 @@ python text_to_image_generation.py \ --prompts "Sailing ship painting by Van Gogh" \ --num_images_per_prompt 28 \ --batch_size 7 \ + --num_inference_steps 30 \ --image_save_dir /tmp/stable_diffusion_xl_images \ --scheduler euler_discrete \ --use_habana \ --use_hpu_graphs \ --gaudi_config Habana/stable-diffusion \ + --sdp_on_bf16 \ --bf16 ``` +> [!NOTE] > HPU graphs are recommended when generating images by batches to get the fastest possible generations. > The first batch of images entails a performance penalty. All subsequent batches will be generated much faster. > You can enable this mode with `--use_hpu_graphs`. -Here is how to generate SDXL images with several prompts: - -```bash -python text_to_image_generation.py \ - --model_name_or_path stabilityai/stable-diffusion-xl-base-1.0 \ - --prompts "Sailing ship painting by Van Gogh" "A shiny flying horse taking off" \ - --num_images_per_prompt 32 \ - --batch_size 8 \ - --image_save_dir /tmp/stable_diffusion_xl_images \ - --scheduler euler_discrete \ - --use_habana \ - --use_hpu_graphs \ - --gaudi_config Habana/stable-diffusion \ - --bf16 -``` - -SDXL combines a second text encoder (OpenCLIP ViT-bigG/14) with the original text encoder to significantly -increase the number of parameters. Here is how to generate images with several prompts for both `prompt` -and `prompt_2` (2nd text encoder), as well as their negative prompts: +SDXL integrates a second text encoder (OpenCLIP ViT-bigG/14), alongside the original Stable Diffusion text encoder. This addition significantly increases the number of parameters, enabling more detailed and descriptive prompts. Below is an example of how to generate images using multiple prompts for both `prompt` (primary text encoder) and `prompt_2` (secondary text encoder), along with their respective negative prompts: ```bash python text_to_image_generation.py \ @@ -221,14 +212,14 @@ python text_to_image_generation.py \ --use_habana \ --use_hpu_graphs \ --gaudi_config Habana/stable-diffusion \ + --sdp_on_bf16 \ --bf16 ``` -Here is how to generate SDXL images with two prompts on two HPUs: +SDXL also supports distributed inferencing with Intel Gaudi accelerators. Below is an example of generating SDXL images in a distributed manner using two prompts on two HPUs: ```bash -python ../gaudi_spawn.py \ - --world_size 2 text_to_image_generation.py \ +python ../gaudi_spawn.py --world_size 2 text_to_image_generation.py \ --model_name_or_path stabilityai/stable-diffusion-xl-base-1.0 \ --prompts "Sailing ship painting by Van Gogh" "A shiny flying horse taking off" \ --prompts_2 "Red tone" "Blue tone" \ @@ -241,29 +232,18 @@ python ../gaudi_spawn.py \ --use_habana \ --use_hpu_graphs \ --gaudi_config Habana/stable-diffusion \ + --sdp_on_bf16 \ --bf16 \ --distributed ``` -Here is how to generate SDXL images with optimized pipeline: -```bash -python text_to_image_generation.py \ - --model_name_or_path stabilityai/stable-diffusion-xl-base-1.0 \ - --prompts "Sailing ship painting by Van Gogh" \ - --num_images_per_prompt 28 \ - --batch_size 7 \ - --image_save_dir /tmp/stable_diffusion_xl_images \ - --scheduler euler_discrete \ - --use_habana \ - --use_hpu_graphs \ - --gaudi_config Habana/stable-diffusion \ - --bf16 \ - --optimize -``` +The performance-optimized SDXL pipeline can be enabled using the `--optimize` option. This option utilizes a more aggressively optimized attention mechanism for enhanced performance. Additionally, it supports running +inference in mixed FP8 precision. -Here is how to generate SDXL images with optimized pipeline in fp8: +Here is how to generate SDXL images with optimized pipeline in FP8 precision: ```bash -QUANT_CONFIG=./quantization/quant_config.json python text_to_image_generation.py \ +QUANT_CONFIG=quantization/stable-diffusion-xl/quantize_config.json \ +python text_to_image_generation.py \ --model_name_or_path stabilityai/stable-diffusion-xl-base-1.0 \ --prompts "Sailing ship painting by Van Gogh" \ --num_images_per_prompt 28 \ @@ -273,17 +253,16 @@ QUANT_CONFIG=./quantization/quant_config.json python text_to_image_generation.py --use_habana \ --use_hpu_graphs \ --gaudi_config Habana/stable-diffusion \ + --sdp_on_bf16 \ --bf16 \ --optimize ``` -> HPU graphs are recommended when generating images by batches to get the fastest possible generations. -> The first batch of images entails a performance penalty. All subsequent batches will be generated much faster. -> You can enable this mode with `--use_hpu_graphs`. - ### SDXL-Turbo -SDXL-Turbo is a distilled version of SDXL 1.0, trained for real-time synthesis. +The knowledge distillation technique can be used to train a distilled version of SDXL, allowing for high-quality +image generation with fewer inference steps. SDXL-Turbo is a distilled version of Stable Diffusion XL 1.0, +optimized for real-time synthesis. Here is how to generate images with multiple prompts: @@ -298,17 +277,16 @@ python text_to_image_generation.py \ --use_habana \ --use_hpu_graphs \ --gaudi_config Habana/stable-diffusion \ + --sdp_on_bf16 \ --bf16 \ --num_inference_steps 1 \ --guidance_scale 1.000001 \ --timestep_spacing trailing ``` -> HPU graphs are recommended when generating images by batches to get the fastest possible generations. -> The first batch of images entails a performance penalty. All subsequent batches will be generated much faster. -> You can enable this mode with `--use_hpu_graphs`. - -> Note: there is a regression with "--guidance_scale 0.0" in current release which will be addressed in later releases. Setting `--guidance_scale` to a value larger than 1 resolves the regression. +> [!WARNING] +> There is a regression with `--guidance_scale 0.0` in current release which will be addressed in later releases. +> Setting `--guidance_scale` to a value larger than 1 resolves the regression. ### Stable Diffusion 3 (SD3) @@ -327,7 +305,6 @@ huggingface-cli login Here is how to generate SD3 images with a single prompt: ```bash -PT_HPU_MAX_COMPOUND_OP_SIZE=1 \ python text_to_image_generation.py \ --model_name_or_path stabilityai/stable-diffusion-3-medium-diffusers \ --prompts "Sailing ship painting by Van Gogh" \ @@ -339,17 +316,57 @@ python text_to_image_generation.py \ --use_habana \ --use_hpu_graphs \ --gaudi_config Habana/stable-diffusion \ + --sdp_on_bf16 \ --bf16 ``` -> For improved performance of the SD3 pipeline on Gaudi, it is recommended to configure the environment -> by setting PT_HPU_MAX_COMPOUND_OP_SIZE to 1. +This model can also be quantized with some ops running in FP8 precision. + +Before quantization, run stats collection using measure mode: + +```bash +QUANT_CONFIG=quantization/stable-diffusion-3/measure_config.json \ +python text_to_image_generation.py \ + --model_name_or_path stabilityai/stable-diffusion-3-medium-diffusers \ + --prompts "Sailing ship painting by Van Gogh" \ + --num_images_per_prompt 10 \ + --batch_size 1 \ + --num_inference_steps 28 \ + --image_save_dir /tmp/stable_diffusion_3_images \ + --scheduler default \ + --use_habana \ + --use_hpu_graphs \ + --gaudi_config Habana/stable-diffusion \ + --sdp_on_bf16 \ + --bf16 \ + --quant_mode measure +``` + +After stats collection, here is how to run SD3 in quantization mode: + +```bash +QUANT_CONFIG=quantization/stable-diffusion-3/quantize_config.json \ +python text_to_image_generation.py \ + --model_name_or_path stabilityai/stable-diffusion-3-medium-diffusers \ + --prompts "Sailing ship painting by Van Gogh" \ + --num_images_per_prompt 10 \ + --batch_size 1 \ + --num_inference_steps 28 \ + --image_save_dir /tmp/stable_diffusion_3_images \ + --scheduler default \ + --use_habana \ + --use_hpu_graphs \ + --gaudi_config Habana/stable-diffusion \ + --sdp_on_bf16 \ + --bf16 \ + --quant_mode quantize +``` ### FLUX.1 FLUX.1 was introduced by Black Forest Labs [here](https://blackforestlabs.ai/announcing-black-forest-labs/). -Here is how to run FLUX.1-schnell model (fast version of FLUX.1): +Here is how to run FLUX.1-schnell model (distilled fast version of FLUX.1): ```bash python text_to_image_generation.py \ @@ -359,10 +376,11 @@ python text_to_image_generation.py \ --batch_size 1 \ --num_inference_steps 4 \ --image_save_dir /tmp/flux_1_images \ - --scheduler flow_match_euler_discrete\ + --scheduler flow_match_euler_discrete \ --use_habana \ --use_hpu_graphs \ --gaudi_config Habana/stable-diffusion \ + --sdp_on_bf16 \ --bf16 ``` @@ -385,10 +403,11 @@ python text_to_image_generation.py \ --batch_size 1 \ --num_inference_steps 30 \ --image_save_dir /tmp/flux_1_images \ - --scheduler flow_match_euler_discrete\ + --scheduler flow_match_euler_discrete \ --use_habana \ --use_hpu_graphs \ --gaudi_config Habana/stable-diffusion \ + --sdp_on_bf16 \ --bf16 ``` @@ -405,10 +424,11 @@ python text_to_image_generation.py \ --batch_size 1 \ --num_inference_steps 30 \ --image_save_dir /tmp/flux_1_images \ - --scheduler flow_match_euler_discrete\ + --scheduler flow_match_euler_discrete \ --use_habana \ --use_hpu_graphs \ --gaudi_config Habana/stable-diffusion \ + --sdp_on_bf16 \ --bf16 \ --quant_mode measure ``` @@ -424,43 +444,28 @@ python text_to_image_generation.py \ --batch_size 1 \ --num_inference_steps 30 \ --image_save_dir /tmp/flux_1_images \ - --scheduler flow_match_euler_discrete\ + --scheduler flow_match_euler_discrete \ --use_habana \ --use_hpu_graphs \ --gaudi_config Habana/stable-diffusion \ + --sdp_on_bf16 \ --bf16 \ --quant_mode quantize ``` ## ControlNet -ControlNet was introduced in [Adding Conditional Control to Text-to-Image Diffusion Models](https://huggingface.co/papers/2302.05543) by Lvmin Zhang and Maneesh Agrawala. -It is a type of model for controlling StableDiffusion by conditioning the model with an additional input image. -Here is how to generate images conditioned by canny edge model: +ControlNet was introduced in [Adding Conditional Control to Text-to-Image Diffusion Models](https://huggingface.co/papers/2302.05543) +by Lvmin Zhang and Maneesh Agrawala, enables conditioning the Stable Diffusion model with an additional input image. This allows for precise control over the composition of generated images using various features such as edges, pose, depth, and more. -```bash -python text_to_image_generation.py \ - --model_name_or_path CompVis/stable-diffusion-v1-4 \ - --controlnet_model_name_or_path lllyasviel/sd-controlnet-canny \ - --prompts "futuristic-looking woman" \ - --control_image https://hf.co/datasets/huggingface/documentation-images/resolve/main/diffusers/input_image_vermeer.png \ - --num_images_per_prompt 28 \ - --batch_size 7 \ - --image_save_dir /tmp/controlnet_images \ - --use_habana \ - --use_hpu_graphs \ - --gaudi_config Habana/stable-diffusion \ - --bf16 -``` - -Here is how to generate images conditioned by canny edge model and with multiple prompts: +Here is how to generate images conditioned by Canny edge model: ```bash python text_to_image_generation.py \ --model_name_or_path CompVis/stable-diffusion-v1-4 \ --controlnet_model_name_or_path lllyasviel/sd-controlnet-canny \ - --prompts "futuristic-looking woman" "a rusty robot" \ + --prompts "futuristic-looking woman" \ --control_image https://hf.co/datasets/huggingface/documentation-images/resolve/main/diffusers/input_image_vermeer.png \ --num_images_per_prompt 28 \ --batch_size 7 \ @@ -468,14 +473,15 @@ python text_to_image_generation.py \ --use_habana \ --use_hpu_graphs \ --gaudi_config Habana/stable-diffusion \ + --sdp_on_bf16 \ --bf16 ``` -Here is how to generate images conditioned by canny edge model and with two prompts on two HPUs: +The ControlNet example can be run with multiple prompts by supplying more than one prompt in the input. +Additionally, it supports distributed execution. Below is an example of generating images conditioned by the Canny edge model using two prompts on two HPUs: ```bash -python ../gaudi_spawn.py \ - --world_size 2 text_to_image_generation.py \ +python ../gaudi_spawn.py --world_size 2 text_to_image_generation.py \ --model_name_or_path CompVis/stable-diffusion-v1-4 \ --controlnet_model_name_or_path lllyasviel/sd-controlnet-canny \ --prompts "futuristic-looking woman" "a rusty robot" \ @@ -486,46 +492,12 @@ python ../gaudi_spawn.py \ --use_habana \ --use_hpu_graphs \ --gaudi_config Habana/stable-diffusion \ + --sdp_on_bf16 \ --bf16 \ --distributed ``` -Here is how to generate images conditioned by open pose model: - -```bash -python text_to_image_generation.py \ - --model_name_or_path CompVis/stable-diffusion-v1-4 \ - --controlnet_model_name_or_path lllyasviel/sd-controlnet-openpose \ - --prompts "Chef in the kitchen" \ - --control_image https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/sd_controlnet/pose.png \ - --control_preprocessing_type "none" \ - --num_images_per_prompt 28 \ - --batch_size 7 \ - --image_save_dir /tmp/controlnet_images \ - --use_habana \ - --use_hpu_graphs \ - --gaudi_config Habana/stable-diffusion \ - --bf16 -``` - -Here is how to generate images with conditioned by canny edge model using Stable Diffusion 2 - -```bash -python text_to_image_generation.py \ - --model_name_or_path stabilityai/stable-diffusion-2-1 \ - --controlnet_model_name_or_path thibaud/controlnet-sd21-canny-diffusers \ - --control_image https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/sd_controlnet/bird_canny.png \ - --control_preprocessing_type "none" \ - --prompts "bird" \ - --seed 0 \ - --num_images_per_prompt 28 \ - --batch_size 7 \ - --image_save_dir /tmp/controlnet-2-1_images \ - --use_habana \ - --use_hpu_graphs \ - --gaudi_config Habana/stable-diffusion-2 \ - --bf16 -``` +These ControlNet examples will preprocess the input image to derive Canny edges. Alternatively, you can use `--control_preprocessing_type none` to supply a preprocessed control image directly, enabling many additional use cases. ## Inpainting @@ -547,6 +519,7 @@ python text_to_image_generation.py \ --use_habana \ --use_hpu_graphs \ --gaudi_config Habana/stable-diffusion \ + --sdp_on_bf16 \ --bf16 ``` @@ -554,7 +527,7 @@ python text_to_image_generation.py \ ```bash python text_to_image_generation.py \ - --model_name_or_path diffusers/stable-diffusion-xl-1.0-inpainting-0.1\ + --model_name_or_path diffusers/stable-diffusion-xl-1.0-inpainting-0.1 \ --base_image https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/inpaint.png \ --mask_image https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/inpaint_mask.png \ --prompts "concept art digital painting of an elven castle, inspired by lord of the rings, highly detailed, 8k" \ @@ -566,46 +539,110 @@ python text_to_image_generation.py \ --use_habana \ --use_hpu_graphs \ --gaudi_config Habana/stable-diffusion \ + --sdp_on_bf16 \ --bf16 ``` -## Image-to-image Generation +## Additional Stable Diffusion-based Inference Techniques + +This section provides examples of additional inference techniques based on Stable Diffusion. For more details, please refer to +[Hugging Face Diffusers documentation](https://huggingface.co/docs/diffusers/main/en/using-diffusers/overview_techniques). -### Single Prompt +### Unconditional Image Generation -Here is how to generate images with one prompt and one image. -Take instruct-pix2pix as an example. +Here is how to perform unconditional image generation on Intel Gaudi. For more details, please refer to the +[Unconditional Image Generation](https://huggingface.co/docs/diffusers/using-diffusers/unconditional_image_generation) +section in the Hugging Face documentation. ```bash -python image_to_image_generation.py \ - --model_name_or_path "timbrooks/instruct-pix2pix" \ - --src_image_path "https://raw.githubusercontent.com/timothybrooks/instruct-pix2pix/main/imgs/example.jpg" \ - --prompts "turn him into cyborg" \ - --num_images_per_prompt 20 \ - --batch_size 4 \ - --guidance_scale 7.5 \ - --image_guidance_scale 1 \ - --num_inference_steps 10 \ - --image_save_dir /tmp/stable_diffusion_images \ +python unconditional_image_generation.py \ + --model_name_or_path "google/ddpm-ema-celebahq-256" \ + --batch_size 16 \ --use_habana \ + --use_gaudi_ddim_scheduler \ --use_hpu_graphs \ - --gaudi_config Habana/stable-diffusion \ + --sdp_on_bf16 \ + --bf16 \ + --save_outputs \ + --output_dir "/tmp/" +``` + +### Controlling Brightness + +Here is an example of how to control brightness. For more information, please refer to the +[Control Brightness](https://huggingface.co/docs/diffusers/main/en/using-diffusers/control_brightness) +section in the Hugging Face documentation. + +```bash +PT_HPU_MAX_COMPOUND_OP_SIZE=1 \ +python text_to_image_generation.py \ + --model_name_or_path ptx0/pseudo-journey-v2 \ + --prompts "A lion in galaxies, spirals, nebulae, stars, smoke, iridescent, intricate detail, octane render, 8k" \ + --num_images_per_prompt 1 \ + --batch_size 1 \ + --use_habana \ + --use_hpu_graphs \ + --image_save_dir /tmp/stable_diffusion_images_brightness \ + --seed 33 \ + --use_zero_snr \ + --guidance_scale 0.7 \ + --timestep_spacing trailing +``` + +### Prompt Weighting + +Here is an example of how to run prompt weighting. For more information, please refer to the +[Weighted Prompts](https://huggingface.co/docs/diffusers/main/en/using-diffusers/weighted_prompts) +section in the Hugging Face documentation. + +```bash +python text_to_image_generation.py \ + --model_name_or_path CompVis/stable-diffusion-v1-4 \ + --prompts "a red cat playing with a ball+++" "a red cat playing with a ball---" \ + --num_images_per_prompt 4 \ + --batch_size 4 \ + --use_habana --use_hpu_graphs \ + --image_save_dir /tmp/stable_diffusion_images_compel \ + --seed 33 \ + --sdp_on_bf16 \ + --bf16 \ + --num_inference_steps 20 \ + --use_compel +``` + +### Controlling Image Quality + +Here is an example of how to improve image quality. For more details, please refer to the +[Image Quality](https://huggingface.co/docs/diffusers/main/en/using-diffusers/image_quality) +section in the Hugging Face documentation. + +```bash +python text_to_image_generation.py \ + --model_name_or_path CompVis/stable-diffusion-v1-4 \ + --prompts "A squirrel eating a burger" \ + --num_images_per_prompt 4 \ + --batch_size 4 \ + --use_habana \ + --image_save_dir /tmp/stable_diffusion_images_freeu \ + --seed 33 \ + --use_freeu \ + --sdp_on_bf16 \ --bf16 ``` -> HPU graphs are recommended when generating images by batches to get the fastest possible generations. -> The first batch of images entails a performance penalty. All subsequent batches will be generated much faster. -> You can enable this mode with `--use_hpu_graphs`. +## Image-to-Image Generation -### Multiple Prompts +Images can also be generated using initial input images to guide the diffusion-based image generation process. -Here is how to generate images with several prompts and one image. +### Stable Diffusion-based Image-to-Image + +Here is how to generate images using a single prompt and an input image with the `timbrooks/instruct-pix2pix` model, which is based on Stable Diffusion: ```bash python image_to_image_generation.py \ --model_name_or_path "timbrooks/instruct-pix2pix" \ --src_image_path "https://raw.githubusercontent.com/timothybrooks/instruct-pix2pix/main/imgs/example.jpg" \ - --prompts "turn him into cyborg" "a strong soldier"\ + --prompts "turn him into cyborg" \ --num_images_per_prompt 20 \ --batch_size 4 \ --guidance_scale 7.5 \ @@ -615,16 +652,18 @@ python image_to_image_generation.py \ --use_habana \ --use_hpu_graphs \ --gaudi_config Habana/stable-diffusion \ + --sdp_on_bf16 \ --bf16 ``` +> [!NOTE] > HPU graphs are recommended when generating images by batches to get the fastest possible generations. > The first batch of images entails a performance penalty. All subsequent batches will be generated much faster. > You can enable this mode with `--use_hpu_graphs`. ### Stable Diffusion XL Refiner -Here is how to generate SDXL images with a single prompt and one image: +Here is how to refine SDXL images using a single image and prompt: ```bash python image_to_image_generation.py \ @@ -639,20 +678,21 @@ python image_to_image_generation.py \ --use_habana \ --use_hpu_graphs \ --gaudi_config Habana/stable-diffusion \ + --sdp_on_bf16 \ --bf16 ``` -### FLUX.1 Image to Image +### FLUX.1 Image-to-Image -Here is how to generate FLUX.1 images with a single prompt and one input image: +Here is how to generate a FLUX.1 image using a single input image and prompt: ```bash python image_to_image_generation.py \ --model_name_or_path "black-forest-labs/FLUX.1-dev" \ --src_image_path "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/cat.png" \ --prompts "cat wizard, gandalf, lord of the rings, detailed, fantasy, cute, adorable, Pixar, Disney, 8k" \ - --num_images_per_prompt 40 \ - --batch_size 10 \ + --num_images_per_prompt 10 \ + --batch_size 1 \ --strength 0.9 \ --guidance_scale 3.5 \ --num_inference_steps 30 \ @@ -660,12 +700,13 @@ python image_to_image_generation.py \ --use_habana \ --use_hpu_graphs \ --gaudi_config Habana/stable-diffusion \ + --sdp_on_bf16 \ --bf16 ``` ### Stable Diffusion Image Variations -Here is how to generate images with one image, it does not accept prompt input +Here is how to generate image variations of a single image (without any input prompts): ```bash python image_to_image_generation.py \ @@ -678,12 +719,13 @@ python image_to_image_generation.py \ --use_habana \ --use_hpu_graphs \ --gaudi_config Habana/stable-diffusion \ + --sdp_on_bf16 \ --bf16 ``` ### Depth to Image Generation -Here is how to generate a depth2img-guided image generation using HPU graphs with BF16: +Here is an example of performing depth-guided image generation: ```bash python depth_to_image_generation.py \ @@ -693,88 +735,24 @@ python depth_to_image_generation.py \ --image_save_dir /tmp/stable_diffusion_images \ --use_habana \ --use_hpu_graphs \ + --sdp_on_bf16 \ --bf16 ``` -## Unconditional Image Generation Example - -Here is how to perform unconditional-image-generation on Gaudi/HPU. - -Original unconditional image generation pipeline is shared in here: [Unconditional Image Generation](https://huggingface.co/docs/diffusers/using-diffusers/unconditional_image_generation) - -```bash -python unconditional_image_generation.py \ - --model_name_or_path "google/ddpm-ema-celebahq-256" \ - --batch_size 16 \ - --use_habana \ - --use_gaudi_ddim_scheduler \ - --use_hpu_graphs \ - --bf16 \ - --save_outputs \ - --output_dir "/tmp/" -``` - -## Additional inference techniques - -Here is how to run the diffusers examples of inference techniques. For more details, -please refer to [Hugging Face Diffusers doc](https://huggingface.co/docs/diffusers/main/en/using-diffusers/overview_techniques). +## Text-to-Video Generation -### Controlling brightness - -Here is how to run the example of controlling brightness. For more details, -please refer to [Hugging Face Diffusers doc](https://huggingface.co/docs/diffusers/main/en/using-diffusers/control_brightness). +This section demonstrates how to use the `GaudiTextToVideoSDPipeline` for text-to-video generation tasks on HPUs. +The pipeline employs a UNet3D structure and generates videos through an iterative denoising process. ```bash -python text_to_image_generation.py \ - --model_name_or_path ptx0/pseudo-journey-v2 \ - --prompts "A lion in galaxies, spirals, nebulae, stars, smoke, iridescent, intricate detail, octane render, 8k" \ - --num_images_per_prompt 1 \ - --batch_size 1 \ +python text_to_video_generation.py \ + --model_name_or_path ali-vilab/text-to-video-ms-1.7b \ + --prompts "An astronaut riding a horse" \ --use_habana \ --use_hpu_graphs \ - --image_save_dir /tmp/stable_diffusion_images_brightness \ - --seed 33 \ - --use_zero_snr \ - --guidance_scale 0.7 \ - --timestep_spacing trailing + --dtype bf16 ``` -### Prompt weighting - -Here is how to run the example of prompt weighting. For more details, -please refer to [Hugging Face Diffusers doc](https://huggingface.co/docs/diffusers/main/en/using-diffusers/weighted_prompts). - -```bash -python text_to_image_generation.py \ - --model_name_or_path CompVis/stable-diffusion-v1-4 \ - --prompts "a red cat playing with a ball+++" "a red cat playing with a ball---" \ - --num_images_per_prompt 4 \ - --batch_size 4 \ - --use_habana --use_hpu_graphs \ - --image_save_dir /tmp/stable_diffusion_images_compel \ - --seed 33 \ - --bf16 \ - --num_inference_steps 20 \ - --use_compel -``` - -### Controlling image quality - -Here is how to run the example of improving image quality. For more details, -please refer to [Hugging Face Diffusers doc](https://huggingface.co/docs/diffusers/main/en/using-diffusers/image_quality). - -```bash -python text_to_image_generation.py \ - --model_name_or_path CompVis/stable-diffusion-v1-4 \ - --prompts "A squirrel eating a burger" \ - --num_images_per_prompt 4 \ - --batch_size 4 \ - --use_habana \ - --image_save_dir /tmp/stable_diffusion_images_freeu \ - --seed 33 \ - --use_freeu \ - --bf16 -``` # Stable Video Diffusion Examples Stable Video Diffusion (SVD) was unveiled in [Stable Video Diffusion Announcement](https://stability.ai/news/stable-video-diffusion-open-ai-video-model) @@ -799,9 +777,11 @@ python image_to_video_generation.py \ --use_habana \ --use_hpu_graphs \ --gaudi_config Habana/stable-diffusion \ + --sdp_on_bf16 \ --bf16 ``` +> [!NOTE] > For improved performance of the image-to-video pipeline on Gaudi, it is recommended to configure the environment > by setting PT_HPU_MAX_COMPOUND_OP_SIZE to 1. @@ -813,53 +793,65 @@ Here is how to generate videos with several image prompts: PT_HPU_MAX_COMPOUND_OP_SIZE=1 \ python image_to_video_generation.py \ --model_name_or_path "stabilityai/stable-video-diffusion-img2vid-xt" \ - --image_path "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/svd/rocket.png" \ - "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/cat.png" \ - "https://hf.co/datasets/huggingface/documentation-images/resolve/main/diffusers/input_image_vermeer.png" \ - "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/sd_controlnet/hf-logo.png" \ + --image_path \ + "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/svd/rocket.png" \ + "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/cat.png" \ + "https://hf.co/datasets/huggingface/documentation-images/resolve/main/diffusers/input_image_vermeer.png" \ + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/sd_controlnet/hf-logo.png" \ --num_videos_per_prompt 1 \ --video_save_dir /tmp/stable_video_diffusion_xt \ --save_frames_as_images \ --use_habana \ --use_hpu_graphs \ --gaudi_config Habana/stable-diffusion \ + --sdp_on_bf16 \ --bf16 ``` +> [!NOTE] > For improved performance of the image-to-video pipeline on Gaudi, it is recommended to configure the environment > by setting PT_HPU_MAX_COMPOUND_OP_SIZE to 1. -### Image-to-video ControlNet +### Image-to-Video ControlNet Here is how to generate video conditioned by depth: -``` +```bash python image_to_video_generation.py \ --model_name_or_path "stabilityai/stable-video-diffusion-img2vid" \ --controlnet_model_name_or_path "CiaraRowles/temporal-controlnet-depth-svd-v1" \ - --control_image_path "https://github.com/CiaraStrawberry/svd-temporal-controlnet/blob/main/validation_demo/depth/frame_0.png?raw=true" \ - "https://github.com/CiaraStrawberry/svd-temporal-controlnet/blob/main/validation_demo/depth/frame_1.png?raw=true" \ - "https://github.com/CiaraStrawberry/svd-temporal-controlnet/blob/main/validation_demo/depth/frame_2.png?raw=true" \ - "https://github.com/CiaraStrawberry/svd-temporal-controlnet/blob/main/validation_demo/depth/frame_3.png?raw=true" \ - "https://github.com/CiaraStrawberry/svd-temporal-controlnet/blob/main/validation_demo/depth/frame_4.png?raw=true" \ - "https://github.com/CiaraStrawberry/svd-temporal-controlnet/blob/main/validation_demo/depth/frame_5.png?raw=true" \ - "https://github.com/CiaraStrawberry/svd-temporal-controlnet/blob/main/validation_demo/depth/frame_6.png?raw=true" \ - "https://github.com/CiaraStrawberry/svd-temporal-controlnet/blob/main/validation_demo/depth/frame_7.png?raw=true" \ - "https://github.com/CiaraStrawberry/svd-temporal-controlnet/blob/main/validation_demo/depth/frame_8.png?raw=true" \ - "https://github.com/CiaraStrawberry/svd-temporal-controlnet/blob/main/validation_demo/depth/frame_9.png?raw=true" \ - "https://github.com/CiaraStrawberry/svd-temporal-controlnet/blob/main/validation_demo/depth/frame_10.png?raw=true" \ - "https://github.com/CiaraStrawberry/svd-temporal-controlnet/blob/main/validation_demo/depth/frame_11.png?raw=true" \ - "https://github.com/CiaraStrawberry/svd-temporal-controlnet/blob/main/validation_demo/depth/frame_12.png?raw=true" \ - "https://github.com/CiaraStrawberry/svd-temporal-controlnet/blob/main/validation_demo/depth/frame_13.png?raw=true" \ + --control_image_path \ + "https://github.com/CiaraStrawberry/svd-temporal-controlnet/blob/main/validation_demo/depth/frame_0.png?raw=true" \ + "https://github.com/CiaraStrawberry/svd-temporal-controlnet/blob/main/validation_demo/depth/frame_1.png?raw=true" \ + "https://github.com/CiaraStrawberry/svd-temporal-controlnet/blob/main/validation_demo/depth/frame_2.png?raw=true" \ + "https://github.com/CiaraStrawberry/svd-temporal-controlnet/blob/main/validation_demo/depth/frame_3.png?raw=true" \ + "https://github.com/CiaraStrawberry/svd-temporal-controlnet/blob/main/validation_demo/depth/frame_4.png?raw=true" \ + "https://github.com/CiaraStrawberry/svd-temporal-controlnet/blob/main/validation_demo/depth/frame_5.png?raw=true" \ + "https://github.com/CiaraStrawberry/svd-temporal-controlnet/blob/main/validation_demo/depth/frame_6.png?raw=true" \ + "https://github.com/CiaraStrawberry/svd-temporal-controlnet/blob/main/validation_demo/depth/frame_7.png?raw=true" \ + "https://github.com/CiaraStrawberry/svd-temporal-controlnet/blob/main/validation_demo/depth/frame_8.png?raw=true" \ + "https://github.com/CiaraStrawberry/svd-temporal-controlnet/blob/main/validation_demo/depth/frame_9.png?raw=true" \ + "https://github.com/CiaraStrawberry/svd-temporal-controlnet/blob/main/validation_demo/depth/frame_10.png?raw=true" \ + "https://github.com/CiaraStrawberry/svd-temporal-controlnet/blob/main/validation_demo/depth/frame_11.png?raw=true" \ + "https://github.com/CiaraStrawberry/svd-temporal-controlnet/blob/main/validation_demo/depth/frame_12.png?raw=true" \ + "https://github.com/CiaraStrawberry/svd-temporal-controlnet/blob/main/validation_demo/depth/frame_13.png?raw=true" \ --image_path "https://github.com/CiaraStrawberry/svd-temporal-controlnet/blob/main/validation_demo/chair.png?raw=true" \ --video_save_dir SVD_controlnet \ --save_frames_as_images \ --use_habana \ --use_hpu_graphs \ --gaudi_config Habana/stable-diffusion \ + --sdp_on_bf16 \ --bf16 \ --num_frames 14 \ --motion_bucket_id=14 \ --width=512 \ --height=512 ``` + +# Important Notes for Gaudi3 Users + +- **Batch Size Limitation**: Due to a known issue, batch sizes for some Stable Diffusion models need to be reduced. + This issue is expected to be resolved in a future release. + +- **Image-to-Video ControlNet**: The Image-to-Video ControlNet command is currently not supported on Gaudi3. diff --git a/examples/stable-diffusion/depth_to_image_generation.py b/examples/stable-diffusion/depth_to_image_generation.py index 570a39b2c3..c32d61a05b 100755 --- a/examples/stable-diffusion/depth_to_image_generation.py +++ b/examples/stable-diffusion/depth_to_image_generation.py @@ -41,7 +41,7 @@ def check_optimum_habana_min_version(*a, **b): # Will error if the minimal version of Optimum Habana is not installed. Remove at your own risks. -check_optimum_habana_min_version("1.14.0.dev0") +check_optimum_habana_min_version("1.16.0.dev0") logger = logging.getLogger(__name__) @@ -172,6 +172,12 @@ def main(): ), ) parser.add_argument("--bf16", action="store_true", help="Whether to perform generation in bf16 precision.") + parser.add_argument( + "--sdp_on_bf16", + action="store_true", + default=False, + help="Allow pyTorch to use reduced precision in the SDPA math backend", + ) parser.add_argument( "--throughput_warmup_steps", type=int, @@ -223,6 +229,7 @@ def main(): "use_habana": args.use_habana, "use_hpu_graphs": args.use_hpu_graphs, "gaudi_config": args.gaudi_config_name, + "sdp_on_bf16": args.sdp_on_bf16, } if args.bf16: diff --git a/examples/stable-diffusion/image_to_image_generation.py b/examples/stable-diffusion/image_to_image_generation.py index a9f2f81930..68e29c97bd 100755 --- a/examples/stable-diffusion/image_to_image_generation.py +++ b/examples/stable-diffusion/image_to_image_generation.py @@ -41,7 +41,7 @@ def check_optimum_habana_min_version(*a, **b): # Will error if the minimal version of Optimum Habana is not installed. Remove at your own risks. -check_optimum_habana_min_version("1.14.0.dev0") +check_optimum_habana_min_version("1.16.0.dev0") logger = logging.getLogger(__name__) @@ -193,6 +193,12 @@ def main(): ), ) parser.add_argument("--bf16", action="store_true", help="Whether to perform generation in bf16 precision.") + parser.add_argument( + "--sdp_on_bf16", + action="store_true", + default=False, + help="Allow pyTorch to use reduced precision in the SDPA math backend", + ) parser.add_argument( "--ldm3d", action="store_true", help="Use LDM3D to generate an image and a depth map from a given text prompt." ) @@ -217,10 +223,10 @@ def main(): args = parser.parse_args() # Set image resolution - res = {} + kwargs_call = {} if args.width > 0 and args.height > 0: - res["width"] = args.width - res["height"] = args.height + kwargs_call["width"] = args.width + kwargs_call["height"] = args.height sdxl_models = ["stable-diffusion-xl", "sdxl"] sdxl = False flux_models = ["FLUX.1"] @@ -230,6 +236,7 @@ def main(): "use_habana": args.use_habana, "use_hpu_graphs": args.use_hpu_graphs, "gaudi_config": args.gaudi_config_name, + "sdp_on_bf16": args.sdp_on_bf16, } # Import selected pipeline @@ -245,7 +252,7 @@ def main(): from optimum.habana.diffusers import GaudiStableDiffusionInstructPix2PixPipeline as Img2ImgPipeline kwargs["safety_checker"] = None - res["image_guidance_scale"] = args.image_guidance_scale + kwargs_call["image_guidance_scale"] = args.image_guidance_scale elif "image-variations" in args.model_name_or_path: from optimum.habana.diffusers import GaudiStableDiffusionImageVariationPipeline as Img2ImgPipeline @@ -284,7 +291,7 @@ def main(): kwargs["torch_dtype"] = torch.bfloat16 if args.throughput_warmup_steps is not None: - kwargs["throughput_warmup_steps"] = args.throughput_warmup_steps + kwargs_call["throughput_warmup_steps"] = args.throughput_warmup_steps pipeline = Img2ImgPipeline.from_pretrained( args.model_name_or_path, @@ -318,7 +325,7 @@ def main(): output_type=args.output_type, profiling_warmup_steps=args.profiling_warmup_steps, profiling_steps=args.profiling_steps, - **res, + **kwargs_call, ) elif flux: outputs = pipeline( @@ -333,7 +340,7 @@ def main(): output_type=args.output_type, profiling_warmup_steps=args.profiling_warmup_steps, profiling_steps=args.profiling_steps, - **res, + **kwargs_call, ) else: outputs = pipeline( @@ -348,7 +355,7 @@ def main(): output_type=args.output_type, profiling_warmup_steps=args.profiling_warmup_steps, profiling_steps=args.profiling_steps, - **res, + **kwargs_call, ) # Save the pipeline in the specified directory if not None @@ -363,12 +370,12 @@ def main(): logger.info(f"Saving images in {image_save_dir.resolve()}...") if args.ldm3d: for i, rgb in enumerate(outputs.rgb): - rgb.save(image_save_dir / f"rgb_{i+1}.png") + rgb.save(image_save_dir / f"rgb_{i + 1}.png") for i, depth in enumerate(outputs.depth): - depth.save(image_save_dir / f"depth_{i+1}.png") + depth.save(image_save_dir / f"depth_{i + 1}.png") else: for i, image in enumerate(outputs.images): - image.save(image_save_dir / f"image_{i+1}.png") + image.save(image_save_dir / f"image_{i + 1}.png") else: logger.warning("--output_type should be equal to 'pil' to save images in --image_save_dir.") diff --git a/examples/stable-diffusion/image_to_video_generation.py b/examples/stable-diffusion/image_to_video_generation.py index 4112a1b39c..bf5cdb5459 100755 --- a/examples/stable-diffusion/image_to_video_generation.py +++ b/examples/stable-diffusion/image_to_video_generation.py @@ -34,7 +34,7 @@ def check_optimum_habana_min_version(*a, **b): # Will error if the minimal version of Optimum Habana is not installed. Remove at your own risks. -check_optimum_habana_min_version("1.14.0.dev0") +check_optimum_habana_min_version("1.16.0.dev0") logger = logging.getLogger(__name__) @@ -177,7 +177,31 @@ def main(): ), ) parser.add_argument("--bf16", action="store_true", help="Whether to perform generation in bf16 precision.") + parser.add_argument( + "--sdp_on_bf16", + action="store_true", + default=False, + help="Allow pyTorch to use reduced precision in the SDPA math backend", + ) parser.add_argument("--num_frames", type=int, default=25, help="The number of video frames to generate.") + parser.add_argument( + "--profiling_warmup_steps", + default=0, + type=int, + help="Number of steps to ignore for profiling.", + ) + parser.add_argument( + "--profiling_steps", + default=0, + type=int, + help="Number of steps to capture for profiling.", + ) + parser.add_argument( + "--throughput_warmup_steps", + type=int, + default=None, + help="Number of steps to ignore for throughput calculation.", + ) args = parser.parse_args() # Setup logging @@ -218,6 +242,7 @@ def main(): "use_habana": args.use_habana, "use_hpu_graphs": args.use_hpu_graphs, "gaudi_config": args.gaudi_config_name, + "sdp_on_bf16": args.sdp_on_bf16, } set_seed(args.seed) @@ -261,6 +286,9 @@ def main(): args.model_name_or_path, **kwargs, ) + kwargs_call = {} + if args.throughput_warmup_steps is not None: + kwargs_call["throughput_warmup_steps"] = args.throughput_warmup_steps # Generate images outputs = pipeline( @@ -277,6 +305,9 @@ def main(): noise_aug_strength=args.noise_aug_strength, decode_chunk_size=args.decode_chunk_size, output_type=args.output_type, + profiling_warmup_steps=args.profiling_warmup_steps, + profiling_steps=args.profiling_steps, + **kwargs_call, ) # Save the pipeline in the specified directory if not None diff --git a/examples/stable-diffusion/quantization/flux/measure_config.json b/examples/stable-diffusion/quantization/flux/measure_config.json index 865078d99f..f90605dba8 100644 --- a/examples/stable-diffusion/quantization/flux/measure_config.json +++ b/examples/stable-diffusion/quantization/flux/measure_config.json @@ -1,5 +1,5 @@ { "method": "HOOKS", "mode": "MEASURE", - "dump_stats_path": "quantization/flux/measure_all/fp8" + "dump_stats_path": "quantization/flux/measure/fp8" } diff --git a/examples/stable-diffusion/quantization/flux/quantize_config.json b/examples/stable-diffusion/quantization/flux/quantize_config.json index 8fdb21fccf..e601db4ba4 100644 --- a/examples/stable-diffusion/quantization/flux/quantize_config.json +++ b/examples/stable-diffusion/quantization/flux/quantize_config.json @@ -2,5 +2,5 @@ "method": "HOOKS", "mode": "QUANTIZE", "scale_method": "maxabs_hw_opt_weight", - "dump_stats_path": "quantization/flux/measure_all/fp8" + "dump_stats_path": "quantization/flux/measure/fp8" } diff --git a/examples/stable-diffusion/quantization/stable-diffusion-3/measure_config.json b/examples/stable-diffusion/quantization/stable-diffusion-3/measure_config.json new file mode 100644 index 0000000000..ebf3baa292 --- /dev/null +++ b/examples/stable-diffusion/quantization/stable-diffusion-3/measure_config.json @@ -0,0 +1,5 @@ +{ + "method": "HOOKS", + "mode": "MEASURE", + "dump_stats_path": "quantization/stable-diffusion-3/measure_all/fp8" +} \ No newline at end of file diff --git a/examples/stable-diffusion/quantization/stable-diffusion-3/quantize_config.json b/examples/stable-diffusion/quantization/stable-diffusion-3/quantize_config.json new file mode 100644 index 0000000000..1fa98ebce0 --- /dev/null +++ b/examples/stable-diffusion/quantization/stable-diffusion-3/quantize_config.json @@ -0,0 +1,6 @@ +{ + "method": "HOOKS", + "mode": "QUANTIZE", + "scale_method": "maxabs_hw_opt_weight", + "dump_stats_path": "quantization/stable-diffusion-3/measure_all/fp8" +} \ No newline at end of file diff --git a/examples/stable-diffusion/quantization/measure/fp8_hooks_maxabs.json b/examples/stable-diffusion/quantization/stable-diffusion-xl/measure/fp8_hooks_maxabs.json similarity index 100% rename from examples/stable-diffusion/quantization/measure/fp8_hooks_maxabs.json rename to examples/stable-diffusion/quantization/stable-diffusion-xl/measure/fp8_hooks_maxabs.json diff --git a/examples/stable-diffusion/quantization/measure/fp8_hooks_maxabs.npz b/examples/stable-diffusion/quantization/stable-diffusion-xl/measure/fp8_hooks_maxabs.npz similarity index 100% rename from examples/stable-diffusion/quantization/measure/fp8_hooks_maxabs.npz rename to examples/stable-diffusion/quantization/stable-diffusion-xl/measure/fp8_hooks_maxabs.npz diff --git a/examples/stable-diffusion/quantization/measure_config.json b/examples/stable-diffusion/quantization/stable-diffusion-xl/measure_config.json old mode 100755 new mode 100644 similarity index 52% rename from examples/stable-diffusion/quantization/measure_config.json rename to examples/stable-diffusion/quantization/stable-diffusion-xl/measure_config.json index 04576eeb46..5a250cad7c --- a/examples/stable-diffusion/quantization/measure_config.json +++ b/examples/stable-diffusion/quantization/stable-diffusion-xl/measure_config.json @@ -2,5 +2,5 @@ "method": "HOOKS", "mode": "MEASURE", "observer": "maxabs", - "dump_stats_path": "./quantization/measure/fp8" + "dump_stats_path": "quantization/stable-diffusion-xl/measure/fp8" } diff --git a/examples/stable-diffusion/quantization/quant_config.json b/examples/stable-diffusion/quantization/stable-diffusion-xl/quantize_config.json old mode 100755 new mode 100644 similarity index 60% rename from examples/stable-diffusion/quantization/quant_config.json rename to examples/stable-diffusion/quantization/stable-diffusion-xl/quantize_config.json index b372905d7f..5d686e659d --- a/examples/stable-diffusion/quantization/quant_config.json +++ b/examples/stable-diffusion/quantization/stable-diffusion-xl/quantize_config.json @@ -3,5 +3,5 @@ "mode": "QUANTIZE", "observer": "maxabs", "scale_method": "maxabs_hw", - "dump_stats_path": "./quantization/measure/fp8" -} \ No newline at end of file + "dump_stats_path": "quantization/stable-diffusion-xl/measure/fp8" +} diff --git a/examples/stable-diffusion/text_to_image_generation.py b/examples/stable-diffusion/text_to_image_generation.py index 77804275cf..b32fc5c3f6 100755 --- a/examples/stable-diffusion/text_to_image_generation.py +++ b/examples/stable-diffusion/text_to_image_generation.py @@ -42,7 +42,7 @@ def check_optimum_habana_min_version(*a, **b): # Will error if the minimal version of Optimum Habana is not installed. Remove at your own risks. -check_optimum_habana_min_version("1.14.0.dev0") +check_optimum_habana_min_version("1.16.0.dev0") logger = logging.getLogger(__name__) @@ -305,6 +305,12 @@ def main(): default=None, help="The file with prompts (for large number of images generation).", ) + parser.add_argument( + "--lora_scale", + type=float, + default=None, + help="A lora scale that will be applied to all LoRA layers of the text encoder if LoRA layers are loaded.", + ) args = parser.parse_args() if args.optimize and not args.use_habana: @@ -380,6 +386,9 @@ def main(): if args.throughput_warmup_steps is not None: kwargs_call["throughput_warmup_steps"] = args.throughput_warmup_steps + if args.lora_scale is not None: + kwargs_call["lora_scale"] = args.lora_scale + negative_prompts = args.negative_prompts if args.distributed: distributed_state = PartialState() @@ -441,8 +450,7 @@ def main(): kwargs_call["quant_mode"] = args.quant_mode # Instantiate a Stable Diffusion pipeline class - import habana_frameworks.torch.core as htcore # noqa: F401 - + quant_config_path = os.getenv("QUANT_CONFIG") if sdxl: # SDXL pipelines if controlnet: @@ -473,7 +481,6 @@ def main(): pipeline.unet.set_default_attn_processor(pipeline.unet) pipeline.to(torch.device("hpu")) - quant_config_path = os.getenv("QUANT_CONFIG") if quant_config_path: import habana_frameworks.torch.core as htcore from neural_compressor.torch.quantization import FP8Config, convert, prepare @@ -685,12 +692,12 @@ def main(): logger.info(f"Saving images in {image_save_dir.resolve()}...") if args.ldm3d: for i, rgb in enumerate(outputs.rgb): - rgb.save(image_save_dir / f"rgb_{i+1}.png") + rgb.save(image_save_dir / f"rgb_{i + 1}.png") for i, depth in enumerate(outputs.depth): - depth.save(image_save_dir / f"depth_{i+1}.png") + depth.save(image_save_dir / f"depth_{i + 1}.png") else: for i, image in enumerate(outputs.images): - image.save(image_save_dir / f"image_{i+1}.png") + image.save(image_save_dir / f"image_{i + 1}.png") else: logger.warning("--output_type should be equal to 'pil' to save images in --image_save_dir.") diff --git a/examples/text-to-video/text_to_video_generation.py b/examples/stable-diffusion/text_to_video_generation.py similarity index 99% rename from examples/text-to-video/text_to_video_generation.py rename to examples/stable-diffusion/text_to_video_generation.py index 4a91359617..8813e321cf 100755 --- a/examples/text-to-video/text_to_video_generation.py +++ b/examples/stable-diffusion/text_to_video_generation.py @@ -37,7 +37,7 @@ def check_optimum_habana_min_version(*a, **b): # Will error if the minimal version of Optimum Habana is not installed. Remove at your own risks. -check_optimum_habana_min_version("1.14.0.dev0") +check_optimum_habana_min_version("1.16.0.dev0") logger = logging.getLogger(__name__) diff --git a/examples/stable-diffusion/training/README.md b/examples/stable-diffusion/training/README.md index a10c194066..4ea85c9e36 100644 --- a/examples/stable-diffusion/training/README.md +++ b/examples/stable-diffusion/training/README.md @@ -18,61 +18,71 @@ limitations under the License. This directory contains scripts that showcase how to perform training/fine-tuning of Stable Diffusion models on Habana Gaudi. - ## Textual Inversion [Textual Inversion](https://arxiv.org/abs/2208.01618) is a method to personalize text2image models like Stable Diffusion on your own images using just 3-5 examples. -The `textual_inversion.py` script shows how to implement the training procedure on Habana Gaudi. - -### Cat Toy Example +The `textual_inversion.py` script shows how to implement the training procedure on Habana Gaudi. In the examples below, we will use a set of cat images from the following dataset: [https://huggingface.co/datasets/diffusers/cat_toy_example](https://huggingface.co/datasets/diffusers/cat_toy_example) -Let's first download this dataset locally: - -```python -from huggingface_hub import snapshot_download -from pathlib import Path -import shutil - -local_dir = './cat' -snapshot_download( - 'diffusers/cat_toy_example', - local_dir=local_dir, - repo_type='dataset', - ignore_patterns='.gitattributes', -) -cache_dir = Path(local_dir, '.cache') -if cache_dir.is_dir(): - shutil.rmtree(cache_dir) +To download this and other example training datasets locally, run: +```bash +python download_train_datasets.py ``` -This will be our training data. Now we can launch the training using: ```bash python textual_inversion.py \ - --pretrained_model_name_or_path CompVis/stable-diffusion-v1-4 \ - --train_data_dir ./cat \ - --learnable_property object \ - --placeholder_token "" \ - --initializer_token toy \ - --resolution 512 \ - --train_batch_size 4 \ - --max_train_steps 3000 \ - --learning_rate 5.0e-04 \ - --scale_lr \ - --lr_scheduler constant \ - --lr_warmup_steps 0 \ - --output_dir /tmp/textual_inversion_cat \ - --save_as_full_pipeline \ - --gaudi_config_name Habana/stable-diffusion \ - --throughput_warmup_steps 3 + --pretrained_model_name_or_path CompVis/stable-diffusion-v1-4 \ + --train_data_dir ./cat \ + --learnable_property object \ + --placeholder_token "" \ + --initializer_token toy \ + --resolution 512 \ + --train_batch_size 4 \ + --max_train_steps 3000 \ + --learning_rate 5.0e-04 \ + --scale_lr \ + --lr_scheduler constant \ + --lr_warmup_steps 0 \ + --output_dir /tmp/textual_inversion_cat \ + --save_as_full_pipeline \ + --gaudi_config_name Habana/stable-diffusion \ + --throughput_warmup_steps 3 +``` + +> [!NOTE] +> Change `--resolution` to 768 if you are using the [stable-diffusion-2](https://huggingface.co/stabilityai/stable-diffusion-2) 768x768 model. + +> [!NOTE] +> As described in [the official paper](https://arxiv.org/abs/2208.01618), only one embedding vector is used for the placeholder token, *e.g.* `""`. +> However, one can also add multiple embedding vectors for the placeholder token to increase the number of fine-tuneable parameters. +> This can help the model to learn more complex details. To use multiple embedding vectors, you can define `--num_vectors` to a number larger than one, +> *e.g.*: `--num_vectors 5`. The saved textual inversion vectors will then be larger in size compared to the default case. + +Once you have trained a model as described above, inference can be done using `GaudiStableDiffusionPipeline`. +Please make sure to include the `placeholder_token` in your prompt so that textual inversion guided inference can take effect. + +You can use `text_to_image_generation.py` sample to run inference with the fine-tuned model: + +```bash +python ../text_to_image_generation.py \ + --model_name_or_path /tmp/textual_inversion_cat \ + --prompts "A backpack" \ + --num_images_per_prompt 5 \ + --batch_size 1 \ + --image_save_dir /tmp/textual_inversion_cat_images \ + --use_habana \ + --use_hpu_graphs \ + --gaudi_config Habana/stable-diffusion \ + --sdp_on_bf16 \ + --bf16 ``` -The following example shows how to run inference using the fine-tuned model: +Alternatively, you can run inference with the fine-tuned model using a simple Python script like this: ```python from optimum.habana.diffusers import GaudiStableDiffusionPipeline @@ -85,6 +95,7 @@ pipe = GaudiStableDiffusionPipeline.from_pretrained( use_habana=True, use_hpu_graphs=True, gaudi_config="Habana/stable-diffusion", + sdp_on_bf16=True, ) prompt = "A backpack" @@ -92,14 +103,6 @@ image = pipe(prompt, num_inference_steps=50, guidance_scale=7.5).images[0] image.save(f"cat-backpack.png") ``` -> Change `--resolution` to 768 if you are using the [stable-diffusion-2](https://huggingface.co/stabilityai/stable-diffusion-2) 768x768 model. - -> As described in [the official paper](https://arxiv.org/abs/2208.01618), only one embedding vector is used for the placeholder token, *e.g.* `""`. -> However, one can also add multiple embedding vectors for the placeholder token to increase the number of fine-tuneable parameters. -> This can help the model to learn more complex details. To use multiple embedding vectors, you can define `--num_vectors` to a number larger than one, -> *e.g.*: `--num_vectors 5`. The saved textual inversion vectors will then be larger in size compared to the default case. - - ## Textual Inversion XL The `textual_inversion_sdxl.py` script shows how to implement textual inversion fine-tuning on Gaudi for XL diffusion models @@ -109,32 +112,52 @@ Assuming the afforemenioned cat toy dataset has been obtained, we can launch tex ```bash python textual_inversion_sdxl.py \ - --pretrained_model_name_or_path stabilityai/stable-diffusion-xl-base-1.0 \ - --train_data_dir ./cat \ - --learnable_property object \ - --placeholder_token "" \ - --initializer_token toy \ - --resolution 768 \ - --train_batch_size 1 \ - --gradient_accumulation_steps 4 \ - --max_train_steps 500 \ - --learning_rate 5.0e-04 \ - --scale_lr \ - --lr_scheduler constant \ - --lr_warmup_steps 0 \ - --output_dir /tmp/textual_inversion_cat_sdxl \ - --save_as_full_pipeline \ - --gaudi_config_name Habana/stable-diffusion \ - --throughput_warmup_steps 3 + --pretrained_model_name_or_path stabilityai/stable-diffusion-xl-base-1.0 \ + --train_data_dir ./cat \ + --learnable_property object \ + --placeholder_token "" \ + --initializer_token toy \ + --resolution 768 \ + --train_batch_size 1 \ + --gradient_accumulation_steps 4 \ + --max_train_steps 500 \ + --learning_rate 5.0e-04 \ + --scale_lr \ + --lr_scheduler constant \ + --lr_warmup_steps 0 \ + --output_dir /tmp/textual_inversion_cat_sdxl \ + --save_as_full_pipeline \ + --gaudi_config_name Habana/stable-diffusion \ + --throughput_warmup_steps 3 ``` -> As described in [the official paper](https://arxiv.org/abs/2208.01618), only one embedding vector is used for the placeholder token, *e.g.* `""`. -> However, one can also add multiple embedding vectors for the placeholder token to increase the number of fine-tuneable parameters. -> This can help the model to learn more complex details. To use multiple embedding vectors, you can define `--num_vectors` to a number larger than one, -> *e.g.*: `--num_vectors 5`. The saved textual inversion vectors will then be larger in size compared to the default case. +> [!NOTE] +> As described in [the official paper](https://arxiv.org/abs/2208.01618), only one embedding vector is used for the placeholder token, +> e.g. `""`. However, one can also add multiple embedding vectors for the placeholder token to increase the number of fine-tuneable +> parameters. This can help the model to learn more complex details. To use multiple embedding vectors, you can define `--num_vectors` to +> a number larger than one, e.g.: `--num_vectors 5`. The saved textual inversion vectors will then be larger in size compared to the default case. The script also supports training of both text encoders of SDXL, so inference can be executed by inserting a placeholder token into one or both prompts. -The following example shows how to run inference using the fine tuned-model with both text encoders, separately and in combination: + +For example, after training you can use `text_to_image_generation.py` sample to run inference with the fine-tuned model as follows: + +```bash +python ../text_to_image_generation.py \ + --model_name_or_path /tmp/textual_inversion_cat_sdxl \ + --prompts "A backpack" \ + --num_images_per_prompt 5 \ + --batch_size 1 \ + --image_save_dir /tmp/textual_inversion_cat_sdxl_images \ + --use_habana \ + --use_hpu_graphs \ + --gaudi_config Habana/stable-diffusion \ + --sdp_on_bf16 \ + --bf16 +``` + +Alternatively, you can run inference with the fine-tuned model using a simple standalone Python script. +The following script can be used to run inference using the fine-tuned model with both text encoders, +separately and in combination: ```python from optimum.habana.diffusers import GaudiStableDiffusionXLPipeline @@ -147,6 +170,7 @@ pipe = GaudiStableDiffusionXLPipeline.from_pretrained( use_habana=True, use_hpu_graphs=True, gaudi_config="Habana/stable-diffusion", + sdp_on_bf16=True, ) prompt = "A backpack" @@ -161,71 +185,77 @@ image = pipe(prompt=prompt, prompt_2=prompt_2, num_inference_steps=50, guidance_ image.save(f"cat-backpack_p1and2.png") ``` -> [!NOTE] -> Change `--resolution` to 768 if you are using [stable-diffusion-2](https://huggingface.co/stabilityai/stable-diffusion-2) 768x768 model. - -> [!NOTE] -> As described in [the official paper](https://arxiv.org/abs/2208.01618), only one embedding vector is used for the placeholder token, -> e.g. `""`. However, one can also add multiple embedding vectors for the placeholder token to increase the number of fine-tuneable -> parameters. This can help the model to learn more complex details. To use multiple embedding vectors, you can define `--num_vectors` to -> a number larger than one, e.g.: `--num_vectors 5`. The saved textual inversion vectors will then be larger in size compared to the default case. - - ## ControlNet Training ControlNet was introduced in [Adding Conditional Control to Text-to-Image Diffusion Models ](https://huggingface.co/papers/2302.05543) by Lvmin Zhang and Maneesh Agrawala. It is a type of model for controlling StableDiffusion by conditioning the model with an additional input image. This example is adapted from [controlnet example in the diffusers repository](https://github.com/huggingface/diffusers/tree/main/examples/controlnet#training). -First, download the conditioning images as shown below: - +To download the example conditioning images locally, run: ```bash -wget https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/controlnet_training/conditioning_image_1.png -wget https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/controlnet_training/conditioning_image_2.png +python download_train_datasets.py ``` Then proceed to training with command: ```bash python train_controlnet.py \ - --pretrained_model_name_or_path=CompVis/stable-diffusion-v1-4\ - --output_dir=/tmp/stable_diffusion1_5 \ - --dataset_name=fusing/fill50k \ - --resolution=512 \ - --learning_rate=1e-5 \ - --validation_image "./conditioning_image_1.png" "./conditioning_image_2.png" \ - --validation_prompt "red circle with blue background" "cyan circle with brown floral background" \ - --train_batch_size=4 \ - --throughput_warmup_steps=3 \ - --use_hpu_graphs \ - --bf16 \ - --trust_remote_code + --pretrained_model_name_or_path=CompVis/stable-diffusion-v1-4\ + --output_dir=/tmp/stable_diffusion1_4 \ + --dataset_name=fusing/fill50k \ + --resolution=512 \ + --learning_rate=1e-5 \ + --validation_image "./cnet/conditioning_image_1.png" "./cnet/conditioning_image_2.png" \ + --validation_prompt "red circle with blue background" "cyan circle with brown floral background" \ + --train_batch_size=4 \ + --throughput_warmup_steps=3 \ + --use_hpu_graphs \ + --sdp_on_bf16 \ + --bf16 \ + --trust_remote_code ``` -### Multi-card Run +### Multi-Card Training You can run these fine-tuning scripts in a distributed fashion as follows: ```bash python ../../gaudi_spawn.py --use_mpi --world_size 8 train_controlnet.py \ - --pretrained_model_name_or_path CompVis/stable-diffusion-v1-4 \ - --output_dir=/tmp/stable_diffusion1_5 \ - --dataset_name=fusing/fill50k \ - --resolution=512 \ - --learning_rate=1e-5 \ - --validation_image "./conditioning_image_1.png" "./conditioning_image_2.png" \ - --validation_prompt "red circle with blue background" "cyan circle with brown floral background" \ - --train_batch_size=4 \ - --throughput_warmup_steps 3 \ - --use_hpu_graphs \ - --bf16 \ - --trust_remote_code + --pretrained_model_name_or_path CompVis/stable-diffusion-v1-4 \ + --output_dir=/tmp/stable_diffusion1_4 \ + --dataset_name=fusing/fill50k \ + --resolution=512 \ + --learning_rate=1e-5 \ + --validation_image "./cnet/conditioning_image_1.png" "./cnet/conditioning_image_2.png" \ + --validation_prompt "red circle with blue background" "cyan circle with brown floral background" \ + --train_batch_size=4 \ + --throughput_warmup_steps 3 \ + --use_hpu_graphs \ + --sdp_on_bf16 \ + --bf16 \ + --trust_remote_code ``` - ### Inference -Once you have trained a model as described right above, inference can be done simply using the `GaudiStableDiffusionPipeline`. -Make sure to include the `placeholder_token` in your prompt. +After training completes, you can use `text_to_image_generation.py` sample to run inference with the fine-tuned ControlNet model: + +```bash +python ../text_to_image_generation.py \ + --model_name_or_path CompVis/stable-diffusion-v1-4 \ + --controlnet_model_name_or_path /tmp/stable_diffusion1_4 \ + --prompts "pale golden rod circle with old lace background" \ + --control_image "./cnet/conditioning_image_1.png" \ + --num_images_per_prompt 5 \ + --batch_size 1 \ + --image_save_dir /tmp/controlnet_images \ + --use_habana \ + --use_hpu_graphs \ + --gaudi_config Habana/stable-diffusion \ + --sdp_on_bf16 \ + --bf16 +``` + +Alternatively, you can run inference using a simple standalone Python script, as shown below: ```python from diffusers import ControlNetModel, UniPCMultistepScheduler @@ -234,7 +264,7 @@ import torch from optimum.habana.diffusers import GaudiStableDiffusionControlNetPipeline base_model_path = "CompVis/stable-diffusion-v1-4" -controlnet_path = "/tmp/stable_diffusion1_5" +controlnet_path = "/tmp/stable_diffusion1_4" controlnet = ControlNetModel.from_pretrained(controlnet_path, torch_dtype=torch.bfloat16) pipe = GaudiStableDiffusionControlNetPipeline.from_pretrained( @@ -244,12 +274,13 @@ pipe = GaudiStableDiffusionControlNetPipeline.from_pretrained( use_habana=True, use_hpu_graphs=True, gaudi_config="Habana/stable-diffusion", + sdp_on_bf16=True, ) # speed up diffusion process with faster scheduler and memory optimization pipe.scheduler = UniPCMultistepScheduler.from_config(pipe.scheduler.config) -control_image = load_image("./conditioning_image_1.png") +control_image = load_image("./cnet/conditioning_image_1.png") prompt = "pale golden rod circle with old lace background" # generate image @@ -260,7 +291,6 @@ image = pipe( image.save("./output.png") ``` - ## Fine-Tuning for Stable Diffusion XL The `train_text_to_image_sdxl.py` script shows how to implement the fine-tuning of Stable Diffusion XL models on Gaudi. @@ -272,100 +302,102 @@ Install the requirements: pip install -r requirements.txt ``` -### Single-card Training +### Single Card Training To train Stable Diffusion XL on a single Gaudi card, use: ```bash python train_text_to_image_sdxl.py \ - --pretrained_model_name_or_path stabilityai/stable-diffusion-xl-base-1.0 \ - --pretrained_vae_model_name_or_path madebyollin/sdxl-vae-fp16-fix \ - --dataset_name lambdalabs/naruto-blip-captions \ - --resolution 512 \ - --crop_resolution 512 \ - --center_crop \ - --random_flip \ - --proportion_empty_prompts=0.2 \ - --train_batch_size 16 \ - --max_train_steps 2500 \ - --learning_rate 1e-05 \ - --max_grad_norm 1 \ - --lr_scheduler constant \ - --lr_warmup_steps 0 \ - --output_dir sdxl_model_output \ - --gaudi_config_name Habana/stable-diffusion \ - --throughput_warmup_steps 3 \ - --dataloader_num_workers 8 \ - --bf16 \ - --use_hpu_graphs_for_training \ - --use_hpu_graphs_for_inference \ - --validation_prompt="a cute naruto creature" \ - --validation_epochs 48 \ - --checkpointing_steps 2500 \ - --logging_step 10 \ - --adjust_throughput + --pretrained_model_name_or_path stabilityai/stable-diffusion-xl-base-1.0 \ + --pretrained_vae_model_name_or_path madebyollin/sdxl-vae-fp16-fix \ + --dataset_name lambdalabs/naruto-blip-captions \ + --resolution 512 \ + --crop_resolution 512 \ + --center_crop \ + --random_flip \ + --proportion_empty_prompts=0.2 \ + --train_batch_size 16 \ + --max_train_steps 2500 \ + --learning_rate 1e-05 \ + --max_grad_norm 1 \ + --lr_scheduler constant \ + --lr_warmup_steps 0 \ + --output_dir sdxl_model_output \ + --gaudi_config_name Habana/stable-diffusion \ + --throughput_warmup_steps 3 \ + --dataloader_num_workers 8 \ + --sdp_on_bf16 \ + --bf16 \ + --use_hpu_graphs_for_training \ + --use_hpu_graphs_for_inference \ + --validation_prompt="a cute naruto creature" \ + --validation_epochs 48 \ + --checkpointing_steps 2500 \ + --logging_step 10 \ + --adjust_throughput ``` - ### Multi-Card Training To train Stable Diffusion XL on a multi-card Gaudi system, use: ```bash PT_HPU_RECIPE_CACHE_CONFIG=/tmp/stdxl_recipe_cache,True,1024 \ python ../../gaudi_spawn.py --world_size 8 --use_mpi train_text_to_image_sdxl.py \ - --pretrained_model_name_or_path stabilityai/stable-diffusion-xl-base-1.0 \ - --pretrained_vae_model_name_or_path madebyollin/sdxl-vae-fp16-fix \ - --dataset_name lambdalabs/naruto-blip-captions \ - --resolution 512 \ - --crop_resolution 512 \ - --center_crop \ - --random_flip \ - --proportion_empty_prompts=0.2 \ - --train_batch_size 16 \ - --max_train_steps 336 \ - --learning_rate 1e-05 \ - --max_grad_norm 1 \ - --lr_scheduler constant \ - --lr_warmup_steps 0 \ - --output_dir sdxl_model_output \ - --gaudi_config_name Habana/stable-diffusion \ - --throughput_warmup_steps 3 \ - --dataloader_num_workers 8 \ - --bf16 \ - --use_hpu_graphs_for_training \ - --use_hpu_graphs_for_inference \ - --validation_prompt="a cute naruto creature" \ - --validation_epochs 48 \ - --checkpointing_steps 336 \ - --mediapipe dataset_sdxl_mediapipe \ - --adjust_throughput + --pretrained_model_name_or_path stabilityai/stable-diffusion-xl-base-1.0 \ + --pretrained_vae_model_name_or_path madebyollin/sdxl-vae-fp16-fix \ + --dataset_name lambdalabs/naruto-blip-captions \ + --resolution 512 \ + --crop_resolution 512 \ + --center_crop \ + --random_flip \ + --proportion_empty_prompts=0.2 \ + --train_batch_size 16 \ + --max_train_steps 336 \ + --learning_rate 1e-05 \ + --max_grad_norm 1 \ + --lr_scheduler constant \ + --lr_warmup_steps 0 \ + --output_dir sdxl_model_output \ + --gaudi_config_name Habana/stable-diffusion \ + --throughput_warmup_steps 3 \ + --dataloader_num_workers 8 \ + --sdp_on_bf16 \ + --bf16 \ + --use_hpu_graphs_for_training \ + --use_hpu_graphs_for_inference \ + --validation_prompt="a cute naruto creature" \ + --validation_epochs 48 \ + --checkpointing_steps 336 \ + --mediapipe dataset_sdxl_mediapipe \ + --adjust_throughput ``` -### Single-Card Training on Gaudi1 +### Single Card Training on Gaudi1 To train Stable Diffusion XL on a single Gaudi1 card, use: ```bash python train_text_to_image_sdxl.py \ - --pretrained_model_name_or_path stabilityai/stable-diffusion-xl-base-1.0 \ - --pretrained_vae_model_name_or_path madebyollin/sdxl-vae-fp16-fix \ - --dataset_name lambdalabs/naruto-blip-captions \ - --resolution 256 \ - --center_crop \ - --random_flip \ - --proportion_empty_prompts=0.2 \ - --train_batch_size 1 \ - --gradient_accumulation_steps 4 \ - --max_train_steps 3000 \ - --learning_rate 1e-05 \ - --max_grad_norm 1 \ - --lr_scheduler constant \ - --lr_warmup_steps 0 \ - --output_dir sdxl_model_output \ - --gaudi_config_name Habana/stable-diffusion \ - --throughput_warmup_steps 3 \ - --use_hpu_graphs_for_training \ - --use_hpu_graphs_for_inference \ - --checkpointing_steps 3000 \ - --bf16 + --pretrained_model_name_or_path stabilityai/stable-diffusion-xl-base-1.0 \ + --pretrained_vae_model_name_or_path madebyollin/sdxl-vae-fp16-fix \ + --dataset_name lambdalabs/naruto-blip-captions \ + --resolution 256 \ + --center_crop \ + --random_flip \ + --proportion_empty_prompts=0.2 \ + --train_batch_size 1 \ + --gradient_accumulation_steps 4 \ + --max_train_steps 3000 \ + --learning_rate 1e-05 \ + --max_grad_norm 1 \ + --lr_scheduler constant \ + --lr_warmup_steps 0 \ + --output_dir sdxl_model_output \ + --gaudi_config_name Habana/stable-diffusion \ + --throughput_warmup_steps 3 \ + --use_hpu_graphs_for_training \ + --use_hpu_graphs_for_inference \ + --checkpointing_steps 3000 \ + --sdp_on_bf16 \ + --bf16 ``` > [!NOTE] @@ -375,6 +407,24 @@ python train_text_to_image_sdxl.py \ > [!NOTE] > `--mediapipe` only works on Gaudi2. +### Inference + +After training is finished, you can run inference using `text_to_image_generation.py` script as follows: + +```bash +python ../text_to_image_generation.py \ + --model_name_or_path sdxl_model_output \ + --prompts "a cute naruto creature" \ + --num_images_per_prompt 5 \ + --batch_size 1 \ + --image_save_dir /tmp/stable_diffusion_xl_images \ + --scheduler euler_discrete \ + --use_habana \ + --use_hpu_graphs \ + --gaudi_config Habana/stable-diffusion \ + --sdp_on_bf16 \ + --bf16 +``` ## DreamBooth @@ -382,28 +432,12 @@ DreamBooth is a technique for personalizing text-to-image models like Stable Dif of a specific subject. The `train_dreambooth.py` script demonstrates how to implement this training process and adapt it for Stable Diffusion. -### Dog Toy Example - For DreamBooth examples we will use a set of dog images from the following dataset: [https://huggingface.co/datasets/diffusers/dog-example](https://huggingface.co/datasets/diffusers/dog-example). -Let's first download this dataset locally: - -```python -from huggingface_hub import snapshot_download -from pathlib import Path -import shutil - -local_dir = './dog' -snapshot_download( - 'diffusers/dog-example', - local_dir=local_dir, - repo_type='dataset', - ignore_patterns='.gitattributes', -) -cache_dir = Path(local_dir, '.cache') -if cache_dir.is_dir(): - shutil.rmtree(cache_dir) +To download this and other example training datasets locally, run: +```bash +python download_train_datasets.py ``` ### Full Model Fine-Tuning @@ -411,26 +445,26 @@ if cache_dir.is_dir(): To launch the multi-card Stable Diffusion training, use: ```bash python ../../gaudi_spawn.py --world_size 8 --use_mpi train_dreambooth.py \ - --pretrained_model_name_or_path="CompVis/stable-diffusion-v1-4" \ - --instance_data_dir="dog" \ - --output_dir="dog_sd" \ - --class_data_dir="path-to-class-images" \ - --with_prior_preservation --prior_loss_weight=1.0 \ - --instance_prompt="a photo of sks dog" \ - --class_prompt="a photo of dog" \ - --resolution=512 \ - --train_batch_size=1 \ - --num_class_images=200 \ - --gradient_accumulation_steps=1 \ - --learning_rate=5e-6 \ - --lr_scheduler="constant" \ - --lr_warmup_steps=0 \ - --max_train_steps=800 \ - --mixed_precision=bf16 \ - --use_hpu_graphs_for_training \ - --use_hpu_graphs_for_inference \ - --gaudi_config_name Habana/stable-diffusion \ - full + --pretrained_model_name_or_path="CompVis/stable-diffusion-v1-4" \ + --instance_data_dir="dog" \ + --output_dir="dog_sd" \ + --class_data_dir="path-to-class-images" \ + --with_prior_preservation --prior_loss_weight=1.0 \ + --instance_prompt="a photo of sks dog" \ + --class_prompt="a photo of dog" \ + --resolution=512 \ + --train_batch_size=1 \ + --num_class_images=200 \ + --gradient_accumulation_steps=1 \ + --learning_rate=5e-6 \ + --lr_scheduler="constant" \ + --lr_warmup_steps=0 \ + --max_train_steps=800 \ + --mixed_precision=bf16 \ + --use_hpu_graphs_for_training \ + --use_hpu_graphs_for_inference \ + --gaudi_config_name Habana/stable-diffusion \ + full ``` Prior preservation is used to prevent overfitting and language drift. For more details, refer to the original paper. @@ -448,27 +482,27 @@ UNet or text encoder. To run the multi-card training, use: ```bash python ../../gaudi_spawn.py --world_size 8 --use_mpi train_dreambooth.py \ - --pretrained_model_name_or_path="CompVis/stable-diffusion-v1-4" \ - --instance_data_dir="dog" \ - --output_dir="dog_sd" \ - --class_data_dir="path-to-class-images" \ - --with_prior_preservation \ - --prior_loss_weight=1.0 \ - --instance_prompt="a photo of sks dog" \ - --class_prompt="a photo of dog" \ - --resolution=512 \ - --train_batch_size=1 \ - --num_class_images=200 \ - --gradient_accumulation_steps=1 \ - --learning_rate=1e-4 \ - --lr_scheduler="constant" \ - --lr_warmup_steps=0 \ - --max_train_steps=800 \ - --mixed_precision=bf16 \ - --use_hpu_graphs_for_training \ - --use_hpu_graphs_for_inference \ - --gaudi_config_name Habana/stable-diffusion \ - lora --unet_r 8 --unet_alpha 8 + --pretrained_model_name_or_path="CompVis/stable-diffusion-v1-4" \ + --instance_data_dir="dog" \ + --output_dir="dog_sd" \ + --class_data_dir="path-to-class-images" \ + --with_prior_preservation \ + --prior_loss_weight=1.0 \ + --instance_prompt="a photo of sks dog" \ + --class_prompt="a photo of dog" \ + --resolution=512 \ + --train_batch_size=1 \ + --num_class_images=200 \ + --gradient_accumulation_steps=1 \ + --learning_rate=1e-4 \ + --lr_scheduler="constant" \ + --lr_warmup_steps=0 \ + --max_train_steps=800 \ + --mixed_precision=bf16 \ + --use_hpu_graphs_for_training \ + --use_hpu_graphs_for_inference \ + --gaudi_config_name Habana/stable-diffusion \ + lora --unet_r 8 --unet_alpha 8 ``` > [!NOTE] > When using PEFT method we can use a much higher learning rate compared to vanilla dreambooth. @@ -498,6 +532,7 @@ python ../text_to_image_generation.py \ --use_habana \ --use_hpu_graphs \ --gaudi_config Habana/stable-diffusion \ + --sdp_on_bf16 \ --bf16 ``` @@ -508,54 +543,70 @@ We can use the same `dog` dataset for the following examples. To launch Stable Diffusion XL LoRA training on a multi-card Gaudi system, use:" ```bash python train_dreambooth_lora_sdxl.py \ - --pretrained_model_name_or_path="stabilityai/stable-diffusion-xl-base-1.0" \ - --instance_data_dir="dog" \ - --pretrained_vae_model_name_or_path="madebyollin/sdxl-vae-fp16-fix" \ - --output_dir="lora-trained-xl" \ - --mixed_precision="bf16" \ - --instance_prompt="a photo of sks dog" \ - --resolution=1024 \ - --train_batch_size=1 \ - --gradient_accumulation_steps=4 \ - --learning_rate=1e-4 \ - --lr_scheduler="constant" \ - --lr_warmup_steps=0 \ - --max_train_steps=500 \ - --validation_prompt="A photo of sks dog in a bucket" \ - --validation_epochs=25 \ - --seed=0 \ - --use_hpu_graphs_for_inference \ - --use_hpu_graphs_for_training \ - --gaudi_config_name Habana/stable-diffusion + --pretrained_model_name_or_path="stabilityai/stable-diffusion-xl-base-1.0" \ + --instance_data_dir="dog" \ + --pretrained_vae_model_name_or_path="madebyollin/sdxl-vae-fp16-fix" \ + --output_dir="lora-trained-xl" \ + --mixed_precision="bf16" \ + --instance_prompt="a photo of sks dog" \ + --resolution=1024 \ + --train_batch_size=1 \ + --gradient_accumulation_steps=4 \ + --learning_rate=1e-4 \ + --lr_scheduler="constant" \ + --lr_warmup_steps=0 \ + --max_train_steps=500 \ + --validation_prompt="A photo of sks dog in a bucket" \ + --validation_epochs=25 \ + --seed=0 \ + --use_hpu_graphs_for_inference \ + --use_hpu_graphs_for_training \ + --gaudi_config_name Habana/stable-diffusion ``` To launch Stable Diffusion XL LoRA training on a multi-card Gaudi system, use:" ```bash python ../../gaudi_spawn.py --world_size 8 --use_mpi train_dreambooth_lora_sdxl.py \ - --pretrained_model_name_or_path="stabilityai/stable-diffusion-xl-base-1.0" \ - --instance_data_dir="dog" \ - --pretrained_vae_model_name_or_path="madebyollin/sdxl-vae-fp16-fix" \ - --output_dir="lora-trained-xl" \ - --mixed_precision="bf16" \ - --instance_prompt="a photo of sks dog" \ - --resolution=1024 \ - --train_batch_size=1 \ - --gradient_accumulation_steps=4 \ - --learning_rate=1e-4 \ - --lr_scheduler="constant" \ - --lr_warmup_steps=0 \ - --max_train_steps=500 \ - --validation_prompt="A photo of sks dog in a bucket" \ - --validation_epochs=25 \ - --seed=0 \ - --use_hpu_graphs_for_inference \ - --use_hpu_graphs_for_training \ - --gaudi_config_name Habana/stable-diffusion + --pretrained_model_name_or_path="stabilityai/stable-diffusion-xl-base-1.0" \ + --instance_data_dir="dog" \ + --pretrained_vae_model_name_or_path="madebyollin/sdxl-vae-fp16-fix" \ + --output_dir="lora-trained-xl" \ + --mixed_precision="bf16" \ + --instance_prompt="a photo of sks dog" \ + --resolution=1024 \ + --train_batch_size=1 \ + --gradient_accumulation_steps=4 \ + --learning_rate=1e-4 \ + --lr_scheduler="constant" \ + --lr_warmup_steps=0 \ + --max_train_steps=500 \ + --validation_prompt="A photo of sks dog in a bucket" \ + --validation_epochs=25 \ + --seed=0 \ + --use_hpu_graphs_for_inference \ + --use_hpu_graphs_for_training \ + --gaudi_config_name Habana/stable-diffusion ``` > [!NOTE] > To use DeepSpeed instead of MPI, replace `--use_mpi` with `--deepspeed` in the previous example -After training completes, you can run inference with a simple python script like this: +After training is completed, you can directly use `text_to_image_generation.py` sample for inference, as shown below: +```bash +python ../text_to_image_generation.py \ + --model_name_or_path stabilityai/stable-diffusion-xl-base-1.0 \ + --lora_id lora-trained-xl \ + --prompts "A picture of a sks dog in a bucket" \ + --num_images_per_prompt 5 \ + --batch_size 1 \ + --image_save_dir /tmp/stable_diffusion_xl_images \ + --use_habana \ + --use_hpu_graphs \ + --gaudi_config Habana/stable-diffusion \ + --sdp_on_bf16 \ + --bf16 +``` + +Alternatively, you can run inference with a simple Python script such as this: ```python import torch from optimum.habana import GaudiConfig @@ -567,6 +618,7 @@ pipe = GaudiStableDiffusionXLPipeline.from_pretrained( use_hpu_graphs=True, use_habana=True, gaudi_config="Habana/stable-diffusion", + sdp_on_bf16=True, ) pipe.load_lora_weights("lora-trained-xl") @@ -582,21 +634,6 @@ image = pipe( image.save("sdxl-lora.png") ``` -Alternatively, you could directly use `text_to_image_generation.py` sample for inference as follows: -```bash -python ../text_to_image_generation.py \ - --model_name_or_path stabilityai/stable-diffusion-xl-base-1.0 \ - --lora_id lora-trained-xl \ - --prompts "A picture of a sks dog in a bucket" \ - --num_images_per_prompt 5 \ - --batch_size 1 \ - --image_save_dir /tmp/stable_diffusion_xl_images \ - --use_habana \ - --use_hpu_graphs \ - --gaudi_config Habana/stable-diffusion \ - --bf16 -``` - ### DreamBooth LoRA Fine-Tuning with FLUX.1-dev We can use the same `dog` dataset for the following examples. @@ -604,60 +641,76 @@ We can use the same `dog` dataset for the following examples. To launch FLUX.1-dev LoRA training on a single Gaudi card, use:" ```bash python train_dreambooth_lora_flux.py \ - --pretrained_model_name_or_path="black-forest-labs/FLUX.1-dev" \ - --dataset="dog" \ - --prompt="a photo of sks dog" \ - --output_dir="dog_lora_flux" \ - --mixed_precision="bf16" \ - --weighting_scheme="none" \ - --resolution=1024 \ - --train_batch_size=1 \ - --learning_rate=1e-4 \ - --guidance_scale=1 \ - --report_to="tensorboard" \ - --gradient_accumulation_steps=4 \ - --gradient_checkpointing \ - --lr_scheduler="constant" \ - --lr_warmup_steps=0 \ - --cache_latents \ - --rank=4 \ - --max_train_steps=500 \ - --seed="0" \ - --use_hpu_graphs_for_inference \ - --use_hpu_graphs_for_training \ - --gaudi_config_name="Habana/stable-diffusion" + --pretrained_model_name_or_path="black-forest-labs/FLUX.1-dev" \ + --dataset="dog" \ + --prompt="a photo of sks dog" \ + --output_dir="dog_lora_flux" \ + --mixed_precision="bf16" \ + --weighting_scheme="none" \ + --resolution=1024 \ + --train_batch_size=1 \ + --learning_rate=1e-4 \ + --guidance_scale=1 \ + --report_to="tensorboard" \ + --gradient_accumulation_steps=4 \ + --gradient_checkpointing \ + --lr_scheduler="constant" \ + --lr_warmup_steps=0 \ + --cache_latents \ + --rank=4 \ + --max_train_steps=500 \ + --seed="0" \ + --use_hpu_graphs_for_inference \ + --use_hpu_graphs_for_training \ + --gaudi_config_name="Habana/stable-diffusion" ``` To launch FLUX.1-dev LoRA training on a multi-card Gaudi system, use:" ```bash python ../../gaudi_spawn.py --world_size 8 --use_mpi train_dreambooth_lora_flux.py \ - --pretrained_model_name_or_path="black-forest-labs/FLUX.1-dev" \ - --dataset="dog" \ - --prompt="a photo of sks dog" \ - --output_dir="dog_lora_flux" \ - --mixed_precision="bf16" \ - --weighting_scheme="none" \ - --resolution=1024 \ - --train_batch_size=1 \ - --learning_rate=1e-4 \ - --guidance_scale=1 \ - --report_to="tensorboard" \ - --gradient_accumulation_steps=4 \ - --gradient_checkpointing \ - --lr_scheduler="constant" \ - --lr_warmup_steps=0 \ - --cache_latents \ - --rank=4 \ - --max_train_steps=500 \ - --seed="0" \ - --use_hpu_graphs_for_inference \ - --use_hpu_graphs_for_training \ - --gaudi_config_name="Habana/stable-diffusion" + --pretrained_model_name_or_path="black-forest-labs/FLUX.1-dev" \ + --dataset="dog" \ + --prompt="a photo of sks dog" \ + --output_dir="dog_lora_flux" \ + --mixed_precision="bf16" \ + --weighting_scheme="none" \ + --resolution=1024 \ + --train_batch_size=1 \ + --learning_rate=1e-4 \ + --guidance_scale=1 \ + --report_to="tensorboard" \ + --gradient_accumulation_steps=4 \ + --gradient_checkpointing \ + --lr_scheduler="constant" \ + --lr_warmup_steps=0 \ + --cache_latents \ + --rank=4 \ + --max_train_steps=500 \ + --seed="0" \ + --use_hpu_graphs_for_inference \ + --use_hpu_graphs_for_training \ + --gaudi_config_name="Habana/stable-diffusion" ``` > [!NOTE] > To use DeepSpeed instead of MPI, replace `--use_mpi` with `--use_deepspeed` in the previous example -After training completes, you can run inference on Gaudi system with a simple python script like this: +After training completes, you could directly use `text_to_image_generation.py` sample for inference as follows: +```bash +python ../text_to_image_generation.py \ + --model_name_or_path "black-forest-labs/FLUX.1-dev" \ + --lora_id dog_lora_flux \ + --prompts "A picture of a sks dog in a bucket" \ + --num_images_per_prompt 5 \ + --batch_size 1 \ + --image_save_dir /tmp/flux_images \ + --use_habana \ + --use_hpu_graphs \ + --gaudi_config Habana/stable-diffusion \ + --sdp_on_bf16 \ + --bf16 +``` + +Alternatively, you can run inference on Gaudi system with a simple Python script like this: ```python import torch from optimum.habana import GaudiConfig @@ -669,6 +722,7 @@ pipe = GaudiFluxPipeline.from_pretrained( use_hpu_graphs=True, use_habana=True, gaudi_config="Habana/stable-diffusion", + sdp_on_bf16=True, ) pipe.load_lora_weights("dog_lora_flux") @@ -682,18 +736,3 @@ image = pipe( ).images[0] image.save("flux-dev.png") ``` - -Alternatively, you could directly use `text_to_image_generation.py` sample for inference as follows: -```bash -python ../text_to_image_generation.py \ - --model_name_or_path "black-forest-labs/FLUX.1-dev" \ - --lora_id dog_lora_flux \ - --prompts "A picture of a sks dog in a bucket" \ - --num_images_per_prompt 5 \ - --batch_size 1 \ - --image_save_dir /tmp/flux_images \ - --use_habana \ - --use_hpu_graphs \ - --gaudi_config Habana/stable-diffusion \ - --bf16 -``` diff --git a/examples/stable-diffusion/training/download_train_datasets.py b/examples/stable-diffusion/training/download_train_datasets.py new file mode 100755 index 0000000000..6ff500c9ef --- /dev/null +++ b/examples/stable-diffusion/training/download_train_datasets.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python +import shutil +from pathlib import Path + +from huggingface_hub import hf_hub_download, snapshot_download + + +# Download Cat-Toy example dataset +local_dir = "./cat" +snapshot_download( + repo_id="diffusers/cat_toy_example", + local_dir=local_dir, + repo_type="dataset", + ignore_patterns=".gitattributes", +) +cache_dir = Path(local_dir, ".cache") +if cache_dir.is_dir(): + shutil.rmtree(cache_dir) + +# Download Dog example dataset +local_dir = "./dog" +snapshot_download( + repo_id="diffusers/dog-example", + local_dir=local_dir, + repo_type="dataset", + ignore_patterns=".gitattributes", +) +cache_dir = Path(local_dir, ".cache") +if cache_dir.is_dir(): + shutil.rmtree(cache_dir) + +# Download ControlNet example images +local_dir = "./cnet" +file_path1 = hf_hub_download( + repo_id="huggingface/documentation-images", + subfolder="diffusers/controlnet_training", + filename="conditioning_image_1.png", + repo_type="dataset", + local_dir=local_dir, +) +file_path2 = hf_hub_download( + repo_id="huggingface/documentation-images", + subfolder="diffusers/controlnet_training", + filename="conditioning_image_2.png", + repo_type="dataset", + local_dir=local_dir, +) +shutil.move(file_path1, local_dir) +shutil.move(file_path2, local_dir) +cache_dir = Path(local_dir, ".cache") +if cache_dir.is_dir(): + shutil.rmtree(cache_dir) +sub_dir = Path(local_dir, "diffusers") +if sub_dir.is_dir(): + shutil.rmtree(sub_dir) diff --git a/examples/stable-diffusion/training/requirements.txt b/examples/stable-diffusion/training/requirements.txt index bf92040ae8..a28ca499b5 100644 --- a/examples/stable-diffusion/training/requirements.txt +++ b/examples/stable-diffusion/training/requirements.txt @@ -1,3 +1,4 @@ imagesize peft == 0.10.0 sentencepiece +compel diff --git a/examples/stable-diffusion/training/textual_inversion.py b/examples/stable-diffusion/training/textual_inversion.py index db488f8749..2f465699b3 100755 --- a/examples/stable-diffusion/training/textual_inversion.py +++ b/examples/stable-diffusion/training/textual_inversion.py @@ -130,6 +130,7 @@ def log_validation(text_encoder, tokenizer, unet, vae, args, accelerator, weight use_habana=True, use_hpu_graphs=True, gaudi_config=args.gaudi_config_name, + sdp_on_bf16=args.sdp_on_bf16, ) pipeline.scheduler = GaudiDDIMScheduler.from_config(pipeline.scheduler.config) pipeline.set_progress_bar_config(disable=True) @@ -415,6 +416,9 @@ def parse_args(): default=None, help="Local path to the Gaudi configuration file or its name on the Hugging Face Hub.", ) + parser.add_argument( + "--sdp_on_bf16", action="store_true", help="Allow pyTorch to use reduced precision in the SDPA math backend" + ) parser.add_argument( "--throughput_warmup_steps", type=int, @@ -883,7 +887,7 @@ def main(): htcore.mark_step() # Let's make sure we don't update any embedding weights besides the newly added token - index_no_updates = torch.ones((len(tokenizer),), dtype=torch.bool) + index_no_updates = torch.ones((len(tokenizer),), dtype=torch.bool, device=accelerator.device) index_no_updates[min(placeholder_token_ids) : max(placeholder_token_ids) + 1] = False with torch.no_grad(): diff --git a/examples/stable-diffusion/training/textual_inversion_sdxl.py b/examples/stable-diffusion/training/textual_inversion_sdxl.py old mode 100644 new mode 100755 index 608ee481ad..3ab6c57602 --- a/examples/stable-diffusion/training/textual_inversion_sdxl.py +++ b/examples/stable-diffusion/training/textual_inversion_sdxl.py @@ -392,6 +392,9 @@ def parse_args(): default=None, help="Local path to the Gaudi configuration file or its name on the Hugging Face Hub.", ) + parser.add_argument( + "--sdp_on_bf16", action="store_true", help="Allow pyTorch to use reduced precision in the SDPA math backend" + ) parser.add_argument( "--throughput_warmup_steps", type=int, @@ -623,6 +626,7 @@ def main(): use_habana=True, use_hpu_graphs=True, gaudi_config=args.gaudi_config_name, + sdp_on_bf16=args.sdp_on_bf16, ) text_encoder_1 = pipeline.text_encoder.to(accelerator.device) text_encoder_2 = pipeline.text_encoder_2.to(accelerator.device) @@ -918,9 +922,9 @@ def main(): htcore.mark_step() # Let's make sure we don't update any embedding weights besides the newly added token - index_no_updates = torch.ones((len(tokenizer_1),), dtype=torch.bool) + index_no_updates = torch.ones((len(tokenizer_1),), dtype=torch.bool, device=accelerator.device) index_no_updates[min(placeholder_token_ids) : max(placeholder_token_ids) + 1] = False - index_no_updates_2 = torch.ones((len(tokenizer_2),), dtype=torch.bool) + index_no_updates_2 = torch.ones((len(tokenizer_2),), dtype=torch.bool, device=accelerator.device) index_no_updates_2[min(placeholder_token_ids_2) : max(placeholder_token_ids_2) + 1] = False with torch.no_grad(): diff --git a/examples/stable-diffusion/training/train_controlnet.py b/examples/stable-diffusion/training/train_controlnet.py index e676ae6ddf..004cee5af5 100755 --- a/examples/stable-diffusion/training/train_controlnet.py +++ b/examples/stable-diffusion/training/train_controlnet.py @@ -68,7 +68,7 @@ def check_optimum_habana_min_version(*a, **b): # Will error if the minimal version of Optimum Habana is not installed. Remove at your own risks. -check_optimum_habana_min_version("1.14.0.dev0") +check_optimum_habana_min_version("1.16.0.dev0") if is_wandb_available(): import wandb @@ -120,6 +120,7 @@ def log_validation( use_habana=True, use_hpu_graphs=args.use_hpu_graphs, gaudi_config=gaudi_config, + sdp_on_bf16=args.sdp_on_bf16, ) pipeline.scheduler = UniPCMultistepScheduler.from_config(pipeline.scheduler.config) pipeline = pipeline.to(accelerator.device) @@ -438,6 +439,12 @@ def parse_args(input_args=None): ' (default), `"wandb"` and `"comet_ml"`. Use `"all"` to report to all integrations.' ), ) + parser.add_argument( + "--sdp_on_bf16", + action="store_true", + default=False, + help="Allow pyTorch to use reduced precision in the SDPA math backend", + ) parser.add_argument( "--bf16", action="store_true", diff --git a/examples/stable-diffusion/training/train_dreambooth_lora_flux.py b/examples/stable-diffusion/training/train_dreambooth_lora_flux.py index 68b5320d19..1117d0a43f 100755 --- a/examples/stable-diffusion/training/train_dreambooth_lora_flux.py +++ b/examples/stable-diffusion/training/train_dreambooth_lora_flux.py @@ -784,7 +784,7 @@ def load_model_hook(models, input_dir): lora_state_dict = FluxPipeline.lora_state_dict(input_dir) transformer_state_dict = { - f'{k.replace("transformer.", "")}': v for k, v in lora_state_dict.items() if k.startswith("transformer.") + f"{k.replace('transformer.', '')}": v for k, v in lora_state_dict.items() if k.startswith("transformer.") } transformer_state_dict = convert_unet_state_dict_to_peft(transformer_state_dict) incompatible_keys = set_peft_model_state_dict(transformer_, transformer_state_dict, adapter_name="default") diff --git a/examples/stable-diffusion/training/train_dreambooth_lora_sdxl.py b/examples/stable-diffusion/training/train_dreambooth_lora_sdxl.py index b177cf12e6..4e96ee8e0d 100755 --- a/examples/stable-diffusion/training/train_dreambooth_lora_sdxl.py +++ b/examples/stable-diffusion/training/train_dreambooth_lora_sdxl.py @@ -94,7 +94,7 @@ def save_model_card( for i, image in enumerate(images): image.save(os.path.join(repo_folder, f"image_{i}.png")) img_str += f""" - - text: '{validation_prompt if validation_prompt else ' ' }' + - text: '{validation_prompt if validation_prompt else " "}' output: url: "image_{i}.png" @@ -1083,7 +1083,7 @@ def load_model_hook(models, input_dir): lora_state_dict, network_alphas = LoraLoaderMixin.lora_state_dict(input_dir) - unet_state_dict = {f'{k.replace("unet.", "")}': v for k, v in lora_state_dict.items() if k.startswith("unet.")} + unet_state_dict = {f"{k.replace('unet.', '')}": v for k, v in lora_state_dict.items() if k.startswith("unet.")} unet_state_dict = convert_unet_state_dict_to_peft(unet_state_dict) incompatible_keys = set_peft_model_state_dict(unet_, unet_state_dict, adapter_name="default") if incompatible_keys is not None: diff --git a/examples/stable-diffusion/training/train_text_to_image_sdxl.py b/examples/stable-diffusion/training/train_text_to_image_sdxl.py index c9d84ae1b9..7bb96e51a1 100755 --- a/examples/stable-diffusion/training/train_text_to_image_sdxl.py +++ b/examples/stable-diffusion/training/train_text_to_image_sdxl.py @@ -491,6 +491,12 @@ def parse_args(input_args=None): default=False, help=("Whether to use bf16 mixed precision."), ) + parser.add_argument( + "--sdp_on_bf16", + action="store_true", + default=False, + help="Allow pyTorch to use reduced precision in the SDPA math backend", + ) parser.add_argument( "--local_rank", type=int, @@ -878,9 +884,9 @@ def main(args): # download the dataset. if args.dataset_name is not None: if len(args.mediapipe) > 0: - assert ( - args.resolution == args.crop_resolution - ), f"To use hardware pipe, --resolution ({args.resolution}) must equal --crop_resolution ({args.crop_resolution})" + assert args.resolution == args.crop_resolution, ( + f"To use hardware pipe, --resolution ({args.resolution}) must equal --crop_resolution ({args.crop_resolution})" + ) if args.local_rank == 0: if not os.path.exists(args.mediapipe): os.mkdir(args.mediapipe) @@ -1421,6 +1427,7 @@ def compute_time_ids(original_size, crops_coords_top_left): use_habana=True, use_hpu_graphs=args.use_hpu_graphs_for_inference, gaudi_config=args.gaudi_config_name, + sdp_on_bf16=args.sdp_on_bf16, ) else: # vae and text encoders are frozen, only need to update unet @@ -1525,7 +1532,7 @@ def compute_time_ids(original_size, crops_coords_top_left): image_save_dir.mkdir(parents=True, exist_ok=True) logger.info(f"Saving images in {image_save_dir.resolve()}...") for i, image in enumerate(images): - image.save(image_save_dir / f"image_{epoch}_{i+1}.png") + image.save(image_save_dir / f"image_{epoch}_{i + 1}.png") else: logger.warning("--output_type should be equal to 'pil' to save images in --image_save_dir.") diff --git a/examples/stable-diffusion/unconditional_image_generation.py b/examples/stable-diffusion/unconditional_image_generation.py index 36e35ff90f..bd70d0e4d6 100755 --- a/examples/stable-diffusion/unconditional_image_generation.py +++ b/examples/stable-diffusion/unconditional_image_generation.py @@ -20,7 +20,7 @@ def check_optimum_habana_min_version(*a, **b): check_min_version("4.45.0") -check_optimum_habana_min_version("1.14.0.dev0") +check_optimum_habana_min_version("1.16.0.dev0") # Setup logging logging.basicConfig( @@ -68,6 +68,12 @@ def main(): action="store_true", help="Whether to use bf16 precision for classification.", ) + parser.add_argument( + "--sdp_on_bf16", + action="store_true", + default=False, + help="Allow pyTorch to use reduced precision in the SDPA math backend", + ) parser.add_argument( "--save_outputs", action="store_true", @@ -104,6 +110,7 @@ def main(): "use_habana": args.use_habana, "use_hpu_graphs": args.use_hpu_graphs, "gaudi_config": gaudi_config, + "sdp_on_bf16": args.sdp_on_bf16, } kwargs_call = {"throughput_warmup_steps": args.throughput_warmup_steps} diff --git a/examples/summarization/README.md b/examples/summarization/README.md index 745b293d69..bdaef78edf 100644 --- a/examples/summarization/README.md +++ b/examples/summarization/README.md @@ -179,65 +179,8 @@ python ../gaudi_spawn.py \ ## Using DeepSpeed -Here is an example on 8 HPUs on Gaudi2/Gaudi3 with DeepSpeed-ZeRO3 to fine-tune [FLAN-T5 XXL](https://huggingface.co/google/flan-t5-xxl): -```bash -PT_HPU_MAX_COMPOUND_OP_SIZE=512 python ../gaudi_spawn.py \ - --world_size 8 --use_deepspeed run_summarization.py \ - --model_name_or_path google/flan-t5-xxl \ - --do_train \ - --do_eval \ - --dataset_name cnn_dailymail \ - --dataset_config '"3.0.0"' \ - --source_prefix '"summarize: "' \ - --output_dir ./tst-summarization \ - --per_device_train_batch_size 22 \ - --per_device_eval_batch_size 22 \ - --learning_rate 1e-4 \ - --num_train_epochs 3 \ - --overwrite_output_dir \ - --predict_with_generate \ - --use_habana \ - --use_lazy_mode \ - --gaudi_config_name Habana/t5 \ - --ignore_pad_token_for_loss False \ - --pad_to_max_length \ - --generation_max_length 129 \ - --save_strategy epoch \ - --throughput_warmup_steps 3 \ - --gradient_checkpointing \ - --adam_epsilon 1e-08 --logging_steps 1 \ - --deepspeed ds_flan_t5_z3_config_bf16.json -``` - -Here is an example on 8 HPUs on Gaudi2 with DeepSpeed-ZeRO2 to fine-tune t5-large: -```bash -PT_HPU_LAZY_MODE=0 python ../gaudi_spawn.py \ - --world_size 8 \ - --use_deepspeed run_summarization.py \ - --deepspeed ../../tests/configs/deepspeed_zero_2.json \ - --do_train \ - --do_eval \ - --overwrite_output_dir \ - --predict_with_generate \ - --use_habana \ - --gaudi_config_name Habana/t5 \ - --ignore_pad_token_for_loss False \ - --pad_to_max_length \ - --save_strategy no \ - --throughput_warmup_steps 15 \ - --model_name_or_path t5-large \ - --source_prefix '"summarize:"' \ - --dataset_name cnn_dailymail \ - --dataset_config '"3.0.0"' \ - --output_dir /tmp/tst-summarization \ - --per_device_train_batch_size 20 \ - --per_device_eval_batch_size 20 \ - --max_train_samples 2000 \ - --torch_compile_backend hpu_backend \ - --torch_compile -``` - -You can look at the [documentation](https://huggingface.co/docs/optimum/habana/usage_guides/deepspeed) for more information about how to use DeepSpeed in Optimum Habana. +You can check the [DeepSpeed](https://github.com/huggingface/optimum-habana/tree/main/examples#deepspeed) section in Optimum Habana examples for how to run DeepSpeed. +You also can look at the [documentation](https://huggingface.co/docs/optimum/habana/usage_guides/deepspeed) for more information about how to use DeepSpeed in Optimum Habana. ## Inference @@ -262,26 +205,8 @@ python run_summarization.py \ --gaudi_config_name Habana/t5 \ --ignore_pad_token_for_loss False \ --pad_to_max_length \ + --throughput_warmup_steps 3 \ --bf16 \ --bf16_full_eval ``` -You can run inference with BART on the CNN-DailyMail dataset on 1 Gaudi card with the following command: -```bash -python run_summarization.py \ - --model_name_or_path facebook/bart-large-cnn \ - --do_predict \ - --dataset_name cnn_dailymail \ - --dataset_config "3.0.0" \ - --output_dir /tmp/tst-summarization \ - --per_device_eval_batch_size 2 \ - --overwrite_output_dir \ - --predict_with_generate \ - --use_habana \ - --use_lazy_mode \ - --use_hpu_graphs_for_inference \ - --gaudi_config_name Habana/bart \ - --ignore_pad_token_for_loss False \ - --pad_to_max_length \ - --num_beams 1 -``` diff --git a/examples/summarization/run_summarization.py b/examples/summarization/run_summarization.py index 28498fc0a2..a14e0e1dea 100755 --- a/examples/summarization/run_summarization.py +++ b/examples/summarization/run_summarization.py @@ -66,7 +66,7 @@ def check_optimum_habana_min_version(*a, **b): # Will error if the minimal version of Transformers and Optimum Habana are not installed. Remove at your own risks. check_min_version("4.45.0") -check_optimum_habana_min_version("1.14.0.dev0") +check_optimum_habana_min_version("1.16.0.dev0") require_version("datasets>=1.8.0", "To fix: pip install -r examples/pytorch/summarization/requirements.txt") @@ -559,9 +559,9 @@ def main(): return if isinstance(tokenizer, tuple(MULTILINGUAL_TOKENIZERS)): - assert ( - data_args.lang is not None - ), f"{tokenizer.__class__.__name__} is a multilingual tokenizer which requires --lang argument" + assert data_args.lang is not None, ( + f"{tokenizer.__class__.__name__} is a multilingual tokenizer which requires --lang argument" + ) tokenizer.src_lang = data_args.lang tokenizer.tgt_lang = data_args.lang diff --git a/examples/text-classification/run_glue.py b/examples/text-classification/run_glue.py index 57bf7cbb05..6a78ecd91e 100755 --- a/examples/text-classification/run_glue.py +++ b/examples/text-classification/run_glue.py @@ -58,7 +58,7 @@ def check_optimum_habana_min_version(*a, **b): # Will error if the minimal version of Transformers and Optimum Habana are not installed. Remove at your own risks. check_min_version("4.45.0") -check_optimum_habana_min_version("1.14.0.dev0") +check_optimum_habana_min_version("1.16.0.dev0") require_version("datasets>=1.8.0", "To fix: pip install -r examples/pytorch/text-classification/requirements.txt") @@ -168,9 +168,9 @@ def __post_init__(self): train_extension = self.train_file.split(".")[-1] assert train_extension in ["csv", "json"], "`train_file` should be a csv or a json file." validation_extension = self.validation_file.split(".")[-1] - assert ( - validation_extension == train_extension - ), "`validation_file` should have the same extension (csv or json) as `train_file`." + assert validation_extension == train_extension, ( + "`validation_file` should have the same extension (csv or json) as `train_file`." + ) @dataclass @@ -338,9 +338,9 @@ def main(): if data_args.test_file is not None: train_extension = data_args.train_file.split(".")[-1] test_extension = data_args.test_file.split(".")[-1] - assert ( - test_extension == train_extension - ), "`test_file` should have the same extension (csv or json) as `train_file`." + assert test_extension == train_extension, ( + "`test_file` should have the same extension (csv or json) as `train_file`." + ) data_files["test"] = data_args.test_file else: raise ValueError("Need either a GLUE task or a test file for `do_predict`.") diff --git a/examples/text-feature-extraction/README.md b/examples/text-feature-extraction/README.md index 9c34ede54a..e46168840b 100644 --- a/examples/text-feature-extraction/README.md +++ b/examples/text-feature-extraction/README.md @@ -28,12 +28,6 @@ python run_feature_extraction.py \ "BERT is a common machine learning architecture for text-based applications." \ "Alexander Hamilton is one of the founding fathers of the United States." \ --use_hpu_graphs \ + --sdp_on_bf16 \ --bf16 ``` - -Models that have been validated: - -- [Supabase/gte-small](https://huggingface.co/Supabase/gte-small) -- [thenlper/gte-small](https://huggingface.co/thenlper/gte-small) -- [thenlper/gte-base](https://huggingface.co/thenlper/gte-base) -- [thenlper/gte-large](https://huggingface.co/thenlper/gte-large) diff --git a/examples/text-feature-extraction/run_feature_extraction.py b/examples/text-feature-extraction/run_feature_extraction.py index 47320b1979..159f36b488 100644 --- a/examples/text-feature-extraction/run_feature_extraction.py +++ b/examples/text-feature-extraction/run_feature_extraction.py @@ -83,6 +83,9 @@ def parse_args(): action="store_true", help="Whether to perform generation in bf16 precision.", ) + parser.add_argument( + "--sdp_on_bf16", action="store_true", help="Allow pyTorch to use reduced precision in the SDPA math backend" + ) parser.add_argument( "--warmup", type=int, @@ -100,6 +103,8 @@ def parse_args(): def main(): args = parse_args() + if args.sdp_on_bf16: + torch._C._set_math_sdp_allow_fp16_bf16_reduction(True) tokenizer = AutoTokenizer.from_pretrained(args.model_name_or_path) model = AutoModel.from_pretrained(args.model_name_or_path).to("hpu") diff --git a/examples/text-generation/README.md b/examples/text-generation/README.md index bd81c7f7fe..5d399f65dd 100755 --- a/examples/text-generation/README.md +++ b/examples/text-generation/README.md @@ -33,7 +33,7 @@ pip install -r requirements_lm_eval.txt Then, if you plan to use [DeepSpeed-inference](https://docs.habana.ai/en/latest/PyTorch/DeepSpeed/Inference_Using_DeepSpeed.html) (e.g. to use BLOOM/BLOOMZ), you should install DeepSpeed as follows: ```bash -pip install git+https://github.com/HabanaAI/DeepSpeed.git@1.18.0 +pip install git+https://github.com/HabanaAI/DeepSpeed.git@1.19.0 ``` @@ -79,7 +79,8 @@ python run_generation.py \ --use_kv_cache \ --max_new_tokens 100 \ --do_sample \ ---prompt "Here is my prompt" +--prompt "Here is my prompt" \ +--sdp_on_bf16 ``` If you want to provide several prompts as inputs, here is how to do it: @@ -91,7 +92,8 @@ python run_generation.py \ --max_new_tokens 100 \ --do_sample \ --batch_size 2 \ ---prompt "Hello world" "How are you?" +--prompt "Hello world" "How are you?" \ +--sdp_on_bf16 ``` > The batch size should be larger than or equal to the number of prompts. Otherwise, only the first N prompts are kept with N being equal to the batch size. @@ -110,7 +112,8 @@ python run_generation.py \ --use_kv_cache \ --num_return_sequences 1 \ --temperature 0 \ ---prompt "Alice and Bob" +--prompt "Alice and Bob" \ +--sdp_on_bf16 ``` ### Benchmark @@ -138,7 +141,8 @@ python ../gaudi_spawn.py --use_deepspeed --world_size 8 run_generation.py \ --batch_size 1 \ --use_hpu_graphs \ --use_kv_cache \ ---max_new_tokens 100 +--max_new_tokens 100 \ +--sdp_on_bf16 ``` You can also run Llama2-70B on Gaudi2 with all optimizations enabled using the following command: @@ -153,7 +157,8 @@ python ../gaudi_spawn.py --use_deepspeed --world_size 8 run_generation.py \ --attn_softmax_bf16 \ --limit_hpu_graphs \ --reuse_cache \ ---trim_logits +--trim_logits \ +--sdp_on_bf16 ``` To run Falcon-7B inference, use the following command: @@ -165,7 +170,8 @@ python run_generation.py \ --use_kv_cache \ --batch_size 1 \ --max_new_tokens 128 \ - --do_sample + --do_sample \ + --sdp_on_bf16 ``` To run Falcon-40B inference on 8 Gaudi2 cards, use the following command: @@ -182,6 +188,20 @@ python ../gaudi_spawn.py --use_deepspeed --world_size 8 run_generation.py \ --flash_attention_causal_mask ``` +To run Llama3-405B inference on 8 Gaudi3 cards use the following command: +```bash +python ../gaudi_spawn.py --use_deepspeed --world_size 8 run_generation.py \ +--model_name_or_path meta-llama/Llama-3.1-405B-Instruct \ +--max_new_tokens 2048 \ +--bf16 \ +--use_hpu_graphs \ +--use_kv_cache \ +--batch_size 1 \ +--do_sample \ +--use_flash_attention \ +--flash_attention_causal_mask +``` + > To be able to run gated models like [StarCoder](https://huggingface.co/bigcode/starcoder), you should: > - have a HF account > - agree to the terms of use of the model in its model card on the HF Hub @@ -196,7 +216,8 @@ python ../gaudi_spawn.py --use_deepspeed --world_size 8 run_generation.py \ > --use_hpu_graphs \ > --use_kv_cache \ > --max_new_tokens 100 \ -> --bf16 +> --bf16 \ +> --sdp_on_bf16 > ``` ### Use any dataset from the Hugging Face Hub @@ -215,7 +236,8 @@ python run_generation.py \ --use_kv_cache \ --dataset_name JulesBelveze/tldr_news \ --column_name content \ ---bf16 +--bf16 \ +--sdp_on_bf16 ``` > The prompt length is limited to 16 tokens. Prompts longer than this will be truncated. @@ -234,7 +256,8 @@ python run_generation.py \ --bf16 \ --max_new_tokens 100 \ --prompt "Here is my prompt" \ ---peft_model yard1/llama-2-7b-sql-lora-test +--peft_model yard1/llama-2-7b-sql-lora-test \ +--sdp_on_bf16 ``` ### Using growing bucket optimization @@ -330,7 +353,7 @@ PT_ENABLE_INT64_SUPPORT=1 PT_HPU_LAZY_MODE=0 python ../gaudi_spawn.py --world_s ### Running with FP8 -Llama2-70b, Llama2-7b, Llama3-70b, Llama3-8b, Mixtral-8x7B, Falcon-7B, Falcon-40B, Falcon-180B and phi-2 in FP8 are enabled using the [Intel Neural Compressor (INC)](https://docs.habana.ai/en/latest/PyTorch/Inference_on_PyTorch/Inference_Using_FP8.html), which provides model measurement and quantization capabilities in PyTorch. From synapse 1.17 / optimum-habana 1.13 release, INC is used by default for measuring and quantization. Habana Quantization Toolkit (HQT), which was used earlier, will be removed in future releases. To use HQT, disable INC by setting the following environment variable: `USE_INC=0`. +Llama2-70b, Llama2-7b, Llama3-70b, Llama3-8b, Mixtral-8x7B, Falcon-7B, Falcon-40B, Falcon-180B, phi-2 and Llama3-405B in FP8 are enabled using the [Intel Neural Compressor (INC)](https://docs.habana.ai/en/latest/PyTorch/Inference_on_PyTorch/Inference_Using_FP8.html), which provides model measurement and quantization capabilities in PyTorch. From synapse 1.17 / optimum-habana 1.13 release, INC is used by default for measuring and quantization. Habana Quantization Toolkit (HQT), which was used earlier, will be removed in future releases. To use HQT, disable INC by setting the following environment variable: `USE_INC=0`. More information on enabling fp8 in SynapseAI is available here: https://docs.habana.ai/en/latest/PyTorch/Inference_on_PyTorch/Inference_Using_FP8.html @@ -453,6 +476,44 @@ QUANT_CONFIG=./quantization_config/maxabs_quant.json python ../gaudi_spawn.py \ --flash_attention_causal_mask ``` +Here is an example to measure the tensor quantization statistics on Llama3-405B with 8 cards: +> Please note that Llama3-405B requires minimum 16 cards Gaudi2 and 8 cards Gaudi3. +```bash +QUANT_CONFIG=./quantization_config/maxabs_measure_include_outputs.json python ../gaudi_spawn.py \ +--use_deepspeed --world_size 8 run_lm_eval.py \ +-o acc_llama3_405b_bs1_quant.txt \ +--model_name_or_path meta-llama/Llama-3.1-405B-Instruct \ +--use_hpu_graphs \ +--use_kv_cache \ +--trim_logits \ +--batch_size 1 \ +--bf16 \ +--reuse_cache \ +--use_flash_attention \ +--flash_attention_recompute \ +--flash_attention_causal_mask +``` + +Here is an example to quantize the model based on previous measurements for Llama3-405B with 8 cards: +> Please note that Llama3-405B requires minimum 16 cards Gaudi2 and 8 cards Gaudi3. +```bash +QUANT_CONFIG=./quantization_config/maxabs_quant.json python ../gaudi_spawn.py \ +--use_deepspeed --world_size 8 run_generation.py \ +--model_name_or_path meta-llama/Llama-3.1-405B-Instruct \ +--use_hpu_graphs \ +--use_kv_cache \ +--limit_hpu_graphs \ +--max_input_tokens 2048 \ +--max_new_tokens 2048 \ +--batch_size 2 \ +--bf16 \ +--reuse_cache \ +--trim_logits \ +--use_flash_attention \ +--flash_attention_recompute \ +--flash_attention_causal_mask +``` + Here is an example to measure the tensor quantization statistics on phi-2 with 1 card: ```bash @@ -491,7 +552,8 @@ QUANT_CONFIG=./quantization_config/maxabs_measure.json python run_generation.py --max_new_tokens 100 \ --batch_size 1 \ --reuse_cache \ ---bf16 +--bf16 \ +--sdp_on_bf16 ``` Here is an example to quantize the model based on previous measurements for gemma with 1 card: @@ -503,7 +565,8 @@ QUANT_CONFIG=./quantization_config/maxabs_quant_gemma.json python run_generation --max_new_tokens 100 \ --batch_size 1 \ --reuse_cache \ ---bf16 +--bf16 \ +--sdp_on_bf16 ``` @@ -513,14 +576,13 @@ Some bf16 models don't fit on one card due to hpu memory limitation, but in fp8 As measurement is being calculated in bf16 precision, to be able to run fp8 model on single card you should use `unify_measurements` script. Here are the steps: 1. Measure the model on a number of cards that are enough for the model to fit in BF16. -2. Quantize the model on the same amount of cards for scales to be saved. -3. Run unify_measurements.py script using the measurement files created after running steps 1 and 2. A unified measurement is then calculated. +2. Run unify_measurements.py script using the measurement files created in step 1. A unified measurement is then calculated. ```bash python quantization_tools/unify_measurements.py -g 01234567 -m *path_to_8x_measurements* -o *path_to_output_1x_measurement* ``` In the above example, the measurements of cards 0-7 will be unified to a single measurement. For example, if you specify `-g 0123 4567`, cards 0-3 and cards 4-7 will be unified in two different measurement files. All different group combinations are supported. -4. Run quantization using the unified measurement file/s. +3. Run quantization using the unified measurement file/s. More information on usage of the unifier script can be found in fp8 Habana docs: https://docs.habana.ai/en/latest/PyTorch/Inference_on_PyTorch/Inference_Using_FP8.html @@ -568,10 +630,8 @@ https://docs.habana.ai/en/latest/PyTorch/Inference_on_PyTorch/Inference_Using_UI Below is an example to load a model with 4bit checkpoints from Hugging Face. Please note that model name is denoted as ``. -Additionally, the below env vars are used for performance optimizations, and are planned to be removed in future version: -`SRAM_SLICER_SHARED_MME_INPUT_EXPANSION_ENABLED=false ENABLE_EXPERIMENTAL_FLAGS=1` + ```bash -SRAM_SLICER_SHARED_MME_INPUT_EXPANSION_ENABLED=false ENABLE_EXPERIMENTAL_FLAGS=1 \ python run_lm_eval.py \ -o acc_load_uint4_model.txt \ --model_name_or_path \ @@ -594,12 +654,10 @@ Currently, only uint4 checkpoints and single-device configurations are supported More information on enabling 4-bit inference in SynapseAI is available here: https://docs.habana.ai/en/latest/PyTorch/Inference_on_PyTorch/Inference_Using_UINT4.html?highlight=inference%20using%20int4#enabling-and-running-uint4-in-pytorch-models. -Below is an example of loading a llama7b model with a 4bit checkpoint quantized in INC. +Below is an example of loading a llama2-7b model with a 4bit checkpoint quantized in INC. Please note that the model checkpoint name is denoted as ``. -Additionally, the following environment variables are used for performance optimizations and are planned to be removed in future versions: -`SRAM_SLICER_SHARED_MME_INPUT_EXPANSION_ENABLED=false ENABLE_EXPERIMENTAL_FLAGS=1` + ```bash -SRAM_SLICER_SHARED_MME_INPUT_EXPANSION_ENABLED=false ENABLE_EXPERIMENTAL_FLAGS=1 \ python run_lm_eval.py \ -o acc_load_uint4_model.txt \ --model_name_or_path meta-llama/Llama-2-7b-hf \ @@ -644,26 +702,18 @@ For more details see [documentation](https://docs.habana.ai/en/latest/PyTorch/Mo ### Running with UINT4 weight quantization using AutoGPTQ - -Llama2-7b in UINT4 weight only quantization is enabled using [AutoGPTQ Fork](https://github.com/HabanaAI/AutoGPTQ), which provides quantization capabilities in PyTorch. +Llama2-7b in UINT4 weight only quantization is enabled using [AutoGPTQ](https://github.com/AutoGPTQ/AutoGPTQ), which provides quantization capabilities in PyTorch. Currently, the support is for UINT4 inference of pre-quantized models only. ```bash -BUILD_CUDA_EXT=0 python -m pip install -vvv --no-build-isolation git+https://github.com/HabanaAI/AutoGPTQ.git +BUILD_CUDA_EXT=0 python -m pip install -vvv --no-build-isolation git+https://github.com/AutoGPTQ/AutoGPTQ ``` -You can run a *UINT4 weight quantized* model using AutoGPTQ by setting the following environment variables: -`SRAM_SLICER_SHARED_MME_INPUT_EXPANSION_ENABLED=false ENABLE_EXPERIMENTAL_FLAGS=true` before running the command, -and by adding the argument `--load_quantized_model_with_autogptq`. - -***Note:*** -Setting the above environment variables improves performance. These variables will be removed in future releases. - +You can run a *UINT4 weight quantized* model using AutoGPTQ by adding the argument `--load_quantized_model_with_autogptq`. Here is an example to run a quantized model : ```bash -SRAM_SLICER_SHARED_MME_INPUT_EXPANSION_ENABLED=false \ -ENABLE_EXPERIMENTAL_FLAGS=true python run_generation.py \ +python run_generation.py \ --attn_softmax_bf16 \ --model_name_or_path \ --use_hpu_graphs \ @@ -678,6 +728,36 @@ ENABLE_EXPERIMENTAL_FLAGS=true python run_generation.py \ --load_quantized_model_with_autogptq ``` +### Running with UINT4 weight quantization using AutoAWQ + +Llama2-7b supports UINT4 weight-only quantization through [AutoAWQ](https://github.com/casper-hansen/AutoAWQ), which offers quantization capabilities in PyTorch. +Currently, this support is limited to UINT4 inference of pre-quantized models only. + +Please run the following command to install AutoAWQ: +```bash +pip install triton==3.1.0 autoawq +``` + +You can run a *UINT4 weight quantized* model using AutoAWQ by including the argument `--load_quantized_model_with_autoawq`. + +Here is an example of how to run a quantized model : +```bash +python run_generation.py \ +--attn_softmax_bf16 \ +--model_name_or_path \ +--use_hpu_graphs \ +--limit_hpu_graphs \ +--use_kv_cache \ +--bucket_size 128 \ +--bucket_internal \ +--trim_logits \ +--max_new_tokens 128 \ +--batch_size 1 \ +--bf16 \ +--load_quantized_model_with_autoawq +``` + + ## Language Model Evaluation Harness The evaluation of LLMs can be done using the `lm_eval.py` script. It utilizes the [LM evaluation harness](https://github.com/EleutherAI/lm-evaluation-harness) diff --git a/examples/text-generation/quantization_config/maxabs_quant_arbitrary.json b/examples/text-generation/quantization_config/maxabs_quant_arbitrary.json new file mode 100644 index 0000000000..0de4d35533 --- /dev/null +++ b/examples/text-generation/quantization_config/maxabs_quant_arbitrary.json @@ -0,0 +1,7 @@ +{ + "method": "HOOKS", + "mode": "QUANTIZE", + "observer": "maxabs", + "scale_method": "maxabs_arbitrary", + "dump_stats_path": "./hqt_output/measure" +} \ No newline at end of file diff --git a/examples/text-generation/quantization_config/maxabs_quant_mixtral.json b/examples/text-generation/quantization_config/maxabs_quant_mixtral.json index 87dc52d08a..caaff8d09e 100644 --- a/examples/text-generation/quantization_config/maxabs_quant_mixtral.json +++ b/examples/text-generation/quantization_config/maxabs_quant_mixtral.json @@ -3,10 +3,7 @@ "mode": "QUANTIZE", "observer": "maxabs", "scale_method": "maxabs_hw", - "allowlist": {"types": [], "names": ["gate","w1","w3","w2"]}, - "blocklist": {"types": [], "names": [ - "model.layers.1.block_sparse_moe.experts.(3|4).w2", - "model.layers.[29-31].block_sparse_moe.experts.[0-7].w2" - ]}, + "allowlist": {"types": [], "names": []}, + "blocklist": {"types": [], "names": ["self_attn"]}, "dump_stats_path": "./hqt_output/measure" } \ No newline at end of file diff --git a/examples/text-generation/quantization_config/pow2_quant.json b/examples/text-generation/quantization_config/pow2_quant.json new file mode 100644 index 0000000000..e1f2eb1c6e --- /dev/null +++ b/examples/text-generation/quantization_config/pow2_quant.json @@ -0,0 +1,7 @@ +{ + "method": "HOOKS", + "mode": "QUANTIZE", + "observer": "maxabs", + "scale_method": "maxabs_pow2", + "dump_stats_path": "./hqt_output/measure" +} diff --git a/examples/text-generation/quantization_config/weight_opt_quant.json b/examples/text-generation/quantization_config/weight_opt_quant.json new file mode 100644 index 0000000000..1ec2dc6b6a --- /dev/null +++ b/examples/text-generation/quantization_config/weight_opt_quant.json @@ -0,0 +1,7 @@ +{ + "method": "HOOKS", + "mode": "QUANTIZE", + "observer": "maxabs", + "scale_method": "maxabs_hw_opt_weight", + "dump_stats_path": "./hqt_output/measure" +} diff --git a/examples/text-generation/quantization_tools/unify_measurements.py b/examples/text-generation/quantization_tools/unify_measurements.py index 4282e4ac49..de2b086c2a 100644 --- a/examples/text-generation/quantization_tools/unify_measurements.py +++ b/examples/text-generation/quantization_tools/unify_measurements.py @@ -6,49 +6,45 @@ import numpy as np -def find_measurement_path(measurement, measurements_dir_path, scales, group_size): +def find_measurement_path(measurement, measurements_dir_path, group_size): measurment_card = measurement + "_" + str(group_size) for measurment_file in os.listdir(measurements_dir_path): filename = os.fsdecode(measurment_file) if not filename.endswith(".json") or "_mod_list" in filename or measurment_card not in filename: continue - if scales: - if "MAXABS" in filename: - return os.path.join(measurements_dir_path, measurment_file) - else: - if "MAXABS" not in filename: - return os.path.join(measurements_dir_path, measurment_file) + if "MAXABS" not in filename: + return os.path.join(measurements_dir_path, measurment_file) -def unify_measurements( - measurement_group, measurements_dir_path, output_path, groups_size, groups_num, group_index, scales=False -): + +def unify_measurements(measurement_group, measurements_dir_path, output_path, groups_size, groups_num, group_index): measurements_paths = [] group_name = "" # save all the jsons paths in the given measurement group for measurement in measurement_group: - measurement_path = find_measurement_path(measurement, measurements_dir_path, scales, groups_size) - measurements_paths.append(measurement_path) + measurement_path = find_measurement_path(measurement, measurements_dir_path, groups_size) + if measurement_path is not None: + measurements_paths.append(measurement_path) group_name += measurement - # save all the jsons content in the given measurement group measurements_jsons = [] for measurement_path in measurements_paths: - with open(measurement_path, "r") as f: - js = json.load(f) - measurements_jsons.append(js["Nodes"]) + if measurement_path is not None: + with open(measurement_path, "r") as f: + js = json.load(f) + measurements_jsons.append(js["Nodes"]) # create a name for the unified json that will be created for this measurement group if groups_num == 1: unified_json_name = ( - find_measurement_path(measurement_group[0], measurements_dir_path, scales, groups_size) + find_measurement_path(measurement_group[0], measurements_dir_path, groups_size) .split("/")[-1] .replace("_" + measurement_group[0] + "_" + str(groups_size), "") ) else: unified_json_name = ( - find_measurement_path(measurement_group[0], measurements_dir_path, scales, groups_size) + find_measurement_path(measurement_group[0], measurements_dir_path, groups_size) .split("/")[-1] .replace( "_" + measurement_group[0] + "_" + str(groups_size), "_" + str(group_index) + "_" + str(groups_num) @@ -74,70 +70,27 @@ def unify_measurements( max_weight = node_values["params"]["weight"] # iterate over all the measurment group and take the maximum for each tensor and its channel - if scales: - for measurement_json in measurements_jsons: - for i in range(0, len(max_inputs)): - max_inputs[i] = max(measurement_json[node_name]["inputs"][i], max_inputs[i]) - if max_outputs is not None: - if isinstance(max_outputs[0], list): - for i in range(0, len(max_outputs)): - for j in range(0, len(max_outputs[i])): - max_outputs[i][j] = max( - measurement_json[node_name]["outputs"][i][j], max_outputs[i][j] - ) - else: - for i in range(0, len(max_outputs)): - max_outputs[i] = max(measurement_json[node_name]["outputs"][i], max_outputs[i]) - if max_weight is not None: - if isinstance(max_weight, dict): - for key, values in max_weight.items(): - for i in range(0, len(values)): - max_weight[key][i] = max( - measurement_json[node_name]["params"]["weight"][key][i], max_weight[key][i] - ) - else: - max_weight = max(measurement_json[node_name]["params"]["weight"], max_weight) - else: - for measurement_json in measurements_jsons: - for i in range(0, len(max_inputs)): - for j in range(0, len(max_inputs[i])): - max_inputs[i][j][0] = max(measurement_json[node_name]["inputs"][i][j][0], max_inputs[i][j][0]) - if max_outputs is not None: - for i in range(0, len(max_outputs)): - max_outputs[i][0] = max(measurement_json[node_name]["outputs"][i][0], max_outputs[i][0]) - if max_weight is not None: - for i in range(0, len(max_weight)): - max_weight[i][0] = max(measurement_json[node_name]["params"]["weight"][i][0], max_weight[i][0]) - - # update the maximum in the unified json - if scales: - for i in range(0, len(max_inputs)): - unified_json["Nodes"][node_name]["inputs"][i] = max_inputs[i] - if max_outputs is not None: - if isinstance(max_outputs[0], list): - for i in range(0, len(max_outputs)): - for j in range(0, len(max_outputs[i])): - unified_json["Nodes"][node_name]["outputs"][i][j] = max_outputs[i][j] - else: - for i in range(0, len(max_outputs)): - unified_json["Nodes"][node_name]["outputs"][i] = max_outputs[i] - if max_weight is not None: - if isinstance(max_weight, dict): - for key, values in max_weight.items(): - for i in range(0, len(values)): - unified_json["Nodes"][node_name]["params"]["weight"][key][i] = max_weight[key][i] - else: - unified_json["Nodes"][node_name]["params"]["weight"] = max_weight - else: + for measurement_json in measurements_jsons: for i in range(0, len(max_inputs)): for j in range(0, len(max_inputs[i])): - unified_json["Nodes"][node_name]["inputs"][i][j][0] = max_inputs[i][j][0] + max_inputs[i][j][0] = max(measurement_json[node_name]["inputs"][i][j][0], max_inputs[i][j][0]) if max_outputs is not None: for i in range(0, len(max_outputs)): - unified_json["Nodes"][node_name]["outputs"][i][0] = max_outputs[i][0] + max_outputs[i][0] = max(measurement_json[node_name]["outputs"][i][0], max_outputs[i][0]) if max_weight is not None: for i in range(0, len(max_weight)): - unified_json["Nodes"][node_name]["params"]["weight"][i][0] = max_weight[i][0] + max_weight[i][0] = max(measurement_json[node_name]["params"]["weight"][i][0], max_weight[i][0]) + + # update the maximum in the unified json + for i in range(0, len(max_inputs)): + for j in range(0, len(max_inputs[i])): + unified_json["Nodes"][node_name]["inputs"][i][j][0] = max_inputs[i][j][0] + if max_outputs is not None: + for i in range(0, len(max_outputs)): + unified_json["Nodes"][node_name]["outputs"][i][0] = max_outputs[i][0] + if max_weight is not None: + for i in range(0, len(max_weight)): + unified_json["Nodes"][node_name]["params"]["weight"][i][0] = max_weight[i][0] global_rank = None local_rank = group_index if groups_num != 1 else -1 mode = "" @@ -153,10 +106,10 @@ def unify_measurements( layers[layer] = {} layers[layer]["inputs"] = [np.array(x) for x in dlayer["inputs"]] if dlayer.get("outputs") is not None: - layers[layer]["outputs"] = np.array(dlayer["outputs"]) + layers[layer]["outputs"] = [np.array(x) for x in dlayer["outputs"]] if dlayer.get("params") is not None and dlayer["params"].get("weight") is not None: layers[layer]["params"] = {} - layers[layer]["params"]["weight"] = np.array(dlayer["params"]["weight"]) + layers[layer]["params"]["weight"] = [np.array(x) for x in dlayer["params"]["weight"]] df = {"GlobalRank": global_rank, "LocalRank": local_rank, "Mode": mode, "Nodes": layers} with open(unified_npz_path, "w"): np.savez(unified_npz_path, df) @@ -196,26 +149,14 @@ def main(args): groups = args.groups num_jsons_drange = 0 - num_jsons_scales = 0 for path in os.listdir(measurements_path): - if path.endswith(".json"): - if "MAXABS" in path: - num_jsons_scales += 1 - elif "mod_list" not in path: - num_jsons_drange += 1 - assert ( - os.path.isdir(measurements_path) - and (num_jsons_drange % len(groups)) == 0 - and (num_jsons_scales % len(groups)) == 0 - ) + if path.endswith(".json") and "MAXABS" not in path and "mod_list" not in path: + num_jsons_drange += 1 + + assert os.path.isdir(measurements_path) and (num_jsons_drange % len(groups)) == 0 for group_index, group in enumerate(groups): - unify_measurements( - group, measurements_path, output_path, num_jsons_drange, len(groups), group_index, scales=False - ) - unify_measurements( - group, measurements_path, output_path, num_jsons_scales, len(groups), group_index, scales=True - ) + unify_measurements(group, measurements_path, output_path, num_jsons_drange, len(groups), group_index) print("finished measurement unifier script") diff --git a/examples/text-generation/requirements_lm_eval.txt b/examples/text-generation/requirements_lm_eval.txt index e632dc1236..322cf204bf 100644 --- a/examples/text-generation/requirements_lm_eval.txt +++ b/examples/text-generation/requirements_lm_eval.txt @@ -1,2 +1 @@ -https://github.com/EleutherAI/lm-evaluation-harness/archive/0bf683b4e6a9df359b3156ba9ba8d62bdd47e0c0.zip -datasets==2.21.0 +https://github.com/EleutherAI/lm-evaluation-harness/archive/refs/tags/v0.4.7.zip diff --git a/examples/text-generation/run_generation.py b/examples/text-generation/run_generation.py index a20783511d..274aa591cd 100755 --- a/examples/text-generation/run_generation.py +++ b/examples/text-generation/run_generation.py @@ -226,6 +226,11 @@ def setup_parser(parser): action="store_true", help="Skip HPU Graph usage for first token to save memory", ) + parser.add_argument( + "--clear_hpu_graphs_cache", + action="store_true", + help="Clear HPU graphs cache", + ) parser.add_argument( "--show_graphs_count", action="store_true", @@ -320,6 +325,9 @@ def setup_parser(parser): action="store_true", help="Run the inference with dataset for specified --n_iterations(default:5)", ) + parser.add_argument( + "--sdp_on_bf16", action="store_true", help="Allow pyTorch to use reduced precision in the SDPA math backend" + ) quant_parser_group = parser.add_mutually_exclusive_group() quant_parser_group.add_argument( @@ -327,6 +335,11 @@ def setup_parser(parser): action="store_true", help="Load an AutoGPTQ quantized checkpoint using AutoGPTQ.", ) + quant_parser_group.add_argument( + "--load_quantized_model_with_autoawq", + action="store_true", + help="Load an AutoAWQ quantized checkpoint using AutoAWQ.", + ) quant_parser_group.add_argument( "--disk_offload", action="store_true", @@ -364,6 +377,8 @@ def setup_parser(parser): args.quant_config = os.getenv("QUANT_CONFIG", "") if args.quant_config and args.load_quantized_model_with_autogptq: raise RuntimeError("Setting both quant_config and load_quantized_model_with_autogptq is unsupported. ") + if args.quant_config and args.load_quantized_model_with_autoawq: + raise RuntimeError("Setting both quant_config and load_quantized_model_with_autoawq is unsupported. ") if args.quant_config == "" and args.disk_offload: logger.warning( @@ -395,6 +410,9 @@ def main(): import habana_frameworks.torch.hpu as torch_hpu + if args.sdp_on_bf16: + torch._C._set_math_sdp_allow_fp16_bf16_reduction(True) + if args.dataset_name is None: # Benchmark over the prompts below if args.prompt: @@ -526,7 +544,7 @@ def compute_valid_sequence_lengths_tensor(input_tokens): profiling_record_shapes=args.profiling_record_shapes, ).cpu() first_token_time = iteration_times[0] + encode_duration - logger.info(f"Time to first token = {first_token_time*1000}ms") + logger.info(f"Time to first token = {first_token_time * 1000}ms") return tokenizer.batch_decode(outputs, skip_special_tokens=True) from optimum.habana.utils import HabanaProfile @@ -541,10 +559,10 @@ def compute_valid_sequence_lengths_tensor(input_tokens): if dyn_prompt_lens is None or len(set(dyn_prompt_lens)) == 1: for i in range(args.warmup): if dyn_prompt_lens is None: - print(f"Warming up iteration {i+1}/{args.warmup}", flush=True) + print(f"Warming up iteration {i + 1}/{args.warmup}", flush=True) generate(None, args.reduce_recompile) else: - print(f"Warming up for shape {dyn_prompt_lens[0]} iteration {i+1}/{args.warmup}", flush=True) + print(f"Warming up for shape {dyn_prompt_lens[0]} iteration {i + 1}/{args.warmup}", flush=True) generate(dyn_prompt_lens[0], args.reduce_recompile) else: if args.bucket_size > 0: @@ -559,7 +577,7 @@ def rounder(x): for i in range(args.warmup): lst = list(range(min_prompt_len, max_sentence_len + 1, args.bucket_size)) for sz in lst: - print(f"Warming up for shape {sz - 1} iteration {i+1}/{args.warmup}", flush=True) + print(f"Warming up for shape {sz - 1} iteration {i + 1}/{args.warmup}", flush=True) generate(sz - 1, args.reduce_recompile) torch_hpu.synchronize() compilation_duration = time.perf_counter() - t0 @@ -586,12 +604,12 @@ def rounder(x): all_inputs = [] all_outputs = [] for i, input_sentence in enumerate(zip(input_sentences)): - print(f"input {i+1}: {input_sentence}") + print(f"input {i + 1}: {input_sentence}") all_inputs.append(input_sentence) for j, output in enumerate( zip(generated[args.num_return_sequences * i : args.num_return_sequences * (i + 1)]) ): - print(f"output {i+1}.{j+1}: {output}") + print(f"output {i + 1}.{j + 1}: {output}") all_outputs.append(output) print() @@ -719,22 +737,21 @@ def generate_dataset(batch): return prompt, outputs # warmup - if prompt_length > 0: - from optimum.habana.utils import HabanaProfile + from optimum.habana.utils import HabanaProfile - # compilation stage disable profiling - HabanaProfile.disable() - # Compilation - logger.info("Graph compilation...") - t0 = time.perf_counter() - for i, batch in enumerate(dataloader): - generate_dataset(batch) - # The first three iterations take longer because of graph compilation - if (i + 1) == 3: - break - torch_hpu.synchronize() - compilation_duration = time.perf_counter() - t0 - HabanaProfile.enable() + # compilation stage disable profiling + HabanaProfile.disable() + # Compilation + logger.info("Graph compilation...") + t0 = time.perf_counter() + for i, batch in enumerate(dataloader): + generate_dataset(batch) + # The first three iterations take longer because of graph compilation + if (i + 1) == 3: + break + torch_hpu.synchronize() + compilation_duration = time.perf_counter() - t0 + HabanaProfile.enable() total_new_tokens_generated = 0 duration = 0 @@ -747,10 +764,10 @@ def generate_dataset(batch): duration += time.perf_counter() - t0 total_new_tokens_generated += args.batch_size * args.max_new_tokens print(separator) - print(f"Batch nĀ°{i+1}") - print(f"Input: {prompt[:args.batch_size]}") + print(f"Batch nĀ°{i + 1}") + print(f"Input: {prompt[: args.batch_size]}") print( - f"Output: {tokenizer.batch_decode(outputs, skip_special_tokens=True)[:args.batch_size*args.num_return_sequences]}" + f"Output: {tokenizer.batch_decode(outputs, skip_special_tokens=True)[: args.batch_size * args.num_return_sequences]}" ) print(separator) if args.run_partial_dataset and args.n_iterations == i + 1: @@ -770,8 +787,7 @@ def generate_dataset(batch): mem = get_hpu_memory_stats() for k, v in mem.items(): print("{:35} = {} GB".format(k[:-5].replace("_", " ").capitalize(), v)) - if prompt_length > 0: - print(f"Graph compilation duration = {compilation_duration} seconds") + print(f"Graph compilation duration = {compilation_duration} seconds") print(separator) if args.quant_config: finalize_quantization(model) diff --git a/examples/text-generation/run_lm_eval.py b/examples/text-generation/run_lm_eval.py index 689860fc7c..cb3ecd80f0 100644 --- a/examples/text-generation/run_lm_eval.py +++ b/examples/text-generation/run_lm_eval.py @@ -14,32 +14,33 @@ # limitations under the License. ############################################################################### -# Copyright (C) 2020-2021 Habana Labs, Ltd. an Intel Company +# Copyright (C) 2020-2025 Habana Labs, Ltd. an Intel Company ############################################################################### import argparse import json -import logging import multiprocessing as mp import os import time +from typing import Literal, Optional -import lm_eval.evaluator -import lm_eval.tasks import psutil import torch import torch.nn.functional as F +from lm_eval import evaluator, utils +from lm_eval.models.huggingface import HFLM, TemplateLM # Local imports from run_generation import setup_parser +from transformers import AutoModelForCausalLM, AutoTokenizer +from transformers.generation import GenerationConfig from utils import finalize_quantization, initialize_model from optimum.habana.utils import get_hpu_memory_stats os.environ.setdefault("TOKENIZERS_PARALLELISM", "false") -logger = logging.getLogger(__name__) - +logger = utils.eval_logger # This hack is a workaround to limitations of lm_eval which always allocates # mp.Pool with max cpu count which explodes on multinode scenarios and for hpu @@ -52,9 +53,7 @@ def LimitedSpawnPool(_): physical_cpu_count = psutil.cpu_count(logical=False) pool_size = physical_cpu_count world_size = int(os.getenv("WORLD_SIZE", 1)) - if world_size == 0: - world_size = 1 - pool_size //= world_size + pool_size //= max(world_size, 1) if (pool_size * world_size) != physical_cpu_count: pool_size -= 1 return spawn_context.Pool(pool_size) @@ -86,22 +85,52 @@ def setup_lm_eval_parser(): default=["hellaswag", "lambada_openai", "piqa", "winogrande"], ) parser.add_argument("--limit_iters", type=int, help="limit examples to run that many iterations", default=None) + parser.add_argument( + "--show_config", + action="store_true", + default=False, + help="If True, shows the the full config of all tasks at the end of the evaluation.", + ) + parser.add_argument("--max_graphs", type=int, help="Maximum number of HPU graphs", default=None) args = setup_parser(parser) return args -class HabanaModelAdapter(lm_eval.base.BaseLM): - def __init__(self, tokenizer, model, args, options): - super().__init__() +class HabanaModelAdapter(HFLM): + def __init__( + self, + tokenizer: AutoTokenizer, + model: AutoModelForCausalLM, + args: argparse.Namespace, + options: GenerationConfig, + backend: Literal["default", "causal", "seq2seq"] = "default", + logits_cache: bool = True, + add_bos_token: Optional[bool] = False, + delta: Optional[str] = None, + **kwargs, + ) -> None: + # To skip cuda code of the HFLM init + TemplateLM.__init__(self) self.tokenizer = tokenizer - self.model = model + self._model = model + self._config = self._model.config self._batch_size = args.batch_size - self.buckets = sorted(args.buckets) + self.buckets: list[int] = sorted(args.buckets) self.options = options - self._device = args.device + self.device_ = args.device + self.pretrained = model + self.peft = args.peft_model + self.delta = delta + # determine which of 'causal' and 'seq2seq' backends to use for HF models + self._get_backend(config=self._config, backend=backend, trust_remote_code=args.trust_remote_code) + self.logits_cache = logits_cache + self.add_bos_token = add_bos_token + self._max_length = options.max_length + self.batch_size_per_gpu = int(args.batch_size) + self.revision = args.model_revision self.model_inputs = {"use_cache": self.options.use_cache} - if self.model.config.model_type in [ + if self._model.config.model_type in [ "llama", "mistral", "falcon", @@ -118,8 +147,18 @@ def __init__(self, tokenizer, model, args, options): "reuse_cache": self.options.reuse_cache, } ) - if self.model.config.model_type in ["llama", "mistral", "qwen2", "falcon", "starcoder2", "gemma", "baichuan"]: - if self.model.config.model_type != "falcon": + + if self.model.config.model_type in [ + "llama", + "mistral", + "qwen2", + "falcon", + "starcoder2", + "gemma", + "baichuan", + "gpt_bigcode", + ]: + if self.model.config.model_type not in ["falcon", "gpt_bigcode"]: self.model_inputs.update( { "attn_softmax_bf16": self.options.attn_softmax_bf16, @@ -132,59 +171,43 @@ def __init__(self, tokenizer, model, args, options): "flash_attention_causal_mask": self.options.flash_attention_causal_mask, } ) + if self.model.config.model_type in ["llama", "qwen2", "baichuan", "gpt_bigcode"]: + self.model_inputs.update({"flash_attention_fast_softmax": self.options.flash_attention_fast_softmax}) if args.warmup: self.warm_up() - def warm_up(self): + def warm_up(self) -> None: for bucket_size in reversed(self.buckets): inps = torch.ones((self._batch_size, bucket_size), dtype=torch.int64) self._model_call(inps) - pass @property - def eot_token_id(self): - return self.model.config.eos_token_id + def eot_token_id(self) -> int: + return self._model.config.eos_token_id @property - def max_length(self): + def max_length(self) -> int: return self.buckets[-1] - @property - def max_gen_toks(self): - raise NotImplementedError() - - @property - def batch_size(self): - return self._batch_size - @property def device(self): # We need to do padding ourselves, otherwise we'll end up with recompilations # Returning 'cpu' to keep tensors on CPU in lm_eval code return "cpu" - def tok_encode(self, string): - return self.tokenizer.encode(string) - - def tok_decode(self, tokens): - return self.tokenizer.decode(tokens) - - def _model_generate(self, context, max_length, eos_token_id): - raise NotImplementedError() - - def find_bucket(self, length): + def find_bucket(self, length: int) -> list[int]: return [b for b in self.buckets if b >= length][0] - def _model_call(self, inps): + def _model_call(self, inps: torch.Tensor) -> torch.Tensor: bs, seq_length = inps.shape padding_length = 0 if self.options.static_shapes: bucket_length = self.find_bucket(seq_length) if self.options.use_cache and self.options.reuse_cache: - self.model.allocate_kv_cache(bs, bucket_length + 1, bucket_length) + self._model.allocate_kv_cache(bs, bucket_length + 1, bucket_length) padding_length = bucket_length - seq_length - inps = F.pad(inps, (0, padding_length), value=self.model.config.pad_token_id) - logits = self.model(inps.to(self._device), **self.model_inputs)["logits"].cpu() + inps = F.pad(inps, (0, padding_length), value=self._model.config.pad_token_id) + logits = self._model(inps.to(self.device_), **self.model_inputs)["logits"].cpu() if self.options.static_shapes and padding_length > 0: logits = logits[:, :-padding_length, :] @@ -192,25 +215,22 @@ def _model_call(self, inps): return logits -def main(): +def main() -> None: + # Modified based on cli_evaluate function in https://github.com/EleutherAI/lm-evaluation-harness/blob/v0.4.7/lm_eval/__main__.py/#L268 args = setup_lm_eval_parser() model, _, tokenizer, generation_config = initialize_model(args, logger) - if args.trust_remote_code: # trust_remote_code fix was introduced in lm_eval 0.4.3 - # https://github.com/EleutherAI/lm-evaluation-harness/pull/1998/files - # We need to cherry-pick the fix manually untill we upgrade (SW-190418) import datasets datasets.config.HF_DATASETS_TRUST_REMOTE_CODE = True - lm_tasks = lm_eval.tasks.get_task_dict(args.tasks) with torch.no_grad(): lm = HabanaModelAdapter(tokenizer, model, args, generation_config) eval_start = time.perf_counter() with torch.no_grad(): - results = lm_eval.evaluator.evaluate(lm, lm_tasks, limit=args.limit_iters) + results = evaluator.simple_evaluate(lm, tasks=args.tasks, limit=args.limit_iters) if args.device == "hpu": import habana_frameworks.torch.hpu as torch_hpu @@ -225,7 +245,13 @@ def main(): mem = get_hpu_memory_stats() for k, v in mem.items(): print("{:35} = {} GB".format(k[:-5].replace("_", " ").capitalize(), v)) - json.dump(results, open(args.output_file, "w"), indent=2) + + json_str = json.dumps(results, indent=2, default=utils.handle_non_serializable, ensure_ascii=False) + with open(args.output_file, "w", encoding="utf-8") as f: + f.write(json_str) + if args.show_config: + print(json_str) + if args.quant_config: finalize_quantization(model) diff --git a/examples/text-generation/text-generation-pipeline/README.md b/examples/text-generation/text-generation-pipeline/README.md index a10792be2a..2aa036ec3a 100644 --- a/examples/text-generation/text-generation-pipeline/README.md +++ b/examples/text-generation/text-generation-pipeline/README.md @@ -22,7 +22,7 @@ The text-generation pipeline can be used to perform text-generation by providing If you plan to use [DeepSpeed-inference](https://docs.habana.ai/en/latest/PyTorch/DeepSpeed/Inference_Using_DeepSpeed.html), you should install DeepSpeed as follows: ```bash -pip install git+https://github.com/HabanaAI/DeepSpeed.git@1.18.0 +pip install git+https://github.com/HabanaAI/DeepSpeed.git@1.19.0 ``` If you would like to use the pipeline with LangChain classes, you can install LangChain as follows: diff --git a/examples/text-generation/text-generation-pipeline/run_pipeline.py b/examples/text-generation/text-generation-pipeline/run_pipeline.py index 43aea65cec..11e542d7a5 100644 --- a/examples/text-generation/text-generation-pipeline/run_pipeline.py +++ b/examples/text-generation/text-generation-pipeline/run_pipeline.py @@ -45,14 +45,14 @@ def main(): duration = 0 for iteration in range(args.n_iterations): - logger.info(f"Running inference iteration {iteration+1}...") + logger.info(f"Running inference iteration {iteration + 1}...") t0 = time.perf_counter() output = pipe(input_sentences) duration += time.perf_counter() - t0 for i, (input_sentence, generated_text) in enumerate(zip(input_sentences, output)): - print(f"Prompt[{iteration+1}][{i+1}]: {input_sentence}") - print(f"Generated Text[{iteration+1}][{i+1}]: {repr(generated_text)}\n") + print(f"Prompt[{iteration + 1}][{i + 1}]: {input_sentence}") + print(f"Generated Text[{iteration + 1}][{i + 1}]: {repr(generated_text)}\n") throughput = args.n_iterations * args.batch_size * args.max_new_tokens / duration print(f"Inference Duration (for {args.n_iterations} iterations): {duration} seconds") diff --git a/examples/text-generation/text-generation-pipeline/run_pipeline_langchain.py b/examples/text-generation/text-generation-pipeline/run_pipeline_langchain.py index 556494cd37..6212e808aa 100644 --- a/examples/text-generation/text-generation-pipeline/run_pipeline_langchain.py +++ b/examples/text-generation/text-generation-pipeline/run_pipeline_langchain.py @@ -87,8 +87,8 @@ def main(): duration += time.perf_counter() - t0 for i, (question, answer) in enumerate(zip(input_questions, responses)): - print(f"Question[{iteration+1}][{i+1}]: {question['question']}") - print(f"Response[{iteration+1}][{i+1}]: {answer}\n") + print(f"Question[{iteration + 1}][{i + 1}]: {question['question']}") + print(f"Response[{iteration + 1}][{i + 1}]: {answer}\n") throughput = args.n_iterations * args.batch_size * args.max_new_tokens / duration print(f"Inference Duration (for {args.n_iterations} iterations): {duration} seconds") diff --git a/examples/text-generation/utils.py b/examples/text-generation/utils.py index 637318b7be..5ef9c5c858 100644 --- a/examples/text-generation/utils.py +++ b/examples/text-generation/utils.py @@ -252,13 +252,14 @@ def setup_model(args, model_dtype, model_kwargs, logger): model = AutoModelForCausalLM.from_pretrained( args.model_name_or_path, torch_dtype=model_dtype, quantization_config=quantization_config, **model_kwargs ) - elif args.load_quantized_model_with_inc: - # TODO: This will be removed in v1.19 Synapse release - # Override neural_compressor _load_remaining_pretrained_weight for the Transformer 4.45 release. - import neural_compressor.torch.algorithms.weight_only.save_load as nc_sl - - nc_sl.WOQModelLoader._load_remaining_pretrained_weight = local_load_remaining_pretrained_weight + elif args.load_quantized_model_with_autoawq: + from transformers import AwqConfig + quantization_config = AwqConfig(bits=4, version="hpu") + model = AutoModelForCausalLM.from_pretrained( + args.model_name_or_path, torch_dtype=model_dtype, quantization_config=quantization_config, **model_kwargs + ) + elif args.load_quantized_model_with_inc: from neural_compressor.torch.quantization import load model = load(model_name_or_path=args.model_name_or_path, format="huggingface", device="hpu", **model_kwargs) @@ -277,9 +278,6 @@ def setup_model(args, model_dtype, model_kwargs, logger): original_model=org_model, **model_kwargs, ) - # TODO: This will be removed in v1.19 Synapse release - # the loaded model should have the same dtype as original_model - model = model.to(model_kwargs["torch_dtype"]) else: if args.assistant_model is not None: assistant_model = AutoModelForCausalLM.from_pretrained( @@ -306,7 +304,8 @@ def setup_model(args, model_dtype, model_kwargs, logger): if check_habana_frameworks_version("1.13.0") and model.config.model_type == "falcon": model = wrap_in_hpu_graph(model, hash_with_views=False) else: - model = wrap_in_hpu_graph(model) + max_graphs = getattr(args, "max_graphs", None) + model = wrap_in_hpu_graph(model, max_graphs=max_graphs) if args.assistant_model is not None: assistant_model = wrap_in_hpu_graph(assistant_model) if _is_peft_model(model): @@ -442,7 +441,8 @@ def setup_distributed_model(args, model_dtype, model_kwargs, logger): # Construct model with fake meta tensors, later will be replaced on devices during ds-inference ckpt load with deepspeed.OnDevice(dtype=model_dtype, device="meta"): if ( - config.rope_scaling + hasattr(config, "rope_scaling") + and config.rope_scaling and config.rope_scaling["rope_type"] == "llama3" and config.max_position_embeddings > 8192 ): @@ -618,6 +618,12 @@ def setup_tokenizer(args, model, assistant_model, logger): ) model.generation_config.eos_token_id = model.generation_config.eos_token_id[-1] + if model.config.model_type == "mpt": + if tokenizer.pad_token is None: + tokenizer.pad_token = tokenizer.eos_token + if model.generation_config.pad_token_id is None: + model.generation_config.pad_token_id = tokenizer.eos_token_id + # Some models like GPT2 do not have a PAD token so we have to set it if necessary if tokenizer.pad_token is None: tokenizer.pad_token = tokenizer.eos_token @@ -655,6 +661,7 @@ def setup_generation_config(args, model, assistant_model, tokenizer): generation_config.trim_logits = args.trim_logits generation_config.attn_softmax_bf16 = args.attn_softmax_bf16 generation_config.limit_hpu_graphs = args.limit_hpu_graphs + generation_config.clear_hpu_graphs_cache = args.clear_hpu_graphs_cache generation_config.reuse_cache = args.reuse_cache generation_config.reduce_recompile = args.reduce_recompile if generation_config.reduce_recompile: @@ -742,47 +749,3 @@ def initialize_model(args, logger): logger.info(f"device: {args.device}, n_hpu: {args.world_size}, bf16: {model_dtype == torch.bfloat16}") logger.info(f"Model initialization took {(init_end - init_start):.3f}s") return model, assistant_model, tokenizer, generation_config - - -# TODO:This will be removed from Synapse v1.19 release. -# This is to override _load_remaining_pretrained_weight for Transformer 4.45 release. -def local_load_remaining_pretrained_weight(self, model): - from transformers.modeling_utils import _load_state_dict_into_meta_model, load_state_dict - - resolved_archive_file = self.kwargs.pop("resolved_archive_file", None) - torch_dtype = self.kwargs.pop("torch_dtype", torch.float32) - dtype_orig = self.kwargs.pop("dtype_orig", None) - offload_folder = self.kwargs.pop("offload_folder", None) - offload_state_dict = self.kwargs.pop("offload_state_dict", False) - - # restore default dtype - if dtype_orig is not None: - torch.set_default_dtype(dtype_orig) - - if not isinstance(resolved_archive_file, list): - resolved_archive_file = [resolved_archive_file] - for shard_file in resolved_archive_file: - state_dict = load_state_dict(shard_file) - - params_dict = { - "model": model, - "state_dict": state_dict, - "start_prefix": "", - "expected_keys": self.loaded_state_dict_keys, - "device_map": {"": self.device}, - "offload_folder": offload_folder, - "state_dict_folder": tempfile.mkdtemp() if offload_state_dict else None, - "state_dict_index": {} if offload_state_dict else None, - "dtype": torch_dtype, - "keep_in_fp32_modules": [], - } - - _load_state_dict_into_meta_model(**params_dict) - - # make sure token embedding weights are still tied if needed - model.tie_weights() - - # Set model in evaluation mode to deactivate DropOut modules by default - model.eval() - - return model diff --git a/examples/text-to-speech/README.md b/examples/text-to-speech/README.md index a1e089f55e..21070d275f 100644 --- a/examples/text-to-speech/README.md +++ b/examples/text-to-speech/README.md @@ -33,8 +33,4 @@ python3 run_pipeline.py \ --text "Hello, my dog is cooler than you!" \ --use_hpu_graphs \ --bf16 -``` -Models that have been validated: - - [microsoft/speecht5_tts](https://huggingface.co/microsoft/speecht5_tts) - - [facebook/hf-seamless-m4t-medium](https://huggingface.co/facebook/hf-seamless-m4t-medium) - - [facebook/mms-tts-eng](https://huggingface.co/facebook/mms-tts-eng) +``` \ No newline at end of file diff --git a/examples/text-to-speech/run_pipeline.py b/examples/text-to-speech/run_pipeline.py index 1d9b53de7d..81546b0cb9 100644 --- a/examples/text-to-speech/run_pipeline.py +++ b/examples/text-to-speech/run_pipeline.py @@ -129,7 +129,7 @@ def main(): text, batch_size=args.batch_size, forward_params=forward_params, generate_kwargs=generate_kwargs ) end = time.time() - logger.info(f"speech = {speech} time = {(end-start) * 1000 / args.n_iterations }ms") + logger.info(f"speech = {speech} time = {(end - start) * 1000 / args.n_iterations}ms") sf.write("speech.wav", speech[0]["audio"].squeeze(), samplerate=speech[0]["sampling_rate"]) diff --git a/examples/text-to-video/requirements.txt b/examples/text-to-video/requirements.txt deleted file mode 100644 index 6ab6d0d570..0000000000 --- a/examples/text-to-video/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -opencv-python-headless diff --git a/examples/translation/run_translation.py b/examples/translation/run_translation.py index c2def132a7..6f55ae1350 100644 --- a/examples/translation/run_translation.py +++ b/examples/translation/run_translation.py @@ -63,7 +63,7 @@ def check_optimum_habana_min_version(*a, **b): # Will error if the minimal version of Transformers and Optimum Habana are not installed. Remove at your own risks. check_min_version("4.45.0") -check_optimum_habana_min_version("1.14.0.dev0") +check_optimum_habana_min_version("1.16.0.dev0") require_version("datasets>=1.8.0", "To fix: pip install -r examples/pytorch/translation/requirements.txt") diff --git a/examples/trl/README.md b/examples/trl/README.md index 750fc82b08..5e488e7072 100644 --- a/examples/trl/README.md +++ b/examples/trl/README.md @@ -39,14 +39,15 @@ $ pip install -U -r requirements.txt --lora_dropout=0.05 \ --lora_target_modules "q_proj" "v_proj" "k_proj" "o_proj" \ --max_seq_length 512 \ - --adam_epsilon 1e-08 + --adam_epsilon 1e-08 \ + --use_flash_attention ``` -2. Supervised fine-tuning of the mistralai/Mixtral-8x7B-v0.1 on 4 cards: +2. Supervised fine-tuning of the mistralai/Mixtral-8x7B-Instruct-v0.1 on 4 cards: ``` DEEPSPEED_HPU_ZERO3_SYNC_MARK_STEP_REQUIRED=1 python ../gaudi_spawn.py --world_size 4 --use_deepspeed sft.py \ - --model_name_or_path mistralai/Mixtral-8x7B-v0.1 \ + --model_name_or_path mistralai/Mixtral-8x7B-Instruct-v0.1 \ --dataset_name "philschmid/dolly-15k-oai-style" \ --subset 'data/' \ --streaming False \ @@ -78,103 +79,10 @@ $ pip install -U -r requirements.txt ### Training -#### For meta-llama/Llama-2-7b-hf - -The following example is for the creation of StackLlaMa 2: a Stack exchange llama-v2-7b model. -There are two main steps to the DPO training process: -1. Supervised fine-tuning of the base llama-v2-7b model to create llama-v2-7b-se: - - ``` - python ../gaudi_spawn.py --world_size 8 --use_mpi sft.py \ - --model_name_or_path meta-llama/Llama-2-7b-hf \ - --dataset_name "lvwerra/stack-exchange-paired" \ - --output_dir="./sft" \ - --max_steps=500 \ - --logging_steps=10 \ - --save_steps=100 \ - --do_train \ - --per_device_train_batch_size=4 \ - --per_device_eval_batch_size=1 \ - --gradient_accumulation_steps=2 \ - --learning_rate=1e-4 \ - --lr_scheduler_type="cosine" \ - --warmup_steps=100 \ - --weight_decay=0.05 \ - --optim="paged_adamw_32bit" \ - --lora_target_modules "q_proj" "v_proj" \ - --bf16 \ - --remove_unused_columns=False \ - --run_name="sft_llama2" \ - --report_to=none \ - --use_habana \ - --use_lazy_mode - ``` - To merge the adaptors to get the final sft merged checkpoint, we can use the `merge_peft_adapter.py` helper script that comes with TRL: - ``` - python merge_peft_adapter.py --base_model_name="meta-llama/Llama-2-7b-hf" --adapter_model_name="sft" --output_name="sft/final_merged_checkpoint" - ``` - -2. Run the DPO trainer using the model saved by the previous step: - ``` - python ../gaudi_spawn.py --world_size 8 --use_mpi dpo.py \ - --model_name_or_path="sft/final_merged_checkpoint" \ - --tokenizer_name_or_path=meta-llama/Llama-2-7b-hf \ - --lora_target_modules "q_proj" "v_proj" "k_proj" "out_proj" "fc_in" "fc_out" "wte" \ - --output_dir="dpo" \ - --report_to=none - ``` - -#### mistralai/Mistral-7B-v0.1 - -1. Supervised fine-tuning of the base Mistral-7B-v0.1 model: - - ``` - DEEPSPEED_HPU_ZERO3_SYNC_MARK_STEP_REQUIRED=1 python ../gaudi_spawn.py --world_size 8 --use_deepspeed sft.py \ - --model_name_or_path mistralai/Mistral-7B-v0.1 \ - --dataset_name "lvwerra/stack-exchange-paired" \ - --deepspeed ../language-modeling/llama2_ds_zero3_config.json \ - --output_dir="./sft" \ - --do_train \ - --max_steps=500 \ - --logging_steps=10 \ - --save_steps=100 \ - --per_device_train_batch_size=1 \ - --per_device_eval_batch_size=1 \ - --gradient_accumulation_steps=2 \ - --learning_rate=1e-4 \ - --lr_scheduler_type="cosine" \ - --warmup_steps=100 \ - --weight_decay=0.05 \ - --optim="paged_adamw_32bit" \ - --lora_target_modules "q_proj" "v_proj" \ - --bf16 \ - --remove_unused_columns=False \ - --run_name="sft_mistral" \ - --report_to=none \ - --use_habana \ - --use_lazy_mode - ``` - To merge the adaptors to get the final sft merged checkpoint, we can use the `merge_peft_adapter.py` helper script that comes with TRL: - - ``` - python merge_peft_adapter.py --base_model_name="mistralai/Mistral-7B-v0.1" --adapter_model_name="sft" --output_name="sft/final_merged_checkpoint" - ``` - -2. Run the DPO trainer using the model saved by the previous step: - ``` - DEEPSPEED_HPU_ZERO3_SYNC_MARK_STEP_REQUIRED=1 python ../gaudi_spawn.py --world_size 8 --use_deepspeed dpo.py \ - --model_name_or_path="sft/final_merged_checkpoint" \ - --tokenizer_name_or_path=mistralai/Mistral-7B-v0.1 \ - --deepspeed ../language-modeling/llama2_ds_zero3_config.json \ - --lora_target_modules "q_proj" "v_proj" "k_proj" "out_proj" "fc_in" "fc_out" "wte" \ - --output_dir="dpo" \ - --max_prompt_length=256 \ - --max_length=512 \ - --report_to=none - ``` - #### For meta-llama/Llama-2-70b-hf +The following example is for the creation of StackLlaMa 2: a Stack exchange llama-v2-70b model. There are two main steps to the DPO training process. + For large model like Llama2-70B, we could use DeepSpeed Zero-3 to enable DPO training in multi-card. steps like: 1. Supervised fine-tuning of the base llama-v2-70b model to create llama-v2-70b-se: @@ -359,8 +267,12 @@ python ddpo.py \ --use_hpu_graphs \ --bf16 \ --hf_hub_model_id="ddpo-finetuned-stable-diffusion" \ - --push_to_hub False + --push_to_hub False \ + --sdp_on_bf16 ``` +> [!NOTE] +> Due to a known issue on Gaudi3, sample_batch_sizes should be changed to 3. The issue will be fixed in the future release. + 2. Inference using the fine-tuned LoRA weights as shown in the example below: ```python diff --git a/examples/trl/ddpo.py b/examples/trl/ddpo.py index 46caf64c49..a2f1f15733 100644 --- a/examples/trl/ddpo.py +++ b/examples/trl/ddpo.py @@ -79,6 +79,9 @@ class ScriptArguments: push_to_hub: bool = field(default=False, metadata={"help": "Whether or not to push the model to the Hub."}) use_habana: bool = field(default=True, metadata={"help": "Whether or not to use HPU."}) use_hpu_graphs: bool = field(default=True, metadata={"help": "Whether or not to use hpu graphs."}) + sdp_on_bf16: bool = field( + default=False, metadata={"help": "Allow pyTorch to use reduced precision in the SDPA math backend."} + ) class MLP(nn.Module): @@ -225,6 +228,7 @@ def image_outputs_logger(image_data, global_step, accelerate_logger): use_habana=args.use_habana, use_hpu_graphs=args.use_hpu_graphs, gaudi_config=gaudi_config, + sdp_on_bf16=args.sdp_on_bf16, ) trainer = GaudiDDPOTrainer( diff --git a/examples/video-classification/run_example.py b/examples/video-classification/run_example.py index b593fb5955..2f78883742 100644 --- a/examples/video-classification/run_example.py +++ b/examples/video-classification/run_example.py @@ -80,7 +80,10 @@ def run( warm_up_epcohs: int, use_hpu_graphs: bool, cast_bf16: bool, + sdp_on_bf16: bool, ): + if sdp_on_bf16: + torch._C._set_math_sdp_allow_fp16_bf16_reduction(True) processor = VideoMAEImageProcessor.from_pretrained(model_name) device = torch.device("hpu") model = VideoMAEForVideoClassification.from_pretrained(model_name) @@ -152,6 +155,9 @@ def main(): action="store_true", help="Whether to perform in bf16 precision.", ) + parser.add_argument( + "--sdp_on_bf16", action="store_true", help="Allow pyTorch to use reduced precision in the SDPA math backend" + ) parser.add_argument( "--log_level", default=None, @@ -176,6 +182,7 @@ def main(): args.warm_up_epochs, args.use_hpu_graphs, args.bf16, + args.sdp_on_bf16, ) diff --git a/examples/text-to-video/README.md b/examples/video-comprehension/README.md similarity index 55% rename from examples/text-to-video/README.md rename to examples/video-comprehension/README.md index 1df4e44e59..da54f26740 100644 --- a/examples/text-to-video/README.md +++ b/examples/video-comprehension/README.md @@ -1,12 +1,9 @@ -# Text to Video Examples - -This directory contains a script that showcases how to use the `GaudiTextToVideoSDPipeline` to run text-to-video generation tasks on HPUs. - -## Requirements +# Examples -First, you should install the requirements: - -```bash -pip install -r requirements.txt -``` +This directory contains example scripts that demonstrate how to perform video comprehension on Gaudi with graph mode. ## Single-HPU inference +### Video-LLaVA Model + ```bash -python3 text_to_video_generation.py \ - --model_name_or_path ali-vilab/text-to-video-ms-1.7b \ - --prompts "An astronaut riding a horse" \ - --use_habana \ +python3 run_example.py \ + --model_name_or_path "LanguageBind/Video-LLaVA-7B-hf" \ + --warmup 3 \ + --n_iterations 5 \ + --batch_size 1 \ --use_hpu_graphs \ - --dtype bf16 + --bf16 \ + --output_dir ./ ``` - Models that have been validated: - - [ali-vilab/text-to-video-ms-1.7b](https://huggingface.co/ali-vilab/text-to-video-ms-1.7b) + - [LanguageBind/Video-LLaVA-7B-hf ](https://huggingface.co/LanguageBind/Video-LLaVA-7B-hf) diff --git a/examples/video-comprehension/requirements.txt b/examples/video-comprehension/requirements.txt new file mode 100644 index 0000000000..7ed65352d9 --- /dev/null +++ b/examples/video-comprehension/requirements.txt @@ -0,0 +1,2 @@ +av == 12.1.0 +sentencepiece == 0.2.0 diff --git a/examples/video-comprehension/run_example.py b/examples/video-comprehension/run_example.py new file mode 100644 index 0000000000..5868bea3e8 --- /dev/null +++ b/examples/video-comprehension/run_example.py @@ -0,0 +1,235 @@ +#!/usr/bin/env python +# coding=utf-8 +# Copyright 2021 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and + +import argparse +import json +import logging +import os +import time +from pathlib import Path + +import av +import numpy as np +import torch +from huggingface_hub import hf_hub_download +from transformers import VideoLlavaProcessor + +from optimum.habana.transformers.modeling_utils import ( + GaudiVideoLlavaForConditionalGeneration, + adapt_transformers_to_gaudi, +) + + +logging.basicConfig( + format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", + datefmt="%m/%d/%Y %H:%M:%S", + level=logging.INFO, +) +logger = logging.getLogger(__name__) + + +def read_video_pyav(container, indices): + frames = [] + container.seek(0) + start_index = indices[0] + end_index = indices[-1] + for i, frame in enumerate(container.decode(video=0)): + if i > end_index: + break + if i >= start_index and i in indices: + frames.append(frame) + return np.stack([x.to_ndarray(format="rgb24") for x in frames]) + + +def main(): + parser = argparse.ArgumentParser() + + parser.add_argument( + "--model_name_or_path", + default=None, + type=str, + help="Path to pre-trained model", + ) + parser.add_argument( + "--video_path", + default=None, + type=str, + nargs="*", + help='Path to video as input. Can be a single string (eg: --image_path "URL1"), or a list of space-separated strings (eg: --video_path "URL1" "URL2")', + ) + parser.add_argument( + "--prompt", + default=None, + type=str, + help='Optional argument to give a prompt of your choice as input. is a single string (eg: --prompt "Hello world")', + ) + parser.add_argument( + "--use_hpu_graphs", + action="store_true", + help="Whether to use HPU graphs or not. Using HPU graphs should give better latencies.", + ) + parser.add_argument("--max_new_tokens", type=int, default=100, help="Number of tokens to generate.") + parser.add_argument( + "--bf16", + action="store_true", + help="Whether to perform generation in bf16 precision.", + ) + parser.add_argument( + "--output_dir", + default=None, + type=str, + help="Output directory to store results in.", + ) + parser.add_argument( + "--token", + default=None, + type=str, + help="The token to use as HTTP bearer authorization for remote files. If not specified, will use the token " + "generated when running `huggingface-cli login` (stored in `~/.huggingface`).", + ) + parser.add_argument("--batch_size", type=int, default=1, help="Input batch size.") + parser.add_argument("--warmup", type=int, default=3, help="Number of warmup iterations for benchmarking.") + parser.add_argument("--n_iterations", type=int, default=5, help="Number of inference iterations for benchmarking.") + parser.add_argument( + "--ignore_eos", + action="store_true", + help="Whether to disable stopping with eos token when calling `generate`.", + ) + parser.add_argument( + "--use_flash_attention", + action="store_true", + help="Whether to enable Habana Flash Attention, provided that the model supports it.", + ) + parser.add_argument( + "--flash_attention_recompute", + action="store_true", + help="Whether to enable Habana Flash Attention in recompute mode on first token generation. This gives an opportunity of splitting graph internally which helps reduce memory consumption.", + ) + + args = parser.parse_args() + + os.environ.setdefault("EXPERIMENTAL_WEIGHT_SHARING", "FALSE") + + if args.video_path is None: + args.video_path = [ + hf_hub_download( + repo_id="raushan-testing-hf/videos-test", filename="sample_demo_1.mp4", repo_type="dataset" + ) + ] + + if args.prompt is None: + args.prompt = ["USER: