added workflow

.github/workflows/check_quality.yaml

@@ -21,7 +21,7 @@ jobs:
       - name: Set up Python 3.10
         uses: actions/setup-python@v3
         with:
-          python-version: '3.10'
+          python-version: "3.10"
 
       - name: Install quality requirements
         run: |

.github/workflows/test_api_cpu.yaml

@@ -21,7 +21,7 @@ jobs:
       - name: Set up Python 3.10
         uses: actions/setup-python@v3
         with:
-          python-version: '3.10'
+          python-version: "3.10"
 
       - name: Install dependencies
         run: |

.github/workflows/test_api_misc.yaml

@@ -21,7 +21,7 @@ jobs:
       - name: Set up Python 3.10
         uses: actions/setup-python@v3
         with:
-          python-version: '3.10'
+          python-version: "3.10"
 
       - name: Install requirements
         run: |

.github/workflows/test_cli_cpu_neural_compressor.yaml

@@ -21,12 +21,12 @@ jobs:
       - name: Set up Python 3.10
         uses: actions/setup-python@v3
         with:
-          python-version: '3.10'
+          python-version: "3.10"
 
-      - name: Install Intel Neural Compressor CPU requirements
+      - name: Install requirements
         run: |
           pip install --upgrade pip
           pip install -e .[testing,neural-compressor,diffusers,timm]
 
-      - name: Run Intel Neural Compressor CPU tests
+      - name: Run CPU tests
         run: pytest -k "cli and cpu and neural_compressor"

.github/workflows/test_cli_cpu_onnxruntime.yaml

@@ -21,13 +21,12 @@ jobs:
       - name: Set up Python 3.10
         uses: actions/setup-python@v3
         with:
-          python-version: '3.10'
+          python-version: "3.10"
 
       - name: Install requirements
         run: |
           pip install --upgrade pip
           pip install -e .[testing,onnxruntime,diffusers,timm]
 
       - name: Run tests
-        run: |
-          pytest -k "cli and cpu and onnxruntime"
+        run: pytest -k "cli and cpu and onnxruntime"

.github/workflows/test_cli_cpu_openvino.yaml

@@ -21,13 +21,12 @@ jobs:
       - name: Set up Python 3.10
         uses: actions/setup-python@v3
         with:
-          python-version: '3.10'
+          python-version: "3.10"
 
       - name: Install requirements
         run: |
           pip install --upgrade pip
           pip install -e .[testing,openvino,diffusers,timm]
 
       - name: Run tests
-        run: |
-          pytest -k "cli and cpu and openvino"
+        run: pytest -k "cli and cpu and openvino"

.github/workflows/test_cli_cpu_py_tgi.yaml

@@ -0,0 +1,32 @@
+name: CLI CPU Py-TGI Tests
+
+on:
+  workflow_dispatch:
+  push:
+    branches: [main]
+  pull_request:
+    types: [opened, reopened, synchronize]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  run_cli_cpu_py_tgi_tests:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v3
+        with:
+          python-version: "3.10"
+
+      - name: Install requirements
+        run: |
+          pip install --upgrade pip
+          pip install -e .[testing,py-tgi]
+
+      - name: Run tests
+        run: pytest -k "cli and cpu and py_tgi"

.github/workflows/test_cli_cpu_pytorch.yaml

@@ -21,13 +21,12 @@ jobs:
       - name: Set up Python 3.10
         uses: actions/setup-python@v3
         with:
-          python-version: '3.10'
+          python-version: "3.10"
 
       - name: Install requirements
         run: |
           pip install --upgrade pip
           pip install -e .[testing,diffusers,timm,peft]
 
       - name: Run tests
-        run: |
-          pytest -k "cli and cpu and pytorch"
+        run: pytest -k "cli and cpu and pytorch"

.github/workflows/test_cli_misc.yaml

@@ -21,7 +21,7 @@ jobs:
       - name: Set up Python 3.10
         uses: actions/setup-python@v3
         with:
-          python-version: '3.10'
+          python-version: "3.10"
 
       - name: Install requirements
         run: |

.github/workflows/test_cli_tensorrt_onnxruntime.yaml

@@ -26,15 +26,20 @@ jobs:
           --tag opt-bench-tensorrt:latest
           .
 
+      - name: Get GPUs with most free memory
+        id: get_devices
+        run: |
+          echo "::set-output name=devices::$(nvidia-smi --query-gpu=memory.free,index --format=csv,noheader,nounits | sort -n -k1 | tail -n 2 | awk -F', ' '{print $2}' | xargs echo -n | sed 's/ /,/g' | awk '{print $0}')"
+
       - name: Run tests
         run: docker run
           --rm
           --pid host
           --shm-size 64G
           --env USE_CUDA="1"
+          --gpus '"device=${{ steps.get_devices.outputs.devices }}"'
           --volume $(pwd):/workspace/optimum-benchmark
           --workdir /workspace/optimum-benchmark
-          --gpus '"device=0,1"'
           --entrypoint /bin/bash
           opt-bench-tensorrt:latest
           -c "pip install -e .[testing,onnxruntime-gpu,diffusers,timm] && pytest -k 'cli and tensorrt and onnxruntime' -x"

tests/configs/_gpt_sweep_.yaml

@@ -2,4 +2,5 @@ hydra:
   sweeper:
     params:
       backend.task: text-generation
+      backend.no_weights: true,false
       backend.model: hf-internal-testing/tiny-random-gpt2,IlyasMoutawwakil/tiny-random-llama

tests/configs/_inference_.yaml

@@ -2,7 +2,11 @@ defaults:
   - benchmark: inference
 
 benchmark:
+  memory: true
+  latency: true
+
   duration: 1
   warmup_runs: 1
-  new_tokens: 2
-  memory: true
+  generate_kwargs:
+    max_new_tokens: 5
+    min_new_tokens: 5

tests/configs/_timm_.yaml

@@ -1,5 +1,7 @@
 backend:
-  library: timm
-  task: image-classification
   model: timm/tiny_vit_21m_224.in1k
 
+hydra:
+  sweeper:
+    params:
+      backend.no_weights: true,false

tests/configs/cuda_training_pytorch_gpt_naive_mp.yaml

@@ -3,7 +3,8 @@ defaults:
   # order of inheritance, last one overrides previous ones
   - _base_ # inherits from base config
   - _training_ # inherits from training config
-  - _gpt_naive_mp_ # inherits from lm naive mp config
+  - _naive_mp_ # inherits from lm naive mp config
+  - _gpt_ # inherits from gpt config
   - _cuda_ # inherits from cpu config
   - _self_ # hydra 1.1 compatibility