test: add pytest entry for worker chart tests

.github/workflows/pre-merge.yml

@@ -52,7 +52,7 @@ jobs:
       timeout-minutes: 3
 
   helm-chart-verification:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     container:
       image: ghcr.io/triton-inference-server/triton_distributed/helm-tester:0.1.1
       options: --tty
@@ -66,7 +66,7 @@ jobs:
     # Allowlist both variants of the mounted source directory.
     - run: git config --global --add safe.directory /__w/triton_distributed/triton_distributed
     - run: git config --global --add safe.directory /workspace
-    - run: pwsh -c ./deploy/Kubernetes/worker/tests/trtllm/test-chart.ps1 --verbose
+    - run: pwsh /workspace/deploy/Kubernetes/worker/tests/trtllm/test-chart.ps1
       timeout-minutes: 2
       working-directory: /workspace
 

deploy/Kubernetes/_build/common.ps1

@@ -105,7 +105,7 @@ function default_local_srcdir {
 }
 
 function default_verbosity {
-  $value = 'MINIMAL'
+  $value = 'NORMAL'
   write-debug "<default_verbosity> -> '${value}'."
   return $value
 }
@@ -236,7 +236,6 @@ function is_installed([string] $command) {
   return $out
 }
 
-
 function is_tty {
   return -not(([System.Console]::IsOutputRedirected) -or ([System.Console]::IsErrorRedirected))
 }
@@ -443,8 +442,7 @@ function write-error([string] $value) {
 
 function write-failed([string] $value) {
   if (is_tty) {
-    write-normal '  Test:' -no_newline
-    write-normal ' [Failed]' $global:colors.test.failed -no_newline
+    write-normal '  [Failed]' $global:colors.test.failed -no_newline
     write-normal " ${value}"
   }
   else {
@@ -497,8 +495,7 @@ function write-normal {
 
 function write-passed([string] $value) {
   if (is_tty) {
-    write-detailed '  Test:' -no_newline
-    write-detailed ' [Passed]' $global:colors.test.passed -no_newline
+    write-detailed '  [Passed]' $global:colors.test.passed -no_newline
     write-detailed " ${value}"
   }
   else {

deploy/Kubernetes/_build/helm-test.ps1

@@ -17,21 +17,122 @@ set-strictmode -version latest
 
 . "$(& git rev-parse --show-toplevel)/deploy/Kubernetes/_build/common.ps1"
 
-function test_helm_chart([string] $chart_path, [string] $tests_path, [object[]] $test_set) {
-  write-debug "<test_helm_chart> chart_path = '${chart_path}'."
-  write-debug "<test_helm_chart> tests_path = '${tests_path}'."
-  write-debug "<test_helm_chart> test_set = [$($test_set.count)]"
+function initialize_test([string] $component_kind, [string] $component_name, [string[]]$params, [object[]] $tests) {
+  write-debug "<initialize_test> component_kind: ${component_kind}."
+  write-debug "<initialize_test> component_name: ${component_name}."
+  write-debug "<initialize_test> params.count: $($params.count)."
+  write-debug "<initialize_test> tests.count: $($tests.count)."
 
-  $chart_path = to_local_path $chart_path
-  $tests_path = to_local_path $tests_path
+  $is_debug = $false
+  $test_filter = @()
+
+  for ($i = 0 ; $i -lt $params.count ; $i += 1) {
+    $arg = $params[$i]
+
+    if ('--debug' -ieq $arg) {
+      $is_debug = $true
+    }
+    elseif ('--list' -ieq $arg)
+    {
+      write-minimal "Available tests:"
+
+      foreach ($test in $tests) {
+        write-minimal "- $($test.name)"
+      }
+
+      exit(0)
+    }
+    elseif (('--test' -ieq $arg) -or ('-t' -ieq $arg))
+    {
+      if ($i + 1 -ge $params.count) {
+        usage_exit "Expected value following ""{$arg}""."
+      }
+
+      $i += 1
+      $test_name = $params[$i]
+      $test_found = $false
+
+      $parts = $test_name.split('/')
+      if ($parts.count -gt 1) {
+        $test_name = $parts[$parts.count - 1]
+      }
+
+      foreach ($test in $tests) {
+        if ($test.name -ieq $test_name) {
+          $test_found = $true
+          break
+        }
+      }
+
+      if (-not $test_found) {
+        usage_exit "Unknown test name ""${test_name}"" provided."
+      }
+
+      $test_filter += $test_name
+    }
+    elseif (('--verbose' -ieq $arg) -or ('-v' -ieq $arg)) {
+      set_verbosity('DETAILED')
+    }
+    else {
+      usage_exit "Unknown option '${arg}'."
+    }
+  }
+
+  $is_debug = $is_debug -or $(is_debug)
+  if ($is_debug) {
+    $DebugPreference = 'Continue'
+  }
+  else {
+    $DebugPreference = 'SilentlyContinue'
+  }
+
+  if (-not ($(get_verbosity) -eq 'MINIMAL' -and $(is_tty))) {
+    set_verbosity('DETAILED')
+  }
+
+  # When a subset of tests has been requested, filter out the not requested tests.
+  if ($test_filter.count -gt 0) {
+    write-debug "<test-chart> selected.count: $($test_filter.count)."
+
+    $replace = @()
+
+    # Find the test that matches each selected item and add it to a replacement list.
+    foreach ($filter in $test_filter) {
+      foreach ($test in $tests) {
+        if ($test.name -ieq $filter) {
+          $replace += $test
+          break
+        }
+      }
+    }
+
+    # Replace the test list with the replacement list.
+    $tests = $replace
+  }
+
+  return @{
+    component = $component_kind
+    name = $component_name
+    is_debug = $is_debug
+    tests = $tests
+  }
+}
+
+function test_helm_chart([object] $config) {
+  write-debug "<test_helm_chart> config.component = '$($config.component)'."
+  write-debug "<test_helm_chart> config.name = '$($config.name)'."
+  write-debug "<test_helm_chart> config.count = [$($config.tests.count)]"
+
+  $chart_path = to_local_path "deploy/Kubernetes/$($config.component)/charts/$($config.name)"
+  $tests_path = to_local_path "deploy/Kubernetes/$($config.component)/tests/$($config.name)"
 
   push-location $chart_path
 
   try {
     $fail_count = 0
     $pass_count = 0
 
-    foreach ($test in $test_set) {
+    foreach ($test in $config.tests) {
       $helm_command = 'helm template test -f ./values.yaml'
       write-debug "<test_helm_chart> helm_command = '${helm_command}'."
 
@@ -82,11 +183,11 @@ function test_helm_chart([string] $chart_path, [string] $tests_path, [object[]]
 
       if ($is_pass) {
         $pass_count += 1
-        write-passed "$($test.name)"
+        write-passed "$($config.component)/$($config.name)/$($test.name)"
       }
       else {
         $fail_count += 1
-        write-failed "$($test.name)"
+        write-failed "$($config.component)/$($config.name)/$($test.name)"
       }
     }
   }
@@ -99,12 +200,12 @@ function test_helm_chart([string] $chart_path, [string] $tests_path, [object[]]
   pop-location
 
   if ($fail_count -gt 0) {
-    write-minimal "Failed: ${fail_count}, Passed: ${pass_count}, Total: $($tests.count)" 'Red'
+    write-minimal "Failed: ${fail_count}, Passed: ${pass_count}, Total: $($config.tests.count)" 'Red'
     return $false
   }
   else
   {
-    write-minimal "Passed: ${pass_count}, Total: $($tests.count)" 'Green'
+    write-minimal "Passed: ${pass_count}, Total: $($config.tests.count)" 'Green'
     return $true
   }
 }

deploy/Kubernetes/worker/tests/trtllm/test-chart.ps1

@@ -17,32 +17,9 @@ set-strictmode -version latest
 
 . "$(& git rev-parse --show-toplevel)/deploy/Kubernetes/_build/helm-test.ps1"
 
-$is_debug = $false
-
-for ($i = 0 ; $i -lt $args.count ; $i += 1) {
-  $arg = $args[$i]
-  if ('--debug' -ieq $arg) {
-    $is_debug = $true
-  }
-  elseif (('--verbose' -ieq $arg) -or ('-v' -ieq $arg)) {
-    set_verbosity('DETAILED')
-  }
-  else {
-    fatal-exit "Unknown option '${arg}'."
-  }
-}
-
-$is_debug = $is_debug -or $(is_debug)
-if ($is_debug) {
-  $DebugPreference = 'Continue'
-}
-else {
-  $DebugPreference = 'SilentlyContinue'
-}
-
 $tests = @(
     @{
-      name = 'worker/trtllm/basic'
+      name = 'basic'
       expected = 0
       matches = @(
           'helm.sh/chart: "triton-distributed_worker-trtllm"[\n\r]{1,2}'
@@ -57,7 +34,7 @@ $tests = @(
       values = @('basic_values.yaml')
     }
     @{
-      name = 'worker/trtllm/basic_error'
+      name = 'basic_error'
       expected = 1
       matches = @(
           '- triton: componentName is required[\n\r]{1,2}'
@@ -67,7 +44,7 @@ $tests = @(
       values = @()
     }
     @{
-      name = 'worker/trtllm/volume_mounts'
+      name = 'volume_mounts'
       expected = 0
       matches = @(
           '- name: mount_w_path[\n\r ]+persistentVolumeClaim:[\n\r ]+claimName: w_path_pvc[\n\r]{1,2}'
@@ -82,7 +59,7 @@ $tests = @(
         )
     }
     @{
-      name = 'worker/trtllm/bad_volume_mounts'
+      name = 'bad_volume_mounts'
       expected = 1
       matches = @(
           '- modelRepository.volumeMounts.0: persistentVolumeClaim is required'
@@ -94,7 +71,7 @@ $tests = @(
         )
     }
     @{
-      name = 'worker/trtllm/host_cache'
+      name = 'host_cache'
       expected = 0
       matches = @(
           'ephemeral-storage: 2Gi[\n\r]{1,2}'
@@ -108,7 +85,7 @@ $tests = @(
         )
     }
     @{
-      name = 'worker/trtllm/host_cache'
+      name = 'non-host_cache'
       expected = 0
       matches = @(
           'ephemeral-storage: 202Gi[\n\r]{1,2}'
@@ -123,10 +100,20 @@ $tests = @(
     }
   )
 
+  $config = initialize_test 'worker' 'trtllm' $args $tests
+
+if ($config.is_debug) {
+  $DebugPreference = 'Continue'
+}
+else {
+  $DebugPreference = 'SilentlyContinue'
+}
+
+# Being w/ the state of not having passed.
 $is_pass = $false
 
 try {
-  $is_pass = test_helm_chart 'deploy/Kubernetes/worker/charts/trtllm' 'deploy/Kubernetes/worker/tests/trtllm' $tests
+  $is_pass = $(test_helm_chart $config)
   write-debug "is_pass: ${is_pass}."
 }
 catch {

deploy/Kubernetes/worker/worker_tests.py

@@ -0,0 +1,76 @@
+# SPDX-FileCopyrightText: Copyright (c) 2024-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import subprocess
+import sys
+
+import pytest
+
+
+@pytest.mark.parametrize(
+    "chart, test",
+    [
+        ("trtllm", "basic"),
+        ("trtllm", "basic_error"),
+        ("trtllm", "volume_mounts"),
+        ("trtllm", "bad_volume_mounts"),
+        ("trtllm", "host_cache"),
+        ("trtllm", "non-host_cache"),
+    ],
+)
+def test_chart(chart, test):
+    cmd_args = [
+        "git",
+        "rev-parse",
+        "--show-toplevel"
+    ]
+
+    repository_root_path = subprocess.check_output(cmd_args).decode("utf-8")
+    repository_root_path = repository_root_path.strip()
+
+    test_chart_path = os.path.join(
+        repository_root_path,
+        "deploy",
+        "Kubernetes",
+        "worker",
+        "tests",
+        chart,
+        "test-chart.ps1",
+    )
+
+    print()
+    print(f"Run {test_chart_path}")
+
+    cmd_args = [
+        "pwsh",
+        "-c",
+        test_chart_path,
+        "--test",
+        test,
+        "--verbose",
+    ]
+
+    assert subprocess.run(cmd_args).returncode == 0
+
+
+if __name__ == "__main__":
+    print(
+        "Error: This script is not indented to executed direct. "
+        "Instead use `pytest worker_tests.py` to execute it.",
+        file=sys.stderr,
+        flush=True,
+    )
+    exit(1)