test: add pytest entry for worker chart tests

.github/workflows/pre-merge.yml

@@ -66,7 +66,8 @@ jobs:
     # Allowlist both variants of the mounted source directory.
     - run: git config --global --add safe.directory /__w/triton_distributed/triton_distributed
     - run: git config --global --add safe.directory /workspace
-    - run: pwsh -c ./deploy/Kubernetes/worker/tests/trtllm/test-chart.ps1 --verbose
+    - run: git config --global --list
+    - run: pwsh -c /workspace/deploy/Kubernetes/worker/tests/trtllm/test-chart.ps1 --verbose
       timeout-minutes: 2
       working-directory: /workspace
 

deploy/Kubernetes/_build/common.ps1

@@ -443,8 +443,7 @@ function write-error([string] $value) {
 
 function write-failed([string] $value) {
   if (is_tty) {
-    write-normal '  Test:' -no_newline
-    write-normal ' [Failed]' $global:colors.test.failed -no_newline
+    write-normal '  [Failed]' $global:colors.test.failed -no_newline
     write-normal " ${value}"
   }
   else {
@@ -497,8 +496,7 @@ function write-normal {
 
 function write-passed([string] $value) {
   if (is_tty) {
-    write-detailed '  Test:' -no_newline
-    write-detailed ' [Passed]' $global:colors.test.passed -no_newline
+    write-detailed '  [Passed]' $global:colors.test.passed -no_newline
     write-detailed " ${value}"
   }
   else {

deploy/Kubernetes/_build/helm-test.ps1

@@ -17,13 +17,13 @@ set-strictmode -version latest
 
 . "$(& git rev-parse --show-toplevel)/deploy/Kubernetes/_build/common.ps1"
 
-function test_helm_chart([string] $chart_path, [string] $tests_path, [object[]] $test_set) {
-  write-debug "<test_helm_chart> chart_path = '${chart_path}'."
-  write-debug "<test_helm_chart> tests_path = '${tests_path}'."
+function test_helm_chart([string] $component_kind, [string] $component_name, [object[]] $test_set) {
+  write-debug "<test_helm_chart> component_kind = '${component_kind}'."
+  write-debug "<test_helm_chart> component_name = '${component_name}'."
   write-debug "<test_helm_chart> test_set = [$($test_set.count)]"
 
-  $chart_path = to_local_path $chart_path
-  $tests_path = to_local_path $tests_path
+  $chart_path = to_local_path "deploy/Kubernetes/${component_kind}/charts/${component_name}"
+  $tests_path = to_local_path "deploy/Kubernetes/${component_kind}/tests/${component_name}"
 
   push-location $chart_path
 
@@ -82,7 +82,7 @@ function test_helm_chart([string] $chart_path, [string] $tests_path, [object[]]
 
       if ($is_pass) {
         $pass_count += 1
-        write-passed "$($test.name)"
+        write-passed "${component_kind}/${component_name}/$($test.name)"
       }
       else {
         $fail_count += 1

deploy/Kubernetes/worker/tests/trtllm/test-chart.ps1

@@ -17,32 +17,9 @@ set-strictmode -version latest
 
 . "$(& git rev-parse --show-toplevel)/deploy/Kubernetes/_build/helm-test.ps1"
 
-$is_debug = $false
-
-for ($i = 0 ; $i -lt $args.count ; $i += 1) {
-  $arg = $args[$i]
-  if ('--debug' -ieq $arg) {
-    $is_debug = $true
-  }
-  elseif (('--verbose' -ieq $arg) -or ('-v' -ieq $arg)) {
-    set_verbosity('DETAILED')
-  }
-  else {
-    fatal-exit "Unknown option '${arg}'."
-  }
-}
-
-$is_debug = $is_debug -or $(is_debug)
-if ($is_debug) {
-  $DebugPreference = 'Continue'
-}
-else {
-  $DebugPreference = 'SilentlyContinue'
-}
-
 $tests = @(
     @{
-      name = 'worker/trtllm/basic'
+      name = 'basic'
       expected = 0
       matches = @(
           'helm.sh/chart: "triton-distributed_worker-trtllm"[\n\r]{1,2}'
@@ -57,7 +34,7 @@ $tests = @(
       values = @('basic_values.yaml')
     }
     @{
-      name = 'worker/trtllm/basic_error'
+      name = 'basic_error'
       expected = 1
       matches = @(
           '- triton: componentName is required[\n\r]{1,2}'
@@ -67,7 +44,7 @@ $tests = @(
       values = @()
     }
     @{
-      name = 'worker/trtllm/volume_mounts'
+      name = 'volume_mounts'
       expected = 0
       matches = @(
           '- name: mount_w_path[\n\r ]+persistentVolumeClaim:[\n\r ]+claimName: w_path_pvc[\n\r]{1,2}'
@@ -82,7 +59,7 @@ $tests = @(
         )
     }
     @{
-      name = 'worker/trtllm/bad_volume_mounts'
+      name = 'bad_volume_mounts'
       expected = 1
       matches = @(
           '- modelRepository.volumeMounts.0: persistentVolumeClaim is required'
@@ -94,7 +71,7 @@ $tests = @(
         )
     }
     @{
-      name = 'worker/trtllm/host_cache'
+      name = 'host_cache'
       expected = 0
       matches = @(
           'ephemeral-storage: 2Gi[\n\r]{1,2}'
@@ -108,8 +85,8 @@ $tests = @(
         )
     }
     @{
-      name = 'worker/trtllm/host_cache'
-      expected = 0
+      name = 'non-host_cache'
+      expected = 1
       matches = @(
           'ephemeral-storage: 202Gi[\n\r]{1,2}'
           '- name: model-host-cache[\n\r ]+emptyDir:[\n\r ]+sizeLimit: 200Gi[\n\r]{1,2}'
@@ -123,10 +100,78 @@ $tests = @(
     }
   )
 
+$is_debug = $false
+$selected = @()
+
+for ($i = 0 ; $i -lt $args.count ; $i += 1) {
+  $arg = $args[$i]
+  if ('--debug' -ieq $arg) {
+    $is_debug = $true
+  }
+  elseif (('--test' -ieq $arg) -or ('-t' -ieq $arg))
+  {
+    if ($i + 1 -ge $args.count) {
+      usage_exit "Expected value following ""{$arg}""."
+    }
+
+    $i += 1
+    $test_name = $args[$i]
+    $test_found = $false
+
+    foreach ($test in $tests) {
+      if ($test.name -ieq $test_name) {
+        $test_found = $true
+        break
+      }
+    }
+
+    if (-not $test_found) {
+      usage_exit "Unknown test name ""${test_name}"" provided."
+    }
+
+    $selected += $test_name
+  }
+  elseif (('--verbose' -ieq $arg) -or ('-v' -ieq $arg)) {
+    set_verbosity('DETAILED')
+  }
+  else {
+    usage_exit "Unknown option '${arg}'."
+  }
+}
+
+$is_debug = $is_debug -or $(is_debug)
+if ($is_debug) {
+  $DebugPreference = 'Continue'
+}
+else {
+  $DebugPreference = 'SilentlyContinue'
+}
+
+# Being w/ the state of not having passed.
 $is_pass = $false
 
 try {
-  $is_pass = test_helm_chart 'deploy/Kubernetes/worker/charts/trtllm' 'deploy/Kubernetes/worker/tests/trtllm' $tests
+  # When a subset of tests has been requested, filter out the not requested tests.
+  if ($selected.count -gt 0) {
+    write-debug "<test-chart> selected.count: $($selected.count)."
+
+    $replace = @()
+
+    # Find the test that matches each selected item and add it to a replacement list.
+    foreach ($item in $selected) {
+      foreach ($test in $tests) {
+        if ($test.name -ieq $item) {
+          $replace += $test
+          break
+        }
+      }
+    }
+
+    # Replace the test list with the replacement list.
+    $tests = $replace
+  }
+
+  $is_pass = $(test_helm_chart 'worker' 'trtllm' $tests)
   write-debug "is_pass: ${is_pass}."
 }
 catch {

deploy/Kubernetes/worker/worker_tests.py

@@ -0,0 +1,76 @@
+# SPDX-FileCopyrightText: Copyright (c) 2024-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import subprocess
+import sys
+
+import pytest
+
+
+@pytest.mark.parametrize(
+    "chart, test",
+    [
+        ("trtllm", "basic"),
+        ("trtllm", "basic_error"),
+        ("trtllm", "volume_mounts"),
+        ("trtllm", "bad_volume_mounts"),
+        ("trtllm", "host_cache"),
+        ("trtllm", "non-host_cache"),
+    ],
+)
+def test_chart(chart, test):
+    cmd_args = [
+        "git",
+        "rev-parse",
+        "--show-toplevel"
+    ]
+
+    repository_root_path = subprocess.check_output(cmd_args).decode("utf-8")
+    repository_root_path = repository_root_path.strip()
+
+    test_chart_path = os.path.join(
+        repository_root_path,
+        "deploy",
+        "Kubernetes",
+        "worker",
+        "tests",
+        chart,
+        "test-chart.ps1",
+    )
+
+    print()
+    print(f"Run {test_chart_path}")
+
+    cmd_args = [
+        "pwsh",
+        "-c",
+        test_chart_path,
+        "--test",
+        test,
+        "--verbose",
+    ]
+
+    assert subprocess.run(cmd_args).returncode == 0
+
+
+if __name__ == "__main__":
+    print(
+        "Error: This script is not indented to executed direct. "
+        "Instead use `pytest worker_tests.py` to execute it.",
+        file=sys.stderr,
+        flush=True,
+    )
+    exit(1)