diff --git a/.github/workflows/pre-merge.yml b/.github/workflows/pre-merge.yml index f794f50a..18b17e19 100644 --- a/.github/workflows/pre-merge.yml +++ b/.github/workflows/pre-merge.yml @@ -52,7 +52,7 @@ jobs: timeout-minutes: 3 helm-chart-verification: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 container: image: ghcr.io/triton-inference-server/triton_distributed/helm-tester:0.1.1 options: --tty @@ -66,7 +66,7 @@ jobs: # Allowlist both variants of the mounted source directory. - run: git config --global --add safe.directory /__w/triton_distributed/triton_distributed - run: git config --global --add safe.directory /workspace - - run: pwsh -c ./deploy/Kubernetes/worker/tests/trtllm/test-chart.ps1 --verbose + - run: pwsh /workspace/deploy/Kubernetes/worker/tests/trtllm/test-chart.ps1 timeout-minutes: 2 working-directory: /workspace diff --git a/deploy/Kubernetes/_build/common.ps1 b/deploy/Kubernetes/_build/common.ps1 index 86885ffc..aad70581 100644 --- a/deploy/Kubernetes/_build/common.ps1 +++ b/deploy/Kubernetes/_build/common.ps1 @@ -105,7 +105,7 @@ function default_local_srcdir { } function default_verbosity { - $value = 'MINIMAL' + $value = 'NORMAL' write-debug " -> '${value}'." return $value } @@ -236,7 +236,6 @@ function is_installed([string] $command) { return $out } - function is_tty { return -not(([System.Console]::IsOutputRedirected) -or ([System.Console]::IsErrorRedirected)) } @@ -443,8 +442,7 @@ function write-error([string] $value) { function write-failed([string] $value) { if (is_tty) { - write-normal ' Test:' -no_newline - write-normal ' [Failed]' $global:colors.test.failed -no_newline + write-normal ' [Failed]' $global:colors.test.failed -no_newline write-normal " ${value}" } else { @@ -497,8 +495,7 @@ function write-normal { function write-passed([string] $value) { if (is_tty) { - write-detailed ' Test:' -no_newline - write-detailed ' [Passed]' $global:colors.test.passed -no_newline + write-detailed ' [Passed]' $global:colors.test.passed -no_newline write-detailed " ${value}" } else { diff --git a/deploy/Kubernetes/_build/helm-test.ps1 b/deploy/Kubernetes/_build/helm-test.ps1 index 38646118..f3a6ef6a 100644 --- a/deploy/Kubernetes/_build/helm-test.ps1 +++ b/deploy/Kubernetes/_build/helm-test.ps1 @@ -17,13 +17,114 @@ set-strictmode -version latest . "$(& git rev-parse --show-toplevel)/deploy/Kubernetes/_build/common.ps1" -function test_helm_chart([string] $chart_path, [string] $tests_path, [object[]] $test_set) { - write-debug " chart_path = '${chart_path}'." - write-debug " tests_path = '${tests_path}'." - write-debug " test_set = [$($test_set.count)]" +function initialize_test([string] $component_kind, [string] $component_name, [string[]]$params, [object[]] $tests) { + write-debug " component_kind: ${component_kind}." + write-debug " component_name: ${component_name}." + write-debug " params.count: $($params.count)." + write-debug " tests.count: $($tests.count)." - $chart_path = to_local_path $chart_path - $tests_path = to_local_path $tests_path + $is_debug = $false + $test_filter = @() + + for ($i = 0 ; $i -lt $params.count ; $i += 1) { + $arg = $params[$i] + + if ('--debug' -ieq $arg) { + $is_debug = $true + } + elseif ('--list' -ieq $arg) + { + write-minimal "Available tests:" + + foreach ($test in $tests) { + write-minimal "- $($test.name)" + } + + exit(0) + } + elseif (('--test' -ieq $arg) -or ('-t' -ieq $arg)) + { + if ($i + 1 -ge $params.count) { + usage_exit "Expected value following ""{$arg}""." + } + + $i += 1 + $test_name = $params[$i] + $test_found = $false + + $parts = $test_name.split('/') + if ($parts.count -gt 1) { + $test_name = $parts[$parts.count - 1] + } + + foreach ($test in $tests) { + if ($test.name -ieq $test_name) { + $test_found = $true + break + } + } + + if (-not $test_found) { + usage_exit "Unknown test name ""${test_name}"" provided." + } + + $test_filter += $test_name + } + elseif (('--verbose' -ieq $arg) -or ('-v' -ieq $arg)) { + set_verbosity('DETAILED') + } + else { + usage_exit "Unknown option '${arg}'." + } + } + + $is_debug = $is_debug -or $(is_debug) + if ($is_debug) { + $DebugPreference = 'Continue' + } + else { + $DebugPreference = 'SilentlyContinue' + } + + if (-not ($(get_verbosity) -eq 'MINIMAL' -and $(is_tty))) { + set_verbosity('DETAILED') + } + + # When a subset of tests has been requested, filter out the not requested tests. + if ($test_filter.count -gt 0) { + write-debug " selected.count: $($test_filter.count)." + + $replace = @() + + # Find the test that matches each selected item and add it to a replacement list. + foreach ($filter in $test_filter) { + foreach ($test in $tests) { + if ($test.name -ieq $filter) { + $replace += $test + break + } + } + } + + # Replace the test list with the replacement list. + $tests = $replace + } + + return @{ + component = $component_kind + name = $component_name + is_debug = $is_debug + tests = $tests + } +} + +function test_helm_chart([object] $config) { + write-debug " config.component = '$($config.component)'." + write-debug " config.name = '$($config.name)'." + write-debug " config.count = [$($config.tests.count)]" + + $chart_path = to_local_path "deploy/Kubernetes/$($config.component)/charts/$($config.name)" + $tests_path = to_local_path "deploy/Kubernetes/$($config.component)/tests/$($config.name)" push-location $chart_path @@ -31,7 +132,7 @@ function test_helm_chart([string] $chart_path, [string] $tests_path, [object[]] $fail_count = 0 $pass_count = 0 - foreach ($test in $test_set) { + foreach ($test in $config.tests) { $helm_command = 'helm template test -f ./values.yaml' write-debug " helm_command = '${helm_command}'." @@ -82,11 +183,11 @@ function test_helm_chart([string] $chart_path, [string] $tests_path, [object[]] if ($is_pass) { $pass_count += 1 - write-passed "$($test.name)" + write-passed "$($config.component)/$($config.name)/$($test.name)" } else { $fail_count += 1 - write-failed "$($test.name)" + write-failed "$($config.component)/$($config.name)/$($test.name)" } } } @@ -99,12 +200,12 @@ function test_helm_chart([string] $chart_path, [string] $tests_path, [object[]] pop-location if ($fail_count -gt 0) { - write-minimal "Failed: ${fail_count}, Passed: ${pass_count}, Total: $($tests.count)" 'Red' + write-minimal "Failed: ${fail_count}, Passed: ${pass_count}, Total: $($config.tests.count)" 'Red' return $false } else { - write-minimal "Passed: ${pass_count}, Total: $($tests.count)" 'Green' + write-minimal "Passed: ${pass_count}, Total: $($config.tests.count)" 'Green' return $true } } diff --git a/deploy/Kubernetes/worker/tests/trtllm/test-chart.ps1 b/deploy/Kubernetes/worker/tests/trtllm/test-chart.ps1 index 43cf8ecd..9372843a 100644 --- a/deploy/Kubernetes/worker/tests/trtllm/test-chart.ps1 +++ b/deploy/Kubernetes/worker/tests/trtllm/test-chart.ps1 @@ -17,32 +17,9 @@ set-strictmode -version latest . "$(& git rev-parse --show-toplevel)/deploy/Kubernetes/_build/helm-test.ps1" -$is_debug = $false - -for ($i = 0 ; $i -lt $args.count ; $i += 1) { - $arg = $args[$i] - if ('--debug' -ieq $arg) { - $is_debug = $true - } - elseif (('--verbose' -ieq $arg) -or ('-v' -ieq $arg)) { - set_verbosity('DETAILED') - } - else { - fatal-exit "Unknown option '${arg}'." - } -} - -$is_debug = $is_debug -or $(is_debug) -if ($is_debug) { - $DebugPreference = 'Continue' -} -else { - $DebugPreference = 'SilentlyContinue' -} - $tests = @( @{ - name = 'worker/trtllm/basic' + name = 'basic' expected = 0 matches = @( 'helm.sh/chart: "triton-distributed_worker-trtllm"[\n\r]{1,2}' @@ -57,7 +34,7 @@ $tests = @( values = @('basic_values.yaml') } @{ - name = 'worker/trtllm/basic_error' + name = 'basic_error' expected = 1 matches = @( '- triton: componentName is required[\n\r]{1,2}' @@ -67,7 +44,7 @@ $tests = @( values = @() } @{ - name = 'worker/trtllm/volume_mounts' + name = 'volume_mounts' expected = 0 matches = @( '- name: mount_w_path[\n\r ]+persistentVolumeClaim:[\n\r ]+claimName: w_path_pvc[\n\r]{1,2}' @@ -82,7 +59,7 @@ $tests = @( ) } @{ - name = 'worker/trtllm/bad_volume_mounts' + name = 'bad_volume_mounts' expected = 1 matches = @( '- modelRepository.volumeMounts.0: persistentVolumeClaim is required' @@ -94,7 +71,7 @@ $tests = @( ) } @{ - name = 'worker/trtllm/host_cache' + name = 'host_cache' expected = 0 matches = @( 'ephemeral-storage: 2Gi[\n\r]{1,2}' @@ -108,7 +85,7 @@ $tests = @( ) } @{ - name = 'worker/trtllm/host_cache' + name = 'non-host_cache' expected = 0 matches = @( 'ephemeral-storage: 202Gi[\n\r]{1,2}' @@ -123,10 +100,20 @@ $tests = @( } ) + $config = initialize_test 'worker' 'trtllm' $args $tests + +if ($config.is_debug) { + $DebugPreference = 'Continue' +} +else { + $DebugPreference = 'SilentlyContinue' +} + +# Being w/ the state of not having passed. $is_pass = $false try { - $is_pass = test_helm_chart 'deploy/Kubernetes/worker/charts/trtllm' 'deploy/Kubernetes/worker/tests/trtllm' $tests + $is_pass = $(test_helm_chart $config) write-debug "is_pass: ${is_pass}." } catch { diff --git a/deploy/Kubernetes/worker/worker_tests.py b/deploy/Kubernetes/worker/worker_tests.py new file mode 100644 index 00000000..5d552323 --- /dev/null +++ b/deploy/Kubernetes/worker/worker_tests.py @@ -0,0 +1,76 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import subprocess +import sys + +import pytest + + +@pytest.mark.parametrize( + "chart, test", + [ + ("trtllm", "basic"), + ("trtllm", "basic_error"), + ("trtllm", "volume_mounts"), + ("trtllm", "bad_volume_mounts"), + ("trtllm", "host_cache"), + ("trtllm", "non-host_cache"), + ], +) +def test_chart(chart, test): + cmd_args = [ + "git", + "rev-parse", + "--show-toplevel" + ] + + repository_root_path = subprocess.check_output(cmd_args).decode("utf-8") + repository_root_path = repository_root_path.strip() + + test_chart_path = os.path.join( + repository_root_path, + "deploy", + "Kubernetes", + "worker", + "tests", + chart, + "test-chart.ps1", + ) + + print() + print(f"Run {test_chart_path}") + + cmd_args = [ + "pwsh", + "-c", + test_chart_path, + "--test", + test, + "--verbose", + ] + + assert subprocess.run(cmd_args).returncode == 0 + + +if __name__ == "__main__": + print( + "Error: This script is not indented to executed direct. " + "Instead use `pytest worker_tests.py` to execute it.", + file=sys.stderr, + flush=True, + ) + exit(1)