Merge branch 'main' into fpetrini-akarale-win-utf16

.github/workflows/build-and-test.yml

@@ -0,0 +1,51 @@
+name: Build And Test
+
+on:
+    pull_request:
+      branches:
+        - main
+      types: [synchronize, opened, reopened, ready_for_review]
+
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    container:
+      image: nvcr.io/nvidia/tritonserver:24.10-py3
+      volumes:
+        - ${{ github.workspace }}:/core
+        # Mount /usr so we can free space
+        - /usr:/host_usr
+      env:
+        AGENT_TOOLSDIRECTORY: "$AGENT_TOOLSDIRECTORY"
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Free space
+        run: |
+          rm -rf \
+            /host_usr/share/dotnet /host_usr/local/lib/android /opt/ghc \
+            /host_usr/local/share/powershell /host_usr/share/swift /host_usr/local/.ghcup \
+            /host_usr/lib/jvm
+          rm -rf "$AGENT_TOOLSDIRECTORY"
+
+      - name: Install dependencies
+        run: |
+          apt update
+          apt install -y --no-install-recommends clang-format-15 cmake libb64-dev rapidjson-dev libre2-dev
+          wget -O /tmp/boost.tar.gz https://archives.boost.io/release/1.80.0/source/boost_1_80_0.tar.gz && (cd /tmp && tar xzf boost.tar.gz) && mv /tmp/boost_1_80_0/boost /usr/include/boost && rm /tmp/boost.tar.gz
+          pip install build pytest
+
+      - name: Build
+        run: |
+          mkdir -p /core/build
+          cd /core/build
+          cmake -DCMAKE_INSTALL_PREFIX:PATH=`pwd`/install -DTRITON_CORE_HEADERS_ONLY=OFF ..
+          make -j8
+
+      - name: Run tests with pytest
+        run: |
+          cd /core
+          python3 -m pip install --force-reinstall build/python/generic/wheel/dist/tritonserver-*.whl
+          pytest python/test -v

include/triton/core/tritonserver.h

@@ -847,9 +847,9 @@ TRITONSERVER_InferenceTraceTensorNew(
 /// \param timestamp The timestamp associated with the trace activity.
 /// \param name The trace activity name.
 /// \return a TRITONSERVER_Error indicating success or failure.
-TRITONSERVER_DECLSPEC TRITONSERVER_Error*
+TRITONSERVER_DECLSPEC struct TRITONSERVER_Error*
 TRITONSERVER_InferenceTraceReportActivity(
-    TRITONSERVER_InferenceTrace* trace, uint64_t timestamp,
+    struct TRITONSERVER_InferenceTrace* trace, uint64_t timestamp,
     const char* activity_name);
 
 /// Delete a trace object.
@@ -1938,9 +1938,9 @@ TRITONSERVER_ServerOptionsSetCudaMemoryPoolByteSize(
 /// \param gpu_device The GPU device to set the CUDA virtual address space size
 /// \param size The size of the CUDA virtual address space.
 /// \return a TRITONSERVER_Error indicating success or failure.
-TRITONSERVER_DECLSPEC TRITONSERVER_Error*
+TRITONSERVER_DECLSPEC struct TRITONSERVER_Error*
 TRITONSERVER_ServerOptionsSetCudaVirtualAddressSize(
-    TRITONSERVER_ServerOptions* options, int gpu_device,
+    struct TRITONSERVER_ServerOptions* options, int gpu_device,
     size_t cuda_virtual_address_size);
 
 /// Deprecated. See TRITONSERVER_ServerOptionsSetCacheConfig instead.

pyproject.toml

@@ -24,6 +24,54 @@
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
+[project]
+name = "tritonserver"
+authors = [{ name = "NVIDIA Inc.", email = "[email protected]" }]
+description = "Triton Inference Server In-Process Python API"
+license = { file = "LICENSE.txt" }
+dynamic = ["version"]
+dependencies = ["numpy<2"]
+classifiers = [
+    "Development Status :: 5 - Production/Stable",
+    "Intended Audience :: Developers",
+    "Intended Audience :: Science/Research",
+    "Intended Audience :: Information Technology",
+    "Topic :: Scientific/Engineering",
+    "Topic :: Scientific/Engineering :: Image Recognition",
+    "Topic :: Scientific/Engineering :: Artificial Intelligence",
+    "Topic :: Software Development :: Libraries",
+    "Topic :: Utilities",
+    "License :: OSI Approved :: BSD License",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.12",
+    "Environment :: Console",
+    "Natural Language :: English",
+    "Operating System :: OS Independent",
+]
+
+[tool.setuptools]
+include-package-data = true
+
+[tool.setuptools.package-data]
+tritonserver = ["_c/triton_bindings.*.so"]
+
+[build-system]
+requires = [
+    "setuptools==75.3.0",
+    "wheel==0.44.0",
+    # For stubgen:
+    "mypy==1.11.0",
+    "numpy<2",
+]
+build-backend = "setuptools.build_meta"
+
+[project.optional-dependencies]
+GPU = ["cupy-cuda12x"]
+test = ["pytest"]
+all = ["tritonserver[GPU]", "tritonserver[test]"]
+
+
 [tool.codespell]
 # note: pre-commit passes explicit lists of files here, which this skip file list doesn't override -
 # this is only to allow you to run codespell interactively

python/CMakeLists.txt

@@ -30,6 +30,7 @@ add_subdirectory(tritonserver)
 file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/TRITON_VERSION ${TRITON_VERSION})
 configure_file(../LICENSE LICENSE.txt COPYONLY)
 configure_file(setup.py setup.py @ONLY)
+configure_file(../pyproject.toml pyproject.toml COPYONLY)
 file(COPY test/  DESTINATION ./test/.)
 
 set(WHEEL_DEPENDS

python/build_wheel.py

@@ -108,17 +108,18 @@ def sed(pattern, replace, source, dest=None):
 
     shutil.copyfile("LICENSE.txt", os.path.join(FLAGS.whl_dir, "LICENSE.txt"))
     shutil.copyfile("setup.py", os.path.join(FLAGS.whl_dir, "setup.py"))
+    shutil.copyfile("pyproject.toml", os.path.join(FLAGS.whl_dir, "pyproject.toml"))
 
     os.chdir(FLAGS.whl_dir)
     print("=== Building wheel")
-    args = ["python3", "setup.py", "bdist_wheel"]
+    args = ["python3", "-m", "build"]
 
     wenv = os.environ.copy()
     wenv["VERSION"] = FLAGS.triton_version
     wenv["TRITON_PYBIND"] = PYBIND_LIB
     p = subprocess.Popen(args, env=wenv)
     p.wait()
-    fail_if(p.returncode != 0, "setup.py failed")
+    fail_if(p.returncode != 0, "Building wheel failed failed")
 
     cpdir("dist", FLAGS.dest_dir)
 

python/setup.py

@@ -25,90 +25,23 @@
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-import os
-import sys
-from itertools import chain
 
-from setuptools import find_packages, setup
+import subprocess
 
-if "--plat-name" in sys.argv:
-    PLATFORM_FLAG = sys.argv[sys.argv.index("--plat-name") + 1]
-else:
-    PLATFORM_FLAG = "any"
+from setuptools import setup
+from setuptools.command.build_py import build_py
 
-if "VERSION" not in os.environ:
-    raise Exception("envvar VERSION must be specified")
 
-VERSION = os.environ["VERSION"]
+class BuildPyCommand(build_py):
+    def run(self):
+        build_py.run(self)
+        # Generate stub files:
+        package_name = self.distribution.metadata.name
+        subprocess.run(
+            ["stubgen", "-p", f"{package_name}._c", "-o", f"{self.build_lib}"],
+            check=True,
+        )
 
-try:
-    from wheel.bdist_wheel import bdist_wheel as _bdist_wheel
 
-    class bdist_wheel(_bdist_wheel):
-        def finalize_options(self):
-            _bdist_wheel.finalize_options(self)
-            self.root_is_pure = False
-
-        def get_tag(self):
-            pyver, abi, plat = "py3", "none", PLATFORM_FLAG
-            return pyver, abi, plat
-
-except ImportError:
-    bdist_wheel = None
-
-this_directory = os.path.abspath(os.path.dirname(__file__))
-
-data_files = [
-    ("", ["LICENSE.txt"]),
-]
-
-# Type checking marker file indicating support for type checkers.
-# https://peps.python.org/pep-0561/
-# Type hints for c extension generated by mypy
-platform_package_data = [
-    os.environ["TRITON_PYBIND"],
-    "py.typed",
-    "_c/__init__.pyi",
-    "_c/triton_bindings.pyi",
-]
-
-gpu_extras = ["cupy-cuda12x"]
-test_extras = ["pytest"]
-all_extras = gpu_extras + test_extras
-
-setup(
-    name="tritonserver",
-    version=VERSION,
-    author="NVIDIA Inc.",
-    author_email="[email protected]",
-    description="Triton Inference Server In-Process Python API",
-    license="BSD",
-    url="https://developer.nvidia.com/nvidia-triton-inference-server",
-    classifiers=[
-        "Development Status :: 5 - Production/Stable",
-        "Intended Audience :: Developers",
-        "Intended Audience :: Science/Research",
-        "Intended Audience :: Information Technology",
-        "Topic :: Scientific/Engineering",
-        "Topic :: Scientific/Engineering :: Image Recognition",
-        "Topic :: Scientific/Engineering :: Artificial Intelligence",
-        "Topic :: Software Development :: Libraries",
-        "Topic :: Utilities",
-        "License :: OSI Approved :: BSD License",
-        "Programming Language :: Python :: 3",
-        "Programming Language :: Python :: 3.10",
-        "Programming Language :: Python :: 3.12",
-        "Environment :: Console",
-        "Natural Language :: English",
-        "Operating System :: OS Independent",
-    ],
-    packages=find_packages(),
-    package_data={
-        "": platform_package_data,
-    },
-    zip_safe=False,
-    cmdclass={"bdist_wheel": bdist_wheel},
-    data_files=data_files,
-    install_requires=["numpy<2"],
-    extras_require={"GPU": gpu_extras, "test": test_extras, "all": all_extras},
-)
+if __name__ == "__main__":
+    setup(cmdclass={"build_py": BuildPyCommand})