From e62131d8ec975354d63f68dcf8675a3b82ccd384 Mon Sep 17 00:00:00 2001 From: Frost Ming Date: Wed, 9 Oct 2024 17:56:37 +0800 Subject: [PATCH 1/6] feat: v2 bento info to support image refactor Signed-off-by: Frost Ming --- src/_bentoml_sdk/images.py | 105 ++++++++++++++++++ src/_bentoml_sdk/service/factory.py | 20 +++- src/bentoml/_internal/bento/bento.py | 94 ++++++++++------ .../_internal/cloud/schemas/modelschemas.py | 6 +- src/bentoml/_internal/container/__init__.py | 4 +- .../container/frontend/dockerfile/__init__.py | 6 +- tests/unit/_internal/bento/test_bento.py | 3 +- 7 files changed, 195 insertions(+), 43 deletions(-) create mode 100644 src/_bentoml_sdk/images.py diff --git a/src/_bentoml_sdk/images.py b/src/_bentoml_sdk/images.py new file mode 100644 index 00000000000..f97b74caa3d --- /dev/null +++ b/src/_bentoml_sdk/images.py @@ -0,0 +1,105 @@ +from __future__ import annotations + +import sys +import typing as t +from pathlib import Path + +import attrs + +from bentoml._internal.bento.bento import ImageInfo +from bentoml._internal.container.frontend.dockerfile import CONTAINER_METADATA +from bentoml._internal.container.frontend.dockerfile import CONTAINER_SUPPORTED_DISTROS +from bentoml.exceptions import BentoMLConfigException + +DEFAULT_PYTHON_VERSION = f"{sys.version_info.major}.{sys.version_info.minor}" + + +@attrs.define +class Image: + """A class defining the environment requirements for bento.""" + + base_image: str + python_version: str = DEFAULT_PYTHON_VERSION + commands: t.List[str] = attrs.field(factory=list) + python_requirements: str = "" + + def requirements_file(self, file_path: str) -> t.Self: + """Add a requirements file to the image. Supports chaining call. + + Example: + + .. code-block:: python + + image = Image("debian:latest").requirements_file("requirements.txt") + """ + self.python_requirements += Path(file_path).read_text() + return self + + def python_packages(self, *packages: str) -> t.Self: + """Add python dependencies to the image. Supports chaining call. + + Example: + + .. code-block:: python + + image = Image("debian:latest")\ + .python_packages("numpy", "pandas")\ + .requirements_file("requirements.txt") + """ + self.python_requirements += "\n".join(packages) + return self + + def run(self, command: str) -> t.Self: + """Add a command to the image. Supports chaining call. + + Example: + + .. code-block:: python + + image = Image("debian:latest").run("echo 'Hello, World!'") + """ + self.commands.append(command) + return self + + def freeze(self) -> ImageInfo: + """Freeze the image to an ImageInfo object for build.""" + return ImageInfo( + base_image=self.base_image, + python_version=self.python_version, + commands=self.commands, + python_requirements=self.python_requirements, + ) + + +@attrs.define +class PythonImage(Image): + base_image: str = "" + distro: str = "debian" + _original_base_image: str = attrs.field(init=False, default="") + + def __attrs_post_init__(self) -> None: + self._original_base_image = self.base_image + if not self.base_image: + if self.distro not in CONTAINER_METADATA: + raise BentoMLConfigException( + f"Unsupported distro: {self.distro}, expected one of {CONTAINER_SUPPORTED_DISTROS}" + ) + + self.base_image = CONTAINER_METADATA[self.distro]["python"]["image"].format( + spec_version=self.python_version + ) + + def system_packages(self, *packages: str) -> t.Self: + if self._original_base_image: + raise BentoMLConfigException( + "system_packages() can only be used in default base image" + ) + if self.distro not in CONTAINER_METADATA: + raise BentoMLConfigException( + f"Unsupported distro: {self.distro}, expected one of {CONTAINER_SUPPORTED_DISTROS}" + ) + self.commands.append( + CONTAINER_METADATA[self.distro]["install_command"].format( + packages=" ".join(packages) + ) + ) diff --git a/src/_bentoml_sdk/service/factory.py b/src/_bentoml_sdk/service/factory.py index 382ac1d304a..91af95936b4 100644 --- a/src/_bentoml_sdk/service/factory.py +++ b/src/_bentoml_sdk/service/factory.py @@ -27,6 +27,7 @@ from bentoml.exceptions import BentoMLConfigException from bentoml.exceptions import BentoMLException +from ..images import Image from ..method import APIMethod from ..models import BentoModel from ..models import HuggingFaceModel @@ -67,6 +68,7 @@ class Service(t.Generic[T]): config: Config inner: type[T] + image: t.Optional[Image] = None bento: t.Optional[Bento] = attrs.field(init=False, default=None) models: list[Model[t.Any]] = attrs.field(factory=list) @@ -406,10 +408,18 @@ def service(inner: type[T], /) -> Service[T]: ... @t.overload -def service(inner: None = ..., /, **kwargs: Unpack[Config]) -> _ServiceDecorator: ... - - -def service(inner: type[T] | None = None, /, **kwargs: Unpack[Config]) -> t.Any: +def service( + inner: None = ..., /, *, image: Image | None = None, **kwargs: Unpack[Config] +) -> _ServiceDecorator: ... + + +def service( + inner: type[T] | None = None, + /, + *, + image: Image | None = None, + **kwargs: Unpack[Config], +) -> t.Any: """Mark a class as a BentoML service. Example: @@ -425,7 +435,7 @@ def predict(self, input: str) -> str: def decorator(inner: type[T]) -> Service[T]: if isinstance(inner, Service): raise TypeError("service() decorator can only be applied once") - return Service(config=config, inner=inner) + return Service(config=config, inner=inner, image=image) return decorator(inner) if inner is not None else decorator diff --git a/src/bentoml/_internal/bento/bento.py b/src/bentoml/_internal/bento/bento.py index 3bfa2cb93c7..2a80005a548 100644 --- a/src/bentoml/_internal/bento/bento.py +++ b/src/bentoml/_internal/bento/bento.py @@ -143,7 +143,7 @@ class Bento(StoreItem): _tag: Tag = attr.field() __fs: FS = attr.field() - _info: BentoInfo + _info: BaseBentoInfo _model_store: ModelStore | None = None _doc: str | None = None @@ -176,7 +176,7 @@ def _fs(self) -> FS: return self.__fs @property - def info(self) -> BentoInfo: + def info(self) -> BaseBentoInfo: return self._info @property @@ -190,23 +190,28 @@ def get_manifest(self) -> BentoManifestSchema: info = self.info models = [str(m.tag) for m in info.all_models] - runners = [ - BentoRunnerSchema( - name=r.name, - runnable_type=r.runnable_type, - models=r.models, - resource_config=( - BentoRunnerResourceSchema( - cpu=r.resource_config.get("cpu"), - nvidia_gpu=r.resource_config.get("nvidia.com/gpu"), - custom_resources=r.resource_config.get("custom_resources"), - ) - if r.resource_config - else None - ), - ) - for r in info.runners - ] + if isinstance(info, BentoInfo): + runners = [ + BentoRunnerSchema( + name=r.name, + runnable_type=r.runnable_type, + models=r.models, + resource_config=( + BentoRunnerResourceSchema( + cpu=r.resource_config.get("cpu"), + nvidia_gpu=r.resource_config.get("nvidia.com/gpu"), + custom_resources=r.resource_config.get("custom_resources"), + ) + if r.resource_config + else None + ), + ) + for r in info.runners + ] + image: ImageInfo | None = None + else: + image = t.cast(ImageInfo, info.image) + runners = [] return BentoManifestSchema( name=info.name, entry_service=info.entry_service, @@ -220,6 +225,7 @@ def get_manifest(self) -> BentoManifestSchema: envs=info.envs, schema=info.schema, version=info.version, + image=image, ) @classmethod @@ -407,7 +413,7 @@ def append_model(model: BentoModelInfo) -> None: def from_fs(cls, item_fs: FS) -> Bento: try: with item_fs.open(BENTO_YAML_FILENAME, "r", encoding="utf-8") as bento_yaml: - info = BentoInfo.from_yaml_file(bento_yaml) + info = BaseBentoInfo.from_yaml_file(bento_yaml) except fs.errors.ResourceNotFound: raise BentoMLException( f"Failed to load bento because it does not contain a '{BENTO_YAML_FILENAME}'" @@ -659,7 +665,7 @@ def get_service_import_str(svc: Service | NewService[t.Any] | str) -> str: @attr.frozen(repr=False) -class BentoInfo: +class BaseBentoInfo: # for backward compatibility in case new fields are added to BentoInfo. __forbid_extra_keys__ = False # omit field in yaml file if it is not provided by the user. @@ -677,18 +683,12 @@ class BentoInfo: factory=dict, converter=normalize_labels_value ) models: t.List[BentoModelInfo] = attr.field(factory=list) - runners: t.List[BentoRunnerInfo] = attr.field(factory=list) # for BentoML 1.2+ SDK entry_service: str = attr.field(factory=str) services: t.List[BentoServiceInfo] = attr.field(factory=list) envs: t.List[BentoEnvSchema] = attr.field(factory=list) schema: t.Dict[str, t.Any] = attr.field(factory=dict) - apis: t.List[BentoApiInfo] = attr.field(factory=list) - docker: DockerOptions = attr.field(factory=lambda: DockerOptions().with_defaults()) - python: PythonOptions = attr.field(factory=lambda: PythonOptions().with_defaults()) - conda: CondaOptions = attr.field(factory=lambda: CondaOptions().with_defaults()) - @property def all_models(self) -> t.List[BentoModelInfo]: model_map = {model.tag: model for model in self.models} @@ -715,7 +715,7 @@ def dump(self, stream: t.IO[t.Any]) -> None: return yaml.safe_dump(self.to_dict(), stream, sort_keys=False) @classmethod - def from_yaml_file(cls, stream: t.IO[t.Any]) -> BentoInfo: + def from_yaml_file(cls, stream: t.IO[t.Any]) -> BaseBentoInfo: try: yaml_content = yaml.safe_load(stream) except yaml.YAMLError as exc: @@ -752,8 +752,12 @@ def from_yaml_file(cls, stream: t.IO[t.Any]) -> BentoInfo: models, ) ) + if yaml_content.get("spec", 1) == 2: + klass = BentoInfoV2 + else: + klass = BentoInfo try: - return bentoml_cattr.structure(yaml_content, cls) + return bentoml_cattr.structure(yaml_content, klass) except KeyError as e: raise BentoMLException(f"Missing field {e} in {BENTO_YAML_FILENAME}") @@ -762,6 +766,30 @@ def validate(self): ... +@attr.frozen(repr=False) +class BentoInfo(BaseBentoInfo): + spec: int = attr.field(default=1, init=False) + runners: t.List[BentoRunnerInfo] = attr.field(factory=list) + apis: t.List[BentoApiInfo] = attr.field(factory=list) + docker: DockerOptions = attr.field(factory=lambda: DockerOptions().with_defaults()) + python: PythonOptions = attr.field(factory=lambda: PythonOptions().with_defaults()) + conda: CondaOptions = attr.field(factory=lambda: CondaOptions().with_defaults()) + + +@attr.frozen(repr=False) +class ImageInfo: + base_image: str = "" + python_version: str = "" + commands: t.List[str] = attr.field(factory=list) + python_requirements: str = "" + + +@attr.frozen(repr=False) +class BentoInfoV2(BaseBentoInfo): + spec: int = attr.field(default=2, init=False) + image: ImageInfo = attr.field(factory=ImageInfo) + + bentoml_cattr.register_unstructure_hook( BentoDependencyInfo, make_dict_unstructure_fn( @@ -788,9 +816,9 @@ def _convert_bento_dependency_info( ) bentoml_cattr.register_structure_hook_func( - lambda cls: inspect.isclass(cls) and issubclass(cls, BentoInfo), + lambda cls: inspect.isclass(cls) and issubclass(cls, BaseBentoInfo), make_dict_structure_fn( - BentoInfo, + BaseBentoInfo, bentoml_cattr, name=override(omit=True), version=override(omit=True), @@ -807,7 +835,7 @@ def _convert_bento_dependency_info( ), ) bentoml_cattr.register_unstructure_hook( - BentoInfo, + BaseBentoInfo, # Ignore tag, tag is saved via the name and version field - make_dict_unstructure_fn(BentoInfo, bentoml_cattr, tag=override(omit=True)), + make_dict_unstructure_fn(BaseBentoInfo, bentoml_cattr, tag=override(omit=True)), ) diff --git a/src/bentoml/_internal/cloud/schemas/modelschemas.py b/src/bentoml/_internal/cloud/schemas/modelschemas.py index 4ef3e287b3b..ec781a7e293 100644 --- a/src/bentoml/_internal/cloud/schemas/modelschemas.py +++ b/src/bentoml/_internal/cloud/schemas/modelschemas.py @@ -7,6 +7,7 @@ import attr from ...bento.bento import BentoServiceInfo +from ...bento.bento import ImageInfo from ...bento.build_config import BentoEnvSchema from ...cloud.schemas.utils import dict_options_converter from ...tag import Tag @@ -77,8 +78,8 @@ class BentoManifestSchema: service: str bentoml_version: str = attr.field(eq=False) size_bytes: int = attr.field(eq=False) - entry_service: str = attr.field(default="") - name: t.Optional[str] = attr.field(default=None) + entry_service: str = "" + name: t.Optional[str] = None apis: t.Dict[str, BentoApiSchema] = attr.field(factory=dict) models: t.List[str] = attr.field(factory=list, eq=False) runners: t.Optional[t.List[BentoRunnerSchema]] = attr.field(factory=list) @@ -87,6 +88,7 @@ class BentoManifestSchema: schema: t.Dict[str, t.Any] = attr.field(factory=dict) version: t.Optional[str] = attr.field(default=None, eq=False) dev: bool = attr.field(default=False, eq=False) + image: t.Optional[ImageInfo] = None @property def tag(self) -> Tag: diff --git a/src/bentoml/_internal/container/__init__.py b/src/bentoml/_internal/container/__init__.py index 03492176c09..adc532162cc 100644 --- a/src/bentoml/_internal/container/__init__.py +++ b/src/bentoml/_internal/container/__init__.py @@ -145,6 +145,7 @@ def construct_containerfile( from _bentoml_sdk.models import BentoModel from _bentoml_sdk.models import HuggingFaceModel + from ..bento.bento import BaseBentoInfo from ..bento.bento import BentoInfo from ..bento.build_config import DockerOptions @@ -154,7 +155,8 @@ def construct_containerfile( with fs.open_fs("temp://") as temp_fs: tempdir = temp_fs.getsyspath("/") with open(bento.path_of("bento.yaml"), "rb") as bento_yaml: - options = BentoInfo.from_yaml_file(bento_yaml) + options = BaseBentoInfo.from_yaml_file(bento_yaml) + assert isinstance(options, BentoInfo), "only v1 bento is supported" # tmpdir is our new build context. fs.mirror.mirror(bento._fs, temp_fs, copy_if_newer=True) diff --git a/src/bentoml/_internal/container/frontend/dockerfile/__init__.py b/src/bentoml/_internal/container/frontend/dockerfile/__init__.py index 31d5a2c1fca..1584654ddc5 100644 --- a/src/bentoml/_internal/container/frontend/dockerfile/__init__.py +++ b/src/bentoml/_internal/container/frontend/dockerfile/__init__.py @@ -21,7 +21,7 @@ # Python supported versions -SUPPORTED_PYTHON_VERSIONS = ["3.8", "3.9", "3.10", "3.11", "3.12"] +SUPPORTED_PYTHON_VERSIONS = ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"] # CUDA supported versions SUPPORTED_CUDA_VERSIONS = [ "12.0.0", @@ -72,6 +72,7 @@ "image": "amazonlinux:2", "supported_architectures": ["amd64", "arm64"], }, + "install_command": "yum update && yum install -y {packages}", }, "ubi8": { "supported_python_versions": ["3.8", "3.9"], @@ -84,6 +85,7 @@ "image": "nvidia/cuda:{spec_version}-cudnn8-runtime-ubi8", "supported_architectures": ["amd64", "arm64", "ppc64le"], }, + "install_command": "yum update && yum install -y {packages}", }, "debian": { "supported_python_versions": SUPPORTED_PYTHON_VERSIONS, @@ -100,6 +102,7 @@ "image": "continuumio/miniconda3:latest", "supported_architectures": SUPPORTED_ARCHITECTURES, }, + "install_command": "apt-get update && apt-get install -y {packages}", }, "alpine": { "supported_python_versions": SUPPORTED_PYTHON_VERSIONS, @@ -112,6 +115,7 @@ "image": "continuumio/miniconda3:4.10.3p0-alpine", "supported_architectures": ["amd64"], }, + "install_command": "apk update && apk add {packages}", }, } diff --git a/tests/unit/_internal/bento/test_bento.py b/tests/unit/_internal/bento/test_bento.py index c3374de29d5..09063b0d079 100644 --- a/tests/unit/_internal/bento/test_bento.py +++ b/tests/unit/_internal/bento/test_bento.py @@ -13,6 +13,7 @@ from bentoml import Tag from bentoml import bentos from bentoml._internal.bento import Bento +from bentoml._internal.bento.bento import BaseBentoInfo from bentoml._internal.bento.bento import BentoApiInfo from bentoml._internal.bento.bento import BentoInfo from bentoml._internal.bento.bento import BentoModelInfo @@ -146,7 +147,7 @@ def test_bento_info(tmpdir: Path): ) with open(bento_yaml_b_filename, encoding="utf-8") as bento_yaml_b: - bentoinfo_b_from_yaml = BentoInfo.from_yaml_file(bento_yaml_b) + bentoinfo_b_from_yaml = BaseBentoInfo.from_yaml_file(bento_yaml_b) assert bentoinfo_b_from_yaml == bentoinfo_b From 3f140997f54efd58001fb3bf5b413fa63059abe5 Mon Sep 17 00:00:00 2001 From: Frost Ming Date: Tue, 22 Oct 2024 18:16:35 +0800 Subject: [PATCH 2/6] feat: generate bento info for v2 Signed-off-by: Frost Ming --- src/_bentoml_sdk/images.py | 136 +++++++++++++++++++++++- src/bentoml/_internal/bento/bento.py | 82 +++++++++----- src/bentoml/_internal/utils/__init__.py | 2 +- 3 files changed, 190 insertions(+), 30 deletions(-) diff --git a/src/_bentoml_sdk/images.py b/src/_bentoml_sdk/images.py index f97b74caa3d..01017a92019 100644 --- a/src/_bentoml_sdk/images.py +++ b/src/_bentoml_sdk/images.py @@ -1,5 +1,8 @@ from __future__ import annotations +import logging +import platform +import subprocess import sys import typing as t from pathlib import Path @@ -7,11 +10,19 @@ import attrs from bentoml._internal.bento.bento import ImageInfo +from bentoml._internal.bento.build_config import BentoBuildConfig +from bentoml._internal.configuration import get_bentoml_requirement +from bentoml._internal.configuration import get_debug_mode +from bentoml._internal.configuration import get_quiet_mode from bentoml._internal.container.frontend.dockerfile import CONTAINER_METADATA from bentoml._internal.container.frontend.dockerfile import CONTAINER_SUPPORTED_DISTROS from bentoml.exceptions import BentoMLConfigException +from bentoml.exceptions import BentoMLException + +logger = logging.getLogger("bentoml.build") DEFAULT_PYTHON_VERSION = f"{sys.version_info.major}.{sys.version_info.minor}" +BENTOML_GIT_DEPENDENCY = "bentoml @ git+https://github.com/bentoml/bentoml.git" @attrs.define @@ -21,6 +32,7 @@ class Image: base_image: str python_version: str = DEFAULT_PYTHON_VERSION commands: t.List[str] = attrs.field(factory=list) + lock_python_packages: bool = True python_requirements: str = "" def requirements_file(self, file_path: str) -> t.Self: @@ -61,15 +73,72 @@ def run(self, command: str) -> t.Self: self.commands.append(command) return self - def freeze(self) -> ImageInfo: + def freeze(self, platform_: str | None = None) -> ImageInfo: """Freeze the image to an ImageInfo object for build.""" + if not self.lock_python_packages: + python_requirements = self.python_requirements + else: + python_requirements = self._freeze_python_requirements(platform_) return ImageInfo( base_image=self.base_image, python_version=self.python_version, commands=self.commands, - python_requirements=self.python_requirements, + python_requirements=python_requirements, ) + def _freeze_python_requirements(self, platform_: str | None = None) -> str: + from tempfile import TemporaryDirectory + + from pip_requirements_parser import RequirementsFile + + with TemporaryDirectory(prefix="bento-reqs-") as parent: + requirements_in = Path(parent).joinpath("requirements.in") + requirements_in.write_text(self.python_requirements) + # XXX: RequirementsFile.from_string() does not work due to bugs + requirements_file = RequirementsFile.from_file(str(requirements_in)) + has_bentoml_req = any( + req.name and req.name.lower() == "bentoml" + for req in requirements_file.requirements + ) + with requirements_in.open("w") as f: + f.write(requirements_file.dumps(preserve_one_empty_line=True)) + if not has_bentoml_req: + req = get_bentoml_requirement() or BENTOML_GIT_DEPENDENCY + f.write(f"{req}\n") + lock_args = [ + str(requirements_in), + "--allow-unsafe", + "--no-header", + f"--output-file={requirements_in.with_suffix('.lock')}", + "--emit-index-url", + "--emit-find-links", + "--no-annotate", + ] + if get_debug_mode(): + lock_args.append("--verbose") + else: + lock_args.append("--quiet") + logger.info("Locking PyPI package versions.") + if platform_: + lock_args.extend(["--python-platform", platform_]) + elif platform.system() != "Linux" or platform.machine() != "x86_64": + logger.info( + "Locking packages for x86_64-unknown-linux-gnu. " + "Pass `--platform` option to specify the platform." + ) + lock_args.extend(["--python-platform", "linux"]) + cmd = [sys.executable, "-m", "uv", "pip", "compile", *lock_args] + try: + subprocess.check_call( + cmd, + text=True, + stderr=subprocess.DEVNULL if get_quiet_mode() else None, + cwd=parent, + ) + except subprocess.CalledProcessError as e: + raise BentoMLException(f"Failed to lock PyPI packages: {e}") from None + return requirements_in.with_suffix(".lock").read_text() + @attrs.define class PythonImage(Image): @@ -103,3 +172,66 @@ def system_packages(self, *packages: str) -> t.Self: packages=" ".join(packages) ) ) + return self + + +def get_image_from_build_config(build_config: BentoBuildConfig) -> Image | None: + if not build_config.conda.is_empty(): + logger.warning( + "conda options are not supported by bento v2, fallback to bento v1" + ) + return None + image = PythonImage() + docker_options = build_config.docker + if docker_options.cuda_version is not None: + logger.warning( + "docker.cuda_version is not supported by bento v2, fallback to bento v1" + ) + return None + if docker_options.dockerfile_template is not None: + logger.warning( + "docker.dockerfile_template is not supported by bento v2, fallback to bento v1" + ) + return None + if docker_options.setup_script is not None: + logger.warning( + "docker.setup_script is not supported by bento v2, fallback to bento v1" + ) + return None + if docker_options.base_image is not None: + image.base_image = docker_options.base_image + if docker_options.distro is not None: + image.distro = docker_options.distro + if docker_options.python_version is not None: + image.python_version = docker_options.python_version + if docker_options.system_packages: + image.system_packages(*docker_options.system_packages) + + python_options = build_config.python.with_defaults() + if python_options.wheels: + logger.warning( + "python.wheels is not supported by bento v2, fallback to bento v1" + ) + return None + image.lock_python_packages = python_options.lock_packages + if python_options.index_url: + image.python_packages(f"--index-url {python_options.index_url}") + if python_options.no_index: + image.python_packages("--no-index") + if python_options.trusted_host: + image.python_packages( + *(f"--trusted-host {h}" for h in python_options.trusted_host) + ) + if python_options.extra_index_url: + image.python_packages( + *(f"--extra-index-url {url}" for url in python_options.extra_index_url) + ) + if python_options.find_links: + image.python_packages( + *(f"--find-links {link}" for link in python_options.find_links) + ) + if python_options.requirements_txt: + image.requirements_file(python_options.requirements_txt) + if python_options.packages: + image.python_packages(*python_options.packages) + return image diff --git a/src/bentoml/_internal/bento/bento.py b/src/bentoml/_internal/bento/bento.py index 2a80005a548..d3eb45574c6 100644 --- a/src/bentoml/_internal/bento/bento.py +++ b/src/bentoml/_internal/bento/bento.py @@ -1,6 +1,5 @@ from __future__ import annotations -import inspect import json import logging import os @@ -239,6 +238,8 @@ def create( bare: bool = False, reload: bool = False, ) -> Bento: + from _bentoml_sdk.images import Image + from _bentoml_sdk.images import get_image_from_build_config from _bentoml_sdk.models import BentoModel from ..service import Service @@ -264,7 +265,7 @@ def create( ) is_legacy = isinstance(svc, Service) # Apply default build options - build_config = build_config.with_defaults() + image: Image | None = None if isinstance(svc, Service): # for < 1.2 @@ -279,6 +280,10 @@ def create( if build_config.name is not None else to_snake_case(svc.name) ) + image = svc.image + if image is None: + image = get_image_from_build_config(build_config) + build_config = build_config.with_defaults() tag = Tag(bento_name, version) if version is None: tag = tag.make_new_version() @@ -335,12 +340,16 @@ def append_model(model: BentoModelInfo) -> None: logger.warn("File size is larger than 10MiB: %s", path) target_fs.makedirs(dir_path, recreate=True) copy_file(ctx_fs, path, target_fs, path) - - # NOTE: we need to generate both Python and Conda - # first to make sure we can generate the Dockerfile correctly. - build_config.python.write_to_bento(bento_fs, build_ctx, platform_=platform) - build_config.conda.write_to_bento(bento_fs, build_ctx) - build_config.docker.write_to_bento(bento_fs, build_ctx, build_config.conda) + if image is None: + # NOTE: we need to generate both Python and Conda + # first to make sure we can generate the Dockerfile correctly. + build_config.python.write_to_bento( + bento_fs, build_ctx, platform_=platform + ) + build_config.conda.write_to_bento(bento_fs, build_ctx) + build_config.docker.write_to_bento( + bento_fs, build_ctx, build_config.conda + ) # Create `readme.md` file if build_config.description is None: @@ -364,10 +373,8 @@ def append_model(model: BentoModelInfo) -> None: with bento_fs.open(fs.path.combine("apis", "schema.json"), "w") as f: json.dump(svc.schema(), f, indent=2) - res = Bento( - tag, - bento_fs, - BentoInfo( + if image is None: + bento_info = BentoInfo( tag=tag, service=svc, # type: ignore # attrs converters do not typecheck entry_service=svc.name, @@ -396,8 +403,28 @@ def append_model(model: BentoModelInfo) -> None: conda=build_config.conda, envs=build_config.envs, schema=svc.schema() if not is_legacy else {}, - ), - ) + ) + else: + bento_info = BentoInfoV2( + tag=tag, + service=svc, # type: ignore # attrs converters do not typecheck + entry_service=svc.name, + labels=build_config.labels, + models=models, + services=( + [ + BentoServiceInfo.from_service(s) + for s in svc.all_services().values() + ] + if not is_legacy + else [] + ), + envs=build_config.envs, + schema=svc.schema() if not is_legacy else {}, + image=image.freeze(platform), + ) + + res = Bento(tag, bento_fs, bento_info) if bare: return res # Create bento.yaml @@ -494,6 +521,7 @@ def total_size( def flush_info(self): with self._fs.open(BENTO_YAML_FILENAME, "w") as bento_yaml: + breakpoint() self.info.dump(bento_yaml) @property @@ -752,7 +780,7 @@ def from_yaml_file(cls, stream: t.IO[t.Any]) -> BaseBentoInfo: models, ) ) - if yaml_content.get("spec", 1) == 2: + if yaml_content.pop("spec", 1) == 2: klass = BentoInfoV2 else: klass = BentoInfo @@ -815,15 +843,6 @@ def _convert_bento_dependency_info( BentoDependencyInfo, _convert_bento_dependency_info ) -bentoml_cattr.register_structure_hook_func( - lambda cls: inspect.isclass(cls) and issubclass(cls, BaseBentoInfo), - make_dict_structure_fn( - BaseBentoInfo, - bentoml_cattr, - name=override(omit=True), - version=override(omit=True), - ), -) bentoml_cattr.register_unstructure_hook( BentoModelInfo, make_dict_unstructure_fn( @@ -834,8 +853,17 @@ def _convert_bento_dependency_info( metadata=override(omit_if_default=True), ), ) -bentoml_cattr.register_unstructure_hook( - BaseBentoInfo, +bentoml_cattr.register_structure_hook_factory( + lambda cls: issubclass(cls, BaseBentoInfo), + lambda cls: make_dict_structure_fn( + cls, + bentoml_cattr, + name=override(omit=True), + version=override(omit=True), + ), +) +bentoml_cattr.register_unstructure_hook_factory( + lambda cls: issubclass(cls, BaseBentoInfo), # Ignore tag, tag is saved via the name and version field - make_dict_unstructure_fn(BaseBentoInfo, bentoml_cattr, tag=override(omit=True)), + lambda cls: make_dict_unstructure_fn(cls, bentoml_cattr, tag=override(omit=True)), ) diff --git a/src/bentoml/_internal/utils/__init__.py b/src/bentoml/_internal/utils/__init__.py index a45c8a8a568..3c41fb15023 100644 --- a/src/bentoml/_internal/utils/__init__.py +++ b/src/bentoml/_internal/utils/__init__.py @@ -606,7 +606,7 @@ def warn_deprecated(message: str, stacklevel: int = 2) -> None: def deprecated( name: str = "", deprecated_since: str = "1.4", suggestion: str = "" -) -> t.Callable[[t.Callable[P, T]], t.Callable[P, T]]: +) -> t.Callable[[C], C]: def decorator(func: t.Callable[P, T]) -> t.Callable[P, T]: obj_name = name or func.__name__ From 77375b4be1f0aecf42e50047b7740a7401c71b21 Mon Sep 17 00:00:00 2001 From: Frost Ming Date: Wed, 23 Oct 2024 11:57:11 +0800 Subject: [PATCH 3/6] fix: dockerfile and containerize Signed-off-by: Frost Ming --- src/_bentoml_impl/docker.py | 60 +++++++++++++ src/bentoml/_internal/bento/bento.py | 24 ++++- src/bentoml/_internal/cloud/bento.py | 38 +------- src/bentoml/_internal/container/__init__.py | 75 +++++++++------- .../frontend/dockerfile/entrypoint.sh | 1 + .../frontend/dockerfile/templates/base_v2.j2 | 90 +++++++++++++++++++ src/bentoml/_internal/container/generate.py | 28 +++--- 7 files changed, 233 insertions(+), 83 deletions(-) create mode 100644 src/_bentoml_impl/docker.py create mode 100644 src/bentoml/_internal/container/frontend/dockerfile/templates/base_v2.j2 diff --git a/src/_bentoml_impl/docker.py b/src/_bentoml_impl/docker.py new file mode 100644 index 00000000000..7a9f0477491 --- /dev/null +++ b/src/_bentoml_impl/docker.py @@ -0,0 +1,60 @@ +from __future__ import annotations + +import importlib +import os +import shlex +import typing as t + +import attrs +from fs.base import FS +from jinja2 import Environment +from jinja2 import FileSystemLoader + +from bentoml._internal.bento.bento import ImageInfo +from bentoml._internal.container.generate import DEFAULT_BENTO_ENVS +from bentoml._internal.container.generate import PREHEAT_PIP_PACKAGES +from bentoml._internal.container.generate import expands_bento_path +from bentoml._internal.container.generate import resolve_package_versions +from bentoml._internal.container.generate import to_bento_field +from bentoml._internal.container.generate import to_options_field + + +def get_templates_variables( + image: ImageInfo, bento_fs: FS, **bento_envs: t.Any +) -> dict[str, t.Any]: + from bentoml._internal.configuration.containers import BentoMLContainer + + bento_envs = {**DEFAULT_BENTO_ENVS, **bento_envs} + options = attrs.asdict(image) + requirement_file = bento_fs.getsyspath("env/python/requirements.txt") + if os.path.exists(requirement_file): + python_packages = resolve_package_versions(requirement_file) + else: + python_packages = {} + pip_preheat_packages = [ + python_packages[k] for k in PREHEAT_PIP_PACKAGES if k in python_packages + ] + return { + **{to_options_field(k): v for k, v in options.items()}, + **{to_bento_field(k): v for k, v in bento_envs.items()}, + "__prometheus_port__": BentoMLContainer.grpc.metrics.port.get(), + "__pip_preheat_packages__": pip_preheat_packages, + } + + +def generate_dockerfile( + image: ImageInfo, bento_fs: FS, *, frontend: str = "dockerfile", **bento_envs: t.Any +) -> str: + templates_path = importlib.import_module( + f"bentoml._internal.container.frontend.{frontend}.templates" + ).__path__ + environment = Environment( + extensions=["jinja2.ext.do", "jinja2.ext.loopcontrols", "jinja2.ext.debug"], + trim_blocks=True, + lstrip_blocks=True, + loader=FileSystemLoader(templates_path, followlinks=True), + ) + environment.filters["bash_quote"] = shlex.quote + environment.globals["expands_bento_path"] = expands_bento_path + template = environment.get_template("base_v2.j2") + return template.render(**get_templates_variables(image, bento_fs, **bento_envs)) diff --git a/src/bentoml/_internal/bento/bento.py b/src/bentoml/_internal/bento/bento.py index d3eb45574c6..a9c8738c698 100644 --- a/src/bentoml/_internal/bento/bento.py +++ b/src/bentoml/_internal/bento/bento.py @@ -423,6 +423,7 @@ def append_model(model: BentoModelInfo) -> None: schema=svc.schema() if not is_legacy else {}, image=image.freeze(platform), ) + bento_info.image.write_to_bento(bento_fs, build_ctx) res = Bento(tag, bento_fs, bento_info) if bare: @@ -521,7 +522,6 @@ def total_size( def flush_info(self): with self._fs.open(BENTO_YAML_FILENAME, "w") as bento_yaml: - breakpoint() self.info.dump(bento_yaml) @property @@ -811,6 +811,28 @@ class ImageInfo: commands: t.List[str] = attr.field(factory=list) python_requirements: str = "" + def write_to_bento(self, bento_fs: FS, build_ctx: str) -> None: + from importlib import resources + + from _bentoml_impl.docker import generate_dockerfile + + py_folder = fs.path.join("env", "python") + bento_fs.makedirs(py_folder, recreate=True) + reqs_txt = fs.path.join(py_folder, "requirements.txt") + bento_fs.writetext(reqs_txt, self.python_requirements) + + docker_folder = fs.path.join("env", "docker") + bento_fs.makedirs(docker_folder, recreate=True) + dockerfile_path = fs.path.join(docker_folder, "Dockerfile") + bento_fs.writetext( + dockerfile_path, generate_dockerfile(self, bento_fs, enable_buildkit=False) + ) + + with resources.path( + "bentoml._internal.container.frontend.dockerfile", "entrypoint.sh" + ) as entrypoint_path: + copy_file_to_fs_folder(str(entrypoint_path), bento_fs, docker_folder) + @attr.frozen(repr=False) class BentoInfoV2(BaseBentoInfo): diff --git a/src/bentoml/_internal/cloud/bento.py b/src/bentoml/_internal/cloud/bento.py index f03bbfe9981..8272b451456 100644 --- a/src/bentoml/_internal/cloud/bento.py +++ b/src/bentoml/_internal/cloud/bento.py @@ -24,11 +24,7 @@ from .base import CallbackIOWrapper from .base import Spinner from .model import ModelAPI -from .schemas.modelschemas import BentoApiSchema -from .schemas.modelschemas import BentoRunnerResourceSchema -from .schemas.modelschemas import BentoRunnerSchema from .schemas.modelschemas import UploadStatus -from .schemas.schemasv1 import BentoManifestSchema from .schemas.schemasv1 import BentoSchema from .schemas.schemasv1 import CompleteMultipartUploadSchema from .schemas.schemasv1 import CompletePartSchema @@ -155,39 +151,7 @@ def push_model(model: Model[t.Any]) -> None: labels: list[LabelItemSchema] = [ LabelItemSchema(key=key, value=value) for key, value in info.labels.items() ] - apis: dict[str, BentoApiSchema] = {} - models = [str(m.tag) for m in info.all_models] - runners = [ - BentoRunnerSchema( - name=r.name, - runnable_type=r.runnable_type, - models=r.models, - resource_config=( - BentoRunnerResourceSchema( - cpu=r.resource_config.get("cpu"), - nvidia_gpu=r.resource_config.get("nvidia.com/gpu"), - custom_resources=r.resource_config.get("custom_resources"), - ) - if r.resource_config - else None - ), - ) - for r in info.runners - ] - manifest = BentoManifestSchema( - name=info.name, - entry_service=info.entry_service, - service=info.service, - bentoml_version=info.bentoml_version, - apis=apis, - models=models, - runners=runners, - size_bytes=bento.total_size(), - services=info.services, - envs=info.envs, - schema=info.schema, - dev=bare, - ) + manifest = bento.get_manifest() if not remote_bento: with self.spinner.spin( text=f'Registering Bento "{bento.tag}" with remote Bento store..' diff --git a/src/bentoml/_internal/container/__init__.py b/src/bentoml/_internal/container/__init__.py index adc532162cc..0398d839b44 100644 --- a/src/bentoml/_internal/container/__init__.py +++ b/src/bentoml/_internal/container/__init__.py @@ -147,6 +147,7 @@ def construct_containerfile( from ..bento.bento import BaseBentoInfo from ..bento.bento import BentoInfo + from ..bento.bento import BentoInfoV2 from ..bento.build_config import DockerOptions dockerfile_path = "env/docker/Dockerfile" @@ -156,7 +157,6 @@ def construct_containerfile( tempdir = temp_fs.getsyspath("/") with open(bento.path_of("bento.yaml"), "rb") as bento_yaml: options = BaseBentoInfo.from_yaml_file(bento_yaml) - assert isinstance(options, BentoInfo), "only v1 bento is supported" # tmpdir is our new build context. fs.mirror.mirror(bento._fs, temp_fs, copy_if_newer=True) @@ -175,37 +175,52 @@ def construct_containerfile( # Dockerfile inside bento, and it is not relevant to # construct_containerfile. Hence it is safe to set it to None here. # See https://github.com/bentoml/BentoML/issues/3399. - docker_attrs = bentoml_cattr.unstructure(options.docker) - if ( - "dockerfile_template" in docker_attrs - and docker_attrs["dockerfile_template"] is not None - ): - # NOTE: if users specify a dockerfile_template, we will - # save it to /env/docker/Dockerfile.template. This is necessary - # for the reconstruction of the Dockerfile. - docker_attrs["dockerfile_template"] = "env/docker/Dockerfile.template" - - dockerfile = generate_containerfile( - docker=DockerOptions(**docker_attrs), - build_ctx=tempdir, - conda=options.conda, - bento_fs=temp_fs, - enable_buildkit=enable_buildkit, - add_header=add_header, - ) - instruction.append(dockerfile) - if features is not None: - diff = set(features).difference(FEATURES) - if len(diff) > 0: - raise InvalidArgument( - f"Available features are: {FEATURES}. Invalid fields from provided: {diff}" - ) - PIP_CACHE_MOUNT = ( - "--mount=type=cache,target=/root/.cache/pip " if enable_buildkit else "" + if isinstance(options, BentoInfo): + docker_attrs = bentoml_cattr.unstructure(options.docker) + if ( + "dockerfile_template" in docker_attrs + and docker_attrs["dockerfile_template"] is not None + ): + # NOTE: if users specify a dockerfile_template, we will + # save it to /env/docker/Dockerfile.template. This is necessary + # for the reconstruction of the Dockerfile. + docker_attrs["dockerfile_template"] = "env/docker/Dockerfile.template" + + dockerfile = generate_containerfile( + docker=DockerOptions(**docker_attrs), + build_ctx=tempdir, + conda=options.conda, + bento_fs=temp_fs, + enable_buildkit=enable_buildkit, + add_header=add_header, ) - instruction.append( - "RUN %spip install bentoml[%s]" % (PIP_CACHE_MOUNT, ",".join(features)) + instruction.append(dockerfile) + if features is not None: + diff = set(features).difference(FEATURES) + if len(diff) > 0: + raise InvalidArgument( + f"Available features are: {FEATURES}. Invalid fields from provided: {diff}" + ) + PIP_CACHE_MOUNT = ( + "--mount=type=cache,target=/root/.cache/pip " + if enable_buildkit + else "" + ) + instruction.append( + "RUN %spip install bentoml[%s]" + % (PIP_CACHE_MOUNT, ",".join(features)) + ) + else: + from _bentoml_impl.docker import generate_dockerfile + + assert isinstance(options, BentoInfoV2) + dockerfile = generate_dockerfile( + options.image, + temp_fs, + enable_buildkit=enable_buildkit, + add_header=add_header, ) + instruction.append(dockerfile) temp_fs.writetext(dockerfile_path, "\n".join(instruction)) yield tempdir, temp_fs.getsyspath(dockerfile_path) diff --git a/src/bentoml/_internal/container/frontend/dockerfile/entrypoint.sh b/src/bentoml/_internal/container/frontend/dockerfile/entrypoint.sh index df1892dd022..3ec4e7c7776 100755 --- a/src/bentoml/_internal/container/frontend/dockerfile/entrypoint.sh +++ b/src/bentoml/_internal/container/frontend/dockerfile/entrypoint.sh @@ -10,6 +10,7 @@ _is_sourced() { } _main() { + [ -d .venv ] && source .venv/bin/activate # For backwards compatibility with the yatai<1.0.0, adapting the old "yatai" command to the new "start" command. if [ "${#}" -gt 0 ] && [ "${1}" = 'python' ] && [ "${2}" = '-m' ] && { [ "${3}" = 'bentoml._internal.server.cli.runner' ] || [ "${3}" = "bentoml._internal.server.cli.api_server" ]; }; then # SC2235, use { } to avoid subshell overhead if [ "${3}" = 'bentoml._internal.server.cli.runner' ]; then diff --git a/src/bentoml/_internal/container/frontend/dockerfile/templates/base_v2.j2 b/src/bentoml/_internal/container/frontend/dockerfile/templates/base_v2.j2 new file mode 100644 index 00000000000..eb89f606001 --- /dev/null +++ b/src/bentoml/_internal/container/frontend/dockerfile/templates/base_v2.j2 @@ -0,0 +1,90 @@ +{# BENTOML INTERNAL #} +{# users can use these values #} +{% import '_macros.j2' as common %} +{% set bento__entrypoint = expands_bento_path('env', 'docker', 'entrypoint.sh', bento_path=bento__path) %} +{% set __enable_buildkit__ = bento__enable_buildkit | default(False) -%} +{% if __enable_buildkit__ %} +# 1.2.1 is the current docker frontend that both buildkitd and kaniko supports. +# syntax = {{ bento__buildkit_frontend }} +# +{% endif %} +{% if bento__add_header %} +# =========================================== +# +# THIS IS A GENERATED DOCKERFILE. DO NOT EDIT +# +# =========================================== +{% endif %} + +# Block SETUP_BENTO_BASE_IMAGE +{% block SETUP_BENTO_BASE_IMAGE %} +FROM {{ __options__base_image }} AS base-container + +ENV LANG=C.UTF-8 + +ENV LC_ALL=C.UTF-8 + +ENV PYTHONIOENCODING=UTF-8 + +ENV PYTHONUNBUFFERED=1 +{% endblock %} + +{% block SETUP_BENTO_USER %} +ARG BENTO_USER={{ bento__user }} +ARG BENTO_USER_UID={{ bento__uid_gid }} +ARG BENTO_USER_GID={{ bento__uid_gid }} +RUN groupadd -g $BENTO_USER_GID -o $BENTO_USER && useradd -m -u $BENTO_USER_UID -g $BENTO_USER_GID -o -r $BENTO_USER +{% endblock %} + +{% block SETUP_BENTO_ENVARS %} + +ARG BENTO_PATH={{ bento__path }} +ENV BENTO_PATH=$BENTO_PATH +ENV BENTOML_HOME={{ bento__home }} +ENV BENTOML_HF_CACHE_DIR={{ bento__path }}/hf-models + +RUN mkdir $BENTO_PATH && chown {{ bento__user }}:{{ bento__user }} $BENTO_PATH -R +WORKDIR $BENTO_PATH +{% endblock %} + +{% block SETUP_BENTO_COMPONENTS %} +{% set required_packages = "curl git" %} +RUN command -v apt-get && apt-get update && apt-get -y install {{ required_packages }} \ + || command -v apk && apk update && apk add {{ required_packages }} \ + || yum update && yum install -y {{ required_packages }} \ + || true + +{% for command in __options__commands %} +RUN {{ command }} +{% endfor %} + +RUN curl -LO https://astral.sh/uv/install.sh && \ + sh install.sh && rm install.sh && mv $HOME/.cargo/bin/uv /usr/local/bin/ && uv venv --python {{ __options__python_version }} +{% set __pip_cache__ = common.mount_cache("/root/.cache/") %} +{% if __pip_preheat_packages__ %} +{% for value in __pip_preheat_packages__ -%} +{% call common.RUN(__enable_buildkit__) -%} {{ __pip_cache__ }} {% endcall -%} uv pip install {{ value|bash_quote }} ; exit 0 +{% endfor -%} +{% endif -%} + +COPY --chown={{ bento__user }}:{{ bento__user }} ./env/python ./env/python/ +# install python packages +{% call common.RUN(__enable_buildkit__) -%} {{ __pip_cache__ }} {% endcall -%} uv pip install -r ./env/python/requirements.txt +COPY --chown={{ bento__user }}:{{ bento__user }} . ./ +{% endblock %} + +# Block SETUP_BENTO_ENTRYPOINT +{% block SETUP_BENTO_ENTRYPOINT %} +# Default port for BentoServer +EXPOSE 3000 + +# Expose Prometheus port +EXPOSE {{ __prometheus_port__ }} + +RUN chmod +x {{ bento__entrypoint }} + +USER bentoml + +ENTRYPOINT [ "{{ bento__entrypoint }}" ] + +{% endblock %} diff --git a/src/bentoml/_internal/container/generate.py b/src/bentoml/_internal/container/generate.py index a25263c3986..d565bb0dc75 100644 --- a/src/bentoml/_internal/container/generate.py +++ b/src/bentoml/_internal/container/generate.py @@ -32,6 +32,16 @@ BENTO_PATH = f"{BENTO_HOME}bento" # 1.2.1 is the current docker frontend that both buildkitd and kaniko supports. BENTO_BUILDKIT_FRONTEND = "docker/dockerfile:1.2.1" +PREHEAT_PIP_PACKAGES = ["torch", "vllm"] +DEFAULT_BENTO_ENVS = { + "uid_gid": BENTO_UID_GID, + "user": BENTO_USER, + "home": BENTO_HOME, + "path": BENTO_PATH, + "add_header": True, + "buildkit_frontend": BENTO_BUILDKIT_FRONTEND, + "enable_buildkit": True, +} def expands_bento_path(*path: str, bento_path: str = BENTO_PATH) -> str: @@ -87,19 +97,7 @@ def get_templates_variables( base_image = spec.image.format(spec_version=docker.cuda_version) # bento__env - default_env = { - "uid_gid": BENTO_UID_GID, - "user": BENTO_USER, - "home": BENTO_HOME, - "path": BENTO_PATH, - "add_header": True, - "buildkit_frontend": BENTO_BUILDKIT_FRONTEND, - "enable_buildkit": True, - } - if bento_env: - default_env.update(bento_env) - - PREHEAT_PIP_PACKAGES = ["torch", "vllm"] + default_env = {**DEFAULT_BENTO_ENVS, **bento_env} return { **{to_options_field(k): v for k, v in docker.to_dict().items()}, @@ -204,7 +202,7 @@ def generate_containerfile( if not os.path.exists(requirement_file): requirement_file = bento_fs.getsyspath("env/python/requirements.txt") if os.path.exists(requirement_file): - python_packages = _resolve_package_versions(requirement_file) + python_packages = resolve_package_versions(requirement_file) else: python_packages = {} @@ -220,7 +218,7 @@ def generate_containerfile( ) -def _resolve_package_versions(requirement: str) -> dict[str, str]: +def resolve_package_versions(requirement: str) -> dict[str, str]: from pip_requirements_parser import RequirementsFile requirements_txt = RequirementsFile.from_file( From 6c0e9a85883fdd5349d287f95fab05a578fb337b Mon Sep 17 00:00:00 2001 From: Frost Ming Date: Wed, 23 Oct 2024 15:32:56 +0800 Subject: [PATCH 4/6] feat: inject build envs Signed-off-by: Frost Ming --- src/_bentoml_sdk/service/factory.py | 17 ++++++++++++++--- src/bentoml/_internal/bento/bento.py | 15 ++++++++++----- src/bentoml/_internal/bento/build_config.py | 2 +- src/bentoml/_internal/container/__init__.py | 1 + .../frontend/dockerfile/templates/base_v2.j2 | 7 +++++++ tests/unit/_internal/bento/test_bento.py | 9 +++++---- 6 files changed, 38 insertions(+), 13 deletions(-) diff --git a/src/_bentoml_sdk/service/factory.py b/src/_bentoml_sdk/service/factory.py index 91af95936b4..5654bcf3c9f 100644 --- a/src/_bentoml_sdk/service/factory.py +++ b/src/_bentoml_sdk/service/factory.py @@ -19,6 +19,7 @@ from bentoml import Runner from bentoml._internal.bento.bento import Bento +from bentoml._internal.bento.build_config import BentoEnvSchema from bentoml._internal.configuration.containers import BentoMLContainer from bentoml._internal.context import ServiceContext from bentoml._internal.models import Model as StoredModel @@ -62,6 +63,10 @@ def wrapper(self: Service[t.Any], *args: P.args, **kwargs: P.kwargs) -> R: return wrapper +def convert_envs(envs: t.List[t.Dict[str, t.Any]]) -> t.List[BentoEnvSchema]: + return [BentoEnvSchema(**env) for env in envs] + + @attrs.define class Service(t.Generic[T]): """A Bentoml service that can be served by BentoML server.""" @@ -69,7 +74,7 @@ class Service(t.Generic[T]): config: Config inner: type[T] image: t.Optional[Image] = None - + envs: t.List[BentoEnvSchema] = attrs.field(factory=list, converter=convert_envs) bento: t.Optional[Bento] = attrs.field(init=False, default=None) models: list[Model[t.Any]] = attrs.field(factory=list) apis: dict[str, APIMethod[..., t.Any]] = attrs.field(factory=dict) @@ -409,7 +414,12 @@ def service(inner: type[T], /) -> Service[T]: ... @t.overload def service( - inner: None = ..., /, *, image: Image | None = None, **kwargs: Unpack[Config] + inner: None = ..., + /, + *, + image: Image | None = None, + envs: list[dict[str, t.Any]] | None = None, + **kwargs: Unpack[Config], ) -> _ServiceDecorator: ... @@ -418,6 +428,7 @@ def service( /, *, image: Image | None = None, + envs: list[dict[str, t.Any]] | None = None, **kwargs: Unpack[Config], ) -> t.Any: """Mark a class as a BentoML service. @@ -435,7 +446,7 @@ def predict(self, input: str) -> str: def decorator(inner: type[T]) -> Service[T]: if isinstance(inner, Service): raise TypeError("service() decorator can only be applied once") - return Service(config=config, inner=inner, image=image) + return Service(config=config, inner=inner, image=image, envs=envs or []) return decorator(inner) if inner is not None else decorator diff --git a/src/bentoml/_internal/bento/bento.py b/src/bentoml/_internal/bento/bento.py index a9c8738c698..23b9846a90a 100644 --- a/src/bentoml/_internal/bento/bento.py +++ b/src/bentoml/_internal/bento/bento.py @@ -237,6 +237,7 @@ def create( platform: t.Optional[str] = None, bare: bool = False, reload: bool = False, + enabled_features: list[str] = Provide[BentoMLContainer.enabled_features], ) -> Bento: from _bentoml_sdk.images import Image from _bentoml_sdk.images import get_image_from_build_config @@ -250,6 +251,7 @@ def create( if build_ctx is None else os.path.realpath(os.path.expanduser(build_ctx)) ) + enable_image = "bento_image" in enabled_features if not os.path.isdir(build_ctx): raise InvalidArgument( f"Bento build context {build_ctx} does not exist or is not a directory." @@ -280,8 +282,10 @@ def create( if build_config.name is not None else to_snake_case(svc.name) ) - image = svc.image - if image is None: + build_config.envs.extend(svc.envs) + if enable_image: + image = svc.image + if image is None and enable_image: image = get_image_from_build_config(build_config) build_config = build_config.with_defaults() tag = Tag(bento_name, version) @@ -423,7 +427,7 @@ def append_model(model: BentoModelInfo) -> None: schema=svc.schema() if not is_legacy else {}, image=image.freeze(platform), ) - bento_info.image.write_to_bento(bento_fs, build_ctx) + bento_info.image.write_to_bento(bento_fs, build_config.envs) res = Bento(tag, bento_fs, bento_info) if bare: @@ -811,7 +815,7 @@ class ImageInfo: commands: t.List[str] = attr.field(factory=list) python_requirements: str = "" - def write_to_bento(self, bento_fs: FS, build_ctx: str) -> None: + def write_to_bento(self, bento_fs: FS, envs: list[BentoEnvSchema]) -> None: from importlib import resources from _bentoml_impl.docker import generate_dockerfile @@ -825,7 +829,8 @@ def write_to_bento(self, bento_fs: FS, build_ctx: str) -> None: bento_fs.makedirs(docker_folder, recreate=True) dockerfile_path = fs.path.join(docker_folder, "Dockerfile") bento_fs.writetext( - dockerfile_path, generate_dockerfile(self, bento_fs, enable_buildkit=False) + dockerfile_path, + generate_dockerfile(self, bento_fs, enable_buildkit=False, envs=envs), ) with resources.path( diff --git a/src/bentoml/_internal/bento/build_config.py b/src/bentoml/_internal/bento/build_config.py index 52ab382211e..ce55ea9f8ca 100644 --- a/src/bentoml/_internal/bento/build_config.py +++ b/src/bentoml/_internal/bento/build_config.py @@ -1013,4 +1013,4 @@ class FilledBentoBuildConfig(BentoBuildConfig): python: PythonOptions conda: CondaOptions models: t.List[ModelSpec] - envs: t.List[t.Dict[str, str]] + envs: t.List[BentoEnvSchema] diff --git a/src/bentoml/_internal/container/__init__.py b/src/bentoml/_internal/container/__init__.py index 0398d839b44..1746216d21c 100644 --- a/src/bentoml/_internal/container/__init__.py +++ b/src/bentoml/_internal/container/__init__.py @@ -219,6 +219,7 @@ def construct_containerfile( temp_fs, enable_buildkit=enable_buildkit, add_header=add_header, + envs=options.envs, ) instruction.append(dockerfile) temp_fs.writetext(dockerfile_path, "\n".join(instruction)) diff --git a/src/bentoml/_internal/container/frontend/dockerfile/templates/base_v2.j2 b/src/bentoml/_internal/container/frontend/dockerfile/templates/base_v2.j2 index eb89f606001..ec1936a5f4d 100644 --- a/src/bentoml/_internal/container/frontend/dockerfile/templates/base_v2.j2 +++ b/src/bentoml/_internal/container/frontend/dockerfile/templates/base_v2.j2 @@ -3,6 +3,7 @@ {% import '_macros.j2' as common %} {% set bento__entrypoint = expands_bento_path('env', 'docker', 'entrypoint.sh', bento_path=bento__path) %} {% set __enable_buildkit__ = bento__enable_buildkit | default(False) -%} +{% set __bento_envs__ = bento__envs | default([]) %} {% if __enable_buildkit__ %} # 1.2.1 is the current docker frontend that both buildkitd and kaniko supports. # syntax = {{ bento__buildkit_frontend }} @@ -43,6 +44,12 @@ ENV BENTO_PATH=$BENTO_PATH ENV BENTOML_HOME={{ bento__home }} ENV BENTOML_HF_CACHE_DIR={{ bento__path }}/hf-models +{% for env in __bento_envs__ %} +ARG {{ env.name }}{% if env.value %}={{ env.value }}{% endif %} + +ENV {{ env.name }}=${{ env.name }} +{% endfor %} + RUN mkdir $BENTO_PATH && chown {{ bento__user }}:{{ bento__user }} $BENTO_PATH -R WORKDIR $BENTO_PATH {% endblock %} diff --git a/tests/unit/_internal/bento/test_bento.py b/tests/unit/_internal/bento/test_bento.py index 09063b0d079..26834509add 100644 --- a/tests/unit/_internal/bento/test_bento.py +++ b/tests/unit/_internal/bento/test_bento.py @@ -94,6 +94,11 @@ def test_bento_info(tmpdir: Path): module: model_b_module creation_time: '{model_creation_time}' alias: model_b_alias +entry_service: '' +services: [] +envs: [] +schema: {{}} +spec: 1 runners: - name: runner_a runnable_type: test_runnable_a @@ -102,10 +107,6 @@ def test_bento_info(tmpdir: Path): - runner_a_model resource_config: cpu: 2 -entry_service: '' -services: [] -envs: [] -schema: {{}} apis: - name: predict input_type: NumpyNdarray From c582dd6bcc15e014ed3db9ed65c6599cec170375 Mon Sep 17 00:00:00 2001 From: Frost Ming Date: Thu, 24 Oct 2024 08:47:12 +0800 Subject: [PATCH 5/6] fix: drop python38 Signed-off-by: Frost Ming --- src/_bentoml_sdk/images.py | 6 ++++-- .../_internal/container/frontend/dockerfile/__init__.py | 4 ++-- src/bentoml/_internal/container/generate.py | 2 +- src/bentoml_cli/containerize.py | 4 ++-- 4 files changed, 9 insertions(+), 7 deletions(-) diff --git a/src/_bentoml_sdk/images.py b/src/_bentoml_sdk/images.py index 01017a92019..9d409b4acef 100644 --- a/src/_bentoml_sdk/images.py +++ b/src/_bentoml_sdk/images.py @@ -153,9 +153,11 @@ def __attrs_post_init__(self) -> None: raise BentoMLConfigException( f"Unsupported distro: {self.distro}, expected one of {CONTAINER_SUPPORTED_DISTROS}" ) - + python_version = self.python_version + if self.distro in ("ubi8",): + python_version = python_version.replace(".", "") self.base_image = CONTAINER_METADATA[self.distro]["python"]["image"].format( - spec_version=self.python_version + spec_version=python_version ) def system_packages(self, *packages: str) -> t.Self: diff --git a/src/bentoml/_internal/container/frontend/dockerfile/__init__.py b/src/bentoml/_internal/container/frontend/dockerfile/__init__.py index 1584654ddc5..631c9465740 100644 --- a/src/bentoml/_internal/container/frontend/dockerfile/__init__.py +++ b/src/bentoml/_internal/container/frontend/dockerfile/__init__.py @@ -21,7 +21,7 @@ # Python supported versions -SUPPORTED_PYTHON_VERSIONS = ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"] +SUPPORTED_PYTHON_VERSIONS = ["3.9", "3.10", "3.11", "3.12", "3.13"] # CUDA supported versions SUPPORTED_CUDA_VERSIONS = [ "12.0.0", @@ -75,7 +75,7 @@ "install_command": "yum update && yum install -y {packages}", }, "ubi8": { - "supported_python_versions": ["3.8", "3.9"], + "supported_python_versions": SUPPORTED_PYTHON_VERSIONS, "supported_cuda_versions": SUPPORTED_CUDA_VERSIONS, "python": { "image": "registry.access.redhat.com/ubi8/python-{spec_version}:1", diff --git a/src/bentoml/_internal/container/generate.py b/src/bentoml/_internal/container/generate.py index d565bb0dc75..8282a2c19da 100644 --- a/src/bentoml/_internal/container/generate.py +++ b/src/bentoml/_internal/container/generate.py @@ -89,7 +89,7 @@ def get_templates_variables( spec = DistroSpec.from_options(docker, conda) python_version = docker.python_version assert docker.distro is not None and python_version is not None - if docker.distro in ("ubi8"): + if docker.distro in ("ubi8",): # ubi8 base images uses "py38" instead of "py3.8" in its image tag python_version = python_version.replace(".", "") base_image = spec.image.format(spec_version=python_version) diff --git a/src/bentoml_cli/containerize.py b/src/bentoml_cli/containerize.py index ce8b21a41da..ecdf3fca6d7 100644 --- a/src/bentoml_cli/containerize.py +++ b/src/bentoml_cli/containerize.py @@ -143,7 +143,7 @@ def decorator(f: F[t.Any]) -> t.Callable[[F[t.Any]], Command]: msg = attrs.pop("help", None) assert msg is not None, "'help' is required." attrs.setdefault("help", " ".join([prepend_msg, msg, append_msg])) - attrs.setdefault("expose_value", True) + attrs.setdefault("expose_value", False) attrs.setdefault("callback", obsolete_callback) return factory.option(*param_decls, **attrs)(f) @@ -419,6 +419,7 @@ def buildx_options_group(f: F[t.Any]): multiple=True, callback=opt_callback, metavar="ARG=VALUE[,ARG=VALUE]", + expose_value=False, ) @click.option( "--run-as-root", @@ -442,7 +443,6 @@ def containerize_command( # type: ignore enable_features: tuple[str] | None, run_as_root: bool, _memoized: dict[str, t.Any], - **kwargs: t.Any, # pylint: disable=unused-argument ) -> None: """Containerizes given Bento into an OCI-compliant container, with any given OCI builder. From ce982cfafbc24cb983356aa327449a2125dcb932 Mon Sep 17 00:00:00 2001 From: Frost Ming Date: Thu, 24 Oct 2024 12:25:59 +0800 Subject: [PATCH 6/6] fix: expose images module Signed-off-by: Frost Ming --- src/bentoml/__init__.py | 3 +++ src/bentoml/_internal/bento/bento.py | 6 ++++++ src/bentoml/images.py | 4 ++++ 3 files changed, 13 insertions(+) create mode 100644 src/bentoml/images.py diff --git a/src/bentoml/__init__.py b/src/bentoml/__init__.py index 3613c6e9b8d..6fbcdf35dd6 100644 --- a/src/bentoml/__init__.py +++ b/src/bentoml/__init__.py @@ -106,6 +106,7 @@ from _bentoml_sdk import api from _bentoml_sdk import depends from _bentoml_sdk import get_current_service + from _bentoml_sdk import images from _bentoml_sdk import mount_asgi_app from _bentoml_sdk import on_deployment from _bentoml_sdk import on_shutdown @@ -233,6 +234,7 @@ models = _LazyLoader("bentoml.models", globals(), "bentoml.models") metrics = _LazyLoader("bentoml.metrics", globals(), "bentoml.metrics") container = _LazyLoader("bentoml.container", globals(), "bentoml.container") + images = _LazyLoader("bentoml.images", globals(), "bentoml.images") client = _LazyLoader("bentoml.client", globals(), "bentoml.client") server = _LazyLoader("bentoml.server", globals(), "bentoml.server") exceptions = _LazyLoader("bentoml.exceptions", globals(), "bentoml.exceptions") @@ -353,6 +355,7 @@ def __getattr__(name: str) -> Any: "runner_service", "api", "task", + "images", "on_shutdown", "on_deployment", "depends", diff --git a/src/bentoml/_internal/bento/bento.py b/src/bentoml/_internal/bento/bento.py index 23b9846a90a..8cd76433445 100644 --- a/src/bentoml/_internal/bento/bento.py +++ b/src/bentoml/_internal/bento/bento.py @@ -285,6 +285,12 @@ def create( build_config.envs.extend(svc.envs) if enable_image: image = svc.image + elif svc.image is not None: + logger.warning( + "BentoML service %s has an image config, but BentoML image feature is not enabled. " + "Please enable it by setting BENTOML_ENABLE_FEATURES=bento_image", + svc.name, + ) if image is None and enable_image: image = get_image_from_build_config(build_config) build_config = build_config.with_defaults() diff --git a/src/bentoml/images.py b/src/bentoml/images.py new file mode 100644 index 00000000000..82555463caf --- /dev/null +++ b/src/bentoml/images.py @@ -0,0 +1,4 @@ +from _bentoml_sdk.images import Image +from _bentoml_sdk.images import PythonImage + +__all__ = ["PythonImage", "Image"]