diff --git a/src/_bentoml_impl/docker.py b/src/_bentoml_impl/docker.py new file mode 100644 index 00000000000..7a9f0477491 --- /dev/null +++ b/src/_bentoml_impl/docker.py @@ -0,0 +1,60 @@ +from __future__ import annotations + +import importlib +import os +import shlex +import typing as t + +import attrs +from fs.base import FS +from jinja2 import Environment +from jinja2 import FileSystemLoader + +from bentoml._internal.bento.bento import ImageInfo +from bentoml._internal.container.generate import DEFAULT_BENTO_ENVS +from bentoml._internal.container.generate import PREHEAT_PIP_PACKAGES +from bentoml._internal.container.generate import expands_bento_path +from bentoml._internal.container.generate import resolve_package_versions +from bentoml._internal.container.generate import to_bento_field +from bentoml._internal.container.generate import to_options_field + + +def get_templates_variables( + image: ImageInfo, bento_fs: FS, **bento_envs: t.Any +) -> dict[str, t.Any]: + from bentoml._internal.configuration.containers import BentoMLContainer + + bento_envs = {**DEFAULT_BENTO_ENVS, **bento_envs} + options = attrs.asdict(image) + requirement_file = bento_fs.getsyspath("env/python/requirements.txt") + if os.path.exists(requirement_file): + python_packages = resolve_package_versions(requirement_file) + else: + python_packages = {} + pip_preheat_packages = [ + python_packages[k] for k in PREHEAT_PIP_PACKAGES if k in python_packages + ] + return { + **{to_options_field(k): v for k, v in options.items()}, + **{to_bento_field(k): v for k, v in bento_envs.items()}, + "__prometheus_port__": BentoMLContainer.grpc.metrics.port.get(), + "__pip_preheat_packages__": pip_preheat_packages, + } + + +def generate_dockerfile( + image: ImageInfo, bento_fs: FS, *, frontend: str = "dockerfile", **bento_envs: t.Any +) -> str: + templates_path = importlib.import_module( + f"bentoml._internal.container.frontend.{frontend}.templates" + ).__path__ + environment = Environment( + extensions=["jinja2.ext.do", "jinja2.ext.loopcontrols", "jinja2.ext.debug"], + trim_blocks=True, + lstrip_blocks=True, + loader=FileSystemLoader(templates_path, followlinks=True), + ) + environment.filters["bash_quote"] = shlex.quote + environment.globals["expands_bento_path"] = expands_bento_path + template = environment.get_template("base_v2.j2") + return template.render(**get_templates_variables(image, bento_fs, **bento_envs)) diff --git a/src/bentoml/_internal/bento/bento.py b/src/bentoml/_internal/bento/bento.py index d3eb45574c6..a9c8738c698 100644 --- a/src/bentoml/_internal/bento/bento.py +++ b/src/bentoml/_internal/bento/bento.py @@ -423,6 +423,7 @@ def append_model(model: BentoModelInfo) -> None: schema=svc.schema() if not is_legacy else {}, image=image.freeze(platform), ) + bento_info.image.write_to_bento(bento_fs, build_ctx) res = Bento(tag, bento_fs, bento_info) if bare: @@ -521,7 +522,6 @@ def total_size( def flush_info(self): with self._fs.open(BENTO_YAML_FILENAME, "w") as bento_yaml: - breakpoint() self.info.dump(bento_yaml) @property @@ -811,6 +811,28 @@ class ImageInfo: commands: t.List[str] = attr.field(factory=list) python_requirements: str = "" + def write_to_bento(self, bento_fs: FS, build_ctx: str) -> None: + from importlib import resources + + from _bentoml_impl.docker import generate_dockerfile + + py_folder = fs.path.join("env", "python") + bento_fs.makedirs(py_folder, recreate=True) + reqs_txt = fs.path.join(py_folder, "requirements.txt") + bento_fs.writetext(reqs_txt, self.python_requirements) + + docker_folder = fs.path.join("env", "docker") + bento_fs.makedirs(docker_folder, recreate=True) + dockerfile_path = fs.path.join(docker_folder, "Dockerfile") + bento_fs.writetext( + dockerfile_path, generate_dockerfile(self, bento_fs, enable_buildkit=False) + ) + + with resources.path( + "bentoml._internal.container.frontend.dockerfile", "entrypoint.sh" + ) as entrypoint_path: + copy_file_to_fs_folder(str(entrypoint_path), bento_fs, docker_folder) + @attr.frozen(repr=False) class BentoInfoV2(BaseBentoInfo): diff --git a/src/bentoml/_internal/cloud/bento.py b/src/bentoml/_internal/cloud/bento.py index f03bbfe9981..8272b451456 100644 --- a/src/bentoml/_internal/cloud/bento.py +++ b/src/bentoml/_internal/cloud/bento.py @@ -24,11 +24,7 @@ from .base import CallbackIOWrapper from .base import Spinner from .model import ModelAPI -from .schemas.modelschemas import BentoApiSchema -from .schemas.modelschemas import BentoRunnerResourceSchema -from .schemas.modelschemas import BentoRunnerSchema from .schemas.modelschemas import UploadStatus -from .schemas.schemasv1 import BentoManifestSchema from .schemas.schemasv1 import BentoSchema from .schemas.schemasv1 import CompleteMultipartUploadSchema from .schemas.schemasv1 import CompletePartSchema @@ -155,39 +151,7 @@ def push_model(model: Model[t.Any]) -> None: labels: list[LabelItemSchema] = [ LabelItemSchema(key=key, value=value) for key, value in info.labels.items() ] - apis: dict[str, BentoApiSchema] = {} - models = [str(m.tag) for m in info.all_models] - runners = [ - BentoRunnerSchema( - name=r.name, - runnable_type=r.runnable_type, - models=r.models, - resource_config=( - BentoRunnerResourceSchema( - cpu=r.resource_config.get("cpu"), - nvidia_gpu=r.resource_config.get("nvidia.com/gpu"), - custom_resources=r.resource_config.get("custom_resources"), - ) - if r.resource_config - else None - ), - ) - for r in info.runners - ] - manifest = BentoManifestSchema( - name=info.name, - entry_service=info.entry_service, - service=info.service, - bentoml_version=info.bentoml_version, - apis=apis, - models=models, - runners=runners, - size_bytes=bento.total_size(), - services=info.services, - envs=info.envs, - schema=info.schema, - dev=bare, - ) + manifest = bento.get_manifest() if not remote_bento: with self.spinner.spin( text=f'Registering Bento "{bento.tag}" with remote Bento store..' diff --git a/src/bentoml/_internal/container/__init__.py b/src/bentoml/_internal/container/__init__.py index adc532162cc..0398d839b44 100644 --- a/src/bentoml/_internal/container/__init__.py +++ b/src/bentoml/_internal/container/__init__.py @@ -147,6 +147,7 @@ def construct_containerfile( from ..bento.bento import BaseBentoInfo from ..bento.bento import BentoInfo + from ..bento.bento import BentoInfoV2 from ..bento.build_config import DockerOptions dockerfile_path = "env/docker/Dockerfile" @@ -156,7 +157,6 @@ def construct_containerfile( tempdir = temp_fs.getsyspath("/") with open(bento.path_of("bento.yaml"), "rb") as bento_yaml: options = BaseBentoInfo.from_yaml_file(bento_yaml) - assert isinstance(options, BentoInfo), "only v1 bento is supported" # tmpdir is our new build context. fs.mirror.mirror(bento._fs, temp_fs, copy_if_newer=True) @@ -175,37 +175,52 @@ def construct_containerfile( # Dockerfile inside bento, and it is not relevant to # construct_containerfile. Hence it is safe to set it to None here. # See https://github.com/bentoml/BentoML/issues/3399. - docker_attrs = bentoml_cattr.unstructure(options.docker) - if ( - "dockerfile_template" in docker_attrs - and docker_attrs["dockerfile_template"] is not None - ): - # NOTE: if users specify a dockerfile_template, we will - # save it to /env/docker/Dockerfile.template. This is necessary - # for the reconstruction of the Dockerfile. - docker_attrs["dockerfile_template"] = "env/docker/Dockerfile.template" - - dockerfile = generate_containerfile( - docker=DockerOptions(**docker_attrs), - build_ctx=tempdir, - conda=options.conda, - bento_fs=temp_fs, - enable_buildkit=enable_buildkit, - add_header=add_header, - ) - instruction.append(dockerfile) - if features is not None: - diff = set(features).difference(FEATURES) - if len(diff) > 0: - raise InvalidArgument( - f"Available features are: {FEATURES}. Invalid fields from provided: {diff}" - ) - PIP_CACHE_MOUNT = ( - "--mount=type=cache,target=/root/.cache/pip " if enable_buildkit else "" + if isinstance(options, BentoInfo): + docker_attrs = bentoml_cattr.unstructure(options.docker) + if ( + "dockerfile_template" in docker_attrs + and docker_attrs["dockerfile_template"] is not None + ): + # NOTE: if users specify a dockerfile_template, we will + # save it to /env/docker/Dockerfile.template. This is necessary + # for the reconstruction of the Dockerfile. + docker_attrs["dockerfile_template"] = "env/docker/Dockerfile.template" + + dockerfile = generate_containerfile( + docker=DockerOptions(**docker_attrs), + build_ctx=tempdir, + conda=options.conda, + bento_fs=temp_fs, + enable_buildkit=enable_buildkit, + add_header=add_header, ) - instruction.append( - "RUN %spip install bentoml[%s]" % (PIP_CACHE_MOUNT, ",".join(features)) + instruction.append(dockerfile) + if features is not None: + diff = set(features).difference(FEATURES) + if len(diff) > 0: + raise InvalidArgument( + f"Available features are: {FEATURES}. Invalid fields from provided: {diff}" + ) + PIP_CACHE_MOUNT = ( + "--mount=type=cache,target=/root/.cache/pip " + if enable_buildkit + else "" + ) + instruction.append( + "RUN %spip install bentoml[%s]" + % (PIP_CACHE_MOUNT, ",".join(features)) + ) + else: + from _bentoml_impl.docker import generate_dockerfile + + assert isinstance(options, BentoInfoV2) + dockerfile = generate_dockerfile( + options.image, + temp_fs, + enable_buildkit=enable_buildkit, + add_header=add_header, ) + instruction.append(dockerfile) temp_fs.writetext(dockerfile_path, "\n".join(instruction)) yield tempdir, temp_fs.getsyspath(dockerfile_path) diff --git a/src/bentoml/_internal/container/frontend/dockerfile/entrypoint.sh b/src/bentoml/_internal/container/frontend/dockerfile/entrypoint.sh index df1892dd022..3ec4e7c7776 100755 --- a/src/bentoml/_internal/container/frontend/dockerfile/entrypoint.sh +++ b/src/bentoml/_internal/container/frontend/dockerfile/entrypoint.sh @@ -10,6 +10,7 @@ _is_sourced() { } _main() { + [ -d .venv ] && source .venv/bin/activate # For backwards compatibility with the yatai<1.0.0, adapting the old "yatai" command to the new "start" command. if [ "${#}" -gt 0 ] && [ "${1}" = 'python' ] && [ "${2}" = '-m' ] && { [ "${3}" = 'bentoml._internal.server.cli.runner' ] || [ "${3}" = "bentoml._internal.server.cli.api_server" ]; }; then # SC2235, use { } to avoid subshell overhead if [ "${3}" = 'bentoml._internal.server.cli.runner' ]; then diff --git a/src/bentoml/_internal/container/frontend/dockerfile/templates/base_v2.j2 b/src/bentoml/_internal/container/frontend/dockerfile/templates/base_v2.j2 new file mode 100644 index 00000000000..eb89f606001 --- /dev/null +++ b/src/bentoml/_internal/container/frontend/dockerfile/templates/base_v2.j2 @@ -0,0 +1,90 @@ +{# BENTOML INTERNAL #} +{# users can use these values #} +{% import '_macros.j2' as common %} +{% set bento__entrypoint = expands_bento_path('env', 'docker', 'entrypoint.sh', bento_path=bento__path) %} +{% set __enable_buildkit__ = bento__enable_buildkit | default(False) -%} +{% if __enable_buildkit__ %} +# 1.2.1 is the current docker frontend that both buildkitd and kaniko supports. +# syntax = {{ bento__buildkit_frontend }} +# +{% endif %} +{% if bento__add_header %} +# =========================================== +# +# THIS IS A GENERATED DOCKERFILE. DO NOT EDIT +# +# =========================================== +{% endif %} + +# Block SETUP_BENTO_BASE_IMAGE +{% block SETUP_BENTO_BASE_IMAGE %} +FROM {{ __options__base_image }} AS base-container + +ENV LANG=C.UTF-8 + +ENV LC_ALL=C.UTF-8 + +ENV PYTHONIOENCODING=UTF-8 + +ENV PYTHONUNBUFFERED=1 +{% endblock %} + +{% block SETUP_BENTO_USER %} +ARG BENTO_USER={{ bento__user }} +ARG BENTO_USER_UID={{ bento__uid_gid }} +ARG BENTO_USER_GID={{ bento__uid_gid }} +RUN groupadd -g $BENTO_USER_GID -o $BENTO_USER && useradd -m -u $BENTO_USER_UID -g $BENTO_USER_GID -o -r $BENTO_USER +{% endblock %} + +{% block SETUP_BENTO_ENVARS %} + +ARG BENTO_PATH={{ bento__path }} +ENV BENTO_PATH=$BENTO_PATH +ENV BENTOML_HOME={{ bento__home }} +ENV BENTOML_HF_CACHE_DIR={{ bento__path }}/hf-models + +RUN mkdir $BENTO_PATH && chown {{ bento__user }}:{{ bento__user }} $BENTO_PATH -R +WORKDIR $BENTO_PATH +{% endblock %} + +{% block SETUP_BENTO_COMPONENTS %} +{% set required_packages = "curl git" %} +RUN command -v apt-get && apt-get update && apt-get -y install {{ required_packages }} \ + || command -v apk && apk update && apk add {{ required_packages }} \ + || yum update && yum install -y {{ required_packages }} \ + || true + +{% for command in __options__commands %} +RUN {{ command }} +{% endfor %} + +RUN curl -LO https://astral.sh/uv/install.sh && \ + sh install.sh && rm install.sh && mv $HOME/.cargo/bin/uv /usr/local/bin/ && uv venv --python {{ __options__python_version }} +{% set __pip_cache__ = common.mount_cache("/root/.cache/") %} +{% if __pip_preheat_packages__ %} +{% for value in __pip_preheat_packages__ -%} +{% call common.RUN(__enable_buildkit__) -%} {{ __pip_cache__ }} {% endcall -%} uv pip install {{ value|bash_quote }} ; exit 0 +{% endfor -%} +{% endif -%} + +COPY --chown={{ bento__user }}:{{ bento__user }} ./env/python ./env/python/ +# install python packages +{% call common.RUN(__enable_buildkit__) -%} {{ __pip_cache__ }} {% endcall -%} uv pip install -r ./env/python/requirements.txt +COPY --chown={{ bento__user }}:{{ bento__user }} . ./ +{% endblock %} + +# Block SETUP_BENTO_ENTRYPOINT +{% block SETUP_BENTO_ENTRYPOINT %} +# Default port for BentoServer +EXPOSE 3000 + +# Expose Prometheus port +EXPOSE {{ __prometheus_port__ }} + +RUN chmod +x {{ bento__entrypoint }} + +USER bentoml + +ENTRYPOINT [ "{{ bento__entrypoint }}" ] + +{% endblock %} diff --git a/src/bentoml/_internal/container/generate.py b/src/bentoml/_internal/container/generate.py index a25263c3986..d565bb0dc75 100644 --- a/src/bentoml/_internal/container/generate.py +++ b/src/bentoml/_internal/container/generate.py @@ -32,6 +32,16 @@ BENTO_PATH = f"{BENTO_HOME}bento" # 1.2.1 is the current docker frontend that both buildkitd and kaniko supports. BENTO_BUILDKIT_FRONTEND = "docker/dockerfile:1.2.1" +PREHEAT_PIP_PACKAGES = ["torch", "vllm"] +DEFAULT_BENTO_ENVS = { + "uid_gid": BENTO_UID_GID, + "user": BENTO_USER, + "home": BENTO_HOME, + "path": BENTO_PATH, + "add_header": True, + "buildkit_frontend": BENTO_BUILDKIT_FRONTEND, + "enable_buildkit": True, +} def expands_bento_path(*path: str, bento_path: str = BENTO_PATH) -> str: @@ -87,19 +97,7 @@ def get_templates_variables( base_image = spec.image.format(spec_version=docker.cuda_version) # bento__env - default_env = { - "uid_gid": BENTO_UID_GID, - "user": BENTO_USER, - "home": BENTO_HOME, - "path": BENTO_PATH, - "add_header": True, - "buildkit_frontend": BENTO_BUILDKIT_FRONTEND, - "enable_buildkit": True, - } - if bento_env: - default_env.update(bento_env) - - PREHEAT_PIP_PACKAGES = ["torch", "vllm"] + default_env = {**DEFAULT_BENTO_ENVS, **bento_env} return { **{to_options_field(k): v for k, v in docker.to_dict().items()}, @@ -204,7 +202,7 @@ def generate_containerfile( if not os.path.exists(requirement_file): requirement_file = bento_fs.getsyspath("env/python/requirements.txt") if os.path.exists(requirement_file): - python_packages = _resolve_package_versions(requirement_file) + python_packages = resolve_package_versions(requirement_file) else: python_packages = {} @@ -220,7 +218,7 @@ def generate_containerfile( ) -def _resolve_package_versions(requirement: str) -> dict[str, str]: +def resolve_package_versions(requirement: str) -> dict[str, str]: from pip_requirements_parser import RequirementsFile requirements_txt = RequirementsFile.from_file(