diff --git a/README.md b/README.md index 9a602ebc5..cd72c4297 100644 --- a/README.md +++ b/README.md @@ -361,6 +361,51 @@ py_image( ) ``` +You can also implement more complex fine layering strategies by using the +`py_layer` rule and its `filter` attribute. For example: +```python +# Suppose that we are synthesizing an image that depends on a complex set +# of libraries that we want to break into layers. +LIBS = [ + "//pkg/complex_library", + # ... +] +# First, we extract all transitive dependencies of LIBS that are under //pkg/common. +py_layer( + name = "common_deps", + deps = LIBS, + filter = "//pkg/common", +) +# Then, we further extract all external dependencies of the deps under //pkg/common. +py_layer( + name = "common_external_deps", + deps = [":common_deps"], + filter = "@", +) +# We also extract all external dependencies of LIBS, which is a superset of +# ":common_external_deps". +py_layer( + name = "external_deps", + deps = LIBS, + filter = "@", +) +# Finally, we create the image, stacking the above filtered layers on top of one +# another in the "layers" attribute. The layers are applied in order, and any +# dependencies already added to the image will not be added again. Therefore, +# ":external_deps" will only add the external dependencies not present in +# ":common_external_deps". +py_image( + name = "image", + deps = LIBS, + layers = [ + ":common_external_deps", + ":common_deps", + ":external_deps", + ], + # ... +) +``` + ### py3_image To use a Python 3 runtime instead of the default of Python 2, use `py3_image`, diff --git a/WORKSPACE b/WORKSPACE index 276f38037..9bbfa1b47 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -66,6 +66,26 @@ load( _py_image_repos() +http_archive( + name = "io_bazel_rules_python", + sha256 = "8b32d2dbb0b0dca02e0410da81499eef8ff051dad167d6931a92579e3b2a1d48", + strip_prefix = "rules_python-8b5d0683a7d878b28fffe464779c8a53659fc645", + urls = ["https://github.com/bazelbuild/rules_python/archive/8b5d0683a7d878b28fffe464779c8a53659fc645.tar.gz"], +) + +load("@io_bazel_rules_python//python:pip.bzl", "pip_import", "pip_repositories") + +pip_repositories() + +pip_import( + name = "pip_deps", + requirements = "//testdata:requirements-pip.txt", +) + +load("@pip_deps//:requirements.bzl", "pip_install") + +pip_install() + load( "//python3:image.bzl", _py3_image_repos = "repositories", diff --git a/cc/image.bzl b/cc/image.bzl index 76aa700d2..9c71d6153 100644 --- a/cc/image.bzl +++ b/cc/image.bzl @@ -19,7 +19,6 @@ The signature of this rule is compatible with cc_binary. load( "//lang:image.bzl", "app_layer", - "dep_layer", ) load( "//container:container.bzl", @@ -78,9 +77,8 @@ def cc_image(name, base = None, deps = [], layers = [], binary = None, **kwargs) base = base or DEFAULT_BASE for index, dep in enumerate(layers): - this_name = "%s.%d" % (name, index) - dep_layer(name = this_name, base = base, dep = dep) - base = this_name + base = app_layer(name = "%s.%d" % (name, index), base = base, dep = dep) + base = app_layer(name = "%s.%d-symlinks" % (name, index), base = base, dep = dep, binary = binary) visibility = kwargs.get("visibility", None) tags = kwargs.get("tags", None) @@ -88,7 +86,6 @@ def cc_image(name, base = None, deps = [], layers = [], binary = None, **kwargs) name = name, base = base, binary = binary, - lang_layers = layers, visibility = visibility, tags = tags, args = kwargs.get("args"), diff --git a/container/BUILD b/container/BUILD index 3046cdca4..aae9f7166 100644 --- a/container/BUILD +++ b/container/BUILD @@ -141,6 +141,7 @@ TEST_TARGETS = [ ":nodejs_image", ":py3_image", ":py_image", + ":py_image_complex", ":rust_image", ":scala_image", ":war_image", diff --git a/container/image.bzl b/container/image.bzl index 0408c9fe4..eeadccf5f 100644 --- a/container/image.bzl +++ b/container/image.bzl @@ -336,6 +336,11 @@ def _impl( unzipped_layers = parent_parts.get("unzipped_layer", []) + [layer.unzipped_layer for layer in layers] layer_diff_ids = [layer.diff_id for layer in layers] diff_ids = parent_parts.get("diff_id", []) + layer_diff_ids + new_files = [f for f in file_map or []] + new_emptyfiles = empty_files or [] + new_symlinks = [f for f in symlinks or []] + parent_transitive_files = parent_parts.get("transitive_files", depset()) + transitive_files = depset(new_files + new_emptyfiles + new_symlinks, transitive = [parent_transitive_files]) # Get the config for the base layer config_file = _get_base_config(ctx, name, base) @@ -393,6 +398,9 @@ def _impl( # At the root of the chain, we support deriving from a tarball # base image. "legacy": parent_parts.get("legacy"), + + # Keep track of all files/emptyfiles/symlinks that we have already added to the image layers. + "transitive_files": transitive_files, } # We support incrementally loading or assembling this single image @@ -482,6 +490,7 @@ _attrs = dict(_layer.attrs.items() + { _outputs = dict(_layer.outputs) _outputs["out"] = "%{name}.tar" + _outputs["digest"] = "%{name}.digest" image = struct( diff --git a/container/image_test.py b/container/image_test.py index 0b8cc5ff9..5fd9b0b3b 100644 --- a/container/image_test.py +++ b/container/image_test.py @@ -40,6 +40,7 @@ def TestBundleImage(name, image_name): class ImageTest(unittest.TestCase): def assertTarballContains(self, tar, paths): + self.maxDiff = None self.assertEqual(paths, tar.getnames()) def assertLayerNContains(self, img, n, paths): @@ -468,16 +469,25 @@ def test_py_image(self): # files to avoid this redundancy. '/app', '/app/testdata', + '/app/testdata/py_image.binary', '/app/testdata/py_image.binary.runfiles', '/app/testdata/py_image.binary.runfiles/io_bazel_rules_docker', - '/app/testdata/py_image.binary.runfiles/io_bazel_rules_docker/testdata', - '/app/testdata/py_image.binary.runfiles/io_bazel_rules_docker/testdata/py_image_library.py', - '/app/testdata/py_image.binary', '/app/testdata/py_image.binary.runfiles/io_bazel_rules_docker/external', ]) - # Check the library layer, which is one below our application layer. + # Below that, we have a layer that generates symlinks for the library layer. self.assertLayerNContains(img, 1, [ + '.', + '/app', + '/app/testdata', + '/app/testdata/py_image.binary.runfiles', + '/app/testdata/py_image.binary.runfiles/io_bazel_rules_docker', + '/app/testdata/py_image.binary.runfiles/io_bazel_rules_docker/testdata', + '/app/testdata/py_image.binary.runfiles/io_bazel_rules_docker/testdata/py_image_library.py', + ]) + + # Check the library layer, which is two below our application layer. + self.assertLayerNContains(img, 2, [ '.', './app', './app/io_bazel_rules_docker', @@ -485,6 +495,146 @@ def test_py_image(self): './app/io_bazel_rules_docker/testdata/py_image_library.py', ]) + def test_py_image_complex(self): + with TestImage('py_image_complex') as img: + # bazel-bin/testdata/py_image_complex-layer.tar + self.assertTopLayerContains(img, [ + '.', + './app', + './app/testdata', + './app/testdata/py_image_complex.binary.runfiles', + './app/testdata/py_image_complex.binary.runfiles/io_bazel_rules_docker', + './app/testdata/py_image_complex.binary.runfiles/io_bazel_rules_docker/testdata', + './app/testdata/py_image_complex.binary.runfiles/io_bazel_rules_docker/testdata/py_image_complex.py', + './app/testdata/py_image_complex.binary.runfiles/io_bazel_rules_docker/testdata/py_image_complex.binary', + './app/testdata/py_image_complex.binary.runfiles/io_bazel_rules_docker/testdata/test', + './app/testdata/py_image_complex.binary.runfiles/io_bazel_rules_docker/testdata/test/__init__.py', + './app/testdata/py_image_complex.binary.runfiles/pypi__six_1_11_0', + './app/testdata/py_image_complex.binary.runfiles/pypi__six_1_11_0/__init__.py', + './app/testdata/py_image_complex.binary.runfiles/__init__.py', + './app/testdata/py_image_complex.binary.runfiles/pypi__addict_2_1_2', + './app/testdata/py_image_complex.binary.runfiles/pypi__addict_2_1_2/__init__.py', + './app/testdata/py_image_complex.binary.runfiles/io_bazel_rules_docker/testdata/__init__.py', + '/app', + '/app/testdata', + '/app/testdata/py_image_complex.binary', + '/app/testdata/py_image_complex.binary.runfiles', + '/app/testdata/py_image_complex.binary.runfiles/io_bazel_rules_docker', + '/app/testdata/py_image_complex.binary.runfiles/io_bazel_rules_docker/external', + ]) + + # bazel-bin/testdata/py_image_complex.3-symlinks-layer.tar + self.assertLayerNContains(img, 1, [ + '.', + '/app', + '/app/testdata', + '/app/testdata/py_image_complex.binary.runfiles', + '/app/testdata/py_image_complex.binary.runfiles/io_bazel_rules_docker', + '/app/testdata/py_image_complex.binary.runfiles/io_bazel_rules_docker/testdata', + '/app/testdata/py_image_complex.binary.runfiles/io_bazel_rules_docker/testdata/py_image_complex_library.py', + '/app/testdata/py_image_complex.binary.runfiles/io_bazel_rules_docker/testdata/py_image_library_using_six.py', + ]) + + # bazel-bin/testdata/py_image_complex.3-layer.tar + self.assertLayerNContains(img, 2, [ + '.', + './app', + './app/io_bazel_rules_docker', + './app/io_bazel_rules_docker/testdata', + './app/io_bazel_rules_docker/testdata/py_image_complex_library.py', + './app/io_bazel_rules_docker/testdata/py_image_library_using_six.py', + ]) + + # bazel-bin/testdata/py_image_complex.2-symlinks-layer.tar + self.assertLayerNContains(img, 3, [ + '.', + '/app', + '/app/testdata', + '/app/testdata/py_image_complex.binary.runfiles', + '/app/testdata/py_image_complex.binary.runfiles/io_bazel_rules_docker', + '/app/testdata/py_image_complex.binary.runfiles/io_bazel_rules_docker/testdata', + '/app/testdata/py_image_complex.binary.runfiles/io_bazel_rules_docker/testdata/test', + '/app/testdata/py_image_complex.binary.runfiles/io_bazel_rules_docker/testdata/test/py_image_library_using_addict.py', + ]) + + # bazel-bin/testdata/py_image_complex.2-layer.tar + self.assertLayerNContains(img, 4, [ + '.', + './app', + './app/io_bazel_rules_docker', + './app/io_bazel_rules_docker/testdata', + './app/io_bazel_rules_docker/testdata/test', + './app/io_bazel_rules_docker/testdata/test/py_image_library_using_addict.py', + ]) + + # bazel-bin/testdata/py_image_complex.1-symlinks-layer.tar + self.assertLayerNContains(img, 5, [ + '.', + '/app', + '/app/testdata', + '/app/testdata/py_image_complex.binary.runfiles', + '/app/testdata/py_image_complex.binary.runfiles/pypi__six_1_11_0', + '/app/testdata/py_image_complex.binary.runfiles/pypi__six_1_11_0/six.py', + '/app/testdata/py_image_complex.binary.runfiles/pypi__six_1_11_0/six-1.11.0.dist-info', + '/app/testdata/py_image_complex.binary.runfiles/pypi__six_1_11_0/six-1.11.0.dist-info/DESCRIPTION.rst', + '/app/testdata/py_image_complex.binary.runfiles/pypi__six_1_11_0/six-1.11.0.dist-info/METADATA', + '/app/testdata/py_image_complex.binary.runfiles/pypi__six_1_11_0/six-1.11.0.dist-info/RECORD', + '/app/testdata/py_image_complex.binary.runfiles/pypi__six_1_11_0/six-1.11.0.dist-info/WHEEL', + '/app/testdata/py_image_complex.binary.runfiles/pypi__six_1_11_0/six-1.11.0.dist-info/metadata.json', + '/app/testdata/py_image_complex.binary.runfiles/pypi__six_1_11_0/six-1.11.0.dist-info/top_level.txt', + ]) + + # bazel-bin/testdata/py_image_complex.1-layer.tar + self.assertLayerNContains(img, 6, [ + '.', + './app', + './app/pypi__six_1_11_0', + './app/pypi__six_1_11_0/six.py', + './app/pypi__six_1_11_0/six-1.11.0.dist-info', + './app/pypi__six_1_11_0/six-1.11.0.dist-info/DESCRIPTION.rst', + './app/pypi__six_1_11_0/six-1.11.0.dist-info/METADATA', + './app/pypi__six_1_11_0/six-1.11.0.dist-info/RECORD', + './app/pypi__six_1_11_0/six-1.11.0.dist-info/WHEEL', + './app/pypi__six_1_11_0/six-1.11.0.dist-info/metadata.json', + './app/pypi__six_1_11_0/six-1.11.0.dist-info/top_level.txt', + ]) + + + # bazel-bin/testdata/py_image_complex.0-symlinks-layer.tar + self.assertLayerNContains(img, 7, [ + '.', + '/app', + '/app/testdata', + '/app/testdata/py_image_complex.binary.runfiles', + '/app/testdata/py_image_complex.binary.runfiles/pypi__addict_2_1_2', + '/app/testdata/py_image_complex.binary.runfiles/pypi__addict_2_1_2/addict', + '/app/testdata/py_image_complex.binary.runfiles/pypi__addict_2_1_2/addict/__init__.py', + '/app/testdata/py_image_complex.binary.runfiles/pypi__addict_2_1_2/addict/addict.py', + '/app/testdata/py_image_complex.binary.runfiles/pypi__addict_2_1_2/addict-2.1.2.dist-info', + '/app/testdata/py_image_complex.binary.runfiles/pypi__addict_2_1_2/addict-2.1.2.dist-info/DESCRIPTION.rst', + '/app/testdata/py_image_complex.binary.runfiles/pypi__addict_2_1_2/addict-2.1.2.dist-info/METADATA', + '/app/testdata/py_image_complex.binary.runfiles/pypi__addict_2_1_2/addict-2.1.2.dist-info/RECORD', + '/app/testdata/py_image_complex.binary.runfiles/pypi__addict_2_1_2/addict-2.1.2.dist-info/WHEEL', + '/app/testdata/py_image_complex.binary.runfiles/pypi__addict_2_1_2/addict-2.1.2.dist-info/metadata.json', + '/app/testdata/py_image_complex.binary.runfiles/pypi__addict_2_1_2/addict-2.1.2.dist-info/top_level.txt', + ]) + + # bazel-bin/testdata/py_image_complex.0-layer.tar + self.assertLayerNContains(img, 8, [ + '.', + './app', + './app/pypi__addict_2_1_2', + './app/pypi__addict_2_1_2/addict', + './app/pypi__addict_2_1_2/addict/__init__.py', + './app/pypi__addict_2_1_2/addict/addict.py', + './app/pypi__addict_2_1_2/addict-2.1.2.dist-info', + './app/pypi__addict_2_1_2/addict-2.1.2.dist-info/DESCRIPTION.rst', + './app/pypi__addict_2_1_2/addict-2.1.2.dist-info/METADATA', + './app/pypi__addict_2_1_2/addict-2.1.2.dist-info/RECORD', + './app/pypi__addict_2_1_2/addict-2.1.2.dist-info/WHEEL', + './app/pypi__addict_2_1_2/addict-2.1.2.dist-info/metadata.json', + './app/pypi__addict_2_1_2/addict-2.1.2.dist-info/top_level.txt', + ]) def test_cc_image(self): with TestImage('cc_image') as img: diff --git a/container/import.bzl b/container/import.bzl index 6f71f606e..b37679435 100644 --- a/container/import.bzl +++ b/container/import.bzl @@ -160,8 +160,14 @@ def _container_import_impl(ctx): container_import = rule( attrs = dict({ "config": attr.label(allow_files = [".json"]), - "manifest": attr.label(allow_files = [".json"], mandatory = False), - "layers": attr.label_list(allow_files = tar_filetype + tgz_filetype, mandatory = True), + "manifest": attr.label( + allow_files = [".json"], + mandatory = False, + ), + "layers": attr.label_list( + allow_files = tar_filetype + tgz_filetype, + mandatory = True, + ), "repository": attr.string(default = "bazel"), }.items() + _hash_tools.items() + _layer_tools.items() + _zip_tools.items()), executable = True, diff --git a/container/layer.bzl b/container/layer.bzl index 5f22c7361..ee46b4e28 100644 --- a/container/layer.bzl +++ b/container/layer.bzl @@ -209,7 +209,10 @@ _layer_attrs = dict({ # Implicit/Undocumented dependencies. "empty_files": attr.string_list(), "empty_dirs": attr.string_list(), - "operating_system": attr.string(default = "linux", mandatory = False), + "operating_system": attr.string( + default = "linux", + mandatory = False, + ), "build_layer": attr.label( default = Label("//container:build_tar"), cfg = "host", diff --git a/container/providers.bzl b/container/providers.bzl index da7c22b46..f1f35ce39 100644 --- a/container/providers.bzl +++ b/container/providers.bzl @@ -14,7 +14,10 @@ """Provider definitions""" # A provider containing information exposed by container_bundle rules -BundleInfo = provider(fields = ["container_images", "stamp"]) +BundleInfo = provider(fields = [ + "container_images", + "stamp", +]) # A provider containing information exposed by container_flatten rules FlattenInfo = provider() @@ -46,3 +49,18 @@ PushInfo = provider(fields = [ "stamp", "stamp_inputs", ]) + +# A provider containing information exposed by filter_layer rules +FilterLayerInfo = provider( + fields = { + "runfiles": "filtered runfiles that should be installed from this layer", + "filtered_depset": "a filtered depset of struct(target=, target_deps=)", + }, +) + +# A provider containing information exposed by filter_aspect +FilterAspectInfo = provider( + fields = { + "depset": "a depset of struct(target=, target_deps=)", + }, +) diff --git a/contrib/idd.bzl b/contrib/idd.bzl index 52282bcd8..2e22070b4 100644 --- a/contrib/idd.bzl +++ b/contrib/idd.bzl @@ -51,11 +51,19 @@ idd( args = ["-v", "-d"] ) """ + idd = rule( - implementation = _impl, attrs = { - "image1": attr.label(mandatory = True, allow_files = container_filetype, single_file = True), - "image2": attr.label(mandatory = True, allow_files = container_filetype, single_file = True), + "image1": attr.label( + mandatory = True, + allow_files = container_filetype, + single_file = True, + ), + "image2": attr.label( + mandatory = True, + allow_files = container_filetype, + single_file = True, + ), "_idd_script": attr.label( default = ":idd", executable = True, @@ -64,4 +72,5 @@ idd = rule( ), }, executable = True, + implementation = _impl, ) diff --git a/d/image.bzl b/d/image.bzl index 09386a906..8c9367cb3 100644 --- a/d/image.bzl +++ b/d/image.bzl @@ -19,7 +19,6 @@ The signature of this rule is compatible with d_binary. load( "//lang:image.bzl", "app_layer", - "dep_layer", ) load( "//cc:image.bzl", @@ -51,9 +50,8 @@ def d_image(name, base = None, deps = [], layers = [], binary = None, **kwargs): base = base or DEFAULT_BASE for index, dep in enumerate(layers): - this_name = "%s_%d" % (name, index) - dep_layer(name = this_name, base = base, dep = dep) - base = this_name + base = app_layer(name = "%s_%d" % (name, index), base = base, dep = dep) + base = app_layer(name = "%s_%d-symlinks" % (name, index), base = base, dep = dep, binary = binary) visibility = kwargs.get("visibility", None) tags = kwargs.get("tags", None) @@ -61,7 +59,6 @@ def d_image(name, base = None, deps = [], layers = [], binary = None, **kwargs): name = name, base = base, binary = binary, - lang_layers = layers, visibility = visibility, tags = tags, args = kwargs.get("args"), diff --git a/go/image.bzl b/go/image.bzl index 57e4ab6f6..f91681e84 100644 --- a/go/image.bzl +++ b/go/image.bzl @@ -19,7 +19,6 @@ The signature of this rule is compatible with go_binary. load( "//lang:image.bzl", "app_layer", - "dep_layer", ) load( "//container:container.bzl", @@ -82,9 +81,8 @@ def go_image(name, base = None, deps = [], layers = [], binary = None, **kwargs) base = base or DEFAULT_BASE for index, dep in enumerate(layers): - this_name = "%s.%d" % (name, index) - dep_layer(name = this_name, base = base, dep = dep) - base = this_name + base = app_layer(name = "%s.%d" % (name, index), base = base, dep = dep) + base = app_layer(name = "%s.%d-symlinks" % (name, index), base = base, dep = dep, binary = binary) visibility = kwargs.get("visibility", None) tags = kwargs.get("tags", None) @@ -92,7 +90,6 @@ def go_image(name, base = None, deps = [], layers = [], binary = None, **kwargs) name = name, base = base, binary = binary, - lang_layers = layers, visibility = visibility, tags = tags, args = kwargs.get("args"), diff --git a/java/image.bzl b/java/image.bzl index 741e8e3fa..19ee6dc2c 100644 --- a/java/image.bzl +++ b/java/image.bzl @@ -26,7 +26,7 @@ load( ) load( "//lang:image.bzl", - "dep_layer_impl", + "app_layer_impl", "layer_file_path", ) @@ -117,26 +117,23 @@ def _jar_dep_layer_impl(ctx): # the data_runfiles. This is probably not ideal -- it would be better # to have the runfiles of the dependencies in the dep_layer, and only # the runfiles of the java_image in the top layer. Doing this without - # also pulling in the JDK deps will requrie extending the dep_layer_impl + # also pulling in the JDK deps will requrie extending the app_layer_impl # with functionality to exclude certain runfiles. Rather than making it # JDK specific, an option would be to add a `skipfiles = ctx.files._jdk` # type option here. The app layer would also need to be updated to # consider data flies include in the dep_layer as "available" so they # aren't duplicated in the top layer. - return dep_layer_impl(ctx, runfiles = java_files_with_data) + return app_layer_impl(ctx, runfiles = java_files_with_data) jar_dep_layer = rule( attrs = dict(_container.image.attrs.items() + { # The base image on which to overlay the dependency layers. "base": attr.label(mandatory = True), # The dependency whose runfiles we're appending. - "dep": attr.label(mandatory = True), + "dep": attr.label(providers = [DefaultInfo]), - # Whether to lay out each dependency in a manner that is agnostic - # of the binary in which it is participating. This can increase - # sharing of the dependency's layer across images, but requires a - # symlink forest in the app layers. - "agnostic_dep_layout": attr.bool(default = True), + # The binary target for which we are synthesizing an image. + "binary": attr.label(mandatory = False), # Override the defaults. "directory": attr.string(default = "/app"), @@ -228,12 +225,6 @@ jar_app_layer = rule( # The main class to invoke on startup. "main_class": attr.string(mandatory = True), - # Whether to lay out each dependency in a manner that is agnostic - # of the binary in which it is participating. This can increase - # sharing of the dependency's layer across images, but requires a - # symlink forest in the app layers. - "agnostic_dep_layout": attr.bool(default = True), - # Whether the classpath should be passed as a file. "_classpath_as_file": attr.bool(default = False), @@ -326,11 +317,8 @@ _war_dep_layer = rule( # The dependency whose runfiles we're appending. "dep": attr.label(mandatory = True), - # Whether to lay out each dependency in a manner that is agnostic - # of the binary in which it is participating. This can increase - # sharing of the dependency's layer across images, but requires a - # symlink forest in the app layers. - "agnostic_dep_layout": attr.bool(default = True), + # The binary target for which we are synthesizing an image. + "binary": attr.label(mandatory = False), # Override the defaults. "directory": attr.string(default = "/jetty/webapps/ROOT/WEB-INF/lib"), @@ -373,12 +361,6 @@ _war_app_layer = rule( "base": attr.label(mandatory = True), "entrypoint": attr.string_list(default = []), - # Whether to lay out each dependency in a manner that is agnostic - # of the binary in which it is participating. This can increase - # sharing of the dependency's layer across images, but requires a - # symlink forest in the app layers. - "agnostic_dep_layout": attr.bool(default = True), - # Override the defaults. "directory": attr.string(default = "/jetty/webapps/ROOT/WEB-INF/lib"), # WE WANT PATHS FLATTENED diff --git a/lang/image.bzl b/lang/image.bzl index d56fdb103..08fee0b49 100644 --- a/lang/image.bzl +++ b/lang/image.bzl @@ -18,13 +18,18 @@ load( "//container:container.bzl", _container = "container", ) +load( + "//container:layer_tools.bzl", + _get_layers = "get_from_target", +) +load("//container:providers.bzl", "FilterAspectInfo", "FilterLayerInfo") def _binary_name(ctx): # For //foo/bar/baz:blah this would translate to # /app/foo/bar/baz/blah return "/".join([ ctx.attr.directory, - ctx.label.package, + ctx.attr.binary.label.package, ctx.attr.binary.label.name, ]) @@ -99,122 +104,80 @@ def layer_file_path(ctx, f): return "/".join([ctx.attr.directory, ctx.workspace_name, f.short_path]) def _default_runfiles(dep): - return dep.default_runfiles.files + if FilterLayerInfo in dep: + return dep[FilterLayerInfo].runfiles.files + else: + return dep.default_runfiles.files def _default_emptyfiles(dep): - return dep.default_runfiles.empty_filenames + if FilterLayerInfo in dep: + return dep[FilterLayerInfo].runfiles.empty_filenames + else: + return dep.default_runfiles.empty_filenames -def dep_layer_impl(ctx, runfiles = None, emptyfiles = None): +def app_layer_impl(ctx, runfiles = None, emptyfiles = None): """Appends a layer for a single dependency's runfiles.""" runfiles = runfiles or _default_runfiles emptyfiles = emptyfiles or _default_emptyfiles + workdir = None - filepath = layer_file_path if ctx.attr.agnostic_dep_layout else _final_file_path - emptyfilepath = _layer_emptyfile_path if ctx.attr.agnostic_dep_layout else _final_emptyfile_path - - return _container.image.implementation( - ctx, - # We use all absolute paths. - directory = "/", - # We put the files from dependency layers into a binary-agnostic - # path to increase the likelihood of layer sharing across images, - # then we symlink them into the appropriate place in the app layer. - # This references the binary package because the file paths are - # relative to it, and normalized by the tarball package. - file_map = { - filepath(ctx, f): f - for f in runfiles(ctx.attr.dep) - }, - empty_files = [ - emptyfilepath(ctx, empty) - for empty in emptyfiles(ctx.attr.dep) - ], - ) - -dep_layer = rule( - attrs = dict(_container.image.attrs.items() + { - # The base image on which to overlay the dependency layers. - "base": attr.label(mandatory = True), - # The dependency whose runfiles we're appending. - "dep": attr.label( - mandatory = True, - allow_files = True, - ), - - # Whether to lay out each dependency in a manner that is agnostic - # of the binary in which it is participating. This can increase - # sharing of the dependency's layer across images, but requires a - # symlink forest in the app layers. - "agnostic_dep_layout": attr.bool(default = True), - # The binary target for which we are synthesizing an image. - # This is needed iff agnostic_dep_layout. - "binary": attr.label(mandatory = False), - - # Override the defaults. - # https://github.com/bazelbuild/bazel/issues/2176 - "data_path": attr.string(default = "."), - "directory": attr.string(default = "/app"), - "legacy_run_behavior": attr.bool(default = False), - }.items()), - executable = True, - outputs = _container.image.outputs, - implementation = dep_layer_impl, -) - -def _app_layer_impl(ctx, runfiles = None, emptyfiles = None): - """Appends the app layer with all remaining runfiles.""" - - runfiles = runfiles or _default_runfiles - emptyfiles = emptyfiles or _default_emptyfiles - workdir = ctx.attr.workdir or "/".join([_runfiles_dir(ctx), ctx.workspace_name]) + parent_parts = _get_layers(ctx, ctx.attr.name, ctx.attr.base) + filepath = _final_file_path if ctx.attr.binary else layer_file_path + emptyfilepath = _final_emptyfile_path if ctx.attr.binary else _layer_emptyfile_path + dep = ctx.attr.dep or ctx.attr.binary + top_layer = ctx.attr.binary and not ctx.attr.dep # Compute the set of runfiles that have been made available # in our base image, tracking absolute paths. - available = {} - for dep in ctx.attr.lang_layers: - available.update({ - _final_file_path(ctx, f): layer_file_path(ctx, f) - for f in runfiles(dep) - }) - available.update({ - _final_emptyfile_path(ctx, f): _layer_emptyfile_path(ctx, f) - for f in emptyfiles(dep) - }) + available = { + f: None + for f in parent_parts.get("transitive_files", depset()) + } # Compute the set of remaining runfiles to include into the # application layer. file_map = { - _final_file_path(ctx, f): f - for f in runfiles(ctx.attr.binary) - if _final_file_path(ctx, f) not in available + filepath(ctx, f): f + for f in runfiles(dep) + if filepath(ctx, f) not in available and layer_file_path(ctx, f) not in available } empty_files = [ - _final_emptyfile_path(ctx, f) - for f in emptyfiles(ctx.attr.binary) - if _final_emptyfile_path(ctx, f) not in available + emptyfilepath(ctx, f) + for f in emptyfiles(dep) + if emptyfilepath(ctx, f) not in available and _layer_emptyfile_path(ctx, f) not in available ] - # For each of the runfiles we aren't including directly into - # the application layer, link to their binary-agnostic - # location from the runfiles path. symlinks = {} - # Include symlinks to available files if they were laid out in a - # binary-agnostic fashion. - if ctx.attr.agnostic_dep_layout: - symlinks.update(available) + # If the caller provided the binary that will eventually form the + # app layer, we can already create symlinks to the runfiles path. + if ctx.attr.binary: + symlinks.update({ + _final_file_path(ctx, f): layer_file_path(ctx, f) + for f in runfiles(dep) + if _final_file_path(ctx, f) not in file_map and _final_file_path(ctx, f) not in available + }) + symlinks.update({ + _final_emptyfile_path(ctx, f): _layer_emptyfile_path(ctx, f) + for f in emptyfiles(dep) + if _final_emptyfile_path(ctx, f) not in empty_files and _final_emptyfile_path(ctx, f) not in available + }) - symlinks.update({ - # Create a symlink from our entrypoint to where it will actually be put - # under runfiles. - _binary_name(ctx): _final_file_path(ctx, ctx.executable.binary), - # Create a directory symlink from /external to the runfiles - # root, since they may be accessed via either path. - _external_dir(ctx): _runfiles_dir(ctx), - }) + entrypoint = None + if top_layer: + entrypoint = ctx.attr.entrypoint + [_binary_name(ctx)] + workdir = ctx.attr.workdir or "/".join([_runfiles_dir(ctx), ctx.workspace_name]) + symlinks.update({ + # Create a symlink from our entrypoint to where it will actually be put + # under runfiles. + _binary_name(ctx): _final_file_path(ctx, ctx.executable.binary), + # Create a directory symlink from /external to the runfiles + # root, since they may be accessed via either path. + _external_dir(ctx): _runfiles_dir(ctx), + }) # args of the form $(location :some_target) are expanded to the path of the underlying file args = [ctx.expand_location(arg, ctx.attr.data) for arg in ctx.attr.args] @@ -231,31 +194,30 @@ def _app_layer_impl(ctx, runfiles = None, emptyfiles = None): # image is `docker run ...`. # Per: https://docs.docker.com/engine/reference/builder/#entrypoint # we should use the "exec" (list) form of entrypoint. - entrypoint = ctx.attr.entrypoint + [_binary_name(ctx)], + entrypoint = entrypoint, cmd = args, ) -app_layer = rule( +_app_layer = rule( attrs = dict(_container.image.attrs.items() + { # The binary target for which we are synthesizing an image. + # If specified, the layer will not be "image agnostic", meaning + # that the runfiles required by "dep" will be created (or symlinked, + # if already found in an agnostic path from the base image) under + # the runfiles dir. "binary": attr.label( - mandatory = True, executable = True, cfg = "target", ), - # The full list of dependencies that have their own layers - # factored into our base. - "lang_layers": attr.label_list(allow_files = True), + # The dependency whose runfiles we're appending. + # If not specified, then the layer will be treated as the top layer, + # and all remaining deps of "binary" will be added under runfiles. + "dep": attr.label(providers = [DefaultInfo]), + # The base image on which to overlay the dependency layers. "base": attr.label(mandatory = True), "entrypoint": attr.string_list(default = []), - # Whether each dependency is laid out in a manner that is agnostic - # of the binary in which it is participating. This can increase - # sharing of the dependency's layer across images, but requires a - # symlink forest in the app layers. - "agnostic_dep_layout": attr.bool(default = True), - # Override the defaults. "data_path": attr.string(default = "."), "workdir": attr.string(default = ""), @@ -265,5 +227,69 @@ app_layer = rule( }.items()), executable = True, outputs = _container.image.outputs, - implementation = _app_layer_impl, + implementation = app_layer_impl, +) + +# Convenience function that instantiates the _app_layer rule and returns +# the name (useful when chaining layers). +def app_layer(name, **kwargs): + _app_layer(name = name, **kwargs) + return name + +def _filter_aspect_impl(target, ctx): + if FilterLayerInfo in target: + # If the aspect propagated along the "deps" attr to another filter layer, + # then take the filtered depset instead of descending further. + return [FilterAspectInfo(depset = target[FilterLayerInfo].filtered_depset)] + + # Collect transitive deps from all children (propagating along "deps" attr). + target_deps = depset(transitive = [dep[FilterAspectInfo].depset for dep in ctx.rule.attr.deps]) + myself = struct(target = target, target_deps = target_deps) + return [ + FilterAspectInfo( + depset = depset(direct = [myself], transitive = [target_deps]), + ), + ] + +# Aspect for collecting dependency info. +_filter_aspect = aspect( + attr_aspects = ["deps"], + implementation = _filter_aspect_impl, +) + +def _filter_layer_rule_impl(ctx): + transitive_deps = ctx.attr.dep[FilterAspectInfo].depset + + runfiles = ctx.runfiles() + filtered_depsets = [] + for dep in transitive_deps: + if str(dep.target.label).startswith(ctx.attr.filter) and str(dep.target.label) != str(ctx.attr.dep.label): + runfiles = runfiles.merge(dep.target.default_runfiles) + filtered_depsets.append(dep.target_deps) + return struct( + providers = [ + FilterLayerInfo( + runfiles = runfiles, + filtered_depset = depset(transitive = filtered_depsets), + ), + ], + # Also forward builtin providers so that the filter_layer() can be used as a normal + # dependency to native targets (e.g. py_library(deps = [])). + py = ctx.attr.dep.py if hasattr(ctx.attr.dep, "py") else None, + ) + +# A rule that allows selecting a subset of transitive dependencies, and using +# them as a layer in an image. +filter_layer = rule( + attrs = { + "dep": attr.label( + providers = [DefaultInfo], + aspects = [_filter_aspect], + mandatory = True, + ), + # Include in this layer only transitive dependencies whose label starts with "filter". + # For example, set filter="@" to include only external dependencies. + "filter": attr.string(default = ""), + }, + implementation = _filter_layer_rule_impl, ) diff --git a/nodejs/image.bzl b/nodejs/image.bzl index b169275f7..2d07fe738 100644 --- a/nodejs/image.bzl +++ b/nodejs/image.bzl @@ -19,7 +19,7 @@ The signature of this rule is compatible with nodejs_binary. load( "//lang:image.bzl", "app_layer", - "dep_layer_impl", + "app_layer_impl", ) load( "//container:container.bzl", @@ -66,7 +66,7 @@ def _emptyfiles(dep): return dep.default_runfiles.empty_filenames + dep.data_runfiles.empty_filenames def _dep_layer_impl(ctx): - return dep_layer_impl(ctx, runfiles = _runfiles, emptyfiles = _emptyfiles) + return app_layer_impl(ctx, runfiles = _runfiles, emptyfiles = _emptyfiles) _dep_layer = rule( attrs = dict(_container.image.attrs.items() + { @@ -78,13 +78,7 @@ _dep_layer = rule( allow_files = True, ), - # Whether to lay out each dependency in a manner that is agnostic - # of the binary in which it is participating. This can increase - # sharing of the dependency's layer across images, but requires a - # symlink forest in the app layers. - "agnostic_dep_layout": attr.bool(default = True), # The binary target for which we are synthesizing an image. - # This is needed iff agnostic_dep_layout. "binary": attr.label(mandatory = False), # Override the defaults. @@ -143,9 +137,7 @@ def nodejs_image( app_layer( name = name, base = base, - agnostic_dep_layout = True, binary = binary_name, - lang_layers = layers, visibility = visibility, tags = tags, args = kwargs.get("args"), diff --git a/python/image.bzl b/python/image.bzl index 449d74819..4f8852812 100644 --- a/python/image.bzl +++ b/python/image.bzl @@ -19,7 +19,7 @@ The signature of this rule is compatible with py_binary. load( "//lang:image.bzl", "app_layer", - "dep_layer", + "filter_layer", ) load( "//container:container.bzl", @@ -58,14 +58,19 @@ DEFAULT_BASE = select({ "//conditions:default": "@py_image_base//image", }) +def py_layer(name, deps, filter = "", **kwargs): + binary_name = name + ".layer-binary" + native.py_library(name = binary_name, deps = deps, **kwargs) + filter_layer(name = name, dep = binary_name, filter = filter) + def py_image(name, base = None, deps = [], layers = [], **kwargs): """Constructs a container image wrapping a py_binary target. - Args: - layers: Augments "deps" with dependencies that should be put into - their own layers. - **kwargs: See py_binary. - """ + Args: + layers: Augments "deps" with dependencies that should be put into + their own layers. + **kwargs: See py_binary. + """ binary_name = name + ".binary" if "main" not in kwargs: @@ -79,9 +84,8 @@ def py_image(name, base = None, deps = [], layers = [], **kwargs): # is placed configurable. base = base or DEFAULT_BASE for index, dep in enumerate(layers): - this_name = "%s.%d" % (name, index) - dep_layer(name = this_name, base = base, dep = dep) - base = this_name + base = app_layer(name = "%s.%d" % (name, index), base = base, dep = dep) + base = app_layer(name = "%s.%d-symlinks" % (name, index), base = base, dep = dep, binary = binary_name) visibility = kwargs.get("visibility", None) tags = kwargs.get("tags", None) @@ -90,7 +94,6 @@ def py_image(name, base = None, deps = [], layers = [], **kwargs): base = base, entrypoint = ["/usr/bin/python"], binary = binary_name, - lang_layers = layers, visibility = visibility, tags = tags, args = kwargs.get("args"), diff --git a/python3/image.bzl b/python3/image.bzl index 1f1f17834..77a50a891 100644 --- a/python3/image.bzl +++ b/python3/image.bzl @@ -19,7 +19,6 @@ The signature of this rule is compatible with py_binary. load( "//lang:image.bzl", "app_layer", - "dep_layer", ) load( "//container:container.bzl", @@ -80,9 +79,8 @@ def py3_image(name, base = None, deps = [], layers = [], **kwargs): # is placed configurable. base = base or DEFAULT_BASE for index, dep in enumerate(layers): - this_name = "%s.%d" % (name, index) - dep_layer(name = this_name, base = base, dep = dep) - base = this_name + base = app_layer(name = "%s.%d" % (name, index), base = base, dep = dep) + base = app_layer(name = "%s.%d-symlinks" % (name, index), base = base, dep = dep, binary = binary_name) visibility = kwargs.get("visibility", None) tags = kwargs.get("tags", None) @@ -91,7 +89,6 @@ def py3_image(name, base = None, deps = [], layers = [], **kwargs): base = base, entrypoint = ["/usr/bin/python"], binary = binary_name, - lang_layers = layers, visibility = visibility, tags = tags, args = kwargs.get("args"), diff --git a/rust/image.bzl b/rust/image.bzl index 2c48be10f..15fd4c09e 100644 --- a/rust/image.bzl +++ b/rust/image.bzl @@ -19,7 +19,6 @@ The signature of this rule is compatible with rust_binary. load( "//lang:image.bzl", "app_layer", - "dep_layer", ) load( "//cc:image.bzl", @@ -51,9 +50,8 @@ def rust_image(name, base = None, deps = [], layers = [], binary = None, **kwarg base = base or DEFAULT_BASE for index, dep in enumerate(layers): - this_name = "%s_%d" % (name, index) - dep_layer(name = this_name, base = base, dep = dep) - base = this_name + base = app_layer(name = "%s_%d" % (name, index), base = base, dep = dep) + base = app_layer(name = "%s_%d-symlinks" % (name, index), base = base, dep = dep, binary = binary) visibility = kwargs.get("visibility", None) tags = kwargs.get("tags", None) @@ -61,7 +59,6 @@ def rust_image(name, base = None, deps = [], layers = [], binary = None, **kwarg name = name, base = base, binary = binary, - lang_layers = layers, visibility = visibility, tags = tags, args = kwargs.get("args"), diff --git a/testdata/BUILD b/testdata/BUILD index 53bd83579..f5f72928c 100644 --- a/testdata/BUILD +++ b/testdata/BUILD @@ -624,7 +624,7 @@ container_image( base = "@distroless_cc//image", ) -load("//python:image.bzl", "py_image") +load("//python:image.bzl", "py_image", "py_layer") py_library( name = "py_image_library", @@ -654,6 +654,53 @@ py_image( main = "py_image.py", ) +load("@pip_deps//:requirements.bzl", "requirement") + +py_library( + name = "py_image_library_using_six", + srcs = ["py_image_library_using_six.py"], + deps = [requirement("six")], +) + +py_library( + name = "py_image_complex_library", + srcs = ["py_image_complex_library.py"], + deps = [ + ":py_image_library_using_six", + "//testdata/test:py_image_library_using_addict", + ], +) + +py_layer( + name = "py_image_complex_filter_by_path", + filter = "//testdata/test", + deps = [":py_image_complex_library"], +) + +py_layer( + name = "py_image_complex_filter_by_path_external_deps", + filter = "@", + deps = [":py_image_complex_filter_by_path"], +) + +py_layer( + name = "py_image_complex_external_deps", + filter = "@", + deps = [":py_image_complex_library"], +) + +py_image( + name = "py_image_complex", + srcs = ["py_image_complex.py"], + layers = [ + ":py_image_complex_filter_by_path_external_deps", # addict + ":py_image_complex_external_deps", # six + ":py_image_complex_filter_by_path", # py_image_library_using_addict + ":py_image_complex_library", # py_image_complex_library + py_image_library_using_six + ], + main = "py_image_complex.py", +) + load("//python3:image.bzl", "py3_image") py3_image( diff --git a/testdata/py_image_complex.py b/testdata/py_image_complex.py new file mode 100644 index 000000000..fbf2a7827 --- /dev/null +++ b/testdata/py_image_complex.py @@ -0,0 +1,22 @@ +# Copyright 2018 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from testdata import py_image_complex_library + +def main(): + print(py_image_complex_library.fn('Calling from main module: ')) + + +if __name__ == '__main__': + main() diff --git a/testdata/py_image_complex_library.py b/testdata/py_image_complex_library.py new file mode 100644 index 000000000..8cc897801 --- /dev/null +++ b/testdata/py_image_complex_library.py @@ -0,0 +1,22 @@ +# Copyright 2017 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from testdata import py_image_library_using_six +from testdata.test import py_image_library_using_addict + +def fn(what_comes_in): + return "\n".join([ + py_image_library_using_six.fn(what_comes_in + "through py_image_complex_library: "), + py_image_library_using_addict.fn(what_comes_in + "through py_image_complex_library: "), + ]) diff --git a/testdata/py_image_library_using_six.py b/testdata/py_image_library_using_six.py new file mode 100644 index 000000000..7c621edc8 --- /dev/null +++ b/testdata/py_image_library_using_six.py @@ -0,0 +1,18 @@ +# Copyright 2018 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import six + +def fn(what_comes_in): + return what_comes_in + "Six version: " + six.__version__ diff --git a/testdata/requirements-pip.txt b/testdata/requirements-pip.txt new file mode 100644 index 000000000..8fe121707 --- /dev/null +++ b/testdata/requirements-pip.txt @@ -0,0 +1,2 @@ +six==1.11.0 +addict==2.1.2 diff --git a/testdata/test/BUILD b/testdata/test/BUILD index 1c9b4f10f..af3c84325 100644 --- a/testdata/test/BUILD +++ b/testdata/test/BUILD @@ -22,3 +22,11 @@ filegroup( name = "test-data", srcs = ["test"], ) + +load("@pip_deps//:requirements.bzl", "requirement") + +py_library( + name = "py_image_library_using_addict", + srcs = ["py_image_library_using_addict.py"], + deps = [requirement("addict")], +) diff --git a/testdata/test/py_image_library_using_addict.py b/testdata/test/py_image_library_using_addict.py new file mode 100644 index 000000000..98a573261 --- /dev/null +++ b/testdata/test/py_image_library_using_addict.py @@ -0,0 +1,18 @@ +# Copyright 2018 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import addict + +def fn(what_comes_in): + return what_comes_in + "Addict version: " + addict.__version__ diff --git a/testing/e2e.sh b/testing/e2e.sh index cb782117f..45679f234 100755 --- a/testing/e2e.sh +++ b/testing/e2e.sh @@ -196,6 +196,17 @@ EOF rm -f output.txt } +function test_py_image_complex() { + cd "${ROOT}" + clear_docker + cat > output.txt <