From 7d7cc2638e2cd37827f1c548346efa2474e9d85a Mon Sep 17 00:00:00 2001 From: Alex Eagle Date: Fri, 24 May 2024 13:01:12 -0700 Subject: [PATCH] feat: default to latest protobuf rather than pinning (#12) * feat: use protobuf v27 * chore: update docs * fix: check in requirements.in * chore: more docs * Update release_prep.sh --- .github/workflows/release_prep.sh | 10 ++++++ MODULE.bazel | 22 ++++-------- README.md | 38 ++++++++------------ WORKSPACE.bazel | 6 ++-- examples/java/BUILD | 2 +- examples/lang_toolchains/BUILD | 9 ++++- examples/lang_toolchains/requirements.in | 1 + examples/lang_toolchains/requirements.txt | 34 +++++++++--------- protoc/private/prebuilt_protoc_toolchain.bzl | 2 ++ 9 files changed, 62 insertions(+), 62 deletions(-) create mode 100644 examples/lang_toolchains/requirements.in diff --git a/.github/workflows/release_prep.sh b/.github/workflows/release_prep.sh index aae8e09..e931644 100755 --- a/.github/workflows/release_prep.sh +++ b/.github/workflows/release_prep.sh @@ -23,6 +23,16 @@ cat << EOF \`\`\`starlark bazel_dep(name = "toolchains_protoc", version = "${TAG:1}") + +# Optional: choose a version of protoc rather than the latest. +protoc = use_extension("@toolchains_protoc//protoc:extensions.bzl", "protoc") +protoc.toolchain( + # Creates a repository to satisfy well-known-types dependencies such as + # deps=["@com_google_protobuf//:any_proto"] + google_protobuf = "com_google_protobuf", + # Pin to any version of protoc + version = "v26.0", +) \`\`\` ## Using WORKSPACE diff --git a/MODULE.bazel b/MODULE.bazel index c4586ca..eb2b720 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -13,7 +13,7 @@ bazel_dep(name = "platforms", version = "0.0.8") protoc = use_extension("//protoc:extensions.bzl", "protoc") protoc.toolchain( google_protobuf = "com_google_protobuf", - version = "v25.3", + version = "LATEST", ) use_repo(protoc, "com_google_protobuf", "toolchains_protoc_hub") @@ -23,15 +23,9 @@ bazel_dep(name = "aspect_bazel_lib", version = "1.32.1", dev_dependency = True) bazel_dep(name = "buildifier_prebuilt", version = "6.1.2", dev_dependency = True) bazel_dep(name = "aspect_rules_py", version = "0.7.1", dev_dependency = True) bazel_dep(name = "rules_java", version = "7.4.0", dev_dependency = True) -bazel_dep(name = "rules_python", version = "0.31.0", dev_dependency = True) - -# # Update to include -# https://github.com/bazelbuild/rules_python/pull/1577 -git_override( - module_name = "rules_python", - commit = "3edcae33dcc43051ca6f200c2d0cb3692df79d03", - remote = "https://github.com/bazelbuild/rules_python.git", -) +bazel_dep(name = "rules_python", version = "0.32.2", dev_dependency = True) +bazel_dep(name = "rules_go", version = "0.48.0", dev_dependency = True) +bazel_dep(name = "rules_uv", version = "0.10.0", dev_dependency = True) register_toolchains( "//examples/lang_toolchains:all", @@ -39,16 +33,14 @@ register_toolchains( ) # Shows how a typical Python user fetches all the dependencies of their app, including the protobuf runtime -pip = use_extension( +dev_pip = use_extension( "@rules_python//python/extensions:pip.bzl", "pip", dev_dependency = True, ) -pip.parse( +dev_pip.parse( hub_name = "pypi", python_version = "3.11", requirements_lock = "//examples/lang_toolchains:requirements.txt", ) -use_repo(pip, "pypi") - -bazel_dep(name = "rules_go", version = "0.48.0") +use_repo(dev_pip, "pypi") diff --git a/README.md b/README.md index 519568c..e4deb90 100644 --- a/README.md +++ b/README.md @@ -18,17 +18,17 @@ Using Protocol Buffers with Bazel has always been difficult. ## Support matrix Minimum versions: + - Bazel: 7.0.0 - rules_proto: 6.0.0 -| Language | Support | Example or Issue | -|----------|---------|------------------| -| Java | Yes | [example](./examples/java) | +| Language | Support | Example or Issue | +| -------- | ------- | ---------------------------- | +| Java | Yes | [example](./examples/java) | | Python | Yes | [example](./examples/python) | -| Rust | | https://github.com/bazelbuild/rules_rust/issues/2627 | -| Go | | https://github.com/bazelbuild/rules_go/issues/3895 | +| Go | Yes | [example](./examples/go) | -Your language missing? Please file the upstream issue and send a PR to update this table! +For all other languages, see https://github.com/bazelbuild/rules_proto/discussions/213 ## Installation @@ -56,43 +56,33 @@ using whatever Bazel rule you chose for interacting with package managers - JavaScript: https://www.npmjs.com/package/protobufjs - Go: https://pkg.go.dev/google.golang.org/protobuf/runtime -Second, declare the toolchain in a `BUILD` file. It looks like the following -(where `LANG`, `--flag_to_protoc`, and `runtime` are replaced +For rulesets that need a "lang toolchain", declare one in a `BUILD` file. +It looks like the following (where `LANG`, `--flag_to_protoc`, and `runtime` are replaced with appropriate values for the language and the label of the runtime you installed). -You can choose a Bazel package where this goes; we recommend `/tools/protoc/BUILD.bazel`. +You can choose a Bazel package where this goes; we recommend `/tools/toolchains/BUILD.bazel`. ```starlark load("@rules_proto//proto:defs.bzl", "proto_lang_toolchain") + proto_lang_toolchain( name = "protoc_LANG_toolchain", command_line = "--flag_to_protoc=%s", progress_message = "Generating LANG proto_library %{label}", runtime = "@some-external//lib", -) -``` - -Finally, in the same `BUILD` file, declare the registration target... - -```starlark -toolchain( - name = "protoc_LANG_toolchain.registration", - toolchain = ":protoc_LANG_toolchain", - # This type should be declared by the language rules: + # This target should be declared by the language rules: toolchain_type = "@rules_LANG//path/to/proto:toolchain_type", ) ``` -...and then register it, either in `MODULE.bazel` or `WORKSPACE`: +Then register the toolchains, either in `MODULE.bazel` or `WORKSPACE`: ```starlark -register_toolchains("//tools/protoc:all") +register_toolchains("//tools/toolchains:all") ``` See `examples` for several language rules like `py_proto_library` and `java_proto_library`. -Note that there is NO dependency on `@com_google_protobuf` anywhere. - ### Troubleshooting What if you still see that protoc is compiling? This means that there is still a transitive dependency on the @@ -110,7 +100,7 @@ What if you still see that protoc is compiling? This means that there is still a 1. That distribution includes the "well known types" such as `timestamp.proto` 1. The protobuf runtimes for each language are distributed to the appropriate package manager such as npm or PyPI. 1. Bazel 7 introduced `--incompatible_enable_proto_toolchain_resolution` to allow us fetch `protoc` rather than re-build it! - That flag ALSO decouples how each built-in language rule (Java, Python, C++, etc.) locates the runtime. + That flag ALSO decouples how each built-in language rule (Java, Python, C++, etc.) locates the runtime. Thanks to that flag, this repo simply contains a toolchain that resolves those pre-built binaries. In the user's repository, there's a small BUILD file where the toolchain is configured. diff --git a/WORKSPACE.bazel b/WORKSPACE.bazel index beab320..1633df2 100644 --- a/WORKSPACE.bazel +++ b/WORKSPACE.bazel @@ -5,7 +5,7 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_jar") # Note: this is simpler than using rules_jvm_external with a maven installation, # but users would probably get this JAR file that way. http_jar( - name = "protobuf-java_3_25_3", - sha256 = "e90d8ddb963b20a972a6a59b5093ade2b07cbe546cab3279aaf4383260385f58", - urls = ["https://repo1.maven.org/maven2/com/google/protobuf/protobuf-java/3.25.3/protobuf-java-3.25.3.jar"], + name = "protobuf-java", + sha256 = "0532ad1024d62361561acaedb974d7d16889e7670b36e23e9321dd6b9d334ef9", + urls = ["https://repo1.maven.org/maven2/com/google/protobuf/protobuf-java/4.27.0-RC3/protobuf-java-4.27.0-RC3.jar"], ) diff --git a/examples/java/BUILD b/examples/java/BUILD index 004b088..43adca8 100644 --- a/examples/java/BUILD +++ b/examples/java/BUILD @@ -4,6 +4,6 @@ java_binary( main_class = "Main", deps = [ "//examples:foo_java_proto", - "@protobuf-java_3_25_3//jar", + "@protobuf-java//jar", ], ) diff --git a/examples/lang_toolchains/BUILD b/examples/lang_toolchains/BUILD index ea56b75..d72db69 100644 --- a/examples/lang_toolchains/BUILD +++ b/examples/lang_toolchains/BUILD @@ -1,4 +1,11 @@ load("@rules_proto//proto:defs.bzl", "proto_lang_toolchain") +load("@rules_uv//uv:pip.bzl", "pip_compile") + +pip_compile( + name = "generate_requirements_txt", + requirements_in = "requirements.in", + requirements_txt = "requirements.txt", +) # Configure protoc to have the right arguments for generating Python stubs. proto_lang_toolchain( @@ -13,6 +20,6 @@ proto_lang_toolchain( name = "protoc_java_toolchain", command_line = "--java_out=%s", progress_message = "Generating Java proto_library %{label}", - runtime = "@protobuf-java_3_25_3//jar", + runtime = "@protobuf-java//jar", toolchain_type = "@rules_java//java/proto:toolchain_type", ) diff --git a/examples/lang_toolchains/requirements.in b/examples/lang_toolchains/requirements.in new file mode 100644 index 0000000..1054a73 --- /dev/null +++ b/examples/lang_toolchains/requirements.in @@ -0,0 +1 @@ +protobuf==5.27.0 diff --git a/examples/lang_toolchains/requirements.txt b/examples/lang_toolchains/requirements.txt index b07e7c5..b22ec81 100644 --- a/examples/lang_toolchains/requirements.txt +++ b/examples/lang_toolchains/requirements.txt @@ -1,19 +1,17 @@ -# -# This file is autogenerated by pip-compile with Python 3.11 -# by the following command: -# -# bazel run //examples/lang_toolchains:requirements.update -# -protobuf==5.26.0 \ - --hash=sha256:2c334550e1cb4efac5c8a3987384bf13a4334abaf5ab59e40479e7b70ecd6b19 \ - --hash=sha256:6ee9d1aa02f951c5ce10bf8c6cfb7604133773038e33f913183c8b5201350600 \ - --hash=sha256:7e47c57303466c867374a17b2b5e99c5a7c8b72a94118e2f28efb599f19b4069 \ - --hash=sha256:82f5870d74c99addfe4152777bdf8168244b9cf0ac65f8eccf045ddfa9d80d9b \ - --hash=sha256:8eef61a90631c21b06b4f492a27e199a269827f046de3bb68b61aa84fcf50905 \ - --hash=sha256:a49b6c5359bf34fb7bf965bf21abfab4476e4527d822ab5289ee3bf73f291159 \ - --hash=sha256:ca825f4eecb8c342d2ec581e6a5ad1ad1a47bededaecd768e0d3451ae4aaac2b \ - --hash=sha256:dfd29f6eb34107dccf289a93d44fb6b131e68888d090b784b691775ac84e8213 \ - --hash=sha256:e184175276edc222e2d5e314a72521e10049938a9a4961fe4bea9b25d073c03f \ - --hash=sha256:efd4f5894c50bd76cbcfdd668cd941021333861ed0f441c78a83d8254a01cc9f \ - --hash=sha256:f9ecc8eb6f18037e0cbf43256db0325d4723f429bca7ef5cd358b7c29d65f628 +# This file was autogenerated by uv via the following command: +# bazel run @@//examples/lang_toolchains:generate_requirements_txt +--index-url https://pypi.org/simple + +protobuf==5.27.0 \ + --hash=sha256:07f2b9a15255e3cf3f137d884af7972407b556a7a220912b252f26dc3121e6bf \ + --hash=sha256:2f83bf341d925650d550b8932b71763321d782529ac0eaf278f5242f513cc04e \ + --hash=sha256:56937f97ae0dcf4e220ff2abb1456c51a334144c9960b23597f044ce99c29c89 \ + --hash=sha256:587be23f1212da7a14a6c65fd61995f8ef35779d4aea9e36aad81f5f3b80aec5 \ + --hash=sha256:673ad60f1536b394b4fa0bcd3146a4130fcad85bfe3b60eaa86d6a0ace0fa374 \ + --hash=sha256:744489f77c29174328d32f8921566fb0f7080a2f064c5137b9d6f4b790f9e0c1 \ + --hash=sha256:7cb65fc8fba680b27cf7a07678084c6e68ee13cab7cace734954c25a43da6d0f \ + --hash=sha256:a17f4d664ea868102feaa30a674542255f9f4bf835d943d588440d1f49a3ed15 \ + --hash=sha256:aabbbcf794fbb4c692ff14ce06780a66d04758435717107c387f12fb477bf0d8 \ + --hash=sha256:b276e3f477ea1eebff3c2e1515136cfcff5ac14519c45f9b4aa2f6a87ea627c4 \ + --hash=sha256:f51f33d305e18646f03acfdb343aac15b8115235af98bc9f844bf9446573827b # via -r examples/lang_toolchains/requirements.in diff --git a/protoc/private/prebuilt_protoc_toolchain.bzl b/protoc/private/prebuilt_protoc_toolchain.bzl index bc62322..738af0b 100644 --- a/protoc/private/prebuilt_protoc_toolchain.bzl +++ b/protoc/private/prebuilt_protoc_toolchain.bzl @@ -24,6 +24,8 @@ GOOGLE_PROTOBUF_DEP_EDGES = { def _prebuilt_protoc_repo_impl(rctx): release_version = rctx.attr.version + if release_version == "LATEST": + release_version = PROTOC_VERSIONS.keys()[0] filename = "{}-{}-{}.zip".format( "protoc",