From c52d80cf04274aefd557e2bc93f64bb71c097645 Mon Sep 17 00:00:00 2001 From: Jason Lunn Date: Mon, 24 Jul 2023 23:13:47 -0700 Subject: [PATCH] Ruby FFI implementation (#13343) Supersedes #11483. Closes #13343 COPYBARA_INTEGRATE_REVIEW=https://github.com/protocolbuffers/protobuf/pull/13343 from protocolbuffers:simultaneous_ffi bcb4bb7842e672acf1a803fbd9abc6a27d00c020 PiperOrigin-RevId: 550782245 --- .github/workflows/test_ruby.yml | 53 +- conformance/BUILD.bazel | 6 +- conformance/failure_list_jruby_ffi.txt | 2 + protobuf_deps.bzl | 4 +- ruby/.gitignore | 8 +- ruby/BUILD.bazel | 158 ++++- ruby/README.md | 33 +- ruby/Rakefile | 48 +- ruby/ext/google/protobuf_c/BUILD.bazel | 103 ++- ruby/ext/google/protobuf_c/Rakefile | 3 + ruby/ext/google/protobuf_c/convert.c | 62 +- ruby/ext/google/protobuf_c/extconf.rb | 3 +- ruby/ext/google/protobuf_c/glue.c | 44 ++ ruby/ext/google/protobuf_c/message.c | 51 +- ruby/ext/google/protobuf_c/shared_convert.c | 87 +++ ruby/ext/google/protobuf_c/shared_convert.h | 49 ++ ruby/ext/google/protobuf_c/shared_message.c | 88 +++ ruby/ext/google/protobuf_c/shared_message.h | 48 ++ ruby/google-protobuf.gemspec | 26 +- ruby/lib/google/BUILD.bazel | 46 +- ruby/lib/google/protobuf.rb | 42 +- ruby/lib/google/protobuf/ffi/descriptor.rb | 177 +++++ .../google/protobuf/ffi/descriptor_pool.rb | 93 +++ .../google/protobuf/ffi/enum_descriptor.rb | 184 +++++ ruby/lib/google/protobuf/ffi/ffi.rb | 236 +++++++ .../google/protobuf/ffi/field_descriptor.rb | 332 +++++++++ .../google/protobuf/ffi/file_descriptor.rb | 71 ++ .../lib/google/protobuf/ffi/internal/arena.rb | 83 +++ .../google/protobuf/ffi/internal/convert.rb | 328 +++++++++ .../protobuf/ffi/internal/pointer_helper.rb | 58 ++ .../protobuf/ffi/internal/type_safety.rb | 48 ++ ruby/lib/google/protobuf/ffi/map.rb | 419 +++++++++++ ruby/lib/google/protobuf/ffi/message.rb | 658 ++++++++++++++++++ ruby/lib/google/protobuf/ffi/object_cache.rb | 53 ++ .../google/protobuf/ffi/oneof_descriptor.rb | 111 +++ .../lib/google/protobuf/ffi/repeated_field.rb | 526 ++++++++++++++ ruby/lib/google/protobuf_ffi.rb | 73 ++ ruby/lib/google/protobuf_native.rb | 43 ++ ruby/lib/google/tasks/ffi.rake | 94 +++ ruby/tests/BUILD.bazel | 9 + ruby/tests/implementation.rb | 37 + ruby/tests/object_cache_test.rb | 2 +- 42 files changed, 4408 insertions(+), 191 deletions(-) create mode 100644 conformance/failure_list_jruby_ffi.txt create mode 100644 ruby/ext/google/protobuf_c/Rakefile create mode 100644 ruby/ext/google/protobuf_c/glue.c create mode 100644 ruby/ext/google/protobuf_c/shared_convert.c create mode 100644 ruby/ext/google/protobuf_c/shared_convert.h create mode 100644 ruby/ext/google/protobuf_c/shared_message.c create mode 100644 ruby/ext/google/protobuf_c/shared_message.h create mode 100644 ruby/lib/google/protobuf/ffi/descriptor.rb create mode 100644 ruby/lib/google/protobuf/ffi/descriptor_pool.rb create mode 100644 ruby/lib/google/protobuf/ffi/enum_descriptor.rb create mode 100644 ruby/lib/google/protobuf/ffi/ffi.rb create mode 100644 ruby/lib/google/protobuf/ffi/field_descriptor.rb create mode 100644 ruby/lib/google/protobuf/ffi/file_descriptor.rb create mode 100644 ruby/lib/google/protobuf/ffi/internal/arena.rb create mode 100644 ruby/lib/google/protobuf/ffi/internal/convert.rb create mode 100644 ruby/lib/google/protobuf/ffi/internal/pointer_helper.rb create mode 100644 ruby/lib/google/protobuf/ffi/internal/type_safety.rb create mode 100644 ruby/lib/google/protobuf/ffi/map.rb create mode 100644 ruby/lib/google/protobuf/ffi/message.rb create mode 100644 ruby/lib/google/protobuf/ffi/object_cache.rb create mode 100644 ruby/lib/google/protobuf/ffi/oneof_descriptor.rb create mode 100644 ruby/lib/google/protobuf/ffi/repeated_field.rb create mode 100644 ruby/lib/google/protobuf_ffi.rb create mode 100644 ruby/lib/google/protobuf_native.rb create mode 100644 ruby/lib/google/tasks/ffi.rake create mode 100644 ruby/tests/implementation.rb diff --git a/.github/workflows/test_ruby.yml b/.github/workflows/test_ruby.yml index 3dfa6b43ce20..045eedfa5b75 100644 --- a/.github/workflows/test_ruby.yml +++ b/.github/workflows/test_ruby.yml @@ -17,17 +17,20 @@ jobs: fail-fast: false matrix: include: - - { name: Ruby 2.7, ruby: ruby-2.7.0, bazel: 5.1.1} + # Test both FFI and Native implementations on the highest and lowest + # Ruby versions for CRuby and JRuby, but only on Bazel 5.x. + - { name: Ruby 2.7, ruby: ruby-2.7.0, bazel: 5.1.1, ffi: NATIVE} + - { name: Ruby 2.7, ruby: ruby-2.7.0, bazel: 5.1.1, ffi: FFI} - { name: Ruby 3.0, ruby: ruby-3.0.2, bazel: 5.1.1} - { name: Ruby 3.1, ruby: ruby-3.1.0, bazel: 5.1.1} - - { name: Ruby 3.2, ruby: ruby-3.2.0, bazel: 5.1.1} - - { name: JRuby 9.2, ruby: jruby-9.2.20.1, bazel: 5.1.1} - - { name: JRuby 9.3, ruby: jruby-9.3.10.0, bazel: 5.1.1} - - { name: JRuby 9.4, ruby: jruby-9.4.3.0, bazel: 5.1.1} + - { name: Ruby 3.2, ruby: ruby-3.2.0, bazel: 5.1.1, ffi: NATIVE} + - { name: Ruby 3.2, ruby: ruby-3.2.0, bazel: 5.1.1, ffi: FFI} + - { name: JRuby 9.4, ruby: jruby-9.4.3.0, bazel: 5.1.1, ffi: NATIVE} + - { name: JRuby 9.4, ruby: jruby-9.4.3.0, bazel: 5.1.1, ffi: FFI} - { name: Ruby 2.7 (Bazel6), ruby: ruby-2.7.0, bazel: 6.0.0} - { name: JRuby 9.4 (Bazel6), ruby: jruby-9.4.3.0, bazel: 6.0.0} - name: Linux ${{ matrix.name }} + name: Linux ${{ matrix.name }}${{ matrix.ffi == 'FFI' && ' FFI' || '' }} runs-on: ubuntu-latest steps: - name: Checkout pending changes @@ -40,7 +43,7 @@ jobs: image: ${{ matrix.image || format('us-docker.pkg.dev/protobuf-build/containers/test/linux/ruby:{0}-{1}-508417e5215994ade7585d28ba3aad681a25fa5d', matrix.ruby, matrix.bazel) }} credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} bazel-cache: ruby_linux/${{ matrix.ruby }}_${{ matrix.bazel }} - bazel: test //ruby/... //ruby/tests:ruby_version --test_env=KOKORO_RUBY_VERSION + bazel: test //ruby/... //ruby/tests:ruby_version --test_env=KOKORO_RUBY_VERSION --test_env=BAZEL=true ${{ matrix.ffi == 'FFI' && '--//ruby:ffi=enabled --test_env=PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION=FFI' || '' }} linux-aarch64: name: Linux aarch64 @@ -78,10 +81,18 @@ jobs: strategy: fail-fast: false # Don't cancel all jobs if one fails. matrix: - # This is the full set of versions we support on MacOS. - version: [ "2.7", "3.0", "3.1", "3.2" ] + include: + # Test both FFI and Native implementations on the highest and lowest + # Ruby versions for CRuby, but only on Bazel 5.x. + # Quote versions numbers otherwise 3.0 will render as 3 + - { version: "2.7", ffi: NATIVE } + - { version: "2.7", ffi: FFI } + - { version: "3.0" } + - { version: "3.1" } + - { version: "3.2", ffi: NATIVE } + - { version: "3.2", ffi: FFI } - name: MacOS Ruby ${{ matrix.version }} + name: MacOS Ruby ${{ matrix.version }}${{ matrix.ffi == 'FFI' && ' FFI' || '' }} runs-on: macos-12 steps: - name: Checkout pending changes @@ -102,23 +113,26 @@ jobs: with: credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} bazel-cache: ruby_macos/${{ matrix.version }} - bazel: test //ruby/... --test_env=KOKORO_RUBY_VERSION=${{ matrix.version }} + bazel: test //ruby/... --test_env=KOKORO_RUBY_VERSION=${{ matrix.version }} --test_env=BAZEL=true ${{ matrix.ffi == 'FFI' && '--//ruby:ffi=enabled --test_env=PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION=FFI' || '' }} test_ruby_gems: strategy: fail-fast: false matrix: include: - - { name: Ruby 2.7, ruby: ruby-2.7.0, bazel: 5.1.1} + # Test both FFI and Native implementations on the highest and lowest + # Ruby versions for CRuby and JRuby, but only on Bazel 5.x. + - { name: Ruby 2.7, ruby: ruby-2.7.0, bazel: 5.1.1, ffi: NATIVE } + - { name: Ruby 2.7, ruby: ruby-2.7.0, bazel: 5.1.1, ffi: FFI } - { name: Ruby 3.0, ruby: ruby-3.0.2, bazel: 5.1.1} - { name: Ruby 3.1, ruby: ruby-3.1.0, bazel: 5.1.1} - - { name: Ruby 3.2, ruby: ruby-3.2.0, bazel: 5.1.1} - - { name: JRuby 9.2, ruby: jruby-9.2.20.1, bazel: 5.1.1} - - { name: JRuby 9.3, ruby: jruby-9.3.10.0, bazel: 5.1.1} - - { name: JRuby 9.4, ruby: jruby-9.4.3.0, bazel: 5.1.1} + - { name: Ruby 3.2, ruby: ruby-3.2.0, bazel: 5.1.1, ffi: NATIVE } + - { name: Ruby 3.2, ruby: ruby-3.2.0, bazel: 5.1.1, ffi: FFI } + - { name: JRuby 9.4, ruby: jruby-9.4.3.0, bazel: 5.1.1, ffi: NATIVE } + - { name: JRuby 9.4, ruby: jruby-9.4.3.0, bazel: 5.1.1, ffi: FFI } - { name: Ruby 2.7 (Bazel6), ruby: ruby-2.7.0, bazel: 6.0.0} - { name: JRuby 9.4 (Bazel6), ruby: jruby-9.4.3.0, bazel: 6.0.0} - name: Install ${{ matrix.name }} + name: Install ${{ matrix.name }}${{ matrix.ffi == 'FFI' && ' FFI' || '' }} runs-on: ubuntu-latest steps: - name: Checkout pending changes @@ -134,8 +148,9 @@ jobs: bash: > bazel --version; ruby --version; - bazel build //ruby:release //:protoc $BAZEL_FLAGS; + bazel build //ruby:release //:protoc ${{ matrix.ffi == 'FFI' && '--//ruby:ffi=enabled' || '' }} $BAZEL_FLAGS; gem install bazel-bin/ruby/google-protobuf-*; bazel-bin/protoc --proto_path=src --proto_path=ruby/tests --proto_path=ruby --ruby_out=ruby tests/test_import_proto2.proto; bazel-bin/protoc --proto_path=src --proto_path=ruby/tests --proto_path=ruby --ruby_out=ruby tests/basic_test.proto; - ruby ruby/tests/basic.rb + ${{ matrix.ffi == 'FFI' && 'PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION=FFI' || '' }} ruby ruby/tests/basic.rb; + ${{ matrix.ffi == 'FFI' && 'PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION=FFI' || '' }} ruby ruby/tests/implementation.rb diff --git a/conformance/BUILD.bazel b/conformance/BUILD.bazel index dde52022816a..c4955042cbce 100644 --- a/conformance/BUILD.bazel +++ b/conformance/BUILD.bazel @@ -24,6 +24,7 @@ exports_files([ "failure_list_python_cpp.txt", "failure_list_ruby.txt", "failure_list_jruby.txt", + "failure_list_jruby_ffi.txt", "text_format_failure_list_cpp.txt", "text_format_failure_list_csharp.txt", "text_format_failure_list_java.txt", @@ -34,6 +35,7 @@ exports_files([ "text_format_failure_list_python_cpp.txt", "text_format_failure_list_ruby.txt", "text_format_failure_list_jruby.txt", + "text_format_failure_list_jruby_ffi.txt", ]) cc_proto_library( @@ -326,12 +328,12 @@ ruby_binary( name = "conformance_ruby", testonly = True, srcs = ["conformance_ruby.rb"], + visibility = ["//ruby:__subpackages__"], deps = [ ":conformance_ruby_proto", "//:test_messages_proto2_ruby_proto", - "//:test_messages_proto3_ruby_proto", + "//:test_messages_proto3_ruby_proto", ], - visibility = ["//ruby:__subpackages__"], ) ################################################################################ diff --git a/conformance/failure_list_jruby_ffi.txt b/conformance/failure_list_jruby_ffi.txt new file mode 100644 index 000000000000..6a487cb293a9 --- /dev/null +++ b/conformance/failure_list_jruby_ffi.txt @@ -0,0 +1,2 @@ +Recommended.Proto3.JsonInput.IgnoreUnknownEnumStringValueInMapValue.ProtobufOutput +Recommended.Proto3.JsonInput.IgnoreUnknownEnumStringValueInRepeatedField.ProtobufOutput \ No newline at end of file diff --git a/protobuf_deps.bzl b/protobuf_deps.bzl index 951d7555df99..e3cd4e1195e4 100644 --- a/protobuf_deps.bzl +++ b/protobuf_deps.bzl @@ -110,8 +110,8 @@ def protobuf_deps(): _github_archive( name = "rules_ruby", repo = "https://github.com/protocolbuffers/rules_ruby", - commit = "8fca842a3006c3d637114aba4f6bf9695bb3a432", - sha256 = "2619f9a23cee6f6a198d9ef284b6f6cbc901545ee9a9aac9ffa6b83dbf17cf0c", + commit = "b7f3e9756f3c45527be27bc38840d5a1ba690436", + sha256 = "347927fd8de6132099fcdc58e8f7eab7bde4eb2fd424546b9cd4f1c6f8f8bad8", ) if not native.existing_rule("rules_jvm_external"): diff --git a/ruby/.gitignore b/ruby/.gitignore index 653309818f32..143b48e92c87 100644 --- a/ruby/.gitignore +++ b/ruby/.gitignore @@ -6,4 +6,10 @@ protobuf-jruby.iml target/ pkg/ tmp/ -tests/google/ \ No newline at end of file +tests/google/ +ext/google/protobuf_c/third_party/utf8_range/utf8_range.h +ext/google/protobuf_c/third_party/utf8_range/range2-sse.c +ext/google/protobuf_c/third_party/utf8_range/range2-neon.c +ext/google/protobuf_c/third_party/utf8_range/naive.c +ext/google/protobuf_c/third_party/utf8_range/LICENSE +lib/google/protobuf/*_pb.rb \ No newline at end of file diff --git a/ruby/BUILD.bazel b/ruby/BUILD.bazel index c581355c5404..cc4b477eedb6 100755 --- a/ruby/BUILD.bazel +++ b/ruby/BUILD.bazel @@ -2,6 +2,8 @@ # # See also code generation logic under /src/google/protobuf/compiler/ruby. +load("@bazel_skylib//lib:selects.bzl", "selects") +load("@bazel_skylib//rules:common_settings.bzl", "string_flag") load("@rules_pkg//:mappings.bzl", "pkg_files", "strip_prefix") load("@rules_ruby//ruby:defs.bzl", "ruby_library") load("//build_defs:internal_shell.bzl", "inline_sh_binary") @@ -13,12 +15,83 @@ load("//:protobuf_version.bzl", "PROTOBUF_RUBY_VERSION") # Ruby Runtime ################################################################################ +string_flag( + name = "ffi", + build_setting_default = "disabled", + values = [ + "enabled", + "disabled", + ], +) + +config_setting( + name = "ffi_enabled", + flag_values = { + ":ffi": "enabled", + }, +) + +config_setting( + name = "ffi_disabled", + flag_values = { + ":ffi": "disabled", + }, +) + +selects.config_setting_group( + name = "jruby_ffi", + match_all = [ + ":ffi_enabled", + "@rules_ruby//ruby/runtime:config_jruby", + ], +) + +selects.config_setting_group( + name = "jruby_native", + match_all = [ + ":ffi_disabled", + "@rules_ruby//ruby/runtime:config_jruby", + ], +) + +selects.config_setting_group( + name = "ruby_ffi", + match_all = [ + ":ffi_enabled", + "@rules_ruby//ruby/runtime:config_ruby", + ], +) + +selects.config_setting_group( + name = "ruby_native", + match_all = [ + ":ffi_disabled", + "@rules_ruby//ruby/runtime:config_ruby", + ], +) + +selects.config_setting_group( + name = "macos_ffi_enabled", + match_all = [ + ":ffi_enabled", + "@platforms//os:osx", + ], +) + +selects.config_setting_group( + name = "linux_ffi_enabled", + match_all = [ + ":ffi_enabled", + "@platforms//os:linux", + ], +) + ruby_library( name = "protobuf", - deps = ["//ruby/lib/google:protobuf_lib"], visibility = [ "//visibility:public", ], + deps = ["//ruby/lib/google:protobuf_lib"], ) # Note: these can be greatly simplified using inline_sh_binary in Bazel 6, @@ -27,18 +100,25 @@ ruby_library( genrule( name = "jruby_release", srcs = [ - "//ruby/lib/google:copy_jar", - "//ruby/lib/google:dist_files", - "//:well_known_ruby_protos", - "google-protobuf.gemspec", + "@utf8_range//:utf8_range_srcs", + "@utf8_range//:LICENSE", + "//ruby/lib/google:copy_jar", + "//ruby/lib/google:dist_files", + "//ruby/ext/google/protobuf_c:dist_files", + "//:well_known_ruby_protos", + "google-protobuf.gemspec", ], - outs = ["google-protobuf-"+PROTOBUF_RUBY_VERSION+"-java.gem"], + outs = ["google-protobuf-" + PROTOBUF_RUBY_VERSION + "-java.gem"], cmd = """ set -eux mkdir tmp for src in $(SRCS); do cp --parents -L "$$src" tmp done + mkdir -p "tmp/ruby/ext/google/protobuf_c/third_party/utf8_range" + for utf in $(execpaths @utf8_range//:utf8_range_srcs) $(execpath @utf8_range//:LICENSE); do + mv "tmp/$$utf" "tmp/ruby/ext/google/protobuf_c/third_party/utf8_range" + done for wkt in $(execpaths //:well_known_ruby_protos); do mv "tmp/$$wkt" "tmp/ruby/lib/google/protobuf/" done @@ -59,14 +139,14 @@ genrule( genrule( name = "ruby_release", srcs = [ - "@utf8_range//:utf8_range_srcs", - "@utf8_range//:LICENSE", - "//:well_known_ruby_protos", - "//ruby/ext/google/protobuf_c:dist_files", - "//ruby/lib/google:dist_files", - "google-protobuf.gemspec", + "@utf8_range//:utf8_range_srcs", + "@utf8_range//:LICENSE", + "//:well_known_ruby_protos", + "//ruby/ext/google/protobuf_c:dist_files", + "//ruby/lib/google:dist_files", + "google-protobuf.gemspec", ], - outs = ["google-protobuf-"+PROTOBUF_RUBY_VERSION+".gem"], + outs = ["google-protobuf-" + PROTOBUF_RUBY_VERSION + ".gem"], cmd = """ set -eux mkdir tmp @@ -102,7 +182,6 @@ filegroup( tags = ["manual"], ) - ################################################################################ # Tests ################################################################################ @@ -111,34 +190,65 @@ filegroup( internal_ruby_proto_library( name = "test_ruby_protos", srcs = ["//ruby/tests:test_protos"], - deps = ["//:well_known_ruby_protos"], - includes = [".", "src", "ruby/tests"], + includes = [ + ".", + "ruby/tests", + "src", + ], visibility = [ "//ruby:__subpackages__", ], + deps = ["//:well_known_ruby_protos"], ) - conformance_test( name = "conformance_test", failure_list = "//conformance:failure_list_ruby.txt", + target_compatible_with = select({ + ":ruby_native": [], + "//conditions:default": ["@platforms//:incompatible"], + }), testee = "//conformance:conformance_ruby", text_format_failure_list = "//conformance:text_format_failure_list_ruby.txt", +) + +conformance_test( + name = "conformance_test_ffi", + env = { + "PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION": "ffi", + }, + failure_list = "//conformance:failure_list_ruby.txt", target_compatible_with = select({ - "@rules_ruby//ruby/runtime:config_ruby": [], + ":ruby_ffi": [], "//conditions:default": ["@platforms//:incompatible"], }), + testee = "//conformance:conformance_ruby", + text_format_failure_list = "//conformance:text_format_failure_list_ruby.txt", ) conformance_test( name = "conformance_test_jruby", failure_list = "//conformance:failure_list_jruby.txt", + target_compatible_with = select({ + ":jruby_native": [], + "//conditions:default": ["@platforms//:incompatible"], + }), testee = "//conformance:conformance_ruby", text_format_failure_list = "//conformance:text_format_failure_list_jruby.txt", +) + +conformance_test( + name = "conformance_test_jruby_ffi", + env = { + "PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION": "ffi", + }, + failure_list = "//conformance:failure_list_jruby_ffi.txt", target_compatible_with = select({ - "@rules_ruby//ruby/runtime:config_jruby": [], + ":jruby_ffi": [], "//conditions:default": ["@platforms//:incompatible"], }), + testee = "//conformance:conformance_ruby", + text_format_failure_list = "//conformance:text_format_failure_list_jruby.txt", ) ################################################################################ @@ -148,15 +258,15 @@ conformance_test( pkg_files( name = "dist_files", srcs = [ - "//ruby/ext/google/protobuf_c:dist_files", - "//ruby/lib/google:dist_files", - "//ruby/src/main/java:dist_files", - "//ruby/tests:dist_files", ".gitignore", "BUILD.bazel", "Gemfile", - "Rakefile", "README.md", + "Rakefile", + "//ruby/ext/google/protobuf_c:dist_files", + "//ruby/lib/google:dist_files", + "//ruby/src/main/java:dist_files", + "//ruby/tests:dist_files", ], strip_prefix = strip_prefix.from_root(""), visibility = ["//pkg:__pkg__"], diff --git a/ruby/README.md b/ruby/README.md index bc94e1d7775a..cba3d58f3c7d 100644 --- a/ruby/README.md +++ b/ruby/README.md @@ -2,10 +2,11 @@ This directory contains the Ruby extension that implements Protocol Buffers functionality in Ruby. The Ruby extension makes use of generated Ruby code that defines message and -enum types in a Ruby DSL. You may write definitions in this DSL directly, but -we recommend using protoc's Ruby generation support with .proto files. The -build process in this directory only installs the extension; you need to -install protoc as well to have Ruby code generation functionality. +enum types in a Ruby DSL. You may write definitions in this DSL directly, but we +recommend using protoc's Ruby generation support with .proto files. The build +process in this directory only installs the extension; you need to install +protoc as well to have Ruby code generation functionality. You can build protoc +from source using `bazel build //:protoc`. Installation from Gem --------------------- @@ -50,6 +51,18 @@ puts MyTestMessage.encode_json(mymessage) Installation from Source (Building Gem) --------------------------------------- + + +Protocol Buffers has a new experimental backend that uses the +[ffi](https://github.com/ffi/ffi) gem to provide a unified C-based +implementation across Ruby interpreters based on +[UPB](https://github.com/protocolbuffers/upb). For now, use of the FFI +implementation is opt-in. If any of the following are true, the traditional +platform-native implementations (MRI-ruby based on CRuby, Java based on JRuby) +are used instead of the new FFI-based implementation: 1. `ffi` and +`ffi-compiler` gems are not installed 2. `PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION` +environment variable has a value other than `FFI` (case-insensitive). 3. FFI is +unable to load the native library at runtime. To build this Ruby extension, you will need: @@ -81,6 +94,12 @@ To run the specs: $ rake test +To run the specs while using the FFI-based implementation: + +``` +$ PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION=FFI rake test +``` + This gem includes the upb parsing and serialization library as a single-file amalgamation. It is up-to-date with upb git commit `535bc2fe2f2b467f59347ffc9449e11e47791257`. @@ -93,6 +112,12 @@ From the project root (rather than the `ruby` directory): $ bazel test //ruby/tests/... ``` +To run tests against the FFI implementation: + +``` +$ bazel test //ruby/tests/... //ruby:ffi=enabled --test_env=PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION=FFI +``` + Version Number Scheme --------------------- diff --git a/ruby/Rakefile b/ruby/Rakefile index c05e1159ea93..85a52dcf845a 100644 --- a/ruby/Rakefile +++ b/ruby/Rakefile @@ -2,6 +2,7 @@ require "rubygems" require "rubygems/package_task" require "rake/extensiontask" unless RUBY_PLATFORM == "java" require "rake/testtask" +import 'lib/google/tasks/ffi.rake' spec = Gem::Specification.load("google-protobuf.gemspec") @@ -68,6 +69,24 @@ unless ENV['IN_DOCKER'] == 'true' or ENV['BAZEL'] == 'true' end end +task :copy_third_party do + unless File.exist? 'ext/google/protobuf_c/third_party/utf8_range' + FileUtils.mkdir_p 'ext/google/protobuf_c/third_party/utf8_range' + # We need utf8_range in-tree. + if ENV['BAZEL'] == 'true' + utf8_root = '../external/utf8_range' + else + utf8_root = '../third_party/utf8_range' + end + %w[ + utf8_range.h naive.c range2-neon.c range2-neon.c range2-sse.c LICENSE + ].each do |file| + FileUtils.cp File.join(utf8_root, file), + "ext/google/protobuf_c/third_party/utf8_range" + end + end +end + if RUBY_PLATFORM == "java" task :clean => :require_mvn do system("mvn --batch-mode clean") @@ -82,20 +101,6 @@ if RUBY_PLATFORM == "java" end else - unless ENV['IN_DOCKER'] == 'true' - # We need utf8_range in-tree. - if ENV['BAZEL'] == 'true' - utf8_root = '../external/utf8_range' - else - utf8_root = '../third_party/utf8_range' - end - FileUtils.mkdir_p("ext/google/protobuf_c") - FileUtils.cp(utf8_root+"/utf8_range.h", "ext/google/protobuf_c") - FileUtils.cp(utf8_root+"/naive.c", "ext/google/protobuf_c") - FileUtils.cp(utf8_root+"/range2-neon.c", "ext/google/protobuf_c") - FileUtils.cp(utf8_root+"/range2-sse.c", "ext/google/protobuf_c") - end - Rake::ExtensionTask.new("protobuf_c", spec) do |ext| unless RUBY_PLATFORM =~ /darwin/ # TODO: also set "no_native to true" for mac if possible. As is, @@ -133,7 +138,7 @@ else ['x86-mingw32', 'x64-mingw32', 'x64-mingw-ucrt', 'x86_64-linux', 'x86-linux'].each do |plat| RakeCompilerDock.sh <<-"EOT", platform: plat bundle && \ - IN_DOCKER=true rake native:#{plat} pkg/#{spec.full_name}-#{plat}.gem RUBY_CC_VERSION=3.1.0:3.0.0:2.7.0:2.6.0 + IN_DOCKER=true rake native:#{plat} pkg/#{spec.full_name}-#{plat}.gem RUBY_CC_VERSION=3.1.0:3.0.0:2.7.0 EOT end end @@ -141,7 +146,7 @@ else if RUBY_PLATFORM =~ /darwin/ task 'gem:native' do system "rake genproto" - system "rake cross native gem RUBY_CC_VERSION=3.1.0:3.0.0:2.7.0:2.6.0" + system "rake cross native gem RUBY_CC_VERSION=3.1.0:3.0.0:2.7.0" end else task 'gem:native' => [:genproto, 'gem:windows', 'gem:java'] @@ -152,6 +157,14 @@ task :genproto => genproto_output task :clean do sh "rm -f #{genproto_output.join(' ')}" + sh "rm -f google-protobuf-*gem" + sh "rm -f Gemfile.lock" + sh "rm -rf pkg" + sh "rm -rf tmp" + # Handles third_party and any platform specific directories built by FFI + Pathname('ext/google/protobuf_c').children.select(&:directory?).each do |dir| + sh "rm -rf #{dir}" + end end Gem::PackageTask.new(spec) do |pkg| @@ -169,7 +182,8 @@ Rake::TestTask.new(:gc_test => ENV['BAZEL'] == 'true' ? [] : :build) do |t| t.test_files = FileList["tests/gc_test.rb"] end -task :build => [:clean, :genproto, :compile] +task :build => [:clean, :genproto, :copy_third_party, :compile, :"ffi-protobuf:default"] +Rake::Task[:gem].enhance [:copy_third_party, :genproto] task :default => [:build] # vim:sw=2:et diff --git a/ruby/ext/google/protobuf_c/BUILD.bazel b/ruby/ext/google/protobuf_c/BUILD.bazel index 1a6b963fad81..36f646f24c88 100644 --- a/ruby/ext/google/protobuf_c/BUILD.bazel +++ b/ruby/ext/google/protobuf_c/BUILD.bazel @@ -6,31 +6,115 @@ package(default_visibility = ["//ruby:__subpackages__"]) cc_library( name = "protobuf_c", - srcs = glob([ - "*.h", - "*.c", - ]), + srcs = [ + "convert.c", + "convert.h", + "defs.c", + "defs.h", + "map.c", + "map.h", + "message.c", + "message.h", + "protobuf.c", + "protobuf.h", + "repeated_field.c", + "repeated_field.h", + "ruby-upb.c", + "ruby-upb.h", + "shared_convert.c", + "shared_convert.h", + "shared_message.c", + "shared_message.h", + "wrap_memcpy.c", + ], + linkstatic = True, + target_compatible_with = select({ + "@rules_ruby//ruby/runtime:config_jruby": ["@platforms//:incompatible"], + "//conditions:default": [], + }), deps = [ "@rules_ruby//ruby/runtime:headers", - "@utf8_range//:utf8_range", + "@utf8_range", + ], + alwayslink = True, +) + +# Needs to be compiled with UPB_BUILD_API in order to expose functions called +# via FFI directly by Ruby. +cc_library( + name = "upb_api", + srcs = [ + "ruby-upb.c", + ], + hdrs = [ + "ruby-upb.h", + ], + copts = ["-fvisibility=hidden"], + linkstatic = False, + local_defines = [ + "UPB_BUILD_API", ], target_compatible_with = select({ - "@rules_ruby//ruby/runtime:config_jruby": ["@platforms//:incompatible"], + "//ruby:ffi_disabled": ["@platforms//:incompatible"], "//conditions:default": [], }), + deps = [ + "@utf8_range", + ], +) + +cc_library( + name = "protobuf_c_ffi", + srcs = [ + "glue.c", + "shared_convert.c", + "shared_convert.h", + "shared_message.c", + "shared_message.h", + ], + copts = [ + "-std=gnu99", + "-O3", + "-Wall", + "-Wsign-compare", + "-Wno-declaration-after-statement", + ], linkstatic = True, - alwayslink = True, + local_defines = [ + "NDEBUG", + ], + target_compatible_with = select({ + "//ruby:ffi_disabled": ["@platforms//:incompatible"], + "//conditions:default": [], + }), + deps = [":upb_api"], + alwayslink = 1, ) apple_binary( - name = "bundle", + name = "ffi_bundle", binary_type = "loadable_bundle", linkopts = [ "-undefined,dynamic_lookup", "-multiply_defined,suppress", ], + minimum_os_version = "10.11", platform_type = "macos", + tags = ["manual"], + deps = [ + ":protobuf_c_ffi", + ], +) + +apple_binary( + name = "bundle", + binary_type = "loadable_bundle", + linkopts = [ + "-undefined,dynamic_lookup", + "-multiply_defined,suppress", + ], minimum_os_version = "10.11", + platform_type = "macos", tags = ["manual"], deps = [ ":protobuf_c", @@ -43,6 +127,7 @@ pkg_files( "*.h", "*.c", "*.rb", + "Rakefile", ]), strip_prefix = strip_prefix.from_root(""), visibility = ["//ruby:__pkg__"], @@ -65,8 +150,8 @@ genrule( staleness_test( name = "test_amalgamation_staleness", outs = [ - "ruby-upb.h", "ruby-upb.c", + "ruby-upb.h", ], generated_pattern = "generated-in/%s", ) diff --git a/ruby/ext/google/protobuf_c/Rakefile b/ruby/ext/google/protobuf_c/Rakefile new file mode 100644 index 000000000000..0d4f84eb3a31 --- /dev/null +++ b/ruby/ext/google/protobuf_c/Rakefile @@ -0,0 +1,3 @@ +import '../../../lib/google/tasks/ffi.rake' + +task default: ['ffi-protobuf:default'] \ No newline at end of file diff --git a/ruby/ext/google/protobuf_c/convert.c b/ruby/ext/google/protobuf_c/convert.c index bdc71599fbe9..65ca11010676 100644 --- a/ruby/ext/google/protobuf_c/convert.c +++ b/ruby/ext/google/protobuf_c/convert.c @@ -41,6 +41,7 @@ #include "message.h" #include "protobuf.h" +#include "shared_convert.h" static upb_StringView Convert_StringData(VALUE str, upb_Arena* arena) { upb_StringView ret; @@ -111,8 +112,7 @@ static int32_t Convert_ToEnum(VALUE value, const char* name, case T_SYMBOL: { const upb_EnumValueDef* ev = upb_EnumDef_FindValueByName(e, rb_id2name(SYM2ID(value))); - if (!ev) - goto unknownval; + if (!ev) goto unknownval; val = upb_EnumValueDef_Number(ev); break; } @@ -255,7 +255,7 @@ VALUE Convert_UpbToRuby(upb_MessageValue upb_val, TypeInfo type_info, case kUpb_CType_UInt64: return ULL2NUM(upb_val.int64_val); case kUpb_CType_Enum: { - const upb_EnumValueDef *ev = upb_EnumDef_FindValueByNumber( + const upb_EnumValueDef* ev = upb_EnumDef_FindValueByNumber( type_info.def.enumdef, upb_val.int32_val); if (ev) { return ID2SYM(rb_intern(upb_EnumValueDef_Name(ev))); @@ -312,50 +312,26 @@ upb_MessageValue Msgval_DeepCopy(upb_MessageValue msgval, TypeInfo type_info, bool Msgval_IsEqual(upb_MessageValue val1, upb_MessageValue val2, TypeInfo type_info) { - switch (type_info.type) { - case kUpb_CType_Bool: - return memcmp(&val1, &val2, 1) == 0; - case kUpb_CType_Float: - case kUpb_CType_Int32: - case kUpb_CType_UInt32: - case kUpb_CType_Enum: - return memcmp(&val1, &val2, 4) == 0; - case kUpb_CType_Double: - case kUpb_CType_Int64: - case kUpb_CType_UInt64: - return memcmp(&val1, &val2, 8) == 0; - case kUpb_CType_String: - case kUpb_CType_Bytes: - return val1.str_val.size == val2.str_val.size && - memcmp(val1.str_val.data, val2.str_val.data, val1.str_val.size) == - 0; - case kUpb_CType_Message: - return Message_Equal(val1.msg_val, val2.msg_val, type_info.def.msgdef); - default: - rb_raise(rb_eRuntimeError, "Internal error, unexpected type"); + upb_Status status; + upb_Status_Clear(&status); + bool return_value = shared_Msgval_IsEqual(val1, val2, type_info.type, + type_info.def.msgdef, &status); + if (upb_Status_IsOk(&status)) { + return return_value; + } else { + rb_raise(rb_eRuntimeError, upb_Status_ErrorMessage(&status)); } } uint64_t Msgval_GetHash(upb_MessageValue val, TypeInfo type_info, uint64_t seed) { - switch (type_info.type) { - case kUpb_CType_Bool: - return _upb_Hash(&val, 1, seed); - case kUpb_CType_Float: - case kUpb_CType_Int32: - case kUpb_CType_UInt32: - case kUpb_CType_Enum: - return _upb_Hash(&val, 4, seed); - case kUpb_CType_Double: - case kUpb_CType_Int64: - case kUpb_CType_UInt64: - return _upb_Hash(&val, 8, seed); - case kUpb_CType_String: - case kUpb_CType_Bytes: - return _upb_Hash(val.str_val.data, val.str_val.size, seed); - case kUpb_CType_Message: - return Message_Hash(val.msg_val, type_info.def.msgdef, seed); - default: - rb_raise(rb_eRuntimeError, "Internal error, unexpected type"); + upb_Status status; + upb_Status_Clear(&status); + uint64_t return_value = shared_Msgval_GetHash( + val, type_info.type, type_info.def.msgdef, seed, &status); + if (upb_Status_IsOk(&status)) { + return return_value; + } else { + rb_raise(rb_eRuntimeError, upb_Status_ErrorMessage(&status)); } } diff --git a/ruby/ext/google/protobuf_c/extconf.rb b/ruby/ext/google/protobuf_c/extconf.rb index b7c439b0b2a6..4bb49bb21570 100755 --- a/ruby/ext/google/protobuf_c/extconf.rb +++ b/ruby/ext/google/protobuf_c/extconf.rb @@ -22,6 +22,7 @@ $srcs = ["protobuf.c", "convert.c", "defs.c", "message.c", "repeated_field.c", "map.c", "ruby-upb.c", "wrap_memcpy.c", - "naive.c", "range2-neon.c", "range2-sse.c"] + "naive.c", "range2-neon.c", "range2-sse.c", "shared_convert.c", + "shared_message.c"] create_makefile(ext_name) diff --git a/ruby/ext/google/protobuf_c/glue.c b/ruby/ext/google/protobuf_c/glue.c new file mode 100644 index 000000000000..ac123beb253c --- /dev/null +++ b/ruby/ext/google/protobuf_c/glue.c @@ -0,0 +1,44 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2023 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// ----------------------------------------------------------------------------- +// Exposing inlined UPB functions. Strictly free of dependencies on +// Ruby interpreter internals. + +#include "ruby-upb.h" + +upb_Arena* Arena_create() { return upb_Arena_Init(NULL, 0, &upb_alloc_global); } + +google_protobuf_FileDescriptorProto* FileDescriptorProto_parse( + const char* serialized_file_proto, size_t length) { + upb_Arena* arena = Arena_create(); + return google_protobuf_FileDescriptorProto_parse(serialized_file_proto, + length, arena); +} diff --git a/ruby/ext/google/protobuf_c/message.c b/ruby/ext/google/protobuf_c/message.c index 4d5a48831866..2efff0eeaf85 100644 --- a/ruby/ext/google/protobuf_c/message.c +++ b/ruby/ext/google/protobuf_c/message.c @@ -35,6 +35,7 @@ #include "map.h" #include "protobuf.h" #include "repeated_field.h" +#include "shared_message.h" static VALUE cParseError = Qnil; static VALUE cAbstractMessage = Qnil; @@ -694,29 +695,13 @@ static VALUE Message_dup(VALUE _self) { // Support function for Message_eq, and also used by other #eq functions. bool Message_Equal(const upb_Message* m1, const upb_Message* m2, const upb_MessageDef* m) { - if (m1 == m2) return true; - - size_t size1, size2; - int encode_opts = - kUpb_EncodeOption_SkipUnknown | kUpb_EncodeOption_Deterministic; - upb_Arena* arena_tmp = upb_Arena_New(); - const upb_MiniTable* layout = upb_MessageDef_MiniTable(m); - - // Compare deterministically serialized payloads with no unknown fields. - char* data1; - char* data2; - upb_EncodeStatus status1 = - upb_Encode(m1, layout, encode_opts, arena_tmp, &data1, &size1); - upb_EncodeStatus status2 = - upb_Encode(m2, layout, encode_opts, arena_tmp, &data2, &size2); - - if (status1 == kUpb_EncodeStatus_Ok && status2 == kUpb_EncodeStatus_Ok) { - bool ret = (size1 == size2) && (memcmp(data1, data2, size1) == 0); - upb_Arena_Free(arena_tmp); - return ret; + upb_Status status; + upb_Status_Clear(&status); + bool return_value = shared_Message_Equal(m1, m2, m, &status); + if (upb_Status_IsOk(&status)) { + return return_value; } else { - upb_Arena_Free(arena_tmp); - rb_raise(cParseError, "Error comparing messages"); + rb_raise(cParseError, upb_Status_ErrorMessage(&status)); } } @@ -741,23 +726,13 @@ static VALUE Message_eq(VALUE _self, VALUE _other) { uint64_t Message_Hash(const upb_Message* msg, const upb_MessageDef* m, uint64_t seed) { - upb_Arena* arena = upb_Arena_New(); - char* data; - size_t size; - - // Hash a deterministically serialized payloads with no unknown fields. - upb_EncodeStatus status = upb_Encode( - msg, upb_MessageDef_MiniTable(m), - kUpb_EncodeOption_SkipUnknown | kUpb_EncodeOption_Deterministic, arena, - &data, &size); - - if (status == kUpb_EncodeStatus_Ok) { - uint64_t ret = _upb_Hash(data, size, seed); - upb_Arena_Free(arena); - return ret; + upb_Status status; + upb_Status_Clear(&status); + uint64_t return_value = shared_Message_Hash(msg, m, seed, &status); + if (upb_Status_IsOk(&status)) { + return return_value; } else { - upb_Arena_Free(arena); - rb_raise(cParseError, "Error calculating hash"); + rb_raise(cParseError, upb_Status_ErrorMessage(&status)); } } diff --git a/ruby/ext/google/protobuf_c/shared_convert.c b/ruby/ext/google/protobuf_c/shared_convert.c new file mode 100644 index 000000000000..d234115300bb --- /dev/null +++ b/ruby/ext/google/protobuf_c/shared_convert.c @@ -0,0 +1,87 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2023 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// ----------------------------------------------------------------------------- +// Ruby <-> upb data conversion functions. Strictly free of dependencies on +// Ruby interpreter internals. + +#include "shared_convert.h" + +bool shared_Msgval_IsEqual(upb_MessageValue val1, upb_MessageValue val2, + upb_CType type, upb_MessageDef* msgdef, + upb_Status* status) { + switch (type) { + case kUpb_CType_Bool: + return memcmp(&val1, &val2, 1) == 0; + case kUpb_CType_Float: + case kUpb_CType_Int32: + case kUpb_CType_UInt32: + case kUpb_CType_Enum: + return memcmp(&val1, &val2, 4) == 0; + case kUpb_CType_Double: + case kUpb_CType_Int64: + case kUpb_CType_UInt64: + return memcmp(&val1, &val2, 8) == 0; + case kUpb_CType_String: + case kUpb_CType_Bytes: + return val1.str_val.size == val2.str_val.size && + memcmp(val1.str_val.data, val2.str_val.data, val1.str_val.size) == + 0; + case kUpb_CType_Message: + return shared_Message_Equal(val1.msg_val, val2.msg_val, msgdef, status); + default: + upb_Status_SetErrorMessage(status, "Internal error, unexpected type"); + } +} + +uint64_t shared_Msgval_GetHash(upb_MessageValue val, upb_CType type, + upb_MessageDef* msgdef, uint64_t seed, + upb_Status* status) { + switch (type) { + case kUpb_CType_Bool: + return _upb_Hash(&val, 1, seed); + case kUpb_CType_Float: + case kUpb_CType_Int32: + case kUpb_CType_UInt32: + case kUpb_CType_Enum: + return _upb_Hash(&val, 4, seed); + case kUpb_CType_Double: + case kUpb_CType_Int64: + case kUpb_CType_UInt64: + return _upb_Hash(&val, 8, seed); + case kUpb_CType_String: + case kUpb_CType_Bytes: + return _upb_Hash(val.str_val.data, val.str_val.size, seed); + case kUpb_CType_Message: + return shared_Message_Hash(val.msg_val, msgdef, seed, status); + default: + upb_Status_SetErrorMessage(status, "Internal error, unexpected type"); + } +} \ No newline at end of file diff --git a/ruby/ext/google/protobuf_c/shared_convert.h b/ruby/ext/google/protobuf_c/shared_convert.h new file mode 100644 index 000000000000..43041e7b0f90 --- /dev/null +++ b/ruby/ext/google/protobuf_c/shared_convert.h @@ -0,0 +1,49 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2023 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// ----------------------------------------------------------------------------- +// Ruby <-> upb data conversion functions. Strictly free of dependencies on +// Ruby interpreter internals. + +#ifndef RUBY_PROTOBUF_SHARED_CONVERT_H_ +#define RUBY_PROTOBUF_SHARED_CONVERT_H_ + +#include "ruby-upb.h" +#include "shared_message.h" + +bool shared_Msgval_IsEqual(upb_MessageValue val1, upb_MessageValue val2, + upb_CType type, upb_MessageDef* msgdef, + upb_Status* status); + +uint64_t shared_Msgval_GetHash(upb_MessageValue val, upb_CType type, + upb_MessageDef* msgdef, uint64_t seed, + upb_Status* status); + +#endif // RUBY_PROTOBUF_SHARED_CONVERT_H_ diff --git a/ruby/ext/google/protobuf_c/shared_message.c b/ruby/ext/google/protobuf_c/shared_message.c new file mode 100644 index 000000000000..f6e422cba7fb --- /dev/null +++ b/ruby/ext/google/protobuf_c/shared_message.c @@ -0,0 +1,88 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2023 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// ----------------------------------------------------------------------------- +// Ruby Message functions. Strictly free of dependencies on +// Ruby interpreter internals. + +#include "shared_message.h" + +// Support function for Message_Hash. Returns a hash value for the given +// message. +uint64_t shared_Message_Hash(const upb_Message* msg, const upb_MessageDef* m, + uint64_t seed, upb_Status* status) { + upb_Arena* arena = upb_Arena_New(); + char* data; + size_t size; + + // Hash a deterministically serialized payloads with no unknown fields. + upb_EncodeStatus encode_status = upb_Encode( + msg, upb_MessageDef_MiniTable(m), + kUpb_EncodeOption_SkipUnknown | kUpb_EncodeOption_Deterministic, arena, + &data, &size); + + if (encode_status == kUpb_EncodeStatus_Ok) { + uint64_t ret = _upb_Hash(data, size, seed); + upb_Arena_Free(arena); + return ret; + } else { + upb_Arena_Free(arena); + upb_Status_SetErrorMessage(status, "Error calculating hash"); + } +} + +// Support function for Message_Equal +bool shared_Message_Equal(const upb_Message* m1, const upb_Message* m2, + const upb_MessageDef* m, upb_Status* status) { + if (m1 == m2) return true; + + size_t size1, size2; + int encode_opts = + kUpb_EncodeOption_SkipUnknown | kUpb_EncodeOption_Deterministic; + upb_Arena* arena_tmp = upb_Arena_New(); + const upb_MiniTable* layout = upb_MessageDef_MiniTable(m); + + // Compare deterministically serialized payloads with no unknown fields. + char* data1; + char* data2; + upb_EncodeStatus status1 = + upb_Encode(m1, layout, encode_opts, arena_tmp, &data1, &size1); + upb_EncodeStatus status2 = + upb_Encode(m2, layout, encode_opts, arena_tmp, &data2, &size2); + + if (status1 == kUpb_EncodeStatus_Ok && status2 == kUpb_EncodeStatus_Ok) { + bool ret = (size1 == size2) && (memcmp(data1, data2, size1) == 0); + upb_Arena_Free(arena_tmp); + return ret; + } else { + upb_Arena_Free(arena_tmp); + upb_Status_SetErrorMessage(status, "Error comparing messages"); + } +} diff --git a/ruby/ext/google/protobuf_c/shared_message.h b/ruby/ext/google/protobuf_c/shared_message.h new file mode 100644 index 000000000000..b84043ed5b57 --- /dev/null +++ b/ruby/ext/google/protobuf_c/shared_message.h @@ -0,0 +1,48 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2023 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// ----------------------------------------------------------------------------- +// Ruby Message functions. Strictly free of dependencies on +// Ruby interpreter internals. + +#ifndef RUBY_PROTOBUF_SHARED_MESSAGE_H_ +#define RUBY_PROTOBUF_SHARED_MESSAGE_H_ + +#include "ruby-upb.h" + +// Returns a hash value for the given message. +uint64_t shared_Message_Hash(const upb_Message* msg, const upb_MessageDef* m, + uint64_t seed, upb_Status* status); + +// Returns true if these two messages are equal. +bool shared_Message_Equal(const upb_Message* m1, const upb_Message* m2, + const upb_MessageDef* m, upb_Status* status); + +#endif // RUBY_PROTOBUF_SHARED_MESSAGE_H_ diff --git a/ruby/google-protobuf.gemspec b/ruby/google-protobuf.gemspec index fbf063167186..296cdb2cbcaf 100644 --- a/ruby/google-protobuf.gemspec +++ b/ruby/google-protobuf.gemspec @@ -10,15 +10,31 @@ Gem::Specification.new do |s| s.email = "protobuf@googlegroups.com" s.metadata = { "source_code_uri" => "https://github.com/protocolbuffers/protobuf/tree/#{git_tag}/ruby" } s.require_paths = ["lib"] - s.files = Dir.glob('lib/**/*.rb') + s.files = Dir.glob('lib/**/*.{rb,rake}') if RUBY_PLATFORM == "java" s.platform = "java" - s.files += ["lib/google/protobuf_java.jar"] + s.files += ["lib/google/protobuf_java.jar"] + + Dir.glob('ext/**/*').reject do |file| + File.basename(file) =~ /^((convert|defs|map|repeated_field)\.[ch]| + BUILD\.bazel|extconf\.rb|wrap_memcpy\.c)$/x + end + s.extensions = ["ext/google/protobuf_c/Rakefile"] + s.add_dependency "ffi", "~>1" + s.add_dependency "ffi-compiler", "~>1" else - s.files += Dir.glob('ext/**/*') - s.extensions= ["ext/google/protobuf_c/extconf.rb"] - s.add_development_dependency "rake-compiler-dock", "= 1.2.1" end + s.files += Dir.glob('ext/**/*').reject do |file| + File.basename(file) =~ /^(BUILD\.bazel)$/ + end + s.extensions = %w[ + ext/google/protobuf_c/extconf.rb + ext/google/protobuf_c/Rakefile + ] + s.add_development_dependency "rake-compiler-dock", "= 1.2.1" + end s.required_ruby_version = '>= 2.5' + s.add_development_dependency "rake", "~> 13" + s.add_development_dependency "ffi", "~>1" + s.add_development_dependency "ffi-compiler", "~>1" s.add_development_dependency "rake-compiler", "~> 1.1.0" s.add_development_dependency "test-unit", '~> 3.0', '>= 3.0.9' end diff --git a/ruby/lib/google/BUILD.bazel b/ruby/lib/google/BUILD.bazel index 4cfa5cbcfe41..18f1c18444ee 100644 --- a/ruby/lib/google/BUILD.bazel +++ b/ruby/lib/google/BUILD.bazel @@ -8,9 +8,16 @@ config_setting( cc_binary( name = "protobuf_c.so", + linkshared = 1, + tags = ["manual"], deps = ["//ruby/ext/google/protobuf_c"], +) + +cc_binary( + name = "libprotobuf_c_ffi.so", linkshared = 1, tags = ["manual"], + deps = ["//ruby/ext/google/protobuf_c:protobuf_c_ffi"], ) # Move the bundle to the location expected by our Ruby files. @@ -22,13 +29,25 @@ genrule( tags = ["manual"], ) +# Move the bundle to the location expected by our Ruby files. +genrule( + name = "copy_ffi_bundle", + srcs = ["//ruby/ext/google/protobuf_c:ffi_bundle"], + outs = ["libprotobuf_c_ffi.bundle"], + cmd = "cp $< $@", + tags = ["manual"], + visibility = [ + "//ruby:__subpackages__", + ], +) + java_binary( name = "protobuf_java_bin", + create_executable = False, + deploy_env = ["@rules_ruby//ruby/runtime:jruby_binary"], runtime_deps = [ - "//ruby/src/main/java:protobuf_java" + "//ruby/src/main/java:protobuf_java", ], - deploy_env = ["@rules_ruby//ruby/runtime:jruby_binary"], - create_executable = False, ) # Move the jar to the location expected by our Ruby files. @@ -46,19 +65,34 @@ ruby_library( srcs = glob([ "**/*.rb", ]), - deps = ["//:well_known_ruby_protos"], - includes = ["ruby/lib"], data = select({ + # Platform native implementations "@rules_ruby//ruby/runtime:config_jruby": ["protobuf_java.jar"], "@platforms//os:osx": ["protobuf_c.bundle"], "//conditions:default": ["protobuf_c.so"], + }) + select({ + # FFI Implementations + "//ruby:macos_ffi_enabled": ["libprotobuf_c_ffi.bundle"], + "//ruby:linux_ffi_enabled": ["libprotobuf_c_ffi.so"], + "//conditions:default": [], }), + includes = ["ruby/lib"], visibility = ["//ruby:__pkg__"], + deps = ["//:well_known_ruby_protos"] + select({ + "//ruby:ffi_enabled": [ + "@protobuf_bundle//:ffi", + "@protobuf_bundle//:ffi-compiler", + ], + "//conditions:default": [], + }), ) pkg_files( name = "dist_files", - srcs = glob(["**/*.rb"]), + srcs = glob([ + "**/*.rb", + "**/*.rake", + ]), strip_prefix = strip_prefix.from_root(""), visibility = ["//ruby:__pkg__"], ) diff --git a/ruby/lib/google/protobuf.rb b/ruby/lib/google/protobuf.rb index 07985f6858aa..ca3e33b58970 100644 --- a/ruby/lib/google/protobuf.rb +++ b/ruby/lib/google/protobuf.rb @@ -39,26 +39,16 @@ module Protobuf class Error < StandardError; end class ParseError < Error; end class TypeError < ::TypeError; end - end -end - -if RUBY_PLATFORM == "java" - require 'json' - require 'google/protobuf_java' -else - begin - require "google/#{RUBY_VERSION.sub(/\.\d+$/, '')}/protobuf_c" - rescue LoadError - require 'google/protobuf_c' - end - -end -require 'google/protobuf/descriptor_dsl' -require 'google/protobuf/repeated_field' - -module Google - module Protobuf + PREFER_FFI = case ENV['PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION'] + when nil, "", /^native$/i + false + when /^ffi$/i + true + else + warn "Unexpected value `#{ENV['PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION']}` for environment variable `PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION`. Should be either \"FFI\", \"NATIVE\"." + false + end def self.encode(msg, options = {}) msg.to_proto(options) @@ -76,5 +66,19 @@ def self.decode_json(klass, json, options = {}) klass.decode_json(json, options) end + IMPLEMENTATION = if PREFER_FFI + begin + require 'google/protobuf_ffi' + :FFI + rescue LoadError + warn "Caught exception `#{$!.message}` while loading FFI implementation of google/protobuf." + warn "Falling back to native implementation." + require 'google/protobuf_native' + :NATIVE + end + else + require 'google/protobuf_native' + :NATIVE + end end end diff --git a/ruby/lib/google/protobuf/ffi/descriptor.rb b/ruby/lib/google/protobuf/ffi/descriptor.rb new file mode 100644 index 000000000000..bd56727c2624 --- /dev/null +++ b/ruby/lib/google/protobuf/ffi/descriptor.rb @@ -0,0 +1,177 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2022 Google Inc. All rights reserved. +# https://developers.google.com/protocol-buffers/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +module Google + module Protobuf + ## + # Message Descriptor - Descriptor for short. + class Descriptor + attr :descriptor_pool, :msg_class + include Enumerable + + # FFI Interface methods and setup + extend ::FFI::DataConverter + native_type ::FFI::Type::POINTER + + class << self + prepend Google::Protobuf::Internal::TypeSafety + include Google::Protobuf::Internal::PointerHelper + + # @param value [Descriptor] Descriptor to convert to an FFI native type + # @param _ [Object] Unused + def to_native(value, _ = nil) + msg_def_ptr = value.nil? ? nil : value.instance_variable_get(:@msg_def) + return ::FFI::Pointer::NULL if msg_def_ptr.nil? + raise "Underlying msg_def was null!" if msg_def_ptr.null? + msg_def_ptr + end + + ## + # @param msg_def [::FFI::Pointer] MsgDef pointer to be wrapped + # @param _ [Object] Unused + def from_native(msg_def, _ = nil) + return nil if msg_def.nil? or msg_def.null? + file_def = Google::Protobuf::FFI.get_message_file_def msg_def + descriptor_from_file_def(file_def, msg_def) + end + end + + def to_native + self.class.to_native(self) + end + + ## + # Great write up of this strategy: + # See https://blog.appsignal.com/2018/08/07/ruby-magic-changing-the-way-ruby-creates-objects.html + def self.new(*arguments, &block) + raise "Descriptor objects may not be created from Ruby." + end + + def to_s + inspect + end + + def inspect + "Descriptor - (not the message class) #{name}" + end + + def file_descriptor + @descriptor_pool.send(:get_file_descriptor, Google::Protobuf::FFI.get_message_file_def(@msg_def)) + end + + def name + @name ||= Google::Protobuf::FFI.get_message_fullname(self) + end + + def each_oneof &block + n = Google::Protobuf::FFI.oneof_count(self) + 0.upto(n-1) do |i| + yield(Google::Protobuf::FFI.get_oneof_by_index(self, i)) + end + nil + end + + def each &block + n = Google::Protobuf::FFI.field_count(self) + 0.upto(n-1) do |i| + yield(Google::Protobuf::FFI.get_field_by_index(self, i)) + end + nil + end + + def lookup(name) + Google::Protobuf::FFI.get_field_by_name(self, name, name.size) + end + + def lookup_oneof(name) + Google::Protobuf::FFI.get_oneof_by_name(self, name, name.size) + end + + def msgclass + @msg_class ||= build_message_class + end + + private + + extend Google::Protobuf::Internal::Convert + + def initialize(msg_def, descriptor_pool) + @msg_def = msg_def + @msg_class = nil + @descriptor_pool = descriptor_pool + end + + def self.private_constructor(msg_def, descriptor_pool) + instance = allocate + instance.send(:initialize, msg_def, descriptor_pool) + instance + end + + def wrapper? + if defined? @wrapper + @wrapper + else + @wrapper = case Google::Protobuf::FFI.get_well_known_type self + when :DoubleValue, :FloatValue, :Int64Value, :UInt64Value, :Int32Value, :UInt32Value, :StringValue, :BytesValue, :BoolValue + true + else + false + end + end + end + + def self.get_message(msg, descriptor, arena) + return nil if msg.nil? or msg.null? + message = OBJECT_CACHE.get(msg.address) + if message.nil? + message = descriptor.msgclass.send(:private_constructor, arena, msg: msg) + end + message + end + + def pool + @descriptor_pool + end + end + + class FFI + # MessageDef + attach_function :new_message_from_def, :upb_Message_New, [Descriptor, Internal::Arena], :Message + attach_function :field_count, :upb_MessageDef_FieldCount, [Descriptor], :int + attach_function :get_message_file_def, :upb_MessageDef_File, [:pointer], :FileDef + attach_function :get_message_fullname, :upb_MessageDef_FullName, [Descriptor], :string + attach_function :get_mini_table, :upb_MessageDef_MiniTable, [Descriptor], MiniTable.ptr + attach_function :oneof_count, :upb_MessageDef_OneofCount, [Descriptor], :int + attach_function :get_well_known_type, :upb_MessageDef_WellKnownType, [Descriptor], WellKnown + attach_function :message_def_syntax, :upb_MessageDef_Syntax, [Descriptor], Syntax + attach_function :find_msg_def_by_name, :upb_MessageDef_FindByNameWithSize, [Descriptor, :string, :size_t, :FieldDefPointer, :OneofDefPointer], :bool + end + end +end diff --git a/ruby/lib/google/protobuf/ffi/descriptor_pool.rb b/ruby/lib/google/protobuf/ffi/descriptor_pool.rb new file mode 100644 index 000000000000..b09cb8fecaa2 --- /dev/null +++ b/ruby/lib/google/protobuf/ffi/descriptor_pool.rb @@ -0,0 +1,93 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2022 Google Inc. All rights reserved. +# https://developers.google.com/protocol-buffers/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +module Google + module Protobuf + class FFI + # DefPool + attach_function :add_serialized_file, :upb_DefPool_AddFile, [:DefPool, :FileDescriptorProto, Status.by_ref], :FileDef + attach_function :free_descriptor_pool, :upb_DefPool_Free, [:DefPool], :void + attach_function :create_descriptor_pool,:upb_DefPool_New, [], :DefPool + attach_function :lookup_enum, :upb_DefPool_FindEnumByName, [:DefPool, :string], EnumDescriptor + attach_function :lookup_msg, :upb_DefPool_FindMessageByName, [:DefPool, :string], Descriptor + # FileDescriptorProto + attach_function :parse, :FileDescriptorProto_parse, [:binary_string, :size_t], :FileDescriptorProto + end + class DescriptorPool + attr :descriptor_pool + attr_accessor :descriptor_class_by_def + + def initialize + @descriptor_pool = ::FFI::AutoPointer.new(Google::Protobuf::FFI.create_descriptor_pool, Google::Protobuf::FFI.method(:free_descriptor_pool)) + @descriptor_class_by_def = {} + + # Should always be the last expression of the initializer to avoid + # leaking references to this object before construction is complete. + Google::Protobuf::OBJECT_CACHE.try_add @descriptor_pool.address, self + end + + def add_serialized_file(file_contents) + # Allocate memory sized to file_contents + memBuf = ::FFI::MemoryPointer.new(:char, file_contents.bytesize) + # Insert the data + memBuf.put_bytes(0, file_contents) + file_descriptor_proto = Google::Protobuf::FFI.parse memBuf, file_contents.bytesize + raise ArgumentError.new("Unable to parse FileDescriptorProto") if file_descriptor_proto.null? + + status = Google::Protobuf::FFI::Status.new + file_descriptor = Google::Protobuf::FFI.add_serialized_file @descriptor_pool, file_descriptor_proto, status + if file_descriptor.null? + raise TypeError.new("Unable to build file to DescriptorPool: #{Google::Protobuf::FFI.error_message(status)}") + else + @descriptor_class_by_def[file_descriptor.address] = FileDescriptor.new file_descriptor, self + end + end + + def lookup name + Google::Protobuf::FFI.lookup_msg(@descriptor_pool, name) || + Google::Protobuf::FFI.lookup_enum(@descriptor_pool, name) + end + + def self.generated_pool + @@generated_pool ||= DescriptorPool.new + end + + private + + # Implementation details below are subject to breaking changes without + # warning and are intended for use only within the gem. + + def get_file_descriptor file_def + return nil if file_def.null? + @descriptor_class_by_def[file_def.address] ||= FileDescriptor.new(file_def, self) + end + end + end +end diff --git a/ruby/lib/google/protobuf/ffi/enum_descriptor.rb b/ruby/lib/google/protobuf/ffi/enum_descriptor.rb new file mode 100644 index 000000000000..37b846df14f0 --- /dev/null +++ b/ruby/lib/google/protobuf/ffi/enum_descriptor.rb @@ -0,0 +1,184 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2022 Google Inc. All rights reserved. +# https://developers.google.com/protocol-buffers/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +module Google + module Protobuf + class EnumDescriptor + attr :descriptor_pool, :enum_def + include Enumerable + + # FFI Interface methods and setup + extend ::FFI::DataConverter + native_type ::FFI::Type::POINTER + + class << self + prepend Google::Protobuf::Internal::TypeSafety + include Google::Protobuf::Internal::PointerHelper + + # @param value [Arena] Arena to convert to an FFI native type + # @param _ [Object] Unused + def to_native(value, _) + value.instance_variable_get(:@enum_def) || ::FFI::Pointer::NULL + end + + ## + # @param enum_def [::FFI::Pointer] EnumDef pointer to be wrapped + # @param _ [Object] Unused + def from_native(enum_def, _) + return nil if enum_def.nil? or enum_def.null? + file_def = Google::Protobuf::FFI.get_message_file_def enum_def + descriptor_from_file_def(file_def, enum_def) + end + end + + def self.new(*arguments, &block) + raise "Descriptor objects may not be created from Ruby." + end + + def file_descriptor + @descriptor_pool.send(:get_file_descriptor, Google::Protobuf::FFI.get_enum_file_descriptor(self)) + end + + def name + Google::Protobuf::FFI.get_enum_fullname(self) + end + + def to_s + inspect + end + + def inspect + "#{self.class.name}: #{name}" + end + + def lookup_name(name) + self.class.send(:lookup_name, self, name) + end + + def lookup_value(number) + self.class.send(:lookup_value, self, number) + end + + def each &block + n = Google::Protobuf::FFI.enum_value_count(self) + 0.upto(n - 1) do |i| + enum_value = Google::Protobuf::FFI.enum_value_by_index(self, i) + yield(Google::Protobuf::FFI.enum_name(enum_value).to_sym, Google::Protobuf::FFI.enum_number(enum_value)) + end + nil + end + + def enummodule + if @module.nil? + @module = build_enum_module + end + @module + end + + private + + def initialize(enum_def, descriptor_pool) + @descriptor_pool = descriptor_pool + @enum_def = enum_def + @module = nil + end + + def self.private_constructor(enum_def, descriptor_pool) + instance = allocate + instance.send(:initialize, enum_def, descriptor_pool) + instance + end + + def self.lookup_value(enum_def, number) + enum_value = Google::Protobuf::FFI.enum_value_by_number(enum_def, number) + if enum_value.null? + nil + else + Google::Protobuf::FFI.enum_name(enum_value).to_sym + end + end + + def self.lookup_name(enum_def, name) + enum_value = Google::Protobuf::FFI.enum_value_by_name(enum_def, name.to_s, name.size) + if enum_value.null? + nil + else + Google::Protobuf::FFI.enum_number(enum_value) + end + end + + def build_enum_module + descriptor = self + dynamic_module = Module.new do + @descriptor = descriptor + + class << self + attr_accessor :descriptor + end + + def self.lookup(number) + descriptor.lookup_value number + end + + def self.resolve(name) + descriptor.lookup_name name + end + end + + self.each do |name, value| + if name[0] < 'A' || name[0] > 'Z' + if name[0] >= 'a' and name[0] <= 'z' + name = name[0].upcase + name[1..-1] # auto capitalize + else + warn( + "Enum value '#{name}' does not start with an uppercase letter " + + "as is required for Ruby constants.") + next + end + end + dynamic_module.const_set(name.to_sym, value) + end + dynamic_module + end + end + + class FFI + # EnumDescriptor + attach_function :get_enum_file_descriptor, :upb_EnumDef_File, [EnumDescriptor], :FileDef + attach_function :enum_value_by_name, :upb_EnumDef_FindValueByNameWithSize,[EnumDescriptor, :string, :size_t], :EnumValueDef + attach_function :enum_value_by_number, :upb_EnumDef_FindValueByNumber, [EnumDescriptor, :int], :EnumValueDef + attach_function :get_enum_fullname, :upb_EnumDef_FullName, [EnumDescriptor], :string + attach_function :enum_value_by_index, :upb_EnumDef_Value, [EnumDescriptor, :int], :EnumValueDef + attach_function :enum_value_count, :upb_EnumDef_ValueCount, [EnumDescriptor], :int + attach_function :enum_name, :upb_EnumValueDef_Name, [:EnumValueDef], :string + attach_function :enum_number, :upb_EnumValueDef_Number, [:EnumValueDef], :int + end + end +end diff --git a/ruby/lib/google/protobuf/ffi/ffi.rb b/ruby/lib/google/protobuf/ffi/ffi.rb new file mode 100644 index 000000000000..69c57b955ab2 --- /dev/null +++ b/ruby/lib/google/protobuf/ffi/ffi.rb @@ -0,0 +1,236 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2022 Google Inc. All rights reserved. +# https://developers.google.com/protocol-buffers/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +module Google + module Protobuf + class FFI + extend ::FFI::Library + # Workaround for Bazel's use of symlinks + JRuby's __FILE__ and `caller` + # that resolves them. + if ENV['BAZEL'] == 'true' + ffi_lib ::FFI::Compiler::Loader.find 'protobuf_c_ffi', ENV['PWD'] + else + ffi_lib ::FFI::Compiler::Loader.find 'protobuf_c_ffi' + end + + ## Map + Upb_Map_Begin = -1 + + ## Encoding Status + Upb_Status_MaxMessage = 127 + Upb_Encode_Deterministic = 1 + Upb_Encode_SkipUnknown = 2 + + ## JSON Encoding options + # When set, emits 0/default values. TODO(haberman): proto3 only? + Upb_JsonEncode_EmitDefaults = 1 + # When set, use normal (snake_case) field names instead of JSON (camelCase) names. + Upb_JsonEncode_UseProtoNames = 2 + # When set, emits enums as their integer values instead of as their names. + Upb_JsonEncode_FormatEnumsAsIntegers = 4 + + ## JSON Decoding options + Upb_JsonDecode_IgnoreUnknown = 1 + + typedef :pointer, :Array + typedef :pointer, :DefPool + typedef :pointer, :EnumValueDef + typedef :pointer, :ExtensionRegistry + typedef :pointer, :FieldDefPointer + typedef :pointer, :FileDef + typedef :pointer, :FileDescriptorProto + typedef :pointer, :Map + typedef :pointer, :Message # Instances of a message + typedef :pointer, :OneofDefPointer + typedef :pointer, :binary_string + if ::FFI::Platform::ARCH == "aarch64" + typedef :u_int8_t, :uint8_t + typedef :u_int16_t, :uint16_t + typedef :u_int32_t, :uint32_t + typedef :u_int64_t, :uint64_t + end + + FieldType = enum( + :double, 1, + :float, + :int64, + :uint64, + :int32, + :fixed64, + :fixed32, + :bool, + :string, + :group, + :message, + :bytes, + :uint32, + :enum, + :sfixed32, + :sfixed64, + :sint32, + :sint64 + ) + + CType = enum( + :bool, 1, + :float, + :int32, + :uint32, + :enum, + :message, + :double, + :int64, + :uint64, + :string, + :bytes + ) + + Label = enum( + :optional, 1, + :required, + :repeated + ) + + Syntax = enum( + :Proto2, 2, + :Proto3 + ) + + # All the different kind of well known type messages. For simplicity of check, + # number wrappers and string wrappers are grouped together. Make sure the + # order and merber of these groups are not changed. + + WellKnown = enum( + :Unspecified, + :Any, + :FieldMask, + :Duration, + :Timestamp, + # number wrappers + :DoubleValue, + :FloatValue, + :Int64Value, + :UInt64Value, + :Int32Value, + :UInt32Value, + # string wrappers + :StringValue, + :BytesValue, + :BoolValue, + :Value, + :ListValue, + :Struct + ) + + DecodeStatus = enum( + :Ok, + :Malformed, # Wire format was corrupt + :OutOfMemory, # Arena alloc failed + :BadUtf8, # String field had bad UTF-8 + :MaxDepthExceeded, # Exceeded UPB_DECODE_MAXDEPTH + + # CheckRequired failed, but the parse otherwise succeeded. + :MissingRequired, + ) + + EncodeStatus = enum( + :Ok, + :OutOfMemory, # Arena alloc failed + :MaxDepthExceeded, # Exceeded UPB_DECODE_MAXDEPTH + + # CheckRequired failed, but the parse otherwise succeeded. + :MissingRequired, + ) + + class StringView < ::FFI::Struct + layout :data, :pointer, + :size, :size_t + end + + class MiniTable < ::FFI::Struct + layout :subs, :pointer, + :fields, :pointer, + :size, :uint16_t, + :field_count, :uint16_t, + :ext, :uint8_t, # upb_ExtMode, declared as uint8_t so sizeof(ext) == 1 + :dense_below, :uint8_t, + :table_mask, :uint8_t, + :required_count, :uint8_t # Required fields have the lowest hasbits. + # To statically initialize the tables of variable length, we need a flexible + # array member, and we need to compile in gnu99 mode (constant initialization + # of flexible array members is a GNU extension, not in C99 unfortunately. */ + # _upb_FastTable_Entry fasttable[]; + end + + class Status < ::FFI::Struct + layout :ok, :bool, + :msg, [:char, Upb_Status_MaxMessage] + + def initialize + super + FFI.clear self + end + end + + class MessageValue < ::FFI::Union + layout :bool_val, :bool, + :float_val, :float, + :double_val, :double, + :int32_val, :int32_t, + :int64_val, :int64_t, + :uint32_val, :uint32_t, + :uint64_val,:uint64_t, + :map_val, :pointer, + :msg_val, :pointer, + :array_val,:pointer, + :str_val, StringView + end + + class MutableMessageValue < ::FFI::Union + layout :map, :Map, + :msg, :Message, + :array, :Array + end + + # Status + attach_function :clear, :upb_Status_Clear, [Status.by_ref], :void + attach_function :error_message, :upb_Status_ErrorMessage, [Status.by_ref], :string + + # Generic + attach_function :memcmp, [:pointer, :pointer, :size_t], :int + attach_function :memcpy, [:pointer, :pointer, :size_t], :int + + # Alternatives to pre-processor macros + def self.decode_max_depth(i) + i << 16 + end + end + end +end diff --git a/ruby/lib/google/protobuf/ffi/field_descriptor.rb b/ruby/lib/google/protobuf/ffi/field_descriptor.rb new file mode 100644 index 000000000000..28b5da04d5bd --- /dev/null +++ b/ruby/lib/google/protobuf/ffi/field_descriptor.rb @@ -0,0 +1,332 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2022 Google Inc. All rights reserved. +# https://developers.google.com/protocol-buffers/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +module Google + module Protobuf + class FieldDescriptor + attr :field_def, :descriptor_pool + + include Google::Protobuf::Internal::Convert + + # FFI Interface methods and setup + extend ::FFI::DataConverter + native_type ::FFI::Type::POINTER + + class << self + prepend Google::Protobuf::Internal::TypeSafety + include Google::Protobuf::Internal::PointerHelper + + # @param value [FieldDescriptor] FieldDescriptor to convert to an FFI native type + # @param _ [Object] Unused + def to_native(value, _) + field_def_ptr = value.instance_variable_get(:@field_def) + warn "Underlying field_def was nil!" if field_def_ptr.nil? + raise "Underlying field_def was null!" if !field_def_ptr.nil? and field_def_ptr.null? + field_def_ptr + end + + ## + # @param field_def [::FFI::Pointer] FieldDef pointer to be wrapped + # @param _ [Object] Unused + def from_native(field_def, _ = nil) + return nil if field_def.nil? or field_def.null? + file_def = Google::Protobuf::FFI.file_def_by_raw_field_def(field_def) + descriptor_from_file_def(file_def, field_def) + end + end + + def self.new(*arguments, &block) + raise "Descriptor objects may not be created from Ruby." + end + + def to_s + inspect + end + + def inspect + "#{self.class.name}: #{name}" + end + + def name + @name ||= Google::Protobuf::FFI.get_full_name(self) + end + + def json_name + @json_name ||= Google::Protobuf::FFI.get_json_name(self) + end + + def number + @number ||= Google::Protobuf::FFI.get_number(self) + end + + def type + @type ||= Google::Protobuf::FFI.get_type(self) + end + + def label + @label ||= Google::Protobuf::FFI::Label[Google::Protobuf::FFI.get_label(self)] + end + + def default + return nil if Google::Protobuf::FFI.is_sub_message(self) + if Google::Protobuf::FFI.is_repeated(self) + message_value = Google::Protobuf::FFI::MessageValue.new + else + message_value = Google::Protobuf::FFI.get_default(self) + end + enum_def = Google::Protobuf::FFI.get_subtype_as_enum(self) + if enum_def.null? + convert_upb_to_ruby message_value, c_type + else + convert_upb_to_ruby message_value, c_type, enum_def + end + end + + def submsg_name + if defined? @submsg_name + @submsg_name + else + @submsg_name = case c_type + when :enum + Google::Protobuf::FFI.get_enum_fullname Google::Protobuf::FFI.get_subtype_as_enum self + when :message + Google::Protobuf::FFI.get_message_fullname Google::Protobuf::FFI.get_subtype_as_message self + else + nil + end + end + end + + ## + # Tests if this field has been set on the argument message. + # + # @param msg [Google::Protobuf::Message] + # @return [Object] Value of the field on this message. + # @raise [TypeError] If the field is not defined on this message. + def get(msg) + if msg.class.descriptor == Google::Protobuf::FFI.get_containing_message_def(self) + msg.send :get_field, self + else + raise TypeError.new "get method called on wrong message type" + end + end + + def subtype + if defined? @subtype + @subtype + else + @subtype = case c_type + when :enum + Google::Protobuf::FFI.get_subtype_as_enum(self) + when :message + Google::Protobuf::FFI.get_subtype_as_message(self) + else + nil + end + end + end + + ## + # Tests if this field has been set on the argument message. + # + # @param msg [Google::Protobuf::Message] + # @return [Boolean] True iff message has this field set + # @raise [TypeError] If this field does not exist on the message + # @raise [ArgumentError] If this field does not track presence + def has?(msg) + if msg.class.descriptor != Google::Protobuf::FFI.get_containing_message_def(self) + raise TypeError.new "has method called on wrong message type" + end + unless has_presence? + raise ArgumentError.new "does not track presence" + end + + Google::Protobuf::FFI.get_message_has msg.instance_variable_get(:@msg), self + end + + ## + # Tests if this field tracks presence. + # + # @return [Boolean] True iff this field tracks presence + def has_presence? + @has_presence ||= Google::Protobuf::FFI.get_has_presence(self) + end + + # @param msg [Google::Protobuf::Message] + def clear(msg) + if msg.class.descriptor != Google::Protobuf::FFI.get_containing_message_def(self) + raise TypeError.new "clear method called on wrong message type" + end + Google::Protobuf::FFI.clear_message_field msg.instance_variable_get(:@msg), self + nil + end + + ## + # call-seq: + # FieldDescriptor.set(message, value) + # + # Sets the value corresponding to this field to the given value on the given + # message. Raises an exception if message is of the wrong type. Performs the + # ordinary type-checks for field setting. + # + # @param msg [Google::Protobuf::Message] + # @param value [Object] + def set(msg, value) + if msg.class.descriptor != Google::Protobuf::FFI.get_containing_message_def(self) + raise TypeError.new "set method called on wrong message type" + end + unless set_value_on_message value, msg.instance_variable_get(:@msg), msg.instance_variable_get(:@arena) + raise RuntimeError.new "allocation failed" + end + nil + end + + def map? + @map ||= Google::Protobuf::FFI.is_map self + end + + def repeated? + @repeated ||= Google::Protobuf::FFI.is_repeated self + end + + def sub_message? + @sub_message ||= Google::Protobuf::FFI.is_sub_message self + end + + def wrapper? + if defined? @wrapper + @wrapper + else + message_descriptor = Google::Protobuf::FFI.get_subtype_as_message(self) + @wrapper = message_descriptor.nil? ? false : message_descriptor.send(:wrapper?) + end + end + + private + + def initialize(field_def, descriptor_pool) + @field_def = field_def + @descriptor_pool = descriptor_pool + end + + def self.private_constructor(field_def, descriptor_pool) + instance = allocate + instance.send(:initialize, field_def, descriptor_pool) + instance + end + + # TODO(jatl) Can this be added to the public API? + def real_containing_oneof + @real_containing_oneof ||= Google::Protobuf::FFI.real_containing_oneof self + end + + # Implementation details below are subject to breaking changes without + # warning and are intended for use only within the gem. + + ## + # Sets the field this FieldDescriptor represents to the given value on the given message. + # @param value [Object] Value to be set + # @param msg [::FFI::Pointer] Pointer the the upb_Message + # @param arena [Arena] Arena of the message that owns msg + def set_value_on_message(value, msg, arena, wrap: false) + message_to_alter = msg + field_def_to_set = self + if map? + raise TypeError.new "Expected map" unless value.is_a? Google::Protobuf::Map + message_descriptor = subtype + + key_field_def = Google::Protobuf::FFI.get_field_by_number(message_descriptor, 1) + key_field_type = Google::Protobuf::FFI.get_type(key_field_def) + raise TypeError.new "Map key type does not match field's key type" unless key_field_type == value.send(:key_type) + + value_field_def = Google::Protobuf::FFI.get_field_by_number(message_descriptor, 2) + value_field_type = Google::Protobuf::FFI.get_type(value_field_def) + raise TypeError.new "Map value type does not match field's value type" unless value_field_type == value.send(:value_type) + + raise TypeError.new "Map value type has wrong message/enum class" unless value_field_def.subtype == value.send(:descriptor) + + arena.fuse(value.send(:arena)) + message_value = Google::Protobuf::FFI::MessageValue.new + message_value[:map_val] = value.send(:map_ptr) + elsif repeated? + raise TypeError.new "Expected repeated field array" unless value.is_a? RepeatedField + raise TypeError.new "Repeated field array has wrong message/enum class" unless value.send(:type) == type + arena.fuse(value.send(:arena)) + message_value = Google::Protobuf::FFI::MessageValue.new + message_value[:array_val] = value.send(:array) + else + if value.nil? and (sub_message? or !real_containing_oneof.nil?) + Google::Protobuf::FFI.clear_message_field message_to_alter, field_def_to_set + return true + end + if wrap + value_field_def = Google::Protobuf::FFI.get_field_by_number subtype, 1 + type_for_conversion = Google::Protobuf::FFI.get_c_type(value_field_def) + raise RuntimeError.new "Not expecting to get a msg or enum when unwrapping" if [:enum, :message].include? type_for_conversion + message_value = convert_ruby_to_upb(value, arena, type_for_conversion, nil) + message_to_alter = Google::Protobuf::FFI.get_mutable_message(msg, self, arena)[:msg] + field_def_to_set = value_field_def + else + message_value = convert_ruby_to_upb(value, arena, c_type, subtype) + end + end + Google::Protobuf::FFI.set_message_field message_to_alter, field_def_to_set, message_value, arena + end + + def c_type + @c_type ||= Google::Protobuf::FFI.get_c_type(self) + end + end + + class FFI + # MessageDef + attach_function :get_field_by_index, :upb_MessageDef_Field, [Descriptor, :int], FieldDescriptor + attach_function :get_field_by_name, :upb_MessageDef_FindFieldByNameWithSize,[Descriptor, :string, :size_t], FieldDescriptor + attach_function :get_field_by_number, :upb_MessageDef_FindFieldByNumber, [Descriptor, :uint32_t], FieldDescriptor + + # FieldDescriptor + attach_function :get_containing_message_def, :upb_FieldDef_ContainingType, [FieldDescriptor], Descriptor + attach_function :get_c_type, :upb_FieldDef_CType, [FieldDescriptor], CType + attach_function :get_default, :upb_FieldDef_Default, [FieldDescriptor], MessageValue.by_value + attach_function :get_subtype_as_enum, :upb_FieldDef_EnumSubDef, [FieldDescriptor], EnumDescriptor + attach_function :get_has_presence, :upb_FieldDef_HasPresence, [FieldDescriptor], :bool + attach_function :is_map, :upb_FieldDef_IsMap, [FieldDescriptor], :bool + attach_function :is_repeated, :upb_FieldDef_IsRepeated, [FieldDescriptor], :bool + attach_function :is_sub_message, :upb_FieldDef_IsSubMessage, [FieldDescriptor], :bool + attach_function :get_json_name, :upb_FieldDef_JsonName, [FieldDescriptor], :string + attach_function :get_label, :upb_FieldDef_Label, [FieldDescriptor], Label + attach_function :get_subtype_as_message, :upb_FieldDef_MessageSubDef, [FieldDescriptor], Descriptor + attach_function :get_full_name, :upb_FieldDef_Name, [FieldDescriptor], :string + attach_function :get_number, :upb_FieldDef_Number, [FieldDescriptor], :uint32_t + attach_function :get_type, :upb_FieldDef_Type, [FieldDescriptor], FieldType + attach_function :file_def_by_raw_field_def, :upb_FieldDef_File, [:pointer], :FileDef + end + end +end diff --git a/ruby/lib/google/protobuf/ffi/file_descriptor.rb b/ruby/lib/google/protobuf/ffi/file_descriptor.rb new file mode 100644 index 000000000000..05968a36e288 --- /dev/null +++ b/ruby/lib/google/protobuf/ffi/file_descriptor.rb @@ -0,0 +1,71 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2022 Google Inc. All rights reserved. +# https://developers.google.com/protocol-buffers/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +module Google + module Protobuf + class FFI + # FileDescriptor + attach_function :file_def_name, :upb_FileDef_Name, [:FileDef], :string + attach_function :file_def_syntax, :upb_FileDef_Syntax, [:FileDef], Syntax + attach_function :file_def_pool, :upb_FileDef_Pool, [:FileDef], :DefPool + end + class FileDescriptor + attr :descriptor_pool, :file_def + + def initialize(file_def, descriptor_pool) + @descriptor_pool = descriptor_pool + @file_def = file_def + end + + def to_s + inspect + end + + def inspect + "#{self.class.name}: #{name}" + end + + def syntax + case Google::Protobuf::FFI.file_def_syntax(@file_def) + when :Proto3 + :proto3 + when :Proto2 + :proto2 + else + nil + end + end + + def name + Google::Protobuf::FFI.file_def_name(@file_def) + end + end + end +end diff --git a/ruby/lib/google/protobuf/ffi/internal/arena.rb b/ruby/lib/google/protobuf/ffi/internal/arena.rb new file mode 100644 index 000000000000..757a84fc9093 --- /dev/null +++ b/ruby/lib/google/protobuf/ffi/internal/arena.rb @@ -0,0 +1,83 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2022 Google Inc. All rights reserved. +# https://developers.google.com/protocol-buffers/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +## +# Implementation details below are subject to breaking changes without +# warning and are intended for use only within the gem. +module Google + module Protobuf + module Internal + class Arena + + # FFI Interface methods and setup + extend ::FFI::DataConverter + native_type ::FFI::Type::POINTER + + class << self + prepend Google::Protobuf::Internal::TypeSafety + + # @param value [Arena] Arena to convert to an FFI native type + # @param _ [Object] Unused + def to_native(value, _) + value.instance_variable_get(:@arena) || ::FFI::Pointer::NULL + end + + ## + # @param value [::FFI::Pointer] Arena pointer to be wrapped + # @param _ [Object] Unused + def from_native(value, _) + new(value) + end + end + + def initialize(pointer) + @arena = ::FFI::AutoPointer.new(pointer, Google::Protobuf::FFI.method(:free_arena)) + end + + def fuse(other_arena) + return if other_arena == self + unless Google::Protobuf::FFI.fuse_arena(self, other_arena) + raise RuntimeError.new "Unable to fuse arenas. This should never happen since Ruby does not use initial blocks" + end + end + end + end + + class FFI + # Arena + attach_function :create_arena, :Arena_create, [], Internal::Arena + attach_function :fuse_arena, :upb_Arena_Fuse, [Internal::Arena, Internal::Arena], :bool + # Argument takes a :pointer rather than a typed Arena here due to + # implementation details of FFI::AutoPointer. + attach_function :free_arena, :upb_Arena_Free, [:pointer], :void + attach_function :arena_malloc, :upb_Arena_Malloc, [Internal::Arena, :size_t], :pointer + end + end +end diff --git a/ruby/lib/google/protobuf/ffi/internal/convert.rb b/ruby/lib/google/protobuf/ffi/internal/convert.rb new file mode 100644 index 000000000000..414598648a1e --- /dev/null +++ b/ruby/lib/google/protobuf/ffi/internal/convert.rb @@ -0,0 +1,328 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2022 Google Inc. All rights reserved. +# https://developers.google.com/protocol-buffers/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +## +# Implementation details below are subject to breaking changes without +# warning and are intended for use only within the gem. +module Google + module Protobuf + module Internal + module Convert + + # Arena should be the + # @param value [Object] Value to convert + # @param arena [Arena] Arena that owns the Message where the MessageValue + # will be set + # @return [Google::Protobuf::FFI::MessageValue] + def convert_ruby_to_upb(value, arena, c_type, msg_or_enum_def) + raise ArgumentError.new "Expected Descriptor or EnumDescriptor, instead got #{msg_or_enum_def.class}" unless [NilClass, Descriptor, EnumDescriptor].include? msg_or_enum_def.class + return_value = Google::Protobuf::FFI::MessageValue.new + case c_type + when :float + raise TypeError.new "Expected number type for float field '#{name}' (given #{value.class})." unless value.respond_to? :to_f + return_value[:float_val] = value.to_f + when :double + raise TypeError.new "Expected number type for double field '#{name}' (given #{value.class})." unless value.respond_to? :to_f + return_value[:double_val] = value.to_f + when :bool + raise TypeError.new "Invalid argument for boolean field '#{name}' (given #{value.class})." unless [TrueClass, FalseClass].include? value.class + return_value[:bool_val] = value + when :string + raise TypeError.new "Invalid argument for string field '#{name}' (given #{value.class})." unless [Symbol, String].include? value.class + begin + string_value = value.to_s.encode("UTF-8") + rescue Encoding::UndefinedConversionError + # TODO(jatl) - why not include the field name here? + raise Encoding::UndefinedConversionError.new "String is invalid UTF-8" + end + return_value[:str_val][:size] = string_value.bytesize + return_value[:str_val][:data] = Google::Protobuf::FFI.arena_malloc(arena, string_value.bytesize) + # TODO(jatl) - how important is it to still use arena malloc, versus the following? + # buffer = ::FFI::MemoryPointer.new(:char, string_value.bytesize) + # buffer.put_bytes(0, string_value) + # return_value[:str_val][:data] = buffer + raise NoMemoryError.new "Cannot allocate #{string_value.bytesize} bytes for string on Arena" if return_value[:str_val][:data].nil? || return_value[:str_val][:data].null? + return_value[:str_val][:data].write_string(string_value) + when :bytes + raise TypeError.new "Invalid argument for bytes field '#{name}' (given #{value.class})." unless value.is_a? String + string_value = value.encode("ASCII-8BIT") + return_value[:str_val][:size] = string_value.bytesize + return_value[:str_val][:data] = Google::Protobuf::FFI.arena_malloc(arena, string_value.bytesize) + raise NoMemoryError.new "Cannot allocate #{string_value.bytesize} bytes for bytes on Arena" if return_value[:str_val][:data].nil? || return_value[:str_val][:data].null? + return_value[:str_val][:data].write_string_length(string_value, string_value.bytesize) + when :message + raise TypeError.new "nil message not allowed here." if value.nil? + if value.is_a? Hash + raise RuntimeError.new "Attempted to initialize message from Hash for field #{name} but have no definition" if msg_or_enum_def.nil? + new_message = msg_or_enum_def.msgclass. + send(:private_constructor, arena, initial_value: value) + return_value[:msg_val] = new_message.instance_variable_get(:@msg) + return return_value + end + + descriptor = value.class.respond_to?(:descriptor) ? value.class.descriptor : nil + if descriptor != msg_or_enum_def + wkt = Google::Protobuf::FFI.get_well_known_type(msg_or_enum_def) + case wkt + when :Timestamp + raise TypeError.new "Invalid type #{value.class} to assign to submessage field '#{name}'." unless value.kind_of? Time + new_message = Google::Protobuf::FFI.new_message_from_def msg_or_enum_def, arena + sec = Google::Protobuf::FFI::MessageValue.new + sec[:int64_val] = value.tv_sec + sec_field_def = Google::Protobuf::FFI.get_field_by_number msg_or_enum_def, 1 + raise "Should be impossible" unless Google::Protobuf::FFI.set_message_field new_message, sec_field_def, sec, arena + nsec_field_def = Google::Protobuf::FFI.get_field_by_number msg_or_enum_def, 2 + nsec = Google::Protobuf::FFI::MessageValue.new + nsec[:int32_val] = value.tv_nsec + raise "Should be impossible" unless Google::Protobuf::FFI.set_message_field new_message, nsec_field_def, nsec, arena + return_value[:msg_val] = new_message + when :Duration + raise TypeError.new "Invalid type #{value.class} to assign to submessage field '#{name}'." unless value.kind_of? Numeric + new_message = Google::Protobuf::FFI.new_message_from_def msg_or_enum_def, arena + sec = Google::Protobuf::FFI::MessageValue.new + sec[:int64_val] = value + sec_field_def = Google::Protobuf::FFI.get_field_by_number msg_or_enum_def, 1 + raise "Should be impossible" unless Google::Protobuf::FFI.set_message_field new_message, sec_field_def, sec, arena + nsec_field_def = Google::Protobuf::FFI.get_field_by_number msg_or_enum_def, 2 + nsec = Google::Protobuf::FFI::MessageValue.new + nsec[:int32_val] = ((value.to_f - value.to_i) * 1000000000).round + raise "Should be impossible" unless Google::Protobuf::FFI.set_message_field new_message, nsec_field_def, nsec, arena + return_value[:msg_val] = new_message + else + raise TypeError.new "Invalid type #{value.class} to assign to submessage field '#{name}'." + end + else + arena.fuse(value.instance_variable_get(:@arena)) + return_value[:msg_val] = value.instance_variable_get :@msg + end + when :enum + return_value[:int32_val] = case value + when Numeric + value.to_i + when String, Symbol + enum_number = EnumDescriptor.send(:lookup_name, msg_or_enum_def, value.to_s) + #TODO(jatl) add the bad value to the error message after tests pass + raise RangeError.new "Unknown symbol value for enum field '#{name}'." if enum_number.nil? + enum_number + else + raise TypeError.new "Expected number or symbol type for enum field '#{name}'." + end + #TODO(jatl) After all tests pass, improve error message across integer type by including actual offending value + when :int32 + raise TypeError.new "Expected number type for integral field '#{name}' (given #{value.class})." unless value.is_a? Numeric + raise RangeError.new "Non-integral floating point value assigned to integer field '#{name}' (given #{value.class})." if value.floor != value + raise RangeError.new "Value assigned to int32 field '#{name}' (given #{value.class}) with more than 32-bits." unless value.to_i.bit_length < 32 + return_value[:int32_val] = value.to_i + when :uint32 + raise TypeError.new "Expected number type for integral field '#{name}' (given #{value.class})." unless value.is_a? Numeric + raise RangeError.new "Non-integral floating point value assigned to integer field '#{name}' (given #{value.class})." if value.floor != value + raise RangeError.new "Assigning negative value to unsigned integer field '#{name}' (given #{value.class})." if value < 0 + raise RangeError.new "Value assigned to uint32 field '#{name}' (given #{value.class}) with more than 32-bits." unless value.to_i.bit_length < 33 + return_value[:uint32_val] = value.to_i + when :int64 + raise TypeError.new "Expected number type for integral field '#{name}' (given #{value.class})." unless value.is_a? Numeric + raise RangeError.new "Non-integral floating point value assigned to integer field '#{name}' (given #{value.class})." if value.floor != value + raise RangeError.new "Value assigned to int64 field '#{name}' (given #{value.class}) with more than 64-bits." unless value.to_i.bit_length < 64 + return_value[:int64_val] = value.to_i + when :uint64 + raise TypeError.new "Expected number type for integral field '#{name}' (given #{value.class})." unless value.is_a? Numeric + raise RangeError.new "Non-integral floating point value assigned to integer field '#{name}' (given #{value.class})." if value.floor != value + raise RangeError.new "Assigning negative value to unsigned integer field '#{name}' (given #{value.class})." if value < 0 + raise RangeError.new "Value assigned to uint64 field '#{name}' (given #{value.class}) with more than 64-bits." unless value.to_i.bit_length < 65 + return_value[:uint64_val] = value.to_i + else + raise RuntimeError.new "Unsupported type #{c_type}" + end + return_value + end + + ## + # Safe to call without an arena if the caller has checked that c_type + # is not :message. + # @param message_value [Google::Protobuf::FFI::MessageValue] Value to be converted. + # @param c_type [Google::Protobuf::FFI::CType] Enum representing the type of message_value + # @param msg_or_enum_def [::FFI::Pointer] Pointer to the MsgDef or EnumDef definition + # @param arena [Google::Protobuf::Internal::Arena] Arena to create Message instances, if needed + def convert_upb_to_ruby(message_value, c_type, msg_or_enum_def = nil, arena = nil) + throw TypeError.new "Expected MessageValue but got #{message_value.class}" unless message_value.is_a? Google::Protobuf::FFI::MessageValue + + case c_type + when :bool + message_value[:bool_val] + when :int32 + message_value[:int32_val] + when :uint32 + message_value[:uint32_val] + when :double + message_value[:double_val] + when :int64 + message_value[:int64_val] + when :uint64 + message_value[:uint64_val] + when :string + if message_value[:str_val][:size].zero? + "" + else + message_value[:str_val][:data].read_string_length(message_value[:str_val][:size]).force_encoding("UTF-8").freeze + end + when :bytes + if message_value[:str_val][:size].zero? + "" + else + message_value[:str_val][:data].read_string_length(message_value[:str_val][:size]).force_encoding("ASCII-8BIT").freeze + end + when :float + message_value[:float_val] + when :enum + EnumDescriptor.send(:lookup_value, msg_or_enum_def, message_value[:int32_val]) || message_value[:int32_val] + when :message + raise "Null Arena for message" if arena.nil? + Descriptor.send(:get_message, message_value[:msg_val], msg_or_enum_def, arena) + else + raise RuntimeError.new "Unexpected type #{c_type}" + end + end + + def to_h_internal(msg, message_descriptor) + return nil if msg.nil? or msg.null? + hash = {} + is_proto2 = Google::Protobuf::FFI.message_def_syntax(message_descriptor) == :Proto2 + message_descriptor.each do |field_descriptor| + # TODO: Legacy behavior, remove when we fix the is_proto2 differences. + if !is_proto2 and + field_descriptor.sub_message? and + !field_descriptor.repeated? and + !Google::Protobuf::FFI.get_message_has(msg, field_descriptor) + hash[field_descriptor.name.to_sym] = nil + next + end + + # Do not include fields that are not present (oneof or optional fields). + if is_proto2 and field_descriptor.has_presence? and !Google::Protobuf::FFI.get_message_has(msg, field_descriptor) + next + end + + message_value = Google::Protobuf::FFI.get_message_value msg, field_descriptor + + # Proto2 omits empty map/repeated fields also. + if field_descriptor.map? + hash_entry = map_create_hash(message_value[:map_val], field_descriptor) + elsif field_descriptor.repeated? + array = message_value[:array_val] + if is_proto2 and (array.null? || Google::Protobuf::FFI.array_size(array).zero?) + next + end + hash_entry = repeated_field_create_array(array, field_descriptor, field_descriptor.type) + else + hash_entry = scalar_create_hash(message_value, field_descriptor.type, field_descriptor: field_descriptor) + end + + hash[field_descriptor.name.to_sym] = hash_entry + + end + + hash + end + + def map_create_hash(map_ptr, field_descriptor) + return {} if map_ptr.nil? or map_ptr.null? + return_value = {} + + message_descriptor = field_descriptor.send(:subtype) + key_field_def = Google::Protobuf::FFI.get_field_by_number(message_descriptor, 1) + key_field_type = Google::Protobuf::FFI.get_type(key_field_def) + + value_field_def = Google::Protobuf::FFI.get_field_by_number(message_descriptor, 2) + value_field_type = Google::Protobuf::FFI.get_type(value_field_def) + + iter = ::FFI::MemoryPointer.new(:size_t, 1) + iter.write(:size_t, Google::Protobuf::FFI::Upb_Map_Begin) + while Google::Protobuf::FFI.map_next(map_ptr, iter) do + iter_size_t = iter.read(:size_t) + key_message_value = Google::Protobuf::FFI.map_key(map_ptr, iter_size_t) + value_message_value = Google::Protobuf::FFI.map_value(map_ptr, iter_size_t) + hash_key = convert_upb_to_ruby(key_message_value, key_field_type) + hash_value = scalar_create_hash(value_message_value, value_field_type, msg_or_enum_descriptor: value_field_def.subtype) + return_value[hash_key] = hash_value + end + return_value + end + + def repeated_field_create_array(array, field_descriptor, type) + return_value = [] + n = (array.nil? || array.null?) ? 0 : Google::Protobuf::FFI.array_size(array) + 0.upto(n - 1) do |i| + message_value = Google::Protobuf::FFI.get_msgval_at(array, i) + return_value << scalar_create_hash(message_value, type, field_descriptor: field_descriptor) + end + return_value + end + + # @param field_descriptor [FieldDescriptor] Descriptor of the field to convert to a hash. + def scalar_create_hash(message_value, type, field_descriptor: nil, msg_or_enum_descriptor: nil) + if [:message, :enum].include? type + if field_descriptor.nil? + if msg_or_enum_descriptor.nil? + raise "scalar_create_hash requires either a FieldDescriptor, MessageDescriptor, or EnumDescriptor as an argument, but received only nil" + end + else + msg_or_enum_descriptor = field_descriptor.subtype + end + if type == :message + to_h_internal(message_value[:msg_val], msg_or_enum_descriptor) + elsif type == :enum + convert_upb_to_ruby message_value, type, msg_or_enum_descriptor + end + else + convert_upb_to_ruby message_value, type + end + end + + def message_value_deep_copy(message_value, type, descriptor, arena) + raise unless message_value.is_a? Google::Protobuf::FFI::MessageValue + new_message_value = Google::Protobuf::FFI::MessageValue.new + case type + when :string, :bytes + # TODO(jatl) - how important is it to still use arena malloc, versus using FFI MemoryPointers? + new_message_value[:str_val][:size] = message_value[:str_val][:size] + new_message_value[:str_val][:data] = Google::Protobuf::FFI.arena_malloc(arena, message_value[:str_val][:size]) + raise NoMemoryError.new "Allocation failed" if new_message_value[:str_val][:data].nil? or new_message_value[:str_val][:data].null? + Google::Protobuf::FFI.memcpy(new_message_value[:str_val][:data], message_value[:str_val][:data], message_value[:str_val][:size]) + when :message + new_message_value[:msg_val] = descriptor.msgclass.send(:deep_copy, message_value[:msg_val], arena).instance_variable_get(:@msg) + else + Google::Protobuf::FFI.memcpy(new_message_value.to_ptr, message_value.to_ptr, Google::Protobuf::FFI::MessageValue.size) + end + new_message_value + end + end + end + end +end diff --git a/ruby/lib/google/protobuf/ffi/internal/pointer_helper.rb b/ruby/lib/google/protobuf/ffi/internal/pointer_helper.rb new file mode 100644 index 000000000000..a1cc15964427 --- /dev/null +++ b/ruby/lib/google/protobuf/ffi/internal/pointer_helper.rb @@ -0,0 +1,58 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2023 Google Inc. All rights reserved. +# https://developers.google.com/protocol-buffers/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +module Google + module Protobuf + module Internal + module PointerHelper + # Utility code to defensively walk the object graph from a file_def to + # the pool, and either retrieve the wrapper object for the given pointer + # or create one. Assumes that the caller is the wrapper class for the + # given pointer and that it implements `private_constructor`. + def descriptor_from_file_def(file_def, pointer) + raise RuntimeError.new "FileDef is nil" if file_def.nil? + raise RuntimeError.new "FileDef is null" if file_def.null? + pool_def = Google::Protobuf::FFI.file_def_pool file_def + raise RuntimeError.new "PoolDef is nil" if pool_def.nil? + raise RuntimeError.new "PoolDef is null" if pool_def.null? + pool = Google::Protobuf::OBJECT_CACHE.get(pool_def.address) + raise "Cannot find pool in ObjectCache!" if pool.nil? + descriptor = pool.descriptor_class_by_def[pointer.address] + if descriptor.nil? + pool.descriptor_class_by_def[pointer.address] = private_constructor(pointer, pool) + else + descriptor + end + end + end + end + end +end + diff --git a/ruby/lib/google/protobuf/ffi/internal/type_safety.rb b/ruby/lib/google/protobuf/ffi/internal/type_safety.rb new file mode 100644 index 000000000000..1752c2939836 --- /dev/null +++ b/ruby/lib/google/protobuf/ffi/internal/type_safety.rb @@ -0,0 +1,48 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2022 Google Inc. All rights reserved. +# https://developers.google.com/protocol-buffers/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# A to_native DataConverter method that raises an error if the value is not of the same type. +# Adapted from to https://www.varvet.com/blog/advanced-topics-in-ruby-ffi/ +module Google + module Protobuf + module Internal + module TypeSafety + def to_native(value, ctx = nil) + if value.kind_of?(self) or value.nil? + super + else + raise TypeError.new "Expected a kind of #{name}, was #{value.class}" + end + end + end + end + end +end + diff --git a/ruby/lib/google/protobuf/ffi/map.rb b/ruby/lib/google/protobuf/ffi/map.rb new file mode 100644 index 000000000000..db1be80a0f98 --- /dev/null +++ b/ruby/lib/google/protobuf/ffi/map.rb @@ -0,0 +1,419 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2022 Google Inc. All rights reserved. +# https://developers.google.com/protocol-buffers/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +module Google + module Protobuf + class FFI + # Map + attach_function :map_clear, :upb_Map_Clear, [:Map], :void + attach_function :map_delete, :upb_Map_Delete, [:Map, MessageValue.by_value, MessageValue.by_ref], :bool + attach_function :map_get, :upb_Map_Get, [:Map, MessageValue.by_value, MessageValue.by_ref], :bool + attach_function :create_map, :upb_Map_New, [Internal::Arena, CType, CType], :Map + attach_function :map_size, :upb_Map_Size, [:Map], :size_t + attach_function :map_set, :upb_Map_Set, [:Map, MessageValue.by_value, MessageValue.by_value, Internal::Arena], :bool + + # MapIterator + attach_function :map_next, :upb_MapIterator_Next, [:Map, :pointer], :bool + attach_function :map_done, :upb_MapIterator_Done, [:Map, :size_t], :bool + attach_function :map_key, :upb_MapIterator_Key, [:Map, :size_t], MessageValue.by_value + attach_function :map_value, :upb_MapIterator_Value, [:Map, :size_t], MessageValue.by_value + end + class Map + include Enumerable + ## + # call-seq: + # Map.new(key_type, value_type, value_typeclass = nil, init_hashmap = {}) + # => new map + # + # Allocates a new Map container. This constructor may be called with 2, 3, or 4 + # arguments. The first two arguments are always present and are symbols (taking + # on the same values as field-type symbols in message descriptors) that + # indicate the type of the map key and value fields. + # + # The supported key types are: :int32, :int64, :uint32, :uint64, :bool, + # :string, :bytes. + # + # The supported value types are: :int32, :int64, :uint32, :uint64, :bool, + # :string, :bytes, :enum, :message. + # + # The third argument, value_typeclass, must be present if value_type is :enum + # or :message. As in RepeatedField#new, this argument must be a message class + # (for :message) or enum module (for :enum). + # + # The last argument, if present, provides initial content for map. Note that + # this may be an ordinary Ruby hashmap or another Map instance with identical + # key and value types. Also note that this argument may be present whether or + # not value_typeclass is present (and it is unambiguously separate from + # value_typeclass because value_typeclass's presence is strictly determined by + # value_type). The contents of this initial hashmap or Map instance are + # shallow-copied into the new Map: the original map is unmodified, but + # references to underlying objects will be shared if the value type is a + # message type. + def self.new(key_type, value_type, value_typeclass = nil, init_hashmap = {}) + instance = allocate + # TODO(jatl) This argument mangling doesn't agree with the type signature, + # but does align with the text of the comments and is required to make unit tests pass. + if init_hashmap.empty? and ![:enum, :message].include?(value_type) + init_hashmap = value_typeclass + value_typeclass = nil + end + instance.send(:initialize, key_type, value_type, value_type_class: value_typeclass, initial_values: init_hashmap) + instance + end + + ## + # call-seq: + # Map.keys => [list_of_keys] + # + # Returns the list of keys contained in the map, in unspecified order. + def keys + return_value = [] + internal_iterator do |iterator| + key_message_value = Google::Protobuf::FFI.map_key(@map_ptr, iterator) + return_value << convert_upb_to_ruby(key_message_value, key_type) + end + return_value + end + + ## + # call-seq: + # Map.values => [list_of_values] + # + # Returns the list of values contained in the map, in unspecified order. + def values + return_value = [] + internal_iterator do |iterator| + value_message_value = Google::Protobuf::FFI.map_value(@map_ptr, iterator) + return_value << convert_upb_to_ruby(value_message_value, value_type, descriptor, arena) + end + return_value + end + + ## + # call-seq: + # Map.[](key) => value + # + # Accesses the element at the given key. Throws an exception if the key type is + # incorrect. Returns nil when the key is not present in the map. + def [](key) + value = Google::Protobuf::FFI::MessageValue.new + key_message_value = convert_ruby_to_upb(key, arena, key_type, nil) + if Google::Protobuf::FFI.map_get(@map_ptr, key_message_value, value) + convert_upb_to_ruby(value, value_type, descriptor, arena) + end + end + + ## + # call-seq: + # Map.[]=(key, value) => value + # + # Inserts or overwrites the value at the given key with the given new value. + # Throws an exception if the key type is incorrect. Returns the new value that + # was just inserted. + def []=(key, value) + raise FrozenError.new "can't modify frozen #{self.class}" if frozen? + key_message_value = convert_ruby_to_upb(key, arena, key_type, nil) + value_message_value = convert_ruby_to_upb(value, arena, value_type, descriptor) + Google::Protobuf::FFI.map_set(@map_ptr, key_message_value, value_message_value, arena) + value + end + + def has_key?(key) + key_message_value = convert_ruby_to_upb(key, arena, key_type, nil) + Google::Protobuf::FFI.map_get(@map_ptr, key_message_value, nil) + end + + ## + # call-seq: + # Map.delete(key) => old_value + # + # Deletes the value at the given key, if any, returning either the old value or + # nil if none was present. Throws an exception if the key is of the wrong type. + def delete(key) + raise FrozenError.new "can't modify frozen #{self.class}" if frozen? + value = Google::Protobuf::FFI::MessageValue.new + key_message_value = convert_ruby_to_upb(key, arena, key_type, nil) + if Google::Protobuf::FFI.map_delete(@map_ptr, key_message_value, value) + convert_upb_to_ruby(value, value_type, descriptor, arena) + else + nil + end + end + + def clear + raise FrozenError.new "can't modify frozen #{self.class}" if frozen? + Google::Protobuf::FFI.map_clear(@map_ptr) + nil + end + + def length + Google::Protobuf::FFI.map_size(@map_ptr) + end + alias size length + + ## + # call-seq: + # Map.dup => new_map + # + # Duplicates this map with a shallow copy. References to all non-primitive + # element objects (e.g., submessages) are shared. + def dup + internal_dup + end + alias clone dup + + ## + # call-seq: + # Map.==(other) => boolean + # + # Compares this map to another. Maps are equal if they have identical key sets, + # and for each key, the values in both maps compare equal. Elements are + # compared as per normal Ruby semantics, by calling their :== methods (or + # performing a more efficient comparison for primitive types). + # + # Maps with dissimilar key types or value types/typeclasses are never equal, + # even if value comparison (for example, between integers and floats) would + # have otherwise indicated that every element has equal value. + def ==(other) + if other.is_a? Hash + other = self.class.send(:private_constructor, key_type, value_type, descriptor, initial_values: other) + elsif !other.is_a? Google::Protobuf::Map + return false + end + + return true if object_id == other.object_id + return false if key_type != other.send(:key_type) or value_type != other.send(:value_type) or descriptor != other.send(:descriptor) or length != other.length + other_map_ptr = other.send(:map_ptr) + each_msg_val do |key_message_value, value_message_value| + other_value = Google::Protobuf::FFI::MessageValue.new + return false unless Google::Protobuf::FFI.map_get(other_map_ptr, key_message_value, other_value) + return false unless Google::Protobuf::FFI.message_value_equal(value_message_value, other_value, value_type, descriptor) + end + true + end + + def hash + return_value = 0 + each_msg_val do |key_message_value, value_message_value| + return_value = Google::Protobuf::FFI.message_value_hash(key_message_value, key_type, nil, return_value) + return_value = Google::Protobuf::FFI.message_value_hash(value_message_value, value_type, descriptor, return_value) + end + return_value + end + + ## + # call-seq: + # Map.to_h => {} + # + # Returns a Ruby Hash object containing all the values within the map + def to_h + return {} if map_ptr.nil? or map_ptr.null? + return_value = {} + each_msg_val do |key_message_value, value_message_value| + hash_key = convert_upb_to_ruby(key_message_value, key_type) + hash_value = scalar_create_hash(value_message_value, value_type, msg_or_enum_descriptor: descriptor) + return_value[hash_key] = hash_value + end + return_value + end + + def inspect + key_value_pairs = [] + each_msg_val do |key_message_value, value_message_value| + key_string = convert_upb_to_ruby(key_message_value, key_type).inspect + if value_type == :message + sub_msg_descriptor = Google::Protobuf::FFI.get_subtype_as_message(descriptor) + value_string = sub_msg_descriptor.msgclass.send(:inspect_internal, value_message_value[:msg_val]) + else + value_string = convert_upb_to_ruby(value_message_value, value_type, descriptor).inspect + end + key_value_pairs << "#{key_string}=>#{value_string}" + end + "{#{key_value_pairs.join(", ")}}" + end + + ## + # call-seq: + # Map.merge(other_map) => map + # + # Copies key/value pairs from other_map into a copy of this map. If a key is + # set in other_map and this map, the value from other_map overwrites the value + # in the new copy of this map. Returns the new copy of this map with merged + # contents. + def merge(other) + internal_merge(other) + end + + ## + # call-seq: + # Map.each(&block) + # + # Invokes &block on each |key, value| pair in the map, in unspecified order. + # Note that Map also includes Enumerable; map thus acts like a normal Ruby + # sequence. + def each &block + each_msg_val do |key_message_value, value_message_value| + key_value = convert_upb_to_ruby(key_message_value, key_type) + value_value = convert_upb_to_ruby(value_message_value, value_type, descriptor, arena) + yield key_value, value_value + end + nil + end + + private + attr :arena, :map_ptr, :key_type, :value_type, :descriptor, :name + + include Google::Protobuf::Internal::Convert + + def internal_iterator + iter = ::FFI::MemoryPointer.new(:size_t, 1) + iter.write(:size_t, Google::Protobuf::FFI::Upb_Map_Begin) + while Google::Protobuf::FFI.map_next(@map_ptr, iter) do + iter_size_t = iter.read(:size_t) + yield iter_size_t + end + end + + def each_msg_val &block + internal_iterator do |iterator| + key_message_value = Google::Protobuf::FFI.map_key(@map_ptr, iterator) + value_message_value = Google::Protobuf::FFI.map_value(@map_ptr, iterator) + yield key_message_value, value_message_value + end + end + + def internal_dup + instance = self.class.send(:private_constructor, key_type, value_type, descriptor, arena: arena) + new_map_ptr = instance.send(:map_ptr) + each_msg_val do |key_message_value, value_message_value| + Google::Protobuf::FFI.map_set(new_map_ptr, key_message_value, value_message_value, arena) + end + instance + end + + def internal_merge_into_self(other) + case other + when Hash + other.each do |key, value| + key_message_value = convert_ruby_to_upb(key, arena, key_type, nil) + value_message_value = convert_ruby_to_upb(value, arena, value_type, descriptor) + Google::Protobuf::FFI.map_set(@map_ptr, key_message_value, value_message_value, arena) + end + when Google::Protobuf::Map + unless key_type == other.send(:key_type) and value_type == other.send(:value_type) and descriptor == other.descriptor + raise ArgumentError.new "Attempt to merge Map with mismatching types" #TODO(jatl) Improve error message by adding type information + end + arena.fuse(other.send(:arena)) + iter = ::FFI::MemoryPointer.new(:size_t, 1) + iter.write(:size_t, Google::Protobuf::FFI::Upb_Map_Begin) + other.send(:each_msg_val) do |key_message_value, value_message_value| + Google::Protobuf::FFI.map_set(@map_ptr, key_message_value, value_message_value, arena) + end + else + raise ArgumentError.new "Unknown type merging into Map" #TODO(jatl) improve this error message by including type information + end + self + end + + def internal_merge(other) + internal_dup.internal_merge_into_self(other) + end + + def initialize(key_type, value_type, value_type_class: nil, initial_values: nil, arena: nil, map: nil, descriptor: nil, name: nil) + @name = name || 'Map' + + unless [:int32, :int64, :uint32, :uint64, :bool, :string, :bytes].include? key_type + raise ArgumentError.new "Invalid key type for map." #TODO(jatl) improve error message to include what type was passed + end + @key_type = key_type + + unless [:int32, :int64, :uint32, :uint64, :bool, :string, :bytes, :enum, :message].include? value_type + raise ArgumentError.new "Invalid value type for map." #TODO(jatl) improve error message to include what type was passed + end + @value_type = value_type + + if !descriptor.nil? + raise ArgumentError "Expected descriptor to be a Descriptor or EnumDescriptor" unless [EnumDescriptor, Descriptor].include? descriptor.class + @descriptor = descriptor + elsif [:message, :enum].include? value_type + raise ArgumentError.new "Expected at least 3 arguments for message/enum." if value_type_class.nil? + descriptor = value_type_class.respond_to?(:descriptor) ? value_type_class.descriptor : nil + raise ArgumentError.new "Type class #{value_type_class} has no descriptor. Please pass a class or enum as returned by the DescriptorPool." if descriptor.nil? + @descriptor = descriptor + else + @descriptor = nil + end + + @arena = arena || Google::Protobuf::FFI.create_arena + @map_ptr = map || Google::Protobuf::FFI.create_map(@arena, @key_type, @value_type) + + internal_merge_into_self(initial_values) unless initial_values.nil? + + # Should always be the last expression of the initializer to avoid + # leaking references to this object before construction is complete. + OBJECT_CACHE.try_add(@map_ptr.address, self) + end + + # @param field [FieldDescriptor] Descriptor of the field where the RepeatedField will be assigned + # @param values [Hash|Map] Initial value; may be nil or empty + # @param arena [Arena] Owning message's arena + def self.construct_for_field(field, arena, value: nil, map: nil) + raise ArgumentError.new "Expected Hash object as initializer value for map field '#{field.name}' (given #{value.class})." unless value.nil? or value.is_a? Hash + instance = allocate + raise ArgumentError.new "Expected field with type :message, instead got #{field.class}" unless field.type == :message + message_descriptor = field.send(:subtype) + key_field_def = Google::Protobuf::FFI.get_field_by_number(message_descriptor, 1) + key_field_type = Google::Protobuf::FFI.get_type(key_field_def) + + value_field_def = Google::Protobuf::FFI.get_field_by_number(message_descriptor, 2) + value_field_type = Google::Protobuf::FFI.get_type(value_field_def) + instance.send(:initialize, key_field_type, value_field_type, initial_values: value, name: field.name, arena: arena, map: map, descriptor: value_field_def.subtype) + instance + end + + def self.private_constructor(key_type, value_type, descriptor, initial_values: nil, arena: nil) + instance = allocate + instance.send(:initialize, key_type, value_type, descriptor: descriptor, initial_values: initial_values, arena: arena) + instance + end + + extend Google::Protobuf::Internal::Convert + + def self.deep_copy(map) + instance = allocate + instance.send(:initialize, map.send(:key_type), map.send(:value_type), descriptor: map.send(:descriptor)) + map.send(:each_msg_val) do |key_message_value, value_message_value| + Google::Protobuf::FFI.map_set(instance.send(:map_ptr), key_message_value, message_value_deep_copy(value_message_value, map.send(:value_type), map.send(:descriptor), instance.send(:arena)), instance.send(:arena)) + end + instance + end + end + end +end diff --git a/ruby/lib/google/protobuf/ffi/message.rb b/ruby/lib/google/protobuf/ffi/message.rb new file mode 100644 index 000000000000..9ec5bc114e0b --- /dev/null +++ b/ruby/lib/google/protobuf/ffi/message.rb @@ -0,0 +1,658 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2023 Google Inc. All rights reserved. +# https://developers.google.com/protocol-buffers/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +# Decorates Descriptor with the `build_message_class` method that defines +# Message classes. +module Google + module Protobuf + class FFI + # Message + attach_function :clear_message_field, :upb_Message_ClearFieldByDef, [:Message, FieldDescriptor], :void + attach_function :get_message_value, :upb_Message_GetFieldByDef, [:Message, FieldDescriptor], MessageValue.by_value + attach_function :get_message_has, :upb_Message_HasFieldByDef, [:Message, FieldDescriptor], :bool + attach_function :set_message_field, :upb_Message_SetFieldByDef, [:Message, FieldDescriptor, MessageValue.by_value, Internal::Arena], :bool + attach_function :encode_message, :upb_Encode, [:Message, MiniTable.by_ref, :size_t, Internal::Arena, :pointer, :pointer], EncodeStatus + attach_function :json_decode_message, :upb_JsonDecode, [:binary_string, :size_t, :Message, Descriptor, :DefPool, :int, Internal::Arena, Status.by_ref], :bool + attach_function :json_encode_message, :upb_JsonEncode, [:Message, Descriptor, :DefPool, :int, :binary_string, :size_t, Status.by_ref], :size_t + attach_function :decode_message, :upb_Decode, [:binary_string, :size_t, :Message, MiniTable.by_ref, :ExtensionRegistry, :int, Internal::Arena], DecodeStatus + attach_function :get_mutable_message, :upb_Message_Mutable, [:Message, FieldDescriptor, Internal::Arena], MutableMessageValue.by_value + attach_function :get_message_which_oneof, :upb_Message_WhichOneof, [:Message, OneofDescriptor], FieldDescriptor + attach_function :message_discard_unknown, :upb_Message_DiscardUnknown, [:Message, Descriptor, :int], :bool + # MessageValue + attach_function :message_value_equal, :shared_Msgval_IsEqual, [MessageValue.by_value, MessageValue.by_value, CType, Descriptor], :bool + attach_function :message_value_hash, :shared_Msgval_GetHash, [MessageValue.by_value, CType, Descriptor, :uint64_t], :uint64_t + end + + class Descriptor + def build_message_class + descriptor = self + Class.new(Google::Protobuf::const_get(:AbstractMessage)) do + @descriptor = descriptor + class << self + attr_accessor :descriptor + private + attr_accessor :oneof_field_names + include ::Google::Protobuf::Internal::Convert + end + + alias original_method_missing method_missing + def method_missing(method_name, *args) + method_missing_internal method_name, *args, mode: :method_missing + end + + def respond_to_missing?(method_name, include_private = false) + method_missing_internal(method_name, mode: :respond_to_missing?) || super + end + + ## + # Public constructor. Automatically allocates from a new Arena. + def self.new(initial_value = nil) + instance = allocate + instance.send(:initialize, initial_value) + instance + end + + def dup + duplicate = self.class.private_constructor(@arena) + mini_table = Google::Protobuf::FFI.get_mini_table(self.class.descriptor) + size = mini_table[:size] + duplicate.instance_variable_get(:@msg).write_string_length(@msg.read_string_length(size), size) + duplicate + end + alias clone dup + + def eql?(other) + return false unless self.class === other + encoding_options = Google::Protobuf::FFI::Upb_Encode_Deterministic | Google::Protobuf::FFI::Upb_Encode_SkipUnknown + temporary_arena = Google::Protobuf::FFI.create_arena + mini_table = Google::Protobuf::FFI.get_mini_table(self.class.descriptor) + size_one = ::FFI::MemoryPointer.new(:size_t, 1) + encoding_one = ::FFI::MemoryPointer.new(:pointer, 1) + encoding_status = Google::Protobuf::FFI.encode_message(@msg, mini_table, encoding_options, temporary_arena, encoding_one.to_ptr, size_one) + raise ParseError.new "Error comparing messages due to #{encoding_status} while encoding LHS of `eql?()`" unless encoding_status == :Ok + + size_two = ::FFI::MemoryPointer.new(:size_t, 1) + encoding_two = ::FFI::MemoryPointer.new(:pointer, 1) + encoding_status = Google::Protobuf::FFI.encode_message(other.instance_variable_get(:@msg), mini_table, encoding_options, temporary_arena, encoding_two.to_ptr, size_two) + raise ParseError.new "Error comparing messages due to #{encoding_status} while encoding RHS of `eql?()`" unless encoding_status == :Ok + + if encoding_one.null? or encoding_two.null? + raise ParseError.new "Error comparing messages" + end + size_one.read(:size_t) == size_two.read(:size_t) and Google::Protobuf::FFI.memcmp(encoding_one.read(:pointer), encoding_two.read(:pointer), size_one.read(:size_t)).zero? + end + alias == eql? + + def hash + encoding_options = Google::Protobuf::FFI::Upb_Encode_Deterministic | Google::Protobuf::FFI::Upb_Encode_SkipUnknown + temporary_arena = Google::Protobuf::FFI.create_arena + mini_table_ptr = Google::Protobuf::FFI.get_mini_table(self.class.descriptor) + size_ptr = ::FFI::MemoryPointer.new(:size_t, 1) + encoding = ::FFI::MemoryPointer.new(:pointer, 1) + encoding_status = Google::Protobuf::FFI.encode_message(@msg, mini_table_ptr, encoding_options, temporary_arena, encoding.to_ptr, size_ptr) + if encoding_status != :Ok or encoding.null? + raise ParseError.new "Error calculating hash" + end + encoding.read(:pointer).read_string(size_ptr.read(:size_t)).hash + end + + def to_h + to_h_internal @msg, self.class.descriptor + end + + ## + # call-seq: + # Message.inspect => string + # + # Returns a human-readable string representing this message. It will be + # formatted as "". Each + # field's value is represented according to its own #inspect method. + def inspect + self.class.inspect_internal @msg + end + + def to_s + self.inspect + end + + ## + # call-seq: + # Message.[](index) => value + # Accesses a field's value by field name. The provided field name + # should be a string. + def [](name) + raise TypeError.new "Expected String for name but got #{name.class}" unless name.is_a? String + index_internal name + end + + ## + # call-seq: + # Message.[]=(index, value) + # Sets a field's value by field name. The provided field name should + # be a string. + # @param name [String] Name of the field to be set + # @param value [Object] Value to set the field to + def []=(name, value) + raise TypeError.new "Expected String for name but got #{name.class}" unless name.is_a? String + index_assign_internal(value, name: name) + end + + ## + # call-seq: + # MessageClass.decode(data, options) => message + # + # Decodes the given data (as a string containing bytes in protocol buffers wire + # format) under the interpretation given by this message class's definition + # and returns a message object with the corresponding field values. + # @param data [String] Binary string in Protobuf wire format to decode + # @param options [Hash] options for the decoder + # @option options [Integer] :recursion_limit Set to maximum decoding depth for message (default is 64) + def self.decode(data, options = {}) + raise ArgumentError.new "Expected hash arguments." unless options.is_a? Hash + raise ArgumentError.new "Expected string for binary protobuf data." unless data.is_a? String + decoding_options = 0 + depth = options[:recursion_limit] + + if depth.is_a? Numeric + decoding_options |= Google::Protobuf::FFI.decode_max_depth(depth.to_i) + end + + message = new + mini_table_ptr = Google::Protobuf::FFI.get_mini_table(message.class.descriptor) + status = Google::Protobuf::FFI.decode_message(data, data.bytesize, message.instance_variable_get(:@msg), mini_table_ptr, nil, decoding_options, message.instance_variable_get(:@arena)) + raise ParseError.new "Error occurred during parsing" unless status == :Ok + message + end + + ## + # call-seq: + # MessageClass.encode(msg, options) => bytes + # + # Encodes the given message object to its serialized form in protocol buffers + # wire format. + # @param options [Hash] options for the encoder + # @option options [Integer] :recursion_limit Set to maximum encoding depth for message (default is 64) + def self.encode(message, options = {}) + raise ArgumentError.new "Message of wrong type." unless message.is_a? self + raise ArgumentError.new "Expected hash arguments." unless options.is_a? Hash + + encoding_options = 0 + depth = options[:recursion_limit] + + if depth.is_a? Numeric + encoding_options |= Google::Protobuf::FFI.decode_max_depth(depth.to_i) + end + + encode_internal(message.instance_variable_get(:@msg), encoding_options) do |encoding, size, _| + if encoding.nil? or encoding.null? + raise RuntimeError.new "Exceeded maximum depth (possibly cycle)" + else + encoding.read_string_length(size).force_encoding("ASCII-8BIT").freeze + end + end + end + + ## + # all-seq: + # MessageClass.decode_json(data, options = {}) => message + # + # Decodes the given data (as a string containing bytes in protocol buffers wire + # format) under the interpretation given by this message class's definition + # and returns a message object with the corresponding field values. + # + # @param options [Hash] options for the decoder + # @option options [Boolean] :ignore_unknown_fields Set true to ignore unknown fields (default is to raise an error) + # @return [Message] + def self.decode_json(data, options = {}) + decoding_options = 0 + unless options.is_a? Hash + if options.respond_to? :to_h + options options.to_h + else + #TODO(jatl) can this error message be improve to include what was received? + raise ArgumentError.new "Expected hash arguments" + end + end + raise ArgumentError.new "Expected string for JSON data." unless data.is_a? String + raise RuntimeError.new "Cannot parse a wrapper directly" if descriptor.send(:wrapper?) + + if options[:ignore_unknown_fields] + decoding_options |= Google::Protobuf::FFI::Upb_JsonDecode_IgnoreUnknown + end + + message = new + pool_def = message.class.descriptor.instance_variable_get(:@descriptor_pool).descriptor_pool + status = Google::Protobuf::FFI::Status.new + unless Google::Protobuf::FFI.json_decode_message(data, data.bytesize, message.instance_variable_get(:@msg), message.class.descriptor, pool_def, decoding_options, message.instance_variable_get(:@arena), status) + raise ParseError.new "Error occurred during parsing: #{Google::Protobuf::FFI.error_message(status)}" + end + message + end + + def self.encode_json(message, options = {}) + encoding_options = 0 + unless options.is_a? Hash + if options.respond_to? :to_h + options = options.to_h + else + #TODO(jatl) can this error message be improve to include what was received? + raise ArgumentError.new "Expected hash arguments" + end + end + + if options[:preserve_proto_fieldnames] + encoding_options |= Google::Protobuf::FFI::Upb_JsonEncode_UseProtoNames + end + if options[:emit_defaults] + encoding_options |= Google::Protobuf::FFI::Upb_JsonEncode_EmitDefaults + end + if options[:format_enums_as_integers] + encoding_options |= Google::Protobuf::FFI::Upb_JsonEncode_FormatEnumsAsIntegers + end + + buffer_size = 1024 + buffer = ::FFI::MemoryPointer.new(:char, buffer_size) + status = Google::Protobuf::FFI::Status.new + msg = message.instance_variable_get(:@msg) + pool_def = message.class.descriptor.instance_variable_get(:@descriptor_pool).descriptor_pool + size = Google::Protobuf::FFI::json_encode_message(msg, message.class.descriptor, pool_def, encoding_options, buffer, buffer_size, status) + unless status[:ok] + raise ParseError.new "Error occurred during encoding: #{Google::Protobuf::FFI.error_message(status)}" + end + + if size >= buffer_size + buffer_size = size + 1 + buffer = ::FFI::MemoryPointer.new(:char, buffer_size) + status.clear + size = Google::Protobuf::FFI::json_encode_message(msg, message.class.descriptor, pool_def, encoding_options, buffer, buffer_size, status) + unless status[:ok] + raise ParseError.new "Error occurred during encoding: #{Google::Protobuf::FFI.error_message(status)}" + end + if size >= buffer_size + raise ParseError.new "Inconsistent JSON encoding sizes - was #{buffer_size - 1}, now #{size}" + end + end + + buffer.read_string_length(size).force_encoding("UTF-8").freeze + end + + private + # Implementation details below are subject to breaking changes without + # warning and are intended for use only within the gem. + + include Google::Protobuf::Internal::Convert + + def self.setup_accessors! + @descriptor.each do |field_descriptor| + field_name = field_descriptor.name + unless instance_methods(true).include?(field_name.to_sym) + #TODO(jatl) - at a high level, dispatching to either + # index_internal or get_field would be logically correct, but slightly slower. + if field_descriptor.map? + define_method(field_name) do + mutable_message_value = Google::Protobuf::FFI.get_mutable_message @msg, field_descriptor, @arena + get_map_field(mutable_message_value[:map], field_descriptor) + end + elsif field_descriptor.repeated? + define_method(field_name) do + mutable_message_value = Google::Protobuf::FFI.get_mutable_message @msg, field_descriptor, @arena + get_repeated_field(mutable_message_value[:array], field_descriptor) + end + elsif field_descriptor.sub_message? + define_method(field_name) do + return nil unless Google::Protobuf::FFI.get_message_has @msg, field_descriptor + mutable_message = Google::Protobuf::FFI.get_mutable_message @msg, field_descriptor, @arena + sub_message = mutable_message[:msg] + sub_message_def = Google::Protobuf::FFI.get_subtype_as_message(field_descriptor) + Descriptor.send(:get_message, sub_message, sub_message_def, @arena) + end + else + c_type = field_descriptor.send(:c_type) + if c_type == :enum + define_method(field_name) do + message_value = Google::Protobuf::FFI.get_message_value @msg, field_descriptor + convert_upb_to_ruby message_value, c_type, Google::Protobuf::FFI.get_subtype_as_enum(field_descriptor) + end + else + define_method(field_name) do + message_value = Google::Protobuf::FFI.get_message_value @msg, field_descriptor + convert_upb_to_ruby message_value, c_type + end + end + end + define_method("#{field_name}=") do |value| + index_assign_internal(value, field_descriptor: field_descriptor) + end + define_method("clear_#{field_name}") do + clear_internal(field_descriptor) + end + if field_descriptor.type == :enum + define_method("#{field_name}_const") do + if field_descriptor.repeated? + return_value = [] + get_field(field_descriptor).send(:each_msg_val) do |msg_val| + return_value << msg_val[:int32_val] + end + return_value + else + message_value = Google::Protobuf::FFI.get_message_value @msg, field_descriptor + message_value[:int32_val] + end + end + end + if !field_descriptor.repeated? and field_descriptor.wrapper? + define_method("#{field_name}_as_value") do + get_field(field_descriptor, unwrap: true) + end + define_method("#{field_name}_as_value=") do |value| + if value.nil? + clear_internal(field_descriptor) + else + index_assign_internal(value, field_descriptor: field_descriptor, wrap: true) + end + end + end + if field_descriptor.has_presence? + define_method("has_#{field_name}?") do + Google::Protobuf::FFI.get_message_has(@msg, field_descriptor) + end + end + end + end + end + + def self.setup_oneof_accessors! + @oneof_field_names = [] + @descriptor.each_oneof do |oneof_descriptor| + self.add_oneof_accessors_for! oneof_descriptor + end + end + def self.add_oneof_accessors_for!(oneof_descriptor) + field_name = oneof_descriptor.name.to_sym + @oneof_field_names << field_name + unless instance_methods(true).include?(field_name) + define_method(field_name) do + field_descriptor = Google::Protobuf::FFI.get_message_which_oneof(@msg, oneof_descriptor) + if field_descriptor.nil? + return + else + return field_descriptor.name.to_sym + end + end + define_method("clear_#{field_name}") do + field_descriptor = Google::Protobuf::FFI.get_message_which_oneof(@msg, oneof_descriptor) + unless field_descriptor.nil? + clear_internal(field_descriptor) + end + end + define_method("has_#{field_name}?") do + !Google::Protobuf::FFI.get_message_which_oneof(@msg, oneof_descriptor).nil? + end + end + end + + setup_accessors! + setup_oneof_accessors! + + def self.private_constructor(arena, msg: nil, initial_value: nil) + instance = allocate + instance.send(:initialize, initial_value, arena, msg) + instance + end + + def self.inspect_field(field_descriptor, c_type, message_value) + if field_descriptor.sub_message? + sub_msg_descriptor = Google::Protobuf::FFI.get_subtype_as_message(field_descriptor) + sub_msg_descriptor.msgclass.send(:inspect_internal, message_value[:msg_val]) + else + convert_upb_to_ruby(message_value, c_type, field_descriptor.subtype).inspect + end + end + + # @param msg [::FFI::Pointer] Pointer to the Message + def self.inspect_internal(msg) + field_output = [] + descriptor.each do |field_descriptor| + next if field_descriptor.has_presence? && !Google::Protobuf::FFI.get_message_has(msg, field_descriptor) + if field_descriptor.map? + # TODO(jatl) Adapted - from map#each_msg_val and map#inspect- can this be refactored to reduce echo without introducing a arena allocation? + message_descriptor = field_descriptor.subtype + key_field_def = Google::Protobuf::FFI.get_field_by_number(message_descriptor, 1) + key_field_type = Google::Protobuf::FFI.get_type(key_field_def) + + value_field_def = Google::Protobuf::FFI.get_field_by_number(message_descriptor, 2) + value_field_type = Google::Protobuf::FFI.get_type(value_field_def) + + message_value = Google::Protobuf::FFI.get_message_value(msg, field_descriptor) + iter = ::FFI::MemoryPointer.new(:size_t, 1) + iter.write(:size_t, Google::Protobuf::FFI::Upb_Map_Begin) + key_value_pairs = [] + while Google::Protobuf::FFI.map_next(message_value[:map_val], iter) do + iter_size_t = iter.read(:size_t) + key_message_value = Google::Protobuf::FFI.map_key(message_value[:map_val], iter_size_t) + value_message_value = Google::Protobuf::FFI.map_value(message_value[:map_val], iter_size_t) + key_string = convert_upb_to_ruby(key_message_value, key_field_type).inspect + value_string = inspect_field(value_field_def, value_field_type, value_message_value) + key_value_pairs << "#{key_string}=>#{value_string}" + end + field_output << "#{field_descriptor.name}: {#{key_value_pairs.join(", ")}}" + elsif field_descriptor.repeated? + # TODO(jatl) Adapted - from repeated_field#each - can this be refactored to reduce echo? + repeated_field_output = [] + message_value = Google::Protobuf::FFI.get_message_value(msg, field_descriptor) + array = message_value[:array_val] + n = array.null? ? 0 : Google::Protobuf::FFI.array_size(array) + 0.upto(n - 1) do |i| + element = Google::Protobuf::FFI.get_msgval_at(array, i) + repeated_field_output << inspect_field(field_descriptor, field_descriptor.send(:c_type), element) + end + field_output << "#{field_descriptor.name}: [#{repeated_field_output.join(", ")}]" + else + message_value = Google::Protobuf::FFI.get_message_value msg, field_descriptor + rendered_value = inspect_field(field_descriptor, field_descriptor.send(:c_type), message_value) + field_output << "#{field_descriptor.name}: #{rendered_value}" + end + end + "<#{name}: #{field_output.join(', ')}>" + end + + def self.deep_copy(msg, arena = nil) + arena ||= Google::Protobuf::FFI.create_arena + encode_internal(msg) do |encoding, size, mini_table_ptr| + message = private_constructor(arena) + if encoding.nil? or encoding.null? or Google::Protobuf::FFI.decode_message(encoding, size, message.instance_variable_get(:@msg), mini_table_ptr, nil, 0, arena) != :Ok + raise ParseError.new "Error occurred copying proto" + end + message + end + end + + def self.encode_internal(msg, encoding_options = 0) + temporary_arena = Google::Protobuf::FFI.create_arena + + mini_table_ptr = Google::Protobuf::FFI.get_mini_table(descriptor) + size_ptr = ::FFI::MemoryPointer.new(:size_t, 1) + pointer_ptr = ::FFI::MemoryPointer.new(:pointer, 1) + encoding_status = Google::Protobuf::FFI.encode_message(msg, mini_table_ptr, encoding_options, temporary_arena, pointer_ptr.to_ptr, size_ptr) + raise "Encoding failed due to #{encoding_status}" unless encoding_status == :Ok + yield pointer_ptr.read(:pointer), size_ptr.read(:size_t), mini_table_ptr + end + + def method_missing_internal(method_name, *args, mode: nil) + raise ArgumentError.new "method_missing_internal called with invalid mode #{mode.inspect}" unless [:respond_to_missing?, :method_missing].include? mode + + #TODO(jatl) not being allowed is not the same thing as not responding, but this is needed to pass tests + if method_name.to_s.end_with? '=' + if self.class.send(:oneof_field_names).include? method_name.to_s[0..-2].to_sym + return false if mode == :respond_to_missing? + raise RuntimeError.new "Oneof accessors are read-only." + end + end + + original_method_missing(method_name, *args) if mode == :method_missing + end + + def clear_internal(field_def) + raise FrozenError.new "can't modify frozen #{self.class}" if frozen? + Google::Protobuf::FFI.clear_message_field(@msg, field_def) + end + + def index_internal(name) + field_descriptor = self.class.descriptor.lookup(name) + get_field field_descriptor unless field_descriptor.nil? + end + + #TODO(jatl) - well known types keeps us on our toes by overloading methods. + # How much of the public API needs to be defended? + def index_assign_internal(value, name: nil, field_descriptor: nil, wrap: false) + raise FrozenError.new "can't modify frozen #{self.class}" if frozen? + if field_descriptor.nil? + field_descriptor = self.class.descriptor.lookup(name) + if field_descriptor.nil? + raise ArgumentError.new "Unknown field: #{name}" + end + end + unless field_descriptor.send :set_value_on_message, value, @msg, @arena, wrap: wrap + raise RuntimeError.new "allocation failed" + end + end + + ## + # @param initial_value [Object] initial value of this Message + # @param arena [Arena] Optional; Arena where this message will be allocated + # @param msg [::FFI::Pointer] Optional; Message to initialize; creates + # one if omitted or nil. + def initialize(initial_value = nil, arena = nil, msg = nil) + @arena = arena || Google::Protobuf::FFI.create_arena + @msg = msg || Google::Protobuf::FFI.new_message_from_def(self.class.descriptor, @arena) + + unless initial_value.nil? + raise ArgumentError.new "Expected hash arguments or message, not #{initial_value.class}" unless initial_value.respond_to? :each + + field_def_ptr = ::FFI::MemoryPointer.new :pointer + oneof_def_ptr = ::FFI::MemoryPointer.new :pointer + + initial_value.each do |key, value| + raise ArgumentError.new "Expected string or symbols as hash keys when initializing proto from hash." unless [String, Symbol].include? key.class + + unless Google::Protobuf::FFI.find_msg_def_by_name self.class.descriptor, key.to_s, key.to_s.bytesize, field_def_ptr, oneof_def_ptr + raise ArgumentError.new "Unknown field name '#{key}' in initialization map entry." + end + raise NotImplementedError.new "Haven't added oneofsupport yet" unless oneof_def_ptr.get_pointer(0).null? + raise NotImplementedError.new "Expected a field def" if field_def_ptr.get_pointer(0).null? + + field_descriptor = FieldDescriptor.from_native field_def_ptr.get_pointer(0) + + next if value.nil? + if field_descriptor.map? + index_assign_internal(Google::Protobuf::Map.send(:construct_for_field, field_descriptor, @arena, value: value), name: key.to_s) + elsif field_descriptor.repeated? + index_assign_internal(RepeatedField.send(:construct_for_field, field_descriptor, @arena, values: value), name: key.to_s) + else + index_assign_internal(value, name: key.to_s) + end + end + end + + # Should always be the last expression of the initializer to avoid + # leaking references to this object before construction is complete. + Google::Protobuf::OBJECT_CACHE.try_add @msg.address, self + end + + ## + # Gets a field of this message identified by the argument definition. + # + # @param field [FieldDescriptor] Descriptor of the field to get + def get_field(field, unwrap: false) + if field.map? + mutable_message_value = Google::Protobuf::FFI.get_mutable_message @msg, field, @arena + get_map_field(mutable_message_value[:map], field) + elsif field.repeated? + mutable_message_value = Google::Protobuf::FFI.get_mutable_message @msg, field, @arena + get_repeated_field(mutable_message_value[:array], field) + elsif field.sub_message? + return nil unless Google::Protobuf::FFI.get_message_has @msg, field + sub_message_def = Google::Protobuf::FFI.get_subtype_as_message(field) + if unwrap + if field.has?(self) + wrapper_message_value = Google::Protobuf::FFI.get_message_value @msg, field + fields = Google::Protobuf::FFI.field_count(sub_message_def) + raise "Sub message has #{fields} fields! Expected exactly 1." unless fields == 1 + value_field_def = Google::Protobuf::FFI.get_field_by_number sub_message_def, 1 + message_value = Google::Protobuf::FFI.get_message_value wrapper_message_value[:msg_val], value_field_def + convert_upb_to_ruby message_value, Google::Protobuf::FFI.get_c_type(value_field_def) + else + nil + end + else + mutable_message = Google::Protobuf::FFI.get_mutable_message @msg, field, @arena + sub_message = mutable_message[:msg] + Descriptor.send(:get_message, sub_message, sub_message_def, @arena) + end + else + c_type = field.send(:c_type) + message_value = Google::Protobuf::FFI.get_message_value @msg, field + if c_type == :enum + convert_upb_to_ruby message_value, c_type, Google::Protobuf::FFI.get_subtype_as_enum(field) + else + convert_upb_to_ruby message_value, c_type + end + end + end + + ## + # @param array [::FFI::Pointer] Pointer to the Array + # @param field [Google::Protobuf::FieldDescriptor] Type of the repeated field + def get_repeated_field(array, field) + return nil if array.nil? or array.null? + repeated_field = OBJECT_CACHE.get(array.address) + if repeated_field.nil? + repeated_field = RepeatedField.send(:construct_for_field, field, @arena, array: array) + end + repeated_field + end + + ## + # @param map [::FFI::Pointer] Pointer to the Map + # @param field [Google::Protobuf::FieldDescriptor] Type of the map field + def get_map_field(map, field) + return nil if map.nil? or map.null? + map_field = OBJECT_CACHE.get(map.address) + if map_field.nil? + map_field = Google::Protobuf::Map.send(:construct_for_field, field, @arena, map: map) + end + map_field + end + end + end + end + end +end diff --git a/ruby/lib/google/protobuf/ffi/object_cache.rb b/ruby/lib/google/protobuf/ffi/object_cache.rb new file mode 100644 index 000000000000..757fc56d837e --- /dev/null +++ b/ruby/lib/google/protobuf/ffi/object_cache.rb @@ -0,0 +1,53 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2022 Google Inc. All rights reserved. +# https://developers.google.com/protocol-buffers/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +module Google + module Protobuf + private + + SIZEOF_LONG = ::FFI::MemoryPointer.new(:long).size + SIZEOF_VALUE = ::FFI::Pointer::SIZE + + def self.interpreter_supports_non_finalized_keys_in_weak_map? + ! defined? JRUBY_VERSION + end + + def self.cache_implementation + if interpreter_supports_non_finalized_keys_in_weak_map? and SIZEOF_LONG >= SIZEOF_VALUE + Google::Protobuf::ObjectCache + else + Google::Protobuf::LegacyObjectCache + end + end + + public + OBJECT_CACHE = cache_implementation.new + end +end \ No newline at end of file diff --git a/ruby/lib/google/protobuf/ffi/oneof_descriptor.rb b/ruby/lib/google/protobuf/ffi/oneof_descriptor.rb new file mode 100644 index 000000000000..c863022cb120 --- /dev/null +++ b/ruby/lib/google/protobuf/ffi/oneof_descriptor.rb @@ -0,0 +1,111 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2022 Google Inc. All rights reserved. +# https://developers.google.com/protocol-buffers/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +module Google + module Protobuf + class OneofDescriptor + attr :descriptor_pool, :oneof_def + include Enumerable + + # FFI Interface methods and setup + extend ::FFI::DataConverter + native_type ::FFI::Type::POINTER + + class << self + prepend Google::Protobuf::Internal::TypeSafety + include Google::Protobuf::Internal::PointerHelper + + # @param value [OneofDescriptor] FieldDescriptor to convert to an FFI native type + # @param _ [Object] Unused + def to_native(value, _ = nil) + oneof_def_ptr = value.instance_variable_get(:@oneof_def) + warn "Underlying oneof_def was nil!" if oneof_def_ptr.nil? + raise "Underlying oneof_def was null!" if !oneof_def_ptr.nil? and oneof_def_ptr.null? + oneof_def_ptr + end + + ## + # @param oneof_def [::FFI::Pointer] OneofDef pointer to be wrapped + # @param _ [Object] Unused + def from_native(oneof_def, _ = nil) + return nil if oneof_def.nil? or oneof_def.null? + message_descriptor = Google::Protobuf::FFI.get_oneof_containing_type oneof_def + raise RuntimeError.new "Message Descriptor is nil" if message_descriptor.nil? + file_def = Google::Protobuf::FFI.get_message_file_def message_descriptor.to_native + descriptor_from_file_def(file_def, oneof_def) + end + end + + def self.new(*arguments, &block) + raise "OneofDescriptor objects may not be created from Ruby." + end + + def name + Google::Protobuf::FFI.get_oneof_name(self) + end + + def each &block + n = Google::Protobuf::FFI.get_oneof_field_count(self) + 0.upto(n-1) do |i| + yield(Google::Protobuf::FFI.get_oneof_field_by_index(self, i)) + end + nil + end + + private + + def initialize(oneof_def, descriptor_pool) + @descriptor_pool = descriptor_pool + @oneof_def = oneof_def + end + + def self.private_constructor(oneof_def, descriptor_pool) + instance = allocate + instance.send(:initialize, oneof_def, descriptor_pool) + instance + end + end + + class FFI + # MessageDef + attach_function :get_oneof_by_name, :upb_MessageDef_FindOneofByNameWithSize, [Descriptor, :string, :size_t], OneofDescriptor + attach_function :get_oneof_by_index, :upb_MessageDef_Oneof, [Descriptor, :int], OneofDescriptor + + # OneofDescriptor + attach_function :get_oneof_name, :upb_OneofDef_Name, [OneofDescriptor], :string + attach_function :get_oneof_field_count, :upb_OneofDef_FieldCount, [OneofDescriptor], :int + attach_function :get_oneof_field_by_index, :upb_OneofDef_Field, [OneofDescriptor, :int], FieldDescriptor + attach_function :get_oneof_containing_type,:upb_OneofDef_ContainingType,[:pointer], Descriptor + + # FieldDescriptor + attach_function :real_containing_oneof, :upb_FieldDef_RealContainingOneof,[FieldDescriptor], OneofDescriptor + end + end +end diff --git a/ruby/lib/google/protobuf/ffi/repeated_field.rb b/ruby/lib/google/protobuf/ffi/repeated_field.rb new file mode 100644 index 000000000000..0b5a67d00532 --- /dev/null +++ b/ruby/lib/google/protobuf/ffi/repeated_field.rb @@ -0,0 +1,526 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2008 Google Inc. All rights reserved. +# https://developers.google.com/protocol-buffers/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +require 'forwardable' + +# +# This class makes RepeatedField act (almost-) like a Ruby Array. +# It has convenience methods that extend the core C or Java based +# methods. +# +# This is a best-effort to mirror Array behavior. Two comments: +# 1) patches always welcome :) +# 2) if performance is an issue, feel free to rewrite the method +# in jruby and C. The source code has plenty of examples +# +# KNOWN ISSUES +# - #[]= doesn't allow less used approaches such as `arr[1, 2] = 'fizz'` +# - #concat should return the orig array +# - #push should accept multiple arguments and push them all at the same time +# +module Google + module Protobuf + class FFI + # Array + attach_function :append_array, :upb_Array_Append, [:Array, MessageValue.by_value, Internal::Arena], :bool + attach_function :get_msgval_at,:upb_Array_Get, [:Array, :size_t], MessageValue.by_value + attach_function :create_array, :upb_Array_New, [Internal::Arena, CType], :Array + attach_function :array_resize, :upb_Array_Resize, [:Array, :size_t, Internal::Arena], :bool + attach_function :array_set, :upb_Array_Set, [:Array, :size_t, MessageValue.by_value], :void + attach_function :array_size, :upb_Array_Size, [:Array], :size_t + end + + class RepeatedField + extend Forwardable + # NOTE: using delegators rather than method_missing to make the + # relationship explicit instead of implicit + def_delegators :to_ary, + :&, :*, :-, :'<=>', + :assoc, :bsearch, :bsearch_index, :combination, :compact, :count, + :cycle, :dig, :drop, :drop_while, :eql?, :fetch, :find_index, :flatten, + :include?, :index, :inspect, :join, + :pack, :permutation, :product, :pretty_print, :pretty_print_cycle, + :rassoc, :repeated_combination, :repeated_permutation, :reverse, + :rindex, :rotate, :sample, :shuffle, :shelljoin, + :to_s, :transpose, :uniq, :| + + include Enumerable + + ## + # call-seq: + # RepeatedField.new(type, type_class = nil, initial_values = []) + # + # Creates a new repeated field. The provided type must be a Ruby symbol, and + # an take on the same values as those accepted by FieldDescriptor#type=. If + # the type is :message or :enum, type_class must be non-nil, and must be the + # Ruby class or module returned by Descriptor#msgclass or + # EnumDescriptor#enummodule, respectively. An initial list of elements may also + # be provided. + def self.new(type, type_class = nil, initial_values = []) + instance = allocate + # TODO(jatl) This argument mangling doesn't agree with the type signature in the comments + # but is required to make unit tests pass; + if type_class.is_a?(Enumerable) and initial_values.empty? and ![:enum, :message].include?(type) + initial_values = type_class + type_class = nil + end + instance.send(:initialize, type, type_class: type_class, initial_values: initial_values) + instance + end + + ## + # call-seq: + # RepeatedField.each(&block) + # + # Invokes the block once for each element of the repeated field. RepeatedField + # also includes Enumerable; combined with this method, the repeated field thus + # acts like an ordinary Ruby sequence. + def each &block + each_msg_val do |element| + yield(convert_upb_to_ruby(element, type, descriptor, arena)) + end + self + end + + def [](*args) + count = length + if args.size < 1 + raise ArgumentError.new "Index or range is a required argument." + end + if args[0].is_a? Range + if args.size > 1 + raise ArgumentError.new "Expected 1 when passing Range argument, but got #{args.size}" + end + range = args[0] + # Handle begin-less and/or endless ranges, when supported. + index_of_first = range.respond_to?(:begin) ? range.begin : range.last + index_of_first = 0 if index_of_first.nil? + end_of_range = range.respond_to?(:end) ? range.end : range.last + index_of_last = end_of_range.nil? ? -1 : end_of_range + + if index_of_last < 0 + index_of_last += count + end + unless range.exclude_end? and !end_of_range.nil? + index_of_last += 1 + end + index_of_first += count if index_of_first < 0 + length = index_of_last - index_of_first + return [] if length.zero? + elsif args[0].is_a? Integer + index_of_first = args[0] + index_of_first += count if index_of_first < 0 + if args.size > 2 + raise ArgumentError.new "Expected 1 or 2 arguments, but got #{args.size}" + end + if args.size == 1 # No length specified, return one element + if array.null? or index_of_first < 0 or index_of_first >= count + return nil + else + return convert_upb_to_ruby(Google::Protobuf::FFI.get_msgval_at(array, index_of_first), type, descriptor, arena) + end + else + length = [args[1],count].min + end + else + raise NotImplementedError + end + + if array.null? or index_of_first < 0 or index_of_first >= count + nil + else + if index_of_first + length > count + length = count - index_of_first + end + if length < 0 + nil + else + subarray(index_of_first, length) + end + end + end + alias at [] + + + def []=(index, value) + raise FrozenError if frozen? + count = length + index += count if index < 0 + return nil if index < 0 + if index >= count + resize(index+1) + empty_message_value = Google::Protobuf::FFI::MessageValue.new # Implicitly clear + count.upto(index-1) do |i| + Google::Protobuf::FFI.array_set(array, i, empty_message_value) + end + end + Google::Protobuf::FFI.array_set(array, index, convert_ruby_to_upb(value, arena, type, descriptor)) + nil + end + + def push(*elements) + raise FrozenError if frozen? + internal_push(*elements) + end + + def <<(element) + raise FrozenError if frozen? + push element + end + + def replace(replacements) + raise FrozenError if frozen? + clear + push(*replacements) + end + + def clear + raise FrozenError if frozen? + resize 0 + self + end + + def length + array.null? ? 0 : Google::Protobuf::FFI.array_size(array) + end + alias size :length + + def dup + instance = self.class.allocate + instance.send(:initialize, type, descriptor: descriptor, arena: arena) + each_msg_val do |element| + instance.send(:append_msg_val, element) + end + instance + end + alias clone dup + + def ==(other) + return true if other.object_id == object_id + if other.is_a? RepeatedField + return false unless other.length == length + each_msg_val_with_index do |msg_val, i| + other_msg_val = Google::Protobuf::FFI.get_msgval_at(other.send(:array), i) + unless Google::Protobuf::FFI.message_value_equal(msg_val, other_msg_val, type, descriptor) + return false + end + end + return true + elsif other.is_a? Enumerable + return to_ary == other.to_a + end + false + end + + ## + # call-seq: + # RepeatedField.to_ary => array + # + # Used when converted implicitly into array, e.g. compared to an Array. + # Also called as a fallback of Object#to_a + def to_ary + return_value = [] + each do |element| + return_value << element + end + return_value + end + + def hash + return_value = 0 + each_msg_val do |msg_val| + return_value = Google::Protobuf::FFI.message_value_hash(msg_val, type, descriptor, return_value) + end + return_value + end + + def +(other) + if other.is_a? RepeatedField + if type != other.instance_variable_get(:@type) or descriptor != other.instance_variable_get(:@descriptor) + raise ArgumentError.new "Attempt to append RepeatedField with different element type." + end + fuse_arena(other.send(:arena)) + super_set = dup + other.send(:each_msg_val) do |msg_val| + super_set.send(:append_msg_val, msg_val) + end + super_set + elsif other.is_a? Enumerable + super_set = dup + super_set.push(*other.to_a) + else + raise ArgumentError.new "Unknown type appending to RepeatedField" + end + end + + def concat(other) + raise ArgumentError.new "Expected Enumerable, but got #{other.class}" unless other.is_a? Enumerable + push(*other.to_a) + end + + def first(n=nil) + if n.nil? + return self[0] + elsif n < 0 + raise ArgumentError, "negative array size" + else + return self[0...n] + end + end + + + def last(n=nil) + if n.nil? + return self[-1] + elsif n < 0 + raise ArgumentError, "negative array size" + else + start = [self.size-n, 0].max + return self[start...self.size] + end + end + + + def pop(n=nil) + if n + results = [] + n.times{ results << pop_one } + return results + else + return pop_one + end + end + + + def empty? + self.size == 0 + end + + # array aliases into enumerable + alias_method :each_index, :each_with_index + alias_method :slice, :[] + alias_method :values_at, :select + alias_method :map, :collect + + + class << self + def define_array_wrapper_method(method_name) + define_method(method_name) do |*args, &block| + arr = self.to_a + result = arr.send(method_name, *args) + self.replace(arr) + return result if result + return block ? block.call : result + end + end + private :define_array_wrapper_method + + + def define_array_wrapper_with_result_method(method_name) + define_method(method_name) do |*args, &block| + # result can be an Enumerator, Array, or nil + # Enumerator can sometimes be returned if a block is an optional argument and it is not passed in + # nil usually specifies that no change was made + result = self.to_a.send(method_name, *args, &block) + if result + new_arr = result.to_a + self.replace(new_arr) + if result.is_a?(Enumerator) + # generate a fresh enum; rewinding the exiting one, in Ruby 2.2, will + # reset the enum with the same length, but all the #next calls will + # return nil + result = new_arr.to_enum + # generate a wrapper enum so any changes which occur by a chained + # enum can be captured + ie = ProxyingEnumerator.new(self, result) + result = ie.to_enum + end + end + result + end + end + private :define_array_wrapper_with_result_method + end + + + %w(delete delete_at shift slice! unshift).each do |method_name| + define_array_wrapper_method(method_name) + end + + + %w(collect! compact! delete_if fill flatten! insert reverse! + rotate! select! shuffle! sort! sort_by! uniq!).each do |method_name| + define_array_wrapper_with_result_method(method_name) + end + alias_method :keep_if, :select! + alias_method :map!, :collect! + alias_method :reject!, :delete_if + + + # propagates changes made by user of enumerator back to the original repeated field. + # This only applies in cases where the calling function which created the enumerator, + # such as #sort!, modifies itself rather than a new array, such as #sort + class ProxyingEnumerator < Struct.new(:repeated_field, :external_enumerator) + def each(*args, &block) + results = [] + external_enumerator.each_with_index do |val, i| + result = yield(val) + results << result + #nil means no change occurred from yield; usually occurs when #to_a is called + if result + repeated_field[i] = result if result != val + end + end + results + end + end + + private + include Google::Protobuf::Internal::Convert + + attr :name, :arena, :array, :type, :descriptor + + def internal_push(*elements) + elements.each do |element| + append_msg_val convert_ruby_to_upb(element, arena, type, descriptor) + end + self + end + + def pop_one + raise FrozenError if frozen? + count = length + return nil if length.zero? + last_element = Google::Protobuf::FFI.get_msgval_at(array, count-1) + return_value = convert_upb_to_ruby(last_element, type, descriptor, arena) + resize(count-1) + return_value + end + + def subarray(start, length) + return_result = [] + (start..(start + length - 1)).each do |i| + element = Google::Protobuf::FFI.get_msgval_at(array, i) + return_result << convert_upb_to_ruby(element, type, descriptor, arena) + end + return_result + end + + def each_msg_val_with_index &block + n = array.null? ? 0 : Google::Protobuf::FFI.array_size(array) + 0.upto(n-1) do |i| + yield Google::Protobuf::FFI.get_msgval_at(array, i), i + end + end + + def each_msg_val &block + each_msg_val_with_index do |msg_val, _| + yield msg_val + end + end + + # @param msg_val [Google::Protobuf::FFI::MessageValue] Value to append + def append_msg_val(msg_val) + unless Google::Protobuf::FFI.append_array(array, msg_val, arena) + raise NoMemoryError.new "Could not allocate room for #{msg_val} in Arena" + end + end + + # @param new_size [Integer] New size of the array + def resize(new_size) + unless Google::Protobuf::FFI.array_resize(array, new_size, arena) + raise NoMemoryError.new "Array resize to #{new_size} failed!" + end + end + + def initialize(type, type_class: nil, initial_values: nil, name: nil, arena: nil, array: nil, descriptor: nil) + @name = name || 'RepeatedField' + raise ArgumentError.new "Expected argument type to be a Symbol" unless type.is_a? Symbol + field_number = Google::Protobuf::FFI::FieldType[type] + raise ArgumentError.new "Unsupported type '#{type}'" if field_number.nil? + if !descriptor.nil? + @descriptor = descriptor + elsif [:message, :enum].include? type + raise ArgumentError.new "Expected at least 2 arguments for message/enum." if type_class.nil? + descriptor = type_class.respond_to?(:descriptor) ? type_class.descriptor : nil + raise ArgumentError.new "Type class #{type_class} has no descriptor. Please pass a class or enum as returned by the DescriptorPool." if descriptor.nil? + @descriptor = descriptor + else + @descriptor = nil + end + @type = type + + @arena = arena || Google::Protobuf::FFI.create_arena + @array = array || Google::Protobuf::FFI.create_array(@arena, @type) + unless initial_values.nil? + unless initial_values.is_a? Enumerable + raise ArgumentError.new "Expected array as initializer value for repeated field '#{name}' (given #{initial_values.class})." + end + internal_push(*initial_values) + end + + # Should always be the last expression of the initializer to avoid + # leaking references to this object before construction is complete. + OBJECT_CACHE.try_add(@array.address, self) + end + + # @param field [FieldDescriptor] Descriptor of the field where the RepeatedField will be assigned + # @param values [Enumerable] Initial values; may be nil or empty + # @param arena [Arena] Owning message's arena + def self.construct_for_field(field, arena, values: nil, array: nil) + instance = allocate + options = {initial_values: values, name: field.name, arena: arena, array: array} + if [:enum, :message].include? field.type + options[:descriptor] = field.subtype + end + instance.send(:initialize, field.type, **options) + instance + end + + def fuse_arena(arena) + arena.fuse(arena) + end + + extend Google::Protobuf::Internal::Convert + + def self.deep_copy(repeated_field) + instance = allocate + instance.send(:initialize, repeated_field.send(:type), descriptor: repeated_field.send(:descriptor)) + instance.send(:resize, repeated_field.length) + new_array = instance.send(:array) + repeated_field.send(:each_msg_val_with_index) do |element, i| + Google::Protobuf::FFI.array_set(new_array, i, message_value_deep_copy(element, repeated_field.send(:type), repeated_field.send(:descriptor), instance.send(:arena))) + end + instance + end + + end + end +end diff --git a/ruby/lib/google/protobuf_ffi.rb b/ruby/lib/google/protobuf_ffi.rb new file mode 100644 index 000000000000..22219c179d39 --- /dev/null +++ b/ruby/lib/google/protobuf_ffi.rb @@ -0,0 +1,73 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2023 Google Inc. All rights reserved. +# https://developers.google.com/protocol-buffers/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +require 'ffi-compiler/loader' +require 'google/protobuf/ffi/ffi' +require 'google/protobuf/ffi/internal/type_safety' +require 'google/protobuf/ffi/internal/pointer_helper' +require 'google/protobuf/ffi/internal/arena' +require 'google/protobuf/ffi/internal/convert' +require 'google/protobuf/ffi/descriptor' +require 'google/protobuf/ffi/enum_descriptor' +require 'google/protobuf/ffi/field_descriptor' +require 'google/protobuf/ffi/oneof_descriptor' +require 'google/protobuf/ffi/descriptor_pool' +require 'google/protobuf/ffi/file_descriptor' +require 'google/protobuf/ffi/map' +require 'google/protobuf/ffi/object_cache' +require 'google/protobuf/ffi/repeated_field' +require 'google/protobuf/ffi/message' +require 'google/protobuf/descriptor_dsl' + +module Google + module Protobuf + def self.deep_copy(object) + case object + when RepeatedField + RepeatedField.send(:deep_copy, object) + when Google::Protobuf::Map + Google::Protobuf::Map.deep_copy(object) + when Google::Protobuf::MessageExts + object.class.send(:deep_copy, object.instance_variable_get(:@msg)) + else + raise NotImplementedError + end + end + + def self.discard_unknown(message) + raise FrozenError if message.frozen? + raise ArgumentError.new "Expected message, got #{message.class} instead." if message.instance_variable_get(:@msg).nil? + unless Google::Protobuf::FFI.message_discard_unknown(message.instance_variable_get(:@msg), message.class.descriptor, 128) + raise RuntimeError.new "Messages nested too deeply." + end + nil + end + end +end \ No newline at end of file diff --git a/ruby/lib/google/protobuf_native.rb b/ruby/lib/google/protobuf_native.rb new file mode 100644 index 000000000000..009983434e86 --- /dev/null +++ b/ruby/lib/google/protobuf_native.rb @@ -0,0 +1,43 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2023 Google Inc. All rights reserved. +# https://developers.google.com/protocol-buffers/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +if RUBY_PLATFORM == "java" + require 'json' + require 'google/protobuf_java' +else + begin + require "google/#{RUBY_VERSION.sub(/\.\d+$/, '')}/protobuf_c" + rescue LoadError + require 'google/protobuf_c' + end +end + +require 'google/protobuf/descriptor_dsl' +require 'google/protobuf/repeated_field' diff --git a/ruby/lib/google/tasks/ffi.rake b/ruby/lib/google/tasks/ffi.rake new file mode 100644 index 000000000000..e8f9b1c7b92b --- /dev/null +++ b/ruby/lib/google/tasks/ffi.rake @@ -0,0 +1,94 @@ +require "ffi-compiler/compile_task" + +# # @param task [FFI::Compiler::CompileTask] task to configure +def configure_common_compile_task(task) + if FileUtils.pwd.include? 'ext' + src_dir = '.' + third_party_path = 'third_party/utf8_range' + else + src_dir = 'ext/google/protobuf_c' + third_party_path = 'ext/google/protobuf_c/third_party/utf8_range' + end + + task.add_include_path third_party_path + task.add_define 'NDEBUG' + task.cflags << "-std=gnu99 -O3" + [ + :convert, :defs, :map, :message, :protobuf, :repeated_field, :wrap_memcpy + ].each { |file| task.exclude << "/#{file}.c" } + task.ext_dir = src_dir + task.source_dirs = [src_dir] + if RbConfig::CONFIG['target_os'] =~ /darwin|linux/ + task.cflags << "-Wall -Wsign-compare -Wno-declaration-after-statement" + end +end + +# FFI::CompilerTask's constructor walks the filesystem at task definition time +# to create subtasks for each source file, so files from third_party must be +# copied into place before the task is defined for it to work correctly. +# TODO(jatl) Is there a sane way to check for generated protos under lib too? +def with_generated_files + expected_path = FileUtils.pwd.include?('ext') ? 'third_party/utf8_range' : 'ext/google/protobuf_c/third_party/utf8_range' + if File.directory?(expected_path) + yield + else + task :default do + # It is possible, especially in cases like the first invocation of + # `rake test` following `rake clean` or a fresh checkout that the + # `copy_third_party` task has been executed since initial task definition. + # If so, run the task definition block now and invoke it explicitly. + if File.directory?(expected_path) + yield + Rake::Task[:default].invoke + else + raise "Missing directory #{File.absolute_path(expected_path)}." + + " Did you forget to run `rake copy_third_party` before building" + + " native extensions?" + end + end + end +end + +desc "Compile Protobuf library for FFI" +namespace "ffi-protobuf" do + with_generated_files do + # Compile Ruby UPB separately in order to limit use of -DUPB_BUILD_API to one + # compilation unit. + desc "Compile UPB library for FFI" + namespace "ffi-upb" do + with_generated_files do + FFI::Compiler::CompileTask.new('ruby-upb') do |c| + configure_common_compile_task c + c.add_define "UPB_BUILD_API" + c.exclude << "/glue.c" + c.exclude << "/shared_message.c" + c.exclude << "/shared_convert.c" + if RbConfig::CONFIG['target_os'] =~ /darwin|linux/ + c.cflags << "-fvisibility=hidden" + end + end + end + end + + FFI::Compiler::CompileTask.new 'protobuf_c_ffi' do |c| + configure_common_compile_task c + # Ruby UPB was already compiled with different flags. + c.exclude << "/range2-neon.c" + c.exclude << "/range2-sse.c" + c.exclude << "/naive.c" + c.exclude << "/ruby-upb.c" + end + + # Setup dependencies so that the .o files generated by building ffi-upb are + # available to link here. + # TODO(jatl) Can this be simplified? Can the single shared library be used + # instead of the object files? + protobuf_c_task = Rake::Task[:default] + protobuf_c_shared_lib_task = Rake::Task[protobuf_c_task.prereqs.last] + ruby_upb_shared_lib_task = Rake::Task[:"ffi-upb:default"].prereqs.first + Rake::Task[ruby_upb_shared_lib_task].prereqs.each do |dependency| + protobuf_c_shared_lib_task.prereqs.prepend dependency + end + end +end + diff --git a/ruby/tests/BUILD.bazel b/ruby/tests/BUILD.bazel index c072cc713647..57566207c0d9 100644 --- a/ruby/tests/BUILD.bazel +++ b/ruby/tests/BUILD.bazel @@ -14,6 +14,15 @@ filegroup( ], ) +ruby_test( + name = "implementation", + srcs = ["implementation.rb"], + deps = [ + "//ruby:protobuf", + "@protobuf_bundle//:test-unit", + ], +) + ruby_test( name = "basic", srcs = ["basic.rb"], diff --git a/ruby/tests/implementation.rb b/ruby/tests/implementation.rb new file mode 100644 index 000000000000..56241fc42b44 --- /dev/null +++ b/ruby/tests/implementation.rb @@ -0,0 +1,37 @@ +require 'google/protobuf' +require 'test/unit' + +class BackendTest < Test::Unit::TestCase + # Verifies the implementation of Protobuf is the preferred one. + # See protobuf.rb for the logic that defines PREFER_FFI. + def test_prefer_ffi_aligns_with_implementation + expected = Google::Protobuf::PREFER_FFI ? :FFI : :NATIVE + assert_equal expected, Google::Protobuf::IMPLEMENTATION + end + + def test_prefer_ffi + unless ENV['PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION'] =~ /ffi/i + omit"FFI implementation requires environment variable PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION=FFI to activate." + end + assert_equal true, Google::Protobuf::PREFER_FFI + end + def test_ffi_implementation + unless ENV['PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION'] =~ /ffi/i + omit "FFI implementation requires environment variable PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION=FFI to activate." + end + assert_equal :FFI, Google::Protobuf::IMPLEMENTATION + end + + def test_prefer_native + if ENV.include?('PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION') and ENV['PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION'] !~ /native/i + omit"Native implementation requires omitting environment variable PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION or setting it to `NATIVE` to activate." + end + assert_equal false, Google::Protobuf::PREFER_FFI + end + def test_native_implementation + if ENV.include?('PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION') and ENV['PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION'] !~ /native/i + omit"Native implementation requires omitting environment variable PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION or setting it to `NATIVE` to activate." + end + assert_equal :NATIVE, Google::Protobuf::IMPLEMENTATION + end +end diff --git a/ruby/tests/object_cache_test.rb b/ruby/tests/object_cache_test.rb index 455a607da9ad..1d9b77f9e71f 100644 --- a/ruby/tests/object_cache_test.rb +++ b/ruby/tests/object_cache_test.rb @@ -4,7 +4,7 @@ class PlatformTest < Test::Unit::TestCase def test_correct_implementation_for_platform omit('OBJECT_CACHE not defined') unless defined? Google::Protobuf::OBJECT_CACHE - if Gem::Version.new(RUBY_VERSION) >= Gem::Version.new('2.7.0') and Google::Protobuf::SIZEOF_LONG >= Google::Protobuf::SIZEOF_VALUE + if Google::Protobuf::SIZEOF_LONG >= Google::Protobuf::SIZEOF_VALUE and not defined? JRUBY_VERSION assert_instance_of Google::Protobuf::ObjectCache, Google::Protobuf::OBJECT_CACHE else assert_instance_of Google::Protobuf::LegacyObjectCache, Google::Protobuf::OBJECT_CACHE