Ruby FFI implementation (#13343)

Supersedes #11483. Closes #13343 COPYBARA_INTEGRATE_REVIEW=#13343 from protocolbuffers:simultaneous_ffi bcb4bb7 PiperOrigin-RevId: 550782245
protocolbuffers · Jul 25, 2023 · c52d80c · c52d80c
1 parent d4f2d48
commit c52d80c
Show file tree

Hide file tree

Showing 42 changed files with 4,408 additions and 191 deletions.
diff --git a/.github/workflows/test_ruby.yml b/.github/workflows/test_ruby.yml
@@ -17,17 +17,20 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - { name: Ruby 2.7, ruby: ruby-2.7.0, bazel: 5.1.1}
+          # Test both FFI and Native implementations on the highest and lowest
+          # Ruby versions for CRuby and JRuby, but only on Bazel 5.x.
+          - { name: Ruby 2.7, ruby: ruby-2.7.0, bazel: 5.1.1, ffi: NATIVE}
+          - { name: Ruby 2.7, ruby: ruby-2.7.0, bazel: 5.1.1, ffi: FFI}
           - { name: Ruby 3.0, ruby: ruby-3.0.2, bazel: 5.1.1}
           - { name: Ruby 3.1, ruby: ruby-3.1.0, bazel: 5.1.1}
-          - { name: Ruby 3.2, ruby: ruby-3.2.0, bazel: 5.1.1}
-          - { name: JRuby 9.2, ruby: jruby-9.2.20.1, bazel: 5.1.1}
-          - { name: JRuby 9.3, ruby: jruby-9.3.10.0, bazel: 5.1.1}
-          - { name: JRuby 9.4, ruby: jruby-9.4.3.0, bazel: 5.1.1}
+          - { name: Ruby 3.2, ruby: ruby-3.2.0, bazel: 5.1.1, ffi: NATIVE}
+          - { name: Ruby 3.2, ruby: ruby-3.2.0, bazel: 5.1.1, ffi: FFI}
+          - { name: JRuby 9.4, ruby: jruby-9.4.3.0, bazel: 5.1.1, ffi: NATIVE}
+          - { name: JRuby 9.4, ruby: jruby-9.4.3.0, bazel: 5.1.1, ffi: FFI}
           - { name: Ruby 2.7 (Bazel6), ruby: ruby-2.7.0, bazel: 6.0.0}
           - { name: JRuby 9.4 (Bazel6), ruby: jruby-9.4.3.0, bazel: 6.0.0}
 
-    name: Linux ${{ matrix.name }}
+    name: Linux ${{ matrix.name }}${{ matrix.ffi == 'FFI' && ' FFI' || '' }}
     runs-on: ubuntu-latest
     steps:
       - name: Checkout pending changes
@@ -40,7 +43,7 @@ jobs:
           image: ${{ matrix.image || format('us-docker.pkg.dev/protobuf-build/containers/test/linux/ruby:{0}-{1}-508417e5215994ade7585d28ba3aad681a25fa5d', matrix.ruby, matrix.bazel) }}
           credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
           bazel-cache: ruby_linux/${{ matrix.ruby }}_${{ matrix.bazel }}
-          bazel: test //ruby/... //ruby/tests:ruby_version --test_env=KOKORO_RUBY_VERSION
+          bazel: test //ruby/... //ruby/tests:ruby_version --test_env=KOKORO_RUBY_VERSION --test_env=BAZEL=true ${{ matrix.ffi == 'FFI' && '--//ruby:ffi=enabled --test_env=PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION=FFI' || '' }}
 
   linux-aarch64:
     name: Linux aarch64
@@ -78,10 +81,18 @@ jobs:
     strategy:
       fail-fast: false   # Don't cancel all jobs if one fails.
       matrix:
-        # This is the full set of versions we support on MacOS.
-        version: [ "2.7", "3.0", "3.1", "3.2" ]
+        include:
+        # Test both FFI and Native implementations on the highest and lowest
+        # Ruby versions for CRuby, but only on Bazel 5.x.
+        # Quote versions numbers otherwise 3.0 will render as 3
+        - { version: "2.7", ffi: NATIVE }
+        - { version: "2.7", ffi: FFI }
+        - { version: "3.0" }
+        - { version: "3.1" }
+        - { version: "3.2", ffi: NATIVE }
+        - { version: "3.2", ffi: FFI }
 
-    name: MacOS Ruby ${{ matrix.version }}
+    name: MacOS Ruby ${{ matrix.version }}${{ matrix.ffi == 'FFI' && ' FFI' || '' }}
     runs-on: macos-12
     steps:
       - name: Checkout pending changes
@@ -102,23 +113,26 @@ jobs:
         with:
           credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
           bazel-cache: ruby_macos/${{ matrix.version }}
-          bazel: test //ruby/... --test_env=KOKORO_RUBY_VERSION=${{ matrix.version }}
+          bazel: test //ruby/... --test_env=KOKORO_RUBY_VERSION=${{ matrix.version }} --test_env=BAZEL=true ${{ matrix.ffi == 'FFI' && '--//ruby:ffi=enabled --test_env=PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION=FFI' || '' }}
 
   test_ruby_gems:
     strategy:
       fail-fast: false
       matrix:
         include:
-          - { name: Ruby 2.7, ruby: ruby-2.7.0, bazel: 5.1.1}
+          # Test both FFI and Native implementations on the highest and lowest
+          # Ruby versions for CRuby and JRuby, but only on Bazel 5.x.
+          - { name: Ruby 2.7, ruby: ruby-2.7.0, bazel: 5.1.1, ffi: NATIVE }
+          - { name: Ruby 2.7, ruby: ruby-2.7.0, bazel: 5.1.1, ffi: FFI }
           - { name: Ruby 3.0, ruby: ruby-3.0.2, bazel: 5.1.1}
           - { name: Ruby 3.1, ruby: ruby-3.1.0, bazel: 5.1.1}
-          - { name: Ruby 3.2, ruby: ruby-3.2.0, bazel: 5.1.1}
-          - { name: JRuby 9.2, ruby: jruby-9.2.20.1, bazel: 5.1.1}
-          - { name: JRuby 9.3, ruby: jruby-9.3.10.0, bazel: 5.1.1}
-          - { name: JRuby 9.4, ruby: jruby-9.4.3.0, bazel: 5.1.1}
+          - { name: Ruby 3.2, ruby: ruby-3.2.0, bazel: 5.1.1, ffi: NATIVE }
+          - { name: Ruby 3.2, ruby: ruby-3.2.0, bazel: 5.1.1, ffi: FFI }
+          - { name: JRuby 9.4, ruby: jruby-9.4.3.0, bazel: 5.1.1, ffi: NATIVE }
+          - { name: JRuby 9.4, ruby: jruby-9.4.3.0, bazel: 5.1.1, ffi: FFI }
           - { name: Ruby 2.7 (Bazel6), ruby: ruby-2.7.0, bazel: 6.0.0}
           - { name: JRuby 9.4 (Bazel6), ruby: jruby-9.4.3.0, bazel: 6.0.0}
-    name: Install ${{ matrix.name }}
+    name: Install ${{ matrix.name }}${{ matrix.ffi == 'FFI' && ' FFI' || '' }}
     runs-on: ubuntu-latest
     steps:
       - name: Checkout pending changes
@@ -134,8 +148,9 @@ jobs:
           bash: >
             bazel --version;
             ruby --version;
-            bazel build //ruby:release //:protoc $BAZEL_FLAGS;
+            bazel build //ruby:release //:protoc ${{ matrix.ffi == 'FFI' && '--//ruby:ffi=enabled' || '' }} $BAZEL_FLAGS;
             gem install bazel-bin/ruby/google-protobuf-*;
             bazel-bin/protoc --proto_path=src --proto_path=ruby/tests --proto_path=ruby --ruby_out=ruby tests/test_import_proto2.proto;
             bazel-bin/protoc --proto_path=src --proto_path=ruby/tests --proto_path=ruby --ruby_out=ruby tests/basic_test.proto;
-            ruby ruby/tests/basic.rb
+            ${{ matrix.ffi == 'FFI' && 'PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION=FFI' || '' }} ruby ruby/tests/basic.rb;
+            ${{ matrix.ffi == 'FFI' && 'PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION=FFI' || '' }} ruby ruby/tests/implementation.rb
diff --git a/conformance/BUILD.bazel b/conformance/BUILD.bazel
@@ -24,6 +24,7 @@ exports_files([
     "failure_list_python_cpp.txt",
     "failure_list_ruby.txt",
     "failure_list_jruby.txt",
+    "failure_list_jruby_ffi.txt",
     "text_format_failure_list_cpp.txt",
     "text_format_failure_list_csharp.txt",
     "text_format_failure_list_java.txt",
@@ -34,6 +35,7 @@ exports_files([
     "text_format_failure_list_python_cpp.txt",
     "text_format_failure_list_ruby.txt",
     "text_format_failure_list_jruby.txt",
+    "text_format_failure_list_jruby_ffi.txt",
 ])
 
 cc_proto_library(
@@ -326,12 +328,12 @@ ruby_binary(
     name = "conformance_ruby",
     testonly = True,
     srcs = ["conformance_ruby.rb"],
+    visibility = ["//ruby:__subpackages__"],
     deps = [
         ":conformance_ruby_proto",
         "//:test_messages_proto2_ruby_proto",
-        "//:test_messages_proto3_ruby_proto",        
+        "//:test_messages_proto3_ruby_proto",
     ],
-    visibility = ["//ruby:__subpackages__"],
 )
 
 ################################################################################

diff --git a/conformance/failure_list_jruby_ffi.txt b/conformance/failure_list_jruby_ffi.txt
@@ -0,0 +1,2 @@
+Recommended.Proto3.JsonInput.IgnoreUnknownEnumStringValueInMapValue.ProtobufOutput
+Recommended.Proto3.JsonInput.IgnoreUnknownEnumStringValueInRepeatedField.ProtobufOutput
diff --git a/protobuf_deps.bzl b/protobuf_deps.bzl
@@ -110,8 +110,8 @@ def protobuf_deps():
         _github_archive(
             name = "rules_ruby",
             repo = "https://github.com/protocolbuffers/rules_ruby",
-            commit = "8fca842a3006c3d637114aba4f6bf9695bb3a432",
-            sha256 = "2619f9a23cee6f6a198d9ef284b6f6cbc901545ee9a9aac9ffa6b83dbf17cf0c",
+            commit = "b7f3e9756f3c45527be27bc38840d5a1ba690436",
+            sha256 = "347927fd8de6132099fcdc58e8f7eab7bde4eb2fd424546b9cd4f1c6f8f8bad8",
         )
 
     if not native.existing_rule("rules_jvm_external"):

diff --git a/ruby/.gitignore b/ruby/.gitignore
@@ -6,4 +6,10 @@ protobuf-jruby.iml
 target/
 pkg/
 tmp/
-tests/google/
+tests/google/
+ext/google/protobuf_c/third_party/utf8_range/utf8_range.h
+ext/google/protobuf_c/third_party/utf8_range/range2-sse.c
+ext/google/protobuf_c/third_party/utf8_range/range2-neon.c
+ext/google/protobuf_c/third_party/utf8_range/naive.c
+ext/google/protobuf_c/third_party/utf8_range/LICENSE
+lib/google/protobuf/*_pb.rb