Merge branch 'main' into relcommon_metadata

substrait-io · Aug 28, 2024 · 9197b45 · 9197b45
2 parents d048ad9 + aab0153
commit 9197b45
Show file tree

Hide file tree

Showing 21 changed files with 760 additions and 67 deletions.
diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
@@ -27,7 +27,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      - uses: bufbuild/buf-setup-action@v1.34.0
+      - uses: bufbuild/buf-setup-action@v1.39.0
         with:
           github_token: ${{ github.token }}
       - run: buf format --diff --exit-code
@@ -36,7 +36,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      - uses: bufbuild/buf-setup-action@v1.34.0
+      - uses: bufbuild/buf-setup-action@v1.39.0
         with:
           github_token: ${{ github.token }}
       - uses: bufbuild/buf-lint-action@v1
@@ -72,7 +72,7 @@ jobs:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
-      - uses: bufbuild/buf-setup-action@v1.34.0
+      - uses: bufbuild/buf-setup-action@v1.39.0
       - uses: actions/setup-node@v4
         with:
           node-version: "20"
@@ -93,7 +93,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      - uses: bufbuild/buf-setup-action@v1.34.0
+      - uses: bufbuild/buf-setup-action@v1.39.0
       - name: Run proto-prefix.py
         run: tools/proto_prefix.py output test proto go_package=github.com/test/proto
       - name: Modify buf config to build rewritten proto files

diff --git a/.github/workflows/pr_breaking.yml b/.github/workflows/pr_breaking.yml
@@ -9,7 +9,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      - uses: bufbuild/buf-setup-action@v1.34.0
+      - uses: bufbuild/buf-setup-action@v1.39.0
         with:
           github_token: ${{ github.token }}
       - name: check for breaking changes

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -33,7 +33,7 @@ jobs:
         with:
           node-version: "20"
 
-      - uses: bufbuild/buf-setup-action@v1.34.0
+      - uses: bufbuild/buf-setup-action@v1.39.0
         with:
           github_token: ${{ github.token }}
 

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,6 +1,62 @@
 Release Notes
 ---
 
+## [0.55.0](https://github.com/substrait-io/substrait/compare/v0.54.0...v0.55.0) (2024-08-18)
+
+### Features
+
+* update interval_day function extensions to include precision param ([#679](https://github.com/substrait-io/substrait/issues/679)) ([28025cb](https://github.com/substrait-io/substrait/commit/28025cbaa8dc3c65b736d8a68fa7070c465fb494))
+
+## [0.54.0](https://github.com/substrait-io/substrait/compare/v0.53.0...v0.54.0) (2024-08-11)
+
+### ⚠ BREAKING CHANGES
+
+* The encoding of IntervalDay literals has changed in a
+strictly backwards incompatible way. However, the logical meaning across
+encoding is maintained using a oneof. Moving a field into a oneof makes
+unset/set to zero unclear with older messages but the fields are defined
+such that the logical meaning of the two is indistinct. If neither
+microseconds nor precision is set, the value can be considered a
+precision 6 value. If you aren't using IntervalDay type, you will not
+need to make any changes.
+* TypeExpression and Parameterized type protobufs (used
+to serialize output derivation) are updated to match the now compound
+nature of IntervalDay. If you use protobuf to serialize output
+derivation that refer to IntervalDay type, you will need to rework that
+logic.
+* JoinRel's type enum now has LEFT_SINGLE
+instead of SINGLE.  Similarly there is now LEFT_ANTI and LEFT_SEMI.
+Other values are available in all join type enums. This affects JSON and
+text formats only (binary plans -- the interoperable part of Substrait --
+will still be compatible before and after this change).
+
+### Features
+
+* add arithmetic function "power" with decimal type ([#660](https://github.com/substrait-io/substrait/issues/660)) ([9af2d66](https://github.com/substrait-io/substrait/commit/9af2d66addc30ef49ed8b570a2bf9c2e1c21bad2))
+* add CSV (text) file support ([#646](https://github.com/substrait-io/substrait/issues/646)) ([5d49e04](https://github.com/substrait-io/substrait/commit/5d49e04325fcdd7c9632bda9e869a71a9d8fa8dc))
+* add precision to IntervalDay and new IntervalCompound type ([#665](https://github.com/substrait-io/substrait/issues/665)) ([e41eff2](https://github.com/substrait-io/substrait/commit/e41eff2cfed5ae6f20d0fde9b6b86da91f9d6542)), closes [#664](https://github.com/substrait-io/substrait/issues/664)
+* normalize the join types ([#662](https://github.com/substrait-io/substrait/issues/662)) ([bed84ec](https://github.com/substrait-io/substrait/commit/bed84ecb6193c22bf2ff83dc3a391ec5a9a3aa68))
+
+## [0.53.0](https://github.com/substrait-io/substrait/compare/v0.52.0...v0.53.0) (2024-08-04)
+
+### ⚠ BREAKING CHANGES
+
+* PrecisionTimestamp(Tz) literal's value is now int64
+instead of uint64
+
+### Features
+
+* add aggregate count functions with decimal return type ([#670](https://github.com/substrait-io/substrait/issues/670)) ([2aa516b](https://github.com/substrait-io/substrait/commit/2aa516bff3b2cc3e5ad262152c98f1d9b15c6765))
+* add arithmetic function "sqrt" and "factorial" with decimal type ([#674](https://github.com/substrait-io/substrait/issues/674)) ([e4f5b68](https://github.com/substrait-io/substrait/commit/e4f5b68981953d3546835572ce566e9586d497be))
+* add arithmetic function for bitwise(AND/OR/XOR) operation with decimal arguments ([#675](https://github.com/substrait-io/substrait/issues/675)) ([a70cf72](https://github.com/substrait-io/substrait/commit/a70cf72425c3a0eed432238c2a8afedab1cc025b))
+* add logarithmic functions with decimal type args ([#669](https://github.com/substrait-io/substrait/issues/669)) ([d9fb1e3](https://github.com/substrait-io/substrait/commit/d9fb1e355e0b378e1b6460f256d724a3aae931d3))
+* add precision timestamp datetime fn variants ([#666](https://github.com/substrait-io/substrait/issues/666)) ([60c93d2](https://github.com/substrait-io/substrait/commit/60c93d28c8e4df3174ba6b3f687a30d256acdcae))
+* clarify the meaning of plans ([#616](https://github.com/substrait-io/substrait/issues/616)) ([c1553df](https://github.com/substrait-io/substrait/commit/c1553dfafa09de1b2441cdb1d22a251a675419a7)), closes [#612](https://github.com/substrait-io/substrait/issues/612) [#613](https://github.com/substrait-io/substrait/issues/613)
+
+### Bug Fixes
+
+* use int64 instead of uint64 for PrecisionTimestamp(Tz) literal value ([#668](https://github.com/substrait-io/substrait/issues/668)) ([da3c74e](https://github.com/substrait-io/substrait/commit/da3c74eccc4978bdaeca4760e98a77aff560e19b))
+
 ## [0.52.0](https://github.com/substrait-io/substrait/compare/v0.51.0...v0.52.0) (2024-07-14)
 
 ### ⚠ BREAKING CHANGES

diff --git a/extensions/functions_aggregate_decimal_output.yaml b/extensions/functions_aggregate_decimal_output.yaml
@@ -0,0 +1,41 @@
+%YAML 1.2
+---
+aggregate_functions:
+  - name: "count"
+    description: Count a set of values. Result is returned as a decimal instead of i64.
+    impls:
+      - args:
+          - name: x
+            value: any
+        options:
+          overflow:
+            values: [SILENT, SATURATE, ERROR]
+        nullability: DECLARED_OUTPUT
+        decomposable: MANY
+        intermediate: decimal<38,0>
+        return: decimal<38,0>
+  - name: "count"
+    description: "Count a set of records (not field referenced). Result is returned as a decimal instead of i64."
+    impls:
+      - options:
+          overflow:
+            values: [SILENT, SATURATE, ERROR]
+        nullability: DECLARED_OUTPUT
+        decomposable: MANY
+        intermediate: decimal<38,0>
+        return: decimal<38,0>
+  - name: "approx_count_distinct"
+    description: >-
+      Calculates the approximate number of rows that contain distinct values of the expression argument using
+      HyperLogLog. This function provides an alternative to the COUNT (DISTINCT expression) function, which
+      returns the exact number of rows that contain distinct values of an expression. APPROX_COUNT_DISTINCT
+      processes large amounts of data significantly faster than COUNT, with negligible deviation from the exact
+      result. Result is returned as a decimal instead of i64.
+    impls:
+      - args:
+          - name: x
+            value: any
+        nullability: DECLARED_OUTPUT
+        decomposable: MANY
+        intermediate: binary
+        return: decimal<38,0>
diff --git a/extensions/functions_arithmetic_decimal.yaml b/extensions/functions_arithmetic_decimal.yaml
@@ -102,14 +102,86 @@ scalar_functions:
           scale_after_borrow = max(init_scale - delta, min_scale)
           scale = init_prec > 38 ? scale_after_borrow : init_scale
           DECIMAL<prec, scale>
-  - 
+  -
     name: "abs"
     description: Calculate the absolute value of the argument.
     impls:
       - args:
           - name: x
             value: decimal<P,S>
         return: decimal<P,S>
+  - name: "bitwise_and"
+    description: >
+      Return the bitwise AND result for two decimal inputs.
+      In inputs scale must be 0 (i.e. only integer types are allowed)
+    impls:
+      - args:
+          - name: x
+            value: "DECIMAL<P1,0>"
+          - name: y
+            value: "DECIMAL<P2,0>"
+        return: |-
+          max_precision = max(P1, P2)
+          DECIMAL<max_precision, 0>
+  - name: "bitwise_or"
+    description: >
+      Return the bitwise OR result for two given decimal inputs.
+      In inputs scale must be 0 (i.e. only integer types are allowed)
+    impls:
+      - args:
+          - name: x
+            value: "DECIMAL<P1,0>"
+          - name: y
+            value: "DECIMAL<P2,0>"
+        return: |-
+          max_precision = max(P1, P2)
+          DECIMAL<max_precision, 0>
+  - name: "bitwise_xor"
+    description: >
+      Return the bitwise XOR result for two given decimal inputs.
+      In inputs scale must be 0 (i.e. only integer types are allowed)
+    impls:
+      - args:
+          - name: x
+            value: "DECIMAL<P1,0>"
+          - name: y
+            value: "DECIMAL<P2,0>"
+        return: |-
+          max_precision = max(P1, P2)
+          DECIMAL<max_precision, 0>
+  - name: "sqrt"
+    description: Square root of the value. Sqrt of 0 is 0 and sqrt of negative values will raise an error.
+    impls:
+      - args:
+          - name: x
+            value: "DECIMAL<P,S>"
+        return: fp64
+  - name: "factorial"
+    description: >
+      Return the factorial of a given decimal input. Scale should be 0 for factorial decimal input.
+      The factorial of 0! is 1 by convention. Negative inputs will raise an error.
+      Input which cause overflow of result will raise an error.
+    impls:
+      - args:
+          - name: "n"
+            value: "DECIMAL<P,0>"
+        return: "DECIMAL<38,0>"
+  -
+    name: "power"
+    description: "Take the power with x as the base and y as exponent.
+    Behavior for complex number result is indicated by option complex_number_result"
+    impls:
+      - args:
+          - name: x
+            value: "DECIMAL<P1,S1>"
+          - name: y
+            value: "DECIMAL<P2,S2>"
+        options:
+          overflow:
+            values: [ SILENT, SATURATE, ERROR ]
+          complex_number_result:
+            values: [ NAN, ERROR ]
+        return: fp64
 
 aggregate_functions:
   - name: "sum"