feat!: Initial Support for FixedSizeBinary

Cargo.toml

@@ -25,7 +25,7 @@ arrow-csv = { version = "51.0.0" }
 bit-iter = { version = "1.1.1" }
 bigdecimal = { version = "0.4.5", default-features = false, features = ["serde"] }
 blake3 = { version = "1.3.3", default-features = false }
-blitzar = { version = "3.4.0" }
+blitzar = { version = "3.5.0" }
 bumpalo = { version = "3.11.0" }
 bytemuck = {version = "1.16.3", features = ["derive"]}
 byte-slice-cast = { version = "1.2.1", default-features = false }

crates/proof-of-sql/benches/bench_append_rows.rs

@@ -13,8 +13,8 @@ use proof_of_sql::{
         commitment::TableCommitment,
         database::{
             owned_table_utility::{
-                bigint, boolean, decimal75, int, int128, owned_table, scalar, smallint,
-                timestamptz, tinyint, varchar,
+                bigint, boolean, decimal75, fixed_size_binary, int, int128, owned_table, scalar,
+                smallint, timestamptz, tinyint, varchar,
             },
             OwnedTable,
         },
@@ -89,6 +89,7 @@ pub fn generate_random_owned_table<S: Scalar>(
         "smallint",
         "int",
         "timestamptz",
+        "fixed_size_binary",
     ];
 
     let mut columns = Vec::new();
@@ -124,6 +125,13 @@ pub fn generate_random_owned_table<S: Scalar>(
                 PoSQLTimeZone::Utc,
                 vec![rng.gen::<i64>(); num_rows],
             )),
+            "fixedsizebinary" => {
+                let byte_width = 16;
+                let data: Vec<Vec<u8>> = (0..num_rows)
+                    .map(|_| (0..byte_width).map(|_| rng.gen::<u8>()).collect())
+                    .collect();
+                columns.push(fixed_size_binary(identifier.deref(), byte_width, data));
+            }
             _ => unreachable!(),
         }
     }

crates/proof-of-sql/src/base/arrow/arrow_array_to_column_conversion.rs

@@ -2,9 +2,9 @@ use super::scalar_and_i256_conversions::convert_i256_to_scalar;
 use crate::base::{database::Column, math::decimal::Precision, scalar::Scalar};
 use arrow::{
     array::{
-        Array, ArrayRef, BooleanArray, Decimal128Array, Decimal256Array, Int16Array, Int32Array,
-        Int64Array, Int8Array, StringArray, TimestampMicrosecondArray, TimestampMillisecondArray,
-        TimestampNanosecondArray, TimestampSecondArray,
+        Array, ArrayRef, BooleanArray, Decimal128Array, Decimal256Array, FixedSizeBinaryArray,
+        Int16Array, Int32Array, Int64Array, Int8Array, StringArray, TimestampMicrosecondArray,
+        TimestampMillisecondArray, TimestampNanosecondArray, TimestampSecondArray,
     },
     datatypes::{i256, DataType, TimeUnit as ArrowTimeUnit},
 };
@@ -375,6 +375,18 @@ impl ArrayRefExt for ArrayRef {
                     })
                 }
             }
+            DataType::FixedSizeBinary(byte_width) => {
+                if let Some(array) = self.as_any().downcast_ref::<FixedSizeBinaryArray>() {
+                    let start_byte = range.start * *byte_width as usize;
+                    let end_byte = range.end * *byte_width as usize;
+                    let byte_slice = &array.values()[start_byte..end_byte];
+                    Ok(Column::FixedSizeBinary(*byte_width, byte_slice))
+                } else {
+                    Err(ArrowArrayToColumnConversionError::UnsupportedType {
+                        datatype: self.data_type().clone(),
+                    })
+                }
+            }
             data_type => Err(ArrowArrayToColumnConversionError::UnsupportedType {
                 datatype: data_type.clone(),
             }),

crates/proof-of-sql/src/base/arrow/column_arrow_conversions.rs

@@ -31,6 +31,7 @@ impl From<&ColumnType> for DataType {
                 };
                 DataType::Timestamp(arrow_timeunit, arrow_timezone)
             }
+            ColumnType::FixedSizeBinary(byte_width) => DataType::FixedSizeBinary(*byte_width),
         }
     }
 }
@@ -63,6 +64,7 @@ impl TryFrom<DataType> for ColumnType {
                 ))
             }
             DataType::Utf8 => Ok(ColumnType::VarChar),
+            DataType::FixedSizeBinary(byte_width) => Ok(ColumnType::FixedSizeBinary(byte_width)),
             _ => Err(format!("Unsupported arrow data type {data_type:?}")),
         }
     }

crates/proof-of-sql/src/base/arrow/owned_and_arrow_conversions.rs

@@ -22,9 +22,9 @@ use crate::base::{
 use alloc::sync::Arc;
 use arrow::{
     array::{
-        ArrayRef, BooleanArray, Decimal128Array, Decimal256Array, Int16Array, Int32Array,
-        Int64Array, Int8Array, StringArray, TimestampMicrosecondArray, TimestampMillisecondArray,
-        TimestampNanosecondArray, TimestampSecondArray,
+        ArrayRef, BooleanArray, Decimal128Array, Decimal256Array, FixedSizeBinaryArray, Int16Array,
+        Int32Array, Int64Array, Int8Array, StringArray, TimestampMicrosecondArray,
+        TimestampMillisecondArray, TimestampNanosecondArray, TimestampSecondArray,
     },
     datatypes::{i256, DataType, Schema, SchemaRef, TimeUnit as ArrowTimeUnit},
     error::ArrowError,
@@ -109,6 +109,9 @@ impl<S: Scalar> From<OwnedColumn<S>> for ArrayRef {
                 PoSQLTimeUnit::Microsecond => Arc::new(TimestampMicrosecondArray::from(col)),
                 PoSQLTimeUnit::Nanosecond => Arc::new(TimestampNanosecondArray::from(col)),
             },
+            OwnedColumn::FixedSizeBinary(byte_width, col) => {
+                Arc::new(FixedSizeBinaryArray::new(byte_width, col.into(), None))
+            }
         }
     }
 }
@@ -149,6 +152,11 @@ impl<S: Scalar> TryFrom<&ArrayRef> for OwnedColumn<S> {
     /// - `Decimal128Array` when converting from `DataType::Decimal128(38, 0)`.
     /// - `Decimal256Array` when converting from `DataType::Decimal256` if precision is less than or equal to 75.
     /// - `StringArray` when converting from `DataType::Utf8`.
+    /// - `TimestampSecondArray` when converting from `DataType::Timestamp(ArrowTimeUnit::Second)`.
+    /// - `TimestampMillisecondArray` when converting from `DataType::Timestamp(ArrowTimeUnit::Millisecond)`.
+    /// - `TimestampMicrosecondArray` when converting from `DataType::Timestamp(ArrowTimeUnit::Microsecond)`.
+    /// - `TimestampNanosecondArray` when converting from `DataType::Timestamp(ArrowTimeUnit::Nanosecond)`.
+    /// - `FixedSizeBinaryArray` when converting from `DataType::FixedSizeBinary`.
     fn try_from(value: &ArrayRef) -> Result<Self, Self::Error> {
         match &value.data_type() {
             // Arrow uses a bit-packed representation for booleans.
@@ -224,6 +232,15 @@ impl<S: Scalar> TryFrom<&ArrayRef> for OwnedColumn<S> {
                     .map(|s| s.unwrap().to_string())
                     .collect(),
             )),
+            DataType::FixedSizeBinary(byte_width) => Ok(Self::FixedSizeBinary(
+                *byte_width,
+                value
+                    .as_any()
+                    .downcast_ref::<FixedSizeBinaryArray>()
+                    .unwrap()
+                    .values()
+                    .to_vec(),
+            )),
             DataType::Timestamp(time_unit, timezone) => match time_unit {
                 ArrowTimeUnit::Second => {
                     let array = value

crates/proof-of-sql/src/base/commitment/column_bounds.rs

@@ -196,7 +196,7 @@ pub struct ColumnBoundsMismatch {
 
 /// Column metadata storing the bounds for column types that have order.
 ///
-/// Other Ord column variants do exist (like Scalar/Boolean).
+/// Other Ord column variants do exist (like Scalar/Boolean, `FixedSizeBinary`, etc).
 /// However, bounding these is useless unless we are performing indexing on these columns.
 /// This functionality only be considered after we support them in the user-facing sql.
 #[derive(Copy, Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
@@ -236,7 +236,8 @@ impl ColumnBounds {
             | CommittableColumn::Decimal75(_, _, _)
             | CommittableColumn::Scalar(_)
             | CommittableColumn::VarChar(_)
-            | CommittableColumn::RangeCheckWord(_) => ColumnBounds::NoOrder,
+            | CommittableColumn::RangeCheckWord(_)
+            | CommittableColumn::FixedSizeBinary(_, _) => ColumnBounds::NoOrder,
         }
     }
 

crates/proof-of-sql/src/base/commitment/column_commitment_metadata.rs

@@ -54,7 +54,8 @@ impl ColumnCommitmentMetadata {
                 ColumnType::Boolean
                 | ColumnType::VarChar
                 | ColumnType::Scalar
-                | ColumnType::Decimal75(..),
+                | ColumnType::Decimal75(..)
+                | ColumnType::FixedSizeBinary(_),
                 ColumnBounds::NoOrder,
             ) => Ok(ColumnCommitmentMetadata {
                 column_type,
@@ -286,6 +287,18 @@ mod tests {
                 bounds: ColumnBounds::NoOrder
             }
         );
+
+        assert_eq!(
+            ColumnCommitmentMetadata::try_new(
+                ColumnType::FixedSizeBinary(16),
+                ColumnBounds::NoOrder
+            )
+            .unwrap(),
+            ColumnCommitmentMetadata {
+                column_type: ColumnType::FixedSizeBinary(16),
+                bounds: ColumnBounds::NoOrder
+            }
+        );
     }
 
     #[test]
@@ -373,6 +386,21 @@ mod tests {
             ),
             Err(InvalidColumnCommitmentMetadata::TypeBoundsMismatch { .. })
         ));
+
+        assert!(matches!(
+            ColumnCommitmentMetadata::try_new(
+                ColumnType::FixedSizeBinary(16),
+                ColumnBounds::BigInt(Bounds::Empty)
+            ),
+            Err(InvalidColumnCommitmentMetadata::TypeBoundsMismatch { .. })
+        ));
+        assert!(matches!(
+            ColumnCommitmentMetadata::try_new(
+                ColumnType::FixedSizeBinary(16),
+                ColumnBounds::Int128(Bounds::Empty)
+            ),
+            Err(InvalidColumnCommitmentMetadata::TypeBoundsMismatch { .. })
+        ));
     }
 
     #[test]
@@ -427,6 +455,26 @@ mod tests {
         assert_eq!(varchar_metadata.column_type(), &ColumnType::VarChar);
         assert_eq!(varchar_metadata.bounds(), &ColumnBounds::NoOrder);
 
+        let byte_width = 16;
+        let fixed_size_binary_data = [
+            vec![0u8; byte_width],
+            vec![1u8; byte_width],
+            vec![2u8; byte_width],
+        ];
+        let fixed_size_binary_column = OwnedColumn::<Curve25519Scalar>::FixedSizeBinary(
+            byte_width as i32,
+            fixed_size_binary_data.concat(),
+        );
+        let committable_fixed_size_binary_column =
+            CommittableColumn::from(&fixed_size_binary_column);
+        let fixed_size_binary_metadata =
+            ColumnCommitmentMetadata::from_column(&committable_fixed_size_binary_column);
+        assert_eq!(
+            fixed_size_binary_metadata.column_type(),
+            &ColumnType::FixedSizeBinary(byte_width as i32)
+        );
+        assert_eq!(fixed_size_binary_metadata.bounds(), &ColumnBounds::NoOrder);
+
         let bigint_column = OwnedColumn::<Curve25519Scalar>::BigInt([1, 2, 3, 1, 0].to_vec());
         let committable_bigint_column = CommittableColumn::from(&bigint_column);
         let bigint_metadata = ColumnCommitmentMetadata::from_column(&committable_bigint_column);
@@ -998,5 +1046,134 @@ mod tests {
         assert!(timestamp_tz_metadata_b
             .try_difference(timestamp_tz_metadata_a)
             .is_err());
+
+        let fixed_size_binary_metadata = ColumnCommitmentMetadata {
+            column_type: ColumnType::FixedSizeBinary(16), // Example byte width
+            bounds: ColumnBounds::NoOrder,
+        };
+
+        // Ensure FixedSizeBinary cannot be unioned with other types
+        assert!(fixed_size_binary_metadata
+            .try_union(boolean_metadata)
+            .is_err());
+        assert!(boolean_metadata
+            .try_union(fixed_size_binary_metadata)
+            .is_err());
+
+        assert!(fixed_size_binary_metadata
+            .try_union(varchar_metadata)
+            .is_err());
+        assert!(varchar_metadata
+            .try_union(fixed_size_binary_metadata)
+            .is_err());
+
+        assert!(fixed_size_binary_metadata
+            .try_union(scalar_metadata)
+            .is_err());
+        assert!(scalar_metadata
+            .try_union(fixed_size_binary_metadata)
+            .is_err());
+
+        assert!(fixed_size_binary_metadata
+            .try_union(tinyint_metadata)
+            .is_err());
+        assert!(tinyint_metadata
+            .try_union(fixed_size_binary_metadata)
+            .is_err());
+
+        assert!(fixed_size_binary_metadata
+            .try_union(smallint_metadata)
+            .is_err());
+        assert!(smallint_metadata
+            .try_union(fixed_size_binary_metadata)
+            .is_err());
+
+        assert!(fixed_size_binary_metadata.try_union(int_metadata).is_err());
+        assert!(int_metadata.try_union(fixed_size_binary_metadata).is_err());
+
+        assert!(fixed_size_binary_metadata
+            .try_union(bigint_metadata)
+            .is_err());
+        assert!(bigint_metadata
+            .try_union(fixed_size_binary_metadata)
+            .is_err());
+
+        assert!(fixed_size_binary_metadata
+            .try_union(int128_metadata)
+            .is_err());
+        assert!(int128_metadata
+            .try_union(fixed_size_binary_metadata)
+            .is_err());
+
+        assert!(fixed_size_binary_metadata
+            .try_union(decimal75_metadata)
+            .is_err());
+        assert!(decimal75_metadata
+            .try_union(fixed_size_binary_metadata)
+            .is_err());
+
+        // Ensure FixedSizeBinary cannot be differenced with other types
+        assert!(fixed_size_binary_metadata
+            .try_difference(boolean_metadata)
+            .is_err());
+        assert!(boolean_metadata
+            .try_difference(fixed_size_binary_metadata)
+            .is_err());
+
+        assert!(fixed_size_binary_metadata
+            .try_difference(varchar_metadata)
+            .is_err());
+        assert!(varchar_metadata
+            .try_difference(fixed_size_binary_metadata)
+            .is_err());
+
+        assert!(fixed_size_binary_metadata
+            .try_difference(scalar_metadata)
+            .is_err());
+        assert!(scalar_metadata
+            .try_difference(fixed_size_binary_metadata)
+            .is_err());
+
+        assert!(fixed_size_binary_metadata
+            .try_difference(tinyint_metadata)
+            .is_err());
+        assert!(tinyint_metadata
+            .try_difference(fixed_size_binary_metadata)
+            .is_err());
+
+        assert!(fixed_size_binary_metadata
+            .try_difference(smallint_metadata)
+            .is_err());
+        assert!(smallint_metadata
+            .try_difference(fixed_size_binary_metadata)
+            .is_err());
+
+        assert!(fixed_size_binary_metadata
+            .try_difference(int_metadata)
+            .is_err());
+        assert!(int_metadata
+            .try_difference(fixed_size_binary_metadata)
+            .is_err());
+
+        assert!(fixed_size_binary_metadata
+            .try_difference(bigint_metadata)
+            .is_err());
+        assert!(bigint_metadata
+            .try_difference(fixed_size_binary_metadata)
+            .is_err());
+
+        assert!(fixed_size_binary_metadata
+            .try_difference(int128_metadata)
+            .is_err());
+        assert!(int128_metadata
+            .try_difference(fixed_size_binary_metadata)
+            .is_err());
+
+        assert!(fixed_size_binary_metadata
+            .try_difference(decimal75_metadata)
+            .is_err());
+        assert!(decimal75_metadata
+            .try_difference(fixed_size_binary_metadata)
+            .is_err());
     }
 }