feat: deny unknown field in options (#184)

* fix: deny unknown options Signed-off-by: usamoi <[email protected]> * test: deny unknown options Signed-off-by: usamoi <[email protected]> --------- Signed-off-by: usamoi <[email protected]>
tensorchord · Dec 15, 2023 · cc1b7d0 · cc1b7d0
1 parent c50912e
commit cc1b7d0
Show file tree

Hide file tree

Showing 19 changed files with 38 additions and 10 deletions.
diff --git a/crates/service/src/algorithms/quantization/mod.rs b/crates/service/src/algorithms/quantization/mod.rs
@@ -15,6 +15,7 @@ use std::sync::Arc;
 use validator::Validate;
 
 #[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
 #[serde(rename_all = "snake_case")]
 pub enum QuantizationOptions {
     Trivial(TrivialQuantizationOptions),

diff --git a/crates/service/src/algorithms/quantization/product.rs b/crates/service/src/algorithms/quantization/product.rs
@@ -16,6 +16,7 @@ use std::sync::Arc;
 use validator::Validate;
 
 #[derive(Debug, Clone, Serialize, Deserialize, Validate)]
+#[serde(deny_unknown_fields)]
 pub struct ProductQuantizationOptions {
     #[serde(default = "ProductQuantizationOptions::default_sample")]
     pub sample: u32,
@@ -40,6 +41,7 @@ impl Default for ProductQuantizationOptions {
 
 #[repr(u16)]
 #[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
 #[serde(rename_all = "snake_case")]
 pub enum ProductQuantizationOptionsRatio {
     X4 = 1,

diff --git a/crates/service/src/algorithms/quantization/scalar.rs b/crates/service/src/algorithms/quantization/scalar.rs
@@ -11,6 +11,7 @@ use std::sync::Arc;
 use validator::Validate;
 
 #[derive(Debug, Clone, Serialize, Deserialize, Validate)]
+#[serde(deny_unknown_fields)]
 pub struct ScalarQuantizationOptions {}
 
 impl Default for ScalarQuantizationOptions {

diff --git a/crates/service/src/algorithms/quantization/trivial.rs b/crates/service/src/algorithms/quantization/trivial.rs
@@ -9,6 +9,7 @@ use std::sync::Arc;
 use validator::Validate;
 
 #[derive(Debug, Clone, Serialize, Deserialize, Validate)]
+#[serde(deny_unknown_fields)]
 pub struct TrivialQuantizationOptions {}
 
 impl Default for TrivialQuantizationOptions {

diff --git a/crates/service/src/index/delete.rs b/crates/service/src/index/delete.rs
@@ -3,6 +3,7 @@ use crate::utils::file_wal::FileWal;
 use dashmap::mapref::entry::Entry;
 use dashmap::DashMap;
 use parking_lot::Mutex;
+use serde::{Deserialize, Serialize};
 use std::path::PathBuf;
 use std::sync::Arc;
 
@@ -79,7 +80,7 @@ impl Delete {
     }
 }
 
-#[derive(Debug, serde::Serialize, serde::Deserialize)]
+#[derive(Debug, Serialize, Deserialize)]
 struct Log {
     key: Pointer,
 }
diff --git a/crates/service/src/index/indexing/flat.rs b/crates/service/src/index/indexing/flat.rs
@@ -10,6 +10,7 @@ use std::sync::Arc;
 use validator::Validate;
 
 #[derive(Debug, Clone, Serialize, Deserialize, Validate)]
+#[serde(deny_unknown_fields)]
 pub struct FlatIndexingOptions {
     #[serde(default)]
     #[validate]

diff --git a/crates/service/src/index/indexing/hnsw.rs b/crates/service/src/index/indexing/hnsw.rs
@@ -10,6 +10,7 @@ use std::{path::PathBuf, sync::Arc};
 use validator::Validate;
 
 #[derive(Debug, Clone, Serialize, Deserialize, Validate)]
+#[serde(deny_unknown_fields)]
 pub struct HnswIndexingOptions {
     #[serde(default = "HnswIndexingOptions::default_m")]
     #[validate(range(min = 4, max = 128))]

diff --git a/crates/service/src/index/indexing/ivf.rs b/crates/service/src/index/indexing/ivf.rs
@@ -11,6 +11,7 @@ use std::sync::Arc;
 use validator::Validate;
 
 #[derive(Debug, Clone, Serialize, Deserialize, Validate)]
+#[serde(deny_unknown_fields)]
 pub struct IvfIndexingOptions {
     #[serde(default = "IvfIndexingOptions::default_least_iterations")]
     #[validate(range(min = 1, max = 1_000_000))]

diff --git a/crates/service/src/index/indexing/mod.rs b/crates/service/src/index/indexing/mod.rs
@@ -16,6 +16,7 @@ use std::sync::Arc;
 use validator::Validate;
 
 #[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
 #[serde(rename_all = "snake_case")]
 pub enum IndexingOptions {
     Flat(FlatIndexingOptions),

diff --git a/crates/service/src/index/mod.rs b/crates/service/src/index/mod.rs
@@ -37,6 +37,7 @@ use validator::Validate;
 pub struct OutdatedError(#[from] pub Option<GrowingSegmentInsertError>);
 
 #[derive(Debug, Clone, Serialize, Deserialize, Validate)]
+#[serde(deny_unknown_fields)]
 pub struct VectorOptions {
     #[validate(range(min = 1, max = 65535))]
     #[serde(rename = "dimensions")]
@@ -48,6 +49,7 @@ pub struct VectorOptions {
 }
 
 #[derive(Debug, Clone, Serialize, Deserialize, Validate)]
+#[serde(deny_unknown_fields)]
 pub struct IndexOptions {
     #[validate]
     pub vector: VectorOptions,
@@ -467,7 +469,7 @@ impl<S: G> IndexView<S> {
     }
 }
 
-#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
 struct IndexStartup {
     sealeds: HashSet<Uuid>,
     growings: HashSet<Uuid>,

diff --git a/crates/service/src/index/optimizing/mod.rs b/crates/service/src/index/optimizing/mod.rs
@@ -6,6 +6,7 @@ use serde::{Deserialize, Serialize};
 use validator::Validate;
 
 #[derive(Debug, Clone, Serialize, Deserialize, Validate)]
+#[serde(deny_unknown_fields)]
 pub struct OptimizingOptions {
     #[serde(default = "OptimizingOptions::default_sealing_secs")]
     #[validate(range(min = 0, max = 60))]

diff --git a/crates/service/src/index/segments/growing.rs b/crates/service/src/index/segments/growing.rs
@@ -7,6 +7,7 @@ use crate::prelude::*;
 use crate::utils::dir_ops::sync_dir;
 use crate::utils::file_wal::FileWal;
 use parking_lot::Mutex;
+use serde::{Deserialize, Serialize};
 use std::cell::UnsafeCell;
 use std::mem::MaybeUninit;
 use std::path::PathBuf;
@@ -199,7 +200,7 @@ impl<S: G> Drop for GrowingSegment<S> {
     }
 }
 
-#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
 struct Log<S: G> {
     vector: Vec<S::Scalar>,
     payload: Payload,

diff --git a/crates/service/src/index/segments/mod.rs b/crates/service/src/index/segments/mod.rs
@@ -9,6 +9,7 @@ use validator::Validate;
 use validator::ValidationError;
 
 #[derive(Debug, Clone, Serialize, Deserialize, Validate)]
+#[serde(deny_unknown_fields)]
 #[validate(schema(function = "Self::validate_0"))]
 pub struct SegmentsOptions {
     #[serde(default = "SegmentsOptions::default_max_growing_segment_size")]

diff --git a/crates/service/src/prelude/global/mod.rs b/crates/service/src/prelude/global/mod.rs
@@ -87,7 +87,7 @@ pub trait FloatCast: Sized {
     }
 }
 
-#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
 pub enum DynamicVector {
     F32(Vec<F32>),
     F16(Vec<F16>),

diff --git a/crates/service/src/utils/vec2.rs b/crates/service/src/utils/vec2.rs
@@ -1,7 +1,7 @@
 use crate::prelude::*;
 use std::ops::{Deref, DerefMut, Index, IndexMut};
 
-#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
+#[derive(Debug, Clone)]
 pub struct Vec2<S: G> {
     dims: u16,
     v: Vec<S::Scalar>,

diff --git a/crates/service/src/worker/mod.rs b/crates/service/src/worker/mod.rs
@@ -10,6 +10,7 @@ use crate::utils::dir_ops::sync_dir;
 use crate::utils::file_atomic::FileAtomic;
 use arc_swap::ArcSwap;
 use parking_lot::Mutex;
+use serde::{Deserialize, Serialize};
 use serde_with::DisplayFromStr;
 use std::collections::HashMap;
 use std::path::PathBuf;
@@ -172,7 +173,7 @@ impl WorkerProtect {
 }
 
 #[serde_with::serde_as]
-#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
 struct WorkerStartup {
     #[serde_as(as = "HashMap<DisplayFromStr, _>")]
     indexes: HashMap<Id, IndexOptions>,

diff --git a/src/index/am_setup.rs b/src/index/am_setup.rs
@@ -1,5 +1,5 @@
 use crate::datatype::typmod::Typmod;
-use serde::{Deserialize, Serialize};
+use serde::Deserialize;
 use service::index::indexing::IndexingOptions;
 use service::index::optimizing::OptimizingOptions;
 use service::index::segments::SegmentsOptions;
@@ -118,7 +118,8 @@ unsafe fn get_parsed_from_varlena(helper: *const pgrx::pg_sys::varlena) -> Parse
     toml::from_str::<Parsed>(cstr.to_str().unwrap()).unwrap()
 }
 
-#[derive(Debug, Clone, Serialize, Deserialize, Default)]
+#[derive(Debug, Clone, Deserialize, Default)]
+#[serde(deny_unknown_fields)]
 struct Parsed {
     #[serde(default)]
     segment: SegmentsOptions,

diff --git a/src/ipc/packet/create.rs b/src/ipc/packet/create.rs
@@ -1,5 +1,4 @@
-use serde::Deserialize;
-use serde::Serialize;
+use serde::{Deserialize, Serialize};
 
 #[derive(Debug, Serialize, Deserialize)]
 pub enum CreatePacket {

diff --git a/tests/sqllogictest/index.slt b/tests/sqllogictest/index.slt
@@ -0,0 +1,12 @@
+statement ok
+DROP TABLE IF EXISTS t;
+
+statement ok
+CREATE TABLE t (val vector(3));
+
+statement ok
+INSERT INTO t (val) SELECT ARRAY[random(), random(), random()]::real[] FROM generate_series(1, 1000);
+
+statement error unknown
+CREATE INDEX ON t USING vectors (val vector_l2_ops)
+WITH (options = "unknown_field = 1");