From f073e99f05bd336131b107eb948f8db6e696087a Mon Sep 17 00:00:00 2001 From: Pascal Seitz Date: Wed, 4 Sep 2024 13:37:32 +0800 Subject: [PATCH] add compat tests --- src/compat_tests.rs | 79 ++++++++++++++++++ src/lib.rs | 9 +- .../compat_tests_data/index_v6/.managed.json | 1 + .../index_v6/.tantivy-meta.lock | 0 .../index_v6/.tantivy-writer.lock | 0 .../00000000000000000000000000000000.fast | Bin 0 -> 146 bytes ...00000000000000000000000000000000.fieldnorm | Bin 0 -> 113 bytes .../00000000000000000000000000000000.idx | Bin 0 -> 130 bytes .../00000000000000000000000000000000.pos | Bin 0 -> 112 bytes .../00000000000000000000000000000000.store | Bin 0 -> 170 bytes .../00000000000000000000000000000000.term | Bin 0 -> 349 bytes tests/compat_tests_data/index_v6/meta.json | 40 +++++++++ 12 files changed, 126 insertions(+), 3 deletions(-) create mode 100644 src/compat_tests.rs create mode 100644 tests/compat_tests_data/index_v6/.managed.json create mode 100644 tests/compat_tests_data/index_v6/.tantivy-meta.lock create mode 100644 tests/compat_tests_data/index_v6/.tantivy-writer.lock create mode 100644 tests/compat_tests_data/index_v6/00000000000000000000000000000000.fast create mode 100644 tests/compat_tests_data/index_v6/00000000000000000000000000000000.fieldnorm create mode 100644 tests/compat_tests_data/index_v6/00000000000000000000000000000000.idx create mode 100644 tests/compat_tests_data/index_v6/00000000000000000000000000000000.pos create mode 100644 tests/compat_tests_data/index_v6/00000000000000000000000000000000.store create mode 100644 tests/compat_tests_data/index_v6/00000000000000000000000000000000.term create mode 100644 tests/compat_tests_data/index_v6/meta.json diff --git a/src/compat_tests.rs b/src/compat_tests.rs new file mode 100644 index 0000000000..eac07f9648 --- /dev/null +++ b/src/compat_tests.rs @@ -0,0 +1,79 @@ +use std::path::PathBuf; + +use collector::TopDocs; +use schema::*; + +use crate::*; + +fn create_index(path: &str) { + let mut schema_builder = Schema::builder(); + let label = schema_builder.add_text_field("label", TEXT | STORED); + let date = schema_builder.add_date_field("date", INDEXED | STORED); + let schema = schema_builder.build(); + std::fs::create_dir_all(path).unwrap(); + let index = Index::create_in_dir(path, schema).unwrap(); + let mut index_writer = index.writer_with_num_threads(1, 20_000_000).unwrap(); + index_writer + .add_document(doc!(label => "dateformat", date => DateTime::from_timestamp_nanos(123456))) + .unwrap(); + index_writer.commit().unwrap(); +} + +#[test] +/// Writes a columnar for the CURRENT_VERSION to disk. +fn create_format() { + let version = INDEX_FORMAT_VERSION.to_string(); + let file_path = path_for_version(&version); + if PathBuf::from(file_path.clone()).exists() { + return; + } + create_index(&file_path); +} + +fn path_for_version(version: &str) -> String { + format!("./tests/compat_tests_data/index_v{}/", version) +} + +#[test] +fn test_format_6() { + let path = path_for_version("6"); + + // Open the index + let index = Index::open_in_dir(path).expect("Failed to open index"); + // dates are truncated to Microseconds in v6 + assert_date_time_precision(&index, DateTimePrecision::Microseconds); +} + +fn assert_date_time_precision(index: &Index, precision: DateTimePrecision) { + // Create a searcher + let reader = index.reader().expect("Failed to create reader"); + let searcher = reader.searcher(); + + let schema = index.schema(); + let label_field = schema.get_field("label").expect("Field 'label' not found"); + let query_parser = query::QueryParser::for_index(index, vec![label_field]); + + let query = query_parser + .parse_query("dateformat") + .expect("Failed to parse query"); + let top_docs = searcher + .search(&query, &TopDocs::with_limit(1)) + .expect("Search failed"); + + assert_eq!(top_docs.len(), 1, "Expected 1 search result"); + + let doc_address = top_docs[0].1; + let retrieved_doc: TantivyDocument = searcher + .doc(doc_address) + .expect("Failed to retrieve document"); + + let date_field = schema.get_field("date").expect("Field 'date' not found"); + let date_value = retrieved_doc + .get_first(date_field) + .expect("Date field not found in document") + .as_datetime() + .unwrap(); + + let expected = DateTime::from_timestamp_nanos(123456).truncate(precision); + assert_eq!(date_value, expected,); +} diff --git a/src/lib.rs b/src/lib.rs index 6ad2981c1a..dd4fc4f52a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -202,12 +202,15 @@ pub mod space_usage; pub mod store; pub mod termdict; +mod docset; mod reader; +#[cfg(test)] +mod compat_tests; + pub use self::reader::{IndexReader, IndexReaderBuilder, ReloadPolicy, Warmer}; pub mod snippet; -mod docset; use std::fmt; pub use census::{Inventory, TrackedObject}; @@ -229,9 +232,9 @@ pub use crate::indexer::{IndexWriter, SingleSegmentIndexWriter}; pub use crate::schema::{Document, TantivyDocument, Term}; /// Index format version. -const INDEX_FORMAT_VERSION: u32 = 6; +pub const INDEX_FORMAT_VERSION: u32 = 6; /// Oldest index format version this tantivy version can read. -const INDEX_FORMAT_OLDEST_SUPPORTED_VERSION: u32 = 4; +pub const INDEX_FORMAT_OLDEST_SUPPORTED_VERSION: u32 = 4; /// Structure version for the index. #[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] diff --git a/tests/compat_tests_data/index_v6/.managed.json b/tests/compat_tests_data/index_v6/.managed.json new file mode 100644 index 0000000000..95b76f5680 --- /dev/null +++ b/tests/compat_tests_data/index_v6/.managed.json @@ -0,0 +1 @@ +["00000000000000000000000000000000.store","00000000000000000000000000000000.fast","00000000000000000000000000000000.fieldnorm","00000000000000000000000000000000.term","00000000000000000000000000000000.idx","meta.json","00000000000000000000000000000000.pos"] diff --git a/tests/compat_tests_data/index_v6/.tantivy-meta.lock b/tests/compat_tests_data/index_v6/.tantivy-meta.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/compat_tests_data/index_v6/.tantivy-writer.lock b/tests/compat_tests_data/index_v6/.tantivy-writer.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/compat_tests_data/index_v6/00000000000000000000000000000000.fast b/tests/compat_tests_data/index_v6/00000000000000000000000000000000.fast new file mode 100644 index 0000000000000000000000000000000000000000..f550fe3aa5fd1dd1edcd6e0992818fff1aa0ef3e GIT binary patch literal 146 zcmZQzKmZm56U<@;k_u2UMj*`u!iD8d)k!r0WXHVkOGB`X61 E06sw-ZU6uP literal 0 HcmV?d00001 diff --git a/tests/compat_tests_data/index_v6/00000000000000000000000000000000.fieldnorm b/tests/compat_tests_data/index_v6/00000000000000000000000000000000.fieldnorm new file mode 100644 index 0000000000000000000000000000000000000000..08af09ea23fe5f7f7fcbd3c7b7346d75f01e9425 GIT binary patch literal 113 zcmZQ%Y-(Ttf`&#$5XlQ9s+G!8i;6Sz^OUTrm2wla@{5$L40M!oGxI~j#6?_vXYgFp{1Fbg@vh!X>AzLWJ^{C1^~;o9>4$q literal 0 HcmV?d00001 diff --git a/tests/compat_tests_data/index_v6/00000000000000000000000000000000.idx b/tests/compat_tests_data/index_v6/00000000000000000000000000000000.idx new file mode 100644 index 0000000000000000000000000000000000000000..a881999d19725a878eafa40d02d0f1ba26598ec2 GIT binary patch literal 130 zcmZQ%fPjWZMi{H90mN$P0tqzm0*Pv+vecsD%=|nht7@g(#H{=xB`X6RrQFOs5X;C| zN2wsOBsl{tn3~j#6?_vXYglfq{jgiIIg-Z79%SOI8L302&e>bN~PV literal 0 HcmV?d00001 diff --git a/tests/compat_tests_data/index_v6/00000000000000000000000000000000.store b/tests/compat_tests_data/index_v6/00000000000000000000000000000000.store new file mode 100644 index 0000000000000000000000000000000000000000..97497908d951c262f85a7ff376a4e0e4c2fb5ae9 GIT binary patch literal 170 zcmY#mU|>*aVqj$Wz|@tJSdyBSUzD3z!Uz;)trlh20Hi?x#BXeEY-ng)1!8J}#DD&CsO1YVNAeND_j#5ElNpc2QFf%VDwIUv3T|8Wy eS*?yza#6C9m8F5Xxrw2%kwI-J&<0CZ1_l5j@+Wox literal 0 HcmV?d00001 diff --git a/tests/compat_tests_data/index_v6/00000000000000000000000000000000.term b/tests/compat_tests_data/index_v6/00000000000000000000000000000000.term new file mode 100644 index 0000000000000000000000000000000000000000..b2df64c6c0fe47be564fd62558d6c8fd397e1d7b GIT binary patch literal 349 zcmZQ#Km!7eM=u;da{JK1qn8+=QqoXb9ZJK*VLFldOfW$p4Q4Yjaakbg1F1z;1C|nC zXka*ifDH#=CMrU85@Rz=4^#^S10w??y3I`uAlZg5Ahiv=K%!cyEVZaOGe1wss#+;G zF)P1F$;v=SDK|3@#4Y^}7v(0F#KX0j)#@lE7bPoM8JZay TnwuCJS{m1e0Zq1KWncgR$89by literal 0 HcmV?d00001 diff --git a/tests/compat_tests_data/index_v6/meta.json b/tests/compat_tests_data/index_v6/meta.json new file mode 100644 index 0000000000..6fa7c001f7 --- /dev/null +++ b/tests/compat_tests_data/index_v6/meta.json @@ -0,0 +1,40 @@ +{ + "index_settings": { + "docstore_compression": "lz4", + "docstore_blocksize": 16384 + }, + "segments": [ + { + "segment_id": "00000000-0000-0000-0000-000000000000", + "max_doc": 1, + "deletes": null + } + ], + "schema": [ + { + "name": "label", + "type": "text", + "options": { + "indexing": { + "record": "position", + "fieldnorms": true, + "tokenizer": "default" + }, + "stored": true, + "fast": false + } + }, + { + "name": "date", + "type": "date", + "options": { + "indexed": true, + "fieldnorms": true, + "fast": false, + "stored": true, + "precision": "seconds" + } + } + ], + "opstamp": 2 +}