diff --git a/src/core/json_utils.rs b/src/core/json_utils.rs index d7ac29ad7d..2a8553acba 100644 --- a/src/core/json_utils.rs +++ b/src/core/json_utils.rs @@ -4,6 +4,7 @@ use rustc_hash::FxHashMap; use crate::postings::{IndexingContext, IndexingPosition, PostingsWriter}; use crate::schema::document::{ReferenceValue, ReferenceValueLeaf, Value}; +use crate::schema::indexing_term::IndexingTerm; use crate::schema::{Field, Type}; use crate::time::format_description::well_known::Rfc3339; use crate::time::{OffsetDateTime, UtcOffset}; @@ -74,7 +75,7 @@ pub(crate) fn index_json_values<'a, V: Value<'a>>( json_visitors: impl Iterator>, text_analyzer: &mut TextAnalyzer, expand_dots_enabled: bool, - term_buffer: &mut Term, + term_buffer: &mut IndexingTerm, postings_writer: &mut dyn PostingsWriter, json_path_writer: &mut JsonPathWriter, ctx: &mut IndexingContext, @@ -103,7 +104,7 @@ fn index_json_object<'a, V: Value<'a>>( doc: DocId, json_visitor: V::ObjectIter, text_analyzer: &mut TextAnalyzer, - term_buffer: &mut Term, + term_buffer: &mut IndexingTerm, json_path_writer: &mut JsonPathWriter, postings_writer: &mut dyn PostingsWriter, ctx: &mut IndexingContext, @@ -130,19 +131,16 @@ fn index_json_value<'a, V: Value<'a>>( doc: DocId, json_value: V, text_analyzer: &mut TextAnalyzer, - term_buffer: &mut Term, + term_buffer: &mut IndexingTerm, json_path_writer: &mut JsonPathWriter, postings_writer: &mut dyn PostingsWriter, ctx: &mut IndexingContext, positions_per_path: &mut IndexingPositionsPerPath, ) { - let set_path_id = |term_buffer: &mut Term, unordered_id: u32| { + let set_path_id = |term_buffer: &mut IndexingTerm, unordered_id: u32| { term_buffer.truncate_value_bytes(0); term_buffer.append_bytes(&unordered_id.to_be_bytes()); }; - let set_type = |term_buffer: &mut Term, typ: Type| { - term_buffer.append_bytes(&[typ.to_code()]); - }; match json_value.as_value() { ReferenceValue::Leaf(leaf) => match leaf { @@ -155,7 +153,7 @@ fn index_json_value<'a, V: Value<'a>>( // TODO: make sure the chain position works out. set_path_id(term_buffer, unordered_id); - set_type(term_buffer, Type::Str); + term_buffer.append_bytes(&[Type::Str.to_code()]); let indexing_position = positions_per_path.get_position_from_id(unordered_id); postings_writer.index_text( doc, @@ -211,18 +209,16 @@ fn index_json_value<'a, V: Value<'a>>( postings_writer.subscribe(doc, 0u32, term_buffer, ctx); } ReferenceValueLeaf::PreTokStr(_) => { - unimplemented!( - "Pre-tokenized string support in dynamic fields is not yet implemented" - ) + unimplemented!("Pre-tokenized string support in JSON fields is not yet implemented") } ReferenceValueLeaf::Bytes(_) => { - unimplemented!("Bytes support in dynamic fields is not yet implemented") + unimplemented!("Bytes support in JSON fields is not yet implemented") } ReferenceValueLeaf::Facet(_) => { - unimplemented!("Facet support in dynamic fields is not yet implemented") + unimplemented!("Facet support in JSON fields is not yet implemented") } ReferenceValueLeaf::IpAddr(_) => { - unimplemented!("IP address support in dynamic fields is not yet implemented") + unimplemented!("IP address support in JSON fields is not yet implemented") } }, ReferenceValue::Array(elements) => { @@ -257,14 +253,12 @@ fn index_json_value<'a, V: Value<'a>>( /// Tries to infer a JSON type from a string and append it to the term. /// /// The term must be json + JSON path. -pub(crate) fn convert_to_fast_value_and_append_to_json_term( - mut term: Term, - phrase: &str, -) -> Option { +pub fn convert_to_fast_value_and_append_to_json_term(mut term: Term, phrase: &str) -> Option { assert_eq!( term.value() - .as_json_value_bytes() + .as_json() .expect("expecting a Term with a json type and json path") + .1 .as_serialized() .len(), 0, @@ -410,8 +404,8 @@ mod tests { term.append_type_and_fast_value(-4i64); assert_eq!( - term.serialized_term(), - b"\x00\x00\x00\x01jcolor\x00i\x7f\xff\xff\xff\xff\xff\xff\xfc" + term.value().as_serialized(), + b"jcolor\x00i\x7f\xff\xff\xff\xff\xff\xff\xfc" ) } @@ -422,8 +416,8 @@ mod tests { term.append_type_and_fast_value(4u64); assert_eq!( - term.serialized_term(), - b"\x00\x00\x00\x01jcolor\x00u\x00\x00\x00\x00\x00\x00\x00\x04" + term.value().as_serialized(), + b"jcolor\x00u\x00\x00\x00\x00\x00\x00\x00\x04" ) } @@ -433,8 +427,8 @@ mod tests { let mut term = term_from_json_paths(field, ["color"].into_iter(), false); term.append_type_and_fast_value(4.0f64); assert_eq!( - term.serialized_term(), - b"\x00\x00\x00\x01jcolor\x00f\xc0\x10\x00\x00\x00\x00\x00\x00" + term.value().as_serialized(), + b"jcolor\x00f\xc0\x10\x00\x00\x00\x00\x00\x00" ) } @@ -444,8 +438,8 @@ mod tests { let mut term = term_from_json_paths(field, ["color"].into_iter(), false); term.append_type_and_fast_value(true); assert_eq!( - term.serialized_term(), - b"\x00\x00\x00\x01jcolor\x00o\x00\x00\x00\x00\x00\x00\x00\x01" + term.value().as_serialized(), + b"jcolor\x00o\x00\x00\x00\x00\x00\x00\x00\x01" ) } diff --git a/src/indexer/segment_writer.rs b/src/indexer/segment_writer.rs index 384a939e6d..497e7471d6 100644 --- a/src/indexer/segment_writer.rs +++ b/src/indexer/segment_writer.rs @@ -1,4 +1,3 @@ -use columnar::MonotonicallyMappableToU64; use common::JsonPathWriter; use itertools::Itertools; use tokenizer_api::BoxTokenStream; @@ -15,7 +14,8 @@ use crate::postings::{ PerFieldPostingsWriter, PostingsWriter, }; use crate::schema::document::{Document, ReferenceValue, Value}; -use crate::schema::{FieldEntry, FieldType, Schema, Term, DATE_TIME_PRECISION_INDEXED}; +use crate::schema::indexing_term::IndexingTerm; +use crate::schema::{FieldEntry, FieldType, Schema}; use crate::store::{StoreReader, StoreWriter}; use crate::tokenizer::{FacetTokenizer, PreTokenizedStream, TextAnalyzer, Tokenizer}; use crate::{DocId, Opstamp, SegmentComponent, TantivyError}; @@ -70,7 +70,7 @@ pub struct SegmentWriter { pub(crate) json_path_writer: JsonPathWriter, pub(crate) doc_opstamps: Vec, per_field_text_analyzers: Vec, - term_buffer: Term, + term_buffer: IndexingTerm, schema: Schema, } @@ -126,7 +126,7 @@ impl SegmentWriter { )?, doc_opstamps: Vec::with_capacity(1_000), per_field_text_analyzers, - term_buffer: Term::with_capacity(16), + term_buffer: IndexingTerm::new(), schema, }) } @@ -195,7 +195,7 @@ impl SegmentWriter { let (term_buffer, ctx) = (&mut self.term_buffer, &mut self.ctx); let postings_writer: &mut dyn PostingsWriter = self.per_field_postings_writers.get_for_field_mut(field); - term_buffer.clear_with_field_and_type(field_entry.field_type().value_type(), field); + term_buffer.clear_with_field(field); match field_entry.field_type() { FieldType::Facet(_) => { @@ -271,8 +271,7 @@ impl SegmentWriter { num_vals += 1; let date_val = value.as_datetime().ok_or_else(make_schema_error)?; - term_buffer - .set_u64(date_val.truncate(DATE_TIME_PRECISION_INDEXED).to_u64()); + term_buffer.set_date(date_val); postings_writer.subscribe(doc_id, 0u32, term_buffer, ctx); } if field_entry.has_fieldnorms() { @@ -332,7 +331,7 @@ impl SegmentWriter { num_vals += 1; let bytes = value.as_bytes().ok_or_else(make_schema_error)?; - term_buffer.set_bytes(bytes); + term_buffer.set_value_bytes(bytes); postings_writer.subscribe(doc_id, 0u32, term_buffer, ctx); } if field_entry.has_fieldnorms() { diff --git a/src/postings/json_postings_writer.rs b/src/postings/json_postings_writer.rs index ed3d5c24f3..b8c84cd29d 100644 --- a/src/postings/json_postings_writer.rs +++ b/src/postings/json_postings_writer.rs @@ -8,9 +8,10 @@ use crate::indexer::path_to_unordered_id::OrderedPathId; use crate::postings::postings_writer::SpecializedPostingsWriter; use crate::postings::recorder::{BufferLender, DocIdRecorder, Recorder}; use crate::postings::{FieldSerializer, IndexingContext, IndexingPosition, PostingsWriter}; -use crate::schema::{Field, Type}; +use crate::schema::indexing_term::IndexingTerm; +use crate::schema::{Field, Type, ValueBytes}; use crate::tokenizer::TokenStream; -use crate::{DocId, Term}; +use crate::DocId; /// The `JsonPostingsWriter` is odd in that it relies on a hidden contract: /// @@ -34,7 +35,7 @@ impl PostingsWriter for JsonPostingsWriter { &mut self, doc: crate::DocId, pos: u32, - term: &crate::Term, + term: &IndexingTerm, ctx: &mut IndexingContext, ) { self.non_str_posting_writer.subscribe(doc, pos, term, ctx); @@ -44,7 +45,7 @@ impl PostingsWriter for JsonPostingsWriter { &mut self, doc_id: DocId, token_stream: &mut dyn TokenStream, - term_buffer: &mut Term, + term_buffer: &mut IndexingTerm, ctx: &mut IndexingContext, indexing_position: &mut IndexingPosition, ) { @@ -66,42 +67,40 @@ impl PostingsWriter for JsonPostingsWriter { ctx: &IndexingContext, serializer: &mut FieldSerializer, ) -> io::Result<()> { - let mut term_buffer = Term::with_capacity(48); + let mut term_buffer = JsonTermSerializer(Vec::with_capacity(48)); let mut buffer_lender = BufferLender::default(); - term_buffer.clear_with_field_and_type(Type::Json, Field::from_field_id(0)); let mut prev_term_id = u32::MAX; let mut term_path_len = 0; // this will be set in the first iteration for (_field, path_id, term, addr) in term_addrs { if prev_term_id != path_id.path_id() { - term_buffer.truncate_value_bytes(0); + term_buffer.clear(); term_buffer.append_path(ordered_id_to_path[path_id.path_id() as usize].as_bytes()); term_buffer.append_bytes(&[JSON_END_OF_PATH]); - term_path_len = term_buffer.len_bytes(); + term_path_len = term_buffer.len(); prev_term_id = path_id.path_id(); } - term_buffer.truncate_value_bytes(term_path_len); + term_buffer.truncate(term_path_len); term_buffer.append_bytes(term); - if let Some(json_value) = term_buffer.value().as_json_value_bytes() { - let typ = json_value.typ(); - if typ == Type::Str { - SpecializedPostingsWriter::::serialize_one_term( - term_buffer.serialized_value_bytes(), - *addr, - doc_id_map, - &mut buffer_lender, - ctx, - serializer, - )?; - } else { - SpecializedPostingsWriter::::serialize_one_term( - term_buffer.serialized_value_bytes(), - *addr, - doc_id_map, - &mut buffer_lender, - ctx, - serializer, - )?; - } + let json_value = ValueBytes::wrap(term); + let typ = json_value.typ(); + if typ == Type::Str { + SpecializedPostingsWriter::::serialize_one_term( + term_buffer.as_bytes(), + *addr, + doc_id_map, + &mut buffer_lender, + ctx, + serializer, + )?; + } else { + SpecializedPostingsWriter::::serialize_one_term( + term_buffer.as_bytes(), + *addr, + doc_id_map, + &mut buffer_lender, + ctx, + serializer, + )?; } } Ok(()) @@ -111,3 +110,40 @@ impl PostingsWriter for JsonPostingsWriter { self.str_posting_writer.total_num_tokens() + self.non_str_posting_writer.total_num_tokens() } } + +struct JsonTermSerializer(Vec); +impl JsonTermSerializer { + #[inline] + pub fn append_path(&mut self, bytes: &[u8]) { + if bytes.contains(&0u8) { + self.0 + .extend(bytes.iter().map(|&b| if b == 0 { b'0' } else { b })); + } else { + self.0.extend_from_slice(bytes); + } + } + + /// Appends value bytes to the Term. + /// + /// This function returns the segment that has just been added. + #[inline] + pub fn append_bytes(&mut self, bytes: &[u8]) -> &mut [u8] { + let len_before = self.0.len(); + self.0.extend_from_slice(bytes); + &mut self.0[len_before..] + } + + fn clear(&mut self) { + self.0.clear(); + } + fn truncate(&mut self, len: usize) { + self.0.truncate(len); + } + fn len(&self) -> usize { + self.0.len() + } + + fn as_bytes(&self) -> &[u8] { + &self.0 + } +} diff --git a/src/postings/postings_writer.rs b/src/postings/postings_writer.rs index 264392889a..5802f74f34 100644 --- a/src/postings/postings_writer.rs +++ b/src/postings/postings_writer.rs @@ -11,7 +11,8 @@ use crate::postings::recorder::{BufferLender, Recorder}; use crate::postings::{ FieldSerializer, IndexingContext, InvertedIndexSerializer, PerFieldPostingsWriter, }; -use crate::schema::{Field, Schema, Term, Type}; +use crate::schema::indexing_term::{get_field_from_indexing_term, IndexingTerm}; +use crate::schema::{Field, Schema, Type}; use crate::tokenizer::{Token, TokenStream, MAX_TOKEN_LEN}; use crate::DocId; @@ -60,14 +61,14 @@ pub(crate) fn serialize_postings( let mut term_offsets: Vec<(Field, OrderedPathId, &[u8], Addr)> = Vec::with_capacity(ctx.term_index.len()); term_offsets.extend(ctx.term_index.iter().map(|(key, addr)| { - let field = Term::wrap(key).field(); + let field = get_field_from_indexing_term(key); if schema.get_field_entry(field).field_type().value_type() == Type::Json { - let byte_range_path = 5..5 + 4; + let byte_range_path = 4..4 + 4; let unordered_id = u32::from_be_bytes(key[byte_range_path.clone()].try_into().unwrap()); let path_id = unordered_id_to_ordered_id[unordered_id as usize]; (field, path_id, &key[byte_range_path.end..], addr) } else { - (field, 0.into(), &key[5..], addr) + (field, 0.into(), &key[4..], addr) } })); // Sort by field, path, and term @@ -114,7 +115,7 @@ pub(crate) trait PostingsWriter: Send + Sync { /// * term - the term /// * ctx - Contains a term hashmap and a memory arena to store all necessary posting list /// information. - fn subscribe(&mut self, doc: DocId, pos: u32, term: &Term, ctx: &mut IndexingContext); + fn subscribe(&mut self, doc: DocId, pos: u32, term: &IndexingTerm, ctx: &mut IndexingContext); /// Serializes the postings on disk. /// The actual serialization format is handled by the `PostingsSerializer`. @@ -132,7 +133,7 @@ pub(crate) trait PostingsWriter: Send + Sync { &mut self, doc_id: DocId, token_stream: &mut dyn TokenStream, - term_buffer: &mut Term, + term_buffer: &mut IndexingTerm, ctx: &mut IndexingContext, indexing_position: &mut IndexingPosition, ) { @@ -203,26 +204,35 @@ impl SpecializedPostingsWriter { impl PostingsWriter for SpecializedPostingsWriter { #[inline] - fn subscribe(&mut self, doc: DocId, position: u32, term: &Term, ctx: &mut IndexingContext) { - debug_assert!(term.serialized_term().len() >= 4); + fn subscribe( + &mut self, + doc: DocId, + position: u32, + term: &IndexingTerm, + ctx: &mut IndexingContext, + ) { + debug_assert!(term.serialized_for_hashmap().len() >= 4); self.total_num_tokens += 1; let (term_index, arena) = (&mut ctx.term_index, &mut ctx.arena); - term_index.mutate_or_create(term.serialized_term(), |opt_recorder: Option| { - if let Some(mut recorder) = opt_recorder { - let current_doc = recorder.current_doc(); - if current_doc != doc { - recorder.close_doc(arena); + term_index.mutate_or_create( + term.serialized_for_hashmap(), + |opt_recorder: Option| { + if let Some(mut recorder) = opt_recorder { + let current_doc = recorder.current_doc(); + if current_doc != doc { + recorder.close_doc(arena); + recorder.new_doc(doc, arena); + } + recorder.record_position(position, arena); + recorder + } else { + let mut recorder = Rec::default(); recorder.new_doc(doc, arena); + recorder.record_position(position, arena); + recorder } - recorder.record_position(position, arena); - recorder - } else { - let mut recorder = Rec::default(); - recorder.new_doc(doc, arena); - recorder.record_position(position, arena); - recorder - } - }); + }, + ); } fn serialize( diff --git a/src/query/fuzzy_query.rs b/src/query/fuzzy_query.rs index a2e3f2a6ba..21b5aace01 100644 --- a/src/query/fuzzy_query.rs +++ b/src/query/fuzzy_query.rs @@ -3,7 +3,7 @@ use once_cell::sync::OnceCell; use tantivy_fst::Automaton; use crate::query::{AutomatonWeight, EnableScoring, Query, Weight}; -use crate::schema::{Term, Type}; +use crate::schema::Term; use crate::TantivyError::InvalidArgument; pub(crate) struct DfaWrapper(pub DFA); @@ -133,40 +133,33 @@ impl FuzzyTermQuery { let term_value = self.term.value(); - let term_text = if term_value.typ() == Type::Json { - if let Some(json_path_type) = term_value.json_path_type() { - if json_path_type != Type::Str { - return Err(InvalidArgument(format!( - "The fuzzy term query requires a string path type for a json term. Found \ - {:?}", - json_path_type - ))); - } + let get_automaton = |term_text: &str| { + if self.prefix { + automaton_builder.build_prefix_dfa(term_text) + } else { + automaton_builder.build_dfa(term_text) } - - std::str::from_utf8(self.term.serialized_value_bytes()).map_err(|_| { - InvalidArgument( - "Failed to convert json term value bytes to utf8 string.".to_string(), - ) - })? - } else { - term_value.as_str().ok_or_else(|| { - InvalidArgument("The fuzzy term query requires a string term.".to_string()) - })? - }; - let automaton = if self.prefix { - automaton_builder.build_prefix_dfa(term_text) - } else { - automaton_builder.build_dfa(term_text) }; - if let Some((json_path_bytes, _)) = term_value.as_json() { + if let Some((json_path_bytes, _term_value)) = term_value.as_json() { + let term_text = + std::str::from_utf8(self.term.serialized_value_bytes()).map_err(|_| { + InvalidArgument( + "Failed to convert json term value bytes to utf8 string.".to_string(), + ) + })?; + + let automaton = get_automaton(term_text); Ok(AutomatonWeight::new_for_json_path( self.term.field(), DfaWrapper(automaton), json_path_bytes, )) } else { + let term_text = term_value.as_str().ok_or_else(|| { + InvalidArgument("The fuzzy term query requires a string term.".to_string()) + })?; + let automaton = get_automaton(term_text); Ok(AutomatonWeight::new( self.term.field(), DfaWrapper(automaton), diff --git a/src/query/phrase_prefix_query/phrase_prefix_query.rs b/src/query/phrase_prefix_query/phrase_prefix_query.rs index 8cbbe637e5..c428d2b30b 100644 --- a/src/query/phrase_prefix_query/phrase_prefix_query.rs +++ b/src/query/phrase_prefix_query/phrase_prefix_query.rs @@ -137,7 +137,7 @@ impl Query for PhrasePrefixQuery { // There are no prefix. Let's just match the suffix. let end_term = if let Some(end_value) = prefix_end(self.prefix.1.serialized_value_bytes()) { - let mut end_term = Term::with_capacity(end_value.len()); + let mut end_term = Term::new(); end_term.set_field_and_type(self.field, self.prefix.1.typ()); end_term.append_bytes(&end_value); Bound::Excluded(end_term) diff --git a/src/schema/indexing_term.rs b/src/schema/indexing_term.rs new file mode 100644 index 0000000000..ffe6494e3c --- /dev/null +++ b/src/schema/indexing_term.rs @@ -0,0 +1,147 @@ +use std::net::Ipv6Addr; + +use columnar::{MonotonicallyMappableToU128, MonotonicallyMappableToU64}; + +use super::date_time_options::DATE_TIME_PRECISION_INDEXED; +use super::Field; +use crate::fastfield::FastValue; +use crate::schema::Type; +use crate::DateTime; + +/// IndexingTerm represents is the serialized information of a term during indexing. +/// It's a serialized representation over different types. +/// +/// It actually wraps a `Vec`. +/// +/// The format is as follow: +/// `[field id: u32][serialized value]` +/// +/// For JSON it equals to: +/// `[field id: u32][path id: u32][type code: u8][serialized value]` +/// +/// The format is chosen to easily partition the terms by field during serialization, as all terms +/// are stored in one hashmap. +#[derive(Clone)] +pub(crate) struct IndexingTerm(Vec); + +/// The number of bytes used as for the field id by `Term`. +const FIELD_ID_LENGTH: usize = 4; + +impl IndexingTerm { + /// Create a new IndexingTerm. + pub fn new() -> IndexingTerm { + let mut data = Vec::with_capacity(FIELD_ID_LENGTH + 32); + data.resize(FIELD_ID_LENGTH, 0u8); + IndexingTerm(data) + } + + /// Is empty if there are no value bytes. + pub fn is_empty(&self) -> bool { + self.0.len() == FIELD_ID_LENGTH + } + + /// Removes the value_bytes and set the field + pub(crate) fn clear_with_field(&mut self, field: Field) { + self.truncate_value_bytes(0); + self.0[0..4].clone_from_slice(field.field_id().to_be_bytes().as_ref()); + } + + /// Sets a u64 value in the term. + /// + /// U64 are serialized using (8-byte) BigEndian + /// representation. + /// The use of BigEndian has the benefit of preserving + /// the natural order of the values. + pub fn set_u64(&mut self, val: u64) { + self.set_fast_value(val); + } + + /// Sets a `DateTime` value in the term. + pub fn set_date(&mut self, val: DateTime) { + self.set_fast_value(val); + } + + /// Sets a `i64` value in the term. + pub fn set_i64(&mut self, val: i64) { + self.set_fast_value(val); + } + + /// Sets a `f64` value in the term. + pub fn set_f64(&mut self, val: f64) { + self.set_fast_value(val); + } + + /// Sets a `bool` value in the term. + pub fn set_bool(&mut self, val: bool) { + self.set_fast_value(val); + } + + fn set_fast_value(&mut self, val: T) { + self.truncate_value_bytes(0); + self.append_fast_value(val); + } + + /// Sets a `Ipv6Addr` value in the term. + pub fn set_ip_addr(&mut self, val: Ipv6Addr) { + self.set_value_bytes(val.to_u128().to_be_bytes().as_ref()); + } + + /// Sets the value bytes of the term. + pub fn set_value_bytes(&mut self, bytes: &[u8]) { + self.truncate_value_bytes(0); + self.0.extend(bytes); + } + + /// Append a type marker + fast value to a term. + /// This is used in JSON type to append a fast value after the path. + /// + /// It will not clear existing bytes. + pub(crate) fn append_type_and_fast_value(&mut self, val: T) { + self.0.push(T::to_type().to_code()); + self.append_fast_value(val) + } + + /// Append a fast value to a term. + /// + /// It will not clear existing bytes. + pub fn append_fast_value(&mut self, val: T) { + let value = if T::to_type() == Type::Date { + DateTime::from_u64(val.to_u64()) + .truncate(DATE_TIME_PRECISION_INDEXED) + .to_u64() + } else { + val.to_u64() + }; + self.0.extend(value.to_be_bytes().as_ref()); + } + + /// Truncates the value bytes of the term. Value and field type stays the same. + pub fn truncate_value_bytes(&mut self, len: usize) { + self.0.truncate(len + FIELD_ID_LENGTH); + } + + /// The length of the bytes. + pub fn len_bytes(&self) -> usize { + self.0.len() - FIELD_ID_LENGTH + } + + /// Appends bytes to the Term. + /// + /// This function returns the segment that has just been added. + #[inline] + pub fn append_bytes(&mut self, bytes: &[u8]) { + self.0.extend_from_slice(bytes); + } + + /// Returns the serialized representation of Term. + /// This includes field_id, value bytes + #[inline] + pub fn serialized_for_hashmap(&self) -> &[u8] { + self.0.as_ref() + } +} + +pub fn get_field_from_indexing_term(bytes: &[u8]) -> Field { + let field_id_bytes: [u8; 4] = bytes[..4].try_into().unwrap(); + Field::from_field_id(u32::from_be_bytes(field_id_bytes)) +} diff --git a/src/schema/mod.rs b/src/schema/mod.rs index b4c3b037e9..56ceb86b71 100644 --- a/src/schema/mod.rs +++ b/src/schema/mod.rs @@ -109,6 +109,7 @@ pub mod document; mod facet; mod facet_options; +pub(crate) mod indexing_term; mod schema; pub(crate) mod term; diff --git a/src/schema/term.rs b/src/schema/term.rs index 3ac5d0ac4b..27afae36d8 100644 --- a/src/schema/term.rs +++ b/src/schema/term.rs @@ -1,4 +1,4 @@ -use std::hash::{Hash, Hasher}; +use std::hash::Hash; use std::net::Ipv6Addr; use std::{fmt, str}; @@ -18,37 +18,45 @@ use crate::DateTime; /// 4 bytes are the field id, and the last byte is the type. /// /// The serialized value `ValueBytes` is considered everything after the 4 first bytes (term id). -#[derive(Clone)] -pub struct Term>(B) -where B: AsRef<[u8]>; +#[derive(Clone, Hash, PartialEq, Ord, PartialOrd, Eq)] +pub struct Term(Vec); +impl Default for Term { + fn default() -> Self { + Self::new() + } +} /// The number of bytes used as metadata by `Term`. const TERM_METADATA_LENGTH: usize = 5; impl Term { - /// Create a new Term with a buffer with a given capacity. - pub fn with_capacity(capacity: usize) -> Term { - let mut data = Vec::with_capacity(TERM_METADATA_LENGTH + capacity); + /// Create a new Term + pub fn new() -> Term { + let mut data = Vec::with_capacity(TERM_METADATA_LENGTH + 32); data.resize(TERM_METADATA_LENGTH, 0u8); Term(data) } pub(crate) fn with_type_and_field(typ: Type, field: Field) -> Term { - let mut term = Self::with_capacity(8); - term.set_field_and_type(field, typ); - term + Self::with_bytes_and_field_and_payload(typ, field, &[]) } fn with_bytes_and_field_and_payload(typ: Type, field: Field, bytes: &[u8]) -> Term { - let mut term = Self::with_capacity(bytes.len()); + let mut term = Self::new(); term.set_field_and_type(field, typ); term.0.extend_from_slice(bytes); term } + /// Sets a fast value in the term. + /// + /// fast values are converted to u64 and then serialized using (8-byte) BigEndian + /// representation. + /// The use of BigEndian has the benefit of preserving + /// the natural order of the values. fn from_fast_value(field: Field, val: &T) -> Term { let mut term = Self::with_type_and_field(T::to_type(), field); - term.set_u64(val.to_u64()); + term.set_bytes(val.to_u64().to_be_bytes().as_ref()); term } @@ -70,7 +78,7 @@ impl Term { /// Builds a term given a field, and a `Ipv6Addr`-value pub fn from_field_ip_addr(field: Field, ip_addr: Ipv6Addr) -> Term { let mut term = Self::with_type_and_field(Type::IpAddr, field); - term.set_ip_addr(ip_addr); + term.set_bytes(ip_addr.to_u128().to_be_bytes().as_ref()); term } @@ -115,52 +123,12 @@ impl Term { Term::with_bytes_and_field_and_payload(Type::Bytes, field, bytes) } - /// Removes the value_bytes and set the field and type code. - pub(crate) fn clear_with_field_and_type(&mut self, typ: Type, field: Field) { - self.truncate_value_bytes(0); - self.set_field_and_type(field, typ); - } - /// Removes the value_bytes and set the type code. pub fn clear_with_type(&mut self, typ: Type) { self.truncate_value_bytes(0); self.0[4] = typ.to_code(); } - /// Sets a u64 value in the term. - /// - /// U64 are serialized using (8-byte) BigEndian - /// representation. - /// The use of BigEndian has the benefit of preserving - /// the natural order of the values. - pub fn set_u64(&mut self, val: u64) { - self.set_fast_value(val); - } - - /// Sets a `i64` value in the term. - pub fn set_i64(&mut self, val: i64) { - self.set_fast_value(val); - } - - /// Sets a `DateTime` value in the term. - pub fn set_date(&mut self, date: DateTime) { - self.set_fast_value(date); - } - - /// Sets a `f64` value in the term. - pub fn set_f64(&mut self, val: f64) { - self.set_fast_value(val); - } - - /// Sets a `bool` value in the term. - pub fn set_bool(&mut self, val: bool) { - self.set_fast_value(val); - } - - fn set_fast_value(&mut self, val: T) { - self.set_bytes(val.to_u64().to_be_bytes().as_ref()); - } - /// Append a type marker + fast value to a term. /// This is used in JSON type to append a fast value after the path. /// @@ -186,13 +154,8 @@ impl Term { self.0.extend(val.as_bytes().as_ref()); } - /// Sets a `Ipv6Addr` value in the term. - pub fn set_ip_addr(&mut self, val: Ipv6Addr) { - self.set_bytes(val.to_u128().to_be_bytes().as_ref()); - } - /// Sets the value of a `Bytes` field. - pub fn set_bytes(&mut self, bytes: &[u8]) { + fn set_bytes(&mut self, bytes: &[u8]) { self.truncate_value_bytes(0); self.0.extend(bytes); } @@ -202,11 +165,6 @@ impl Term { self.0.truncate(len + TERM_METADATA_LENGTH); } - /// The length of the bytes. - pub fn len_bytes(&self) -> usize { - self.0.len() - TERM_METADATA_LENGTH - } - /// Appends value bytes to the Term. /// /// This function returns the segment that has just been added. @@ -217,57 +175,19 @@ impl Term { &mut self.0[len_before..] } - /// Appends json path bytes to the Term. - /// If the path contains 0 bytes, they are replaced by a "0" string. - /// The 0 byte is used to mark the end of the path. - /// - /// This function returns the segment that has just been added. - #[inline] - pub fn append_path(&mut self, bytes: &[u8]) -> &mut [u8] { - let len_before = self.0.len(); - if bytes.contains(&0u8) { - self.0 - .extend(bytes.iter().map(|&b| if b == 0 { b'0' } else { b })); - } else { - self.0.extend_from_slice(bytes); - } - &mut self.0[len_before..] - } -} - -impl Term -where B: AsRef<[u8]> -{ - /// Wraps a object holding bytes - pub fn wrap(data: B) -> Term { - Term(data) - } - /// Return the type of the term. pub fn typ(&self) -> Type { self.value().typ() } - /// Returns the field. - pub fn field(&self) -> Field { - let field_id_bytes: [u8; 4] = (&self.0.as_ref()[..4]).try_into().unwrap(); - Field::from_field_id(u32::from_be_bytes(field_id_bytes)) - } - /// Returns the serialized representation of the value. /// (this does neither include the field id nor the value type.) /// /// If the term is a string, its value is utf-8 encoded. /// If the term is a u64, its value is encoded according /// to `byteorder::BigEndian`. - pub fn serialized_value_bytes(&self) -> &[u8] { - &self.0.as_ref()[TERM_METADATA_LENGTH..] - } - - /// Returns the value of the term. - /// address or JSON path + value. (this does not include the field.) - pub fn value(&self) -> ValueBytes<&[u8]> { - ValueBytes::wrap(&self.0.as_ref()[4..]) + pub(crate) fn serialized_value_bytes(&self) -> &[u8] { + &self.0[TERM_METADATA_LENGTH..] } /// Returns the serialized representation of Term. @@ -276,9 +196,22 @@ where B: AsRef<[u8]> /// Do NOT rely on this byte representation in the index. /// This value is likely to change in the future. #[inline] + #[cfg(test)] pub fn serialized_term(&self) -> &[u8] { self.0.as_ref() } + + /// Returns the field. + pub fn field(&self) -> Field { + let field_id_bytes: [u8; 4] = (&self.0[..4]).try_into().unwrap(); + Field::from_field_id(u32::from_be_bytes(field_id_bytes)) + } + + /// Returns the value of the term. + /// address or JSON path + value. (this does not include the field.) + pub fn value(&self) -> ValueBytes<&[u8]> { + ValueBytes::wrap(&self.0[4..]) + } } /// ValueBytes represents a serialized value. @@ -309,18 +242,10 @@ where B: AsRef<[u8]> } /// Return the type of the term. - pub fn typ(&self) -> Type { + pub(crate) fn typ(&self) -> Type { Type::from_code(self.typ_code()).expect("The term has an invalid type code") } - /// Returns the `u64` value stored in a term. - /// - /// Returns `None` if the term is not of the u64 type, or if the term byte representation - /// is invalid. - pub fn as_u64(&self) -> Option { - self.get_fast_type::() - } - fn get_fast_type(&self) -> Option { if self.typ() != T::to_type() { return None; @@ -330,38 +255,6 @@ where B: AsRef<[u8]> Some(T::from_u64(value_u64)) } - /// Returns the `i64` value stored in a term. - /// - /// Returns `None` if the term is not of the i64 type, or if the term byte representation - /// is invalid. - pub fn as_i64(&self) -> Option { - self.get_fast_type::() - } - - /// Returns the `f64` value stored in a term. - /// - /// Returns `None` if the term is not of the f64 type, or if the term byte representation - /// is invalid. - pub fn as_f64(&self) -> Option { - self.get_fast_type::() - } - - /// Returns the `bool` value stored in a term. - /// - /// Returns `None` if the term is not of the bool type, or if the term byte representation - /// is invalid. - pub fn as_bool(&self) -> Option { - self.get_fast_type::() - } - - /// Returns the `Date` value stored in a term. - /// - /// Returns `None` if the term is not of the Date type, or if the term byte representation - /// is invalid. - pub fn as_date(&self) -> Option { - self.get_fast_type::() - } - /// Returns the text associated with the term. /// /// Returns `None` if the field is not of string type @@ -377,7 +270,7 @@ where B: AsRef<[u8]> /// /// Returns `None` if the field is not of facet type /// or if the bytes are not valid utf-8. - pub fn as_facet(&self) -> Option { + pub(crate) fn as_facet(&self) -> Option { if self.typ() != Type::Facet { return None; } @@ -388,7 +281,7 @@ where B: AsRef<[u8]> /// Returns the bytes associated with the term. /// /// Returns `None` if the field is not of bytes type. - pub fn as_bytes(&self) -> Option<&[u8]> { + pub(crate) fn as_bytes(&self) -> Option<&[u8]> { if self.typ() != Type::Bytes { return None; } @@ -396,7 +289,7 @@ where B: AsRef<[u8]> } /// Returns a `Ipv6Addr` value from the term. - pub fn as_ip_addr(&self) -> Option { + pub(crate) fn as_ip_addr(&self) -> Option { if self.typ() != Type::IpAddr { return None; } @@ -404,15 +297,6 @@ where B: AsRef<[u8]> Some(Ipv6Addr::from_u128(ip_u128)) } - /// Returns the json path type. - /// - /// Returns `None` if the value is not JSON. - pub fn json_path_type(&self) -> Option { - let json_value_bytes = self.as_json_value_bytes()?; - - Some(json_value_bytes.typ()) - } - /// Returns the json path bytes (including the JSON_END_OF_PATH byte), /// and the encoded ValueBytes after the json path. /// @@ -429,18 +313,6 @@ where B: AsRef<[u8]> Some((json_path_bytes, ValueBytes::wrap(term))) } - /// Returns the encoded ValueBytes after the json path. - /// - /// Returns `None` if the value is not JSON. - pub(crate) fn as_json_value_bytes(&self) -> Option> { - if self.typ() != Type::Json { - return None; - } - let bytes = self.value_bytes(); - let pos = bytes.iter().cloned().position(|b| b == JSON_END_OF_PATH)?; - Some(ValueBytes::wrap(&bytes[pos + 1..])) - } - /// Returns the serialized value of ValueBytes without the type. fn value_bytes(&self) -> &[u8] { &self.0.as_ref()[1..] @@ -463,20 +335,20 @@ where B: AsRef<[u8]> write_opt(f, s)?; } Type::U64 => { - write_opt(f, self.as_u64())?; + write_opt(f, self.get_fast_type::())?; } Type::I64 => { - write_opt(f, self.as_i64())?; + write_opt(f, self.get_fast_type::())?; } Type::F64 => { - write_opt(f, self.as_f64())?; + write_opt(f, self.get_fast_type::())?; } Type::Bool => { - write_opt(f, self.as_bool())?; + write_opt(f, self.get_fast_type::())?; } // TODO pretty print these types too. Type::Date => { - write_opt(f, self.as_date())?; + write_opt(f, self.get_fast_type::())?; } Type::Facet => { write_opt(f, self.as_facet())?; @@ -502,40 +374,6 @@ where B: AsRef<[u8]> } } -impl Ord for Term -where B: AsRef<[u8]> -{ - fn cmp(&self, other: &Self) -> std::cmp::Ordering { - self.serialized_term().cmp(other.serialized_term()) - } -} - -impl PartialOrd for Term -where B: AsRef<[u8]> -{ - fn partial_cmp(&self, other: &Self) -> Option { - Some(self.cmp(other)) - } -} - -impl PartialEq for Term -where B: AsRef<[u8]> -{ - fn eq(&self, other: &Self) -> bool { - self.serialized_term() == other.serialized_term() - } -} - -impl Eq for Term where B: AsRef<[u8]> {} - -impl Hash for Term -where B: AsRef<[u8]> -{ - fn hash(&self, state: &mut H) { - self.0.as_ref().hash(state) - } -} - fn write_opt(f: &mut fmt::Formatter, val_opt: Option) -> fmt::Result { if let Some(val) = val_opt { write!(f, "{val:?}")?; @@ -543,13 +381,11 @@ fn write_opt(f: &mut fmt::Formatter, val_opt: Option) -> Ok(()) } -impl fmt::Debug for Term -where B: AsRef<[u8]> -{ +impl fmt::Debug for Term { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { let field_id = self.field().field_id(); write!(f, "Term(field={field_id}, ")?; - let value_bytes = ValueBytes::wrap(&self.0.as_ref()[4..]); + let value_bytes = ValueBytes::wrap(&self.0[4..]); value_bytes.debug_value_bytes(f)?; write!(f, ")",)?; Ok(()) @@ -571,38 +407,4 @@ mod tests { assert_eq!(term.typ(), Type::Str); assert_eq!(term.value().as_str(), Some("test")) } - - /// Size (in bytes) of the buffer of a fast value (u64, i64, f64, or date) term. - /// + + - /// - /// - is a big endian encoded u32 field id - /// - 's most significant bit expresses whether the term is a json term or not - /// The remaining 7 bits are used to encode the type of the value. - /// If this is a JSON term, the type is the type of the leaf of the json. - /// - /// - is, if this is not the json term, a binary representation specific to the type. - /// If it is a JSON Term, then it is prepended with the path that leads to this leaf value. - const FAST_VALUE_TERM_LEN: usize = 4 + 1 + 8; - - #[test] - pub fn test_term_u64() { - let mut schema_builder = Schema::builder(); - let count_field = schema_builder.add_u64_field("count", INDEXED); - let term = Term::from_field_u64(count_field, 983u64); - assert_eq!(term.field(), count_field); - assert_eq!(term.typ(), Type::U64); - assert_eq!(term.serialized_term().len(), FAST_VALUE_TERM_LEN); - assert_eq!(term.value().as_u64(), Some(983u64)) - } - - #[test] - pub fn test_term_bool() { - let mut schema_builder = Schema::builder(); - let bool_field = schema_builder.add_bool_field("bool", INDEXED); - let term = Term::from_field_bool(bool_field, true); - assert_eq!(term.field(), bool_field); - assert_eq!(term.typ(), Type::Bool); - assert_eq!(term.serialized_term().len(), FAST_VALUE_TERM_LEN); - assert_eq!(term.value().as_bool(), Some(true)) - } }