diff --git a/src/rust/src/bin/write.rs b/src/rust/src/bin/write.rs index 32774a3..689a9b1 100644 --- a/src/rust/src/bin/write.rs +++ b/src/rust/src/bin/write.rs @@ -37,7 +37,7 @@ fn write_file() -> Result<(), Box> { } } } - let mut fcb = FcbWriter::new(cj, header_options, features.first(), Some(&attr_schema))?; + let mut fcb = FcbWriter::new(cj, header_options, features.first(), Some(attr_schema))?; fcb.write_feature()?; for feature in features.iter().skip(1) { fcb.add_feature(feature)?; diff --git a/src/rust/src/cj_utils.rs b/src/rust/src/cj_utils.rs index 3f53525..e2afc56 100644 --- a/src/rust/src/cj_utils.rs +++ b/src/rust/src/cj_utils.rs @@ -44,21 +44,20 @@ impl CityJSONReader for &str { fn parse_cityjson(mut source: T, cj_type: CJTypeKind) -> Result { let mut lines = source.read_lines(); - let first_line = lines.next().ok_or_else(|| anyhow!("Empty input"))??; - - let mut cjj: CityJSON = serde_json::from_str(&first_line)?; - match cj_type { CJTypeKind::Normal => { - for line in lines { - let mut feature: CityJSONFeature = serde_json::from_str(&line?)?; - cjj.add_cjfeature(&mut feature); - } - cjj.remove_duplicate_vertices(); - Ok(CJType::Normal(cjj)) + let content = lines.collect::>>()?.join("\n"); + + let cj: CityJSON = serde_json::from_str(&content)?; + Ok(CJType::Normal(cj)) } CJTypeKind::Seq => { + // Read first line as CityJSON metadata + let first_line = lines.next().ok_or_else(|| anyhow!("Empty input"))??; + let cj: CityJSON = serde_json::from_str(&first_line)?; + + // Read remaining lines as CityJSONFeatures let features: Result> = lines .map(|line| -> Result<_> { let line = line?; @@ -67,7 +66,7 @@ fn parse_cityjson(mut source: T, cj_type: CJTypeKind) -> Resu .collect(); Ok(CJType::Seq(CityJSONSeq { - cj: cjj, + cj, features: features?, })) } @@ -89,22 +88,18 @@ pub fn read_cityjson_from_reader( #[cfg(test)] mod tests { + use std::{fs::File, path::PathBuf}; + use super::*; - use std::io::Cursor; #[test] fn test_read_from_memory() -> Result<()> { - let data = r#"{"type":"CityJSON","version":"1.1"} -{"type":"CityJSONFeature","id":"feature1"} -{"type":"CityJSONFeature","id":"feature2"}"#; - - let reader = BufReader::new(Cursor::new(data)); + let input_file = + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/data/small.city.jsonl"); + let reader = BufReader::new(File::open(input_file)?); let result = read_cityjson_from_reader(reader, CJTypeKind::Seq)?; - if let CJType::Seq(seq) = result { - assert_eq!(seq.features.len(), 2); - assert_eq!(seq.features[0].id, "feature1"); - assert_eq!(seq.features[1].id, "feature2"); + assert_eq!(seq.features.len(), 3); } else { panic!("Expected Seq type"); } diff --git a/src/rust/src/fcb_serde/fcb_deserializer.rs b/src/rust/src/fcb_serde/fcb_deserializer.rs index 87c06b4..5a79101 100644 --- a/src/rust/src/fcb_serde/fcb_deserializer.rs +++ b/src/rust/src/fcb_serde/fcb_deserializer.rs @@ -140,6 +140,10 @@ pub fn decode_attributes( columns: flatbuffers::Vector<'_, flatbuffers::ForwardsUOffset>>, attributes: flatbuffers::Vector<'_, u8>, ) -> serde_json::Value { + if attributes.is_empty() { + return serde_json::Value::Object(serde_json::Map::new()); + } + let mut map = serde_json::Map::new(); let bytes = attributes.bytes(); let mut offset = 0; @@ -288,14 +292,13 @@ pub fn to_cj_feature( .collect::>() }); - let mut attributes = None; - if root_attr_schema.is_none() && co.columns().is_none() { - attributes = None; + let attributes = if root_attr_schema.is_none() && co.columns().is_none() { + None } else { - attributes = co.attributes().map(|a| { + co.attributes().map(|a| { decode_attributes(co.columns().unwrap_or(root_attr_schema.unwrap()), a) - }); - } + }) + }; let children_roles = co .children_roles() diff --git a/src/rust/src/main.rs b/src/rust/src/main.rs index 4f95bdd..b8ef082 100644 --- a/src/rust/src/main.rs +++ b/src/rust/src/main.rs @@ -75,14 +75,21 @@ fn serialize(input: &str, output: &str) -> Result<()> { }; let CityJSONSeq { cj, features } = cj_seq; - let mut attr_schema = AttributeSchema::new(); - for feature in features.iter() { - for (_, co) in feature.city_objects.iter() { - if let Some(attributes) = &co.attributes { - attr_schema.add_attributes(attributes); + let attr_schema = { + let mut schema = AttributeSchema::new(); + for feature in features.iter() { + for (_, co) in feature.city_objects.iter() { + if let Some(attributes) = &co.attributes { + schema.add_attributes(attributes); + } } } - } + if schema.is_empty() { + None + } else { + Some(schema) + } + }; let header_metadata = HeaderMetadata { features_count: features.len() as u64, @@ -91,16 +98,7 @@ fn serialize(input: &str, output: &str) -> Result<()> { write_index: false, header_metadata, }); - let mut fcb = FcbWriter::new( - cj, - header_options, - None, - if attr_schema.is_empty() { - None - } else { - Some(&attr_schema) - }, - )?; + let mut fcb = FcbWriter::new(cj, header_options, None, attr_schema)?; fcb.write_feature()?; for feature in features.iter() { @@ -125,14 +123,12 @@ fn deserialize(input: &str, output: &str) -> Result<()> { // Write header writeln!(writer, "{}", serde_json::to_string(&cj)?)?; - let root_attr_schema = header.columns(); // Write features let feat_count = header.features_count(); let mut feat_num = 0; while let Ok(Some(feat_buf)) = fcb_reader.next() { - let feature = feat_buf.cur_feature(); - let cj_feature = fcb_deserializer::to_cj_feature(feature, None)?; - writeln!(writer, "{}", serde_json::to_string(&cj_feature)?)?; + let feature = feat_buf.cur_cj_feature()?; + writeln!(writer, "{}", serde_json::to_string(&feature)?)?; feat_num += 1; if feat_num >= feat_count { diff --git a/src/rust/src/reader/mod.rs b/src/rust/src/reader/mod.rs index a7f827d..7d10b12 100644 --- a/src/rust/src/reader/mod.rs +++ b/src/rust/src/reader/mod.rs @@ -350,7 +350,7 @@ impl FeatureIter { to_cj_feature(fcb_feature, root_attr_schema) } - pub fn get_features(&mut self, out: impl Write) -> Result<()> { + pub fn get_features(&mut self, _: impl Write) -> Result<()> { // println!("get features"); // let mut count = 0; diff --git a/src/rust/src/writer/attribute.rs b/src/rust/src/writer/attribute.rs index c61e968..0b7ec61 100644 --- a/src/rust/src/writer/attribute.rs +++ b/src/rust/src/writer/attribute.rs @@ -75,17 +75,28 @@ pub fn attr_size(coltype: &ColumnType, colval: &Value) -> usize { } pub fn encode_attributes_with_schema(attr: &Value, schema: &AttributeSchema) -> Vec { + if !attr.is_object() || attr.as_object().unwrap().is_empty() || attr.is_null() { + return Vec::new(); + } + let mut out = Vec::new(); let mut sorted_schema: Vec<_> = schema.iter().collect(); sorted_schema.sort_by_key(|(_, (index, _))| *index); for (name, (index, coltype)) in sorted_schema { - let (_, val) = attr - .as_object() - .unwrap() - .iter() - .find(|(k, _)| *k == name) - .unwrap(); + let (_, val) = { + let attr_obj = attr.as_object(); + if let Some(attr_obj) = attr_obj { + let value = attr_obj.iter().find(|(k, _)| *k == name); + if let Some(value) = value { + (value.0, value.1) + } else { + return Vec::new(); + } + } else { + return Vec::new(); + } + }; if val.is_null() { continue; @@ -201,12 +212,12 @@ mod tests { attr_schema.add_attributes(&json_data["attributes"]); // Check if the schema contains the expected keys and types - assert_eq!(attr_schema.get("int").unwrap().1, ColumnType::Int); - assert_eq!(attr_schema.get("uint").unwrap().1, ColumnType::UInt); + assert_eq!(attr_schema.get("int").unwrap().1, ColumnType::Long); + assert_eq!(attr_schema.get("uint").unwrap().1, ColumnType::ULong); assert_eq!(attr_schema.get("bool").unwrap().1, ColumnType::Bool); - assert_eq!(attr_schema.get("float").unwrap().1, ColumnType::Float); + assert_eq!(attr_schema.get("float").unwrap().1, ColumnType::Double); assert_eq!(attr_schema.get("string").unwrap().1, ColumnType::String); - assert_eq!(attr_schema.get("array").unwrap().1, ColumnType::Json); + assert_eq!(attr_schema.get("array").unwrap().1, ColumnType::Json); //TODO: check if this is correct assert_eq!(attr_schema.get("json").unwrap().1, ColumnType::Json); Ok(()) diff --git a/src/rust/src/writer/mod.rs b/src/rust/src/writer/mod.rs index 3341cf3..fad4d17 100644 --- a/src/rust/src/writer/mod.rs +++ b/src/rust/src/writer/mod.rs @@ -24,7 +24,6 @@ pub struct FcbWriter<'a> { header_writer: HeaderWriter<'a>, /// Optional writer for features feat_writer: Option>, - attr_schema: AttributeSchema, } @@ -44,18 +43,17 @@ impl<'a> FcbWriter<'a> { cj: CityJSON, header_option: Option, first_feature: Option<&'a CityJSONFeature>, - attr_schema: Option<&AttributeSchema>, + attr_schema: Option, ) -> Result { - let owned_schema = AttributeSchema::new(); - let attr_schema = attr_schema.unwrap_or(&owned_schema); + let attr_schema = attr_schema.unwrap_or_default(); - let header_writer = HeaderWriter::new(cj, header_option, attr_schema.clone()); // if attr_schema is None, instantiate an empty one + let header_writer = HeaderWriter::new(cj, header_option, attr_schema.clone()); let feat_writer = first_feature.map(|feat| FeatureWriter::new(feat, attr_schema.clone())); Ok(Self { header_writer, feat_writer, tmpout: BufWriter::new(tempfile::tempfile()?), - attr_schema: attr_schema.clone(), + attr_schema, }) } @@ -82,10 +80,15 @@ impl<'a> FcbWriter<'a> { /// /// A Result indicating success or failure of the operation pub fn add_feature(&mut self, feature: &'a CityJSONFeature) -> Result<()> { + if self.feat_writer.is_none() { + self.feat_writer = Some(FeatureWriter::new(feature, self.attr_schema.clone())); + } + if let Some(feat_writer) = &mut self.feat_writer { feat_writer.add_feature(feature); self.write_feature()?; } + Ok(()) } diff --git a/src/rust/tests/e2e.rs b/src/rust/tests/e2e.rs index 29a301d..070c14f 100644 --- a/src/rust/tests/e2e.rs +++ b/src/rust/tests/e2e.rs @@ -52,7 +52,7 @@ fn test_cityjson_serialization_cycle() -> Result<()> { header_metadata, }), original_cj_seq.features.first(), - Some(&attr_schema), + Some(attr_schema), )?; fcb.write_feature()?; for feature in original_cj_seq.features.iter().skip(1) { diff --git a/src/rust/tests/serde.rs b/src/rust/tests/serde.rs index 032653d..1391f45 100644 --- a/src/rust/tests/serde.rs +++ b/src/rust/tests/serde.rs @@ -11,39 +11,99 @@ use serde_json::json; #[test] fn test_attribute_serialization() -> Result<()> { - let json_data = json!({ - "attributes": { - "int": -10, - "uint": 5, - "bool": true, - "float": 1.0, - "string": "hoge", - "array": [1, 2, 3], - "json": { - "hoge": "fuga" - }, - "exceptional": null - } - }); - let schema = json!({ - "attributes": { - "int": -10, - "uint": 5, - "bool": true, - "float": 1.0, - "string": "hoge", - "array": [1, 2, 3], - "json": { - "hoge": "fuga" - }, - "exceptional": 1000 - } - }); - let attrs = &json_data["attributes"]; - let attr_schema = &schema["attributes"]; + let test_cases = vec![ + // Case 1: Same schema + ( + json!({ + "attributes": { + "int": -10, + "uint": 5, + "bool": true, + "float": 1.0, + "string": "hoge", + "array": [1, 2, 3], + "json": { + "hoge": "fuga" + }, + } + }), + json!({ + "attributes": { + "int": -10, + "uint": 5, + "bool": true, + "float": 1.0, + "string": "hoge", + "array": [1, 2, 3], + "json": { + "hoge": "fuga" + }, + } + }), + "same schema", + ), + // Case 2: JSON with null value + ( + json!({ + "attributes": { + "int": -10, + "uint": 5, + "bool": true, + "float": 1.0, + "string": "hoge", + "array": [1, 2, 3], + "json": { + "hoge": "fuga" + }, + "exception": null + } + }), + json!({ + "attributes": { + "int": -10, + "uint": 5, + "bool": true, + "float": 1.0, + "string": "hoge", + "array": [1, 2, 3], + "json": { + "hoge": "fuga" + }, + "exception": 1000 + } + }), + "JSON with null value", + ), + // Case 3: JSON is empty + ( + json!({ + "attributes": {} + }), + json!({ + "attributes": { + "int": -10, + "uint": 5, + "bool": true, + "float": 1.0, + "string": "hoge", + "array": [1, 2, 3], + "json": { + "hoge": "fuga" + }, + "exception": 1000 + } + }), + "JSON is empty", + ), + ]; - // Test case 1: Using common schema - { + for (json_data, schema, test_name) in test_cases { + println!("Testing case: {}", test_name); + + let attrs = &json_data["attributes"]; + let attr_schema = &schema["attributes"]; + + // Create and encode with schema let mut fbb = FlatBufferBuilder::new(); let mut common_schema = AttributeSchema::new(); common_schema.add_attributes(attr_schema); @@ -61,13 +121,16 @@ fn test_attribute_serialization() -> Result<()> { ) }; fbb.finish(header, None); + + // Decode and verify let finished_data = fbb.finished_data(); let header_buf = root_as_header(finished_data).unwrap(); + let mut fbb = FlatBufferBuilder::new(); let feature = { let (attr_buf, _) = to_fcb_attribute(&mut fbb, attrs, &common_schema); let city_object = { - let id = fbb.create_string("hoge"); + let id = fbb.create_string("test"); CityObject::create( &mut fbb, &CityObjectArgs { @@ -78,7 +141,7 @@ fn test_attribute_serialization() -> Result<()> { ) }; let objects = fbb.create_vector(&[city_object]); - let cf_id = fbb.create_string("hoge"); + let cf_id = fbb.create_string("test_feature"); CityFeature::create( &mut fbb, &CityFeatureArgs { @@ -94,11 +157,13 @@ fn test_attribute_serialization() -> Result<()> { let finished_data = fbb.finished_data(); let feature_buf = root_as_city_feature(finished_data).unwrap(); let attributes = feature_buf.objects().unwrap().get(0).attributes().unwrap(); - // Verify encoded data - assert!(!attributes.is_empty()); let decoded = decode_attributes(header_buf.columns().unwrap(), attributes); - assert_eq!(attrs, &decoded); + assert_eq!( + attrs, &decoded, + "decoded data should match original for {}", + test_name + ); } Ok(())