feat: encode and decode attributes

src/rust/src/bin/write.rs

@@ -37,7 +37,7 @@ fn write_file() -> Result<(), Box<dyn Error>> {
                 }
             }
         }
-        let mut fcb = FcbWriter::new(cj, header_options, features.first(), Some(&attr_schema))?;
+        let mut fcb = FcbWriter::new(cj, header_options, features.first(), Some(attr_schema))?;
         fcb.write_feature()?;
         for feature in features.iter().skip(1) {
             fcb.add_feature(feature)?;

src/rust/src/cj_utils.rs

@@ -44,21 +44,20 @@ impl CityJSONReader for &str {
 fn parse_cityjson<T: CityJSONReader>(mut source: T, cj_type: CJTypeKind) -> Result<CJType> {
     let mut lines = source.read_lines();
 
-    let first_line = lines.next().ok_or_else(|| anyhow!("Empty input"))??;
-
-    let mut cjj: CityJSON = serde_json::from_str(&first_line)?;
-
     match cj_type {
         CJTypeKind::Normal => {
-            for line in lines {
-                let mut feature: CityJSONFeature = serde_json::from_str(&line?)?;
-                cjj.add_cjfeature(&mut feature);
-            }
-            cjj.remove_duplicate_vertices();
-            Ok(CJType::Normal(cjj))
+            let content = lines.collect::<Result<Vec<_>>>()?.join("\n");
+
+            let cj: CityJSON = serde_json::from_str(&content)?;
+            Ok(CJType::Normal(cj))
         }
 
         CJTypeKind::Seq => {
+            // Read first line as CityJSON metadata
+            let first_line = lines.next().ok_or_else(|| anyhow!("Empty input"))??;
+            let cj: CityJSON = serde_json::from_str(&first_line)?;
+
+            // Read remaining lines as CityJSONFeatures
             let features: Result<Vec<_>> = lines
                 .map(|line| -> Result<_> {
                     let line = line?;
@@ -67,7 +66,7 @@ fn parse_cityjson<T: CityJSONReader>(mut source: T, cj_type: CJTypeKind) -> Resu
                 .collect();
 
             Ok(CJType::Seq(CityJSONSeq {
-                cj: cjj,
+                cj,
                 features: features?,
             }))
         }
@@ -89,22 +88,18 @@ pub fn read_cityjson_from_reader<R: Read>(
 
 #[cfg(test)]
 mod tests {
+    use std::{fs::File, path::PathBuf};
+
     use super::*;
-    use std::io::Cursor;
 
     #[test]
     fn test_read_from_memory() -> Result<()> {
-        let data = r#"{"type":"CityJSON","version":"1.1"}
-{"type":"CityJSONFeature","id":"feature1"}
-{"type":"CityJSONFeature","id":"feature2"}"#;
-
-        let reader = BufReader::new(Cursor::new(data));
+        let input_file =
+            PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/data/small.city.jsonl");
+        let reader = BufReader::new(File::open(input_file)?);
         let result = read_cityjson_from_reader(reader, CJTypeKind::Seq)?;
-
         if let CJType::Seq(seq) = result {
-            assert_eq!(seq.features.len(), 2);
-            assert_eq!(seq.features[0].id, "feature1");
-            assert_eq!(seq.features[1].id, "feature2");
+            assert_eq!(seq.features.len(), 3);
         } else {
             panic!("Expected Seq type");
         }

src/rust/src/fcb_serde/fcb_deserializer.rs

@@ -140,6 +140,10 @@ pub fn decode_attributes(
     columns: flatbuffers::Vector<'_, flatbuffers::ForwardsUOffset<Column<'_>>>,
     attributes: flatbuffers::Vector<'_, u8>,
 ) -> serde_json::Value {
+    if attributes.is_empty() {
+        return serde_json::Value::Object(serde_json::Map::new());
+    }
+
     let mut map = serde_json::Map::new();
     let bytes = attributes.bytes();
     let mut offset = 0;
@@ -288,14 +292,13 @@ pub fn to_cj_feature(
                         .collect::<Vec<_>>()
                 });
 
-                let mut attributes = None;
-                if root_attr_schema.is_none() && co.columns().is_none() {
-                    attributes = None;
+                let attributes = if root_attr_schema.is_none() && co.columns().is_none() {
+                    None
                 } else {
-                    attributes = co.attributes().map(|a| {
+                    co.attributes().map(|a| {
                         decode_attributes(co.columns().unwrap_or(root_attr_schema.unwrap()), a)
-                    });
-                }
+                    })
+                };
 
                 let children_roles = co
                     .children_roles()

src/rust/src/main.rs

@@ -75,14 +75,21 @@ fn serialize(input: &str, output: &str) -> Result<()> {
     };
 
     let CityJSONSeq { cj, features } = cj_seq;
-    let mut attr_schema = AttributeSchema::new();
-    for feature in features.iter() {
-        for (_, co) in feature.city_objects.iter() {
-            if let Some(attributes) = &co.attributes {
-                attr_schema.add_attributes(attributes);
+    let attr_schema = {
+        let mut schema = AttributeSchema::new();
+        for feature in features.iter() {
+            for (_, co) in feature.city_objects.iter() {
+                if let Some(attributes) = &co.attributes {
+                    schema.add_attributes(attributes);
+                }
             }
         }
-    }
+        if schema.is_empty() {
+            None
+        } else {
+            Some(schema)
+        }
+    };
 
     let header_metadata = HeaderMetadata {
         features_count: features.len() as u64,
@@ -91,16 +98,7 @@ fn serialize(input: &str, output: &str) -> Result<()> {
         write_index: false,
         header_metadata,
     });
-    let mut fcb = FcbWriter::new(
-        cj,
-        header_options,
-        None,
-        if attr_schema.is_empty() {
-            None
-        } else {
-            Some(&attr_schema)
-        },
-    )?;
+    let mut fcb = FcbWriter::new(cj, header_options, None, attr_schema)?;
     fcb.write_feature()?;
 
     for feature in features.iter() {
@@ -125,14 +123,12 @@ fn deserialize(input: &str, output: &str) -> Result<()> {
     // Write header
     writeln!(writer, "{}", serde_json::to_string(&cj)?)?;
 
-    let root_attr_schema = header.columns();
     // Write features
     let feat_count = header.features_count();
     let mut feat_num = 0;
     while let Ok(Some(feat_buf)) = fcb_reader.next() {
-        let feature = feat_buf.cur_feature();
-        let cj_feature = fcb_deserializer::to_cj_feature(feature, None)?;
-        writeln!(writer, "{}", serde_json::to_string(&cj_feature)?)?;
+        let feature = feat_buf.cur_cj_feature()?;
+        writeln!(writer, "{}", serde_json::to_string(&feature)?)?;
 
         feat_num += 1;
         if feat_num >= feat_count {

src/rust/src/reader/mod.rs

@@ -350,7 +350,7 @@ impl<R: Read + Seek> FeatureIter<R, Seekable> {
         to_cj_feature(fcb_feature, root_attr_schema)
     }
 
-    pub fn get_features(&mut self, out: impl Write) -> Result<()> {
+    pub fn get_features(&mut self, _: impl Write) -> Result<()> {
         // println!("get features");
         // let mut count = 0;
 

src/rust/src/writer/attribute.rs

@@ -75,17 +75,28 @@ pub fn attr_size(coltype: &ColumnType, colval: &Value) -> usize {
 }
 
 pub fn encode_attributes_with_schema(attr: &Value, schema: &AttributeSchema) -> Vec<u8> {
+    if !attr.is_object() || attr.as_object().unwrap().is_empty() || attr.is_null() {
+        return Vec::new();
+    }
+
     let mut out = Vec::new();
     let mut sorted_schema: Vec<_> = schema.iter().collect();
     sorted_schema.sort_by_key(|(_, (index, _))| *index);
 
     for (name, (index, coltype)) in sorted_schema {
-        let (_, val) = attr
-            .as_object()
-            .unwrap()
-            .iter()
-            .find(|(k, _)| *k == name)
-            .unwrap();
+        let (_, val) = {
+            let attr_obj = attr.as_object();
+            if let Some(attr_obj) = attr_obj {
+                let value = attr_obj.iter().find(|(k, _)| *k == name);
+                if let Some(value) = value {
+                    (value.0, value.1)
+                } else {
+                    return Vec::new();
+                }
+            } else {
+                return Vec::new();
+            }
+        };
 
         if val.is_null() {
             continue;
@@ -201,12 +212,12 @@ mod tests {
         attr_schema.add_attributes(&json_data["attributes"]);
 
         // Check if the schema contains the expected keys and types
-        assert_eq!(attr_schema.get("int").unwrap().1, ColumnType::Int);
-        assert_eq!(attr_schema.get("uint").unwrap().1, ColumnType::UInt);
+        assert_eq!(attr_schema.get("int").unwrap().1, ColumnType::Long);
+        assert_eq!(attr_schema.get("uint").unwrap().1, ColumnType::ULong);
         assert_eq!(attr_schema.get("bool").unwrap().1, ColumnType::Bool);
-        assert_eq!(attr_schema.get("float").unwrap().1, ColumnType::Float);
+        assert_eq!(attr_schema.get("float").unwrap().1, ColumnType::Double);
         assert_eq!(attr_schema.get("string").unwrap().1, ColumnType::String);
-        assert_eq!(attr_schema.get("array").unwrap().1, ColumnType::Json);
+        assert_eq!(attr_schema.get("array").unwrap().1, ColumnType::Json); //TODO: check if this is correct
         assert_eq!(attr_schema.get("json").unwrap().1, ColumnType::Json);
 
         Ok(())

src/rust/src/writer/mod.rs

@@ -24,7 +24,6 @@ pub struct FcbWriter<'a> {
     header_writer: HeaderWriter<'a>,
     /// Optional writer for features
     feat_writer: Option<FeatureWriter<'a>>,
-
     attr_schema: AttributeSchema,
 }
 
@@ -44,18 +43,17 @@ impl<'a> FcbWriter<'a> {
         cj: CityJSON,
         header_option: Option<HeaderWriterOptions>,
         first_feature: Option<&'a CityJSONFeature>,
-        attr_schema: Option<&AttributeSchema>,
+        attr_schema: Option<AttributeSchema>,
     ) -> Result<Self> {
-        let owned_schema = AttributeSchema::new();
-        let attr_schema = attr_schema.unwrap_or(&owned_schema);
+        let attr_schema = attr_schema.unwrap_or_default();
 
-        let header_writer = HeaderWriter::new(cj, header_option, attr_schema.clone()); // if attr_schema is None, instantiate an empty one
+        let header_writer = HeaderWriter::new(cj, header_option, attr_schema.clone());
         let feat_writer = first_feature.map(|feat| FeatureWriter::new(feat, attr_schema.clone()));
         Ok(Self {
             header_writer,
             feat_writer,
             tmpout: BufWriter::new(tempfile::tempfile()?),
-            attr_schema: attr_schema.clone(),
+            attr_schema,
         })
     }
 
@@ -82,10 +80,15 @@ impl<'a> FcbWriter<'a> {
     ///
     /// A Result indicating success or failure of the operation
     pub fn add_feature(&mut self, feature: &'a CityJSONFeature) -> Result<()> {
+        if self.feat_writer.is_none() {
+            self.feat_writer = Some(FeatureWriter::new(feature, self.attr_schema.clone()));
+        }
+
         if let Some(feat_writer) = &mut self.feat_writer {
             feat_writer.add_feature(feature);
             self.write_feature()?;
         }
+
         Ok(())
     }
 

src/rust/tests/e2e.rs

@@ -52,7 +52,7 @@ fn test_cityjson_serialization_cycle() -> Result<()> {
                 header_metadata,
             }),
             original_cj_seq.features.first(),
-            Some(&attr_schema),
+            Some(attr_schema),
         )?;
         fcb.write_feature()?;
         for feature in original_cj_seq.features.iter().skip(1) {