From 41946f2dca87aa8937c63a116fabb789ca2d782e Mon Sep 17 00:00:00 2001 From: HideBa Date: Thu, 23 Jan 2025 13:00:16 +0100 Subject: [PATCH] support wasm build --- scripts/gen_rust.sh | 2 +- src/rust/.rust-analyzer | 3 + src/rust/Cargo.toml | 5 +- src/rust/cli/Cargo.toml | 2 +- src/rust/cli/src/main.rs | 1 - src/rust/fcb_core/Cargo.toml | 7 +- src/rust/fcb_core/benches/read.rs | 1 + src/rust/fcb_core/src/bin/write.rs | 1 - src/rust/fcb_core/src/cj_utils.rs | 1 - src/rust/fcb_core/src/const_vars.rs | 3 - src/rust/fcb_core/src/fb/feature_generated.rs | 28 +- src/rust/fcb_core/src/fb/header_generated.rs | 26 +- src/rust/fcb_core/src/lib.rs | 10 +- src/rust/fcb_core/src/reader/deserializer.rs | 2 +- src/rust/fcb_core/src/reader/geom_decoder.rs | 8 +- src/rust/fcb_core/src/reader/mod.rs | 13 +- src/rust/fcb_core/src/writer/attribute.rs | 174 +++++++++- src/rust/fcb_core/src/writer/geom_encoder.rs | 2 +- src/rust/fcb_core/src/writer/header_writer.rs | 6 +- src/rust/fcb_core/src/writer/mod.rs | 4 +- src/rust/fcb_core/src/writer/serializer.rs | 28 +- src/rust/fcb_core/tests/e2e.rs | 5 +- src/rust/fcb_core/tests/http.rs | 16 +- src/rust/fcb_core/tests/read.rs | 11 +- src/rust/fcb_core/tests/serde.rs | 170 ---------- src/rust/makefile | 35 +- src/rust/packed_rtree/Cargo.toml | 1 + src/rust/packed_rtree/src/lib.rs | 320 +++++++----------- src/rust/src/lib.rs | 1 + src/rust/wasm/Cargo.toml | 7 +- src/rust/wasm/src/gloo_client.rs | 4 - src/rust/wasm/src/lib.rs | 115 +------ src/rust/wasm/src/range_client.rs | 39 --- 33 files changed, 432 insertions(+), 619 deletions(-) create mode 100644 src/rust/.rust-analyzer delete mode 100644 src/rust/fcb_core/tests/serde.rs delete mode 100644 src/rust/wasm/src/range_client.rs diff --git a/scripts/gen_rust.sh b/scripts/gen_rust.sh index 8188501..0f8d350 100644 --- a/scripts/gen_rust.sh +++ b/scripts/gen_rust.sh @@ -1,2 +1,2 @@ #!/bin/bash -flatc --rust -o ./src/rust/src src/fbs/*.fbs +flatc --rust -o ./src/rust/fcb_core/src/fb src/fbs/*.fbs diff --git a/src/rust/.rust-analyzer b/src/rust/.rust-analyzer new file mode 100644 index 0000000..c4dc4fb --- /dev/null +++ b/src/rust/.rust-analyzer @@ -0,0 +1,3 @@ +exclude = [ + "src/rust/fcb_core/src/fb/*", +] diff --git a/src/rust/Cargo.toml b/src/rust/Cargo.toml index d8175bf..a2fa3c5 100644 --- a/src/rust/Cargo.toml +++ b/src/rust/Cargo.toml @@ -26,6 +26,10 @@ criterion = { version = "0.5.1", features = ["async_tokio", "html_reports"] } memory-stats = "1.2.0" pretty_assertions = "1.4.1" tokio = { version = "1.43.0", features = ["macros", "rt-multi-thread"] } +rand = "0.8.5" +log = "0.4" + +#---WASM dependencies--- getrandom = { version = "0.2.15", features = ["js"] } gloo-net = "0.6.0" js-sys = "0.3.77" @@ -33,7 +37,6 @@ wasm-bindgen = "0.2" wasm-bindgen-futures = "0.4.50" serde-wasm-bindgen = "0.6.5" console_error_panic_hook = "0.1.7" -log = "0.4" console_log = "0.2" [dependencies] diff --git a/src/rust/cli/Cargo.toml b/src/rust/cli/Cargo.toml index 28989b9..64794c4 100644 --- a/src/rust/cli/Cargo.toml +++ b/src/rust/cli/Cargo.toml @@ -4,7 +4,7 @@ version = "0.1.0" edition = "2021" [dependencies] -fcb_core = { path = "../fcb_core", default-features = false } +fcb_core = { path = "../fcb_core" } clap = { workspace = true } anyhow = { workspace = true } serde = { workspace = true } diff --git a/src/rust/cli/src/main.rs b/src/rust/cli/src/main.rs index 68e29fa..c02fcc9 100644 --- a/src/rust/cli/src/main.rs +++ b/src/rust/cli/src/main.rs @@ -97,7 +97,6 @@ fn serialize(input: &str, output: &str) -> Result<()> { index_node_size: 16, }); let mut fcb = FcbWriter::new(cj, header_options, attr_schema)?; - fcb.write_feature()?; for feature in features.iter() { fcb.add_feature(feature)?; diff --git a/src/rust/fcb_core/Cargo.toml b/src/rust/fcb_core/Cargo.toml index 1f95e04..c39ffbc 100644 --- a/src/rust/fcb_core/Cargo.toml +++ b/src/rust/fcb_core/Cargo.toml @@ -66,8 +66,5 @@ harness = false async-trait = { workspace = true } memory-stats = { workspace = true } pretty_assertions = { workspace = true } - -# [target.'cfg(not(feature = "wasm"))'.dev-dependencies] -# criterion = { workspace = true, features = ["async_tokio", "html_reports"] } -# [target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies] -# tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } +criterion = { workspace = true, features = ["async_tokio", "html_reports"] } +tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } diff --git a/src/rust/fcb_core/benches/read.rs b/src/rust/fcb_core/benches/read.rs index 4b04ef3..423ff1b 100644 --- a/src/rust/fcb_core/benches/read.rs +++ b/src/rust/fcb_core/benches/read.rs @@ -43,6 +43,7 @@ fn read_fcb(path: &str) -> Result<(u64, u64, u64)> { } /// Read FCB file and count geometry types +#[allow(dead_code)] fn read_fcb_as_cj(path: &str) -> Result<(u64, u64, u64)> { let input_file = File::open(path)?; let inputreader = BufReader::new(input_file); diff --git a/src/rust/fcb_core/src/bin/write.rs b/src/rust/fcb_core/src/bin/write.rs index 66d7f5f..6bc7a9f 100644 --- a/src/rust/fcb_core/src/bin/write.rs +++ b/src/rust/fcb_core/src/bin/write.rs @@ -38,7 +38,6 @@ fn write_file() -> Result<(), Box> { } } let mut fcb = FcbWriter::new(cj, header_options, Some(attr_schema))?; - fcb.write_feature()?; for feature in features.iter() { fcb.add_feature(feature)?; } diff --git a/src/rust/fcb_core/src/cj_utils.rs b/src/rust/fcb_core/src/cj_utils.rs index e31fa80..0381bf8 100644 --- a/src/rust/fcb_core/src/cj_utils.rs +++ b/src/rust/fcb_core/src/cj_utils.rs @@ -99,7 +99,6 @@ pub fn read_cityjson_from_reader( /// let test_data = include_str!("../tests/data/small.city.jsonl"); /// let result = read_cityjson(test_data, CJTypeKind::Seq)?; /// ``` - #[cfg(test)] mod tests { use std::{fs::File, path::PathBuf}; diff --git a/src/rust/fcb_core/src/const_vars.rs b/src/rust/fcb_core/src/const_vars.rs index 67eefce..91257d4 100644 --- a/src/rust/fcb_core/src/const_vars.rs +++ b/src/rust/fcb_core/src/const_vars.rs @@ -12,6 +12,3 @@ pub const MAGIC_BYTES_SIZE: usize = 8; // Size of header size pub const HEADER_SIZE_SIZE: usize = 4; - -// // Offset of header size -// pub(crate) const HEADER_SIZE_OFFSET: usize = MAGIC_BYTES_SIZE + HEADER_SIZE_SIZE; diff --git a/src/rust/fcb_core/src/fb/feature_generated.rs b/src/rust/fcb_core/src/fb/feature_generated.rs index 567914d..4619852 100644 --- a/src/rust/fcb_core/src/fb/feature_generated.rs +++ b/src/rust/fcb_core/src/fb/feature_generated.rs @@ -2,9 +2,7 @@ // @generated -use crate::fb::*; -use core::cmp::Ordering; -use core::mem; +use crate::header_generated::*; extern crate flatbuffers; use self::flatbuffers::{EndianScalar, Follow}; @@ -505,12 +503,8 @@ impl flatbuffers::SimpleToVerifyInSlice for GeometryType {} // struct Vertex, aligned to 4 #[repr(transparent)] #[derive(Clone, Copy, PartialEq)] +#[derive(Default)] pub struct Vertex(pub [u8; 12]); -impl Default for Vertex { - fn default() -> Self { - Self([0; 12]) - } -} impl core::fmt::Debug for Vertex { fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { f.debug_struct("Vertex") @@ -551,7 +545,7 @@ impl<'a> flatbuffers::Verifiable for Vertex { v: &mut flatbuffers::Verifier, pos: usize, ) -> Result<(), flatbuffers::InvalidFlatbuffer> { - use self::flatbuffers::Verifiable; + v.in_buffer::(pos) } } @@ -753,7 +747,7 @@ impl flatbuffers::Verifiable for CityFeature<'_> { v: &mut flatbuffers::Verifier, pos: usize, ) -> Result<(), flatbuffers::InvalidFlatbuffer> { - use self::flatbuffers::Verifiable; + v.visit_table(pos)? .visit_field::>("id", Self::VT_ID, true)? .visit_field:: { >, pub vertices: Option>>, } -impl<'a> Default for CityFeatureArgs<'a> { +impl Default for CityFeatureArgs<'_> { #[inline] fn default() -> Self { CityFeatureArgs { @@ -1037,7 +1031,7 @@ impl flatbuffers::Verifiable for CityObject<'_> { v: &mut flatbuffers::Verifier, pos: usize, ) -> Result<(), flatbuffers::InvalidFlatbuffer> { - use self::flatbuffers::Verifiable; + v.visit_table(pos)? .visit_field::("type_", Self::VT_TYPE_, false)? .visit_field::>("id", Self::VT_ID, true)? @@ -1091,7 +1085,7 @@ pub struct CityObjectArgs<'a> { flatbuffers::WIPOffset>>, >, } -impl<'a> Default for CityObjectArgs<'a> { +impl Default for CityObjectArgs<'_> { #[inline] fn default() -> Self { CityObjectArgs { @@ -1409,7 +1403,7 @@ impl flatbuffers::Verifiable for Geometry<'_> { v: &mut flatbuffers::Verifier, pos: usize, ) -> Result<(), flatbuffers::InvalidFlatbuffer> { - use self::flatbuffers::Verifiable; + v.visit_table(pos)? .visit_field::("type_", Self::VT_TYPE_, false)? .visit_field::>("lod", Self::VT_LOD, false)? @@ -1465,7 +1459,7 @@ pub struct GeometryArgs<'a> { >, >, } -impl<'a> Default for GeometryArgs<'a> { +impl Default for GeometryArgs<'_> { #[inline] fn default() -> Self { GeometryArgs { @@ -1676,7 +1670,7 @@ impl flatbuffers::Verifiable for SemanticObject<'_> { v: &mut flatbuffers::Verifier, pos: usize, ) -> Result<(), flatbuffers::InvalidFlatbuffer> { - use self::flatbuffers::Verifiable; + v.visit_table(pos)? .visit_field::("type_", Self::VT_TYPE_, false)? .visit_field::>>( @@ -1700,7 +1694,7 @@ pub struct SemanticObjectArgs<'a> { pub children: Option>>, pub parent: Option, } -impl<'a> Default for SemanticObjectArgs<'a> { +impl Default for SemanticObjectArgs<'_> { #[inline] fn default() -> Self { SemanticObjectArgs { diff --git a/src/rust/fcb_core/src/fb/header_generated.rs b/src/rust/fcb_core/src/fb/header_generated.rs index b276213..adc867d 100644 --- a/src/rust/fcb_core/src/fb/header_generated.rs +++ b/src/rust/fcb_core/src/fb/header_generated.rs @@ -2,8 +2,6 @@ // @generated -use core::cmp::Ordering; -use core::mem; extern crate flatbuffers; use self::flatbuffers::{EndianScalar, Follow}; @@ -158,12 +156,8 @@ impl flatbuffers::SimpleToVerifyInSlice for ColumnType {} // struct Vector, aligned to 8 #[repr(transparent)] #[derive(Clone, Copy, PartialEq)] +#[derive(Default)] pub struct Vector(pub [u8; 24]); -impl Default for Vector { - fn default() -> Self { - Self([0; 24]) - } -} impl core::fmt::Debug for Vector { fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { f.debug_struct("Vector") @@ -204,7 +198,7 @@ impl<'a> flatbuffers::Verifiable for Vector { v: &mut flatbuffers::Verifier, pos: usize, ) -> Result<(), flatbuffers::InvalidFlatbuffer> { - use self::flatbuffers::Verifiable; + v.in_buffer::(pos) } } @@ -356,7 +350,7 @@ impl<'a> flatbuffers::Verifiable for Transform { v: &mut flatbuffers::Verifier, pos: usize, ) -> Result<(), flatbuffers::InvalidFlatbuffer> { - use self::flatbuffers::Verifiable; + v.in_buffer::(pos) } } @@ -446,7 +440,7 @@ impl<'a> flatbuffers::Verifiable for GeographicalExtent { v: &mut flatbuffers::Verifier, pos: usize, ) -> Result<(), flatbuffers::InvalidFlatbuffer> { - use self::flatbuffers::Verifiable; + v.in_buffer::(pos) } } @@ -665,7 +659,7 @@ impl flatbuffers::Verifiable for Column<'_> { v: &mut flatbuffers::Verifier, pos: usize, ) -> Result<(), flatbuffers::InvalidFlatbuffer> { - use self::flatbuffers::Verifiable; + v.visit_table(pos)? .visit_field::("index", Self::VT_INDEX, false)? .visit_field::>("name", Self::VT_NAME, true)? @@ -703,7 +697,7 @@ pub struct ColumnArgs<'a> { pub primary_key: bool, pub metadata: Option>, } -impl<'a> Default for ColumnArgs<'a> { +impl Default for ColumnArgs<'_> { #[inline] fn default() -> Self { ColumnArgs { @@ -907,7 +901,7 @@ impl flatbuffers::Verifiable for ReferenceSystem<'_> { v: &mut flatbuffers::Verifier, pos: usize, ) -> Result<(), flatbuffers::InvalidFlatbuffer> { - use self::flatbuffers::Verifiable; + v.visit_table(pos)? .visit_field::>( "authority", @@ -931,7 +925,7 @@ pub struct ReferenceSystemArgs<'a> { pub code: i32, pub code_string: Option>, } -impl<'a> Default for ReferenceSystemArgs<'a> { +impl Default for ReferenceSystemArgs<'_> { #[inline] fn default() -> Self { ReferenceSystemArgs { @@ -1356,7 +1350,7 @@ impl flatbuffers::Verifiable for Header<'_> { v: &mut flatbuffers::Verifier, pos: usize, ) -> Result<(), flatbuffers::InvalidFlatbuffer> { - use self::flatbuffers::Verifiable; + v.visit_table(pos)? .visit_field::("transform", Self::VT_TRANSFORM, false)? .visit_field:: { pub attributes: Option>>, pub version: Option>, } -impl<'a> Default for HeaderArgs<'a> { +impl Default for HeaderArgs<'_> { #[inline] fn default() -> Self { HeaderArgs { diff --git a/src/rust/fcb_core/src/lib.rs b/src/rust/fcb_core/src/lib.rs index 9d9926b..5a648b6 100644 --- a/src/rust/fcb_core/src/lib.rs +++ b/src/rust/fcb_core/src/lib.rs @@ -1,7 +1,5 @@ -#![allow(clippy::manual_range_contains)] - mod cj_utils; -pub mod const_vars; +mod const_vars; mod error; mod fb; #[allow(dead_code, unused_imports, clippy::all, warnings)] @@ -14,16 +12,14 @@ mod writer; pub use cj_utils::*; pub use const_vars::*; -// pub use fb as fb_generated; pub use fb::*; +pub use reader::*; +pub use writer::*; #[cfg(feature = "http")] #[cfg(feature = "wasm")] pub use http_reader::*; -pub use reader::*; -pub use writer::*; - pub fn check_magic_bytes(bytes: &[u8]) -> bool { bytes[0..3] == MAGIC_BYTES[0..3] && bytes[4..7] == MAGIC_BYTES[4..7] && bytes[3] <= VERSION } diff --git a/src/rust/fcb_core/src/reader/deserializer.rs b/src/rust/fcb_core/src/reader/deserializer.rs index 82a08a7..abd3ec2 100644 --- a/src/rust/fcb_core/src/reader/deserializer.rs +++ b/src/rust/fcb_core/src/reader/deserializer.rs @@ -135,7 +135,7 @@ pub(crate) fn to_cj_co_type(co_type: CityObjectType) -> String { } } -pub fn decode_attributes( +pub(crate) fn decode_attributes( columns: flatbuffers::Vector<'_, flatbuffers::ForwardsUOffset>>, attributes: flatbuffers::Vector<'_, u8>, ) -> serde_json::Value { diff --git a/src/rust/fcb_core/src/reader/geom_decoder.rs b/src/rust/fcb_core/src/reader/geom_decoder.rs index 33494b4..eb99849 100644 --- a/src/rust/fcb_core/src/reader/geom_decoder.rs +++ b/src/rust/fcb_core/src/reader/geom_decoder.rs @@ -21,7 +21,7 @@ struct PartLists<'a> { /// # Returns /// /// The reconstructed CityJSON boundaries structure -pub fn decode( +pub(crate) fn decode( solids: &[u32], shells: &[u32], surfaces: &[u32], @@ -164,7 +164,9 @@ pub fn decode( /// # Returns /// /// Vector of CityJSON semantic surface definitions -pub fn decode_semantics_surfaces(semantics_objects: &[SemanticObject]) -> Vec { +pub(crate) fn decode_semantics_surfaces( + semantics_objects: &[SemanticObject], +) -> Vec { let surfaces = semantics_objects.iter().map(|s| { let surface_type_str = match s.type_() { SemanticSurfaceType::RoofSurface => "RoofSurface", @@ -310,7 +312,7 @@ fn decode_semantics_( /// # Returns /// /// Complete CityJSON semantics structure with surfaces and values -pub fn decode_semantics( +pub(crate) fn decode_semantics( solids: &[u32], shells: &[u32], geometry_type: GeometryType, diff --git a/src/rust/fcb_core/src/reader/mod.rs b/src/rust/fcb_core/src/reader/mod.rs index 33ff9e1..e0ae61d 100644 --- a/src/rust/fcb_core/src/reader/mod.rs +++ b/src/rust/fcb_core/src/reader/mod.rs @@ -63,6 +63,11 @@ impl FcbReader { Ok(reader) } + /// Open a reader without verifying the FlatBuffers data. + /// + /// # Safety + /// This function skips FlatBuffers verification. The caller must ensure that the input data + /// is valid and properly formatted to avoid undefined behavior. pub unsafe fn open_unchecked(reader: R) -> Result> { Self::read_header(reader, false) } @@ -77,7 +82,7 @@ impl FcbReader { let mut size_buf: [u8; 4] = [0; 4]; // MEMO: 4 bytes for size prefix. This is comvention for FlatBuffers's size_prefixed_root reader.read_exact(&mut size_buf)?; let header_size = u32::from_le_bytes(size_buf) as usize; - if header_size > HEADER_MAX_BUFFER_SIZE || header_size < 8 { + if !((8..=HEADER_MAX_BUFFER_SIZE).contains(&header_size)) { return Err(anyhow!("Illegal header size: {header_size}")); } @@ -298,6 +303,7 @@ impl FeatureIter { todo!("implement") } + #[allow(clippy::should_implement_trait)] pub fn next(&mut self) -> Result> { self.advance()?; if self.get().is_some() { @@ -328,6 +334,7 @@ impl FeatureIter { self.buffer.feature() } + #[allow(clippy::should_implement_trait)] pub fn next(&mut self) -> Result> { self.advance()?; if self.get().is_some() { @@ -390,10 +397,6 @@ impl FeatureIter { self.buffer.header().columns() } - // pub fn features(&self) -> CityFeature { - // self.buffer.feature() - // } - pub fn features_count(&self) -> Option { self.count } diff --git a/src/rust/fcb_core/src/writer/attribute.rs b/src/rust/fcb_core/src/writer/attribute.rs index b49289f..5e44c93 100644 --- a/src/rust/fcb_core/src/writer/attribute.rs +++ b/src/rust/fcb_core/src/writer/attribute.rs @@ -49,7 +49,7 @@ fn guess_type(value: &Value) -> Option { } } -pub fn attr_size(coltype: &ColumnType, colval: &Value) -> usize { +pub(crate) fn attr_size(coltype: &ColumnType, colval: &Value) -> usize { match *coltype { ColumnType::Byte => size_of::(), ColumnType::UByte => size_of::(), @@ -67,14 +67,14 @@ pub fn attr_size(coltype: &ColumnType, colval: &Value) -> usize { } ColumnType::Json => { let json = serde_json::to_string(colval).unwrap_or_default(); - size_of::() + json.as_bytes().len() + size_of::() + json.len() } ColumnType::Binary => size_of::() + colval.as_str().unwrap().len(), //TODO: check if this is correct _ => unreachable!(), } } -pub fn encode_attributes_with_schema(attr: &Value, schema: &AttributeSchema) -> Vec { +pub(crate) fn encode_attributes_with_schema(attr: &Value, schema: &AttributeSchema) -> Vec { if !attr.is_object() || attr.as_object().unwrap().is_empty() || attr.is_null() { return Vec::new(); } @@ -185,9 +185,17 @@ pub fn encode_attributes_with_schema(attr: &Value, schema: &AttributeSchema) -> #[cfg(test)] mod tests { + use crate::{ + deserializer::decode_attributes, + root_as_city_feature, root_as_header, + serializer::{to_columns, to_fcb_attribute}, + CityFeature, CityFeatureArgs, CityObject, CityObjectArgs, Header, HeaderArgs, + }; + use super::*; use anyhow::Result; + use flatbuffers::FlatBufferBuilder; use pretty_assertions::assert_eq; use serde_json::json; @@ -223,4 +231,164 @@ mod tests { Ok(()) } + + #[test] + fn test_attribute_serialization() -> Result<()> { + let test_cases = vec![ + // Case 1: Same schema + ( + json!({ + "attributes": { + "int": -10, + "uint": 5, + "bool": true, + "float": 1.0, + "string": "hoge", + "array": [1, 2, 3], + "json": { + "hoge": "fuga" + }, + } + }), + json!({ + "attributes": { + "int": -10, + "uint": 5, + "bool": true, + "float": 1.0, + "string": "hoge", + "array": [1, 2, 3], + "json": { + "hoge": "fuga" + }, + } + }), + "same schema", + ), + // Case 2: JSON with null value + ( + json!({ + "attributes": { + "int": -10, + "uint": 5, + "bool": true, + "float": 1.0, + "string": "hoge", + "array": [1, 2, 3], + "json": { + "hoge": "fuga" + }, + "exception": null + } + }), + json!({ + "attributes": { + "int": -10, + "uint": 5, + "bool": true, + "float": 1.0, + "string": "hoge", + "array": [1, 2, 3], + "json": { + "hoge": "fuga" + }, + "exception": 1000 + } + }), + "JSON with null value", + ), + // Case 3: JSON is empty + ( + json!({ + "attributes": {} + }), + json!({ + "attributes": { + "int": -10, + "uint": 5, + "bool": true, + "float": 1.0, + "string": "hoge", + "array": [1, 2, 3], + "json": { + "hoge": "fuga" + }, + "exception": 1000 + } + }), + "JSON is empty", + ), + ]; + + for (json_data, schema, test_name) in test_cases { + println!("Testing case: {}", test_name); + + let attrs = &json_data["attributes"]; + let attr_schema = &schema["attributes"]; + + // Create and encode with schema + let mut fbb = FlatBufferBuilder::new(); + let mut common_schema = AttributeSchema::new(); + common_schema.add_attributes(attr_schema); + + let columns = to_columns(&mut fbb, &common_schema); + let header = { + let version = fbb.create_string("1.0.0"); + Header::create( + &mut fbb, + &HeaderArgs { + version: Some(version), + columns: Some(columns), + ..Default::default() + }, + ) + }; + fbb.finish(header, None); + + // Decode and verify + let finished_data = fbb.finished_data(); + let header_buf = root_as_header(finished_data).unwrap(); + + let mut fbb = FlatBufferBuilder::new(); + let feature = { + let (attr_buf, _) = to_fcb_attribute(&mut fbb, attrs, &common_schema); + let city_object = { + let id = fbb.create_string("test"); + CityObject::create( + &mut fbb, + &CityObjectArgs { + id: Some(id), + attributes: Some(attr_buf), + ..Default::default() + }, + ) + }; + let objects = fbb.create_vector(&[city_object]); + let cf_id = fbb.create_string("test_feature"); + CityFeature::create( + &mut fbb, + &CityFeatureArgs { + id: Some(cf_id), + objects: Some(objects), + ..Default::default() + }, + ) + }; + + fbb.finish(feature, None); + + let finished_data = fbb.finished_data(); + let feature_buf = root_as_city_feature(finished_data).unwrap(); + let attributes = feature_buf.objects().unwrap().get(0).attributes().unwrap(); + + let decoded = decode_attributes(header_buf.columns().unwrap(), attributes); + assert_eq!( + attrs, &decoded, + "decoded data should match original for {}", + test_name + ); + } + + Ok(()) + } } diff --git a/src/rust/fcb_core/src/writer/geom_encoder.rs b/src/rust/fcb_core/src/writer/geom_encoder.rs index 9273d4d..f719a1e 100644 --- a/src/rust/fcb_core/src/writer/geom_encoder.rs +++ b/src/rust/fcb_core/src/writer/geom_encoder.rs @@ -175,7 +175,7 @@ fn encode_semantics_values( /// # Arguments /// /// * `semantics` - Reference to the CityJSON Semantics object containing surfaces and values -pub fn encode_semantics(semantics: &CjSemantics) -> GMSemantics { +pub(crate) fn encode_semantics(semantics: &CjSemantics) -> GMSemantics { let mut values = Vec::new(); let _ = encode_semantics_values(&semantics.values, &mut values); diff --git a/src/rust/fcb_core/src/writer/header_writer.rs b/src/rust/fcb_core/src/writer/header_writer.rs index 3a73d4d..8444ce3 100644 --- a/src/rust/fcb_core/src/writer/header_writer.rs +++ b/src/rust/fcb_core/src/writer/header_writer.rs @@ -44,7 +44,7 @@ impl<'a> HeaderWriter<'a> { /// /// * `cj` - The CityJSON data to write /// * `header_options` - Optional configuration for the header writing process - pub fn new( + pub(super) fn new( cj: CityJSON, header_options: Option, attr_schema: AttributeSchema, @@ -58,7 +58,7 @@ impl<'a> HeaderWriter<'a> { /// /// * `options` - Configuration for the header writing process /// * `cj` - The CityJSON data to write - pub fn new_with_options( + fn new_with_options( mut options: HeaderWriterOptions, cj: CityJSON, attr_schema: AttributeSchema, @@ -83,7 +83,7 @@ impl<'a> HeaderWriter<'a> { /// # Returns /// /// A size-prefixed FlatBuffer containing the serialized header - pub fn finish_to_header(mut self) -> Vec { + pub(super) fn finish_to_header(mut self) -> Vec { let header = to_fcb_header( &mut self.fbb, &self.cj, diff --git a/src/rust/fcb_core/src/writer/mod.rs b/src/rust/fcb_core/src/writer/mod.rs index 8db60d1..bbd0332 100644 --- a/src/rust/fcb_core/src/writer/mod.rs +++ b/src/rust/fcb_core/src/writer/mod.rs @@ -74,7 +74,7 @@ impl<'a> FcbWriter<'a> { /// # Returns /// /// A Result indicating success or failure of the write operation - pub fn write_feature(&mut self) -> Result<()> { + fn write_feature(&mut self) -> Result<()> { if let Some(feat_writer) = &mut self.feat_writer { let mut node = feat_writer.bbox.clone(); node.offset = self.feat_offsets.len() as u64; @@ -152,7 +152,7 @@ impl<'a> FcbWriter<'a> { offset += feat.size as u64; node }) - .collect(); + .collect::>(); let tree = PackedRTree::build(&index_nodes, &extent, index_node_size)?; tree.stream_write(&mut out)?; } diff --git a/src/rust/fcb_core/src/writer/serializer.rs b/src/rust/fcb_core/src/writer/serializer.rs index 83b13a8..9f3ecf5 100644 --- a/src/rust/fcb_core/src/writer/serializer.rs +++ b/src/rust/fcb_core/src/writer/serializer.rs @@ -23,7 +23,6 @@ use super::header_writer::HeaderWriterOptions; /// ----------------------------------- /// Serializer for Header /// ----------------------------------- - /// Converts a CityJSON header into FlatBuffers format /// /// # Arguments @@ -31,7 +30,7 @@ use super::header_writer::HeaderWriterOptions; /// * `fbb` - FlatBuffers builder instance /// * `cj` - CityJSON data containing header information /// * `header_metadata` - Additional metadata for the header -pub fn to_fcb_header<'a>( +pub(super) fn to_fcb_header<'a>( fbb: &mut flatbuffers::FlatBufferBuilder<'a>, cj: &CityJSON, header_options: HeaderWriterOptions, @@ -138,7 +137,7 @@ pub fn to_fcb_header<'a>( /// # Arguments /// /// * `geographical_extent` - Array of 6 values [minx, miny, minz, maxx, maxy, maxz] -pub(crate) fn to_geographical_extent(geographical_extent: &[f64; 6]) -> GeographicalExtent { +pub(super) fn to_geographical_extent(geographical_extent: &[f64; 6]) -> GeographicalExtent { let min = Vector::new( geographical_extent[0], geographical_extent[1], @@ -157,7 +156,7 @@ pub(crate) fn to_geographical_extent(geographical_extent: &[f64; 6]) -> Geograph /// # Arguments /// /// * `transform` - CityJSON transform containing scale and translate values -pub(crate) fn to_transform(transform: &CjTransform) -> Transform { +pub(super) fn to_transform(transform: &CjTransform) -> Transform { let scale = Vector::new(transform.scale[0], transform.scale[1], transform.scale[2]); let translate = Vector::new( transform.translate[0], @@ -173,7 +172,7 @@ pub(crate) fn to_transform(transform: &CjTransform) -> Transform { /// /// * `fbb` - FlatBuffers builder instance /// * `metadata` - CityJSON metadata containing reference system information -pub(crate) fn to_reference_system<'a>( +pub(super) fn to_reference_system<'a>( fbb: &mut FlatBufferBuilder<'a>, ref_system: &CjReferenceSystem, ) -> flatbuffers::WIPOffset> { @@ -260,7 +259,6 @@ fn to_point_of_contact<'a>( /// ----------------------------------- /// Serializer for CityJSONFeature /// ----------------------------------- - /// Creates a CityFeature in FlatBuffers format /// /// # Arguments @@ -269,7 +267,7 @@ fn to_point_of_contact<'a>( /// * `id` - Feature identifier /// * `objects` - Vector of city objects /// * `vertices` - Vector of vertex coordinates -pub fn to_fcb_city_feature<'a>( +pub(super) fn to_fcb_city_feature<'a>( fbb: &mut flatbuffers::FlatBufferBuilder<'a>, id: &str, city_feature: &CityJSONFeature, @@ -343,7 +341,7 @@ pub fn to_fcb_city_feature<'a>( /// * `fbb` - FlatBuffers builder instance /// * `id` - Object identifier /// * `co` - CityJSON city object -pub(crate) fn to_city_object<'a>( +pub(super) fn to_city_object<'a>( fbb: &mut flatbuffers::FlatBufferBuilder<'a>, id: &str, co: &CjCityObject, @@ -421,7 +419,7 @@ pub(crate) fn to_city_object<'a>( /// # Arguments /// /// * `co_type` - String representation of CityJSON object type -pub(crate) fn to_co_type(co_type: &str) -> CityObjectType { +pub(super) fn to_co_type(co_type: &str) -> CityObjectType { match co_type { "Bridge" => CityObjectType::Bridge, "BridgePart" => CityObjectType::BridgePart, @@ -470,7 +468,7 @@ pub(crate) fn to_co_type(co_type: &str) -> CityObjectType { /// # Arguments /// /// * `geometry_type` - CityJSON geometry type -pub(crate) fn to_geom_type(geometry_type: &CjGeometryType) -> GeometryType { +pub(super) fn to_geom_type(geometry_type: &CjGeometryType) -> GeometryType { match geometry_type { CjGeometryType::MultiPoint => GeometryType::MultiPoint, CjGeometryType::MultiLineString => GeometryType::MultiLineString, @@ -488,7 +486,7 @@ pub(crate) fn to_geom_type(geometry_type: &CjGeometryType) -> GeometryType { /// # Arguments /// /// * `ss_type` - String representation of semantic surface type -pub(crate) fn to_semantic_surface_type(ss_type: &str) -> SemanticSurfaceType { +pub(super) fn to_semantic_surface_type(ss_type: &str) -> SemanticSurfaceType { match ss_type { "RoofSurface" => SemanticSurfaceType::RoofSurface, "GroundSurface" => SemanticSurfaceType::GroundSurface, @@ -520,7 +518,7 @@ pub(crate) fn to_semantic_surface_type(ss_type: &str) -> SemanticSurfaceType { /// /// * `fbb` - FlatBuffers builder instance /// * `geometry` - CityJSON geometry object -pub fn to_geometry<'a>( +pub(crate) fn to_geometry<'a>( fbb: &mut flatbuffers::FlatBufferBuilder<'a>, geometry: &CjGeometry, ) -> flatbuffers::WIPOffset> { @@ -585,7 +583,7 @@ pub fn to_geometry<'a>( ) } -pub fn to_columns<'a>( +pub(super) fn to_columns<'a>( fbb: &mut FlatBufferBuilder<'a>, attr_schema: &AttributeSchema, ) -> flatbuffers::WIPOffset>>> { @@ -609,7 +607,7 @@ pub fn to_columns<'a>( fbb.create_vector(&columns_vec) } -pub fn to_fcb_attribute<'a>( +pub(super) fn to_fcb_attribute<'a>( fbb: &mut FlatBufferBuilder<'a>, attr: &Value, schema: &AttributeSchema, @@ -661,7 +659,7 @@ mod tests { // Create FlatBuffer and encode let mut fbb = FlatBufferBuilder::new(); - let (city_feature, feat_node) = + let (city_feature, _) = to_fcb_city_feature(&mut fbb, "test_id", &cj_city_feature, &attr_schema); fbb.finish(city_feature, None); diff --git a/src/rust/fcb_core/tests/e2e.rs b/src/rust/fcb_core/tests/e2e.rs index 9692d9a..2c3e623 100644 --- a/src/rust/fcb_core/tests/e2e.rs +++ b/src/rust/fcb_core/tests/e2e.rs @@ -11,6 +11,7 @@ use std::{ io::{BufReader, BufWriter}, path::PathBuf, }; +use tempfile::NamedTempFile; #[test] fn test_cityjson_serialization_cycle() -> Result<()> { @@ -20,7 +21,8 @@ fn test_cityjson_serialization_cycle() -> Result<()> { .join("tests") .join("data") .join("small.city.jsonl"); - let temp_fcb = manifest_dir.join("temp").join("test_e2e.fcb"); + + let temp_fcb = NamedTempFile::new()?; // Read original CityJSONSeq let input_file = File::open(input_file)?; @@ -52,7 +54,6 @@ fn test_cityjson_serialization_cycle() -> Result<()> { }), Some(attr_schema), )?; - fcb.write_feature()?; for feature in original_cj_seq.features.iter() { fcb.add_feature(feature)?; } diff --git a/src/rust/fcb_core/tests/http.rs b/src/rust/fcb_core/tests/http.rs index 82bb359..189ed91 100644 --- a/src/rust/fcb_core/tests/http.rs +++ b/src/rust/fcb_core/tests/http.rs @@ -1,8 +1,4 @@ -use std::{ - error::Error, - fs::File, - io::{BufWriter, Write}, -}; +use std::error::Error; use fcb_core::{deserializer::to_cj_metadata, HttpFcbReader}; @@ -17,20 +13,24 @@ async fn read_http_file(path: &str) -> Result<(), Box> { let header = iter.header(); let cj = to_cj_metadata(&header)?; - let mut writer = BufWriter::new(File::create("delft_http.city.jsonl")?); - writeln!(writer, "{}", serde_json::to_string(&cj)?)?; + // let mut writer = BufWriter::new(File::create("delft_http.city.jsonl")?); + // writeln!(writer, "{}", serde_json::to_string(&cj)?)?; let mut feat_num = 0; let feat_count = header.features_count(); + let mut features = Vec::new(); while let Some(feature) = iter.next().await? { let cj_feature = feature.cj_feature()?; - writeln!(writer, "{}", serde_json::to_string(&cj_feature)?)?; + features.push(cj_feature); + // writeln!(writer, "{}", serde_json::to_string(&cj_feature)?)?; feat_num += 1; if feat_num >= feat_count { break; } } + println!("cj: {:?}", cj); + println!("features: {:?}", features); // TODO: add more tests Ok(()) } diff --git a/src/rust/fcb_core/tests/read.rs b/src/rust/fcb_core/tests/read.rs index 43c0dad..0b35b86 100644 --- a/src/rust/fcb_core/tests/read.rs +++ b/src/rust/fcb_core/tests/read.rs @@ -5,7 +5,10 @@ use std::{fs::File, io::BufReader, path::PathBuf}; #[test] fn read_bbox() -> Result<()> { let manifest_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")); - let input_file = manifest_dir.join("tests").join("data").join("delft.fcb"); + let input_file = manifest_dir + .join("tests") + .join("data") + .join("delft_bbox.fcb"); let mut filein = BufReader::new(File::open(input_file.clone())?); let minx = -200000.0; @@ -25,6 +28,12 @@ fn read_bbox() -> Result<()> { features.push(cj_feat); } + println!("bbox_cnt: {}", bbox_cnt); + println!( + "fcb.header().features_count(): {}", + fcb.header().features_count() + ); + assert!(bbox_cnt < fcb.header().features_count()); let mut count_to_check = 0; diff --git a/src/rust/fcb_core/tests/serde.rs b/src/rust/fcb_core/tests/serde.rs deleted file mode 100644 index a21db5a..0000000 --- a/src/rust/fcb_core/tests/serde.rs +++ /dev/null @@ -1,170 +0,0 @@ -use anyhow::Result; -use fcb_core::{ - attribute::{AttributeSchema, AttributeSchemaMethods}, - deserializer::decode_attributes, - root_as_city_feature, root_as_header, - serializer::{to_columns, to_fcb_attribute}, - CityFeature, CityFeatureArgs, CityObject, CityObjectArgs, Header, HeaderArgs, -}; -use flatbuffers::FlatBufferBuilder; -use serde_json::json; - -#[test] -fn test_attribute_serialization() -> Result<()> { - let test_cases = vec![ - // Case 1: Same schema - ( - json!({ - "attributes": { - "int": -10, - "uint": 5, - "bool": true, - "float": 1.0, - "string": "hoge", - "array": [1, 2, 3], - "json": { - "hoge": "fuga" - }, - } - }), - json!({ - "attributes": { - "int": -10, - "uint": 5, - "bool": true, - "float": 1.0, - "string": "hoge", - "array": [1, 2, 3], - "json": { - "hoge": "fuga" - }, - } - }), - "same schema", - ), - // Case 2: JSON with null value - ( - json!({ - "attributes": { - "int": -10, - "uint": 5, - "bool": true, - "float": 1.0, - "string": "hoge", - "array": [1, 2, 3], - "json": { - "hoge": "fuga" - }, - "exception": null - } - }), - json!({ - "attributes": { - "int": -10, - "uint": 5, - "bool": true, - "float": 1.0, - "string": "hoge", - "array": [1, 2, 3], - "json": { - "hoge": "fuga" - }, - "exception": 1000 - } - }), - "JSON with null value", - ), - // Case 3: JSON is empty - ( - json!({ - "attributes": {} - }), - json!({ - "attributes": { - "int": -10, - "uint": 5, - "bool": true, - "float": 1.0, - "string": "hoge", - "array": [1, 2, 3], - "json": { - "hoge": "fuga" - }, - "exception": 1000 - } - }), - "JSON is empty", - ), - ]; - - for (json_data, schema, test_name) in test_cases { - println!("Testing case: {}", test_name); - - let attrs = &json_data["attributes"]; - let attr_schema = &schema["attributes"]; - - // Create and encode with schema - let mut fbb = FlatBufferBuilder::new(); - let mut common_schema = AttributeSchema::new(); - common_schema.add_attributes(attr_schema); - - let columns = to_columns(&mut fbb, &common_schema); - let header = { - let version = fbb.create_string("1.0.0"); - Header::create( - &mut fbb, - &HeaderArgs { - version: Some(version), - columns: Some(columns), - ..Default::default() - }, - ) - }; - fbb.finish(header, None); - - // Decode and verify - let finished_data = fbb.finished_data(); - let header_buf = root_as_header(finished_data).unwrap(); - - let mut fbb = FlatBufferBuilder::new(); - let feature = { - let (attr_buf, _) = to_fcb_attribute(&mut fbb, attrs, &common_schema); - let city_object = { - let id = fbb.create_string("test"); - CityObject::create( - &mut fbb, - &CityObjectArgs { - id: Some(id), - attributes: Some(attr_buf), - ..Default::default() - }, - ) - }; - let objects = fbb.create_vector(&[city_object]); - let cf_id = fbb.create_string("test_feature"); - CityFeature::create( - &mut fbb, - &CityFeatureArgs { - id: Some(cf_id), - objects: Some(objects), - ..Default::default() - }, - ) - }; - - fbb.finish(feature, None); - - let finished_data = fbb.finished_data(); - let feature_buf = root_as_city_feature(finished_data).unwrap(); - let attributes = feature_buf.objects().unwrap().get(0).attributes().unwrap(); - - let decoded = decode_attributes(header_buf.columns().unwrap(), attributes); - assert_eq!( - attrs, &decoded, - "decoded data should match original for {}", - test_name - ); - } - - Ok(()) -} diff --git a/src/rust/makefile b/src/rust/makefile index c62fefe..17ab235 100644 --- a/src/rust/makefile +++ b/src/rust/makefile @@ -1,27 +1,42 @@ - - .PHONY: pre-commit pre-commit: + make check-common + make check-wasm + +.PHONY: check-common +check-common: cargo fmt - cargo clippy --fix --allow-dirty --all-features - cargo machete --all-features - cargo nextest run --all-features - cargo check --all-features - cargo build --release --all-features + cargo clippy --fix --allow-dirty --workspace --all-targets --all-features --exclude wasm + cargo clippy --fix --allow-dirty -p wasm --target wasm32-unknown-unknown +# cargo machete --all-features --workspace --exclude wasm + cargo nextest run --all-features --workspace --exclude wasm + cargo check --all-features --workspace --exclude wasm + cargo build --workspace --all-features --exclude wasm + +.PHONY: check-wasm +check-wasm: + cargo clippy --fix --allow-dirty -p wasm --target wasm32-unknown-unknown +# cargo nextest run -p wasm --target wasm32-unknown-unknown + cargo check -p wasm --target wasm32-unknown-unknown + cargo build -p wasm --target wasm32-unknown-unknown .PHONY: ser ser: - cargo run --all-features --bin flatcitybuf_cli ser -i fcb_core/tests/data/delft.city.jsonl -o temp/delft_attr.fcb + cargo run -p fcb_cli ser -i fcb_core/tests/data/delft.city.jsonl -o temp/delft_attr.fcb # cargo run --bin flatcitybuf_cli serialize -i tests/data/delft.city.jsonl -o temp/delft.fcb .PHONY: deser deser: - cargo run --all-features --bin flatcitybuf_cli deser -i temp/delft_attr.fcb -o temp/delft_attr.city.jsonl + cargo run -p fcb_cli deser -i temp/delft_attr.fcb -o temp/delft_attr.city.jsonl # cargo run --bin flatcitybuf_cli deserialize -i temp/small.fcb -o temp/small.city.jsonl .PHONY: bench bench: - cargo bench --bench read + cargo bench -p fcb_core --bench read + +.PHONY: build-fcb_core +build-fcb_core: + cargo build --release --bin flatcitybuf_cli .PHONY: wasm-build wasm-build: diff --git a/src/rust/packed_rtree/Cargo.toml b/src/rust/packed_rtree/Cargo.toml index 9f42b8b..96f097f 100644 --- a/src/rust/packed_rtree/Cargo.toml +++ b/src/rust/packed_rtree/Cargo.toml @@ -14,3 +14,4 @@ http-range-client = { workspace = true, optional = true, default-features = fals tracing = { workspace = true } [dev-dependencies] +rand = { workspace = true } diff --git a/src/rust/packed_rtree/src/lib.rs b/src/rust/packed_rtree/src/lib.rs index a152c83..bbb6a61 100644 --- a/src/rust/packed_rtree/src/lib.rs +++ b/src/rust/packed_rtree/src/lib.rs @@ -17,7 +17,7 @@ use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt}; use core::f64; #[cfg(feature = "http")] use http_range_client::{AsyncBufferedHttpRangeClient, AsyncHttpRangeClient}; -use std::cmp::{max, min}; +use std::cmp::min; use std::collections::VecDeque; use std::io::{Cursor, Read, Seek, SeekFrom, Write}; use std::mem::size_of; @@ -300,7 +300,7 @@ impl PackedRTree { fn init(&mut self, node_size: u16) -> Result<()> { assert!(node_size >= 2, "Node size must be at least 2"); assert!(self.num_leaf_nodes > 0, "Cannot create empty tree"); - self.branching_factor = min(max(node_size, 2u16), 65535u16); + self.branching_factor = node_size.clamp(2u16, 65535u16); self.level_bounds = PackedRTree::generate_level_bounds(self.num_leaf_nodes, self.branching_factor); let num_nodes = self @@ -326,7 +326,7 @@ impl PackedRTree { let mut num_nodes = n; level_num_nodes.push(n); loop { - n = (n + node_size as usize - 1) / node_size as usize; + n = n.div_ceil(node_size as usize); num_nodes += n; level_num_nodes.push(n); if n == 1 { @@ -402,7 +402,7 @@ impl PackedRTree { self.node_items.len() } - pub fn build(nodes: &Vec, extent: &NodeItem, node_size: u16) -> Result { + pub fn build(nodes: &[NodeItem], extent: &NodeItem, node_size: u16) -> Result { let mut tree = PackedRTree { extent: extent.clone(), node_items: Vec::new(), @@ -420,7 +420,7 @@ impl PackedRTree { } pub fn from_buf(data: impl Read, num_items: usize, node_size: u16) -> Result { - let node_size = min(max(node_size, 2u16), 65535u16); + let node_size = node_size.clamp(2u16, 65535u16); let level_bounds = PackedRTree::generate_level_bounds(num_items, node_size); let num_nodes = level_bounds .first() @@ -696,7 +696,7 @@ impl PackedRTree { pub fn index_size(num_items: usize, node_size: u16) -> usize { assert!(node_size >= 2, "Node size must be at least 2"); assert!(num_items > 0, "Cannot create empty tree"); - let node_size_min = min(max(node_size, 2), 65535) as usize; + let node_size_min = node_size.clamp(2, 65535) as usize; // limit so that resulting size in bytes can be represented by uint64_t // assert!( // num_items <= 1 << 56, @@ -705,7 +705,7 @@ impl PackedRTree { let mut n = num_items; let mut num_nodes = n; loop { - n = (n + node_size_min - 1) / node_size_min; + n = n.div_ceil(node_size_min); num_nodes += n; if n == 1 { break; @@ -778,47 +778,6 @@ pub mod http { #[cfg(feature = "http")] pub(crate) use http::*; -// mod inspect { -// use super::*; -// use geozero::{ColumnValue, FeatureProcessor}; - -// impl PackedRTree { -// pub fn process_index( -// &self, -// processor: &mut P, -// ) -> geozero::error::Result<()> { -// processor.dataset_begin(Some("PackedRTree"))?; -// let mut fid = 0; -// for (levelno, level) in self.level_bounds.iter().rev().enumerate() { -// for pos in level.clone() { -// let node = &self.node_items[pos]; -// processor.feature_begin(fid)?; -// processor.properties_begin()?; -// let _ = -// processor.property(0, "levelno", &ColumnValue::ULong(levelno as u64))?; -// let _ = processor.property(1, "pos", &ColumnValue::ULong(pos as u64))?; -// let _ = processor.property(2, "offset", &ColumnValue::ULong(node.offset))?; -// processor.properties_end()?; -// processor.geometry_begin()?; -// processor.polygon_begin(true, 1, 0)?; -// processor.linestring_begin(false, 5, 0)?; -// processor.xy(node.min_x, node.min_y, 0)?; -// processor.xy(node.min_x, node.max_y, 1)?; -// processor.xy(node.max_x, node.max_y, 2)?; -// processor.xy(node.max_x, node.min_y, 3)?; -// processor.xy(node.min_x, node.min_y, 4)?; -// processor.linestring_end(false, 0)?; -// processor.polygon_end(true, 0)?; -// processor.geometry_end()?; -// processor.feature_end(fid)?; -// fid += 1; -// } -// } -// processor.dataset_end() -// } -// } -// } - #[cfg(test)] mod tests { use super::*; @@ -848,147 +807,126 @@ mod tests { Ok(()) } - // #[test] - // fn tree_19items_roundtrip_stream_search() -> Result<()> { - // let mut nodes = vec![ - // NodeItem::bounds(0.0, 0.0, 1.0, 1.0), - // NodeItem::bounds(2.0, 2.0, 3.0, 3.0), - // NodeItem::bounds(100.0, 100.0, 110.0, 110.0), - // NodeItem::bounds(101.0, 101.0, 111.0, 111.0), - // NodeItem::bounds(102.0, 102.0, 112.0, 112.0), - // NodeItem::bounds(103.0, 103.0, 113.0, 113.0), - // NodeItem::bounds(104.0, 104.0, 114.0, 114.0), - // NodeItem::bounds(10010.0, 10010.0, 10110.0, 10110.0), - // NodeItem::bounds(10010.0, 10010.0, 10110.0, 10110.0), - // NodeItem::bounds(10010.0, 10010.0, 10110.0, 10110.0), - // NodeItem::bounds(10010.0, 10010.0, 10110.0, 10110.0), - // NodeItem::bounds(10010.0, 10010.0, 10110.0, 10110.0), - // NodeItem::bounds(10010.0, 10010.0, 10110.0, 10110.0), - // NodeItem::bounds(10010.0, 10010.0, 10110.0, 10110.0), - // NodeItem::bounds(10010.0, 10010.0, 10110.0, 10110.0), - // NodeItem::bounds(10010.0, 10010.0, 10110.0, 10110.0), - // NodeItem::bounds(10010.0, 10010.0, 10110.0, 10110.0), - // NodeItem::bounds(10010.0, 10010.0, 10110.0, 10110.0), - // NodeItem::bounds(10010.0, 10010.0, 10110.0, 10110.0), - // ]; - - // let extent = calc_extent(&nodes); - // hilbert_sort(&mut nodes, &extent); - // let mut offset = 0; - // for node in &mut nodes { - // node.offset = offset; - // offset += size_of::() as u64; - // } - // let tree = PackedRTree::build(&nodes, &extent, PackedRTree::DEFAULT_NODE_SIZE)?; - // let list = tree.search(102.0, 102.0, 103.0, 103.0)?; - // assert_eq!(list.len(), 4); - - // let indexes: Vec = list.iter().map(|item| item.index).collect(); - // let expected: Vec = vec![13, 14, 15, 16]; - // assert_eq!(indexes, expected); - - // let mut tree_data: Vec = Vec::new(); - // let res = tree.stream_write(&mut tree_data); - // assert!(res.is_ok()); - // assert_eq!(tree_data.len(), (nodes.len() + 3) * size_of::()); - // assert_eq!(size_of::(), 40); - - // let tree2 = PackedRTree::from_buf( - // &mut &tree_data[..], - // nodes.len(), - // PackedRTree::DEFAULT_NODE_SIZE, - // )?; - // let list = tree2.search(102.0, 102.0, 103.0, 103.0)?; - // assert_eq!(list.len(), 4); - - // let indexes: Vec = list.iter().map(|item| item.index).collect(); - // let expected: Vec = vec![13, 14, 15, 16]; - // assert_eq!(indexes, expected); - - // let mut reader = Cursor::new(&tree_data); - // let list = PackedRTree::stream_search( - // &mut reader, - // nodes.len(), - // PackedRTree::DEFAULT_NODE_SIZE, - // 102.0, - // 102.0, - // 103.0, - // 103.0, - // )?; - // assert_eq!(list.len(), 4); - - // let indexes: Vec = list.iter().map(|item| item.index).collect(); - // let expected: Vec = vec![13, 14, 15, 16]; - // assert_eq!(indexes, expected); - - // Ok(()) - // } - - // #[test] - // fn tree_100_000_items_in_denmark() -> Result<()> { - // use rand::distributions::{Distribution, Uniform}; - - // let unifx = Uniform::from(466379..708929); - // let unify = Uniform::from(6096801..6322352); - // let mut rng = rand::thread_rng(); - - // let mut nodes = Vec::new(); - // for _ in 0..100000 { - // let x = unifx.sample(&mut rng) as f64; - // let y = unify.sample(&mut rng) as f64; - // nodes.push(NodeItem::bounds(x, y, x, y)); - // } - - // let extent = calc_extent(&nodes); - // hilbert_sort(&mut nodes, &extent); - // let tree = PackedRTree::build(&nodes, &extent, PackedRTree::DEFAULT_NODE_SIZE)?; - // let list = tree.search(690407.0, 6063692.0, 811682.0, 6176467.0)?; - - // for i in 0..list.len() { - // assert!(nodes[list[i].index] - // .intersects(&NodeItem::bounds(690407.0, 6063692.0, 811682.0, 6176467.0))); - // } - - // let mut tree_data: Vec = Vec::new(); - // let res = tree.stream_write(&mut tree_data); - // assert!(res.is_ok()); - - // let mut reader = Cursor::new(&tree_data); - // let list2 = PackedRTree::stream_search( - // &mut reader, - // nodes.len(), - // PackedRTree::DEFAULT_NODE_SIZE, - // 690407.0, - // 6063692.0, - // 811682.0, - // 6176467.0, - // )?; - // assert_eq!(list2.len(), list.len()); - // for i in 0..list2.len() { - // assert!(nodes[list2[i].index] - // .intersects(&NodeItem::bounds(690407.0, 6063692.0, 811682.0, 6176467.0))); - // } - // Ok(()) - // } - - // #[test] - // fn tree_processing() -> Result<()> { - // use geozero::geojson::GeoJsonWriter; - // use std::io::BufWriter; - // use tempfile::tempfile; - - // let mut nodes = Vec::new(); - // nodes.push(NodeItem::bounds(0.0, 0.0, 1.0, 1.0)); - // nodes.push(NodeItem::bounds(2.0, 2.0, 3.0, 3.0)); - // let extent = calc_extent(&nodes); - // let mut offset = 0; - // for node in &mut nodes { - // node.offset = offset; - // offset += size_of::() as u64; - // } - // let tree = PackedRTree::build(&nodes, &extent, PackedRTree::DEFAULT_NODE_SIZE)?; - // let mut fout = BufWriter::new(tempfile()?); - // tree.process_index(&mut GeoJsonWriter::new(&mut fout))?; - // Ok(()) - // } + #[test] + fn tree_19items_roundtrip_stream_search() -> Result<()> { + let mut nodes = vec![ + NodeItem::bounds(0.0, 0.0, 1.0, 1.0), + NodeItem::bounds(2.0, 2.0, 3.0, 3.0), + NodeItem::bounds(100.0, 100.0, 110.0, 110.0), + NodeItem::bounds(101.0, 101.0, 111.0, 111.0), + NodeItem::bounds(102.0, 102.0, 112.0, 112.0), + NodeItem::bounds(103.0, 103.0, 113.0, 113.0), + NodeItem::bounds(104.0, 104.0, 114.0, 114.0), + NodeItem::bounds(10010.0, 10010.0, 10110.0, 10110.0), + NodeItem::bounds(10010.0, 10010.0, 10110.0, 10110.0), + NodeItem::bounds(10010.0, 10010.0, 10110.0, 10110.0), + NodeItem::bounds(10010.0, 10010.0, 10110.0, 10110.0), + NodeItem::bounds(10010.0, 10010.0, 10110.0, 10110.0), + NodeItem::bounds(10010.0, 10010.0, 10110.0, 10110.0), + NodeItem::bounds(10010.0, 10010.0, 10110.0, 10110.0), + NodeItem::bounds(10010.0, 10010.0, 10110.0, 10110.0), + NodeItem::bounds(10010.0, 10010.0, 10110.0, 10110.0), + NodeItem::bounds(10010.0, 10010.0, 10110.0, 10110.0), + NodeItem::bounds(10010.0, 10010.0, 10110.0, 10110.0), + NodeItem::bounds(10010.0, 10010.0, 10110.0, 10110.0), + ]; + + let extent = calc_extent(&nodes); + hilbert_sort(&mut nodes, &extent); + let mut offset = 0; + for node in &mut nodes { + node.offset = offset; + offset += size_of::() as u64; + } + let tree = PackedRTree::build(&nodes, &extent, PackedRTree::DEFAULT_NODE_SIZE)?; + let list = tree.search(102.0, 102.0, 103.0, 103.0)?; + assert_eq!(list.len(), 4); + + let indexes: Vec = list.iter().map(|item| item.index).collect(); + let expected: Vec = vec![13, 14, 15, 16]; + assert_eq!(indexes, expected); + + let mut tree_data: Vec = Vec::new(); + let res = tree.stream_write(&mut tree_data); + assert!(res.is_ok()); + assert_eq!(tree_data.len(), (nodes.len() + 3) * size_of::()); + assert_eq!(size_of::(), 40); + + let tree2 = PackedRTree::from_buf( + &mut &tree_data[..], + nodes.len(), + PackedRTree::DEFAULT_NODE_SIZE, + )?; + let list = tree2.search(102.0, 102.0, 103.0, 103.0)?; + assert_eq!(list.len(), 4); + + let indexes: Vec = list.iter().map(|item| item.index).collect(); + let expected: Vec = vec![13, 14, 15, 16]; + assert_eq!(indexes, expected); + + let mut reader = Cursor::new(&tree_data); + let list = PackedRTree::stream_search( + &mut reader, + nodes.len(), + PackedRTree::DEFAULT_NODE_SIZE, + 102.0, + 102.0, + 103.0, + 103.0, + )?; + assert_eq!(list.len(), 4); + + let indexes: Vec = list.iter().map(|item| item.index).collect(); + let expected: Vec = vec![13, 14, 15, 16]; + assert_eq!(indexes, expected); + + Ok(()) + } + + #[test] + fn tree_100_000_items_in_denmark() -> Result<()> { + use rand::distributions::{Distribution, Uniform}; + + let unifx = Uniform::from(466379..708929); + let unify = Uniform::from(6096801..6322352); + let mut rng = rand::thread_rng(); + + let mut nodes = Vec::new(); + for _ in 0..100000 { + let x = unifx.sample(&mut rng) as f64; + let y = unify.sample(&mut rng) as f64; + nodes.push(NodeItem::bounds(x, y, x, y)); + } + + let extent = calc_extent(&nodes); + hilbert_sort(&mut nodes, &extent); + let tree = PackedRTree::build(&nodes, &extent, PackedRTree::DEFAULT_NODE_SIZE)?; + let list = tree.search(690407.0, 6063692.0, 811682.0, 6176467.0)?; + + for i in 0..list.len() { + assert!(nodes[list[i].index] + .intersects(&NodeItem::bounds(690407.0, 6063692.0, 811682.0, 6176467.0))); + } + + let mut tree_data: Vec = Vec::new(); + let res = tree.stream_write(&mut tree_data); + assert!(res.is_ok()); + + let mut reader = Cursor::new(&tree_data); + let list2 = PackedRTree::stream_search( + &mut reader, + nodes.len(), + PackedRTree::DEFAULT_NODE_SIZE, + 690407.0, + 6063692.0, + 811682.0, + 6176467.0, + )?; + assert_eq!(list2.len(), list.len()); + for i in 0..list2.len() { + assert!(nodes[list2[i].index] + .intersects(&NodeItem::bounds(690407.0, 6063692.0, 811682.0, 6176467.0))); + } + Ok(()) + } } diff --git a/src/rust/src/lib.rs b/src/rust/src/lib.rs index e69de29..8b13789 100644 --- a/src/rust/src/lib.rs +++ b/src/rust/src/lib.rs @@ -0,0 +1 @@ + diff --git a/src/rust/wasm/Cargo.toml b/src/rust/wasm/Cargo.toml index d5bc1b7..001e798 100644 --- a/src/rust/wasm/Cargo.toml +++ b/src/rust/wasm/Cargo.toml @@ -7,6 +7,10 @@ edition = "2021" [lib] crate-type = ["cdylib"] +[build] +target = "wasm32-unknown-unknown" + + [dependencies] packed_rtree = { path = "../packed_rtree" } bytes = { workspace = true } @@ -18,12 +22,9 @@ js-sys = { workspace = true } wasm-bindgen = { workspace = true } wasm-bindgen-futures = { workspace = true } byteorder = { workspace = true } -cjseq = { workspace = true } tempfile = { workspace = true } serde_json = { workspace = true } serde = { workspace = true, features = ["derive"] } -anyhow = { workspace = true } -fallible-streaming-iterator = { workspace = true } clap = { workspace = true } tracing = { workspace = true } serde-wasm-bindgen = { workspace = true } diff --git a/src/rust/wasm/src/gloo_client.rs b/src/rust/wasm/src/gloo_client.rs index a37b406..5f45448 100644 --- a/src/rust/wasm/src/gloo_client.rs +++ b/src/rust/wasm/src/gloo_client.rs @@ -39,10 +39,6 @@ impl AsyncHttpRangeClient for WasmHttpClient { .await .map_err(|e| HttpError::HttpError(format!("failed to send request: {}", e)))?; if let Some(val) = response.headers().get(header) { - // let v = val - // .to_str() - // .map_err(|e| HttpError::HttpError(e.to_string()))?; - // Ok(Some(v.to_string())) Ok(Some(val.to_string())) } else { Ok(None) diff --git a/src/rust/wasm/src/lib.rs b/src/rust/wasm/src/lib.rs index febe9a3..a56311e 100644 --- a/src/rust/wasm/src/lib.rs +++ b/src/rust/wasm/src/lib.rs @@ -1,17 +1,16 @@ -use console_error_panic_hook::set_once; +#![cfg(target_arch = "wasm32")] use console_log::init_with_level; use fcb_core::deserializer::{to_cj_feature, to_cj_metadata}; -use fcb_core::{feature_generated, header_generated, size_prefixed_root_as_header, Header}; +use fcb_core::{size_prefixed_root_as_header, Header}; +// #[cfg(target_arch = "wasm32")] use gloo_client::WasmHttpClient; -use gloo_net::http::{Request as GlooRequest, Response}; -use js_sys::Uint8Array; -use log::{error, info}; +#[cfg(target_arch = "wasm32")] +use log::{debug, info, trace}; use serde_wasm_bindgen::to_value; use wasm_bindgen::prelude::*; use byteorder::{ByteOrder, LittleEndian}; use bytes::{BufMut, Bytes, BytesMut}; -use cjseq::CityJSONFeature; use fcb_core::city_buffer::FcbBuffer; use fcb_core::{ check_magic_bytes, size_prefixed_root_as_city_feature, HEADER_MAX_BUFFER_SIZE, @@ -21,15 +20,12 @@ use fcb_core::{ use std::fmt::Error; use std::result::Result; -use http_range_client::{ - AsyncBufferedHttpRangeClient, AsyncHttpRangeClient, BufferedHttpRangeClient, -}; +use http_range_client::{AsyncBufferedHttpRangeClient, AsyncHttpRangeClient}; use packed_rtree::{http::HttpRange, http::HttpSearchResultItem, NodeItem, PackedRTree}; use std::collections::VecDeque; use std::ops::Range; -use tracing::debug; -use tracing::trace; + mod gloo_client; // The largest request we'll speculatively make. @@ -57,16 +53,9 @@ pub struct AsyncFeatureIter { count: usize, } -// impl WasmFcbReader { -// pub async fn new(url: String) -> Result { -// let client = WasmHttpClient::new(url).await?; -// Self::_open(client).await -// } -// } - #[wasm_bindgen] impl HttpFcbReader { - #[wasm_bindgen(constructor, start)] + #[wasm_bindgen(constructor)] pub async fn new(url: String) -> Result { println!("open===: {:?}", url); console_error_panic_hook::set_once(); @@ -125,7 +114,7 @@ impl HttpFcbReader { read_bytes += HEADER_SIZE_SIZE; let header_size = LittleEndian::read_u32(&bytes) as usize; - if header_size > HEADER_MAX_BUFFER_SIZE || header_size < 8 { + if !(8..=HEADER_MAX_BUFFER_SIZE).contains(&header_size) { // minimum size check avoids panic in FlatBuffers header decoding return Err(JsValue::from_str(&format!( "IllegalHeaderSize: {header_size}" @@ -288,13 +277,11 @@ impl AsyncFeatureIter { .map_err(|e| JsValue::from_str(&e.to_string()))?; let cj_feature = to_cj_feature(feature, self._header().columns()) .map_err(|e| JsValue::from_str(&e.to_string()))?; + Ok(Some(to_value(&cj_feature)?)) } - /// Return current feature - pub fn cur_feature(&self) -> Result { - self.cur_feature() - } + #[wasm_bindgen] pub fn cur_cj_feature(&self) -> Result { let cj_feature = to_cj_feature(self.fbs.feature(), self._header().columns()) .map_err(|e| JsValue::from_str(&e.to_string()))?; @@ -489,83 +476,3 @@ impl FeatureBatch { Ok(Some(feature_buffer.freeze())) } } - -// #[cfg(test)] -// mod tests { -// use crate::HttpFcbReader; - -// #[tokio::test] -// async fn fgb_max_request_size() { -// let (fgb, stats) = HttpFcbReader::mock_from_file("../../test/data/UScounties.fgb") -// .await -// .unwrap(); - -// { -// // The read guard needs to be in a scoped block, else we won't release the lock and the test will hang when -// // the actual FGB client code tries to update the stats. -// let stats = stats.read().unwrap(); -// assert_eq!(stats.request_count, 1); -// // This number might change a little if the test data or logic changes, but they should be in the same ballpark. -// assert_eq!(stats.bytes_requested, 12944); -// } - -// // This bbox covers a large swathe of the dataset. The idea is that at least one request should be limited by the -// // max request size `DEFAULT_HTTP_FETCH_SIZE`, but that we should still have a reasonable number of requests. -// let mut iter = fgb.select_bbox(-118.0, 42.0, -100.0, 47.0).await.unwrap(); - -// let mut feature_count = 0; -// while let Some(_feature) = iter.next().await.unwrap() { -// feature_count += 1; -// } -// assert_eq!(feature_count, 169); - -// { -// // The read guard needs to be in a scoped block, else we won't release the lock and the test will hang when -// // the actual FGB client code tries to update the stats. -// let stats = stats.read().unwrap(); -// // These numbers might change a little if the test data or logic changes, but they should be in the same ballpark. -// assert_eq!(stats.request_count, 5); -// assert_eq!(stats.bytes_requested, 2131152); -// } -// } -// } - -// #[wasm_bindgen] -// pub async fn fetch_partial(url: &str, start: u64, end: u64) -> Result { -// // Construct the "Range" header, e.g. "bytes=0-1023" -// let range_header_value = format!("bytes={}-{}", start, end); - -// // Perform a GET request using the `fetch` API under the hood -// let response = Request::get(url) -// .header("Range", &range_header_value) -// .send() -// .await -// .map_err(|err| JsValue::from_str(&err.to_string()))?; - -// // If the server supports partial requests, often you'll get status 206 -// // However, some servers might return 200 if they don't handle partial fetches -// if !response.ok() { -// // We'll forward the status as an error -// return Err(JsValue::from_str(&format!( -// "HTTP status: {}", -// response.status() -// ))); -// } - -// // Retrieve the bytes from the response -// let bytes = response -// .binary() -// .await -// .map_err(|err| JsValue::from_str(&err.to_string()))?; - -// // Convert them to a JavaScript `Uint8Array` so JS code can read them -// let array = Uint8Array::from(bytes.as_slice()); - -// // Return the Uint8Array as a `JsValue` -// Ok(array.into()) -// } - -// #[wasm_bindgen] -// pub fn hello() { -// println!("Hello, world!"); -// } diff --git a/src/rust/wasm/src/range_client.rs b/src/rust/wasm/src/range_client.rs deleted file mode 100644 index 9250a4c..0000000 --- a/src/rust/wasm/src/range_client.rs +++ /dev/null @@ -1,39 +0,0 @@ -// use crate::HttpFcbReader; -// use anyhow::Result; -// use bytes::Bytes; -// use gloo_net::http::Request; -// use http_range_client::AsyncHttpRangeClient; -// use std::fs::File; -// use std::io::{BufReader, Read, Seek, SeekFrom}; -// use std::ops::Range; -// use std::path::PathBuf; -// use std::sync::{Arc, RwLock}; - -// use wasm_bindgen::prelude::*; - -// #[wasm_bindgen] -// pub struct WasmHttpClient { -// inner: GlooRequest, -// } - -// // impl HttpFcbReader { -// // pub async fn open(url: &str) -> Result<> { -// // trace!("starting: opening http reader, reading header"); - -// // // let stats = Arc::new(RwLock::new(RequestStats::new())); -// // // let http_client = MockHttpRangeClient::new(path, stats.clone()); -// // // let client = http_range_client::AsyncBufferedHttpRangeClient::with(http_client, path); -// // // Ok((Self::_open(client).await?, stats)) -// // todo!("implement me") -// // } -// // } - -// impl AsyncHttpRangeClient for WasmHttpClient { -// async fn get_range(&self, url: &str, range: &str) -> Result { -// todo!("implement me") -// } - -// async fn head_response_header(&self, url: &str, header: &str) -> Result> { -// todo!("implement me") -// } -// }