wip: re-structure folder and build wasm

.gitignore

@@ -58,6 +58,7 @@ __pycache__/
 
 temp/
 
-src/rust/benchmark_data/
-
+benchmark_data/
+src/rust/fcb_core/tests/data/
+!src/rust/fcb_core/tests/data/.gitkeep
 .cursorrules

.vscode/settings.json

@@ -11,5 +11,6 @@
   "python.pythonPath": "${workspaceFolder}/src/py/.venv/bin/python",
   "rust-analyzer.server.extraEnv": {
     "RUST_LOG": "debug"
-  }
+  },
+  "liveServer.settings.port": 5501
 }

src/rust/Cargo.toml

@@ -3,49 +3,37 @@ name = "flatcitybuf"
 version = "0.1.0"
 edition = "2021"
 
+[workspace]
+members = ["cli", "fcb_core", "packed_rtree", "wasm"]
+resolver = "2"
 
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
-[features]
-default = []
-http = ["http-range-client", "bytes", "reqwest"]
-
-[dependencies]
+[workspace.dependencies]
+async-trait = "0.1.85"
 flatbuffers = "24.3.25"
 byteorder = "1.5.0"
 cjseq = { git = "https://github.com/HideBa/cjseq", branch = "develop" }
 tempfile = "3.14.0"
 serde_json = "1.0.133"
+serde = "1.0.200"
 anyhow = "1.0.95"
 fallible-streaming-iterator = "0.1.9"
 clap = "4.5.23"
-http-range-client = { version = "0.9.0", optional = true, default-features = false, features = [
-  "reqwest-async",
-] }
-reqwest = { version = "0.12.12", optional = true }
-bytes = { version = "1.9.0", optional = true }
+http-range-client = { version = "0.9.0", default-features = false }
+reqwest = { version = "0.12.12" }
 tracing = "0.1.41"
-tokio = "1.43.0"
-
-[lib]
-name = "flatcitybuf"
-path = "src/lib.rs"
-
-
-[[bin]]
-name = "read"
-path = "src/bin/read.rs"
-
-[[bin]]
-name = "flatcitybuf_cli"
-path = "src/main.rs"
-
-[dev-dependencies]
-async-trait = "0.1.85"
+bytes = "1.9.0"
 criterion = { version = "0.5.1", features = ["async_tokio", "html_reports"] }
 memory-stats = "1.2.0"
 pretty_assertions = "1.4.1"
 tokio = { version = "1.43.0", features = ["macros", "rt-multi-thread"] }
+getrandom = { version = "0.2.15", features = ["js"] }
+gloo-net = "0.6.0"
+js-sys = "0.3.77"
+wasm-bindgen = "0.2"
+wasm-bindgen-futures = "0.4.50"
+serde-wasm-bindgen = "0.6.5"
+console_error_panic_hook = "0.1.7"
+log = "0.4"
+console_log = "0.2"
 
-[[bench]]
-name = "read"
-harness = false
+[dependencies]

src/rust/cli/Cargo.toml

@@ -0,0 +1,11 @@
+[package]
+name = "fcb_cli"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+fcb_core = { path = "../fcb_core", default-features = false }
+clap = { workspace = true }
+anyhow = { workspace = true }
+serde = { workspace = true }
+serde_json = { workspace = true }

src/rust/src/main.rs → src/rust/cli/src/main.rs

@@ -1,6 +1,6 @@
 use anyhow::Result;
 use clap::{Parser, Subcommand};
-use flatcitybuf::{
+use fcb_core::{
     attribute::{AttributeSchema, AttributeSchemaMethods},
     deserializer,
     header_writer::HeaderWriterOptions,

src/rust/fcb_core/Cargo.toml

@@ -0,0 +1,73 @@
+[package]
+name = "fcb_core"
+version = "0.1.0"
+edition = "2021"
+
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+[features]
+default = ["http"]
+http = ["http-range-client", "reqwest", "bytes"]
+wasm = [
+  "wasm-bindgen",
+  "wasm-bindgen-futures",
+  "gloo-net",
+  "js-sys",
+  "getrandom",
+]
+
+[dependencies]
+bytes = { workspace = true, optional = true }
+flatbuffers = { workspace = true }
+byteorder = { workspace = true }
+cjseq = { workspace = true }
+tempfile = { workspace = true }
+serde_json = { workspace = true }
+anyhow = { workspace = true }
+fallible-streaming-iterator = { workspace = true }
+clap = { workspace = true }
+tracing = { workspace = true }
+packed_rtree = { path = "../packed_rtree" }
+http-range-client = { workspace = true, default-features = false, optional = true, features = [
+  "reqwest-async",
+] }
+reqwest = { workspace = true, optional = true }
+
+[target.'cfg(feature = "http")'.dependencies]
+tokio = { workspace = true }
+http-range-client = { workspace = true, default-features = false, optional = true, features = [
+  "reqwest-async",
+] }
+reqwest = { workspace = true, optional = true }
+getrandom = { workspace = true, optional = true, default-features = false, features = [
+  "js",
+] }
+async-trait = { workspace = true, optional = true, default-features = false }
+gloo-net = { workspace = true, optional = true }
+js-sys = { workspace = true, optional = true }
+wasm-bindgen = { workspace = true, optional = true }
+wasm-bindgen-futures = { workspace = true, optional = true }
+
+
+[lib]
+name = "fcb_core"
+path = "src/lib.rs"
+
+[[bin]]
+name = "read"
+path = "src/bin/read.rs"
+
+
+[[bench]]
+name = "read"
+harness = false
+
+[dev-dependencies]
+async-trait = { workspace = true }
+memory-stats = { workspace = true }
+pretty_assertions = { workspace = true }
+
+# [target.'cfg(not(feature = "wasm"))'.dev-dependencies]
+# criterion = { workspace = true, features = ["async_tokio", "html_reports"] }
+# [target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies]
+# tokio = { workspace = true, features = ["macros", "rt-multi-thread"] }

src/rust/benches/read.rs → src/rust/fcb_core/benches/read.rs

@@ -1,7 +1,7 @@
 use anyhow::Result;
 use cjseq::{CityJSON, CityJSONFeature};
 use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion};
-use flatcitybuf::{FcbReader, GeometryType};
+use fcb_core::{FcbReader, GeometryType};
 use std::{
     collections::HashMap,
     fs::File,

src/rust/src/bin/read.rs → src/rust/fcb_core/src/bin/read.rs

@@ -1,5 +1,5 @@
-use flatcitybuf::deserializer::to_cj_metadata;
-use flatcitybuf::FcbReader;
+use fcb_core::deserializer::to_cj_metadata;
+use fcb_core::FcbReader;
 use std::error::Error;
 use std::fs::File;
 use std::io::{BufReader, BufWriter, Write};

src/rust/src/bin/write.rs → src/rust/fcb_core/src/bin/write.rs

@@ -1,6 +1,8 @@
-use flatcitybuf::attribute::{AttributeSchema, AttributeSchemaMethods};
-use flatcitybuf::header_writer::HeaderWriterOptions;
-use flatcitybuf::{read_cityjson_from_reader, CJType, CJTypeKind, CityJSONSeq, FcbWriter};
+use fcb_core::{
+    attribute::{AttributeSchema, AttributeSchemaMethods},
+    header_writer::HeaderWriterOptions,
+    read_cityjson_from_reader, CJType, CJTypeKind, CityJSONSeq, FcbWriter,
+};
 use std::error::Error;
 use std::fs::File;
 use std::io::{BufReader, BufWriter};

src/rust/fcb_core/src/const_vars.rs

@@ -0,0 +1,17 @@
+// Current version of FlatCityBuf
+pub const VERSION: u8 = 1;
+
+// Magic bytes for FlatCityBuf
+pub const MAGIC_BYTES: [u8; 8] = [b'f', b'c', b'b', VERSION, b'f', b'c', b'b', 0];
+
+// Maximum buffer size for header
+pub const HEADER_MAX_BUFFER_SIZE: usize = 1024 * 1024 * 512; // 512MB
+
+// Size of magic bytes
+pub const MAGIC_BYTES_SIZE: usize = 8;
+
+// Size of header size
+pub const HEADER_SIZE_SIZE: usize = 4;
+
+// // Offset of header size
+// pub(crate) const HEADER_SIZE_OFFSET: usize = MAGIC_BYTES_SIZE + HEADER_SIZE_SIZE;

src/rust/src/feature_generated.rs → src/rust/fcb_core/src/fb/feature_generated.rs

@@ -2,7 +2,7 @@
 
 // @generated
 
-use crate::header_generated::*;
+use crate::fb::*;
 use core::cmp::Ordering;
 use core::mem;
 

src/rust/fcb_core/src/fb/mod.rs

@@ -0,0 +1,5 @@
+pub mod feature_generated;
+pub mod header_generated;
+
+pub use feature_generated::*;
+pub use header_generated::*;

src/rust/fcb_core/src/http_reader/mock_http_range_client.rs

@@ -0,0 +1,107 @@
+// use crate::http_reader::HttpFcbReader;
+// use anyhow::Result;
+// use bytes::Bytes;
+// use http_range_client;
+// use std::fs::File;
+// use std::io::{BufReader, Read, Seek, SeekFrom};
+// use std::ops::Range;
+// use std::path::PathBuf;
+// use std::sync::{Arc, RwLock};
+// use tracing::trace;
+
+// impl HttpFcbReader<MockHttpRangeClient> {
+//     /// NOTE: For debugging expediency, this test class often prefers panics over returning a result.
+//     pub async fn mock_from_file(
+//         path: &str,
+//     ) -> Result<(
+//         HttpFcbReader<MockHttpRangeClient>,
+//         Arc<RwLock<RequestStats>>,
+//     )> {
+//         trace!("starting: opening http reader, reading header");
+
+//         let stats = Arc::new(RwLock::new(RequestStats::new()));
+//         let http_client = MockHttpRangeClient::new(path, stats.clone());
+//         let client = http_range_client::AsyncBufferedHttpRangeClient::with(http_client, path);
+//         Ok((Self::_open(client).await?, stats))
+//     }
+// }
+
+// /// NOTE: For debugging expediency, this test class often prefers panics over returning a result.
+// pub(crate) struct MockHttpRangeClient {
+//     path: PathBuf,
+//     stats: Arc<RwLock<RequestStats>>,
+// }
+
+// pub(crate) struct RequestStats {
+//     pub request_count: u64,
+//     pub bytes_requested: u64,
+// }
+
+// impl RequestStats {
+//     fn new() -> Self {
+//         Self {
+//             request_count: 0,
+//             bytes_requested: 0,
+//         }
+//     }
+// }
+
+// #[async_trait::async_trait]
+// impl AsyncHttpRangeClient for MockHttpRangeClient {
+//     async fn get_range(&self, url: &str, range: &str) -> http_range_client::Result<Bytes> {
+//         assert_eq!(url, self.path.to_str().unwrap());
+
+//         /// This is a hack, but we need the start and length of the range
+//         /// since all we're given is the pre-formatted range string, we
+//         /// need to parse it into its components
+//         ///
+//         /// For expediency, this test code panics rather than returns a result.
+//         fn parse_range_header(range: &str) -> Range<u64> {
+//             let bytes = range.strip_prefix("bytes=").unwrap();
+//             let parts: Vec<&str> = bytes.split('-').collect();
+//             assert!(parts.len() == 2);
+//             let start = parts[0].parse().expect("should have valid start range");
+//             let end: u64 = parts[1].parse().expect("should have valid end range");
+//             // Range headers are *inclusive*
+//             start..(end + 1)
+//         }
+
+//         let range = parse_range_header(range);
+//         let request_length = range.end - range.start;
+
+//         let mut stats = self
+//             .stats
+//             .write()
+//             .expect("test code does not handle actual concurrency");
+
+//         stats.request_count += 1;
+//         stats.bytes_requested += request_length;
+
+//         let mut file_reader = BufReader::new(File::open(&self.path).unwrap());
+//         file_reader
+//             .seek(SeekFrom::Start(range.start))
+//             .expect("unable to seek test reader");
+//         let mut output = vec![0; request_length as usize];
+//         file_reader
+//             .read_exact(&mut output)
+//             .expect("failed to read from test reader");
+//         Ok(Bytes::from(output))
+//     }
+
+//     async fn head_response_header(
+//         &self,
+//         _url: &str,
+//         _header: &str,
+//     ) -> http_range_client::Result<Option<String>> {
+//         unimplemented!()
+//     }
+// }
+
+// impl MockHttpRangeClient {
+//     fn new(path: &str, stats: Arc<RwLock<RequestStats>>) -> Self {
+//         Self {
+//             path: path.into(),
+//             stats,
+//         }
+//     }
+// }