Skip to content

Commit

Permalink
Shapefile sinkの改善 (#467)
Browse files Browse the repository at this point in the history 
Shapefile Sink に以下の改善を行いました：

- 全カラムが .dbf に含まれるようにする。順序もスキーマ通りになるようにする。
    - 属性を作る部分のコードが大分減った。
- 数値を Numeric 型で格納。長さやdecimal point ? は今後調整可能。
- 複数種類の地物がある場合に、マルチスレッド (rayon) で処理する。


![Untitled](https://github.com/MIERUNE/PLATEAU-GIS-Converter/assets/5351911/9de6a1bf-0bd6-40ac-abc6-dc1443480304)

Close: #427

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit


- **新機能**
    - Cesium JavaScriptライブラリと関連するCSSファイルをバージョン1.114から1.115に更新しました。
- **ドキュメント**
    - CesiumJSライブラリのバージョンを1.114から1.115に更新する手順をドキュメントに追加しました。
- **リファクタ**
    - Shapefileの属性を処理するコードにおいて、異なる属性タイプを扱うロジックを改善しました。
    - ShapefileSinkの実行方法を更新し、並列処理とエラーハンドリングを強化しました。

<!-- end of auto-generated comment: release notes by coderabbit.ai -->
  • Loading branch information
@ciscorn
ciscorn authored Mar 13, 2024
1 parent 9b36273 commit 72506ec
Show file tree
Hide file tree
Showing 3 changed files with 189 additions and 231 deletions.
2 changes: 1 addition & 1 deletion nusamai/Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ version.workspace = true
edition = "2021"

[dependencies]
indexmap = { version = "2.2.5", features = ["serde"] }
indexmap = { version = "2.2.5", features = ["serde", "rayon"] }
rayon = "1.9.0"
serde = { version = "1.0.197", features = ["derive"] }
nusamai-plateau = { path = "../nusamai-plateau" }
Expand Down
227 changes: 81 additions & 146 deletions nusamai/src/sink/shapefile/attributes.rs
View file
Original file line number Diff line number Diff line change
@@ -1,205 +1,124 @@
use chrono::Datelike;
use hashbrown::HashMap;
use nusamai_citygml::schema::DataTypeDef;
use nusamai_citygml::schema::FeatureTypeDef;
use nusamai_citygml::schema::TypeDef;
use shapefile::dbase::{self, Date, FieldValue, Record};

use nusamai_citygml::object::Map;
use nusamai_citygml::object::Value;
use nusamai_citygml::schema::TypeRef;
use shapefile::Shape;

pub struct FieldInfo {
field_type: TypeRef,
size: u8,
}

pub type FieldInfoMap = HashMap<String, FieldInfo>;
pub type Features = Vec<(Shape, Map)>;

pub fn make_table_builder(fields: &FieldInfoMap) -> dbase::TableWriterBuilder {
pub fn make_table_builder(
typedef: &TypeDef,
) -> (dbase::TableWriterBuilder, HashMap<String, FieldValue>) {
let mut builder = dbase::TableWriterBuilder::new();
let mut defaults = HashMap::new();

let attributes = match typedef {
TypeDef::Feature(FeatureTypeDef { attributes, .. }) => {
let key = "id";
builder = builder.add_character_field(key.try_into().unwrap(), 255);
defaults.insert(key.into(), FieldValue::Character(None));
attributes
}
TypeDef::Data(DataTypeDef { attributes, .. }) => attributes,
TypeDef::Property(_) => unreachable!(),
};

for (field_name, field_info) in fields {
for (field_name, attr) in attributes {
let name = field_name.as_str().try_into().unwrap(); // FIXME: handle errors
let key = field_name.to_string();

match field_info.field_type {
TypeRef::String | TypeRef::Code | TypeRef::URI => {
builder = builder.add_character_field(name, field_info.size);
match attr.type_ref {
TypeRef::String | TypeRef::Code | TypeRef::URI | TypeRef::JsonString(_) => {
builder = builder.add_character_field(name, 255);
defaults.insert(key, FieldValue::Character(None));
}
TypeRef::Integer | TypeRef::NonNegativeInteger => {
builder = builder.add_integer_field(name);
builder = builder.add_numeric_field(name, 11, 0);
defaults.insert(key, FieldValue::Numeric(None));
}
TypeRef::Double | TypeRef::Measure => {
builder = builder.add_float_field(name, 50, 10);
builder = builder.add_numeric_field(name, 18, 6);
defaults.insert(key, FieldValue::Numeric(None));
}
TypeRef::Boolean => {
builder = builder.add_logical_field(name);
builder = builder.add_character_field(name, 6);
defaults.insert(key, FieldValue::Character(None));
}
TypeRef::Date => {
builder = builder.add_date_field(name);
defaults.insert(key, FieldValue::Date(None));
}
TypeRef::DateTime => {
// todo
}
TypeRef::Point => {
// todo
}
TypeRef::Unknown => {
// todo
unreachable!();
}
TypeRef::Named(_) => {
// todo
}
TypeRef::JsonString(_) => {
// todo
}
TypeRef::DateTime => {
// todo
unreachable!();
}
}
}

builder
(builder, defaults)
}

pub fn fill_missing_fields(attributes: &mut Map, field_info: &FieldInfoMap) {
for (field_name, field_info) in field_info {
if !attributes.contains_key(field_name.as_str()) {
match field_info.field_type {
TypeRef::String | TypeRef::Code | TypeRef::URI => {
attributes.insert(field_name.clone(), Value::String("".to_string()));
}
TypeRef::Integer | TypeRef::NonNegativeInteger => {
attributes.insert(field_name.clone(), Value::Integer(0));
}
TypeRef::Double | TypeRef::Measure => {
attributes.insert(field_name.clone(), Value::Double(0.0));
}
TypeRef::Boolean => {
attributes.insert(field_name.clone(), Value::String("".to_string()));
}
TypeRef::Date => {
attributes.insert(field_name.clone(), Value::String("".to_string()));
}
TypeRef::Point => {
// todo
}
TypeRef::Unknown => {
// todo
}
TypeRef::Named(_) => {
// todo
}
TypeRef::JsonString(_) => {
// todo
}
TypeRef::DateTime => {
// todo
}
}
}
}
}

pub fn make_field_list(features: &Features) -> FieldInfoMap {
let mut fields: FieldInfoMap = Default::default();
pub fn attributes_to_record(
attributes: Map,
fields_default: &HashMap<String, FieldValue>,
) -> Record {
let mut record = dbase::Record::default();

for (_, attributes) in features {
for (field_name, field_value) in attributes {
match field_value {
Value::String(_) | Value::Code(_) | Value::Uri(_) => {
fields.insert(
field_name.clone(),
FieldInfo {
field_type: TypeRef::String,
size: 255,
},
);
}
Value::Integer(_) | Value::NonNegativeInteger(_) => {
fields.insert(
field_name.clone(),
FieldInfo {
field_type: TypeRef::Integer,
size: 4,
},
);
}
Value::Double(_) | Value::Measure(_) => {
fields.insert(
field_name.clone(),
FieldInfo {
field_type: TypeRef::Double,
size: 8,
},
);
}
Value::Boolean(_) => {
fields.insert(
field_name.clone(),
FieldInfo {
field_type: TypeRef::Boolean,
size: 1,
},
);
}
Value::Date(_) => {
fields.insert(
field_name.clone(),
FieldInfo {
field_type: TypeRef::Date,
size: 8,
},
);
}
Value::Point(_) => {
// todo
}
Value::Array(_) => {
// todo
}
Value::Object(_) => {
// todo
}
}
// Fill in with default values for attributes that are not present
for (name, default) in fields_default {
if !attributes.contains_key(name) {
record.insert(name.to_string(), default.clone());
}
}
fields
}

pub fn attributes_to_record(attributes: Map) -> Record {
let mut record = dbase::Record::default();

for (attr_name, attr_value) in attributes {
match attr_value {
Value::String(s) => {
// Shapefile string type can only store up to 255 characters.
if s.len() > 255 {
log::warn!("{} value too long, truncating to 255 characters", attr_name);
record.insert(attr_name, FieldValue::Character(Some(s[0..255].to_owned())));
} else {
record.insert(attr_name, FieldValue::Character(Some(s.to_owned())));
}
// Shapefile cannot store string longer than 254 bytes
let s = trim_string_bytes(s, 254);
record.insert(attr_name, FieldValue::Character(Some(s)));
}
Value::Code(c) => {
// value of the code
record.insert(attr_name, FieldValue::Character(Some(c.value().to_owned())));
}
Value::Integer(i) => {
record.insert(attr_name, FieldValue::Integer(i.to_owned() as i32));
record.insert(attr_name, FieldValue::Numeric(Some(i as f64)));
}
Value::NonNegativeInteger(i) => {
record.insert(attr_name, FieldValue::Integer(i.to_owned() as i32));
record.insert(attr_name, FieldValue::Numeric(Some(i as f64)));
}
// Handle as Float
Value::Double(d) => {
record.insert(attr_name, FieldValue::Float(Some(d.to_owned() as f32)));
}
// Handle as Float
Value::Measure(m) => {
record.insert(
attr_name,
FieldValue::Float(Some(m.value().to_owned() as f32)),
FieldValue::Numeric(match d.is_nan() {
true => None,
false => Some(d),
}),
);
}
Value::Measure(m) => {
record.insert(attr_name, FieldValue::Numeric(Some(m.value())));
}
Value::Boolean(b) => {
record.insert(attr_name, FieldValue::Logical(Some(b.to_owned())));
record.insert(
attr_name,
FieldValue::Character(Some(match b {
true => "true".to_string(),
false => "false".to_string(),
})),
);
}
Value::Uri(u) => {
record.insert(
Expand Down Expand Up @@ -228,3 +147,19 @@ pub fn attributes_to_record(attributes: Map) -> Record {

record
}

fn trim_string_bytes(s: String, n: usize) -> String {
let bytes = s.as_bytes();
if bytes.len() <= n {
return s;
}
log::warn!("string is too long, truncating to {} characters", n);
match std::str::from_utf8(&bytes[..n]) {
Ok(valid_str) => valid_str.to_string(),
Err(e) => {
let valid_up_to = e.valid_up_to();
let valid_str = std::str::from_utf8(&bytes[..valid_up_to]).unwrap();
valid_str.to_string()
}
}
}
Loading

0 comments on commit 72506ec

Please sign in to comment.