diff --git a/README.md b/README.md index a12f3b5..10bc164 100644 --- a/README.md +++ b/README.md @@ -60,6 +60,12 @@ sort: - field: "name" order: "ASC" +# optional additional associations (that you need in your deserialization script) +# note: entity_path associations are already added by default +# only applied on export +associations: + - "cover" + # mappings can either be # - by entity_path # - by key @@ -75,6 +81,16 @@ mappings: entity_path: "stock" - file_column: "tax id" entity_path: "taxId" + - file_column: "tax rate" + # entity path can resolve "To-One-Associations" of any depth + entity_path: "tax.taxRate" + - file_column: "manufacturer name" + # They can also use the optional chaining '?.' operator to fall back to null + # if the association is null + entity_path: "manufacturer?.name" + - file_column: "manufacturer id" + # for importing, you also need the association id in the association object + entity_path: "manufacturer?.id" - file_column: "gross price EUR" key: "gross_price_eur" - file_column: "net price EUR" diff --git a/profiles/manufacturer.yaml b/profiles/manufacturer.yaml index e4c6f40..94155c6 100644 --- a/profiles/manufacturer.yaml +++ b/profiles/manufacturer.yaml @@ -9,3 +9,5 @@ mappings: entity_path: "description" - file_column: "website" entity_path: "link" + - file_column: "logo" + entity_path: "media?.url" diff --git a/src/api.rs b/src/api.rs index cf1e3ea..5341d72 100644 --- a/src/api.rs +++ b/src/api.rs @@ -1,6 +1,7 @@ use crate::config::Credentials; use anyhow::anyhow; -use reqwest::{Client, Response, StatusCode}; +use reqwest::header::{HeaderMap, HeaderValue}; +use reqwest::{header, Client, Response, StatusCode}; use serde::{Deserialize, Serialize}; use serde_json::json; use std::collections::BTreeMap; @@ -20,7 +21,14 @@ pub struct SwClient { impl SwClient { pub async fn new(credentials: Credentials, in_flight_limit: usize) -> anyhow::Result { - let client = Client::builder().timeout(Duration::from_secs(10)).build()?; + let mut default_headers = HeaderMap::default(); + // This header is needed, otherwise the response would be "application/vnd.api+json" (by default) + // and that doesn't have the association data as part of the entity object + default_headers.insert(header::ACCEPT, HeaderValue::from_static("application/json")); + let client = Client::builder() + .timeout(Duration::from_secs(10)) + .default_headers(default_headers) + .build()?; let credentials = Arc::new(credentials); let auth_response = Self::authenticate(&client, credentials.as_ref()).await?; @@ -311,15 +319,10 @@ pub struct SwErrorSource { #[derive(Debug, Deserialize)] pub struct SwListResponse { - pub data: Vec, + pub data: Vec, } -#[derive(Debug, Deserialize)] -pub struct SwListEntity { - pub id: String, - pub r#type: String, - pub attributes: serde_json::Map, -} +pub type Entity = serde_json::Map; #[derive(Debug, Serialize)] pub struct Criteria { diff --git a/src/config.rs b/src/config.rs index b92df27..8d650f5 100644 --- a/src/config.rs +++ b/src/config.rs @@ -1,5 +1,6 @@ use crate::api::{CriteriaFilter, CriteriaSorting}; use serde::{Deserialize, Serialize}; +use std::collections::HashSet; #[derive(Debug, Clone, Serialize, Deserialize)] pub struct Credentials { @@ -15,6 +16,8 @@ pub struct Schema { pub filter: Vec, #[serde(default = "Vec::new")] pub sort: Vec, + #[serde(default = "HashSet::new")] + pub associations: HashSet, pub mappings: Vec, #[serde(default = "String::new")] pub serialize_script: String, diff --git a/src/data/export.rs b/src/data/export.rs index b137b94..ae9170b 100644 --- a/src/data/export.rs +++ b/src/data/export.rs @@ -7,6 +7,16 @@ use tokio::task::JoinHandle; /// Might block, so should be used with `task::spawn_blocking` pub async fn export(context: Arc) -> anyhow::Result<()> { + if !context.associations.is_empty() { + println!("Using associations: {:#?}", context.associations); + } + if !context.schema.filter.is_empty() { + println!("Using filter: {:#?}", context.schema.filter); + } + if !context.schema.sort.is_empty() { + println!("Using sort: {:#?}", context.schema.sort); + } + let mut total = context .sw_client .get_total(&context.schema.entity, &context.schema.filter) diff --git a/src/data/transform.rs b/src/data/transform.rs index def1c04..1fa7909 100644 --- a/src/data/transform.rs +++ b/src/data/transform.rs @@ -1,4 +1,4 @@ -use crate::api::SwListEntity; +use crate::api::Entity; use crate::config::Mapping; use crate::SyncContext; use anyhow::Context; @@ -88,6 +88,7 @@ pub fn deserialize_row( serde_json::Value::String(raw_value.to_owned()) }; + // ToDo: insert path must be considered and create child json objects (maps) entity.insert(path_mapping.entity_path.clone(), json_value); } Mapping::ByScript(_script_mapping) => { @@ -100,15 +101,12 @@ pub fn deserialize_row( } /// Serialize a single entity (as json object) into a single row (string columns) -pub fn serialize_entity( - entity: SwListEntity, - context: &SyncContext, -) -> anyhow::Result> { +pub fn serialize_entity(entity: Entity, context: &SyncContext) -> anyhow::Result> { let script_row = if let Some(serialize) = &context.scripting_environment.serialize { let engine = &context.scripting_environment.engine; let mut scope = Scope::new(); - let script_entity = rhai::serde::to_dynamic(&entity.attributes)?; + let script_entity = rhai::serde::to_dynamic(&entity)?; scope.push_dynamic("entity", script_entity); let row_dynamic = rhai::Map::new(); scope.push("row", row_dynamic); @@ -127,14 +125,12 @@ pub fn serialize_entity( for mapping in &context.schema.mappings { match mapping { Mapping::ByPath(path_mapping) => { - let value = match path_mapping.entity_path.as_ref() { - "id" => &serde_json::Value::String(entity.id.to_string()), - path => entity.attributes.get(path).context(format!( - "could not get field path '{}' specified in mapping, entity attributes:\n{}", - path, - serde_json::to_string_pretty(&entity.attributes).unwrap() - ))?, - }; + let value = entity.get_by_path(&path_mapping.entity_path) + .context(format!( + "could not get field path '{}' specified in mapping (you might try the optional chaining operator '?.' to fallback to null), entity attributes:\n{}", + path_mapping.entity_path, + serde_json::to_string_pretty(&entity).unwrap()) + )?; let value_str = match value { serde_json::Value::String(s) => s.clone(), @@ -231,3 +227,122 @@ fn get_base_engine() -> Engine { engine } + +trait EntityPath { + /// Search for a value inside a json object tree by a given path. + /// Example path `object.child.attribute` + /// Path with null return, if not existing: `object?.child?.attribute` + fn get_by_path(&self, path: &str) -> Option<&serde_json::Value>; + + /// Insert a value into a given path + fn insert_by_path(&mut self, path: &str, value: serde_json::Value); +} + +impl EntityPath for Entity { + // based on the pointer implementation in serde_json::Value + fn get_by_path(&self, path: &str) -> Option<&serde_json::Value> { + if path.is_empty() { + return None; + } + + let tokens = path.split('.'); + let mut optional_chain = tokens.clone().map(|token| token.ends_with('?')); + let mut tokens = tokens.map(|t| t.trim_end_matches('?')); + + // initial access happens on map + let first_token = tokens.next()?; + let first_optional = optional_chain.next()?; + let mut value = match self.get(first_token) { + Some(v) => v, + None => { + if first_optional { + return Some(&serde_json::Value::Null); + } else { + return None; + } + } + }; + + // the question mark refers to the current token and allows it to be undefined + // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Operators/Optional_chaining + for (token, is_optional) in tokens.zip(optional_chain) { + value = match value { + serde_json::Value::Object(map) => match map.get(token) { + Some(v) => v, + None => { + return if is_optional { + Some(&serde_json::Value::Null) + } else { + None + } + } + }, + serde_json::Value::Null => { + return Some(&serde_json::Value::Null); + } + _ => { + return None; + } + } + } + + Some(value) + } + + fn insert_by_path(&mut self, path: &str, value: serde_json::Value) { + if path.is_empty() { + panic!("empty entity_path encountered"); + } + + let mut tokens = path.split('.').map(|t| t.trim_end_matches('?')); + + let first_token = tokens.next().expect("has a value because non empty"); + + todo!("implement me and write tests") + } +} + +#[cfg(test)] +mod tests { + use crate::data::transform::EntityPath; + use serde_json::{json, Number, Value}; + + #[test] + fn test_get_by_path() { + let child = json!({ + "attribute": 42, + "hello": null + }); + let entity = json!({ + "child": child, + "fizz": "buzz", + "hello": null, + }); + + let entity = match entity { + Value::Object(map) => map, + _ => unreachable!(), + }; + + assert_eq!( + entity.get_by_path("fizz"), + Some(&Value::String("buzz".into())) + ); + assert_eq!(entity.get_by_path("child"), Some(&child)); + assert_eq!(entity.get_by_path("bar"), None); + assert_eq!( + entity.get_by_path("child.attribute"), + Some(&Value::Number(Number::from(42))) + ); + assert_eq!(entity.get_by_path("child.bar"), None); + assert_eq!(entity.get_by_path("child.fizz.bar"), None); + + // optional chaining + assert_eq!(entity.get_by_path("child?.bar"), None); + assert_eq!(entity.get_by_path("child?.bar?.fizz"), Some(&Value::Null)); + assert_eq!(entity.get_by_path("child?.attribute?.fizz"), None); // invalid access (attribute is not a map) + assert_eq!(entity.get_by_path("hello?.bar"), Some(&Value::Null)); + assert_eq!(entity.get_by_path("child.hello"), Some(&Value::Null)); + assert_eq!(entity.get_by_path("child.hello?.bar"), Some(&Value::Null)); + } +} diff --git a/src/main.rs b/src/main.rs index 04b1dd8..e0956b3 100644 --- a/src/main.rs +++ b/src/main.rs @@ -3,6 +3,7 @@ use crate::config::{Credentials, Mapping, Schema}; use crate::data::{export, import, prepare_scripting_environment, ScriptingEnvironment}; use anyhow::Context; use clap::{ArgAction, Parser, Subcommand}; +use std::collections::HashSet; use std::path::PathBuf; use std::sync::Arc; use std::time::Instant; @@ -79,7 +80,7 @@ pub struct SyncContext { pub limit: Option, pub verbose: bool, pub scripting_environment: ScriptingEnvironment, - pub associations: Vec, + pub associations: HashSet, } #[tokio::main] @@ -157,17 +158,14 @@ async fn create_context( .await .context("No provided schema file not found")?; let schema: Schema = serde_yaml::from_str(&serialized_schema)?; - let mut associations = vec![]; + let mut associations = schema.associations.clone(); for mapping in &schema.mappings { if let Mapping::ByPath(by_path) = mapping { if let Some((association, _field)) = by_path.entity_path.rsplit_once('.') { - associations.push(association.to_owned()); + associations.insert(association.trim_end_matches('?').to_owned()); } } } - if !associations.is_empty() { - println!("Detected associations: {:#?}", associations); - } let serialized_credentials = tokio::fs::read_to_string("./.credentials.toml") .await