Implement to-one-associations on export

README.md

@@ -60,6 +60,12 @@ sort:
   - field: "name"
     order: "ASC"
 
+# optional additional associations (that you need in your deserialization script)
+# note: entity_path associations are already added by default
+# only applied on export
+associations:
+  - "cover"
+
 # mappings can either be
 # - by entity_path
 # - by key
@@ -75,6 +81,16 @@ mappings:
     entity_path: "stock"
   - file_column: "tax id"
     entity_path: "taxId"
+  - file_column: "tax rate"
+    # entity path can resolve "To-One-Associations" of any depth
+    entity_path: "tax.taxRate"
+  - file_column: "manufacturer name"
+    # They can also use the optional chaining '?.' operator to fall back to null
+    # if the association is null
+    entity_path: "manufacturer?.name"
+  - file_column: "manufacturer id"
+    # for importing, you also need the association id in the association object
+    entity_path: "manufacturer?.id"
   - file_column: "gross price EUR"
     key: "gross_price_eur"
   - file_column: "net price EUR"

profiles/manufacturer.yaml

@@ -9,3 +9,5 @@ mappings:
     entity_path: "description"
   - file_column: "website"
     entity_path: "link"
+  - file_column: "logo"
+    entity_path: "media?.url"

src/api.rs

@@ -1,6 +1,7 @@
 use crate::config::Credentials;
 use anyhow::anyhow;
-use reqwest::{Client, Response, StatusCode};
+use reqwest::header::{HeaderMap, HeaderValue};
+use reqwest::{header, Client, Response, StatusCode};
 use serde::{Deserialize, Serialize};
 use serde_json::json;
 use std::collections::BTreeMap;
@@ -20,7 +21,14 @@ pub struct SwClient {
 
 impl SwClient {
     pub async fn new(credentials: Credentials, in_flight_limit: usize) -> anyhow::Result<Self> {
-        let client = Client::builder().timeout(Duration::from_secs(10)).build()?;
+        let mut default_headers = HeaderMap::default();
+        // This header is needed, otherwise the response would be "application/vnd.api+json" (by default)
+        // and that doesn't have the association data as part of the entity object
+        default_headers.insert(header::ACCEPT, HeaderValue::from_static("application/json"));
+        let client = Client::builder()
+            .timeout(Duration::from_secs(10))
+            .default_headers(default_headers)
+            .build()?;
         let credentials = Arc::new(credentials);
         let auth_response = Self::authenticate(&client, credentials.as_ref()).await?;
 
@@ -311,15 +319,10 @@ pub struct SwErrorSource {
 
 #[derive(Debug, Deserialize)]
 pub struct SwListResponse {
-    pub data: Vec<SwListEntity>,
+    pub data: Vec<Entity>,
 }
 
-#[derive(Debug, Deserialize)]
-pub struct SwListEntity {
-    pub id: String,
-    pub r#type: String,
-    pub attributes: serde_json::Map<String, serde_json::Value>,
-}
+pub type Entity = serde_json::Map<String, serde_json::Value>;
 
 #[derive(Debug, Serialize)]
 pub struct Criteria {

src/config.rs

@@ -1,5 +1,6 @@
 use crate::api::{CriteriaFilter, CriteriaSorting};
 use serde::{Deserialize, Serialize};
+use std::collections::HashSet;
 
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct Credentials {
@@ -15,6 +16,8 @@ pub struct Schema {
     pub filter: Vec<CriteriaFilter>,
     #[serde(default = "Vec::new")]
     pub sort: Vec<CriteriaSorting>,
+    #[serde(default = "HashSet::new")]
+    pub associations: HashSet<String>,
     pub mappings: Vec<Mapping>,
     #[serde(default = "String::new")]
     pub serialize_script: String,

src/data/export.rs

@@ -7,6 +7,16 @@ use tokio::task::JoinHandle;
 
 /// Might block, so should be used with `task::spawn_blocking`
 pub async fn export(context: Arc<SyncContext>) -> anyhow::Result<()> {
+    if !context.associations.is_empty() {
+        println!("Using associations: {:#?}", context.associations);
+    }
+    if !context.schema.filter.is_empty() {
+        println!("Using filter: {:#?}", context.schema.filter);
+    }
+    if !context.schema.sort.is_empty() {
+        println!("Using sort: {:#?}", context.schema.sort);
+    }
+
     let mut total = context
         .sw_client
         .get_total(&context.schema.entity, &context.schema.filter)

src/data/transform.rs

@@ -1,4 +1,4 @@
-use crate::api::SwListEntity;
+use crate::api::Entity;
 use crate::config::Mapping;
 use crate::SyncContext;
 use anyhow::Context;
@@ -88,6 +88,7 @@ pub fn deserialize_row(
                     serde_json::Value::String(raw_value.to_owned())
                 };
 
+                // ToDo: insert path must be considered and create child json objects (maps)
                 entity.insert(path_mapping.entity_path.clone(), json_value);
             }
             Mapping::ByScript(_script_mapping) => {
@@ -100,15 +101,12 @@ pub fn deserialize_row(
 }
 
 /// Serialize a single entity (as json object) into a single row (string columns)
-pub fn serialize_entity(
-    entity: SwListEntity,
-    context: &SyncContext,
-) -> anyhow::Result<Vec<String>> {
+pub fn serialize_entity(entity: Entity, context: &SyncContext) -> anyhow::Result<Vec<String>> {
     let script_row = if let Some(serialize) = &context.scripting_environment.serialize {
         let engine = &context.scripting_environment.engine;
 
         let mut scope = Scope::new();
-        let script_entity = rhai::serde::to_dynamic(&entity.attributes)?;
+        let script_entity = rhai::serde::to_dynamic(&entity)?;
         scope.push_dynamic("entity", script_entity);
         let row_dynamic = rhai::Map::new();
         scope.push("row", row_dynamic);
@@ -127,14 +125,12 @@ pub fn serialize_entity(
     for mapping in &context.schema.mappings {
         match mapping {
             Mapping::ByPath(path_mapping) => {
-                let value = match path_mapping.entity_path.as_ref() {
-                    "id" => &serde_json::Value::String(entity.id.to_string()),
-                    path => entity.attributes.get(path).context(format!(
-                        "could not get field path '{}' specified in mapping, entity attributes:\n{}",
-                        path,
-                        serde_json::to_string_pretty(&entity.attributes).unwrap()
-                    ))?,
-                };
+                let value = entity.get_by_path(&path_mapping.entity_path)
+                    .context(format!(
+                        "could not get field path '{}' specified in mapping (you might try the optional chaining operator '?.' to fallback to null), entity attributes:\n{}",
+                        path_mapping.entity_path,
+                        serde_json::to_string_pretty(&entity).unwrap())
+                    )?;
 
                 let value_str = match value {
                     serde_json::Value::String(s) => s.clone(),
@@ -231,3 +227,122 @@ fn get_base_engine() -> Engine {
 
     engine
 }
+
+trait EntityPath {
+    /// Search for a value inside a json object tree by a given path.
+    /// Example path `object.child.attribute`
+    /// Path with null return, if not existing: `object?.child?.attribute`
+    fn get_by_path(&self, path: &str) -> Option<&serde_json::Value>;
+
+    /// Insert a value into a given path
+    fn insert_by_path(&mut self, path: &str, value: serde_json::Value);
+}
+
+impl EntityPath for Entity {
+    // based on the pointer implementation in serde_json::Value
+    fn get_by_path(&self, path: &str) -> Option<&serde_json::Value> {
+        if path.is_empty() {
+            return None;
+        }
+
+        let tokens = path.split('.');
+        let mut optional_chain = tokens.clone().map(|token| token.ends_with('?'));
+        let mut tokens = tokens.map(|t| t.trim_end_matches('?'));
+
+        // initial access happens on map
+        let first_token = tokens.next()?;
+        let first_optional = optional_chain.next()?;
+        let mut value = match self.get(first_token) {
+            Some(v) => v,
+            None => {
+                if first_optional {
+                    return Some(&serde_json::Value::Null);
+                } else {
+                    return None;
+                }
+            }
+        };
+
+        // the question mark refers to the current token and allows it to be undefined
+        // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Operators/Optional_chaining
+        for (token, is_optional) in tokens.zip(optional_chain) {
+            value = match value {
+                serde_json::Value::Object(map) => match map.get(token) {
+                    Some(v) => v,
+                    None => {
+                        return if is_optional {
+                            Some(&serde_json::Value::Null)
+                        } else {
+                            None
+                        }
+                    }
+                },
+                serde_json::Value::Null => {
+                    return Some(&serde_json::Value::Null);
+                }
+                _ => {
+                    return None;
+                }
+            }
+        }
+
+        Some(value)
+    }
+
+    fn insert_by_path(&mut self, path: &str, value: serde_json::Value) {
+        if path.is_empty() {
+            panic!("empty entity_path encountered");
+        }
+
+        let mut tokens = path.split('.').map(|t| t.trim_end_matches('?'));
+
+        let first_token = tokens.next().expect("has a value because non empty");
+
+        todo!("implement me and write tests")
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::data::transform::EntityPath;
+    use serde_json::{json, Number, Value};
+
+    #[test]
+    fn test_get_by_path() {
+        let child = json!({
+            "attribute": 42,
+            "hello": null
+        });
+        let entity = json!({
+            "child": child,
+            "fizz": "buzz",
+            "hello": null,
+        });
+
+        let entity = match entity {
+            Value::Object(map) => map,
+            _ => unreachable!(),
+        };
+
+        assert_eq!(
+            entity.get_by_path("fizz"),
+            Some(&Value::String("buzz".into()))
+        );
+        assert_eq!(entity.get_by_path("child"), Some(&child));
+        assert_eq!(entity.get_by_path("bar"), None);
+        assert_eq!(
+            entity.get_by_path("child.attribute"),
+            Some(&Value::Number(Number::from(42)))
+        );
+        assert_eq!(entity.get_by_path("child.bar"), None);
+        assert_eq!(entity.get_by_path("child.fizz.bar"), None);
+
+        // optional chaining
+        assert_eq!(entity.get_by_path("child?.bar"), None);
+        assert_eq!(entity.get_by_path("child?.bar?.fizz"), Some(&Value::Null));
+        assert_eq!(entity.get_by_path("child?.attribute?.fizz"), None); // invalid access (attribute is not a map)
+        assert_eq!(entity.get_by_path("hello?.bar"), Some(&Value::Null));
+        assert_eq!(entity.get_by_path("child.hello"), Some(&Value::Null));
+        assert_eq!(entity.get_by_path("child.hello?.bar"), Some(&Value::Null));
+    }
+}

src/main.rs

@@ -3,6 +3,7 @@ use crate::config::{Credentials, Mapping, Schema};
 use crate::data::{export, import, prepare_scripting_environment, ScriptingEnvironment};
 use anyhow::Context;
 use clap::{ArgAction, Parser, Subcommand};
+use std::collections::HashSet;
 use std::path::PathBuf;
 use std::sync::Arc;
 use std::time::Instant;
@@ -79,7 +80,7 @@ pub struct SyncContext {
     pub limit: Option<u64>,
     pub verbose: bool,
     pub scripting_environment: ScriptingEnvironment,
-    pub associations: Vec<String>,
+    pub associations: HashSet<String>,
 }
 
 #[tokio::main]
@@ -157,17 +158,14 @@ async fn create_context(
         .await
         .context("No provided schema file not found")?;
     let schema: Schema = serde_yaml::from_str(&serialized_schema)?;
-    let mut associations = vec![];
+    let mut associations = schema.associations.clone();
     for mapping in &schema.mappings {
         if let Mapping::ByPath(by_path) = mapping {
             if let Some((association, _field)) = by_path.entity_path.rsplit_once('.') {
-                associations.push(association.to_owned());
+                associations.insert(association.trim_end_matches('?').to_owned());
             }
         }
     }
-    if !associations.is_empty() {
-        println!("Detected associations: {:#?}", associations);
-    }
 
     let serialized_credentials = tokio::fs::read_to_string("./.credentials.toml")
         .await