From c2b4dc37392fa21caf41238a31f1d4f97fc7f240 Mon Sep 17 00:00:00 2001 From: Malte Janz Date: Wed, 3 Jul 2024 17:17:44 +0200 Subject: [PATCH] Some refactoring for readability --- src/api/filter.rs | 2 + src/api/mod.rs | 8 +- src/cli.rs | 64 ++++++ src/{config.rs => config_file.rs} | 14 ++ src/data/export.rs | 3 + src/data/import.rs | 35 ++-- src/data/mod.rs | 5 +- src/data/{transform.rs => transform/mod.rs} | 212 +++----------------- src/data/transform/script.rs | 200 ++++++++++++++++++ src/data/validate.rs | 88 ++++---- src/main.rs | 67 +------ 11 files changed, 380 insertions(+), 318 deletions(-) create mode 100644 src/cli.rs rename src/{config.rs => config_file.rs} (81%) rename src/data/{transform.rs => transform/mod.rs} (61%) create mode 100644 src/data/transform/script.rs diff --git a/src/api/filter.rs b/src/api/filter.rs index a13fb12..05123b4 100644 --- a/src/api/filter.rs +++ b/src/api/filter.rs @@ -1,3 +1,5 @@ +//! Data structures to build criteria objects for the shopware API + use serde::{Deserialize, Serialize}; use std::collections::BTreeMap; diff --git a/src/api/mod.rs b/src/api/mod.rs index 3ef581b..1cef6f3 100644 --- a/src/api/mod.rs +++ b/src/api/mod.rs @@ -1,7 +1,9 @@ +//! Everything needed for communicating with the Shopware API + pub mod filter; use crate::api::filter::{Criteria, CriteriaFilter}; -use crate::config::Credentials; +use crate::config_file::Credentials; use anyhow::anyhow; use reqwest::header::{HeaderMap, HeaderValue}; use reqwest::{header, Client, Response, StatusCode}; @@ -98,9 +100,7 @@ impl SwClient { Ok(()) } - pub async fn entity_schema( - &self, - ) -> Result, SwApiError> { + pub async fn entity_schema(&self) -> Result { // ToDo: implement retry on auth fail let access_token = self.access_token.lock().unwrap().clone(); let response = { diff --git a/src/cli.rs b/src/cli.rs new file mode 100644 index 0000000..c229554 --- /dev/null +++ b/src/cli.rs @@ -0,0 +1,64 @@ +//! Definitions for the CLI commands, arguments and help texts +//! +//! Makes heavy use of https://docs.rs/clap/latest/clap/ + +use clap::{Parser, Subcommand}; +use std::path::PathBuf; + +#[derive(Parser)] +#[command(version, about, long_about = None)] +pub struct Cli { + #[command(subcommand)] + pub command: Commands, +} + +#[derive(Subcommand)] +pub enum Commands { + /// Authenticate with a given shopware shop via integration admin API. + /// Credentials are stored in .credentials.toml in the current working directory. + Auth { + /// base URL of the shop + #[arg(short, long)] + domain: String, + + /// access_key_id + #[arg(short, long)] + id: String, + + /// access_key_secret + #[arg(short, long)] + secret: String, + }, + + /// Import data into shopware or export data to a file + Sync { + /// Mode (import or export) + #[arg(value_enum, short, long)] + mode: SyncMode, + + /// Path to profile schema.yaml + #[arg(short, long)] + schema: PathBuf, + + /// Path to data file + #[arg(short, long)] + file: PathBuf, + + /// Maximum amount of entities, can be used for debugging + #[arg(short, long)] + limit: Option, + + // Verbose output, used for debugging + // #[arg(short, long, action = ArgAction::SetTrue)] + // verbose: bool, + /// How many requests can be "in-flight" at the same time + #[arg(short, long, default_value = "8")] + in_flight_limit: usize, + }, +} + +#[derive(Debug, Clone, Copy, clap::ValueEnum)] +pub enum SyncMode { + Import, + Export, +} diff --git a/src/config.rs b/src/config_file.rs similarity index 81% rename from src/config.rs rename to src/config_file.rs index 0d73ca9..a88205f 100644 --- a/src/config.rs +++ b/src/config_file.rs @@ -1,3 +1,10 @@ +//! Definitions for the `schema.yaml` and `.credentials.toml` files +//! +//! Allows deserialization into a proper typed structure from these files +//! or also write these typed structures to a file (in case of `.credentials.toml`) +//! +//! Utilizes https://serde.rs/ + use crate::api::filter::{CriteriaFilter, CriteriaSorting}; use serde::{Deserialize, Serialize}; use std::collections::HashSet; @@ -12,15 +19,22 @@ pub struct Credentials { #[derive(Debug, Deserialize)] pub struct Schema { pub entity: String, + #[serde(default = "Vec::new")] pub filter: Vec, + #[serde(default = "Vec::new")] pub sort: Vec, + + /// Are unique thanks to `HashSet` #[serde(default = "HashSet::new")] pub associations: HashSet, + pub mappings: Vec, + #[serde(default = "String::new")] pub serialize_script: String, + #[serde(default = "String::new")] pub deserialize_script: String, } diff --git a/src/data/export.rs b/src/data/export.rs index 2d0ca4a..7816925 100644 --- a/src/data/export.rs +++ b/src/data/export.rs @@ -1,3 +1,5 @@ +//! Everything related to exporting data out of shopware + use crate::api::filter::Criteria; use crate::data::transform::serialize_entity; use crate::SyncContext; @@ -68,6 +70,7 @@ async fn write_to_file( csv_writer.write_record(get_header_line(context))?; for handle in worker_handles { + // ToDo: we might want to handle the errors more gracefully here and don't stop on first error let (page, rows) = handle.await??; println!("writing page {}", page); diff --git a/src/data/import.rs b/src/data/import.rs index 58b7cb5..ba631ca 100644 --- a/src/data/import.rs +++ b/src/data/import.rs @@ -1,6 +1,9 @@ -use crate::api::{SwApiError, SyncAction}; +//! Everything related to import data into shopware + +use crate::api::{Entity, SwApiError, SyncAction}; use crate::data::transform::deserialize_row; use crate::SyncContext; +use anyhow::anyhow; use itertools::Itertools; use std::sync::Arc; @@ -11,27 +14,32 @@ pub async fn import(context: Arc) -> anyhow::Result<()> { .from_path(&context.file)?; let headers = csv_reader.headers()?.clone(); + // create an iterator, that processes (CSV) rows (StringRecord) into (usize, anyhow::Result) + // where the former is the row index let iter = csv_reader .into_records() - .map(|r| { - let result = r.expect("failed reading CSV row"); - - deserialize_row(&headers, result, &context).expect("deserialize failed") - // ToDo improve error handling + .map(|r| match r { + Ok(row) => deserialize_row(&headers, row, &context), + Err(e) => Err(anyhow!(e)), }) .enumerate() .take(context.limit.unwrap_or(u64::MAX) as usize); + // iterate in chunks of 500 or less let mut join_handles = vec![]; for sync_values in &iter.chunks(500) { - let (mut row_indices, mut chunk): ( - Vec, - Vec>, - ) = sync_values.unzip(); + let (mut row_indices, chunk): (Vec, Vec>) = + sync_values.unzip(); + + // for now fail on first invalid row + // currently the most likely deserialization failure is not finding the column / CSV header + // ToDo: we might want to handle the errors more gracefully here and don't stop on first error + let mut valid_chunk = chunk.into_iter().collect::>>()?; + // submit sync task let context = Arc::clone(&context); join_handles.push(tokio::spawn(async move { - match context.sw_client.sync(&context.schema.entity, SyncAction::Upsert, &chunk).await { + match context.sw_client.sync(&context.schema.entity, SyncAction::Upsert, &valid_chunk).await { Ok(()) => Ok(()), Err(SwApiError::Server(_, body)) => { for err in body.errors.iter().rev() { @@ -40,7 +48,7 @@ pub async fn import(context: Arc) -> anyhow::Result<()> { let entry: usize = entry_str.parse().expect("error pointer should contain usize"); let row_index = row_indices.remove(entry); - let row = chunk.remove(entry); + let row = valid_chunk.remove(entry); println!( "server validation error on row {}: {} Remaining pointer '{}' ignored payload:\n{}", row_index + 2, @@ -50,13 +58,14 @@ pub async fn import(context: Arc) -> anyhow::Result<()> { ); } // retry - context.sw_client.sync(&context.schema.entity, SyncAction::Upsert, &chunk).await + context.sw_client.sync(&context.schema.entity, SyncAction::Upsert, &valid_chunk).await }, Err(e) => Err(e), } })); } + // wait for all the sync tasks to finish for join_handle in join_handles { join_handle.await??; } diff --git a/src/data/mod.rs b/src/data/mod.rs index 85a029b..8aa229d 100644 --- a/src/data/mod.rs +++ b/src/data/mod.rs @@ -3,8 +3,9 @@ mod import; mod transform; mod validate; +// reexport the important functions / structs as part of this module pub use export::export; pub use import::import; -pub use transform::prepare_scripting_environment; -pub use transform::ScriptingEnvironment; +pub use transform::script::prepare_scripting_environment; +pub use transform::script::ScriptingEnvironment; pub use validate::validate_paths_for_entity; diff --git a/src/data/transform.rs b/src/data/transform/mod.rs similarity index 61% rename from src/data/transform.rs rename to src/data/transform/mod.rs index 6902159..b5ccfef 100644 --- a/src/data/transform.rs +++ b/src/data/transform/mod.rs @@ -1,64 +1,24 @@ +//! Everything related to data transformations + +pub mod script; + use crate::api::Entity; -use crate::config::Mapping; +use crate::config_file::Mapping; use crate::SyncContext; use anyhow::Context; use csv::StringRecord; -use rhai::packages::{BasicArrayPackage, CorePackage, MoreStringPackage, Package}; -use rhai::{Engine, Position, Scope, AST}; use std::str::FromStr; -/// Deserialize a single row of the input file into a json object +/// Deserialize a single row of the input (CSV) file into a json object pub fn deserialize_row( headers: &StringRecord, row: StringRecord, context: &SyncContext, -) -> anyhow::Result> { - let mut entity = if let Some(deserialize) = &context.scripting_environment.deserialize { - let engine = &context.scripting_environment.engine; - - let mut scope = Scope::new(); - - // build row object - let mut script_row = rhai::Map::new(); - let script_mappings = context.schema.mappings.iter().filter_map(|m| match m { - Mapping::ByScript(s) => Some(s), - _ => None, - }); - for mapping in script_mappings { - let column_index = headers - .iter() - .position(|h| h == mapping.file_column) - .context(format!( - "Can't find column '{}' in CSV headers", - mapping.file_column - ))?; - - let value = row - .get(column_index) - .context("failed to get column of row")?; - - script_row.insert(mapping.key.as_str().into(), value.into()); - } - - scope.push_constant("row", script_row); - let entity_dynamic = rhai::Map::new(); - scope.push("entity", entity_dynamic); - - engine.run_ast_with_scope(&mut scope, deserialize)?; - - let row_result: rhai::Map = scope - .get_value("entity") - .expect("row should exist in script scope"); - let mut entity_after_script = serde_json::Map::with_capacity(context.schema.mappings.len()); - for (key, value) in row_result { - let json_value: serde_json::Value = rhai::serde::from_dynamic(&value)?; - entity_after_script.insert(key.to_string(), json_value); - } - - entity_after_script - } else { - serde_json::Map::with_capacity(context.schema.mappings.len()) - }; +) -> anyhow::Result { + // Either run deserialize script or create initial empty entity object + let mut entity = context + .scripting_environment + .run_deserialize(headers, &row, context)?; for mapping in &context.schema.mappings { match mapping { @@ -74,19 +34,7 @@ pub fn deserialize_row( let raw_value = row .get(column_index) .context("failed to get column of row")?; - let raw_value_lowercase = raw_value.to_lowercase(); - - let json_value = if raw_value_lowercase == "null" || raw_value.trim().is_empty() { - serde_json::Value::Null - } else if raw_value_lowercase == "true" { - serde_json::Value::Bool(true) - } else if raw_value_lowercase == "false" { - serde_json::Value::Bool(false) - } else if let Ok(number) = serde_json::Number::from_str(raw_value) { - serde_json::Value::Number(number) - } else { - serde_json::Value::String(raw_value.to_owned()) - }; + let json_value = get_json_value_from_string(raw_value); entity.insert_by_path(&path_mapping.entity_path, json_value); } @@ -101,26 +49,9 @@ pub fn deserialize_row( /// Serialize a single entity (as json object) into a single row (string columns) pub fn serialize_entity(entity: Entity, context: &SyncContext) -> anyhow::Result> { - let script_row = if let Some(serialize) = &context.scripting_environment.serialize { - let engine = &context.scripting_environment.engine; - - let mut scope = Scope::new(); - let script_entity = rhai::serde::to_dynamic(&entity)?; - scope.push_dynamic("entity", script_entity); - let row_dynamic = rhai::Map::new(); - scope.push("row", row_dynamic); - - engine.run_ast_with_scope(&mut scope, serialize)?; - - let row_result: rhai::Map = scope - .get_value("row") - .expect("row should exist in script scope"); - row_result - } else { - rhai::Map::new() - }; - + let script_row = context.scripting_environment.run_serialize(&entity)?; let mut row = Vec::with_capacity(context.schema.mappings.len()); + for mapping in &context.schema.mappings { match mapping { Mapping::ByPath(path_mapping) => { @@ -155,111 +86,18 @@ pub fn serialize_entity(entity: Entity, context: &SyncContext) -> anyhow::Result Ok(row) } -#[derive(Debug)] -pub struct ScriptingEnvironment { - engine: Engine, - serialize: Option, - deserialize: Option, -} - -pub fn prepare_scripting_environment( - raw_serialize_script: &str, - raw_deserialize_script: &str, -) -> anyhow::Result { - let engine = get_base_engine(); - let serialize_ast = if !raw_serialize_script.is_empty() { - let ast = engine - .compile(raw_serialize_script) - .context("serialize_script compilation failed")?; - Some(ast) - } else { - None - }; - let deserialize_ast = if !raw_deserialize_script.is_empty() { - let ast = engine - .compile(raw_deserialize_script) - .context("serialize_script compilation failed")?; - Some(ast) +fn get_json_value_from_string(raw_input: &str) -> serde_json::Value { + let raw_input_lowercase = raw_input.to_lowercase(); + if raw_input_lowercase == "null" || raw_input.trim().is_empty() { + serde_json::Value::Null + } else if raw_input_lowercase == "true" { + serde_json::Value::Bool(true) + } else if raw_input_lowercase == "false" { + serde_json::Value::Bool(false) + } else if let Ok(number) = serde_json::Number::from_str(raw_input) { + serde_json::Value::Number(number) } else { - None - }; - - Ok(ScriptingEnvironment { - engine, - serialize: serialize_ast, - deserialize: deserialize_ast, - }) -} - -fn get_base_engine() -> Engine { - let mut engine = Engine::new_raw(); - // Default print/debug implementations - engine.on_print(|text| println!("{text}")); - engine.on_debug(|text, source, pos| match (source, pos) { - (Some(source), Position::NONE) => println!("{source} | {text}"), - (Some(source), pos) => println!("{source} @ {pos:?} | {text}"), - (None, Position::NONE) => println!("{text}"), - (None, pos) => println!("{pos:?} | {text}"), - }); - - let core_package = CorePackage::new(); - core_package.register_into_engine(&mut engine); - let string_package = MoreStringPackage::new(); - string_package.register_into_engine(&mut engine); - let array_package = BasicArrayPackage::new(); - array_package.register_into_engine(&mut engine); - - // ToDo: add custom utility functions to engine - engine.register_fn("get_default", script::get_default); - - // Some reference implementations below - /* - engine.register_type::(); - engine.register_fn("uuid", scripts::uuid); - engine.register_fn("uuidFromStr", scripts::uuid_from_str); - - engine.register_type::(); - engine.register_fn("map", scripts::Mapper::map); - engine.register_fn("get", scripts::Mapper::get); - - engine.register_type::(); - engine.register_fn("fetchFirst", scripts::DB::fetch_first); - */ - - engine -} - -/// utilities for inside scripts -mod script { - /// Imitate - /// https://github.com/shopware/shopware/blob/03cfe8cca937e6e45c9c3e15821d1449dfd01d82/src/Core/Defaults.php - pub fn get_default(name: &str) -> String { - match name { - "LANGUAGE_SYSTEM" => "2fbb5fe2e29a4d70aa5854ce7ce3e20b".to_string(), - "LIVE_VERSION" => "0fa91ce3e96a4bc2be4bd9ce752c3425".to_string(), - "CURRENCY" => "b7d2554b0ce847cd82f3ac9bd1c0dfca".to_string(), - "SALES_CHANNEL_TYPE_API" => "f183ee5650cf4bdb8a774337575067a6".to_string(), - "SALES_CHANNEL_TYPE_STOREFRONT" => "8a243080f92e4c719546314b577cf82b".to_string(), - "SALES_CHANNEL_TYPE_PRODUCT_COMPARISON" => "ed535e5722134ac1aa6524f73e26881b".to_string(), - "STORAGE_DATE_TIME_FORMAT" => "Y-m-d H:i:s.v".to_string(), - "STORAGE_DATE_FORMAT" => "Y-m-d".to_string(), - "CMS_PRODUCT_DETAIL_PAGE" => "7a6d253a67204037966f42b0119704d5".to_string(), - n => panic!( - "get_default called with '{}' but there is no such definition. Have a look into Shopware/src/Core/Defaults.php. Available constants: {:?}", - n, - vec![ - "LANGUAGE_SYSTEM", - "LIVE_VERSION", - "CURRENCY", - "SALES_CHANNEL_TYPE_API", - "SALES_CHANNEL_TYPE_STOREFRONT", - "SALES_CHANNEL_TYPE_PRODUCT_COMPARISON", - "STORAGE_DATE_TIME_FORMAT", - "STORAGE_DATE_FORMAT", - "CMS_PRODUCT_DETAIL_PAGE", - ] - ) - } + serde_json::Value::String(raw_input.to_owned()) } } diff --git a/src/data/transform/script.rs b/src/data/transform/script.rs new file mode 100644 index 0000000..f5190be --- /dev/null +++ b/src/data/transform/script.rs @@ -0,0 +1,200 @@ +//! Everything scripting related + +use crate::api::Entity; +use crate::config_file::Mapping; +use crate::SyncContext; +use anyhow::Context; +use csv::StringRecord; +use rhai::packages::{BasicArrayPackage, CorePackage, MoreStringPackage, Package}; +use rhai::{Engine, Position, Scope, AST}; + +#[derive(Debug)] +pub struct ScriptingEnvironment { + pub engine: Engine, + pub serialize: Option, + pub deserialize: Option, +} + +impl ScriptingEnvironment { + /// Just returns a default value if there is no script + pub fn run_deserialize( + &self, + headers: &StringRecord, + row: &StringRecord, + context: &SyncContext, + ) -> anyhow::Result { + let Some(deserialize_script) = &self.deserialize else { + return Ok(Entity::with_capacity(context.schema.mappings.len())); + }; + + // build row object + let mut script_row = rhai::Map::new(); + let script_mappings = context.schema.mappings.iter().filter_map(|m| match m { + Mapping::ByScript(s) => Some(s), + _ => None, + }); + for mapping in script_mappings { + let column_index = headers + .iter() + .position(|h| h == mapping.file_column) + .context(format!( + "Can't find column '{}' in CSV headers", + mapping.file_column + ))?; + + let value = row + .get(column_index) + .context("failed to get column of row")?; + + script_row.insert(mapping.key.as_str().into(), value.into()); + } + + // run the script + let mut scope = Scope::new(); + scope.push_constant("row", script_row); + let entity_dynamic = rhai::Map::new(); + scope.push("entity", entity_dynamic); + + self.engine + .run_ast_with_scope(&mut scope, deserialize_script)?; + + // get the entity out of the script + let row_result: rhai::Map = scope + .get_value("entity") + .expect("row should exist in script scope"); + let mut entity_after_script = Entity::with_capacity(context.schema.mappings.len()); + for (key, value) in row_result { + let json_value: serde_json::Value = rhai::serde::from_dynamic(&value)?; + entity_after_script.insert(key.to_string(), json_value); + } + + Ok(entity_after_script) + } + + /// Just returns a default value if there is no script + pub fn run_serialize(&self, entity: &Entity) -> anyhow::Result { + let Some(serialize_script) = &self.serialize else { + return Ok(rhai::Map::new()); + }; + + let mut scope = Scope::new(); + let script_entity = rhai::serde::to_dynamic(entity)?; + scope.push_dynamic("entity", script_entity); + let row_dynamic = rhai::Map::new(); + scope.push("row", row_dynamic); + + self.engine + .run_ast_with_scope(&mut scope, serialize_script)?; + + let row_result: rhai::Map = scope + .get_value("row") + .expect("row should exist in script scope"); + Ok(row_result) + } +} + +pub fn prepare_scripting_environment( + raw_serialize_script: &str, + raw_deserialize_script: &str, +) -> anyhow::Result { + let engine = get_base_engine(); + let serialize_ast = if !raw_serialize_script.is_empty() { + let ast = engine + .compile(raw_serialize_script) + .context("serialize_script compilation failed")?; + Some(ast) + } else { + None + }; + let deserialize_ast = if !raw_deserialize_script.is_empty() { + let ast = engine + .compile(raw_deserialize_script) + .context("serialize_script compilation failed")?; + Some(ast) + } else { + None + }; + + Ok(ScriptingEnvironment { + engine, + serialize: serialize_ast, + deserialize: deserialize_ast, + }) +} + +fn get_base_engine() -> Engine { + let mut engine = Engine::new_raw(); + // Default print/debug implementations + engine.on_print(|text| println!("{text}")); + engine.on_debug(|text, source, pos| match (source, pos) { + (Some(source), Position::NONE) => println!("{source} | {text}"), + (Some(source), pos) => println!("{source} @ {pos:?} | {text}"), + (None, Position::NONE) => println!("{text}"), + (None, pos) => println!("{pos:?} | {text}"), + }); + + let core_package = CorePackage::new(); + core_package.register_into_engine(&mut engine); + let string_package = MoreStringPackage::new(); + string_package.register_into_engine(&mut engine); + let array_package = BasicArrayPackage::new(); + array_package.register_into_engine(&mut engine); + + // ToDo: add custom utility functions to engine + engine.register_fn("get_default", inside_script::get_default); + + // Some reference implementations below + /* + engine.register_type::(); + engine.register_fn("uuid", scripts::uuid); + engine.register_fn("uuidFromStr", scripts::uuid_from_str); + + engine.register_type::(); + engine.register_fn("map", scripts::Mapper::map); + engine.register_fn("get", scripts::Mapper::get); + + engine.register_type::(); + engine.register_fn("fetchFirst", scripts::DB::fetch_first); + */ + + engine +} + +/// Utilities for inside scripts +/// +/// Important, don't use the type `String` as function parameters, see +/// https://rhai.rs/book/rust/strings.html +mod inside_script { + use rhai::ImmutableString; + + /// Imitate + /// https://github.com/shopware/shopware/blob/03cfe8cca937e6e45c9c3e15821d1449dfd01d82/src/Core/Defaults.php + pub fn get_default(name: &str) -> ImmutableString { + match name { + "LANGUAGE_SYSTEM" => "2fbb5fe2e29a4d70aa5854ce7ce3e20b".into(), + "LIVE_VERSION" => "0fa91ce3e96a4bc2be4bd9ce752c3425".into(), + "CURRENCY" => "b7d2554b0ce847cd82f3ac9bd1c0dfca".into(), + "SALES_CHANNEL_TYPE_API" => "f183ee5650cf4bdb8a774337575067a6".into(), + "SALES_CHANNEL_TYPE_STOREFRONT" => "8a243080f92e4c719546314b577cf82b".into(), + "SALES_CHANNEL_TYPE_PRODUCT_COMPARISON" => "ed535e5722134ac1aa6524f73e26881b".into(), + "STORAGE_DATE_TIME_FORMAT" => "Y-m-d H:i:s.v".into(), + "STORAGE_DATE_FORMAT" => "Y-m-d".into(), + "CMS_PRODUCT_DETAIL_PAGE" => "7a6d253a67204037966f42b0119704d5".into(), + n => panic!( + "get_default called with '{}' but there is no such definition. Have a look into Shopware/src/Core/Defaults.php. Available constants: {:?}", + n, + vec![ + "LANGUAGE_SYSTEM", + "LIVE_VERSION", + "CURRENCY", + "SALES_CHANNEL_TYPE_API", + "SALES_CHANNEL_TYPE_STOREFRONT", + "SALES_CHANNEL_TYPE_PRODUCT_COMPARISON", + "STORAGE_DATE_TIME_FORMAT", + "STORAGE_DATE_FORMAT", + "CMS_PRODUCT_DETAIL_PAGE", + ] + ) + } + } +} diff --git a/src/data/validate.rs b/src/data/validate.rs index cbfcfcb..3f94417 100644 --- a/src/data/validate.rs +++ b/src/data/validate.rs @@ -1,10 +1,11 @@ -use crate::config::Mapping; +use crate::api::Entity; +use crate::config_file::{EntityPathMapping, Mapping}; /// Validate paths for entity pub fn validate_paths_for_entity( entity: &str, mappings: &Vec, - api_schema: &serde_json::Map, + api_schema: &Entity, ) -> anyhow::Result<()> { // if entity name is not set in api_schema throw an exception if !api_schema.contains_key(entity) { @@ -46,7 +47,7 @@ pub fn validate_paths_for_entity( let path = path[1..].join("."); // create a new mapping with the new path - let mapping = Mapping::ByPath(crate::config::EntityPathMapping { + let mapping = Mapping::ByPath(EntityPathMapping { file_column: path_mapping.file_column.clone(), entity_path: path, }); @@ -60,17 +61,16 @@ pub fn validate_paths_for_entity( #[cfg(test)] mod tests { + use crate::config_file::{EntityPathMapping, Mapping}; use serde_json::json; #[test] fn validate_non_existent_entity() { let entity = "nonexistent"; - let mapping = vec![crate::config::Mapping::ByPath( - crate::config::EntityPathMapping { - file_column: "manufacturer id".to_string(), - entity_path: "manufacturerId".to_string(), - }, - )]; + let mapping = vec![Mapping::ByPath(EntityPathMapping { + file_column: "manufacturer id".to_string(), + entity_path: "manufacturerId".to_string(), + })]; let api_schema = json!({ "product": { } @@ -90,12 +90,10 @@ mod tests { #[test] fn validate_non_existent_simple_path() { let entity = "product"; - let mapping = vec![crate::config::Mapping::ByPath( - crate::config::EntityPathMapping { - file_column: "manufacturer id".to_string(), - entity_path: "manufacturerId".to_string(), - }, - )]; + let mapping = vec![Mapping::ByPath(EntityPathMapping { + file_column: "manufacturer id".to_string(), + entity_path: "manufacturerId".to_string(), + })]; let api_schema = json!({ "product": { } @@ -115,12 +113,10 @@ mod tests { #[test] fn validate_existing_simple_path() { let entity = "product"; - let mapping = vec![crate::config::Mapping::ByPath( - crate::config::EntityPathMapping { - file_column: "manufacturer id".to_string(), - entity_path: "manufacturerId".to_string(), - }, - )]; + let mapping = vec![Mapping::ByPath(EntityPathMapping { + file_column: "manufacturer id".to_string(), + entity_path: "manufacturerId".to_string(), + })]; let api_schema = json!({ "product": { "entity": "product", @@ -144,12 +140,10 @@ mod tests { #[test] fn validate_non_existent_association() { let entity = "product"; - let mapping = vec![crate::config::Mapping::ByPath( - crate::config::EntityPathMapping { - file_column: "manufacturer name".to_string(), - entity_path: "manufacturer.name".to_string(), - }, - )]; + let mapping = vec![Mapping::ByPath(EntityPathMapping { + file_column: "manufacturer name".to_string(), + entity_path: "manufacturer.name".to_string(), + })]; let api_schema = json!({ "product": { "entity": "product", @@ -175,12 +169,10 @@ mod tests { #[test] fn validate_existing_association() { let entity = "product"; - let mapping = vec![crate::config::Mapping::ByPath( - crate::config::EntityPathMapping { - file_column: "manufacturer name".to_string(), - entity_path: "manufacturer.name".to_string(), - }, - )]; + let mapping = vec![Mapping::ByPath(EntityPathMapping { + file_column: "manufacturer name".to_string(), + entity_path: "manufacturer.name".to_string(), + })]; let api_schema = json!({ "product": { "entity": "product", @@ -213,12 +205,10 @@ mod tests { #[test] fn validate_valid_optional_value() { let entity = "product"; - let mapping = vec![crate::config::Mapping::ByPath( - crate::config::EntityPathMapping { - file_column: "manufacturer name".to_string(), - entity_path: "manufacturer?.name".to_string(), - }, - )]; + let mapping = vec![Mapping::ByPath(EntityPathMapping { + file_column: "manufacturer name".to_string(), + entity_path: "manufacturer?.name".to_string(), + })]; let api_schema = json!({ "product": { "entity": "product", @@ -251,12 +241,10 @@ mod tests { #[test] fn validate_invalid_optional_value() { let entity = "product"; - let mapping = vec![crate::config::Mapping::ByPath( - crate::config::EntityPathMapping { - file_column: "manufacturer name".to_string(), - entity_path: "manufacturer?.name".to_string(), - }, - )]; + let mapping = vec![Mapping::ByPath(EntityPathMapping { + file_column: "manufacturer name".to_string(), + entity_path: "manufacturer?.name".to_string(), + })]; let api_schema = json!({ "product": { "entity": "product", @@ -291,12 +279,10 @@ mod tests { #[test] fn validate_valid_nested_association() { let entity = "product"; - let mapping = vec![crate::config::Mapping::ByPath( - crate::config::EntityPathMapping { - file_column: "tax country".to_string(), - entity_path: "tax.country.name".to_string(), - }, - )]; + let mapping = vec![Mapping::ByPath(EntityPathMapping { + file_column: "tax country".to_string(), + entity_path: "tax.country.name".to_string(), + })]; let api_schema = json!({ "product": { "entity": "product", diff --git a/src/main.rs b/src/main.rs index 6faabaa..8470e84 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,76 +1,20 @@ use crate::api::SwClient; -use crate::config::{Credentials, Mapping, Schema}; +use crate::cli::{Cli, Commands, SyncMode}; +use crate::config_file::{Credentials, Mapping, Schema}; use crate::data::validate_paths_for_entity; use crate::data::{export, import, prepare_scripting_environment, ScriptingEnvironment}; use anyhow::Context; -use clap::{Parser, Subcommand}; +use clap::Parser; use std::collections::HashSet; use std::path::PathBuf; use std::sync::Arc; use std::time::Instant; mod api; -mod config; +mod cli; +mod config_file; mod data; -#[derive(Parser)] -#[command(version, about, long_about = None)] -struct Cli { - #[command(subcommand)] - command: Commands, -} - -#[derive(Subcommand)] -enum Commands { - /// Authenticate with a given shopware shop via integration admin API. - /// Credentials are stored in .credentials.toml in the current working directory. - Auth { - /// base URL of the shop - #[arg(short, long)] - domain: String, - - /// access_key_id - #[arg(short, long)] - id: String, - - /// access_key_secret - #[arg(short, long)] - secret: String, - }, - - /// Import data into shopware or export data to a file - Sync { - /// Mode (import or export) - #[arg(value_enum, short, long)] - mode: SyncMode, - - /// Path to profile schema.yaml - #[arg(short, long)] - schema: PathBuf, - - /// Path to data file - #[arg(short, long)] - file: PathBuf, - - /// Maximum amount of entities, can be used for debugging - #[arg(short, long)] - limit: Option, - - // Verbose output, used for debugging - // #[arg(short, long, action = ArgAction::SetTrue)] - // verbose: bool, - /// How many requests can be "in-flight" at the same time - #[arg(short, long, default_value = "8")] - in_flight_limit: usize, - }, -} - -#[derive(Debug, Clone, Copy, clap::ValueEnum)] -pub enum SyncMode { - Import, - Export, -} - #[derive(Debug)] pub struct SyncContext { pub sw_client: SwClient, @@ -109,6 +53,7 @@ async fn main() -> anyhow::Result<()> { .await?; println!("Imported successfully"); + println!("You might want to run the indexers in your shop now. Go to Settings -> System -> Caches & indexes"); } SyncMode::Export => { tokio::task::spawn_blocking(|| async move { export(Arc::new(context)).await })