From 9346359dfaf5b1c93ba1be19237e26733bc16fc0 Mon Sep 17 00:00:00 2001 From: Malte Janz Date: Wed, 3 Jul 2024 17:17:44 +0200 Subject: [PATCH] Some refactoring for readability --- src/api/filter.rs | 2 + src/api/mod.rs | 4 +- src/cli.rs | 64 +++++++++++ src/{config.rs => config_file.rs} | 14 +++ src/data/export.rs | 2 + src/data/import.rs | 2 + src/data/mod.rs | 5 +- src/data/{transform.rs => transform/mod.rs} | 119 ++------------------ src/data/transform/script.rs | 118 +++++++++++++++++++ src/main.rs | 66 +---------- 10 files changed, 220 insertions(+), 176 deletions(-) create mode 100644 src/cli.rs rename src/{config.rs => config_file.rs} (81%) rename src/data/{transform.rs => transform/mod.rs} (78%) create mode 100644 src/data/transform/script.rs diff --git a/src/api/filter.rs b/src/api/filter.rs index a13fb12..05123b4 100644 --- a/src/api/filter.rs +++ b/src/api/filter.rs @@ -1,3 +1,5 @@ +//! Data structures to build criteria objects for the shopware API + use serde::{Deserialize, Serialize}; use std::collections::BTreeMap; diff --git a/src/api/mod.rs b/src/api/mod.rs index 3ef581b..72ed0df 100644 --- a/src/api/mod.rs +++ b/src/api/mod.rs @@ -1,7 +1,9 @@ +//! Everything needed for communicating with the Shopware API + pub mod filter; use crate::api::filter::{Criteria, CriteriaFilter}; -use crate::config::Credentials; +use crate::config_file::Credentials; use anyhow::anyhow; use reqwest::header::{HeaderMap, HeaderValue}; use reqwest::{header, Client, Response, StatusCode}; diff --git a/src/cli.rs b/src/cli.rs new file mode 100644 index 0000000..c229554 --- /dev/null +++ b/src/cli.rs @@ -0,0 +1,64 @@ +//! Definitions for the CLI commands, arguments and help texts +//! +//! Makes heavy use of https://docs.rs/clap/latest/clap/ + +use clap::{Parser, Subcommand}; +use std::path::PathBuf; + +#[derive(Parser)] +#[command(version, about, long_about = None)] +pub struct Cli { + #[command(subcommand)] + pub command: Commands, +} + +#[derive(Subcommand)] +pub enum Commands { + /// Authenticate with a given shopware shop via integration admin API. + /// Credentials are stored in .credentials.toml in the current working directory. + Auth { + /// base URL of the shop + #[arg(short, long)] + domain: String, + + /// access_key_id + #[arg(short, long)] + id: String, + + /// access_key_secret + #[arg(short, long)] + secret: String, + }, + + /// Import data into shopware or export data to a file + Sync { + /// Mode (import or export) + #[arg(value_enum, short, long)] + mode: SyncMode, + + /// Path to profile schema.yaml + #[arg(short, long)] + schema: PathBuf, + + /// Path to data file + #[arg(short, long)] + file: PathBuf, + + /// Maximum amount of entities, can be used for debugging + #[arg(short, long)] + limit: Option, + + // Verbose output, used for debugging + // #[arg(short, long, action = ArgAction::SetTrue)] + // verbose: bool, + /// How many requests can be "in-flight" at the same time + #[arg(short, long, default_value = "8")] + in_flight_limit: usize, + }, +} + +#[derive(Debug, Clone, Copy, clap::ValueEnum)] +pub enum SyncMode { + Import, + Export, +} diff --git a/src/config.rs b/src/config_file.rs similarity index 81% rename from src/config.rs rename to src/config_file.rs index 0d73ca9..a88205f 100644 --- a/src/config.rs +++ b/src/config_file.rs @@ -1,3 +1,10 @@ +//! Definitions for the `schema.yaml` and `.credentials.toml` files +//! +//! Allows deserialization into a proper typed structure from these files +//! or also write these typed structures to a file (in case of `.credentials.toml`) +//! +//! Utilizes https://serde.rs/ + use crate::api::filter::{CriteriaFilter, CriteriaSorting}; use serde::{Deserialize, Serialize}; use std::collections::HashSet; @@ -12,15 +19,22 @@ pub struct Credentials { #[derive(Debug, Deserialize)] pub struct Schema { pub entity: String, + #[serde(default = "Vec::new")] pub filter: Vec, + #[serde(default = "Vec::new")] pub sort: Vec, + + /// Are unique thanks to `HashSet` #[serde(default = "HashSet::new")] pub associations: HashSet, + pub mappings: Vec, + #[serde(default = "String::new")] pub serialize_script: String, + #[serde(default = "String::new")] pub deserialize_script: String, } diff --git a/src/data/export.rs b/src/data/export.rs index 2d0ca4a..68aebda 100644 --- a/src/data/export.rs +++ b/src/data/export.rs @@ -1,3 +1,5 @@ +//! Everything related to exporting data out of shopware + use crate::api::filter::Criteria; use crate::data::transform::serialize_entity; use crate::SyncContext; diff --git a/src/data/import.rs b/src/data/import.rs index 58b7cb5..c0bf0f7 100644 --- a/src/data/import.rs +++ b/src/data/import.rs @@ -1,3 +1,5 @@ +//! Everything related to import data into shopware + use crate::api::{SwApiError, SyncAction}; use crate::data::transform::deserialize_row; use crate::SyncContext; diff --git a/src/data/mod.rs b/src/data/mod.rs index 9ca38ca..44d6584 100644 --- a/src/data/mod.rs +++ b/src/data/mod.rs @@ -2,7 +2,8 @@ mod export; mod import; mod transform; +// reexport the important functions / structs as part of this module pub use export::export; pub use import::import; -pub use transform::prepare_scripting_environment; -pub use transform::ScriptingEnvironment; +pub use transform::script::prepare_scripting_environment; +pub use transform::script::ScriptingEnvironment; diff --git a/src/data/transform.rs b/src/data/transform/mod.rs similarity index 78% rename from src/data/transform.rs rename to src/data/transform/mod.rs index 6902159..7cf67e5 100644 --- a/src/data/transform.rs +++ b/src/data/transform/mod.rs @@ -1,13 +1,16 @@ +//! Everything related to data transformations + +pub mod script; + use crate::api::Entity; -use crate::config::Mapping; +use crate::config_file::Mapping; use crate::SyncContext; use anyhow::Context; use csv::StringRecord; -use rhai::packages::{BasicArrayPackage, CorePackage, MoreStringPackage, Package}; -use rhai::{Engine, Position, Scope, AST}; +use rhai::Scope; use std::str::FromStr; -/// Deserialize a single row of the input file into a json object +/// Deserialize a single row of the input (CSV) file into a json object pub fn deserialize_row( headers: &StringRecord, row: StringRecord, @@ -155,114 +158,6 @@ pub fn serialize_entity(entity: Entity, context: &SyncContext) -> anyhow::Result Ok(row) } -#[derive(Debug)] -pub struct ScriptingEnvironment { - engine: Engine, - serialize: Option, - deserialize: Option, -} - -pub fn prepare_scripting_environment( - raw_serialize_script: &str, - raw_deserialize_script: &str, -) -> anyhow::Result { - let engine = get_base_engine(); - let serialize_ast = if !raw_serialize_script.is_empty() { - let ast = engine - .compile(raw_serialize_script) - .context("serialize_script compilation failed")?; - Some(ast) - } else { - None - }; - let deserialize_ast = if !raw_deserialize_script.is_empty() { - let ast = engine - .compile(raw_deserialize_script) - .context("serialize_script compilation failed")?; - Some(ast) - } else { - None - }; - - Ok(ScriptingEnvironment { - engine, - serialize: serialize_ast, - deserialize: deserialize_ast, - }) -} - -fn get_base_engine() -> Engine { - let mut engine = Engine::new_raw(); - // Default print/debug implementations - engine.on_print(|text| println!("{text}")); - engine.on_debug(|text, source, pos| match (source, pos) { - (Some(source), Position::NONE) => println!("{source} | {text}"), - (Some(source), pos) => println!("{source} @ {pos:?} | {text}"), - (None, Position::NONE) => println!("{text}"), - (None, pos) => println!("{pos:?} | {text}"), - }); - - let core_package = CorePackage::new(); - core_package.register_into_engine(&mut engine); - let string_package = MoreStringPackage::new(); - string_package.register_into_engine(&mut engine); - let array_package = BasicArrayPackage::new(); - array_package.register_into_engine(&mut engine); - - // ToDo: add custom utility functions to engine - engine.register_fn("get_default", script::get_default); - - // Some reference implementations below - /* - engine.register_type::(); - engine.register_fn("uuid", scripts::uuid); - engine.register_fn("uuidFromStr", scripts::uuid_from_str); - - engine.register_type::(); - engine.register_fn("map", scripts::Mapper::map); - engine.register_fn("get", scripts::Mapper::get); - - engine.register_type::(); - engine.register_fn("fetchFirst", scripts::DB::fetch_first); - */ - - engine -} - -/// utilities for inside scripts -mod script { - /// Imitate - /// https://github.com/shopware/shopware/blob/03cfe8cca937e6e45c9c3e15821d1449dfd01d82/src/Core/Defaults.php - pub fn get_default(name: &str) -> String { - match name { - "LANGUAGE_SYSTEM" => "2fbb5fe2e29a4d70aa5854ce7ce3e20b".to_string(), - "LIVE_VERSION" => "0fa91ce3e96a4bc2be4bd9ce752c3425".to_string(), - "CURRENCY" => "b7d2554b0ce847cd82f3ac9bd1c0dfca".to_string(), - "SALES_CHANNEL_TYPE_API" => "f183ee5650cf4bdb8a774337575067a6".to_string(), - "SALES_CHANNEL_TYPE_STOREFRONT" => "8a243080f92e4c719546314b577cf82b".to_string(), - "SALES_CHANNEL_TYPE_PRODUCT_COMPARISON" => "ed535e5722134ac1aa6524f73e26881b".to_string(), - "STORAGE_DATE_TIME_FORMAT" => "Y-m-d H:i:s.v".to_string(), - "STORAGE_DATE_FORMAT" => "Y-m-d".to_string(), - "CMS_PRODUCT_DETAIL_PAGE" => "7a6d253a67204037966f42b0119704d5".to_string(), - n => panic!( - "get_default called with '{}' but there is no such definition. Have a look into Shopware/src/Core/Defaults.php. Available constants: {:?}", - n, - vec![ - "LANGUAGE_SYSTEM", - "LIVE_VERSION", - "CURRENCY", - "SALES_CHANNEL_TYPE_API", - "SALES_CHANNEL_TYPE_STOREFRONT", - "SALES_CHANNEL_TYPE_PRODUCT_COMPARISON", - "STORAGE_DATE_TIME_FORMAT", - "STORAGE_DATE_FORMAT", - "CMS_PRODUCT_DETAIL_PAGE", - ] - ) - } - } -} - trait EntityPath { /// Search for a value inside a json object tree by a given path. /// Example path `object.child.attribute` diff --git a/src/data/transform/script.rs b/src/data/transform/script.rs new file mode 100644 index 0000000..3499581 --- /dev/null +++ b/src/data/transform/script.rs @@ -0,0 +1,118 @@ +//! Everything scripting related + +use anyhow::Context; +use rhai::packages::{BasicArrayPackage, CorePackage, MoreStringPackage, Package}; +use rhai::{Engine, Position, AST}; + +#[derive(Debug)] +pub struct ScriptingEnvironment { + pub engine: Engine, + pub serialize: Option, + pub deserialize: Option, +} + +pub fn prepare_scripting_environment( + raw_serialize_script: &str, + raw_deserialize_script: &str, +) -> anyhow::Result { + let engine = get_base_engine(); + let serialize_ast = if !raw_serialize_script.is_empty() { + let ast = engine + .compile(raw_serialize_script) + .context("serialize_script compilation failed")?; + Some(ast) + } else { + None + }; + let deserialize_ast = if !raw_deserialize_script.is_empty() { + let ast = engine + .compile(raw_deserialize_script) + .context("serialize_script compilation failed")?; + Some(ast) + } else { + None + }; + + Ok(ScriptingEnvironment { + engine, + serialize: serialize_ast, + deserialize: deserialize_ast, + }) +} + +fn get_base_engine() -> Engine { + let mut engine = Engine::new_raw(); + // Default print/debug implementations + engine.on_print(|text| println!("{text}")); + engine.on_debug(|text, source, pos| match (source, pos) { + (Some(source), Position::NONE) => println!("{source} | {text}"), + (Some(source), pos) => println!("{source} @ {pos:?} | {text}"), + (None, Position::NONE) => println!("{text}"), + (None, pos) => println!("{pos:?} | {text}"), + }); + + let core_package = CorePackage::new(); + core_package.register_into_engine(&mut engine); + let string_package = MoreStringPackage::new(); + string_package.register_into_engine(&mut engine); + let array_package = BasicArrayPackage::new(); + array_package.register_into_engine(&mut engine); + + // ToDo: add custom utility functions to engine + engine.register_fn("get_default", inside_script::get_default); + + // Some reference implementations below + /* + engine.register_type::(); + engine.register_fn("uuid", scripts::uuid); + engine.register_fn("uuidFromStr", scripts::uuid_from_str); + + engine.register_type::(); + engine.register_fn("map", scripts::Mapper::map); + engine.register_fn("get", scripts::Mapper::get); + + engine.register_type::(); + engine.register_fn("fetchFirst", scripts::DB::fetch_first); + */ + + engine +} + +/// Utilities for inside scripts +/// +/// Important, don't use the type `String` as function parameters, see +/// https://rhai.rs/book/rust/strings.html +mod inside_script { + use rhai::ImmutableString; + + /// Imitate + /// https://github.com/shopware/shopware/blob/03cfe8cca937e6e45c9c3e15821d1449dfd01d82/src/Core/Defaults.php + pub fn get_default(name: &str) -> ImmutableString { + match name { + "LANGUAGE_SYSTEM" => "2fbb5fe2e29a4d70aa5854ce7ce3e20b".into(), + "LIVE_VERSION" => "0fa91ce3e96a4bc2be4bd9ce752c3425".into(), + "CURRENCY" => "b7d2554b0ce847cd82f3ac9bd1c0dfca".into(), + "SALES_CHANNEL_TYPE_API" => "f183ee5650cf4bdb8a774337575067a6".into(), + "SALES_CHANNEL_TYPE_STOREFRONT" => "8a243080f92e4c719546314b577cf82b".into(), + "SALES_CHANNEL_TYPE_PRODUCT_COMPARISON" => "ed535e5722134ac1aa6524f73e26881b".into(), + "STORAGE_DATE_TIME_FORMAT" => "Y-m-d H:i:s.v".into(), + "STORAGE_DATE_FORMAT" => "Y-m-d".into(), + "CMS_PRODUCT_DETAIL_PAGE" => "7a6d253a67204037966f42b0119704d5".into(), + n => panic!( + "get_default called with '{}' but there is no such definition. Have a look into Shopware/src/Core/Defaults.php. Available constants: {:?}", + n, + vec![ + "LANGUAGE_SYSTEM", + "LIVE_VERSION", + "CURRENCY", + "SALES_CHANNEL_TYPE_API", + "SALES_CHANNEL_TYPE_STOREFRONT", + "SALES_CHANNEL_TYPE_PRODUCT_COMPARISON", + "STORAGE_DATE_TIME_FORMAT", + "STORAGE_DATE_FORMAT", + "CMS_PRODUCT_DETAIL_PAGE", + ] + ) + } + } +} diff --git a/src/main.rs b/src/main.rs index 3b1eea8..bfa8a87 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,75 +1,19 @@ use crate::api::SwClient; -use crate::config::{Credentials, Mapping, Schema}; +use crate::cli::{Cli, Commands, SyncMode}; +use crate::config_file::{Credentials, Mapping, Schema}; use crate::data::{export, import, prepare_scripting_environment, ScriptingEnvironment}; use anyhow::Context; -use clap::{Parser, Subcommand}; +use clap::Parser; use std::collections::HashSet; use std::path::PathBuf; use std::sync::Arc; use std::time::Instant; mod api; -mod config; +mod cli; +mod config_file; mod data; -#[derive(Parser)] -#[command(version, about, long_about = None)] -struct Cli { - #[command(subcommand)] - command: Commands, -} - -#[derive(Subcommand)] -enum Commands { - /// Authenticate with a given shopware shop via integration admin API. - /// Credentials are stored in .credentials.toml in the current working directory. - Auth { - /// base URL of the shop - #[arg(short, long)] - domain: String, - - /// access_key_id - #[arg(short, long)] - id: String, - - /// access_key_secret - #[arg(short, long)] - secret: String, - }, - - /// Import data into shopware or export data to a file - Sync { - /// Mode (import or export) - #[arg(value_enum, short, long)] - mode: SyncMode, - - /// Path to profile schema.yaml - #[arg(short, long)] - schema: PathBuf, - - /// Path to data file - #[arg(short, long)] - file: PathBuf, - - /// Maximum amount of entities, can be used for debugging - #[arg(short, long)] - limit: Option, - - // Verbose output, used for debugging - // #[arg(short, long, action = ArgAction::SetTrue)] - // verbose: bool, - /// How many requests can be "in-flight" at the same time - #[arg(short, long, default_value = "8")] - in_flight_limit: usize, - }, -} - -#[derive(Debug, Clone, Copy, clap::ValueEnum)] -pub enum SyncMode { - Import, - Export, -} - #[derive(Debug)] pub struct SyncContext { pub sw_client: SwClient,