Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

light/superlight to replace initial #17

Merged
merged 1 commit into from
Feb 25, 2024
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 8 additions & 8 deletions .zetch.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

6 changes: 2 additions & 4 deletions dev_scripts/py_rust.sh
Original file line number Diff line number Diff line change
@@ -21,10 +21,8 @@ ensure_venv () {
source ./py_rust/.venv/bin/activate
fi

./dev_scripts/utils.sh py_install_if_missing typing-extensions
./dev_scripts/utils.sh py_install_if_missing maturin
./dev_scripts/utils.sh py_install_if_missing pyright
./dev_scripts/utils.sh py_install_if_missing pytest
# Install any dev requirements that aren't managed by maturin:
pip install -r ./py_rust/dev_requirements.txt

./dev_scripts/utils.sh py_install_if_missing ruff
}
1 change: 0 additions & 1 deletion py_rust/Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion py_rust/Cargo.toml
Original file line number Diff line number Diff line change
@@ -17,7 +17,7 @@ path = "src/lib.rs"
colored = '2'
tracing = "0.1"
error-stack = "0.4"
bitbazaar = { version = "0.0.38", features = ["cli", "timing", "clap"] }
bitbazaar = { version = "0.0.38", features = ["cli", "timing"] }
pyo3 = { version = '0.20.0', features = ['extension-module', 'chrono', 'generate-import-lib'] }
parking_lot = { version = "0.12", features = ['deadlock_detection', 'serde'] }
strum = { version = '0.25', features = ['derive'] }
6 changes: 6 additions & 0 deletions py_rust/dev_requirements.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
maturin==1.4.0
typing-extensions==4.9.0
pyright==1.1.351
pytest==8.0.1
pytest-xdist==3.5.0
pytest-profiling==1.7.0
47 changes: 44 additions & 3 deletions py_rust/src/args.rs
Original file line number Diff line number Diff line change
@@ -37,7 +37,7 @@ pub fn get_version_info() -> String {
}
}

#[derive(Debug, Parser)]
#[derive(Clone, Debug, Parser)]
#[command(
author,
name = "zetch",
@@ -49,7 +49,7 @@ pub struct Args {
#[command(subcommand)]
pub command: Command,
#[clap(flatten)]
pub log_level_args: bitbazaar::log::ClapLogLevelArgs,
pub log_level_args: ClapLogLevelArgs,
/// The config file to use. Note if render command, relative and not found from working directory, will search entered root directory.
#[arg(
short,
@@ -61,7 +61,7 @@ pub struct Args {
pub config: PathBuf,
}

#[derive(Debug, clap::Subcommand)]
#[derive(Clone, Debug, clap::Subcommand)]
pub enum Command {
/// Render all templates found whilst traversing the given root (default).
Render(RenderCommand),
@@ -95,9 +95,19 @@ pub struct RenderCommand {
/// The target directory to search and render.
#[clap(default_value = ".")]
pub root: PathBuf,

/// No tasks will be run, cli vars will be ignored and treated as empty strings if "light" defaults not specified.
#[arg(short, long, default_value = "false")]
pub light: bool,

/// Same as light, but user defined custom extensions are also treated as empty strings, pure rust speeds!
#[arg(long, default_value = "false")]
pub superlight: bool,

/// Force write all rendered files, ignore existing lockfile.
#[arg(short, long, default_value = "false")]
pub force: bool,

/// Comma separated list of env ctx vars to ignore defaults for and raise if not in env. E.g. --ban-defaults FOO,BAR...
///
/// If no vars are provided, all defaults will be ignored.
@@ -218,3 +228,34 @@ pub enum HelpFormat {
Text,
Json,
}

/// A simple clap argument group for controlling the log level for cli usage.
#[derive(Clone, Debug, clap::Args)]
pub struct ClapLogLevelArgs {
/// Enable verbose logging.
#[arg(
short,
long,
global = true,
group = "verbosity",
help_heading = "Log levels"
)]
pub verbose: bool,
/// Print diagnostics, but nothing else.
#[arg(
short,
long,
global = true,
group = "verbosity",
help_heading = "Log levels"
)]
/// Disable all logging (but still exit with status code "1" upon detecting diagnostics).
#[arg(
short,
long,
global = true,
group = "verbosity",
help_heading = "Log levels"
)]
pub silent: bool,
}
2 changes: 1 addition & 1 deletion py_rust/src/config/context.rs
Original file line number Diff line number Diff line change
@@ -66,7 +66,7 @@ impl CtxEnvVar {
pub struct CtxCliVar {
pub commands: Vec<String>,
pub coerce: Option<Coerce>,
pub initial: Option<serde_json::Value>,
pub light: Option<serde_json::Value>,
}

impl CtxCliVar {
4 changes: 2 additions & 2 deletions py_rust/src/config/schema.json
Original file line number Diff line number Diff line change
@@ -148,8 +148,8 @@
},
"minItems": 1
},
"initial": {
"description": "You might find one of your commands fails on a fresh build if zetch hasn't run yet due to dependencies on other zetched files, i.e. a circular dependency. When no lockfile is found, or --force is used, if any cli variable has initial set zetch will run twice, on the first run cli vars will use their initials where they have them, on the second render the real commands will compute the values."
"light": {
"description": "The value to use when in rendering in --light or --superlight mode. If not set, the var will be treated as an empty string."
},
"coerce": {
"type": "string",
14 changes: 10 additions & 4 deletions py_rust/src/custom_exts/py_interface.rs
Original file line number Diff line number Diff line change
@@ -50,6 +50,14 @@ pub fn load_custom_exts(exts: &[String], state: &State) -> Result<HashMap<String
Python::with_gil(|py| {
// Pythonize a copy of the context and add to the global PY_CONTEXT so its usable from zetch.context():
let mut py_ctx = PY_CONTEXT.lock();

if py_ctx.is_some() {
return Err(zerr!(
Zerr::InternalError,
"Custom extensions loaded more than once."
));
}

*py_ctx = Some(pythonize(py, &state.ctx).change_context(Zerr::InternalError)?);

let syspath: &PyList = py
@@ -110,10 +118,8 @@ pub fn load_custom_exts(exts: &[String], state: &State) -> Result<HashMap<String
Ok::<_, error_stack::Report<Zerr>>(())
})?;

// Extract a copy of the user funcs to add to minijinja env:
// Note copying as env might be created multiple times (e.g. initial)
// TODO: instead of this maybe reusing an env? In general need a bit of a refactor!
Ok(PY_USER_FUNCS.lock().clone())
// Extra the loaded user funcs, this fn is checked to only run once. So no need to clone and maintain global var.
Ok(std::mem::take(&mut *PY_USER_FUNCS.lock()))
}

pub fn mini_values_to_py_params(
7 changes: 5 additions & 2 deletions py_rust/src/render/debug.rs
Original file line number Diff line number Diff line change
@@ -1,8 +1,11 @@
use crate::state::State;
use std::collections::HashMap;

use crate::config::conf::Config;

#[derive(Debug, serde::Serialize)]
pub struct Debug {
pub state: State,
pub conf: Config,
pub ctx: HashMap<String, serde_json::Value>,
pub written: Vec<String>,
pub identical: Vec<String>,
pub matched_templates: Vec<String>,
63 changes: 36 additions & 27 deletions py_rust/src/render/mini_env.rs
Original file line number Diff line number Diff line change
@@ -60,33 +60,42 @@ pub fn new_mini_env<'a>(root: &Path, state: &'a State) -> Result<minijinja::Envi
));
}

// Add the rust-wrapped python fn to the minijinja environment:
env.add_function(
name.clone(),
move |
values: minijinja::value::Rest<minijinja::Value>|
-> core::result::Result<minijinja::Value, minijinja::Error> {
let result =
Python::with_gil(|py| -> Result<serde_json::Value, Zerr> {
let (py_args, py_kwargs) = py_interface::mini_values_to_py_params(py, values)?;
let py_result = py_fn
.call(py, py_args, py_kwargs)
.map_err(|e: PyErr| zerr!(Zerr::CustomPyFunctionError, "{}", e))?;
let rustified: serde_json::Value =
depythonize(py_result.as_ref(py)).change_context(Zerr::CustomPyFunctionError).attach_printable_lazy(|| {
"Failed to convert python result to a rust-like value."
})?;
Ok(rustified)
});
match result {
Err(e) => Err(minijinja::Error::new(
minijinja::ErrorKind::InvalidOperation,
format!("Failed to call custom filter '{}'. Err: \n{:?}", name, e),
)),
Ok(result) => Ok(minijinja::Value::from_serializable(&result)),
}
},
)
// If superlight, add a pseudo fn that returns an empty string
if state.superlight {
let empty_str = minijinja::Value::from_safe_string("".to_string());
env.add_function(
name.clone(),
move |_values: minijinja::value::Rest<minijinja::Value>| empty_str.clone(),
);
} else {
// Add the rust-wrapped python fn to the minijinja environment:
env.add_function(
name.clone(),
move |
values: minijinja::value::Rest<minijinja::Value>|
-> core::result::Result<minijinja::Value, minijinja::Error> {
let result =
Python::with_gil(|py| -> Result<serde_json::Value, Zerr> {
let (py_args, py_kwargs) = py_interface::mini_values_to_py_params(py, values)?;
let py_result = py_fn
.call(py, py_args, py_kwargs)
.map_err(|e: PyErr| zerr!(Zerr::CustomPyFunctionError, "{}", e))?;
let rustified: serde_json::Value =
depythonize(py_result.as_ref(py)).change_context(Zerr::CustomPyFunctionError).attach_printable_lazy(|| {
"Failed to convert python result to a rust-like value."
})?;
Ok(rustified)
});
match result {
Err(e) => Err(minijinja::Error::new(
minijinja::ErrorKind::InvalidOperation,
format!("Failed to call custom filter '{}'. Err: \n{:?}", name, e),
)),
Ok(result) => Ok(minijinja::Value::from_serializable(&result)),
}
},
)
}
}

Ok(env)
45 changes: 16 additions & 29 deletions py_rust/src/render/mod.rs
Original file line number Diff line number Diff line change
@@ -23,43 +23,26 @@ pub fn render(args: &crate::args::Args, render_args: &RenderCommand) -> Result<b
self::lockfile::Lockfile::load(render_args.root.clone(), render_args.force)
});

// TODO double prints what's that about

let mut state = State::new(args)?;
state.load_all_vars()?;
debug!("State: {:#?}", state);

// If newly created, should use cli initials if any have them:
let use_cli_initials = lockfile.newly_created
&& state
.conf
.context
.cli
.iter()
.any(|(_, v)| v.initial.is_some());
state.load_all_vars(Some(render_args), use_cli_initials)?;

// Need to run twice and rebuild config with real cli vars if initials used in the first state build:
let (written, identical) = if use_cli_initials {
warn!("Lockfile newly created/force updated and some cli vars have initials, will double render and use initials first time round.");
// Conf from second as that has the real cli vars, template info from the first as the second will be inaccurate due to the first having run.
let (first_written, first_identical) = render_inner(&state, render_args, &mut lockfile)?;

// Reload with real cli vars:
state.load_all_vars(Some(render_args), false)?;

// Re-render:
render_inner(&state, render_args, &mut lockfile)?;
(first_written, first_identical)
} else {
render_inner(&state, render_args, &mut lockfile)?
};
let (written, identical) = render_inner(&state, render_args, &mut lockfile)?;

// Run post-tasks:
state.conf.tasks.run_post(&state)?;
// Run post-tasks only if not light/superlight:
if !state.light {
state.conf.tasks.run_post(&state)?;
}

timeit!("Syncing lockfile", { lockfile.sync() })?;

// Write only when hidden cli flag --debug is set, to allow testing internals from python without having to setup custom interfaces:
if render_args.debug {
let debug = debug::Debug {
state: state.clone(),
conf: state.conf.clone(),
ctx: state.ctx.clone(),
written: written
.iter()
.map(|t| t.out_path.display().to_string())
@@ -85,7 +68,11 @@ pub fn render(args: &crate::args::Args, render_args: &RenderCommand) -> Result<b
.change_context(Zerr::InternalError)?;
}

let num_tasks = state.conf.tasks.pre.len() + state.conf.tasks.post.len();
let num_tasks = if state.light {
0
} else {
state.conf.tasks.pre.len() + state.conf.tasks.post.len()
};
println!(
"{} {} template{} written, {} identical.{} Lockfile {}. {} elapsed.",
"zetch:".bold(),
8 changes: 0 additions & 8 deletions py_rust/src/run.rs
Original file line number Diff line number Diff line change
@@ -68,14 +68,6 @@ pub fn run() -> Result<(), Zerr> {
.change_context(Zerr::InternalError)?;
}

// Stdout if enabled:
if let Some(level) = args.log_level_args.level() {
builder = builder
.stdout(true, true)
.level_from(level)
.change_context(Zerr::InternalError)?;
}

// Build and set as global logger:
let log = builder.build().change_context(Zerr::InternalError)?;
log.register_global().change_context(Zerr::InternalError)?;
211 changes: 120 additions & 91 deletions py_rust/src/state/active_state.rs
Original file line number Diff line number Diff line change
@@ -4,25 +4,27 @@ use std::{
};

use bitbazaar::timeit;
use serde::{Deserialize, Serialize};

use super::parent_state::load_parent_state;
use crate::{
args::RenderCommand,
config::{conf::Config, context::CtxCliVar},
prelude::*,
};
use crate::{args::Command, config::conf::Config, prelude::*};

#[derive(Clone, Debug, Serialize, Deserialize)]
#[derive(Debug)]
pub struct State {
pub args: crate::args::Args,

/// Raw config decoded from the config file.
pub conf: Config,
pub final_config_path: PathBuf,

/// True if this state has been loaded in from a parent process:
pub from_parent: bool,
pub pre_tasks_run: bool,
// The currently loaded values, this starts of empty until explicitly loaded using load_var() or load_all_vars():
/// The currently loaded values, this starts of empty until explicitly loaded using load_var() or load_all_vars():
pub ctx: HashMap<String, serde_json::Value>,

/// True if --light or --superlight
pub light: bool,

/// True if --superlight
pub superlight: bool,
}

impl State {
@@ -31,56 +33,73 @@ impl State {
/// Will automatically run the pre-tasks if needed.
pub fn new(args: &crate::args::Args) -> Result<State, Zerr> {
// If running as a subprocess of a zetch command and is applicable (i.e. in a post task),
// then use its config directly to prevent recursion errors and avoid unnecessary processing):
if let Some(mut tmp_parent_state) = load_parent_state()? {
tmp_parent_state.from_parent = true;
return Ok(tmp_parent_state);
}
// then use some of its state (ctx/config) to prevent recursion errors and avoid unnecessary processing):
let state = if let Some(parent_shared_state) = load_parent_state()? {
Self {
args: args.clone(),
conf: parent_shared_state.conf,
ctx: parent_shared_state.ctx,
final_config_path: parent_shared_state.final_config_path,
light: false,
superlight: false,
}
} else {
let final_config_path = build_final_config_path(
&args.config,
if let crate::args::Command::Render(render) = &args.command {
Some(&render.root)
} else {
None
},
)?;

let final_config_path = build_final_config_path(
&args.config,
if let crate::args::Command::Render(render) = &args.command {
Some(&render.root)
let conf = timeit!("Config processing", {
Config::from_toml(&final_config_path)
})?;

// Run the pre-tasks if applicable to the active command.
// Note this won't be run if in child process (which makes sense), due to above return.
let command_expecting_tasks = match &args.command {
crate::args::Command::Render(_) | crate::args::Command::Var(_) => true,
crate::args::Command::Read(_)
| crate::args::Command::Put(_)
| crate::args::Command::Del(_)
| crate::args::Command::Init(_)
| crate::args::Command::ReplaceMatcher(_)
| crate::args::Command::Version { .. } => false,
};

// Set light to true if --light or --superlight, and superlight to true if --superlight:
let (light, superlight) = if let crate::args::Command::Render(render) = &args.command {
(render.light || render.superlight, render.superlight)
} else {
None
},
)?;
(false, false)
};

let conf = timeit!("Config processing", {
Config::from_toml(&final_config_path)
})?;
// Run pre-tasks if the right type of command and not running in light/superlight mode:
if command_expecting_tasks && !light {
conf.tasks.run_pre(&final_config_path)?;
}

// Run the pre-tasks if applicable to the active command.
// Note this won't be run if in child process (which makes sense), due to above return.
let run_pre_tasks = match &args.command {
crate::args::Command::Render(_) | crate::args::Command::Var(_) => true,
crate::args::Command::Read(_)
| crate::args::Command::Put(_)
| crate::args::Command::Del(_)
| crate::args::Command::Init(_)
| crate::args::Command::ReplaceMatcher(_)
| crate::args::Command::Version { .. } => false,
Self {
args: args.clone(),
conf,
ctx: HashMap::new(),
final_config_path,
light,
superlight,
}
};
if run_pre_tasks {
conf.tasks.run_pre(&final_config_path)?;
}

Ok(Self {
conf,
ctx: HashMap::new(),
final_config_path,
from_parent: false,
pre_tasks_run: false,
})
Ok(state)
}

/// Load a new context var, returning a reference to the value, and storing in state.ctx.
/// This will also internally manage running pre tasks.
pub fn load_var(
&mut self,
var: &str,
default_banned: bool, // TODO want to internalise into state
use_cli_initials: bool, // TODO want to internalise into state
default_banned: bool, // TODO want to internalise into state
) -> Result<&serde_json::Value, Zerr> {
// If already exists use:
if self.ctx.contains_key(var) {
@@ -93,7 +112,16 @@ impl State {
} else if let Some(value) = self.conf.context.env.get(var) {
value.read(var, default_banned)
} else if let Some(value) = self.conf.context.cli.get(var) {
read_cli_var(use_cli_initials, value, &self.final_config_path)
// In light mode use the user provided default or an empty string, rather than running a user command:
if self.light {
if let Some(light_val) = &value.light {
Ok(light_val.clone())
} else {
Ok(serde_json::Value::String("".to_string()))
}
} else {
value.read(&self.final_config_path)
}
} else {
// Otherwise something wrong in userland:
return Err(zerr!(
@@ -113,11 +141,7 @@ impl State {
}

/// Load all context vars.
pub fn load_all_vars(
&mut self,
render_args: Option<&RenderCommand>, // TODO want to internalise
use_cli_initials: bool, // TODO want to internalise
) -> Result<(), Zerr> {
pub fn load_all_vars(&mut self) -> Result<(), Zerr> {
timeit!(
"Context value extraction (including user task & cli env scripting)",
{
@@ -130,13 +154,14 @@ impl State {
.cloned()
.collect::<Vec<String>>()
{
self.load_var(&key, false, use_cli_initials)?;
self.load_var(&key, false)?;
}

// Env vars:
// If some env defaults banned, validate list and convert to a hashset for faster lookup:
let banned_env_defaults: Option<HashSet<String>> = if let Some(render_args) =
render_args
let banned_env_defaults: Option<HashSet<String>> = if let Command::Render(
render_args,
) = &self.args.command
{
if let Some(banned) = render_args.ban_defaults.as_ref() {
// If no vars provided, ban all defaults:
@@ -196,11 +221,9 @@ impl State {
} else {
false
},
use_cli_initials,
)?;
}

// TODO still don't like all of this, would like to improve:
// External commands can be extremely slow compared to the rest of the library,
// try and remedy a bit by running them in parallel:
let mut handles = vec![];
@@ -212,52 +235,58 @@ impl State {
.cloned()
.collect::<Vec<String>>()
{
// can't use load_var() as wanting to make parallel:
let value = self.conf.context.cli.get(&key).unwrap().clone();
let final_config_path = self.final_config_path.clone();
handles.push(std::thread::spawn(
move || -> Result<(String, serde_json::Value), Zerr> {
timeit!(format!("Cli var processing: '{}'", &key).as_str(), {
Ok((
key,
read_cli_var(use_cli_initials, &value, &final_config_path)?,
))
})
},
));
// can't use load_var() as wanting to make parallel, so repeating logic here in a way that can be executed in parallel:

// If already exists skip:
if self.ctx.contains_key(&key) {
continue;
}

let var = self.conf.context.cli.get(&key).unwrap();
// If light mode, need to use the light user replacement otherwise an empty string: (no need for threads)
if self.light {
let value = if let Some(light_val) = &var.light {
light_val.clone()
} else {
serde_json::Value::String("".to_string())
};
self.ctx.insert(key, value);
} else {
let final_config_path = self.final_config_path.to_path_buf();
let var = var.clone();
handles.push(std::thread::spawn(
move || -> Result<(String, serde_json::Value), Zerr> {
timeit!(format!("Cli var processing: '{}'", &key).as_str(), {
Ok((key, var.read(&final_config_path)?))
})
},
));
}
}

for handle in handles {
// TODO what's this unwrap about?
let (key, value) =
handle.join().unwrap().change_context(Zerr::InternalError)?;
// Add to context:
self.ctx.insert(key, value);
match handle.join() {
Ok(fn_result) => {
let (key, value) = fn_result?;
self.ctx.insert(key, value);
}
Err(thread_err) => {
return Err(
zerr!(Zerr::InternalError, "Error reading thread result.",)
.attach_printable(format!("Thread error: {:?}", thread_err)),
);
}
}
}

Ok(())
}
)?;

// TODO replacement:
// debug!("Processed state: \n{:#?}", state);

Ok(())
}
}

fn read_cli_var(
use_cli_initials: bool,
var: &CtxCliVar,
final_config_path: &Path,
) -> Result<serde_json::Value, Zerr> {
if use_cli_initials && var.initial.is_some() {
Ok(var.initial.clone().unwrap())
} else {
var.read(final_config_path)
}
}

/// Get the final config path, errors if path doesn't exist.
/// For render subcommand usage, if the config path is relative and doesn't exist to run directory, will also check relative to root directory.
fn build_final_config_path(base_path: &Path, render_root: Option<&Path>) -> Result<PathBuf, Zerr> {
29 changes: 25 additions & 4 deletions py_rust/src/state/parent_state.rs
Original file line number Diff line number Diff line change
@@ -1,23 +1,44 @@
use std::path::{Path, PathBuf};
use std::{
collections::HashMap,
path::{Path, PathBuf},
};

use serde::{Deserialize, Serialize};
use tempfile::NamedTempFile;

use super::State;
use crate::{config::tasks::parent_task_active, prelude::*};
use crate::{
config::{conf::Config, tasks::parent_task_active},
prelude::*,
};

pub static CACHED_STATE_ENV_VAR: &str = "ZETCH_TMP_STORED_CONFIG_PATH";

/// The parts of State that should be stored between parent/child.
#[derive(Debug, Serialize, Deserialize)]
pub struct StoredParentState {
pub conf: Config,
pub ctx: HashMap<String, serde_json::Value>,
pub final_config_path: PathBuf,
}

/// Cache the config in a temporary file, used in e.g. subcommands that might read the config.
///
/// Returns the PathBuf to the temporary file.
pub fn store_parent_state(state: &State) -> Result<PathBuf, Zerr> {
let state = StoredParentState {
conf: state.conf.clone(),
ctx: state.ctx.clone(),
final_config_path: state.final_config_path.clone(),
};

let temp = NamedTempFile::new().change_context(Zerr::InternalError)?;
serde_json::to_writer(&temp, state).change_context(Zerr::InternalError)?;
serde_json::to_writer(&temp, &state).change_context(Zerr::InternalError)?;
Ok(temp.path().to_path_buf())
}

/// Load the cached state if it's available, return None otherwise.
pub fn load_parent_state() -> Result<Option<State>, Zerr> {
pub fn load_parent_state() -> Result<Option<StoredParentState>, Zerr> {
// If not in a task, parent state shouldn't be set or used:
if !parent_task_active() {
return Ok(None);
2 changes: 1 addition & 1 deletion py_rust/src/var.rs
Original file line number Diff line number Diff line change
@@ -9,7 +9,7 @@ pub fn read_var(args: &crate::args::Args, read: &VarCommand) -> Result<(), Zerr>
let mut state = State::new(args)?;

// Only need to load the specific target:
let target = state.load_var(read.var.as_str(), false, false)?;
let target = state.load_var(read.var.as_str(), false)?;

// Handle different output formats:
match read.output {
2 changes: 1 addition & 1 deletion py_rust/tests/helpers/types.py
Original file line number Diff line number Diff line change
@@ -6,7 +6,7 @@
class CliCtx(tp.TypedDict):
commands: "list[str]"
coerce: tp.NotRequired[Coerce_T]
initial: tp.NotRequired[tp.Any]
light: tp.NotRequired[tp.Any]


class EnvCtx(tp.TypedDict):
23 changes: 11 additions & 12 deletions py_rust/tests/helpers/utils.py
Original file line number Diff line number Diff line change
@@ -34,31 +34,30 @@ def check_single(
expected: tp.Union[str, tp.Callable[[str], bool]],
file_type="txt",
extra_args: tp.Optional["list[str]"] = None,
extra_templates_written: tp.Optional["list[pathlib.Path]"] = None,
ignore_extra_written: bool = False,
):
template = manager.tmpfile(content=contents, suffix=".zetch.{}".format(file_type))

rendered_info = cli.render(manager.root_dir, config_file, extra_args=extra_args)
result = rendered_info["debug"]

# Should return the correct compiled file:
expected_written = [remove_template(template)] + (
[remove_template(template) for template in extra_templates_written]
if extra_templates_written
else []
)
assert sorted(result["written"]) == sorted(expected_written), (
sorted(result["written"]),
sorted(expected_written),
)
assert result["identical"] == [], result["identical"]
if not ignore_extra_written:
expected_written = [remove_template(template)]
assert result["written"] == expected_written, (
result["written"],
expected_written,
)
assert result["identical"] == [], result["identical"]

# Original shouldn't have changed:
with open(template, "r") as file:
assert contents == file.read()

# Compiled should match expected:
with open(result["written"][0], "r") as file:
out_path = get_out_path(template)
assert out_path is not None
with open(out_path, "r") as file:
output = file.read()
if isinstance(expected, str):
assert output == expected, (output, expected)
50 changes: 0 additions & 50 deletions py_rust/tests/render/test_cli_initial.py

This file was deleted.

13 changes: 8 additions & 5 deletions py_rust/tests/render/test_custom_extensions.py
Original file line number Diff line number Diff line change
@@ -415,6 +415,8 @@ def test_custom_ext_multi_render():
"""Check custom extensions don't break when renderer runs multiple times (e.g. when cli var initials used).
This test was made for a real bug where extensions were lost after the first renderer usage.
NOTE: the renderer no longer runs multiple times so this test is redundant, better to keep though! Switched initial to light as that's the replacement.
"""
with TmpFileManager() as manager:
ext = manager.tmpfile(
@@ -433,15 +435,15 @@ def capitalize(s):
"engine": {"custom_extensions": [str(ext)]},
"context": {
"cli": {
"var_with_initial": {
"var_with_light": {
"commands": ["echo bar"],
"initial": "init",
"light": "init",
}
}
},
}
),
"{{ capitalize('foo') }} {{ var_with_initial }}",
"{{ capitalize('foo') }} {{ var_with_light }}",
"Foo bar",
)

@@ -470,9 +472,10 @@ def capitalize(s):
# DONE probably remove most engine config, maybe making top level if minimal enough, we don't want to mess with files and error early (so enforce no_undefined and keep_trailing_newline)
# DONE fix schema - not sure why its not working
# DONE: order everything in the lockfile to prevent diffs when nothings actually changed.
# TODO: Before thinking about modes/light etc as other solutions to circular deps. I think by default the repeated rendering should be improved to be a first class citizen, then, every run, static+env rendered in first + initials when not existing in lockfile + others clis as empty strings + custom extensions as empty strings, only then a second render with cli commands, this could get rid of a bunch of circular dep problems natively.
# TODO some sort of heavy/light/modes solution to caching values and not recomputing, maybe also for ban-defaults etc. maybe a modes top level config section, where a mode can override any config, set ban-defaults etc, need to think, but also need a way to only run certain post and pre in certain modes, need to think on best api.
# TODO: modes which should be the way ban-defaults are done, should be able to control task rendering too.
# TODO: static, env default and cli light should all have the same syntax, either the object with value|coerce, and just raw string which is treated as such (can have same schema object and use CliStaticVar for all).
# TODO think about interop with jinja,cookiecutter,copier,etc
# TODO decide and document optimal formatting, probably using scolvins and making sure it can working with custom extensions.
# TODO fix the conch parser rust incompatibility upstream somehow
# TODO for read put and delete, should compare with dasel to make sure not missing anything key
# TODO: (NOTE probably not needed to stablise and have docs published) context parent child hierarchy, config files should be processed deep down and can give different variables in different areas.
197 changes: 197 additions & 0 deletions py_rust/tests/render/test_light_superlight.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,197 @@
import os
import pathlib
import typing as tp

import pytest

from ..helpers import utils
from ..helpers.tmp_file_manager import TmpFileManager
from ..helpers.types import InputConfig

# light/superlight TODO:
# - At the moment if e.g. filters like |items are used, light values will have to be configured to prevent breaking these filters, need some way to internally properly render these out.
# - Same as above but for e.g. for loops.
# - If above can be solved, extending that to allow full partial rendering, i.e. only sections not containing cli vars are rendered would make this a very powerful feature.
# - If above 3 can be implemented, --light can be run automatically on a cli var failing, to automatically fix render cycles. This would be kind've possible before, but would then require the user to make all their cli vars light compatible (given the above current restriction with filters, for loops etc.).


@pytest.mark.parametrize(
"config, src, expected",
[
# Should be replaced by empty string when no light value set:
({"context": {"cli": {"VAR": {"commands": ["echo foo"]}}}}, "var: {{ VAR }}", "var: "),
# Light value should work:
(
{"context": {"cli": {"VAR": {"commands": ["echo foo"], "light": "LIGHT"}}}},
"var: {{ VAR }}",
"var: LIGHT",
),
# Env and static vars should work:
(
{
"context": {
"static": {"VAR_STATIC": {"value": "STATIC"}},
"env": {"VAR_ENV": {"default": "ENV"}},
"cli": {
"VAR": {
"commands": ["echo foo"],
"light": "LIGHT",
}
},
}
},
"{{ VAR }} {{ VAR_ENV }} {{ VAR_STATIC }}",
"LIGHT ENV STATIC",
),
# Pre and post tasks should be completely ignored:
(
{
"tasks": {
"pre": [{"commands": ["exit 1"]}],
"post": [{"commands": ["exit 1"]}],
},
"context": {"cli": {"VAR": {"commands": ["echo foo"]}}},
},
"var: {{ VAR }}",
"var: ",
),
],
)
def test_light_superlight_shared(config: InputConfig, src: str, expected: str):
"""Confirm shared behaviour between light and superlight."""

def test(superlight: bool):
with TmpFileManager() as manager:
utils.check_single(
manager,
manager.create_cfg(config),
src,
expected,
extra_args=["--superlight" if superlight else "--light"],
)

# These should be the same for superlight and light modes:
test(False)
test(True)


@pytest.mark.parametrize(
"config, src, expected, cb",
[
# Custom extensions should still be respected:
(
{
"engine": {"custom_extensions": ["./ext.py"]},
},
"out: {{ add_2(2) }}",
"out: 4",
lambda m: m.tmpfile(
"""import zetch
@zetch.register_function
def add_2(num):
return num + 2
""",
full_name="ext.py",
),
),
],
)
def test_light_only(
config: InputConfig, src: str, expected: str, cb: tp.Callable[[TmpFileManager], tp.Any]
):
"""Confirm shared behaviour between light and superlight."""
with TmpFileManager() as manager:
cb(manager)
utils.check_single(
manager,
manager.create_cfg(config),
src,
expected,
extra_args=["--light"],
)


@pytest.mark.parametrize(
"config, src, expected, cb",
[
# Custom extensions shouldn't be run and just return empty strings:
(
{
"engine": {"custom_extensions": ["./ext.py"]},
},
"out: {{ add_2(2) }}",
"out: ",
lambda m: m.tmpfile(
"""import zetch
@zetch.register_function
def add_2(num):
return num + 2
""",
full_name="ext.py",
),
),
],
)
def test_superlight_only(
config: InputConfig, src: str, expected: str, cb: tp.Callable[[TmpFileManager], tp.Any]
):
"""Confirm shared behaviour between light and superlight."""
with TmpFileManager() as manager:
cb(manager)
utils.check_single(
manager,
manager.create_cfg(config),
src,
expected,
extra_args=["--superlight"],
)


def test_light_fixes_circular_dep():
"""Make sure that running with --light or --superlight, then running again normally would fix a circular dependency in cli commands."""

def run(light: bool):
with TmpFileManager() as manager:
manager.tmpfile("Hello, World!", full_name="circ_dep.zetch.txt")
config: InputConfig = {
"context": {
"cli": {
"VAR": {
"commands": [
'{} "{}"'.format(
utils.cat_cmd_cross(),
utils.str_path_for_tmpl_writing(
pathlib.Path(os.path.join(manager.root_dir, "circ_dep.txt"))
),
)
],
"light": "LIGHT",
},
}
}
}

if light:
utils.check_single(
manager,
manager.create_cfg(config),
"{{ VAR }}",
"LIGHT",
extra_args=["--light"],
ignore_extra_written=True,
)

utils.check_single(
manager,
manager.create_cfg(config),
"{{ VAR }}",
"Hello, World!",
ignore_extra_written=True,
)

# Should fail due to circular dependency without a previous light/superlight run:
with pytest.raises(ValueError, match=utils.no_file_err_cross("circ_dep.txt")):
run(False)

# Should work with light/superlight:
run(True)
12 changes: 4 additions & 8 deletions py_rust/tests/test_config_valid.py
Original file line number Diff line number Diff line change
@@ -216,11 +216,7 @@ def test_read_config(

debug = cli.render(manager.root_dir, manager.tmpfile(cfg_str, suffix=".toml"))["debug"]
# Some things moved to "conf" that used to be on state:
out = (
debug["state"][config_var]
if config_var in debug["state"]
else debug["state"]["conf"][config_var]
)
out = debug[config_var] if config_var in debug else debug["conf"][config_var]
assert out == final_expected


@@ -299,7 +295,7 @@ def test_valid_coercion(as_type: tp.Any, input_val: tp.Any, expected: tp.Any):
manager.create_cfg(
{"context": {"static": {"FOO": {"value": input_val, "coerce": as_type}}}}
),
)["debug"]["state"]["ctx"]["FOO"]
)["debug"]["ctx"]["FOO"]
== expected
)

@@ -324,7 +320,7 @@ def test_valid_coercion(as_type: tp.Any, input_val: tp.Any, expected: tp.Any):
}
}
),
)["debug"]["state"]["ctx"]["FOO"]
)["debug"]["ctx"]["FOO"]
== expected
)

@@ -340,6 +336,6 @@ def test_valid_coercion(as_type: tp.Any, input_val: tp.Any, expected: tp.Any):
manager.create_cfg(
{"context": {"env": {"FOO": {"env_name": "FOO", "coerce": as_type}}}},
),
)["debug"]["state"]["ctx"]["FOO"]
)["debug"]["ctx"]["FOO"]
== expected
)