Skip to content

Commit

Permalink
internal: Merge engines into the cache crate. (#1447)
Browse files Browse the repository at this point in the history
* Merge engines.

* Better API.

* Add helper.
  • Loading branch information
milesj committed Apr 26, 2024
1 parent 4f28308 commit 3be106e
Show file tree
Hide file tree
Showing 27 changed files with 276 additions and 136 deletions.
3 changes: 2 additions & 1 deletion Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion crates/cli/src/queries/hash.rs
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ use tracing::debug;
pub async fn query_hash(workspace: &Workspace, hash: &str) -> AppResult<(String, String)> {
debug!("Querying for hash manifest with {}", color::hash(hash));

for file in fs::read_dir(&workspace.hash_engine.hashes_dir)? {
for file in fs::read_dir(&workspace.cache_engine.hash.hashes_dir)? {
let path = file.path();
let name = fs::file_name(&path).replace(".json", "");

Expand Down
1 change: 1 addition & 0 deletions crates/cli/tests/run_test.rs
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ fn extract_hash_from_run(fixture: &Path, target_id: &str) -> String {
let engine = CacheEngine::new(fixture).unwrap();
let cache: RunTargetState = json::read_file(
engine
.state
.states_dir
.join(target_id.replace(':', "/"))
.join("lastRun.json"),
Expand Down
20 changes: 13 additions & 7 deletions crates/core/action-pipeline/src/actions/install_deps.rs
Original file line number Diff line number Diff line change
Expand Up @@ -114,7 +114,8 @@ pub async fn install_deps(
let manifest_path = working_dir.join(&manifest);
let lockfile_path = working_dir.join(&lockfile);
let mut hasher = workspace
.hash_engine
.cache_engine
.hash
.create_hasher(format!("Install {} deps", runtime.label()));
let mut last_modified = 0;

Expand Down Expand Up @@ -145,16 +146,21 @@ pub async fn install_deps(
}

// Install dependencies in the working directory
let hash = workspace.hash_engine.save_manifest(hasher)?;
let hash = workspace.cache_engine.hash.save_manifest(hasher)?;

let state_path = format!("deps{runtime}.json");
let mut state = workspace.cache_engine.cache_state::<DependenciesState>(
if let Some(project) = &project {
project.get_cache_dir().join(state_path)
let mut state = workspace
.cache_engine
.state
.load_state::<DependenciesState>(if let Some(project) = &project {
workspace
.cache_engine
.state
.get_project_dir(&project.id)
.join(state_path)
} else {
PathBuf::from(state_path)
},
)?;
})?;

Check warning on line 163 in crates/core/action-pipeline/src/actions/install_deps.rs

View check run for this annotation

Codecov / codecov/patch

crates/core/action-pipeline/src/actions/install_deps.rs#L163

Added line #L163 was not covered by tests

if hash != state.data.last_hash
|| last_modified == 0
Expand Down
3 changes: 2 additions & 1 deletion crates/core/action-pipeline/src/actions/setup_tool.rs
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,8 @@ pub async fn setup_tool(

let mut state = workspace
.cache_engine
.cache_state::<ToolState>(format!("tool{}-{}.json", runtime, runtime.requirement))?;
.state
.load_state::<ToolState>(format!("tool{}-{}.json", runtime, runtime.requirement))?;

// Install and setup the specific tool + version in the toolchain!
let installed_count = PlatformManager::write()
Expand Down
3 changes: 2 additions & 1 deletion crates/core/action-pipeline/src/actions/sync_project.rs
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,8 @@ pub async fn sync_project(
// Create a snapshot for tasks to reference
workspace
.cache_engine
.write_state(project.get_cache_dir().join("snapshot.json"), project)?;
.state
.save_project_snapshot(&project.id, project)?;

// Collect all project dependencies so we can pass them along.
// We can't pass the graph itself because of circuler references between crates!
Expand Down
10 changes: 5 additions & 5 deletions crates/core/action-pipeline/src/subscribers/local_cache.rs
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ impl Subscriber for LocalCacheSubscriber {
// We only check for the archive, as the manifest is purely for local debugging!
Event::TargetOutputCacheCheck { hash, .. } => {
if get_cache_mode().is_readable()
&& workspace.hash_engine.get_archive_path(hash).exists()
&& workspace.cache_engine.hash.get_archive_path(hash).exists()
{
return Ok(EventFlow::Return("local-cache".into()));
}
Expand All @@ -44,8 +44,8 @@ impl Subscriber for LocalCacheSubscriber {
task,
..
} => {
let state_dir = workspace.cache_engine.states_dir.join(task.get_cache_dir());
let archive_path = workspace.hash_engine.get_archive_path(hash);
let state_dir = workspace.cache_engine.state.get_target_dir(&task.target);
let archive_path = workspace.cache_engine.hash.get_archive_path(hash);
let output_paths = task
.outputs
.iter()
Expand All @@ -64,8 +64,8 @@ impl Subscriber for LocalCacheSubscriber {
task,
..
} => {
let state_dir = workspace.cache_engine.states_dir.join(task.get_cache_dir());
let archive_path = workspace.hash_engine.get_archive_path(hash);
let state_dir = workspace.cache_engine.state.get_target_dir(&task.target);
let archive_path = workspace.cache_engine.hash.get_archive_path(hash);
let output_paths = task
.outputs
.iter()
Expand Down
2 changes: 1 addition & 1 deletion crates/core/action-pipeline/src/subscribers/moonbase.rs
Original file line number Diff line number Diff line change
Expand Up @@ -499,7 +499,7 @@ impl Subscriber for MoonbaseSubscriber {
Event::TargetOutputHydrating { hash, .. } => {
if get_cache_mode().is_readable() {
if let Some(download_url) = self.download_urls.get(*hash) {
let archive_file = workspace.hash_engine.get_archive_path(hash);
let archive_file = workspace.cache_engine.hash.get_archive_path(hash);

Check warning on line 502 in crates/core/action-pipeline/src/subscribers/moonbase.rs

View check run for this annotation

Codecov / codecov/patch

crates/core/action-pipeline/src/subscribers/moonbase.rs#L502

Added line #L502 was not covered by tests

trace!(
target: LOG_TARGET,
Expand Down
36 changes: 18 additions & 18 deletions crates/core/actions/src/sync_codeowners.rs
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
use moon_cache_item::CommonState;
use moon_codeowners::{CodeownersGenerator, CodeownersHash};
use moon_config::CodeownersOrderBy;
use moon_project_graph::ProjectGraph;
Expand All @@ -10,8 +9,7 @@ pub async fn sync_codeowners(
project_graph: &ProjectGraph,
force: bool,
) -> miette::Result<PathBuf> {
let cache_engine = &workspace.cache_engine;
let hash_engine = &workspace.hash_engine;
let mut generator = CodeownersGenerator::new(&workspace.root, workspace.config.vcs.provider)?;

// Sort the projects based on config
let mut projects = project_graph.get_all_unexpanded();
Expand All @@ -22,37 +20,39 @@ pub async fn sync_codeowners(
CodeownersOrderBy::ProjectName => a.id.cmp(&d.id),
});

// Generate the codeowners file
// Generate a hash for the codeowners file
let mut codeowners_hash = CodeownersHash::new(&workspace.config.codeowners);
let mut codeowners = CodeownersGenerator::new(&workspace.root, workspace.config.vcs.provider)?;

if !workspace.config.codeowners.global_paths.is_empty() {
codeowners.add_workspace_entries(&workspace.config.codeowners)?;
generator.add_workspace_entries(&workspace.config.codeowners)?;

Check warning on line 27 in crates/core/actions/src/sync_codeowners.rs

View check run for this annotation

Codecov / codecov/patch

crates/core/actions/src/sync_codeowners.rs#L27

Added line #L27 was not covered by tests
}

for project in &projects {
for project in projects {
if !project.config.owners.paths.is_empty() {
codeowners_hash.add_project(&project.id, &project.config.owners);

codeowners.add_project_entry(
generator.add_project_entry(

Check warning on line 34 in crates/core/actions/src/sync_codeowners.rs

View check run for this annotation

Codecov / codecov/patch

crates/core/actions/src/sync_codeowners.rs#L34

Added line #L34 was not covered by tests
&project.id,
project.source.as_str(),
&project.config.owners,
)?;
}
}

let file_path = codeowners.file_path.clone();

// Check the cache before writing the file
let mut state = cache_engine.cache_state::<CommonState>("codeowners.json")?;
let hash = hash_engine.save_manifest_without_hasher("CODEOWNERS", &codeowners_hash)?;
let file_path = generator.file_path.clone();

if force || hash != state.data.last_hash {
codeowners.generate()?;

state.data.last_hash = hash;
state.save()?;
// Force run the generator and bypass cache
if force {
generator.generate()?;

Check warning on line 46 in crates/core/actions/src/sync_codeowners.rs

View check run for this annotation

Codecov / codecov/patch

crates/core/actions/src/sync_codeowners.rs#L46

Added line #L46 was not covered by tests
}
// Only generate if the hash has changed
else {
workspace
.cache_engine
.execute_if_changed("codeowners.json", codeowners_hash, || async {
generator.generate()
})
.await?;

Check warning on line 55 in crates/core/actions/src/sync_codeowners.rs

View check run for this annotation

Codecov / codecov/patch

crates/core/actions/src/sync_codeowners.rs#L55

Added line #L55 was not covered by tests
}

Ok(file_path)
Expand Down
28 changes: 13 additions & 15 deletions crates/core/actions/src/sync_vcs_hooks.rs
Original file line number Diff line number Diff line change
@@ -1,11 +1,14 @@
use moon_cache_item::CommonState;
use moon_vcs_hooks::{HooksGenerator, HooksHash};
use moon_workspace::Workspace;

pub async fn sync_vcs_hooks(workspace: &Workspace, force: bool) -> miette::Result<()> {
let vcs_config = &workspace.config.vcs;
let cache_engine = &workspace.cache_engine;
let hash_engine = &workspace.hash_engine;
let generator = HooksGenerator::new(&workspace.root, &workspace.vcs, vcs_config);

// Force run the generator and bypass cache
if force {
return generator.generate().await;

Check warning on line 10 in crates/core/actions/src/sync_vcs_hooks.rs

View check run for this annotation

Codecov / codecov/patch

crates/core/actions/src/sync_vcs_hooks.rs#L10

Added line #L10 was not covered by tests
}

// Hash all the hook commands
let mut hooks_hash = HooksHash::new(&vcs_config.manager);
Expand All @@ -14,18 +17,13 @@ pub async fn sync_vcs_hooks(workspace: &Workspace, force: bool) -> miette::Resul
hooks_hash.add_hook(hook_name, commands);
}

// Check the cache before creating the files
let mut state = cache_engine.cache_state::<CommonState>("vcsHooks.json")?;
let hash = hash_engine.save_manifest_without_hasher("VCS hooks", &hooks_hash)?;

if force || hash != state.data.last_hash {
HooksGenerator::new(&workspace.root, &workspace.vcs, vcs_config)
.generate()
.await?;

state.data.last_hash = hash;
state.save()?;
}
// Only generate if the hash has changed
workspace
.cache_engine
.execute_if_changed("vcsHooks.json", hooks_hash, || async {
generator.generate().await
})
.await?;

Ok(())
}
Expand Down
4 changes: 1 addition & 3 deletions crates/core/moon/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -212,9 +212,7 @@ pub async fn build_project_graph(workspace: &mut Workspace) -> miette::Result<Pr

pub async fn generate_project_graph(workspace: &mut Workspace) -> miette::Result<ProjectGraph> {
let context = create_project_graph_context(workspace).await;
let builder =
ProjectGraphBuilder::generate(context, &workspace.cache_engine, &workspace.hash_engine)
.await?;
let builder = ProjectGraphBuilder::generate(context, &workspace.cache_engine).await?;

let graph = builder.build().await?;

Expand Down
12 changes: 7 additions & 5 deletions crates/core/runner/src/runner.rs
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,8 @@ impl<'a> Runner<'a> {
) -> miette::Result<Runner<'a>> {
let mut cache = workspace
.cache_engine
.cache_state::<RunTargetState>(task.get_cache_dir().join("lastRun.json"))?;
.state
.load_target_state::<RunTargetState>(&task.target)?;

if cache.data.target.is_empty() {
cache.data.target = task.target.to_string();
Expand Down Expand Up @@ -420,8 +421,8 @@ impl<'a> Runner<'a> {
path::to_string(
self.workspace
.cache_engine
.states_dir
.join(self.project.get_cache_dir().join("snapshot.json")),
.state
.get_project_snapshot_path(&self.project.id),
)?,
);

Expand Down Expand Up @@ -551,7 +552,8 @@ impl<'a> Runner<'a> {
) -> miette::Result<Option<HydrateFrom>> {
let mut hasher = self
.workspace
.hash_engine
.cache_engine
.hash
.create_hasher(format!("Run {} target", self.task.target));

self.hash_common_target(context, &mut hasher).await?;
Expand Down Expand Up @@ -595,7 +597,7 @@ impl<'a> Runner<'a> {
self.cache.data.hash = hash.clone();

// Refresh the hash manifest
self.workspace.hash_engine.save_manifest(hasher)?;
self.workspace.cache_engine.hash.save_manifest(hasher)?;

// Check if that hash exists in the cache
if let EventFlow::Return(value) = self
Expand Down
4 changes: 3 additions & 1 deletion nextgen/api/src/launchpad.rs
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,9 @@ impl Launchpad {
moon_env: &MoonEnvironment,
bypass_cache: bool,
) -> miette::Result<Option<VersionCheck>> {
let mut state = cache_engine.cache_state::<CurrentVersionState>("moonVersion.json")?;
let mut state = cache_engine
.state
.load_state::<CurrentVersionState>("moonVersion.json")?;

if let Some(last_check) = state.data.last_check {

Check warning on line 73 in nextgen/api/src/launchpad.rs

View check run for this annotation

Codecov / codecov/patch

nextgen/api/src/launchpad.rs#L69-L73

Added lines #L69 - L73 were not covered by tests
if (last_check + ALERT_PAUSE_DURATION) > SystemTime::now() && !bypass_cache {
Expand Down
2 changes: 2 additions & 0 deletions nextgen/cache/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@ publish = false
[dependencies]
moon_cache_item = { path = "../cache-item" }
moon_common = { path = "../common" }
moon_hash = { path = "../hash" }
moon_target = { path = "../target" }
moon_time = { path = "../time" }
miette = { workspace = true }
serde = { workspace = true }
Expand Down
Loading

0 comments on commit 3be106e

Please sign in to comment.