Skip to content

Commit

Permalink
feat: add build time executed modules to stats (#6829)
Browse files Browse the repository at this point in the history
* feat: add build time executed modules

* feat: add build time executed runtime modules
  • Loading branch information
LingyuCoder authored Jun 18, 2024
1 parent 371a8c8 commit b3147a9
Show file tree
Hide file tree
Showing 15 changed files with 746 additions and 84 deletions.
1 change: 1 addition & 0 deletions crates/node_binding/binding.d.ts
Original file line number Diff line number Diff line change
Expand Up @@ -556,6 +556,7 @@ export interface JsStatsModule {
postOrderIndex?: number
built: boolean
codeGenerated: boolean
buildTimeExecuted: boolean
cached: boolean
cacheable: boolean
optional: boolean
Expand Down
2 changes: 2 additions & 0 deletions crates/rspack_binding_values/src/stats.rs
Original file line number Diff line number Diff line change
Expand Up @@ -245,6 +245,7 @@ pub struct JsStatsModule {
pub post_order_index: Option<u32>,
pub built: bool,
pub code_generated: bool,
pub build_time_executed: bool,
pub cached: bool,
pub cacheable: bool,
pub optional: bool,
Expand Down Expand Up @@ -316,6 +317,7 @@ impl TryFrom<rspack_core::StatsModule<'_>> for JsStatsModule {
post_order_index: stats.post_order_index,
built: stats.built,
code_generated: stats.code_generated,
build_time_executed: stats.build_time_executed,
cached: stats.cached,
cacheable: stats.cacheable,
optional: stats.optional,
Expand Down
6 changes: 5 additions & 1 deletion crates/rspack_core/src/compiler/compilation.rs
Original file line number Diff line number Diff line change
Expand Up @@ -157,6 +157,7 @@ pub struct Compilation {
pub code_generation_results: CodeGenerationResults,
pub built_modules: IdentifierSet,
pub code_generated_modules: IdentifierSet,
pub build_time_executed_modules: IdentifierSet,
pub old_cache: Arc<OldCache>,
pub code_splitting_cache: CodeSplittingCache,
pub hash: Option<RspackHashDigest>,
Expand Down Expand Up @@ -238,6 +239,7 @@ impl Compilation {
code_generation_results: Default::default(),
built_modules: Default::default(),
code_generated_modules: Default::default(),
build_time_executed_modules: Default::default(),
old_cache,
code_splitting_cache: Default::default(),
hash: None,
Expand Down Expand Up @@ -1002,7 +1004,9 @@ impl Compilation {
}

// take built_modules
self.built_modules = self.make_artifact.take_built_modules();
self
.built_modules
.extend(self.make_artifact.take_built_modules());
Ok(())
}

Expand Down
4 changes: 2 additions & 2 deletions crates/rspack_core/src/compiler/make/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -33,10 +33,10 @@ pub struct MakeArtifact {
}

impl MakeArtifact {
fn get_module_graph(&self) -> ModuleGraph {
pub fn get_module_graph(&self) -> ModuleGraph {
ModuleGraph::new(vec![&self.module_graph_partial], None)
}
fn get_module_graph_mut(&mut self) -> ModuleGraph {
pub fn get_module_graph_mut(&mut self) -> ModuleGraph {
ModuleGraph::new(vec![], Some(&mut self.module_graph_partial))
}
// TODO remove it
Expand Down
2 changes: 1 addition & 1 deletion crates/rspack_core/src/compiler/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ use tracing::instrument;

pub use self::compilation::*;
pub use self::hmr::{collect_changed_modules, CompilationRecords};
pub use self::module_executor::{ExecuteModuleId, ModuleExecutor};
pub use self::module_executor::{ExecuteModuleId, ExecutedRuntimeModule, ModuleExecutor};
use crate::old_cache::Cache as OldCache;
use crate::{
fast_set, BoxPlugin, CompilerOptions, Logger, PluginDriver, ResolverFactory, SharedPluginDriver,
Expand Down
76 changes: 69 additions & 7 deletions crates/rspack_core/src/compiler/module_executor/execute.rs
Original file line number Diff line number Diff line change
@@ -1,8 +1,10 @@
use std::{iter::once, sync::atomic::AtomicU32};

use itertools::Itertools;
use rayon::prelude::*;
use rspack_error::Result;
use rspack_identifier::IdentifierSet;
use rspack_identifier::{Identifier, IdentifierSet};
use rustc_hash::FxHashMap as HashMap;
use rustc_hash::FxHashSet as HashSet;
use tokio::{runtime::Handle, sync::oneshot::Sender};

Expand All @@ -12,9 +14,19 @@ use crate::{
utils::task_loop::{Task, TaskResult, TaskType},
Chunk, ChunkGraph, ChunkKind, CodeGenerationDataAssetInfo, CodeGenerationDataFilename,
CodeGenerationResult, CompilationAsset, CompilationAssets, DependencyId, EntryOptions,
Entrypoint, RuntimeSpec, SourceType,
Entrypoint, ModuleType, RuntimeSpec, SourceType,
};

#[derive(Debug, Clone)]
pub struct ExecutedRuntimeModule {
pub identifier: Identifier,
pub name: String,
pub name_for_condition: Option<String>,
pub module_type: ModuleType,
pub size: f64,
pub cacheable: bool,
}

static EXECUTE_MODULE_ID: AtomicU32 = AtomicU32::new(0);
pub type ExecuteModuleId = u32;

Expand All @@ -25,6 +37,7 @@ pub struct ExecuteModuleResult {
pub context_dependencies: HashSet<std::path::PathBuf>,
pub missing_dependencies: HashSet<std::path::PathBuf>,
pub build_dependencies: HashSet<std::path::PathBuf>,
pub code_generated_modules: IdentifierSet,
pub assets: HashSet<String>,
pub id: ExecuteModuleId,
}
Expand All @@ -34,7 +47,12 @@ pub struct ExecuteTask {
pub entry_dep_id: DependencyId,
pub public_path: Option<String>,
pub base_uri: Option<String>,
pub result_sender: Sender<(Result<ExecuteModuleResult>, CompilationAssets)>,
pub result_sender: Sender<(
Result<ExecuteModuleResult>,
CompilationAssets,
IdentifierSet,
Vec<ExecutedRuntimeModule>,
)>,
}

impl Task<MakeTaskContext> for ExecuteTask {
Expand Down Expand Up @@ -143,7 +161,16 @@ impl Task<MakeTaskContext> for ExecuteTask {
// replace code_generation_results is the same reason
compilation.chunk_graph = chunk_graph;

compilation.code_generation_modules(&mut None, false, modules.par_iter().copied())?;
let code_generation_results =
compilation.code_generation_modules(&mut None, false, modules.par_iter().copied())?;

code_generation_results
.iter()
.for_each(|module_identifier| {
compilation
.code_generated_modules
.insert(*module_identifier);
});

Handle::current().block_on(async {
compilation
Expand All @@ -170,14 +197,19 @@ impl Task<MakeTaskContext> for ExecuteTask {
.collect::<Vec<_>>()
);

let mut runtime_module_size = HashMap::default();
for runtime_id in &runtime_modules {
let runtime_module = compilation
.runtime_modules
.get(runtime_id)
.expect("runtime module exist");

let result =
CodeGenerationResult::default().with_javascript(runtime_module.generate(&compilation)?);
let runtime_module_source = runtime_module.generate(&compilation)?;
runtime_module_size.insert(
runtime_module.identifier(),
runtime_module_source.size() as f64,
);
let result = CodeGenerationResult::default().with_javascript(runtime_module_source);
let result_id = result.id;

compilation
Expand All @@ -187,6 +219,9 @@ impl Task<MakeTaskContext> for ExecuteTask {
compilation
.code_generation_results
.add(*runtime_id, runtime.clone(), result_id);
compilation
.code_generated_modules
.insert(runtime_module.identifier());
}

let codegen_results = compilation.code_generation_results.clone();
Expand Down Expand Up @@ -257,12 +292,39 @@ impl Task<MakeTaskContext> for ExecuteTask {
};

let assets = std::mem::take(compilation.assets_mut());
let code_generated_modules = std::mem::take(&mut compilation.code_generated_modules);
if let Ok(ref mut result) = execute_result {
result.assets = assets.keys().cloned().collect::<HashSet<_>>();
}
let executed_runtime_modules = runtime_modules
.iter()
.map(|runtime_id| {
let runtime_module = compilation
.runtime_modules
.get(runtime_id)
.expect("runtime module exist");
let identifier = runtime_module.identifier();
ExecutedRuntimeModule {
identifier,
name: runtime_module.name().to_string(),
name_for_condition: runtime_module.name_for_condition().map(|n| n.to_string()),
module_type: *runtime_module.module_type(),
cacheable: runtime_module.cacheable(),
size: runtime_module_size
.get(&identifier)
.map(|s| s.to_owned())
.unwrap_or(0 as f64),
}
})
.collect_vec();
context.recovery_from_temp_compilation(compilation);
result_sender
.send((execute_result, assets))
.send((
execute_result,
assets,
code_generated_modules,
executed_runtime_modules,
))
.expect("should send result success");
Ok(vec![])
}
Expand Down
29 changes: 27 additions & 2 deletions crates/rspack_core/src/compiler/module_executor/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,12 @@ mod entry;
mod execute;
mod overwrite;

use dashmap::mapref::entry::Entry;
use dashmap::DashMap;
use dashmap::{mapref::entry::Entry, DashSet};
pub use execute::ExecuteModuleId;
pub use execute::ExecutedRuntimeModule;
use rspack_error::Result;
use rspack_identifier::Identifier;
use tokio::sync::{
mpsc::{unbounded_channel, UnboundedSender},
oneshot,
Expand All @@ -32,6 +34,8 @@ pub struct ModuleExecutor {
event_sender: Option<UnboundedSender<Event>>,
stop_receiver: Option<oneshot::Receiver<MakeArtifact>>,
assets: DashMap<String, CompilationAsset>,
code_generated_modules: DashSet<Identifier>,
pub executed_runtime_modules: DashMap<Identifier, ExecutedRuntimeModule>,
}

impl ModuleExecutor {
Expand Down Expand Up @@ -107,6 +111,16 @@ impl ModuleExecutor {

let diagnostics = self.make_artifact.take_diagnostics();
compilation.extend_diagnostics(diagnostics);

let built_modules = self.make_artifact.take_built_modules();
for id in built_modules {
compilation.built_modules.insert(id);
}

let code_generated_modules = std::mem::take(&mut self.code_generated_modules);
for id in code_generated_modules {
compilation.code_generated_modules.insert(id);
}
}

#[allow(clippy::too_many_arguments)]
Expand Down Expand Up @@ -149,12 +163,23 @@ impl ModuleExecutor {
},
))
.expect("should success");
let (execute_result, assets) = rx.await.expect("should receiver success");
let (execute_result, assets, code_generated_modules, executed_runtime_modules) =
rx.await.expect("should receiver success");

for (key, value) in assets {
self.assets.insert(key, value);
}

for id in code_generated_modules {
self.code_generated_modules.insert(id);
}

for runtime_module in executed_runtime_modules {
self
.executed_runtime_modules
.insert(runtime_module.identifier, runtime_module);
}

execute_result
}
}
Loading

2 comments on commit b3147a9

@rspack-bot
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

📝 Benchmark detail: Open

Name Base (2024-06-18 553f785) Current Change
10000_development-mode + exec 2.21 s ± 22 ms 2.21 s ± 32 ms -0.07 %
10000_development-mode_hmr + exec 737 ms ± 12 ms 737 ms ± 7.8 ms +0.06 %
10000_production-mode + exec 2.57 s ± 22 ms 2.58 s ± 27 ms +0.41 %
arco-pro_development-mode + exec 1.94 s ± 68 ms 1.9 s ± 54 ms -1.80 %
arco-pro_development-mode_hmr + exec 441 ms ± 2.1 ms 442 ms ± 1.9 ms +0.17 %
arco-pro_production-mode + exec 3.53 s ± 85 ms 3.51 s ± 92 ms -0.35 %
threejs_development-mode_10x + exec 1.46 s ± 11 ms 1.47 s ± 19 ms +0.45 %
threejs_development-mode_10x_hmr + exec 799 ms ± 3.3 ms 789 ms ± 11 ms -1.20 %
threejs_production-mode_10x + exec 4.76 s ± 33 ms 4.78 s ± 20 ms +0.35 %

@rspack-bot
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

📝 Ran ecosystem CI: Open

suite result
modernjs ❌ failure
_selftest ✅ success
nx ✅ success
rspress ✅ success
rsbuild ✅ success
compat ✅ success
examples ✅ success

Please sign in to comment.