From d87966de01a3ff296c35205b896ff93734b52ac0 Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Mon, 7 Oct 2024 12:57:10 +0200 Subject: [PATCH] add persistentCaching config option --- crates/napi/Cargo.toml | 4 +- crates/napi/src/next_api/project.rs | 39 +++-- crates/napi/src/next_api/utils.rs | 139 +++++++++++++++--- crates/napi/src/turbotrace.rs | 48 +++--- .../next/src/build/collect-build-traces.ts | 5 +- packages/next/src/build/index.ts | 1 + .../next/src/build/swc/generated-native.d.ts | 8 +- packages/next/src/build/swc/index.ts | 16 +- packages/next/src/build/swc/types.ts | 16 +- packages/next/src/server/config-schema.ts | 1 + packages/next/src/server/config-shared.ts | 5 + .../src/server/dev/hot-reloader-turbopack.ts | 1 + 12 files changed, 207 insertions(+), 76 deletions(-) diff --git a/crates/napi/Cargo.toml b/crates/napi/Cargo.toml index 604dd24188771..cae43aa15ca31 100644 --- a/crates/napi/Cargo.toml +++ b/crates/napi/Cargo.toml @@ -37,8 +37,6 @@ __internal_dhat-heap = ["dhat"] # effectively does nothing. __internal_dhat-ad-hoc = ["dhat"] -new-backend = ["dep:turbo-tasks-backend"] - # Enable specific tls features per-target. [target.'cfg(all(target_os = "windows", target_arch = "aarch64"))'.dependencies] next-core = { workspace = true, features = ["native-tls"] } @@ -107,7 +105,7 @@ lightningcss-napi = { workspace = true } tokio = { workspace = true, features = ["full"] } turbo-tasks = { workspace = true } turbo-tasks-memory = { workspace = true } -turbo-tasks-backend = { workspace = true, optional = true } +turbo-tasks-backend = { workspace = true } turbo-tasks-fs = { workspace = true } next-api = { workspace = true } next-build = { workspace = true } diff --git a/crates/napi/src/next_api/project.rs b/crates/napi/src/next_api/project.rs index ee2c720dac863..e4e195c5c8b79 100644 --- a/crates/napi/src/next_api/project.rs +++ b/crates/napi/src/next_api/project.rs @@ -22,7 +22,7 @@ use rand::Rng; use tokio::{io::AsyncWriteExt, time::Instant}; use tracing::Instrument; use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt, EnvFilter, Registry}; -use turbo_tasks::{Completion, RcStr, ReadRef, TransientInstance, TurboTasks, UpdateInfo, Vc}; +use turbo_tasks::{Completion, RcStr, ReadRef, TransientInstance, UpdateInfo, Vc}; use turbo_tasks_fs::{DiskFileSystem, FileContent, FileSystem, FileSystemPath}; use turbopack_core::{ diagnostics::PlainDiagnostic, @@ -44,7 +44,7 @@ use super::{ endpoint::ExternalEndpoint, utils::{ create_turbo_tasks, get_diagnostics, get_issues, subscribe, NapiDiagnostic, NapiIssue, - NextBackend, RootTask, TurbopackResult, VcArc, + NextTurboTasks, RootTask, TurbopackResult, VcArc, }, }; use crate::register; @@ -188,6 +188,8 @@ pub struct NapiDefineEnv { #[napi(object)] pub struct NapiTurboEngineOptions { + /// Use the new backend with persistent caching enabled. + pub persistent_caching: Option, /// An upper bound of memory that turbopack will attempt to stay under. pub memory_limit: Option, } @@ -272,7 +274,7 @@ impl From for DefineEnv { } pub struct ProjectInstance { - turbo_tasks: Arc>, + turbo_tasks: NextTurboTasks, container: Vc, exit_receiver: tokio::sync::Mutex>, } @@ -338,13 +340,20 @@ pub async fn project_new( .memory_limit .map(|m| m as usize) .unwrap_or(usize::MAX); - let turbo_tasks = create_turbo_tasks(PathBuf::from(&options.dist_dir), memory_limit)?; - #[cfg(not(feature = "new-backend"))] - { + let persistent_caching = turbo_engine_options.persistent_caching.unwrap_or_default(); + let turbo_tasks = create_turbo_tasks( + PathBuf::from(&options.dist_dir), + persistent_caching, + memory_limit, + )?; + if !persistent_caching { use std::io::Write; let stats_path = std::env::var_os("NEXT_TURBOPACK_TASK_STATISTICS"); if let Some(stats_path) = stats_path { - let task_stats = turbo_tasks.backend().task_statistics().enable().clone(); + let Some(backend) = turbo_tasks.memory_backend() else { + return Err(anyhow!("task statistics require a memory backend").into()); + }; + let task_stats = backend.task_statistics().enable().clone(); exit.on_exit(async move { tokio::task::spawn_blocking(move || { let mut file = std::fs::File::create(&stats_path) @@ -498,11 +507,7 @@ struct NapiRoute { } impl NapiRoute { - fn from_route( - pathname: String, - value: Route, - turbo_tasks: &Arc>, - ) -> Self { + fn from_route(pathname: String, value: Route, turbo_tasks: &NextTurboTasks) -> Self { let convert_endpoint = |endpoint: Vc>| { Some(External::new(ExternalEndpoint(VcArc::new( turbo_tasks.clone(), @@ -566,10 +571,7 @@ struct NapiMiddleware { } impl NapiMiddleware { - fn from_middleware( - value: &Middleware, - turbo_tasks: &Arc>, - ) -> Result { + fn from_middleware(value: &Middleware, turbo_tasks: &NextTurboTasks) -> Result { Ok(NapiMiddleware { endpoint: External::new(ExternalEndpoint(VcArc::new( turbo_tasks.clone(), @@ -586,10 +588,7 @@ struct NapiInstrumentation { } impl NapiInstrumentation { - fn from_instrumentation( - value: &Instrumentation, - turbo_tasks: &Arc>, - ) -> Result { + fn from_instrumentation(value: &Instrumentation, turbo_tasks: &NextTurboTasks) -> Result { Ok(NapiInstrumentation { node_js: External::new(ExternalEndpoint(VcArc::new( turbo_tasks.clone(), diff --git a/crates/napi/src/next_api/utils.rs b/crates/napi/src/next_api/utils.rs index a5c4284c73c66..20b776c1863ac 100644 --- a/crates/napi/src/next_api/utils.rs +++ b/crates/napi/src/next_api/utils.rs @@ -1,4 +1,6 @@ -use std::{collections::HashMap, future::Future, ops::Deref, path::PathBuf, sync::Arc}; +use std::{ + collections::HashMap, future::Future, ops::Deref, path::PathBuf, sync::Arc, time::Duration, +}; use anyhow::{anyhow, Context, Result}; use napi::{ @@ -7,7 +9,9 @@ use napi::{ JsFunction, JsObject, JsUnknown, NapiRaw, NapiValue, Status, }; use serde::Serialize; -use turbo_tasks::{ReadRef, TaskId, TryJoinIterExt, TurboTasks, Vc}; +use turbo_tasks::{ + trace::TraceRawVcs, ReadRef, TaskId, TryJoinIterExt, TurboTasks, UpdateInfo, Vc, +}; use turbo_tasks_fs::FileContent; use turbopack_core::{ diagnostics::{Diagnostic, DiagnosticContextExt, PlainDiagnostic}, @@ -18,41 +22,136 @@ use turbopack_core::{ use crate::util::log_internal_error_and_inform; -#[cfg(not(feature = "new-backend"))] -pub type NextBackend = turbo_tasks_memory::MemoryBackend; -#[cfg(feature = "new-backend")] -pub type NextBackend = turbo_tasks_backend::TurboTasksBackend; +#[derive(Clone)] +pub enum NextTurboTasks { + Memory(Arc>), + PersistentCaching(Arc>), +} + +impl NextTurboTasks { + pub fn dispose_root_task(&self, task: TaskId) { + match self { + NextTurboTasks::Memory(turbo_tasks) => turbo_tasks.dispose_root_task(task), + NextTurboTasks::PersistentCaching(turbo_tasks) => turbo_tasks.dispose_root_task(task), + } + } + + pub fn spawn_root_task(&self, functor: F) -> TaskId + where + T: Send, + F: Fn() -> Fut + Send + Sync + Clone + 'static, + Fut: Future>> + Send, + { + match self { + NextTurboTasks::Memory(turbo_tasks) => turbo_tasks.spawn_root_task(functor), + NextTurboTasks::PersistentCaching(turbo_tasks) => turbo_tasks.spawn_root_task(functor), + } + } + + pub async fn run_once( + &self, + future: impl Future> + Send + 'static, + ) -> Result { + match self { + NextTurboTasks::Memory(turbo_tasks) => turbo_tasks.run_once(future).await, + NextTurboTasks::PersistentCaching(turbo_tasks) => turbo_tasks.run_once(future).await, + } + } + + pub fn spawn_once_task(&self, future: Fut) -> TaskId + where + T: Send, + Fut: Future>> + Send + 'static, + { + match self { + NextTurboTasks::Memory(turbo_tasks) => turbo_tasks.spawn_once_task(future), + NextTurboTasks::PersistentCaching(turbo_tasks) => turbo_tasks.spawn_once_task(future), + } + } + + pub async fn aggregated_update_info( + &self, + aggregation: Duration, + timeout: Duration, + ) -> Option { + match self { + NextTurboTasks::Memory(turbo_tasks) => { + turbo_tasks + .aggregated_update_info(aggregation, timeout) + .await + } + NextTurboTasks::PersistentCaching(turbo_tasks) => { + turbo_tasks + .aggregated_update_info(aggregation, timeout) + .await + } + } + } + + pub async fn get_or_wait_aggregated_update_info(&self, aggregation: Duration) -> UpdateInfo { + match self { + NextTurboTasks::Memory(turbo_tasks) => { + turbo_tasks + .get_or_wait_aggregated_update_info(aggregation) + .await + } + NextTurboTasks::PersistentCaching(turbo_tasks) => { + turbo_tasks + .get_or_wait_aggregated_update_info(aggregation) + .await + } + } + } + + pub fn memory_backend(&self) -> Option<&turbo_tasks_memory::MemoryBackend> { + match self { + NextTurboTasks::Memory(turbo_tasks) => Some(turbo_tasks.backend()), + NextTurboTasks::PersistentCaching(_) => None, + } + } + + pub async fn stop_and_wait(&self) { + match self { + NextTurboTasks::Memory(turbo_tasks) => turbo_tasks.stop_and_wait().await, + NextTurboTasks::PersistentCaching(turbo_tasks) => turbo_tasks.stop_and_wait().await, + } + } +} -#[allow(unused_variables, reason = "feature-gated")] pub fn create_turbo_tasks( output_path: PathBuf, + persistent_caching: bool, memory_limit: usize, -) -> Result>> { - #[cfg(not(feature = "new-backend"))] - let backend = TurboTasks::new(turbo_tasks_memory::MemoryBackend::new(memory_limit)); - #[cfg(feature = "new-backend")] - let backend = TurboTasks::new(turbo_tasks_backend::TurboTasksBackend::new(Arc::new( - turbo_tasks_backend::LmdbBackingStorage::new(&output_path.join("cache/turbopack"))?, - ))); - Ok(backend) +) -> Result { + Ok(if persistent_caching { + NextTurboTasks::PersistentCaching(TurboTasks::new( + turbo_tasks_backend::TurboTasksBackend::new(Arc::new( + turbo_tasks_backend::LmdbBackingStorage::new(&output_path.join("cache/turbopack"))?, + )), + )) + } else { + NextTurboTasks::Memory(TurboTasks::new(turbo_tasks_memory::MemoryBackend::new( + memory_limit, + ))) + }) } /// A helper type to hold both a Vc operation and the TurboTasks root process. /// Without this, we'd need to pass both individually all over the place #[derive(Clone)] pub struct VcArc { - turbo_tasks: Arc>, + turbo_tasks: NextTurboTasks, /// The Vc. Must be resolved, otherwise you are referencing an inactive /// operation. vc: T, } impl VcArc { - pub fn new(turbo_tasks: Arc>, vc: T) -> Self { + pub fn new(turbo_tasks: NextTurboTasks, vc: T) -> Self { Self { turbo_tasks, vc } } - pub fn turbo_tasks(&self) -> &Arc> { + pub fn turbo_tasks(&self) -> &NextTurboTasks { &self.turbo_tasks } } @@ -75,7 +174,7 @@ pub fn serde_enum_to_string(value: &T) -> Result { /// The root of our turbopack computation. pub struct RootTask { #[allow(dead_code)] - turbo_tasks: Arc>, + turbo_tasks: NextTurboTasks, #[allow(dead_code)] task_id: Option, } @@ -319,7 +418,7 @@ impl ToNapiValue for TurbopackResult { } pub fn subscribe> + Send, V: ToNapiValue>( - turbo_tasks: Arc>, + turbo_tasks: NextTurboTasks, func: JsFunction, handler: impl 'static + Sync + Send + Clone + Fn() -> F, mapper: impl 'static + Sync + Send + FnMut(ThreadSafeCallContext) -> napi::Result>, diff --git a/crates/napi/src/turbotrace.rs b/crates/napi/src/turbotrace.rs index fef534c04b550..95db996f00122 100644 --- a/crates/napi/src/turbotrace.rs +++ b/crates/napi/src/turbotrace.rs @@ -2,46 +2,52 @@ use std::{path::PathBuf, sync::Arc}; use napi::bindgen_prelude::*; use node_file_trace::{start, Args}; -use turbo_tasks::TurboTasks; use turbopack::{ module_options::{EcmascriptOptionsContext, ModuleOptionsContext}, resolve_options_context::ResolveOptionsContext, }; -use crate::next_api::utils::{self, NextBackend}; +use crate::next_api::utils::{self, NextTurboTasks}; #[napi] pub fn create_turbo_tasks( output_path: String, + persistent_caching: bool, memory_limit: Option, -) -> External>> { +) -> External { let limit = memory_limit.map(|u| u as usize).unwrap_or(usize::MAX); - let turbo_tasks = utils::create_turbo_tasks(PathBuf::from(&output_path), limit) - .expect("Failed to create TurboTasks"); + let turbo_tasks = + utils::create_turbo_tasks(PathBuf::from(&output_path), persistent_caching, limit) + .expect("Failed to create TurboTasks"); External::new_with_size_hint(turbo_tasks, limit) } #[napi] pub async fn run_turbo_tracing( options: Buffer, - turbo_tasks: External>>, + turbo_tasks: External, ) -> napi::Result> { let args: Args = serde_json::from_slice(options.as_ref())?; - let files = start( - Arc::new(args), - turbo_tasks.clone(), - Some(ModuleOptionsContext { - ecmascript: EcmascriptOptionsContext { - enable_types: true, - ..Default::default() - }, - enable_mdx: true, + let args = Arc::new(args); + let module_options = Some(ModuleOptionsContext { + ecmascript: EcmascriptOptionsContext { + enable_types: true, ..Default::default() - }), - Some(ResolveOptionsContext { - ..Default::default() - }), - ) - .await?; + }, + enable_mdx: true, + ..Default::default() + }); + let resolve_options = Some(ResolveOptionsContext { + ..Default::default() + }); + let files = match &*turbo_tasks { + NextTurboTasks::Memory(turbo_tasks) => { + start(args, turbo_tasks.clone(), module_options, resolve_options).await? + } + NextTurboTasks::PersistentCaching(turbo_tasks) => { + start(args, turbo_tasks.clone(), module_options, resolve_options).await? + } + }; + Ok(files.into_iter().map(|f| f.to_string()).collect()) } diff --git a/packages/next/src/build/collect-build-traces.ts b/packages/next/src/build/collect-build-traces.ts index 75639427ed0cf..e8035c02ca900 100644 --- a/packages/next/src/build/collect-build-traces.ts +++ b/packages/next/src/build/collect-build-traces.ts @@ -27,7 +27,7 @@ import { normalizeAppPath } from '../shared/lib/router/utils/app-paths' import isError from '../lib/is-error' import type { NodeFileTraceReasons } from '@vercel/nft' import type { RoutesUsingEdgeRuntime } from './utils' -import type { ExternalObject, TurboTasks } from './swc/generated-native' +import type { ExternalObject, NextTurboTasks } from './swc/generated-native' const debug = debugOriginal('next:build:build-traces') @@ -108,7 +108,7 @@ export async function collectBuildTraces({ }) { const startTime = Date.now() debug('starting build traces') - let turboTasksForTrace: ExternalObject + let turboTasksForTrace: ExternalObject let bindings = await loadBindings() const runTurbotrace = async function () { @@ -120,6 +120,7 @@ export async function collectBuildTraces({ let turbotraceFiles: string[] | undefined turboTasksForTrace = bindings.turbo.createTurboTasks( distDir, + false, (config.experimental.turbotrace?.memoryLimit ?? TURBO_TRACE_DEFAULT_MEMORY_LIMIT) * 1024 * diff --git a/packages/next/src/build/index.ts b/packages/next/src/build/index.ts index 64a5ca0bf6063..eb651d013b0ed 100644 --- a/packages/next/src/build/index.ts +++ b/packages/next/src/build/index.ts @@ -1409,6 +1409,7 @@ export default async function build( browserslistQuery: supportedBrowsers.join(', '), }, { + persistentCaching: config.experimental.turbo?.persistentCaching, memoryLimit: config.experimental.turbo?.memoryLimit, } ) diff --git a/packages/next/src/build/swc/generated-native.d.ts b/packages/next/src/build/swc/generated-native.d.ts index 2c9def68d3b7b..c7f991907a4ef 100644 --- a/packages/next/src/build/swc/generated-native.d.ts +++ b/packages/next/src/build/swc/generated-native.d.ts @@ -4,6 +4,7 @@ import type { TurbopackResult } from './types' export type TurboTasks = { readonly __tag: unique symbol } export type ExternalEndpoint = { readonly __tag: unique symbol } +export type NextTurboTasks = { readonly __tag: unique symbol } export type RefCell = { readonly __tag: unique symbol } export type NapiRouteHas = { type: string @@ -172,6 +173,8 @@ export interface NapiDefineEnv { nodejs: Array } export interface NapiTurboEngineOptions { + /** Use the new backend with persistent caching enabled. */ + persistentCaching?: boolean /** An upper bound of memory that turbopack will attempt to stay under. */ memoryLimit?: number } @@ -364,11 +367,12 @@ export interface NapiRewrite { } export function createTurboTasks( outputPath: string, + persistentCaching: boolean, memoryLimit?: number | undefined | null -): ExternalObject +): ExternalObject export function runTurboTracing( options: Buffer, - turboTasks: ExternalObject + turboTasks: ExternalObject ): Promise> export function getTargetTriple(): string export function initHeapProfiler(): ExternalObject diff --git a/packages/next/src/build/swc/index.ts b/packages/next/src/build/swc/index.ts index 0b35aee09bd5c..f275b6df134af 100644 --- a/packages/next/src/build/swc/index.ts +++ b/packages/next/src/build/swc/index.ts @@ -26,7 +26,7 @@ import type { ExternalObject, NapiPartialProjectOptions, NapiProjectOptions, - TurboTasks, + NextTurboTasks, } from './generated-native' import type { Binding, @@ -1055,8 +1055,9 @@ async function loadWasm(importPath = '') { }, createTurboTasks: function ( _outputPath: string, + _persistentCaching: boolean, _memoryLimit?: number | undefined - ): ExternalObject { + ): ExternalObject { throw new Error( '`turbo.createTurboTasks` is not supported by the wasm bindings.' ) @@ -1228,7 +1229,7 @@ function loadNative(importPath?: string) { initHeapProfiler: bindings.initHeapProfiler, teardownHeapProfiler: bindings.teardownHeapProfiler, turbo: { - startTrace(options = {}, turboTasks: ExternalObject) { + startTrace(options = {}, turboTasks: ExternalObject) { initHeapProfiler() return (customBindings ?? bindings).runTurboTracing( toBuffer({ exact: true, ...options }), @@ -1237,9 +1238,14 @@ function loadNative(importPath?: string) { }, createTurboTasks( outputPath: string, + persistentCaching: boolean, memoryLimit?: number - ): ExternalObject { - return bindings.createTurboTasks(outputPath, memoryLimit) + ): ExternalObject { + return bindings.createTurboTasks( + outputPath, + persistentCaching, + memoryLimit + ) }, createProject: bindingToApi(customBindings ?? bindings, false), startTurbopackTraceServer(traceFilePath) { diff --git a/packages/next/src/build/swc/types.ts b/packages/next/src/build/swc/types.ts index 76eaf9a02e112..c0a0ad0b74efc 100644 --- a/packages/next/src/build/swc/types.ts +++ b/packages/next/src/build/swc/types.ts @@ -1,15 +1,20 @@ import type { NextConfigComplete } from '../../server/config-shared' import type { __ApiPreviewProps } from '../../server/api-utils' -import type { ExternalObject, RefCell, TurboTasks } from './generated-native' +import type { + ExternalObject, + NextTurboTasks, + RefCell, +} from './generated-native' export interface Binding { isWasm: boolean turbo: { - startTrace(options: any, turboTasks: ExternalObject): any + startTrace(options: any, turboTasks: ExternalObject): any createTurboTasks( outputPath: string, + persistentCaching: boolean, memoryLimit?: number - ): ExternalObject + ): ExternalObject createProject( options: ProjectOptions, turboEngineOptions?: TurboEngineOptions @@ -107,6 +112,11 @@ export type TurbopackResult = T & { } export interface TurboEngineOptions { + /** + * Use the new backend with persistent caching enabled. + */ + persistentCaching?: boolean + /** * An upper bound of memory that turbopack will attempt to stay under. */ diff --git a/packages/next/src/server/config-schema.ts b/packages/next/src/server/config-schema.ts index df208132c65f1..f335d83233ef1 100644 --- a/packages/next/src/server/config-schema.ts +++ b/packages/next/src/server/config-schema.ts @@ -391,6 +391,7 @@ export const configSchema: zod.ZodType = z.lazy(() => resolveExtensions: z.array(z.string()).optional(), useSwcCss: z.boolean().optional(), treeShaking: z.boolean().optional(), + persistentCaching: z.boolean().optional(), memoryLimit: z.number().optional(), moduleIdStrategy: z.enum(['named', 'deterministic']).optional(), }) diff --git a/packages/next/src/server/config-shared.ts b/packages/next/src/server/config-shared.ts index 257e81ae041d1..1bb98ec08579e 100644 --- a/packages/next/src/server/config-shared.ts +++ b/packages/next/src/server/config-shared.ts @@ -154,6 +154,11 @@ export interface ExperimentalTurboOptions { */ memoryLimit?: number + /** + * Enable persistent caching for the turbopack dev server and build. + */ + persistentCaching?: boolean + /** * Enable tree shaking for the turbopack dev server and build. */ diff --git a/packages/next/src/server/dev/hot-reloader-turbopack.ts b/packages/next/src/server/dev/hot-reloader-turbopack.ts index f35251289c010..b2ad33abd7f60 100644 --- a/packages/next/src/server/dev/hot-reloader-turbopack.ts +++ b/packages/next/src/server/dev/hot-reloader-turbopack.ts @@ -176,6 +176,7 @@ export async function createHotReloaderTurbopack( browserslistQuery: supportedBrowsers.join(', '), }, { + persistentCaching: opts.nextConfig.experimental.turbo?.persistentCaching, memoryLimit: opts.nextConfig.experimental.turbo?.memoryLimit, } )