Seperated build directory from target directory

This commits implements the seperation of the intermidate artifact
directory (called "build directory") from the target directory. (see #14125)

src/cargo/core/compiler/build_runner/compilation_files.rs

@@ -217,7 +217,7 @@ impl<'a, 'gctx: 'a> CompilationFiles<'a, 'gctx> {
         } else if unit.target.is_custom_build() {
             self.build_script_dir(unit)
         } else if unit.target.is_example() {
-            self.layout(unit.kind).examples().to_path_buf()
+            self.layout(unit.kind).build_examples().to_path_buf()
         } else if unit.artifact.is_true() {
             self.artifact_dir(unit)
         } else {

src/cargo/core/compiler/custom_build.rs

@@ -1124,7 +1124,7 @@ fn prepare_metabuild(
     let path = unit
         .pkg
         .manifest()
-        .metabuild_path(build_runner.bcx.ws.target_dir());
+        .metabuild_path(build_runner.bcx.ws.build_dir());
     paths::create_dir_all(path.parent().unwrap())?;
     paths::write_if_changed(path, &output)?;
     Ok(())

src/cargo/core/compiler/fingerprint/mod.rs

@@ -1857,7 +1857,7 @@ pub fn dep_info_loc(build_runner: &mut BuildRunner<'_, '_>, unit: &Unit) -> Path
 /// Returns an absolute path that target directory.
 /// All paths are rewritten to be relative to this.
 fn target_root(build_runner: &BuildRunner<'_, '_>) -> PathBuf {
-    build_runner.bcx.ws.target_dir().into_path_unlocked()
+    build_runner.bcx.ws.build_dir().into_path_unlocked()
 }
 
 /// Reads the value from the old fingerprint hash file and compare.

src/cargo/core/compiler/future_incompat.rs

@@ -166,7 +166,7 @@ impl OnDiskReports {
         }
         let on_disk = serde_json::to_vec(&self).unwrap();
         if let Err(e) = ws
-            .target_dir()
+            .build_dir()
             .open_rw_exclusive_create(
                 FUTURE_INCOMPAT_FILE,
                 ws.gctx(),
@@ -191,7 +191,7 @@ impl OnDiskReports {
 
     /// Loads the on-disk reports.
     pub fn load(ws: &Workspace<'_>) -> CargoResult<OnDiskReports> {
-        let report_file = match ws.target_dir().open_ro_shared(
+        let report_file = match ws.build_dir().open_ro_shared(
             FUTURE_INCOMPAT_FILE,
             ws.gctx(),
             "Future incompatible report",

src/cargo/core/compiler/layout.rs

@@ -128,13 +128,20 @@ pub struct Layout {
     fingerprint: PathBuf,
     /// The directory for examples: `$dest/examples`
     examples: PathBuf,
+    /// The directory for pre-uplifted examples: `$build-dir/debug/examples`
+    build_examples: PathBuf,
     /// The directory for rustdoc output: `$root/doc`
     doc: PathBuf,
     /// The directory for temporary data of integration tests and benches: `$dest/tmp`
     tmp: PathBuf,
     /// The lockfile for a build (`.cargo-lock`). Will be unlocked when this
     /// struct is `drop`ped.
     _lock: FileLock,
+    /// Same as `_lock` but for the build directory.
+    ///
+    /// Will be `None` when the build-dir and target-dir are the same path as we cannot
+    /// lock the same path twice.
+    _build_lock: Option<FileLock>,
 }
 
 impl Layout {
@@ -150,15 +157,22 @@ impl Layout {
         dest: &str,
     ) -> CargoResult<Layout> {
         let mut root = ws.target_dir();
+        let mut build_root = ws.build_dir();
         if let Some(target) = target {
             root.push(target.short_name());
+            build_root.push(target.short_name());
         }
+        let build_dest = build_root.join(dest);
         let dest = root.join(dest);
         // If the root directory doesn't already exist go ahead and create it
         // here. Use this opportunity to exclude it from backups as well if the
         // system supports it since this is a freshly created folder.
         //
         paths::create_dir_all_excluded_from_backups_atomic(root.as_path_unlocked())?;
+        if root != build_root {
+            paths::create_dir_all_excluded_from_backups_atomic(build_root.as_path_unlocked())?;
+        }
+
         // Now that the excluded from backups target root is created we can create the
         // actual destination (sub)subdirectory.
         paths::create_dir_all(dest.as_path_unlocked())?;
@@ -167,23 +181,37 @@ impl Layout {
         // directory, so just lock the entire thing for the duration of this
         // compile.
         let lock = dest.open_rw_exclusive_create(".cargo-lock", ws.gctx(), "build directory")?;
+
+        let build_lock = if root != build_root {
+            Some(build_dest.open_rw_exclusive_create(
+                ".cargo-lock",
+                ws.gctx(),
+                "build directory",
+            )?)
+        } else {
+            None
+        };
         let root = root.into_path_unlocked();
+        let build_root = build_root.into_path_unlocked();
         let dest = dest.into_path_unlocked();
-        let deps = dest.join("deps");
+        let build_dest = build_dest.as_path_unlocked();
+        let deps = build_dest.join("deps");
         let artifact = deps.join("artifact");
 
         Ok(Layout {
             deps,
-            build: dest.join("build"),
+            build: build_dest.join("build"),
             artifact,
-            incremental: dest.join("incremental"),
-            fingerprint: dest.join(".fingerprint"),
+            incremental: build_dest.join("incremental"),
+            fingerprint: build_dest.join(".fingerprint"),
             examples: dest.join("examples"),
+            build_examples: build_dest.join("examples"),
             doc: root.join("doc"),
-            tmp: root.join("tmp"),
+            tmp: build_root.join("tmp"),
             root,
             dest,
             _lock: lock,
+            _build_lock: build_lock,
         })
     }
 
@@ -193,6 +221,7 @@ impl Layout {
         paths::create_dir_all(&self.incremental)?;
         paths::create_dir_all(&self.fingerprint)?;
         paths::create_dir_all(&self.examples)?;
+        paths::create_dir_all(&self.build_examples)?;
         paths::create_dir_all(&self.build)?;
 
         Ok(())
@@ -210,6 +239,10 @@ impl Layout {
     pub fn examples(&self) -> &Path {
         &self.examples
     }
+    /// Fetch the build examples path.
+    pub fn build_examples(&self) -> &Path {
+        &self.build_examples
+    }
     /// Fetch the doc path.
     pub fn doc(&self) -> &Path {
         &self.doc

src/cargo/core/compiler/mod.rs

@@ -297,7 +297,7 @@ fn rustc(
     let exec = exec.clone();
 
     let root_output = build_runner.files().host_dest().to_path_buf();
-    let target_dir = build_runner.bcx.ws.target_dir().into_path_unlocked();
+    let target_dir = build_runner.bcx.ws.build_dir().into_path_unlocked();
     let pkg_root = unit.pkg.root().to_path_buf();
     let cwd = rustc
         .get_cwd()
@@ -555,7 +555,7 @@ fn link_targets(
         let path = unit
             .pkg
             .manifest()
-            .metabuild_path(build_runner.bcx.ws.target_dir());
+            .metabuild_path(build_runner.bcx.ws.build_dir());
         target.set_src_path(TargetSourcePath::Path(path));
     }
 

src/cargo/core/workspace.rs

@@ -419,9 +419,7 @@ impl<'gctx> Workspace<'gctx> {
         if !self.gctx().cli_unstable().build_dir {
             return self.target_dir();
         }
-        self.build_dir
-            .clone()
-            .unwrap_or_else(|| self.default_target_dir())
+        self.build_dir.clone().unwrap_or_else(|| self.target_dir())
     }
 
     fn default_target_dir(&self) -> Filesystem {

src/cargo/ops/cargo_clean.rs

@@ -41,6 +41,7 @@ pub struct CleanContext<'gctx> {
 /// Cleans various caches.
 pub fn clean(ws: &Workspace<'_>, opts: &CleanOptions<'_>) -> CargoResult<()> {
     let mut target_dir = ws.target_dir();
+    let mut build_dir = ws.build_dir();
     let gctx = opts.gctx;
     let mut clean_ctx = CleanContext::new(gctx);
     clean_ctx.dry_run = opts.dry_run;
@@ -67,6 +68,7 @@ pub fn clean(ws: &Workspace<'_>, opts: &CleanOptions<'_>) -> CargoResult<()> {
             // that profile.
             let dir_name = profiles.get_dir_name();
             target_dir = target_dir.join(dir_name);
+            build_dir = build_dir.join(dir_name);
         }
 
         // If we have a spec, then we need to delete some packages, otherwise, just
@@ -75,7 +77,15 @@ pub fn clean(ws: &Workspace<'_>, opts: &CleanOptions<'_>) -> CargoResult<()> {
         // Note that we don't bother grabbing a lock here as we're just going to
         // blow it all away anyway.
         if opts.spec.is_empty() {
-            clean_ctx.remove_paths(&[target_dir.into_path_unlocked()])?;
+            let paths: &[PathBuf] = if gctx.cli_unstable().build_dir && build_dir != target_dir {
+                &[
+                    target_dir.into_path_unlocked(),
+                    build_dir.into_path_unlocked(),
+                ]
+            } else {
+                &[target_dir.into_path_unlocked()]
+            };
+            clean_ctx.remove_paths(paths)?;
         } else {
             clean_specs(
                 &mut clean_ctx,
@@ -222,7 +232,7 @@ fn clean_specs(
                         .rustc_outputs(mode, target.kind(), triple)?;
                     let (dir, uplift_dir) = match target.kind() {
                         TargetKind::ExampleBin | TargetKind::ExampleLib(..) => {
-                            (layout.examples(), Some(layout.examples()))
+                            (layout.build_examples(), Some(layout.examples()))
                         }
                         // Tests/benchmarks are never uplifted.
                         TargetKind::Test | TargetKind::Bench => (layout.deps(), None),

src/cargo/ops/cargo_package/mod.rs

@@ -217,7 +217,7 @@ fn do_package<'a>(
     };
 
     let mut local_reg = if ws.gctx().cli_unstable().package_workspace {
-        let reg_dir = ws.target_dir().join("package").join("tmp-registry");
+        let reg_dir = ws.build_dir().join("package").join("tmp-registry");
         sid.map(|sid| TmpRegistry::new(ws.gctx(), reg_dir, sid))
             .transpose()?
     } else {

src/cargo/ops/cargo_package/verify.rs

@@ -45,9 +45,14 @@ pub fn run_verify(
 
     tar.file().seek(SeekFrom::Start(0))?;
     let f = GzDecoder::new(tar.file());
-    let dst = tar
-        .parent()
-        .join(&format!("{}-{}", pkg.name(), pkg.version()));
+    let dst = if gctx.cli_unstable().build_dir {
+        ws.build_dir()
+            .as_path_unlocked()
+            .join(&format!("package/{}-{}", pkg.name(), pkg.version()))
+    } else {
+        tar.parent()
+            .join(&format!("{}-{}", pkg.name(), pkg.version()))
+    };
     if dst.exists() {
         paths::remove_dir_all(&dst)?;
     }
@@ -63,7 +68,14 @@ pub fn run_verify(
     let mut src = PathSource::new(&dst, id, ws.gctx());
     let new_pkg = src.root_package()?;
     let pkg_fingerprint = hash_all(&dst)?;
-    let mut ws = Workspace::ephemeral(new_pkg, gctx, None, true)?;
+
+    let target_dir = if gctx.cli_unstable().build_dir {
+        Some(ws.build_dir())
+    } else {
+        None
+    };
+
+    let mut ws = Workspace::ephemeral(new_pkg, gctx, target_dir, true)?;
     if let Some(local_reg) = local_reg {
         ws.add_local_overlay(
             local_reg.upstream,

src/cargo/util/context/mod.rs

@@ -421,11 +421,8 @@ impl GlobalContext {
 
     /// Gets the path to the `rustc` executable.
     pub fn load_global_rustc(&self, ws: Option<&Workspace<'_>>) -> CargoResult<Rustc> {
-        let cache_location = ws.map(|ws| {
-            ws.target_dir()
-                .join(".rustc_info.json")
-                .into_path_unlocked()
-        });
+        let cache_location =
+            ws.map(|ws| ws.build_dir().join(".rustc_info.json").into_path_unlocked());
         let wrapper = self.maybe_get_tool("rustc_wrapper", &self.build_config()?.rustc_wrapper);
         let rustc_workspace_wrapper = self.maybe_get_tool(
             "rustc_workspace_wrapper",

src/cargo/util/workspace.rs

@@ -114,7 +114,7 @@ pub fn print_available_tests(ws: &Workspace<'_>, options: &CompileOptions) -> Ca
 pub fn path_args(ws: &Workspace<'_>, unit: &Unit) -> (PathBuf, PathBuf) {
     let src = match unit.target.src_path() {
         TargetSourcePath::Path(path) => path.to_path_buf(),
-        TargetSourcePath::Metabuild => unit.pkg.manifest().metabuild_path(ws.target_dir()),
+        TargetSourcePath::Metabuild => unit.pkg.manifest().metabuild_path(ws.build_dir()),
     };
     assert!(src.is_absolute());
     if unit.pkg.package_id().source_id().is_path() {