macOS: Always pass SDK root when linking with cc, and pass it via SDKROOT env var

compiler/rustc_codegen_ssa/messages.ftl

@@ -8,7 +8,65 @@ codegen_ssa_apple_deployment_target_invalid =
 codegen_ssa_apple_deployment_target_too_low =
     deployment target in {$env_var} was set to {$version}, but the minimum supported by `rustc` is {$os_min}
 
-codegen_ssa_apple_sdk_error_sdk_path = failed to get {$sdk_name} SDK path: {$error}
+codegen_ssa_apple_sdk_error_failed_reading =
+    failed reading `{$path}` while looking for SDK root: {$error}
+
+codegen_ssa_apple_sdk_error_missing =
+    failed finding SDK for platform `{$sdk_name}`. It looks like you have not installed Xcode?
+
+    { $sdk_name ->
+        [MacOSX] You should install Xcode via the App Store, or run `xcode-select --install` to install the Command Line Tools if you only intend on developing for macOS.
+        *[other] You should install Xcode via the App Store.
+    }
+
+codegen_ssa_apple_sdk_error_missing_commandline_tools =
+    failed finding SDK at `{$sdkroot}` in Command Line Tools installation.
+
+    { $sdk_name ->
+        [MacOSX] Perhaps you need to reinstall it with `xcode-select --install`?
+       *[other] When compiling for iOS, tvOS, visionOS or watchOS, you will need a full installation of Xcode.
+    }
+
+codegen_ssa_apple_sdk_error_missing_cross_compile_non_macos =
+    failed finding Apple SDK with name `{$sdk_name}`.
+
+    The SDK is needed by the linker to know where to find symbols in system libraries and for embedding the SDK version in the final object file.
+
+    The SDK can be downloaded and extracted from https://developer.apple.com/download/all/?q=xcode (requires an Apple ID).
+
+    The full Xcode bundle should contain the SDK in `Xcode.app/Contents/Developer/Platforms/{$sdk_name}.platform/Developer/SDKs/{$sdk_name}.sdk`{ $sdk_name ->
+        [MacOSX] , but downloading just the Command Line Tools for Xcode should also be sufficient to obtain the macOS SDK.
+       *[other] .
+    }
+
+    You will then need to tell `rustc` about it using the `SDKROOT` environment variables.
+
+    Furthermore, you might need to install a linker capable of linking Mach-O files, or at least ensure that `rustc` is configured to use the bundled `lld`.
+
+    { $sdk_name ->
+        [MacOSX] {""}
+       *[other] Beware that cross-compilation to iOS, tvOS, visionOS or watchOS is generally ill supported on non-macOS hosts.
+    }
+
+codegen_ssa_apple_sdk_error_missing_developer_dir =
+    failed finding SDK inside active developer directory `{$dir}` set by the DEVELOPER_DIR environment variable. Looked in:
+    - `{$sdkroot}`
+    - `{$sdkroot_bare}`
+
+codegen_ssa_apple_sdk_error_missing_xcode =
+    failed finding SDK at `{$sdkroot}` in Xcode installation.
+
+    { $sdk_name ->
+        [MacOSX] {""}
+       *[other] Perhaps you need a newer version of Xcode?
+    }
+
+codegen_ssa_apple_sdk_error_missing_xcode_select =
+    failed finding SDK inside active developer directory `{$dir}` set by `xcode-select`. Looked in:
+    - `{$sdkroot}`
+    - `{$sdkroot_bare}`
+
+    Consider using `sudo xcode-select --switch path/to/Xcode.app` or `sudo xcode-select --reset` to select a valid path.
 
 codegen_ssa_archive_build_failure = failed to build archive at `{$path}`: {$error}
 
@@ -348,8 +406,6 @@ codegen_ssa_unknown_atomic_ordering = unknown ordering in atomic intrinsic
 
 codegen_ssa_unknown_reuse_kind = unknown cgu-reuse-kind `{$kind}` specified
 
-codegen_ssa_unsupported_arch = unsupported arch `{$arch}` for os `{$os}`
-
 codegen_ssa_unsupported_link_self_contained = option `-C link-self-contained` is not supported on this target
 
 codegen_ssa_use_cargo_directive = use the `cargo:rustc-link-lib` directive to specify the native libraries to link with Cargo (see https://doc.rust-lang.org/cargo/reference/build-scripts.html#rustc-link-lib)

compiler/rustc_codegen_ssa/src/back/apple.rs

@@ -1,15 +1,34 @@
-use std::env;
 use std::fmt::{Display, from_fn};
+use std::io::ErrorKind;
 use std::num::ParseIntError;
+use std::path::{Path, PathBuf};
+use std::{env, fs};
 
 use rustc_session::Session;
 use rustc_target::spec::Target;
 
-use crate::errors::AppleDeploymentTarget;
+use crate::errors::{AppleDeploymentTarget, AppleSdkError};
 
 #[cfg(test)]
 mod tests;
 
+pub(super) fn sdk_name(target: &Target) -> &'static str {
+    match (&*target.os, &*target.abi) {
+        ("ios", "") => "iPhoneOS",
+        ("ios", "sim") => "iPhoneSimulator",
+        // Mac Catalyst uses the macOS SDK
+        ("ios", "macabi") => "MacOSX",
+        ("macos", "") => "MacOSX",
+        ("tvos", "") => "AppleTVOS",
+        ("tvos", "sim") => "AppleTVSimulator",
+        ("visionos", "") => "XROS",
+        ("visionos", "sim") => "XRSimulator",
+        ("watchos", "") => "WatchOS",
+        ("watchos", "sim") => "WatchSimulator",
+        (os, abi) => unreachable!("invalid os '{os}' / abi '{abi}' combination for Apple target"),
+    }
+}
+
 pub(super) fn macho_platform(target: &Target) -> u32 {
     match (&*target.os, &*target.abi) {
         ("macos", _) => object::macho::PLATFORM_MACOS,
@@ -169,3 +188,132 @@ pub(super) fn add_version_to_llvm_target(
         format!("{arch}-{vendor}-{os}{major}.{minor}.{patch}")
     }
 }
+
+// TOCTOU is not _really_ an issue with our use of `try_exists` in here, we mostly use it for
+// diagnostics, and these directories are global state that the user can change anytime anyhow in
+// ways that are going to interfere much more with the compilation process.
+fn try_exists(path: &Path) -> Result<bool, AppleSdkError> {
+    path.try_exists().map_err(|error| AppleSdkError::FailedReading { path: path.to_owned(), error })
+}
+
+/// Get the SDK path for an SDK under `/Library/Developer/CommandLineTools`.
+fn sdk_root_in_sdks_dir(sdks_dir: impl Into<PathBuf>, sdk_name: &str) -> PathBuf {
+    let mut path = sdks_dir.into();
+    path.push("SDKs");
+    path.push(sdk_name);
+    path.set_extension("sdk");
+    path
+}
+
+/// Get the SDK path for an SDK under `/Applications/Xcode.app/Contents/Developer`.
+fn sdk_root_in_developer_dir(developer_dir: impl Into<PathBuf>, sdk_name: &str) -> PathBuf {
+    let mut path = developer_dir.into();
+    path.push("Platforms");
+    path.push(sdk_name);
+    path.set_extension("platform");
+    path.push("Developer");
+    path.push("SDKs");
+    path.push(sdk_name);
+    path.set_extension("sdk");
+    path
+}
+
+/// Find a SDK root from the user's environment for the given SDK name.
+///
+/// We do this by searching (purely by names in the filesystem, without reading SDKSettings.json)
+/// for a matching SDK in the following places:
+/// - `DEVELOPER_DIR`
+/// - `/var/db/xcode_select_link`
+/// - `/Applications/Xcode.app`
+/// - `/Library/Developer/CommandLineTools`
+///
+/// This does roughly the same thing as `xcrun -sdk $sdk_name -show-sdk-path` (see `man xcrun` for
+/// a few details on the search algorithm).
+///
+/// The reason why we implement this logic ourselves is:
+/// - Reading these directly is faster than spawning a new process.
+/// - `xcrun` can be fairly slow to start up after a reboot.
+/// - In the future, we will be able to integrate this better with the compiler's change tracking
+///   mechanisms, allowing rebuilds when the involved env vars and paths here change. See #118204.
+/// - It's easier for us to emit better error messages.
+///
+/// Though a downside is that `xcrun` might be expanded in the future to check more places, and then
+/// `rustc` would have to be changed to keep up. Furthermore, `xcrun`'s exact algorithm is
+/// undocumented, so it might be doing more things than we do here.
+pub(crate) fn find_sdk_root(sdk_name: &'static str) -> Result<PathBuf, AppleSdkError> {
+    // Only try this if the host OS is macOS.
+    if !cfg!(target_os = "macos") {
+        return Err(AppleSdkError::MissingCrossCompileNonMacOS { sdk_name });
+    }
+
+    // NOTE: We could consider walking upwards in `SDKROOT` assuming Xcode directory structure, but
+    // that isn't what `xcrun` does, and might still not yield the desired result (e.g. if using an
+    // old SDK to compile for an old ARM iOS arch, we don't want `rustc` to pick a macOS SDK from
+    // the old Xcode).
+
+    // Try reading from `DEVELOPER_DIR`.
+    if let Some(dir) = std::env::var_os("DEVELOPER_DIR") {
+        let dir = PathBuf::from(dir);
+        let sdkroot = sdk_root_in_developer_dir(&dir, sdk_name);
+
+        if try_exists(&sdkroot)? {
+            return Ok(sdkroot);
+        } else {
+            let sdkroot_bare = sdk_root_in_sdks_dir(&dir, sdk_name);
+            if try_exists(&sdkroot_bare)? {
+                return Ok(sdkroot_bare);
+            } else {
+                return Err(AppleSdkError::MissingDeveloperDir { dir, sdkroot, sdkroot_bare });
+            }
+        }
+    }
+
+    // Next, try to read the link that `xcode-select` sets.
+    //
+    // FIXME(madsmtm): Support cases where `/var/db/xcode_select_link` contains a relative path?
+    let path = Path::new("/var/db/xcode_select_link");
+    match fs::read_link(path) {
+        Ok(dir) => {
+            let sdkroot = sdk_root_in_developer_dir(&dir, sdk_name);
+            if try_exists(&sdkroot)? {
+                return Ok(sdkroot);
+            } else {
+                let sdkroot_bare = sdk_root_in_sdks_dir(&dir, sdk_name);
+                if try_exists(&sdkroot_bare)? {
+                    return Ok(sdkroot_bare);
+                } else {
+                    return Err(AppleSdkError::MissingXcodeSelect { dir, sdkroot, sdkroot_bare });
+                }
+            }
+        }
+        Err(err) if err.kind() == ErrorKind::NotFound => {
+            // Intentionally ignore not found errors, if `xcode-select --reset` is called the
+            // link will not exist.
+        }
+        Err(error) => return Err(AppleSdkError::FailedReading { path: path.into(), error }),
+    }
+
+    // Next, fall back to reading from `/Applications/Xcode.app`.
+    let dir = Path::new("/Applications/Xcode.app/Contents/Developer");
+    if try_exists(dir)? {
+        let sdkroot = sdk_root_in_developer_dir(dir, sdk_name);
+        if try_exists(&sdkroot)? {
+            return Ok(sdkroot);
+        } else {
+            return Err(AppleSdkError::MissingXcode { sdkroot, sdk_name });
+        }
+    }
+
+    // Finally, fall back to reading from `/Library/Developer/CommandLineTools`.
+    let dir = Path::new("/Library/Developer/CommandLineTools");
+    if try_exists(dir)? {
+        let sdkroot = sdk_root_in_sdks_dir(dir, sdk_name);
+        if try_exists(&sdkroot)? {
+            return Ok(sdkroot);
+        } else {
+            return Err(AppleSdkError::MissingCommandlineTools { sdkroot, sdk_name });
+        }
+    }
+
+    Err(AppleSdkError::Missing { sdk_name })
+}

compiler/rustc_codegen_ssa/src/back/apple/tests.rs

@@ -1,4 +1,8 @@
-use super::{add_version_to_llvm_target, parse_version};
+use std::io;
+use std::path::PathBuf;
+use std::process::Command;
+
+use super::{add_version_to_llvm_target, find_sdk_root, parse_version};
 
 #[test]
 fn test_add_version_to_llvm_target() {
@@ -19,3 +23,56 @@ fn test_parse_version() {
     assert_eq!(parse_version("10.12.6"), Ok((10, 12, 6)));
     assert_eq!(parse_version("9999.99.99"), Ok((9999, 99, 99)));
 }
+
+fn find_sdk_root_xcrun(sdk_name: &str) -> io::Result<PathBuf> {
+    let output = Command::new("xcrun")
+        .arg("-sdk")
+        .arg(sdk_name.to_lowercase())
+        .arg("-show-sdk-path")
+        .output()?;
+    if output.status.success() {
+        // FIXME(madsmtm): If using this for real, we should not error on non-UTF-8 paths.
+        let output = std::str::from_utf8(&output.stdout).unwrap();
+        Ok(PathBuf::from(output.trim()))
+    } else {
+        let error = String::from_utf8(output.stderr);
+        let error = format!("process exit with error: {}", error.unwrap());
+        Err(io::Error::new(io::ErrorKind::Other, error))
+    }
+}
+
+/// Ensure that our `find_sdk_root` matches `xcrun`'s behaviour.
+///
+/// `xcrun` is quite slow the first time it's run after a reboot, so this test may take some time.
+#[test]
+#[cfg_attr(not(target_os = "macos"), ignore = "xcrun is only available on macOS")]
+fn test_find_sdk_root() {
+    let sdks = [
+        "MacOSX",
+        "AppleTVOS",
+        "AppleTVSimulator",
+        "iPhoneOS",
+        "iPhoneSimulator",
+        "WatchOS",
+        "WatchSimulator",
+        "XROS",
+        "XRSimulator",
+    ];
+    for sdk_name in sdks {
+        if let Ok(expected) = find_sdk_root_xcrun(sdk_name) {
+            // `xcrun` prefers `MacOSX14.0.sdk` over `MacOSX.sdk`, so let's compare canonical paths.
+            let expected = std::fs::canonicalize(expected).unwrap();
+            let actual = find_sdk_root(sdk_name).unwrap();
+            let actual = std::fs::canonicalize(actual).unwrap();
+            assert_eq!(expected, actual);
+        } else {
+            // The macOS SDK must always be findable in Rust's CI.
+            //
+            // The other SDKs are allowed to not be found in the current developer directory when
+            // running this test.
+            if sdk_name == "MacOSX" {
+                panic!("Could not find macOS SDK with `xcrun -sdk macosx -show-sdk-path`");
+            }
+        }
+    }
+}