sccache now tracks compiler hits/misses on a per language basis. (#1895)

Previously sccache was actually tracking hits and misses on a compiler id basis, and guessing on the language. This is an issue for compilers like `clang` which support C, C++, and CUDA. In that case we want to clearly state what language we are getting hit or misses on to help user diagnose issues. Co-authored-by: Sylvestre Ledru <[email protected]>
mozilla · Oct 17, 2023 · b28d686 · b28d686
1 parent eb292cd
commit b28d686
Show file tree

Hide file tree

Showing 13 changed files with 198 additions and 98 deletions.
diff --git a/src/compiler/c.rs b/src/compiler/c.rs
@@ -15,7 +15,7 @@
 use crate::cache::FileObjectSource;
 use crate::compiler::{
     Cacheable, ColorMode, Compilation, CompileCommand, Compiler, CompilerArguments, CompilerHasher,
-    CompilerKind, HashResult,
+    CompilerKind, HashResult, Language,
 };
 #[cfg(feature = "dist-client")]
 use crate::compiler::{DistPackagers, NoopOutputsRewriter};
@@ -62,18 +62,6 @@ where
     compiler: I,
 }
 
-#[derive(Debug, PartialEq, Eq, Clone, Copy)]
-pub enum Language {
-    C,
-    Cxx,
-    GenericHeader,
-    CHeader,
-    CxxHeader,
-    ObjectiveC,
-    ObjectiveCxx,
-    Cuda,
-}
-
 /// Artifact produced by a C/C++ compiler.
 #[derive(Clone, Debug, PartialEq, Eq)]
 pub struct ArtifactDescriptor {
@@ -129,43 +117,6 @@ impl ParsedArguments {
     }
 }
 
-impl Language {
-    pub fn from_file_name(file: &Path) -> Option<Self> {
-        match file.extension().and_then(|e| e.to_str()) {
-            // gcc: https://gcc.gnu.org/onlinedocs/gcc/Overall-Options.html
-            Some("c") => Some(Language::C),
-            // Could be C or C++
-            Some("h") => Some(Language::GenericHeader),
-            // TODO i
-            Some("C") | Some("cc") | Some("cp") | Some("cpp") | Some("CPP") | Some("cxx")
-            | Some("c++") => Some(Language::Cxx),
-            // TODO ii
-            Some("H") | Some("hh") | Some("hp") | Some("hpp") | Some("HPP") | Some("hxx")
-            | Some("h++") | Some("tcc") => Some(Language::CxxHeader),
-            Some("m") => Some(Language::ObjectiveC),
-            // TODO mi
-            Some("M") | Some("mm") => Some(Language::ObjectiveCxx),
-            // TODO mii
-            Some("cu") => Some(Language::Cuda),
-            e => {
-                trace!("Unknown source extension: {}", e.unwrap_or("(None)"));
-                None
-            }
-        }
-    }
-
-    pub fn as_str(self) -> &'static str {
-        match self {
-            Language::C | Language::CHeader => "c",
-            Language::Cxx | Language::CxxHeader => "c++",
-            Language::GenericHeader => "c/c++",
-            Language::ObjectiveC => "objc",
-            Language::ObjectiveCxx => "objc++",
-            Language::Cuda => "cuda",
-        }
-    }
-}
-
 /// A generic implementation of the `Compilation` trait for C/C++ compilers.
 struct CCompilation<I: CCompilerImpl> {
     parsed_args: ParsedArguments,
@@ -443,6 +394,10 @@ where
     fn box_clone(&self) -> Box<dyn CompilerHasher<T>> {
         Box::new((*self).clone())
     }
+
+    fn language(&self) -> Language {
+        self.parsed_args.language
+    }
 }
 
 impl<I: CCompilerImpl> Compilation for CCompilation<I> {

diff --git a/src/compiler/clang.rs b/src/compiler/clang.rs
@@ -15,11 +15,11 @@
 #![allow(unused_imports, dead_code, unused_variables)]
 
 use crate::compiler::args::*;
-use crate::compiler::c::{
-    ArtifactDescriptor, CCompilerImpl, CCompilerKind, Language, ParsedArguments,
-};
+use crate::compiler::c::{ArtifactDescriptor, CCompilerImpl, CCompilerKind, ParsedArguments};
 use crate::compiler::gcc::ArgData::*;
-use crate::compiler::{gcc, write_temp_file, Cacheable, CompileCommand, CompilerArguments};
+use crate::compiler::{
+    gcc, write_temp_file, Cacheable, CompileCommand, CompilerArguments, Language,
+};
 use crate::mock_command::{CommandCreator, CommandCreatorSync, RunCommand};
 use crate::util::{run_input_output, OsStrExt};
 use crate::{counted_array, dist};

diff --git a/src/compiler/compiler.rs b/src/compiler/compiler.rs
@@ -102,12 +102,71 @@ pub enum CompilerKind {
     Rust,
 }
 
-impl CompilerKind {
-    pub fn lang_kind(&self) -> String {
+#[derive(Debug, PartialEq, Eq, Clone, Copy)]
+pub enum Language {
+    C,
+    Cxx,
+    GenericHeader,
+    CHeader,
+    CxxHeader,
+    ObjectiveC,
+    ObjectiveCxx,
+    Cuda,
+    Rust,
+}
+
+impl Language {
+    pub fn from_file_name(file: &Path) -> Option<Self> {
+        match file.extension().and_then(|e| e.to_str()) {
+            // gcc: https://gcc.gnu.org/onlinedocs/gcc/Overall-Options.html
+            Some("c") => Some(Language::C),
+            // Could be C or C++
+            Some("h") => Some(Language::GenericHeader),
+            // TODO i
+            Some("C") | Some("cc") | Some("cp") | Some("cpp") | Some("CPP") | Some("cxx")
+            | Some("c++") => Some(Language::Cxx),
+            // TODO ii
+            Some("H") | Some("hh") | Some("hp") | Some("hpp") | Some("HPP") | Some("hxx")
+            | Some("h++") | Some("tcc") => Some(Language::CxxHeader),
+            Some("m") => Some(Language::ObjectiveC),
+            // TODO mi
+            Some("M") | Some("mm") => Some(Language::ObjectiveCxx),
+            // TODO mii
+            Some("cu") => Some(Language::Cuda),
+            // TODO cy
+            Some("rs") => Some(Language::Rust),
+            e => {
+                trace!("Unknown source extension: {}", e.unwrap_or("(None)"));
+                None
+            }
+        }
+    }
+
+    pub fn as_str(self) -> &'static str {
         match self {
-            CompilerKind::C(CCompilerKind::Nvcc) => "CUDA",
-            CompilerKind::C(_) => "C/C++",
-            CompilerKind::Rust => "Rust",
+            Language::C | Language::CHeader => "c",
+            Language::Cxx | Language::CxxHeader => "c++",
+            Language::GenericHeader => "c/c++",
+            Language::ObjectiveC => "objc",
+            Language::ObjectiveCxx => "objc++",
+            Language::Cuda => "cuda",
+            Language::Rust => "rust",
+        }
+    }
+}
+
+impl CompilerKind {
+    pub fn lang_kind(&self, lang: &Language) -> String {
+        match lang {
+            Language::C
+            | Language::CHeader
+            | Language::Cxx
+            | Language::CxxHeader
+            | Language::GenericHeader
+            | Language::ObjectiveC
+            | Language::ObjectiveCxx => "C/C++",
+            Language::Cuda => "CUDA",
+            Language::Rust => "Rust",
         }
         .to_string()
     }
@@ -427,6 +486,8 @@ where
     fn output_pretty(&self) -> Cow<'_, str>;
 
     fn box_clone(&self) -> Box<dyn CompilerHasher<T>>;
+
+    fn language(&self) -> Language;
 }
 
 #[cfg(not(feature = "dist-client"))]

diff --git a/src/compiler/diab.rs b/src/compiler/diab.rs
@@ -17,10 +17,8 @@ use crate::compiler::args::{
     ArgDisposition, ArgInfo, ArgToStringResult, ArgsIter, Argument, FromArg, IntoArg,
     NormalizedDisposition, PathTransformerFn, SearchableArgInfo,
 };
-use crate::compiler::c::{
-    ArtifactDescriptor, CCompilerImpl, CCompilerKind, Language, ParsedArguments,
-};
-use crate::compiler::{Cacheable, ColorMode, CompileCommand, CompilerArguments};
+use crate::compiler::c::{ArtifactDescriptor, CCompilerImpl, CCompilerKind, ParsedArguments};
+use crate::compiler::{Cacheable, ColorMode, CompileCommand, CompilerArguments, Language};
 use crate::errors::*;
 use crate::mock_command::{CommandCreatorSync, RunCommand};
 use crate::util::{run_input_output, OsStrExt};

diff --git a/src/compiler/gcc.rs b/src/compiler/gcc.rs
@@ -13,10 +13,8 @@
 // limitations under the License.
 
 use crate::compiler::args::*;
-use crate::compiler::c::{
-    ArtifactDescriptor, CCompilerImpl, CCompilerKind, Language, ParsedArguments,
-};
-use crate::compiler::{clang, Cacheable, ColorMode, CompileCommand, CompilerArguments};
+use crate::compiler::c::{ArtifactDescriptor, CCompilerImpl, CCompilerKind, ParsedArguments};
+use crate::compiler::{clang, Cacheable, ColorMode, CompileCommand, CompilerArguments, Language};
 use crate::mock_command::{CommandCreatorSync, RunCommand};
 use crate::util::{run_input_output, OsStrExt};
 use crate::{counted_array, dist};
@@ -367,6 +365,7 @@ where
                     "objective-c" => Some(Language::ObjectiveC),
                     "objective-c++" => Some(Language::ObjectiveCxx),
                     "cu" => Some(Language::Cuda),
+                    "rs" => Some(Language::Rust),
                     "cuda" => Some(Language::Cuda),
                     _ => cannot_cache!("-x"),
                 };
@@ -616,6 +615,7 @@ fn language_to_gcc_arg(lang: Language) -> Option<&'static str> {
         Language::ObjectiveC => Some("objective-c"),
         Language::ObjectiveCxx => Some("objective-c++"),
         Language::Cuda => Some("cu"),
+        Language::Rust => None,          // Let the compiler decide
         Language::GenericHeader => None, // Let the compiler decide
     }
 }

diff --git a/src/compiler/msvc.rs b/src/compiler/msvc.rs
@@ -13,11 +13,9 @@
 // limitations under the License.
 
 use crate::compiler::args::*;
-use crate::compiler::c::{
-    ArtifactDescriptor, CCompilerImpl, CCompilerKind, Language, ParsedArguments,
-};
+use crate::compiler::c::{ArtifactDescriptor, CCompilerImpl, CCompilerKind, ParsedArguments};
 use crate::compiler::{
-    clang, gcc, write_temp_file, Cacheable, ColorMode, CompileCommand, CompilerArguments,
+    clang, gcc, write_temp_file, Cacheable, ColorMode, CompileCommand, CompilerArguments, Language,
 };
 use crate::mock_command::{CommandCreatorSync, RunCommand};
 use crate::util::{run_input_output, OsStrExt};

diff --git a/src/compiler/nvcc.rs b/src/compiler/nvcc.rs
@@ -16,11 +16,11 @@
 #![allow(unused_imports, dead_code, unused_variables)]
 
 use crate::compiler::args::*;
-use crate::compiler::c::{
-    ArtifactDescriptor, CCompilerImpl, CCompilerKind, Language, ParsedArguments,
-};
+use crate::compiler::c::{ArtifactDescriptor, CCompilerImpl, CCompilerKind, ParsedArguments};
 use crate::compiler::gcc::ArgData::*;
-use crate::compiler::{gcc, write_temp_file, Cacheable, CompileCommand, CompilerArguments};
+use crate::compiler::{
+    gcc, write_temp_file, Cacheable, CompileCommand, CompilerArguments, Language,
+};
 use crate::mock_command::{CommandCreator, CommandCreatorSync, RunCommand};
 use crate::util::{run_input_output, OsStrExt};
 use crate::{counted_array, dist};

diff --git a/src/compiler/nvhpc.rs b/src/compiler/nvhpc.rs
@@ -16,11 +16,11 @@
 #![allow(unused_imports, dead_code, unused_variables)]
 
 use crate::compiler::args::*;
-use crate::compiler::c::{
-    ArtifactDescriptor, CCompilerImpl, CCompilerKind, Language, ParsedArguments,
-};
+use crate::compiler::c::{ArtifactDescriptor, CCompilerImpl, CCompilerKind, ParsedArguments};
 use crate::compiler::gcc::ArgData::*;
-use crate::compiler::{gcc, write_temp_file, Cacheable, CompileCommand, CompilerArguments};
+use crate::compiler::{
+    gcc, write_temp_file, Cacheable, CompileCommand, CompilerArguments, Language,
+};
 use crate::mock_command::{CommandCreator, CommandCreatorSync, RunCommand};
 use crate::util::{run_input_output, OsStrExt};
 use crate::{counted_array, dist};

diff --git a/src/compiler/rust.rs b/src/compiler/rust.rs
@@ -16,7 +16,7 @@ use crate::cache::FileObjectSource;
 use crate::compiler::args::*;
 use crate::compiler::{
     c::ArtifactDescriptor, Cacheable, ColorMode, Compilation, CompileCommand, Compiler,
-    CompilerArguments, CompilerHasher, CompilerKind, CompilerProxy, HashResult,
+    CompilerArguments, CompilerHasher, CompilerKind, CompilerProxy, HashResult, Language,
 };
 #[cfg(feature = "dist-client")]
 use crate::compiler::{DistPackagers, OutputsRewriter};
@@ -1589,6 +1589,10 @@ where
     fn box_clone(&self) -> Box<dyn CompilerHasher<T>> {
         Box::new((*self).clone())
     }
+
+    fn language(&self) -> Language {
+        Language::Rust
+    }
 }
 
 impl Compilation for RustCompilation {

diff --git a/src/compiler/tasking_vx.rs b/src/compiler/tasking_vx.rs
@@ -19,8 +19,8 @@ use crate::{
             ArgDisposition, ArgInfo, ArgToStringResult, ArgsIter, Argument, FromArg, IntoArg,
             NormalizedDisposition, PathTransformerFn, SearchableArgInfo,
         },
-        c::{ArtifactDescriptor, CCompilerImpl, CCompilerKind, Language, ParsedArguments},
-        Cacheable, ColorMode, CompileCommand, CompilerArguments,
+        c::{ArtifactDescriptor, CCompilerImpl, CCompilerKind, ParsedArguments},
+        Cacheable, ColorMode, CompileCommand, CompilerArguments, Language,
     },
     counted_array, dist,
     errors::*,

diff --git a/src/server.rs b/src/server.rs
@@ -15,7 +15,7 @@
 use crate::cache::{storage_from_config, Storage};
 use crate::compiler::{
     get_compiler_info, CacheControl, CompileResult, Compiler, CompilerArguments, CompilerHasher,
-    CompilerKind, CompilerProxy, DistType, MissType,
+    CompilerKind, CompilerProxy, DistType, Language, MissType,
 };
 #[cfg(feature = "dist-client")]
 use crate::config;
@@ -1164,6 +1164,7 @@ where
         let color_mode = hasher.color_mode();
         let me = self.clone();
         let kind = compiler.kind();
+        let lang = hasher.language();
         let creator = self.creator.clone();
         let storage = self.storage.clone();
         let pool = self.rt.clone();
@@ -1199,12 +1200,12 @@ where
                         CompileResult::Error => {
                             debug!("compile result: cache error");
 
-                            stats.cache_errors.increment(&kind);
+                            stats.cache_errors.increment(&kind, &lang);
                         }
                         CompileResult::CacheHit(duration) => {
                             debug!("compile result: cache hit");
 
-                            stats.cache_hits.increment(&kind);
+                            stats.cache_hits.increment(&kind, &lang);
                             stats.cache_read_hit_duration += duration;
                         }
                         CompileResult::CacheMiss(miss_type, dist_type, duration, future) => {
@@ -1229,18 +1230,18 @@ where
                                     stats.cache_timeouts += 1;
                                 }
                                 MissType::CacheReadError => {
-                                    stats.cache_errors.increment(&kind);
+                                    stats.cache_errors.increment(&kind, &lang);
                                 }
                             }
-                            stats.cache_misses.increment(&kind);
+                            stats.cache_misses.increment(&kind, &lang);
                             stats.compiler_write_duration += duration;
                             debug!("stats after compile result: {stats:?}");
                             cache_write = Some(future);
                         }
                         CompileResult::NotCacheable => {
                             debug!("compile result: not cacheable");
 
-                            stats.cache_misses.increment(&kind);
+                            stats.cache_misses.increment(&kind, &lang);
                             stats.non_cacheable_compilations += 1;
                         }
                         CompileResult::CompileFailed => {
@@ -1298,7 +1299,7 @@ where
                                     error!("[{:?}] \t{}", out_pretty, e);
                                     let _ = writeln!(error, "sccache: caused by: {}", e);
                                 }
-                                stats.cache_errors.increment(&kind);
+                                stats.cache_errors.increment(&kind, &lang);
                                 //TODO: figure out a better way to communicate this?
                                 res.retcode = Some(-2);
                                 res.stderr = error.into_bytes();
@@ -1353,9 +1354,9 @@ pub struct PerLanguageCount {
 }
 
 impl PerLanguageCount {
-    fn increment(&mut self, kind: &CompilerKind) {
-        let key = kind.lang_kind();
-        let count = self.counts.entry(key).or_insert(0);
+    fn increment(&mut self, kind: &CompilerKind, lang: &Language) {
+        let lang_key = kind.lang_kind(lang);
+        let count = self.counts.entry(lang_key).or_insert(0);
         *count += 1;
     }