Add command to build a graph from a CST and extract CLI commands into…

… the runtime library (NomicFoundation#983)

Closes NomicFoundation#981 

1. Exposes the `metaslang_graph_builder` functionality in the generated
languages runtime, parameterized with the language `KindTypes`.
2. Exercise the graph builder via a test in `testlang`.
3. Extracts the CLI command from `slang_solidity` into the runtime crate
to be able to easily reuse it from other generated languages.
4. Adds a new `build-graph` command to the common CLI commands to run a
source file through an arbitrary `.msgb` file and build a graph.
5. Expose the language's root non-terminal kind in the public API.

Items 1, 2 and 4 are only available when building with the private
feature flag `__graph_builder`.

.changeset/forty-deers-arrive.md

@@ -0,0 +1,5 @@
+---
+"@nomicfoundation/slang": patch
+---
+
+Expose the language root non-terminal kind at `Language.rootKind()`.

crates/codegen/runtime/cargo/Cargo.toml

@@ -13,7 +13,9 @@ codegen_runtime_generator = { workspace = true }
 
 [dependencies]
 ariadne = { workspace = true, optional = true }
+clap = { workspace = true, optional = true }
 metaslang_cst = { workspace = true }
+metaslang_graph_builder = { workspace = true, optional = true }
 napi = { workspace = true, optional = true }
 napi-derive = { workspace = true, optional = true }
 semver = { workspace = true }
@@ -28,8 +30,11 @@ thiserror = { workspace = true }
 [features]
 default = ["slang_napi_interfaces"]
 slang_napi_interfaces = ["dep:napi", "dep:napi-derive", "dep:serde_json"]
+cli = ["dep:clap", "dep:serde_json", "__private_ariadne"]
 # Only used by the `slang_solidity` CLI
 __private_ariadne = ["dep:ariadne"]
+# For internal development only
+__graph_builder = ["dep:metaslang_graph_builder"]
 
 [lints]
 workspace = true

crates/codegen/runtime/cargo/src/runtime/cli/commands/build_graph.rs

@@ -0,0 +1,57 @@
+use std::fs;
+use std::path::PathBuf;
+
+use semver::Version;
+
+use super::parse::parse_source_file;
+use super::CommandError;
+use crate::graph_builder::{
+    ExecutionConfig, File as GraphBuilderFile, Functions, NoCancellation, Variables,
+};
+
+pub fn execute(
+    file_path_string: &str,
+    version: Version,
+    msgb_path_string: &str,
+    output_json: bool,
+    debug: bool,
+) -> Result<(), CommandError> {
+    let parse_output = parse_source_file(file_path_string, version, |_| ())?;
+    let msgb = parse_graph_builder(msgb_path_string)?;
+
+    let functions = Functions::stdlib();
+    let variables = Variables::new();
+    let mut execution_config = ExecutionConfig::new(&functions, &variables);
+    if debug {
+        execution_config = execution_config.debug_attributes(
+            "_location".into(),
+            "_variable".into(),
+            "_match".into(),
+        );
+    }
+
+    let tree = parse_output.create_tree_cursor();
+    let graph = msgb.execute(&tree, &execution_config, &NoCancellation)?;
+
+    if output_json {
+        graph.display_json(None)?;
+    } else {
+        print!("{}", graph.pretty_print());
+    }
+
+    Ok(())
+}
+
+fn parse_graph_builder(msgb_path_string: &str) -> Result<GraphBuilderFile, CommandError> {
+    let msgb_path = PathBuf::from(&msgb_path_string)
+        .canonicalize()
+        .map_err(|_| CommandError::FileNotFound(msgb_path_string.to_string()))?;
+
+    let msgb_source = fs::read_to_string(&msgb_path)?;
+    GraphBuilderFile::from_str(&msgb_source).map_err(|parser_error| {
+        let error_message = parser_error
+            .display_pretty(&msgb_path, &msgb_source)
+            .to_string();
+        CommandError::ParseFailed(error_message)
+    })
+}

crates/codegen/runtime/cargo/src/runtime/cli/commands/mod.rs

@@ -0,0 +1,24 @@
+use thiserror::Error;
+
+#[cfg(feature = "__graph_builder")]
+pub mod build_graph;
+pub mod parse;
+
+#[derive(Debug, Error)]
+pub enum CommandError {
+    #[error("File not found: {0:?}")]
+    FileNotFound(String),
+
+    #[error(transparent)]
+    Io(#[from] std::io::Error),
+
+    #[error(transparent)]
+    LanguageError(#[from] crate::language::Error),
+
+    #[error("Parsing failed: {0}")]
+    ParseFailed(String),
+
+    #[cfg(feature = "__graph_builder")]
+    #[error(transparent)]
+    ExecutionFailed(#[from] crate::graph_builder::ExecutionError),
+}

crates/codegen/runtime/cargo/src/runtime/cli/commands/parse.rs

@@ -0,0 +1,52 @@
+use std::fs;
+use std::path::PathBuf;
+
+use semver::Version;
+
+use super::CommandError;
+use crate::diagnostic;
+use crate::language::Language;
+use crate::parse_output::ParseOutput;
+
+pub fn execute(file_path_string: &str, version: Version, json: bool) -> Result<(), CommandError> {
+    parse_source_file(file_path_string, version, |output| {
+        if json {
+            let root_node = output.tree();
+            let json = serde_json::to_string_pretty(&root_node).expect("JSON serialization failed");
+            println!("{json}");
+        }
+    })
+    .map(|_| ())
+}
+
+pub(crate) fn parse_source_file<F>(
+    file_path_string: &str,
+    version: Version,
+    run_before_checking: F,
+) -> Result<ParseOutput, CommandError>
+where
+    F: FnOnce(&ParseOutput),
+{
+    let file_path = PathBuf::from(&file_path_string)
+        .canonicalize()
+        .map_err(|_| CommandError::FileNotFound(file_path_string.to_string()))?;
+
+    let input = fs::read_to_string(file_path)?;
+    let language = Language::new(version)?;
+    let parse_output = language.parse(Language::ROOT_KIND, &input);
+
+    run_before_checking(&parse_output);
+
+    if parse_output.is_valid() {
+        Ok(parse_output)
+    } else {
+        const COLOR: bool = true;
+        let report = parse_output
+            .errors()
+            .iter()
+            .map(|error| diagnostic::render(error, file_path_string, &input, COLOR))
+            .collect::<Vec<_>>()
+            .join("\n");
+        Err(CommandError::ParseFailed(report))
+    }
+}

crates/codegen/runtime/cargo/src/runtime/cli/mod.rs

@@ -0,0 +1,73 @@
+use std::process::ExitCode;
+
+use clap::Subcommand;
+use semver::Version;
+
+pub mod commands;
+
+#[derive(Subcommand, Debug)]
+pub enum Commands {
+    /// Parses a source file, and outputs any syntax errors, or a JSON concrete syntax tree
+    Parse {
+        /// File path to the source file to parse
+        file_path: String,
+
+        /// The language version to use for parsing
+        #[arg(short, long)]
+        version: Version,
+
+        /// Print the concrete syntax tree as JSON
+        #[clap(long)]
+        json: bool,
+    },
+
+    // This is only intended for internal development
+    #[cfg(feature = "__graph_builder")]
+    /// Parses a source file and builds a graph executing the instructions from the builder file (*.msgb)
+    BuildGraph {
+        /// File path to the source file to parse
+        file_path: String,
+
+        /// The language version to use for parsing
+        #[arg(short, long)]
+        version: Version,
+
+        /// The graph buider (.msgb) file to use
+        msgb_path: String,
+
+        /// Print the graph as JSON
+        #[clap(long)]
+        json: bool,
+
+        /// Include debug info (location, variable and match) in the built graph
+        #[clap(long)]
+        debug: bool,
+    },
+}
+
+impl Commands {
+    pub fn execute(self) -> ExitCode {
+        let command_result = match self {
+            Commands::Parse {
+                file_path,
+                version,
+                json,
+            } => commands::parse::execute(&file_path, version, json),
+            #[cfg(feature = "__graph_builder")]
+            Commands::BuildGraph {
+                file_path,
+                version,
+                msgb_path,
+                json,
+                debug,
+            } => commands::build_graph::execute(&file_path, version, &msgb_path, json, debug),
+        };
+        match command_result {
+            Ok(()) => ExitCode::SUCCESS,
+            Err(error) => {
+                eprintln!("{error}");
+                ExitCode::FAILURE
+            }
+        }
+    }
+}

crates/codegen/runtime/cargo/src/runtime/language.rs.jinja2

@@ -67,6 +67,8 @@ impl Language {
         {%- endif -%}
     ];
 
+    pub const ROOT_KIND: NonterminalKind = NonterminalKind::{{ model.kinds.root_kind }};
+
     pub fn new(version: Version) -> std::result::Result<Self, Error> {
         if Self::SUPPORTED_VERSIONS.binary_search(&version).is_ok() {
             Ok(Self {
@@ -295,6 +297,11 @@ impl Language {
         return Self::SUPPORTED_VERSIONS.iter().map(|v| v.to_string()).collect();
     }
 
+    #[napi(js_name = "rootKind", ts_return_type = "kinds.NonterminalKind", catch_unwind)]
+    pub fn root_kind_napi() -> NonterminalKind {
+        Self::ROOT_KIND
+    }
+
     #[napi(js_name = "parse", ts_return_type = "parse_output.ParseOutput", catch_unwind)]
     pub fn parse_napi(
         &self,

crates/codegen/runtime/cargo/src/runtime/mod.rs

@@ -13,6 +13,9 @@ pub mod parse_output;
 #[cfg(feature = "slang_napi_interfaces")]
 pub mod napi_interface;
 
+#[cfg(feature = "cli")]
+pub mod cli;
+
 mod metaslang_cst {
     #[derive(Clone, Debug, PartialEq, Eq, serde::Serialize)]
     // These derives are because default #[derive(...)] on a generic type implements only the trait
@@ -69,3 +72,14 @@ pub mod text_index {
     use metaslang_cst::text_index;
     pub use text_index::{TextIndex, TextRange, TextRangeExtensions};
 }
+
+#[cfg(feature = "__graph_builder")]
+pub mod graph_builder {
+    use metaslang_graph_builder::ast;
+    pub use metaslang_graph_builder::functions::Functions;
+    pub use metaslang_graph_builder::{ExecutionConfig, ExecutionError, NoCancellation, Variables};
+
+    use super::metaslang_cst::KindTypes;
+
+    pub type File = ast::File<KindTypes>;
+}

crates/codegen/runtime/generator/src/kinds.rs

@@ -14,10 +14,12 @@ pub struct KindsModel {
     trivia_scanner_names: BTreeSet<Identifier>,
     /// Defines `EdgeLabel` enum variants.
     labels: BTreeSet<Identifier>,
-    /// Built-in labels for edges
+    /// Built-in labels for edges.
     built_in_labels: &'static [&'static str],
     // Defines the `LexicalContext(Type)` enum and type-level variants.
     lexical_contexts: BTreeSet<Identifier>,
+    /// Defines the root `NonterminalKind` for a source file of the language.
+    root_kind: Identifier,
 }
 
 impl Default for KindsModel {
@@ -29,6 +31,7 @@ impl Default for KindsModel {
             labels: BTreeSet::default(),
             built_in_labels: BuiltInLabel::VARIANTS,
             lexical_contexts: BTreeSet::default(),
+            root_kind: Identifier::from("Stub1"),
         }
     }
 }
@@ -103,12 +106,15 @@ impl KindsModel {
             .chain(std::iter::once(Identifier::from("Default")))
             .collect();
 
+        let root_kind = language.root_item.clone();
+
         KindsModel {
             nonterminal_kinds,
             terminal_kinds,
             trivia_scanner_names,
             labels,
             lexical_contexts,
+            root_kind,
             ..Self::default()
         }
     }

crates/infra/cli/src/commands/publish/cargo/mod.rs

@@ -14,6 +14,7 @@ use crate::commands::publish::DryRun;
 const USER_FACING_CRATES: &[&str] = &[
     // Sorted by dependency order (from dependencies to dependents):
     "metaslang_cst",
+    "metaslang_graph_builder",
     "slang_solidity",
 ];