Add command to build a graph from a CST and extract CLI commands into the runtime library

.changeset/forty-deers-arrive.md

@@ -0,0 +1,5 @@
+---
+"@nomicfoundation/slang": patch
+---
+
+Expose the language root non-terminal kind at `Language.rootKind()`.

crates/codegen/runtime/cargo/Cargo.toml

@@ -13,7 +13,9 @@ codegen_runtime_generator = { workspace = true }
 
 [dependencies]
 ariadne = { workspace = true, optional = true }
+clap = { workspace = true, optional = true }
 metaslang_cst = { workspace = true }
+metaslang_graph_builder = { workspace = true, optional = true }
 napi = { workspace = true, optional = true }
 napi-derive = { workspace = true, optional = true }
 semver = { workspace = true }
@@ -28,8 +30,11 @@ thiserror = { workspace = true }
 [features]
 default = ["slang_napi_interfaces"]
 slang_napi_interfaces = ["dep:napi", "dep:napi-derive", "dep:serde_json"]
+cli = ["dep:clap", "dep:serde_json", "__private_ariadne"]
 # Only used by the `slang_solidity` CLI
 __private_ariadne = ["dep:ariadne"]
+# For internal development only
+__graph_builder = ["dep:metaslang_graph_builder"]
 
 [lints]
 workspace = true

crates/codegen/runtime/cargo/src/runtime/cli/commands/build_graph.rs

@@ -0,0 +1,57 @@
+use std::fs;
+use std::path::PathBuf;
+
+use semver::Version;
+
+use super::parse::parse_source_file;
+use super::CommandError;
+use crate::graph_builder::{
+ ExecutionConfig, File as GraphBuilderFile, Functions, NoCancellation, Variables,
+};
+
+pub fn execute(
+ file_path_string: &str,
+ version: Version,
+ msgb_path_string: &str,
+ output_json: bool,
+ debug: bool,
+) -> Result<(), CommandError> {
+ let parse_output = parse_source_file(file_path_string, version, |_| ())?;
+ let msgb = parse_graph_builder(msgb_path_string)?;
+
+ let functions = Functions::stdlib();
+ let variables = Variables::new();
+ let mut execution_config = ExecutionConfig::new(&functions, &variables);
+ if debug {
+ execution_config = execution_config.debug_attributes(
+ "_location".into(),
+ "_variable".into(),
+ "_match".into(),
+ );
+ }
+
+ let tree = parse_output.create_tree_cursor();
+ let graph = msgb.execute(&tree, &execution_config, &NoCancellation)?;
+
+ if output_json {
+ graph.display_json(None)?;
+ } else {
+ print!("{}", graph.pretty_print());
+ }
+
+ Ok(())
+}
+
+fn parse_graph_builder(msgb_path_string: &str) -> Result<GraphBuilderFile, CommandError> {
+ let msgb_path = PathBuf::from(&msgb_path_string)
+ .canonicalize()
+ .map_err(|_| CommandError::FileNotFound(msgb_path_string.to_string()))?;
+
+ let msgb_source = fs::read_to_string(&msgb_path)?;
+ GraphBuilderFile::from_str(&msgb_source).map_err(|parser_error| {
+ let error_message = parser_error
+ .display_pretty(&msgb_path, &msgb_source)
+ .to_string();
+ CommandError::ParseFailed(error_message)
+ })
+}

crates/codegen/runtime/cargo/src/runtime/cli/commands/mod.rs

@@ -0,0 +1,24 @@
+use thiserror::Error;
+
+#[cfg(feature = "__graph_builder")]
+pub mod build_graph;
+pub mod parse;
+
+#[derive(Debug, Error)]
+pub enum CommandError {
+ #[error("File not found: {0:?}")]
+ FileNotFound(String),
+
+ #[error(transparent)]
+ Io(#[from] std::io::Error),
+
+ #[error(transparent)]
+ LanguageError(#[from] crate::language::Error),
+
+ #[error("Parsing failed: {0}")]
+ ParseFailed(String),
+
+ #[cfg(feature = "__graph_builder")]
+ #[error(transparent)]
+ ExecutionFailed(#[from] crate::graph_builder::ExecutionError),
+}

crates/codegen/runtime/cargo/src/runtime/cli/commands/parse.rs

@@ -0,0 +1,52 @@
+use std::fs;
+use std::path::PathBuf;
+
+use semver::Version;
+
+use super::CommandError;
+use crate::diagnostic;
+use crate::language::Language;
+use crate::parse_output::ParseOutput;
+
+pub fn execute(file_path_string: &str, version: Version, json: bool) -> Result<(), CommandError> {
+ parse_source_file(file_path_string, version, |output| {
+ if json {
+ let root_node = output.tree();
+ let json = serde_json::to_string_pretty(&root_node).expect("JSON serialization failed");
+ println!("{json}");
+ }
+ })
+ .map(|_| ())
+}
+
+pub(crate) fn parse_source_file<F>(
+ file_path_string: &str,
+ version: Version,
+ run_before_checking: F,
+) -> Result<ParseOutput, CommandError>
+where
+ F: FnOnce(&ParseOutput),
+{
+ let file_path = PathBuf::from(&file_path_string)
+ .canonicalize()
+ .map_err(|_| CommandError::FileNotFound(file_path_string.to_string()))?;
+
+ let input = fs::read_to_string(file_path)?;
+ let language = Language::new(version)?;
+ let parse_output = language.parse(Language::ROOT_KIND, &input);
+
+ run_before_checking(&parse_output);
+
+ if parse_output.is_valid() {
+ Ok(parse_output)
+ } else {
+ const COLOR: bool = true;
+ let report = parse_output
+ .errors()
+ .iter()
+ .map(|error| diagnostic::render(error, file_path_string, &input, COLOR))
+ .collect::<Vec<_>>()
+ .join("\n");
+ Err(CommandError::ParseFailed(report))
+ }
+}

crates/codegen/runtime/cargo/src/runtime/cli/mod.rs

@@ -0,0 +1,73 @@
+use std::process::ExitCode;
+
+use clap::Subcommand;
+use semver::Version;
+
+pub mod commands;
+
+#[derive(Subcommand, Debug)]
+pub enum Commands {
+ /// Parses a source file, and outputs any syntax errors, or a JSON concrete syntax tree
+ Parse {
+ /// File path to the source file to parse
+ file_path: String,
+
+ /// The language version to use for parsing
+ #[arg(short, long)]
+ version: Version,
+
+ /// Print the concrete syntax tree as JSON
+ #[clap(long)]
+ json: bool,
+ },
+
+ // This is only intended for internal development
+ #[cfg(feature = "__graph_builder")]
+ /// Parses a source file and builds a graph executing the instructions from the builder file (*.msgb)
+ BuildGraph {
+ /// File path to the source file to parse
+ file_path: String,
+
+ /// The language version to use for parsing
+ #[arg(short, long)]
+ version: Version,
+
+ /// The graph buider (.msgb) file to use
+ msgb_path: String,
+
+ /// Print the graph as JSON
+ #[clap(long)]
+ json: bool,
+
+ /// Include debug info (location, variable and match) in the built graph
+ #[clap(long)]
+ debug: bool,
+ },
+}
+
+impl Commands {
+ pub fn execute(self) -> ExitCode {
+ let command_result = match self {
+ Commands::Parse {
+ file_path,
+ version,
+ json,
+ } => commands::parse::execute(&file_path, version, json),
+ #[cfg(feature = "__graph_builder")]
+ Commands::BuildGraph {
+ file_path,
+ version,
+ msgb_path,
+ json,
+ debug,
+ } => commands::build_graph::execute(&file_path, version, &msgb_path, json, debug),
+ };
+ match command_result {
+ Ok(()) => ExitCode::SUCCESS,
+ Err(error) => {
+ eprintln!("{error}");
+ ExitCode::FAILURE
+ }
+ }
+ }
+}

crates/codegen/runtime/cargo/src/runtime/language.rs.jinja2

@@ -67,6 +67,8 @@ impl Language {
  {%- endif -%}
  ];
 
+ pub const ROOT_KIND: NonterminalKind = NonterminalKind::{{ model.kinds.root_kind }};
+
  pub fn new(version: Version) -> std::result::Result<Self, Error> {
  if Self::SUPPORTED_VERSIONS.binary_search(&version).is_ok() {
  Ok(Self {
@@ -295,6 +297,11 @@ impl Language {
  return Self::SUPPORTED_VERSIONS.iter().map(|v| v.to_string()).collect();
  }
 
+ #[napi(js_name = "rootKind", ts_return_type = "kinds.NonterminalKind", catch_unwind)]
+ pub fn root_kind_napi() -> NonterminalKind {
+ Self::ROOT_KIND
+ }
+
  #[napi(js_name = "parse", ts_return_type = "parse_output.ParseOutput", catch_unwind)]
  pub fn parse_napi(
  &self,

crates/codegen/runtime/cargo/src/runtime/mod.rs

@@ -13,6 +13,9 @@ pub mod parse_output;
 #[cfg(feature = "slang_napi_interfaces")]
 pub mod napi_interface;
 
+#[cfg(feature = "cli")]
+pub mod cli;
+
 mod metaslang_cst {
  #[derive(Clone, Debug, PartialEq, Eq, serde::Serialize)]
  // These derives are because default #[derive(...)] on a generic type implements only the trait
@@ -69,3 +72,14 @@ pub mod text_index {
  use metaslang_cst::text_index;
  pub use text_index::{TextIndex, TextRange, TextRangeExtensions};
 }
+
+#[cfg(feature = "__graph_builder")]
+pub mod graph_builder {
+ use metaslang_graph_builder::ast;
+ pub use metaslang_graph_builder::functions::Functions;
+ pub use metaslang_graph_builder::{ExecutionConfig, ExecutionError, NoCancellation, Variables};
+
+ use super::metaslang_cst::KindTypes;
+
+ pub type File = ast::File<KindTypes>;
+}

crates/codegen/runtime/generator/src/kinds.rs

@@ -14,10 +14,12 @@ pub struct KindsModel {
  trivia_scanner_names: BTreeSet<Identifier>,
  /// Defines `EdgeLabel` enum variants.
  labels: BTreeSet<Identifier>,
- /// Built-in labels for edges
+ /// Built-in labels for edges.
  built_in_labels: &'static [&'static str],
  // Defines the `LexicalContext(Type)` enum and type-level variants.
  lexical_contexts: BTreeSet<Identifier>,
+ /// Defines the root `NonterminalKind` for a source file of the language.
+ root_kind: Identifier,
 }
 
 impl Default for KindsModel {
@@ -29,6 +31,7 @@ impl Default for KindsModel {
  labels: BTreeSet::default(),
  built_in_labels: BuiltInLabel::VARIANTS,
  lexical_contexts: BTreeSet::default(),
+ root_kind: Identifier::from("Stub1"),
  }
  }
 }
@@ -103,12 +106,15 @@ impl KindsModel {
  .chain(std::iter::once(Identifier::from("Default")))
  .collect();
 
+ let root_kind = language.root_item.clone();
+
  KindsModel {
  nonterminal_kinds,
  terminal_kinds,
  trivia_scanner_names,
  labels,
  lexical_contexts,
+ root_kind,
  ..Self::default()
  }
  }

crates/infra/cli/src/commands/publish/cargo/mod.rs

@@ -14,6 +14,7 @@ use crate::commands::publish::DryRun;
 const USER_FACING_CRATES: &[&str] = &[
  // Sorted by dependency order (from dependencies to dependents):
  "metaslang_cst",
+ "metaslang_graph_builder",
  "slang_solidity",
 ];