From 36cca11a3c0623d95b891899ec17782f1079f9e6 Mon Sep 17 00:00:00 2001 From: matt rice Date: Wed, 19 Aug 2026 03:37:02 -0700 Subject: [PATCH 1/7] Initial constructors for codegen module --- lrlex/src/lib/codegen.rs | 297 +++++++++++++++++++++++++++++++++++++ lrlex/src/lib/ctbuilder.rs | 194 ++++++++++-------------- lrlex/src/lib/mod.rs | 1 + 3 files changed, 377 insertions(+), 115 deletions(-) create mode 100644 lrlex/src/lib/codegen.rs diff --git a/lrlex/src/lib/codegen.rs b/lrlex/src/lib/codegen.rs new file mode 100644 index 000000000..6ea4a3c74 --- /dev/null +++ b/lrlex/src/lib/codegen.rs @@ -0,0 +1,297 @@ +use cfgrammar::{ + Location, Span, + header::{GrmtoolsSectionParser, Header, HeaderError, HeaderValue}, + markmap::MergeError, +}; +use lrpar::LexerTypes; + +use crate::{LRNonStreamingLexerDef, LexBuildError, LexFlags, LexerKind}; +use proc_macro2::Ident; +use std::{collections::HashMap, fmt, marker::PhantomData, path::Path}; + +pub(crate) enum LexerSrcEnvError { + GrmtoolsSectionParseError(Vec>), + GrmtoolsSectionMergeError(MergeError>>), + GrmtoolsSectionLookupError(HeaderError), + MissingModName, + LexBuildErrors(Vec), +} + +impl From>> for LexerSrcEnvError { + fn from(it: Vec>) -> Self { + LexerSrcEnvError::GrmtoolsSectionParseError(it) + } +} + +impl From>>> for LexerSrcEnvError { + fn from(it: MergeError>>) -> Self { + LexerSrcEnvError::GrmtoolsSectionMergeError(it) + } +} + +impl From> for LexerSrcEnvError { + fn from(it: HeaderError) -> Self { + LexerSrcEnvError::GrmtoolsSectionLookupError(it) + } +} + +impl fmt::Display for LexerSrcEnvError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(&match self { + Self::GrmtoolsSectionParseError(errs) => errs + .iter() + .map(|e| e.to_string()) + .collect::>() + .join("\n"), + Self::GrmtoolsSectionMergeError(e) => e.to_string(), + Self::GrmtoolsSectionLookupError(e) => e.to_string(), + Self::MissingModName => "Code generator requires a mod name".to_string(), + Self::LexBuildErrors(_) => "Lex build error".to_string(), + }) + } +} + +pub(crate) struct LexerSrcEnv<'a, LexerTypesT> +where + LexerTypesT: LexerTypes, + usize: num_traits::AsPrimitive, +{ + src: &'a str, + fallback_modname: Option, + header: Header, + phantom_storaget: PhantomData, +} + +pub(crate) struct LexerBuildEnvArgs { + mod_name: Option, + lexerkind: Option, +} + +impl LexerBuildEnvArgs { + pub(crate) fn new() -> Self { + Self { + mod_name: None, + lexerkind: None, + } + } + + pub(crate) fn mod_name(mut self, mod_name: Option) -> Self { + self.mod_name = mod_name; + self + } + + pub(crate) fn lexerkind(mut self, lexerkind: Option) -> Self { + self.lexerkind = lexerkind; + self + } +} + +pub(crate) struct LexerBuildEnv +where + LexerTypesT: LexerTypes, + usize: num_traits::AsPrimitive, +{ + mod_name: String, + lexerkind: LexerKind, + header: Header, + lex_flags: LexFlags, + lexerdef: LRNonStreamingLexerDef, +} + +pub(crate) struct LexerCodegen +where + LexerTypesT: LexerTypes, + usize: num_traits::AsPrimitive, +{ + rule_ids_map: Option>, + #[expect(dead_code)] + timestamp: String, +} + +impl<'a, LexerTypesT> LexerSrcEnv<'a, LexerTypesT> +where + LexerTypesT: LexerTypes, + usize: num_traits::AsPrimitive, +{ + pub(crate) fn new_with_header( + src: &'a str, + path: Option<&Path>, + header: Header, + ) -> LexerSrcEnv<'a, LexerTypesT> { + let fallback_modname = if let Some(lexerp) = path { + // The user hasn't specified a module name, so we create one automatically: what we + // do is strip off all the filename extensions (note that it's likely that inp ends + // with `l.rs`, so we potentially have to strip off more than one extension) and + // then add `_l` to the end. + let mut stem = lexerp.to_str().unwrap(); + loop { + let new_stem = Path::new(stem).file_stem().unwrap().to_str().unwrap(); + if stem == new_stem { + break; + } + stem = new_stem; + } + Some(format!("{}_l", stem)) + } else { + None + }; + LexerSrcEnv { + src, + fallback_modname, + header, + phantom_storaget: PhantomData, + } + } + + fn merge_headers(&mut self) -> Result<(), LexerSrcEnvError> { + let (parsed_header, _) = self.parse_header()?; + Ok(self.header.merge_from(parsed_header)?) + } + + /// Looks up the `mod_name` from the `args`, and defaults to + /// the `{filename}_l` with any file extenstion stripped off. + fn resolve_mod_name(&self, args: &LexerBuildEnvArgs) -> Result { + match &args.mod_name { + Some(s) => Ok(s.to_owned()), + None => self + .fallback_modname + .as_ref() + .ok_or(LexerSrcEnvError::MissingModName) + .map(|s| s.to_string()), + } + } + + fn parse_header(&self) -> Result<(Header, usize), Vec>> { + GrmtoolsSectionParser::new(self.src, false).parse() + } + + pub(crate) fn build_env( + mut self, + args: LexerBuildEnvArgs, + ) -> Result, LexerSrcEnvError> + where + LexerTypesT: LexerTypes, + usize: num_traits::AsPrimitive, + LexerTypesT::StorageT: TryFrom, + { + self.merge_headers()?; + let mod_name = self.resolve_mod_name(&args)?; + self.header.mark_used(&"lexerkind".to_string()); + let lexerkind = match args.lexerkind { + Some(lexerkind) => lexerkind, + None => { + if let Some(HeaderValue(_, lk_val)) = self.header.get("lexerkind") { + LexerKind::try_from(lk_val)? + } else { + LexerKind::LRNonStreamingLexer + } + } + }; + let lex_flags = LexFlags::try_from(&mut self.header)?; + let (lexerdef, lex_flags): (LRNonStreamingLexerDef, LexFlags) = match lexerkind + { + LexerKind::LRNonStreamingLexer => { + let lexerdef = + LRNonStreamingLexerDef::::new_with_options(self.src, lex_flags) + .map_err(LexerSrcEnvError::LexBuildErrors)?; + + let lex_flags = lexerdef.lex_flags().cloned(); + (lexerdef, lex_flags.unwrap()) + } + }; + Ok(LexerBuildEnv { + mod_name, + lexerkind, + header: self.header, + lex_flags, + lexerdef, + }) + } +} + +pub(crate) enum LexerBuildEnvError {} + +impl fmt::Display for LexerBuildEnvError { + fn fmt(&self, _f: &mut fmt::Formatter<'_>) -> fmt::Result { + Ok(()) + } +} + +impl LexerBuildEnv +where + LexerTypesT: LexerTypes, + usize: num_traits::AsPrimitive, + LexerTypesT::StorageT: TryFrom, +{ + pub(crate) fn lexerkind(&self) -> &LexerKind { + &self.lexerkind + } + + pub(crate) fn header(&self) -> &Header { + &self.header + } + + pub(crate) fn mod_name(&self) -> &String { + &self.mod_name + } + + pub(crate) fn lexerdef(&self) -> &LRNonStreamingLexerDef { + &self.lexerdef + } + + pub(crate) fn lexerdef_mut(&mut self) -> &mut LRNonStreamingLexerDef { + &mut self.lexerdef + } + + pub(crate) fn lex_flags(&self) -> &LexFlags { + &self.lex_flags + } + + pub(crate) fn code_generator( + &self, + rule_ids_map: Option>, + timestamp: &str, + ) -> Result, LexerBuildEnvError> { + Ok(LexerCodegen { + rule_ids_map, + timestamp: timestamp.to_string(), + }) + } +} + +pub(crate) enum LexerCodegenError { + InvalidRustIdentifierModName { mod_name: String, error: syn::Error }, +} + +impl fmt::Display for LexerCodegenError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(&match self { + Self::InvalidRustIdentifierModName { mod_name, error } => { + format!("mod_name '{mod_name}' is not a valid rust identifier due to '{error}'") + } + }) + } +} + +impl LexerCodegen +where + LexerTypesT: LexerTypes, + usize: num_traits::AsPrimitive, + LexerTypesT::StorageT: TryFrom, +{ + pub(crate) fn rule_ids_map(&self) -> Option<&HashMap> { + self.rule_ids_map.as_ref() + } + + pub(crate) fn gen_mod_name( + &self, + build_env: &LexerBuildEnv, + ) -> Result { + syn::parse_str::(build_env.mod_name()).map_err(|e| { + LexerCodegenError::InvalidRustIdentifierModName { + mod_name: build_env.mod_name().clone(), + error: e, + } + }) + } +} diff --git a/lrlex/src/lib/ctbuilder.rs b/lrlex/src/lib/ctbuilder.rs index 325e6bab4..4d9082abc 100644 --- a/lrlex/src/lib/ctbuilder.rs +++ b/lrlex/src/lib/ctbuilder.rs @@ -1,10 +1,7 @@ //! Build grammars at run-time. use cfgrammar::{ - header::{ - GrmtoolsSectionParser, Header, HeaderError, HeaderErrorKind, HeaderValue, Namespaced, - Setting, Value, - }, + header::{Header, HeaderError, HeaderErrorKind, HeaderValue, Namespaced, Setting, Value}, markmap::MergeBehavior, span::{Location, Span}, }; @@ -33,7 +30,10 @@ use std::{ }; use wincode::SchemaWrite; -use crate::{DefaultLexerTypes, LRNonStreamingLexer, LRNonStreamingLexerDef, LexFlags, LexerDef}; +use crate::{ + DefaultLexerTypes, LRNonStreamingLexer, LexFlags, LexerDef, + codegen::{LexerBuildEnvArgs, LexerSrcEnv, LexerSrcEnvError}, +}; const RUST_FILE_EXT: &str = "rs"; @@ -496,63 +496,47 @@ where let lex_src = read_to_string(lexerp) .map_err(|e| format!("When reading '{}': {e}", lexerp.display()))?; let lex_diag = SpannedDiagnosticFormatter::new(&lex_src, lexerp); - let mut header = self.header; - let (parsed_header, _) = GrmtoolsSectionParser::new(&lex_src, false) - .parse() - .map_err(|es| { - let mut out = String::new(); - out.push_str(&format!( - "\n{ERROR}{}\n", - lex_diag.file_location_msg(" parsing the `%grmtools` section", None) - )); - for e in es { - out.push_str(&indent(" ", &lex_diag.format_error(e).to_string())); - out.push('\n'); - } - ErrorString(out) - })?; - header.merge_from(parsed_header)?; - header.mark_used(&"lexerkind".to_string()); - let lexerkind = match self.lexerkind { - Some(lexerkind) => lexerkind, - None => { - if let Some(HeaderValue(_, lk_val)) = header.get("lexerkind") { - LexerKind::try_from(lk_val)? - } else { - LexerKind::LRNonStreamingLexer - } - } - }; - #[cfg(test)] - if let Some(inspect_lexerkind_cb) = self.inspect_lexerkind_cb { - inspect_lexerkind_cb(&lexerkind)? - } - let (lexerdef, lex_flags): (LRNonStreamingLexerDef, LexFlags) = - match lexerkind { - LexerKind::LRNonStreamingLexer => { - let lex_flags = LexFlags::try_from(&mut header)?; - let lexerdef = LRNonStreamingLexerDef::::new_with_options( - &lex_src, lex_flags, - ) - .map_err(|errs| { + let args = LexerBuildEnvArgs::new() + .mod_name(self.mod_name.map(|s| s.to_string())) + .lexerkind(self.lexerkind); + let mut build_env = + LexerSrcEnv::::new_with_header(&lex_src, Some(lexerp), self.header) + .build_env(args) + .map_err(|e| match e { + LexerSrcEnvError::GrmtoolsSectionParseError(es) => { let mut out = String::new(); out.push_str(&format!( "\n{ERROR}{}\n", - lex_diag.file_location_msg("", None) + lex_diag.file_location_msg(" parsing the `%grmtools` section", None) + )); + for e in es { + out.push_str(&indent(" ", &lex_diag.format_error(e).to_string())); + out.push('\n'); + } + ErrorString(out) + } + LexerSrcEnvError::LexBuildErrors(errs) => { + let mut out = String::new(); + out.push_str(&format!( + "\n{ERROR}{}\n", + lex_diag.file_location_msg(" building the lexer", None) )); for e in errs { out.push_str(&indent(" ", &lex_diag.format_error(e).to_string())); out.push('\n'); } ErrorString(out) - })?; - let lex_flags = lexerdef.lex_flags().cloned(); - (lexerdef, lex_flags.unwrap()) - } - }; + } + e => ErrorString(e.to_string()), + })?; + + #[cfg(test)] + if let Some(inspect_lexerkind_cb) = self.inspect_lexerkind_cb { + inspect_lexerkind_cb(build_env.lexerkind())? + } let ct_parser = if let Some(ref lrcfg) = self.lrpar_config { - let mut closure_lexerdef = lexerdef.clone(); + let mut closure_lexerdef = build_env.lexerdef().clone(); let mut ctp = CTParserBuilder::::new().inspect_rt(Box::new( move |yacc_header, rtpb, rule_ids_map, grm_path| { let owned_map = rule_ids_map @@ -632,34 +616,39 @@ where None }; - let mut lexerdef = Box::new(lexerdef); - let unused_header_values = header.unused(); + let unused_header_values = build_env.header().unused(); if !unused_header_values.is_empty() { return Err( format!("Unused header values: {}", unused_header_values.join(", ")).into(), ); } - let (mut missing_from_lexer, missing_from_parser) = match self.rule_ids_map { - Some(ref rim) => { - // Convert from HashMap to HashMap<&str, _> - let owned_map = rim - .iter() - .map(|(x, y)| (&**x, *y)) - .collect::>(); - let (x, y) = lexerdef.set_rule_ids_spanned(&owned_map); - ( - x.map(|a| a.iter().map(|&b| b.to_string()).collect::>()), - y.map(|a| { - a.iter() - .map(|(b, span)| (b.to_string(), *span)) - .collect::>() - }), - ) + let code_gen = build_env + .code_generator(self.rule_ids_map, env!("VERGEN_BUILD_TIMESTAMP")) + .map_err(|e| match e {})?; + let (mut missing_from_lexer, missing_from_parser) = { + let lexerdef = Box::new(build_env.lexerdef_mut()); + match code_gen.rule_ids_map() { + Some(rim) => { + // Convert from HashMap to HashMap<&str, _> + let owned_map = rim + .iter() + .map(|(x, y)| (&**x, *y)) + .collect::>(); + let (x, y) = lexerdef.set_rule_ids_spanned(&owned_map); + ( + x.map(|a| a.iter().map(|&b| b.to_string()).collect::>()), + y.map(|a| { + a.iter() + .map(|(b, span)| (b.to_string(), *span)) + .collect::>() + }), + ) + } + None => (None, None), } - None => (None, None), }; - + let lexerdef = build_env.lexerdef(); if let Some(mut mfl) = missing_from_lexer.take() { for tok in &lexerdef.expected_missing_tokens { mfl.remove(tok.as_str()); @@ -760,34 +749,9 @@ where fs::remove_file(outp).ok(); panic!(); } - - let mod_name = match self.mod_name { - Some(s) => s.to_owned(), - None => { - // The user hasn't specified a module name, so we create one automatically: what we - // do is strip off all the filename extensions (note that it's likely that inp ends - // with `l.rs`, so we potentially have to strip off more than one extension) and - // then add `_l` to the end. - let mut stem = lexerp.to_str().unwrap(); - loop { - let new_stem = Path::new(stem).file_stem().unwrap().to_str().unwrap(); - if stem == new_stem { - break; - } - stem = new_stem; - } - format!("{}_l", stem) - } - }; - let mod_name = - match syn::parse_str::(&mod_name) { - Ok(s) => s, - Err(e) => return Err(format!( - "CTLexerBuilder::mod_name(\"{}\") is not a valid rust identifier due to '{}'", - mod_name, e - ) - .into()), - }; + let mod_name = code_gen + .gen_mod_name(&build_env) + .map_err(|e| ErrorString(e.to_string()))?; let mut lexerdef_func_impl = { let LexFlags { allow_wholeline_comments, @@ -802,19 +766,19 @@ where size_limit, dfa_size_limit, nest_limit, - } = lex_flags; - let allow_wholeline_comments = QuoteOption(allow_wholeline_comments); - let dot_matches_new_line = QuoteOption(dot_matches_new_line); - let multi_line = QuoteOption(multi_line); - let octal = QuoteOption(octal); - let posix_escapes = QuoteOption(posix_escapes); - let case_insensitive = QuoteOption(case_insensitive); - let unicode = QuoteOption(unicode); - let swap_greed = QuoteOption(swap_greed); - let ignore_whitespace = QuoteOption(ignore_whitespace); - let size_limit = QuoteOption(size_limit); - let dfa_size_limit = QuoteOption(dfa_size_limit); - let nest_limit = QuoteOption(nest_limit); + } = build_env.lex_flags(); + let allow_wholeline_comments = QuoteOption(allow_wholeline_comments.as_ref()); + let dot_matches_new_line = QuoteOption(dot_matches_new_line.as_ref()); + let multi_line = QuoteOption(multi_line.as_ref()); + let octal = QuoteOption(octal.as_ref()); + let posix_escapes = QuoteOption(posix_escapes.as_ref()); + let case_insensitive = QuoteOption(case_insensitive.as_ref()); + let unicode = QuoteOption(unicode.as_ref()); + let swap_greed = QuoteOption(swap_greed.as_ref()); + let ignore_whitespace = QuoteOption(ignore_whitespace.as_ref()); + let size_limit = QuoteOption(size_limit.as_ref()); + let dfa_size_limit = QuoteOption(dfa_size_limit.as_ref()); + let nest_limit = QuoteOption(nest_limit.as_ref()); // Code gen for the lexerdef() `lex_flags` variable. quote! { @@ -859,7 +823,7 @@ where let rules = vec![#(#rules),*]; }); } - let lexerdef_ty = match lexerkind { + let lexerdef_ty = match build_env.lexerkind() { LexerKind::LRNonStreamingLexer => { quote!(::lrlex::LRNonStreamingLexerDef) } @@ -870,7 +834,7 @@ where }); let mut token_consts = TokenStream::new(); - if let Some(rim) = self.rule_ids_map { + if let Some(rim) = code_gen.rule_ids_map() { let mut rim_sorted = Vec::from_iter(rim.iter()); rim_sorted.sort_by_key(|(k, _)| *k); for (name, id) in rim_sorted { @@ -1547,7 +1511,7 @@ A "A" let err_string = e.to_string(); assert_eq!( err_string, - "CTLexerBuilder::mod_name(\"contains-a-dash_l\") is not a valid rust identifier due to 'unexpected token'" + "mod_name 'contains-a-dash_l' is not a valid rust identifier due to 'unexpected token'" ); } } diff --git a/lrlex/src/lib/mod.rs b/lrlex/src/lib/mod.rs index f32cf641a..000d44259 100644 --- a/lrlex/src/lib/mod.rs +++ b/lrlex/src/lib/mod.rs @@ -14,6 +14,7 @@ use std::{error::Error, fmt}; +mod codegen; mod ctbuilder; #[doc(hidden)] pub mod defaults; From 5327b867eca8b1549a6d5c8fded1ad95c6c6279d Mon Sep 17 00:00:00 2001 From: matt rice Date: Wed, 19 Aug 2026 03:53:46 -0700 Subject: [PATCH 2/7] Move lex_flags codegen to module --- lrlex/src/lib/codegen.rs | 66 +++++++++++++++++++++++++++++++++++++- lrlex/src/lib/ctbuilder.rs | 51 +++-------------------------- 2 files changed, 69 insertions(+), 48 deletions(-) diff --git a/lrlex/src/lib/codegen.rs b/lrlex/src/lib/codegen.rs index 6ea4a3c74..1c8c6a773 100644 --- a/lrlex/src/lib/codegen.rs +++ b/lrlex/src/lib/codegen.rs @@ -6,7 +6,8 @@ use cfgrammar::{ use lrpar::LexerTypes; use crate::{LRNonStreamingLexerDef, LexBuildError, LexFlags, LexerKind}; -use proc_macro2::Ident; +use proc_macro2::{Ident, TokenStream}; +use quote::{ToTokens, TokenStreamExt, quote}; use std::{collections::HashMap, fmt, marker::PhantomData, path::Path}; pub(crate) enum LexerSrcEnvError { @@ -294,4 +295,67 @@ where } }) } + + pub(crate) fn gen_lex_flags_decl(&self, build_env: &LexerBuildEnv) -> TokenStream { + let LexFlags { + allow_wholeline_comments, + dot_matches_new_line, + multi_line, + octal, + posix_escapes, + case_insensitive, + unicode, + swap_greed, + ignore_whitespace, + size_limit, + dfa_size_limit, + nest_limit, + } = build_env.lex_flags(); + let allow_wholeline_comments = QuoteOption(allow_wholeline_comments.as_ref()); + let dot_matches_new_line = QuoteOption(dot_matches_new_line.as_ref()); + let multi_line = QuoteOption(multi_line.as_ref()); + let octal = QuoteOption(octal.as_ref()); + let posix_escapes = QuoteOption(posix_escapes.as_ref()); + let case_insensitive = QuoteOption(case_insensitive.as_ref()); + let unicode = QuoteOption(unicode.as_ref()); + let swap_greed = QuoteOption(swap_greed.as_ref()); + let ignore_whitespace = QuoteOption(ignore_whitespace.as_ref()); + let size_limit = QuoteOption(size_limit.as_ref()); + let dfa_size_limit = QuoteOption(dfa_size_limit.as_ref()); + let nest_limit = QuoteOption(nest_limit.as_ref()); + + // Code gen for the lexerdef() `lex_flags` variable. + quote! { + let mut lex_flags = ::lrlex::DEFAULT_LEX_FLAGS; + lex_flags.allow_wholeline_comments = #allow_wholeline_comments.or(::lrlex::DEFAULT_LEX_FLAGS.allow_wholeline_comments); + lex_flags.dot_matches_new_line = #dot_matches_new_line.or(::lrlex::DEFAULT_LEX_FLAGS.dot_matches_new_line); + lex_flags.multi_line = #multi_line.or(::lrlex::DEFAULT_LEX_FLAGS.multi_line); + lex_flags.octal = #octal.or(::lrlex::DEFAULT_LEX_FLAGS.octal); + lex_flags.posix_escapes = #posix_escapes.or(::lrlex::DEFAULT_LEX_FLAGS.posix_escapes); + lex_flags.case_insensitive = #case_insensitive.or(::lrlex::DEFAULT_LEX_FLAGS.case_insensitive); + lex_flags.unicode = #unicode.or(::lrlex::DEFAULT_LEX_FLAGS.unicode); + lex_flags.swap_greed = #swap_greed.or(::lrlex::DEFAULT_LEX_FLAGS.swap_greed); + lex_flags.ignore_whitespace = #ignore_whitespace.or(::lrlex::DEFAULT_LEX_FLAGS.ignore_whitespace); + lex_flags.size_limit = #size_limit.or(::lrlex::DEFAULT_LEX_FLAGS.size_limit); + lex_flags.dfa_size_limit = #dfa_size_limit.or(::lrlex::DEFAULT_LEX_FLAGS.dfa_size_limit); + lex_flags.nest_limit = #nest_limit.or(::lrlex::DEFAULT_LEX_FLAGS.nest_limit); + let lex_flags = lex_flags; + } + } +} + +/// The quote impl of `ToTokens` for `Option` prints an empty string for `None` +/// and the inner value for `Some(inner_value)`. +/// +/// This wrapper instead emits both `Some` and `None` variants. +/// See: [quote #20](https://github.com/dtolnay/quote/issues/20) +struct QuoteOption(Option); + +impl ToTokens for QuoteOption { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append_all(match self.0 { + Some(ref t) => quote! { ::std::option::Option::Some(#t) }, + None => quote! { ::std::option::Option::None }, + }); + } } diff --git a/lrlex/src/lib/ctbuilder.rs b/lrlex/src/lib/ctbuilder.rs index 4d9082abc..9fb43a03e 100644 --- a/lrlex/src/lib/ctbuilder.rs +++ b/lrlex/src/lib/ctbuilder.rs @@ -31,7 +31,7 @@ use std::{ use wincode::SchemaWrite; use crate::{ - DefaultLexerTypes, LRNonStreamingLexer, LexFlags, LexerDef, + DefaultLexerTypes, LRNonStreamingLexer, LexerDef, codegen::{LexerBuildEnvArgs, LexerSrcEnv, LexerSrcEnvError}, }; @@ -178,6 +178,7 @@ pub enum RustEdition { /// /// This wrapper instead emits both `Some` and `None` variants. /// See: [quote #20](https://github.com/dtolnay/quote/issues/20) +// FIXME Remove in the next patch. struct QuoteOption(Option); impl ToTokens for QuoteOption { @@ -752,52 +753,7 @@ where let mod_name = code_gen .gen_mod_name(&build_env) .map_err(|e| ErrorString(e.to_string()))?; - let mut lexerdef_func_impl = { - let LexFlags { - allow_wholeline_comments, - dot_matches_new_line, - multi_line, - octal, - posix_escapes, - case_insensitive, - unicode, - swap_greed, - ignore_whitespace, - size_limit, - dfa_size_limit, - nest_limit, - } = build_env.lex_flags(); - let allow_wholeline_comments = QuoteOption(allow_wholeline_comments.as_ref()); - let dot_matches_new_line = QuoteOption(dot_matches_new_line.as_ref()); - let multi_line = QuoteOption(multi_line.as_ref()); - let octal = QuoteOption(octal.as_ref()); - let posix_escapes = QuoteOption(posix_escapes.as_ref()); - let case_insensitive = QuoteOption(case_insensitive.as_ref()); - let unicode = QuoteOption(unicode.as_ref()); - let swap_greed = QuoteOption(swap_greed.as_ref()); - let ignore_whitespace = QuoteOption(ignore_whitespace.as_ref()); - let size_limit = QuoteOption(size_limit.as_ref()); - let dfa_size_limit = QuoteOption(dfa_size_limit.as_ref()); - let nest_limit = QuoteOption(nest_limit.as_ref()); - - // Code gen for the lexerdef() `lex_flags` variable. - quote! { - let mut lex_flags = ::lrlex::DEFAULT_LEX_FLAGS; - lex_flags.allow_wholeline_comments = #allow_wholeline_comments.or(::lrlex::DEFAULT_LEX_FLAGS.allow_wholeline_comments); - lex_flags.dot_matches_new_line = #dot_matches_new_line.or(::lrlex::DEFAULT_LEX_FLAGS.dot_matches_new_line); - lex_flags.multi_line = #multi_line.or(::lrlex::DEFAULT_LEX_FLAGS.multi_line); - lex_flags.octal = #octal.or(::lrlex::DEFAULT_LEX_FLAGS.octal); - lex_flags.posix_escapes = #posix_escapes.or(::lrlex::DEFAULT_LEX_FLAGS.posix_escapes); - lex_flags.case_insensitive = #case_insensitive.or(::lrlex::DEFAULT_LEX_FLAGS.case_insensitive); - lex_flags.unicode = #unicode.or(::lrlex::DEFAULT_LEX_FLAGS.unicode); - lex_flags.swap_greed = #swap_greed.or(::lrlex::DEFAULT_LEX_FLAGS.swap_greed); - lex_flags.ignore_whitespace = #ignore_whitespace.or(::lrlex::DEFAULT_LEX_FLAGS.ignore_whitespace); - lex_flags.size_limit = #size_limit.or(::lrlex::DEFAULT_LEX_FLAGS.size_limit); - lex_flags.dfa_size_limit = #dfa_size_limit.or(::lrlex::DEFAULT_LEX_FLAGS.dfa_size_limit); - lex_flags.nest_limit = #nest_limit.or(::lrlex::DEFAULT_LEX_FLAGS.nest_limit); - let lex_flags = lex_flags; - } - }; + let mut lexerdef_func_impl = code_gen.gen_lex_flags_decl(&build_env); { let start_states = lexerdef.iter_start_states(); let rules = lexerdef.iter_rules().map(|r| { @@ -823,6 +779,7 @@ where let rules = vec![#(#rules),*]; }); } + let lexerdef_ty = match build_env.lexerkind() { LexerKind::LRNonStreamingLexer => { quote!(::lrlex::LRNonStreamingLexerDef) From 2b835c708886b81bc003515d4cc28da93745e5f5 Mon Sep 17 00:00:00 2001 From: matt rice Date: Wed, 19 Aug 2026 04:02:20 -0700 Subject: [PATCH 3/7] Move rule and start_state codegen to module --- lrlex/src/lib/codegen.rs | 58 +++++++++++++++++++++++++++++++-- lrlex/src/lib/ctbuilder.rs | 66 ++------------------------------------ 2 files changed, 58 insertions(+), 66 deletions(-) diff --git a/lrlex/src/lib/codegen.rs b/lrlex/src/lib/codegen.rs index 1c8c6a773..9d05e34ae 100644 --- a/lrlex/src/lib/codegen.rs +++ b/lrlex/src/lib/codegen.rs @@ -5,7 +5,7 @@ use cfgrammar::{ }; use lrpar::LexerTypes; -use crate::{LRNonStreamingLexerDef, LexBuildError, LexFlags, LexerKind}; +use crate::{LRNonStreamingLexerDef, LexBuildError, LexFlags, LexerDef, LexerKind}; use proc_macro2::{Ident, TokenStream}; use quote::{ToTokens, TokenStreamExt, quote}; use std::{collections::HashMap, fmt, marker::PhantomData, path::Path}; @@ -278,7 +278,7 @@ impl LexerCodegen where LexerTypesT: LexerTypes, usize: num_traits::AsPrimitive, - LexerTypesT::StorageT: TryFrom, + LexerTypesT::StorageT: TryFrom + ToTokens, { pub(crate) fn rule_ids_map(&self) -> Option<&HashMap> { self.rule_ids_map.as_ref() @@ -342,6 +342,39 @@ where let lex_flags = lex_flags; } } + + pub(crate) fn gen_rules_val(&self, build_env: &LexerBuildEnv) -> TokenStream { + let rules = build_env.lexerdef.iter_rules().map(|r| { + let tok_id = QuoteOption(r.tok_id); + let n = QuoteOption(r.name().map(QuoteToString)); + let target_state = QuoteOption(r.target_state().map(|(x, y)| QuoteTuple((x, y)))); + let n_span = r.name_span(); + let regex = QuoteToString(&r.re_str); + let start_states = r.start_states(); + // Code gen to construct a rule. + // + // We cannot `impl ToToken for Rule` because `Rule` never stores `lex_flags`, + // Thus we reference the local lex_flags variable bound earlier. + quote! { + Rule::new(::lrlex::unstable_api::InternalPublicApi, #tok_id, #n, #n_span, #regex, + vec![#(#start_states),*], #target_state, &lex_flags).unwrap() + } + }); + // Code gen for `lexerdef()`s rules and the stack of `start_states`. + quote! { + let rules = vec![#(#rules),*]; + } + } + + pub(crate) fn gen_start_states_val( + &self, + build_env: &LexerBuildEnv, + ) -> TokenStream { + let start_states = build_env.lexerdef.iter_start_states(); + quote! { + let start_states: Vec = vec![#(#start_states),*]; + } + } } /// The quote impl of `ToTokens` for `Option` prints an empty string for `None` @@ -359,3 +392,24 @@ impl ToTokens for QuoteOption { }); } } + +/// The wrapped `&str` value will be emitted with a call to `to_string()` +struct QuoteToString<'a>(&'a str); + +impl ToTokens for QuoteToString<'_> { + fn to_tokens(&self, tokens: &mut TokenStream) { + let x = &self.0; + tokens.append_all(quote! { #x.to_string() }); + } +} + +/// This wrapper adds a missing impl of `ToTokens` for tuples. +/// For a tuple `(a, b)` emits `(a.to_tokens(), b.to_tokens())` +struct QuoteTuple(T); + +impl ToTokens for QuoteTuple<(A, B)> { + fn to_tokens(&self, tokens: &mut TokenStream) { + let (a, b) = &self.0; + tokens.append_all(quote!((#a, #b))); + } +} diff --git a/lrlex/src/lib/ctbuilder.rs b/lrlex/src/lib/ctbuilder.rs index 9fb43a03e..10fc022d3 100644 --- a/lrlex/src/lib/ctbuilder.rs +++ b/lrlex/src/lib/ctbuilder.rs @@ -173,44 +173,6 @@ pub enum RustEdition { Rust2021, } -/// The quote impl of `ToTokens` for `Option` prints an empty string for `None` -/// and the inner value for `Some(inner_value)`. -/// -/// This wrapper instead emits both `Some` and `None` variants. -/// See: [quote #20](https://github.com/dtolnay/quote/issues/20) -// FIXME Remove in the next patch. -struct QuoteOption(Option); - -impl ToTokens for QuoteOption { - fn to_tokens(&self, tokens: &mut TokenStream) { - tokens.append_all(match self.0 { - Some(ref t) => quote! { ::std::option::Option::Some(#t) }, - None => quote! { ::std::option::Option::None }, - }); - } -} - -/// This wrapper adds a missing impl of `ToTokens` for tuples. -/// For a tuple `(a, b)` emits `(a.to_tokens(), b.to_tokens())` -struct QuoteTuple(T); - -impl ToTokens for QuoteTuple<(A, B)> { - fn to_tokens(&self, tokens: &mut TokenStream) { - let (a, b) = &self.0; - tokens.append_all(quote!((#a, #b))); - } -} - -/// The wrapped `&str` value will be emitted with a call to `to_string()` -struct QuoteToString<'a>(&'a str); - -impl ToTokens for QuoteToString<'_> { - fn to_tokens(&self, tokens: &mut TokenStream) { - let x = &self.0; - tokens.append_all(quote! { #x.to_string() }); - } -} - /// A string which uses `Display` for it's `Debug` impl. struct ErrorString(String); impl fmt::Display for ErrorString { @@ -754,32 +716,8 @@ where .gen_mod_name(&build_env) .map_err(|e| ErrorString(e.to_string()))?; let mut lexerdef_func_impl = code_gen.gen_lex_flags_decl(&build_env); - { - let start_states = lexerdef.iter_start_states(); - let rules = lexerdef.iter_rules().map(|r| { - let tok_id = QuoteOption(r.tok_id); - let n = QuoteOption(r.name().map(QuoteToString)); - let target_state = - QuoteOption(r.target_state().map(|(x, y)| QuoteTuple((x, y)))); - let n_span = r.name_span(); - let regex = QuoteToString(&r.re_str); - let start_states = r.start_states(); - // Code gen to construct a rule. - // - // We cannot `impl ToToken for Rule` because `Rule` never stores `lex_flags`, - // Thus we reference the local lex_flags variable bound earlier. - quote! { - Rule::new(::lrlex::unstable_api::InternalPublicApi, #tok_id, #n, #n_span, #regex, - vec![#(#start_states),*], #target_state, &lex_flags).unwrap() - } - }); - // Code gen for `lexerdef()`s rules and the stack of `start_states`. - lexerdef_func_impl.append_all(quote! { - let start_states: Vec = vec![#(#start_states),*]; - let rules = vec![#(#rules),*]; - }); - } - + lexerdef_func_impl.append_all(code_gen.gen_start_states_val(&build_env)); + lexerdef_func_impl.append_all(code_gen.gen_rules_val(&build_env)); let lexerdef_ty = match build_env.lexerkind() { LexerKind::LRNonStreamingLexer => { quote!(::lrlex::LRNonStreamingLexerDef) From ebb6efd9528dff9a1c9d4a5810f73a6d60d77536 Mon Sep 17 00:00:00 2001 From: matt rice Date: Wed, 19 Aug 2026 04:18:28 -0700 Subject: [PATCH 4/7] Move codegen for building of lexerdef to module --- lrlex/src/lib/codegen.rs | 18 ++++++++++++++++++ lrlex/src/lib/ctbuilder.rs | 10 ++-------- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/lrlex/src/lib/codegen.rs b/lrlex/src/lib/codegen.rs index 9d05e34ae..4a5a59f1b 100644 --- a/lrlex/src/lib/codegen.rs +++ b/lrlex/src/lib/codegen.rs @@ -375,6 +375,24 @@ where let start_states: Vec = vec![#(#start_states),*]; } } + + pub(crate) fn gen_lexerdef_ty(&self, build_env: &LexerBuildEnv) -> TokenStream { + match build_env.lexerkind() { + LexerKind::LRNonStreamingLexer => { + quote!(::lrlex::LRNonStreamingLexerDef) + } + } + } + + pub(crate) fn gen_instantiate_lexerdef( + &self, + build_env: &LexerBuildEnv, + ) -> TokenStream { + let lexerdef_ty = self.gen_lexerdef_ty(build_env); + quote! { + #lexerdef_ty::from_rules(start_states, rules) + } + } } /// The quote impl of `ToTokens` for `Option` prints an empty string for `None` diff --git a/lrlex/src/lib/ctbuilder.rs b/lrlex/src/lib/ctbuilder.rs index 10fc022d3..6f1058ad4 100644 --- a/lrlex/src/lib/ctbuilder.rs +++ b/lrlex/src/lib/ctbuilder.rs @@ -718,15 +718,9 @@ where let mut lexerdef_func_impl = code_gen.gen_lex_flags_decl(&build_env); lexerdef_func_impl.append_all(code_gen.gen_start_states_val(&build_env)); lexerdef_func_impl.append_all(code_gen.gen_rules_val(&build_env)); - let lexerdef_ty = match build_env.lexerkind() { - LexerKind::LRNonStreamingLexer => { - quote!(::lrlex::LRNonStreamingLexerDef) - } - }; // Code gen for the lexerdef() return value referencing variables bound earlier. - lexerdef_func_impl.append_all(quote! { - #lexerdef_ty::from_rules(start_states, rules) - }); + lexerdef_func_impl.append_all(code_gen.gen_instantiate_lexerdef(&build_env)); + let lexerdef_ty = code_gen.gen_lexerdef_ty(&build_env); let mut token_consts = TokenStream::new(); if let Some(rim) = code_gen.rule_ids_map() { From 6a8d2433a1f5a79a290bf83a2eb3500f24dff97d Mon Sep 17 00:00:00 2001 From: matt rice Date: Wed, 19 Aug 2026 05:58:30 -0700 Subject: [PATCH 5/7] Move generation of token consts to codegen module --- lrlex/src/lib/codegen.rs | 32 ++++++++++++++++++++++++++++++-- lrlex/src/lib/ctbuilder.rs | 26 ++------------------------ 2 files changed, 32 insertions(+), 26 deletions(-) diff --git a/lrlex/src/lib/codegen.rs b/lrlex/src/lib/codegen.rs index 4a5a59f1b..3433b0c92 100644 --- a/lrlex/src/lib/codegen.rs +++ b/lrlex/src/lib/codegen.rs @@ -7,8 +7,14 @@ use lrpar::LexerTypes; use crate::{LRNonStreamingLexerDef, LexBuildError, LexFlags, LexerDef, LexerKind}; use proc_macro2::{Ident, TokenStream}; -use quote::{ToTokens, TokenStreamExt, quote}; -use std::{collections::HashMap, fmt, marker::PhantomData, path::Path}; +use quote::{ToTokens, TokenStreamExt, format_ident, quote}; +use regex::Regex; +use std::{ + any::type_name, collections::HashMap, fmt, marker::PhantomData, path::Path, sync::LazyLock, +}; + +static RE_TOKEN_ID: LazyLock = + LazyLock::new(|| Regex::new(r"^[a-zA-Z_][a-zA-Z_0-9]*$").unwrap()); pub(crate) enum LexerSrcEnvError { GrmtoolsSectionParseError(Vec>), @@ -393,6 +399,28 @@ where #lexerdef_ty::from_rules(start_states, rules) } } + + pub(crate) fn gen_token_consts(&self) -> TokenStream { + let mut token_consts = TokenStream::new(); + if let Some(rim) = self.rule_ids_map() { + let mut rim_sorted = Vec::from_iter(rim.iter()); + rim_sorted.sort_by_key(|(k, _)| *k); + for (name, id) in rim_sorted { + if RE_TOKEN_ID.is_match(name) { + let tok_ident = format_ident!("N_{}", name.to_ascii_uppercase()); + let storaget = + str::parse::(type_name::()).unwrap(); + // Code gen for the constant token values. + let tok_const = quote! { + #[allow(dead_code)] + pub const #tok_ident: #storaget = #id; + }; + token_consts.extend(tok_const) + } + } + } + token_consts + } } /// The quote impl of `ToTokens` for `Option` prints an empty string for `None` diff --git a/lrlex/src/lib/ctbuilder.rs b/lrlex/src/lib/ctbuilder.rs index 6f1058ad4..5c5229354 100644 --- a/lrlex/src/lib/ctbuilder.rs +++ b/lrlex/src/lib/ctbuilder.rs @@ -13,8 +13,6 @@ use lrpar::{ use num_traits::{AsPrimitive, PrimInt, Unsigned}; use proc_macro2::{Ident, TokenStream}; use quote::{ToTokens, TokenStreamExt, format_ident, quote}; -use regex::Regex; -use std::marker::PhantomData; use std::{ any::type_name, borrow::Borrow, @@ -25,6 +23,7 @@ use std::{ fs::{self, File, create_dir_all, read_to_string}, hash::Hash, io::Write, + marker::PhantomData, path::{Path, PathBuf}, sync::{LazyLock, Mutex}, }; @@ -40,9 +39,6 @@ const RUST_FILE_EXT: &str = "rs"; const ERROR: &str = "[Error]"; const WARNING: &str = "[Warning]"; -static RE_TOKEN_ID: LazyLock = - LazyLock::new(|| Regex::new(r"^[a-zA-Z_][a-zA-Z_0-9]*$").unwrap()); - static GENERATED_PATHS: LazyLock>> = LazyLock::new(|| Mutex::new(HashSet::new())); @@ -721,25 +717,7 @@ where // Code gen for the lexerdef() return value referencing variables bound earlier. lexerdef_func_impl.append_all(code_gen.gen_instantiate_lexerdef(&build_env)); let lexerdef_ty = code_gen.gen_lexerdef_ty(&build_env); - - let mut token_consts = TokenStream::new(); - if let Some(rim) = code_gen.rule_ids_map() { - let mut rim_sorted = Vec::from_iter(rim.iter()); - rim_sorted.sort_by_key(|(k, _)| *k); - for (name, id) in rim_sorted { - if RE_TOKEN_ID.is_match(name) { - let tok_ident = format_ident!("N_{}", name.to_ascii_uppercase()); - let storaget = - str::parse::(type_name::()).unwrap(); - // Code gen for the constant token values. - let tok_const = quote! { - #[allow(dead_code)] - pub const #tok_ident: #storaget = #id; - }; - token_consts.extend(tok_const) - } - } - } + let token_consts = code_gen.gen_token_consts(); let token_consts = token_consts.into_iter(); let out_tokens = { let lexerdef_param = str::parse::(type_name::()).unwrap(); From d3a9e39422c977c7f13e3591a538e64eaea51db2 Mon Sep 17 00:00:00 2001 From: matt rice Date: Wed, 19 Aug 2026 06:13:34 -0700 Subject: [PATCH 6/7] Move module generation code to module --- lrlex/src/lib/codegen.rs | 42 ++++++++++++++++++++++++++++++++++++-- lrlex/src/lib/ctbuilder.rs | 36 +++++++------------------------- 2 files changed, 47 insertions(+), 31 deletions(-) diff --git a/lrlex/src/lib/codegen.rs b/lrlex/src/lib/codegen.rs index 3433b0c92..130620996 100644 --- a/lrlex/src/lib/codegen.rs +++ b/lrlex/src/lib/codegen.rs @@ -5,7 +5,7 @@ use cfgrammar::{ }; use lrpar::LexerTypes; -use crate::{LRNonStreamingLexerDef, LexBuildError, LexFlags, LexerDef, LexerKind}; +use crate::{LRNonStreamingLexerDef, LexBuildError, LexFlags, LexerDef, LexerKind, Visibility}; use proc_macro2::{Ident, TokenStream}; use quote::{ToTokens, TokenStreamExt, format_ident, quote}; use regex::Regex; @@ -72,6 +72,7 @@ where pub(crate) struct LexerBuildEnvArgs { mod_name: Option, lexerkind: Option, + visibility: Visibility, } impl LexerBuildEnvArgs { @@ -79,6 +80,7 @@ impl LexerBuildEnvArgs { Self { mod_name: None, lexerkind: None, + visibility: Visibility::Private, } } @@ -91,6 +93,11 @@ impl LexerBuildEnvArgs { self.lexerkind = lexerkind; self } + + pub(crate) fn visibility(mut self, visibility: Visibility) -> Self { + self.visibility = visibility; + self + } } pub(crate) struct LexerBuildEnv @@ -102,6 +109,7 @@ where lexerkind: LexerKind, header: Header, lex_flags: LexFlags, + visibility: Visibility, lexerdef: LRNonStreamingLexerDef, } @@ -111,7 +119,7 @@ where usize: num_traits::AsPrimitive, { rule_ids_map: Option>, - #[expect(dead_code)] + #[expect(unused)] timestamp: String, } @@ -209,6 +217,7 @@ where Ok(LexerBuildEnv { mod_name, lexerkind, + visibility: args.visibility, header: self.header, lex_flags, lexerdef, @@ -421,6 +430,35 @@ where } token_consts } + + pub(crate) fn generate_lex_module( + &self, + build_env: &LexerBuildEnv, + ) -> Result { + let mod_name = self.gen_mod_name(build_env)?; + let mut lexerdef_func_impl = self.gen_lex_flags_decl(build_env); + lexerdef_func_impl.append_all(self.gen_start_states_val(build_env)); + lexerdef_func_impl.append_all(self.gen_rules_val(build_env)); + // Code gen for the lexerdef() return value referencing variables bound earlier. + lexerdef_func_impl.append_all(self.gen_instantiate_lexerdef(build_env)); + let lexerdef_ty = self.gen_lexerdef_ty(build_env); + let token_consts = self.gen_token_consts(); + let token_consts = token_consts.into_iter(); + let lexerdef_param = str::parse::(type_name::()).unwrap(); + let mod_vis = &build_env.visibility; + // Code gen for the generated module. + Ok(quote! { + #mod_vis mod #mod_name { + use ::lrlex::{LexerDef, Rule, StartState}; + #[allow(dead_code)] + pub fn lexerdef() -> #lexerdef_ty<#lexerdef_param> { + #lexerdef_func_impl + } + + #(#token_consts)* + } + }) + } } /// The quote impl of `ToTokens` for `Option` prints an empty string for `None` diff --git a/lrlex/src/lib/ctbuilder.rs b/lrlex/src/lib/ctbuilder.rs index 5c5229354..4e81783dc 100644 --- a/lrlex/src/lib/ctbuilder.rs +++ b/lrlex/src/lib/ctbuilder.rs @@ -12,7 +12,7 @@ use lrpar::{ }; use num_traits::{AsPrimitive, PrimInt, Unsigned}; use proc_macro2::{Ident, TokenStream}; -use quote::{ToTokens, TokenStreamExt, format_ident, quote}; +use quote::{ToTokens, format_ident, quote}; use std::{ any::type_name, borrow::Borrow, @@ -457,7 +457,8 @@ where let lex_diag = SpannedDiagnosticFormatter::new(&lex_src, lexerp); let args = LexerBuildEnvArgs::new() .mod_name(self.mod_name.map(|s| s.to_string())) - .lexerkind(self.lexerkind); + .lexerkind(self.lexerkind) + .visibility(self.visibility); let mut build_env = LexerSrcEnv::::new_with_header(&lex_src, Some(lexerp), self.header) .build_env(args) @@ -708,35 +709,12 @@ where fs::remove_file(outp).ok(); panic!(); } - let mod_name = code_gen - .gen_mod_name(&build_env) - .map_err(|e| ErrorString(e.to_string()))?; - let mut lexerdef_func_impl = code_gen.gen_lex_flags_decl(&build_env); - lexerdef_func_impl.append_all(code_gen.gen_start_states_val(&build_env)); - lexerdef_func_impl.append_all(code_gen.gen_rules_val(&build_env)); - // Code gen for the lexerdef() return value referencing variables bound earlier. - lexerdef_func_impl.append_all(code_gen.gen_instantiate_lexerdef(&build_env)); - let lexerdef_ty = code_gen.gen_lexerdef_ty(&build_env); - let token_consts = code_gen.gen_token_consts(); - let token_consts = token_consts.into_iter(); - let out_tokens = { - let lexerdef_param = str::parse::(type_name::()).unwrap(); - let mod_vis = self.visibility; - // Code gen for the generated module. - quote! { - #mod_vis mod #mod_name { - use ::lrlex::{LexerDef, Rule, StartState}; - #[allow(dead_code)] - pub fn lexerdef() -> #lexerdef_ty<#lexerdef_param> { - #lexerdef_func_impl - } - #(#token_consts)* - } - } - }; // Try and run a code formatter on the generated code. - let unformatted = out_tokens.to_string(); + let unformatted = code_gen + .generate_lex_module(&build_env) + .map_err(|e| ErrorString(e.to_string()))? + .to_string(); let mut outs = String::new(); // Record the time that this version of lrlex was built. If the source code changes and rustc // forces a recompile, this will change this value, causing anything which depends on this From c2960af20f298d7f5b4108fd82a7890f5ee2a426 Mon Sep 17 00:00:00 2001 From: matt rice Date: Wed, 19 Aug 2026 06:24:48 -0700 Subject: [PATCH 7/7] Move code string output generation to module --- lrlex/src/lib/codegen.rs | 28 ++++++++++++++++++++++++++-- lrlex/src/lib/ctbuilder.rs | 20 +++----------------- 2 files changed, 29 insertions(+), 19 deletions(-) diff --git a/lrlex/src/lib/codegen.rs b/lrlex/src/lib/codegen.rs index 130620996..570745345 100644 --- a/lrlex/src/lib/codegen.rs +++ b/lrlex/src/lib/codegen.rs @@ -10,7 +10,12 @@ use proc_macro2::{Ident, TokenStream}; use quote::{ToTokens, TokenStreamExt, format_ident, quote}; use regex::Regex; use std::{ - any::type_name, collections::HashMap, fmt, marker::PhantomData, path::Path, sync::LazyLock, + any::type_name, + collections::HashMap, + fmt::{self, Write as _}, + marker::PhantomData, + path::Path, + sync::LazyLock, }; static RE_TOKEN_ID: LazyLock = @@ -119,7 +124,6 @@ where usize: num_traits::AsPrimitive, { rule_ids_map: Option>, - #[expect(unused)] timestamp: String, } @@ -459,6 +463,26 @@ where } }) } + + pub(crate) fn generate( + &self, + build_env: &LexerBuildEnv, + ) -> Result { + // Try and run a code formatter on the generated code. + let unformatted = self.generate_lex_module(build_env)?.to_string(); + let mut outs = String::new(); + // Record the time that this version of lrlex was built. If the source code changes and rustc + // forces a recompile, this will change this value, causing anything which depends on this + // build of lrlex to be recompiled too. + let timestamp = &self.timestamp; + write!(outs, "// lrlex build time: {}\n\n", quote!(#timestamp),).ok(); + outs.push_str( + &syn::parse_str(&unformatted) + .map(|syntax_tree| prettyplease::unparse(&syntax_tree)) + .unwrap_or(unformatted), + ); + Ok(outs) + } } /// The quote impl of `ToTokens` for `Option` prints an empty string for `None` diff --git a/lrlex/src/lib/ctbuilder.rs b/lrlex/src/lib/ctbuilder.rs index 4e81783dc..9040a8638 100644 --- a/lrlex/src/lib/ctbuilder.rs +++ b/lrlex/src/lib/ctbuilder.rs @@ -709,23 +709,9 @@ where fs::remove_file(outp).ok(); panic!(); } - - // Try and run a code formatter on the generated code. - let unformatted = code_gen - .generate_lex_module(&build_env) - .map_err(|e| ErrorString(e.to_string()))? - .to_string(); - let mut outs = String::new(); - // Record the time that this version of lrlex was built. If the source code changes and rustc - // forces a recompile, this will change this value, causing anything which depends on this - // build of lrlex to be recompiled too. - let timestamp = env!("VERGEN_BUILD_TIMESTAMP"); - write!(outs, "// lrlex build time: {}\n\n", quote!(#timestamp),).ok(); - outs.push_str( - &syn::parse_str(&unformatted) - .map(|syntax_tree| prettyplease::unparse(&syntax_tree)) - .unwrap_or(unformatted), - ); + let outs = code_gen + .generate(&build_env) + .map_err(|e| ErrorString(e.to_string()))?; // If the file we're about to write out already exists with the same contents, then we // don't overwrite it (since that will force a recompile of the file, and relinking of the // binary etc).