From e822ba6c0dd4f390beb84ed76fe8fbd009664a1d Mon Sep 17 00:00:00 2001 From: rzvxa Date: Tue, 30 Jul 2024 22:52:49 +0330 Subject: [PATCH] refactor(ast_codegen): abstract passes and generators behind `Runner` trait. --- .../ast_codegen/src/generators/ast_builder.rs | 6 +- tasks/ast_codegen/src/generators/ast_kind.rs | 6 +- .../src/generators/impl_get_span.rs | 6 +- tasks/ast_codegen/src/generators/mod.rs | 24 ++++ tasks/ast_codegen/src/generators/visit.rs | 11 +- tasks/ast_codegen/src/linker.rs | 110 ----------------- tasks/ast_codegen/src/main.rs | 112 ++++++++++++------ tasks/ast_codegen/src/passes/build_schema.rs | 18 +++ tasks/ast_codegen/src/passes/linker.rs | 93 +++++++++++++++ tasks/ast_codegen/src/passes/mod.rs | 66 +++++++++++ tasks/ast_codegen/src/schema.rs | 4 +- 11 files changed, 302 insertions(+), 154 deletions(-) delete mode 100644 tasks/ast_codegen/src/linker.rs create mode 100644 tasks/ast_codegen/src/passes/build_schema.rs create mode 100644 tasks/ast_codegen/src/passes/linker.rs create mode 100644 tasks/ast_codegen/src/passes/mod.rs diff --git a/tasks/ast_codegen/src/generators/ast_builder.rs b/tasks/ast_codegen/src/generators/ast_builder.rs index 5c23e464dda43..0b3e94d8f9535 100644 --- a/tasks/ast_codegen/src/generators/ast_builder.rs +++ b/tasks/ast_codegen/src/generators/ast_builder.rs @@ -20,7 +20,11 @@ use crate::{ CodegenCtx, Generator, GeneratorOutput, TypeRef, }; -pub struct AstBuilderGenerator; +use super::define_generator; + +define_generator! { + pub struct AstBuilderGenerator; +} impl Generator for AstBuilderGenerator { fn name(&self) -> &'static str { diff --git a/tasks/ast_codegen/src/generators/ast_kind.rs b/tasks/ast_codegen/src/generators/ast_kind.rs index dec9fbd2c09fe..bc4c0b807b006 100644 --- a/tasks/ast_codegen/src/generators/ast_kind.rs +++ b/tasks/ast_codegen/src/generators/ast_kind.rs @@ -7,9 +7,11 @@ use crate::{ GeneratorOutput, TypeRef, }; -use super::generated_header; +use super::{define_generator, generated_header}; -pub struct AstKindGenerator; +define_generator! { + pub struct AstKindGenerator; +} pub const BLACK_LIST: [&str; 61] = [ "Expression", diff --git a/tasks/ast_codegen/src/generators/impl_get_span.rs b/tasks/ast_codegen/src/generators/impl_get_span.rs index 275d72dc7f3b3..471f3c49d5179 100644 --- a/tasks/ast_codegen/src/generators/impl_get_span.rs +++ b/tasks/ast_codegen/src/generators/impl_get_span.rs @@ -9,9 +9,11 @@ use crate::{ CodegenCtx, Generator, GeneratorOutput, }; -use super::generated_header; +use super::{define_generator, generated_header}; -pub struct ImplGetSpanGenerator; +define_generator! { + pub struct ImplGetSpanGenerator; +} impl Generator for ImplGetSpanGenerator { fn name(&self) -> &'static str { diff --git a/tasks/ast_codegen/src/generators/mod.rs b/tasks/ast_codegen/src/generators/mod.rs index d11242f37229d..7d747df9bf43f 100644 --- a/tasks/ast_codegen/src/generators/mod.rs +++ b/tasks/ast_codegen/src/generators/mod.rs @@ -43,3 +43,27 @@ pub use ast_builder::AstBuilderGenerator; pub use ast_kind::AstKindGenerator; pub use impl_get_span::ImplGetSpanGenerator; pub use visit::{VisitGenerator, VisitMutGenerator}; + +use crate::{CodegenCtx, GeneratorOutput}; + +pub trait Generator { + fn name(&self) -> &'static str; + fn generate(&mut self, ctx: &CodegenCtx) -> GeneratorOutput; +} + +macro_rules! define_generator { + ($vis:vis struct $ident:ident $($lifetime:lifetime)? $($rest:tt)*) => { + $vis struct $ident $($lifetime)? $($rest)* + impl $($lifetime)? $crate::Runner for $ident $($lifetime)? { + fn name(&self) -> &'static str { + $crate::Generator::name(self) + } + + fn run(&mut self, ctx: &$crate::CodegenCtx) -> $crate::Result<$crate::GeneratorOutput> { + Ok(self.generate(ctx)) + } + } + }; +} + +pub(crate) use define_generator; diff --git a/tasks/ast_codegen/src/generators/visit.rs b/tasks/ast_codegen/src/generators/visit.rs index bc786104c7837..d9ae1ec99f35a 100644 --- a/tasks/ast_codegen/src/generators/visit.rs +++ b/tasks/ast_codegen/src/generators/visit.rs @@ -17,10 +17,15 @@ use crate::{ CodegenCtx, Generator, GeneratorOutput, TypeRef, }; -use super::generated_header; +use super::{define_generator, generated_header}; -pub struct VisitGenerator; -pub struct VisitMutGenerator; +define_generator! { + pub struct VisitGenerator; +} + +define_generator! { + pub struct VisitMutGenerator; +} impl Generator for VisitGenerator { fn name(&self) -> &'static str { diff --git a/tasks/ast_codegen/src/linker.rs b/tasks/ast_codegen/src/linker.rs deleted file mode 100644 index ba713b0b90caf..0000000000000 --- a/tasks/ast_codegen/src/linker.rs +++ /dev/null @@ -1,110 +0,0 @@ -use std::collections::VecDeque; - -use syn::parse_quote; - -use super::{CodegenCtx, Cow, Inherit, Itertools, RType, Result}; - -pub trait Linker: Sized { - fn link(self, linker: impl FnMut(&mut RType, &Self) -> Result) -> Result; -} - -pub trait Unresolved { - fn unresolved(&self) -> bool; - - // TODO: remove me - #[allow(dead_code)] - fn resolved(&self) -> bool { - !self.unresolved() - } -} - -impl Unresolved for Inherit { - fn unresolved(&self) -> bool { - matches!(self, Self::Unlinked(_)) - } -} - -impl Unresolved for Vec { - fn unresolved(&self) -> bool { - self.iter().any(Unresolved::unresolved) - } -} - -impl Linker for CodegenCtx { - fn link(self, mut linker: impl FnMut(&mut RType, &Self) -> Result) -> Result { - // we sort by `TypeId` so we always have the same ordering as how it is written in the rust. - let mut unresolved = self - .ident_table - .iter() - .sorted_by_key(|it| it.1) - .map(|it| it.0) - .collect::>(); - - while let Some(next) = unresolved.pop_back() { - let next_id = *self.type_id(next).unwrap(); - - let val = &mut self.ty_table[next_id].borrow_mut(); - - if !linker(val, &self)? { - // for now we don't have entangled dependencies so we just add unresolved item back - // to the list so we revisit it again at the end. - unresolved.push_front(next); - } - } - Ok(self) - } -} - -/// Returns false if can't resolve at the moment -/// # Panics -/// On invalid inheritance. -#[allow(clippy::unnecessary_wraps)] -pub fn linker(ty: &mut RType, ctx: &CodegenCtx) -> Result { - // Exit early if it isn't an enum, We only link to resolve enum inheritance! - let RType::Enum(ty) = ty else { - return Ok(true); - }; - - // Exit early if there is this enum doesn't use enum inheritance - if ty.meta.inherits.is_empty() { - return Ok(true); - } - - let inherits = ty - .meta - .inherits - .drain(..) - .map(|it| match it { - Inherit::Unlinked(ref sup) => { - let linkee = ctx.find(&Cow::Owned(sup.to_string())).unwrap(); - let linkee = linkee.borrow(); - let inherit_value = format!(r#""{}""#, linkee.ident().unwrap()); - let variants = match &*linkee { - RType::Enum(enum_) => { - if enum_.meta.inherits.unresolved() { - return Err(it); - } - enum_.item.variants.clone().into_iter().map(|mut v| { - v.attrs = vec![parse_quote!(#[inherit = #inherit_value])]; - v - }) - } - _ => { - panic!("invalid inheritance, you can only inherit from enums and in enums.") - } - }; - ty.item.variants.extend(variants.clone()); - Ok(Inherit::Linked { - super_: linkee.as_type().unwrap(), - variants: variants.collect(), - }) - } - Inherit::Linked { .. } => Ok(it), - }) - .collect::>>(); - let unresolved = inherits.iter().any(std::result::Result::is_err); - - ty.meta.inherits = inherits.into_iter().map(|it| it.unwrap_or_else(|it| it)).collect(); - - Ok(!unresolved) -} diff --git a/tasks/ast_codegen/src/main.rs b/tasks/ast_codegen/src/main.rs index 871c117931352..2dbee92bc8644 100644 --- a/tasks/ast_codegen/src/main.rs +++ b/tasks/ast_codegen/src/main.rs @@ -3,23 +3,25 @@ const AST_CRATE: &str = "crates/oxc_ast"; mod defs; mod fmt; mod generators; -mod linker; mod markers; +mod passes; mod schema; mod util; -use std::{borrow::Cow, cell::RefCell, collections::HashMap, io::Read, path::PathBuf, rc::Rc}; +use std::{cell::RefCell, collections::HashMap, io::Read, path::PathBuf, rc::Rc}; use bpaf::{Bpaf, Parser}; use fmt::{cargo_fmt, pprint}; use itertools::Itertools; +use passes::{BuildSchema, Linker, Pass}; use proc_macro2::TokenStream; use syn::parse_file; use defs::TypeDef; -use generators::{AstBuilderGenerator, AstKindGenerator, VisitGenerator, VisitMutGenerator}; -use linker::{linker, Linker}; -use schema::{Inherit, Module, REnum, RStruct, RType, Schema}; +use generators::{ + AstBuilderGenerator, AstKindGenerator, Generator, VisitGenerator, VisitMutGenerator, +}; +use schema::{Module, REnum, RStruct, RType, Schema}; use util::{write_all_to, NormalizeError}; use crate::generators::ImplGetSpanGenerator; @@ -34,22 +36,19 @@ type TypeRef = Rc>; #[derive(Default)] struct AstCodegen { files: Vec, - generators: Vec>, -} - -trait Generator { - fn name(&self) -> &'static str; - fn generate(&mut self, ctx: &CodegenCtx) -> GeneratorOutput; + runners: Vec>, } type GeneratedStream = (/* output path */ PathBuf, TokenStream); +type DataStream = (/* output path */ PathBuf, Vec); // TODO: remove me #[allow(dead_code)] #[derive(Debug, Clone)] enum GeneratorOutput { None, - Info(String), + Info(Vec), + Data(DataStream), Stream(GeneratedStream), } @@ -64,7 +63,7 @@ impl GeneratorOutput { assert!(self.is_none()); } - pub fn to_info(&self) -> &String { + pub fn to_info(&self) -> &[u8] { if let Self::Info(it) = self { it } else { @@ -72,6 +71,14 @@ impl GeneratorOutput { } } + pub fn to_data(&self) -> &DataStream { + if let Self::Data(it) = self { + it + } else { + panic!(); + } + } + pub fn to_stream(&self) -> &GeneratedStream { if let Self::Stream(it) = self { it @@ -80,7 +87,7 @@ impl GeneratorOutput { } } - pub fn into_info(self) -> String { + pub fn into_info(self) -> Vec { if let Self::Info(it) = self { it } else { @@ -88,6 +95,14 @@ impl GeneratorOutput { } } + pub fn into_data(self) -> DataStream { + if let Self::Data(it) = self { + it + } else { + panic!(); + } + } + pub fn into_stream(self) -> GeneratedStream { if let Self::Stream(it) = self { it @@ -100,7 +115,8 @@ impl GeneratorOutput { struct CodegenCtx { ty_table: TypeTable, ident_table: IdentTable, - schema: Schema, + schema: RefCell, + mods: RefCell>, } struct CodegenResult { @@ -109,7 +125,7 @@ struct CodegenResult { } impl CodegenCtx { - fn new(mods: Vec) -> Result { + fn new(mods: Vec) -> Self { // worst case len let len = mods.iter().fold(0, |acc, it| acc + it.items.len()); let defs = mods.iter().flat_map(|it| it.items.iter()); @@ -125,11 +141,12 @@ impl CodegenCtx { } } - let mut me = Self { ty_table, ident_table, schema: Schema::default() }.link(linker)?; - for m in mods { - m.build_in(&mut me.schema)?; + Self { + ty_table, + ident_table, + schema: RefCell::new(Schema::default()), + mods: RefCell::new(mods), } - Ok(me) } fn find(&self, key: &TypeName) -> Option { @@ -141,6 +158,11 @@ impl CodegenCtx { } } +trait Runner { + fn name(&self) -> &'static str; + fn run(&mut self, ctx: &CodegenCtx) -> Result; +} + impl AstCodegen { #[must_use] fn add_file

(mut self, path: P) -> Self @@ -152,11 +174,20 @@ impl AstCodegen { } #[must_use] - fn with(mut self, generator: G) -> Self + fn pass

(mut self, pass: P) -> Self where - G: Generator + 'static, + P: Pass + Runner + 'static, { - self.generators.push(Box::new(generator)); + self.runners.push(Box::new(pass)); + self + } + + #[must_use] + fn gen(mut self, generator: G) -> Self + where + G: Generator + Runner + 'static, + { + self.runners.push(Box::new(generator)); self } @@ -171,15 +202,15 @@ impl AstCodegen { .map_ok(Module::analyze) .collect::>>>()??; - let ctx = CodegenCtx::new(modules)?; + let ctx = CodegenCtx::new(modules); let outputs = self - .generators + .runners .into_iter() - .map(|mut gen| (gen.name(), gen.generate(&ctx))) - .collect_vec(); + .map(|mut runner| runner.run(&ctx).map(|res| (runner.name(), res))) + .collect::>>()?; - Ok(CodegenResult { outputs, schema: ctx.schema }) + Ok(CodegenResult { outputs, schema: ctx.schema.into_inner() }) } } @@ -201,6 +232,13 @@ fn write_generated_streams( Ok(()) } +fn write_data_streams(streams: impl IntoIterator) -> std::io::Result<()> { + for (path, content) in streams { + write_all_to(&content, path.into_os_string().to_str().unwrap())?; + } + Ok(()) +} + #[derive(Debug, Bpaf)] pub struct CliOptions { /// Runs all generators but won't write anything down. @@ -219,18 +257,24 @@ fn main() -> std::result::Result<(), Box> { let CodegenResult { outputs, schema } = files() .fold(AstCodegen::default(), AstCodegen::add_file) - .with(AstKindGenerator) - .with(AstBuilderGenerator) - .with(ImplGetSpanGenerator) - .with(VisitGenerator) - .with(VisitMutGenerator) + .pass(Linker) + .pass(BuildSchema) + .gen(AstKindGenerator) + .gen(AstBuilderGenerator) + .gen(ImplGetSpanGenerator) + .gen(VisitGenerator) + .gen(VisitMutGenerator) .generate()?; - let (streams, _): (Vec<_>, Vec<_>) = + let (streams, outputs): (Vec<_>, Vec<_>) = outputs.into_iter().partition(|it| matches!(it.1, GeneratorOutput::Stream(_))); + let (binaries, _): (Vec<_>, Vec<_>) = + outputs.into_iter().partition(|it| matches!(it.1, GeneratorOutput::Data(_))); + if !cli_options.dry_run { write_generated_streams(streams.into_iter().map(|it| it.1.into_stream()))?; + write_data_streams(binaries.into_iter().map(|it| it.1.into_data()))?; } if !cli_options.no_fmt { diff --git a/tasks/ast_codegen/src/passes/build_schema.rs b/tasks/ast_codegen/src/passes/build_schema.rs new file mode 100644 index 0000000000000..0b62e61d0e383 --- /dev/null +++ b/tasks/ast_codegen/src/passes/build_schema.rs @@ -0,0 +1,18 @@ +use super::{define_pass, Pass}; + +define_pass! { + pub struct BuildSchema; +} + +impl Pass for BuildSchema { + fn name(&self) -> &'static str { + stringify!(BuildSchema) + } + + fn once(&mut self, ctx: &crate::CodegenCtx) -> crate::Result { + for m in ctx.mods.borrow_mut().iter_mut() { + m.build_in(&mut ctx.schema.borrow_mut())?; + } + Ok(true) + } +} diff --git a/tasks/ast_codegen/src/passes/linker.rs b/tasks/ast_codegen/src/passes/linker.rs new file mode 100644 index 0000000000000..480bb8edc0943 --- /dev/null +++ b/tasks/ast_codegen/src/passes/linker.rs @@ -0,0 +1,93 @@ +use std::borrow::Cow; + +use syn::parse_quote; + +use crate::{schema::Inherit, util::NormalizeError}; + +use super::{define_pass, CodegenCtx, Pass, RType, Result}; + +pub trait Unresolved { + fn unresolved(&self) -> bool; + + // TODO: remove me + #[allow(dead_code)] + fn resolved(&self) -> bool { + !self.unresolved() + } +} + +impl Unresolved for Inherit { + fn unresolved(&self) -> bool { + matches!(self, Self::Unlinked(_)) + } +} + +impl Unresolved for Vec { + fn unresolved(&self) -> bool { + self.iter().any(Unresolved::unresolved) + } +} + +define_pass! { + pub struct Linker; +} + +impl Pass for Linker { + fn name(&self) -> &'static str { + stringify!(Linker) + } + + /// # Panics + /// On invalid inheritance. + fn each(&mut self, ty: &mut RType, ctx: &CodegenCtx) -> crate::Result { + // Exit early if it isn't an enum, We only link to resolve enum inheritance! + let RType::Enum(ty) = ty else { + return Ok(true); + }; + + // Exit early if there is this enum doesn't use enum inheritance + if ty.meta.inherits.is_empty() { + return Ok(true); + } + + let inherits = ty + .meta + .inherits + .drain(..) + .map(|it| match it { + Inherit::Unlinked(ref sup) => { + let linkee = ctx + .find(&Cow::Owned(sup.to_string())) + .normalize_with(format!("Unknown type {sup:?}"))?; + let linkee = linkee.borrow(); + let inherit_value = format!(r#""{}""#, linkee.ident().unwrap()); + let variants = match &*linkee { + RType::Enum(enum_) => { + if enum_.meta.inherits.unresolved() { + return Ok(Err(it)); + } + enum_.item.variants.clone().into_iter().map(|mut v| { + v.attrs = vec![parse_quote!(#[inherit = #inherit_value])]; + v + }) + } + _ => { + panic!("invalid inheritance, you can only inherit from enums and in enums.") + } + }; + ty.item.variants.extend(variants.clone()); + Ok(Ok(Inherit::Linked { + super_: linkee.as_type().unwrap(), + variants: variants.collect(), + })) + } + Inherit::Linked { .. } => Ok(Ok(it)), + }) + .collect::>>>()?; + let unresolved = inherits.iter().any(std::result::Result::is_err); + + ty.meta.inherits = inherits.into_iter().map(|it| it.unwrap_or_else(|it| it)).collect(); + + Ok(!unresolved) + } +} diff --git a/tasks/ast_codegen/src/passes/mod.rs b/tasks/ast_codegen/src/passes/mod.rs new file mode 100644 index 0000000000000..3d56fc7153dc9 --- /dev/null +++ b/tasks/ast_codegen/src/passes/mod.rs @@ -0,0 +1,66 @@ +mod build_schema; +mod linker; + +use std::collections::VecDeque; + +use itertools::Itertools; + +use crate::{schema::RType, CodegenCtx, Result}; + +pub use build_schema::BuildSchema; +pub use linker::Linker; + +pub trait Pass { + fn name(&self) -> &'static str; + + /// Returns false if can't resolve + fn once(&mut self, _ctx: &CodegenCtx) -> Result { + Ok(true) + } + + /// Returns false if can't resolve + fn each(&mut self, _ty: &mut RType, _ctx: &CodegenCtx) -> Result { + Ok(true) + } + + fn call(&mut self, ctx: &CodegenCtx) -> Result { + // call once + if !self.once(ctx)? { + return Ok(false); + } + + // call each + // we sort by `TypeId` so we always have the same ordering as how it is written in the rust. + let mut unresolved = + ctx.ident_table.iter().sorted_by_key(|it| it.1).map(|it| it.0).collect::>(); + + while let Some(next) = unresolved.pop_back() { + let next_id = *ctx.type_id(next).unwrap(); + + let val = &mut ctx.ty_table[next_id].borrow_mut(); + + if !self.each(val, ctx)? { + unresolved.push_front(next); + } + } + Ok(unresolved.is_empty()) + } +} + +macro_rules! define_pass { + ($vis:vis struct $ident:ident $($lifetime:lifetime)? $($rest:tt)*) => { + $vis struct $ident $($lifetime)? $($rest)* + impl $($lifetime)? $crate::Runner for $ident $($lifetime)? { + fn name(&self) -> &'static str { + $crate::Pass::name(self) + } + + fn run(&mut self, ctx: &$crate::CodegenCtx) -> $crate::Result<$crate::GeneratorOutput> { + self.call(ctx)?; + Ok($crate::GeneratorOutput::None) + } + } + }; +} + +pub(crate) use define_pass; diff --git a/tasks/ast_codegen/src/schema.rs b/tasks/ast_codegen/src/schema.rs index abf9764f14e45..7829891c87fec 100644 --- a/tasks/ast_codegen/src/schema.rs +++ b/tasks/ast_codegen/src/schema.rs @@ -259,12 +259,12 @@ impl Module { Ok(self) } - pub fn build_in(self, schema: &mut Schema) -> Result<()> { + pub fn build_in(&self, schema: &mut Schema) -> Result<()> { if !self.loaded { return Err(String::from(LOAD_ERROR)); } - schema.definitions.extend(self.items.into_iter().filter_map(|it| (&*it.borrow()).into())); + schema.definitions.extend(self.items.iter().filter_map(|it| (&*it.borrow()).into())); Ok(()) } }