feat(ast_tools): add dedicated Derive trait. (#5278)

In an effort toward the implementation of #5256, this PR allows us to have a separately generated "derive" file for each crate.
This also eliminates a bunch of boilerplate when writing new "derive" generators and generally makes it more approachable.

.github/.generated_ast_watch_list.yml

@@ -13,9 +13,9 @@ src:
   - 'crates/oxc_ast/src/generated/assert_layouts.rs'
   - 'crates/oxc_ast/src/generated/ast_kind.rs'
   - 'crates/oxc_ast/src/generated/ast_builder.rs'
+  - 'crates/oxc_ast/src/generated/visit.rs'
+  - 'crates/oxc_ast/src/generated/visit_mut.rs'
   - 'crates/oxc_ast/src/generated/derive_clone_in.rs'
   - 'crates/oxc_ast/src/generated/derive_get_span.rs'
   - 'crates/oxc_ast/src/generated/derive_get_span_mut.rs'
-  - 'crates/oxc_ast/src/generated/visit.rs'
-  - 'crates/oxc_ast/src/generated/visit_mut.rs'
   - 'tasks/ast_codegen/src/**'

crates/oxc_ast/src/ast/mod.rs

@@ -175,11 +175,11 @@
 //!
 //! If you are seeing compile-time errors in `src/ast/macros.rs`, this will be the cause.
 
-mod js;
-mod jsx;
-mod literal;
-mod macros;
-mod ts;
+pub(crate) mod js;
+pub(crate) mod jsx;
+pub(crate) mod literal;
+pub(crate) mod macros;
+pub(crate) mod ts;
 
 use macros::inherit_variants;
 

crates/oxc_ast/src/generated/derive_clone_in.rs

@@ -1,12 +1,21 @@
 // Auto-generated code, DO NOT EDIT DIRECTLY!
-// To edit this generated file you have to edit `tasks/ast_tools/src/generators/derive_clone_in.rs`
+// To edit this generated file you have to edit `tasks/ast_tools/src/derives/clone_in.rs`
 
 #![allow(clippy::default_trait_access)]
 
 use oxc_allocator::{Allocator, CloneIn};
 
 #[allow(clippy::wildcard_imports)]
-use crate::ast::*;
+use crate::ast::js::*;
+
+#[allow(clippy::wildcard_imports)]
+use crate::ast::jsx::*;
+
+#[allow(clippy::wildcard_imports)]
+use crate::ast::literal::*;
+
+#[allow(clippy::wildcard_imports)]
+use crate::ast::ts::*;
 
 impl<'alloc> CloneIn<'alloc> for BooleanLiteral {
     type Cloned = BooleanLiteral;

crates/oxc_ast/src/generated/derive_get_span.rs

@@ -1,12 +1,21 @@
 // Auto-generated code, DO NOT EDIT DIRECTLY!
-// To edit this generated file you have to edit `tasks/ast_tools/src/generators/derive_get_span.rs`
+// To edit this generated file you have to edit `tasks/ast_tools/src/derives/get_span.rs`
 
 #![allow(clippy::match_same_arms)]
 
-use oxc_span::GetSpan;
+use oxc_span::{GetSpan, Span};
 
 #[allow(clippy::wildcard_imports)]
-use crate::ast::*;
+use crate::ast::js::*;
+
+#[allow(clippy::wildcard_imports)]
+use crate::ast::jsx::*;
+
+#[allow(clippy::wildcard_imports)]
+use crate::ast::literal::*;
+
+#[allow(clippy::wildcard_imports)]
+use crate::ast::ts::*;
 
 impl GetSpan for BooleanLiteral {
     #[inline]

crates/oxc_ast/src/generated/derive_get_span_mut.rs

@@ -1,12 +1,21 @@
 // Auto-generated code, DO NOT EDIT DIRECTLY!
-// To edit this generated file you have to edit `tasks/ast_tools/src/generators/derive_get_span.rs`
+// To edit this generated file you have to edit `tasks/ast_tools/src/derives/get_span.rs`
 
 #![allow(clippy::match_same_arms)]
 
-use oxc_span::GetSpanMut;
+use oxc_span::{GetSpanMut, Span};
 
 #[allow(clippy::wildcard_imports)]
-use crate::ast::*;
+use crate::ast::js::*;
+
+#[allow(clippy::wildcard_imports)]
+use crate::ast::jsx::*;
+
+#[allow(clippy::wildcard_imports)]
+use crate::ast::literal::*;
+
+#[allow(clippy::wildcard_imports)]
+use crate::ast::ts::*;
 
 impl GetSpanMut for BooleanLiteral {
     #[inline]

tasks/ast_tools/src/codegen.rs

@@ -1,12 +1,16 @@
 use std::{cell::RefCell, collections::HashMap, path::PathBuf};
 
 use itertools::Itertools;
+use proc_macro2::TokenStream;
 
 use crate::{
+    derives::{Derive, DeriveOutput},
+    fmt::pretty_print,
     generators::{Generator, GeneratorOutput},
     passes::Pass,
     rust_ast::{self, AstRef},
     schema::{lower_ast_types, Schema, TypeDef},
+    util::write_all_to,
     Result, TypeId,
 };
 
@@ -15,16 +19,51 @@ pub struct AstCodegen {
     files: Vec<PathBuf>,
     passes: Vec<Box<dyn Runner<Output = (), Context = EarlyCtx>>>,
     generators: Vec<Box<dyn Runner<Output = GeneratorOutput, Context = LateCtx>>>,
+    derives: Vec<Box<dyn Runner<Output = DeriveOutput, Context = LateCtx>>>,
 }
 
 pub struct AstCodegenResult {
     pub schema: Schema,
-    pub outputs: Vec<(/* generator name */ &'static str, /* output */ GeneratorOutput)>,
+    pub outputs: Vec<SideEffect>,
+}
+
+pub struct SideEffect(/* path */ pub PathBuf, /* output */ pub Vec<u8>);
+
+impl SideEffect {
+    /// Apply the side-effect
+    pub fn apply(self) -> std::io::Result<()> {
+        let Self(path, data) = self;
+        let path = path.into_os_string();
+        let path = path.to_str().unwrap();
+        write_all_to(&data, path)?;
+        Ok(())
+    }
+
+    #[allow(clippy::unnecessary_wraps)]
+    pub fn path(&self) -> Option<String> {
+        let Self(path, _) = self;
+        let path = path.to_string_lossy();
+        Some(path.replace('\\', "/"))
+    }
+}
+
+impl From<(PathBuf, TokenStream)> for SideEffect {
+    fn from((path, stream): (PathBuf, TokenStream)) -> Self {
+        let content = pretty_print(&stream);
+        Self(path, content.as_bytes().into())
+    }
+}
+
+impl From<GeneratorOutput> for SideEffect {
+    fn from(output: GeneratorOutput) -> Self {
+        Self::from((output.0, output.1))
+    }
 }
 
 pub trait Runner {
     type Context;
     type Output;
+    #[allow(dead_code)]
     fn name(&self) -> &'static str;
     fn run(&mut self, ctx: &Self::Context) -> Result<Self::Output>;
 }
@@ -116,15 +155,24 @@ impl AstCodegen {
     }
 
     #[must_use]
-    pub fn gen<G>(mut self, generator: G) -> Self
+    pub fn generate<G>(mut self, generator: G) -> Self
     where
         G: Generator + Runner<Output = GeneratorOutput, Context = LateCtx> + 'static,
     {
         self.generators.push(Box::new(generator));
         self
     }
 
-    pub fn generate(self) -> Result<AstCodegenResult> {
+    #[must_use]
+    pub fn derive<D>(mut self, derive: D) -> Self
+    where
+        D: Derive + Runner<Output = DeriveOutput, Context = LateCtx> + 'static,
+    {
+        self.derives.push(Box::new(derive));
+        self
+    }
+
+    pub fn run(self) -> Result<AstCodegenResult> {
         let modules = self
             .files
             .into_iter()
@@ -140,17 +188,42 @@ impl AstCodegen {
             _ = self
                 .passes
                 .into_iter()
-                .map(|mut runner| runner.run(&ctx).map(|res| (runner.name(), res)))
+                .map(|mut runner| runner.run(&ctx))
                 .collect::<Result<Vec<_>>>()?;
             ctx.into_late_ctx()
         };
 
+        let derives = self
+            .derives
+            .into_iter()
+            .map(|mut runner| runner.run(&ctx))
+            .map_ok(|output| output.0.into_iter().map(SideEffect::from))
+            .flatten_ok();
+
         let outputs = self
             .generators
             .into_iter()
-            .map(|mut runner| runner.run(&ctx).map(|res| (runner.name(), res)))
+            .map(|mut runner| runner.run(&ctx))
+            .map_ok(SideEffect::from)
+            .chain(derives)
             .collect::<Result<Vec<_>>>()?;
 
         Ok(AstCodegenResult { outputs, schema: ctx.schema })
     }
 }
+
+/// Creates a generated file warning + required information for a generated file.
+macro_rules! generated_header {
+    () => {{
+        let file = file!().replace("\\", "/");
+        // TODO add generation date, AST source hash, etc here.
+        let edit_comment = format!("@ To edit this generated file you have to edit `{file}`");
+        quote::quote! {
+            //!@ Auto-generated code, DO NOT EDIT DIRECTLY!
+            #![doc = #edit_comment]
+            //!@@line_break
+        }
+    }};
+}
+
+pub(crate) use generated_header;

tasks/ast_tools/src/generators/derive_clone_in.rs → tasks/ast_tools/src/derives/clone_in.rs

@@ -6,49 +6,34 @@ use syn::Ident;
 use crate::{
     codegen::LateCtx,
     markers::CloneInAttribute,
-    output,
     schema::{EnumDef, GetIdent, StructDef, TypeDef},
-    GeneratorOutput,
 };
 
-use super::{define_generator, generated_header, Generator};
+use super::{define_derive, Derive, DeriveOutput};
 
-define_generator! {
+define_derive! {
     pub struct DeriveCloneIn;
 }
 
-impl Generator for DeriveCloneIn {
-    fn generate(&mut self, ctx: &LateCtx) -> GeneratorOutput {
-        let impls: Vec<TokenStream> = ctx
-            .schema()
-            .into_iter()
-            .filter(|def| def.generates_derive("CloneIn"))
-            .map(|def| match &def {
-                TypeDef::Enum(it) => derive_enum(it),
-                TypeDef::Struct(it) => derive_struct(it),
-            })
-            .collect();
-
-        let header = generated_header!();
-
-        GeneratorOutput::Stream((
-            output(crate::AST_CRATE, "derive_clone_in.rs"),
-            quote! {
-                #header
-
-                #![allow(clippy::default_trait_access)]
+impl Derive for DeriveCloneIn {
+    fn trait_name() -> &'static str {
+        "CloneIn"
+    }
 
-                ///@@line_break
-                use oxc_allocator::{Allocator, CloneIn};
+    fn derive(&mut self, def: &TypeDef, _: &LateCtx) -> TokenStream {
+        match &def {
+            TypeDef::Enum(it) => derive_enum(it),
+            TypeDef::Struct(it) => derive_struct(it),
+        }
+    }
 
-                ///@@line_break
-                #[allow(clippy::wildcard_imports)]
-                use crate::ast::*;
+    fn prelude() -> TokenStream {
+        quote! {
+            #![allow(clippy::default_trait_access)]
 
-                ///@@line_break
-                #(#impls)*
-            },
-        ))
+            ///@@line_break
+            use oxc_allocator::{Allocator, CloneIn};
+        }
     }
 }
 
@@ -106,7 +91,6 @@ fn impl_clone_in(
 ) -> TokenStream {
     if has_lifetime {
         quote! {
-            ///@@line_break
             impl <'old_alloc, 'new_alloc> CloneIn<'new_alloc> for #ty_ident<'old_alloc> {
                 type Cloned = #ty_ident<'new_alloc>;
                 fn clone_in(&self, #alloc_ident: &'new_alloc Allocator) -> Self::Cloned {
@@ -116,7 +100,6 @@ fn impl_clone_in(
         }
     } else {
         quote! {
-            ///@@line_break
             impl <'alloc> CloneIn<'alloc> for #ty_ident {
                 type Cloned = #ty_ident;
                 fn clone_in(&self, #alloc_ident: &'alloc Allocator) -> Self::Cloned {