feat(ast_tools): generate list of helper attributes in oxc_ast_macros crate (#8852)

Custom attributes used by generators (e.g. `#[visit]`) need to be added to the dummy `Ast` derive macro in `oxc_ast_macros`. Do this automatically, rather than requiring the person writing the generator to remember to do it.

Author: overlookmotel

.github/.generated_ast_watch_list.yml

@@ -20,6 +20,7 @@ src:
   - 'crates/oxc_ast/src/generated/get_id.rs'
   - 'crates/oxc_ast/src/generated/visit.rs'
   - 'crates/oxc_ast/src/generated/visit_mut.rs'
+  - 'crates/oxc_ast_macros/src/lib.rs'
   - 'crates/oxc_regular_expression/src/ast.rs'
   - 'crates/oxc_regular_expression/src/generated/derive_clone_in.rs'
   - 'crates/oxc_regular_expression/src/generated/derive_content_eq.rs'

crates/oxc_ast_macros/src/lib.rs

@@ -84,7 +84,7 @@ pub fn ast(_args: TokenStream, input: TokenStream) -> TokenStream {
 /// Its only purpose is to allow the occurrence of helper attributes used in `tasks/ast_tools`.
 ///
 /// Read [`macro@ast`] for further details.
-#[proc_macro_derive(Ast, attributes(scope, visit, span, generate_derive, clone_in, estree, ts))]
+#[proc_macro_derive(Ast, attributes(clone_in, estree, generate_derive, scope, span, ts, visit))]
 pub fn ast_derive(_input: TokenStream) -> TokenStream {
     TokenStream::new()
 }

tasks/ast_tools/src/codegen.rs

@@ -73,6 +73,11 @@ impl Codegen {
     pub fn attr_processor(&self, attr_name: &str) -> Option<(AttrProcessor, AttrPositions)> {
         self.attr_processors.get(attr_name).copied()
     }
+
+    /// Get all attributes which derives and generators handle.
+    pub fn attrs(&self) -> Vec<&'static str> {
+        self.attr_processors.keys().copied().collect()
+    }
 }
 
 /// Runner trait.

tasks/ast_tools/src/main.rs

@@ -147,7 +147,6 @@
 //! * Implement [`Generator::attrs`] / [`Derive::attrs`] to declare the generator's custom attributes.
 //! * Implement [`Generator::parse_attr`] / [`Derive::parse_attr`] to parse those attributes
 //!   and mutate the "extension" types in [`Schema`] as required.
-//! * Add the attributes' names to the list on `ast_derive` in `crates/oxc_ast_macros/src/lib.rs`.
 //!
 //! #### Attributes
 //!
@@ -170,7 +169,7 @@
 //! [`AttrLocation`]: parse::attr::AttrLocation
 //! [`AttrPart`]: parse::attr::AttrPart
 
-use std::fmt::Write;
+use std::{fmt::Write, fs};
 
 use bpaf::{Bpaf, Parser};
 use rayon::prelude::*;
@@ -209,6 +208,9 @@ static SOURCE_PATHS: &[&str] = &[
 /// Path to `oxc_ast` crate
 const AST_CRATE: &str = "crates/oxc_ast";
 
+/// Path to `oxc_ast_macros` crate's `lib.rs` file
+const AST_MACROS_LIB_PATH: &str = "crates/oxc_ast_macros/src/lib.rs";
+
 /// Path to write TS type definitions to
 const TYPESCRIPT_DEFINITIONS_PATH: &str = "npm/oxc-types/types.d.ts";
 
@@ -288,6 +290,9 @@ fn main() {
 
     logln!("All Derives and Generators... Done!");
 
+    // Edit `lib.rs` in `oxc_ast_macros` crate
+    outputs.push(generate_updated_proc_macro(&codegen));
+
     // Add CI filter file to outputs
     outputs.sort_unstable_by(|o1, o2| o1.path.cmp(&o2.path));
     outputs.push(generate_ci_filter(&outputs));
@@ -326,3 +331,26 @@ fn generate_ci_filter(outputs: &[RawOutput]) -> RawOutput {
 
     Output::Yaml { path: GITHUB_WATCH_LIST_PATH.to_string(), code }.into_raw(file!())
 }
+
+/// Update the list of helper attributes for `Ast` derive proc macro in `oxc_ast_macros` crate
+/// to include all attrs which generators/derives utilize.
+///
+/// Unfortunately we can't add a separate generated file for this, as proc macros can only be declared
+/// in the main `lib.rs` of a proc macro crate. So we have to edit the existing file.
+fn generate_updated_proc_macro(codegen: &Codegen) -> RawOutput {
+    // Get all attrs which derives/generators use
+    let mut attrs = codegen.attrs();
+    attrs.push("generate_derive");
+    attrs.sort_unstable();
+    let attrs = attrs.join(", ");
+
+    // Load `oxc_ast_macros` crate's `lib.rs` file.
+    // Substitute list of used attrs into `#[proc_macro_derive(Ast, attributes(...))]`.
+    let code = fs::read_to_string(AST_MACROS_LIB_PATH).unwrap();
+    let (start, end) = code.split_once("#[proc_macro_derive(").unwrap();
+    let (_, end) = end.split_once(")]").unwrap();
+    assert!(end.starts_with("\npub fn ast_derive("));
+    let code = format!("{start}#[proc_macro_derive(Ast, attributes({attrs}))]{end}");
+
+    Output::RustString { path: AST_MACROS_LIB_PATH.to_string(), code }.into_raw("")
+}

tasks/ast_tools/src/output/mod.rs

@@ -9,7 +9,7 @@ mod javascript;
 mod rust;
 mod yaml;
 use javascript::print_javascript;
-use rust::print_rust;
+use rust::{print_rust, rust_fmt};
 use yaml::print_yaml;
 
 /// Get path for an output.
@@ -32,6 +32,7 @@ fn add_header(code: &str, generator_path: &str, comment_start: &str) -> String {
 #[expect(dead_code)]
 pub enum Output {
     Rust { path: String, tokens: TokenStream },
+    RustString { path: String, code: String },
     Javascript { path: String, code: String },
     Yaml { path: String, code: String },
     Raw { path: String, code: String },
@@ -49,6 +50,10 @@ impl Output {
                 let code = print_rust(tokens, &generator_path);
                 (path, code)
             }
+            Self::RustString { path, code } => {
+                let code = rust_fmt(&code);
+                (path, code)
+            }
             Self::Javascript { path, code } => {
                 let code = print_javascript(&code, &generator_path);
                 (path, code)

tasks/ast_tools/src/output/rust.rs

@@ -21,7 +21,7 @@ pub fn print_rust(tokens: TokenStream, generator_path: &str) -> String {
 /// Format Rust code with `rustfmt`.
 ///
 /// Does not format on disk - interfaces with `rustfmt` via stdin/stdout.
-fn rust_fmt(source_text: &str) -> String {
+pub fn rust_fmt(source_text: &str) -> String {
     let mut rustfmt = Command::new("rustfmt")
         .stdin(Stdio::piped())
         .stdout(Stdio::piped())