diff --git a/Cargo.lock b/Cargo.lock index 5cef54b81..31f782f78 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2233,6 +2233,7 @@ version = "0.3.0" dependencies = [ "anyhow", "fs-err", + "futures", "git2", "grit-util", "homedir", diff --git a/crates/cli/src/commands/patterns_test.rs b/crates/cli/src/commands/patterns_test.rs index 11b8c866e..6e2b41a60 100644 --- a/crates/cli/src/commands/patterns_test.rs +++ b/crates/cli/src/commands/patterns_test.rs @@ -2,6 +2,7 @@ use colored::Colorize; use dashmap::{DashMap, ReadOnlyView}; use log::{debug, info}; +use marzano_core::analysis::get_dependents_of_target_patterns_by_traversal_from_src; use marzano_core::api::MatchResult; use marzano_gritmodule::config::{GritPatternSample, GritPatternTestInfo}; use marzano_gritmodule::formatting::format_rich_files; @@ -30,24 +31,24 @@ use super::patterns::PatternsTestArgs; use anyhow::{anyhow, bail, Context as _, Result}; use std::collections::HashMap; + use std::{path::Path, time::Duration}; -use marzano_core::pattern_compiler::compiler::get_dependents_of_target_patterns_by_traversal_from_src; use marzano_gritmodule::searcher::collect_from_file; use notify::{self, RecursiveMode}; use notify_debouncer_mini::{new_debouncer_opt, Config}; +pub enum AggregatedTestResult { + SomeFailed(String), + AllPassed, +} + pub async fn get_marzano_pattern_test_results( patterns: Vec, libs: &PatternsDirectory, args: &PatternsTestArgs, output: OutputFormat, -) -> Result<()> { - if patterns.is_empty() { - bail!("No testable patterns found. To test a pattern, make sure it is defined in .grit/grit.yaml or a .md file in your .grit/patterns directory."); - } - info!("Found {} testable patterns.", patterns.len()); - +) -> Result { let resolver = GritModuleResolver::new(); let final_results: DashMap> = DashMap::new(); @@ -201,7 +202,7 @@ pub async fn get_marzano_pattern_test_results( if args.update { update_results(&final_results, patterns)?; - return Ok(()); + return Ok(AggregatedTestResult::AllPassed); } let final_results = final_results.into_read_only(); @@ -213,15 +214,15 @@ pub async fn get_marzano_pattern_test_results( .values() .any(|v| v.iter().any(|r| !r.result.is_pass())) { - bail!( + return Ok(AggregatedTestResult::SomeFailed(format!( "{} out of {} samples failed.", final_results .values() .flatten() .filter(|r| !r.result.is_pass()) .count(), - total - ) + total, + ))); }; info!("✓ All {} samples passed.", total); } @@ -254,7 +255,7 @@ pub async fn get_marzano_pattern_test_results( bail!("Output format not supported for this command"); } } - Ok(()) + Ok(AggregatedTestResult::AllPassed) } pub(crate) async fn run_patterns_test( @@ -284,12 +285,138 @@ pub(crate) async fn run_patterns_test( let testable_patterns = collect_testable_patterns(patterns); - get_marzano_pattern_test_results(testable_patterns.clone(), &libs, &arg, flags.clone().into()) - .await?; + if testable_patterns.is_empty() { + bail!("No testable patterns found. To test a pattern, make sure it is defined in .grit/grit.yaml or a .md file in your .grit/patterns directory."); + } + info!("Found {} testable patterns.", testable_patterns.len()); + + let first_result = get_marzano_pattern_test_results( + testable_patterns.clone(), + &libs, + &arg, + flags.clone().into(), + ) + .await?; if arg.watch { + if let AggregatedTestResult::SomeFailed(message) = first_result { + println!("{}", message); + } let _ = enable_watch_mode(testable_patterns, &libs, &arg, flags.into()).await; + Ok(()) + } else { + match first_result { + AggregatedTestResult::SomeFailed(message) => bail!(message), + AggregatedTestResult::AllPassed => Ok(()), + } + } +} + +fn print_watch_start(path: &Path) { + log::info!( + "\nWatching for changes to {}", + format!("{}", path.display()).bold().underline() + ); +} + +async fn test_modified_path( + modified_file_path: &Path, + testable_patterns: &Vec, + testable_patterns_map: &HashMap<&String, &GritPatternTestInfo>, + libs: &PatternsDirectory, + args: &PatternsTestArgs, + output: OutputFormat, +) -> Result<()> { + let ignore_path = [".grit/.gritmodules", ".grit/.gitignore", ".log"]; + + if !modified_file_path.is_file() { + return Ok(()); + } + let modified_file_path = modified_file_path.to_string_lossy().to_string(); + + //temporary fix, until notify crate adds support for ignoring paths + for path in &ignore_path { + if modified_file_path.contains(path) { + return Ok(()); + } + } + let (modified_patterns, deleted_patterns) = + get_modified_and_deleted_patterns(&modified_file_path, testable_patterns).await?; + + if modified_patterns.is_empty() && deleted_patterns.is_empty() { + log::error!( + "{}", + format!( + "\nFile {} was modified, but no changed patterns were found.", + modified_file_path.bold().underline() + ) + .bold() + ); + return Ok(()); + } + + let deleted_patterns_names = deleted_patterns + .iter() + .map(|p| p.local_name.as_ref().unwrap()) + .collect::>(); + + let mut patterns_to_test = modified_patterns.clone(); + let mut patterns_to_test_names = patterns_to_test + .iter() + .map(|p| p.local_name.clone().unwrap()) + .collect::>(); + + if !modified_patterns.is_empty() { + let modified_patterns_dependents_names = + get_dependents_of_target_patterns(libs, testable_patterns, &modified_patterns)?; + for name in &modified_patterns_dependents_names { + if !deleted_patterns_names.contains(&name) && !patterns_to_test_names.contains(name) { + patterns_to_test.push((*testable_patterns_map.get(name).unwrap()).clone()); + patterns_to_test_names.push(name.to_owned()); + } + } + } + + if !deleted_patterns.is_empty() { + let deleted_patterns_dependents_names = + get_dependents_of_target_patterns(libs, testable_patterns, &deleted_patterns)?; + for name in &deleted_patterns_dependents_names { + if !deleted_patterns_names.contains(&name) && !patterns_to_test_names.contains(name) { + patterns_to_test.push((*testable_patterns_map.get(name).unwrap()).clone()); + patterns_to_test_names.push(name.to_owned()); + } + } } + + if patterns_to_test_names.is_empty() { + log::error!( + "{}", + format!( + "\nFile {} was modified, but no testable pattern changes were found.", + modified_file_path.bold().underline() + ) + .bold() + ); + return Ok(()); + } + log::info!( + "{}", + format!( + "\nFile {} was modified, retesting {} patterns:", + modified_file_path.bold().underline(), + patterns_to_test_names.len() + ) + .bold() + ); + + let res = + get_marzano_pattern_test_results(patterns_to_test, libs, args, output.clone()).await?; + match res { + AggregatedTestResult::SomeFailed(message) => { + log::error!("{}", message.to_string().bold().red()); + } + AggregatedTestResult::AllPassed => {} + }; Ok(()) } @@ -300,7 +427,6 @@ async fn enable_watch_mode( output: OutputFormat, ) -> Result<()> { let path = Path::new(".grit"); - let ignore_path = [".grit/.gritmodules", ".grit/.gitignore", ".log"]; // setup debouncer let (tx, rx) = std::sync::mpsc::channel(); // notify backend configuration @@ -313,7 +439,7 @@ async fn enable_watch_mode( let mut debouncer = new_debouncer_opt::<_, notify::PollWatcher>(debouncer_config, tx)?; debouncer.watcher().watch(path, RecursiveMode::Recursive)?; - log::info!("\n[Watch Mode] Enabled on path: {}", path.display()); + print_watch_start(path); let testable_patterns_map = testable_patterns .iter() @@ -326,90 +452,26 @@ async fn enable_watch_mode( Ok(event) => { let modified_file_path = &event.first().unwrap().path; - if !modified_file_path.is_file() { - continue; - } - let modified_file_path = modified_file_path - .clone() - .into_os_string() - .into_string() - .unwrap(); - - //temporary fix, until notify crate adds support for ignoring paths - for path in &ignore_path { - if modified_file_path.contains(path) { - continue; - } - } - log::info!("\n[Watch Mode] File modified: {:?}", modified_file_path); - let (modified_patterns, deleted_patterns) = - get_modified_and_deleted_patterns(&modified_file_path, &testable_patterns) - .await?; - - if modified_patterns.is_empty() && deleted_patterns.is_empty() { - log::info!("[Watch Mode] No patterns changed.\n"); - continue; - } - - let deleted_patterns_names = deleted_patterns - .iter() - .map(|p| p.local_name.as_ref().unwrap()) - .collect::>(); - - let mut patterns_to_test = modified_patterns.clone(); - let mut patterns_to_test_names = patterns_to_test - .iter() - .map(|p| p.local_name.clone().unwrap()) - .collect::>(); - - if !modified_patterns.is_empty() { - let modified_patterns_dependents_names = get_dependents_of_target_patterns( - libs, - &testable_patterns, - &modified_patterns, - )?; - for name in &modified_patterns_dependents_names { - if !deleted_patterns_names.contains(&name) - && !patterns_to_test_names.contains(name) - { - patterns_to_test - .push((*testable_patterns_map.get(name).unwrap()).clone()); - patterns_to_test_names.push(name.to_owned()); - } - } - } - - if !deleted_patterns.is_empty() { - let deleted_patterns_dependents_names = get_dependents_of_target_patterns( - libs, - &testable_patterns, - &deleted_patterns, - )?; - for name in &deleted_patterns_dependents_names { - if !deleted_patterns_names.contains(&name) - && !patterns_to_test_names.contains(name) - { - patterns_to_test - .push((*testable_patterns_map.get(name).unwrap()).clone()); - patterns_to_test_names.push(name.to_owned()); - } + let retest = test_modified_path( + modified_file_path, + &testable_patterns, + &testable_patterns_map, + libs, + args, + output.clone(), + ) + .await; + match retest { + Ok(_) => {} + Err(error) => { + log::error!("Error: {error:?}") } } - log::info!( - "[Watch Mode] Pattern(s) to test: {:?}", - patterns_to_test_names - ); - if patterns_to_test_names.is_empty() { - continue; - } - - let _ = - get_marzano_pattern_test_results(patterns_to_test, libs, args, output.clone()) - .await; + print_watch_start(path); } Err(error) => { - log::error!("[Watch Mode] Error: {error:?}") + log::error!("Error: {error:?}") } } } @@ -454,17 +516,17 @@ fn get_dependents_of_target_patterns( } async fn get_modified_and_deleted_patterns( - modified_path: &String, + modified_path: &str, testable_patterns: &Vec, ) -> Result<(Vec, Vec)> { let path = Path::new(modified_path); let file_patterns = collect_from_file(path, &None).await.unwrap_or(vec![]); let modified_patterns = get_grit_pattern_test_info(file_patterns); + let modified_pattern_names = modified_patterns + .iter() + .map(|p| p.local_name.as_ref().unwrap()) + .collect::>(); - let mut modified_pattern_names = >::new(); - for pattern in &modified_patterns { - modified_pattern_names.push(pattern.local_name.as_ref().unwrap()); - } //modified_patterns = patterns which are updated/edited or newly created. //deleted_patterns = patterns which are deleted. Only remaining dependents of deleted_patterns should gets tested. let mut deleted_patterns = >::new(); diff --git a/crates/cli/src/commands/plumbing.rs b/crates/cli/src/commands/plumbing.rs index ea697a08c..431ef4203 100644 --- a/crates/cli/src/commands/plumbing.rs +++ b/crates/cli/src/commands/plumbing.rs @@ -261,7 +261,7 @@ pub(crate) async fn run_plumbing( let cwd = std::env::current_dir()?; let libs = get_grit_files_from(Some(cwd)).await?; - get_marzano_pattern_test_results( + let res = get_marzano_pattern_test_results( patterns, &libs, &PatternsTestArgs { @@ -273,7 +273,13 @@ pub(crate) async fn run_plumbing( }, parent.into(), ) - .await + .await?; + match res { + super::patterns_test::AggregatedTestResult::SomeFailed(message) => { + Err(anyhow::anyhow!(message)) + } + super::patterns_test::AggregatedTestResult::AllPassed => Ok(()), + } } } } diff --git a/crates/cli_bin/fixtures/.grit/.gitignore b/crates/cli_bin/fixtures/.grit/.gitignore deleted file mode 100644 index e4fdfb17c..000000000 --- a/crates/cli_bin/fixtures/.grit/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -.gritmodules* -*.log diff --git a/crates/cli_bin/fixtures/.grit/grit.yaml b/crates/cli_bin/fixtures/.grit/grit.yaml deleted file mode 100644 index 758e168a6..000000000 --- a/crates/cli_bin/fixtures/.grit/grit.yaml +++ /dev/null @@ -1,81 +0,0 @@ -version: 0.0.1 -patterns: - - name: github.com/getgrit/stdlib#* - - name: our_cargo_use_long_dependency - level: error - body: | - language toml - - cargo_use_long_dependency() where $filename <: not includes or { - "language-submodules", - "language-metavariables" - } - - name: cargo_use_long_dependency - level: error - body: | - language toml - - `[dependencies] - $deps` where { - $filename <: or { includes "Cargo.toml", includes "cargo.toml" }, - $deps <: some bubble `$name = $version` where { - $version <: string(), - $version => `{ version = $version }`, - } - } - - name: no_treesitter_in_grit_crates - description: | - The `grit-pattern-matcher` and `grit-util` crates should remain free of - TreeSitter dependencies. This also implies they cannot have dependencies on any - of the `marzano-*` crates, since those *can* have TreeSitter dependencies. - level: error - body: | - language toml - - `[dependencies] - $deps` where { - $filename <: or { includes "Cargo.toml", includes "cargo.toml" }, - $absolute_filename <: or { - includes "crates/grit-pattern-matcher", - includes "crates/grit-util" - }, - $deps <: some bubble `$name = $specifier` where $name <: or { - includes "tree_sitter", - includes "tree-sitter", - includes "marzano" - } - } - - name: no_println_in_lsp - description: Don't use println!() in LSP code, it breaks the LSP stdio protocol. - level: error - body: | - engine marzano(0.1) - language rust - - `println!($_)` => . where { - $filename <: not includes "test.rs", - $absolute_filename <: includes "lsp", - } - - name: no_println_in_core - description: Don't use println or other debugging macros in core code. - level: error - body: | - engine marzano(0.1) - language rust - - `println!($args)` as $print where { - $outcome = ., - or { - $absolute_filename <: includes "crates/core", - $absolute_filename <: includes "crates/gritmodule", - $absolute_filename <: includes "crates/util", - and { $absolute_filename <: includes "crates/cli/", $outcome = `log::info!($args)` } - }, - // Allow tests and build utils - $absolute_filename <: not includes or { - "tests/", - "build.rs", - "test" - }, - $print <: not within `mod tests { $_ }` - } => $outcome diff --git a/crates/cli_bin/fixtures/patterns_test/.grit/grit.yaml b/crates/cli_bin/fixtures/patterns_test/.grit/grit.yaml index f1a3fc508..5221b94f2 100644 --- a/crates/cli_bin/fixtures/patterns_test/.grit/grit.yaml +++ b/crates/cli_bin/fixtures/patterns_test/.grit/grit.yaml @@ -1,2 +1,10 @@ version: 0.0.1 -patterns: [] \ No newline at end of file +patterns: + - name: simple_js_watch + body: | + `foo` => `bar` + samples: + - input: | + console.log(foo) + output: | + console.log(bar) diff --git a/crates/cli_bin/tests/patterns_test.rs b/crates/cli_bin/tests/patterns_test.rs index bbca569d4..e8e1989f5 100644 --- a/crates/cli_bin/tests/patterns_test.rs +++ b/crates/cli_bin/tests/patterns_test.rs @@ -320,18 +320,20 @@ fn tests_python_pattern_with_file_name() -> Result<()> { } #[test] -fn patterns_test_watch_mode_case_patterns_changed() -> Result<()> { +fn test_watch_mode_changed() -> Result<()> { let (tx, rx) = mpsc::channel(); - let (temp_dir, temp_grit_dir) = get_fixture(".grit", false)?; - let test_yaml_path = temp_grit_dir.join("grit.yaml"); - let temp_dir_path = temp_dir.path().to_owned(); + let (temp_dir, temp_fixture_path) = get_fixture("patterns_test", false)?; + let test_yaml_path = temp_fixture_path.join(".grit/grit.yaml"); + + println!("temp_fixture_path: {:?}", test_yaml_path); + println!("temp_dir_path: {:?}", temp_dir.into_path()); let _cmd_handle = thread::spawn(move || { let mut cmd = get_test_process_cmd() .unwrap() - .args(&["patterns", "test", "--watch"]) - .current_dir(&temp_dir_path) + .args(["patterns", "test", "--watch"]) + .current_dir(&temp_fixture_path) .stdout(Stdio::piped()) .stderr(Stdio::piped()) .spawn() @@ -339,27 +341,33 @@ fn patterns_test_watch_mode_case_patterns_changed() -> Result<()> { let stdout = BufReader::new(cmd.stdout.take().unwrap()); let stderr = BufReader::new(cmd.stderr.take().unwrap()); - for line in stdout.lines().chain(stderr.lines()) { - if let Ok(line) = line { - tx.send(line).unwrap(); - } + for line in stdout.lines().chain(stderr.lines()).flatten() { + println!("LINE: {:?}", line); + tx.send(line).unwrap(); } }); - thread::sleep(Duration::from_secs(1)); + thread::sleep(Duration::from_secs(3)); + + // Update it + let old_content = fs::read_to_string(&test_yaml_path).expect("Unable to read the file"); + let new_content = old_content.replace("console.log(bar)", "console.log(bad)"); + fs::write(&test_yaml_path, new_content)?; + thread::sleep(Duration::from_secs(3)); - let content = fs::read_to_string(&test_yaml_path).expect("Unable to read the file"); - fs::write(&test_yaml_path, content)?; - thread::sleep(Duration::from_secs(1)); + // Then reset it + fs::write(&test_yaml_path, old_content)?; + thread::sleep(Duration::from_secs(3)); let mut output = Vec::new(); while let Ok(line) = rx.try_recv() { output.push(line); } let expected_output = vec![ - "[Watch Mode] Enabled on path: .grit", - "[Watch Mode] File modified: \".grit/grit.yaml\"", - "[Watch Mode] Pattern(s) to test: [\"our_cargo_use_long_dependency\", \"cargo_use_long_dependency\", \"no_treesitter_in_grit_crates\", \"no_println_in_lsp\", \"no_println_in_core\"]", - "Found 5 testable patterns.", + "Found 2 testable patterns.", + "Watching for changes", + "✓ All 7 samples passed.", + ".grit/grit.yaml was modified", + "1 out of 1 samples failed.", ]; for expected_line in expected_output { assert!( @@ -368,22 +376,33 @@ fn patterns_test_watch_mode_case_patterns_changed() -> Result<()> { expected_line ); } + + assert!( + output + .iter() + .filter(|line| line.contains("samples passed")) + .count() + == 2 + ); + Ok(()) } #[test] -fn patterns_test_watch_mode_case_no_pattern_to_test() -> Result<()> { +fn test_watch_mode_deleted() -> Result<()> { let (tx, rx) = mpsc::channel(); - let (temp_dir, temp_grit_dir) = get_fixture(".grit", false)?; - let test_yaml_path = temp_grit_dir.join("grit.yaml"); - let temp_dir_path = temp_dir.path().to_owned(); + let (temp_dir, temp_fixture_path) = get_fixture("patterns_test", false)?; + let test_yaml_path = temp_fixture_path.join(".grit/grit.yaml"); + + println!("temp_fixture_path: {:?}", test_yaml_path); + println!("temp_dir_path: {:?}", temp_dir.into_path()); let _cmd_handle = thread::spawn(move || { let mut cmd = get_test_process_cmd() .unwrap() - .args(&["patterns", "test", "--watch"]) - .current_dir(&temp_dir_path) + .args(["patterns", "test", "--watch"]) + .current_dir(&temp_fixture_path) .stdout(Stdio::piped()) .stderr(Stdio::piped()) .spawn() @@ -391,26 +410,27 @@ fn patterns_test_watch_mode_case_no_pattern_to_test() -> Result<()> { let stdout = BufReader::new(cmd.stdout.take().unwrap()); let stderr = BufReader::new(cmd.stderr.take().unwrap()); - for line in stdout.lines().chain(stderr.lines()) { - if let Ok(line) = line { - tx.send(line).unwrap(); - } + for line in stdout.lines().chain(stderr.lines()).flatten() { + println!("LINE: {:?}", line); + tx.send(line).unwrap(); } }); - thread::sleep(Duration::from_secs(1)); + thread::sleep(Duration::from_secs(3)); - fs::write(&test_yaml_path, "")?; - thread::sleep(Duration::from_secs(1)); + // Update it to an empty config + fs::write(&test_yaml_path, "patterns: []")?; + thread::sleep(Duration::from_secs(3)); let mut output = Vec::new(); while let Ok(line) = rx.try_recv() { output.push(line); } - let expected_output = vec![ - "[Watch Mode] Enabled on path: .grit", - "[Watch Mode] File modified: \".grit/grit.yaml\"", - "[Watch Mode] Pattern(s) to test: []", + "Found 2 testable patterns.", + "Watching for changes", + "✓ All 7 samples passed.", + ".grit/grit.yaml was modified", + "no testable pattern changes were found", ]; for expected_line in expected_output { assert!( @@ -419,5 +439,6 @@ fn patterns_test_watch_mode_case_no_pattern_to_test() -> Result<()> { expected_line ); } + Ok(()) } diff --git a/crates/core/src/analysis.rs b/crates/core/src/analysis.rs index 26e9b9270..788d84967 100644 --- a/crates/core/src/analysis.rs +++ b/crates/core/src/analysis.rs @@ -6,6 +6,10 @@ use std::collections::BTreeMap; use std::ffi::OsStr; use std::path::Path; +use grit_pattern_matcher::constants::DEFAULT_FILE_NAME; + +use crate::pattern_compiler::compiler::{defs_to_filenames, DefsToFilenames}; + /// Walks the call tree and returns true if the predicate is true for any node. /// This is potentially error-prone, so not entirely recommended fn walk_call_tree( @@ -93,6 +97,91 @@ pub fn defines_itself(root: &NodeWithSource, root_name: &str) -> Result { Ok(false) } +/// Using source code alone, find dependents of a pattern. +/// This is *NOT* a good implementation and has numerous performance issues, +/// but it is good enough for `grit patterns test --watch` +/// +/// Consider refactoring if this is used in a more performance-critical context +pub fn get_dependents_of_target_patterns_by_traversal_from_src( + libs: &BTreeMap, + src: &str, + parser: &mut MarzanoGritParser, + target_patterns: &[&String], +) -> Result> { + let mut dependents = >::new(); + let node_like = "nodeLike"; + let predicate_call = "predicateCall"; + + let tree = parser.parse_file(src, Some(Path::new(DEFAULT_FILE_NAME)))?; + + let DefsToFilenames { + patterns: pattern_file, + predicates: predicate_file, + functions: function_file, + foreign_functions: foreign_file, + } = defs_to_filenames(libs, parser, tree.root_node())?; + + let name_to_filename: BTreeMap<&String, &String> = pattern_file + .iter() + .chain(predicate_file.iter()) + .chain(function_file.iter()) + .chain(foreign_file.iter()) + .collect(); + + let mut traversed_stack = >::new(); + let mut stack: Vec = vec![tree]; + while let Some(tree) = stack.pop() { + let root = tree.root_node(); + let cursor = root.walk(); + + for n in traverse(cursor, Order::Pre).filter(|n| { + n.node.is_named() && (n.node.kind() == node_like || n.node.kind() == predicate_call) + }) { + let name = n + .child_by_field_name("name") + .ok_or_else(|| anyhow!("missing name of nodeLike"))?; + let name = name.text()?; + let name = name.trim().to_string(); + + if target_patterns.contains(&&name) { + while let Some(e) = traversed_stack.pop() { + dependents.push(e); + } + } + if let Some(file_name) = name_to_filename.get(&name) { + if let Some(tree) = find_child_tree_definition( + file_name, + parser, + libs, + &mut traversed_stack, + &name, + )? { + stack.push(tree); + } + } + } + } + Ok(dependents) +} + +/// Attempt to find where a pattern is defined +fn find_child_tree_definition( + file_name: &str, + parser: &mut MarzanoGritParser, + libs: &BTreeMap, + traversed_stack: &mut Vec, + name: &str, +) -> Result> { + if !traversed_stack.contains(&name.to_string()) { + if let Some(file_body) = libs.get(file_name) { + traversed_stack.push(name.to_owned()); + let tree = parser.parse_file(file_body, Some(Path::new(file_name)))?; + return Ok(Some(tree)); + } + } + Ok(None) +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/core/src/pattern_compiler/compiler.rs b/crates/core/src/pattern_compiler/compiler.rs index 8a48401c1..8d9afe4bb 100644 --- a/crates/core/src/pattern_compiler/compiler.rs +++ b/crates/core/src/pattern_compiler/compiler.rs @@ -372,14 +372,14 @@ pub(crate) fn get_definitions( foreign_function_definitions, }) } -struct DefsToFilenames { - patterns: BTreeMap, - predicates: BTreeMap, - functions: BTreeMap, - foreign_functions: BTreeMap, +pub(crate) struct DefsToFilenames { + pub(crate) patterns: BTreeMap, + pub(crate) predicates: BTreeMap, + pub(crate) functions: BTreeMap, + pub(crate) foreign_functions: BTreeMap, } -fn defs_to_filenames( +pub(crate) fn defs_to_filenames( libs: &BTreeMap, parser: &mut MarzanoGritParser, root: NodeWithSource, @@ -567,85 +567,6 @@ fn find_definition_if_exists( Ok(None) } -pub fn get_dependents_of_target_patterns_by_traversal_from_src( - libs: &BTreeMap, - src: &str, - parser: &mut MarzanoGritParser, - target_patterns: &[&String], -) -> Result> { - let mut dependents = >::new(); - let node_like = "nodeLike"; - let predicate_call = "predicateCall"; - - let tree = parser.parse_file(src, Some(Path::new(DEFAULT_FILE_NAME)))?; - - let DefsToFilenames { - patterns: pattern_file, - predicates: predicate_file, - functions: function_file, - foreign_functions: foreign_file, - } = defs_to_filenames(libs, parser, tree.root_node())?; - - let name_to_filename: BTreeMap<&String, &String> = pattern_file - .iter() - .chain(predicate_file.iter()) - .chain(function_file.iter()) - .chain(foreign_file.iter()) - .collect(); - - let mut traversed_stack = >::new(); - let mut stack: Vec = vec![tree]; - while let Some(tree) = stack.pop() { - let root = tree.root_node(); - let cursor = root.walk(); - - for n in traverse(cursor, Order::Pre).filter(|n| { - n.node.is_named() && (n.node.kind() == node_like || n.node.kind() == predicate_call) - }) { - let name = n - .child_by_field_name("name") - .ok_or_else(|| anyhow!("missing name of nodeLike"))?; - let name = name.text()?; - let name = name.trim().to_string(); - - if target_patterns.contains(&&name) { - while let Some(e) = traversed_stack.pop() { - dependents.push(e); - } - } - if let Some(file_name) = name_to_filename.get(&name) { - if let Some(tree) = find_child_tree_definition( - file_name, - parser, - libs, - &mut traversed_stack, - &name, - )? { - stack.push(tree); - } - } - } - } - Ok(dependents) -} - -fn find_child_tree_definition( - file_name: &str, - parser: &mut MarzanoGritParser, - libs: &BTreeMap, - traversed_stack: &mut Vec, - name: &str, -) -> Result> { - if !traversed_stack.contains(&name.to_string()) { - if let Some(file_body) = libs.get(file_name) { - traversed_stack.push(name.to_owned()); - let tree = parser.parse_file(file_body, Some(Path::new(file_name)))?; - return Ok(Some(tree)); - } - } - Ok(None) -} - pub struct CompilationResult { pub compilation_warnings: AnalysisLogs, pub problem: Problem, diff --git a/crates/gritmodule/Cargo.toml b/crates/gritmodule/Cargo.toml index b44827894..60b2df13a 100644 --- a/crates/gritmodule/Cargo.toml +++ b/crates/gritmodule/Cargo.toml @@ -21,6 +21,7 @@ tree-sitter = { path = "../../vendor/tree-sitter-facade", package = "tree-sitter serde = { version = "1.0.164", features = ["derive"] } serde_yaml = { version = "0.9.25" } anyhow = { version = "1.0.70" } +futures = { version = "0.3.29" } rand = { version = "0.8.5" } git2 = { version = "0.17.2", default-features = false, features = [ "vendored-openssl", diff --git a/crates/gritmodule/src/config.rs b/crates/gritmodule/src/config.rs index 2b8979ca1..e071fe3e3 100644 --- a/crates/gritmodule/src/config.rs +++ b/crates/gritmodule/src/config.rs @@ -298,7 +298,7 @@ impl Ord for ResolvedGritDefinition { pub fn pattern_config_to_model( pattern: GritDefinitionConfig, - source: &Option, + source: Option<&ModuleRepo>, ) -> Result { let mut split_name = pattern.name.split('#'); let repo = split_name.next(); @@ -320,7 +320,7 @@ pub fn pattern_config_to_model( Some(split_repo.collect::>().join("/")) }; if defined_local_name.is_none() { - source.clone() + source.cloned() } else if host.is_none() || full_name.is_none() { None } else { diff --git a/crates/gritmodule/src/parser.rs b/crates/gritmodule/src/parser.rs index 37eb8b878..d2cf612d9 100644 --- a/crates/gritmodule/src/parser.rs +++ b/crates/gritmodule/src/parser.rs @@ -71,14 +71,15 @@ impl PatternFileExt { ) }) } - PatternFileExt::Yaml => get_patterns_from_yaml(file, source_module, root, "") - .await - .with_context(|| { + PatternFileExt::Yaml => { + let res = get_patterns_from_yaml(file, source_module.as_ref(), root, "").await; + res.with_context(|| { format!( "Failed to parse yaml pattern {}", extract_relative_file_path(file, root) ) - }), + }) + } } } diff --git a/crates/gritmodule/src/resolver.rs b/crates/gritmodule/src/resolver.rs index f4ba854dd..4925f8db1 100644 --- a/crates/gritmodule/src/resolver.rs +++ b/crates/gritmodule/src/resolver.rs @@ -535,8 +535,7 @@ async fn get_grit_files_for_module( Some(config) => { if let Some(module) = module { let repo_root = find_repo_root_from(repo_path).await?; - get_patterns_from_yaml(&config, &Some(module.to_owned()), &repo_root, repo_dir) - .await? + get_patterns_from_yaml(&config, Some(module), &repo_root, repo_dir).await? } else { vec![] } @@ -587,8 +586,7 @@ async fn resolve_patterns_for_module( Some(config) => { if let Some(module) = module { let repo_root = find_repo_root_from(repo_path).await?; - get_patterns_from_yaml(&config, &Some(module.to_owned()), &repo_root, repo_dir) - .await? + get_patterns_from_yaml(&config, Some(module), &repo_root, repo_dir).await? } else { vec![] } diff --git a/crates/gritmodule/src/yaml.rs b/crates/gritmodule/src/yaml.rs index beea33572..a240c73e4 100644 --- a/crates/gritmodule/src/yaml.rs +++ b/crates/gritmodule/src/yaml.rs @@ -1,11 +1,12 @@ use anyhow::{bail, Result}; +use futures::{future::BoxFuture, FutureExt as _}; use grit_util::Position; use marzano_util::rich_path::RichFile; use std::{ collections::HashSet, path::{Path, PathBuf}, }; -use tokio::{fs, task}; +use tokio::fs; use crate::{ config::{ @@ -58,56 +59,49 @@ pub fn get_grit_config(source: &str, source_path: &str) -> Result { Ok(new_config) } -pub async fn get_patterns_from_yaml( - file: &RichFile, - source_module: &Option, - root: &Option, - repo_dir: &str, -) -> Result> { - let grit_path = extract_relative_file_path(file, root); - let mut config = get_grit_config(&file.content, &grit_path)?; - - for pattern in config.patterns.iter_mut() { - pattern.kind = Some(DefinitionKind::Pattern); - let offset = file.content.find(&pattern.name).unwrap_or(0); - pattern.position = Some(Position::from_byte_index(&file.content, offset)); - } - - let patterns = config - .patterns - .into_iter() - .map(|pattern| pattern_config_to_model(pattern, source_module)) - .collect(); +pub fn get_patterns_from_yaml<'a>( + file: &'a RichFile, + source_module: Option<&'a ModuleRepo>, + root: &'a Option, + repo_dir: &'a str, +) -> BoxFuture<'a, Result>> { + async move { + let grit_path = extract_relative_file_path(file, root); + let mut config = get_grit_config(&file.content, &grit_path)?; + + for pattern in config.patterns.iter_mut() { + pattern.kind = Some(DefinitionKind::Pattern); + let offset = file.content.find(&pattern.name).unwrap_or(0); + pattern.position = Some(Position::from_byte_index(&file.content, offset)); + } - if config.pattern_files.is_none() { - return patterns; - } + let patterns: Result> = config + .patterns + .into_iter() + .map(|pattern| pattern_config_to_model(pattern, source_module)) + .collect(); + let mut patterns = patterns?; - let mut patterns = patterns?; - let mut file_readers = Vec::new(); + if config.pattern_files.is_none() { + return Ok(patterns); + } - for pattern_file in config.pattern_files.unwrap() { - let pattern_file = PathBuf::from(repo_dir) - .join(REPO_CONFIG_DIR_NAME) - .join(&pattern_file.file); - let extension = PatternFileExt::from_path(&pattern_file); - if extension.is_none() { - continue; + for pattern_file in config.pattern_files.unwrap() { + let pattern_file = PathBuf::from(repo_dir) + .join(REPO_CONFIG_DIR_NAME) + .join(&pattern_file.file); + let extension = PatternFileExt::from_path(&pattern_file); + if extension.is_none() { + continue; + } + let extension = extension.unwrap(); + let source_module = source_module.cloned(); + patterns.extend(get_patterns_from_file(pattern_file, source_module, extension).await?); } - let extension = extension.unwrap(); - let source_module = source_module.clone(); - file_readers.push(task::spawn_blocking(move || { - tokio::runtime::Runtime::new().unwrap().block_on(async { - get_patterns_from_file(pattern_file, source_module, extension).await - }) - })); - } - for file_reader in file_readers { - patterns.extend(file_reader.await??); + Ok(patterns) } - - Ok(patterns) + .boxed() } pub fn extract_grit_modules(content: &str, path: &str) -> Result> { @@ -215,8 +209,7 @@ github: "# .to_string(), }; - let repo = Default::default(); - let patterns = get_patterns_from_yaml(&grit_yaml, &repo, &None, "getgrit/rewriter") + let patterns = get_patterns_from_yaml(&grit_yaml, None, &None, "getgrit/rewriter") .await .unwrap(); assert_eq!(patterns.len(), 4);