diff --git a/lychee-bin/src/commands/check.rs b/lychee-bin/src/commands/check.rs index 4451a2240d..c724eaad19 100644 --- a/lychee-bin/src/commands/check.rs +++ b/lychee-bin/src/commands/check.rs @@ -17,10 +17,11 @@ use lychee_lib::{ResponseBody, Status}; use crate::formatters::get_progress_formatter; use crate::formatters::response::ResponseFormatter; +use crate::formatters::stats::ResponseStats; use crate::formatters::suggestion::Suggestion; use crate::parse::parse_duration_secs; use crate::progress::Progress; -use crate::{ExitCode, cache::Cache, stats::ResponseStats}; +use crate::{ExitCode, cache::Cache}; use super::CommandParams; diff --git a/lychee-bin/src/formatters/host_stats/compact.rs b/lychee-bin/src/formatters/host_stats/compact.rs index d312e0c60f..15f44df7c4 100644 --- a/lychee-bin/src/formatters/host_stats/compact.rs +++ b/lychee-bin/src/formatters/host_stats/compact.rs @@ -1,23 +1,17 @@ -use anyhow::Result; -use std::{ - collections::HashMap, - fmt::{self, Display}, -}; +use std::fmt::{self, Display}; use crate::formatters::color::{DIM, NORMAL, color}; -use lychee_lib::ratelimit::HostStats; +use lychee_lib::ratelimit::HostStatsMap; -use super::HostStatsFormatter; - -struct CompactHostStats { - host_stats: HashMap, +pub(crate) struct CompactHostStats { + pub(crate) host_stats: Option, } impl Display for CompactHostStats { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - if self.host_stats.is_empty() { + let Some(host_stats) = &self.host_stats else { return Ok(()); - } + }; writeln!(f)?; writeln!(f, "šŸ“Š Per-host Statistics")?; @@ -26,7 +20,7 @@ impl Display for CompactHostStats { color!(f, DIM, "{}", separator)?; writeln!(f)?; - let sorted_hosts = super::sort_host_stats(&self.host_stats); + let sorted_hosts = host_stats.sorted(); // Calculate optimal hostname width based on longest hostname let max_hostname_len = sorted_hosts @@ -60,18 +54,3 @@ impl Display for CompactHostStats { Ok(()) } } - -pub(crate) struct Compact; - -impl Compact { - pub(crate) const fn new() -> Self { - Self - } -} - -impl HostStatsFormatter for Compact { - fn format(&self, host_stats: HashMap) -> Result { - let compact = CompactHostStats { host_stats }; - Ok(compact.to_string()) - } -} diff --git a/lychee-bin/src/formatters/host_stats/detailed.rs b/lychee-bin/src/formatters/host_stats/detailed.rs index f62d248138..6415a7e4a3 100644 --- a/lychee-bin/src/formatters/host_stats/detailed.rs +++ b/lychee-bin/src/formatters/host_stats/detailed.rs @@ -1,29 +1,21 @@ -use anyhow::Result; -use std::{ - collections::HashMap, - fmt::{self, Display}, -}; +use std::fmt::{self, Display}; -use lychee_lib::ratelimit::HostStats; +use lychee_lib::ratelimit::HostStatsMap; -use super::HostStatsFormatter; - -struct DetailedHostStats { - host_stats: HashMap, +pub(crate) struct DetailedHostStats { + pub(crate) host_stats: Option, } impl Display for DetailedHostStats { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - if self.host_stats.is_empty() { + let Some(host_stats) = &self.host_stats else { return Ok(()); - } + }; writeln!(f, "\nšŸ“Š Per-host Statistics")?; writeln!(f, "---------------------")?; - let sorted_hosts = super::sort_host_stats(&self.host_stats); - - for (hostname, stats) in sorted_hosts { + for (hostname, stats) in host_stats.sorted() { writeln!(f, "\nHost: {hostname}")?; writeln!(f, " Total requests: {}", stats.total_requests)?; writeln!( @@ -69,18 +61,3 @@ impl Display for DetailedHostStats { Ok(()) } } - -pub(crate) struct Detailed; - -impl Detailed { - pub(crate) const fn new() -> Self { - Self - } -} - -impl HostStatsFormatter for Detailed { - fn format(&self, host_stats: HashMap) -> Result { - let detailed = DetailedHostStats { host_stats }; - Ok(detailed.to_string()) - } -} diff --git a/lychee-bin/src/formatters/host_stats/json.rs b/lychee-bin/src/formatters/host_stats/json.rs deleted file mode 100644 index fcb66b89df..0000000000 --- a/lychee-bin/src/formatters/host_stats/json.rs +++ /dev/null @@ -1,51 +0,0 @@ -use anyhow::{Context, Result}; -use serde_json::json; -use std::collections::HashMap; - -use super::HostStatsFormatter; -use lychee_lib::ratelimit::HostStats; - -pub(crate) struct Json; - -impl Json { - pub(crate) const fn new() -> Self { - Self - } -} - -impl HostStatsFormatter for Json { - /// Format host stats as JSON object - fn format(&self, host_stats: HashMap) -> Result { - // Convert HostStats to a more JSON-friendly format - let json_stats: HashMap = host_stats - .into_iter() - .map(|(hostname, stats)| { - let json_value = json!({ - "total_requests": stats.total_requests, - "successful_requests": stats.successful_requests, - "success_rate": stats.success_rate(), - "rate_limited": stats.rate_limited, - "client_errors": stats.client_errors, - "server_errors": stats.server_errors, - "median_request_time_ms": stats.median_request_time() - .map(|d| { - #[allow(clippy::cast_possible_truncation)] - let millis = d.as_millis() as u64; - millis - }), - "cache_hits": stats.cache_hits, - "cache_misses": stats.cache_misses, - "cache_hit_rate": stats.cache_hit_rate(), - "status_codes": stats.status_codes - }); - (hostname, json_value) - }) - .collect(); - - let output = json!({ - "host_statistics": json_stats - }); - - serde_json::to_string_pretty(&output).context("Cannot format host stats as JSON") - } -} diff --git a/lychee-bin/src/formatters/host_stats/markdown.rs b/lychee-bin/src/formatters/host_stats/markdown.rs index 3543911a9e..b57b06447a 100644 --- a/lychee-bin/src/formatters/host_stats/markdown.rs +++ b/lychee-bin/src/formatters/host_stats/markdown.rs @@ -1,16 +1,29 @@ -use std::{ - collections::HashMap, - fmt::{self, Display}, -}; +use std::fmt::{self, Display}; -use super::HostStatsFormatter; -use anyhow::Result; -use lychee_lib::ratelimit::HostStats; +use lychee_lib::ratelimit::HostStatsMap; use tabled::{ Table, Tabled, settings::{Alignment, Modify, Style, object::Segment}, }; +pub(crate) struct MarkdownHostStats { + pub(crate) host_stats: Option, +} + +impl Display for MarkdownHostStats { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let Some(host_stats) = &self.host_stats else { + return Ok(()); + }; + + writeln!(f, "\n## Per-host Statistics")?; + writeln!(f)?; + writeln!(f, "{}", host_stats_table(host_stats))?; + + Ok(()) + } +} + #[derive(Tabled)] struct HostStatsTableEntry { #[tabled(rename = "Host")] @@ -25,8 +38,8 @@ struct HostStatsTableEntry { cache_hit_rate: String, } -fn host_stats_table(host_stats: &HashMap) -> String { - let sorted_hosts = super::sort_host_stats(host_stats); +fn host_stats_table(host_stats: &HostStatsMap) -> String { + let sorted_hosts = host_stats.sorted(); let entries: Vec = sorted_hosts .into_iter() @@ -55,34 +68,3 @@ fn host_stats_table(host_stats: &HashMap) -> String { .with(style) .to_string() } - -struct MarkdownHostStats(HashMap); - -impl Display for MarkdownHostStats { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - if self.0.is_empty() { - return Ok(()); - } - - writeln!(f, "\n## Per-host Statistics")?; - writeln!(f)?; - writeln!(f, "{}", host_stats_table(&self.0))?; - - Ok(()) - } -} - -pub(crate) struct Markdown; - -impl Markdown { - pub(crate) const fn new() -> Self { - Self {} - } -} - -impl HostStatsFormatter for Markdown { - fn format(&self, host_stats: HashMap) -> Result { - let markdown = MarkdownHostStats(host_stats); - Ok(markdown.to_string()) - } -} diff --git a/lychee-bin/src/formatters/host_stats/mod.rs b/lychee-bin/src/formatters/host_stats/mod.rs index 205f5581d4..e3dfbb3b0c 100644 --- a/lychee-bin/src/formatters/host_stats/mod.rs +++ b/lychee-bin/src/formatters/host_stats/mod.rs @@ -1,28 +1,7 @@ mod compact; mod detailed; -mod json; mod markdown; -pub(crate) use compact::Compact; -pub(crate) use detailed::Detailed; -pub(crate) use json::Json; -pub(crate) use markdown::Markdown; - -use anyhow::Result; -use lychee_lib::ratelimit::HostStats; -use std::collections::HashMap; - -/// Trait for formatting per-host statistics in different output formats -pub(crate) trait HostStatsFormatter { - /// Format the host statistics and return them as a string - fn format(&self, host_stats: HashMap) -> Result; -} - -/// Sort host statistics by request count (descending order) -/// This matches the display order we want in the output -fn sort_host_stats(host_stats: &HashMap) -> Vec<(&String, &HostStats)> { - let mut sorted_hosts: Vec<_> = host_stats.iter().collect(); - // Sort by total requests (descending) - sorted_hosts.sort_by_key(|(_, stats)| std::cmp::Reverse(stats.total_requests)); - sorted_hosts -} +pub(crate) use compact::CompactHostStats; +pub(crate) use detailed::DetailedHostStats; +pub(crate) use markdown::MarkdownHostStats; diff --git a/lychee-bin/src/formatters/mod.rs b/lychee-bin/src/formatters/mod.rs index de36c32bb6..faacd107e1 100644 --- a/lychee-bin/src/formatters/mod.rs +++ b/lychee-bin/src/formatters/mod.rs @@ -6,7 +6,7 @@ pub(crate) mod response; pub(crate) mod stats; pub(crate) mod suggestion; -use self::{host_stats::HostStatsFormatter, response::ResponseFormatter, stats::StatsFormatter}; +use self::{response::ResponseFormatter, stats::StatsFormatter}; use crate::options::{OutputMode, StatsFormat}; use supports_color::Stream; @@ -42,19 +42,6 @@ pub(crate) fn get_progress_formatter(mode: &OutputMode) -> Box Box { - match format { - StatsFormat::Compact | StatsFormat::Raw => Box::new(host_stats::Compact::new()), // Use compact for raw - StatsFormat::Detailed => Box::new(host_stats::Detailed::new()), - StatsFormat::Json => Box::new(host_stats::Json::new()), - StatsFormat::Markdown => Box::new(host_stats::Markdown::new()), - } -} - /// Create a response formatter based on the given format option pub(crate) fn get_response_formatter(mode: &OutputMode) -> Box { // Checks if color is supported in current environment or NO_COLOR is set (https://no-color.org) diff --git a/lychee-bin/src/formatters/stats/compact.rs b/lychee-bin/src/formatters/stats/compact.rs index 66876874ce..5599db644d 100644 --- a/lychee-bin/src/formatters/stats/compact.rs +++ b/lychee-bin/src/formatters/stats/compact.rs @@ -6,8 +6,13 @@ use std::{ time::Duration, }; -use crate::formatters::color::{BOLD_GREEN, BOLD_PINK, BOLD_YELLOW, DIM, NORMAL, color}; -use crate::{formatters::get_response_formatter, options, stats::ResponseStats}; +use crate::formatters::{ + color::{BOLD_GREEN, BOLD_PINK, BOLD_YELLOW, DIM, NORMAL, color}, + get_response_formatter, + host_stats::CompactHostStats, + stats::{OutputStats, ResponseStats}, +}; +use crate::options; use super::StatsFormatter; @@ -107,87 +112,54 @@ impl Compact { } impl StatsFormatter for Compact { - fn format(&self, stats: ResponseStats) -> Result> { - let compact = CompactResponseStats { - stats, + fn format(&self, stats: OutputStats) -> Result { + let response_stats = CompactResponseStats { + stats: stats.response_stats, mode: self.mode.clone(), }; - Ok(Some(compact.to_string())) + let host_stats = CompactHostStats { + host_stats: stats.host_stats, + }; + + Ok(format!("{response_stats}\n{host_stats}")) } } #[cfg(test)] mod tests { - use crate::formatters::stats::StatsFormatter; - use crate::{options::OutputMode, stats::ResponseStats}; - use http::StatusCode; - use lychee_lib::{InputSource, ResponseBody, Status, Uri}; - use std::collections::{HashMap, HashSet}; - use url::Url; + use crate::formatters::stats::{StatsFormatter, get_dummy_stats}; + use crate::options::OutputMode; + use regex::Regex; use super::*; #[test] fn test_formatter() { - // A couple of dummy successes - let mut success_map: HashMap> = HashMap::new(); - - success_map.insert( - InputSource::RemoteUrl(Box::new(Url::parse("https://example.com").unwrap())), - HashSet::from_iter(vec![ResponseBody { - uri: Uri::from(Url::parse("https://example.com").unwrap()), - status: Status::Ok(StatusCode::OK), - }]), - ); - - let err1 = ResponseBody { - uri: Uri::try_from("https://github.com/mre/idiomatic-rust-doesnt-exist-man").unwrap(), - status: Status::Ok(StatusCode::NOT_FOUND), - }; + let formatter = Compact::new(OutputMode::Plain); + let result = formatter.format(get_dummy_stats()).unwrap(); - let err2 = ResponseBody { - uri: Uri::try_from("https://github.com/mre/boom").unwrap(), - status: Status::Ok(StatusCode::INTERNAL_SERVER_ERROR), - }; + // Remove color codes for better readability of the expected result + let without_color_codes = Regex::new(r"\u{1b}\[[0-9;]*m") + .unwrap() + .replace_all(&result, "") + .to_string(); - let mut error_map: HashMap> = HashMap::new(); - let source = InputSource::RemoteUrl(Box::new(Url::parse("https://example.com").unwrap())); - error_map.insert(source, HashSet::from_iter(vec![err1, err2])); - - let stats = ResponseStats { - total: 1, - successful: 1, - errors: 2, - unknown: 0, - excludes: 0, - timeouts: 0, - duration_secs: 0, - error_map, - suggestion_map: HashMap::default(), - redirect_map: HashMap::default(), - unsupported: 0, - redirects: 0, - cached: 0, - success_map, - excluded_map: HashMap::default(), - detailed_stats: false, - }; - - let formatter = Compact::new(OutputMode::Plain); + assert_eq!( + without_color_codes, + "Issues found in 1 input. Find details below. - let result = formatter.format(stats).unwrap().unwrap(); +[https://example.com/]: +[404] https://github.com/mre/idiomatic-rust-doesnt-exist-man | 404 Not Found: Not Found - println!("{result}"); +ℹ Suggestions +https://original.dev/ --> https://suggestion.dev/ - assert!(result.contains("šŸ” 1 Total")); - assert!(result.contains("āœ… 1 OK")); - assert!(result.contains("🚫 2 Errors")); +šŸ” 2 Total (in 0s) āœ… 0 OK 🚫 1 Error šŸ”€ 1 Redirects - assert!(result.contains("[https://example.com/]:")); - assert!( - result - .contains("https://github.com/mre/idiomatic-rust-doesnt-exist-man | 404 Not Found") +šŸ“Š Per-host Statistics +──────────────────────────────────────────────────────────── +example.com │ 5 reqs │ 60.0% success │ N/A median │ 20.0% cached +" ); - assert!(result.contains("https://github.com/mre/boom | 500 Internal Server Error")); } } diff --git a/lychee-bin/src/formatters/stats/detailed.rs b/lychee-bin/src/formatters/stats/detailed.rs index 58c2cb7fcb..ef3f9bad00 100644 --- a/lychee-bin/src/formatters/stats/detailed.rs +++ b/lychee-bin/src/formatters/stats/detailed.rs @@ -1,5 +1,12 @@ use super::StatsFormatter; -use crate::{formatters::get_response_formatter, options, stats::ResponseStats}; +use crate::{ + formatters::{ + get_response_formatter, + host_stats::DetailedHostStats, + stats::{OutputStats, ResponseStats}, + }, + options, +}; use anyhow::Result; use lychee_lib::InputSource; @@ -102,87 +109,28 @@ impl Detailed { } impl StatsFormatter for Detailed { - fn format(&self, stats: ResponseStats) -> Result> { - let detailed = DetailedResponseStats { - stats, + fn format(&self, stats: OutputStats) -> Result { + let response_stats = DetailedResponseStats { + stats: stats.response_stats, mode: self.mode.clone(), }; - Ok(Some(detailed.to_string())) + let host_stats = DetailedHostStats { + host_stats: stats.host_stats, + }; + + Ok(format!("{response_stats}\n{host_stats}")) } } #[cfg(test)] mod tests { use super::*; - use crate::{formatters::suggestion::Suggestion, options::OutputMode}; - use http::StatusCode; - use lychee_lib::{InputSource, Redirects, ResponseBody, Status}; - use std::collections::{HashMap, HashSet}; - use url::Url; + use crate::{formatters::stats::get_dummy_stats, options::OutputMode}; #[test] fn test_detailed_formatter() { - let source = InputSource::RemoteUrl(Box::new(Url::parse("https://example.com").unwrap())); - let error_map = HashMap::from([( - source.clone(), - HashSet::from([ - ResponseBody { - uri: "https://github.com/mre/idiomatic-rust-doesnt-exist-man" - .try_into() - .unwrap(), - status: Status::Ok(StatusCode::NOT_FOUND), - }, - ResponseBody { - uri: "https://github.com/mre/boom".try_into().unwrap(), - status: Status::Ok(StatusCode::INTERNAL_SERVER_ERROR), - }, - ]), - )]); - - let suggestion_map = HashMap::from([( - source.clone(), - HashSet::from([Suggestion { - original: "https://original.dev".try_into().unwrap(), - suggestion: "https://suggestion.dev".try_into().unwrap(), - }]), - )]); - - let redirect_map = HashMap::from([( - source, - HashSet::from([ResponseBody { - uri: "https://redirected.dev".try_into().unwrap(), - status: Status::Redirected( - StatusCode::OK, - Redirects::from(vec![ - Url::parse("https://1.dev").unwrap(), - Url::parse("https://2.dev").unwrap(), - Url::parse("http://redirected.dev").unwrap(), - ]), - ), - }]), - )]); - - let stats = ResponseStats { - total: 2, - successful: 0, - errors: 2, - unknown: 0, - excludes: 0, - timeouts: 0, - duration_secs: 0, - unsupported: 0, - redirects: 0, - cached: 0, - suggestion_map, - redirect_map, - success_map: HashMap::default(), - error_map, - excluded_map: HashMap::default(), - detailed_stats: true, - }; - let formatter = Detailed::new(OutputMode::Plain); - let result = formatter.format(stats).unwrap().unwrap(); + let result = formatter.format(get_dummy_stats()).unwrap(); assert_eq!( result, @@ -191,14 +139,13 @@ mod tests { šŸ” Total............2 āœ… Successful.......0 ā³ Timeouts.........0 -šŸ”€ Redirected.......0 +šŸ”€ Redirected.......1 šŸ‘» Excluded.........0 ā“ Unknown..........0 -🚫 Errors...........2 -ā›” Unsupported......2 +🚫 Errors...........1 +ā›” Unsupported......1 Errors in https://example.com/ -[500] https://github.com/mre/boom | 500 Internal Server Error: Internal Server Error [404] https://github.com/mre/idiomatic-rust-doesnt-exist-man | 404 Not Found: Not Found Suggestions in https://example.com/ @@ -207,6 +154,18 @@ https://original.dev/ --> https://suggestion.dev/ Redirects in https://example.com/ https://redirected.dev/ | Redirect: Followed 2 redirects resolving to the final status of: OK. Redirects: https://1.dev/ --> https://2.dev/ --> http://redirected.dev/ + + +šŸ“Š Per-host Statistics +--------------------- + +Host: example.com + Total requests: 5 + Successful: 3 (60.0%) + Rate limited: 1 (429 Too Many Requests) + Server errors (5xx): 1 + Cache hit rate: 20.0% + Cache hits: 1, misses: 4 " ); } diff --git a/lychee-bin/src/formatters/stats/json.rs b/lychee-bin/src/formatters/stats/json.rs index 0e6e9b49f5..de230ab1f7 100644 --- a/lychee-bin/src/formatters/stats/json.rs +++ b/lychee-bin/src/formatters/stats/json.rs @@ -1,7 +1,7 @@ use anyhow::{Context, Result}; use super::StatsFormatter; -use crate::stats::ResponseStats; +use crate::formatters::stats::OutputStats; pub(crate) struct Json; @@ -13,9 +13,87 @@ impl Json { impl StatsFormatter for Json { /// Format stats as JSON object - fn format(&self, stats: ResponseStats) -> Result> { - serde_json::to_string_pretty(&stats) - .map(Some) - .context("Cannot format stats as JSON") + fn format(&self, stats: OutputStats) -> Result { + serde_json::to_string_pretty(&stats).context("Cannot format stats as JSON") + } +} + +#[cfg(test)] +mod tests { + use crate::formatters::stats::{Json, StatsFormatter, get_dummy_stats}; + + #[test] + fn test_json_formatter() { + let formatter = Json::new(); + let result = formatter.format(get_dummy_stats()).unwrap(); + + assert_eq!( + result, + r#"{ + "total": 2, + "successful": 0, + "unknown": 0, + "unsupported": 0, + "timeouts": 0, + "redirects": 1, + "excludes": 0, + "errors": 1, + "cached": 0, + "success_map": {}, + "error_map": { + "https://example.com/": [ + { + "url": "https://github.com/mre/idiomatic-rust-doesnt-exist-man", + "status": { + "text": "404 Not Found", + "code": 404 + } + } + ] + }, + "suggestion_map": { + "https://example.com/": [ + { + "original": "https://original.dev/", + "suggestion": "https://suggestion.dev/" + } + ] + }, + "redirect_map": { + "https://example.com/": [ + { + "url": "https://redirected.dev/", + "status": { + "text": "Redirect", + "code": 200, + "redirects": [ + "https://1.dev/", + "https://2.dev/", + "http://redirected.dev/" + ] + } + } + ] + }, + "excluded_map": {}, + "duration_secs": 0, + "detailed_stats": true, + "host_stats": { + "example.com": { + "total_requests": 5, + "successful_requests": 3, + "success_rate": 0.6, + "rate_limited": 1, + "client_errors": 0, + "server_errors": 1, + "median_request_time_ms": null, + "cache_hits": 1, + "cache_misses": 4, + "cache_hit_rate": 0.2, + "status_codes": {} + } + } +}"# + ); } } diff --git a/lychee-bin/src/formatters/stats/markdown.rs b/lychee-bin/src/formatters/stats/markdown.rs index 6feddb4c0f..3d8f96d63b 100644 --- a/lychee-bin/src/formatters/stats/markdown.rs +++ b/lychee-bin/src/formatters/stats/markdown.rs @@ -13,7 +13,10 @@ use tabled::{ settings::{Alignment, Modify, Style, object::Segment}, }; -use crate::stats::ResponseStats; +use crate::formatters::{ + host_stats::MarkdownHostStats, + stats::{OutputStats, ResponseStats}, +}; #[derive(Tabled)] struct StatsTableEntry { @@ -154,19 +157,22 @@ impl Markdown { } impl StatsFormatter for Markdown { - fn format(&self, stats: ResponseStats) -> Result> { - let markdown = MarkdownResponseStats(stats); - Ok(Some(markdown.to_string())) + fn format(&self, stats: OutputStats) -> Result { + let response_stats = MarkdownResponseStats(stats.response_stats); + let host_stats = MarkdownHostStats { + host_stats: stats.host_stats, + }; + + Ok(format!("{response_stats}\n{host_stats}")) } } #[cfg(test)] mod tests { use http::StatusCode; - use lychee_lib::{CacheStatus, InputSource, Redirects, Response, ResponseBody, Status, Uri}; - use reqwest::Url; + use lychee_lib::{CacheStatus, ResponseBody, Status, Uri}; - use crate::formatters::suggestion::Suggestion; + use crate::formatters::stats::get_dummy_stats; use super::*; @@ -219,40 +225,7 @@ mod tests { #[test] fn test_render_summary() { - let mut stats = ResponseStats::default(); - - // Add cached error - stats.add(Response::new( - Uri::try_from("http://127.0.0.1").unwrap(), - Status::Cached(CacheStatus::Error(Some(404))), - InputSource::Stdin, - )); - - // Add suggestion - stats - .suggestion_map - .entry((InputSource::Stdin).clone()) - .or_default() - .insert(Suggestion { - suggestion: Url::parse("https://example.com/suggestion").unwrap(), - original: Url::parse("https://example.com/original").unwrap(), - }); - - // Add redirect - stats.add(Response::new( - Uri::try_from("http://redirected.dev").unwrap(), - Status::Redirected( - StatusCode::OK, - Redirects::from(vec![ - Url::parse("https://1.dev").unwrap(), - Url::parse("https://2.dev").unwrap(), - Url::parse("http://redirected.dev").unwrap(), - ]), - ), - InputSource::Stdin, - )); - - let summary = MarkdownResponseStats(stats); + let summary = MarkdownResponseStats(get_dummy_stats().response_stats); let expected = "# Summary | Status | Count | @@ -268,21 +241,21 @@ mod tests { ## Errors per input -### Errors in stdin +### Errors in https://example.com/ -* [404] | Error (cached) +* [404] | 404 Not Found: Not Found ## Redirects per input -### Redirects in stdin +### Redirects in https://example.com/ -* [200] | Redirect: Followed 2 redirects resolving to the final status of: OK. Redirects: https://1.dev/ --> https://2.dev/ --> http://redirected.dev/ +* [200] | Redirect: Followed 2 redirects resolving to the final status of: OK. Redirects: https://1.dev/ --> https://2.dev/ --> http://redirected.dev/ ## Suggestions per input -### Suggestions in stdin +### Suggestions in https://example.com/ -* https://example.com/original --> https://example.com/suggestion +* https://original.dev/ --> https://suggestion.dev/ "; assert_eq!(summary.to_string(), expected.to_string()); } diff --git a/lychee-bin/src/formatters/stats/mod.rs b/lychee-bin/src/formatters/stats/mod.rs index 8d6cb559e9..869d7f9113 100644 --- a/lychee-bin/src/formatters/stats/mod.rs +++ b/lychee-bin/src/formatters/stats/mod.rs @@ -3,12 +3,16 @@ mod detailed; mod json; mod markdown; mod raw; +mod response; pub(crate) use compact::Compact; + pub(crate) use detailed::Detailed; pub(crate) use json::Json; pub(crate) use markdown::Markdown; pub(crate) use raw::Raw; +pub(crate) use response::ResponseStats; +use serde::Serialize; use std::{ collections::{HashMap, HashSet}, @@ -17,30 +21,33 @@ use std::{ io::{Write, stdout}, }; -use crate::{formatters::get_stats_formatter, options::Config, stats::ResponseStats}; +use crate::{formatters::get_stats_formatter, options::Config}; use anyhow::{Context, Result}; -use lychee_lib::InputSource; +use lychee_lib::{InputSource, ratelimit::HostStatsMap}; + +#[derive(Default, Serialize)] +pub(crate) struct OutputStats { + #[serde(flatten)] + pub(crate) response_stats: ResponseStats, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) host_stats: Option, +} pub(crate) trait StatsFormatter { /// Format the stats of all responses and write them to stdout - fn format(&self, stats: ResponseStats) -> Result>; + fn format(&self, stats: OutputStats) -> Result; } /// If configured to do so, output response statistics to stdout or the specified output file. -pub(crate) fn output_response_statistics(stats: ResponseStats, config: &Config) -> Result<()> { - let is_empty = stats.is_empty(); +pub(crate) fn output_statistics(stats: OutputStats, config: &Config) -> Result<()> { let formatter = get_stats_formatter(&config.format, &config.mode); - if let Some(formatted_stats) = formatter.format(stats)? { - if let Some(output) = &config.output { - fs::write(output, formatted_stats).context("Cannot write status output to file")?; - } else { - if config.verbose.log_level() >= log::Level::Info && !is_empty { - // separate summary from the verbose list of links above with a newline - writeln!(stdout())?; - } - // we assume that the formatted stats don't have a final newline - writeln!(stdout(), "{formatted_stats}")?; - } + let formatted_stats = formatter.format(stats)?; + + if let Some(output) = &config.output { + fs::write(output, formatted_stats).context("Cannot write status output to file")?; + } else { + // we assume that the formatted stats don't have a final newline + writeln!(stdout(), "{formatted_stats}")?; } Ok(()) } @@ -72,6 +79,86 @@ where entries } +#[cfg(test)] +fn get_dummy_stats() -> OutputStats { + use http::StatusCode; + use lychee_lib::{Redirects, ResponseBody, Status, ratelimit::HostStats}; + use url::Url; + + use crate::formatters::suggestion::Suggestion; + + let source = InputSource::RemoteUrl(Box::new(Url::parse("https://example.com").unwrap())); + let error_map = HashMap::from([( + source.clone(), + HashSet::from([ResponseBody { + uri: "https://github.com/mre/idiomatic-rust-doesnt-exist-man" + .try_into() + .unwrap(), + status: Status::Ok(StatusCode::NOT_FOUND), + }]), + )]); + + let suggestion_map = HashMap::from([( + source.clone(), + HashSet::from([Suggestion { + original: "https://original.dev".try_into().unwrap(), + suggestion: "https://suggestion.dev".try_into().unwrap(), + }]), + )]); + + let redirect_map = HashMap::from([( + source, + HashSet::from([ResponseBody { + uri: "https://redirected.dev".try_into().unwrap(), + status: Status::Redirected( + StatusCode::OK, + Redirects::from(vec![ + Url::parse("https://1.dev").unwrap(), + Url::parse("https://2.dev").unwrap(), + Url::parse("http://redirected.dev").unwrap(), + ]), + ), + }]), + )]); + + let response_stats = ResponseStats { + total: 2, + successful: 0, + errors: 1, + unknown: 0, + excludes: 0, + timeouts: 0, + duration_secs: 0, + unsupported: 0, + redirects: 1, + cached: 0, + suggestion_map, + redirect_map, + success_map: HashMap::default(), + error_map, + excluded_map: HashMap::default(), + detailed_stats: true, + }; + + let host_stats = Some(HostStatsMap::from(HashMap::from([( + String::from("example.com"), + HostStats { + total_requests: 5, + successful_requests: 3, + rate_limited: 1, + server_errors: 1, + cache_hits: 1, + cache_misses: 4, + ..Default::default() + }, + )]))); + + OutputStats { + response_stats, + host_stats, + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/lychee-bin/src/formatters/stats/raw.rs b/lychee-bin/src/formatters/stats/raw.rs index 0ea496ee8d..6189177ac9 100644 --- a/lychee-bin/src/formatters/stats/raw.rs +++ b/lychee-bin/src/formatters/stats/raw.rs @@ -1,7 +1,7 @@ use anyhow::Result; use super::StatsFormatter; -use crate::stats::ResponseStats; +use crate::formatters::stats::OutputStats; pub(crate) struct Raw; impl Raw { @@ -12,7 +12,7 @@ impl Raw { impl StatsFormatter for Raw { /// Don't print stats in raw mode - fn format(&self, _stats: ResponseStats) -> Result> { - Ok(None) + fn format(&self, _: OutputStats) -> Result { + Ok(String::new()) } } diff --git a/lychee-bin/src/stats.rs b/lychee-bin/src/formatters/stats/response.rs similarity index 99% rename from lychee-bin/src/stats.rs rename to lychee-bin/src/formatters/stats/response.rs index e7614affb1..6257984ee7 100644 --- a/lychee-bin/src/stats.rs +++ b/lychee-bin/src/formatters/stats/response.rs @@ -117,8 +117,9 @@ impl ResponseStats { } #[inline] + #[cfg(test)] /// Check if no responses were received - pub(crate) const fn is_empty(&self) -> bool { + const fn is_empty(&self) -> bool { self.total == 0 } } diff --git a/lychee-bin/src/host_stats.rs b/lychee-bin/src/host_stats.rs deleted file mode 100644 index a5ddbf118f..0000000000 --- a/lychee-bin/src/host_stats.rs +++ /dev/null @@ -1,26 +0,0 @@ -use anyhow::{Context, Result}; -use lychee_lib::ratelimit::HostPool; - -use crate::{formatters::get_host_stats_formatter, options::Config}; - -/// If configured to do so, output per-host statistics to stdout or the specified output file. -pub(crate) fn output_per_host_statistics(host_pool: &HostPool, config: &Config) -> Result<()> { - if !config.host_stats { - return Ok(()); - } - - let host_stats = host_pool.all_host_stats(); - let host_stats_formatter = get_host_stats_formatter(&config.format, &config.mode); - let formatted_host_stats = host_stats_formatter.format(host_stats)?; - - if let Some(output) = &config.output { - // For file output, append to the existing output - let mut file_content = std::fs::read_to_string(output).unwrap_or_default(); - file_content.push_str(&formatted_host_stats); - std::fs::write(output, file_content).context("Cannot write host stats to output file")?; - } else { - print!("{formatted_host_stats}"); - } - - Ok(()) -} diff --git a/lychee-bin/src/main.rs b/lychee-bin/src/main.rs index 1e3d910e27..2acc653272 100644 --- a/lychee-bin/src/main.rs +++ b/lychee-bin/src/main.rs @@ -85,21 +85,17 @@ mod client; mod commands; mod files_from; mod formatters; -mod host_stats; mod options; mod parse; mod progress; -mod stats; mod time; mod verbosity; -use crate::formatters::stats::output_response_statistics; -use crate::stats::ResponseStats; +use crate::formatters::stats::{OutputStats, ResponseStats, output_statistics}; use crate::{ cache::{Cache, StoreExt}, formatters::duration::Duration, generate::generate, - host_stats::output_per_host_statistics, options::{Config, LYCHEE_CACHE_FILE, LYCHEE_IGNORE_FILE, LycheeOptions}, }; @@ -393,10 +389,14 @@ async fn run(opts: &LycheeOptions) -> Result { let exit_code = if opts.config.dump { commands::dump(params).await? } else { - let (stats, cache, exit_code, host_pool) = commands::check(params).await?; - github_warning(&stats, &opts.config); - output_response_statistics(stats, &opts.config)?; - output_per_host_statistics(&host_pool, &opts.config)?; + let (response_stats, cache, exit_code, host_pool) = commands::check(params).await?; + github_warning(&response_stats, &opts.config); + + let stats = OutputStats { + response_stats, + host_stats: opts.config.host_stats.then_some(host_pool.all_host_stats()), + }; + output_statistics(stats, &opts.config)?; if opts.config.cache { cache.store(LYCHEE_CACHE_FILE)?; diff --git a/lychee-lib/src/ratelimit/host/mod.rs b/lychee-lib/src/ratelimit/host/mod.rs index 50b8b1ad3e..488531c6eb 100644 --- a/lychee-lib/src/ratelimit/host/mod.rs +++ b/lychee-lib/src/ratelimit/host/mod.rs @@ -6,4 +6,4 @@ mod stats; pub use host::Host; pub use key::HostKey; -pub use stats::HostStats; +pub use stats::{HostStats, HostStatsMap}; diff --git a/lychee-lib/src/ratelimit/host/stats.rs b/lychee-lib/src/ratelimit/host/stats.rs index c78ec43623..b57ee34d9c 100644 --- a/lychee-lib/src/ratelimit/host/stats.rs +++ b/lychee-lib/src/ratelimit/host/stats.rs @@ -1,6 +1,30 @@ use std::collections::HashMap; use std::time::{Duration, Instant}; +use serde::Serialize; +use serde::ser::SerializeStruct; + +/// A [`HashMap`] mapping hosts to their [`HostStats`] +#[derive(Debug, Default, Serialize)] +pub struct HostStatsMap(HashMap); + +impl HostStatsMap { + /// Sort host statistics by request count (descending order) + /// This matches the display order we want in the output + #[must_use] + pub fn sorted(&self) -> Vec<(String, HostStats)> { + let mut sorted_hosts: Vec<_> = self.0.clone().into_iter().collect(); + sorted_hosts.sort_by_key(|(_, stats)| std::cmp::Reverse(stats.total_requests)); + sorted_hosts + } +} + +impl From> for HostStatsMap { + fn from(value: HashMap) -> Self { + Self(value) + } +} + /// Record and report statistics for a [`crate::ratelimit::Host`] #[derive(Debug, Clone, Default)] pub struct HostStats { @@ -178,6 +202,29 @@ impl HostStats { } } +impl Serialize for HostStats { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + let median_request_time_ms = self.median_request_time().map(|d| d.as_millis()); + + let mut s = serializer.serialize_struct("HostStats", 11)?; + s.serialize_field("total_requests", &self.total_requests)?; + s.serialize_field("successful_requests", &self.successful_requests)?; + s.serialize_field("success_rate", &self.success_rate())?; + s.serialize_field("rate_limited", &self.rate_limited)?; + s.serialize_field("client_errors", &self.client_errors)?; + s.serialize_field("server_errors", &self.server_errors)?; + s.serialize_field("median_request_time_ms", &median_request_time_ms)?; + s.serialize_field("cache_hits", &self.cache_hits)?; + s.serialize_field("cache_misses", &self.cache_misses)?; + s.serialize_field("cache_hit_rate", &self.cache_hit_rate())?; + s.serialize_field("status_codes", &self.status_codes)?; + s.end() + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/lychee-lib/src/ratelimit/mod.rs b/lychee-lib/src/ratelimit/mod.rs index fac500207b..f56a4a505c 100644 --- a/lychee-lib/src/ratelimit/mod.rs +++ b/lychee-lib/src/ratelimit/mod.rs @@ -18,7 +18,7 @@ mod host; mod pool; pub use config::{HostConfig, HostConfigs, RateLimitConfig}; -pub use host::{Host, HostKey, HostStats}; +pub use host::{Host, HostKey, HostStats, HostStatsMap}; use http::HeaderMap; pub use pool::{ClientMap, HostPool}; use reqwest::Response; diff --git a/lychee-lib/src/ratelimit/pool.rs b/lychee-lib/src/ratelimit/pool.rs index df6d5027dc..a7357b5ece 100644 --- a/lychee-lib/src/ratelimit/pool.rs +++ b/lychee-lib/src/ratelimit/pool.rs @@ -4,7 +4,9 @@ use reqwest::{Client, Request}; use std::collections::HashMap; use std::sync::Arc; -use crate::ratelimit::{CacheableResponse, Host, HostConfigs, HostKey, HostStats, RateLimitConfig}; +use crate::ratelimit::{ + CacheableResponse, Host, HostConfigs, HostKey, HostStats, HostStatsMap, RateLimitConfig, +}; use crate::types::Result; use crate::{ErrorKind, Uri}; @@ -130,15 +132,17 @@ impl HostPool { /// Returns a `HashMap` mapping hostnames to their statistics. /// Only hosts that have had requests will be included. #[must_use] - pub fn all_host_stats(&self) -> HashMap { - self.hosts - .iter() - .map(|entry| { - let hostname = entry.key().to_string(); - let stats = entry.value().stats(); - (hostname, stats) - }) - .collect() + pub fn all_host_stats(&self) -> HostStatsMap { + HostStatsMap::from( + self.hosts + .iter() + .map(|entry| { + let hostname = entry.key().to_string(); + let stats = entry.value().stats(); + (hostname, stats) + }) + .collect::>(), + ) } /// Get the number of host instances that have been created, @@ -299,15 +303,4 @@ mod tests { // We can't easily test removal of existing hosts without making actual requests // due to the async nature of host creation, but the basic functionality works } - - #[test] - fn test_all_host_stats() { - let pool = HostPool::default(); - - // No hosts initially - let stats = pool.all_host_stats(); - assert!(stats.is_empty()); - - // Stats would be populated after actual requests are made to create hosts - } }