Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions lychee-bin/tests/data_uris.rs
Original file line number Diff line number Diff line change
Expand Up @@ -53,4 +53,16 @@ mod cli {
let output = std::str::from_utf8(&output.stdout).unwrap();
assert_eq!(output.lines().count(), 5);
}

#[test]
fn test_check_data_uris() {
let input = fixtures_path().join("TEST_DATA_URIS.html");

cargo_bin_cmd!()
.arg(input)
.arg("--exclude-loopback")
.assert()
.success()
.stdout(contains("3 Unsupported"));
}
}
2 changes: 1 addition & 1 deletion lychee-lib/src/checker/website.rs
Original file line number Diff line number Diff line change
Expand Up @@ -254,7 +254,7 @@ impl WebsiteChecker {

let request = match request {
Ok(r) => r,
Err(e) => return e.into(),
Err(e) => return Status::from(e),
};

let status = ClientRequestChains::new(vec![&self.plugin_request_chain, default_chain])
Expand Down
29 changes: 10 additions & 19 deletions lychee-lib/src/ratelimit/host/key.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,6 @@ use serde::Deserialize;
use std::fmt;
use url::Url;

use crate::ErrorKind;
use crate::types::Result;

/// A type-safe representation of a hostname for rate limiting purposes.
///
/// This extracts and normalizes hostnames from URLs to ensure consistent
Expand Down Expand Up @@ -37,30 +34,24 @@ impl HostKey {
}
}

impl TryFrom<&Url> for HostKey {
type Error = ErrorKind;

fn try_from(url: &Url) -> Result<Self> {
let host = url.host_str().ok_or_else(|| ErrorKind::InvalidUrlHost)?;
impl From<&Url> for HostKey {
fn from(url: &Url) -> Self {
let host = url.host_str().unwrap_or("unknown-host.internal");

// Normalize to lowercase for consistent lookup
Ok(HostKey(host.to_lowercase()))
HostKey(host.to_lowercase())
}
}

impl TryFrom<&crate::Uri> for HostKey {
type Error = ErrorKind;

fn try_from(uri: &crate::Uri) -> Result<Self> {
Self::try_from(&uri.url)
impl From<&crate::Uri> for HostKey {
fn from(uri: &crate::Uri) -> Self {
Self::from(&uri.url)
}
}

impl TryFrom<Url> for HostKey {
type Error = ErrorKind;

fn try_from(url: Url) -> Result<Self> {
HostKey::try_from(&url)
impl From<Url> for HostKey {
fn from(url: Url) -> Self {
Self::from(&url)
}
}

Expand Down
16 changes: 5 additions & 11 deletions lychee-lib/src/ratelimit/pool.rs
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ impl HostPool {
/// - The underlying HTTP request fails
pub(crate) async fn execute_request(&self, request: Request) -> Result<CacheableResponse> {
let url = request.url();
let host_key = HostKey::try_from(url)?;
let host_key = HostKey::from(url);
let host = self.get_or_create_host(host_key);
host.execute_request(request).await
}
Expand All @@ -82,7 +82,7 @@ impl HostPool {
/// - The request URI has no valid hostname
/// - The request fails to build
pub fn build_request(&self, method: Method, uri: &Uri) -> Result<Request> {
let host_key = HostKey::try_from(uri)?;
let host_key = HostKey::from(uri);
let host = self.get_or_create_host(host_key);
host.get_client()
.request(method, uri.url.clone())
Expand Down Expand Up @@ -195,15 +195,9 @@ impl HostPool {
/// since this is handled internally.
pub fn record_persistent_cache_hit(&self, uri: &crate::Uri) {
if !uri.is_file() && !uri.is_mail() {
match crate::ratelimit::HostKey::try_from(uri) {
Ok(key) => {
let host = self.get_or_create_host(key);
host.record_persistent_cache_hit();
}
Err(e) => {
log::debug!("Failed to record cache hit for {uri}: {e}");
}
}
let key = crate::ratelimit::HostKey::from(uri);
let host = self.get_or_create_host(key);
host.record_persistent_cache_hit();
}
}
}
Expand Down
5 changes: 4 additions & 1 deletion lychee-lib/src/types/status.rs
Original file line number Diff line number Diff line change
Expand Up @@ -298,7 +298,10 @@ impl Status {

impl From<ErrorKind> for Status {
fn from(e: ErrorKind) -> Self {
Self::Error(e)
match e {
ErrorKind::BuildRequestClient(e) => Status::from(e),
_ => Self::Error(e),
}
}
}

Expand Down
Loading