Skip to content

Commit 53748f3

Browse files
committed
Remove deduplication after source tracking
After source tracking, adding a Request to a HashSet is ineffective because it contains a unique location for each request. As an added side-effect, removing the deduplication means URLs will be sent to the stream in file order, which is nice.
1 parent b9b0ac8 commit 53748f3

1 file changed

Lines changed: 8 additions & 11 deletions

File tree

lychee-lib/src/utils/request.rs

Lines changed: 8 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -83,9 +83,9 @@ fn try_parse_into_uri(
8383
Ok(url.into())
8484
}
8585

86-
/// Create requests out of the collected URLs.
87-
/// Returns a vector of valid URLs and errors. Valid URLs are deduplicated,
88-
/// request errors are not deduplicated.
86+
/// Create requests out of the collected URLs. Returns a vector of valid URLs
87+
/// and errors. URLs are not deduplicated because repeated URLs may occur at
88+
/// different source locations.
8989
///
9090
/// If a URLs is ignored (because of the current settings),
9191
/// it will not be added to the results.
@@ -114,26 +114,23 @@ pub(crate) fn create(
114114
let fallback_base = fallback_base.use_fs_root_as_origin();
115115
let base = source_base.or_fallback(&fallback_base);
116116

117-
let mut requests = HashSet::<Request>::new();
118-
let mut errors = Vec::<RequestError>::new();
117+
let mut vec = vec![];
119118

120119
for raw_uri in uris {
121120
let result = create_request(&raw_uri, source, root_dir, base, extractor);
122121
match result {
123122
Ok(request) => {
124-
requests.insert(request);
123+
vec.push(Ok(request));
125124
}
126-
Err(e) => errors.push(RequestError::CreateRequestItem(
125+
Err(e) => vec.push(Err(RequestError::CreateRequestItem(
127126
raw_uri.clone(),
128127
source.clone(),
129128
e,
130-
)),
129+
))),
131130
}
132131
}
133132

134-
(requests.into_iter().map(Result::Ok))
135-
.chain(errors.into_iter().map(Result::Err))
136-
.collect()
133+
vec
137134
}
138135

139136
#[cfg(test)]

0 commit comments

Comments
 (0)