From 1120e2e87a290e1641cfcb165ee8f9ac20c36ddf Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 24 Feb 2026 11:01:57 -0600 Subject: [PATCH 1/5] Pre-allocate the buffer based on the expected `Content-Length` with the Rust HTTP client --- rust/src/http_client.rs | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/rust/src/http_client.rs b/rust/src/http_client.rs index 9bbdff8b454..44d328ac73f 100644 --- a/rust/src/http_client.rs +++ b/rust/src/http_client.rs @@ -16,6 +16,7 @@ use std::{collections::HashMap, future::Future, sync::OnceLock}; use anyhow::Context; use futures::TryStreamExt; +use http::header; use once_cell::sync::OnceCell; use pyo3::{create_exception, exceptions::PyException, prelude::*}; use reqwest::RequestBuilder; @@ -235,8 +236,37 @@ impl HttpClient { let status = response.status(); + // Find the expected `Content-Length` so we can pre-allocate the buffer + // necessary to read the response. + let content_length = { + let content_length = response + .headers() + .get(reqwest::header::CONTENT_LENGTH) + .and_then(|value| value.to_str().ok()) + .and_then(|s| s.parse::().ok()); + + // Sanity check that the request isn't too large from the information + // they told us (may be inaccurate so we also check below as we actually + // read the bytes) + if let Some(content_length_bytes) = content_length { + if content_length_bytes > response_limit { + Err(anyhow::anyhow!( + "Response size (defined by `Content-Length`) too large" + ))?; + } + } + + content_length + }; + let mut stream = response.bytes_stream(); - let mut buffer = Vec::new(); + // Pre-allocate the buffer based on the expected `Content-Length` + let mut buffer = Vec::with_capacity( + content_length + // Default to pre-allocating nothing when the request doesn't have a + // `Content-Length` header + .unwrap_or(0), + ); while let Some(chunk) = stream.try_next().await.context("reading body")? { if buffer.len() + chunk.len() > response_limit { Err(anyhow::anyhow!("Response size too large"))?; From d61e217be7588ba2e9b02806b3695bd44c3746c4 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 24 Feb 2026 11:08:45 -0600 Subject: [PATCH 2/5] Add changelog --- changelog.d/19498.misc | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/19498.misc diff --git a/changelog.d/19498.misc b/changelog.d/19498.misc new file mode 100644 index 00000000000..7d048c283d6 --- /dev/null +++ b/changelog.d/19498.misc @@ -0,0 +1 @@ +Pre-allocate the buffer based on the expected `Content-Length` with the Rust HTTP client. From 2ca3ca3ed3635e738efe1d60a90250a3bc5124c9 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 24 Feb 2026 11:11:03 -0600 Subject: [PATCH 3/5] Fix lints --- rust/src/http_client.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/rust/src/http_client.rs b/rust/src/http_client.rs index 44d328ac73f..092c9455d5e 100644 --- a/rust/src/http_client.rs +++ b/rust/src/http_client.rs @@ -16,7 +16,6 @@ use std::{collections::HashMap, future::Future, sync::OnceLock}; use anyhow::Context; use futures::TryStreamExt; -use http::header; use once_cell::sync::OnceCell; use pyo3::{create_exception, exceptions::PyException, prelude::*}; use reqwest::RequestBuilder; From 9ccc7fc2f6b6d9c9f1d3a4c8fbc564636614ef22 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Fri, 27 Feb 2026 14:30:54 -0600 Subject: [PATCH 4/5] Better comments --- rust/src/http_client.rs | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/rust/src/http_client.rs b/rust/src/http_client.rs index 092c9455d5e..97708f1e199 100644 --- a/rust/src/http_client.rs +++ b/rust/src/http_client.rs @@ -236,7 +236,15 @@ impl HttpClient { let status = response.status(); // Find the expected `Content-Length` so we can pre-allocate the buffer - // necessary to read the response. + // necessary to read the response. It's expected that not every request will + // have a `Content-Length` header. + // + // `response.content_length()` does exist but the "value does not directly + // represents the value of the `Content-Length` header, but rather the size + // of the response’s body" + // (https://docs.rs/reqwest/latest/reqwest/struct.Response.html#method.content_length) + // and we want to avoid reading the entire body at this point because we + // purposely stream it below until the `response_limit`. let content_length = { let content_length = response .headers() @@ -258,6 +266,8 @@ impl HttpClient { content_length }; + // Stream the response to avoid allocating a giant object on the server + // above our expected `response_limit`. let mut stream = response.bytes_stream(); // Pre-allocate the buffer based on the expected `Content-Length` let mut buffer = Vec::with_capacity( From e9e75ba0a212e7e923b271b41372142150ab63ca Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Fri, 27 Feb 2026 14:39:59 -0600 Subject: [PATCH 5/5] Use `response.headers().typed_get::()` See https://github.com/element-hq/synapse/pull/19498#discussion_r2863569606 --- rust/src/http_client.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/rust/src/http_client.rs b/rust/src/http_client.rs index 97708f1e199..dd37a10426b 100644 --- a/rust/src/http_client.rs +++ b/rust/src/http_client.rs @@ -16,6 +16,7 @@ use std::{collections::HashMap, future::Future, sync::OnceLock}; use anyhow::Context; use futures::TryStreamExt; +use headers::HeaderMapExt; use once_cell::sync::OnceCell; use pyo3::{create_exception, exceptions::PyException, prelude::*}; use reqwest::RequestBuilder; @@ -248,9 +249,9 @@ impl HttpClient { let content_length = { let content_length = response .headers() - .get(reqwest::header::CONTENT_LENGTH) - .and_then(|value| value.to_str().ok()) - .and_then(|s| s.parse::().ok()); + .typed_get::() + // We need a `usize` for the `Vec::with_capacity(...)` usage below + .and_then(|content_length| content_length.0.try_into().ok()); // Sanity check that the request isn't too large from the information // they told us (may be inaccurate so we also check below as we actually