snapshot of https://github.com/mountain-loop/yaak.git (main, 2026-09-23; blobs > 131072B omitted)
Some checks failed
Lint and Test / Lint/Test (push) Has been cancelled
Generate Sponsors README / deploy (push) Has been skipped

This commit is contained in:
upstream-mirror
2026-09-23 17:39:41 +05:00
commit 906fb231cc
1544 changed files with 164131 additions and 0 deletions

View File

@@ -0,0 +1,43 @@
[package]
name = "yaak-http"
version = "0.1.0"
edition = "2024"
publish = false
[dependencies]
async-compression = { version = "0.4", features = ["tokio", "gzip", "deflate", "brotli", "zstd"] }
async-trait = "0.1"
brotli = "7"
bytes = "1.11.1"
cookie = "0.18.1"
flate2 = "1"
futures-util = "0.3"
http-body = "1"
url = "2"
zstd = "0.13"
hyper-util = { version = "0.1.17", default-features = false, features = ["client-legacy"] }
log = { workspace = true }
mime_guess = "2.0.5"
native-tls = { version = "0.2", features = ["alpn"] }
reqwest = { workspace = true, features = [
"rustls-no-provider",
"native-tls",
"socks",
"http2",
"stream",
] }
serde = { workspace = true, features = ["derive"] }
serde_json = { workspace = true }
thiserror = { workspace = true }
tokio = { workspace = true, features = ["macros", "rt", "fs", "io-util"] }
tokio-util = { version = "0.7", features = ["codec", "io", "io-util"] }
tower-service = "0.3.3"
urlencoding = "2.1.3"
yaak-common = { workspace = true }
yaak-models = { workspace = true }
yaak-templates = { workspace = true }
yaak-tls = { workspace = true }
[dev-dependencies]
tempfile = "3"
tokio = { workspace = true, features = ["net"] }

View File

@@ -0,0 +1,78 @@
use std::io;
use std::pin::Pin;
use std::task::{Context, Poll};
use tokio::io::{AsyncRead, ReadBuf};
/// A stream that chains multiple AsyncRead sources together
pub(crate) struct ChainedReader {
readers: Vec<ReaderType>,
current_index: usize,
current_reader: Option<Box<dyn AsyncRead + Send + Unpin + 'static>>,
}
#[derive(Clone)]
pub(crate) enum ReaderType {
Bytes(Vec<u8>),
FilePath(String),
}
impl ChainedReader {
pub(crate) fn new(readers: Vec<ReaderType>) -> Self {
Self { readers, current_index: 0, current_reader: None }
}
}
impl AsyncRead for ChainedReader {
fn poll_read(
mut self: Pin<&mut Self>,
cx: &mut Context<'_>,
buf: &mut ReadBuf<'_>,
) -> Poll<io::Result<()>> {
loop {
// Try to read from current reader if we have one
if let Some(ref mut reader) = self.current_reader {
let before_len = buf.filled().len();
return match Pin::new(reader).poll_read(cx, buf) {
Poll::Ready(Ok(())) => {
if buf.filled().len() == before_len && buf.remaining() > 0 {
// Current reader is exhausted, move to next
self.current_reader = None;
continue;
}
Poll::Ready(Ok(()))
}
Poll::Ready(Err(e)) => Poll::Ready(Err(e)),
Poll::Pending => Poll::Pending,
};
}
// We need to get the next reader
if self.current_index >= self.readers.len() {
// No more readers
return Poll::Ready(Ok(()));
}
// Get the next reader
let reader_type = self.readers[self.current_index].clone();
self.current_index += 1;
match reader_type {
ReaderType::Bytes(bytes) => {
self.current_reader = Some(Box::new(io::Cursor::new(bytes)));
}
ReaderType::FilePath(path) => {
// We need to handle file opening synchronously in poll_read
// This is a limitation - we'll use blocking file open
match std::fs::File::open(&path) {
Ok(file) => {
// Convert std File to tokio File
let tokio_file = tokio::fs::File::from_std(file);
self.current_reader = Some(Box::new(tokio_file));
}
Err(e) => return Poll::Ready(Err(e)),
}
}
}
}
}
}

View File

@@ -0,0 +1,504 @@
use crate::dns::{AddressFilter, LocalhostResolver};
use crate::error::Result;
use log::{debug, info, warn};
use reqwest::{Client, ClientBuilder, Proxy, redirect};
use std::sync::{Arc, Mutex};
use yaak_models::models::{DnsOverride, HttpVersion};
use yaak_tls::{
ClientCertificateConfig, NativeClientIdentity, get_tls_config, load_native_client_identity,
};
pub const HTTP2_MAX_RESPONSE_HEADER_LIST_SIZE: u32 = 1024 * 1024;
/// hyper caps HTTP/1 responses at 100 headers, which real servers exceed.
/// https://yaak.app/feedback/posts/when-response-headers-exceed-a-certain-count-hyper-throws-error
pub const HTTP1_MAX_RESPONSE_HEADERS: usize = 1024;
fn client_builder() -> ClientBuilder {
Client::builder()
.http2_max_header_list_size(HTTP2_MAX_RESPONSE_HEADER_LIST_SIZE)
.http1_max_headers(HTTP1_MAX_RESPONSE_HEADERS)
}
#[derive(Clone)]
pub struct ConfiguredClient {
inner: Client,
}
impl ConfiguredClient {
pub(crate) fn build_default() -> Result<Self> {
Ok(Self { inner: client_builder().build()? })
}
pub(crate) fn from_inner(inner: Client) -> Self {
Self { inner }
}
pub(crate) fn inner(&self) -> &Client {
&self.inner
}
}
/// Build a native-tls connector for maximum compatibility when certificate
/// validation is disabled. Unlike rustls, native-tls uses the OS TLS stack
/// (Secure Transport on macOS, SChannel on Windows, OpenSSL on Linux) which
/// supports TLS 1.0+ for legacy servers.
fn build_native_tls_connector(
client_cert: Option<ClientCertificateConfig>,
http_version: HttpVersion,
) -> Result<native_tls::TlsConnector> {
let mut builder = native_tls::TlsConnector::builder();
builder.danger_accept_invalid_certs(true);
builder.danger_accept_invalid_hostnames(true);
builder.min_protocol_version(Some(native_tls::Protocol::Tlsv10));
// reqwest cannot add ALPN to a connector it did not build, so without this
// the native path would silently negotiate HTTP/1.1 for every request.
match http_version {
HttpVersion::Auto => builder.request_alpns(&["h2", "http/1.1"]),
HttpVersion::Http1 => builder.request_alpns(&["http/1.1"]),
HttpVersion::Http2 => builder.request_alpns(&["h2"]),
};
if let Some(identity) = build_native_tls_identity(client_cert)? {
builder.identity(identity);
}
Ok(builder.build()?)
}
/// Serializes PKCS#12 imports. On macOS native-tls imports every identity into
/// one process-wide temporary keychain, and it releases its own lock before
/// importing, so concurrent imports fail with an opaque OSStatus.
static IDENTITY_IMPORT: Mutex<()> = Mutex::new(());
fn build_native_tls_identity(
client_cert: Option<ClientCertificateConfig>,
) -> Result<Option<native_tls::Identity>> {
let Some(material) = load_native_client_identity(client_cert)? else {
return Ok(None);
};
let _guard = IDENTITY_IMPORT.lock().unwrap_or_else(|e| e.into_inner());
Ok(Some(match material {
NativeClientIdentity::Pkcs12 { data, password } => {
native_tls::Identity::from_pkcs12(&data, &password)?
}
NativeClientIdentity::Pkcs8 { chain_pem, key_pem } => {
native_tls::Identity::from_pkcs8(&chain_pem, &key_pem)?
}
}))
}
#[derive(Clone)]
pub struct HttpConnectionProxySettingAuth {
pub user: String,
pub password: String,
}
#[derive(Clone)]
pub enum HttpConnectionProxySetting {
Disabled,
System,
Enabled {
http: String,
https: String,
auth: Option<HttpConnectionProxySettingAuth>,
bypass: String,
},
}
#[derive(Clone)]
pub struct HttpConnectionOptions {
pub id: String,
pub validate_certificates: bool,
pub http_version: HttpVersion,
pub proxy: HttpConnectionProxySetting,
pub client_certificate: Option<ClientCertificateConfig>,
pub dns_overrides: Vec<DnsOverride>,
/// Refuse connections to addresses a hostname resolves to. `None` means
/// every resolved address is connectable, which is what the desktop wants:
/// a user sending to their own machine or their own network is the point.
/// A hosted sender is the caller that supplies one.
pub address_filter: Option<AddressFilter>,
}
impl HttpConnectionOptions {
/// Build a reqwest Client and return it along with the DNS resolver.
/// The resolver is returned separately so it can be configured per-request
/// to emit DNS timing events to the appropriate channel.
pub fn build_client(&self) -> Result<(ConfiguredClient, Arc<LocalhostResolver>)> {
let mut client = client_builder()
.connection_verbose(true)
.redirect(redirect::Policy::none())
// Decompression is handled by HttpTransaction, not reqwest
.no_gzip()
.no_brotli()
.no_deflate()
.referer(false)
.tls_info(true)
// Disable connection pooling to ensure DNS resolution happens on each request
// This is needed so we can emit DNS timing events for each request
.pool_max_idle_per_host(0);
match self.http_version {
HttpVersion::Auto => {}
HttpVersion::Http1 => client = client.http1_only(),
HttpVersion::Http2 => client = client.http2_prior_knowledge(),
}
// Configure TLS
if self.validate_certificates {
// Use rustls with platform certificate verification (TLS 1.2+ only)
let mut config = get_tls_config(true, true, self.client_certificate.clone())?;
// A forced version must also constrain ALPN, or the server may
// negotiate a protocol the client then refuses to speak
match self.http_version {
HttpVersion::Auto => {}
HttpVersion::Http1 => config.alpn_protocols = vec![b"http/1.1".to_vec()],
HttpVersion::Http2 => config.alpn_protocols = vec![b"h2".to_vec()],
}
client = client.use_preconfigured_tls(config);
} else {
// Use native TLS for maximum compatibility (supports TLS 1.0+)
let connector =
build_native_tls_connector(self.client_certificate.clone(), self.http_version)?;
client = client.use_preconfigured_tls(connector);
}
// Configure DNS resolver - keep a reference to configure per-request
let resolver = LocalhostResolver::with_address_filter(
self.dns_overrides.clone(),
self.address_filter.clone(),
);
client = client.dns_resolver(resolver.clone());
// Configure proxy
match self.proxy.clone() {
HttpConnectionProxySetting::System => { /* Default */ }
HttpConnectionProxySetting::Disabled => {
client = client.no_proxy();
}
HttpConnectionProxySetting::Enabled { http, https, auth, bypass } => {
for p in build_enabled_proxy(http, https, auth, bypass) {
client = client.proxy(p)
}
}
}
info!(
"Building new HTTP client validate_certificates={} client_cert={}",
self.validate_certificates,
self.client_certificate.is_some()
);
Ok((ConfiguredClient::from_inner(client.build()?), resolver))
}
}
fn build_enabled_proxy(
http: String,
https: String,
auth: Option<HttpConnectionProxySettingAuth>,
bypass: String,
) -> Vec<Proxy> {
debug!("Using proxy http={http} https={https} bypass={bypass}");
let mut proxies = Vec::new();
if !http.is_empty() {
match Proxy::http(http) {
Ok(mut proxy) => {
if let Some(HttpConnectionProxySettingAuth { user, password }) = auth.clone() {
debug!("Using http proxy auth");
proxy = proxy.basic_auth(user.as_str(), password.as_str());
}
proxies.push(proxy.no_proxy(reqwest::NoProxy::from_string(&bypass)));
}
Err(e) => {
warn!("Failed to apply http proxy {e:?}");
}
};
}
if !https.is_empty() {
match Proxy::https(https) {
Ok(mut proxy) => {
if let Some(HttpConnectionProxySettingAuth { user, password }) = auth {
debug!("Using https proxy auth");
proxy = proxy.basic_auth(user.as_str(), password.as_str());
}
proxies.push(proxy.no_proxy(reqwest::NoProxy::from_string(&bypass)));
}
Err(e) => {
warn!("Failed to apply https proxy {e:?}");
}
};
}
proxies
}
#[cfg(test)]
mod client_certificate_tests {
use super::*;
use yaak_tls::get_tls_config;
// Self-signed throwaway certificates, generated only for these tests.
const RSA_CRT: &str = r#"-----BEGIN CERTIFICATE-----
MIIDEzCCAfugAwIBAgIUI3A1FyWKpilH/z1Ehnd68J9bpeEwDQYJKoZIhvcNAQEL
BQAwGDEWMBQGA1UEAwwNeWFhay10ZXN0LXJzYTAgFw0yNjA4MTQyMDQ2MjRaGA8y
MTI2MDcyMTIwNDYyNFowGDEWMBQGA1UEAwwNeWFhay10ZXN0LXJzYTCCASIwDQYJ
KoZIhvcNAQEBBQADggEPADCCAQoCggEBAK6o+exvMUs02ChHi81ILcaDkEaCzgFP
of6yDTF2m7jU1CUVwAcH4R0OKPetPVVqxPF+b/+LMdIeRIT/F5yaJgzWI6LQGmkZ
2hBrqQwuHKJq7g01+OuY0nQ13j/6HKRNKGdOiQRsKomMao1tcFYo1QMgD3lvhroO
Kr+zXPG8BP1T+b3taYyc9apeNaCouVBlY8cHk5/JxDjzOAP2s2oWBUGMSIFdjvQ+
h0oCJ5MavHXC9mMf/7nUi9bdfClZKfitUOuLiPf4qAjPW0zsyj+6IVzOG69ds5CV
/yM5B30fZxpRBqOlklisLW3r+aU1X8PsUeNyM31n8HfUzJMN+QBVmAkCAwEAAaNT
MFEwHQYDVR0OBBYEFBN40FBTuNs2ITuWp54ydZjYEx8YMB8GA1UdIwQYMBaAFBN4
0FBTuNs2ITuWp54ydZjYEx8YMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQEL
BQADggEBAHdr7OgkQueHqkV31MyZAZQqOkEn6SnNo1lKwCdl5Axby2gNbSCdZLey
sHoQwEI1nbeRjXhK/Un+UB/aPBSpNTXjh7kLC6EhEj+xL1moUqcxEBxpSHY6awim
vAqRVnxA9IeZPDQy6si6W7nomaZzvdS5YBEDh9xFwsfbo/HGaHIGtngQwnglVKt2
XedN38Z71J0ZPPdBcod5trSkJGJwsh5q4i9h1TbLvVT4Am/3IU/WsPldmQs654cn
Vaj38PL32yhzo2HUUndf+4XRQezQXcqgAZBqJTpZShXXoJuvqYobB9gdeS8HvBoa
c9aYuXK5sppEg3haLax8XRWjhra/yAI=
-----END CERTIFICATE-----"#;
const RSA_PKCS1_KEY: &str = r#"-----BEGIN RSA PRIVATE KEY-----
MIIEpAIBAAKCAQEArqj57G8xSzTYKEeLzUgtxoOQRoLOAU+h/rINMXabuNTUJRXA
BwfhHQ4o9609VWrE8X5v/4sx0h5EhP8XnJomDNYjotAaaRnaEGupDC4comruDTX4
65jSdDXeP/ocpE0oZ06JBGwqiYxqjW1wVijVAyAPeW+Gug4qv7Nc8bwE/VP5ve1p
jJz1ql41oKi5UGVjxweTn8nEOPM4A/azahYFQYxIgV2O9D6HSgInkxq8dcL2Yx//
udSL1t18KVkp+K1Q64uI9/ioCM9bTOzKP7ohXM4br12zkJX/IzkHfR9nGlEGo6WS
WKwtbev5pTVfw+xR43IzfWfwd9TMkw35AFWYCQIDAQABAoIBAATW9PvpR6BrnFr0
QwM/i4MrbJhgB1cFIIFeL3/Kkf10YyoorjY+VB89g4Lto/++cmOndaoup3blO8HD
j6hneLzU0dyw/a8+so4kwnL8l1YkdovpcbLVw5dX0cdE/vZYa/wMK4aA7gK0GRvb
oIuNwCZkjA9x1TXl/PxsXpu/1fc5j/CUj0Kk3f/5Rj5cxM/nLqxVvd+EzVgt9TVv
fyspUPK0uRz2PK4ZXIVFAbJXVcpV3V3nY7UGjy8OukOVfEbAXtYeSq/eoI4H5Biu
gS1AnfST8ESe8r4OKl0Ii1sdhVS9etmBx+/EQjyjAGDe9Q38AgJL/6kPR7fuDdvU
SJ1ZM20CgYEA2NR01F2Yt8Z8/+02v213NPK+WYqFfOq7qusnBYdcibaivbszFNKO
kAzkaiWs4seou7ga9BbF5/kKjSlx2ud3Ks548IYg6/05XddXdiN4MT2rNn9m3IAm
zis5Sk/Elhw6HIilBJJgISjQVNj19nq1o2YTCSrvlccZNqn205DAEtUCgYEAzjZT
8OBaLcF1sEGY7hpVOeOe7377dG5B0ZO3k/q9IeW8f/W0xP6rcxdK+Rdun25Uig9H
36t2VYpboWTh1adTRUQRgcTFlFjulZilvofvFIgErLpRELqSghrd/O+0e1DPvx1w
YiMjZ5VMfvoOIvoYrNtgAmRBrkq/ohDvFEIGgmUCgYAPTh/ZBapL/pTAM+xTYtSx
RhktlNuLT75jeCnO+BkOF3gxUE9wvtQVUvOkkng7ocBFT9+HLzxU/X1DLZO90ezV
drGOuMkGH1+3QgYIbsSDJUk6lY+bLOiQUPjASBUmS2PGs9aCFhr2/DyIYLAr78l2
eTQKx58VwXIEK8cic+s66QKBgQC0VdyIvZr/gr0KPAOizpKTwpS+u1zqEHYs8rLL
ja6TE1cKzHSfBlwnlUoyliRe9tyltAFWAJvG6O2DMjcxYlg3LfTleJCVUEStvMXN
3xDc8qqr53B3YcI4V4ik53f9k8lqSDN9D1+p+W3haYqtWev0VxEaZlTuOF5oO8jo
/Wi15QKBgQDOc9FRdEPrW6KEA9QJROjt8vEmElWgJu14KxMpK8Jq6HgZ2/CqC6Hl
wdphddrz57mWSuJtrAnmbSiiGyw3uUZFSQV8CLW12k5qaulxR9ODdosNjaEjJ/Rl
yUXONJ+DpurH+/ZzRKM5q1AqMG54EeX/WoCQbBOPB9zfYJJ1AKuZ5g==
-----END RSA PRIVATE KEY-----"#;
const RSA_PKCS8_KEY: &str = r#"-----BEGIN PRIVATE KEY-----
MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCuqPnsbzFLNNgo
R4vNSC3Gg5BGgs4BT6H+sg0xdpu41NQlFcAHB+EdDij3rT1VasTxfm//izHSHkSE
/xecmiYM1iOi0BppGdoQa6kMLhyiau4NNfjrmNJ0Nd4/+hykTShnTokEbCqJjGqN
bXBWKNUDIA95b4a6Diq/s1zxvAT9U/m97WmMnPWqXjWgqLlQZWPHB5OfycQ48zgD
9rNqFgVBjEiBXY70PodKAieTGrx1wvZjH/+51IvW3XwpWSn4rVDri4j3+KgIz1tM
7Mo/uiFczhuvXbOQlf8jOQd9H2caUQajpZJYrC1t6/mlNV/D7FHjcjN9Z/B31MyT
DfkAVZgJAgMBAAECggEABNb0++lHoGucWvRDAz+LgytsmGAHVwUggV4vf8qR/XRj
KiiuNj5UHz2Dgu2j/75yY6d1qi6nduU7wcOPqGd4vNTR3LD9rz6yjiTCcvyXViR2
i+lxstXDl1fRx0T+9lhr/AwrhoDuArQZG9ugi43AJmSMD3HVNeX8/Gxem7/V9zmP
8JSPQqTd//lGPlzEz+curFW934TNWC31NW9/KylQ8rS5HPY8rhlchUUBsldVylXd
XedjtQaPLw66Q5V8RsBe1h5Kr96gjgfkGK6BLUCd9JPwRJ7yvg4qXQiLWx2FVL16
2YHH78RCPKMAYN71DfwCAkv/qQ9Ht+4N29RInVkzbQKBgQDY1HTUXZi3xnz/7Ta/
bXc08r5ZioV86ruq6ycFh1yJtqK9uzMU0o6QDORqJazix6i7uBr0FsXn+QqNKXHa
53cqznjwhiDr/Tld11d2I3gxPas2f2bcgCbOKzlKT8SWHDociKUEkmAhKNBU2PX2
erWjZhMJKu+Vxxk2qfbTkMAS1QKBgQDONlPw4FotwXWwQZjuGlU5457vfvt0bkHR
k7eT+r0h5bx/9bTE/qtzF0r5F26fblSKD0ffq3ZViluhZOHVp1NFRBGBxMWUWO6V
mKW+h+8UiASsulEQupKCGt3877R7UM+/HXBiIyNnlUx++g4i+his22ACZEGuSr+i
EO8UQgaCZQKBgA9OH9kFqkv+lMAz7FNi1LFGGS2U24tPvmN4Kc74GQ4XeDFQT3C+
1BVS86SSeDuhwEVP34cvPFT9fUMtk73R7NV2sY64yQYfX7dCBghuxIMlSTqVj5ss
6JBQ+MBIFSZLY8az1oIWGvb8PIhgsCvvyXZ5NArHnxXBcgQrxyJz6zrpAoGBALRV
3Ii9mv+CvQo8A6LOkpPClL67XOoQdizyssuNrpMTVwrMdJ8GXCeVSjKWJF723KW0
AVYAm8bo7YMyNzFiWDct9OV4kJVQRK28xc3fENzyqqvncHdhwjhXiKTnd/2TyWpI
M30PX6n5beFpiq1Z6/RXERpmVO44Xmg7yOj9aLXlAoGBAM5z0VF0Q+tbooQD1AlE
6O3y8SYSVaAm7XgrEykrwmroeBnb8KoLoeXB2mF12vPnuZZK4m2sCeZtKKIbLDe5
RkVJBXwItbXaTmpq6XFH04N2iw2NoSMn9GXJRc40n4Om6sf79nNEozmrUCowbngR
5f9agJBsE48H3N9gknUAq5nm
-----END PRIVATE KEY-----"#;
const EC_CRT: &str = r#"-----BEGIN CERTIFICATE-----
MIIBhTCCASugAwIBAgIUB8703dqXCUOJQbhbyaMUMbVFOjwwCgYIKoZIzj0EAwIw
FzEVMBMGA1UEAwwMeWFhay10ZXN0LWVjMCAXDTI2MDgxNDIwNDYyNFoYDzIxMjYw
NzIxMjA0NjI0WjAXMRUwEwYDVQQDDAx5YWFrLXRlc3QtZWMwWTATBgcqhkjOPQIB
BggqhkjOPQMBBwNCAATCYYKhzgHEaRaGsYVjJSoXvoroL8qe1yeEA0VtfxFzMBg+
+bkPQ0nCtMyFfvQQtXWYIakxzsWJyhI8wPjUj6QSo1MwUTAdBgNVHQ4EFgQUKq40
Hl+2DziVkBVR/tGsPj9FRo0wHwYDVR0jBBgwFoAUKq40Hl+2DziVkBVR/tGsPj9F
Ro0wDwYDVR0TAQH/BAUwAwEB/zAKBggqhkjOPQQDAgNIADBFAiEAj1dx5XLl9iCZ
rD0CW+a3RTluxQ5icXno9WJ9qaS6L08CIFx2t0y9znQr7n5x+SmfXbfZtkDola8e
8nEZga/HXSeu
-----END CERTIFICATE-----"#;
const EC_SEC1_KEY: &str = r#"-----BEGIN EC PRIVATE KEY-----
MHcCAQEEIIoiiZ/hb4h6eHkZUVBTQFz7KLrVKJqQtWee2ygOjijNoAoGCCqGSM49
AwEHoUQDQgAEwmGCoc4BxGkWhrGFYyUqF76K6C/KntcnhANFbX8RczAYPvm5D0NJ
wrTMhX70ELV1mCGpMc7FicoSPMD41I+kEg==
-----END EC PRIVATE KEY-----"#;
const EC_PKCS8_KEY: &str = r#"-----BEGIN PRIVATE KEY-----
MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgiiKJn+FviHp4eRlR
UFNAXPsoutUompC1Z57bKA6OKM2hRANCAATCYYKhzgHEaRaGsYVjJSoXvoroL8qe
1yeEA0VtfxFzMBg++bkPQ0nCtMyFfvQQtXWYIakxzsWJyhI8wPjUj6QS
-----END PRIVATE KEY-----"#;
/// Every key format the rustls path accepts, in the encodings users
/// actually have on disk.
fn key_formats() -> Vec<(&'static str, &'static str, &'static str)> {
vec![
("RSA cert with PKCS#8 key", RSA_CRT, RSA_PKCS8_KEY),
("RSA cert with PKCS#1 key", RSA_CRT, RSA_PKCS1_KEY),
("EC cert with PKCS#8 key", EC_CRT, EC_PKCS8_KEY),
("EC cert with SEC1 key", EC_CRT, EC_SEC1_KEY),
]
}
fn write_pair(dir: &tempfile::TempDir, crt: &str, key: &str) -> ClientCertificateConfig {
let crt_path = dir.path().join("client.crt");
let key_path = dir.path().join("client.key");
std::fs::write(&crt_path, crt).unwrap();
std::fs::write(&key_path, key).unwrap();
ClientCertificateConfig {
crt_file: Some(crt_path.to_str().unwrap().to_string()),
key_file: Some(key_path.to_str().unwrap().to_string()),
pfx_file: None,
passphrase: None,
}
}
#[test]
fn native_tls_accepts_every_format_rustls_accepts() {
for (name, crt, key) in key_formats() {
let dir = tempfile::TempDir::new().unwrap();
let config = write_pair(&dir, crt, key);
get_tls_config(false, true, Some(config.clone()))
.unwrap_or_else(|e| panic!("rustls rejected {name}: {e}"));
let identity = build_native_tls_identity(Some(config))
.unwrap_or_else(|e| panic!("native-tls rejected {name}: {e}"));
assert!(identity.is_some(), "no identity built for {name}");
}
}
#[test]
fn a_leading_header_line_before_the_key_is_tolerated() {
let dir = tempfile::TempDir::new().unwrap();
let key = format!("Bag Attributes: friendlyName=client\n{RSA_PKCS8_KEY}");
let config = write_pair(&dir, RSA_CRT, &key);
assert!(build_native_tls_identity(Some(config)).unwrap().is_some());
}
#[test]
fn no_identity_is_built_without_a_configured_certificate() {
assert!(build_native_tls_identity(None).unwrap().is_none());
let empty = ClientCertificateConfig {
crt_file: Some("".into()),
key_file: Some("".into()),
pfx_file: Some("".into()),
passphrase: None,
};
assert!(build_native_tls_identity(Some(empty)).unwrap().is_none());
}
}
#[cfg(test)]
mod header_limit_tests {
use super::*;
use tokio::io::{AsyncReadExt, AsyncWriteExt};
use tokio::net::TcpListener;
/// Accepts one connection and replies with a response head of exactly
/// `total_headers` fields. The trailing `Content-Length` is one of them,
/// because hyper counts every field against its limit.
async fn serve_response_with_header_count(total_headers: usize) -> String {
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
tokio::spawn(async move {
let (mut socket, _) = listener.accept().await.unwrap();
// Read the request head so the client is not writing into a closed socket
let mut buf = [0u8; 4096];
loop {
let n = socket.read(&mut buf).await.unwrap();
if n == 0 || buf[..n].windows(4).any(|w| w == b"\r\n\r\n") {
break;
}
}
let mut response = String::from("HTTP/1.1 200 OK\r\n");
for i in 0..total_headers - 1 {
response.push_str(&format!("X-Test-{i}: value-{i}\r\n"));
}
response.push_str("Content-Length: 0\r\n\r\n");
// A response the client rejects mid-parse closes the socket under
// us, so a failed write here is an expected outcome, not a fault
let _ = socket.write_all(response.as_bytes()).await;
let _ = socket.flush().await;
});
format!("http://{addr}/")
}
fn options() -> HttpConnectionOptions {
HttpConnectionOptions {
id: "test".to_string(),
validate_certificates: true,
http_version: HttpVersion::Http1,
proxy: HttpConnectionProxySetting::Disabled,
client_certificate: None,
dns_overrides: Vec::new(),
address_filter: None,
}
}
/// hyper defaults to 100 header fields per HTTP/1 response and fails the
/// whole request past that, which is what users hit in the field.
#[tokio::test]
async fn responses_with_more_than_100_headers_are_accepted() {
let url = serve_response_with_header_count(150).await;
let (client, _resolver) = options().build_client().unwrap();
let response = client
.inner()
.get(&url)
.send()
.await
.expect("request with 150 response headers should succeed");
assert_eq!(response.status(), 200);
assert_eq!(response.headers().len(), 150);
for i in 0..149 {
assert_eq!(
response.headers().get(format!("x-test-{i}")).unwrap(),
format!("value-{i}").as_str(),
);
}
}
/// Pins both sides of the boundary. The sizes are written out rather than
/// derived from `HTTP1_MAX_RESPONSE_HEADERS`, so retuning the limit trips
/// this test instead of silently moving with it.
#[tokio::test]
async fn the_limit_is_exactly_1024_header_fields() {
let (client, _resolver) = options().build_client().unwrap();
let at_limit = serve_response_with_header_count(1024).await;
let response = client
.inner()
.get(&at_limit)
.send()
.await
.expect("a response with 1024 header fields should succeed");
assert_eq!(response.headers().len(), 1024);
let over_limit = serve_response_with_header_count(1025).await;
assert!(client.inner().get(&over_limit).send().await.is_err());
}
}

View File

@@ -0,0 +1,581 @@
//! Custom cookie handling for HTTP requests
//!
//! This module provides cookie storage and matching functionality that was previously
//! delegated to reqwest. It implements RFC 6265 cookie domain and path matching.
use log::debug;
use std::sync::{Arc, Mutex};
use std::time::{Duration, SystemTime, UNIX_EPOCH};
use url::Url;
use yaak_models::models::{Cookie, CookieDomain, CookieExpires, CookieSameSite};
/// A thread-safe cookie store that can be shared across requests
#[derive(Debug, Clone)]
pub struct CookieStore {
cookies: Arc<Mutex<Vec<Cookie>>>,
}
impl Default for CookieStore {
fn default() -> Self {
Self::new()
}
}
impl CookieStore {
/// Create a new empty cookie store
pub fn new() -> Self {
Self { cookies: Arc::new(Mutex::new(Vec::new())) }
}
/// Create a cookie store from existing cookies
pub fn from_cookies(cookies: Vec<Cookie>) -> Self {
Self { cookies: Arc::new(Mutex::new(cookies)) }
}
/// Get all cookies (for persistence)
pub fn get_all_cookies(&self) -> Vec<Cookie> {
self.cookies.lock().unwrap().clone()
}
/// Get the Cookie header value for the given URL
pub fn get_cookie_header(&self, url: &Url) -> Option<String> {
let cookies = self.cookies.lock().unwrap();
let now = SystemTime::now();
let matching_cookies: Vec<_> = cookies
.iter()
.filter(|cookie| self.cookie_matches(cookie, url, &now))
.map(|cookie| (cookie.name.clone(), cookie.value.clone()))
.collect();
if matching_cookies.is_empty() {
None
} else {
Some(
matching_cookies
.into_iter()
.map(|(name, value)| format!("{}={}", name, value))
.collect::<Vec<_>>()
.join("; "),
)
}
}
/// Parse Set-Cookie headers and add cookies to the store
pub fn store_cookies_from_response(&self, url: &Url, set_cookie_headers: &[String]) {
let mut cookies = self.cookies.lock().unwrap();
for header_value in set_cookie_headers {
if let Some(cookie) = parse_set_cookie(header_value, url) {
// Remove any existing cookie with the same name and domain
cookies.retain(|existing| !cookies_match(existing, &cookie));
debug!("Storing cookie: {} for domain {:?}", cookie.name, cookie.domain);
cookies.push(cookie);
}
}
}
/// Check if a cookie matches the given URL
fn cookie_matches(&self, cookie: &Cookie, url: &Url, now: &SystemTime) -> bool {
// Check expiration
if let CookieExpires::AtUtc(expiry_str) = &cookie.expires {
if let Ok(expiry) = parse_cookie_date(expiry_str) {
if expiry < *now {
return false;
}
}
}
// Check domain
let url_host = match url.host_str() {
Some(h) => h.to_lowercase(),
None => return false,
};
let domain_matches = match &cookie.domain {
CookieDomain::HostOnly(domain) => url_host == domain.to_lowercase(),
CookieDomain::Suffix(domain) => {
let domain_lower = domain.to_lowercase();
url_host == domain_lower || url_host.ends_with(&format!(".{}", domain_lower))
}
// NotPresent and Empty should never occur in practice since we always set domain
// when parsing Set-Cookie headers. Treat as non-matching to be safe.
CookieDomain::NotPresent | CookieDomain::Empty => false,
};
if !domain_matches {
return false;
}
// Check path
let url_path = url.path();
path_matches(url_path, &cookie.path)
}
}
/// Get a stored cookie value by name, optionally scoped to an exact stored domain.
pub fn get_cookie_value_from_jar(
cookies: impl IntoIterator<Item = Cookie>,
name: &str,
domain: Option<&str>,
) -> Option<String> {
let domain = domain.and_then(normalize_cookie_domain_filter);
cookies.into_iter().find_map(|cookie| {
if cookie.name != name {
return None;
}
if let Some(domain) = domain.as_deref() {
if !cookie_domain_matches_filter(&cookie.domain, domain) {
return None;
}
}
Some(cookie.value)
})
}
/// Parse name=value from a cookie string (raw_cookie format)
#[cfg(test)]
fn parse_cookie_name_value(raw_cookie: &str) -> Option<(String, String)> {
// The raw_cookie typically looks like "name=value" or "name=value; attr1; attr2=..."
let first_part = raw_cookie.split(';').next()?;
let mut parts = first_part.splitn(2, '=');
let name = parts.next()?.trim().to_string();
let value = parts.next().unwrap_or("").trim().to_string();
if name.is_empty() { None } else { Some((name, value)) }
}
fn normalize_cookie_domain_filter(domain: &str) -> Option<String> {
let domain = domain.trim().trim_start_matches('.').to_lowercase();
if domain.is_empty() { None } else { Some(domain) }
}
fn cookie_domain_matches_filter(cookie_domain: &CookieDomain, domain: &str) -> bool {
match cookie_domain {
CookieDomain::HostOnly(cookie_domain) | CookieDomain::Suffix(cookie_domain) => {
normalize_cookie_domain_filter(cookie_domain).is_some_and(|d| d == domain)
}
CookieDomain::NotPresent | CookieDomain::Empty => false,
}
}
/// Parse a Set-Cookie header into a Cookie
fn parse_set_cookie(header_value: &str, request_url: &Url) -> Option<Cookie> {
let parsed = cookie::Cookie::parse(header_value).ok()?;
// Determine domain
let domain = if let Some(domain_attr) = parsed.domain() {
// Domain attribute present - this is a suffix match
let domain = domain_attr.trim_start_matches('.').to_lowercase();
// Reject single-component domains (TLDs) except localhost
if is_single_component_domain(&domain) && !is_localhost(&domain) {
debug!("Rejecting cookie with single-component domain: {}", domain);
return None;
}
CookieDomain::Suffix(domain)
} else {
// No domain attribute - host-only cookie
CookieDomain::HostOnly(request_url.host_str().unwrap_or("").to_lowercase())
};
// Determine expiration
let expires = if let Some(max_age) = parsed.max_age() {
let duration = Duration::from_secs(max_age.whole_seconds().max(0) as u64);
let expiry = SystemTime::now() + duration;
let expiry_secs = expiry.duration_since(UNIX_EPOCH).unwrap_or_default().as_secs();
CookieExpires::AtUtc(format!("{}", expiry_secs))
} else if let Some(expires_time) = parsed.expires() {
match expires_time {
cookie::Expiration::DateTime(dt) => {
let timestamp = dt.unix_timestamp();
CookieExpires::AtUtc(format!("{}", timestamp))
}
cookie::Expiration::Session => CookieExpires::SessionEnd,
}
} else {
CookieExpires::SessionEnd
};
// Determine path
let path = if let Some(path_attr) = parsed.path() {
path_attr.to_string()
} else {
// Default path is the directory of the request URI
default_cookie_path(request_url.path())
};
let same_site = parsed.same_site().map(|same_site| match same_site {
cookie::SameSite::Strict => CookieSameSite::Strict,
cookie::SameSite::Lax => CookieSameSite::Lax,
cookie::SameSite::None => CookieSameSite::None,
});
Some(Cookie {
name: parsed.name().to_string(),
value: parsed.value().to_string(),
domain,
expires,
path,
secure: parsed.secure().unwrap_or(false),
http_only: parsed.http_only().unwrap_or(false),
same_site,
})
}
/// Get the default cookie path from a request path (RFC 6265 Section 5.1.4)
fn default_cookie_path(request_path: &str) -> String {
if request_path.is_empty() || !request_path.starts_with('/') {
return "/".to_string();
}
// Find the last slash
if let Some(last_slash) = request_path.rfind('/') {
if last_slash == 0 { "/".to_string() } else { request_path[..last_slash].to_string() }
} else {
"/".to_string()
}
}
/// Check if a request path matches a cookie path (RFC 6265 Section 5.1.4)
fn path_matches(request_path: &str, cookie_path: &str) -> bool {
if request_path == cookie_path {
return true;
}
if request_path.starts_with(cookie_path) {
// Cookie path must end with / or the char after cookie_path in request_path must be /
if cookie_path.ends_with('/') {
return true;
}
if request_path.chars().nth(cookie_path.len()) == Some('/') {
return true;
}
}
false
}
/// Check if two cookies match (same name and domain)
fn cookies_match(a: &Cookie, b: &Cookie) -> bool {
if a.name != b.name {
return false;
}
// Check domain match
match (&a.domain, &b.domain) {
(CookieDomain::HostOnly(d1), CookieDomain::HostOnly(d2)) => {
d1.to_lowercase() == d2.to_lowercase()
}
(CookieDomain::Suffix(d1), CookieDomain::Suffix(d2)) => {
d1.to_lowercase() == d2.to_lowercase()
}
_ => false,
}
}
/// Parse a cookie date string (Unix timestamp in our format)
fn parse_cookie_date(date_str: &str) -> Result<SystemTime, ()> {
let timestamp: i64 = date_str.parse().map_err(|_| ())?;
let duration = Duration::from_secs(timestamp.max(0) as u64);
Ok(UNIX_EPOCH + duration)
}
/// Check if a domain is a single-component domain (TLD)
/// e.g., "com", "org", "net" - domains without any dots
fn is_single_component_domain(domain: &str) -> bool {
// Empty or only dots
let trimmed = domain.trim_matches('.');
if trimmed.is_empty() {
return true;
}
// IPv6 addresses use colons, not dots - don't consider them single-component
if domain.contains(':') {
return false;
}
!trimmed.contains('.')
}
/// Check if a domain is localhost or a localhost variant
fn is_localhost(domain: &str) -> bool {
let lower = domain.to_lowercase();
lower == "localhost"
|| lower.ends_with(".localhost")
|| lower == "127.0.0.1"
|| lower == "::1"
|| lower == "[::1]"
}
#[cfg(test)]
mod tests {
use super::*;
fn cookie(raw_cookie: &str, domain: CookieDomain) -> Cookie {
let (name, value) = parse_cookie_name_value(raw_cookie).unwrap();
Cookie {
name,
value,
domain,
expires: CookieExpires::SessionEnd,
path: "/".to_string(),
secure: false,
http_only: false,
same_site: None,
}
}
#[test]
fn test_parse_cookie_name_value() {
assert_eq!(
parse_cookie_name_value("session=abc123"),
Some(("session".to_string(), "abc123".to_string()))
);
assert_eq!(
parse_cookie_name_value("name=value; Path=/; HttpOnly"),
Some(("name".to_string(), "value".to_string()))
);
assert_eq!(parse_cookie_name_value("empty="), Some(("empty".to_string(), "".to_string())));
assert_eq!(parse_cookie_name_value(""), None);
}
#[test]
fn test_path_matches() {
assert!(path_matches("/", "/"));
assert!(path_matches("/foo", "/"));
assert!(path_matches("/foo/bar", "/foo"));
assert!(path_matches("/foo/bar", "/foo/"));
assert!(!path_matches("/foobar", "/foo"));
assert!(!path_matches("/foo", "/foo/bar"));
}
#[test]
fn test_default_cookie_path() {
assert_eq!(default_cookie_path("/"), "/");
assert_eq!(default_cookie_path("/foo"), "/");
assert_eq!(default_cookie_path("/foo/bar"), "/foo");
assert_eq!(default_cookie_path("/foo/bar/baz"), "/foo/bar");
assert_eq!(default_cookie_path(""), "/");
}
#[test]
fn test_cookie_store_basic() {
let store = CookieStore::new();
let url = Url::parse("https://example.com/path").unwrap();
// Initially empty
assert!(store.get_cookie_header(&url).is_none());
// Add a cookie
store.store_cookies_from_response(&url, &["session=abc123".to_string()]);
// Should now have the cookie
let header = store.get_cookie_header(&url);
assert_eq!(header, Some("session=abc123".to_string()));
}
#[test]
fn test_cookie_domain_matching() {
let store = CookieStore::new();
let url = Url::parse("https://example.com/").unwrap();
// Cookie with domain attribute (suffix match)
store.store_cookies_from_response(
&url,
&["domain_cookie=value; Domain=example.com".to_string()],
);
// Should match example.com
assert!(store.get_cookie_header(&url).is_some());
// Should match subdomain
let subdomain_url = Url::parse("https://sub.example.com/").unwrap();
assert!(store.get_cookie_header(&subdomain_url).is_some());
// Should not match different domain
let other_url = Url::parse("https://other.com/").unwrap();
assert!(store.get_cookie_header(&other_url).is_none());
}
#[test]
fn test_cookie_path_matching() {
let store = CookieStore::new();
let url = Url::parse("https://example.com/api/v1").unwrap();
// Cookie with path
store.store_cookies_from_response(&url, &["api_cookie=value; Path=/api".to_string()]);
// Should match /api/v1
assert!(store.get_cookie_header(&url).is_some());
// Should match /api
let api_url = Url::parse("https://example.com/api").unwrap();
assert!(store.get_cookie_header(&api_url).is_some());
// Should not match /other
let other_url = Url::parse("https://example.com/other").unwrap();
assert!(store.get_cookie_header(&other_url).is_none());
}
#[test]
fn test_cookie_replacement() {
let store = CookieStore::new();
let url = Url::parse("https://example.com/").unwrap();
// Add a cookie
store.store_cookies_from_response(&url, &["session=old".to_string()]);
assert_eq!(store.get_cookie_header(&url), Some("session=old".to_string()));
// Replace with new value
store.store_cookies_from_response(&url, &["session=new".to_string()]);
assert_eq!(store.get_cookie_header(&url), Some("session=new".to_string()));
// Should only have one cookie
assert_eq!(store.get_all_cookies().len(), 1);
}
#[test]
fn test_get_cookie_value_preserves_name_only_first_match() {
let cookies = vec![
cookie("co-auth=", CookieDomain::HostOnly("foo.example.com".to_string())),
cookie("co-auth=token", CookieDomain::Suffix("example.com".to_string())),
];
assert_eq!(get_cookie_value_from_jar(cookies, "co-auth", None), Some("".to_string()));
}
#[test]
fn test_get_cookie_value_matches_domain() {
let cookies = vec![
cookie("co-auth=", CookieDomain::HostOnly("foo.example.com".to_string())),
cookie("co-auth=token", CookieDomain::Suffix("example.com".to_string())),
];
assert_eq!(
get_cookie_value_from_jar(cookies, "co-auth", Some("example.com")),
Some("token".to_string())
);
}
#[test]
fn test_get_cookie_value_normalizes_domain_filter() {
let cookies = vec![cookie(
"co-auth=token",
CookieDomain::Suffix("Example.COM".to_string()),
)];
assert_eq!(
get_cookie_value_from_jar(cookies, "co-auth", Some(" .example.com ")),
Some("token".to_string())
);
}
#[test]
fn test_get_cookie_value_requires_exact_stored_domain_match() {
let cookies = vec![cookie(
"co-auth=token",
CookieDomain::HostOnly("foo.example.com".to_string()),
)];
assert_eq!(get_cookie_value_from_jar(cookies, "co-auth", Some("example.com")), None);
}
#[test]
fn test_is_single_component_domain() {
// Single-component domains (TLDs)
assert!(is_single_component_domain("com"));
assert!(is_single_component_domain("org"));
assert!(is_single_component_domain("net"));
assert!(is_single_component_domain("localhost")); // Still single-component, but allowed separately
// Multi-component domains
assert!(!is_single_component_domain("example.com"));
assert!(!is_single_component_domain("sub.example.com"));
assert!(!is_single_component_domain("co.uk"));
// Edge cases
assert!(is_single_component_domain("")); // Empty is treated as single-component
assert!(is_single_component_domain(".")); // Only dots
assert!(is_single_component_domain("..")); // Only dots
// IPv6 addresses (have colons, not dots)
assert!(!is_single_component_domain("::1")); // IPv6 localhost
assert!(!is_single_component_domain("[::1]")); // Bracketed IPv6
assert!(!is_single_component_domain("2001:db8::1")); // IPv6 address
}
#[test]
fn test_is_localhost() {
// Localhost variants
assert!(is_localhost("localhost"));
assert!(is_localhost("LOCALHOST")); // Case-insensitive
assert!(is_localhost("sub.localhost"));
assert!(is_localhost("app.sub.localhost"));
// IP localhost
assert!(is_localhost("127.0.0.1"));
assert!(is_localhost("::1"));
assert!(is_localhost("[::1]"));
// Not localhost
assert!(!is_localhost("example.com"));
assert!(!is_localhost("localhost.com")); // .com domain, not localhost
assert!(!is_localhost("notlocalhost"));
}
#[test]
fn test_reject_tld_cookies() {
let store = CookieStore::new();
let url = Url::parse("https://example.com/").unwrap();
// Try to set a cookie with Domain=com (TLD)
store.store_cookies_from_response(&url, &["bad=cookie; Domain=com".to_string()]);
// Should be rejected - no cookies stored
assert_eq!(store.get_all_cookies().len(), 0);
assert!(store.get_cookie_header(&url).is_none());
}
#[test]
fn test_allow_localhost_cookies() {
let store = CookieStore::new();
let url = Url::parse("http://localhost:3000/").unwrap();
// Cookie with Domain=localhost should be allowed
store.store_cookies_from_response(&url, &["session=abc; Domain=localhost".to_string()]);
// Should be accepted
assert_eq!(store.get_all_cookies().len(), 1);
assert!(store.get_cookie_header(&url).is_some());
}
#[test]
fn test_allow_127_0_0_1_cookies() {
let store = CookieStore::new();
let url = Url::parse("http://127.0.0.1:8080/").unwrap();
// Cookie without Domain attribute (host-only) should work
store.store_cookies_from_response(&url, &["session=xyz".to_string()]);
// Should be accepted
assert_eq!(store.get_all_cookies().len(), 1);
assert!(store.get_cookie_header(&url).is_some());
}
#[test]
fn test_allow_normal_domain_cookies() {
let store = CookieStore::new();
let url = Url::parse("https://example.com/").unwrap();
// Cookie with valid domain should be allowed
store.store_cookies_from_response(&url, &["session=abc; Domain=example.com".to_string()]);
// Should be accepted
assert_eq!(store.get_all_cookies().len(), 1);
assert!(store.get_cookie_header(&url).is_some());
}
}

View File

@@ -0,0 +1,188 @@
use crate::error::{Error, Result};
use async_compression::tokio::bufread::{
BrotliDecoder, DeflateDecoder as AsyncDeflateDecoder, GzipDecoder,
ZstdDecoder as AsyncZstdDecoder,
};
use flate2::read::{DeflateDecoder, GzDecoder};
use std::io::Read;
use tokio::io::{AsyncBufRead, AsyncRead};
/// Supported compression encodings
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ContentEncoding {
Gzip,
Deflate,
Brotli,
Zstd,
Identity,
}
impl ContentEncoding {
/// Parse a Content-Encoding header value into an encoding type.
/// Returns Identity for unknown or missing encodings.
pub fn from_header(value: Option<&str>) -> Self {
match value.map(|s| s.trim().to_lowercase()).as_deref() {
Some("gzip") | Some("x-gzip") => ContentEncoding::Gzip,
Some("deflate") => ContentEncoding::Deflate,
Some("br") => ContentEncoding::Brotli,
Some("zstd") => ContentEncoding::Zstd,
_ => ContentEncoding::Identity,
}
}
}
/// Result of decompression, containing both the decompressed data and size info
#[derive(Debug)]
pub struct DecompressResult {
pub data: Vec<u8>,
pub compressed_size: u64,
pub decompressed_size: u64,
}
/// Decompress data based on the Content-Encoding.
/// Returns the original data unchanged if encoding is Identity or unknown.
pub fn decompress(data: Vec<u8>, encoding: ContentEncoding) -> Result<DecompressResult> {
let compressed_size = data.len() as u64;
let decompressed = match encoding {
ContentEncoding::Identity => data,
ContentEncoding::Gzip => decompress_gzip(&data)?,
ContentEncoding::Deflate => decompress_deflate(&data)?,
ContentEncoding::Brotli => decompress_brotli(&data)?,
ContentEncoding::Zstd => decompress_zstd(&data)?,
};
let decompressed_size = decompressed.len() as u64;
Ok(DecompressResult { data: decompressed, compressed_size, decompressed_size })
}
fn decompress_gzip(data: &[u8]) -> Result<Vec<u8>> {
let mut decoder = GzDecoder::new(data);
let mut decompressed = Vec::new();
decoder
.read_to_end(&mut decompressed)
.map_err(|e| Error::DecompressionError(format!("gzip decompression failed: {}", e)))?;
Ok(decompressed)
}
fn decompress_deflate(data: &[u8]) -> Result<Vec<u8>> {
let mut decoder = DeflateDecoder::new(data);
let mut decompressed = Vec::new();
decoder
.read_to_end(&mut decompressed)
.map_err(|e| Error::DecompressionError(format!("deflate decompression failed: {}", e)))?;
Ok(decompressed)
}
fn decompress_brotli(data: &[u8]) -> Result<Vec<u8>> {
let mut decompressed = Vec::new();
brotli::BrotliDecompress(&mut std::io::Cursor::new(data), &mut decompressed)
.map_err(|e| Error::DecompressionError(format!("brotli decompression failed: {}", e)))?;
Ok(decompressed)
}
fn decompress_zstd(data: &[u8]) -> Result<Vec<u8>> {
zstd::stream::decode_all(std::io::Cursor::new(data))
.map_err(|e| Error::DecompressionError(format!("zstd decompression failed: {}", e)))
}
/// Create a streaming decompressor that wraps an async reader.
/// Returns an AsyncRead that decompresses data on-the-fly.
pub fn streaming_decoder<R: AsyncBufRead + Unpin + Send + 'static>(
reader: R,
encoding: ContentEncoding,
) -> Box<dyn AsyncRead + Unpin + Send> {
match encoding {
ContentEncoding::Identity => Box::new(reader),
ContentEncoding::Gzip => Box::new(GzipDecoder::new(reader)),
ContentEncoding::Deflate => Box::new(AsyncDeflateDecoder::new(reader)),
ContentEncoding::Brotli => Box::new(BrotliDecoder::new(reader)),
ContentEncoding::Zstd => Box::new(AsyncZstdDecoder::new(reader)),
}
}
#[cfg(test)]
mod tests {
use super::*;
use flate2::Compression;
use flate2::write::GzEncoder;
use std::io::Write;
#[test]
fn test_content_encoding_from_header() {
assert_eq!(ContentEncoding::from_header(Some("gzip")), ContentEncoding::Gzip);
assert_eq!(ContentEncoding::from_header(Some("x-gzip")), ContentEncoding::Gzip);
assert_eq!(ContentEncoding::from_header(Some("GZIP")), ContentEncoding::Gzip);
assert_eq!(ContentEncoding::from_header(Some("deflate")), ContentEncoding::Deflate);
assert_eq!(ContentEncoding::from_header(Some("br")), ContentEncoding::Brotli);
assert_eq!(ContentEncoding::from_header(Some("zstd")), ContentEncoding::Zstd);
assert_eq!(ContentEncoding::from_header(Some("identity")), ContentEncoding::Identity);
assert_eq!(ContentEncoding::from_header(Some("unknown")), ContentEncoding::Identity);
assert_eq!(ContentEncoding::from_header(None), ContentEncoding::Identity);
}
#[test]
fn test_decompress_identity() {
let data = b"hello world".to_vec();
let result = decompress(data.clone(), ContentEncoding::Identity).unwrap();
assert_eq!(result.data, data);
assert_eq!(result.compressed_size, 11);
assert_eq!(result.decompressed_size, 11);
}
#[test]
fn test_decompress_gzip() {
// Compress some data with gzip
let original = b"hello world, this is a test of gzip compression";
let mut encoder = GzEncoder::new(Vec::new(), Compression::default());
encoder.write_all(original).unwrap();
let compressed = encoder.finish().unwrap();
let result = decompress(compressed.clone(), ContentEncoding::Gzip).unwrap();
assert_eq!(result.data, original);
assert_eq!(result.compressed_size, compressed.len() as u64);
assert_eq!(result.decompressed_size, original.len() as u64);
}
#[test]
fn test_decompress_deflate() {
// Compress some data with deflate
let original = b"hello world, this is a test of deflate compression";
let mut encoder = flate2::write::DeflateEncoder::new(Vec::new(), Compression::default());
encoder.write_all(original).unwrap();
let compressed = encoder.finish().unwrap();
let result = decompress(compressed.clone(), ContentEncoding::Deflate).unwrap();
assert_eq!(result.data, original);
assert_eq!(result.compressed_size, compressed.len() as u64);
assert_eq!(result.decompressed_size, original.len() as u64);
}
#[test]
fn test_decompress_brotli() {
// Compress some data with brotli
let original = b"hello world, this is a test of brotli compression";
let mut compressed = Vec::new();
let mut writer = brotli::CompressorWriter::new(&mut compressed, 4096, 4, 22);
writer.write_all(original).unwrap();
drop(writer);
let result = decompress(compressed.clone(), ContentEncoding::Brotli).unwrap();
assert_eq!(result.data, original);
assert_eq!(result.compressed_size, compressed.len() as u64);
assert_eq!(result.decompressed_size, original.len() as u64);
}
#[test]
fn test_decompress_zstd() {
// Compress some data with zstd
let original = b"hello world, this is a test of zstd compression";
let compressed = zstd::stream::encode_all(std::io::Cursor::new(original), 3).unwrap();
let result = decompress(compressed.clone(), ContentEncoding::Zstd).unwrap();
assert_eq!(result.data, original);
assert_eq!(result.compressed_size, compressed.len() as u64);
assert_eq!(result.decompressed_size, original.len() as u64);
}
}

225
crates/yaak-http/src/dns.rs Normal file
View File

@@ -0,0 +1,225 @@
use crate::sender::HttpResponseEvent;
use hyper_util::client::legacy::connect::dns::{
GaiResolver as HyperGaiResolver, Name as HyperName,
};
use log::info;
use reqwest::dns::{Addrs, Name, Resolve, Resolving};
use std::collections::HashMap;
use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr};
use std::str::FromStr;
use std::sync::Arc;
use std::time::Instant;
use tokio::sync::{RwLock, mpsc};
use tower_service::Service;
use yaak_models::models::DnsOverride;
/// Stores resolved addresses for a hostname override
#[derive(Clone)]
pub struct ResolvedOverride {
pub ipv4: Vec<Ipv4Addr>,
pub ipv6: Vec<Ipv6Addr>,
}
/// A veto on the addresses a hostname resolves to, consulted after resolution
/// and before any connection is made. Returning `Err` refuses the whole lookup
/// with that message; a hostname is never partially allowed.
///
/// A hosted sender uses this to refuse private and metadata ranges no matter
/// what name they hide behind. Checking here rather than on the URL is what
/// catches a public hostname that resolves to an internal address.
pub type AddressFilter = Arc<dyn Fn(IpAddr) -> std::result::Result<(), String> + Send + Sync>;
#[derive(Clone)]
pub struct LocalhostResolver {
fallback: HyperGaiResolver,
event_tx: Arc<RwLock<Option<mpsc::Sender<HttpResponseEvent>>>>,
overrides: Arc<HashMap<String, ResolvedOverride>>,
address_filter: Option<AddressFilter>,
}
impl LocalhostResolver {
pub fn new(dns_overrides: Vec<DnsOverride>) -> Arc<Self> {
Self::with_address_filter(dns_overrides, None)
}
pub fn with_address_filter(
dns_overrides: Vec<DnsOverride>,
address_filter: Option<AddressFilter>,
) -> Arc<Self> {
let resolver = HyperGaiResolver::new();
// Pre-parse DNS overrides into a lookup map
let mut overrides = HashMap::new();
for o in dns_overrides {
if !o.enabled {
continue;
}
let hostname = o.hostname.to_lowercase();
let ipv4: Vec<Ipv4Addr> =
o.ipv4.iter().filter_map(|s| s.parse::<Ipv4Addr>().ok()).collect();
let ipv6: Vec<Ipv6Addr> =
o.ipv6.iter().filter_map(|s| s.parse::<Ipv6Addr>().ok()).collect();
// Only add if at least one address is valid
if !ipv4.is_empty() || !ipv6.is_empty() {
overrides.insert(hostname, ResolvedOverride { ipv4, ipv6 });
}
}
Arc::new(Self {
fallback: resolver,
event_tx: Arc::new(RwLock::new(None)),
overrides: Arc::new(overrides),
address_filter,
})
}
/// Apply the address filter, if any, to a resolved address list.
fn filter_addrs(
filter: &Option<AddressFilter>,
addrs: &[SocketAddr],
) -> std::result::Result<(), Box<dyn std::error::Error + Send + Sync>> {
if let Some(filter) = filter {
for addr in addrs {
if let Err(reason) = filter(addr.ip()) {
return Err(Box::new(std::io::Error::other(reason)));
}
}
}
Ok(())
}
/// Set the event sender for the current request.
/// This should be called before each request to direct DNS events
/// to the appropriate channel.
pub async fn set_event_sender(&self, tx: Option<mpsc::Sender<HttpResponseEvent>>) {
let mut guard = self.event_tx.write().await;
*guard = tx;
}
}
impl Resolve for LocalhostResolver {
fn resolve(&self, name: Name) -> Resolving {
let host = name.as_str().to_lowercase();
let event_tx = self.event_tx.clone();
let overrides = self.overrides.clone();
let address_filter = self.address_filter.clone();
info!("DNS resolve called for: {}", host);
// Check for DNS override first
if let Some(resolved) = overrides.get(&host) {
log::debug!("DNS override found for: {}", host);
let hostname = host.clone();
let mut addrs: Vec<SocketAddr> = Vec::new();
// Add IPv4 addresses
for ip in &resolved.ipv4 {
addrs.push(SocketAddr::new(IpAddr::V4(*ip), 0));
}
// Add IPv6 addresses
for ip in &resolved.ipv6 {
addrs.push(SocketAddr::new(IpAddr::V6(*ip), 0));
}
let addresses: Vec<String> = addrs.iter().map(|a| a.ip().to_string()).collect();
return Box::pin(async move {
Self::filter_addrs(&address_filter, &addrs)?;
// Emit DNS event for override
let guard = event_tx.read().await;
if let Some(tx) = guard.as_ref() {
let _ = tx
.send(HttpResponseEvent::DnsResolved {
hostname,
addresses,
duration: 0,
overridden: true,
})
.await;
}
Ok::<Addrs, Box<dyn std::error::Error + Send + Sync>>(Box::new(addrs.into_iter()))
});
}
// Check for .localhost suffix
let is_localhost = host.ends_with(".localhost");
if is_localhost {
let hostname = host.clone();
// Port 0 is fine; reqwest replaces it with the URL's explicit
// port or the scheme's default (80/443, etc.).
let addrs: Vec<SocketAddr> = vec![
SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 0),
SocketAddr::new(IpAddr::V6(Ipv6Addr::LOCALHOST), 0),
];
let addresses: Vec<String> = addrs.iter().map(|a| a.ip().to_string()).collect();
return Box::pin(async move {
Self::filter_addrs(&address_filter, &addrs)?;
// Emit DNS event for localhost resolution
let guard = event_tx.read().await;
if let Some(tx) = guard.as_ref() {
let _ = tx
.send(HttpResponseEvent::DnsResolved {
hostname,
addresses,
duration: 0,
overridden: false,
})
.await;
}
Ok::<Addrs, Box<dyn std::error::Error + Send + Sync>>(Box::new(addrs.into_iter()))
});
}
// Fall back to system DNS
let mut fallback = self.fallback.clone();
let name_str = name.as_str().to_string();
let hostname = host.clone();
Box::pin(async move {
let start = Instant::now();
let result = match HyperName::from_str(&name_str) {
Ok(n) => fallback.call(n).await,
Err(e) => return Err(Box::new(e) as Box<dyn std::error::Error + Send + Sync>),
};
let duration = start.elapsed().as_millis() as u64;
match result {
Ok(addrs) => {
// Collect addresses for event emission
let addr_vec: Vec<SocketAddr> = addrs.collect();
Self::filter_addrs(&address_filter, &addr_vec)?;
let addresses: Vec<String> =
addr_vec.iter().map(|a| a.ip().to_string()).collect();
// Emit DNS event
let guard = event_tx.read().await;
if let Some(tx) = guard.as_ref() {
let _ = tx
.send(HttpResponseEvent::DnsResolved {
hostname,
addresses,
duration,
overridden: false,
})
.await;
}
Ok(Box::new(addr_vec.into_iter()) as Addrs)
}
Err(err) => Err(Box::new(err) as Box<dyn std::error::Error + Send + Sync>),
}
})
}
}

View File

@@ -0,0 +1,43 @@
use serde::{Serialize, Serializer};
use thiserror::Error;
#[derive(Error, Debug)]
pub enum Error {
#[error("Client error: {0:?}")]
Client(#[from] reqwest::Error),
#[error(transparent)]
TlsError(#[from] yaak_tls::error::Error),
#[error("Native TLS error: {0}")]
NativeTlsError(#[from] native_tls::Error),
#[error("IO error: {0}")]
IoError(#[from] std::io::Error),
#[error("Request failed with {0:?}")]
RequestError(String),
#[error("Request canceled")]
RequestCanceledError,
#[error("Timeout of {0:?} reached")]
RequestTimeout(std::time::Duration),
#[error("Decompression error: {0}")]
DecompressionError(String),
#[error("Failed to read response body: {0}")]
BodyReadError(String),
}
impl Serialize for Error {
fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
where
S: Serializer,
{
serializer.serialize_str(self.to_string().as_ref())
}
}
pub type Result<T> = std::result::Result<T, Error>;

View File

@@ -0,0 +1,16 @@
mod chained_reader;
pub mod client;
pub mod cookies;
pub mod decompress;
pub mod dns;
pub mod error;
pub mod manager;
mod proto;
pub mod sender;
pub mod tee_reader;
pub mod transaction;
pub mod types;
// Moved to yaak-models so the browser's wasm host can render requests with the
// same code; re-exported here so existing callers keep their path.
pub use yaak_models::path_placeholders;

View File

@@ -0,0 +1,53 @@
use crate::client::{ConfiguredClient, HttpConnectionOptions};
use crate::dns::LocalhostResolver;
use crate::error::Result;
use std::collections::BTreeMap;
use std::sync::Arc;
use std::time::{Duration, Instant};
use tokio::sync::RwLock;
/// A cached HTTP client along with its DNS resolver.
/// The resolver is needed to set the event sender per-request.
pub struct CachedClient {
pub client: ConfiguredClient,
pub resolver: Arc<LocalhostResolver>,
}
pub struct HttpConnectionManager {
connections: Arc<RwLock<BTreeMap<String, (CachedClient, Instant)>>>,
ttl: Duration,
}
impl HttpConnectionManager {
pub fn new() -> Self {
Self {
connections: Arc::new(RwLock::new(BTreeMap::new())),
ttl: Duration::from_secs(10 * 60),
}
}
pub async fn get_client(&self, opt: &HttpConnectionOptions) -> Result<CachedClient> {
let mut connections = self.connections.write().await;
// The key must include any per-request option that changes how the
// client is built, or a send after a settings change reuses a client
// built with the old value for up to the cache TTL.
let id = format!("{}::{}::{}", opt.id, opt.validate_certificates, opt.http_version);
// Clean old connections
connections.retain(|_, (_, last_used)| last_used.elapsed() <= self.ttl);
if let Some((cached, last_used)) = connections.get_mut(&id) {
*last_used = Instant::now();
return Ok(CachedClient {
client: cached.client.clone(),
resolver: cached.resolver.clone(),
});
}
let (client, resolver) = opt.build_client()?;
let cached = CachedClient { client: client.clone(), resolver: resolver.clone() };
connections.insert(id.into(), (cached, Instant::now()));
Ok(CachedClient { client, resolver })
}
}

View File

@@ -0,0 +1,29 @@
use reqwest::Url;
use std::str::FromStr;
pub(crate) fn ensure_proto(url_str: &str) -> String {
if url_str.is_empty() {
return "".to_string();
}
if url_str.starts_with("http://") || url_str.starts_with("https://") {
return url_str.to_string();
}
// Url::from_str will fail without a proto, so add one
let parseable_url = format!("http://{}", url_str);
if let Ok(u) = Url::from_str(parseable_url.as_str()) {
match u.host() {
Some(host) => {
let h = host.to_string();
// These TLDs force HTTPS
if h.ends_with(".app") || h.ends_with(".dev") || h.ends_with(".page") {
return format!("https://{url_str}");
}
}
None => {}
}
}
format!("http://{url_str}")
}

View File

@@ -0,0 +1,641 @@
use crate::decompress::{ContentEncoding, streaming_decoder};
use crate::error::{Error, Result};
use crate::types::{SendableBody, SendableHttpRequest};
use async_trait::async_trait;
use bytes::Bytes;
use futures_util::StreamExt;
use http_body::{Body as HttpBody, Frame, SizeHint};
use reqwest::{Method, Version};
use std::fmt::Display;
use std::pin::Pin;
use std::task::{Context, Poll};
use std::time::Duration;
use tokio::io::{AsyncRead, AsyncReadExt, BufReader, ReadBuf};
use tokio::sync::mpsc;
use tokio_util::io::StreamReader;
#[derive(Debug, Clone)]
pub enum RedirectBehavior {
/// 307/308: Method and body are preserved
Preserve,
/// 303 or 301/302 with POST: Method changed to GET, body dropped
DropBody,
}
#[derive(Debug, Clone)]
pub enum HttpResponseEvent {
Setting {
name: String,
value: String,
source_model: Option<String>,
source_id: Option<String>,
source_name: Option<String>,
},
Info(String),
Redirect {
url: String,
status: u16,
behavior: RedirectBehavior,
dropped_body: bool,
dropped_headers: Vec<String>,
},
SendUrl {
method: String,
scheme: String,
username: String,
password: String,
host: String,
port: u16,
path: String,
query: String,
fragment: String,
},
ReceiveUrl {
version: Version,
status: String,
},
HeaderUp(String, String),
HeaderDown(String, String),
ChunkSent {
bytes: usize,
},
ChunkReceived {
bytes: usize,
},
DnsResolved {
hostname: String,
addresses: Vec<String>,
duration: u64,
overridden: bool,
},
}
impl Display for HttpResponseEvent {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
HttpResponseEvent::Setting { name, value, .. } => {
write!(f, "* Setting {}={}", name, value)
}
HttpResponseEvent::Info(s) => write!(f, "* {}", s),
HttpResponseEvent::Redirect {
url,
status,
behavior,
dropped_body,
dropped_headers,
} => {
let behavior_str = match behavior {
RedirectBehavior::Preserve => "preserve",
RedirectBehavior::DropBody => "drop body",
};
let body_str = if *dropped_body { ", body dropped" } else { "" };
let headers_str = if dropped_headers.is_empty() {
String::new()
} else {
format!(", headers dropped: {}", dropped_headers.join(", "))
};
write!(
f,
"* Redirect {} -> {} ({}{}{})",
status, url, behavior_str, body_str, headers_str
)
}
HttpResponseEvent::SendUrl {
method,
scheme,
username,
password,
host,
port,
path,
query,
fragment,
} => {
let auth_str = if username.is_empty() && password.is_empty() {
String::new()
} else {
format!("{}:{}@", username, password)
};
let query_str =
if query.is_empty() { String::new() } else { format!("?{}", query) };
let fragment_str =
if fragment.is_empty() { String::new() } else { format!("#{}", fragment) };
write!(
f,
"> {} {}://{}{}:{}{}{}{}",
method, scheme, auth_str, host, port, path, query_str, fragment_str
)
}
HttpResponseEvent::ReceiveUrl { version, status } => {
write!(f, "< {} {}", version_to_str(version), status)
}
HttpResponseEvent::HeaderUp(name, value) => write!(f, "> {}: {}", name, value),
HttpResponseEvent::HeaderDown(name, value) => write!(f, "< {}: {}", name, value),
HttpResponseEvent::ChunkSent { bytes } => write!(f, "> [{} bytes sent]", bytes),
HttpResponseEvent::ChunkReceived { bytes } => write!(f, "< [{} bytes received]", bytes),
HttpResponseEvent::DnsResolved { hostname, addresses, duration, overridden } => {
if *overridden {
write!(f, "* DNS override {} -> {}", hostname, addresses.join(", "))
} else {
write!(
f,
"* DNS resolved {} to {} ({}ms)",
hostname,
addresses.join(", "),
duration
)
}
}
}
}
}
impl From<HttpResponseEvent> for yaak_models::models::HttpResponseEventData {
fn from(event: HttpResponseEvent) -> Self {
use yaak_models::models::HttpResponseEventData as D;
match event {
HttpResponseEvent::Setting { name, value, source_model, source_id, source_name } => {
D::Setting { name, value, source_model, source_id, source_name }
}
HttpResponseEvent::Info(message) => D::Info { message },
HttpResponseEvent::Redirect {
url,
status,
behavior,
dropped_body,
dropped_headers,
} => D::Redirect {
url,
status,
behavior: match behavior {
RedirectBehavior::Preserve => "preserve".to_string(),
RedirectBehavior::DropBody => "drop_body".to_string(),
},
dropped_body,
dropped_headers,
},
HttpResponseEvent::SendUrl {
method,
scheme,
username,
password,
host,
port,
path,
query,
fragment,
} => {
D::SendUrl { method, scheme, username, password, host, port, path, query, fragment }
}
HttpResponseEvent::ReceiveUrl { version, status } => {
D::ReceiveUrl { version: format!("{:?}", version), status }
}
HttpResponseEvent::HeaderUp(name, value) => D::HeaderUp { name, value },
HttpResponseEvent::HeaderDown(name, value) => D::HeaderDown { name, value },
HttpResponseEvent::ChunkSent { bytes } => D::ChunkSent { bytes },
HttpResponseEvent::ChunkReceived { bytes } => D::ChunkReceived { bytes },
HttpResponseEvent::DnsResolved { hostname, addresses, duration, overridden } => {
D::DnsResolved { hostname, addresses, duration, overridden }
}
}
}
}
/// Statistics about the body after consumption
#[derive(Debug, Default, Clone)]
pub struct BodyStats {
/// Size of the body as received over the wire (before decompression)
pub size_compressed: u64,
/// Size of the body after decompression
pub size_decompressed: u64,
}
/// An AsyncRead wrapper that sends chunk events as data is read
pub struct TrackingRead<R> {
inner: R,
event_tx: mpsc::Sender<HttpResponseEvent>,
ended: bool,
}
impl<R> TrackingRead<R> {
pub fn new(inner: R, event_tx: mpsc::Sender<HttpResponseEvent>) -> Self {
Self { inner, event_tx, ended: false }
}
}
impl<R: AsyncRead + Unpin> AsyncRead for TrackingRead<R> {
fn poll_read(
mut self: Pin<&mut Self>,
cx: &mut Context<'_>,
buf: &mut ReadBuf<'_>,
) -> Poll<std::io::Result<()>> {
let before = buf.filled().len();
let result = Pin::new(&mut self.inner).poll_read(cx, buf);
if let Poll::Ready(Ok(())) = &result {
let bytes_read = buf.filled().len() - before;
if bytes_read > 0 {
// Ignore send errors - receiver may have been dropped or channel is full
let _ =
self.event_tx.try_send(HttpResponseEvent::ChunkReceived { bytes: bytes_read });
} else if !self.ended {
self.ended = true;
}
}
result
}
}
/// Type alias for the body stream
type BodyStream = Pin<Box<dyn AsyncRead + Send>>;
/// HTTP response with deferred body consumption.
/// Headers are available immediately after send(), body can be consumed in different ways.
/// Note: Debug is manually implemented since BodyStream doesn't implement Debug.
pub struct HttpResponse {
/// HTTP status code
pub status: u16,
/// HTTP status reason phrase (e.g., "OK", "Not Found")
pub status_reason: Option<String>,
/// Response headers (Vec to support multiple headers with same name, e.g., Set-Cookie)
pub headers: Vec<(String, String)>,
/// Request headers (Vec to support multiple headers with same name)
pub request_headers: Vec<(String, String)>,
/// Content-Length from headers (may differ from actual body size)
pub content_length: Option<u64>,
/// Final URL (after redirects)
pub url: String,
/// Remote address of the server
pub remote_addr: Option<String>,
/// HTTP version (e.g., "HTTP/1.1", "HTTP/2")
pub version: Option<String>,
/// The body stream (consumed when calling bytes(), text(), write_to_file(), or drain())
body_stream: Option<BodyStream>,
/// Content-Encoding for decompression
encoding: ContentEncoding,
}
impl std::fmt::Debug for HttpResponse {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("HttpResponse")
.field("status", &self.status)
.field("status_reason", &self.status_reason)
.field("headers", &self.headers)
.field("content_length", &self.content_length)
.field("url", &self.url)
.field("remote_addr", &self.remote_addr)
.field("version", &self.version)
.field("body_stream", &"<stream>")
.field("encoding", &self.encoding)
.finish()
}
}
impl HttpResponse {
/// Create a new HttpResponse with an unconsumed body stream
#[allow(clippy::too_many_arguments)]
pub fn new(
status: u16,
status_reason: Option<String>,
headers: Vec<(String, String)>,
request_headers: Vec<(String, String)>,
content_length: Option<u64>,
url: String,
remote_addr: Option<String>,
version: Option<String>,
body_stream: BodyStream,
encoding: ContentEncoding,
) -> Self {
Self {
status,
status_reason,
headers,
request_headers,
content_length,
url,
remote_addr,
version,
body_stream: Some(body_stream),
encoding,
}
}
/// Consume the body and return it as bytes (loads entire body into memory).
/// Also decompresses the body if Content-Encoding is set.
pub async fn bytes(mut self) -> Result<(Vec<u8>, BodyStats)> {
let stream = self.body_stream.take().ok_or_else(|| {
Error::RequestError("Response body has already been consumed".to_string())
})?;
let buf_reader = BufReader::new(stream);
let mut decoder = streaming_decoder(buf_reader, self.encoding);
let mut decompressed = Vec::new();
let mut bytes_read = 0u64;
// Read through the decoder in chunks to track compressed size
let mut buf = [0u8; 8192];
loop {
match decoder.read(&mut buf).await {
Ok(0) => break,
Ok(n) => {
decompressed.extend_from_slice(&buf[..n]);
bytes_read += n as u64;
}
Err(e) => {
return Err(Error::BodyReadError(e.to_string()));
}
}
}
let stats = BodyStats {
// For now, we can't easily track compressed size when streaming through decoder
// Use content_length as an approximation, or decompressed size if identity encoding
size_compressed: self.content_length.unwrap_or(bytes_read),
size_decompressed: decompressed.len() as u64,
};
Ok((decompressed, stats))
}
/// Consume the body and return it as a UTF-8 string.
pub async fn text(self) -> Result<(String, BodyStats)> {
let (bytes, stats) = self.bytes().await?;
let text = String::from_utf8(bytes)
.map_err(|e| Error::RequestError(format!("Response is not valid UTF-8: {}", e)))?;
Ok((text, stats))
}
/// Take the body stream for manual consumption.
/// Returns an AsyncRead that decompresses on-the-fly if Content-Encoding is set.
/// The caller is responsible for reading and processing the stream.
pub fn into_body_stream(&mut self) -> Result<Box<dyn AsyncRead + Unpin + Send>> {
let stream = self.body_stream.take().ok_or_else(|| {
Error::RequestError("Response body has already been consumed".to_string())
})?;
let buf_reader = BufReader::new(stream);
let decoder = streaming_decoder(buf_reader, self.encoding);
Ok(decoder)
}
/// Discard the body without reading it (useful for redirects).
pub async fn drain(mut self) -> Result<()> {
let stream = self.body_stream.take().ok_or_else(|| {
Error::RequestError("Response body has already been consumed".to_string())
})?;
// Just read and discard all bytes
let mut reader = stream;
let mut buf = [0u8; 8192];
loop {
match reader.read(&mut buf).await {
Ok(0) => break,
Ok(_) => continue,
Err(e) => {
return Err(Error::RequestError(format!(
"Failed to drain response body: {}",
e
)));
}
}
}
Ok(())
}
}
/// Trait for sending HTTP requests
#[async_trait]
pub trait HttpSender: Send + Sync {
/// Send an HTTP request and return the response with headers.
/// The body is not consumed until you call bytes(), text(), write_to_file(), or drain().
/// Events are sent through the provided channel.
async fn send(
&self,
request: SendableHttpRequest,
event_tx: mpsc::Sender<HttpResponseEvent>,
) -> Result<HttpResponse>;
}
/// Reqwest-based implementation of HttpSender
pub struct ReqwestSender {
client: crate::client::ConfiguredClient,
}
impl ReqwestSender {
/// Create a new ReqwestSender with a default client
pub fn new() -> Result<Self> {
let client = crate::client::ConfiguredClient::build_default()?;
Ok(Self { client })
}
/// Create a new ReqwestSender with a configured client
pub fn with_client(client: crate::client::ConfiguredClient) -> Self {
Self { client }
}
}
#[async_trait]
impl HttpSender for ReqwestSender {
async fn send(
&self,
request: SendableHttpRequest,
event_tx: mpsc::Sender<HttpResponseEvent>,
) -> Result<HttpResponse> {
// Helper to send events (ignores errors if receiver is dropped or channel is full)
let send_event = |event: HttpResponseEvent| {
let _ = event_tx.try_send(event);
};
// Parse the HTTP method
let method = Method::from_bytes(request.method.as_bytes())
.map_err(|e| Error::RequestError(format!("Invalid HTTP method: {}", e)))?;
// Build the request
let mut req_builder = self.client.inner().request(method, &request.url);
// Add headers
for header in request.headers {
if header.0.is_empty() {
continue;
}
req_builder = req_builder.header(&header.0, &header.1);
}
// Configure timeout
if let Some(d) = request.options.timeout
&& !d.is_zero()
{
req_builder = req_builder.timeout(d);
}
// Add body
match request.body {
None => {}
Some(SendableBody::Bytes(bytes)) => {
req_builder = req_builder.body(bytes);
}
Some(SendableBody::Stream { data, content_length }) => {
// Convert AsyncRead stream to reqwest Body. If content length is
// known, wrap with a SizedBody so hyper can set Content-Length
// automatically (for both HTTP/1.1 and HTTP/2).
let stream = tokio_util::io::ReaderStream::new(data);
let body = if let Some(len) = content_length {
reqwest::Body::wrap(SizedBody::new(stream, len))
} else {
reqwest::Body::wrap_stream(stream)
};
req_builder = req_builder.body(body);
}
}
// Send the request
let sendable_req = req_builder.build()?;
send_event(HttpResponseEvent::SendUrl {
method: sendable_req.method().to_string(),
scheme: sendable_req.url().scheme().to_string(),
username: sendable_req.url().username().to_string(),
password: sendable_req.url().password().unwrap_or_default().to_string(),
host: sendable_req.url().host_str().unwrap_or_default().to_string(),
port: sendable_req.url().port_or_known_default().unwrap_or(0),
path: sendable_req.url().path().to_string(),
query: sendable_req.url().query().unwrap_or_default().to_string(),
fragment: sendable_req.url().fragment().unwrap_or_default().to_string(),
});
let mut request_headers = Vec::new();
for (name, value) in sendable_req.headers() {
let v = value.to_str().unwrap_or_default().to_string();
request_headers.push((name.to_string(), v.clone()));
send_event(HttpResponseEvent::HeaderUp(name.to_string(), v));
}
send_event(HttpResponseEvent::Info("Sending request to server".to_string()));
// Map some errors to our own, so they look nicer
let response = self.client.inner().execute(sendable_req).await.map_err(|e| {
if reqwest::Error::is_timeout(&e) {
Error::RequestTimeout(
request.options.timeout.unwrap_or(Duration::from_secs(0)).clone(),
)
} else {
Error::Client(e)
}
})?;
let status = response.status().as_u16();
let status_reason = response.status().canonical_reason().map(|s| s.to_string());
let url = response.url().to_string();
let remote_addr = response.remote_addr().map(|a| a.to_string());
let version = Some(version_to_str(&response.version()));
let content_length = response.content_length();
send_event(HttpResponseEvent::ReceiveUrl {
version: response.version(),
status: response.status().to_string(),
});
// Extract headers (use Vec to preserve duplicates like Set-Cookie)
let mut headers = Vec::new();
for (key, value) in response.headers() {
if let Ok(v) = value.to_str() {
send_event(HttpResponseEvent::HeaderDown(key.to_string(), v.to_string()));
headers.push((key.to_string(), v.to_string()));
}
}
// Determine content encoding for decompression
// HTTP headers are case-insensitive, so we need to search for any casing
let encoding = ContentEncoding::from_header(
headers
.iter()
.find(|(k, _)| k.eq_ignore_ascii_case("content-encoding"))
.map(|(_, v)| v.as_str()),
);
// Get the byte stream instead of loading into memory
let byte_stream = response.bytes_stream();
// Convert the stream to an AsyncRead
let stream_reader = StreamReader::new(
byte_stream.map(|result| result.map_err(|e| std::io::Error::other(e))),
);
// Wrap the stream with tracking to emit chunk received events via the same channel
let tracking_reader = TrackingRead::new(stream_reader, event_tx);
let body_stream: BodyStream = Box::pin(tracking_reader);
Ok(HttpResponse::new(
status,
status_reason,
headers,
request_headers,
content_length,
url,
remote_addr,
version,
body_stream,
encoding,
))
}
}
/// A wrapper around a byte stream that reports a known content length via
/// `size_hint()`. This lets hyper set the `Content-Length` header
/// automatically based on the body size, without us having to add it as an
/// explicit header — which can cause duplicate `Content-Length` headers and
/// break HTTP/2.
struct SizedBody<S> {
stream: std::sync::Mutex<S>,
remaining: u64,
}
impl<S> SizedBody<S> {
fn new(stream: S, content_length: u64) -> Self {
Self { stream: std::sync::Mutex::new(stream), remaining: content_length }
}
}
impl<S> HttpBody for SizedBody<S>
where
S: futures_util::Stream<Item = std::result::Result<Bytes, std::io::Error>>
+ Send
+ Unpin
+ 'static,
{
type Data = Bytes;
type Error = std::io::Error;
fn poll_frame(
self: Pin<&mut Self>,
cx: &mut Context<'_>,
) -> Poll<Option<std::result::Result<Frame<Self::Data>, Self::Error>>> {
let this = self.get_mut();
let mut stream = this.stream.lock().unwrap();
match stream.poll_next_unpin(cx) {
Poll::Ready(Some(Ok(chunk))) => {
this.remaining = this.remaining.saturating_sub(chunk.len() as u64);
Poll::Ready(Some(Ok(Frame::data(chunk))))
}
Poll::Ready(Some(Err(e))) => Poll::Ready(Some(Err(e))),
Poll::Ready(None) => Poll::Ready(None),
Poll::Pending => Poll::Pending,
}
}
fn size_hint(&self) -> SizeHint {
SizeHint::with_exact(self.remaining)
}
}
fn version_to_str(version: &Version) -> String {
match *version {
Version::HTTP_09 => "HTTP/0.9".to_string(),
Version::HTTP_10 => "HTTP/1.0".to_string(),
Version::HTTP_11 => "HTTP/1.1".to_string(),
Version::HTTP_2 => "HTTP/2".to_string(),
Version::HTTP_3 => "HTTP/3".to_string(),
_ => "unknown".to_string(),
}
}

View File

@@ -0,0 +1,159 @@
use std::io;
use std::pin::Pin;
use std::task::{Context, Poll};
use tokio::io::{AsyncRead, ReadBuf};
use tokio::sync::mpsc;
/// A reader that forwards all read data to a channel while also returning it to the caller.
/// This allows capturing request body data as it's being sent.
/// Uses an unbounded channel to ensure all data is captured without blocking the request.
pub struct TeeReader<R> {
inner: R,
tx: mpsc::UnboundedSender<Vec<u8>>,
}
impl<R> TeeReader<R> {
pub fn new(inner: R, tx: mpsc::UnboundedSender<Vec<u8>>) -> Self {
Self { inner, tx }
}
}
impl<R: AsyncRead + Unpin> AsyncRead for TeeReader<R> {
fn poll_read(
mut self: Pin<&mut Self>,
cx: &mut Context<'_>,
buf: &mut ReadBuf<'_>,
) -> Poll<io::Result<()>> {
let before_len = buf.filled().len();
match Pin::new(&mut self.inner).poll_read(cx, buf) {
Poll::Ready(Ok(())) => {
let after_len = buf.filled().len();
if after_len > before_len {
// Data was read, send a copy to the channel
let data = buf.filled()[before_len..after_len].to_vec();
// Send to unbounded channel - this never blocks
// Ignore error if receiver is closed
let _ = self.tx.send(data);
}
Poll::Ready(Ok(()))
}
Poll::Ready(Err(e)) => Poll::Ready(Err(e)),
Poll::Pending => Poll::Pending,
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::io::Cursor;
use tokio::io::AsyncReadExt;
#[tokio::test]
async fn test_tee_reader_captures_all_data() {
let data = b"Hello, World!";
let cursor = Cursor::new(data.to_vec());
let (tx, mut rx) = mpsc::unbounded_channel();
let mut tee = TeeReader::new(cursor, tx);
let mut output = Vec::new();
tee.read_to_end(&mut output).await.unwrap();
// Verify the reader returns the correct data
assert_eq!(output, data);
// Verify the channel received the data
let mut captured = Vec::new();
while let Ok(chunk) = rx.try_recv() {
captured.extend(chunk);
}
assert_eq!(captured, data);
}
#[tokio::test]
async fn test_tee_reader_with_chunked_reads() {
let data = b"ABCDEFGHIJKLMNOPQRSTUVWXYZ";
let cursor = Cursor::new(data.to_vec());
let (tx, mut rx) = mpsc::unbounded_channel();
let mut tee = TeeReader::new(cursor, tx);
// Read in small chunks
let mut buf = [0u8; 5];
let mut output = Vec::new();
loop {
let n = tee.read(&mut buf).await.unwrap();
if n == 0 {
break;
}
output.extend_from_slice(&buf[..n]);
}
// Verify the reader returns the correct data
assert_eq!(output, data);
// Verify the channel received all chunks
let mut captured = Vec::new();
while let Ok(chunk) = rx.try_recv() {
captured.extend(chunk);
}
assert_eq!(captured, data);
}
#[tokio::test]
async fn test_tee_reader_empty_data() {
let data: Vec<u8> = vec![];
let cursor = Cursor::new(data.clone());
let (tx, mut rx) = mpsc::unbounded_channel();
let mut tee = TeeReader::new(cursor, tx);
let mut output = Vec::new();
tee.read_to_end(&mut output).await.unwrap();
// Verify empty output
assert!(output.is_empty());
// Verify no data was sent to channel
assert!(rx.try_recv().is_err());
}
#[tokio::test]
async fn test_tee_reader_works_when_receiver_dropped() {
let data = b"Hello, World!";
let cursor = Cursor::new(data.to_vec());
let (tx, rx) = mpsc::unbounded_channel();
// Drop the receiver before reading
drop(rx);
let mut tee = TeeReader::new(cursor, tx);
let mut output = Vec::new();
// Should still work even though receiver is dropped
tee.read_to_end(&mut output).await.unwrap();
assert_eq!(output, data);
}
#[tokio::test]
async fn test_tee_reader_large_data() {
// Test with 1MB of data
let data: Vec<u8> = (0..1024 * 1024).map(|i| (i % 256) as u8).collect();
let cursor = Cursor::new(data.clone());
let (tx, mut rx) = mpsc::unbounded_channel();
let mut tee = TeeReader::new(cursor, tx);
let mut output = Vec::new();
tee.read_to_end(&mut output).await.unwrap();
// Verify the reader returns the correct data
assert_eq!(output, data);
// Verify the channel received all data
let mut captured = Vec::new();
while let Ok(chunk) = rx.try_recv() {
captured.extend(chunk);
}
assert_eq!(captured, data);
}
}

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1 @@
This is a test file!