snapshot of https://github.com/mountain-loop/yaak.git (main, 2026-09-23; blobs > 131072B omitted)
This commit is contained in:
43
crates/yaak-http/Cargo.toml
Normal file
43
crates/yaak-http/Cargo.toml
Normal file
@@ -0,0 +1,43 @@
|
||||
[package]
|
||||
name = "yaak-http"
|
||||
version = "0.1.0"
|
||||
edition = "2024"
|
||||
publish = false
|
||||
|
||||
[dependencies]
|
||||
async-compression = { version = "0.4", features = ["tokio", "gzip", "deflate", "brotli", "zstd"] }
|
||||
async-trait = "0.1"
|
||||
brotli = "7"
|
||||
bytes = "1.11.1"
|
||||
cookie = "0.18.1"
|
||||
flate2 = "1"
|
||||
futures-util = "0.3"
|
||||
http-body = "1"
|
||||
url = "2"
|
||||
zstd = "0.13"
|
||||
hyper-util = { version = "0.1.17", default-features = false, features = ["client-legacy"] }
|
||||
log = { workspace = true }
|
||||
mime_guess = "2.0.5"
|
||||
native-tls = { version = "0.2", features = ["alpn"] }
|
||||
reqwest = { workspace = true, features = [
|
||||
"rustls-no-provider",
|
||||
"native-tls",
|
||||
"socks",
|
||||
"http2",
|
||||
"stream",
|
||||
] }
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
serde_json = { workspace = true }
|
||||
thiserror = { workspace = true }
|
||||
tokio = { workspace = true, features = ["macros", "rt", "fs", "io-util"] }
|
||||
tokio-util = { version = "0.7", features = ["codec", "io", "io-util"] }
|
||||
tower-service = "0.3.3"
|
||||
urlencoding = "2.1.3"
|
||||
yaak-common = { workspace = true }
|
||||
yaak-models = { workspace = true }
|
||||
yaak-templates = { workspace = true }
|
||||
yaak-tls = { workspace = true }
|
||||
|
||||
[dev-dependencies]
|
||||
tempfile = "3"
|
||||
tokio = { workspace = true, features = ["net"] }
|
||||
78
crates/yaak-http/src/chained_reader.rs
Normal file
78
crates/yaak-http/src/chained_reader.rs
Normal file
@@ -0,0 +1,78 @@
|
||||
use std::io;
|
||||
use std::pin::Pin;
|
||||
use std::task::{Context, Poll};
|
||||
use tokio::io::{AsyncRead, ReadBuf};
|
||||
|
||||
/// A stream that chains multiple AsyncRead sources together
|
||||
pub(crate) struct ChainedReader {
|
||||
readers: Vec<ReaderType>,
|
||||
current_index: usize,
|
||||
current_reader: Option<Box<dyn AsyncRead + Send + Unpin + 'static>>,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub(crate) enum ReaderType {
|
||||
Bytes(Vec<u8>),
|
||||
FilePath(String),
|
||||
}
|
||||
|
||||
impl ChainedReader {
|
||||
pub(crate) fn new(readers: Vec<ReaderType>) -> Self {
|
||||
Self { readers, current_index: 0, current_reader: None }
|
||||
}
|
||||
}
|
||||
|
||||
impl AsyncRead for ChainedReader {
|
||||
fn poll_read(
|
||||
mut self: Pin<&mut Self>,
|
||||
cx: &mut Context<'_>,
|
||||
buf: &mut ReadBuf<'_>,
|
||||
) -> Poll<io::Result<()>> {
|
||||
loop {
|
||||
// Try to read from current reader if we have one
|
||||
if let Some(ref mut reader) = self.current_reader {
|
||||
let before_len = buf.filled().len();
|
||||
return match Pin::new(reader).poll_read(cx, buf) {
|
||||
Poll::Ready(Ok(())) => {
|
||||
if buf.filled().len() == before_len && buf.remaining() > 0 {
|
||||
// Current reader is exhausted, move to next
|
||||
self.current_reader = None;
|
||||
continue;
|
||||
}
|
||||
Poll::Ready(Ok(()))
|
||||
}
|
||||
Poll::Ready(Err(e)) => Poll::Ready(Err(e)),
|
||||
Poll::Pending => Poll::Pending,
|
||||
};
|
||||
}
|
||||
|
||||
// We need to get the next reader
|
||||
if self.current_index >= self.readers.len() {
|
||||
// No more readers
|
||||
return Poll::Ready(Ok(()));
|
||||
}
|
||||
|
||||
// Get the next reader
|
||||
let reader_type = self.readers[self.current_index].clone();
|
||||
self.current_index += 1;
|
||||
|
||||
match reader_type {
|
||||
ReaderType::Bytes(bytes) => {
|
||||
self.current_reader = Some(Box::new(io::Cursor::new(bytes)));
|
||||
}
|
||||
ReaderType::FilePath(path) => {
|
||||
// We need to handle file opening synchronously in poll_read
|
||||
// This is a limitation - we'll use blocking file open
|
||||
match std::fs::File::open(&path) {
|
||||
Ok(file) => {
|
||||
// Convert std File to tokio File
|
||||
let tokio_file = tokio::fs::File::from_std(file);
|
||||
self.current_reader = Some(Box::new(tokio_file));
|
||||
}
|
||||
Err(e) => return Poll::Ready(Err(e)),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
504
crates/yaak-http/src/client.rs
Normal file
504
crates/yaak-http/src/client.rs
Normal file
@@ -0,0 +1,504 @@
|
||||
use crate::dns::{AddressFilter, LocalhostResolver};
|
||||
use crate::error::Result;
|
||||
use log::{debug, info, warn};
|
||||
use reqwest::{Client, ClientBuilder, Proxy, redirect};
|
||||
use std::sync::{Arc, Mutex};
|
||||
use yaak_models::models::{DnsOverride, HttpVersion};
|
||||
use yaak_tls::{
|
||||
ClientCertificateConfig, NativeClientIdentity, get_tls_config, load_native_client_identity,
|
||||
};
|
||||
|
||||
pub const HTTP2_MAX_RESPONSE_HEADER_LIST_SIZE: u32 = 1024 * 1024;
|
||||
|
||||
/// hyper caps HTTP/1 responses at 100 headers, which real servers exceed.
|
||||
/// https://yaak.app/feedback/posts/when-response-headers-exceed-a-certain-count-hyper-throws-error
|
||||
pub const HTTP1_MAX_RESPONSE_HEADERS: usize = 1024;
|
||||
|
||||
fn client_builder() -> ClientBuilder {
|
||||
Client::builder()
|
||||
.http2_max_header_list_size(HTTP2_MAX_RESPONSE_HEADER_LIST_SIZE)
|
||||
.http1_max_headers(HTTP1_MAX_RESPONSE_HEADERS)
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct ConfiguredClient {
|
||||
inner: Client,
|
||||
}
|
||||
|
||||
impl ConfiguredClient {
|
||||
pub(crate) fn build_default() -> Result<Self> {
|
||||
Ok(Self { inner: client_builder().build()? })
|
||||
}
|
||||
|
||||
pub(crate) fn from_inner(inner: Client) -> Self {
|
||||
Self { inner }
|
||||
}
|
||||
|
||||
pub(crate) fn inner(&self) -> &Client {
|
||||
&self.inner
|
||||
}
|
||||
}
|
||||
|
||||
/// Build a native-tls connector for maximum compatibility when certificate
|
||||
/// validation is disabled. Unlike rustls, native-tls uses the OS TLS stack
|
||||
/// (Secure Transport on macOS, SChannel on Windows, OpenSSL on Linux) which
|
||||
/// supports TLS 1.0+ for legacy servers.
|
||||
fn build_native_tls_connector(
|
||||
client_cert: Option<ClientCertificateConfig>,
|
||||
http_version: HttpVersion,
|
||||
) -> Result<native_tls::TlsConnector> {
|
||||
let mut builder = native_tls::TlsConnector::builder();
|
||||
builder.danger_accept_invalid_certs(true);
|
||||
builder.danger_accept_invalid_hostnames(true);
|
||||
builder.min_protocol_version(Some(native_tls::Protocol::Tlsv10));
|
||||
// reqwest cannot add ALPN to a connector it did not build, so without this
|
||||
// the native path would silently negotiate HTTP/1.1 for every request.
|
||||
match http_version {
|
||||
HttpVersion::Auto => builder.request_alpns(&["h2", "http/1.1"]),
|
||||
HttpVersion::Http1 => builder.request_alpns(&["http/1.1"]),
|
||||
HttpVersion::Http2 => builder.request_alpns(&["h2"]),
|
||||
};
|
||||
|
||||
if let Some(identity) = build_native_tls_identity(client_cert)? {
|
||||
builder.identity(identity);
|
||||
}
|
||||
|
||||
Ok(builder.build()?)
|
||||
}
|
||||
|
||||
/// Serializes PKCS#12 imports. On macOS native-tls imports every identity into
|
||||
/// one process-wide temporary keychain, and it releases its own lock before
|
||||
/// importing, so concurrent imports fail with an opaque OSStatus.
|
||||
static IDENTITY_IMPORT: Mutex<()> = Mutex::new(());
|
||||
|
||||
fn build_native_tls_identity(
|
||||
client_cert: Option<ClientCertificateConfig>,
|
||||
) -> Result<Option<native_tls::Identity>> {
|
||||
let Some(material) = load_native_client_identity(client_cert)? else {
|
||||
return Ok(None);
|
||||
};
|
||||
|
||||
let _guard = IDENTITY_IMPORT.lock().unwrap_or_else(|e| e.into_inner());
|
||||
Ok(Some(match material {
|
||||
NativeClientIdentity::Pkcs12 { data, password } => {
|
||||
native_tls::Identity::from_pkcs12(&data, &password)?
|
||||
}
|
||||
NativeClientIdentity::Pkcs8 { chain_pem, key_pem } => {
|
||||
native_tls::Identity::from_pkcs8(&chain_pem, &key_pem)?
|
||||
}
|
||||
}))
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct HttpConnectionProxySettingAuth {
|
||||
pub user: String,
|
||||
pub password: String,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub enum HttpConnectionProxySetting {
|
||||
Disabled,
|
||||
System,
|
||||
Enabled {
|
||||
http: String,
|
||||
https: String,
|
||||
auth: Option<HttpConnectionProxySettingAuth>,
|
||||
bypass: String,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct HttpConnectionOptions {
|
||||
pub id: String,
|
||||
pub validate_certificates: bool,
|
||||
pub http_version: HttpVersion,
|
||||
pub proxy: HttpConnectionProxySetting,
|
||||
pub client_certificate: Option<ClientCertificateConfig>,
|
||||
pub dns_overrides: Vec<DnsOverride>,
|
||||
/// Refuse connections to addresses a hostname resolves to. `None` means
|
||||
/// every resolved address is connectable, which is what the desktop wants:
|
||||
/// a user sending to their own machine or their own network is the point.
|
||||
/// A hosted sender is the caller that supplies one.
|
||||
pub address_filter: Option<AddressFilter>,
|
||||
}
|
||||
|
||||
impl HttpConnectionOptions {
|
||||
/// Build a reqwest Client and return it along with the DNS resolver.
|
||||
/// The resolver is returned separately so it can be configured per-request
|
||||
/// to emit DNS timing events to the appropriate channel.
|
||||
pub fn build_client(&self) -> Result<(ConfiguredClient, Arc<LocalhostResolver>)> {
|
||||
let mut client = client_builder()
|
||||
.connection_verbose(true)
|
||||
.redirect(redirect::Policy::none())
|
||||
// Decompression is handled by HttpTransaction, not reqwest
|
||||
.no_gzip()
|
||||
.no_brotli()
|
||||
.no_deflate()
|
||||
.referer(false)
|
||||
.tls_info(true)
|
||||
// Disable connection pooling to ensure DNS resolution happens on each request
|
||||
// This is needed so we can emit DNS timing events for each request
|
||||
.pool_max_idle_per_host(0);
|
||||
|
||||
match self.http_version {
|
||||
HttpVersion::Auto => {}
|
||||
HttpVersion::Http1 => client = client.http1_only(),
|
||||
HttpVersion::Http2 => client = client.http2_prior_knowledge(),
|
||||
}
|
||||
|
||||
// Configure TLS
|
||||
if self.validate_certificates {
|
||||
// Use rustls with platform certificate verification (TLS 1.2+ only)
|
||||
let mut config = get_tls_config(true, true, self.client_certificate.clone())?;
|
||||
// A forced version must also constrain ALPN, or the server may
|
||||
// negotiate a protocol the client then refuses to speak
|
||||
match self.http_version {
|
||||
HttpVersion::Auto => {}
|
||||
HttpVersion::Http1 => config.alpn_protocols = vec![b"http/1.1".to_vec()],
|
||||
HttpVersion::Http2 => config.alpn_protocols = vec![b"h2".to_vec()],
|
||||
}
|
||||
client = client.use_preconfigured_tls(config);
|
||||
} else {
|
||||
// Use native TLS for maximum compatibility (supports TLS 1.0+)
|
||||
let connector =
|
||||
build_native_tls_connector(self.client_certificate.clone(), self.http_version)?;
|
||||
client = client.use_preconfigured_tls(connector);
|
||||
}
|
||||
|
||||
// Configure DNS resolver - keep a reference to configure per-request
|
||||
let resolver = LocalhostResolver::with_address_filter(
|
||||
self.dns_overrides.clone(),
|
||||
self.address_filter.clone(),
|
||||
);
|
||||
client = client.dns_resolver(resolver.clone());
|
||||
|
||||
// Configure proxy
|
||||
match self.proxy.clone() {
|
||||
HttpConnectionProxySetting::System => { /* Default */ }
|
||||
HttpConnectionProxySetting::Disabled => {
|
||||
client = client.no_proxy();
|
||||
}
|
||||
HttpConnectionProxySetting::Enabled { http, https, auth, bypass } => {
|
||||
for p in build_enabled_proxy(http, https, auth, bypass) {
|
||||
client = client.proxy(p)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
info!(
|
||||
"Building new HTTP client validate_certificates={} client_cert={}",
|
||||
self.validate_certificates,
|
||||
self.client_certificate.is_some()
|
||||
);
|
||||
|
||||
Ok((ConfiguredClient::from_inner(client.build()?), resolver))
|
||||
}
|
||||
}
|
||||
|
||||
fn build_enabled_proxy(
|
||||
http: String,
|
||||
https: String,
|
||||
auth: Option<HttpConnectionProxySettingAuth>,
|
||||
bypass: String,
|
||||
) -> Vec<Proxy> {
|
||||
debug!("Using proxy http={http} https={https} bypass={bypass}");
|
||||
|
||||
let mut proxies = Vec::new();
|
||||
|
||||
if !http.is_empty() {
|
||||
match Proxy::http(http) {
|
||||
Ok(mut proxy) => {
|
||||
if let Some(HttpConnectionProxySettingAuth { user, password }) = auth.clone() {
|
||||
debug!("Using http proxy auth");
|
||||
proxy = proxy.basic_auth(user.as_str(), password.as_str());
|
||||
}
|
||||
proxies.push(proxy.no_proxy(reqwest::NoProxy::from_string(&bypass)));
|
||||
}
|
||||
Err(e) => {
|
||||
warn!("Failed to apply http proxy {e:?}");
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
if !https.is_empty() {
|
||||
match Proxy::https(https) {
|
||||
Ok(mut proxy) => {
|
||||
if let Some(HttpConnectionProxySettingAuth { user, password }) = auth {
|
||||
debug!("Using https proxy auth");
|
||||
proxy = proxy.basic_auth(user.as_str(), password.as_str());
|
||||
}
|
||||
proxies.push(proxy.no_proxy(reqwest::NoProxy::from_string(&bypass)));
|
||||
}
|
||||
Err(e) => {
|
||||
warn!("Failed to apply https proxy {e:?}");
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
proxies
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod client_certificate_tests {
|
||||
use super::*;
|
||||
use yaak_tls::get_tls_config;
|
||||
|
||||
// Self-signed throwaway certificates, generated only for these tests.
|
||||
const RSA_CRT: &str = r#"-----BEGIN CERTIFICATE-----
|
||||
MIIDEzCCAfugAwIBAgIUI3A1FyWKpilH/z1Ehnd68J9bpeEwDQYJKoZIhvcNAQEL
|
||||
BQAwGDEWMBQGA1UEAwwNeWFhay10ZXN0LXJzYTAgFw0yNjA4MTQyMDQ2MjRaGA8y
|
||||
MTI2MDcyMTIwNDYyNFowGDEWMBQGA1UEAwwNeWFhay10ZXN0LXJzYTCCASIwDQYJ
|
||||
KoZIhvcNAQEBBQADggEPADCCAQoCggEBAK6o+exvMUs02ChHi81ILcaDkEaCzgFP
|
||||
of6yDTF2m7jU1CUVwAcH4R0OKPetPVVqxPF+b/+LMdIeRIT/F5yaJgzWI6LQGmkZ
|
||||
2hBrqQwuHKJq7g01+OuY0nQ13j/6HKRNKGdOiQRsKomMao1tcFYo1QMgD3lvhroO
|
||||
Kr+zXPG8BP1T+b3taYyc9apeNaCouVBlY8cHk5/JxDjzOAP2s2oWBUGMSIFdjvQ+
|
||||
h0oCJ5MavHXC9mMf/7nUi9bdfClZKfitUOuLiPf4qAjPW0zsyj+6IVzOG69ds5CV
|
||||
/yM5B30fZxpRBqOlklisLW3r+aU1X8PsUeNyM31n8HfUzJMN+QBVmAkCAwEAAaNT
|
||||
MFEwHQYDVR0OBBYEFBN40FBTuNs2ITuWp54ydZjYEx8YMB8GA1UdIwQYMBaAFBN4
|
||||
0FBTuNs2ITuWp54ydZjYEx8YMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQEL
|
||||
BQADggEBAHdr7OgkQueHqkV31MyZAZQqOkEn6SnNo1lKwCdl5Axby2gNbSCdZLey
|
||||
sHoQwEI1nbeRjXhK/Un+UB/aPBSpNTXjh7kLC6EhEj+xL1moUqcxEBxpSHY6awim
|
||||
vAqRVnxA9IeZPDQy6si6W7nomaZzvdS5YBEDh9xFwsfbo/HGaHIGtngQwnglVKt2
|
||||
XedN38Z71J0ZPPdBcod5trSkJGJwsh5q4i9h1TbLvVT4Am/3IU/WsPldmQs654cn
|
||||
Vaj38PL32yhzo2HUUndf+4XRQezQXcqgAZBqJTpZShXXoJuvqYobB9gdeS8HvBoa
|
||||
c9aYuXK5sppEg3haLax8XRWjhra/yAI=
|
||||
-----END CERTIFICATE-----"#;
|
||||
|
||||
const RSA_PKCS1_KEY: &str = r#"-----BEGIN RSA PRIVATE KEY-----
|
||||
MIIEpAIBAAKCAQEArqj57G8xSzTYKEeLzUgtxoOQRoLOAU+h/rINMXabuNTUJRXA
|
||||
BwfhHQ4o9609VWrE8X5v/4sx0h5EhP8XnJomDNYjotAaaRnaEGupDC4comruDTX4
|
||||
65jSdDXeP/ocpE0oZ06JBGwqiYxqjW1wVijVAyAPeW+Gug4qv7Nc8bwE/VP5ve1p
|
||||
jJz1ql41oKi5UGVjxweTn8nEOPM4A/azahYFQYxIgV2O9D6HSgInkxq8dcL2Yx//
|
||||
udSL1t18KVkp+K1Q64uI9/ioCM9bTOzKP7ohXM4br12zkJX/IzkHfR9nGlEGo6WS
|
||||
WKwtbev5pTVfw+xR43IzfWfwd9TMkw35AFWYCQIDAQABAoIBAATW9PvpR6BrnFr0
|
||||
QwM/i4MrbJhgB1cFIIFeL3/Kkf10YyoorjY+VB89g4Lto/++cmOndaoup3blO8HD
|
||||
j6hneLzU0dyw/a8+so4kwnL8l1YkdovpcbLVw5dX0cdE/vZYa/wMK4aA7gK0GRvb
|
||||
oIuNwCZkjA9x1TXl/PxsXpu/1fc5j/CUj0Kk3f/5Rj5cxM/nLqxVvd+EzVgt9TVv
|
||||
fyspUPK0uRz2PK4ZXIVFAbJXVcpV3V3nY7UGjy8OukOVfEbAXtYeSq/eoI4H5Biu
|
||||
gS1AnfST8ESe8r4OKl0Ii1sdhVS9etmBx+/EQjyjAGDe9Q38AgJL/6kPR7fuDdvU
|
||||
SJ1ZM20CgYEA2NR01F2Yt8Z8/+02v213NPK+WYqFfOq7qusnBYdcibaivbszFNKO
|
||||
kAzkaiWs4seou7ga9BbF5/kKjSlx2ud3Ks548IYg6/05XddXdiN4MT2rNn9m3IAm
|
||||
zis5Sk/Elhw6HIilBJJgISjQVNj19nq1o2YTCSrvlccZNqn205DAEtUCgYEAzjZT
|
||||
8OBaLcF1sEGY7hpVOeOe7377dG5B0ZO3k/q9IeW8f/W0xP6rcxdK+Rdun25Uig9H
|
||||
36t2VYpboWTh1adTRUQRgcTFlFjulZilvofvFIgErLpRELqSghrd/O+0e1DPvx1w
|
||||
YiMjZ5VMfvoOIvoYrNtgAmRBrkq/ohDvFEIGgmUCgYAPTh/ZBapL/pTAM+xTYtSx
|
||||
RhktlNuLT75jeCnO+BkOF3gxUE9wvtQVUvOkkng7ocBFT9+HLzxU/X1DLZO90ezV
|
||||
drGOuMkGH1+3QgYIbsSDJUk6lY+bLOiQUPjASBUmS2PGs9aCFhr2/DyIYLAr78l2
|
||||
eTQKx58VwXIEK8cic+s66QKBgQC0VdyIvZr/gr0KPAOizpKTwpS+u1zqEHYs8rLL
|
||||
ja6TE1cKzHSfBlwnlUoyliRe9tyltAFWAJvG6O2DMjcxYlg3LfTleJCVUEStvMXN
|
||||
3xDc8qqr53B3YcI4V4ik53f9k8lqSDN9D1+p+W3haYqtWev0VxEaZlTuOF5oO8jo
|
||||
/Wi15QKBgQDOc9FRdEPrW6KEA9QJROjt8vEmElWgJu14KxMpK8Jq6HgZ2/CqC6Hl
|
||||
wdphddrz57mWSuJtrAnmbSiiGyw3uUZFSQV8CLW12k5qaulxR9ODdosNjaEjJ/Rl
|
||||
yUXONJ+DpurH+/ZzRKM5q1AqMG54EeX/WoCQbBOPB9zfYJJ1AKuZ5g==
|
||||
-----END RSA PRIVATE KEY-----"#;
|
||||
|
||||
const RSA_PKCS8_KEY: &str = r#"-----BEGIN PRIVATE KEY-----
|
||||
MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCuqPnsbzFLNNgo
|
||||
R4vNSC3Gg5BGgs4BT6H+sg0xdpu41NQlFcAHB+EdDij3rT1VasTxfm//izHSHkSE
|
||||
/xecmiYM1iOi0BppGdoQa6kMLhyiau4NNfjrmNJ0Nd4/+hykTShnTokEbCqJjGqN
|
||||
bXBWKNUDIA95b4a6Diq/s1zxvAT9U/m97WmMnPWqXjWgqLlQZWPHB5OfycQ48zgD
|
||||
9rNqFgVBjEiBXY70PodKAieTGrx1wvZjH/+51IvW3XwpWSn4rVDri4j3+KgIz1tM
|
||||
7Mo/uiFczhuvXbOQlf8jOQd9H2caUQajpZJYrC1t6/mlNV/D7FHjcjN9Z/B31MyT
|
||||
DfkAVZgJAgMBAAECggEABNb0++lHoGucWvRDAz+LgytsmGAHVwUggV4vf8qR/XRj
|
||||
KiiuNj5UHz2Dgu2j/75yY6d1qi6nduU7wcOPqGd4vNTR3LD9rz6yjiTCcvyXViR2
|
||||
i+lxstXDl1fRx0T+9lhr/AwrhoDuArQZG9ugi43AJmSMD3HVNeX8/Gxem7/V9zmP
|
||||
8JSPQqTd//lGPlzEz+curFW934TNWC31NW9/KylQ8rS5HPY8rhlchUUBsldVylXd
|
||||
XedjtQaPLw66Q5V8RsBe1h5Kr96gjgfkGK6BLUCd9JPwRJ7yvg4qXQiLWx2FVL16
|
||||
2YHH78RCPKMAYN71DfwCAkv/qQ9Ht+4N29RInVkzbQKBgQDY1HTUXZi3xnz/7Ta/
|
||||
bXc08r5ZioV86ruq6ycFh1yJtqK9uzMU0o6QDORqJazix6i7uBr0FsXn+QqNKXHa
|
||||
53cqznjwhiDr/Tld11d2I3gxPas2f2bcgCbOKzlKT8SWHDociKUEkmAhKNBU2PX2
|
||||
erWjZhMJKu+Vxxk2qfbTkMAS1QKBgQDONlPw4FotwXWwQZjuGlU5457vfvt0bkHR
|
||||
k7eT+r0h5bx/9bTE/qtzF0r5F26fblSKD0ffq3ZViluhZOHVp1NFRBGBxMWUWO6V
|
||||
mKW+h+8UiASsulEQupKCGt3877R7UM+/HXBiIyNnlUx++g4i+his22ACZEGuSr+i
|
||||
EO8UQgaCZQKBgA9OH9kFqkv+lMAz7FNi1LFGGS2U24tPvmN4Kc74GQ4XeDFQT3C+
|
||||
1BVS86SSeDuhwEVP34cvPFT9fUMtk73R7NV2sY64yQYfX7dCBghuxIMlSTqVj5ss
|
||||
6JBQ+MBIFSZLY8az1oIWGvb8PIhgsCvvyXZ5NArHnxXBcgQrxyJz6zrpAoGBALRV
|
||||
3Ii9mv+CvQo8A6LOkpPClL67XOoQdizyssuNrpMTVwrMdJ8GXCeVSjKWJF723KW0
|
||||
AVYAm8bo7YMyNzFiWDct9OV4kJVQRK28xc3fENzyqqvncHdhwjhXiKTnd/2TyWpI
|
||||
M30PX6n5beFpiq1Z6/RXERpmVO44Xmg7yOj9aLXlAoGBAM5z0VF0Q+tbooQD1AlE
|
||||
6O3y8SYSVaAm7XgrEykrwmroeBnb8KoLoeXB2mF12vPnuZZK4m2sCeZtKKIbLDe5
|
||||
RkVJBXwItbXaTmpq6XFH04N2iw2NoSMn9GXJRc40n4Om6sf79nNEozmrUCowbngR
|
||||
5f9agJBsE48H3N9gknUAq5nm
|
||||
-----END PRIVATE KEY-----"#;
|
||||
|
||||
const EC_CRT: &str = r#"-----BEGIN CERTIFICATE-----
|
||||
MIIBhTCCASugAwIBAgIUB8703dqXCUOJQbhbyaMUMbVFOjwwCgYIKoZIzj0EAwIw
|
||||
FzEVMBMGA1UEAwwMeWFhay10ZXN0LWVjMCAXDTI2MDgxNDIwNDYyNFoYDzIxMjYw
|
||||
NzIxMjA0NjI0WjAXMRUwEwYDVQQDDAx5YWFrLXRlc3QtZWMwWTATBgcqhkjOPQIB
|
||||
BggqhkjOPQMBBwNCAATCYYKhzgHEaRaGsYVjJSoXvoroL8qe1yeEA0VtfxFzMBg+
|
||||
+bkPQ0nCtMyFfvQQtXWYIakxzsWJyhI8wPjUj6QSo1MwUTAdBgNVHQ4EFgQUKq40
|
||||
Hl+2DziVkBVR/tGsPj9FRo0wHwYDVR0jBBgwFoAUKq40Hl+2DziVkBVR/tGsPj9F
|
||||
Ro0wDwYDVR0TAQH/BAUwAwEB/zAKBggqhkjOPQQDAgNIADBFAiEAj1dx5XLl9iCZ
|
||||
rD0CW+a3RTluxQ5icXno9WJ9qaS6L08CIFx2t0y9znQr7n5x+SmfXbfZtkDola8e
|
||||
8nEZga/HXSeu
|
||||
-----END CERTIFICATE-----"#;
|
||||
|
||||
const EC_SEC1_KEY: &str = r#"-----BEGIN EC PRIVATE KEY-----
|
||||
MHcCAQEEIIoiiZ/hb4h6eHkZUVBTQFz7KLrVKJqQtWee2ygOjijNoAoGCCqGSM49
|
||||
AwEHoUQDQgAEwmGCoc4BxGkWhrGFYyUqF76K6C/KntcnhANFbX8RczAYPvm5D0NJ
|
||||
wrTMhX70ELV1mCGpMc7FicoSPMD41I+kEg==
|
||||
-----END EC PRIVATE KEY-----"#;
|
||||
|
||||
const EC_PKCS8_KEY: &str = r#"-----BEGIN PRIVATE KEY-----
|
||||
MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgiiKJn+FviHp4eRlR
|
||||
UFNAXPsoutUompC1Z57bKA6OKM2hRANCAATCYYKhzgHEaRaGsYVjJSoXvoroL8qe
|
||||
1yeEA0VtfxFzMBg++bkPQ0nCtMyFfvQQtXWYIakxzsWJyhI8wPjUj6QS
|
||||
-----END PRIVATE KEY-----"#;
|
||||
|
||||
/// Every key format the rustls path accepts, in the encodings users
|
||||
/// actually have on disk.
|
||||
fn key_formats() -> Vec<(&'static str, &'static str, &'static str)> {
|
||||
vec![
|
||||
("RSA cert with PKCS#8 key", RSA_CRT, RSA_PKCS8_KEY),
|
||||
("RSA cert with PKCS#1 key", RSA_CRT, RSA_PKCS1_KEY),
|
||||
("EC cert with PKCS#8 key", EC_CRT, EC_PKCS8_KEY),
|
||||
("EC cert with SEC1 key", EC_CRT, EC_SEC1_KEY),
|
||||
]
|
||||
}
|
||||
|
||||
fn write_pair(dir: &tempfile::TempDir, crt: &str, key: &str) -> ClientCertificateConfig {
|
||||
let crt_path = dir.path().join("client.crt");
|
||||
let key_path = dir.path().join("client.key");
|
||||
std::fs::write(&crt_path, crt).unwrap();
|
||||
std::fs::write(&key_path, key).unwrap();
|
||||
ClientCertificateConfig {
|
||||
crt_file: Some(crt_path.to_str().unwrap().to_string()),
|
||||
key_file: Some(key_path.to_str().unwrap().to_string()),
|
||||
pfx_file: None,
|
||||
passphrase: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn native_tls_accepts_every_format_rustls_accepts() {
|
||||
for (name, crt, key) in key_formats() {
|
||||
let dir = tempfile::TempDir::new().unwrap();
|
||||
let config = write_pair(&dir, crt, key);
|
||||
|
||||
get_tls_config(false, true, Some(config.clone()))
|
||||
.unwrap_or_else(|e| panic!("rustls rejected {name}: {e}"));
|
||||
|
||||
let identity = build_native_tls_identity(Some(config))
|
||||
.unwrap_or_else(|e| panic!("native-tls rejected {name}: {e}"));
|
||||
assert!(identity.is_some(), "no identity built for {name}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_leading_header_line_before_the_key_is_tolerated() {
|
||||
let dir = tempfile::TempDir::new().unwrap();
|
||||
let key = format!("Bag Attributes: friendlyName=client\n{RSA_PKCS8_KEY}");
|
||||
let config = write_pair(&dir, RSA_CRT, &key);
|
||||
assert!(build_native_tls_identity(Some(config)).unwrap().is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_identity_is_built_without_a_configured_certificate() {
|
||||
assert!(build_native_tls_identity(None).unwrap().is_none());
|
||||
|
||||
let empty = ClientCertificateConfig {
|
||||
crt_file: Some("".into()),
|
||||
key_file: Some("".into()),
|
||||
pfx_file: Some("".into()),
|
||||
passphrase: None,
|
||||
};
|
||||
assert!(build_native_tls_identity(Some(empty)).unwrap().is_none());
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod header_limit_tests {
|
||||
use super::*;
|
||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||
use tokio::net::TcpListener;
|
||||
|
||||
/// Accepts one connection and replies with a response head of exactly
|
||||
/// `total_headers` fields. The trailing `Content-Length` is one of them,
|
||||
/// because hyper counts every field against its limit.
|
||||
async fn serve_response_with_header_count(total_headers: usize) -> String {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
|
||||
tokio::spawn(async move {
|
||||
let (mut socket, _) = listener.accept().await.unwrap();
|
||||
|
||||
// Read the request head so the client is not writing into a closed socket
|
||||
let mut buf = [0u8; 4096];
|
||||
loop {
|
||||
let n = socket.read(&mut buf).await.unwrap();
|
||||
if n == 0 || buf[..n].windows(4).any(|w| w == b"\r\n\r\n") {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
let mut response = String::from("HTTP/1.1 200 OK\r\n");
|
||||
for i in 0..total_headers - 1 {
|
||||
response.push_str(&format!("X-Test-{i}: value-{i}\r\n"));
|
||||
}
|
||||
response.push_str("Content-Length: 0\r\n\r\n");
|
||||
|
||||
// A response the client rejects mid-parse closes the socket under
|
||||
// us, so a failed write here is an expected outcome, not a fault
|
||||
let _ = socket.write_all(response.as_bytes()).await;
|
||||
let _ = socket.flush().await;
|
||||
});
|
||||
|
||||
format!("http://{addr}/")
|
||||
}
|
||||
|
||||
fn options() -> HttpConnectionOptions {
|
||||
HttpConnectionOptions {
|
||||
id: "test".to_string(),
|
||||
validate_certificates: true,
|
||||
http_version: HttpVersion::Http1,
|
||||
proxy: HttpConnectionProxySetting::Disabled,
|
||||
client_certificate: None,
|
||||
dns_overrides: Vec::new(),
|
||||
address_filter: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// hyper defaults to 100 header fields per HTTP/1 response and fails the
|
||||
/// whole request past that, which is what users hit in the field.
|
||||
#[tokio::test]
|
||||
async fn responses_with_more_than_100_headers_are_accepted() {
|
||||
let url = serve_response_with_header_count(150).await;
|
||||
let (client, _resolver) = options().build_client().unwrap();
|
||||
|
||||
let response = client
|
||||
.inner()
|
||||
.get(&url)
|
||||
.send()
|
||||
.await
|
||||
.expect("request with 150 response headers should succeed");
|
||||
|
||||
assert_eq!(response.status(), 200);
|
||||
assert_eq!(response.headers().len(), 150);
|
||||
for i in 0..149 {
|
||||
assert_eq!(
|
||||
response.headers().get(format!("x-test-{i}")).unwrap(),
|
||||
format!("value-{i}").as_str(),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Pins both sides of the boundary. The sizes are written out rather than
|
||||
/// derived from `HTTP1_MAX_RESPONSE_HEADERS`, so retuning the limit trips
|
||||
/// this test instead of silently moving with it.
|
||||
#[tokio::test]
|
||||
async fn the_limit_is_exactly_1024_header_fields() {
|
||||
let (client, _resolver) = options().build_client().unwrap();
|
||||
|
||||
let at_limit = serve_response_with_header_count(1024).await;
|
||||
let response = client
|
||||
.inner()
|
||||
.get(&at_limit)
|
||||
.send()
|
||||
.await
|
||||
.expect("a response with 1024 header fields should succeed");
|
||||
assert_eq!(response.headers().len(), 1024);
|
||||
|
||||
let over_limit = serve_response_with_header_count(1025).await;
|
||||
assert!(client.inner().get(&over_limit).send().await.is_err());
|
||||
}
|
||||
}
|
||||
581
crates/yaak-http/src/cookies.rs
Normal file
581
crates/yaak-http/src/cookies.rs
Normal file
@@ -0,0 +1,581 @@
|
||||
//! Custom cookie handling for HTTP requests
|
||||
//!
|
||||
//! This module provides cookie storage and matching functionality that was previously
|
||||
//! delegated to reqwest. It implements RFC 6265 cookie domain and path matching.
|
||||
|
||||
use log::debug;
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::{Duration, SystemTime, UNIX_EPOCH};
|
||||
use url::Url;
|
||||
use yaak_models::models::{Cookie, CookieDomain, CookieExpires, CookieSameSite};
|
||||
|
||||
/// A thread-safe cookie store that can be shared across requests
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct CookieStore {
|
||||
cookies: Arc<Mutex<Vec<Cookie>>>,
|
||||
}
|
||||
|
||||
impl Default for CookieStore {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
impl CookieStore {
|
||||
/// Create a new empty cookie store
|
||||
pub fn new() -> Self {
|
||||
Self { cookies: Arc::new(Mutex::new(Vec::new())) }
|
||||
}
|
||||
|
||||
/// Create a cookie store from existing cookies
|
||||
pub fn from_cookies(cookies: Vec<Cookie>) -> Self {
|
||||
Self { cookies: Arc::new(Mutex::new(cookies)) }
|
||||
}
|
||||
|
||||
/// Get all cookies (for persistence)
|
||||
pub fn get_all_cookies(&self) -> Vec<Cookie> {
|
||||
self.cookies.lock().unwrap().clone()
|
||||
}
|
||||
|
||||
/// Get the Cookie header value for the given URL
|
||||
pub fn get_cookie_header(&self, url: &Url) -> Option<String> {
|
||||
let cookies = self.cookies.lock().unwrap();
|
||||
let now = SystemTime::now();
|
||||
|
||||
let matching_cookies: Vec<_> = cookies
|
||||
.iter()
|
||||
.filter(|cookie| self.cookie_matches(cookie, url, &now))
|
||||
.map(|cookie| (cookie.name.clone(), cookie.value.clone()))
|
||||
.collect();
|
||||
|
||||
if matching_cookies.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(
|
||||
matching_cookies
|
||||
.into_iter()
|
||||
.map(|(name, value)| format!("{}={}", name, value))
|
||||
.collect::<Vec<_>>()
|
||||
.join("; "),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/// Parse Set-Cookie headers and add cookies to the store
|
||||
pub fn store_cookies_from_response(&self, url: &Url, set_cookie_headers: &[String]) {
|
||||
let mut cookies = self.cookies.lock().unwrap();
|
||||
|
||||
for header_value in set_cookie_headers {
|
||||
if let Some(cookie) = parse_set_cookie(header_value, url) {
|
||||
// Remove any existing cookie with the same name and domain
|
||||
cookies.retain(|existing| !cookies_match(existing, &cookie));
|
||||
debug!("Storing cookie: {} for domain {:?}", cookie.name, cookie.domain);
|
||||
cookies.push(cookie);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Check if a cookie matches the given URL
|
||||
fn cookie_matches(&self, cookie: &Cookie, url: &Url, now: &SystemTime) -> bool {
|
||||
// Check expiration
|
||||
if let CookieExpires::AtUtc(expiry_str) = &cookie.expires {
|
||||
if let Ok(expiry) = parse_cookie_date(expiry_str) {
|
||||
if expiry < *now {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Check domain
|
||||
let url_host = match url.host_str() {
|
||||
Some(h) => h.to_lowercase(),
|
||||
None => return false,
|
||||
};
|
||||
|
||||
let domain_matches = match &cookie.domain {
|
||||
CookieDomain::HostOnly(domain) => url_host == domain.to_lowercase(),
|
||||
CookieDomain::Suffix(domain) => {
|
||||
let domain_lower = domain.to_lowercase();
|
||||
url_host == domain_lower || url_host.ends_with(&format!(".{}", domain_lower))
|
||||
}
|
||||
// NotPresent and Empty should never occur in practice since we always set domain
|
||||
// when parsing Set-Cookie headers. Treat as non-matching to be safe.
|
||||
CookieDomain::NotPresent | CookieDomain::Empty => false,
|
||||
};
|
||||
|
||||
if !domain_matches {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Check path
|
||||
let url_path = url.path();
|
||||
|
||||
path_matches(url_path, &cookie.path)
|
||||
}
|
||||
}
|
||||
|
||||
/// Get a stored cookie value by name, optionally scoped to an exact stored domain.
|
||||
pub fn get_cookie_value_from_jar(
|
||||
cookies: impl IntoIterator<Item = Cookie>,
|
||||
name: &str,
|
||||
domain: Option<&str>,
|
||||
) -> Option<String> {
|
||||
let domain = domain.and_then(normalize_cookie_domain_filter);
|
||||
|
||||
cookies.into_iter().find_map(|cookie| {
|
||||
if cookie.name != name {
|
||||
return None;
|
||||
}
|
||||
|
||||
if let Some(domain) = domain.as_deref() {
|
||||
if !cookie_domain_matches_filter(&cookie.domain, domain) {
|
||||
return None;
|
||||
}
|
||||
}
|
||||
|
||||
Some(cookie.value)
|
||||
})
|
||||
}
|
||||
|
||||
/// Parse name=value from a cookie string (raw_cookie format)
|
||||
#[cfg(test)]
|
||||
fn parse_cookie_name_value(raw_cookie: &str) -> Option<(String, String)> {
|
||||
// The raw_cookie typically looks like "name=value" or "name=value; attr1; attr2=..."
|
||||
let first_part = raw_cookie.split(';').next()?;
|
||||
let mut parts = first_part.splitn(2, '=');
|
||||
let name = parts.next()?.trim().to_string();
|
||||
let value = parts.next().unwrap_or("").trim().to_string();
|
||||
|
||||
if name.is_empty() { None } else { Some((name, value)) }
|
||||
}
|
||||
|
||||
fn normalize_cookie_domain_filter(domain: &str) -> Option<String> {
|
||||
let domain = domain.trim().trim_start_matches('.').to_lowercase();
|
||||
if domain.is_empty() { None } else { Some(domain) }
|
||||
}
|
||||
|
||||
fn cookie_domain_matches_filter(cookie_domain: &CookieDomain, domain: &str) -> bool {
|
||||
match cookie_domain {
|
||||
CookieDomain::HostOnly(cookie_domain) | CookieDomain::Suffix(cookie_domain) => {
|
||||
normalize_cookie_domain_filter(cookie_domain).is_some_and(|d| d == domain)
|
||||
}
|
||||
CookieDomain::NotPresent | CookieDomain::Empty => false,
|
||||
}
|
||||
}
|
||||
|
||||
/// Parse a Set-Cookie header into a Cookie
|
||||
fn parse_set_cookie(header_value: &str, request_url: &Url) -> Option<Cookie> {
|
||||
let parsed = cookie::Cookie::parse(header_value).ok()?;
|
||||
|
||||
// Determine domain
|
||||
let domain = if let Some(domain_attr) = parsed.domain() {
|
||||
// Domain attribute present - this is a suffix match
|
||||
let domain = domain_attr.trim_start_matches('.').to_lowercase();
|
||||
|
||||
// Reject single-component domains (TLDs) except localhost
|
||||
if is_single_component_domain(&domain) && !is_localhost(&domain) {
|
||||
debug!("Rejecting cookie with single-component domain: {}", domain);
|
||||
return None;
|
||||
}
|
||||
|
||||
CookieDomain::Suffix(domain)
|
||||
} else {
|
||||
// No domain attribute - host-only cookie
|
||||
CookieDomain::HostOnly(request_url.host_str().unwrap_or("").to_lowercase())
|
||||
};
|
||||
|
||||
// Determine expiration
|
||||
let expires = if let Some(max_age) = parsed.max_age() {
|
||||
let duration = Duration::from_secs(max_age.whole_seconds().max(0) as u64);
|
||||
let expiry = SystemTime::now() + duration;
|
||||
let expiry_secs = expiry.duration_since(UNIX_EPOCH).unwrap_or_default().as_secs();
|
||||
CookieExpires::AtUtc(format!("{}", expiry_secs))
|
||||
} else if let Some(expires_time) = parsed.expires() {
|
||||
match expires_time {
|
||||
cookie::Expiration::DateTime(dt) => {
|
||||
let timestamp = dt.unix_timestamp();
|
||||
CookieExpires::AtUtc(format!("{}", timestamp))
|
||||
}
|
||||
cookie::Expiration::Session => CookieExpires::SessionEnd,
|
||||
}
|
||||
} else {
|
||||
CookieExpires::SessionEnd
|
||||
};
|
||||
|
||||
// Determine path
|
||||
let path = if let Some(path_attr) = parsed.path() {
|
||||
path_attr.to_string()
|
||||
} else {
|
||||
// Default path is the directory of the request URI
|
||||
default_cookie_path(request_url.path())
|
||||
};
|
||||
|
||||
let same_site = parsed.same_site().map(|same_site| match same_site {
|
||||
cookie::SameSite::Strict => CookieSameSite::Strict,
|
||||
cookie::SameSite::Lax => CookieSameSite::Lax,
|
||||
cookie::SameSite::None => CookieSameSite::None,
|
||||
});
|
||||
|
||||
Some(Cookie {
|
||||
name: parsed.name().to_string(),
|
||||
value: parsed.value().to_string(),
|
||||
domain,
|
||||
expires,
|
||||
path,
|
||||
secure: parsed.secure().unwrap_or(false),
|
||||
http_only: parsed.http_only().unwrap_or(false),
|
||||
same_site,
|
||||
})
|
||||
}
|
||||
|
||||
/// Get the default cookie path from a request path (RFC 6265 Section 5.1.4)
|
||||
fn default_cookie_path(request_path: &str) -> String {
|
||||
if request_path.is_empty() || !request_path.starts_with('/') {
|
||||
return "/".to_string();
|
||||
}
|
||||
|
||||
// Find the last slash
|
||||
if let Some(last_slash) = request_path.rfind('/') {
|
||||
if last_slash == 0 { "/".to_string() } else { request_path[..last_slash].to_string() }
|
||||
} else {
|
||||
"/".to_string()
|
||||
}
|
||||
}
|
||||
|
||||
/// Check if a request path matches a cookie path (RFC 6265 Section 5.1.4)
|
||||
fn path_matches(request_path: &str, cookie_path: &str) -> bool {
|
||||
if request_path == cookie_path {
|
||||
return true;
|
||||
}
|
||||
|
||||
if request_path.starts_with(cookie_path) {
|
||||
// Cookie path must end with / or the char after cookie_path in request_path must be /
|
||||
if cookie_path.ends_with('/') {
|
||||
return true;
|
||||
}
|
||||
if request_path.chars().nth(cookie_path.len()) == Some('/') {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
false
|
||||
}
|
||||
|
||||
/// Check if two cookies match (same name and domain)
|
||||
fn cookies_match(a: &Cookie, b: &Cookie) -> bool {
|
||||
if a.name != b.name {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Check domain match
|
||||
match (&a.domain, &b.domain) {
|
||||
(CookieDomain::HostOnly(d1), CookieDomain::HostOnly(d2)) => {
|
||||
d1.to_lowercase() == d2.to_lowercase()
|
||||
}
|
||||
(CookieDomain::Suffix(d1), CookieDomain::Suffix(d2)) => {
|
||||
d1.to_lowercase() == d2.to_lowercase()
|
||||
}
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
/// Parse a cookie date string (Unix timestamp in our format)
|
||||
fn parse_cookie_date(date_str: &str) -> Result<SystemTime, ()> {
|
||||
let timestamp: i64 = date_str.parse().map_err(|_| ())?;
|
||||
let duration = Duration::from_secs(timestamp.max(0) as u64);
|
||||
Ok(UNIX_EPOCH + duration)
|
||||
}
|
||||
|
||||
/// Check if a domain is a single-component domain (TLD)
|
||||
/// e.g., "com", "org", "net" - domains without any dots
|
||||
fn is_single_component_domain(domain: &str) -> bool {
|
||||
// Empty or only dots
|
||||
let trimmed = domain.trim_matches('.');
|
||||
if trimmed.is_empty() {
|
||||
return true;
|
||||
}
|
||||
// IPv6 addresses use colons, not dots - don't consider them single-component
|
||||
if domain.contains(':') {
|
||||
return false;
|
||||
}
|
||||
!trimmed.contains('.')
|
||||
}
|
||||
|
||||
/// Check if a domain is localhost or a localhost variant
|
||||
fn is_localhost(domain: &str) -> bool {
|
||||
let lower = domain.to_lowercase();
|
||||
lower == "localhost"
|
||||
|| lower.ends_with(".localhost")
|
||||
|| lower == "127.0.0.1"
|
||||
|| lower == "::1"
|
||||
|| lower == "[::1]"
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn cookie(raw_cookie: &str, domain: CookieDomain) -> Cookie {
|
||||
let (name, value) = parse_cookie_name_value(raw_cookie).unwrap();
|
||||
Cookie {
|
||||
name,
|
||||
value,
|
||||
domain,
|
||||
expires: CookieExpires::SessionEnd,
|
||||
path: "/".to_string(),
|
||||
secure: false,
|
||||
http_only: false,
|
||||
same_site: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_cookie_name_value() {
|
||||
assert_eq!(
|
||||
parse_cookie_name_value("session=abc123"),
|
||||
Some(("session".to_string(), "abc123".to_string()))
|
||||
);
|
||||
assert_eq!(
|
||||
parse_cookie_name_value("name=value; Path=/; HttpOnly"),
|
||||
Some(("name".to_string(), "value".to_string()))
|
||||
);
|
||||
assert_eq!(parse_cookie_name_value("empty="), Some(("empty".to_string(), "".to_string())));
|
||||
assert_eq!(parse_cookie_name_value(""), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_path_matches() {
|
||||
assert!(path_matches("/", "/"));
|
||||
assert!(path_matches("/foo", "/"));
|
||||
assert!(path_matches("/foo/bar", "/foo"));
|
||||
assert!(path_matches("/foo/bar", "/foo/"));
|
||||
assert!(!path_matches("/foobar", "/foo"));
|
||||
assert!(!path_matches("/foo", "/foo/bar"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_default_cookie_path() {
|
||||
assert_eq!(default_cookie_path("/"), "/");
|
||||
assert_eq!(default_cookie_path("/foo"), "/");
|
||||
assert_eq!(default_cookie_path("/foo/bar"), "/foo");
|
||||
assert_eq!(default_cookie_path("/foo/bar/baz"), "/foo/bar");
|
||||
assert_eq!(default_cookie_path(""), "/");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_cookie_store_basic() {
|
||||
let store = CookieStore::new();
|
||||
let url = Url::parse("https://example.com/path").unwrap();
|
||||
|
||||
// Initially empty
|
||||
assert!(store.get_cookie_header(&url).is_none());
|
||||
|
||||
// Add a cookie
|
||||
store.store_cookies_from_response(&url, &["session=abc123".to_string()]);
|
||||
|
||||
// Should now have the cookie
|
||||
let header = store.get_cookie_header(&url);
|
||||
assert_eq!(header, Some("session=abc123".to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_cookie_domain_matching() {
|
||||
let store = CookieStore::new();
|
||||
let url = Url::parse("https://example.com/").unwrap();
|
||||
|
||||
// Cookie with domain attribute (suffix match)
|
||||
store.store_cookies_from_response(
|
||||
&url,
|
||||
&["domain_cookie=value; Domain=example.com".to_string()],
|
||||
);
|
||||
|
||||
// Should match example.com
|
||||
assert!(store.get_cookie_header(&url).is_some());
|
||||
|
||||
// Should match subdomain
|
||||
let subdomain_url = Url::parse("https://sub.example.com/").unwrap();
|
||||
assert!(store.get_cookie_header(&subdomain_url).is_some());
|
||||
|
||||
// Should not match different domain
|
||||
let other_url = Url::parse("https://other.com/").unwrap();
|
||||
assert!(store.get_cookie_header(&other_url).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_cookie_path_matching() {
|
||||
let store = CookieStore::new();
|
||||
let url = Url::parse("https://example.com/api/v1").unwrap();
|
||||
|
||||
// Cookie with path
|
||||
store.store_cookies_from_response(&url, &["api_cookie=value; Path=/api".to_string()]);
|
||||
|
||||
// Should match /api/v1
|
||||
assert!(store.get_cookie_header(&url).is_some());
|
||||
|
||||
// Should match /api
|
||||
let api_url = Url::parse("https://example.com/api").unwrap();
|
||||
assert!(store.get_cookie_header(&api_url).is_some());
|
||||
|
||||
// Should not match /other
|
||||
let other_url = Url::parse("https://example.com/other").unwrap();
|
||||
assert!(store.get_cookie_header(&other_url).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_cookie_replacement() {
|
||||
let store = CookieStore::new();
|
||||
let url = Url::parse("https://example.com/").unwrap();
|
||||
|
||||
// Add a cookie
|
||||
store.store_cookies_from_response(&url, &["session=old".to_string()]);
|
||||
assert_eq!(store.get_cookie_header(&url), Some("session=old".to_string()));
|
||||
|
||||
// Replace with new value
|
||||
store.store_cookies_from_response(&url, &["session=new".to_string()]);
|
||||
assert_eq!(store.get_cookie_header(&url), Some("session=new".to_string()));
|
||||
|
||||
// Should only have one cookie
|
||||
assert_eq!(store.get_all_cookies().len(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_get_cookie_value_preserves_name_only_first_match() {
|
||||
let cookies = vec![
|
||||
cookie("co-auth=", CookieDomain::HostOnly("foo.example.com".to_string())),
|
||||
cookie("co-auth=token", CookieDomain::Suffix("example.com".to_string())),
|
||||
];
|
||||
|
||||
assert_eq!(get_cookie_value_from_jar(cookies, "co-auth", None), Some("".to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_get_cookie_value_matches_domain() {
|
||||
let cookies = vec![
|
||||
cookie("co-auth=", CookieDomain::HostOnly("foo.example.com".to_string())),
|
||||
cookie("co-auth=token", CookieDomain::Suffix("example.com".to_string())),
|
||||
];
|
||||
|
||||
assert_eq!(
|
||||
get_cookie_value_from_jar(cookies, "co-auth", Some("example.com")),
|
||||
Some("token".to_string())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_get_cookie_value_normalizes_domain_filter() {
|
||||
let cookies = vec![cookie(
|
||||
"co-auth=token",
|
||||
CookieDomain::Suffix("Example.COM".to_string()),
|
||||
)];
|
||||
|
||||
assert_eq!(
|
||||
get_cookie_value_from_jar(cookies, "co-auth", Some(" .example.com ")),
|
||||
Some("token".to_string())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_get_cookie_value_requires_exact_stored_domain_match() {
|
||||
let cookies = vec![cookie(
|
||||
"co-auth=token",
|
||||
CookieDomain::HostOnly("foo.example.com".to_string()),
|
||||
)];
|
||||
|
||||
assert_eq!(get_cookie_value_from_jar(cookies, "co-auth", Some("example.com")), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_is_single_component_domain() {
|
||||
// Single-component domains (TLDs)
|
||||
assert!(is_single_component_domain("com"));
|
||||
assert!(is_single_component_domain("org"));
|
||||
assert!(is_single_component_domain("net"));
|
||||
assert!(is_single_component_domain("localhost")); // Still single-component, but allowed separately
|
||||
|
||||
// Multi-component domains
|
||||
assert!(!is_single_component_domain("example.com"));
|
||||
assert!(!is_single_component_domain("sub.example.com"));
|
||||
assert!(!is_single_component_domain("co.uk"));
|
||||
|
||||
// Edge cases
|
||||
assert!(is_single_component_domain("")); // Empty is treated as single-component
|
||||
assert!(is_single_component_domain(".")); // Only dots
|
||||
assert!(is_single_component_domain("..")); // Only dots
|
||||
|
||||
// IPv6 addresses (have colons, not dots)
|
||||
assert!(!is_single_component_domain("::1")); // IPv6 localhost
|
||||
assert!(!is_single_component_domain("[::1]")); // Bracketed IPv6
|
||||
assert!(!is_single_component_domain("2001:db8::1")); // IPv6 address
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_is_localhost() {
|
||||
// Localhost variants
|
||||
assert!(is_localhost("localhost"));
|
||||
assert!(is_localhost("LOCALHOST")); // Case-insensitive
|
||||
assert!(is_localhost("sub.localhost"));
|
||||
assert!(is_localhost("app.sub.localhost"));
|
||||
|
||||
// IP localhost
|
||||
assert!(is_localhost("127.0.0.1"));
|
||||
assert!(is_localhost("::1"));
|
||||
assert!(is_localhost("[::1]"));
|
||||
|
||||
// Not localhost
|
||||
assert!(!is_localhost("example.com"));
|
||||
assert!(!is_localhost("localhost.com")); // .com domain, not localhost
|
||||
assert!(!is_localhost("notlocalhost"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_reject_tld_cookies() {
|
||||
let store = CookieStore::new();
|
||||
let url = Url::parse("https://example.com/").unwrap();
|
||||
|
||||
// Try to set a cookie with Domain=com (TLD)
|
||||
store.store_cookies_from_response(&url, &["bad=cookie; Domain=com".to_string()]);
|
||||
|
||||
// Should be rejected - no cookies stored
|
||||
assert_eq!(store.get_all_cookies().len(), 0);
|
||||
assert!(store.get_cookie_header(&url).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_allow_localhost_cookies() {
|
||||
let store = CookieStore::new();
|
||||
let url = Url::parse("http://localhost:3000/").unwrap();
|
||||
|
||||
// Cookie with Domain=localhost should be allowed
|
||||
store.store_cookies_from_response(&url, &["session=abc; Domain=localhost".to_string()]);
|
||||
|
||||
// Should be accepted
|
||||
assert_eq!(store.get_all_cookies().len(), 1);
|
||||
assert!(store.get_cookie_header(&url).is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_allow_127_0_0_1_cookies() {
|
||||
let store = CookieStore::new();
|
||||
let url = Url::parse("http://127.0.0.1:8080/").unwrap();
|
||||
|
||||
// Cookie without Domain attribute (host-only) should work
|
||||
store.store_cookies_from_response(&url, &["session=xyz".to_string()]);
|
||||
|
||||
// Should be accepted
|
||||
assert_eq!(store.get_all_cookies().len(), 1);
|
||||
assert!(store.get_cookie_header(&url).is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_allow_normal_domain_cookies() {
|
||||
let store = CookieStore::new();
|
||||
let url = Url::parse("https://example.com/").unwrap();
|
||||
|
||||
// Cookie with valid domain should be allowed
|
||||
store.store_cookies_from_response(&url, &["session=abc; Domain=example.com".to_string()]);
|
||||
|
||||
// Should be accepted
|
||||
assert_eq!(store.get_all_cookies().len(), 1);
|
||||
assert!(store.get_cookie_header(&url).is_some());
|
||||
}
|
||||
}
|
||||
188
crates/yaak-http/src/decompress.rs
Normal file
188
crates/yaak-http/src/decompress.rs
Normal file
@@ -0,0 +1,188 @@
|
||||
use crate::error::{Error, Result};
|
||||
use async_compression::tokio::bufread::{
|
||||
BrotliDecoder, DeflateDecoder as AsyncDeflateDecoder, GzipDecoder,
|
||||
ZstdDecoder as AsyncZstdDecoder,
|
||||
};
|
||||
use flate2::read::{DeflateDecoder, GzDecoder};
|
||||
use std::io::Read;
|
||||
use tokio::io::{AsyncBufRead, AsyncRead};
|
||||
|
||||
/// Supported compression encodings
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum ContentEncoding {
|
||||
Gzip,
|
||||
Deflate,
|
||||
Brotli,
|
||||
Zstd,
|
||||
Identity,
|
||||
}
|
||||
|
||||
impl ContentEncoding {
|
||||
/// Parse a Content-Encoding header value into an encoding type.
|
||||
/// Returns Identity for unknown or missing encodings.
|
||||
pub fn from_header(value: Option<&str>) -> Self {
|
||||
match value.map(|s| s.trim().to_lowercase()).as_deref() {
|
||||
Some("gzip") | Some("x-gzip") => ContentEncoding::Gzip,
|
||||
Some("deflate") => ContentEncoding::Deflate,
|
||||
Some("br") => ContentEncoding::Brotli,
|
||||
Some("zstd") => ContentEncoding::Zstd,
|
||||
_ => ContentEncoding::Identity,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Result of decompression, containing both the decompressed data and size info
|
||||
#[derive(Debug)]
|
||||
pub struct DecompressResult {
|
||||
pub data: Vec<u8>,
|
||||
pub compressed_size: u64,
|
||||
pub decompressed_size: u64,
|
||||
}
|
||||
|
||||
/// Decompress data based on the Content-Encoding.
|
||||
/// Returns the original data unchanged if encoding is Identity or unknown.
|
||||
pub fn decompress(data: Vec<u8>, encoding: ContentEncoding) -> Result<DecompressResult> {
|
||||
let compressed_size = data.len() as u64;
|
||||
|
||||
let decompressed = match encoding {
|
||||
ContentEncoding::Identity => data,
|
||||
ContentEncoding::Gzip => decompress_gzip(&data)?,
|
||||
ContentEncoding::Deflate => decompress_deflate(&data)?,
|
||||
ContentEncoding::Brotli => decompress_brotli(&data)?,
|
||||
ContentEncoding::Zstd => decompress_zstd(&data)?,
|
||||
};
|
||||
|
||||
let decompressed_size = decompressed.len() as u64;
|
||||
|
||||
Ok(DecompressResult { data: decompressed, compressed_size, decompressed_size })
|
||||
}
|
||||
|
||||
fn decompress_gzip(data: &[u8]) -> Result<Vec<u8>> {
|
||||
let mut decoder = GzDecoder::new(data);
|
||||
let mut decompressed = Vec::new();
|
||||
decoder
|
||||
.read_to_end(&mut decompressed)
|
||||
.map_err(|e| Error::DecompressionError(format!("gzip decompression failed: {}", e)))?;
|
||||
Ok(decompressed)
|
||||
}
|
||||
|
||||
fn decompress_deflate(data: &[u8]) -> Result<Vec<u8>> {
|
||||
let mut decoder = DeflateDecoder::new(data);
|
||||
let mut decompressed = Vec::new();
|
||||
decoder
|
||||
.read_to_end(&mut decompressed)
|
||||
.map_err(|e| Error::DecompressionError(format!("deflate decompression failed: {}", e)))?;
|
||||
Ok(decompressed)
|
||||
}
|
||||
|
||||
fn decompress_brotli(data: &[u8]) -> Result<Vec<u8>> {
|
||||
let mut decompressed = Vec::new();
|
||||
brotli::BrotliDecompress(&mut std::io::Cursor::new(data), &mut decompressed)
|
||||
.map_err(|e| Error::DecompressionError(format!("brotli decompression failed: {}", e)))?;
|
||||
Ok(decompressed)
|
||||
}
|
||||
|
||||
fn decompress_zstd(data: &[u8]) -> Result<Vec<u8>> {
|
||||
zstd::stream::decode_all(std::io::Cursor::new(data))
|
||||
.map_err(|e| Error::DecompressionError(format!("zstd decompression failed: {}", e)))
|
||||
}
|
||||
|
||||
/// Create a streaming decompressor that wraps an async reader.
|
||||
/// Returns an AsyncRead that decompresses data on-the-fly.
|
||||
pub fn streaming_decoder<R: AsyncBufRead + Unpin + Send + 'static>(
|
||||
reader: R,
|
||||
encoding: ContentEncoding,
|
||||
) -> Box<dyn AsyncRead + Unpin + Send> {
|
||||
match encoding {
|
||||
ContentEncoding::Identity => Box::new(reader),
|
||||
ContentEncoding::Gzip => Box::new(GzipDecoder::new(reader)),
|
||||
ContentEncoding::Deflate => Box::new(AsyncDeflateDecoder::new(reader)),
|
||||
ContentEncoding::Brotli => Box::new(BrotliDecoder::new(reader)),
|
||||
ContentEncoding::Zstd => Box::new(AsyncZstdDecoder::new(reader)),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use flate2::Compression;
|
||||
use flate2::write::GzEncoder;
|
||||
use std::io::Write;
|
||||
|
||||
#[test]
|
||||
fn test_content_encoding_from_header() {
|
||||
assert_eq!(ContentEncoding::from_header(Some("gzip")), ContentEncoding::Gzip);
|
||||
assert_eq!(ContentEncoding::from_header(Some("x-gzip")), ContentEncoding::Gzip);
|
||||
assert_eq!(ContentEncoding::from_header(Some("GZIP")), ContentEncoding::Gzip);
|
||||
assert_eq!(ContentEncoding::from_header(Some("deflate")), ContentEncoding::Deflate);
|
||||
assert_eq!(ContentEncoding::from_header(Some("br")), ContentEncoding::Brotli);
|
||||
assert_eq!(ContentEncoding::from_header(Some("zstd")), ContentEncoding::Zstd);
|
||||
assert_eq!(ContentEncoding::from_header(Some("identity")), ContentEncoding::Identity);
|
||||
assert_eq!(ContentEncoding::from_header(Some("unknown")), ContentEncoding::Identity);
|
||||
assert_eq!(ContentEncoding::from_header(None), ContentEncoding::Identity);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_decompress_identity() {
|
||||
let data = b"hello world".to_vec();
|
||||
let result = decompress(data.clone(), ContentEncoding::Identity).unwrap();
|
||||
assert_eq!(result.data, data);
|
||||
assert_eq!(result.compressed_size, 11);
|
||||
assert_eq!(result.decompressed_size, 11);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_decompress_gzip() {
|
||||
// Compress some data with gzip
|
||||
let original = b"hello world, this is a test of gzip compression";
|
||||
let mut encoder = GzEncoder::new(Vec::new(), Compression::default());
|
||||
encoder.write_all(original).unwrap();
|
||||
let compressed = encoder.finish().unwrap();
|
||||
|
||||
let result = decompress(compressed.clone(), ContentEncoding::Gzip).unwrap();
|
||||
assert_eq!(result.data, original);
|
||||
assert_eq!(result.compressed_size, compressed.len() as u64);
|
||||
assert_eq!(result.decompressed_size, original.len() as u64);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_decompress_deflate() {
|
||||
// Compress some data with deflate
|
||||
let original = b"hello world, this is a test of deflate compression";
|
||||
let mut encoder = flate2::write::DeflateEncoder::new(Vec::new(), Compression::default());
|
||||
encoder.write_all(original).unwrap();
|
||||
let compressed = encoder.finish().unwrap();
|
||||
|
||||
let result = decompress(compressed.clone(), ContentEncoding::Deflate).unwrap();
|
||||
assert_eq!(result.data, original);
|
||||
assert_eq!(result.compressed_size, compressed.len() as u64);
|
||||
assert_eq!(result.decompressed_size, original.len() as u64);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_decompress_brotli() {
|
||||
// Compress some data with brotli
|
||||
let original = b"hello world, this is a test of brotli compression";
|
||||
let mut compressed = Vec::new();
|
||||
let mut writer = brotli::CompressorWriter::new(&mut compressed, 4096, 4, 22);
|
||||
writer.write_all(original).unwrap();
|
||||
drop(writer);
|
||||
|
||||
let result = decompress(compressed.clone(), ContentEncoding::Brotli).unwrap();
|
||||
assert_eq!(result.data, original);
|
||||
assert_eq!(result.compressed_size, compressed.len() as u64);
|
||||
assert_eq!(result.decompressed_size, original.len() as u64);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_decompress_zstd() {
|
||||
// Compress some data with zstd
|
||||
let original = b"hello world, this is a test of zstd compression";
|
||||
let compressed = zstd::stream::encode_all(std::io::Cursor::new(original), 3).unwrap();
|
||||
|
||||
let result = decompress(compressed.clone(), ContentEncoding::Zstd).unwrap();
|
||||
assert_eq!(result.data, original);
|
||||
assert_eq!(result.compressed_size, compressed.len() as u64);
|
||||
assert_eq!(result.decompressed_size, original.len() as u64);
|
||||
}
|
||||
}
|
||||
225
crates/yaak-http/src/dns.rs
Normal file
225
crates/yaak-http/src/dns.rs
Normal file
@@ -0,0 +1,225 @@
|
||||
use crate::sender::HttpResponseEvent;
|
||||
use hyper_util::client::legacy::connect::dns::{
|
||||
GaiResolver as HyperGaiResolver, Name as HyperName,
|
||||
};
|
||||
use log::info;
|
||||
use reqwest::dns::{Addrs, Name, Resolve, Resolving};
|
||||
use std::collections::HashMap;
|
||||
use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr};
|
||||
use std::str::FromStr;
|
||||
use std::sync::Arc;
|
||||
use std::time::Instant;
|
||||
use tokio::sync::{RwLock, mpsc};
|
||||
use tower_service::Service;
|
||||
use yaak_models::models::DnsOverride;
|
||||
|
||||
/// Stores resolved addresses for a hostname override
|
||||
#[derive(Clone)]
|
||||
pub struct ResolvedOverride {
|
||||
pub ipv4: Vec<Ipv4Addr>,
|
||||
pub ipv6: Vec<Ipv6Addr>,
|
||||
}
|
||||
|
||||
/// A veto on the addresses a hostname resolves to, consulted after resolution
|
||||
/// and before any connection is made. Returning `Err` refuses the whole lookup
|
||||
/// with that message; a hostname is never partially allowed.
|
||||
///
|
||||
/// A hosted sender uses this to refuse private and metadata ranges no matter
|
||||
/// what name they hide behind. Checking here rather than on the URL is what
|
||||
/// catches a public hostname that resolves to an internal address.
|
||||
pub type AddressFilter = Arc<dyn Fn(IpAddr) -> std::result::Result<(), String> + Send + Sync>;
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct LocalhostResolver {
|
||||
fallback: HyperGaiResolver,
|
||||
event_tx: Arc<RwLock<Option<mpsc::Sender<HttpResponseEvent>>>>,
|
||||
overrides: Arc<HashMap<String, ResolvedOverride>>,
|
||||
address_filter: Option<AddressFilter>,
|
||||
}
|
||||
|
||||
impl LocalhostResolver {
|
||||
pub fn new(dns_overrides: Vec<DnsOverride>) -> Arc<Self> {
|
||||
Self::with_address_filter(dns_overrides, None)
|
||||
}
|
||||
|
||||
pub fn with_address_filter(
|
||||
dns_overrides: Vec<DnsOverride>,
|
||||
address_filter: Option<AddressFilter>,
|
||||
) -> Arc<Self> {
|
||||
let resolver = HyperGaiResolver::new();
|
||||
|
||||
// Pre-parse DNS overrides into a lookup map
|
||||
let mut overrides = HashMap::new();
|
||||
for o in dns_overrides {
|
||||
if !o.enabled {
|
||||
continue;
|
||||
}
|
||||
let hostname = o.hostname.to_lowercase();
|
||||
|
||||
let ipv4: Vec<Ipv4Addr> =
|
||||
o.ipv4.iter().filter_map(|s| s.parse::<Ipv4Addr>().ok()).collect();
|
||||
|
||||
let ipv6: Vec<Ipv6Addr> =
|
||||
o.ipv6.iter().filter_map(|s| s.parse::<Ipv6Addr>().ok()).collect();
|
||||
|
||||
// Only add if at least one address is valid
|
||||
if !ipv4.is_empty() || !ipv6.is_empty() {
|
||||
overrides.insert(hostname, ResolvedOverride { ipv4, ipv6 });
|
||||
}
|
||||
}
|
||||
|
||||
Arc::new(Self {
|
||||
fallback: resolver,
|
||||
event_tx: Arc::new(RwLock::new(None)),
|
||||
overrides: Arc::new(overrides),
|
||||
address_filter,
|
||||
})
|
||||
}
|
||||
|
||||
/// Apply the address filter, if any, to a resolved address list.
|
||||
fn filter_addrs(
|
||||
filter: &Option<AddressFilter>,
|
||||
addrs: &[SocketAddr],
|
||||
) -> std::result::Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
if let Some(filter) = filter {
|
||||
for addr in addrs {
|
||||
if let Err(reason) = filter(addr.ip()) {
|
||||
return Err(Box::new(std::io::Error::other(reason)));
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Set the event sender for the current request.
|
||||
/// This should be called before each request to direct DNS events
|
||||
/// to the appropriate channel.
|
||||
pub async fn set_event_sender(&self, tx: Option<mpsc::Sender<HttpResponseEvent>>) {
|
||||
let mut guard = self.event_tx.write().await;
|
||||
*guard = tx;
|
||||
}
|
||||
}
|
||||
|
||||
impl Resolve for LocalhostResolver {
|
||||
fn resolve(&self, name: Name) -> Resolving {
|
||||
let host = name.as_str().to_lowercase();
|
||||
let event_tx = self.event_tx.clone();
|
||||
let overrides = self.overrides.clone();
|
||||
let address_filter = self.address_filter.clone();
|
||||
|
||||
info!("DNS resolve called for: {}", host);
|
||||
|
||||
// Check for DNS override first
|
||||
if let Some(resolved) = overrides.get(&host) {
|
||||
log::debug!("DNS override found for: {}", host);
|
||||
let hostname = host.clone();
|
||||
let mut addrs: Vec<SocketAddr> = Vec::new();
|
||||
|
||||
// Add IPv4 addresses
|
||||
for ip in &resolved.ipv4 {
|
||||
addrs.push(SocketAddr::new(IpAddr::V4(*ip), 0));
|
||||
}
|
||||
|
||||
// Add IPv6 addresses
|
||||
for ip in &resolved.ipv6 {
|
||||
addrs.push(SocketAddr::new(IpAddr::V6(*ip), 0));
|
||||
}
|
||||
|
||||
let addresses: Vec<String> = addrs.iter().map(|a| a.ip().to_string()).collect();
|
||||
|
||||
return Box::pin(async move {
|
||||
Self::filter_addrs(&address_filter, &addrs)?;
|
||||
|
||||
// Emit DNS event for override
|
||||
let guard = event_tx.read().await;
|
||||
if let Some(tx) = guard.as_ref() {
|
||||
let _ = tx
|
||||
.send(HttpResponseEvent::DnsResolved {
|
||||
hostname,
|
||||
addresses,
|
||||
duration: 0,
|
||||
overridden: true,
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
Ok::<Addrs, Box<dyn std::error::Error + Send + Sync>>(Box::new(addrs.into_iter()))
|
||||
});
|
||||
}
|
||||
|
||||
// Check for .localhost suffix
|
||||
let is_localhost = host.ends_with(".localhost");
|
||||
if is_localhost {
|
||||
let hostname = host.clone();
|
||||
// Port 0 is fine; reqwest replaces it with the URL's explicit
|
||||
// port or the scheme's default (80/443, etc.).
|
||||
let addrs: Vec<SocketAddr> = vec![
|
||||
SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 0),
|
||||
SocketAddr::new(IpAddr::V6(Ipv6Addr::LOCALHOST), 0),
|
||||
];
|
||||
|
||||
let addresses: Vec<String> = addrs.iter().map(|a| a.ip().to_string()).collect();
|
||||
|
||||
return Box::pin(async move {
|
||||
Self::filter_addrs(&address_filter, &addrs)?;
|
||||
|
||||
// Emit DNS event for localhost resolution
|
||||
let guard = event_tx.read().await;
|
||||
if let Some(tx) = guard.as_ref() {
|
||||
let _ = tx
|
||||
.send(HttpResponseEvent::DnsResolved {
|
||||
hostname,
|
||||
addresses,
|
||||
duration: 0,
|
||||
overridden: false,
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
Ok::<Addrs, Box<dyn std::error::Error + Send + Sync>>(Box::new(addrs.into_iter()))
|
||||
});
|
||||
}
|
||||
|
||||
// Fall back to system DNS
|
||||
let mut fallback = self.fallback.clone();
|
||||
let name_str = name.as_str().to_string();
|
||||
let hostname = host.clone();
|
||||
|
||||
Box::pin(async move {
|
||||
let start = Instant::now();
|
||||
|
||||
let result = match HyperName::from_str(&name_str) {
|
||||
Ok(n) => fallback.call(n).await,
|
||||
Err(e) => return Err(Box::new(e) as Box<dyn std::error::Error + Send + Sync>),
|
||||
};
|
||||
|
||||
let duration = start.elapsed().as_millis() as u64;
|
||||
|
||||
match result {
|
||||
Ok(addrs) => {
|
||||
// Collect addresses for event emission
|
||||
let addr_vec: Vec<SocketAddr> = addrs.collect();
|
||||
Self::filter_addrs(&address_filter, &addr_vec)?;
|
||||
let addresses: Vec<String> =
|
||||
addr_vec.iter().map(|a| a.ip().to_string()).collect();
|
||||
|
||||
// Emit DNS event
|
||||
let guard = event_tx.read().await;
|
||||
if let Some(tx) = guard.as_ref() {
|
||||
let _ = tx
|
||||
.send(HttpResponseEvent::DnsResolved {
|
||||
hostname,
|
||||
addresses,
|
||||
duration,
|
||||
overridden: false,
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
Ok(Box::new(addr_vec.into_iter()) as Addrs)
|
||||
}
|
||||
Err(err) => Err(Box::new(err) as Box<dyn std::error::Error + Send + Sync>),
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
43
crates/yaak-http/src/error.rs
Normal file
43
crates/yaak-http/src/error.rs
Normal file
@@ -0,0 +1,43 @@
|
||||
use serde::{Serialize, Serializer};
|
||||
use thiserror::Error;
|
||||
|
||||
#[derive(Error, Debug)]
|
||||
pub enum Error {
|
||||
#[error("Client error: {0:?}")]
|
||||
Client(#[from] reqwest::Error),
|
||||
|
||||
#[error(transparent)]
|
||||
TlsError(#[from] yaak_tls::error::Error),
|
||||
|
||||
#[error("Native TLS error: {0}")]
|
||||
NativeTlsError(#[from] native_tls::Error),
|
||||
|
||||
#[error("IO error: {0}")]
|
||||
IoError(#[from] std::io::Error),
|
||||
|
||||
#[error("Request failed with {0:?}")]
|
||||
RequestError(String),
|
||||
|
||||
#[error("Request canceled")]
|
||||
RequestCanceledError,
|
||||
|
||||
#[error("Timeout of {0:?} reached")]
|
||||
RequestTimeout(std::time::Duration),
|
||||
|
||||
#[error("Decompression error: {0}")]
|
||||
DecompressionError(String),
|
||||
|
||||
#[error("Failed to read response body: {0}")]
|
||||
BodyReadError(String),
|
||||
}
|
||||
|
||||
impl Serialize for Error {
|
||||
fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
|
||||
where
|
||||
S: Serializer,
|
||||
{
|
||||
serializer.serialize_str(self.to_string().as_ref())
|
||||
}
|
||||
}
|
||||
|
||||
pub type Result<T> = std::result::Result<T, Error>;
|
||||
16
crates/yaak-http/src/lib.rs
Normal file
16
crates/yaak-http/src/lib.rs
Normal file
@@ -0,0 +1,16 @@
|
||||
mod chained_reader;
|
||||
pub mod client;
|
||||
pub mod cookies;
|
||||
pub mod decompress;
|
||||
pub mod dns;
|
||||
pub mod error;
|
||||
pub mod manager;
|
||||
mod proto;
|
||||
pub mod sender;
|
||||
pub mod tee_reader;
|
||||
pub mod transaction;
|
||||
pub mod types;
|
||||
|
||||
// Moved to yaak-models so the browser's wasm host can render requests with the
|
||||
// same code; re-exported here so existing callers keep their path.
|
||||
pub use yaak_models::path_placeholders;
|
||||
53
crates/yaak-http/src/manager.rs
Normal file
53
crates/yaak-http/src/manager.rs
Normal file
@@ -0,0 +1,53 @@
|
||||
use crate::client::{ConfiguredClient, HttpConnectionOptions};
|
||||
use crate::dns::LocalhostResolver;
|
||||
use crate::error::Result;
|
||||
use std::collections::BTreeMap;
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, Instant};
|
||||
use tokio::sync::RwLock;
|
||||
|
||||
/// A cached HTTP client along with its DNS resolver.
|
||||
/// The resolver is needed to set the event sender per-request.
|
||||
pub struct CachedClient {
|
||||
pub client: ConfiguredClient,
|
||||
pub resolver: Arc<LocalhostResolver>,
|
||||
}
|
||||
|
||||
pub struct HttpConnectionManager {
|
||||
connections: Arc<RwLock<BTreeMap<String, (CachedClient, Instant)>>>,
|
||||
ttl: Duration,
|
||||
}
|
||||
|
||||
impl HttpConnectionManager {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
connections: Arc::new(RwLock::new(BTreeMap::new())),
|
||||
ttl: Duration::from_secs(10 * 60),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn get_client(&self, opt: &HttpConnectionOptions) -> Result<CachedClient> {
|
||||
let mut connections = self.connections.write().await;
|
||||
// The key must include any per-request option that changes how the
|
||||
// client is built, or a send after a settings change reuses a client
|
||||
// built with the old value for up to the cache TTL.
|
||||
let id = format!("{}::{}::{}", opt.id, opt.validate_certificates, opt.http_version);
|
||||
|
||||
// Clean old connections
|
||||
connections.retain(|_, (_, last_used)| last_used.elapsed() <= self.ttl);
|
||||
|
||||
if let Some((cached, last_used)) = connections.get_mut(&id) {
|
||||
*last_used = Instant::now();
|
||||
return Ok(CachedClient {
|
||||
client: cached.client.clone(),
|
||||
resolver: cached.resolver.clone(),
|
||||
});
|
||||
}
|
||||
|
||||
let (client, resolver) = opt.build_client()?;
|
||||
let cached = CachedClient { client: client.clone(), resolver: resolver.clone() };
|
||||
connections.insert(id.into(), (cached, Instant::now()));
|
||||
|
||||
Ok(CachedClient { client, resolver })
|
||||
}
|
||||
}
|
||||
29
crates/yaak-http/src/proto.rs
Normal file
29
crates/yaak-http/src/proto.rs
Normal file
@@ -0,0 +1,29 @@
|
||||
use reqwest::Url;
|
||||
use std::str::FromStr;
|
||||
|
||||
pub(crate) fn ensure_proto(url_str: &str) -> String {
|
||||
if url_str.is_empty() {
|
||||
return "".to_string();
|
||||
}
|
||||
|
||||
if url_str.starts_with("http://") || url_str.starts_with("https://") {
|
||||
return url_str.to_string();
|
||||
}
|
||||
|
||||
// Url::from_str will fail without a proto, so add one
|
||||
let parseable_url = format!("http://{}", url_str);
|
||||
if let Ok(u) = Url::from_str(parseable_url.as_str()) {
|
||||
match u.host() {
|
||||
Some(host) => {
|
||||
let h = host.to_string();
|
||||
// These TLDs force HTTPS
|
||||
if h.ends_with(".app") || h.ends_with(".dev") || h.ends_with(".page") {
|
||||
return format!("https://{url_str}");
|
||||
}
|
||||
}
|
||||
None => {}
|
||||
}
|
||||
}
|
||||
|
||||
format!("http://{url_str}")
|
||||
}
|
||||
641
crates/yaak-http/src/sender.rs
Normal file
641
crates/yaak-http/src/sender.rs
Normal file
@@ -0,0 +1,641 @@
|
||||
use crate::decompress::{ContentEncoding, streaming_decoder};
|
||||
use crate::error::{Error, Result};
|
||||
use crate::types::{SendableBody, SendableHttpRequest};
|
||||
use async_trait::async_trait;
|
||||
use bytes::Bytes;
|
||||
use futures_util::StreamExt;
|
||||
use http_body::{Body as HttpBody, Frame, SizeHint};
|
||||
use reqwest::{Method, Version};
|
||||
use std::fmt::Display;
|
||||
use std::pin::Pin;
|
||||
use std::task::{Context, Poll};
|
||||
use std::time::Duration;
|
||||
use tokio::io::{AsyncRead, AsyncReadExt, BufReader, ReadBuf};
|
||||
use tokio::sync::mpsc;
|
||||
use tokio_util::io::StreamReader;
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum RedirectBehavior {
|
||||
/// 307/308: Method and body are preserved
|
||||
Preserve,
|
||||
/// 303 or 301/302 with POST: Method changed to GET, body dropped
|
||||
DropBody,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum HttpResponseEvent {
|
||||
Setting {
|
||||
name: String,
|
||||
value: String,
|
||||
source_model: Option<String>,
|
||||
source_id: Option<String>,
|
||||
source_name: Option<String>,
|
||||
},
|
||||
Info(String),
|
||||
Redirect {
|
||||
url: String,
|
||||
status: u16,
|
||||
behavior: RedirectBehavior,
|
||||
dropped_body: bool,
|
||||
dropped_headers: Vec<String>,
|
||||
},
|
||||
SendUrl {
|
||||
method: String,
|
||||
scheme: String,
|
||||
username: String,
|
||||
password: String,
|
||||
host: String,
|
||||
port: u16,
|
||||
path: String,
|
||||
query: String,
|
||||
fragment: String,
|
||||
},
|
||||
ReceiveUrl {
|
||||
version: Version,
|
||||
status: String,
|
||||
},
|
||||
HeaderUp(String, String),
|
||||
HeaderDown(String, String),
|
||||
ChunkSent {
|
||||
bytes: usize,
|
||||
},
|
||||
ChunkReceived {
|
||||
bytes: usize,
|
||||
},
|
||||
DnsResolved {
|
||||
hostname: String,
|
||||
addresses: Vec<String>,
|
||||
duration: u64,
|
||||
overridden: bool,
|
||||
},
|
||||
}
|
||||
|
||||
impl Display for HttpResponseEvent {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
match self {
|
||||
HttpResponseEvent::Setting { name, value, .. } => {
|
||||
write!(f, "* Setting {}={}", name, value)
|
||||
}
|
||||
HttpResponseEvent::Info(s) => write!(f, "* {}", s),
|
||||
HttpResponseEvent::Redirect {
|
||||
url,
|
||||
status,
|
||||
behavior,
|
||||
dropped_body,
|
||||
dropped_headers,
|
||||
} => {
|
||||
let behavior_str = match behavior {
|
||||
RedirectBehavior::Preserve => "preserve",
|
||||
RedirectBehavior::DropBody => "drop body",
|
||||
};
|
||||
let body_str = if *dropped_body { ", body dropped" } else { "" };
|
||||
let headers_str = if dropped_headers.is_empty() {
|
||||
String::new()
|
||||
} else {
|
||||
format!(", headers dropped: {}", dropped_headers.join(", "))
|
||||
};
|
||||
write!(
|
||||
f,
|
||||
"* Redirect {} -> {} ({}{}{})",
|
||||
status, url, behavior_str, body_str, headers_str
|
||||
)
|
||||
}
|
||||
HttpResponseEvent::SendUrl {
|
||||
method,
|
||||
scheme,
|
||||
username,
|
||||
password,
|
||||
host,
|
||||
port,
|
||||
path,
|
||||
query,
|
||||
fragment,
|
||||
} => {
|
||||
let auth_str = if username.is_empty() && password.is_empty() {
|
||||
String::new()
|
||||
} else {
|
||||
format!("{}:{}@", username, password)
|
||||
};
|
||||
let query_str =
|
||||
if query.is_empty() { String::new() } else { format!("?{}", query) };
|
||||
let fragment_str =
|
||||
if fragment.is_empty() { String::new() } else { format!("#{}", fragment) };
|
||||
write!(
|
||||
f,
|
||||
"> {} {}://{}{}:{}{}{}{}",
|
||||
method, scheme, auth_str, host, port, path, query_str, fragment_str
|
||||
)
|
||||
}
|
||||
HttpResponseEvent::ReceiveUrl { version, status } => {
|
||||
write!(f, "< {} {}", version_to_str(version), status)
|
||||
}
|
||||
HttpResponseEvent::HeaderUp(name, value) => write!(f, "> {}: {}", name, value),
|
||||
HttpResponseEvent::HeaderDown(name, value) => write!(f, "< {}: {}", name, value),
|
||||
HttpResponseEvent::ChunkSent { bytes } => write!(f, "> [{} bytes sent]", bytes),
|
||||
HttpResponseEvent::ChunkReceived { bytes } => write!(f, "< [{} bytes received]", bytes),
|
||||
HttpResponseEvent::DnsResolved { hostname, addresses, duration, overridden } => {
|
||||
if *overridden {
|
||||
write!(f, "* DNS override {} -> {}", hostname, addresses.join(", "))
|
||||
} else {
|
||||
write!(
|
||||
f,
|
||||
"* DNS resolved {} to {} ({}ms)",
|
||||
hostname,
|
||||
addresses.join(", "),
|
||||
duration
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<HttpResponseEvent> for yaak_models::models::HttpResponseEventData {
|
||||
fn from(event: HttpResponseEvent) -> Self {
|
||||
use yaak_models::models::HttpResponseEventData as D;
|
||||
match event {
|
||||
HttpResponseEvent::Setting { name, value, source_model, source_id, source_name } => {
|
||||
D::Setting { name, value, source_model, source_id, source_name }
|
||||
}
|
||||
HttpResponseEvent::Info(message) => D::Info { message },
|
||||
HttpResponseEvent::Redirect {
|
||||
url,
|
||||
status,
|
||||
behavior,
|
||||
dropped_body,
|
||||
dropped_headers,
|
||||
} => D::Redirect {
|
||||
url,
|
||||
status,
|
||||
behavior: match behavior {
|
||||
RedirectBehavior::Preserve => "preserve".to_string(),
|
||||
RedirectBehavior::DropBody => "drop_body".to_string(),
|
||||
},
|
||||
dropped_body,
|
||||
dropped_headers,
|
||||
},
|
||||
HttpResponseEvent::SendUrl {
|
||||
method,
|
||||
scheme,
|
||||
username,
|
||||
password,
|
||||
host,
|
||||
port,
|
||||
path,
|
||||
query,
|
||||
fragment,
|
||||
} => {
|
||||
D::SendUrl { method, scheme, username, password, host, port, path, query, fragment }
|
||||
}
|
||||
HttpResponseEvent::ReceiveUrl { version, status } => {
|
||||
D::ReceiveUrl { version: format!("{:?}", version), status }
|
||||
}
|
||||
HttpResponseEvent::HeaderUp(name, value) => D::HeaderUp { name, value },
|
||||
HttpResponseEvent::HeaderDown(name, value) => D::HeaderDown { name, value },
|
||||
HttpResponseEvent::ChunkSent { bytes } => D::ChunkSent { bytes },
|
||||
HttpResponseEvent::ChunkReceived { bytes } => D::ChunkReceived { bytes },
|
||||
HttpResponseEvent::DnsResolved { hostname, addresses, duration, overridden } => {
|
||||
D::DnsResolved { hostname, addresses, duration, overridden }
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Statistics about the body after consumption
|
||||
#[derive(Debug, Default, Clone)]
|
||||
pub struct BodyStats {
|
||||
/// Size of the body as received over the wire (before decompression)
|
||||
pub size_compressed: u64,
|
||||
/// Size of the body after decompression
|
||||
pub size_decompressed: u64,
|
||||
}
|
||||
|
||||
/// An AsyncRead wrapper that sends chunk events as data is read
|
||||
pub struct TrackingRead<R> {
|
||||
inner: R,
|
||||
event_tx: mpsc::Sender<HttpResponseEvent>,
|
||||
ended: bool,
|
||||
}
|
||||
|
||||
impl<R> TrackingRead<R> {
|
||||
pub fn new(inner: R, event_tx: mpsc::Sender<HttpResponseEvent>) -> Self {
|
||||
Self { inner, event_tx, ended: false }
|
||||
}
|
||||
}
|
||||
|
||||
impl<R: AsyncRead + Unpin> AsyncRead for TrackingRead<R> {
|
||||
fn poll_read(
|
||||
mut self: Pin<&mut Self>,
|
||||
cx: &mut Context<'_>,
|
||||
buf: &mut ReadBuf<'_>,
|
||||
) -> Poll<std::io::Result<()>> {
|
||||
let before = buf.filled().len();
|
||||
let result = Pin::new(&mut self.inner).poll_read(cx, buf);
|
||||
if let Poll::Ready(Ok(())) = &result {
|
||||
let bytes_read = buf.filled().len() - before;
|
||||
if bytes_read > 0 {
|
||||
// Ignore send errors - receiver may have been dropped or channel is full
|
||||
let _ =
|
||||
self.event_tx.try_send(HttpResponseEvent::ChunkReceived { bytes: bytes_read });
|
||||
} else if !self.ended {
|
||||
self.ended = true;
|
||||
}
|
||||
}
|
||||
result
|
||||
}
|
||||
}
|
||||
|
||||
/// Type alias for the body stream
|
||||
type BodyStream = Pin<Box<dyn AsyncRead + Send>>;
|
||||
|
||||
/// HTTP response with deferred body consumption.
|
||||
/// Headers are available immediately after send(), body can be consumed in different ways.
|
||||
/// Note: Debug is manually implemented since BodyStream doesn't implement Debug.
|
||||
pub struct HttpResponse {
|
||||
/// HTTP status code
|
||||
pub status: u16,
|
||||
/// HTTP status reason phrase (e.g., "OK", "Not Found")
|
||||
pub status_reason: Option<String>,
|
||||
/// Response headers (Vec to support multiple headers with same name, e.g., Set-Cookie)
|
||||
pub headers: Vec<(String, String)>,
|
||||
/// Request headers (Vec to support multiple headers with same name)
|
||||
pub request_headers: Vec<(String, String)>,
|
||||
/// Content-Length from headers (may differ from actual body size)
|
||||
pub content_length: Option<u64>,
|
||||
/// Final URL (after redirects)
|
||||
pub url: String,
|
||||
/// Remote address of the server
|
||||
pub remote_addr: Option<String>,
|
||||
/// HTTP version (e.g., "HTTP/1.1", "HTTP/2")
|
||||
pub version: Option<String>,
|
||||
|
||||
/// The body stream (consumed when calling bytes(), text(), write_to_file(), or drain())
|
||||
body_stream: Option<BodyStream>,
|
||||
/// Content-Encoding for decompression
|
||||
encoding: ContentEncoding,
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for HttpResponse {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.debug_struct("HttpResponse")
|
||||
.field("status", &self.status)
|
||||
.field("status_reason", &self.status_reason)
|
||||
.field("headers", &self.headers)
|
||||
.field("content_length", &self.content_length)
|
||||
.field("url", &self.url)
|
||||
.field("remote_addr", &self.remote_addr)
|
||||
.field("version", &self.version)
|
||||
.field("body_stream", &"<stream>")
|
||||
.field("encoding", &self.encoding)
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
impl HttpResponse {
|
||||
/// Create a new HttpResponse with an unconsumed body stream
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub fn new(
|
||||
status: u16,
|
||||
status_reason: Option<String>,
|
||||
headers: Vec<(String, String)>,
|
||||
request_headers: Vec<(String, String)>,
|
||||
content_length: Option<u64>,
|
||||
url: String,
|
||||
remote_addr: Option<String>,
|
||||
version: Option<String>,
|
||||
body_stream: BodyStream,
|
||||
encoding: ContentEncoding,
|
||||
) -> Self {
|
||||
Self {
|
||||
status,
|
||||
status_reason,
|
||||
headers,
|
||||
request_headers,
|
||||
content_length,
|
||||
url,
|
||||
remote_addr,
|
||||
version,
|
||||
body_stream: Some(body_stream),
|
||||
encoding,
|
||||
}
|
||||
}
|
||||
|
||||
/// Consume the body and return it as bytes (loads entire body into memory).
|
||||
/// Also decompresses the body if Content-Encoding is set.
|
||||
pub async fn bytes(mut self) -> Result<(Vec<u8>, BodyStats)> {
|
||||
let stream = self.body_stream.take().ok_or_else(|| {
|
||||
Error::RequestError("Response body has already been consumed".to_string())
|
||||
})?;
|
||||
|
||||
let buf_reader = BufReader::new(stream);
|
||||
let mut decoder = streaming_decoder(buf_reader, self.encoding);
|
||||
|
||||
let mut decompressed = Vec::new();
|
||||
let mut bytes_read = 0u64;
|
||||
|
||||
// Read through the decoder in chunks to track compressed size
|
||||
let mut buf = [0u8; 8192];
|
||||
loop {
|
||||
match decoder.read(&mut buf).await {
|
||||
Ok(0) => break,
|
||||
Ok(n) => {
|
||||
decompressed.extend_from_slice(&buf[..n]);
|
||||
bytes_read += n as u64;
|
||||
}
|
||||
Err(e) => {
|
||||
return Err(Error::BodyReadError(e.to_string()));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let stats = BodyStats {
|
||||
// For now, we can't easily track compressed size when streaming through decoder
|
||||
// Use content_length as an approximation, or decompressed size if identity encoding
|
||||
size_compressed: self.content_length.unwrap_or(bytes_read),
|
||||
size_decompressed: decompressed.len() as u64,
|
||||
};
|
||||
|
||||
Ok((decompressed, stats))
|
||||
}
|
||||
|
||||
/// Consume the body and return it as a UTF-8 string.
|
||||
pub async fn text(self) -> Result<(String, BodyStats)> {
|
||||
let (bytes, stats) = self.bytes().await?;
|
||||
let text = String::from_utf8(bytes)
|
||||
.map_err(|e| Error::RequestError(format!("Response is not valid UTF-8: {}", e)))?;
|
||||
Ok((text, stats))
|
||||
}
|
||||
|
||||
/// Take the body stream for manual consumption.
|
||||
/// Returns an AsyncRead that decompresses on-the-fly if Content-Encoding is set.
|
||||
/// The caller is responsible for reading and processing the stream.
|
||||
pub fn into_body_stream(&mut self) -> Result<Box<dyn AsyncRead + Unpin + Send>> {
|
||||
let stream = self.body_stream.take().ok_or_else(|| {
|
||||
Error::RequestError("Response body has already been consumed".to_string())
|
||||
})?;
|
||||
|
||||
let buf_reader = BufReader::new(stream);
|
||||
let decoder = streaming_decoder(buf_reader, self.encoding);
|
||||
|
||||
Ok(decoder)
|
||||
}
|
||||
|
||||
/// Discard the body without reading it (useful for redirects).
|
||||
pub async fn drain(mut self) -> Result<()> {
|
||||
let stream = self.body_stream.take().ok_or_else(|| {
|
||||
Error::RequestError("Response body has already been consumed".to_string())
|
||||
})?;
|
||||
|
||||
// Just read and discard all bytes
|
||||
let mut reader = stream;
|
||||
let mut buf = [0u8; 8192];
|
||||
loop {
|
||||
match reader.read(&mut buf).await {
|
||||
Ok(0) => break,
|
||||
Ok(_) => continue,
|
||||
Err(e) => {
|
||||
return Err(Error::RequestError(format!(
|
||||
"Failed to drain response body: {}",
|
||||
e
|
||||
)));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Trait for sending HTTP requests
|
||||
#[async_trait]
|
||||
pub trait HttpSender: Send + Sync {
|
||||
/// Send an HTTP request and return the response with headers.
|
||||
/// The body is not consumed until you call bytes(), text(), write_to_file(), or drain().
|
||||
/// Events are sent through the provided channel.
|
||||
async fn send(
|
||||
&self,
|
||||
request: SendableHttpRequest,
|
||||
event_tx: mpsc::Sender<HttpResponseEvent>,
|
||||
) -> Result<HttpResponse>;
|
||||
}
|
||||
|
||||
/// Reqwest-based implementation of HttpSender
|
||||
pub struct ReqwestSender {
|
||||
client: crate::client::ConfiguredClient,
|
||||
}
|
||||
|
||||
impl ReqwestSender {
|
||||
/// Create a new ReqwestSender with a default client
|
||||
pub fn new() -> Result<Self> {
|
||||
let client = crate::client::ConfiguredClient::build_default()?;
|
||||
Ok(Self { client })
|
||||
}
|
||||
|
||||
/// Create a new ReqwestSender with a configured client
|
||||
pub fn with_client(client: crate::client::ConfiguredClient) -> Self {
|
||||
Self { client }
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl HttpSender for ReqwestSender {
|
||||
async fn send(
|
||||
&self,
|
||||
request: SendableHttpRequest,
|
||||
event_tx: mpsc::Sender<HttpResponseEvent>,
|
||||
) -> Result<HttpResponse> {
|
||||
// Helper to send events (ignores errors if receiver is dropped or channel is full)
|
||||
let send_event = |event: HttpResponseEvent| {
|
||||
let _ = event_tx.try_send(event);
|
||||
};
|
||||
|
||||
// Parse the HTTP method
|
||||
let method = Method::from_bytes(request.method.as_bytes())
|
||||
.map_err(|e| Error::RequestError(format!("Invalid HTTP method: {}", e)))?;
|
||||
|
||||
// Build the request
|
||||
let mut req_builder = self.client.inner().request(method, &request.url);
|
||||
|
||||
// Add headers
|
||||
for header in request.headers {
|
||||
if header.0.is_empty() {
|
||||
continue;
|
||||
}
|
||||
req_builder = req_builder.header(&header.0, &header.1);
|
||||
}
|
||||
|
||||
// Configure timeout
|
||||
if let Some(d) = request.options.timeout
|
||||
&& !d.is_zero()
|
||||
{
|
||||
req_builder = req_builder.timeout(d);
|
||||
}
|
||||
|
||||
// Add body
|
||||
match request.body {
|
||||
None => {}
|
||||
Some(SendableBody::Bytes(bytes)) => {
|
||||
req_builder = req_builder.body(bytes);
|
||||
}
|
||||
Some(SendableBody::Stream { data, content_length }) => {
|
||||
// Convert AsyncRead stream to reqwest Body. If content length is
|
||||
// known, wrap with a SizedBody so hyper can set Content-Length
|
||||
// automatically (for both HTTP/1.1 and HTTP/2).
|
||||
let stream = tokio_util::io::ReaderStream::new(data);
|
||||
let body = if let Some(len) = content_length {
|
||||
reqwest::Body::wrap(SizedBody::new(stream, len))
|
||||
} else {
|
||||
reqwest::Body::wrap_stream(stream)
|
||||
};
|
||||
req_builder = req_builder.body(body);
|
||||
}
|
||||
}
|
||||
|
||||
// Send the request
|
||||
let sendable_req = req_builder.build()?;
|
||||
send_event(HttpResponseEvent::SendUrl {
|
||||
method: sendable_req.method().to_string(),
|
||||
scheme: sendable_req.url().scheme().to_string(),
|
||||
username: sendable_req.url().username().to_string(),
|
||||
password: sendable_req.url().password().unwrap_or_default().to_string(),
|
||||
host: sendable_req.url().host_str().unwrap_or_default().to_string(),
|
||||
port: sendable_req.url().port_or_known_default().unwrap_or(0),
|
||||
path: sendable_req.url().path().to_string(),
|
||||
query: sendable_req.url().query().unwrap_or_default().to_string(),
|
||||
fragment: sendable_req.url().fragment().unwrap_or_default().to_string(),
|
||||
});
|
||||
|
||||
let mut request_headers = Vec::new();
|
||||
for (name, value) in sendable_req.headers() {
|
||||
let v = value.to_str().unwrap_or_default().to_string();
|
||||
request_headers.push((name.to_string(), v.clone()));
|
||||
send_event(HttpResponseEvent::HeaderUp(name.to_string(), v));
|
||||
}
|
||||
send_event(HttpResponseEvent::Info("Sending request to server".to_string()));
|
||||
|
||||
// Map some errors to our own, so they look nicer
|
||||
let response = self.client.inner().execute(sendable_req).await.map_err(|e| {
|
||||
if reqwest::Error::is_timeout(&e) {
|
||||
Error::RequestTimeout(
|
||||
request.options.timeout.unwrap_or(Duration::from_secs(0)).clone(),
|
||||
)
|
||||
} else {
|
||||
Error::Client(e)
|
||||
}
|
||||
})?;
|
||||
|
||||
let status = response.status().as_u16();
|
||||
let status_reason = response.status().canonical_reason().map(|s| s.to_string());
|
||||
let url = response.url().to_string();
|
||||
let remote_addr = response.remote_addr().map(|a| a.to_string());
|
||||
let version = Some(version_to_str(&response.version()));
|
||||
let content_length = response.content_length();
|
||||
|
||||
send_event(HttpResponseEvent::ReceiveUrl {
|
||||
version: response.version(),
|
||||
status: response.status().to_string(),
|
||||
});
|
||||
|
||||
// Extract headers (use Vec to preserve duplicates like Set-Cookie)
|
||||
let mut headers = Vec::new();
|
||||
for (key, value) in response.headers() {
|
||||
if let Ok(v) = value.to_str() {
|
||||
send_event(HttpResponseEvent::HeaderDown(key.to_string(), v.to_string()));
|
||||
headers.push((key.to_string(), v.to_string()));
|
||||
}
|
||||
}
|
||||
|
||||
// Determine content encoding for decompression
|
||||
// HTTP headers are case-insensitive, so we need to search for any casing
|
||||
let encoding = ContentEncoding::from_header(
|
||||
headers
|
||||
.iter()
|
||||
.find(|(k, _)| k.eq_ignore_ascii_case("content-encoding"))
|
||||
.map(|(_, v)| v.as_str()),
|
||||
);
|
||||
|
||||
// Get the byte stream instead of loading into memory
|
||||
let byte_stream = response.bytes_stream();
|
||||
|
||||
// Convert the stream to an AsyncRead
|
||||
let stream_reader = StreamReader::new(
|
||||
byte_stream.map(|result| result.map_err(|e| std::io::Error::other(e))),
|
||||
);
|
||||
|
||||
// Wrap the stream with tracking to emit chunk received events via the same channel
|
||||
let tracking_reader = TrackingRead::new(stream_reader, event_tx);
|
||||
let body_stream: BodyStream = Box::pin(tracking_reader);
|
||||
|
||||
Ok(HttpResponse::new(
|
||||
status,
|
||||
status_reason,
|
||||
headers,
|
||||
request_headers,
|
||||
content_length,
|
||||
url,
|
||||
remote_addr,
|
||||
version,
|
||||
body_stream,
|
||||
encoding,
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
/// A wrapper around a byte stream that reports a known content length via
|
||||
/// `size_hint()`. This lets hyper set the `Content-Length` header
|
||||
/// automatically based on the body size, without us having to add it as an
|
||||
/// explicit header — which can cause duplicate `Content-Length` headers and
|
||||
/// break HTTP/2.
|
||||
struct SizedBody<S> {
|
||||
stream: std::sync::Mutex<S>,
|
||||
remaining: u64,
|
||||
}
|
||||
|
||||
impl<S> SizedBody<S> {
|
||||
fn new(stream: S, content_length: u64) -> Self {
|
||||
Self { stream: std::sync::Mutex::new(stream), remaining: content_length }
|
||||
}
|
||||
}
|
||||
|
||||
impl<S> HttpBody for SizedBody<S>
|
||||
where
|
||||
S: futures_util::Stream<Item = std::result::Result<Bytes, std::io::Error>>
|
||||
+ Send
|
||||
+ Unpin
|
||||
+ 'static,
|
||||
{
|
||||
type Data = Bytes;
|
||||
type Error = std::io::Error;
|
||||
|
||||
fn poll_frame(
|
||||
self: Pin<&mut Self>,
|
||||
cx: &mut Context<'_>,
|
||||
) -> Poll<Option<std::result::Result<Frame<Self::Data>, Self::Error>>> {
|
||||
let this = self.get_mut();
|
||||
let mut stream = this.stream.lock().unwrap();
|
||||
match stream.poll_next_unpin(cx) {
|
||||
Poll::Ready(Some(Ok(chunk))) => {
|
||||
this.remaining = this.remaining.saturating_sub(chunk.len() as u64);
|
||||
Poll::Ready(Some(Ok(Frame::data(chunk))))
|
||||
}
|
||||
Poll::Ready(Some(Err(e))) => Poll::Ready(Some(Err(e))),
|
||||
Poll::Ready(None) => Poll::Ready(None),
|
||||
Poll::Pending => Poll::Pending,
|
||||
}
|
||||
}
|
||||
|
||||
fn size_hint(&self) -> SizeHint {
|
||||
SizeHint::with_exact(self.remaining)
|
||||
}
|
||||
}
|
||||
|
||||
fn version_to_str(version: &Version) -> String {
|
||||
match *version {
|
||||
Version::HTTP_09 => "HTTP/0.9".to_string(),
|
||||
Version::HTTP_10 => "HTTP/1.0".to_string(),
|
||||
Version::HTTP_11 => "HTTP/1.1".to_string(),
|
||||
Version::HTTP_2 => "HTTP/2".to_string(),
|
||||
Version::HTTP_3 => "HTTP/3".to_string(),
|
||||
_ => "unknown".to_string(),
|
||||
}
|
||||
}
|
||||
159
crates/yaak-http/src/tee_reader.rs
Normal file
159
crates/yaak-http/src/tee_reader.rs
Normal file
@@ -0,0 +1,159 @@
|
||||
use std::io;
|
||||
use std::pin::Pin;
|
||||
use std::task::{Context, Poll};
|
||||
use tokio::io::{AsyncRead, ReadBuf};
|
||||
use tokio::sync::mpsc;
|
||||
|
||||
/// A reader that forwards all read data to a channel while also returning it to the caller.
|
||||
/// This allows capturing request body data as it's being sent.
|
||||
/// Uses an unbounded channel to ensure all data is captured without blocking the request.
|
||||
pub struct TeeReader<R> {
|
||||
inner: R,
|
||||
tx: mpsc::UnboundedSender<Vec<u8>>,
|
||||
}
|
||||
|
||||
impl<R> TeeReader<R> {
|
||||
pub fn new(inner: R, tx: mpsc::UnboundedSender<Vec<u8>>) -> Self {
|
||||
Self { inner, tx }
|
||||
}
|
||||
}
|
||||
|
||||
impl<R: AsyncRead + Unpin> AsyncRead for TeeReader<R> {
|
||||
fn poll_read(
|
||||
mut self: Pin<&mut Self>,
|
||||
cx: &mut Context<'_>,
|
||||
buf: &mut ReadBuf<'_>,
|
||||
) -> Poll<io::Result<()>> {
|
||||
let before_len = buf.filled().len();
|
||||
|
||||
match Pin::new(&mut self.inner).poll_read(cx, buf) {
|
||||
Poll::Ready(Ok(())) => {
|
||||
let after_len = buf.filled().len();
|
||||
if after_len > before_len {
|
||||
// Data was read, send a copy to the channel
|
||||
let data = buf.filled()[before_len..after_len].to_vec();
|
||||
// Send to unbounded channel - this never blocks
|
||||
// Ignore error if receiver is closed
|
||||
let _ = self.tx.send(data);
|
||||
}
|
||||
Poll::Ready(Ok(()))
|
||||
}
|
||||
Poll::Ready(Err(e)) => Poll::Ready(Err(e)),
|
||||
Poll::Pending => Poll::Pending,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::io::Cursor;
|
||||
use tokio::io::AsyncReadExt;
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_tee_reader_captures_all_data() {
|
||||
let data = b"Hello, World!";
|
||||
let cursor = Cursor::new(data.to_vec());
|
||||
let (tx, mut rx) = mpsc::unbounded_channel();
|
||||
|
||||
let mut tee = TeeReader::new(cursor, tx);
|
||||
let mut output = Vec::new();
|
||||
tee.read_to_end(&mut output).await.unwrap();
|
||||
|
||||
// Verify the reader returns the correct data
|
||||
assert_eq!(output, data);
|
||||
|
||||
// Verify the channel received the data
|
||||
let mut captured = Vec::new();
|
||||
while let Ok(chunk) = rx.try_recv() {
|
||||
captured.extend(chunk);
|
||||
}
|
||||
assert_eq!(captured, data);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_tee_reader_with_chunked_reads() {
|
||||
let data = b"ABCDEFGHIJKLMNOPQRSTUVWXYZ";
|
||||
let cursor = Cursor::new(data.to_vec());
|
||||
let (tx, mut rx) = mpsc::unbounded_channel();
|
||||
|
||||
let mut tee = TeeReader::new(cursor, tx);
|
||||
|
||||
// Read in small chunks
|
||||
let mut buf = [0u8; 5];
|
||||
let mut output = Vec::new();
|
||||
loop {
|
||||
let n = tee.read(&mut buf).await.unwrap();
|
||||
if n == 0 {
|
||||
break;
|
||||
}
|
||||
output.extend_from_slice(&buf[..n]);
|
||||
}
|
||||
|
||||
// Verify the reader returns the correct data
|
||||
assert_eq!(output, data);
|
||||
|
||||
// Verify the channel received all chunks
|
||||
let mut captured = Vec::new();
|
||||
while let Ok(chunk) = rx.try_recv() {
|
||||
captured.extend(chunk);
|
||||
}
|
||||
assert_eq!(captured, data);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_tee_reader_empty_data() {
|
||||
let data: Vec<u8> = vec![];
|
||||
let cursor = Cursor::new(data.clone());
|
||||
let (tx, mut rx) = mpsc::unbounded_channel();
|
||||
|
||||
let mut tee = TeeReader::new(cursor, tx);
|
||||
let mut output = Vec::new();
|
||||
tee.read_to_end(&mut output).await.unwrap();
|
||||
|
||||
// Verify empty output
|
||||
assert!(output.is_empty());
|
||||
|
||||
// Verify no data was sent to channel
|
||||
assert!(rx.try_recv().is_err());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_tee_reader_works_when_receiver_dropped() {
|
||||
let data = b"Hello, World!";
|
||||
let cursor = Cursor::new(data.to_vec());
|
||||
let (tx, rx) = mpsc::unbounded_channel();
|
||||
|
||||
// Drop the receiver before reading
|
||||
drop(rx);
|
||||
|
||||
let mut tee = TeeReader::new(cursor, tx);
|
||||
let mut output = Vec::new();
|
||||
|
||||
// Should still work even though receiver is dropped
|
||||
tee.read_to_end(&mut output).await.unwrap();
|
||||
assert_eq!(output, data);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_tee_reader_large_data() {
|
||||
// Test with 1MB of data
|
||||
let data: Vec<u8> = (0..1024 * 1024).map(|i| (i % 256) as u8).collect();
|
||||
let cursor = Cursor::new(data.clone());
|
||||
let (tx, mut rx) = mpsc::unbounded_channel();
|
||||
|
||||
let mut tee = TeeReader::new(cursor, tx);
|
||||
let mut output = Vec::new();
|
||||
tee.read_to_end(&mut output).await.unwrap();
|
||||
|
||||
// Verify the reader returns the correct data
|
||||
assert_eq!(output, data);
|
||||
|
||||
// Verify the channel received all data
|
||||
let mut captured = Vec::new();
|
||||
while let Ok(chunk) = rx.try_recv() {
|
||||
captured.extend(chunk);
|
||||
}
|
||||
assert_eq!(captured, data);
|
||||
}
|
||||
}
|
||||
1089
crates/yaak-http/src/transaction.rs
Normal file
1089
crates/yaak-http/src/transaction.rs
Normal file
File diff suppressed because it is too large
Load Diff
1262
crates/yaak-http/src/types.rs
Normal file
1262
crates/yaak-http/src/types.rs
Normal file
File diff suppressed because it is too large
Load Diff
1
crates/yaak-http/tests/test.txt
Normal file
1
crates/yaak-http/tests/test.txt
Normal file
@@ -0,0 +1 @@
|
||||
This is a test file!
|
||||
Reference in New Issue
Block a user