snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)

seaweed-volume/src/server/debug.rs

@@ -0,0 +1,159 @@
+use axum::Router;
+use axum::body::Body;
+use axum::extract::Query;
+use axum::http::{StatusCode, header};
+use axum::response::{IntoResponse, Response};
+use axum::routing::{any, get};
+use pprof::protos::Message;
+use serde::Deserialize;
+
+#[derive(Deserialize, Default)]
+struct ProfileQuery {
+    seconds: Option<u64>,
+}
+
+pub fn build_debug_router() -> Router {
+    Router::new()
+        .route("/debug/pprof/", get(pprof_index_handler))
+        .route("/debug/pprof/cmdline", get(pprof_cmdline_handler))
+        .route("/debug/pprof/profile", get(pprof_profile_handler))
+        .route("/debug/pprof/symbol", any(pprof_symbol_handler))
+        .route("/debug/pprof/trace", get(pprof_trace_handler))
+}
+
+async fn pprof_index_handler() -> Response {
+    let body = concat!(
+        "<html><head><title>/debug/pprof/</title></head><body>",
+        "<a href=\"cmdline\">cmdline</a><br>",
+        "<a href=\"profile\">profile</a><br>",
+        "<a href=\"symbol\">symbol</a><br>",
+        "<a href=\"trace\">trace</a><br>",
+        "</body></html>",
+    );
+    (
+        StatusCode::OK,
+        [(header::CONTENT_TYPE, "text/html; charset=utf-8")],
+        body,
+    )
+        .into_response()
+}
+
+async fn pprof_cmdline_handler() -> Response {
+    let body = std::env::args().collect::<Vec<_>>().join("\0");
+    (
+        StatusCode::OK,
+        [(header::CONTENT_TYPE, "text/plain; charset=utf-8")],
+        body,
+    )
+        .into_response()
+}
+
+async fn pprof_profile_handler(Query(query): Query<ProfileQuery>) -> Response {
+    let seconds = query.seconds.unwrap_or(30).clamp(1, 300);
+    let guard = match pprof::ProfilerGuard::new(100) {
+        Ok(guard) => guard,
+        Err(e) => {
+            return (
+                StatusCode::INTERNAL_SERVER_ERROR,
+                format!("failed to start profiler: {}", e),
+            )
+                .into_response();
+        }
+    };
+
+    tokio::time::sleep(std::time::Duration::from_secs(seconds)).await;
+
+    let report = match guard.report().build() {
+        Ok(report) => report,
+        Err(e) => {
+            return (
+                StatusCode::INTERNAL_SERVER_ERROR,
+                format!("failed to build profile report: {}", e),
+            )
+                .into_response();
+        }
+    };
+
+    let profile = match report.pprof() {
+        Ok(profile) => profile,
+        Err(e) => {
+            return (
+                StatusCode::INTERNAL_SERVER_ERROR,
+                format!("failed to encode profile: {}", e),
+            )
+                .into_response();
+        }
+    };
+
+    let mut bytes = Vec::new();
+    if let Err(e) = profile.encode(&mut bytes) {
+        return (
+            StatusCode::INTERNAL_SERVER_ERROR,
+            format!("failed to serialize profile: {}", e),
+        )
+            .into_response();
+    }
+
+    (
+        StatusCode::OK,
+        [(header::CONTENT_TYPE, "application/octet-stream")],
+        bytes,
+    )
+        .into_response()
+}
+
+async fn pprof_symbol_handler() -> Response {
+    (
+        StatusCode::OK,
+        [(header::CONTENT_TYPE, "text/plain; charset=utf-8")],
+        "num_symbols: 0\n",
+    )
+        .into_response()
+}
+
+async fn pprof_trace_handler(Query(query): Query<ProfileQuery>) -> Response {
+    let seconds = query.seconds.unwrap_or(1).clamp(1, 30);
+    tokio::time::sleep(std::time::Duration::from_secs(seconds)).await;
+    Response::builder()
+        .status(StatusCode::OK)
+        .header(header::CONTENT_TYPE, "application/octet-stream")
+        .body(Body::from(Vec::<u8>::new()))
+        .unwrap()
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use axum::http::Request;
+    use tower::ServiceExt;
+
+    #[tokio::test]
+    async fn test_debug_index_route() {
+        let app = build_debug_router();
+        let response = app
+            .oneshot(
+                Request::builder()
+                    .uri("/debug/pprof/")
+                    .body(Body::empty())
+                    .unwrap(),
+            )
+            .await
+            .unwrap();
+        assert_eq!(response.status(), StatusCode::OK);
+    }
+
+    #[tokio::test]
+    async fn test_debug_cmdline_route() {
+        let app = build_debug_router();
+        let response = app
+            .oneshot(
+                Request::builder()
+                    .uri("/debug/pprof/cmdline")
+                    .body(Body::empty())
+                    .unwrap(),
+            )
+            .await
+            .unwrap();
+        assert_eq!(response.status(), StatusCode::OK);
+    }
+}

seaweed-volume/src/server/grpc_client.rs

@@ -0,0 +1,370 @@
+use std::error::Error;
+use std::fmt;
+use std::time::Duration;
+
+use hyper::http::Uri;
+use tonic::transport::{Certificate, Channel, ClientTlsConfig, Endpoint, Identity};
+
+use crate::config::VolumeServerConfig;
+
+pub const GRPC_MAX_MESSAGE_SIZE: usize = 1 << 30;
+const GRPC_KEEPALIVE_INTERVAL: Duration = Duration::from_secs(60);
+const GRPC_KEEPALIVE_TIMEOUT: Duration = Duration::from_secs(20);
+const GRPC_INITIAL_WINDOW_SIZE: u32 = 16 * 1024 * 1024;
+
+#[derive(Clone, Debug)]
+pub struct OutgoingGrpcTlsConfig {
+    cert_pem: String,
+    key_pem: String,
+    ca_pem: String,
+}
+
+#[derive(Debug)]
+pub struct GrpcClientError(String);
+
+impl fmt::Display for GrpcClientError {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.write_str(&self.0)
+    }
+}
+
+impl Error for GrpcClientError {}
+
+pub fn load_outgoing_grpc_tls(
+    config: &VolumeServerConfig,
+) -> Result<Option<OutgoingGrpcTlsConfig>, GrpcClientError> {
+    // prefer a dedicated client certificate: CAs may issue certs with only one of the serverAuth/clientAuth EKUs
+    let (cert_file, key_file) = if !config.grpc_client_cert_file.is_empty()
+        && !config.grpc_client_key_file.is_empty()
+    {
+        (&config.grpc_client_cert_file, &config.grpc_client_key_file)
+    } else {
+        if !config.grpc_client_cert_file.is_empty() || !config.grpc_client_key_file.is_empty() {
+            tracing::warn!(
+                "grpc.volume.client_cert and grpc.volume.client_key must both be set, falling back to grpc.volume.cert and grpc.volume.key"
+            );
+        }
+        (&config.grpc_cert_file, &config.grpc_key_file)
+    };
+    if cert_file.is_empty() || key_file.is_empty() || config.grpc_ca_file.is_empty() {
+        return Ok(None);
+    }
+
+    let cert_pem = std::fs::read_to_string(cert_file).map_err(|e| {
+        GrpcClientError(format!(
+            "Failed to read outgoing gRPC cert '{}': {}",
+            cert_file, e
+        ))
+    })?;
+    let key_pem = std::fs::read_to_string(key_file).map_err(|e| {
+        GrpcClientError(format!(
+            "Failed to read outgoing gRPC key '{}': {}",
+            key_file, e
+        ))
+    })?;
+    let ca_pem = std::fs::read_to_string(&config.grpc_ca_file).map_err(|e| {
+        GrpcClientError(format!(
+            "Failed to read outgoing gRPC CA '{}': {}",
+            config.grpc_ca_file, e
+        ))
+    })?;
+
+    Ok(Some(OutgoingGrpcTlsConfig {
+        cert_pem,
+        key_pem,
+        ca_pem,
+    }))
+}
+
+pub fn grpc_endpoint_uri(grpc_host_port: &str, tls: Option<&OutgoingGrpcTlsConfig>) -> String {
+    let scheme = if tls.is_some() { "https" } else { "http" };
+    format!("{}://{}", scheme, grpc_host_port)
+}
+
+pub fn build_grpc_endpoint(
+    grpc_host_port: &str,
+    tls: Option<&OutgoingGrpcTlsConfig>,
+) -> Result<Endpoint, GrpcClientError> {
+    let uri = grpc_endpoint_uri(grpc_host_port, tls);
+    let mut endpoint = Channel::from_shared(uri.clone())
+        .map_err(|e| GrpcClientError(format!("invalid gRPC endpoint {}: {}", uri, e)))?
+        .http2_keep_alive_interval(GRPC_KEEPALIVE_INTERVAL)
+        .keep_alive_timeout(GRPC_KEEPALIVE_TIMEOUT)
+        .keep_alive_while_idle(false)
+        .initial_stream_window_size(Some(GRPC_INITIAL_WINDOW_SIZE))
+        .initial_connection_window_size(Some(GRPC_INITIAL_WINDOW_SIZE))
+        .http2_adaptive_window(false);
+
+    if let Some(tls) = tls {
+        let parsed = uri
+            .parse::<Uri>()
+            .map_err(|e| GrpcClientError(format!("invalid gRPC endpoint {}: {}", uri, e)))?;
+        let host = parsed
+            .host()
+            .ok_or_else(|| GrpcClientError(format!("missing host in gRPC endpoint {}", uri)))?;
+        let tls_config = ClientTlsConfig::new()
+            .identity(Identity::from_pem(
+                tls.cert_pem.clone(),
+                tls.key_pem.clone(),
+            ))
+            .ca_certificate(Certificate::from_pem(tls.ca_pem.clone()))
+            .domain_name(host.to_string());
+        endpoint = endpoint.tls_config(tls_config).map_err(|e| {
+            GrpcClientError(format!("configure gRPC TLS for {} failed: {}", uri, e))
+        })?;
+    }
+
+    Ok(endpoint)
+}
+
+/// Parse a SeaweedFS server address (`"ip:port.grpcPort"` or
+/// `"ip:port"`) into the `host:grpcPort` form `build_grpc_endpoint`
+/// expects. With the trailing `.grpcPort` segment, that segment IS
+/// the gRPC port; without it, the gRPC port is `port + 10000`
+/// (SeaweedFS's HTTP↔gRPC port-offset convention).
+///
+/// Shared between `grpc_server.rs` and the distributed-EC-read path
+/// in `store_ec.rs` — keep this as the single source of truth so the
+/// HTTP↔gRPC port translation can't drift between callers.
+pub fn parse_grpc_address(source: &str) -> Result<String, String> {
+    let colon_idx = source
+        .rfind(':')
+        .ok_or_else(|| format!("cannot parse address: {}", source))?;
+    let host = &source[..colon_idx];
+    let port_part = &source[colon_idx + 1..];
+
+    if let Some(dot_idx) = port_part.rfind('.') {
+        // Format: "ip:port.grpcPort". Validate BOTH ports as u16
+        // so a malformed HTTP port (e.g. `host:abc.18080`) is
+        // rejected here rather than tripping a downstream
+        // `build_grpc_endpoint` URI parse failure with a less
+        // useful error.
+        let http_port = &port_part[..dot_idx];
+        let grpc_port = &port_part[dot_idx + 1..];
+        http_port
+            .parse::<u16>()
+            .map_err(|e| format!("invalid http port {:?}: {}", http_port, e))?;
+        grpc_port
+            .parse::<u16>()
+            .map_err(|e| format!("invalid grpc port {:?}: {}", grpc_port, e))?;
+        return Ok(format!("{}:{}", host, grpc_port));
+    }
+
+    // Format: "ip:port" → grpc = port + 10000. Reject inputs whose
+    // implicit grpc port would overflow the TCP port range (e.g.
+    // `host:60000` produces 70000 — invalid). Without this check
+    // the cast silently wraps and the endpoint call later fails
+    // with an opaque connection error.
+    let port: u16 = port_part
+        .parse()
+        .map_err(|e| format!("invalid port {:?}: {}", port_part, e))?;
+    let grpc_port = port as u32 + 10000;
+    if grpc_port > u16::MAX as u32 {
+        return Err(format!(
+            "implicit grpc port out of range: {} + 10000 = {}",
+            port, grpc_port
+        ));
+    }
+    Ok(format!("{}:{}", host, grpc_port))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::{build_grpc_endpoint, grpc_endpoint_uri, load_outgoing_grpc_tls};
+    use crate::config::{NeedleMapKind, ReadMode, VolumeServerConfig};
+    use crate::security::tls::TlsPolicy;
+
+    const TEST_CERT_PEM: &str = "-----BEGIN CERTIFICATE-----\nMIIBPDCB76ADAgECAhRuRPQgeAu43BT/M7EfAWSdapVdYDAFBgMrZXAwFDESMBAG\nA1UEAwwJbG9jYWxob3N0MB4XDTI2MDcwNTE2MTUyOVoXDTM2MDcwMjE2MTUyOVow\nFDESMBAGA1UEAwwJbG9jYWxob3N0MCowBQYDK2VwAyEAr/3bNIFI+8V32oCiY6y+\nXRFmZpdNQ2g//VtRkT+nQg+jUzBRMB0GA1UdDgQWBBTsy9tLf1zPiXCQfgci6zNi\ndEzRSjAfBgNVHSMEGDAWgBTsy9tLf1zPiXCQfgci6zNidEzRSjAPBgNVHRMBAf8E\nBTADAQH/MAUGAytlcANBAIvsdw0IbvOBBkb9cd7BfMJfIP9pQQrAL03pCRWJFnFh\nSysaLVgFXI4T078IiaM874oO+iB+5vNbWEpc7CkGow4=\n-----END CERTIFICATE-----\n";
+    const TEST_KEY_PEM: &str = "-----BEGIN PRIVATE KEY-----\nMC4CAQAwBQYDK2VwBCIEIHbyn71Kk+Y7KT3sBctit7uZpErpoH6qDbFj6P8qGaZH\n-----END PRIVATE KEY-----\n";
+
+    #[test]
+    fn test_build_grpc_endpoint_with_tls_resolves_crypto_provider() {
+        crate::security::tls::install_default_crypto_provider();
+        let tls = super::OutgoingGrpcTlsConfig {
+            cert_pem: TEST_CERT_PEM.to_string(),
+            key_pem: TEST_KEY_PEM.to_string(),
+            ca_pem: TEST_CERT_PEM.to_string(),
+        };
+        let endpoint = build_grpc_endpoint("127.0.0.1:19333", Some(&tls)).unwrap();
+        assert_eq!(endpoint.uri().scheme_str(), Some("https"));
+    }
+
+    fn sample_config() -> VolumeServerConfig {
+        VolumeServerConfig {
+            port: 8080,
+            grpc_port: 18080,
+            public_port: 8080,
+            ip: "127.0.0.1".to_string(),
+            bind_ip: String::new(),
+            public_url: "127.0.0.1:8080".to_string(),
+            id: String::new(),
+            masters: vec![],
+            pre_stop_seconds: 0,
+            idle_timeout: 0,
+            data_center: String::new(),
+            rack: String::new(),
+            index_type: NeedleMapKind::InMemory,
+            disk_type: String::new(),
+            folders: vec![],
+            folder_max_limits: vec![],
+            folder_tags: vec![],
+            min_free_spaces: vec![],
+            disk_types: vec![],
+            idx_folder: String::new(),
+            white_list: vec![],
+            fix_jpg_orientation: false,
+            read_mode: ReadMode::Local,
+            allow_untrusted_remote_endpoints: false,
+            cpu_profile: String::new(),
+            mem_profile: String::new(),
+            compaction_byte_per_second: 0,
+            maintenance_byte_per_second: 0,
+            file_size_limit_bytes: 0,
+            concurrent_upload_limit: 0,
+            concurrent_download_limit: 0,
+            inflight_upload_data_timeout: std::time::Duration::from_secs(0),
+            inflight_download_data_timeout: std::time::Duration::from_secs(0),
+            has_slow_read: false,
+            read_buffer_size_mb: 0,
+            ldb_timeout: 0,
+            pprof: false,
+            metrics_port: 0,
+            metrics_ip: String::new(),
+            debug: false,
+            debug_port: 0,
+            ui_enabled: false,
+            jwt_signing_key: vec![],
+            jwt_signing_expires_seconds: 0,
+            jwt_read_signing_key: vec![],
+            jwt_read_signing_expires_seconds: 0,
+            https_cert_file: String::new(),
+            https_key_file: String::new(),
+            https_ca_file: String::new(),
+            https_client_enabled: false,
+            https_client_cert_file: String::new(),
+            https_client_key_file: String::new(),
+            https_client_ca_file: String::new(),
+            grpc_cert_file: String::new(),
+            grpc_key_file: String::new(),
+            grpc_client_cert_file: String::new(),
+            grpc_client_key_file: String::new(),
+            grpc_ca_file: String::new(),
+            grpc_allowed_wildcard_domain: String::new(),
+            grpc_volume_allowed_common_names: vec![],
+            tls_policy: TlsPolicy::default(),
+            enable_write_queue: false,
+            security_file: String::new(),
+        }
+    }
+
+    #[test]
+    fn test_grpc_endpoint_uri_uses_https_when_tls_enabled() {
+        let tls = super::OutgoingGrpcTlsConfig {
+            cert_pem: "cert".to_string(),
+            key_pem: "key".to_string(),
+            ca_pem: "ca".to_string(),
+        };
+        assert_eq!(
+            grpc_endpoint_uri("master.example.com:19333", Some(&tls)),
+            "https://master.example.com:19333"
+        );
+    }
+
+    #[test]
+    fn test_load_outgoing_grpc_tls_requires_cert_key_and_ca() {
+        let mut config = sample_config();
+        config.grpc_cert_file = "/tmp/client.pem".to_string();
+        assert!(load_outgoing_grpc_tls(&config).unwrap().is_none());
+    }
+
+    fn write_pem_files(dir: &tempfile::TempDir, config: &mut VolumeServerConfig) {
+        let write = |name: &str, content: &str| {
+            let path = dir.path().join(name);
+            std::fs::write(&path, content).unwrap();
+            path.to_str().unwrap().to_string()
+        };
+        config.grpc_cert_file = write("server.pem", "server-cert");
+        config.grpc_key_file = write("server.key", "server-key");
+        config.grpc_ca_file = write("ca.pem", "ca");
+    }
+
+    #[test]
+    fn test_load_outgoing_grpc_tls_prefers_client_cert() {
+        let dir = tempfile::TempDir::new().unwrap();
+        let mut config = sample_config();
+        write_pem_files(&dir, &mut config);
+        let client_cert = dir.path().join("client.pem");
+        let client_key = dir.path().join("client.key");
+        std::fs::write(&client_cert, "client-cert").unwrap();
+        std::fs::write(&client_key, "client-key").unwrap();
+        config.grpc_client_cert_file = client_cert.to_str().unwrap().to_string();
+        config.grpc_client_key_file = client_key.to_str().unwrap().to_string();
+
+        let tls = load_outgoing_grpc_tls(&config).unwrap().unwrap();
+        assert_eq!(tls.cert_pem, "client-cert");
+        assert_eq!(tls.key_pem, "client-key");
+    }
+
+    #[test]
+    fn test_load_outgoing_grpc_tls_falls_back_to_server_cert() {
+        let dir = tempfile::TempDir::new().unwrap();
+        let mut config = sample_config();
+        write_pem_files(&dir, &mut config);
+
+        let tls = load_outgoing_grpc_tls(&config).unwrap().unwrap();
+        assert_eq!(tls.cert_pem, "server-cert");
+        assert_eq!(tls.key_pem, "server-key");
+    }
+
+    #[test]
+    fn test_build_grpc_endpoint_without_tls_uses_http_scheme() {
+        let endpoint = build_grpc_endpoint("127.0.0.1:19333", None).unwrap();
+        assert_eq!(endpoint.uri().scheme_str(), Some("http"));
+    }
+
+    #[test]
+    fn test_parse_grpc_address_dotted_form() {
+        use super::parse_grpc_address;
+        assert_eq!(
+            parse_grpc_address("127.0.0.1:8080.18080").unwrap(),
+            "127.0.0.1:18080"
+        );
+    }
+
+    #[test]
+    fn test_parse_grpc_address_implicit_form_adds_10000() {
+        use super::parse_grpc_address;
+        assert_eq!(
+            parse_grpc_address("127.0.0.1:8080").unwrap(),
+            "127.0.0.1:18080"
+        );
+    }
+
+    #[test]
+    fn test_parse_grpc_address_rejects_non_numeric_http_port_in_dotted_form() {
+        use super::parse_grpc_address;
+        let err = parse_grpc_address("host:abc.18080").unwrap_err();
+        assert!(err.contains("invalid http port"), "{}", err);
+    }
+
+    #[test]
+    fn test_parse_grpc_address_rejects_non_numeric_grpc_port_in_dotted_form() {
+        use super::parse_grpc_address;
+        let err = parse_grpc_address("host:8080.xyz").unwrap_err();
+        assert!(err.contains("invalid grpc port"), "{}", err);
+    }
+
+    #[test]
+    fn test_parse_grpc_address_rejects_implicit_port_that_overflows() {
+        use super::parse_grpc_address;
+        let err = parse_grpc_address("127.0.0.1:60000").unwrap_err();
+        assert!(err.contains("out of range"), "{}", err);
+    }
+
+    #[test]
+    fn test_parse_grpc_address_rejects_input_without_colon() {
+        use super::parse_grpc_address;
+        let err = parse_grpc_address("hostname").unwrap_err();
+        assert!(err.contains("cannot parse"), "{}", err);
+    }
+}

seaweed-volume/src/server/memory_status.rs

@@ -0,0 +1,102 @@
+use crate::pb::volume_server_pb;
+
+pub fn collect_mem_status() -> volume_server_pb::MemStatus {
+    #[allow(unused_mut)]
+    let mut mem = volume_server_pb::MemStatus {
+        goroutines: 1,
+        ..Default::default()
+    };
+
+    #[cfg(target_os = "linux")]
+    {
+        if let Some((all, free)) = get_system_memory_linux() {
+            mem.all = all;
+            mem.free = free;
+            mem.used = all.saturating_sub(free);
+        }
+
+        if let Some(status) = read_process_status_linux() {
+            if status.threads > 0 {
+                mem.goroutines = status.threads as i32;
+            }
+            if let Some(rss) = status.rss {
+                mem.self_ = rss;
+            }
+            if let Some(heap) = status.data.or(status.rss) {
+                mem.heap = heap;
+            }
+            if let Some(stack) = status.stack {
+                mem.stack = stack;
+            }
+        }
+    }
+
+    mem
+}
+
+#[cfg(target_os = "linux")]
+fn get_system_memory_linux() -> Option<(u64, u64)> {
+    unsafe {
+        let mut info: libc::sysinfo = std::mem::zeroed();
+        if libc::sysinfo(&mut info) == 0 {
+            let unit = info.mem_unit as u64;
+            let total = info.totalram as u64 * unit;
+            let free = info.freeram as u64 * unit;
+            return Some((total, free));
+        }
+    }
+    None
+}
+
+#[cfg(target_os = "linux")]
+#[derive(Default)]
+struct ProcessStatus {
+    threads: u64,
+    rss: Option<u64>,
+    data: Option<u64>,
+    stack: Option<u64>,
+}
+
+#[cfg(target_os = "linux")]
+fn read_process_status_linux() -> Option<ProcessStatus> {
+    let status = std::fs::read_to_string("/proc/self/status").ok()?;
+    let mut out = ProcessStatus::default();
+
+    for line in status.lines() {
+        if let Some(value) = line.strip_prefix("Threads:") {
+            out.threads = value.trim().parse().ok()?;
+            continue;
+        }
+        if let Some(value) = parse_proc_status_kib_field(line, "VmRSS:") {
+            out.rss = Some(value);
+            continue;
+        }
+        if let Some(value) = parse_proc_status_kib_field(line, "VmData:") {
+            out.data = Some(value);
+            continue;
+        }
+        if let Some(value) = parse_proc_status_kib_field(line, "VmStk:") {
+            out.stack = Some(value);
+        }
+    }
+
+    Some(out)
+}
+
+#[cfg(target_os = "linux")]
+fn parse_proc_status_kib_field(line: &str, prefix: &str) -> Option<u64> {
+    let raw = line.strip_prefix(prefix)?.trim();
+    let value = raw.strip_suffix(" kB").unwrap_or(raw).trim();
+    value.parse::<u64>().ok().map(|kib| kib * 1024)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::collect_mem_status;
+
+    #[test]
+    fn test_collect_mem_status_reports_live_process_state() {
+        let mem = collect_mem_status();
+        assert!(mem.goroutines > 0);
+    }
+}

seaweed-volume/src/server/mod.rs

@@ -0,0 +1,15 @@
+#[cfg(unix)]
+pub mod debug;
+pub mod grpc_client;
+pub mod grpc_server;
+pub mod handlers;
+pub mod heartbeat;
+pub mod memory_status;
+#[cfg(unix)]
+pub mod profiling;
+pub mod request_id;
+pub mod server_stats;
+pub mod store_ec;
+pub mod ui;
+pub mod volume_server;
+pub mod write_queue;

seaweed-volume/src/server/profiling.rs

@@ -0,0 +1,190 @@
+use std::fs::File;
+use std::io::Write;
+use std::path::PathBuf;
+
+use pprof::protos::Message;
+
+use crate::config::VolumeServerConfig;
+
+const GO_CPU_PROFILE_FREQUENCY: i32 = 100;
+const GO_PPROF_BLOCKLIST: [&str; 4] = ["libc", "libgcc", "pthread", "vdso"];
+
+pub struct CpuProfileSession {
+    output_path: PathBuf,
+    guard: pprof::ProfilerGuard<'static>,
+}
+
+impl CpuProfileSession {
+    pub fn start(config: &VolumeServerConfig) -> Result<Option<Self>, String> {
+        if config.cpu_profile.is_empty() {
+            if !config.mem_profile.is_empty() && !config.pprof {
+                tracing::warn!(
+                    "--memprofile is not yet supported in the Rust volume server; ignoring '{}'",
+                    config.mem_profile
+                );
+            }
+            return Ok(None);
+        }
+
+        if config.pprof {
+            tracing::info!(
+                "--pprof is enabled; ignoring --cpuprofile '{}' and --memprofile '{}'",
+                config.cpu_profile,
+                config.mem_profile
+            );
+            return Ok(None);
+        }
+
+        if !config.mem_profile.is_empty() {
+            tracing::warn!(
+                "--memprofile is not yet supported in the Rust volume server; only --cpuprofile '{}' will be written",
+                config.cpu_profile
+            );
+        }
+
+        let guard = pprof::ProfilerGuardBuilder::default()
+            .frequency(GO_CPU_PROFILE_FREQUENCY)
+            .blocklist(&GO_PPROF_BLOCKLIST)
+            .build()
+            .map_err(|e| {
+                format!(
+                    "Failed to start CPU profiler '{}': {}",
+                    config.cpu_profile, e
+                )
+            })?;
+
+        Ok(Some(Self {
+            output_path: PathBuf::from(&config.cpu_profile),
+            guard,
+        }))
+    }
+
+    pub fn finish(self) -> Result<(), String> {
+        let report = self
+            .guard
+            .report()
+            .build()
+            .map_err(|e| format!("Failed to build CPU profile report: {}", e))?;
+        let profile = report
+            .pprof()
+            .map_err(|e| format!("Failed to encode CPU profile report: {}", e))?;
+
+        let mut bytes = Vec::new();
+        profile
+            .encode(&mut bytes)
+            .map_err(|e| format!("Failed to serialize CPU profile report: {}", e))?;
+
+        let mut file = File::create(&self.output_path).map_err(|e| {
+            format!(
+                "Failed to create CPU profile '{}': {}",
+                self.output_path.display(),
+                e
+            )
+        })?;
+        file.write_all(&bytes).map_err(|e| {
+            format!(
+                "Failed to write CPU profile '{}': {}",
+                self.output_path.display(),
+                e
+            )
+        })?;
+        file.flush().map_err(|e| {
+            format!(
+                "Failed to flush CPU profile '{}': {}",
+                self.output_path.display(),
+                e
+            )
+        })?;
+
+        Ok(())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::CpuProfileSession;
+    use crate::config::{NeedleMapKind, ReadMode, VolumeServerConfig};
+    use crate::security::tls::TlsPolicy;
+
+    fn sample_config() -> VolumeServerConfig {
+        VolumeServerConfig {
+            port: 8080,
+            grpc_port: 18080,
+            public_port: 8080,
+            ip: "127.0.0.1".to_string(),
+            bind_ip: "127.0.0.1".to_string(),
+            public_url: "127.0.0.1:8080".to_string(),
+            id: "127.0.0.1:8080".to_string(),
+            masters: vec![],
+            pre_stop_seconds: 0,
+            idle_timeout: 0,
+            data_center: String::new(),
+            rack: String::new(),
+            index_type: NeedleMapKind::InMemory,
+            disk_type: String::new(),
+            folders: vec!["/tmp".to_string()],
+            folder_max_limits: vec![8],
+            folder_tags: vec![vec![]],
+            min_free_spaces: vec![],
+            disk_types: vec![String::new()],
+            idx_folder: String::new(),
+            white_list: vec![],
+            fix_jpg_orientation: false,
+            read_mode: ReadMode::Local,
+            allow_untrusted_remote_endpoints: false,
+            cpu_profile: String::new(),
+            mem_profile: String::new(),
+            compaction_byte_per_second: 0,
+            maintenance_byte_per_second: 0,
+            file_size_limit_bytes: 0,
+            concurrent_upload_limit: 0,
+            concurrent_download_limit: 0,
+            inflight_upload_data_timeout: std::time::Duration::from_secs(0),
+            inflight_download_data_timeout: std::time::Duration::from_secs(0),
+            has_slow_read: false,
+            read_buffer_size_mb: 4,
+            ldb_timeout: 0,
+            pprof: false,
+            metrics_port: 0,
+            metrics_ip: String::new(),
+            debug: false,
+            debug_port: 0,
+            ui_enabled: false,
+            jwt_signing_key: vec![],
+            jwt_signing_expires_seconds: 0,
+            jwt_read_signing_key: vec![],
+            jwt_read_signing_expires_seconds: 0,
+            https_cert_file: String::new(),
+            https_key_file: String::new(),
+            https_ca_file: String::new(),
+            https_client_enabled: false,
+            https_client_cert_file: String::new(),
+            https_client_key_file: String::new(),
+            https_client_ca_file: String::new(),
+            grpc_cert_file: String::new(),
+            grpc_key_file: String::new(),
+            grpc_client_cert_file: String::new(),
+            grpc_client_key_file: String::new(),
+            grpc_ca_file: String::new(),
+            grpc_allowed_wildcard_domain: String::new(),
+            grpc_volume_allowed_common_names: vec![],
+            tls_policy: TlsPolicy::default(),
+            enable_write_queue: false,
+            security_file: String::new(),
+        }
+    }
+
+    #[test]
+    fn test_cpu_profile_session_skips_when_disabled() {
+        let config = sample_config();
+        assert!(CpuProfileSession::start(&config).unwrap().is_none());
+    }
+
+    #[test]
+    fn test_cpu_profile_session_skips_when_pprof_enabled() {
+        let mut config = sample_config();
+        config.cpu_profile = "/tmp/cpu.pb".to_string();
+        config.pprof = true;
+        assert!(CpuProfileSession::start(&config).unwrap().is_none());
+    }
+}

seaweed-volume/src/server/request_id.rs

@@ -0,0 +1,137 @@
+use std::future::Future;
+use std::pin::Pin;
+use std::task::{Context, Poll};
+
+use hyper::http::{self, HeaderValue};
+use tonic::metadata::MetadataValue;
+use tonic::{Request, Status};
+use tower::{Layer, Service};
+
+tokio::task_local! {
+    static CURRENT_REQUEST_ID: String;
+}
+
+#[derive(Clone, Debug, Default)]
+pub struct GrpcRequestIdLayer;
+
+#[derive(Clone, Debug)]
+pub struct GrpcRequestIdService<S> {
+    inner: S,
+}
+
+impl<S> Layer<S> for GrpcRequestIdLayer {
+    type Service = GrpcRequestIdService<S>;
+
+    fn layer(&self, inner: S) -> Self::Service {
+        GrpcRequestIdService { inner }
+    }
+}
+
+impl<S, B> Service<http::Request<B>> for GrpcRequestIdService<S>
+where
+    S: Service<http::Request<B>, Response = http::Response<tonic::body::Body>> + Send + 'static,
+    S::Future: Send + 'static,
+    B: Send + 'static,
+{
+    type Response = http::Response<tonic::body::Body>;
+    type Error = S::Error;
+    type Future = Pin<Box<dyn Future<Output = Result<Self::Response, Self::Error>> + Send>>;
+
+    fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
+        self.inner.poll_ready(cx)
+    }
+
+    fn call(&mut self, mut request: http::Request<B>) -> Self::Future {
+        let request_id = match request.headers().get("x-amz-request-id") {
+            Some(value) => match value.to_str() {
+                Ok(value) if !value.is_empty() => value.to_owned(),
+                _ => generate_grpc_request_id(),
+            },
+            None => generate_grpc_request_id(),
+        };
+
+        if let Ok(value) = HeaderValue::from_str(&request_id) {
+            request.headers_mut().insert("x-amz-request-id", value);
+        }
+
+        let future = self.inner.call(request);
+
+        Box::pin(async move {
+            let mut response: http::Response<tonic::body::Body> =
+                scope_request_id(request_id.clone(), future).await?;
+            if let Ok(value) = HeaderValue::from_str(&request_id) {
+                response.headers_mut().insert("x-amz-request-id", value);
+            }
+            Ok(response)
+        })
+    }
+}
+
+pub async fn scope_request_id<F, T>(request_id: String, future: F) -> T
+where
+    F: Future<Output = T>,
+{
+    CURRENT_REQUEST_ID.scope(request_id, future).await
+}
+
+pub fn current_request_id() -> Option<String> {
+    CURRENT_REQUEST_ID.try_with(Clone::clone).ok()
+}
+
+pub fn outgoing_request_id_interceptor(mut request: Request<()>) -> Result<Request<()>, Status> {
+    if let Some(request_id) = current_request_id() {
+        let value = MetadataValue::try_from(request_id.as_str())
+            .map_err(|_| Status::internal("invalid scoped request id"))?;
+        request.metadata_mut().insert("x-amz-request-id", value);
+    }
+    Ok(request)
+}
+
+pub fn generate_http_request_id() -> String {
+    use rand::RngExt;
+
+    let nanos = std::time::SystemTime::now()
+        .duration_since(std::time::UNIX_EPOCH)
+        .unwrap_or_default()
+        .as_nanos() as u64;
+    let rand_val: u32 = rand::rng().random();
+    format!("{:X}{:08X}", nanos, rand_val)
+}
+
+fn generate_grpc_request_id() -> String {
+    uuid::Uuid::new_v4().to_string()
+}
+
+#[cfg(test)]
+mod tests {
+    use super::{current_request_id, outgoing_request_id_interceptor, scope_request_id};
+    use tonic::Request;
+
+    #[tokio::test]
+    async fn test_scope_request_id_exposes_current_value() {
+        let request_id = "req-123".to_string();
+        let current = scope_request_id(
+            request_id.clone(),
+            async move { current_request_id().unwrap() },
+        )
+        .await;
+        assert_eq!(current, request_id);
+    }
+
+    #[tokio::test]
+    async fn test_outgoing_request_id_interceptor_propagates_scope() {
+        let request = scope_request_id("req-456".to_string(), async move {
+            outgoing_request_id_interceptor(Request::new(())).unwrap()
+        })
+        .await;
+        assert_eq!(
+            request
+                .metadata()
+                .get("x-amz-request-id")
+                .unwrap()
+                .to_str()
+                .unwrap(),
+            "req-456"
+        );
+    }
+}

seaweed-volume/src/server/server_stats.rs

@@ -0,0 +1,248 @@
+use chrono::{Datelike, Local, Timelike};
+use serde::Serialize;
+use std::sync::{LazyLock, Mutex};
+use std::time::Instant;
+
+static START_TIME: LazyLock<Instant> = LazyLock::new(Instant::now);
+static SERVER_STATS: LazyLock<ServerStats> = LazyLock::new(ServerStats::default);
+
+#[derive(Default)]
+pub struct ServerStats {
+    inner: Mutex<ServerStatsInner>,
+}
+
+#[derive(Default)]
+struct ServerStatsInner {
+    requests: DurationCounter,
+    connections: DurationCounter,
+    assign_requests: DurationCounter,
+    read_requests: DurationCounter,
+    write_requests: DurationCounter,
+    delete_requests: DurationCounter,
+    bytes_in: DurationCounter,
+    bytes_out: DurationCounter,
+}
+
+#[derive(Clone, Serialize)]
+#[serde(rename_all = "PascalCase")]
+pub struct ServerStatsSnapshot {
+    pub requests: DurationCounterSnapshot,
+    pub connections: DurationCounterSnapshot,
+    pub assign_requests: DurationCounterSnapshot,
+    pub read_requests: DurationCounterSnapshot,
+    pub write_requests: DurationCounterSnapshot,
+    pub delete_requests: DurationCounterSnapshot,
+    pub bytes_in: DurationCounterSnapshot,
+    pub bytes_out: DurationCounterSnapshot,
+}
+
+#[derive(Clone, Serialize)]
+#[serde(rename_all = "PascalCase")]
+pub struct DurationCounterSnapshot {
+    pub minute_counter: RoundRobinCounterSnapshot,
+    pub hour_counter: RoundRobinCounterSnapshot,
+    pub day_counter: RoundRobinCounterSnapshot,
+    pub week_counter: RoundRobinCounterSnapshot,
+}
+
+#[derive(Clone, Serialize)]
+#[serde(rename_all = "PascalCase")]
+pub struct RoundRobinCounterSnapshot {
+    pub last_index: i32,
+    pub values: Vec<i64>,
+    pub counts: Vec<i64>,
+}
+
+#[derive(Clone)]
+struct DurationCounter {
+    minute_counter: RoundRobinCounter,
+    hour_counter: RoundRobinCounter,
+    day_counter: RoundRobinCounter,
+    week_counter: RoundRobinCounter,
+}
+
+#[derive(Clone)]
+struct RoundRobinCounter {
+    last_index: i32,
+    values: Vec<i64>,
+    counts: Vec<i64>,
+}
+
+impl Default for DurationCounter {
+    fn default() -> Self {
+        Self {
+            minute_counter: RoundRobinCounter::new(60),
+            hour_counter: RoundRobinCounter::new(60),
+            day_counter: RoundRobinCounter::new(24),
+            week_counter: RoundRobinCounter::new(7),
+        }
+    }
+}
+
+impl RoundRobinCounter {
+    fn new(slots: usize) -> Self {
+        Self {
+            last_index: -1,
+            values: vec![0; slots],
+            counts: vec![0; slots],
+        }
+    }
+
+    fn add(&mut self, index: usize, val: i64) {
+        if index >= self.values.len() {
+            return;
+        }
+        while self.last_index != index as i32 {
+            self.last_index = (self.last_index + 1).rem_euclid(self.values.len() as i32);
+            self.values[self.last_index as usize] = 0;
+            self.counts[self.last_index as usize] = 0;
+        }
+        self.values[index] += val;
+        self.counts[index] += 1;
+    }
+
+    fn snapshot(&self) -> RoundRobinCounterSnapshot {
+        RoundRobinCounterSnapshot {
+            last_index: self.last_index,
+            values: self.values.clone(),
+            counts: self.counts.clone(),
+        }
+    }
+}
+
+impl DurationCounter {
+    fn add_now(&mut self, val: i64) {
+        let now = Local::now();
+        self.minute_counter.add(now.second() as usize, val);
+        self.hour_counter.add(now.minute() as usize, val);
+        self.day_counter.add(now.hour() as usize, val);
+        self.week_counter
+            .add(now.weekday().num_days_from_sunday() as usize, val);
+    }
+
+    fn snapshot(&self) -> DurationCounterSnapshot {
+        DurationCounterSnapshot {
+            minute_counter: self.minute_counter.snapshot(),
+            hour_counter: self.hour_counter.snapshot(),
+            day_counter: self.day_counter.snapshot(),
+            week_counter: self.week_counter.snapshot(),
+        }
+    }
+}
+
+impl ServerStatsInner {
+    fn snapshot(&self) -> ServerStatsSnapshot {
+        ServerStatsSnapshot {
+            requests: self.requests.snapshot(),
+            connections: self.connections.snapshot(),
+            assign_requests: self.assign_requests.snapshot(),
+            read_requests: self.read_requests.snapshot(),
+            write_requests: self.write_requests.snapshot(),
+            delete_requests: self.delete_requests.snapshot(),
+            bytes_in: self.bytes_in.snapshot(),
+            bytes_out: self.bytes_out.snapshot(),
+        }
+    }
+}
+
+impl ServerStats {
+    fn update<F>(&self, update: F)
+    where
+        F: FnOnce(&mut ServerStatsInner),
+    {
+        let mut inner = self.inner.lock().unwrap();
+        update(&mut inner);
+    }
+
+    fn snapshot(&self) -> ServerStatsSnapshot {
+        self.inner.lock().unwrap().snapshot()
+    }
+}
+
+impl RoundRobinCounterSnapshot {
+    pub fn to_list(&self) -> Vec<i64> {
+        if self.values.is_empty() {
+            return Vec::new();
+        }
+        let mut ret = Vec::with_capacity(self.values.len());
+        let mut index = self.last_index;
+        let mut step = self.values.len();
+        while step > 0 {
+            step -= 1;
+            index += 1;
+            if index >= self.values.len() as i32 {
+                index = 0;
+            }
+            ret.push(self.values[index as usize]);
+        }
+        ret
+    }
+}
+
+pub fn init_process_start() {
+    LazyLock::force(&START_TIME);
+    LazyLock::force(&SERVER_STATS);
+}
+
+pub fn uptime_string() -> String {
+    let secs = START_TIME.elapsed().as_secs();
+    let hours = secs / 3600;
+    let minutes = (secs % 3600) / 60;
+    let seconds = secs % 60;
+    let mut out = String::new();
+    if hours > 0 {
+        out.push_str(&format!("{}h", hours));
+    }
+    if hours > 0 || minutes > 0 {
+        out.push_str(&format!("{}m", minutes));
+    }
+    out.push_str(&format!("{}s", seconds));
+    out
+}
+
+pub fn snapshot() -> ServerStatsSnapshot {
+    SERVER_STATS.snapshot()
+}
+
+pub fn record_request_open() {
+    SERVER_STATS.update(|inner| inner.requests.add_now(1));
+}
+
+pub fn record_request_close() {
+    SERVER_STATS.update(|inner| inner.requests.add_now(-1));
+}
+
+pub fn record_connection_open() {
+    SERVER_STATS.update(|inner| inner.connections.add_now(1));
+}
+
+pub fn record_connection_close() {
+    SERVER_STATS.update(|inner| inner.connections.add_now(-1));
+}
+
+pub fn record_read_request() {
+    SERVER_STATS.update(|inner| inner.read_requests.add_now(1));
+}
+
+pub fn record_write_request() {
+    SERVER_STATS.update(|inner| inner.write_requests.add_now(1));
+}
+
+pub fn record_delete_request() {
+    SERVER_STATS.update(|inner| inner.delete_requests.add_now(1));
+}
+
+pub fn record_bytes_in(bytes: i64) {
+    SERVER_STATS.update(|inner| inner.bytes_in.add_now(bytes));
+}
+
+pub fn record_bytes_out(bytes: i64) {
+    SERVER_STATS.update(|inner| inner.bytes_out.add_now(bytes));
+}
+
+#[cfg(test)]
+pub fn reset_for_tests() {
+    LazyLock::force(&START_TIME);
+    let mut inner = SERVER_STATS.inner.lock().unwrap();
+    *inner = ServerStatsInner::default();
+}

seaweed-volume/src/server/ui.rs

@@ -0,0 +1,506 @@
+use std::fmt::Write as _;
+
+use crate::server::server_stats;
+use crate::server::volume_server::VolumeServerState;
+use crate::storage::store::Store;
+
+pub struct EmbeddedAsset {
+    pub content_type: &'static str,
+    pub bytes: &'static [u8],
+}
+
+struct UiDiskRow {
+    dir: String,
+    disk_type: String,
+    all: u64,
+    free: u64,
+    used: u64,
+}
+
+struct UiVolumeRow {
+    id: u32,
+    collection: String,
+    disk_type: String,
+    size: u64,
+    file_count: i64,
+    delete_count: i64,
+    deleted_byte_count: u64,
+    ttl: String,
+    read_only: bool,
+    version: u32,
+    remote_storage_name: String,
+    remote_storage_key: String,
+}
+
+struct UiEcShardRow {
+    shard_id: u8,
+    size: u64,
+}
+
+struct UiEcVolumeRow {
+    volume_id: u32,
+    collection: String,
+    size: u64,
+    shards: Vec<UiEcShardRow>,
+    created_at: String,
+}
+
+pub fn favicon_asset() -> EmbeddedAsset {
+    EmbeddedAsset {
+        content_type: "image/x-icon",
+        bytes: include_bytes!(concat!(
+            env!("CARGO_MANIFEST_DIR"),
+            "/../weed/static/favicon.ico"
+        )),
+    }
+}
+
+pub fn lookup_static_asset(path: &str) -> Option<EmbeddedAsset> {
+    let path = path.trim_start_matches('/');
+    let asset = match path {
+        "bootstrap/3.3.1/css/bootstrap.min.css" => EmbeddedAsset {
+            content_type: "text/css; charset=utf-8",
+            bytes: include_bytes!(concat!(
+                env!("CARGO_MANIFEST_DIR"),
+                "/../weed/static/bootstrap/3.3.1/css/bootstrap.min.css"
+            )),
+        },
+        "bootstrap/3.3.1/fonts/glyphicons-halflings-regular.eot" => EmbeddedAsset {
+            content_type: "application/vnd.ms-fontobject",
+            bytes: include_bytes!(concat!(
+                env!("CARGO_MANIFEST_DIR"),
+                "/../weed/static/bootstrap/3.3.1/fonts/glyphicons-halflings-regular.eot"
+            )),
+        },
+        "bootstrap/3.3.1/fonts/glyphicons-halflings-regular.svg" => EmbeddedAsset {
+            content_type: "image/svg+xml",
+            bytes: include_bytes!(concat!(
+                env!("CARGO_MANIFEST_DIR"),
+                "/../weed/static/bootstrap/3.3.1/fonts/glyphicons-halflings-regular.svg"
+            )),
+        },
+        "bootstrap/3.3.1/fonts/glyphicons-halflings-regular.ttf" => EmbeddedAsset {
+            content_type: "font/ttf",
+            bytes: include_bytes!(concat!(
+                env!("CARGO_MANIFEST_DIR"),
+                "/../weed/static/bootstrap/3.3.1/fonts/glyphicons-halflings-regular.ttf"
+            )),
+        },
+        "bootstrap/3.3.1/fonts/glyphicons-halflings-regular.woff" => EmbeddedAsset {
+            content_type: "font/woff",
+            bytes: include_bytes!(concat!(
+                env!("CARGO_MANIFEST_DIR"),
+                "/../weed/static/bootstrap/3.3.1/fonts/glyphicons-halflings-regular.woff"
+            )),
+        },
+        "images/folder.gif" => EmbeddedAsset {
+            content_type: "image/gif",
+            bytes: include_bytes!(concat!(
+                env!("CARGO_MANIFEST_DIR"),
+                "/../weed/static/images/folder.gif"
+            )),
+        },
+        "javascript/jquery-3.6.0.min.js" => EmbeddedAsset {
+            content_type: "application/javascript; charset=utf-8",
+            bytes: include_bytes!(concat!(
+                env!("CARGO_MANIFEST_DIR"),
+                "/../weed/static/javascript/jquery-3.6.0.min.js"
+            )),
+        },
+        "javascript/jquery-sparklines/2.1.2/jquery.sparkline.min.js" => EmbeddedAsset {
+            content_type: "application/javascript; charset=utf-8",
+            bytes: include_bytes!(concat!(
+                env!("CARGO_MANIFEST_DIR"),
+                "/../weed/static/javascript/jquery-sparklines/2.1.2/jquery.sparkline.min.js"
+            )),
+        },
+        "seaweed50x50.png" => EmbeddedAsset {
+            content_type: "image/png",
+            bytes: include_bytes!(concat!(
+                env!("CARGO_MANIFEST_DIR"),
+                "/../weed/static/seaweed50x50.png"
+            )),
+        },
+        _ => return None,
+    };
+    Some(asset)
+}
+
+pub fn render_volume_server_html(state: &VolumeServerState) -> String {
+    let counters = server_stats::snapshot();
+    let (disk_rows, volume_rows, remote_volume_rows, ec_volume_rows) = {
+        let store = state.store.read().unwrap();
+        collect_ui_data(&store)
+    };
+
+    let masters = if state.master_urls.is_empty() {
+        "[]".to_string()
+    } else {
+        format!("[{}]", state.master_urls.join(" "))
+    };
+    let uptime = server_stats::uptime_string();
+    let read_week = join_i64(&counters.read_requests.week_counter.to_list());
+    let read_day = join_i64(&counters.read_requests.day_counter.to_list());
+    let read_hour = join_i64(&counters.read_requests.hour_counter.to_list());
+    let read_minute = join_i64(&counters.read_requests.minute_counter.to_list());
+
+    let mut disk_rows_html = String::new();
+    for disk in &disk_rows {
+        let _ = write!(
+            disk_rows_html,
+            "<tr><td>{}</td><td>{}</td><td>{}</td><td>{}</td><td>{:.2}%</td></tr>",
+            escape_html(&disk.dir),
+            escape_html(&disk.disk_type),
+            bytes_to_human_readable(disk.all),
+            bytes_to_human_readable(disk.free),
+            percent_from(disk.all, disk.used),
+        );
+    }
+
+    let mut volume_rows_html = String::new();
+    for volume in &volume_rows {
+        let _ = write!(
+            volume_rows_html,
+            "<tr><td><code>{}</code></td><td>{}</td><td>{}</td><td>{}</td><td>{}</td><td>{} / {}</td><td>{}</td><td>{}</td><td>{}</td></tr>",
+            volume.id,
+            escape_html(&volume.collection),
+            escape_html(&volume.disk_type),
+            bytes_to_human_readable(volume.size),
+            volume.file_count,
+            volume.delete_count,
+            bytes_to_human_readable(volume.deleted_byte_count),
+            escape_html(&volume.ttl),
+            volume.read_only,
+            volume.version,
+        );
+    }
+
+    let remote_section = if remote_volume_rows.is_empty() {
+        String::new()
+    } else {
+        let mut remote_rows_html = String::new();
+        for volume in &remote_volume_rows {
+            let _ = write!(
+                remote_rows_html,
+                "<tr><td><code>{}</code></td><td>{}</td><td>{}</td><td>{}</td><td>{} / {}</td><td>{}</td><td>{}</td></tr>",
+                volume.id,
+                escape_html(&volume.collection),
+                bytes_to_human_readable(volume.size),
+                volume.file_count,
+                volume.delete_count,
+                bytes_to_human_readable(volume.deleted_byte_count),
+                escape_html(&volume.remote_storage_name),
+                escape_html(&volume.remote_storage_key),
+            );
+        }
+        format!(
+            r#"<div class="row">
+            <h2>Remote Volumes</h2>
+            <table class="table table-striped">
+                <thead>
+                    <tr>
+                        <th>Id</th>
+                        <th>Collection</th>
+                        <th>Size</th>
+                        <th>Files</th>
+                        <th>Trash</th>
+                        <th>Remote</th>
+                        <th>Key</th>
+                    </tr>
+                </thead>
+                <tbody>{}</tbody>
+            </table>
+        </div>"#,
+            remote_rows_html
+        )
+    };
+
+    let ec_section = if ec_volume_rows.is_empty() {
+        String::new()
+    } else {
+        let mut ec_rows_html = String::new();
+        for ec in &ec_volume_rows {
+            let mut shard_labels = String::new();
+            for shard in &ec.shards {
+                let _ = write!(
+                    shard_labels,
+                    "<span class=\"label label-info\" style=\"margin-right: 5px;\">{}: {}</span>",
+                    shard.shard_id,
+                    bytes_to_human_readable(shard.size)
+                );
+            }
+            let _ = write!(
+                ec_rows_html,
+                "<tr><td><code>{}</code></td><td>{}</td><td>{}</td><td>{}</td><td>{}</td></tr>",
+                ec.volume_id,
+                escape_html(&ec.collection),
+                bytes_to_human_readable(ec.size),
+                shard_labels,
+                escape_html(&ec.created_at),
+            );
+        }
+        format!(
+            r#"<div class="row">
+            <h2>Erasure Coding Shards</h2>
+            <table class="table table-striped">
+                <thead>
+                    <tr>
+                        <th>Id</th>
+                        <th>Collection</th>
+                        <th>Total Size</th>
+                        <th>Shard Details</th>
+                        <th>CreatedAt</th>
+                    </tr>
+                </thead>
+                <tbody>{}</tbody>
+            </table>
+        </div>"#,
+            ec_rows_html
+        )
+    };
+
+    format!(
+        r#"<!DOCTYPE html>
+<html>
+<head>
+    <title>SeaweedFS {version}</title>
+    <link rel="stylesheet" href="/seaweedfsstatic/bootstrap/3.3.1/css/bootstrap.min.css">
+    <script type="text/javascript" src="/seaweedfsstatic/javascript/jquery-3.6.0.min.js"></script>
+    <script type="text/javascript" src="/seaweedfsstatic/javascript/jquery-sparklines/2.1.2/jquery.sparkline.min.js"></script>
+    <script type="text/javascript">
+        $(function () {{
+            var periods = ['second', 'minute', 'hour', 'day'];
+            for (var i = 0; i < periods.length; i++) {{
+                var period = periods[i];
+                $('.inlinesparkline-' + period).sparkline('html', {{
+                    type: 'line',
+                    barColor: 'red',
+                    tooltipSuffix: ' request per ' + period
+                }});
+            }}
+        }});
+    </script>
+    <style>
+        #jqstooltip {{
+            height: 28px !important;
+            width: 150px !important;
+        }}
+    </style>
+</head>
+<body>
+    <div class="container">
+        <div class="page-header">
+            <h1>
+                <a href="https://github.com/seaweedfs/seaweedfs"><img src="/seaweedfsstatic/seaweed50x50.png" alt="SeaweedFS"></a>
+                SeaweedFS <small>{version}</small>
+            </h1>
+        </div>
+
+        <div class="row">
+            <div class="col-sm-6">
+                <h2>Disk Stats</h2>
+                <table class="table table-striped">
+                    <thead>
+                        <tr>
+                            <th>Path</th>
+                            <th>Disk</th>
+                            <th>Total</th>
+                            <th>Free</th>
+                            <th>Usage</th>
+                        </tr>
+                    </thead>
+                    <tbody>{disk_rows_html}</tbody>
+                </table>
+            </div>
+
+            <div class="col-sm-6">
+                <h2>System Stats</h2>
+                <table class="table table-condensed table-striped">
+                    <tr><th>Masters</th><td>{masters}</td></tr>
+                    <tr><th>Weekly # ReadRequests</th><td><span class="inlinesparkline-day">{read_week}</span></td></tr>
+                    <tr><th>Daily # ReadRequests</th><td><span class="inlinesparkline-hour">{read_day}</span></td></tr>
+                    <tr><th>Hourly # ReadRequests</th><td><span class="inlinesparkline-minute">{read_hour}</span></td></tr>
+                    <tr><th>Last Minute # ReadRequests</th><td><span class="inlinesparkline-second">{read_minute}</span></td></tr>
+                    <tr><th>Up Time</th><td>{uptime}</td></tr>
+                </table>
+            </div>
+        </div>
+
+        <div class="row">
+            <h2>Volumes</h2>
+            <table class="table table-striped">
+                <thead>
+                    <tr>
+                        <th>Id</th>
+                        <th>Collection</th>
+                        <th>Disk</th>
+                        <th>Data Size</th>
+                        <th>Files</th>
+                        <th>Trash</th>
+                        <th>TTL</th>
+                        <th>ReadOnly</th>
+                        <th>Version</th>
+                    </tr>
+                </thead>
+                <tbody>{volume_rows_html}</tbody>
+            </table>
+        </div>
+
+        {remote_section}
+        {ec_section}
+    </div>
+</body>
+</html>"#,
+        version = escape_html(crate::version::version()),
+        disk_rows_html = disk_rows_html,
+        masters = escape_html(&masters),
+        read_week = read_week,
+        read_day = read_day,
+        read_hour = read_hour,
+        read_minute = read_minute,
+        uptime = escape_html(&uptime),
+        volume_rows_html = volume_rows_html,
+        remote_section = remote_section,
+        ec_section = ec_section,
+    )
+}
+
+fn collect_ui_data(
+    store: &Store,
+) -> (
+    Vec<UiDiskRow>,
+    Vec<UiVolumeRow>,
+    Vec<UiVolumeRow>,
+    Vec<UiEcVolumeRow>,
+) {
+    let mut disk_rows = Vec::new();
+    let mut volumes = Vec::new();
+    let mut remote_volumes = Vec::new();
+    let mut ec_volumes = Vec::new();
+
+    for loc in &store.locations {
+        let dir = absolute_display_path(&loc.directory);
+        let (all, free) = crate::storage::disk_location::get_disk_stats(&dir);
+        disk_rows.push(UiDiskRow {
+            dir,
+            disk_type: loc.disk_type.to_string(),
+            all,
+            free,
+            used: all.saturating_sub(free),
+        });
+
+        for (_, volume) in loc.volumes() {
+            let (remote_storage_name, remote_storage_key) = volume.remote_storage_name_key();
+            let row = UiVolumeRow {
+                id: volume.id.0,
+                collection: volume.collection.clone(),
+                disk_type: loc.disk_type.to_string(),
+                size: volume.content_size(),
+                file_count: volume.file_count(),
+                delete_count: volume.deleted_count(),
+                deleted_byte_count: volume.deleted_size(),
+                ttl: volume.super_block.ttl.to_string(),
+                read_only: volume.is_read_only(),
+                version: volume.version().0 as u32,
+                remote_storage_name,
+                remote_storage_key,
+            };
+            if row.remote_storage_name.is_empty() {
+                volumes.push(row);
+            } else {
+                remote_volumes.push(row);
+            }
+        }
+
+        for (_, ec_volume) in loc.ec_volumes() {
+            let mut shards = Vec::new();
+            let mut total_size = 0u64;
+            let mut created_at = String::from("-");
+            for shard in ec_volume.shards.iter().flatten() {
+                let shard_size = shard.file_size().max(0) as u64;
+                total_size = total_size.saturating_add(shard_size);
+                shards.push(UiEcShardRow {
+                    shard_id: shard.shard_id,
+                    size: shard_size,
+                });
+                if created_at == "-"
+                    && let Ok(metadata) = std::fs::metadata(shard.file_name())
+                    && let Ok(modified) = metadata.modified()
+                {
+                    let ts: chrono::DateTime<chrono::Local> = modified.into();
+                    created_at = ts.format("%Y-%m-%d %H:%M").to_string();
+                }
+            }
+            let preferred_size = ec_volume.dat_file_size.max(0) as u64;
+            ec_volumes.push(UiEcVolumeRow {
+                volume_id: ec_volume.volume_id.0,
+                collection: ec_volume.collection.clone(),
+                size: preferred_size.max(total_size),
+                shards,
+                created_at,
+            });
+        }
+    }
+
+    disk_rows.sort_by(|left, right| left.dir.cmp(&right.dir));
+    volumes.sort_by_key(|row| row.id);
+    remote_volumes.sort_by_key(|row| row.id);
+    ec_volumes.sort_by_key(|row| row.volume_id);
+
+    (disk_rows, volumes, remote_volumes, ec_volumes)
+}
+
+fn absolute_display_path(path: &str) -> String {
+    let p = std::path::Path::new(path);
+    if p.is_absolute() {
+        return path.to_string();
+    }
+    std::env::current_dir()
+        .map(|cwd| cwd.join(p).to_string_lossy().to_string())
+        .unwrap_or_else(|_| path.to_string())
+}
+
+fn join_i64(values: &[i64]) -> String {
+    values
+        .iter()
+        .map(std::string::ToString::to_string)
+        .collect::<Vec<_>>()
+        .join(",")
+}
+
+fn percent_from(total: u64, part: u64) -> f64 {
+    if total == 0 {
+        return 0.0;
+    }
+    (part as f64 / total as f64) * 100.0
+}
+
+fn bytes_to_human_readable(bytes: u64) -> String {
+    const UNIT: u64 = 1024;
+    if bytes < UNIT {
+        return format!("{} B", bytes);
+    }
+
+    let mut div = UNIT;
+    let mut exp = 0usize;
+    let mut n = bytes / UNIT;
+    while n >= UNIT {
+        div *= UNIT;
+        n /= UNIT;
+        exp += 1;
+    }
+
+    format!(
+        "{:.2} {}iB",
+        bytes as f64 / div as f64,
+        ["K", "M", "G", "T", "P", "E"][exp]
+    )
+}
+
+fn escape_html(input: &str) -> String {
+    input
+        .replace('&', "&amp;")
+        .replace('<', "&lt;")
+        .replace('>', "&gt;")
+        .replace('"', "&quot;")
+}

seaweed-volume/src/server/volume_server.rs

@@ -0,0 +1,542 @@
+//! VolumeServer: the main HTTP server for volume operations.
+//!
+//! Routes:
+//!   GET/HEAD /{vid},{fid}     — read a file
+//!   POST/PUT /{vid},{fid}     — write a file
+//!   DELETE   /{vid},{fid}     — delete a file
+//!   GET      /status          — server status
+//!   GET      /healthz         — health check
+//!
+//! Matches Go's server/volume_server.go.
+
+use std::net::SocketAddr;
+use std::sync::atomic::{AtomicBool, AtomicI64, AtomicU32, Ordering};
+use std::sync::{Arc, RwLock};
+
+use axum::{
+    Router,
+    extract::{Request, State, connect_info::ConnectInfo},
+    http::{HeaderValue, Method, StatusCode, header},
+    middleware::{self, Next},
+    response::{IntoResponse, Response},
+    routing::{any, get},
+};
+
+use crate::config::ReadMode;
+use crate::security::Guard;
+use crate::storage::store::Store;
+
+use super::grpc_client::OutgoingGrpcTlsConfig;
+use super::handlers;
+use super::write_queue::WriteQueue;
+
+#[derive(Clone, Debug, Default)]
+pub struct RuntimeMetricsConfig {
+    pub push_gateway: crate::metrics::PushGatewayConfig,
+}
+
+/// Shared state for the volume server.
+pub struct VolumeServerState {
+    pub store: RwLock<Store>,
+    pub guard: RwLock<Guard>,
+    pub is_stopping: RwLock<bool>,
+    /// Maintenance mode flag.
+    pub maintenance: AtomicBool,
+    /// State version — incremented on each SetState call.
+    pub state_version: AtomicU32,
+    /// Throttling: concurrent upload/download limits (in bytes, 0 = disabled).
+    pub concurrent_upload_limit: i64,
+    pub concurrent_download_limit: i64,
+    pub inflight_upload_data_timeout: std::time::Duration,
+    pub inflight_download_data_timeout: std::time::Duration,
+    /// Current in-flight upload/download bytes.
+    pub inflight_upload_bytes: AtomicI64,
+    pub inflight_download_bytes: AtomicI64,
+    /// Notify waiters when inflight bytes decrease.
+    pub upload_notify: tokio::sync::Notify,
+    pub download_notify: tokio::sync::Notify,
+    /// Data center name from config.
+    pub data_center: String,
+    /// Rack name from config.
+    pub rack: String,
+    /// File size limit in bytes (0 = no limit).
+    pub file_size_limit_bytes: i64,
+    /// Default IO rate limit for maintenance copy/replication work.
+    pub maintenance_byte_per_second: i64,
+    /// Whether the server is connected to master (heartbeat active).
+    pub is_heartbeating: AtomicBool,
+    /// Whether master addresses are configured.
+    pub has_master: bool,
+    /// Seconds to wait before shutting down servers (graceful drain).
+    pub pre_stop_seconds: u32,
+    /// Notify heartbeat to send an immediate update when volume state changes.
+    pub volume_state_notify: tokio::sync::Notify,
+    /// Optional batched write queue for improved throughput under load.
+    pub write_queue: std::sync::OnceLock<WriteQueue>,
+    /// Registry of S3 tier backends for tiered storage operations.
+    pub s3_tier_registry: std::sync::RwLock<crate::remote_storage::s3_tier::S3TierRegistry>,
+    /// Read mode: local, proxy, or redirect for non-local volumes.
+    pub read_mode: ReadMode,
+    /// If true, FetchAndWriteNeedle skips remote S3 endpoint validation,
+    /// allowing arbitrary (incl. loopback / link-local / metadata) hosts.
+    pub allow_untrusted_remote_endpoints: bool,
+    /// First master address for volume lookups (e.g., "localhost:9333").
+    pub master_url: String,
+    /// Seed master addresses for UI rendering.
+    pub master_urls: Vec<String>,
+    /// Canonical http `host:port` form of every configured seed master.
+    /// Built once at construction so Ping admission stays O(1). Mirrors
+    /// Go's `seedMasterSet` on `VolumeServer`.
+    pub seed_master_set: std::collections::HashSet<String>,
+    /// Current master this server is heartbeating with, in canonical http
+    /// `host:port` form. Empty when no heartbeat connection is active. The
+    /// heartbeat goroutine writes; admission reads — the lock keeps them
+    /// from racing on a leader change. Mirrors Go's `currentMaster` plus
+    /// `currentMasterLock`.
+    pub current_master_url: tokio::sync::RwLock<String>,
+    /// This server's own address (ip:port) for filtering self from lookup results.
+    pub self_url: String,
+    /// HTTP client for proxy requests and master lookups.
+    pub http_client: reqwest::Client,
+    /// Scheme used for outgoing master and peer HTTP requests ("http" or "https").
+    pub outgoing_http_scheme: String,
+    /// Optional client TLS material for outgoing gRPC connections.
+    pub outgoing_grpc_tls: Option<OutgoingGrpcTlsConfig>,
+    /// Metrics push settings learned from master heartbeat responses.
+    pub metrics_runtime: std::sync::RwLock<RuntimeMetricsConfig>,
+    pub metrics_notify: tokio::sync::Notify,
+    /// Whether JPEG uploads should be normalized using EXIF orientation.
+    pub fix_jpg_orientation: bool,
+    /// Read tuning flags for large-file streaming.
+    pub has_slow_read: bool,
+    pub read_buffer_size_bytes: usize,
+    /// Path to security.toml — stored for SIGHUP reload.
+    pub security_file: String,
+    /// Original CLI whitelist entries — stored for SIGHUP reload.
+    pub cli_white_list: Vec<String>,
+    /// Path to state.pb file for persisting VolumeServerState across restarts.
+    pub state_file_path: String,
+}
+
+impl VolumeServerState {
+    /// Check if the server is in maintenance mode; return gRPC error if so.
+    pub fn check_maintenance(&self) -> Result<(), tonic::Status> {
+        if self.maintenance.load(Ordering::Relaxed) {
+            let id = self.store.read().unwrap().id.clone();
+            return Err(tonic::Status::unavailable(format!(
+                "volume server {} is in maintenance mode",
+                id
+            )));
+        }
+        Ok(())
+    }
+
+    /// Build the seed master set from a list of raw `host:port[.grpcPort]`
+    /// addresses, normalised the same way Go's `pb.ServerAddress.ToHttpAddress`
+    /// does (drop the `.grpcPort` suffix, preserve everything else).
+    pub fn build_seed_master_set(master_urls: &[String]) -> std::collections::HashSet<String> {
+        master_urls
+            .iter()
+            .map(|m| to_http_address(m).into_owned())
+            .collect()
+    }
+
+    /// Returns true iff `target` (normalised to canonical http `host:port`)
+    /// is a master this server already knows about. Volume servers do not
+    /// keep a peer-volume or peer-filer list, so Ping is scoped to masters.
+    /// Mirrors Go's `VolumeServer.isKnownPingTarget`.
+    pub async fn is_known_ping_target(&self, target: &str, target_type: &str) -> bool {
+        if target_type != "master" {
+            return false;
+        }
+        let key = to_http_address(target).into_owned();
+        if key.is_empty() {
+            return false;
+        }
+        let current = self.current_master_url.read().await.clone();
+        if !current.is_empty() && current == key {
+            return true;
+        }
+        self.seed_master_set.contains(&key)
+    }
+}
+
+pub fn build_metrics_router() -> Router {
+    Router::new().route("/metrics", get(handlers::metrics_handler))
+}
+
+pub fn normalize_outgoing_http_url(scheme: &str, raw_target: &str) -> Result<String, String> {
+    if raw_target.starts_with("http://") || raw_target.starts_with("https://") {
+        let mut url = reqwest::Url::parse(raw_target)
+            .map_err(|e| format!("invalid url {}: {}", raw_target, e))?;
+        url.set_scheme(scheme)
+            .map_err(|_| format!("invalid scheme {}", scheme))?;
+        return Ok(url.to_string());
+    }
+    Ok(format!("{}://{}", scheme, raw_target))
+}
+
+/// Convert a SeaweedFS server address to its HTTP `host:port` form.
+///
+/// Mirrors Go's `pb.ServerAddress.ToHttpAddress()`: SeaweedFS encodes a
+/// server's gRPC port by appending `.grpcPort` to the HTTP port (e.g.
+/// `host:9333.19333`). For HTTP requests we want only the HTTP `host:port`.
+/// - `host:port.grpcPort` -> `host:port`
+/// - `host:port` -> `host:port` (unchanged)
+/// - Anything that does not look like `host:port[.grpcPort]` is returned unchanged.
+///
+/// Returns a `Cow<str>` so the common (no-suffix) case borrows from `addr`
+/// without allocating; only the rewrite branch produces a new `String`.
+pub fn to_http_address(addr: &str) -> std::borrow::Cow<'_, str> {
+    let Some(ports_sep_index) = addr.rfind(':') else {
+        return std::borrow::Cow::Borrowed(addr);
+    };
+    let ports = &addr[ports_sep_index + 1..];
+    if let Some(dot_idx) = ports.rfind('.') {
+        let http_port = &ports[..dot_idx];
+        let grpc_port = &ports[dot_idx + 1..];
+        // Only strip the suffix when both parts parse as real ports — leave
+        // anything else (e.g. "host:abc.def") untouched so bad config surfaces
+        // rather than being silently rewritten. Mirrors the validation already
+        // done in `to_grpc_address` for the inverse direction.
+        if let (Ok(_), Ok(_)) = (http_port.parse::<u16>(), grpc_port.parse::<u16>()) {
+            return std::borrow::Cow::Owned(addr[..ports_sep_index + 1 + dot_idx].to_string());
+        }
+    }
+    std::borrow::Cow::Borrowed(addr)
+}
+
+fn request_remote_addr(request: &Request) -> Option<SocketAddr> {
+    request
+        .extensions()
+        .get::<ConnectInfo<SocketAddr>>()
+        .map(|info| info.0)
+}
+
+fn request_is_whitelisted(state: &VolumeServerState, request: &Request) -> bool {
+    request_remote_addr(request)
+        .map(|remote_addr| {
+            state
+                .guard
+                .read()
+                .unwrap()
+                .check_whitelist(&remote_addr.to_string())
+        })
+        .unwrap_or(true)
+}
+
+/// Middleware: set Server header, echo x-amz-request-id, set CORS if Origin present.
+async fn common_headers_middleware(request: Request, next: Next) -> Response {
+    let origin = request.headers().get("origin").cloned();
+    let request_id = super::request_id::generate_http_request_id();
+
+    let mut response =
+        super::request_id::scope_request_id(
+            request_id.clone(),
+            async move { next.run(request).await },
+        )
+        .await;
+
+    let headers = response.headers_mut();
+    if let Ok(val) = HeaderValue::from_str(crate::version::server_header()) {
+        headers.insert("Server", val);
+    }
+
+    if let Ok(val) = HeaderValue::from_str(&request_id) {
+        headers.insert("X-Request-Id", val.clone());
+        headers.insert("x-amz-request-id", val);
+    }
+
+    if origin.is_some() {
+        headers.insert("Access-Control-Allow-Origin", HeaderValue::from_static("*"));
+        headers.insert(
+            "Access-Control-Allow-Credentials",
+            HeaderValue::from_static("true"),
+        );
+    }
+
+    response
+}
+
+/// Admin store handler — dispatches based on HTTP method.
+/// Matches Go's privateStoreHandler: GET/HEAD → read, POST/PUT → write,
+/// DELETE → delete, OPTIONS → CORS headers, anything else → 400.
+async fn admin_store_handler(state: State<Arc<VolumeServerState>>, request: Request) -> Response {
+    let start = std::time::Instant::now();
+    let method = request.method().clone();
+    let mut method_str = method.as_str().to_string();
+    let request_bytes = request
+        .headers()
+        .get(header::CONTENT_LENGTH)
+        .and_then(|value| value.to_str().ok())
+        .and_then(|value| value.parse::<i64>().ok())
+        .filter(|value| *value > 0)
+        .unwrap_or(0);
+    super::server_stats::record_request_open();
+    crate::metrics::INFLIGHT_REQUESTS_GAUGE
+        .with_label_values(&[&method_str])
+        .inc();
+    let whitelist_rejected = matches!(method, Method::POST | Method::PUT | Method::DELETE)
+        && !request_is_whitelisted(&state, &request);
+    let response = match method.clone() {
+        _ if whitelist_rejected => StatusCode::UNAUTHORIZED.into_response(),
+        Method::GET | Method::HEAD => {
+            super::server_stats::record_read_request();
+            handlers::get_or_head_handler_from_request(state, request).await
+        }
+        Method::POST | Method::PUT => {
+            super::server_stats::record_write_request();
+            if request_bytes > 0 {
+                super::server_stats::record_bytes_in(request_bytes);
+            }
+            handlers::post_handler(state, request).await
+        }
+        Method::DELETE => {
+            super::server_stats::record_delete_request();
+            handlers::delete_handler(state, request).await
+        }
+        Method::OPTIONS => {
+            super::server_stats::record_read_request();
+            admin_options_response()
+        }
+        _ => {
+            let method_name = request.method().to_string();
+            let query = request.uri().query().map(|q| q.to_string());
+            method_str = "INVALID".to_string();
+            handlers::json_error_with_query(
+                StatusCode::BAD_REQUEST,
+                format!("unsupported method {}", method_name),
+                query.as_deref(),
+            )
+        }
+    };
+    if method == Method::GET
+        && let Some(response_bytes) = response
+            .headers()
+            .get(header::CONTENT_LENGTH)
+            .and_then(|value| value.to_str().ok())
+            .and_then(|value| value.parse::<i64>().ok())
+            .filter(|value| *value > 0)
+    {
+        super::server_stats::record_bytes_out(response_bytes);
+    }
+    super::server_stats::record_request_close();
+    crate::metrics::INFLIGHT_REQUESTS_GAUGE
+        .with_label_values(&[&method_str])
+        .dec();
+    crate::metrics::REQUEST_COUNTER
+        .with_label_values(&[&method_str, response.status().as_str()])
+        .inc();
+    crate::metrics::REQUEST_DURATION
+        .with_label_values(&[&method_str])
+        .observe(start.elapsed().as_secs_f64());
+    response
+}
+
+/// Public store handler — dispatches based on HTTP method.
+/// Matches Go's publicReadOnlyHandler: GET/HEAD → read, OPTIONS → CORS,
+/// anything else → 200 (passthrough no-op).
+async fn public_store_handler(state: State<Arc<VolumeServerState>>, request: Request) -> Response {
+    let start = std::time::Instant::now();
+    let method = request.method().clone();
+    let method_str = method.as_str().to_string();
+    super::server_stats::record_request_open();
+    crate::metrics::INFLIGHT_REQUESTS_GAUGE
+        .with_label_values(&[&method_str])
+        .inc();
+    let response = match method.clone() {
+        Method::GET | Method::HEAD => {
+            super::server_stats::record_read_request();
+            handlers::get_or_head_handler_from_request(state, request).await
+        }
+        Method::OPTIONS => {
+            super::server_stats::record_read_request();
+            public_options_response()
+        }
+        _ => StatusCode::OK.into_response(),
+    };
+    if method == Method::GET
+        && let Some(response_bytes) = response
+            .headers()
+            .get(header::CONTENT_LENGTH)
+            .and_then(|value| value.to_str().ok())
+            .and_then(|value| value.parse::<i64>().ok())
+            .filter(|value| *value > 0)
+    {
+        super::server_stats::record_bytes_out(response_bytes);
+    }
+    super::server_stats::record_request_close();
+    crate::metrics::INFLIGHT_REQUESTS_GAUGE
+        .with_label_values(&[&method_str])
+        .dec();
+    crate::metrics::REQUEST_COUNTER
+        .with_label_values(&[&method_str, response.status().as_str()])
+        .inc();
+    crate::metrics::REQUEST_DURATION
+        .with_label_values(&[&method_str])
+        .observe(start.elapsed().as_secs_f64());
+    response
+}
+
+/// Build OPTIONS response for admin port.
+fn admin_options_response() -> Response {
+    let mut response = StatusCode::OK.into_response();
+    let headers = response.headers_mut();
+    headers.insert(
+        "Access-Control-Allow-Methods",
+        HeaderValue::from_static("PUT, POST, GET, DELETE, OPTIONS"),
+    );
+    headers.insert(
+        "Access-Control-Allow-Headers",
+        HeaderValue::from_static("*"),
+    );
+    response
+}
+
+/// Build OPTIONS response for public port.
+fn public_options_response() -> Response {
+    let mut response = StatusCode::OK.into_response();
+    let headers = response.headers_mut();
+    headers.insert(
+        "Access-Control-Allow-Methods",
+        HeaderValue::from_static("GET, OPTIONS"),
+    );
+    headers.insert(
+        "Access-Control-Allow-Headers",
+        HeaderValue::from_static("*"),
+    );
+    response
+}
+
+/// Build the admin (private) HTTP router — supports all operations.
+/// UI route is only registered when no signing keys are configured,
+/// matching Go's `if signingKey == "" || enableUiAccess` check.
+pub fn build_admin_router(state: Arc<VolumeServerState>) -> Router {
+    let guard = state.guard.read().unwrap();
+    // This helper can only derive the default Go behavior from the guard state:
+    // UI stays enabled when the write signing key is empty. The explicit
+    // `access.ui` override is handled by `build_admin_router_with_ui(...)`.
+    let ui_enabled = guard.signing_key.0.is_empty();
+    drop(guard);
+    build_admin_router_with_ui(state, ui_enabled)
+}
+
+/// Build the admin router with an explicit UI exposure flag.
+pub fn build_admin_router_with_ui(state: Arc<VolumeServerState>, ui_enabled: bool) -> Router {
+    let mut router = Router::new()
+        .route("/status", get(handlers::status_handler))
+        .route("/healthz", get(handlers::healthz_handler))
+        .route("/favicon.ico", get(handlers::favicon_handler))
+        .route(
+            "/seaweedfsstatic/{*path}",
+            get(handlers::static_asset_handler),
+        )
+        .route("/", any(admin_store_handler))
+        .route("/{path}", any(admin_store_handler))
+        .route("/{vid}/{fid}", any(admin_store_handler))
+        .route("/{vid}/{fid}/{filename}", any(admin_store_handler))
+        .fallback(admin_store_handler);
+    if ui_enabled {
+        // Note: /stats/* endpoints are commented out in Go's volume_server.go (L130-134).
+        // Only the UI endpoint is registered when UI access is enabled.
+        router = router.route("/ui/index.html", get(handlers::ui_handler));
+    }
+    router
+        .layer(middleware::from_fn(common_headers_middleware))
+        .with_state(state)
+}
+
+/// Build the public (read-only) HTTP router — only GET/HEAD.
+pub fn build_public_router(state: Arc<VolumeServerState>) -> Router {
+    Router::new()
+        .route("/favicon.ico", get(handlers::favicon_handler))
+        .route(
+            "/seaweedfsstatic/{*path}",
+            get(handlers::static_asset_handler),
+        )
+        .route("/", any(public_store_handler))
+        .route("/{path}", any(public_store_handler))
+        .route("/{vid}/{fid}", any(public_store_handler))
+        .route("/{vid}/{fid}/{filename}", any(public_store_handler))
+        .fallback(public_store_handler)
+        .layer(middleware::from_fn(common_headers_middleware))
+        .with_state(state)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_to_http_address_strips_grpc_port_suffix() {
+        assert_eq!(to_http_address("10.0.0.1:9333.19333"), "10.0.0.1:9333");
+        assert_eq!(
+            to_http_address("master.local:9333.19333"),
+            "master.local:9333"
+        );
+        assert_eq!(to_http_address("10.85.183.6:5300.6300"), "10.85.183.6:5300");
+    }
+
+    #[test]
+    fn test_to_http_address_passthrough_without_grpc_suffix() {
+        assert_eq!(to_http_address("10.0.0.1:9333"), "10.0.0.1:9333");
+        assert_eq!(to_http_address("master.local:9333"), "master.local:9333");
+    }
+
+    #[test]
+    fn test_to_http_address_returns_input_when_unparseable() {
+        assert_eq!(to_http_address(""), "");
+        assert_eq!(to_http_address("no-port"), "no-port");
+        // Trailing colon: nothing after the separator, treat as unparseable.
+        assert_eq!(to_http_address("host:"), "host:");
+    }
+
+    #[test]
+    fn test_to_http_address_borrows_when_unchanged_and_owns_when_stripped() {
+        // The common case (no suffix) must not allocate.
+        let result = to_http_address("10.0.0.1:9333");
+        assert!(matches!(result, std::borrow::Cow::Borrowed(_)));
+        // Stripping requires a new string.
+        let result = to_http_address("10.0.0.1:9333.19333");
+        assert!(matches!(result, std::borrow::Cow::Owned(_)));
+        assert_eq!(result, "10.0.0.1:9333");
+        // Unparseable / passthrough also borrows.
+        let result = to_http_address("host:abc.def");
+        assert!(matches!(result, std::borrow::Cow::Borrowed(_)));
+    }
+
+    #[test]
+    fn test_to_http_address_keeps_non_numeric_dotted_suffix() {
+        // The dotted form is only valid when both sides are real port numbers.
+        // Otherwise the address is malformed config (e.g. a hostname like
+        // "host:abc.def"), and silently rewriting it would just hide the bug.
+        assert_eq!(to_http_address("host:abc.def"), "host:abc.def");
+        assert_eq!(to_http_address("host:9333.notaport"), "host:9333.notaport");
+        assert_eq!(
+            to_http_address("host:notaport.19333"),
+            "host:notaport.19333"
+        );
+        // Out-of-range ports must not be silently truncated either.
+        assert_eq!(to_http_address("host:99999.19333"), "host:99999.19333");
+    }
+
+    #[test]
+    fn test_to_http_address_handles_bracketed_ipv6_literals() {
+        // The function uses `rfind(':')`, so for bracketed IPv6 the port
+        // separator is correctly identified as the colon AFTER the closing
+        // bracket — making IPv4 and IPv6 behave the same.
+        assert_eq!(to_http_address("[::1]:9333.19333"), "[::1]:9333");
+        assert_eq!(
+            to_http_address("[2001:db8::10]:5300.6300"),
+            "[2001:db8::10]:5300"
+        );
+        // Plain bracketed IPv6 without a dotted suffix is borrowed unchanged.
+        let result = to_http_address("[2001:db8::1]:9333");
+        assert!(matches!(result, std::borrow::Cow::Borrowed(_)));
+        assert_eq!(result, "[2001:db8::1]:9333");
+        // Non-numeric / out-of-range / missing suffix all preserve the input.
+        assert_eq!(to_http_address("[::1]:"), "[::1]:");
+        assert_eq!(to_http_address("[::1]:abc.def"), "[::1]:abc.def");
+        assert_eq!(to_http_address("[::1]:99999.19333"), "[::1]:99999.19333");
+    }
+}

seaweed-volume/src/server/write_queue.rs

@@ -0,0 +1,339 @@
+//! Async batched write processing for the volume server.
+//!
+//! Instead of each upload handler directly calling `write_needle`, writes are
+//! submitted to a queue. A background worker drains the queue in batches (up to
+//! 128 entries), groups them by volume ID, and processes them together under a
+//! single store lock. Requests that asked for `fsync` are flushed by
+//! `write_needle` itself, one flush per durable write.
+
+use std::sync::Arc;
+
+use tokio::sync::{mpsc, oneshot};
+use tracing::debug;
+
+use crate::storage::needle::needle::Needle;
+use crate::storage::types::{Size, VolumeId};
+use crate::storage::volume::VolumeError;
+
+use super::volume_server::VolumeServerState;
+
+/// Result of a single write operation: (offset, size, is_unchanged).
+pub type WriteResult = Result<(u64, Size, bool), VolumeError>;
+
+/// The needles queued for one volume, each with the durability it asked for.
+type VolumeBatch = Vec<(Needle, bool, oneshot::Sender<WriteResult>)>;
+
+/// A request to write a needle, submitted to the write queue.
+pub struct WriteRequest {
+    pub volume_id: VolumeId,
+    pub needle: Needle,
+    pub fsync: bool,
+    pub response_tx: oneshot::Sender<WriteResult>,
+}
+
+/// Maximum number of write requests to batch together.
+const MAX_BATCH_SIZE: usize = 128;
+
+/// Maximum bytes to accumulate per batch before breaking (matches Go's 4MB limit).
+/// This prevents large writes from accumulating unbounded latency.
+const MAX_BATCH_BYTES: usize = 4 * 1024 * 1024;
+
+/// Handle for submitting write requests to the background worker.
+#[derive(Clone)]
+pub struct WriteQueue {
+    tx: mpsc::Sender<WriteRequest>,
+}
+
+impl WriteQueue {
+    /// Create a new write queue and spawn the background worker.
+    ///
+    /// `capacity` controls the channel buffer size (backpressure kicks in when full).
+    /// The worker holds a reference to `state` for accessing the store.
+    pub fn new(state: Arc<VolumeServerState>, capacity: usize) -> Self {
+        let (tx, rx) = mpsc::channel(capacity);
+        let worker = WriteQueueWorker { rx, state };
+        tokio::spawn(worker.run());
+        WriteQueue { tx }
+    }
+
+    /// Submit a write request and wait for the result.
+    ///
+    /// Returns `Err` if the worker has shut down or the response channel was dropped.
+    pub async fn submit(&self, volume_id: VolumeId, needle: Needle, fsync: bool) -> WriteResult {
+        let (response_tx, response_rx) = oneshot::channel();
+        let request = WriteRequest {
+            volume_id,
+            needle,
+            fsync,
+            response_tx,
+        };
+
+        // Send to queue; this awaits if the channel is full (backpressure).
+        if self.tx.send(request).await.is_err() {
+            return Err(VolumeError::Io(std::io::Error::new(
+                std::io::ErrorKind::BrokenPipe,
+                "write queue worker has shut down",
+            )));
+        }
+
+        // Wait for the worker to process our request.
+        match response_rx.await {
+            Ok(result) => result,
+            Err(_) => Err(VolumeError::Io(std::io::Error::new(
+                std::io::ErrorKind::BrokenPipe,
+                "write queue worker dropped response channel",
+            ))),
+        }
+    }
+}
+
+/// Background worker that drains write requests and processes them in batches.
+struct WriteQueueWorker {
+    rx: mpsc::Receiver<WriteRequest>,
+    state: Arc<VolumeServerState>,
+}
+
+impl WriteQueueWorker {
+    async fn run(mut self) {
+        debug!("write queue worker started");
+
+        loop {
+            // Wait for the first request (blocks until one arrives or channel closes).
+            let first = match self.rx.recv().await {
+                Some(req) => req,
+                None => {
+                    debug!("write queue channel closed, worker exiting");
+                    return;
+                }
+            };
+
+            // Drain as many additional requests as available, up to MAX_BATCH_SIZE
+            // or MAX_BATCH_BYTES (matches Go: 128 requests or 4MB, whichever comes first).
+            let mut batch = Vec::with_capacity(MAX_BATCH_SIZE);
+            let mut batch_bytes: usize = first.needle.data.len();
+            batch.push(first);
+
+            while batch.len() < MAX_BATCH_SIZE && batch_bytes < MAX_BATCH_BYTES {
+                match self.rx.try_recv() {
+                    Ok(req) => {
+                        batch_bytes += req.needle.data.len();
+                        batch.push(req);
+                    }
+                    Err(_) => break,
+                }
+            }
+
+            let batch_size = batch.len();
+            debug!("processing write batch of {} requests", batch_size);
+
+            // Process the batch in spawn_blocking since write_needle does file I/O.
+            let state = self.state.clone();
+            let _ = tokio::task::spawn_blocking(move || {
+                process_batch(state, batch);
+            })
+            .await;
+        }
+    }
+}
+
+/// Process a batch of write requests, grouped by volume ID.
+///
+/// Groups writes by volume to minimize the number of store lock acquisitions,
+/// then sends results back via each request's oneshot channel.
+fn process_batch(state: Arc<VolumeServerState>, batch: Vec<WriteRequest>) {
+    // Group requests by volume ID for efficient processing.
+    // We use a Vec of (VolumeId, Vec<(Needle, Sender)>) to preserve order
+    // and avoid requiring Hash on VolumeId.
+    let mut groups: Vec<(VolumeId, VolumeBatch)> = Vec::new();
+
+    for req in batch {
+        let vid = req.volume_id;
+        if let Some(group) = groups.iter_mut().find(|(v, _)| *v == vid) {
+            group.1.push((req.needle, req.fsync, req.response_tx));
+        } else {
+            groups.push((vid, vec![(req.needle, req.fsync, req.response_tx)]));
+        }
+    }
+
+    // Process each volume group under a single store lock.
+    let mut store = state.store.write().unwrap();
+
+    for (vid, entries) in groups {
+        for (mut needle, fsync, response_tx) in entries {
+            let result = store.write_volume_needle(vid, &mut needle, fsync);
+            // Send result back; ignore error if receiver dropped.
+            let _ = response_tx.send(result);
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::storage::types::VolumeId;
+
+    /// Helper to create a minimal VolumeServerState for testing.
+    fn make_test_state() -> Arc<VolumeServerState> {
+        use crate::security::{Guard, SigningKey};
+        use crate::server::volume_server::RuntimeMetricsConfig;
+        use crate::storage::needle_map::NeedleMapKind;
+        use crate::storage::store::Store;
+        use std::sync::RwLock;
+        use std::sync::atomic::{AtomicBool, AtomicI64, AtomicU32};
+
+        let store = Store::new(NeedleMapKind::InMemory);
+        let guard = Guard::new(&[], SigningKey(vec![]), 0, SigningKey(vec![]), 0);
+
+        Arc::new(VolumeServerState {
+            store: RwLock::new(store),
+            guard: RwLock::new(guard),
+            is_stopping: RwLock::new(false),
+            maintenance: AtomicBool::new(false),
+            state_version: AtomicU32::new(0),
+            concurrent_upload_limit: 0,
+            concurrent_download_limit: 0,
+            inflight_upload_data_timeout: std::time::Duration::ZERO,
+            inflight_download_data_timeout: std::time::Duration::ZERO,
+            inflight_upload_bytes: AtomicI64::new(0),
+            inflight_download_bytes: AtomicI64::new(0),
+            upload_notify: tokio::sync::Notify::new(),
+            download_notify: tokio::sync::Notify::new(),
+            data_center: String::new(),
+            rack: String::new(),
+            file_size_limit_bytes: 0,
+            maintenance_byte_per_second: 0,
+            is_heartbeating: AtomicBool::new(false),
+            has_master: false,
+            pre_stop_seconds: 0,
+            volume_state_notify: tokio::sync::Notify::new(),
+            write_queue: std::sync::OnceLock::new(),
+            s3_tier_registry: std::sync::RwLock::new(
+                crate::remote_storage::s3_tier::S3TierRegistry::new(),
+            ),
+            read_mode: crate::config::ReadMode::Local,
+            allow_untrusted_remote_endpoints: false,
+            master_url: String::new(),
+            master_urls: Vec::new(),
+            seed_master_set: std::collections::HashSet::new(),
+            current_master_url: tokio::sync::RwLock::new(String::new()),
+            self_url: String::new(),
+            http_client: reqwest::Client::new(),
+            outgoing_http_scheme: "http".to_string(),
+            outgoing_grpc_tls: None,
+            metrics_runtime: std::sync::RwLock::new(RuntimeMetricsConfig::default()),
+            metrics_notify: tokio::sync::Notify::new(),
+            fix_jpg_orientation: false,
+            has_slow_read: true,
+            read_buffer_size_bytes: 4 * 1024 * 1024,
+            security_file: String::new(),
+            cli_white_list: vec![],
+            state_file_path: String::new(),
+        })
+    }
+
+    #[tokio::test]
+    async fn test_write_queue_submit_no_volume() {
+        // Submit a write to a non-existent volume -- should return VolumeError::NotFound.
+        let state = make_test_state();
+        let queue = WriteQueue::new(state, MAX_BATCH_SIZE);
+
+        let needle = Needle {
+            id: 1.into(),
+            cookie: 0x12345678.into(),
+            data: vec![1, 2, 3],
+            data_size: 3,
+            ..Needle::default()
+        };
+
+        let result = queue.submit(VolumeId(999), needle, false).await;
+        assert!(result.is_err());
+        match result {
+            Err(VolumeError::NotFound) => {} // expected
+            other => panic!("expected NotFound, got {:?}", other),
+        }
+    }
+
+    #[tokio::test]
+    async fn test_write_queue_concurrent_submissions() {
+        // Submit multiple concurrent writes -- all should complete (with errors since no volume).
+        let state = make_test_state();
+        let queue = WriteQueue::new(state, MAX_BATCH_SIZE);
+
+        let mut handles = Vec::new();
+        for i in 0..10u64 {
+            let q = queue.clone();
+            handles.push(tokio::spawn(async move {
+                let needle = Needle {
+                    id: i.into(),
+                    cookie: 0xABCD.into(),
+                    data: vec![i as u8; 10],
+                    data_size: 10,
+                    ..Needle::default()
+                };
+                q.submit(VolumeId(1), needle, false).await
+            }));
+        }
+
+        for handle in handles {
+            let result = handle.await.unwrap();
+            // All should fail with NotFound since there's no volume 1
+            assert!(matches!(result, Err(VolumeError::NotFound)));
+        }
+    }
+
+    #[tokio::test]
+    async fn test_write_queue_batching() {
+        // Verify that many concurrent writes get processed (testing the batching path).
+        let state = make_test_state();
+        let queue = WriteQueue::new(state, MAX_BATCH_SIZE);
+
+        // Submit MAX_BATCH_SIZE requests concurrently
+        let mut handles = Vec::new();
+        for i in 0..MAX_BATCH_SIZE as u64 {
+            let q = queue.clone();
+            handles.push(tokio::spawn(async move {
+                let needle = Needle {
+                    id: i.into(),
+                    cookie: 0x1111.into(),
+                    data: vec![0u8; 4],
+                    data_size: 4,
+                    ..Needle::default()
+                };
+                q.submit(VolumeId(42), needle, false).await
+            }));
+        }
+
+        let mut results = Vec::new();
+        for handle in handles {
+            results.push(handle.await.unwrap());
+        }
+
+        // All should complete (with NotFound errors since no volume exists)
+        assert_eq!(results.len(), MAX_BATCH_SIZE);
+        for r in results {
+            assert!(matches!(r, Err(VolumeError::NotFound)));
+        }
+    }
+
+    #[tokio::test]
+    async fn test_write_queue_dropped_sender() {
+        // When the queue is dropped, subsequent submits should fail gracefully.
+        let state = make_test_state();
+        let queue = WriteQueue::new(state, 1);
+
+        // Clone then drop the original -- the worker keeps running via its rx handle.
+        let queue2 = queue.clone();
+        drop(queue);
+
+        // This should still work since the worker is alive.
+        let needle = Needle {
+            id: 1.into(),
+            cookie: 0.into(),
+            data: vec![],
+            data_size: 0,
+            ..Needle::default()
+        };
+        let result = queue2.submit(VolumeId(1), needle, false).await;
+        assert!(result.is_err()); // NotFound is fine -- the point is it doesn't panic
+    }
+}