snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)

seaweed-worker/crates/lance/Cargo.toml

@@ -0,0 +1,55 @@
+[package]
+name = "weed-lance-worker"
+version.workspace = true
+edition.workspace = true
+rust-version.workspace = true
+description = "SeaweedFS maintenance worker for Lance tables"
+
+[lib]
+name = "weed_lance_worker"
+
+# The binary is not named for lance: it is the Rust side of `weed worker`, and
+# the job families it registers will outgrow this crate. When a second one
+# arrives the bin target moves to a crate of its own under the same name.
+[[bin]]
+name = "weed-worker"
+path = "src/main.rs"
+
+[dependencies]
+seaweed-worker-core = { path = "../core" }
+seaweed-worker-sort = { path = "../sort" }
+prometheus.workspace = true
+# Only the S3 backend: the other object stores lance enables by default are
+# build time this worker never spends.
+lance = { version = "10", default-features = false, features = ["aws"] }
+lance-index = "10"
+# For the manifest's own fragment type, which the sort marker summarises.
+lance-table = "10"
+# For the execution options the sort needs; the scanner's own stream helper
+# takes the default, which has spilling off.
+lance-datafusion = "10"
+arrow-schema = "58"
+arrow-cast = "58"
+chrono = "0.4"
+futures = "0.3"
+anyhow.workspace = true
+async-trait.workspace = true
+clap = { version = "4", features = ["derive", "env"] }
+reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] }
+rustls = "0.23"
+serde = { version = "1", features = ["derive"] }
+serde_json = "1"
+tokio.workspace = true
+tracing.workspace = true
+tracing-subscriber.workspace = true
+
+[dev-dependencies]
+tokio = { workspace = true }
+tonic.workspace = true
+arrow-array = "58"
+arrow-schema = "58"
+arrow-cast = "58"
+lance-linalg = "10"
+
+[lints]
+workspace = true

seaweed-worker/crates/lance/src/catalog/mod.rs

@@ -0,0 +1,10 @@
+//! How the worker finds Lance tables and gets at their bytes.
+//!
+//! It goes through the Lance namespace rather than the filer: the namespace is
+//! the catalog of record, it already knows which tables are Lance, and asking it
+//! to describe a table with vend_credentials is how the worker gets storage
+//! credentials without holding any of its own.
+
+pub mod namespace;
+
+pub use namespace::{NamespaceClient, TableDescription, parse_id};

seaweed-worker/crates/lance/src/catalog/namespace.rs

@@ -0,0 +1,129 @@
+use std::collections::HashMap;
+
+use std::time::Duration;
+
+use anyhow::{Context, Result};
+use serde::Deserialize;
+
+/// The delimiter the Lance namespace joins identifier parts with.
+const DELIMITER: &str = "$";
+
+#[derive(Debug, Deserialize)]
+pub struct TableDescription {
+    pub location: String,
+    #[serde(default)]
+    pub storage_options: HashMap<String, String>,
+    #[serde(default)]
+    pub managed_versioning: bool,
+}
+
+#[derive(Debug, Deserialize)]
+struct ListNamespacesResponse {
+    #[serde(default)]
+    namespaces: Vec<String>,
+}
+
+#[derive(Debug, Deserialize)]
+struct ListTablesResponse {
+    #[serde(default)]
+    tables: Vec<String>,
+}
+
+/// A thin client for the operations this worker needs. It deliberately does not
+/// wrap the whole spec: a maintenance worker lists, describes, and commits.
+pub struct NamespaceClient {
+    base_url: String,
+    http: reqwest::Client,
+}
+
+/// A namespace call that has not answered by now is not going to. Without this
+/// a gateway that accepts the connection and then goes quiet holds a detection
+/// slot open forever, and the sweep never finishes.
+const REQUEST_TIMEOUT: Duration = Duration::from_secs(30);
+const CONNECT_TIMEOUT: Duration = Duration::from_secs(10);
+
+impl NamespaceClient {
+    pub fn new(base_url: impl Into<String>) -> Self {
+        let http = reqwest::Client::builder()
+            .timeout(REQUEST_TIMEOUT)
+            .connect_timeout(CONNECT_TIMEOUT)
+            .build()
+            // The builder only fails on a bad TLS backend, which would break
+            // every call anyway; a client with no timeouts is worse than a panic
+            // at startup, so keep the default only as a last resort.
+            .unwrap_or_else(|err| {
+                tracing::warn!("falling back to an untimed HTTP client: {err}");
+                reqwest::Client::new()
+            });
+        Self {
+            base_url: base_url.into().trim_end_matches('/').to_string(),
+            http,
+        }
+    }
+
+    /// Every table the namespace holds, as delimiter-joined identifiers.
+    pub async fn list_all_tables(&self) -> Result<Vec<String>> {
+        let url = format!("{}/v1/table", self.base_url);
+        let response: ListTablesResponse = self
+            .http
+            .get(&url)
+            .send()
+            .await
+            .context("list tables")?
+            .error_for_status()?
+            .json()
+            .await?;
+        Ok(response.tables)
+    }
+
+    /// Child namespaces of `id`; the root lists table buckets.
+    pub async fn list_namespaces(&self, id: &[String]) -> Result<Vec<String>> {
+        let url = format!("{}/v1/namespace/{}/list", self.base_url, encode_id(id));
+        let response: ListNamespacesResponse = self
+            .http
+            .get(&url)
+            .send()
+            .await
+            .context("list namespaces")?
+            .error_for_status()?
+            .json()
+            .await?;
+        Ok(response.namespaces)
+    }
+
+    /// Resolve a table to a location and the credentials to reach it. The
+    /// credentials expire, so a long compaction re-describes rather than
+    /// carrying one set for the whole job.
+    pub async fn describe_table(&self, id: &[String]) -> Result<TableDescription> {
+        let url = format!("{}/v1/table/{}/describe", self.base_url, encode_id(id));
+        let body = serde_json::json!({ "id": id, "vend_credentials": true });
+        let description: TableDescription = self
+            .http
+            .post(&url)
+            .json(&body)
+            .send()
+            .await
+            .context("describe table")?
+            .error_for_status()?
+            .json()
+            .await?;
+        Ok(description)
+    }
+}
+
+fn encode_id(id: &[String]) -> String {
+    if id.is_empty() {
+        DELIMITER.to_string()
+    } else {
+        id.join(DELIMITER)
+    }
+}
+
+/// Splits a delimiter-joined identifier back into parts.
+pub fn parse_id(encoded: &str) -> Vec<String> {
+    encoded
+        .split(DELIMITER)
+        .filter(|part| !part.is_empty())
+        .map(|part| part.to_string())
+        .collect()
+}

seaweed-worker/crates/lance/src/dataset.rs

@@ -0,0 +1,98 @@
+//! Opening a Lance dataset with credentials the namespace vended.
+//!
+//! The worker holds no storage credentials of its own: it asks the namespace to
+//! describe a table with `vend_credentials`, and the `storage_options` that come
+//! back are handed to lance as-is. They expire, so a job that runs longer than
+//! their lifetime re-describes rather than carrying one set throughout.
+
+use std::collections::HashMap;
+
+use anyhow::{Context, Result};
+use lance::dataset::Dataset;
+use lance::dataset::builder::DatasetBuilder;
+
+use crate::catalog::{NamespaceClient, TableDescription};
+
+/// A table the worker is about to work on.
+pub struct OpenTable {
+    pub id: Vec<String>,
+    pub location: String,
+    pub dataset: Dataset,
+}
+
+/// What detection needs to decide whether a table is worth a job. Reading it
+/// opens the dataset but touches no data files.
+pub struct TableStats {
+    pub fragments: usize,
+    pub version: u64,
+    pub total_versions: usize,
+    pub rows: usize,
+    /// The Arrow schema as JSON, which is the only description of this table
+    /// anything outside the format can produce.
+    pub schema: Option<String>,
+}
+
+/// Storage options an operator supplies for deployments that vend none.
+pub type FallbackOptions = HashMap<String, String>;
+
+/// Resolve a table through the namespace and open it.
+pub async fn open(
+    client: &NamespaceClient,
+    id: &[String],
+    fallback: &FallbackOptions,
+) -> Result<OpenTable> {
+    let description = client.describe_table(id).await?;
+    let dataset = open_at(&description, fallback).await?;
+    Ok(OpenTable {
+        id: id.to_vec(),
+        location: description.location,
+        dataset,
+    })
+}
+
+/// The namespace vends object_store's own option names, so they pass straight
+/// through. What it vends always wins: the fallback exists because a deployment
+/// without STS vends no credentials at all, and then the worker has no other way
+/// to reach the data.
+async fn open_at(description: &TableDescription, fallback: &FallbackOptions) -> Result<Dataset> {
+    let mut options: HashMap<String, String> = fallback.clone();
+    options.extend(description.storage_options.clone());
+    DatasetBuilder::from_uri(&description.location)
+        .with_storage_options(options)
+        .load()
+        .await
+        .with_context(|| format!("open lance dataset at {}", description.location))
+}
+
+impl OpenTable {
+    pub async fn stats(&self) -> Result<TableStats> {
+        let versions = self.dataset.versions().await?;
+        Ok(TableStats {
+            fragments: self.dataset.get_fragments().len(),
+            version: self.dataset.version().version,
+            total_versions: versions.len(),
+            rows: self.dataset.count_rows(None).await.unwrap_or(0),
+            schema: schema_json(&self.dataset),
+        })
+    }
+}
+
+/// Renders the dataset's schema as JSON. Best effort: a schema that will not
+/// serialise is not a reason to fail a maintenance sweep.
+fn schema_json(dataset: &Dataset) -> Option<String> {
+    let arrow: arrow_schema::Schema = dataset.schema().into();
+    serde_json::to_string(
+        &arrow
+            .fields()
+            .iter()
+            .map(|f| {
+                serde_json::json!({
+                    "name": f.name(),
+                    "type": f.data_type().to_string(),
+                    "nullable": f.is_nullable(),
+                })
+            })
+            .collect::<Vec<_>>(),
+    )
+    .ok()
+}

seaweed-worker/crates/lance/src/jobs/cleanup.rs

@@ -0,0 +1,336 @@
+use std::collections::HashMap;
+
+use anyhow::{Context, Result, anyhow};
+use async_trait::async_trait;
+use chrono::{Duration, Utc};
+use lance::dataset::cleanup::{CleanupPolicy, cleanup_old_versions};
+use seaweed_worker_core::config_form::{form, int_or, int_value, number_field};
+use seaweed_worker_core::pb::{
+    ConfigValue, DetectionComplete, DetectionProposals, ExecuteJobRequest, JobCompleted,
+    JobProgressUpdate, JobProposal, JobResult, JobTypeCapability, JobTypeDescriptor,
+    RunDetectionRequest,
+};
+use seaweed_worker_core::{DetectionSender, ExecutionSender, JobHandler};
+use tracing::warn;
+
+use crate::catalog::{NamespaceClient, parse_id};
+use crate::dataset;
+use crate::jobs::{clamp, string_list, table_id};
+
+pub const JOB_TYPE: &str = "lance_cleanup_versions";
+
+const DEFAULT_RETAIN_HOURS: i64 = 168;
+const DEFAULT_MIN_VERSIONS: i64 = 5;
+
+// The ranges the form offers. Duration::hours also panics far outside this one.
+const MAX_RETAIN_HOURS: i64 = 8760;
+const MAX_MIN_VERSIONS: i64 = 1000;
+
+/// Lance keeps every version until something removes it. Lance can also do this
+/// itself through auto-cleanup, so this job is for deployments that would rather
+/// the cluster owned the policy than each writer.
+pub struct CleanupVersionsHandler {
+    namespace_url: String,
+    fallback: dataset::FallbackOptions,
+    metrics: Option<crate::metrics::LanceMetrics>,
+}
+
+impl CleanupVersionsHandler {
+    pub fn new(namespace_url: String) -> Self {
+        Self {
+            namespace_url,
+            fallback: dataset::FallbackOptions::new(),
+            metrics: None,
+        }
+    }
+
+    pub fn with_metrics(mut self, metrics: Option<crate::metrics::LanceMetrics>) -> Self {
+        self.metrics = metrics;
+        self
+    }
+
+    pub fn with_fallback(mut self, fallback: dataset::FallbackOptions) -> Self {
+        self.fallback = fallback;
+        self
+    }
+
+    fn client(&self) -> NamespaceClient {
+        NamespaceClient::new(self.namespace_url.clone())
+    }
+}
+
+/// The oldest version that may be removed while still leaving `min_versions`
+/// behind, or None when the table has no more than the floor.
+async fn version_floor(table: &dataset::OpenTable, min_versions: usize) -> Result<Option<u64>> {
+    let mut versions: Vec<u64> = table
+        .dataset
+        .versions()
+        .await?
+        .iter()
+        .map(|version| version.version)
+        .collect();
+    if versions.len() <= min_versions {
+        return Ok(None);
+    }
+    versions.sort_unstable();
+    Ok(Some(versions[versions.len() - min_versions]))
+}
+
+#[async_trait]
+impl JobHandler for CleanupVersionsHandler {
+    fn capability(&self) -> JobTypeCapability {
+        JobTypeCapability {
+            job_type: JOB_TYPE.to_string(),
+            can_detect: true,
+            can_execute: true,
+            max_detection_concurrency: 1,
+            max_execution_concurrency: 1,
+            display_name: "Lance Version Cleanup".to_string(),
+            description: "Remove old Lance versions and the files only they referenced".to_string(),
+            weight: 10,
+        }
+    }
+
+    fn descriptor(&self) -> JobTypeDescriptor {
+        let mut defaults: HashMap<String, ConfigValue> = HashMap::new();
+        defaults.insert("retain_hours".to_string(), int_value(DEFAULT_RETAIN_HOURS));
+        defaults.insert(
+            "min_versions_to_keep".to_string(),
+            int_value(DEFAULT_MIN_VERSIONS),
+        );
+
+        JobTypeDescriptor {
+            job_type: JOB_TYPE.to_string(),
+            display_name: "Lance Version Cleanup".to_string(),
+            description: "Age out Lance versions the table no longer needs".to_string(),
+            icon: "fas fa-broom".to_string(),
+            descriptor_version: 1,
+            worker_config_form: Some(form(
+                "lance-cleanup-worker",
+                "Version cleanup",
+                vec![
+                    number_field(
+                        "retain_hours",
+                        "Retain versions for (hours)",
+                        "Versions younger than this are always kept",
+                        1,
+                        8760,
+                    ),
+                    number_field(
+                        "min_versions_to_keep",
+                        "Minimum versions",
+                        "Never leave a table with fewer versions than this, whatever their age",
+                        1,
+                        1000,
+                    ),
+                ],
+                defaults.clone(),
+            )),
+            worker_default_values: defaults,
+            ..Default::default()
+        }
+    }
+
+    async fn detect(
+        &self,
+        request: &RunDetectionRequest,
+        sender: &dyn DetectionSender,
+    ) -> Result<()> {
+        let min_versions = clamp(
+            int_or(
+                &request.worker_config_values,
+                "min_versions_to_keep",
+                DEFAULT_MIN_VERSIONS,
+            ),
+            1,
+            MAX_MIN_VERSIONS,
+        ) as usize;
+        let client = self.client();
+        let tables = client.list_all_tables().await?;
+
+        let mut proposals = Vec::new();
+        for encoded in &tables {
+            let id = parse_id(encoded);
+            let table = match dataset::open(&client, &id, &self.fallback).await {
+                Ok(table) => table,
+                Err(err) => {
+                    if let Some(counters) = &self.metrics {
+                        counters.worker.object_skipped(JOB_TYPE, "open");
+                    }
+                    warn!("skipping {encoded}: {err:#}");
+                    continue;
+                }
+            };
+            if let Some(counters) = &self.metrics {
+                counters.worker.object_seen(JOB_TYPE);
+            }
+            let stats = match table.stats().await {
+                Ok(stats) => stats,
+                Err(err) => {
+                    if let Some(counters) = &self.metrics {
+                        counters.worker.object_skipped(JOB_TYPE, "stats");
+                    }
+                    warn!("skipping {encoded}: reading its stats failed: {err:#}");
+                    continue;
+                }
+            };
+            // Age is decided at execution against the retention window; a table
+            // at or under the floor cannot lose a version whatever its age, so
+            // proposing one would only produce a job with nothing to do.
+            if stats.total_versions <= min_versions {
+                continue;
+            }
+            let mut parameters: HashMap<String, ConfigValue> = HashMap::new();
+            parameters.insert("table_id".to_string(), string_list(&id));
+            proposals.push(JobProposal {
+                proposal_id: format!("{JOB_TYPE}:{encoded}"),
+                dedupe_key: format!("{JOB_TYPE}:{encoded}"),
+                job_type: JOB_TYPE.to_string(),
+                summary: format!("Clean up {encoded} ({} versions)", stats.total_versions),
+                detail: format!(
+                    "{} versions retained, above the {min_versions} floor; \
+                     those outside the retention window can go",
+                    stats.total_versions
+                ),
+                parameters,
+                ..Default::default()
+            });
+        }
+
+        let total = proposals.len() as i32;
+        sender.send_proposals(DetectionProposals {
+            request_id: request.request_id.clone(),
+            job_type: JOB_TYPE.to_string(),
+            proposals,
+            has_more: false,
+        })?;
+        sender.send_complete(DetectionComplete {
+            request_id: request.request_id.clone(),
+            job_type: JOB_TYPE.to_string(),
+            success: true,
+            error_message: String::new(),
+            total_proposals: total,
+        })?;
+        Ok(())
+    }
+
+    async fn execute(
+        &self,
+        request: &ExecuteJobRequest,
+        sender: &dyn ExecutionSender,
+    ) -> Result<()> {
+        let job = request
+            .job
+            .as_ref()
+            .ok_or_else(|| anyhow!("execute request carried no job"))?;
+        let id = table_id(&job.parameters)
+            .ok_or_else(|| anyhow!("job {} carried no table_id", job.job_id))?;
+        let retain_hours = clamp(
+            int_or(
+                &request.worker_config_values,
+                "retain_hours",
+                DEFAULT_RETAIN_HOURS,
+            ),
+            0,
+            MAX_RETAIN_HOURS,
+        );
+        let min_versions = clamp(
+            int_or(
+                &request.worker_config_values,
+                "min_versions_to_keep",
+                DEFAULT_MIN_VERSIONS,
+            ),
+            1,
+            MAX_MIN_VERSIONS,
+        ) as usize;
+
+        let client = self.client();
+        let table = dataset::open(&client, &id, &self.fallback).await?;
+        let before = table.stats().await?;
+
+        // The floor is a promise about how much history survives, so it has to
+        // be applied here and not only when the job was proposed: by the time it
+        // runs, versions may have aged past the retention window, and age alone
+        // would take the table below what the operator asked to keep.
+        let Some(floor) = version_floor(&table, min_versions).await? else {
+            sender.send_completed(JobCompleted {
+                request_id: request.request_id.clone(),
+                job_id: job.job_id.clone(),
+                job_type: JOB_TYPE.to_string(),
+                success: true,
+                result: Some(JobResult {
+                    summary: format!(
+                        "kept all {} versions; the {min_versions} version floor leaves none to remove",
+                        before.total_versions
+                    ),
+                    ..Default::default()
+                }),
+                ..Default::default()
+            })?;
+            return Ok(());
+        };
+
+        sender.send_progress(JobProgressUpdate {
+            request_id: request.request_id.clone(),
+            job_id: job.job_id.clone(),
+            job_type: JOB_TYPE.to_string(),
+            progress_percent: 10.0,
+            stage: format!("cleaning up {} versions", before.total_versions),
+            ..Default::default()
+        })?;
+
+        let policy = CleanupPolicy {
+            before_timestamp: Some(Utc::now() - Duration::hours(retain_hours)),
+            // should_clean() ANDs its clauses, so a version has to be both older
+            // than the window and below the floor to go.
+            before_version: Some(floor),
+            // Files this dataset cannot account for are left alone: they may
+            // belong to a writer that has not committed yet, and deleting them
+            // would corrupt a commit in flight.
+            delete_unverified: false,
+            // A tagged version is pinned on purpose, so refuse rather than
+            // silently dropping what someone named.
+            error_if_tagged_old_versions: true,
+            ..Default::default()
+        };
+        let stats = cleanup_old_versions(&table.dataset, policy)
+            .await
+            .with_context(|| format!("clean up versions of {}", table.location))?;
+
+        if let Some(counters) = &self.metrics {
+            counters.versions_removed.inc_by(stats.old_versions as u64);
+            counters.bytes_reclaimed.inc_by(stats.bytes_removed as u64);
+        }
+
+        let mut output: HashMap<String, ConfigValue> = HashMap::new();
+        output.insert(
+            "old_versions_removed".to_string(),
+            int_value(stats.old_versions as i64),
+        );
+        output.insert(
+            "bytes_removed".to_string(),
+            int_value(stats.bytes_removed as i64),
+        );
+        output.insert(
+            "data_files_removed".to_string(),
+            int_value(stats.data_files_removed as i64),
+        );
+
+        sender.send_completed(JobCompleted {
+            request_id: request.request_id.clone(),
+            job_id: job.job_id.clone(),
+            job_type: JOB_TYPE.to_string(),
+            success: true,
+            result: Some(JobResult {
+                output_values: output,
+                summary: format!(
+                    "removed {} versions and {} bytes",
+                    stats.old_versions, stats.bytes_removed
+                ),
+                ..Default::default()
+            }),
+            ..Default::default()
+        })?;
+        Ok(())
+    }
+}

seaweed-worker/crates/lance/src/jobs/compact.rs

@@ -0,0 +1,318 @@
+use std::collections::HashMap;
+
+use anyhow::{Context, Result, anyhow};
+use async_trait::async_trait;
+use lance::dataset::optimize::{CompactionOptions, compact_files};
+use seaweed_worker_core::config_form::{form, int_or, int_value, number_field, string_value};
+use seaweed_worker_core::pb::{
+    ConfigValue, DetectionComplete, DetectionProposals, ExecuteJobRequest, JobCompleted,
+    JobProgressUpdate, JobProposal, JobResult, JobTypeCapability, JobTypeDescriptor,
+    RunDetectionRequest, WorkerObservations,
+};
+use seaweed_worker_core::{DetectionSender, ExecutionSender, JobHandler};
+use tracing::warn;
+
+use crate::catalog::{NamespaceClient, parse_id};
+use crate::dataset;
+use crate::jobs::{FORMAT, clamp, observation, string_list, table_id};
+
+pub const JOB_TYPE: &str = "lance_compact";
+
+const DEFAULT_TARGET_ROWS: i64 = 1_048_576;
+const DEFAULT_MIN_FRAGMENTS: i64 = 8;
+
+// The ranges the descriptor's form offers. These are also the values that stay
+// meaningful: a table needs two fragments before merging them means anything,
+// and a fragment target below a thousand rows defeats the purpose of the job.
+// Both are cast to usize, where a negative would arrive as an enormous number.
+const TARGET_ROWS_FLOOR: i64 = 1024;
+const TARGET_ROWS_CEILING: i64 = 16_777_216;
+const MIN_FRAGMENTS_FLOOR: i64 = 2;
+const MIN_FRAGMENTS_CEILING: i64 = 4096;
+
+/// Lance writes one fragment per write batch, so a table fed by small appends
+/// accumulates small files the same way an Iceberg table does.
+pub struct CompactHandler {
+    namespace_url: String,
+    fallback: dataset::FallbackOptions,
+    metrics: Option<crate::metrics::LanceMetrics>,
+}
+
+impl CompactHandler {
+    pub fn new(namespace_url: String) -> Self {
+        Self {
+            namespace_url,
+            fallback: dataset::FallbackOptions::new(),
+            metrics: None,
+        }
+    }
+
+    pub fn with_metrics(mut self, metrics: Option<crate::metrics::LanceMetrics>) -> Self {
+        self.metrics = metrics;
+        self
+    }
+
+    /// Storage options to use where the namespace vends none.
+    pub fn with_fallback(mut self, fallback: dataset::FallbackOptions) -> Self {
+        self.fallback = fallback;
+        self
+    }
+
+    fn client(&self) -> NamespaceClient {
+        NamespaceClient::new(self.namespace_url.clone())
+    }
+}
+
+#[async_trait]
+impl JobHandler for CompactHandler {
+    fn capability(&self) -> JobTypeCapability {
+        JobTypeCapability {
+            job_type: JOB_TYPE.to_string(),
+            can_detect: true,
+            can_execute: true,
+            max_detection_concurrency: 1,
+            max_execution_concurrency: 1,
+            display_name: "Lance Compaction".to_string(),
+            description: "Merge small Lance fragments into fewer, larger ones".to_string(),
+            weight: 20,
+        }
+    }
+
+    fn descriptor(&self) -> JobTypeDescriptor {
+        let mut defaults: HashMap<String, ConfigValue> = HashMap::new();
+        defaults.insert(
+            "target_rows_per_fragment".to_string(),
+            int_value(DEFAULT_TARGET_ROWS),
+        );
+        defaults.insert(
+            "min_fragments".to_string(),
+            int_value(DEFAULT_MIN_FRAGMENTS),
+        );
+
+        JobTypeDescriptor {
+            job_type: JOB_TYPE.to_string(),
+            display_name: "Lance Compaction".to_string(),
+            description: "Compact fragments of Lance tables".to_string(),
+            icon: "fas fa-compress".to_string(),
+            descriptor_version: 1,
+            worker_config_form: Some(form(
+                "lance-compact-worker",
+                "Compaction",
+                vec![
+                    number_field(
+                        "target_rows_per_fragment",
+                        "Target rows per fragment",
+                        "Rows to aim for when rewriting fragments",
+                        1024,
+                        16_777_216,
+                    ),
+                    number_field(
+                        "min_fragments",
+                        "Minimum fragments",
+                        "Leave a table alone until it has at least this many fragments",
+                        2,
+                        4096,
+                    ),
+                ],
+                defaults.clone(),
+            )),
+            worker_default_values: defaults,
+            ..Default::default()
+        }
+    }
+
+    /// Propose a job for every table with more fragments than the operator is
+    /// willing to leave alone. Opening a dataset reads its manifest, not its
+    /// data, so this stays cheap across a catalog.
+    async fn detect(
+        &self,
+        request: &RunDetectionRequest,
+        sender: &dyn DetectionSender,
+    ) -> Result<()> {
+        let min_fragments = clamp(
+            int_or(
+                &request.worker_config_values,
+                "min_fragments",
+                DEFAULT_MIN_FRAGMENTS,
+            ),
+            MIN_FRAGMENTS_FLOOR,
+            MIN_FRAGMENTS_CEILING,
+        ) as usize;
+        let client = self.client();
+        let tables = client.list_all_tables().await?;
+
+        let mut proposals = Vec::new();
+        let mut observations = Vec::new();
+        for encoded in &tables {
+            let id = parse_id(encoded);
+            let table = match dataset::open(&client, &id, &self.fallback).await {
+                Ok(table) => table,
+                Err(err) => {
+                    // A table that cannot be opened is the next run's problem,
+                    // not a reason to abandon the whole sweep.
+                    if let Some(counters) = &self.metrics {
+                        counters.worker.object_skipped(JOB_TYPE, "open");
+                    }
+                    warn!("skipping {encoded}: {err:#}");
+                    continue;
+                }
+            };
+            // One unreadable table must not end the sweep: the tables already
+            // read would lose their proposals, and admin would get no
+            // completion for this request at all.
+            if let Some(counters) = &self.metrics {
+                counters.worker.object_seen(JOB_TYPE);
+            }
+            let stats = match table.stats().await {
+                Ok(stats) => stats,
+                Err(err) => {
+                    if let Some(counters) = &self.metrics {
+                        counters.worker.object_skipped(JOB_TYPE, "stats");
+                    }
+                    warn!("skipping {encoded}: reading its stats failed: {err:#}");
+                    continue;
+                }
+            };
+            // Logged because "detection proposed nothing" is otherwise
+            // indistinguishable from a table the worker could not read.
+            tracing::info!(
+                "compaction detection: {encoded} has {} fragments, threshold {min_fragments}",
+                stats.fragments
+            );
+
+            let mut attributes: HashMap<String, ConfigValue> = HashMap::new();
+            attributes.insert("fragments".to_string(), int_value(stats.fragments as i64));
+            attributes.insert("version".to_string(), int_value(stats.version as i64));
+            attributes.insert(
+                "versions".to_string(),
+                int_value(stats.total_versions as i64),
+            );
+            attributes.insert("rows".to_string(), int_value(stats.rows as i64));
+            if let Some(schema) = stats.schema.clone() {
+                attributes.insert("schema".to_string(), string_value(schema));
+            }
+            observations.push(observation(&id, FORMAT, attributes));
+
+            if stats.fragments < min_fragments {
+                continue;
+            }
+            let mut parameters: HashMap<String, ConfigValue> = HashMap::new();
+            parameters.insert("table_id".to_string(), string_list(&id));
+            proposals.push(JobProposal {
+                proposal_id: format!("{JOB_TYPE}:{encoded}"),
+                dedupe_key: format!("{JOB_TYPE}:{encoded}"),
+                job_type: JOB_TYPE.to_string(),
+                summary: format!("Compact {encoded} ({} fragments)", stats.fragments),
+                detail: format!(
+                    "{} fragments at version {}, above the {min_fragments} the policy allows",
+                    stats.fragments, stats.version
+                ),
+                parameters,
+                ..Default::default()
+            });
+        }
+
+        if !observations.is_empty() {
+            sender.send_observations(WorkerObservations {
+                job_type: JOB_TYPE.to_string(),
+                observations,
+            })?;
+        }
+
+        let total = proposals.len() as i32;
+        sender.send_proposals(DetectionProposals {
+            request_id: request.request_id.clone(),
+            job_type: JOB_TYPE.to_string(),
+            proposals,
+            has_more: false,
+        })?;
+        sender.send_complete(DetectionComplete {
+            request_id: request.request_id.clone(),
+            job_type: JOB_TYPE.to_string(),
+            success: true,
+            error_message: String::new(),
+            total_proposals: total,
+        })?;
+        Ok(())
+    }
+
+    async fn execute(
+        &self,
+        request: &ExecuteJobRequest,
+        sender: &dyn ExecutionSender,
+    ) -> Result<()> {
+        let job = request
+            .job
+            .as_ref()
+            .ok_or_else(|| anyhow!("execute request carried no job"))?;
+        let id = table_id(&job.parameters)
+            .ok_or_else(|| anyhow!("job {} carried no table_id", job.job_id))?;
+        let target_rows = clamp(
+            int_or(
+                &request.worker_config_values,
+                "target_rows_per_fragment",
+                DEFAULT_TARGET_ROWS,
+            ),
+            TARGET_ROWS_FLOOR,
+            TARGET_ROWS_CEILING,
+        ) as usize;
+
+        let client = self.client();
+        // Re-resolve rather than trusting the location detection saw: the table
+        // may have been repointed, and the vended credentials have expired.
+        let mut table = dataset::open(&client, &id, &self.fallback).await?;
+        let before = table.stats().await?;
+
+        sender.send_progress(JobProgressUpdate {
+            request_id: request.request_id.clone(),
+            job_id: job.job_id.clone(),
+            job_type: JOB_TYPE.to_string(),
+            progress_percent: 10.0,
+            stage: format!("compacting {} fragments", before.fragments),
+            ..Default::default()
+        })?;
+
+        let options = CompactionOptions {
+            target_rows_per_fragment: target_rows,
+            ..Default::default()
+        };
+        let metrics = compact_files(&mut table.dataset, options, None)
+            .await
+            .with_context(|| format!("compact {}", table.location))?;
+
+        if let Some(counters) = &self.metrics {
+            counters
+                .fragments_removed
+                .inc_by(metrics.fragments_removed as u64);
+        }
+
+        let after = table.stats().await?;
+        let mut output: HashMap<String, ConfigValue> = HashMap::new();
+        output.insert(
+            "fragments_removed".to_string(),
+            int_value(metrics.fragments_removed as i64),
+        );
+        output.insert(
+            "fragments_added".to_string(),
+            int_value(metrics.fragments_added as i64),
+        );
+        output.insert(
+            "files_removed".to_string(),
+            int_value(metrics.files_removed as i64),
+        );
+
+        sender.send_completed(JobCompleted {
+            request_id: request.request_id.clone(),
+            job_id: job.job_id.clone(),
+            job_type: JOB_TYPE.to_string(),
+            success: true,
+            result: Some(JobResult {
+                output_values: output,
+                summary: format!("{} fragments became {}", before.fragments, after.fragments),
+                ..Default::default()
+            }),
+            ..Default::default()
+        })?;
+        Ok(())
+    }
+}

seaweed-worker/crates/lance/src/jobs/indices.rs

@@ -0,0 +1,275 @@
+use std::collections::HashMap;
+
+use anyhow::{Context, Result, anyhow};
+use async_trait::async_trait;
+use lance::index::DatasetIndexExt;
+use lance_index::optimize::OptimizeOptions;
+use seaweed_worker_core::config_form::{form, int_or, int_value, number_field};
+use seaweed_worker_core::pb::{
+    ConfigValue, DetectionComplete, DetectionProposals, ExecuteJobRequest, JobCompleted,
+    JobProgressUpdate, JobProposal, JobResult, JobTypeCapability, JobTypeDescriptor,
+    RunDetectionRequest,
+};
+use seaweed_worker_core::{DetectionSender, ExecutionSender, JobHandler};
+use tracing::warn;
+
+use crate::catalog::{NamespaceClient, parse_id};
+use crate::dataset::{self, OpenTable};
+use crate::jobs::{clamp, string_list, table_id};
+
+pub const JOB_TYPE: &str = "lance_optimize_indices";
+
+const DEFAULT_MAX_UNINDEXED_ROWS: i64 = 100_000;
+
+// Zero is a real setting - reindex as soon as any row is uncovered - so the
+// floor is what stays meaningful rather than what the form offers. The ceiling
+// is the form's, and the point of both is that this is cast to u64: a negative
+// would arrive as an enormous budget and mean "never reindex".
+const MAX_UNINDEXED_FLOOR: i64 = 0;
+const MAX_UNINDEXED_CEILING: i64 = 100_000_000;
+
+/// Rows written after an index was built are not covered by it, so a vector
+/// search quietly misses them. This is the job with no Iceberg equivalent, and
+/// the reason a neglected Lance table is a correctness problem rather than a
+/// slow one.
+pub struct OptimizeIndicesHandler {
+    namespace_url: String,
+    fallback: dataset::FallbackOptions,
+    metrics: Option<crate::metrics::LanceMetrics>,
+}
+
+impl OptimizeIndicesHandler {
+    pub fn new(namespace_url: String) -> Self {
+        Self {
+            namespace_url,
+            fallback: dataset::FallbackOptions::new(),
+            metrics: None,
+        }
+    }
+
+    pub fn with_metrics(mut self, metrics: Option<crate::metrics::LanceMetrics>) -> Self {
+        self.metrics = metrics;
+        self
+    }
+
+    pub fn with_fallback(mut self, fallback: dataset::FallbackOptions) -> Self {
+        self.fallback = fallback;
+        self
+    }
+
+    fn client(&self) -> NamespaceClient {
+        NamespaceClient::new(self.namespace_url.clone())
+    }
+}
+
+/// Rows no index covers, summed across a table's indices. A table with no
+/// indices at all has nothing to optimize, which is different from a table whose
+/// indices have fallen behind.
+async fn unindexed_rows(table: &OpenTable) -> Result<Option<u64>> {
+    let indices = table.dataset.load_indices().await?;
+    if indices.is_empty() {
+        return Ok(None);
+    }
+
+    let mut worst = 0u64;
+    let mut names: Vec<String> = indices.iter().map(|index| index.name.clone()).collect();
+    names.sort();
+    names.dedup();
+    for name in names {
+        let raw = table.dataset.index_statistics(&name).await?;
+        let stats: serde_json::Value = serde_json::from_str(&raw)
+            .with_context(|| format!("parse index statistics for {name}"))?;
+        let unindexed = stats
+            .get("num_unindexed_rows")
+            .and_then(|value| value.as_u64())
+            .unwrap_or(0);
+        worst = worst.max(unindexed);
+    }
+    Ok(Some(worst))
+}
+
+#[async_trait]
+impl JobHandler for OptimizeIndicesHandler {
+    fn capability(&self) -> JobTypeCapability {
+        JobTypeCapability {
+            job_type: JOB_TYPE.to_string(),
+            can_detect: true,
+            can_execute: true,
+            max_detection_concurrency: 1,
+            max_execution_concurrency: 1,
+            display_name: "Lance Index Optimization".to_string(),
+            description: "Extend indices to cover rows written since they were built".to_string(),
+            weight: 30,
+        }
+    }
+
+    fn descriptor(&self) -> JobTypeDescriptor {
+        let mut defaults: HashMap<String, ConfigValue> = HashMap::new();
+        defaults.insert(
+            "max_unindexed_rows".to_string(),
+            int_value(DEFAULT_MAX_UNINDEXED_ROWS),
+        );
+
+        JobTypeDescriptor {
+            job_type: JOB_TYPE.to_string(),
+            display_name: "Lance Index Optimization".to_string(),
+            description: "Keep vector and scalar indices covering the whole table".to_string(),
+            icon: "fas fa-magnifying-glass-chart".to_string(),
+            descriptor_version: 1,
+            worker_config_form: Some(form(
+                "lance-indices-worker",
+                "Index optimization",
+                vec![number_field(
+                    "max_unindexed_rows",
+                    "Unindexed row budget",
+                    "Reindex once a table has more rows than this outside its indices",
+                    1_000,
+                    100_000_000,
+                )],
+                defaults.clone(),
+            )),
+            worker_default_values: defaults,
+            ..Default::default()
+        }
+    }
+
+    async fn detect(
+        &self,
+        request: &RunDetectionRequest,
+        sender: &dyn DetectionSender,
+    ) -> Result<()> {
+        let budget = clamp(
+            int_or(
+                &request.worker_config_values,
+                "max_unindexed_rows",
+                DEFAULT_MAX_UNINDEXED_ROWS,
+            ),
+            MAX_UNINDEXED_FLOOR,
+            MAX_UNINDEXED_CEILING,
+        ) as u64;
+        let client = self.client();
+        let tables = client.list_all_tables().await?;
+
+        let mut proposals = Vec::new();
+        for encoded in &tables {
+            let id = parse_id(encoded);
+            let table = match dataset::open(&client, &id, &self.fallback).await {
+                Ok(table) => table,
+                Err(err) => {
+                    if let Some(counters) = &self.metrics {
+                        counters.worker.object_skipped(JOB_TYPE, "open");
+                    }
+                    warn!("skipping {encoded}: {err:#}");
+                    continue;
+                }
+            };
+            if let Some(counters) = &self.metrics {
+                counters.worker.object_seen(JOB_TYPE);
+            }
+            let unindexed = match unindexed_rows(&table).await {
+                Ok(Some(unindexed)) => unindexed,
+                Ok(None) => continue,
+                Err(err) => {
+                    if let Some(counters) = &self.metrics {
+                        counters.worker.object_skipped(JOB_TYPE, "index_stats");
+                    }
+                    warn!("skipping {encoded}: reading its index stats failed: {err:#}");
+                    continue;
+                }
+            };
+            if unindexed <= budget {
+                continue;
+            }
+            let mut parameters: HashMap<String, ConfigValue> = HashMap::new();
+            parameters.insert("table_id".to_string(), string_list(&id));
+            proposals.push(JobProposal {
+                proposal_id: format!("{JOB_TYPE}:{encoded}"),
+                dedupe_key: format!("{JOB_TYPE}:{encoded}"),
+                job_type: JOB_TYPE.to_string(),
+                summary: format!("Reindex {encoded} ({unindexed} rows uncovered)"),
+                detail: format!(
+                    "{unindexed} rows sit outside the indices, above the {budget} the policy allows; \
+                     a search of this table misses them"
+                ),
+                parameters,
+                ..Default::default()
+            });
+        }
+
+        let total = proposals.len() as i32;
+        sender.send_proposals(DetectionProposals {
+            request_id: request.request_id.clone(),
+            job_type: JOB_TYPE.to_string(),
+            proposals,
+            has_more: false,
+        })?;
+        sender.send_complete(DetectionComplete {
+            request_id: request.request_id.clone(),
+            job_type: JOB_TYPE.to_string(),
+            success: true,
+            error_message: String::new(),
+            total_proposals: total,
+        })?;
+        Ok(())
+    }
+
+    async fn execute(
+        &self,
+        request: &ExecuteJobRequest,
+        sender: &dyn ExecutionSender,
+    ) -> Result<()> {
+        let job = request
+            .job
+            .as_ref()
+            .ok_or_else(|| anyhow!("execute request carried no job"))?;
+        let id = table_id(&job.parameters)
+            .ok_or_else(|| anyhow!("job {} carried no table_id", job.job_id))?;
+
+        let client = self.client();
+        let mut table = dataset::open(&client, &id, &self.fallback).await?;
+        let before = unindexed_rows(&table).await?.unwrap_or(0);
+
+        sender.send_progress(JobProgressUpdate {
+            request_id: request.request_id.clone(),
+            job_id: job.job_id.clone(),
+            job_type: JOB_TYPE.to_string(),
+            progress_percent: 10.0,
+            stage: format!("indexing {before} uncovered rows"),
+            ..Default::default()
+        })?;
+
+        // Merging every delta keeps read latency from drifting as the table is
+        // reindexed again and again; leaving them unmerged is how an index ends
+        // up fast to write and slow to search.
+        table
+            .dataset
+            .optimize_indices(&OptimizeOptions::default())
+            .await
+            .with_context(|| format!("optimize indices of {}", table.location))?;
+
+        let after = unindexed_rows(&table).await?.unwrap_or(0);
+        if let Some(counters) = &self.metrics {
+            counters.rows_indexed.inc_by(before.saturating_sub(after));
+        }
+        let mut output: HashMap<String, ConfigValue> = HashMap::new();
+        output.insert(
+            "unindexed_rows_before".to_string(),
+            int_value(before as i64),
+        );
+        output.insert("unindexed_rows_after".to_string(), int_value(after as i64));
+
+        sender.send_completed(JobCompleted {
+            request_id: request.request_id.clone(),
+            job_id: job.job_id.clone(),
+            job_type: JOB_TYPE.to_string(),
+            success: true,
+            result: Some(JobResult {
+                output_values: output,
+                summary: format!("{before} uncovered rows became {after}"),
+                ..Default::default()
+            }),
+            ..Default::default()
+        })?;
+        Ok(())
+    }
+}

seaweed-worker/crates/lance/src/jobs/mod.rs

@@ -0,0 +1,118 @@
+//! One module per job type. Each declares its capability and the settings form
+//! admin renders for it, then does the work.
+//!
+//! Detection opens each table and reads its manifest rather than its data, so a
+//! sweep across a catalog stays cheap. Execution re-resolves the table instead
+//! of trusting what detection saw: it may have been repointed, and the vended
+//! credentials expire.
+
+pub mod cleanup;
+pub mod compact;
+pub mod indices;
+pub mod sort;
+
+use std::collections::HashMap;
+use std::sync::Arc;
+
+use seaweed_worker_core::JobHandler;
+use seaweed_worker_core::pb::{ConfigValue, ObjectObservation, StringList, config_value::Kind};
+
+use crate::catalog::parse_id;
+
+/// A table identifier travels in a proposal's parameters and comes back on the
+/// job, so both sides agree on one encoding.
+pub(crate) fn string_list(parts: &[String]) -> ConfigValue {
+    ConfigValue {
+        kind: Some(Kind::StringList(StringList {
+            values: parts.to_vec(),
+        })),
+    }
+}
+
+pub(crate) fn table_id(parameters: &HashMap<String, ConfigValue>) -> Option<Vec<String>> {
+    match parameters.get("table_id")?.kind.as_ref()? {
+        Kind::StringList(list) => Some(list.values.clone()),
+        Kind::StringValue(encoded) => Some(parse_id(encoded)),
+        _ => None,
+    }
+}
+
+/// Every handler this worker serves. A worker process may serve several job
+/// types, which is why WorkerHello carries a list.
+pub fn handlers(
+    namespace_url: String,
+    fallback: crate::dataset::FallbackOptions,
+    metrics: Option<crate::metrics::LanceMetrics>,
+) -> Vec<Arc<dyn JobHandler>> {
+    vec![
+        Arc::new(
+            compact::CompactHandler::new(namespace_url.clone())
+                .with_fallback(fallback.clone())
+                .with_metrics(metrics.clone()),
+        ),
+        Arc::new(
+            indices::OptimizeIndicesHandler::new(namespace_url.clone())
+                .with_fallback(fallback.clone())
+                .with_metrics(metrics.clone()),
+        ),
+        Arc::new(
+            sort::SortHandler::new(namespace_url.clone())
+                .with_fallback(fallback.clone())
+                .with_metrics(metrics.clone()),
+        ),
+        Arc::new(
+            cleanup::CleanupVersionsHandler::new(namespace_url)
+                .with_fallback(fallback)
+                .with_metrics(metrics),
+        ),
+    ]
+}
+
+/// Holds a configured value to the range its form offers. A value from outside
+/// it is one the UI could not have produced, and every one of these is cast to
+/// an unsigned type: a negative arrives as an enormous number, which silently
+/// turns a threshold into "never" rather than failing loudly.
+pub(crate) fn clamp(value: i64, low: i64, high: i64) -> i64 {
+    value.max(low).min(high)
+}
+
+/// The format the catalog records for the tables this worker maintains.
+pub const FORMAT: &str = "LANCE";
+
+/// Builds the observation a detection sweep reports for one table. Detection
+/// has already opened the dataset to decide whether it needs work, so saying
+/// what it saw costs nothing, and for a format the cluster cannot read this is
+/// the only description of the table anything can produce.
+pub(crate) fn observation(
+    id: &[String],
+    format: &str,
+    attributes: HashMap<String, ConfigValue>,
+) -> ObjectObservation {
+    ObjectObservation {
+        object_id: id.to_vec(),
+        object_kind: "table".to_string(),
+        format: format.to_string(),
+        attributes,
+        observed_at: Some(std::time::SystemTime::now().into()),
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    // Every configured threshold is cast to an unsigned type before use. A
+    // negative one would arrive as an enormous number and quietly mean "never",
+    // which looks exactly like a worker with nothing to do.
+    #[test]
+    fn clamp_keeps_a_negative_from_wrapping() {
+        assert_eq!(clamp(-1, 2, 4096) as usize, 2);
+        assert_eq!(clamp(i64::MIN, 0, 100_000_000) as u64, 0);
+    }
+
+    #[test]
+    fn clamp_holds_the_ceiling_and_passes_the_middle() {
+        assert_eq!(clamp(i64::MAX, 0, 8760), 8760);
+        assert_eq!(clamp(168, 0, 8760), 168);
+    }
+}

seaweed-worker/crates/lance/src/jobs/sort.rs

@@ -0,0 +1,488 @@
+use std::collections::HashMap;
+use std::sync::Arc;
+
+use anyhow::{Context, Result, anyhow};
+use async_trait::async_trait;
+use lance::dataset::scanner::ColumnOrdering;
+use lance::dataset::transaction::Operation;
+use lance::dataset::write::{CommitBuilder, InsertBuilder};
+use lance::dataset::{WriteDestination, WriteMode, WriteParams};
+use lance::index::DatasetIndexExt;
+use lance_datafusion::exec::{LanceExecutionOptions, execute_plan};
+use seaweed_worker_core::config_form::{form, int_or, int_value, string_or, string_value};
+use seaweed_worker_core::pb::{
+    ConfigValue, DetectionComplete, DetectionProposals, ExecuteJobRequest, JobCompleted,
+    JobProgressUpdate, JobProposal, JobResult, JobTypeCapability, JobTypeDescriptor,
+    RunDetectionRequest, WorkerObservations,
+};
+use seaweed_worker_core::{DetectionSender, ExecutionSender, JobHandler};
+use seaweed_worker_sort::{
+    CONFIG_MAX_ROWS_PER_FILE, CONFIG_MEMORY_BUDGET_MB, CONFIG_MIN_UNSORTED_ROWS,
+    CONFIG_SORT_FIELDS, DECLARED_FIELDS_KEY, FragmentSummary, SortSpec, SortState, config_fields,
+    resolve, verdict,
+};
+use tracing::warn;
+
+use crate::catalog::{NamespaceClient, parse_id};
+use crate::dataset;
+use crate::jobs::{FORMAT, clamp, observation, string_list, table_id};
+
+pub const JOB_TYPE: &str = "lance_sort";
+
+const DEFAULT_MIN_UNSORTED_ROWS: i64 = 1_048_576;
+const DEFAULT_MEMORY_BUDGET_MB: i64 = 512;
+const DEFAULT_MAX_ROWS_PER_FILE: i64 = 1_048_576;
+
+// The ranges the shared form offers, repeated here because a value from outside
+// one is a value the UI could not have produced and every one of these is cast
+// to an unsigned type on the way in.
+const MIN_UNSORTED_ROWS_FLOOR: i64 = 1;
+const MIN_UNSORTED_ROWS_CEILING: i64 = 1_000_000_000;
+const MEMORY_BUDGET_FLOOR: i64 = 64;
+const MEMORY_BUDGET_CEILING: i64 = 1_048_576;
+const MAX_ROWS_PER_FILE_FLOOR: i64 = 1024;
+const MAX_ROWS_PER_FILE_CEILING: i64 = 16_777_216;
+
+const BYTES_PER_MB: u64 = 1024 * 1024;
+
+/// Rewrites a table so its rows are stored in the order its fields declare,
+/// which is what lets a range scan read a few files instead of all of them.
+///
+/// Lance appends fragments in write order and has no notion of a sorted table,
+/// so nothing but a rewrite establishes that order, and nothing but another
+/// rewrite restores it once rows have been appended.
+pub struct SortHandler {
+    namespace_url: String,
+    fallback: dataset::FallbackOptions,
+    metrics: Option<crate::metrics::LanceMetrics>,
+}
+
+impl SortHandler {
+    pub fn new(namespace_url: String) -> Self {
+        Self {
+            namespace_url,
+            fallback: dataset::FallbackOptions::new(),
+            metrics: None,
+        }
+    }
+
+    pub fn with_metrics(mut self, metrics: Option<crate::metrics::LanceMetrics>) -> Self {
+        self.metrics = metrics;
+        self
+    }
+
+    /// Storage options to use where the namespace vends none.
+    pub fn with_fallback(mut self, fallback: dataset::FallbackOptions) -> Self {
+        self.fallback = fallback;
+        self
+    }
+
+    fn client(&self) -> NamespaceClient {
+        NamespaceClient::new(self.namespace_url.clone())
+    }
+}
+
+/// The fragments a dataset holds, in fragment order, as the sort crate reads
+/// them: the data files each one names, and the live rows it holds.
+///
+/// Appends only add fragments after the existing ones and lance hands out
+/// fragment ids monotonically even across an overwrite, so ordering by id makes
+/// "the fragments the sort wrote are still the first ones" a prefix test.
+fn fragment_summaries(dataset: &lance::Dataset) -> Vec<FragmentSummary> {
+    let mut fragments = dataset.get_fragments();
+    fragments.sort_by_key(|fragment| fragment.id());
+    fragments
+        .iter()
+        .map(|fragment| summarize(fragment.metadata()))
+        .collect()
+}
+
+/// The manifest's own account of one fragment. `num_rows` is physical rows less
+/// deletions, and None where the manifest does not record them.
+fn summarize(fragment: &lance_table::format::Fragment) -> FragmentSummary {
+    FragmentSummary {
+        files: fragment
+            .files
+            .iter()
+            .map(|file| file.path.clone())
+            .collect(),
+        rows: fragment.num_rows().map(|rows| rows as u64),
+    }
+}
+
+/// The scan ordering for a spec. Lance validates the column names against the
+/// schema when the ordering is set, so a spec naming a column the table does
+/// not have fails before anything is written.
+fn orderings(spec: &SortSpec) -> Vec<ColumnOrdering> {
+    spec.fields
+        .iter()
+        .map(|field| ColumnOrdering {
+            ascending: !field.descending,
+            nulls_first: field.nulls_first,
+            column_name: field.path.clone(),
+        })
+        .collect()
+}
+
+#[async_trait]
+impl JobHandler for SortHandler {
+    fn capability(&self) -> JobTypeCapability {
+        JobTypeCapability {
+            job_type: JOB_TYPE.to_string(),
+            can_detect: true,
+            can_execute: true,
+            max_detection_concurrency: 1,
+            max_execution_concurrency: 1,
+            display_name: "Lance Sort".to_string(),
+            description: "Rewrite Lance tables in the order their fields declare. The rewrite replaces every fragment, so indices do not survive it and want rebuilding afterwards.".to_string(),
+            weight: 40,
+        }
+    }
+
+    fn descriptor(&self) -> JobTypeDescriptor {
+        let mut defaults: HashMap<String, ConfigValue> = HashMap::new();
+        defaults.insert(CONFIG_SORT_FIELDS.to_string(), string_value(""));
+        defaults.insert(
+            CONFIG_MIN_UNSORTED_ROWS.to_string(),
+            int_value(DEFAULT_MIN_UNSORTED_ROWS),
+        );
+        defaults.insert(
+            CONFIG_MEMORY_BUDGET_MB.to_string(),
+            int_value(DEFAULT_MEMORY_BUDGET_MB),
+        );
+        defaults.insert(
+            CONFIG_MAX_ROWS_PER_FILE.to_string(),
+            int_value(DEFAULT_MAX_ROWS_PER_FILE),
+        );
+
+        JobTypeDescriptor {
+            job_type: JOB_TYPE.to_string(),
+            display_name: "Lance Sort".to_string(),
+            description: "Sort Lance tables by their declared fields".to_string(),
+            icon: "fas fa-sort".to_string(),
+            descriptor_version: 1,
+            worker_config_form: Some(form(
+                "lance-sort-worker",
+                "Sort",
+                config_fields(),
+                defaults.clone(),
+            )),
+            worker_default_values: defaults,
+            ..Default::default()
+        }
+    }
+
+    /// Propose a job for every table whose declared order the data no longer
+    /// follows. Opening a dataset reads its manifest and its configuration, not
+    /// its data, so this stays cheap across a catalog.
+    async fn detect(
+        &self,
+        request: &RunDetectionRequest,
+        sender: &dyn DetectionSender,
+    ) -> Result<()> {
+        let configured = string_or(&request.worker_config_values, CONFIG_SORT_FIELDS, "");
+        let min_unsorted_rows = clamp(
+            int_or(
+                &request.worker_config_values,
+                CONFIG_MIN_UNSORTED_ROWS,
+                DEFAULT_MIN_UNSORTED_ROWS,
+            ),
+            MIN_UNSORTED_ROWS_FLOOR,
+            MIN_UNSORTED_ROWS_CEILING,
+        ) as u64;
+
+        let client = self.client();
+        let tables = client.list_all_tables().await?;
+
+        let mut proposals = Vec::new();
+        let mut observations = Vec::new();
+        for encoded in &tables {
+            let id = parse_id(encoded);
+            let table = match dataset::open(&client, &id, &self.fallback).await {
+                Ok(table) => table,
+                Err(err) => {
+                    // One unreadable table must not end the sweep: the tables
+                    // already read would lose their proposals.
+                    if let Some(counters) = &self.metrics {
+                        counters.worker.object_skipped(JOB_TYPE, "open");
+                    }
+                    warn!("skipping {encoded}: {err:#}");
+                    continue;
+                }
+            };
+            if let Some(counters) = &self.metrics {
+                counters.worker.object_seen(JOB_TYPE);
+            }
+            let stats = match table.stats().await {
+                Ok(stats) => stats,
+                Err(err) => {
+                    if let Some(counters) = &self.metrics {
+                        counters.worker.object_skipped(JOB_TYPE, "stats");
+                    }
+                    warn!("skipping {encoded}: reading its stats failed: {err:#}");
+                    continue;
+                }
+            };
+
+            let config = table.dataset.config().clone();
+            let spec = match resolve(
+                config.get(DECLARED_FIELDS_KEY).map(String::as_str),
+                &configured,
+            ) {
+                Ok(Some(spec)) => spec,
+                Ok(None) => {
+                    // Not every table is one an operator wants sorted, and a
+                    // table with no order named anywhere is not a failure.
+                    if let Some(counters) = &self.metrics {
+                        counters.worker.object_skipped(JOB_TYPE, "no_sort_order");
+                    }
+                    continue;
+                }
+                Err(err) => {
+                    if let Some(counters) = &self.metrics {
+                        counters.worker.object_skipped(JOB_TYPE, "sort_order");
+                    }
+                    warn!("skipping {encoded}: {err:#}");
+                    continue;
+                }
+            };
+
+            let state = SortState::from_config(&config);
+            let decision = verdict(
+                &spec,
+                &state,
+                &fragment_summaries(&table.dataset),
+                min_unsorted_rows,
+            );
+
+            let mut attributes: HashMap<String, ConfigValue> = HashMap::new();
+            attributes.insert("rows".to_string(), int_value(stats.rows as i64));
+            attributes.insert("version".to_string(), int_value(stats.version as i64));
+            attributes.insert("fragments".to_string(), int_value(stats.fragments as i64));
+            attributes.insert("sort_fields".to_string(), string_value(spec.to_string()));
+            attributes.insert("sort_state".to_string(), string_value(decision.reason()));
+            if let Some(schema) = stats.schema.clone() {
+                attributes.insert("schema".to_string(), string_value(schema));
+            }
+            observations.push(observation(&id, FORMAT, attributes));
+
+            // Logged because "detection proposed nothing" is otherwise
+            // indistinguishable from a table the worker could not read.
+            tracing::info!(
+                "sort detection: {encoded} has {} rows in {spec} order: {}",
+                stats.rows,
+                decision.reason()
+            );
+
+            if !decision.needs_sort() {
+                continue;
+            }
+            let mut parameters: HashMap<String, ConfigValue> = HashMap::new();
+            parameters.insert("table_id".to_string(), string_list(&id));
+            proposals.push(JobProposal {
+                proposal_id: format!("{JOB_TYPE}:{encoded}"),
+                dedupe_key: format!("{JOB_TYPE}:{encoded}"),
+                job_type: JOB_TYPE.to_string(),
+                summary: format!("Sort {encoded} by {spec}"),
+                detail: format!("{} rows, {}", stats.rows, decision.reason()),
+                parameters,
+                ..Default::default()
+            });
+        }
+
+        if !observations.is_empty() {
+            sender.send_observations(WorkerObservations {
+                job_type: JOB_TYPE.to_string(),
+                observations,
+            })?;
+        }
+
+        let total = proposals.len() as i32;
+        sender.send_proposals(DetectionProposals {
+            request_id: request.request_id.clone(),
+            job_type: JOB_TYPE.to_string(),
+            proposals,
+            has_more: false,
+        })?;
+        sender.send_complete(DetectionComplete {
+            request_id: request.request_id.clone(),
+            job_type: JOB_TYPE.to_string(),
+            success: true,
+            error_message: String::new(),
+            total_proposals: total,
+        })?;
+        Ok(())
+    }
+
+    async fn execute(
+        &self,
+        request: &ExecuteJobRequest,
+        sender: &dyn ExecutionSender,
+    ) -> Result<()> {
+        let job = request
+            .job
+            .as_ref()
+            .ok_or_else(|| anyhow!("execute request carried no job"))?;
+        let id = table_id(&job.parameters)
+            .ok_or_else(|| anyhow!("job {} carried no table_id", job.job_id))?;
+
+        let configured = string_or(&request.worker_config_values, CONFIG_SORT_FIELDS, "");
+        let memory_budget_mb = clamp(
+            int_or(
+                &request.worker_config_values,
+                CONFIG_MEMORY_BUDGET_MB,
+                DEFAULT_MEMORY_BUDGET_MB,
+            ),
+            MEMORY_BUDGET_FLOOR,
+            MEMORY_BUDGET_CEILING,
+        ) as u64;
+        let max_rows_per_file = clamp(
+            int_or(
+                &request.worker_config_values,
+                CONFIG_MAX_ROWS_PER_FILE,
+                DEFAULT_MAX_ROWS_PER_FILE,
+            ),
+            MAX_ROWS_PER_FILE_FLOOR,
+            MAX_ROWS_PER_FILE_CEILING,
+        ) as usize;
+
+        let client = self.client();
+        // Re-resolve rather than trusting what detection saw: the table may have
+        // been repointed, its declared order changed, and the vended credentials
+        // have expired.
+        let table = dataset::open(&client, &id, &self.fallback).await?;
+        let before = table.stats().await?;
+        let config = table.dataset.config().clone();
+        let spec = resolve(
+            config.get(DECLARED_FIELDS_KEY).map(String::as_str),
+            &configured,
+        )?
+        .ok_or_else(|| anyhow!("neither the table nor this worker names an order to sort by"))?;
+
+        sender.send_progress(JobProgressUpdate {
+            request_id: request.request_id.clone(),
+            job_id: job.job_id.clone(),
+            job_type: JOB_TYPE.to_string(),
+            progress_percent: 10.0,
+            stage: format!("sorting {} rows by {spec}", before.rows),
+            ..Default::default()
+        })?;
+
+        let mut scanner = table.dataset.scan();
+        scanner
+            .order_by(Some(orderings(&spec)))
+            .with_context(|| format!("order the scan of {} by {spec}", table.location))?;
+        let plan = scanner
+            .create_plan()
+            .await
+            .context("plan the ordered scan")?;
+
+        // Spilling is off in LanceExecutionOptions::default(), and the scanner's
+        // own stream helper takes that default, so the options are built here
+        // instead: a sort that cannot spill is one bounded by memory, which is
+        // the whole thing this job exists to avoid.
+        let stream = execute_plan(
+            plan,
+            LanceExecutionOptions {
+                use_spilling: true,
+                mem_pool_size: Some(memory_budget_mb * BYTES_PER_MB),
+                ..Default::default()
+            },
+        )
+        .context("run the ordered scan")?;
+
+        let params = WriteParams {
+            mode: WriteMode::Overwrite,
+            max_rows_per_file,
+            ..Default::default()
+        };
+        let destination = Arc::new(table.dataset.clone());
+        let mut transaction = InsertBuilder::new(WriteDestination::Dataset(destination.clone()))
+            .with_params(&params)
+            .execute_uncommitted_stream(stream)
+            .await
+            .context("write the sorted fragments")?;
+
+        // The marker rides in the same commit as the data it describes:
+        // Operation::Overwrite is the one place lance takes configuration
+        // values alongside fragments, and a marker written as a second commit
+        // would be lost if the worker died in between, re-sorting the whole
+        // table on the next sweep.
+        match &mut transaction.operation {
+            Operation::Overwrite {
+                config_upsert_values,
+                fragments,
+                ..
+            } => {
+                // The files these fragments name were written a moment ago and
+                // keep their names whatever version this commit becomes, which
+                // is what lets the marker travel inside the commit it describes.
+                let written: Vec<FragmentSummary> = fragments.iter().map(summarize).collect();
+                *config_upsert_values =
+                    Some(SortState::record(&spec, &written, before.rows as u64));
+            }
+            other => {
+                return Err(anyhow!(
+                    "a sorted rewrite produced {other} instead of an overwrite"
+                ));
+            }
+        }
+
+        let sorted = CommitBuilder::new(WriteDestination::Dataset(destination))
+            .execute(transaction)
+            .await
+            .context("commit the sorted rewrite")?;
+
+        if let Some(counters) = &self.metrics {
+            counters.rows_sorted.inc_by(before.rows as u64);
+        }
+
+        // An overwrite carries fragments, schema and configuration — not index
+        // metadata — so a table that had indices no longer does. Saying so is
+        // the difference between a slow table and a mystery.
+        let dropped_indices = match table.dataset.load_indices().await {
+            Ok(indices) => indices.len(),
+            Err(err) => {
+                warn!("could not read the indices of {}: {err:#}", table.location);
+                0
+            }
+        };
+        if dropped_indices > 0 {
+            warn!(
+                "{} lost {dropped_indices} index/indices to the sorted rewrite; run the index job to rebuild them",
+                table.location
+            );
+        }
+
+        let fragments_after = sorted.get_fragments().len();
+        let mut output: HashMap<String, ConfigValue> = HashMap::new();
+        output.insert("rows_sorted".to_string(), int_value(before.rows as i64));
+        output.insert("sort_fields".to_string(), string_value(spec.to_string()));
+        output.insert(
+            "fragments_before".to_string(),
+            int_value(before.fragments as i64),
+        );
+        output.insert(
+            "fragments_after".to_string(),
+            int_value(fragments_after as i64),
+        );
+        output.insert(
+            "indices_dropped".to_string(),
+            int_value(dropped_indices as i64),
+        );
+
+        sender.send_completed(JobCompleted {
+            request_id: request.request_id.clone(),
+            job_id: job.job_id.clone(),
+            job_type: JOB_TYPE.to_string(),
+            success: true,
+            result: Some(JobResult {
+                output_values: output,
+                summary: format!("{} rows rewritten in {spec} order", before.rows),
+                ..Default::default()
+            }),
+            ..Default::default()
+        })?;
+        Ok(())
+    }
+}

seaweed-worker/crates/lance/src/lib.rs

@@ -0,0 +1,15 @@
+//! Maintenance jobs for Lance tables.
+//!
+//! A Lance dataset needs three things done to it over time: its fragments
+//! compacted, its indices extended to cover rows written after they were built,
+//! and its old versions removed. None can run in the Go worker, because all
+//! three read and rewrite Lance files. This crate is the worker that can.
+
+pub mod catalog;
+pub mod dataset;
+pub mod jobs;
+pub mod metrics;
+pub mod preview;
+pub mod tls;
+
+pub use jobs::handlers;

seaweed-worker/crates/lance/src/main.rs

@@ -0,0 +1,193 @@
+use std::time::Duration;
+
+use anyhow::{Context, Result};
+use clap::Parser;
+use std::net::{IpAddr, SocketAddr};
+use std::sync::Arc;
+
+use seaweed_worker_core::{Metrics, Registry, TlsOptions, WorkerOptions};
+use weed_lance_worker::handlers;
+use weed_lance_worker::metrics::LanceMetrics;
+
+/// Mirrors `weed worker`'s flags, because this is the same contract from another
+/// language and an operator should not have to learn a second set of names.
+#[derive(Parser, Debug)]
+#[command(name = "weed-worker", about = "SeaweedFS maintenance worker")]
+struct Args {
+    /// Admin server gRPC address.
+    #[arg(long, default_value = "localhost:23646", env = "WEED_ADMIN")]
+    admin: String,
+
+    /// Worker identity reported to admin.
+    #[arg(long, default_value = "lance-worker", env = "WEED_WORKER_ID")]
+    id: String,
+
+    /// Lance namespace the worker lists tables from.
+    #[arg(
+        long,
+        default_value = "http://localhost:9101",
+        env = "WEED_LANCE_NAMESPACE"
+    )]
+    namespace: String,
+
+    #[arg(long, default_value = "10", env = "WEED_HEARTBEAT_SECONDS")]
+    heartbeat_seconds: u64,
+
+    #[arg(long, default_value = "1")]
+    max_concurrency: i32,
+
+    /// Storage credentials to use where the namespace vends none. A gateway
+    /// with STS configured vends its own and these are ignored.
+    #[arg(long, env = "WEED_S3_ACCESS_KEY")]
+    access_key: Option<String>,
+
+    #[arg(long, env = "WEED_S3_SECRET_KEY")]
+    secret_key: Option<String>,
+
+    /// mTLS for the admin stream, the same certificates the Go worker reads
+    /// from the [grpc.worker] section of security.toml. All three together, or
+    /// none, in which case the stream is plaintext.
+    #[arg(long, env = "WEED_GRPC_CA")]
+    tls_ca: Option<String>,
+
+    #[arg(long, env = "WEED_GRPC_CLIENT_CERT")]
+    tls_cert: Option<String>,
+
+    #[arg(long, env = "WEED_GRPC_CLIENT_KEY")]
+    tls_key: Option<String>,
+
+    /// Name to verify admin's certificate against, when it is not the address
+    /// this worker dials.
+    #[arg(long)]
+    tls_server_name: Option<String>,
+
+    /// Serve /health, /ready and /metrics on this port, the way
+    /// `weed worker -metricsPort` does. Zero, the default, serves nothing.
+    #[arg(long, default_value = "0", env = "WEED_METRICS_PORT")]
+    metrics_port: u16,
+
+    /// Address the metrics server binds. Loopback by default, since the
+    /// endpoint is unauthenticated.
+    #[arg(long, default_value = "127.0.0.1", env = "WEED_METRICS_IP")]
+    metrics_ip: String,
+}
+
+impl Args {
+    /// The TLS configuration, or an error when the three certificate paths do
+    /// not arrive together: a CA on its own would silently give one-way TLS,
+    /// which the cluster's mutual setup refuses anyway.
+    fn tls(&self) -> Result<Option<TlsOptions>> {
+        match (
+            self.tls_ca.clone(),
+            self.tls_cert.clone(),
+            self.tls_key.clone(),
+        ) {
+            (None, None, None) => Ok(None),
+            (Some(ca_path), Some(client_cert_path), Some(client_key_path)) => {
+                Ok(Some(TlsOptions {
+                    ca_path,
+                    client_cert_path,
+                    client_key_path,
+                    server_name: self.tls_server_name.clone(),
+                }))
+            }
+            _ => Err(anyhow::anyhow!(
+                "--tls-ca, --tls-cert and --tls-key must be given together"
+            )),
+        }
+    }
+}
+
+/// Builds the metrics bind address. Parsing the host separately is what makes an
+/// IPv6 literal work: "::1" and 9327 joined with a colon is not an address, and
+/// formatting them that way turns `--metrics-ip ::1` into a startup failure.
+/// Brackets are accepted too, since that is how the same address is written in a
+/// URL and an operator will reasonably try it.
+fn metrics_address(ip: &str, port: u16) -> Result<SocketAddr> {
+    let host = ip.trim().trim_start_matches('[').trim_end_matches(']');
+    let parsed: IpAddr = host
+        .parse()
+        .with_context(|| format!("parse the metrics address {ip}"))?;
+    Ok(SocketAddr::new(parsed, port))
+}
+
+#[tokio::main]
+async fn main() -> Result<()> {
+    weed_lance_worker::tls::install_default_crypto_provider();
+
+    tracing_subscriber::fmt()
+        .with_env_filter(
+            tracing_subscriber::EnvFilter::try_from_default_env()
+                .unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("info")),
+        )
+        .init();
+
+    let args = Args::parse();
+    let tls = args.tls()?;
+    let options = WorkerOptions {
+        admin_address: args.admin,
+        worker_id: args.id,
+        heartbeat_interval: Duration::from_secs(args.heartbeat_seconds),
+        max_detection_concurrency: args.max_concurrency,
+        max_execution_concurrency: args.max_concurrency,
+        tls,
+        ..Default::default()
+    };
+
+    let mut fallback = weed_lance_worker::dataset::FallbackOptions::new();
+    if let (Some(access), Some(secret)) = (args.access_key, args.secret_key) {
+        fallback.insert("aws_access_key_id".to_string(), access);
+        fallback.insert("aws_secret_access_key".to_string(), secret);
+    }
+
+    let metrics = Metrics::new(&options.worker_id, &options.worker_version)?;
+    let lance_metrics = LanceMetrics::new(&metrics)?;
+    if args.metrics_port > 0 {
+        seaweed_worker_core::metrics::spawn(
+            metrics.clone(),
+            metrics_address(&args.metrics_ip, args.metrics_port)?,
+        );
+    }
+
+    let mut registry = Registry::new().with_preview(Arc::new(
+        weed_lance_worker::preview::LancePreview::new(args.namespace.clone(), fallback.clone()),
+    ));
+    for handler in handlers(args.namespace, fallback, Some(lance_metrics)) {
+        registry = registry.register(handler);
+    }
+
+    seaweed_worker_core::stream::run_with_metrics(options, registry, metrics).await
+}
+
+#[cfg(test)]
+mod tests {
+    use super::metrics_address;
+
+    #[test]
+    fn metrics_address_takes_ipv4_ipv6_and_brackets() {
+        assert_eq!(
+            metrics_address("127.0.0.1", 9327).unwrap().to_string(),
+            "127.0.0.1:9327"
+        );
+        // Joining these with a colon gives "::1:9327", which is not an address.
+        assert_eq!(
+            metrics_address("::1", 9327).unwrap().to_string(),
+            "[::1]:9327"
+        );
+        assert_eq!(
+            metrics_address("[::1]", 9327).unwrap().to_string(),
+            "[::1]:9327"
+        );
+        assert_eq!(
+            metrics_address("0.0.0.0", 9327).unwrap().to_string(),
+            "0.0.0.0:9327"
+        );
+    }
+
+    #[test]
+    fn metrics_address_rejects_a_hostname() {
+        // Binding takes an address, not a name; saying so beats a confusing
+        // failure inside the server.
+        assert!(metrics_address("localhost", 9327).is_err());
+    }
+}

seaweed-worker/crates/lance/src/metrics.rs

@@ -0,0 +1,50 @@
+//! What the Lance jobs reclaimed, in numbers a scrape can add up.
+//!
+//! The generic worker metrics say a job ran and how long it took. These say what
+//! it did: a compaction sweep that runs every minute and removes nothing is a
+//! different thing from one that never runs, and only these tell them apart.
+
+use anyhow::Result;
+use prometheus::IntCounter;
+use seaweed_worker_core::Metrics;
+
+/// Counters the Lance jobs raise, registered on the worker's own registry so
+/// they are served from the same endpoint.
+#[derive(Clone)]
+pub struct LanceMetrics {
+    /// The worker's own metrics, so a job can also record what it read and what
+    /// it had to skip - the pair that tells "nothing needed doing" apart from
+    /// "nothing could be read".
+    pub worker: Metrics,
+    pub fragments_removed: IntCounter,
+    pub rows_indexed: IntCounter,
+    pub rows_sorted: IntCounter,
+    pub versions_removed: IntCounter,
+    pub bytes_reclaimed: IntCounter,
+}
+
+impl LanceMetrics {
+    pub fn new(metrics: &Metrics) -> Result<Self> {
+        Ok(Self {
+            worker: metrics.clone(),
+            fragments_removed: metrics.counter(
+                "lance_fragments_removed_total",
+                "Fragments merged away by compaction.",
+            )?,
+            rows_indexed: metrics.counter(
+                "lance_rows_indexed_total",
+                "Rows brought under an index that did not cover them.",
+            )?,
+            rows_sorted: metrics.counter(
+                "lance_rows_sorted_total",
+                "Rows rewritten into their table's declared order.",
+            )?,
+            versions_removed: metrics.counter(
+                "lance_versions_removed_total",
+                "Dataset versions removed by cleanup.",
+            )?,
+            bytes_reclaimed: metrics
+                .counter("lance_bytes_reclaimed_total", "Bytes freed by cleanup.")?,
+        })
+    }
+}

seaweed-worker/crates/lance/src/preview.rs

@@ -0,0 +1,86 @@
+//! Sample rows of a Lance table, for a page that cannot read the format.
+//!
+//! Admin renders an Iceberg table by reading its Parquet files directly. There
+//! is no Go Lance reader, so for Lance the worker does the reading and hands
+//! back text.
+
+use anyhow::{Context, Result};
+use arrow_cast::display::{ArrayFormatter, FormatOptions};
+use async_trait::async_trait;
+use futures::TryStreamExt;
+use seaweed_worker_core::{Preview, PreviewProvider};
+
+use crate::catalog::NamespaceClient;
+use crate::dataset::{self, FallbackOptions};
+use crate::jobs::FORMAT;
+
+pub struct LancePreview {
+    namespace_url: String,
+    fallback: FallbackOptions,
+}
+
+impl LancePreview {
+    pub fn new(namespace_url: String, fallback: FallbackOptions) -> Self {
+        Self {
+            namespace_url,
+            fallback,
+        }
+    }
+}
+
+#[async_trait]
+impl PreviewProvider for LancePreview {
+    fn format(&self) -> &str {
+        FORMAT
+    }
+
+    async fn preview(&self, object_id: &[String], row_limit: usize) -> Result<Preview> {
+        let client = NamespaceClient::new(self.namespace_url.clone());
+        let table = dataset::open(&client, object_id, &self.fallback).await?;
+
+        let total_rows = table.dataset.count_rows(None).await.unwrap_or(0) as i64;
+        let mut scanner = table.dataset.scan();
+        scanner.limit(Some(row_limit as i64), None)?;
+
+        let batches: Vec<_> = scanner
+            .try_into_stream()
+            .await?
+            .try_collect()
+            .await
+            .with_context(|| format!("read rows from {}", table.location))?;
+
+        let columns = table
+            .dataset
+            .schema()
+            .fields
+            .iter()
+            .map(|field| field.name.clone())
+            .collect();
+
+        let mut rows = Vec::new();
+        for batch in &batches {
+            // ArrayFormatter renders each Arrow type the way its own tooling
+            // does, so a vector column reads as a vector rather than as bytes.
+            let formatters = batch
+                .columns()
+                .iter()
+                .map(|array| ArrayFormatter::try_new(array.as_ref(), &FormatOptions::default()))
+                .collect::<Result<Vec<_>, _>>()?;
+            for index in 0..batch.num_rows() {
+                rows.push(
+                    formatters
+                        .iter()
+                        .map(|formatter| formatter.value(index).to_string())
+                        .collect(),
+                );
+            }
+        }
+        rows.truncate(row_limit);
+
+        Ok(Preview {
+            columns,
+            rows,
+            total_rows,
+        })
+    }
+}

seaweed-worker/crates/lance/src/tls.rs

@@ -0,0 +1,35 @@
+use rustls::crypto::aws_lc_rs;
+
+// aws-lc-rs and ring both get linked transitively (lance's aws backend pulls
+// aws-lc-rs, reqwest's rustls-tls pulls ring), so rustls can't auto-select a
+// provider and tonic's client TLS panics on first use. Pin the default to
+// aws-lc-rs, matching the Rust volume server. Idempotent.
+pub fn install_default_crypto_provider() {
+    let _ = aws_lc_rs::default_provider().install_default();
+}
+
+#[cfg(test)]
+mod tests {
+    use tonic::transport::{Certificate, Channel, ClientTlsConfig, Identity};
+
+    use super::install_default_crypto_provider;
+
+    const TEST_CERT_PEM: &str = "-----BEGIN CERTIFICATE-----\nMIIBPDCB76ADAgECAhRuRPQgeAu43BT/M7EfAWSdapVdYDAFBgMrZXAwFDESMBAG\nA1UEAwwJbG9jYWxob3N0MB4XDTI2MDcwNTE2MTUyOVoXDTM2MDcwMjE2MTUyOVow\nFDESMBAGA1UEAwwJbG9jYWxob3N0MCowBQYDK2VwAyEAr/3bNIFI+8V32oCiY6y+\nXRFmZpdNQ2g//VtRkT+nQg+jUzBRMB0GA1UdDgQWBBTsy9tLf1zPiXCQfgci6zNi\ndEzRSjAfBgNVHSMEGDAWgBTsy9tLf1zPiXCQfgci6zNidEzRSjAPBgNVHRMBAf8E\nBTADAQH/MAUGAytlcANBAIvsdw0IbvOBBkb9cd7BfMJfIP9pQQrAL03pCRWJFnFh\nSysaLVgFXI4T078IiaM874oO+iB+5vNbWEpc7CkGow4=\n-----END CERTIFICATE-----\n";
+    const TEST_KEY_PEM: &str = "-----BEGIN PRIVATE KEY-----\nMC4CAQAwBQYDK2VwBCIEIHbyn71Kk+Y7KT3sBctit7uZpErpoH6qDbFj6P8qGaZH\n-----END PRIVATE KEY-----\n";
+
+    // Without install_default_crypto_provider this panics in the lance crate,
+    // where both aws-lc-rs and ring are linked. Mirrors the volume server's
+    // test_build_grpc_endpoint_with_tls_resolves_crypto_provider.
+    #[tokio::test]
+    async fn tls_channel_builds_after_crypto_provider_installed() {
+        install_default_crypto_provider();
+        let config = ClientTlsConfig::new()
+            .ca_certificate(Certificate::from_pem(TEST_CERT_PEM))
+            .identity(Identity::from_pem(TEST_CERT_PEM, TEST_KEY_PEM));
+        let endpoint = Channel::from_shared("https://127.0.0.1:9")
+            .unwrap()
+            .tls_config(config)
+            .unwrap();
+        assert_eq!(endpoint.uri().scheme_str(), Some("https"));
+    }
+}

seaweed-worker/crates/lance/tests/common/mod.rs

@@ -0,0 +1,97 @@
+//! What every integration test in this crate needs to drive a handler: a sender
+//! that keeps what the handler sent, and the two settings that make a handler
+//! usable against a test gateway.
+//!
+//! Each test binary compiles this module on its own and uses part of it.
+#![allow(dead_code)]
+
+use std::collections::HashMap;
+use std::sync::Mutex;
+
+use anyhow::Result;
+use seaweed_worker_core::pb::{
+    ActivityEvent, ConfigValue, DetectionComplete, DetectionProposals, JobCompleted,
+    JobProgressUpdate, JobProposal, ObjectObservation, WorkerObservations, config_value::Kind,
+};
+use seaweed_worker_core::{DetectionSender, ExecutionSender};
+
+/// Keeps everything a handler reports, so a test can assert on it afterwards.
+#[derive(Default)]
+pub struct Recorder {
+    pub proposals: Mutex<Vec<JobProposal>>,
+    pub observations: Mutex<Vec<ObjectObservation>>,
+    pub completed: Mutex<Vec<JobCompleted>>,
+}
+
+impl DetectionSender for Recorder {
+    fn send_proposals(&self, proposals: DetectionProposals) -> Result<()> {
+        self.proposals.lock().unwrap().extend(proposals.proposals);
+        Ok(())
+    }
+    fn send_complete(&self, _complete: DetectionComplete) -> Result<()> {
+        Ok(())
+    }
+    fn send_activity(&self, _activity: ActivityEvent) -> Result<()> {
+        Ok(())
+    }
+    fn send_observations(&self, observations: WorkerObservations) -> Result<()> {
+        self.observations
+            .lock()
+            .unwrap()
+            .extend(observations.observations);
+        Ok(())
+    }
+}
+
+impl ExecutionSender for Recorder {
+    fn send_progress(&self, _progress: JobProgressUpdate) -> Result<()> {
+        Ok(())
+    }
+    fn send_completed(&self, completed: JobCompleted) -> Result<()> {
+        self.completed.lock().unwrap().push(completed);
+        Ok(())
+    }
+}
+
+/// The gateway to run against, or nothing, in which case the caller skips:
+/// these handlers rewrite real files, and there is nothing to learn from them
+/// against a fake.
+pub fn namespace_url() -> Option<String> {
+    std::env::var("WEED_LANCE_NAMESPACE")
+        .ok()
+        .filter(|s| !s.is_empty())
+}
+
+pub fn int_config(name: &str, value: i64) -> HashMap<String, ConfigValue> {
+    let mut values = HashMap::new();
+    values.insert(
+        name.to_string(),
+        ConfigValue {
+            kind: Some(Kind::Int64Value(value)),
+        },
+    );
+    values
+}
+
+/// What to reach the data with where the namespace vends nothing of its own -
+/// a gateway without STS vends no credentials, and one bound to a wildcard
+/// address vends no endpoint either. Taken from the environment so a harness
+/// can point these tests at a gateway that does check what it is given.
+pub fn fallback() -> weed_lance_worker::dataset::FallbackOptions {
+    let mut options = weed_lance_worker::dataset::FallbackOptions::new();
+    let from_env =
+        |name: &str, default: &str| std::env::var(name).unwrap_or_else(|_| default.to_string());
+    options.insert(
+        "aws_access_key_id".to_string(),
+        from_env("AWS_ACCESS_KEY_ID", "any"),
+    );
+    options.insert(
+        "aws_secret_access_key".to_string(),
+        from_env("AWS_SECRET_ACCESS_KEY", "any"),
+    );
+    if let Ok(endpoint) = std::env::var("AWS_ENDPOINT_URL") {
+        options.insert("aws_endpoint".to_string(), endpoint);
+        options.insert("allow_http".to_string(), "true".to_string());
+    }
+    options
+}

seaweed-worker/crates/lance/tests/compaction.rs

@@ -0,0 +1,540 @@
+//! Drives the compaction handler against a live namespace.
+//!
+//! Skipped unless WEED_LANCE_NAMESPACE names one, the way the Go integration
+//! tests skip without Docker: compaction rewrites real files, and there is
+//! nothing to learn from it against a fake.
+
+use std::collections::HashMap;
+
+use anyhow::Result;
+use seaweed_worker_core::pb::{
+    ExecuteJobRequest, JobSpec, RunDetectionRequest, config_value::Kind,
+};
+use seaweed_worker_core::{JobHandler, PreviewProvider};
+use weed_lance_worker::catalog::NamespaceClient;
+use weed_lance_worker::jobs::cleanup::CleanupVersionsHandler;
+use weed_lance_worker::jobs::compact::{CompactHandler, JOB_TYPE};
+use weed_lance_worker::jobs::indices::OptimizeIndicesHandler;
+use weed_lance_worker::preview::LancePreview;
+
+mod common;
+use common::{Recorder, fallback, int_config, namespace_url};
+
+/// These tests drive one live gateway and one shared catalog: `list_all_tables`
+/// sweeps everything, so a table another test is writing shows up in this test's
+/// detection. Rust runs a binary's tests concurrently, so take a lock.
+static GATEWAY: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(());
+
+/// Declares a table through the namespace and writes `fragments` one-row
+/// appends into it, so a test brings its own state instead of depending on
+/// whatever a previous run left behind.
+async fn seed_fragmented_table(url: &str, name: &str, fragments: usize) -> Result<String> {
+    seed_table(url, name, fragments, 1, false).await
+}
+
+/// Writes `batches` appends of `rows_each` into a freshly declared table, and
+/// optionally builds a vector index after the first batch so the later ones are
+/// rows no index covers.
+async fn seed_table(
+    url: &str,
+    name: &str,
+    batches: usize,
+    rows_each: usize,
+    with_index: bool,
+) -> Result<String> {
+    use arrow_array::{
+        FixedSizeListArray, Float32Array, Int64Array, RecordBatch, RecordBatchIterator,
+    };
+    use arrow_schema::{DataType, Field, Schema};
+    use lance::dataset::{Dataset, WriteMode, WriteParams};
+    use lance::io::{ObjectStoreParams, StorageOptionsAccessor};
+    use std::sync::Arc;
+
+    // Declaring is the namespace's job, not the worker's, so the test asks for
+    // it directly rather than widening the client the worker uses. The bucket and
+    // namespace come first: a table cannot be declared under a parent that does
+    // not exist, and a test that assumes one is a test that only passes twice.
+    let http = reqwest::Client::new();
+    for parent in ["vec", "vec$ml"] {
+        http.post(format!("{url}/v1/namespace/{parent}/create"))
+            .json(&serde_json::json!({"mode": "EXIST_OK"}))
+            .send()
+            .await?
+            .error_for_status()?;
+    }
+    let encoded = format!("vec$ml${name}");
+    http.post(format!("{url}/v1/table/{encoded}/declare"))
+        .json(&serde_json::json!({}))
+        .send()
+        .await?
+        .error_for_status()?;
+
+    let client = NamespaceClient::new(url.to_string());
+    let id = vec!["vec".to_string(), "ml".to_string(), name.to_string()];
+    let description = client.describe_table(&id).await?;
+
+    let mut options = description.storage_options.clone();
+    options.extend(fallback());
+    const DIM: i32 = 16;
+    let schema = Arc::new(Schema::new(vec![
+        Field::new("id", DataType::Int64, false),
+        Field::new(
+            "vec",
+            DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), DIM),
+            false,
+        ),
+    ]));
+
+    for i in 0..batches {
+        let ids: Vec<i64> = (0..rows_each).map(|r| (i * rows_each + r) as i64).collect();
+        let values: Vec<f32> = ids
+            .iter()
+            .flat_map(|id| (0..DIM).map(move |d| (*id as f32) + d as f32))
+            .collect();
+        let vectors = FixedSizeListArray::new(
+            Arc::new(Field::new("item", DataType::Float32, true)),
+            DIM,
+            Arc::new(Float32Array::from(values)),
+            None,
+        );
+        let batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![Arc::new(Int64Array::from(ids)), Arc::new(vectors)],
+        )?;
+        let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone());
+        let params = WriteParams {
+            mode: if i == 0 {
+                WriteMode::Overwrite
+            } else {
+                WriteMode::Append
+            },
+            store_params: Some(ObjectStoreParams {
+                storage_options_accessor: Some(std::sync::Arc::new(
+                    StorageOptionsAccessor::with_static_options(options.clone()),
+                )),
+                ..Default::default()
+            }),
+            ..Default::default()
+        };
+        let dataset = Dataset::write(reader, description.location.as_str(), Some(params)).await?;
+
+        // The index is built after the first batch, so everything appended
+        // afterwards is a row it does not cover.
+        if with_index && i == 0 {
+            use lance::index::DatasetIndexExt;
+            use lance::index::vector::VectorIndexParams;
+            use lance_index::IndexType;
+            use lance_index::vector::{ivf::IvfBuildParams, pq::PQBuildParams};
+
+            let mut dataset = dataset;
+            let params = VectorIndexParams::with_ivf_pq_params(
+                lance_linalg::distance::MetricType::L2,
+                IvfBuildParams::new(1),
+                PQBuildParams::new(4, 8),
+            );
+            dataset
+                .create_index(&["vec"], IndexType::Vector, None, &params, true)
+                .await?;
+        }
+    }
+    Ok(encoded)
+}
+
+/// A table with more fragments than the policy allows is proposed, and running
+/// the proposal leaves it with fewer than it started with.
+#[tokio::test]
+async fn compacts_a_fragmented_table() {
+    let _gateway = GATEWAY.lock().await;
+    let Some(url) = namespace_url() else {
+        eprintln!("WEED_LANCE_NAMESPACE is unset, skipping");
+        return;
+    };
+    // Seeded here rather than by a script, so the test is repeatable: a previous
+    // run compacts the table it depended on.
+    let encoded = seed_fragmented_table(&url, "compactme", 12)
+        .await
+        .expect("seed a fragmented table");
+    let handler = CompactHandler::new(url).with_fallback(fallback());
+    let recorder = Recorder::default();
+
+    let request = RunDetectionRequest {
+        request_id: "detect-1".to_string(),
+        job_type: JOB_TYPE.to_string(),
+        worker_config_values: int_config("min_fragments", 4),
+        ..Default::default()
+    };
+    handler
+        .detect(&request, &recorder)
+        .await
+        .expect("detection failed");
+
+    let proposals = recorder.proposals.lock().unwrap().clone();
+    assert!(
+        !proposals.is_empty(),
+        "expected a proposal for the fragmented table"
+    );
+    let proposal = proposals
+        .iter()
+        .find(|p| p.summary.contains(encoded.as_str()))
+        .expect("no proposal for the seeded table");
+
+    // Detection opened the dataset to decide, so it reports what it saw. This is
+    // the only description of a Lance table anything outside the format can give.
+    let observations = recorder.observations.lock().unwrap().clone();
+    let observed = observations
+        .iter()
+        .find(|o| o.object_id.last().map(String::as_str) == Some("compactme"))
+        .expect("detection reported no observation for the seeded table");
+    assert_eq!(observed.format, "LANCE");
+    for attribute in ["fragments", "rows", "versions", "schema"] {
+        assert!(
+            observed.attributes.contains_key(attribute),
+            "observation is missing {attribute}: {:?}",
+            observed.attributes.keys().collect::<Vec<_>>()
+        );
+    }
+
+    let execute = ExecuteJobRequest {
+        request_id: "execute-1".to_string(),
+        job: Some(JobSpec {
+            job_id: "job-1".to_string(),
+            job_type: JOB_TYPE.to_string(),
+            parameters: proposal.parameters.clone(),
+            ..Default::default()
+        }),
+        worker_config_values: int_config("target_rows_per_fragment", 1_048_576),
+        ..Default::default()
+    };
+    handler
+        .execute(&execute, &recorder)
+        .await
+        .expect("execution failed");
+
+    let completed = recorder.completed.lock().unwrap().clone();
+    let result = completed.first().expect("no completion reported");
+    assert!(
+        result.success,
+        "compaction reported failure: {}",
+        result.error_message
+    );
+    let summary = result
+        .result
+        .as_ref()
+        .map(|r| r.summary.clone())
+        .unwrap_or_default();
+    assert!(
+        summary.contains("became"),
+        "completion carried no fragment counts: {summary}"
+    );
+    eprintln!("compaction result: {summary}");
+}
+
+/// A table with more versions than the floor is proposed, and running the job
+/// reports what it removed. The compaction test above leaves one behind.
+#[tokio::test]
+async fn cleans_up_old_versions() {
+    let _gateway = GATEWAY.lock().await;
+    let Some(url) = namespace_url() else {
+        eprintln!("WEED_LANCE_NAMESPACE is unset, skipping");
+        return;
+    };
+    let encoded = seed_table(&url, "cleanme", 6, 4, false)
+        .await
+        .expect("seed a table with versions to clean");
+
+    let handler = CleanupVersionsHandler::new(url).with_fallback(fallback());
+    let recorder = Recorder::default();
+
+    let request = RunDetectionRequest {
+        request_id: "detect-cleanup".to_string(),
+        job_type: "lance_cleanup_versions".to_string(),
+        worker_config_values: int_config("min_versions_to_keep", 2),
+        ..Default::default()
+    };
+    handler
+        .detect(&request, &recorder)
+        .await
+        .expect("detection failed");
+    let proposals = recorder.proposals.lock().unwrap().clone();
+    let proposal = proposals
+        .iter()
+        .find(|p| p.summary.contains(encoded.as_str()))
+        .cloned()
+        .expect("no cleanup proposal for the seeded table");
+
+    // Retain nothing, so every version outside the current one is fair game and
+    // the job has something to report rather than a no-op.
+    let execute = ExecuteJobRequest {
+        request_id: "execute-cleanup".to_string(),
+        job: Some(JobSpec {
+            job_id: "job-cleanup".to_string(),
+            job_type: "lance_cleanup_versions".to_string(),
+            parameters: proposal.parameters.clone(),
+            ..Default::default()
+        }),
+        worker_config_values: int_config("retain_hours", 0),
+        ..Default::default()
+    };
+    handler
+        .execute(&execute, &recorder)
+        .await
+        .expect("cleanup failed");
+
+    let completed = recorder.completed.lock().unwrap().clone();
+    let result = completed.last().expect("no completion reported");
+    assert!(
+        result.success,
+        "cleanup reported failure: {}",
+        result.error_message
+    );
+    eprintln!(
+        "cleanup result: {}",
+        result.result.as_ref().unwrap().summary
+    );
+}
+
+/// A table with no indices has nothing to optimize, so detection proposes
+/// nothing rather than queueing work that would do nothing.
+#[tokio::test]
+async fn skips_tables_without_indices() {
+    let _gateway = GATEWAY.lock().await;
+    let Some(url) = namespace_url() else {
+        eprintln!("WEED_LANCE_NAMESPACE is unset, skipping");
+        return;
+    };
+    let encoded = seed_table(&url, "noindex", 2, 8, false)
+        .await
+        .expect("seed a table without an index");
+
+    let handler = OptimizeIndicesHandler::new(url).with_fallback(fallback());
+    let recorder = Recorder::default();
+
+    let request = RunDetectionRequest {
+        request_id: "detect-indices".to_string(),
+        job_type: "lance_optimize_indices".to_string(),
+        worker_config_values: int_config("max_unindexed_rows", 1),
+        ..Default::default()
+    };
+    handler
+        .detect(&request, &recorder)
+        .await
+        .expect("detection failed");
+    // Judge this table only: the catalog holds every other test's tables too,
+    // and an indexed one with uncovered rows is supposed to be proposed.
+    assert!(
+        !recorder
+            .proposals
+            .lock()
+            .unwrap()
+            .iter()
+            .any(|p| p.summary.contains(encoded.as_str())),
+        "a table with no indices must not be proposed for reindexing"
+    );
+}
+
+/// The job with no Iceberg equivalent: rows appended after an index was built
+/// are invisible to a search of it until this runs. Needs a table with an index
+/// and rows outside it, which `indexed.py` in the scratchpad seeds.
+#[tokio::test]
+async fn reindexes_rows_an_index_does_not_cover() {
+    let _gateway = GATEWAY.lock().await;
+    let Some(url) = namespace_url() else {
+        eprintln!("WEED_LANCE_NAMESPACE is unset, skipping");
+        return;
+    };
+    let encoded = seed_table(&url, "reindexme", 2, 512, true)
+        .await
+        .expect("seed an indexed table with uncovered rows");
+    let handler = OptimizeIndicesHandler::new(url).with_fallback(fallback());
+    let recorder = Recorder::default();
+
+    let request = RunDetectionRequest {
+        request_id: "detect-reindex".to_string(),
+        job_type: "lance_optimize_indices".to_string(),
+        worker_config_values: int_config("max_unindexed_rows", 100),
+        ..Default::default()
+    };
+    handler
+        .detect(&request, &recorder)
+        .await
+        .expect("detection failed");
+    let proposals = recorder.proposals.lock().unwrap().clone();
+    let proposal = proposals
+        .iter()
+        .find(|p| p.summary.contains(encoded.as_str()))
+        .cloned()
+        .expect("the seeded indexed table was not proposed");
+
+    let execute = ExecuteJobRequest {
+        request_id: "execute-reindex".to_string(),
+        job: Some(JobSpec {
+            job_id: "job-reindex".to_string(),
+            job_type: "lance_optimize_indices".to_string(),
+            parameters: proposal.parameters.clone(),
+            ..Default::default()
+        }),
+        ..Default::default()
+    };
+    handler
+        .execute(&execute, &recorder)
+        .await
+        .expect("reindex failed");
+
+    let completed = recorder.completed.lock().unwrap().clone();
+    let result = completed.last().expect("no completion reported");
+    assert!(
+        result.success,
+        "reindex reported failure: {}",
+        result.error_message
+    );
+    let output = &result.result.as_ref().unwrap().output_values;
+    let after = match output
+        .get("unindexed_rows_after")
+        .and_then(|v| v.kind.as_ref())
+    {
+        Some(Kind::Int64Value(value)) => *value,
+        other => panic!("no unindexed_rows_after in {other:?}"),
+    };
+    assert_eq!(
+        after, 0,
+        "rows are still outside the index after optimizing"
+    );
+    eprintln!(
+        "reindex result: {}",
+        result.result.as_ref().unwrap().summary
+    );
+}
+
+/// The UI's whole reason for asking a worker: admin cannot read a Lance table,
+/// so the rows have to come back already rendered.
+#[tokio::test]
+async fn previews_rows_of_a_table() {
+    let _gateway = GATEWAY.lock().await;
+    let Some(url) = namespace_url() else {
+        eprintln!("set WEED_LANCE_NAMESPACE to run this test");
+        return;
+    };
+    seed_table(&url, "previewme", 2, 3, false)
+        .await
+        .expect("seed a table to preview");
+
+    let provider = LancePreview::new(url, fallback());
+    let id = vec!["vec".to_string(), "ml".to_string(), "previewme".to_string()];
+    let preview = provider.preview(&id, 4).await.expect("preview the table");
+
+    assert_eq!(preview.columns, vec!["id".to_string(), "vec".to_string()]);
+    assert_eq!(preview.total_rows, 6, "total is the table, not the sample");
+    assert_eq!(preview.rows.len(), 4, "row_limit bounds the sample");
+    assert!(
+        preview.rows[0][1].starts_with('['),
+        "a vector column should render as a list, got {:?}",
+        preview.rows[0][1]
+    );
+}
+
+/// The claim that removed managed versioning is not "one writer wins the
+/// conditional PUT" - that is only the mechanism. It is that concurrent writers
+/// lose nothing: the loser sees the conflict, rebases, and commits again. Eight
+/// writers appending at once must leave all eight batches in the table.
+#[tokio::test]
+async fn concurrent_writers_keep_every_commit() {
+    let _gateway = GATEWAY.lock().await;
+    let Some(url) = namespace_url() else {
+        eprintln!("set WEED_LANCE_NAMESPACE to run this test");
+        return;
+    };
+    const WRITERS: i64 = 8;
+    const ROWS_EACH: i64 = 4;
+
+    seed_table(&url, "racers", 1, ROWS_EACH as usize, false)
+        .await
+        .expect("seed the table the writers will append to");
+
+    let client = NamespaceClient::new(url.clone());
+    let id = vec!["vec".to_string(), "ml".to_string(), "racers".to_string()];
+    let description = client.describe_table(&id).await.expect("describe");
+    let mut options = description.storage_options.clone();
+    options.extend(fallback());
+
+    let writes = (0..WRITERS).map(|writer| {
+        let location = description.location.clone();
+        let options = options.clone();
+        tokio::spawn(
+            async move { append_rows(&location, &options, writer * 1000, ROWS_EACH).await },
+        )
+    });
+    for (writer, handle) in writes.enumerate() {
+        handle
+            .await
+            .expect("writer panicked")
+            .unwrap_or_else(|err| panic!("writer {writer} failed to commit: {err:#}"));
+    }
+
+    let table = weed_lance_worker::dataset::open(&client, &id, &fallback())
+        .await
+        .expect("reopen the table");
+    let rows = table.dataset.count_rows(None).await.expect("count rows");
+    let expected = (ROWS_EACH + WRITERS * ROWS_EACH) as usize;
+    assert_eq!(
+        rows, expected,
+        "concurrent commits lost data: {rows} rows, want {expected}"
+    );
+}
+
+/// Appends one batch to an existing dataset, the way an independent writer would.
+async fn append_rows(
+    location: &str,
+    options: &HashMap<String, String>,
+    first_id: i64,
+    rows: i64,
+) -> Result<()> {
+    use arrow_array::{
+        FixedSizeListArray, Float32Array, Int64Array, RecordBatch, RecordBatchIterator,
+    };
+    use arrow_schema::{DataType, Field, Schema};
+    use lance::dataset::{Dataset, WriteMode, WriteParams};
+    use lance::io::{ObjectStoreParams, StorageOptionsAccessor};
+    use std::sync::Arc;
+
+    const DIM: i32 = 16;
+    let schema = Arc::new(Schema::new(vec![
+        Field::new("id", DataType::Int64, false),
+        Field::new(
+            "vec",
+            DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), DIM),
+            false,
+        ),
+    ]));
+    let ids: Vec<i64> = (0..rows).map(|r| first_id + r).collect();
+    let values: Vec<f32> = ids
+        .iter()
+        .flat_map(|id| (0..DIM).map(move |d| (*id as f32) + d as f32))
+        .collect();
+    let vectors = FixedSizeListArray::new(
+        Arc::new(Field::new("item", DataType::Float32, true)),
+        DIM,
+        Arc::new(Float32Array::from(values)),
+        None,
+    );
+    let batch = RecordBatch::try_new(
+        schema.clone(),
+        vec![Arc::new(Int64Array::from(ids)), Arc::new(vectors)],
+    )?;
+    let params = WriteParams {
+        mode: WriteMode::Append,
+        store_params: Some(ObjectStoreParams {
+            storage_options_accessor: Some(Arc::new(StorageOptionsAccessor::with_static_options(
+                options.clone(),
+            ))),
+            ..Default::default()
+        }),
+        ..Default::default()
+    };
+    Dataset::write(
+        RecordBatchIterator::new(vec![Ok(batch)], schema.clone()),
+        location,
+        Some(params),
+    )
+    .await?;
+    Ok(())
+}

seaweed-worker/crates/lance/tests/lifecycle.rs

@@ -0,0 +1,123 @@
+//! Maintains one table someone else wrote, and nothing else.
+//!
+//! The Go suite in test/s3tables/lifecycle drives a table through its whole
+//! life - declared through the namespace, written by a Lance client, maintained,
+//! read again, dropped - and this is the maintenance step of it, so that step
+//! goes through the handlers a deployed worker runs rather than through the two
+//! lance calls they wrap. The table to work on comes in as WEED_LANCE_TABLE
+//! because the table is the Go test's, seeded and checked there.
+//!
+//! Skipped, like the rest of this crate's integration tests, unless
+//! WEED_LANCE_NAMESPACE names a live gateway.
+
+use std::collections::HashMap;
+
+use seaweed_worker_core::JobHandler;
+use seaweed_worker_core::pb::{ConfigValue, ExecuteJobRequest, JobSpec, RunDetectionRequest};
+use weed_lance_worker::jobs::cleanup::{self, CleanupVersionsHandler};
+use weed_lance_worker::jobs::compact::{CompactHandler, JOB_TYPE as COMPACT_JOB_TYPE};
+
+mod common;
+use common::{Recorder, fallback, int_config, namespace_url};
+
+fn table() -> Option<String> {
+    std::env::var("WEED_LANCE_TABLE")
+        .ok()
+        .filter(|s| !s.is_empty())
+}
+
+/// Compacts the named table's fragments, then drops the versions compaction
+/// left behind. Both go through the handler's own detect-then-execute path: a
+/// proposal the worker would not have made is not one worth running.
+#[tokio::test]
+async fn maintains_the_named_table() {
+    let (Some(url), Some(name)) = (namespace_url(), table()) else {
+        eprintln!("WEED_LANCE_NAMESPACE or WEED_LANCE_TABLE is unset, skipping");
+        return;
+    };
+
+    // A table written as a fragment per append is the case this exists for, so
+    // anything above one fragment is worth merging here.
+    let mut config = int_config("min_fragments", 2);
+    config.extend(int_config("target_rows_per_fragment", 1_048_576));
+    let compact = CompactHandler::new(url.clone()).with_fallback(fallback());
+    run(&compact, COMPACT_JOB_TYPE, &name, config, true).await;
+
+    // Keep the current version and retain nothing else, so the versions
+    // compaction superseded are actually removed rather than counted.
+    let mut config = int_config("min_versions_to_keep", 1);
+    config.extend(int_config("retain_hours", 0));
+    let cleanup = CleanupVersionsHandler::new(url).with_fallback(fallback());
+    run(&cleanup, cleanup::JOB_TYPE, &name, config, true).await;
+}
+
+/// Detects, finds the proposal for `name`, and executes it. The config carries
+/// every key both halves read; each ignores what it does not know.
+async fn run<H: JobHandler>(
+    handler: &H,
+    job_type: &str,
+    name: &str,
+    config: HashMap<String, ConfigValue>,
+    required: bool,
+) {
+    let recorder = Recorder::default();
+    handler
+        .detect(
+            &RunDetectionRequest {
+                request_id: format!("detect-{job_type}"),
+                job_type: job_type.to_string(),
+                worker_config_values: config.clone(),
+                ..Default::default()
+            },
+            &recorder,
+        )
+        .await
+        .unwrap_or_else(|err| panic!("{job_type} detection failed: {err}"));
+
+    let proposals = recorder.proposals.lock().unwrap().clone();
+    let Some(proposal) = proposals.iter().find(|p| p.summary.contains(name)) else {
+        assert!(
+            !required,
+            "{job_type} proposed nothing for {name}, out of {} proposals",
+            proposals.len()
+        );
+        eprintln!("{job_type}: nothing to do for {name}");
+        return;
+    };
+
+    handler
+        .execute(
+            &ExecuteJobRequest {
+                request_id: format!("execute-{job_type}"),
+                job: Some(JobSpec {
+                    job_id: format!("job-{job_type}"),
+                    job_type: job_type.to_string(),
+                    parameters: proposal.parameters.clone(),
+                    ..Default::default()
+                }),
+                worker_config_values: config,
+                ..Default::default()
+            },
+            &recorder,
+        )
+        .await
+        .unwrap_or_else(|err| panic!("{job_type} execution failed: {err}"));
+
+    let completed = recorder.completed.lock().unwrap().clone();
+    let result = completed
+        .last()
+        .unwrap_or_else(|| panic!("{job_type} reported no completion"));
+    assert!(
+        result.success,
+        "{job_type} reported failure: {}",
+        result.error_message
+    );
+    eprintln!(
+        "{job_type}: {}",
+        result
+            .result
+            .as_ref()
+            .map(|r| r.summary.clone())
+            .unwrap_or_default()
+    );
+}

seaweed-worker/crates/lance/tests/sort.rs

@@ -0,0 +1,504 @@
+//! Drives the sort handler against a live namespace.
+//!
+//! Skipped unless WEED_LANCE_NAMESPACE names one, like the other integration
+//! tests here: the job rewrites every fragment of a real table and commits, and
+//! the commit is the half worth testing.
+
+use anyhow::Result;
+use seaweed_worker_core::JobHandler;
+use seaweed_worker_core::pb::{
+    ConfigValue, ExecuteJobRequest, JobSpec, RunDetectionRequest, config_value::Kind,
+};
+use weed_lance_worker::catalog::NamespaceClient;
+use weed_lance_worker::jobs::sort::{JOB_TYPE, SortHandler};
+
+mod common;
+use common::{Recorder, fallback, namespace_url};
+
+/// One live gateway and one shared catalog, and `list_all_tables` sweeps
+/// everything, so these tests take a lock the way the compaction ones do.
+static GATEWAY: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(());
+
+fn config(values: &[(&str, ConfigValue)]) -> std::collections::HashMap<String, ConfigValue> {
+    values
+        .iter()
+        .map(|(name, value)| (name.to_string(), value.clone()))
+        .collect()
+}
+
+fn int(value: i64) -> ConfigValue {
+    ConfigValue {
+        kind: Some(Kind::Int64Value(value)),
+    }
+}
+
+fn text(value: &str) -> ConfigValue {
+    ConfigValue {
+        kind: Some(Kind::StringValue(value.to_string())),
+    }
+}
+
+/// Declares a table and writes `batches` appends whose ids descend, so the rows
+/// land in an order no sort would produce. `declared` is written into the
+/// dataset's own configuration, which is where a table says how it wants to be
+/// sorted.
+async fn seed_unsorted_table(
+    url: &str,
+    name: &str,
+    batches: i64,
+    rows_each: i64,
+    declared: Option<&str>,
+) -> Result<String> {
+    use arrow_array::{Int64Array, RecordBatch, RecordBatchIterator};
+    use arrow_schema::{DataType, Field, Schema};
+    use lance::dataset::{Dataset, WriteMode, WriteParams};
+    use lance::io::{ObjectStoreParams, StorageOptionsAccessor};
+    use std::sync::Arc;
+
+    let http = reqwest::Client::new();
+    for parent in ["vec", "vec$ml"] {
+        http.post(format!("{url}/v1/namespace/{parent}/create"))
+            .json(&serde_json::json!({"mode": "EXIST_OK"}))
+            .send()
+            .await?
+            .error_for_status()?;
+    }
+    let encoded = format!("vec$ml${name}");
+    http.post(format!("{url}/v1/table/{encoded}/declare"))
+        .json(&serde_json::json!({}))
+        .send()
+        .await?
+        .error_for_status()?;
+
+    let client = NamespaceClient::new(url.to_string());
+    let id = vec!["vec".to_string(), "ml".to_string(), name.to_string()];
+    let description = client.describe_table(&id).await?;
+    let mut options = description.storage_options.clone();
+    options.extend(fallback());
+
+    let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int64, false)]));
+    let total = batches * rows_each;
+    let mut dataset = None;
+    for batch_index in 0..batches {
+        // Ids descend across and within the batches, so the physical order is
+        // the reverse of the order the table declares.
+        let ids: Vec<i64> = (0..rows_each)
+            .map(|row| total - 1 - (batch_index * rows_each + row))
+            .collect();
+        let batch = RecordBatch::try_new(schema.clone(), vec![Arc::new(Int64Array::from(ids))])?;
+        let params = WriteParams {
+            mode: if batch_index == 0 {
+                WriteMode::Overwrite
+            } else {
+                WriteMode::Append
+            },
+            store_params: Some(ObjectStoreParams {
+                storage_options_accessor: Some(Arc::new(
+                    StorageOptionsAccessor::with_static_options(options.clone()),
+                )),
+                ..Default::default()
+            }),
+            ..Default::default()
+        };
+        dataset = Some(
+            Dataset::write(
+                RecordBatchIterator::new(vec![Ok(batch)], schema.clone()),
+                description.location.as_str(),
+                Some(params),
+            )
+            .await?,
+        );
+    }
+
+    if let Some(declared) = declared {
+        let mut dataset = dataset.expect("at least one batch was written");
+        dataset
+            .update_config([("seaweedfs.sort.fields", declared)])
+            .await?;
+    }
+    Ok(encoded)
+}
+
+/// Reads the ids back in the order they are stored, which is what a sort is
+/// supposed to change.
+async fn stored_ids(url: &str, name: &str) -> Result<Vec<i64>> {
+    use arrow_array::Int64Array;
+    use futures::TryStreamExt;
+
+    let client = NamespaceClient::new(url.to_string());
+    let id = vec!["vec".to_string(), "ml".to_string(), name.to_string()];
+    let table = weed_lance_worker::dataset::open(&client, &id, &fallback()).await?;
+    let stream = table.dataset.scan().try_into_stream().await?;
+    let batches: Vec<_> = stream.try_collect().await?;
+    let mut ids = Vec::new();
+    for batch in batches {
+        let column = batch
+            .column_by_name("id")
+            .expect("the seeded table has an id column")
+            .as_any()
+            .downcast_ref::<Int64Array>()
+            .expect("id is an Int64 column")
+            .clone();
+        ids.extend(column.values().iter().copied());
+    }
+    Ok(ids)
+}
+
+/// The whole loop: a table declaring an order its rows do not follow is
+/// proposed, sorting it stores the rows in that order, and the marker the same
+/// commit carries is what stops the next sweep proposing it again.
+#[tokio::test]
+async fn sorts_a_table_and_stops_proposing_it() {
+    let _gateway = GATEWAY.lock().await;
+    let Some(url) = namespace_url() else {
+        eprintln!("WEED_LANCE_NAMESPACE is unset, skipping");
+        return;
+    };
+    let encoded = seed_unsorted_table(&url, "sortme", 4, 250, Some("id asc"))
+        .await
+        .expect("seed an unsorted table");
+
+    let before = stored_ids(&url, "sortme").await.expect("read the ids back");
+    assert!(
+        before.windows(2).any(|pair| pair[0] > pair[1]),
+        "the seeded table was already sorted, so the test proves nothing"
+    );
+
+    let handler = SortHandler::new(url.clone()).with_fallback(fallback());
+    let recorder = Recorder::default();
+    let detection = RunDetectionRequest {
+        request_id: "detect-sort".to_string(),
+        job_type: JOB_TYPE.to_string(),
+        worker_config_values: config(&[("min_unsorted_rows", int(1))]),
+        ..Default::default()
+    };
+    handler
+        .detect(&detection, &recorder)
+        .await
+        .expect("detection failed");
+
+    let proposal = recorder
+        .proposals
+        .lock()
+        .unwrap()
+        .iter()
+        .find(|p| p.summary.contains(encoded.as_str()))
+        .cloned()
+        .expect("the unsorted table was not proposed");
+    assert!(
+        proposal.summary.contains("id asc"),
+        "the proposal does not say what order it would use: {}",
+        proposal.summary
+    );
+
+    let execute = ExecuteJobRequest {
+        request_id: "execute-sort".to_string(),
+        job: Some(JobSpec {
+            job_id: "job-sort".to_string(),
+            job_type: JOB_TYPE.to_string(),
+            parameters: proposal.parameters.clone(),
+            ..Default::default()
+        }),
+        worker_config_values: config(&[
+            ("memory_budget_mb", int(64)),
+            ("max_rows_per_file", int(1024)),
+        ]),
+        ..Default::default()
+    };
+    handler
+        .execute(&execute, &recorder)
+        .await
+        .expect("sorting failed");
+
+    let completed = recorder.completed.lock().unwrap().clone();
+    let result = completed.last().expect("no completion reported");
+    assert!(
+        result.success,
+        "the sort reported failure: {}",
+        result.error_message
+    );
+
+    let after = stored_ids(&url, "sortme")
+        .await
+        .expect("read the sorted ids back");
+    assert_eq!(after.len(), before.len(), "the sort lost or invented rows");
+    assert!(
+        after.windows(2).all(|pair| pair[0] <= pair[1]),
+        "the rows are not stored in ascending id order"
+    );
+
+    // The marker rode in the same commit as the data, so a second sweep finds
+    // the table up to date without anything else having written it.
+    let second = Recorder::default();
+    handler
+        .detect(&detection, &second)
+        .await
+        .expect("second detection failed");
+    assert!(
+        !second
+            .proposals
+            .lock()
+            .unwrap()
+            .iter()
+            .any(|p| p.summary.contains(encoded.as_str())),
+        "a table that was just sorted must not be proposed again"
+    );
+    eprintln!("sort result: {}", result.result.as_ref().unwrap().summary);
+}
+
+/// A table that declares no order, with a worker configuring none either, is
+/// not something to rewrite on a guess.
+#[tokio::test]
+async fn leaves_a_table_that_declares_no_order_alone() {
+    let _gateway = GATEWAY.lock().await;
+    let Some(url) = namespace_url() else {
+        eprintln!("WEED_LANCE_NAMESPACE is unset, skipping");
+        return;
+    };
+    let encoded = seed_unsorted_table(&url, "unsorted", 2, 16, None)
+        .await
+        .expect("seed a table with no declared order");
+
+    let handler = SortHandler::new(url.clone()).with_fallback(fallback());
+    let recorder = Recorder::default();
+    handler
+        .detect(
+            &RunDetectionRequest {
+                request_id: "detect-none".to_string(),
+                job_type: JOB_TYPE.to_string(),
+                worker_config_values: config(&[
+                    ("min_unsorted_rows", int(1)),
+                    ("sort_fields", text("")),
+                ]),
+                ..Default::default()
+            },
+            &recorder,
+        )
+        .await
+        .expect("detection failed");
+
+    assert!(
+        !recorder
+            .proposals
+            .lock()
+            .unwrap()
+            .iter()
+            .any(|p| p.summary.contains(encoded.as_str())),
+        "a table with no order named anywhere must not be proposed"
+    );
+}
+
+/// The case a row-count threshold cannot see: the table replaced by a write of
+/// its own, growing by far less than `min_unsorted_rows`. Every row moved, so
+/// the table is unsorted, and detection has to say so however small the delta.
+#[tokio::test]
+async fn proposes_a_replacement_that_barely_changes_the_row_count() {
+    let _gateway = GATEWAY.lock().await;
+    let Some(url) = namespace_url() else {
+        eprintln!("WEED_LANCE_NAMESPACE is unset, skipping");
+        return;
+    };
+    let encoded = seed_unsorted_table(&url, "replaceme", 4, 250, Some("id asc"))
+        .await
+        .expect("seed an unsorted table");
+
+    let handler = SortHandler::new(url.clone()).with_fallback(fallback());
+    // A threshold no realistic append would cross, so nothing but the identity
+    // of the data can be what proposes this table.
+    let detection = RunDetectionRequest {
+        request_id: "detect-replace".to_string(),
+        job_type: JOB_TYPE.to_string(),
+        worker_config_values: config(&[("min_unsorted_rows", int(1_000_000))]),
+        ..Default::default()
+    };
+
+    let first = Recorder::default();
+    handler.detect(&detection, &first).await.expect("detection");
+    let proposal = first
+        .proposals
+        .lock()
+        .unwrap()
+        .iter()
+        .find(|p| p.summary.contains(encoded.as_str()))
+        .cloned()
+        .expect("a never-sorted table was not proposed");
+
+    handler
+        .execute(
+            &ExecuteJobRequest {
+                request_id: "execute-replace".to_string(),
+                job: Some(JobSpec {
+                    job_id: "job-replace".to_string(),
+                    job_type: JOB_TYPE.to_string(),
+                    parameters: proposal.parameters.clone(),
+                    ..Default::default()
+                }),
+                worker_config_values: config(&[("max_rows_per_file", int(1024))]),
+                ..Default::default()
+            },
+            &first,
+        )
+        .await
+        .expect("sorting failed");
+
+    // Sorted, and left alone by the next sweep.
+    let after_sort = Recorder::default();
+    handler
+        .detect(&detection, &after_sort)
+        .await
+        .expect("detection");
+    assert!(
+        !after_sort
+            .proposals
+            .lock()
+            .unwrap()
+            .iter()
+            .any(|p| p.summary.contains(encoded.as_str())),
+        "a table that was just sorted must not be proposed again"
+    );
+
+    // Now replace the data: 1002 rows where there were 1000, shuffled again.
+    seed_unsorted_table(&url, "replaceme", 3, 334, Some("id asc"))
+        .await
+        .expect("replace the table's data");
+
+    let after_replace = Recorder::default();
+    handler
+        .detect(&detection, &after_replace)
+        .await
+        .expect("detection");
+    assert!(
+        after_replace
+            .proposals
+            .lock()
+            .unwrap()
+            .iter()
+            .any(|p| p.summary.contains(encoded.as_str())),
+        "a replaced table must be proposed even when the row count barely moved"
+    );
+}
+
+/// Deletions must not hide appended rows. This is the arithmetic against a real
+/// manifest rather than a hand-built one: the sorted fragments keep their files
+/// and lose most of their rows, a smaller batch is appended after them, and the
+/// table has to be proposed *because rows were appended* — not because it looks
+/// replaced, and not left alone because it now holds fewer rows than when it
+/// was sorted.
+#[tokio::test]
+async fn deletions_do_not_hide_appended_rows() {
+    let _gateway = GATEWAY.lock().await;
+    let Some(url) = namespace_url() else {
+        eprintln!("WEED_LANCE_NAMESPACE is unset, skipping");
+        return;
+    };
+    let encoded = seed_unsorted_table(&url, "deleteme", 4, 250, Some("id asc"))
+        .await
+        .expect("seed an unsorted table");
+
+    let handler = SortHandler::new(url.clone()).with_fallback(fallback());
+    let detection = RunDetectionRequest {
+        request_id: "detect-delete".to_string(),
+        job_type: JOB_TYPE.to_string(),
+        worker_config_values: config(&[("min_unsorted_rows", int(100))]),
+        ..Default::default()
+    };
+
+    let recorder = Recorder::default();
+    handler
+        .detect(&detection, &recorder)
+        .await
+        .expect("detection");
+    let proposal = recorder
+        .proposals
+        .lock()
+        .unwrap()
+        .iter()
+        .find(|p| p.summary.contains(encoded.as_str()))
+        .cloned()
+        .expect("a never-sorted table was not proposed");
+    handler
+        .execute(
+            &ExecuteJobRequest {
+                request_id: "execute-delete".to_string(),
+                job: Some(JobSpec {
+                    job_id: "job-delete".to_string(),
+                    job_type: JOB_TYPE.to_string(),
+                    parameters: proposal.parameters.clone(),
+                    ..Default::default()
+                }),
+                worker_config_values: config(&[("max_rows_per_file", int(1024))]),
+                ..Default::default()
+            },
+            &recorder,
+        )
+        .await
+        .expect("sorting failed");
+
+    // Delete 900 of the 1000 sorted rows, then append 200 unsorted ones. The
+    // table is smaller than when it was sorted, so net growth reads as zero.
+    let client = NamespaceClient::new(url.clone());
+    let id = vec!["vec".to_string(), "ml".to_string(), "deleteme".to_string()];
+    let mut table = weed_lance_worker::dataset::open(&client, &id, &fallback())
+        .await
+        .expect("reopen the sorted table");
+    table
+        .dataset
+        .delete("id < 900")
+        .await
+        .expect("delete most of the sorted rows");
+    append_ids(&url, "deleteme", 5_000, 200)
+        .await
+        .expect("append rows after the sorted ones");
+
+    let after = Recorder::default();
+    handler.detect(&detection, &after).await.expect("detection");
+    let proposal = after
+        .proposals
+        .lock()
+        .unwrap()
+        .iter()
+        .find(|p| p.summary.contains(encoded.as_str()))
+        .cloned()
+        .expect("a table with 200 unsorted rows appended was not proposed");
+    assert!(
+        proposal.detail.contains("200 rows appended"),
+        "expected the appended rows to be counted, got {:?}",
+        proposal.detail
+    );
+}
+
+/// Appends one batch of ascending ids to an existing table.
+async fn append_ids(url: &str, name: &str, first_id: i64, rows: i64) -> Result<()> {
+    use arrow_array::{Int64Array, RecordBatch, RecordBatchIterator};
+    use arrow_schema::{DataType, Field, Schema};
+    use lance::dataset::{Dataset, WriteMode, WriteParams};
+    use lance::io::{ObjectStoreParams, StorageOptionsAccessor};
+    use std::sync::Arc;
+
+    let client = NamespaceClient::new(url.to_string());
+    let id = vec!["vec".to_string(), "ml".to_string(), name.to_string()];
+    let description = client.describe_table(&id).await?;
+    let mut options = description.storage_options.clone();
+    options.extend(fallback());
+
+    let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int64, false)]));
+    let ids: Vec<i64> = (0..rows).map(|row| first_id + row).collect();
+    let batch = RecordBatch::try_new(schema.clone(), vec![Arc::new(Int64Array::from(ids))])?;
+    let params = WriteParams {
+        mode: WriteMode::Append,
+        store_params: Some(ObjectStoreParams {
+            storage_options_accessor: Some(Arc::new(StorageOptionsAccessor::with_static_options(
+                options,
+            ))),
+            ..Default::default()
+        }),
+        ..Default::default()
+    };
+    Dataset::write(
+        RecordBatchIterator::new(vec![Ok(batch)], schema.clone()),
+        description.location.as_str(),
+        Some(params),
+    )
+    .await?;
+    Ok(())
+}