snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)

seaweed-worker/crates/lance/tests/common/mod.rs

@@ -0,0 +1,97 @@
+//! What every integration test in this crate needs to drive a handler: a sender
+//! that keeps what the handler sent, and the two settings that make a handler
+//! usable against a test gateway.
+//!
+//! Each test binary compiles this module on its own and uses part of it.
+#![allow(dead_code)]
+
+use std::collections::HashMap;
+use std::sync::Mutex;
+
+use anyhow::Result;
+use seaweed_worker_core::pb::{
+    ActivityEvent, ConfigValue, DetectionComplete, DetectionProposals, JobCompleted,
+    JobProgressUpdate, JobProposal, ObjectObservation, WorkerObservations, config_value::Kind,
+};
+use seaweed_worker_core::{DetectionSender, ExecutionSender};
+
+/// Keeps everything a handler reports, so a test can assert on it afterwards.
+#[derive(Default)]
+pub struct Recorder {
+    pub proposals: Mutex<Vec<JobProposal>>,
+    pub observations: Mutex<Vec<ObjectObservation>>,
+    pub completed: Mutex<Vec<JobCompleted>>,
+}
+
+impl DetectionSender for Recorder {
+    fn send_proposals(&self, proposals: DetectionProposals) -> Result<()> {
+        self.proposals.lock().unwrap().extend(proposals.proposals);
+        Ok(())
+    }
+    fn send_complete(&self, _complete: DetectionComplete) -> Result<()> {
+        Ok(())
+    }
+    fn send_activity(&self, _activity: ActivityEvent) -> Result<()> {
+        Ok(())
+    }
+    fn send_observations(&self, observations: WorkerObservations) -> Result<()> {
+        self.observations
+            .lock()
+            .unwrap()
+            .extend(observations.observations);
+        Ok(())
+    }
+}
+
+impl ExecutionSender for Recorder {
+    fn send_progress(&self, _progress: JobProgressUpdate) -> Result<()> {
+        Ok(())
+    }
+    fn send_completed(&self, completed: JobCompleted) -> Result<()> {
+        self.completed.lock().unwrap().push(completed);
+        Ok(())
+    }
+}
+
+/// The gateway to run against, or nothing, in which case the caller skips:
+/// these handlers rewrite real files, and there is nothing to learn from them
+/// against a fake.
+pub fn namespace_url() -> Option<String> {
+    std::env::var("WEED_LANCE_NAMESPACE")
+        .ok()
+        .filter(|s| !s.is_empty())
+}
+
+pub fn int_config(name: &str, value: i64) -> HashMap<String, ConfigValue> {
+    let mut values = HashMap::new();
+    values.insert(
+        name.to_string(),
+        ConfigValue {
+            kind: Some(Kind::Int64Value(value)),
+        },
+    );
+    values
+}
+
+/// What to reach the data with where the namespace vends nothing of its own -
+/// a gateway without STS vends no credentials, and one bound to a wildcard
+/// address vends no endpoint either. Taken from the environment so a harness
+/// can point these tests at a gateway that does check what it is given.
+pub fn fallback() -> weed_lance_worker::dataset::FallbackOptions {
+    let mut options = weed_lance_worker::dataset::FallbackOptions::new();
+    let from_env =
+        |name: &str, default: &str| std::env::var(name).unwrap_or_else(|_| default.to_string());
+    options.insert(
+        "aws_access_key_id".to_string(),
+        from_env("AWS_ACCESS_KEY_ID", "any"),
+    );
+    options.insert(
+        "aws_secret_access_key".to_string(),
+        from_env("AWS_SECRET_ACCESS_KEY", "any"),
+    );
+    if let Ok(endpoint) = std::env::var("AWS_ENDPOINT_URL") {
+        options.insert("aws_endpoint".to_string(), endpoint);
+        options.insert("allow_http".to_string(), "true".to_string());
+    }
+    options
+}

seaweed-worker/crates/lance/tests/compaction.rs

@@ -0,0 +1,540 @@
+//! Drives the compaction handler against a live namespace.
+//!
+//! Skipped unless WEED_LANCE_NAMESPACE names one, the way the Go integration
+//! tests skip without Docker: compaction rewrites real files, and there is
+//! nothing to learn from it against a fake.
+
+use std::collections::HashMap;
+
+use anyhow::Result;
+use seaweed_worker_core::pb::{
+    ExecuteJobRequest, JobSpec, RunDetectionRequest, config_value::Kind,
+};
+use seaweed_worker_core::{JobHandler, PreviewProvider};
+use weed_lance_worker::catalog::NamespaceClient;
+use weed_lance_worker::jobs::cleanup::CleanupVersionsHandler;
+use weed_lance_worker::jobs::compact::{CompactHandler, JOB_TYPE};
+use weed_lance_worker::jobs::indices::OptimizeIndicesHandler;
+use weed_lance_worker::preview::LancePreview;
+
+mod common;
+use common::{Recorder, fallback, int_config, namespace_url};
+
+/// These tests drive one live gateway and one shared catalog: `list_all_tables`
+/// sweeps everything, so a table another test is writing shows up in this test's
+/// detection. Rust runs a binary's tests concurrently, so take a lock.
+static GATEWAY: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(());
+
+/// Declares a table through the namespace and writes `fragments` one-row
+/// appends into it, so a test brings its own state instead of depending on
+/// whatever a previous run left behind.
+async fn seed_fragmented_table(url: &str, name: &str, fragments: usize) -> Result<String> {
+    seed_table(url, name, fragments, 1, false).await
+}
+
+/// Writes `batches` appends of `rows_each` into a freshly declared table, and
+/// optionally builds a vector index after the first batch so the later ones are
+/// rows no index covers.
+async fn seed_table(
+    url: &str,
+    name: &str,
+    batches: usize,
+    rows_each: usize,
+    with_index: bool,
+) -> Result<String> {
+    use arrow_array::{
+        FixedSizeListArray, Float32Array, Int64Array, RecordBatch, RecordBatchIterator,
+    };
+    use arrow_schema::{DataType, Field, Schema};
+    use lance::dataset::{Dataset, WriteMode, WriteParams};
+    use lance::io::{ObjectStoreParams, StorageOptionsAccessor};
+    use std::sync::Arc;
+
+    // Declaring is the namespace's job, not the worker's, so the test asks for
+    // it directly rather than widening the client the worker uses. The bucket and
+    // namespace come first: a table cannot be declared under a parent that does
+    // not exist, and a test that assumes one is a test that only passes twice.
+    let http = reqwest::Client::new();
+    for parent in ["vec", "vec$ml"] {
+        http.post(format!("{url}/v1/namespace/{parent}/create"))
+            .json(&serde_json::json!({"mode": "EXIST_OK"}))
+            .send()
+            .await?
+            .error_for_status()?;
+    }
+    let encoded = format!("vec$ml${name}");
+    http.post(format!("{url}/v1/table/{encoded}/declare"))
+        .json(&serde_json::json!({}))
+        .send()
+        .await?
+        .error_for_status()?;
+
+    let client = NamespaceClient::new(url.to_string());
+    let id = vec!["vec".to_string(), "ml".to_string(), name.to_string()];
+    let description = client.describe_table(&id).await?;
+
+    let mut options = description.storage_options.clone();
+    options.extend(fallback());
+    const DIM: i32 = 16;
+    let schema = Arc::new(Schema::new(vec![
+        Field::new("id", DataType::Int64, false),
+        Field::new(
+            "vec",
+            DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), DIM),
+            false,
+        ),
+    ]));
+
+    for i in 0..batches {
+        let ids: Vec<i64> = (0..rows_each).map(|r| (i * rows_each + r) as i64).collect();
+        let values: Vec<f32> = ids
+            .iter()
+            .flat_map(|id| (0..DIM).map(move |d| (*id as f32) + d as f32))
+            .collect();
+        let vectors = FixedSizeListArray::new(
+            Arc::new(Field::new("item", DataType::Float32, true)),
+            DIM,
+            Arc::new(Float32Array::from(values)),
+            None,
+        );
+        let batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![Arc::new(Int64Array::from(ids)), Arc::new(vectors)],
+        )?;
+        let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone());
+        let params = WriteParams {
+            mode: if i == 0 {
+                WriteMode::Overwrite
+            } else {
+                WriteMode::Append
+            },
+            store_params: Some(ObjectStoreParams {
+                storage_options_accessor: Some(std::sync::Arc::new(
+                    StorageOptionsAccessor::with_static_options(options.clone()),
+                )),
+                ..Default::default()
+            }),
+            ..Default::default()
+        };
+        let dataset = Dataset::write(reader, description.location.as_str(), Some(params)).await?;
+
+        // The index is built after the first batch, so everything appended
+        // afterwards is a row it does not cover.
+        if with_index && i == 0 {
+            use lance::index::DatasetIndexExt;
+            use lance::index::vector::VectorIndexParams;
+            use lance_index::IndexType;
+            use lance_index::vector::{ivf::IvfBuildParams, pq::PQBuildParams};
+
+            let mut dataset = dataset;
+            let params = VectorIndexParams::with_ivf_pq_params(
+                lance_linalg::distance::MetricType::L2,
+                IvfBuildParams::new(1),
+                PQBuildParams::new(4, 8),
+            );
+            dataset
+                .create_index(&["vec"], IndexType::Vector, None, &params, true)
+                .await?;
+        }
+    }
+    Ok(encoded)
+}
+
+/// A table with more fragments than the policy allows is proposed, and running
+/// the proposal leaves it with fewer than it started with.
+#[tokio::test]
+async fn compacts_a_fragmented_table() {
+    let _gateway = GATEWAY.lock().await;
+    let Some(url) = namespace_url() else {
+        eprintln!("WEED_LANCE_NAMESPACE is unset, skipping");
+        return;
+    };
+    // Seeded here rather than by a script, so the test is repeatable: a previous
+    // run compacts the table it depended on.
+    let encoded = seed_fragmented_table(&url, "compactme", 12)
+        .await
+        .expect("seed a fragmented table");
+    let handler = CompactHandler::new(url).with_fallback(fallback());
+    let recorder = Recorder::default();
+
+    let request = RunDetectionRequest {
+        request_id: "detect-1".to_string(),
+        job_type: JOB_TYPE.to_string(),
+        worker_config_values: int_config("min_fragments", 4),
+        ..Default::default()
+    };
+    handler
+        .detect(&request, &recorder)
+        .await
+        .expect("detection failed");
+
+    let proposals = recorder.proposals.lock().unwrap().clone();
+    assert!(
+        !proposals.is_empty(),
+        "expected a proposal for the fragmented table"
+    );
+    let proposal = proposals
+        .iter()
+        .find(|p| p.summary.contains(encoded.as_str()))
+        .expect("no proposal for the seeded table");
+
+    // Detection opened the dataset to decide, so it reports what it saw. This is
+    // the only description of a Lance table anything outside the format can give.
+    let observations = recorder.observations.lock().unwrap().clone();
+    let observed = observations
+        .iter()
+        .find(|o| o.object_id.last().map(String::as_str) == Some("compactme"))
+        .expect("detection reported no observation for the seeded table");
+    assert_eq!(observed.format, "LANCE");
+    for attribute in ["fragments", "rows", "versions", "schema"] {
+        assert!(
+            observed.attributes.contains_key(attribute),
+            "observation is missing {attribute}: {:?}",
+            observed.attributes.keys().collect::<Vec<_>>()
+        );
+    }
+
+    let execute = ExecuteJobRequest {
+        request_id: "execute-1".to_string(),
+        job: Some(JobSpec {
+            job_id: "job-1".to_string(),
+            job_type: JOB_TYPE.to_string(),
+            parameters: proposal.parameters.clone(),
+            ..Default::default()
+        }),
+        worker_config_values: int_config("target_rows_per_fragment", 1_048_576),
+        ..Default::default()
+    };
+    handler
+        .execute(&execute, &recorder)
+        .await
+        .expect("execution failed");
+
+    let completed = recorder.completed.lock().unwrap().clone();
+    let result = completed.first().expect("no completion reported");
+    assert!(
+        result.success,
+        "compaction reported failure: {}",
+        result.error_message
+    );
+    let summary = result
+        .result
+        .as_ref()
+        .map(|r| r.summary.clone())
+        .unwrap_or_default();
+    assert!(
+        summary.contains("became"),
+        "completion carried no fragment counts: {summary}"
+    );
+    eprintln!("compaction result: {summary}");
+}
+
+/// A table with more versions than the floor is proposed, and running the job
+/// reports what it removed. The compaction test above leaves one behind.
+#[tokio::test]
+async fn cleans_up_old_versions() {
+    let _gateway = GATEWAY.lock().await;
+    let Some(url) = namespace_url() else {
+        eprintln!("WEED_LANCE_NAMESPACE is unset, skipping");
+        return;
+    };
+    let encoded = seed_table(&url, "cleanme", 6, 4, false)
+        .await
+        .expect("seed a table with versions to clean");
+
+    let handler = CleanupVersionsHandler::new(url).with_fallback(fallback());
+    let recorder = Recorder::default();
+
+    let request = RunDetectionRequest {
+        request_id: "detect-cleanup".to_string(),
+        job_type: "lance_cleanup_versions".to_string(),
+        worker_config_values: int_config("min_versions_to_keep", 2),
+        ..Default::default()
+    };
+    handler
+        .detect(&request, &recorder)
+        .await
+        .expect("detection failed");
+    let proposals = recorder.proposals.lock().unwrap().clone();
+    let proposal = proposals
+        .iter()
+        .find(|p| p.summary.contains(encoded.as_str()))
+        .cloned()
+        .expect("no cleanup proposal for the seeded table");
+
+    // Retain nothing, so every version outside the current one is fair game and
+    // the job has something to report rather than a no-op.
+    let execute = ExecuteJobRequest {
+        request_id: "execute-cleanup".to_string(),
+        job: Some(JobSpec {
+            job_id: "job-cleanup".to_string(),
+            job_type: "lance_cleanup_versions".to_string(),
+            parameters: proposal.parameters.clone(),
+            ..Default::default()
+        }),
+        worker_config_values: int_config("retain_hours", 0),
+        ..Default::default()
+    };
+    handler
+        .execute(&execute, &recorder)
+        .await
+        .expect("cleanup failed");
+
+    let completed = recorder.completed.lock().unwrap().clone();
+    let result = completed.last().expect("no completion reported");
+    assert!(
+        result.success,
+        "cleanup reported failure: {}",
+        result.error_message
+    );
+    eprintln!(
+        "cleanup result: {}",
+        result.result.as_ref().unwrap().summary
+    );
+}
+
+/// A table with no indices has nothing to optimize, so detection proposes
+/// nothing rather than queueing work that would do nothing.
+#[tokio::test]
+async fn skips_tables_without_indices() {
+    let _gateway = GATEWAY.lock().await;
+    let Some(url) = namespace_url() else {
+        eprintln!("WEED_LANCE_NAMESPACE is unset, skipping");
+        return;
+    };
+    let encoded = seed_table(&url, "noindex", 2, 8, false)
+        .await
+        .expect("seed a table without an index");
+
+    let handler = OptimizeIndicesHandler::new(url).with_fallback(fallback());
+    let recorder = Recorder::default();
+
+    let request = RunDetectionRequest {
+        request_id: "detect-indices".to_string(),
+        job_type: "lance_optimize_indices".to_string(),
+        worker_config_values: int_config("max_unindexed_rows", 1),
+        ..Default::default()
+    };
+    handler
+        .detect(&request, &recorder)
+        .await
+        .expect("detection failed");
+    // Judge this table only: the catalog holds every other test's tables too,
+    // and an indexed one with uncovered rows is supposed to be proposed.
+    assert!(
+        !recorder
+            .proposals
+            .lock()
+            .unwrap()
+            .iter()
+            .any(|p| p.summary.contains(encoded.as_str())),
+        "a table with no indices must not be proposed for reindexing"
+    );
+}
+
+/// The job with no Iceberg equivalent: rows appended after an index was built
+/// are invisible to a search of it until this runs. Needs a table with an index
+/// and rows outside it, which `indexed.py` in the scratchpad seeds.
+#[tokio::test]
+async fn reindexes_rows_an_index_does_not_cover() {
+    let _gateway = GATEWAY.lock().await;
+    let Some(url) = namespace_url() else {
+        eprintln!("WEED_LANCE_NAMESPACE is unset, skipping");
+        return;
+    };
+    let encoded = seed_table(&url, "reindexme", 2, 512, true)
+        .await
+        .expect("seed an indexed table with uncovered rows");
+    let handler = OptimizeIndicesHandler::new(url).with_fallback(fallback());
+    let recorder = Recorder::default();
+
+    let request = RunDetectionRequest {
+        request_id: "detect-reindex".to_string(),
+        job_type: "lance_optimize_indices".to_string(),
+        worker_config_values: int_config("max_unindexed_rows", 100),
+        ..Default::default()
+    };
+    handler
+        .detect(&request, &recorder)
+        .await
+        .expect("detection failed");
+    let proposals = recorder.proposals.lock().unwrap().clone();
+    let proposal = proposals
+        .iter()
+        .find(|p| p.summary.contains(encoded.as_str()))
+        .cloned()
+        .expect("the seeded indexed table was not proposed");
+
+    let execute = ExecuteJobRequest {
+        request_id: "execute-reindex".to_string(),
+        job: Some(JobSpec {
+            job_id: "job-reindex".to_string(),
+            job_type: "lance_optimize_indices".to_string(),
+            parameters: proposal.parameters.clone(),
+            ..Default::default()
+        }),
+        ..Default::default()
+    };
+    handler
+        .execute(&execute, &recorder)
+        .await
+        .expect("reindex failed");
+
+    let completed = recorder.completed.lock().unwrap().clone();
+    let result = completed.last().expect("no completion reported");
+    assert!(
+        result.success,
+        "reindex reported failure: {}",
+        result.error_message
+    );
+    let output = &result.result.as_ref().unwrap().output_values;
+    let after = match output
+        .get("unindexed_rows_after")
+        .and_then(|v| v.kind.as_ref())
+    {
+        Some(Kind::Int64Value(value)) => *value,
+        other => panic!("no unindexed_rows_after in {other:?}"),
+    };
+    assert_eq!(
+        after, 0,
+        "rows are still outside the index after optimizing"
+    );
+    eprintln!(
+        "reindex result: {}",
+        result.result.as_ref().unwrap().summary
+    );
+}
+
+/// The UI's whole reason for asking a worker: admin cannot read a Lance table,
+/// so the rows have to come back already rendered.
+#[tokio::test]
+async fn previews_rows_of_a_table() {
+    let _gateway = GATEWAY.lock().await;
+    let Some(url) = namespace_url() else {
+        eprintln!("set WEED_LANCE_NAMESPACE to run this test");
+        return;
+    };
+    seed_table(&url, "previewme", 2, 3, false)
+        .await
+        .expect("seed a table to preview");
+
+    let provider = LancePreview::new(url, fallback());
+    let id = vec!["vec".to_string(), "ml".to_string(), "previewme".to_string()];
+    let preview = provider.preview(&id, 4).await.expect("preview the table");
+
+    assert_eq!(preview.columns, vec!["id".to_string(), "vec".to_string()]);
+    assert_eq!(preview.total_rows, 6, "total is the table, not the sample");
+    assert_eq!(preview.rows.len(), 4, "row_limit bounds the sample");
+    assert!(
+        preview.rows[0][1].starts_with('['),
+        "a vector column should render as a list, got {:?}",
+        preview.rows[0][1]
+    );
+}
+
+/// The claim that removed managed versioning is not "one writer wins the
+/// conditional PUT" - that is only the mechanism. It is that concurrent writers
+/// lose nothing: the loser sees the conflict, rebases, and commits again. Eight
+/// writers appending at once must leave all eight batches in the table.
+#[tokio::test]
+async fn concurrent_writers_keep_every_commit() {
+    let _gateway = GATEWAY.lock().await;
+    let Some(url) = namespace_url() else {
+        eprintln!("set WEED_LANCE_NAMESPACE to run this test");
+        return;
+    };
+    const WRITERS: i64 = 8;
+    const ROWS_EACH: i64 = 4;
+
+    seed_table(&url, "racers", 1, ROWS_EACH as usize, false)
+        .await
+        .expect("seed the table the writers will append to");
+
+    let client = NamespaceClient::new(url.clone());
+    let id = vec!["vec".to_string(), "ml".to_string(), "racers".to_string()];
+    let description = client.describe_table(&id).await.expect("describe");
+    let mut options = description.storage_options.clone();
+    options.extend(fallback());
+
+    let writes = (0..WRITERS).map(|writer| {
+        let location = description.location.clone();
+        let options = options.clone();
+        tokio::spawn(
+            async move { append_rows(&location, &options, writer * 1000, ROWS_EACH).await },
+        )
+    });
+    for (writer, handle) in writes.enumerate() {
+        handle
+            .await
+            .expect("writer panicked")
+            .unwrap_or_else(|err| panic!("writer {writer} failed to commit: {err:#}"));
+    }
+
+    let table = weed_lance_worker::dataset::open(&client, &id, &fallback())
+        .await
+        .expect("reopen the table");
+    let rows = table.dataset.count_rows(None).await.expect("count rows");
+    let expected = (ROWS_EACH + WRITERS * ROWS_EACH) as usize;
+    assert_eq!(
+        rows, expected,
+        "concurrent commits lost data: {rows} rows, want {expected}"
+    );
+}
+
+/// Appends one batch to an existing dataset, the way an independent writer would.
+async fn append_rows(
+    location: &str,
+    options: &HashMap<String, String>,
+    first_id: i64,
+    rows: i64,
+) -> Result<()> {
+    use arrow_array::{
+        FixedSizeListArray, Float32Array, Int64Array, RecordBatch, RecordBatchIterator,
+    };
+    use arrow_schema::{DataType, Field, Schema};
+    use lance::dataset::{Dataset, WriteMode, WriteParams};
+    use lance::io::{ObjectStoreParams, StorageOptionsAccessor};
+    use std::sync::Arc;
+
+    const DIM: i32 = 16;
+    let schema = Arc::new(Schema::new(vec![
+        Field::new("id", DataType::Int64, false),
+        Field::new(
+            "vec",
+            DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), DIM),
+            false,
+        ),
+    ]));
+    let ids: Vec<i64> = (0..rows).map(|r| first_id + r).collect();
+    let values: Vec<f32> = ids
+        .iter()
+        .flat_map(|id| (0..DIM).map(move |d| (*id as f32) + d as f32))
+        .collect();
+    let vectors = FixedSizeListArray::new(
+        Arc::new(Field::new("item", DataType::Float32, true)),
+        DIM,
+        Arc::new(Float32Array::from(values)),
+        None,
+    );
+    let batch = RecordBatch::try_new(
+        schema.clone(),
+        vec![Arc::new(Int64Array::from(ids)), Arc::new(vectors)],
+    )?;
+    let params = WriteParams {
+        mode: WriteMode::Append,
+        store_params: Some(ObjectStoreParams {
+            storage_options_accessor: Some(Arc::new(StorageOptionsAccessor::with_static_options(
+                options.clone(),
+            ))),
+            ..Default::default()
+        }),
+        ..Default::default()
+    };
+    Dataset::write(
+        RecordBatchIterator::new(vec![Ok(batch)], schema.clone()),
+        location,
+        Some(params),
+    )
+    .await?;
+    Ok(())
+}

seaweed-worker/crates/lance/tests/lifecycle.rs

@@ -0,0 +1,123 @@
+//! Maintains one table someone else wrote, and nothing else.
+//!
+//! The Go suite in test/s3tables/lifecycle drives a table through its whole
+//! life - declared through the namespace, written by a Lance client, maintained,
+//! read again, dropped - and this is the maintenance step of it, so that step
+//! goes through the handlers a deployed worker runs rather than through the two
+//! lance calls they wrap. The table to work on comes in as WEED_LANCE_TABLE
+//! because the table is the Go test's, seeded and checked there.
+//!
+//! Skipped, like the rest of this crate's integration tests, unless
+//! WEED_LANCE_NAMESPACE names a live gateway.
+
+use std::collections::HashMap;
+
+use seaweed_worker_core::JobHandler;
+use seaweed_worker_core::pb::{ConfigValue, ExecuteJobRequest, JobSpec, RunDetectionRequest};
+use weed_lance_worker::jobs::cleanup::{self, CleanupVersionsHandler};
+use weed_lance_worker::jobs::compact::{CompactHandler, JOB_TYPE as COMPACT_JOB_TYPE};
+
+mod common;
+use common::{Recorder, fallback, int_config, namespace_url};
+
+fn table() -> Option<String> {
+    std::env::var("WEED_LANCE_TABLE")
+        .ok()
+        .filter(|s| !s.is_empty())
+}
+
+/// Compacts the named table's fragments, then drops the versions compaction
+/// left behind. Both go through the handler's own detect-then-execute path: a
+/// proposal the worker would not have made is not one worth running.
+#[tokio::test]
+async fn maintains_the_named_table() {
+    let (Some(url), Some(name)) = (namespace_url(), table()) else {
+        eprintln!("WEED_LANCE_NAMESPACE or WEED_LANCE_TABLE is unset, skipping");
+        return;
+    };
+
+    // A table written as a fragment per append is the case this exists for, so
+    // anything above one fragment is worth merging here.
+    let mut config = int_config("min_fragments", 2);
+    config.extend(int_config("target_rows_per_fragment", 1_048_576));
+    let compact = CompactHandler::new(url.clone()).with_fallback(fallback());
+    run(&compact, COMPACT_JOB_TYPE, &name, config, true).await;
+
+    // Keep the current version and retain nothing else, so the versions
+    // compaction superseded are actually removed rather than counted.
+    let mut config = int_config("min_versions_to_keep", 1);
+    config.extend(int_config("retain_hours", 0));
+    let cleanup = CleanupVersionsHandler::new(url).with_fallback(fallback());
+    run(&cleanup, cleanup::JOB_TYPE, &name, config, true).await;
+}
+
+/// Detects, finds the proposal for `name`, and executes it. The config carries
+/// every key both halves read; each ignores what it does not know.
+async fn run<H: JobHandler>(
+    handler: &H,
+    job_type: &str,
+    name: &str,
+    config: HashMap<String, ConfigValue>,
+    required: bool,
+) {
+    let recorder = Recorder::default();
+    handler
+        .detect(
+            &RunDetectionRequest {
+                request_id: format!("detect-{job_type}"),
+                job_type: job_type.to_string(),
+                worker_config_values: config.clone(),
+                ..Default::default()
+            },
+            &recorder,
+        )
+        .await
+        .unwrap_or_else(|err| panic!("{job_type} detection failed: {err}"));
+
+    let proposals = recorder.proposals.lock().unwrap().clone();
+    let Some(proposal) = proposals.iter().find(|p| p.summary.contains(name)) else {
+        assert!(
+            !required,
+            "{job_type} proposed nothing for {name}, out of {} proposals",
+            proposals.len()
+        );
+        eprintln!("{job_type}: nothing to do for {name}");
+        return;
+    };
+
+    handler
+        .execute(
+            &ExecuteJobRequest {
+                request_id: format!("execute-{job_type}"),
+                job: Some(JobSpec {
+                    job_id: format!("job-{job_type}"),
+                    job_type: job_type.to_string(),
+                    parameters: proposal.parameters.clone(),
+                    ..Default::default()
+                }),
+                worker_config_values: config,
+                ..Default::default()
+            },
+            &recorder,
+        )
+        .await
+        .unwrap_or_else(|err| panic!("{job_type} execution failed: {err}"));
+
+    let completed = recorder.completed.lock().unwrap().clone();
+    let result = completed
+        .last()
+        .unwrap_or_else(|| panic!("{job_type} reported no completion"));
+    assert!(
+        result.success,
+        "{job_type} reported failure: {}",
+        result.error_message
+    );
+    eprintln!(
+        "{job_type}: {}",
+        result
+            .result
+            .as_ref()
+            .map(|r| r.summary.clone())
+            .unwrap_or_default()
+    );
+}

seaweed-worker/crates/lance/tests/sort.rs

@@ -0,0 +1,504 @@
+//! Drives the sort handler against a live namespace.
+//!
+//! Skipped unless WEED_LANCE_NAMESPACE names one, like the other integration
+//! tests here: the job rewrites every fragment of a real table and commits, and
+//! the commit is the half worth testing.
+
+use anyhow::Result;
+use seaweed_worker_core::JobHandler;
+use seaweed_worker_core::pb::{
+    ConfigValue, ExecuteJobRequest, JobSpec, RunDetectionRequest, config_value::Kind,
+};
+use weed_lance_worker::catalog::NamespaceClient;
+use weed_lance_worker::jobs::sort::{JOB_TYPE, SortHandler};
+
+mod common;
+use common::{Recorder, fallback, namespace_url};
+
+/// One live gateway and one shared catalog, and `list_all_tables` sweeps
+/// everything, so these tests take a lock the way the compaction ones do.
+static GATEWAY: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(());
+
+fn config(values: &[(&str, ConfigValue)]) -> std::collections::HashMap<String, ConfigValue> {
+    values
+        .iter()
+        .map(|(name, value)| (name.to_string(), value.clone()))
+        .collect()
+}
+
+fn int(value: i64) -> ConfigValue {
+    ConfigValue {
+        kind: Some(Kind::Int64Value(value)),
+    }
+}
+
+fn text(value: &str) -> ConfigValue {
+    ConfigValue {
+        kind: Some(Kind::StringValue(value.to_string())),
+    }
+}
+
+/// Declares a table and writes `batches` appends whose ids descend, so the rows
+/// land in an order no sort would produce. `declared` is written into the
+/// dataset's own configuration, which is where a table says how it wants to be
+/// sorted.
+async fn seed_unsorted_table(
+    url: &str,
+    name: &str,
+    batches: i64,
+    rows_each: i64,
+    declared: Option<&str>,
+) -> Result<String> {
+    use arrow_array::{Int64Array, RecordBatch, RecordBatchIterator};
+    use arrow_schema::{DataType, Field, Schema};
+    use lance::dataset::{Dataset, WriteMode, WriteParams};
+    use lance::io::{ObjectStoreParams, StorageOptionsAccessor};
+    use std::sync::Arc;
+
+    let http = reqwest::Client::new();
+    for parent in ["vec", "vec$ml"] {
+        http.post(format!("{url}/v1/namespace/{parent}/create"))
+            .json(&serde_json::json!({"mode": "EXIST_OK"}))
+            .send()
+            .await?
+            .error_for_status()?;
+    }
+    let encoded = format!("vec$ml${name}");
+    http.post(format!("{url}/v1/table/{encoded}/declare"))
+        .json(&serde_json::json!({}))
+        .send()
+        .await?
+        .error_for_status()?;
+
+    let client = NamespaceClient::new(url.to_string());
+    let id = vec!["vec".to_string(), "ml".to_string(), name.to_string()];
+    let description = client.describe_table(&id).await?;
+    let mut options = description.storage_options.clone();
+    options.extend(fallback());
+
+    let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int64, false)]));
+    let total = batches * rows_each;
+    let mut dataset = None;
+    for batch_index in 0..batches {
+        // Ids descend across and within the batches, so the physical order is
+        // the reverse of the order the table declares.
+        let ids: Vec<i64> = (0..rows_each)
+            .map(|row| total - 1 - (batch_index * rows_each + row))
+            .collect();
+        let batch = RecordBatch::try_new(schema.clone(), vec![Arc::new(Int64Array::from(ids))])?;
+        let params = WriteParams {
+            mode: if batch_index == 0 {
+                WriteMode::Overwrite
+            } else {
+                WriteMode::Append
+            },
+            store_params: Some(ObjectStoreParams {
+                storage_options_accessor: Some(Arc::new(
+                    StorageOptionsAccessor::with_static_options(options.clone()),
+                )),
+                ..Default::default()
+            }),
+            ..Default::default()
+        };
+        dataset = Some(
+            Dataset::write(
+                RecordBatchIterator::new(vec![Ok(batch)], schema.clone()),
+                description.location.as_str(),
+                Some(params),
+            )
+            .await?,
+        );
+    }
+
+    if let Some(declared) = declared {
+        let mut dataset = dataset.expect("at least one batch was written");
+        dataset
+            .update_config([("seaweedfs.sort.fields", declared)])
+            .await?;
+    }
+    Ok(encoded)
+}
+
+/// Reads the ids back in the order they are stored, which is what a sort is
+/// supposed to change.
+async fn stored_ids(url: &str, name: &str) -> Result<Vec<i64>> {
+    use arrow_array::Int64Array;
+    use futures::TryStreamExt;
+
+    let client = NamespaceClient::new(url.to_string());
+    let id = vec!["vec".to_string(), "ml".to_string(), name.to_string()];
+    let table = weed_lance_worker::dataset::open(&client, &id, &fallback()).await?;
+    let stream = table.dataset.scan().try_into_stream().await?;
+    let batches: Vec<_> = stream.try_collect().await?;
+    let mut ids = Vec::new();
+    for batch in batches {
+        let column = batch
+            .column_by_name("id")
+            .expect("the seeded table has an id column")
+            .as_any()
+            .downcast_ref::<Int64Array>()
+            .expect("id is an Int64 column")
+            .clone();
+        ids.extend(column.values().iter().copied());
+    }
+    Ok(ids)
+}
+
+/// The whole loop: a table declaring an order its rows do not follow is
+/// proposed, sorting it stores the rows in that order, and the marker the same
+/// commit carries is what stops the next sweep proposing it again.
+#[tokio::test]
+async fn sorts_a_table_and_stops_proposing_it() {
+    let _gateway = GATEWAY.lock().await;
+    let Some(url) = namespace_url() else {
+        eprintln!("WEED_LANCE_NAMESPACE is unset, skipping");
+        return;
+    };
+    let encoded = seed_unsorted_table(&url, "sortme", 4, 250, Some("id asc"))
+        .await
+        .expect("seed an unsorted table");
+
+    let before = stored_ids(&url, "sortme").await.expect("read the ids back");
+    assert!(
+        before.windows(2).any(|pair| pair[0] > pair[1]),
+        "the seeded table was already sorted, so the test proves nothing"
+    );
+
+    let handler = SortHandler::new(url.clone()).with_fallback(fallback());
+    let recorder = Recorder::default();
+    let detection = RunDetectionRequest {
+        request_id: "detect-sort".to_string(),
+        job_type: JOB_TYPE.to_string(),
+        worker_config_values: config(&[("min_unsorted_rows", int(1))]),
+        ..Default::default()
+    };
+    handler
+        .detect(&detection, &recorder)
+        .await
+        .expect("detection failed");
+
+    let proposal = recorder
+        .proposals
+        .lock()
+        .unwrap()
+        .iter()
+        .find(|p| p.summary.contains(encoded.as_str()))
+        .cloned()
+        .expect("the unsorted table was not proposed");
+    assert!(
+        proposal.summary.contains("id asc"),
+        "the proposal does not say what order it would use: {}",
+        proposal.summary
+    );
+
+    let execute = ExecuteJobRequest {
+        request_id: "execute-sort".to_string(),
+        job: Some(JobSpec {
+            job_id: "job-sort".to_string(),
+            job_type: JOB_TYPE.to_string(),
+            parameters: proposal.parameters.clone(),
+            ..Default::default()
+        }),
+        worker_config_values: config(&[
+            ("memory_budget_mb", int(64)),
+            ("max_rows_per_file", int(1024)),
+        ]),
+        ..Default::default()
+    };
+    handler
+        .execute(&execute, &recorder)
+        .await
+        .expect("sorting failed");
+
+    let completed = recorder.completed.lock().unwrap().clone();
+    let result = completed.last().expect("no completion reported");
+    assert!(
+        result.success,
+        "the sort reported failure: {}",
+        result.error_message
+    );
+
+    let after = stored_ids(&url, "sortme")
+        .await
+        .expect("read the sorted ids back");
+    assert_eq!(after.len(), before.len(), "the sort lost or invented rows");
+    assert!(
+        after.windows(2).all(|pair| pair[0] <= pair[1]),
+        "the rows are not stored in ascending id order"
+    );
+
+    // The marker rode in the same commit as the data, so a second sweep finds
+    // the table up to date without anything else having written it.
+    let second = Recorder::default();
+    handler
+        .detect(&detection, &second)
+        .await
+        .expect("second detection failed");
+    assert!(
+        !second
+            .proposals
+            .lock()
+            .unwrap()
+            .iter()
+            .any(|p| p.summary.contains(encoded.as_str())),
+        "a table that was just sorted must not be proposed again"
+    );
+    eprintln!("sort result: {}", result.result.as_ref().unwrap().summary);
+}
+
+/// A table that declares no order, with a worker configuring none either, is
+/// not something to rewrite on a guess.
+#[tokio::test]
+async fn leaves_a_table_that_declares_no_order_alone() {
+    let _gateway = GATEWAY.lock().await;
+    let Some(url) = namespace_url() else {
+        eprintln!("WEED_LANCE_NAMESPACE is unset, skipping");
+        return;
+    };
+    let encoded = seed_unsorted_table(&url, "unsorted", 2, 16, None)
+        .await
+        .expect("seed a table with no declared order");
+
+    let handler = SortHandler::new(url.clone()).with_fallback(fallback());
+    let recorder = Recorder::default();
+    handler
+        .detect(
+            &RunDetectionRequest {
+                request_id: "detect-none".to_string(),
+                job_type: JOB_TYPE.to_string(),
+                worker_config_values: config(&[
+                    ("min_unsorted_rows", int(1)),
+                    ("sort_fields", text("")),
+                ]),
+                ..Default::default()
+            },
+            &recorder,
+        )
+        .await
+        .expect("detection failed");
+
+    assert!(
+        !recorder
+            .proposals
+            .lock()
+            .unwrap()
+            .iter()
+            .any(|p| p.summary.contains(encoded.as_str())),
+        "a table with no order named anywhere must not be proposed"
+    );
+}
+
+/// The case a row-count threshold cannot see: the table replaced by a write of
+/// its own, growing by far less than `min_unsorted_rows`. Every row moved, so
+/// the table is unsorted, and detection has to say so however small the delta.
+#[tokio::test]
+async fn proposes_a_replacement_that_barely_changes_the_row_count() {
+    let _gateway = GATEWAY.lock().await;
+    let Some(url) = namespace_url() else {
+        eprintln!("WEED_LANCE_NAMESPACE is unset, skipping");
+        return;
+    };
+    let encoded = seed_unsorted_table(&url, "replaceme", 4, 250, Some("id asc"))
+        .await
+        .expect("seed an unsorted table");
+
+    let handler = SortHandler::new(url.clone()).with_fallback(fallback());
+    // A threshold no realistic append would cross, so nothing but the identity
+    // of the data can be what proposes this table.
+    let detection = RunDetectionRequest {
+        request_id: "detect-replace".to_string(),
+        job_type: JOB_TYPE.to_string(),
+        worker_config_values: config(&[("min_unsorted_rows", int(1_000_000))]),
+        ..Default::default()
+    };
+
+    let first = Recorder::default();
+    handler.detect(&detection, &first).await.expect("detection");
+    let proposal = first
+        .proposals
+        .lock()
+        .unwrap()
+        .iter()
+        .find(|p| p.summary.contains(encoded.as_str()))
+        .cloned()
+        .expect("a never-sorted table was not proposed");
+
+    handler
+        .execute(
+            &ExecuteJobRequest {
+                request_id: "execute-replace".to_string(),
+                job: Some(JobSpec {
+                    job_id: "job-replace".to_string(),
+                    job_type: JOB_TYPE.to_string(),
+                    parameters: proposal.parameters.clone(),
+                    ..Default::default()
+                }),
+                worker_config_values: config(&[("max_rows_per_file", int(1024))]),
+                ..Default::default()
+            },
+            &first,
+        )
+        .await
+        .expect("sorting failed");
+
+    // Sorted, and left alone by the next sweep.
+    let after_sort = Recorder::default();
+    handler
+        .detect(&detection, &after_sort)
+        .await
+        .expect("detection");
+    assert!(
+        !after_sort
+            .proposals
+            .lock()
+            .unwrap()
+            .iter()
+            .any(|p| p.summary.contains(encoded.as_str())),
+        "a table that was just sorted must not be proposed again"
+    );
+
+    // Now replace the data: 1002 rows where there were 1000, shuffled again.
+    seed_unsorted_table(&url, "replaceme", 3, 334, Some("id asc"))
+        .await
+        .expect("replace the table's data");
+
+    let after_replace = Recorder::default();
+    handler
+        .detect(&detection, &after_replace)
+        .await
+        .expect("detection");
+    assert!(
+        after_replace
+            .proposals
+            .lock()
+            .unwrap()
+            .iter()
+            .any(|p| p.summary.contains(encoded.as_str())),
+        "a replaced table must be proposed even when the row count barely moved"
+    );
+}
+
+/// Deletions must not hide appended rows. This is the arithmetic against a real
+/// manifest rather than a hand-built one: the sorted fragments keep their files
+/// and lose most of their rows, a smaller batch is appended after them, and the
+/// table has to be proposed *because rows were appended* — not because it looks
+/// replaced, and not left alone because it now holds fewer rows than when it
+/// was sorted.
+#[tokio::test]
+async fn deletions_do_not_hide_appended_rows() {
+    let _gateway = GATEWAY.lock().await;
+    let Some(url) = namespace_url() else {
+        eprintln!("WEED_LANCE_NAMESPACE is unset, skipping");
+        return;
+    };
+    let encoded = seed_unsorted_table(&url, "deleteme", 4, 250, Some("id asc"))
+        .await
+        .expect("seed an unsorted table");
+
+    let handler = SortHandler::new(url.clone()).with_fallback(fallback());
+    let detection = RunDetectionRequest {
+        request_id: "detect-delete".to_string(),
+        job_type: JOB_TYPE.to_string(),
+        worker_config_values: config(&[("min_unsorted_rows", int(100))]),
+        ..Default::default()
+    };
+
+    let recorder = Recorder::default();
+    handler
+        .detect(&detection, &recorder)
+        .await
+        .expect("detection");
+    let proposal = recorder
+        .proposals
+        .lock()
+        .unwrap()
+        .iter()
+        .find(|p| p.summary.contains(encoded.as_str()))
+        .cloned()
+        .expect("a never-sorted table was not proposed");
+    handler
+        .execute(
+            &ExecuteJobRequest {
+                request_id: "execute-delete".to_string(),
+                job: Some(JobSpec {
+                    job_id: "job-delete".to_string(),
+                    job_type: JOB_TYPE.to_string(),
+                    parameters: proposal.parameters.clone(),
+                    ..Default::default()
+                }),
+                worker_config_values: config(&[("max_rows_per_file", int(1024))]),
+                ..Default::default()
+            },
+            &recorder,
+        )
+        .await
+        .expect("sorting failed");
+
+    // Delete 900 of the 1000 sorted rows, then append 200 unsorted ones. The
+    // table is smaller than when it was sorted, so net growth reads as zero.
+    let client = NamespaceClient::new(url.clone());
+    let id = vec!["vec".to_string(), "ml".to_string(), "deleteme".to_string()];
+    let mut table = weed_lance_worker::dataset::open(&client, &id, &fallback())
+        .await
+        .expect("reopen the sorted table");
+    table
+        .dataset
+        .delete("id < 900")
+        .await
+        .expect("delete most of the sorted rows");
+    append_ids(&url, "deleteme", 5_000, 200)
+        .await
+        .expect("append rows after the sorted ones");
+
+    let after = Recorder::default();
+    handler.detect(&detection, &after).await.expect("detection");
+    let proposal = after
+        .proposals
+        .lock()
+        .unwrap()
+        .iter()
+        .find(|p| p.summary.contains(encoded.as_str()))
+        .cloned()
+        .expect("a table with 200 unsorted rows appended was not proposed");
+    assert!(
+        proposal.detail.contains("200 rows appended"),
+        "expected the appended rows to be counted, got {:?}",
+        proposal.detail
+    );
+}
+
+/// Appends one batch of ascending ids to an existing table.
+async fn append_ids(url: &str, name: &str, first_id: i64, rows: i64) -> Result<()> {
+    use arrow_array::{Int64Array, RecordBatch, RecordBatchIterator};
+    use arrow_schema::{DataType, Field, Schema};
+    use lance::dataset::{Dataset, WriteMode, WriteParams};
+    use lance::io::{ObjectStoreParams, StorageOptionsAccessor};
+    use std::sync::Arc;
+
+    let client = NamespaceClient::new(url.to_string());
+    let id = vec!["vec".to_string(), "ml".to_string(), name.to_string()];
+    let description = client.describe_table(&id).await?;
+    let mut options = description.storage_options.clone();
+    options.extend(fallback());
+
+    let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int64, false)]));
+    let ids: Vec<i64> = (0..rows).map(|row| first_id + row).collect();
+    let batch = RecordBatch::try_new(schema.clone(), vec![Arc::new(Int64Array::from(ids))])?;
+    let params = WriteParams {
+        mode: WriteMode::Append,
+        store_params: Some(ObjectStoreParams {
+            storage_options_accessor: Some(Arc::new(StorageOptionsAccessor::with_static_options(
+                options,
+            ))),
+            ..Default::default()
+        }),
+        ..Default::default()
+    };
+    Dataset::write(
+        RecordBatchIterator::new(vec![Ok(batch)], schema.clone()),
+        description.location.as_str(),
+        Some(params),
+    )
+    .await?;
+    Ok(())
+}