snapshot of https://github.com/ActivityWatch/aw-server-rust.git (master, 2026-09-16; blobs > 131072B omitted)

aw-datastore/Cargo.toml

@@ -0,0 +1,41 @@
+[package]
+name = "aw-datastore"
+version = "0.1.0"
+authors = ["Johan Bjäreholt <johan@bjareho.lt>"]
+edition = "2021"
+
+[features]
+default = ["bundled"]
+# Use bundled SQLite (default, no encryption support)
+bundled = ["rusqlite/bundled"]
+# Use bundled SQLCipher for encrypted databases (mutually exclusive with 'bundled')
+# Build with: cargo build --no-default-features --features encryption
+# Requires OpenSSL. Use 'encryption-vendored' to vendor OpenSSL as well.
+encryption = ["rusqlite/bundled-sqlcipher", "dep:zeroize"]
+# Like 'encryption' but also vendors OpenSSL (fully self-contained)
+encryption-vendored = ["rusqlite/bundled-sqlcipher-vendored-openssl", "dep:zeroize"]
+legacy_import_tests = []
+
+[dependencies]
+dirs = "6"
+serde = "1.0"
+serde_json = "1.0"
+chrono = { version = "0.4", features = ["serde"] }
+rusqlite = { version = "0.30", features = ["chrono", "serde_json"] }
+mpsc_requests = "0.3"
+log = "0.4"
+zeroize = { version = "1", optional = true, features = ["alloc"] }
+
+aw-models = { path = "../aw-models" }
+aw-transform = { path = "../aw-transform" }
+regex = "1"
+
+[dev-dependencies]
+criterion = "0.5.1"
+tempfile = "3"
+# Used by migration tests to construct databases with old schema versions
+rusqlite = { version = "0.30", features = ["chrono", "serde_json"] }
+
+[[bench]]
+name = "interval_reads"
+harness = false

aw-datastore/benches/interval_reads.rs

@@ -0,0 +1,80 @@
+use aw_datastore::DatastoreInstance;
+use aw_models::{Bucket, BucketMetadata};
+use chrono::{DateTime, Utc};
+use criterion::{criterion_group, criterion_main, BatchSize, Criterion};
+use rusqlite::Connection;
+
+fn interval_reads(c: &mut Criterion) {
+    let conn = Connection::open_in_memory().unwrap();
+    let mut ds = DatastoreInstance::new(&conn, true).unwrap();
+    ds.create_bucket(
+        &conn,
+        Bucket {
+            bid: None,
+            id: "bench".into(),
+            _type: "test".into(),
+            client: "test".into(),
+            hostname: "test".into(),
+            created: Some(Utc::now()),
+            data: Default::default(),
+            metadata: BucketMetadata::default(),
+            events: None,
+            last_updated: None,
+        },
+    )
+    .unwrap();
+    conn.execute_batch(
+        "WITH RECURSIVE seq(n) AS (SELECT 1 UNION ALL SELECT n+1 FROM seq WHERE n<1000000)
+        INSERT INTO events(bucketrow, starttime, endtime, data)
+        SELECT 1, n*1000000000, n*1000000000+500000000, '{\"app\":\"browser\"}' FROM seq;",
+    )
+    .unwrap();
+    let mut ds = DatastoreInstance::new(&conn, true).unwrap();
+    let mut group = c.benchmark_group("interval_reads");
+    for (name, start, end) in [
+        ("recent", 999_000, 1_000_000),
+        ("middle", 500_000, 501_000),
+        ("early", 0, 1_000),
+    ] {
+        let start = Some(DateTime::from_timestamp(start, 0).unwrap());
+        let end = Some(DateTime::from_timestamp(end, 0).unwrap());
+        group.bench_function(name, |b| {
+            b.iter(|| ds.get_events(&conn, "bench", start, end, None).unwrap())
+        });
+    }
+    group.bench_function("latest_one", |b| {
+        b.iter(|| ds.get_events(&conn, "bench", None, None, Some(1)).unwrap())
+    });
+    let start = Some(DateTime::from_timestamp(999_000, 0).unwrap());
+    group.bench_function("recent_count", |b| {
+        b.iter(|| ds.get_event_count(&conn, "bench", start, None).unwrap())
+    });
+    group.finish();
+}
+
+fn index_write_cost(c: &mut Criterion) {
+    let mut group = c.benchmark_group("index_write_cost");
+    for second_index in [false, true] {
+        group.bench_function(if second_index { "two_indexes" } else { "one_index" }, |b| {
+            b.iter_batched(|| {
+                let conn = Connection::open_in_memory().unwrap();
+                conn.execute_batch("CREATE TABLE events(id INTEGER PRIMARY KEY AUTOINCREMENT,
+                    bucketrow INTEGER NOT NULL, starttime INTEGER NOT NULL, endtime INTEGER NOT NULL, data TEXT NOT NULL);
+                    CREATE INDEX by_start ON events(bucketrow, starttime DESC, endtime);").unwrap();
+                if second_index {
+                    conn.execute_batch("CREATE INDEX by_end ON events(bucketrow, endtime, starttime DESC);").unwrap();
+                }
+                conn
+            }, |conn| {
+                conn.execute_batch("WITH RECURSIVE seq(n) AS (SELECT 1 UNION ALL SELECT n+1 FROM seq WHERE n<10000)
+                    INSERT INTO events(bucketrow,starttime,endtime,data)
+                    SELECT 1,n*1000000000,n*1000000000+500000000,'{}' FROM seq;").unwrap();
+                conn
+            }, BatchSize::SmallInput);
+        });
+    }
+    group.finish();
+}
+
+criterion_group!(benches, interval_reads, index_write_cost);
+criterion_main!(benches);

aw-datastore/examples/export_memory.rs

@@ -0,0 +1,61 @@
+//! Reproducible export-memory comparison using synthetic, in-memory data only.
+//! Build with `cargo build -p aw-datastore --example export_memory --release`.
+//! Run the resulting executable under a memory profiler with `stream 100000`
+//! or `materialized 100000`. No existing database or server is opened.
+use aw_datastore::DatastoreInstance;
+use aw_models::{Bucket, BucketMetadata, BucketsExport, TryVec};
+use chrono::Utc;
+use rusqlite::Connection;
+use std::{hint::black_box, time::Instant};
+
+fn main() {
+    let mode = std::env::args().nth(1).unwrap_or_else(|| "stream".into());
+    assert!(matches!(mode.as_str(), "stream" | "materialized"));
+    let count: u32 = std::env::args()
+        .nth(2)
+        .unwrap_or_else(|| "100000".into())
+        .parse()
+        .unwrap();
+    let conn = Connection::open_in_memory().unwrap();
+    let mut ds = DatastoreInstance::new(&conn, true).unwrap();
+    ds.create_bucket(
+        &conn,
+        Bucket {
+            bid: None,
+            id: "synthetic".into(),
+            _type: "test".into(),
+            client: "test".into(),
+            hostname: "test".into(),
+            created: Some(Utc::now()),
+            data: Default::default(),
+            metadata: BucketMetadata::default(),
+            events: None,
+            last_updated: None,
+        },
+    )
+    .unwrap();
+    let payload = serde_json::json!({"app": "browser", "title": "x".repeat(256)}).to_string();
+    conn.execute(
+        "WITH RECURSIVE seq(n) AS (SELECT 1 UNION ALL SELECT n+1 FROM seq WHERE n<?1)
+        INSERT INTO events(bucketrow,starttime,endtime,data)
+        SELECT 1,n*1000000000,n*1000000000+500000000,?2 FROM seq",
+        rusqlite::params![count, payload],
+    )
+    .unwrap();
+    let mut ds = DatastoreInstance::new(&conn, true).unwrap();
+    let start = Instant::now();
+    if mode == "stream" {
+        ds.write_export(&conn, None, std::io::sink()).unwrap();
+    } else {
+        let mut buckets = ds.get_buckets();
+        for (id, bucket) in &mut buckets {
+            bucket.events = Some(TryVec::new(
+                ds.get_events(&conn, id, None, None, None).unwrap(),
+            ));
+        }
+        let export = BucketsExport { buckets };
+        let body = serde_json::to_string(&export).unwrap();
+        black_box((&export, &body));
+    }
+    println!("{mode}: {count} synthetic events, {:?}", start.elapsed());
+}

aw-datastore/src/export.rs

@@ -0,0 +1,220 @@
+use std::{collections::HashMap, io::Write};
+
+use aw_models::Bucket;
+use rusqlite::Connection;
+use serde::{
+    ser::{Error, SerializeMap, SerializeSeq},
+    Serialize, Serializer,
+};
+
+use crate::DatastoreError;
+
+struct EventRows<'a> {
+    conn: &'a Connection,
+    bucket: &'a Bucket,
+}
+
+impl Serialize for EventRows<'_> {
+    fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
+        let mut stmt = self
+            .conn
+            .prepare_cached(
+                "SELECT id, starttime, endtime, data
+             FROM events INDEXED BY events_bucketrow_starttime_endtime_index
+             WHERE bucketrow = ?1 AND endtime >= 0 AND starttime <= ?2
+             ORDER BY starttime DESC, endtime ASC, id ASC",
+            )
+            .map_err(S::Error::custom)?;
+        let mut rows = stmt
+            .query(rusqlite::params![self.bucket.bid.unwrap(), i64::MAX])
+            .map_err(S::Error::custom)?;
+        let mut seq = serializer.serialize_seq(None)?;
+        while let Some(row) = rows.next().map_err(S::Error::custom)? {
+            // Match get_events' default range, clipping and corrupt-row policy.
+            match crate::datastore::parse_event_row(row, Some((0, i64::MAX))) {
+                Ok(event) => seq.serialize_element(&event)?,
+                Err(err) => warn!("Corrupt event in bucket {}: {}", self.bucket.id, err),
+            }
+        }
+        seq.end()
+    }
+}
+
+struct ExportBucket<'a> {
+    conn: &'a Connection,
+    bucket: &'a Bucket,
+}
+
+impl Serialize for ExportBucket<'_> {
+    fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
+        // Reuse Bucket's field names and serialization rules, replacing only
+        // events. This allocation contains bucket metadata, never event rows.
+        let metadata = serde_json::to_value(self.bucket).map_err(S::Error::custom)?;
+        let metadata = metadata
+            .as_object()
+            .ok_or_else(|| S::Error::custom("invalid bucket metadata"))?;
+        let mut map = serializer.serialize_map(Some(metadata.len()))?;
+        for (key, value) in metadata {
+            if key != "events" {
+                map.serialize_entry(key, value)?;
+            }
+        }
+        map.serialize_entry(
+            "events",
+            &EventRows {
+                conn: self.conn,
+                bucket: self.bucket,
+            },
+        )?;
+        map.end()
+    }
+}
+
+struct ExportBuckets<'a> {
+    conn: &'a Connection,
+    buckets: &'a HashMap<String, Bucket>,
+    selected: Option<&'a str>,
+}
+
+impl Serialize for ExportBuckets<'_> {
+    fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
+        let mut map = serializer.serialize_map(None)?;
+        for (id, bucket) in self.buckets {
+            if self.selected.is_none() || self.selected == Some(id.as_str()) {
+                map.serialize_entry(
+                    id,
+                    &ExportBucket {
+                        conn: self.conn,
+                        bucket,
+                    },
+                )?;
+            }
+        }
+        map.end()
+    }
+}
+
+pub(crate) fn write_export(
+    conn: &Connection,
+    buckets: &HashMap<String, Bucket>,
+    selected: Option<&str>,
+    writer: impl Write,
+) -> Result<(), DatastoreError> {
+    if let Some(id) = selected {
+        if !buckets.contains_key(id) {
+            return Err(DatastoreError::NoSuchBucket(id.to_owned()));
+        }
+    }
+    #[derive(Serialize)]
+    struct Export<'a> {
+        buckets: ExportBuckets<'a>,
+    }
+    serde_json::to_writer(
+        writer,
+        &Export {
+            buckets: ExportBuckets {
+                conn,
+                buckets,
+                selected,
+            },
+        },
+    )
+    .map_err(|err| DatastoreError::InternalError(format!("Failed to write export: {err}")))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::DatastoreInstance;
+    use aw_models::{BucketMetadata, BucketsExport, Event, TryVec};
+    use chrono::{DateTime, Duration};
+
+    fn setup() -> (Connection, DatastoreInstance) {
+        let conn = Connection::open_in_memory().unwrap();
+        let mut ds = DatastoreInstance::new(&conn, true).unwrap();
+        for id in ["populated", "empty"] {
+            ds.create_bucket(
+                &conn,
+                Bucket {
+                    bid: None,
+                    id: id.into(),
+                    _type: "test".into(),
+                    client: "test".into(),
+                    hostname: "host".into(),
+                    created: None,
+                    data: Default::default(),
+                    metadata: BucketMetadata::default(),
+                    events: None,
+                    last_updated: None,
+                },
+            )
+            .unwrap();
+        }
+        let events = [(-10, 20), (5, 20), (5, 10), (5, 10), (30, 0)]
+            .into_iter()
+            .map(|(start, duration)| {
+                Event::new(
+                    DateTime::from_timestamp(start, 0).unwrap(),
+                    Duration::seconds(duration),
+                    serde_json::from_value(
+                        serde_json::json!({"text": "quotes \" and unicode ☀", "nested": [1, true]}),
+                    )
+                    .unwrap(),
+                )
+            })
+            .collect();
+        ds.insert_events(&conn, "populated", events).unwrap();
+        conn.execute("INSERT INTO events(bucketrow,starttime,endtime,data) VALUES(1,6000000000,7000000000,'invalid json')", []).unwrap();
+        (conn, ds)
+    }
+
+    #[test]
+    fn streamed_json_matches_materialized_exports() {
+        let (conn, mut ds) = setup();
+        for selected in [None, Some("populated"), Some("empty")] {
+            let mut buckets = ds.get_buckets();
+            buckets.retain(|id, _| selected.is_none() || selected == Some(id.as_str()));
+            for (id, bucket) in &mut buckets {
+                bucket.events = Some(TryVec::new(
+                    ds.get_events(&conn, id, None, None, None).unwrap(),
+                ));
+            }
+            let expected = serde_json::to_value(BucketsExport { buckets }).unwrap();
+            let mut output = Vec::new();
+            ds.write_export(&conn, selected, &mut output).unwrap();
+            assert_eq!(
+                serde_json::from_slice::<serde_json::Value>(&output).unwrap(),
+                expected
+            );
+        }
+    }
+
+    #[test]
+    fn missing_bucket_and_writer_failures_propagate() {
+        let (conn, ds) = setup();
+        let mut output = Vec::new();
+        assert!(matches!(
+            ds.write_export(&conn, Some("missing"), &mut output),
+            Err(DatastoreError::NoSuchBucket(_))
+        ));
+        assert!(output.is_empty());
+        struct FailingWriter(usize);
+        impl Write for FailingWriter {
+            fn write(&mut self, bytes: &[u8]) -> std::io::Result<usize> {
+                if self.0 == 0 {
+                    return Err(std::io::Error::new(std::io::ErrorKind::Other, "disk full"));
+                }
+                let n = bytes.len().min(self.0);
+                self.0 -= n;
+                Ok(n)
+            }
+            fn flush(&mut self) -> std::io::Result<()> {
+                Ok(())
+            }
+        }
+        assert!(matches!(
+            ds.write_export(&conn, None, FailingWriter(500)),
+            Err(DatastoreError::InternalError(_))
+        ));
+    }
+}

aw-datastore/src/legacy_import.rs

@@ -0,0 +1,230 @@
+use crate::datastore::DatastoreInstance;
+use rusqlite::Connection;
+
+#[derive(Debug, Clone)]
+pub enum LegacyDatastoreImportError {
+    #[allow(dead_code)]
+    SQLPrepareError(String),
+    #[allow(dead_code)]
+    SQLMapError(String),
+}
+
+pub fn legacy_import(
+    new_ds: &mut DatastoreInstance,
+    new_conn: &Connection,
+) -> Result<(), LegacyDatastoreImportError> {
+    import::legacy_import(new_ds, new_conn)
+}
+
+#[cfg(not(target_os = "android"))]
+mod import {
+    use std::path::PathBuf;
+
+    use rusqlite::types::ToSql;
+    use rusqlite::Connection;
+
+    use chrono::DateTime;
+    use chrono::Duration;
+    use chrono::Utc;
+
+    use aw_models::Bucket;
+    use aw_models::BucketMetadata;
+    use aw_models::Event;
+
+    use crate::datastore::DatastoreInstance;
+
+    use super::LegacyDatastoreImportError;
+
+    fn dbfile_path() -> PathBuf {
+        dirs::data_dir()
+            .expect("Unable to read user data dir")
+            .join("activitywatch")
+            .join("aw-server")
+            .join("peewee-sqlite.v2.db")
+    }
+
+    fn get_legacy_buckets(conn: &Connection) -> Result<Vec<Bucket>, LegacyDatastoreImportError> {
+        let mut stmt = match conn
+            .prepare("SELECT key, id, type, client, hostname, created FROM bucketmodel")
+        {
+            Ok(stmt) => stmt,
+            Err(err) => return Err(LegacyDatastoreImportError::SQLPrepareError(err.to_string())),
+        };
+        let bucket_rows = match stmt.query_map(&[] as &[&dyn ToSql], |row| {
+            Ok(Bucket {
+                bid: row.get(0)?,
+                id: row.get(1)?,
+                _type: row.get(2)?,
+                client: row.get(3)?,
+                hostname: row.get(4)?,
+                created: row.get(5)?,
+                data: json_map! {},
+                events: None,
+                last_updated: None,
+                metadata: BucketMetadata {
+                    start: None,
+                    end: None,
+                },
+            })
+        }) {
+            Ok(buckets) => buckets,
+            Err(err) => {
+                return Err(LegacyDatastoreImportError::SQLMapError(format!(
+                    "Failed to query get_legacy_buckets SQL statement: {err:?}"
+                )))
+            }
+        };
+
+        let mut buckets = Vec::new();
+        for bucket_res in bucket_rows {
+            match bucket_res {
+                Ok(bucket) => buckets.push(bucket),
+                Err(err) => panic!("{err:?}"),
+            }
+        }
+        Ok(buckets)
+    }
+
+    fn get_legacy_events(
+        conn: &Connection,
+        bucket_id: i64,
+    ) -> Result<Vec<Event>, LegacyDatastoreImportError> {
+        let mut stmt = match conn.prepare(
+            "
+                SELECT timestamp, duration, datastr
+                FROM eventmodel
+                WHERE bucket_id = ?1
+                ORDER BY timestamp DESC
+            ;",
+        ) {
+            Ok(stmt) => stmt,
+            Err(err) => return Err(LegacyDatastoreImportError::SQLPrepareError(err.to_string())),
+        };
+
+        let rows = match stmt.query_map([&bucket_id], |row| {
+            let timestamp_str: String = row.get(0)?;
+            let duration_float: f64 = row.get(1)?;
+            let data_str: String = row.get(2)?;
+            Ok((timestamp_str, duration_float, data_str))
+        }) {
+            Ok(rows) => rows,
+            Err(err) => {
+                return Err(LegacyDatastoreImportError::SQLMapError(format!(
+                    "Failed to query get_legacy_events SQL statement: {err:?}"
+                )))
+            }
+        };
+        let mut list = Vec::new();
+        for row in rows {
+            match row {
+                Ok((timestamp_str, duration_float, data_str)) => {
+                    let timestamp_str = timestamp_str.replace(' ', "T");
+                    let timestamp = match DateTime::parse_from_rfc3339(&timestamp_str) {
+                        Ok(timestamp) => timestamp.with_timezone(&Utc),
+                        Err(err) => panic!("Timestamp string {timestamp_str}: {err:?}"),
+                    };
+
+                    let duration_ns = (duration_float * 1_000_000_000.0) as i64;
+
+                    let data: serde_json::map::Map<String, serde_json::Value> =
+                        match serde_json::from_str(&data_str) {
+                            Ok(data) => data,
+                            Err(err) => {
+                                warn!(
+                                    "Unable to parse JSON data in event from bucket {}\n{}\n{}",
+                                    bucket_id, err, data_str
+                                );
+                                continue;
+                            }
+                        };
+
+                    let event = Event {
+                        id: None,
+                        timestamp,
+                        duration: Duration::nanoseconds(duration_ns),
+                        data,
+                    };
+                    list.push(event)
+                }
+                Err(err) => panic!("Corrupt event in bucket {bucket_id}: {err}"),
+            };
+        }
+        Ok(list)
+    }
+
+    pub fn legacy_import(
+        new_ds: &mut DatastoreInstance,
+        new_conn: &Connection,
+    ) -> Result<(), LegacyDatastoreImportError> {
+        let legacy_db_path = dbfile_path();
+        if !legacy_db_path.exists() {
+            info!("Did not find an old database, skipping legacy import");
+            return Ok(());
+        }
+        info!("Importing legacy DB");
+        let legacy_conn =
+            Connection::open(legacy_db_path).expect("Unable to open corrupt legacy db file");
+
+        let buckets = get_legacy_buckets(&legacy_conn)?;
+        for bucket in &buckets {
+            println!("Importing legacy bucket: {bucket:?}");
+            match new_ds.create_bucket(new_conn, bucket.clone()) {
+                Ok(_) => (),
+                Err(err) => panic!("Failed to create bucket '{}': {:?}", bucket.id, err),
+            };
+            let events = get_legacy_events(&legacy_conn, bucket.bid.unwrap())?;
+            let num_events = events.len(); // Save len before lending events to insert_events
+            println!("Importing {} events for {}", num_events, bucket.id);
+            match new_ds.insert_events(new_conn, &bucket.id, events) {
+                Ok(_) => (),
+                Err(err) => panic!(
+                    "Failed to insert events to bucket '{}': {:?}",
+                    bucket.id, err
+                ),
+            };
+            //assert_eq!(new_ds.get_events(new_conn, &bucket.id, None, None, None).unwrap().len(), num_events);
+        }
+        assert_eq!(new_ds.get_buckets().len(), buckets.len());
+        Ok(())
+    }
+
+    /* This test is disabled because it requires manual set-up of a old aw-server database
+     * Can be run with:
+     * cargo test --features legacy_import,legacy_import_tests */
+    #[test]
+    #[cfg_attr(not(feature = "legacy_import_tests"), ignore)]
+    fn test_legacy_import() {
+        assert!(dbfile_path().exists());
+        let mut new_conn =
+            Connection::open_in_memory().expect("Unable to open corrupt legacy db file");
+        let mut ds = DatastoreInstance::new(&mut new_conn, true).unwrap();
+        assert!(
+            ds.ensure_legacy_import(&new_conn).unwrap(),
+            "Failed to ensure legacy import"
+        );
+        let buckets = ds.get_buckets();
+        assert!(!buckets.is_empty());
+        let mut num_events = 0;
+        for (bucket_id, _bucket) in buckets {
+            let events = ds
+                .get_events(&new_conn, &bucket_id, None, None, Some(1000))
+                .unwrap();
+            num_events += events.len();
+        }
+        assert!(num_events > 0);
+    }
+}
+
+#[cfg(target_os = "android")]
+mod import {
+    use super::LegacyDatastoreImportError;
+    use crate::datastore::DatastoreInstance;
+    use rusqlite::Connection;
+
+    pub fn legacy_import(
+        _new_ds: &mut DatastoreInstance,
+        _new_conn: &Connection,
+    ) -> Result<(), LegacyDatastoreImportError> {
+        Ok(())
+    }
+}

aw-datastore/src/lib.rs

@@ -0,0 +1,61 @@
+#[macro_use]
+extern crate log;
+
+use std::fmt;
+
+#[macro_export]
+macro_rules! json_map {
+    { $( $key:literal : $value:expr),* } => {{
+        use serde_json::Value;
+        use serde_json::map::Map;
+        #[allow(unused_mut)]
+        let mut map : Map<String, Value> = Map::new();
+        $(
+          map.insert( $key.to_string(), json!($value) );
+        )*
+        map
+    }};
+}
+
+mod datastore;
+mod export;
+mod legacy_import;
+mod privacy_filter;
+mod worker;
+
+pub use self::datastore::DatastoreInstance;
+pub use self::worker::Datastore;
+
+#[derive(Clone)]
+pub enum DatastoreMethod {
+    Memory(),
+    File(String),
+    /// Encrypted SQLite file using SQLCipher. Only available with the
+    /// `encryption` or `encryption-vendored` feature flags.
+    #[cfg(any(feature = "encryption", feature = "encryption-vendored"))]
+    FileEncrypted(String, zeroize::Zeroizing<String>), // (path, key)
+}
+
+impl fmt::Debug for DatastoreMethod {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            DatastoreMethod::Memory() => write!(f, "Memory()"),
+            DatastoreMethod::File(p) => write!(f, "File({p:?})"),
+            #[cfg(any(feature = "encryption", feature = "encryption-vendored"))]
+            DatastoreMethod::FileEncrypted(p, _) => write!(f, "FileEncrypted({p:?}, <redacted>)"),
+        }
+    }
+}
+
+/* TODO: Implement this as a proper error */
+#[derive(Debug, Clone)]
+pub enum DatastoreError {
+    NoSuchBucket(String),
+    BucketAlreadyExists(String),
+    NoSuchKey(String),
+    MpscError,
+    InternalError(String),
+    // Errors specific to when migrate is disabled
+    Uninitialized(String),
+    OldDbVersion(String),
+}

aw-datastore/src/privacy_filter.rs

@@ -0,0 +1,422 @@
+use std::sync::OnceLock;
+
+use serde::{Deserialize, Serialize};
+use serde_json::map::Map;
+use serde_json::Value;
+
+use aw_models::Event;
+
+/// Action to take when a rule matches an event.
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
+#[serde(rename_all = "snake_case")]
+pub enum PrivacyFilterAction {
+    /// Drop the entire event from storage
+    Drop,
+    /// Redact a specific field's value with a replacement
+    Redact,
+}
+
+/// A single privacy filter rule.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct PrivacyFilterRule {
+    /// If true, this rule is active
+    pub enabled: bool,
+    /// Only apply to buckets whose ID starts with this prefix (e.g. "aw-watcher-window")
+    pub bucket_prefix: Option<String>,
+    /// Dotted path to the event data field to check (e.g. "title")
+    pub field: Option<String>,
+    /// Regex pattern to match against the field value
+    pub pattern: String,
+    /// What to do when matched
+    pub action: PrivacyFilterAction,
+    /// Replacement text for the redact action
+    pub replacement: Option<String>,
+    /// Pre-compiled regex, populated lazily on first match. Not serialized.
+    #[serde(skip)]
+    regex_cache: OnceLock<Option<regex::Regex>>,
+}
+
+impl PartialEq for PrivacyFilterRule {
+    fn eq(&self, other: &Self) -> bool {
+        self.enabled == other.enabled
+            && self.bucket_prefix == other.bucket_prefix
+            && self.field == other.field
+            && self.pattern == other.pattern
+            && self.action == other.action
+            && self.replacement == other.replacement
+    }
+}
+
+impl PrivacyFilterRule {
+    /// Check if this rule matches a given event in a given bucket.
+    pub fn matches(&self, bucket_id: &str, event: &Event) -> bool {
+        if !self.enabled {
+            return false;
+        }
+
+        // Check bucket prefix if specified
+        if let Some(ref prefix) = self.bucket_prefix {
+            if !bucket_id.starts_with(prefix) {
+                return false;
+            }
+        }
+
+        // Check field pattern if specified
+        if let Some(ref field_path) = self.field {
+            let field_value = resolve_field(&event.data, field_path);
+            match field_value {
+                Some(Value::String(s)) => {
+                    // Compile the regex once and cache it for subsequent calls.
+                    let re = self
+                        .regex_cache
+                        .get_or_init(|| regex::Regex::new(&self.pattern).ok());
+                    re.as_ref().map(|re| re.is_match(s)).unwrap_or(false)
+                }
+                Some(_) | None => false,
+            }
+        } else {
+            true
+        }
+    }
+
+    /// Apply this rule's action to an event.
+    /// Returns None if dropped, Some(event) if kept (possibly redacted).
+    pub fn apply<'a>(&self, event: &'a mut Event) -> Option<&'a mut Event> {
+        match self.action {
+            PrivacyFilterAction::Drop => None,
+            PrivacyFilterAction::Redact => {
+                if let Some(ref replacement) = self.replacement {
+                    if let Some(ref field_path) = self.field {
+                        set_field(
+                            &mut event.data,
+                            field_path,
+                            Value::String(replacement.clone()),
+                        );
+                    }
+                }
+                Some(event)
+            }
+        }
+    }
+}
+
+/// Engine that holds and applies privacy filter rules.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct PrivacyFilterEngine {
+    rules: Vec<PrivacyFilterRule>,
+}
+
+impl PrivacyFilterEngine {
+    pub fn new(rules: Vec<PrivacyFilterRule>) -> Self {
+        PrivacyFilterEngine { rules }
+    }
+
+    /// Example rules for common sensitive data patterns.
+    /// Not applied automatically — use `new()` with these rules to opt in.
+    #[allow(dead_code)]
+    pub fn with_defaults() -> Self {
+        let rules = vec![
+            PrivacyFilterRule {
+                enabled: true,
+                bucket_prefix: Some("aw-watcher-window".to_string()),
+                field: Some("title".to_string()),
+                pattern: r"(?i)(private browsing|incognito)".to_string(),
+                action: PrivacyFilterAction::Drop,
+                replacement: None,
+                regex_cache: OnceLock::new(),
+            },
+            PrivacyFilterRule {
+                enabled: true,
+                bucket_prefix: Some("aw-watcher-window".to_string()),
+                field: Some("title".to_string()),
+                pattern: r"(?i).*banking.*".to_string(),
+                action: PrivacyFilterAction::Redact,
+                replacement: Some("REDACTED".to_string()),
+                regex_cache: OnceLock::new(),
+            },
+        ];
+        PrivacyFilterEngine { rules }
+    }
+
+    /// Parse rules from a JSON string (as stored in settings).
+    pub fn from_json(json_str: &str) -> Result<Self, String> {
+        let rules: Vec<PrivacyFilterRule> = serde_json::from_str(json_str)
+            .map_err(|e| format!("Failed to parse privacy filter rules: {e}"))?;
+        for rule in &rules {
+            if rule.action == PrivacyFilterAction::Redact
+                && rule.replacement.as_deref().is_none_or(str::is_empty)
+            {
+                return Err(format!(
+                    "Redact rule with pattern {:?} is missing `replacement` — add a non-empty replacement string or use action=drop",
+                    rule.pattern
+                ));
+            }
+            if rule.action == PrivacyFilterAction::Redact && rule.field.is_none() {
+                return Err(format!(
+                    "Redact rule with pattern {:?} is missing `field` — specify which data field to redact (e.g. \"title\")",
+                    rule.pattern
+                ));
+            }
+            if rule.action == PrivacyFilterAction::Drop && rule.field.is_none() {
+                return Err(format!(
+                    "Drop rule with pattern {:?} is missing `field` — without a field path the pattern is never evaluated and every event in the matching bucket is dropped (specify a dotted field path, e.g. \"title\")",
+                    rule.pattern
+                ));
+            }
+            if let Err(e) = regex::Regex::new(&rule.pattern) {
+                return Err(format!(
+                    "Rule with pattern {:?} has an invalid regex: {e}",
+                    rule.pattern
+                ));
+            }
+        }
+        Ok(PrivacyFilterEngine { rules })
+    }
+
+    /// Serialize rules to JSON string.
+    #[allow(dead_code)]
+    pub fn to_json(&self) -> Result<String, String> {
+        serde_json::to_string(&self.rules)
+            .map_err(|e| format!("Failed to serialize privacy filter rules: {e}"))
+    }
+
+    /// Filter a single event for a given bucket.
+    /// Applies all matching rules. Returns None if dropped, Some (possibly redacted) event if kept.
+    pub fn filter_event(&self, bucket_id: &str, event: Event) -> Option<Event> {
+        let mut event = event;
+        for rule in &self.rules {
+            if rule.matches(bucket_id, &event) {
+                match rule.apply(&mut event) {
+                    Some(_) => {}        // Redacted — continue applying other rules
+                    None => return None, // Dropped
+                }
+            }
+        }
+        Some(event)
+    }
+
+    /// Filter a batch of events for a given bucket.
+    pub fn filter_events(&self, bucket_id: &str, events: Vec<Event>) -> Vec<Event> {
+        events
+            .into_iter()
+            .filter_map(|e| self.filter_event(bucket_id, e))
+            .collect()
+    }
+}
+
+/// Resolve a dotted field path (e.g. "title", "data.url") from a serde_json Map.
+fn resolve_field<'a>(data: &'a Map<String, Value>, path: &str) -> Option<&'a Value> {
+    let parts: Vec<&str> = path.split('.').collect();
+    let mut current: &Map<String, Value> = data;
+    for (i, part) in parts.iter().enumerate() {
+        let val = current.get(*part)?;
+        if i == parts.len() - 1 {
+            return Some(val);
+        }
+        match val {
+            Value::Object(map) => current = map,
+            _ => return None,
+        }
+    }
+    None
+}
+
+/// Set a field value at a dotted path in a serde_json Map.
+fn set_field(data: &mut Map<String, Value>, path: &str, value: Value) {
+    let parts: Vec<&str> = path.split('.').collect();
+    let mut current: &mut Map<String, Value> = data;
+    for (i, part) in parts.iter().enumerate() {
+        if i == parts.len() - 1 {
+            current.insert(part.to_string(), value);
+            return;
+        }
+        current = match current
+            .entry(part.to_string())
+            .or_insert_with(|| Value::Object(Map::new()))
+            .as_object_mut()
+        {
+            Some(m) => m,
+            None => return, // intermediate segment is not an object — skip silently
+        };
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use chrono::Utc;
+    use serde_json::json;
+
+    fn test_event(title: &str) -> Event {
+        Event {
+            id: None,
+            timestamp: Utc::now(),
+            duration: chrono::Duration::seconds(1),
+            data: json_map! {"title": json!(title), "app": json!("Firefox")},
+        }
+    }
+
+    #[test]
+    fn test_drop_incognito_window() {
+        let engine = PrivacyFilterEngine::with_defaults();
+        let event = test_event("Private Browsing - Mozilla Firefox");
+        let rule = &engine.rules[0];
+        assert!(rule.matches("aw-watcher-window", &event));
+    }
+
+    #[test]
+    fn test_allow_normal_window() {
+        let engine = PrivacyFilterEngine::with_defaults();
+        let event = test_event("GitHub - Mozilla Firefox");
+        let rule = &engine.rules[0];
+        assert!(!rule.matches("aw-watcher-window", &event));
+    }
+
+    #[test]
+    fn test_redact_banking() {
+        let engine = PrivacyFilterEngine::with_defaults();
+        let mut event = test_event("Online Banking - My Account Balance");
+        let rule = &engine.rules[1];
+        assert!(rule.matches("aw-watcher-window", &event));
+        let result = rule.apply(&mut event);
+        assert!(result.is_some());
+        assert_eq!(
+            result.unwrap().data.get("title").unwrap().as_str().unwrap(),
+            "REDACTED"
+        );
+    }
+
+    #[test]
+    fn test_bucket_scoping() {
+        let rule = PrivacyFilterRule {
+            enabled: true,
+            bucket_prefix: Some("aw-watcher-window".to_string()),
+            field: Some("title".to_string()),
+            pattern: r"(?i)(private browsing|incognito)".to_string(),
+            action: PrivacyFilterAction::Drop,
+            replacement: None,
+            regex_cache: OnceLock::new(),
+        };
+        let event = test_event("Private Browsing - Mozilla Firefox");
+        assert!(rule.matches("aw-watcher-window", &event));
+        assert!(!rule.matches("aw-watcher-afk", &event));
+    }
+
+    #[test]
+    fn test_disabled_rule() {
+        let rule = PrivacyFilterRule {
+            enabled: false,
+            bucket_prefix: Some("aw-watcher-window".to_string()),
+            field: Some("title".to_string()),
+            pattern: ".*".to_string(),
+            action: PrivacyFilterAction::Drop,
+            replacement: None,
+            regex_cache: OnceLock::new(),
+        };
+        let event = test_event("Anything at all");
+        assert!(!rule.matches("aw-watcher-window", &event));
+    }
+
+    #[test]
+    fn test_invalid_regex_no_panic() {
+        let rule = PrivacyFilterRule {
+            enabled: true,
+            bucket_prefix: Some("aw-watcher-window".to_string()),
+            field: Some("title".to_string()),
+            pattern: r"[invalid".to_string(),
+            action: PrivacyFilterAction::Drop,
+            replacement: None,
+            regex_cache: OnceLock::new(),
+        };
+        let event = test_event("test");
+        assert!(!rule.matches("aw-watcher-window", &event));
+    }
+
+    #[test]
+    fn test_set_field_no_panic_on_non_object_intermediate() {
+        let mut data = serde_json::Map::new();
+        data.insert(
+            "title".to_string(),
+            Value::String("flat string".to_string()),
+        );
+        // "title" is a string, not an object — setting "title.nested" should not panic
+        set_field(
+            &mut data,
+            "title.nested",
+            Value::String("value".to_string()),
+        );
+        // title should remain unchanged
+        assert_eq!(data.get("title").unwrap().as_str().unwrap(), "flat string");
+    }
+
+    #[test]
+    fn test_from_json_redact_without_replacement_is_error() {
+        let json = r#"[{"enabled":true,"pattern":"(?i)secret","action":"redact","field":"title"}]"#;
+        let result = PrivacyFilterEngine::from_json(json);
+        assert!(
+            result.is_err(),
+            "Redact rule without replacement must fail from_json"
+        );
+        assert!(result.unwrap_err().contains("replacement"));
+    }
+
+    #[test]
+    fn test_from_json_redact_with_empty_replacement_is_error() {
+        let json = r#"[{"enabled":true,"pattern":"(?i)secret","action":"redact","field":"title","replacement":""}]"#;
+        let result = PrivacyFilterEngine::from_json(json);
+        assert!(
+            result.is_err(),
+            "Redact rule with empty replacement must fail from_json"
+        );
+        assert!(result.unwrap_err().contains("replacement"));
+    }
+
+    #[test]
+    fn test_from_json_redact_without_field_is_error() {
+        let json = r#"[{"enabled":true,"pattern":"(?i)secret","action":"redact","replacement":"REDACTED"}]"#;
+        let result = PrivacyFilterEngine::from_json(json);
+        assert!(
+            result.is_err(),
+            "Redact rule without field must fail from_json"
+        );
+        assert!(result.unwrap_err().contains("field"));
+    }
+
+    #[test]
+    fn test_from_json_invalid_regex_is_error() {
+        let json = r#"[{"enabled":true,"pattern":"[unclosed","action":"drop","field":"title"}]"#;
+        let result = PrivacyFilterEngine::from_json(json);
+        assert!(
+            result.is_err(),
+            "Rule with invalid regex must fail from_json"
+        );
+    }
+
+    #[test]
+    fn test_from_json_drop_without_field_is_error() {
+        let json = r#"[{"enabled":true,"pattern":"(?i)incognito","action":"drop"}]"#;
+        let result = PrivacyFilterEngine::from_json(json);
+        assert!(
+            result.is_err(),
+            "Drop rule without field must fail from_json"
+        );
+        assert!(result.unwrap_err().contains("field"));
+    }
+
+    #[test]
+    fn test_drop_action() {
+        let rule = PrivacyFilterRule {
+            enabled: true,
+            bucket_prefix: None,
+            field: Some("title".to_string()),
+            pattern: ".*".to_string(),
+            action: PrivacyFilterAction::Drop,
+            replacement: None,
+            regex_cache: OnceLock::new(),
+        };
+        let mut event = test_event("anything");
+        assert!(rule.matches("any-bucket", &event));
+        let result = rule.apply(&mut event);
+        assert!(result.is_none(), "Drop action should return None");
+    }
+}

aw-datastore/src/worker.rs

@@ -0,0 +1,768 @@
+use std::collections::HashMap;
+use std::fmt;
+use std::thread;
+use std::{
+    fs::File,
+    io::{BufWriter, Write},
+};
+
+use chrono::DateTime;
+use chrono::Duration;
+use chrono::Utc;
+
+use rusqlite::Connection;
+use rusqlite::DropBehavior;
+use rusqlite::Transaction;
+use rusqlite::TransactionBehavior;
+
+use aw_models::Bucket;
+use aw_models::Event;
+
+use crate::privacy_filter::PrivacyFilterEngine;
+use crate::DatastoreError;
+use crate::DatastoreInstance;
+use crate::DatastoreMethod;
+
+type RequestSender = mpsc_requests::RequestSender<Command, Result<Response, DatastoreError>>;
+type RequestReceiver = mpsc_requests::RequestReceiver<Command, Result<Response, DatastoreError>>;
+
+#[derive(Clone)]
+pub struct Datastore {
+    requester: RequestSender,
+}
+
+impl fmt::Debug for Datastore {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "Datastore()")
+    }
+}
+
+/*
+ * TODO:
+ * - Allow read requests to go straight through a read-only db connection instead of requesting the
+ * worker thread for better performance?
+ * TODO: Add an separate "Import" request which does an import with an transaction
+ */
+
+#[allow(clippy::large_enum_variant)]
+#[derive(Debug)]
+pub enum Response {
+    Export(File, Option<String>),
+    Empty(),
+    Bucket(Bucket),
+    BucketMap(HashMap<String, Bucket>),
+    Event(Event),
+    EventList(Vec<Event>),
+    Count(i64),
+    KeyValue(String),
+    KeyValues(HashMap<String, String>),
+}
+
+#[allow(clippy::large_enum_variant)]
+#[derive(Debug)]
+pub enum Command {
+    Export(Option<String>, File),
+    CreateBucket(Bucket),
+    DeleteBucket(String),
+    GetBucket(String),
+    GetBuckets(),
+    InsertEvents(String, Vec<Event>),
+    Heartbeat(String, Event, f64),
+    GetEvent(String, i64),
+    GetEvents(
+        String,
+        Option<DateTime<Utc>>,
+        Option<DateTime<Utc>>,
+        Option<u64>,
+        bool,
+    ),
+    GetEventCount(String, Option<DateTime<Utc>>, Option<DateTime<Utc>>),
+    DeleteEventsById(String, Vec<i64>),
+    ForceCommit(),
+    GetKeyValues(String),
+    GetKeyValue(String),
+    SetKeyValue(String, String),
+    DeleteKeyValue(String),
+    RefreshPrivacyFilter(),
+    RenameBucket(String, String),
+    MigrateHostname(String),
+    MigrateTestBucketNames(),
+    Close(),
+}
+
+/// Key the webui writes via POST /0/settings/privacy_filters.
+/// The worker's in-memory PrivacyFilterEngine is the only thing that actually
+/// filters inserts/heartbeats, so every write/delete of this key (and startup)
+/// must reload the engine. RefreshPrivacyFilter exists for explicit reloads.
+const PRIVACY_FILTERS_KEY: &str = "settings.privacy_filters";
+
+fn _unwrap_empty_response(response: Response) -> Result<(), DatastoreError> {
+    match response {
+        Response::Empty() => Ok(()),
+        _ => panic!("Invalid response"),
+    }
+}
+
+struct DatastoreWorker {
+    responder: RequestReceiver,
+    legacy_import: bool,
+    quit: bool,
+    uncommitted_events: usize,
+    commit: bool,
+    last_heartbeat: HashMap<String, Option<Event>>,
+    privacy_engine: PrivacyFilterEngine,
+}
+
+impl DatastoreWorker {
+    pub fn new(
+        responder: mpsc_requests::RequestReceiver<Command, Result<Response, DatastoreError>>,
+        legacy_import: bool,
+    ) -> Self {
+        DatastoreWorker {
+            responder,
+            legacy_import,
+            quit: false,
+            uncommitted_events: 0,
+            commit: false,
+            last_heartbeat: HashMap::new(),
+            privacy_engine: PrivacyFilterEngine::new(vec![]),
+        }
+    }
+
+    /// Replace the in-memory engine from `settings.privacy_filters`.
+    /// Only an absent key clears the engine, so deleting the setting actually
+    /// disables filtering. Parse errors and query errors keep the previous
+    /// engine: unfiltering on a bad save or a transient database error would
+    /// silently store the events these rules exist to keep out.
+    fn reload_privacy_engine(&mut self, ds: &DatastoreInstance, conn: &Connection) {
+        match ds.get_key_value(conn, PRIVACY_FILTERS_KEY) {
+            Ok(json_str) => match PrivacyFilterEngine::from_json(&json_str) {
+                Ok(engine) => self.privacy_engine = engine,
+                Err(e) => warn!("Failed to parse privacy_filters setting: {e}"),
+            },
+            Err(DatastoreError::NoSuchKey(_)) => {
+                self.privacy_engine = PrivacyFilterEngine::new(vec![]);
+            }
+            Err(e) => warn!("Failed to load privacy_filters setting: {e:?}"),
+        }
+    }
+
+    fn work_loop(&mut self, method: DatastoreMethod) {
+        // Open SQLite connection
+        let mut conn = match &method {
+            DatastoreMethod::Memory() => {
+                Connection::open_in_memory().expect("Failed to create in-memory datastore")
+            }
+            DatastoreMethod::File(path) => {
+                Connection::open(path).expect("Failed to create datastore")
+            }
+            #[cfg(any(feature = "encryption", feature = "encryption-vendored"))]
+            DatastoreMethod::FileEncrypted(path, key) => {
+                let conn = Connection::open(path).expect("Failed to create encrypted datastore");
+                conn.pragma_update(None, "key", key.as_str())
+                    .expect("Failed to set SQLCipher encryption key");
+                // PRAGMA key always succeeds even with a wrong passphrase; the
+                // first real SQL query is what fails. Read user_version immediately
+                // to surface an incorrect key as a clear error rather than an
+                // opaque panic later.
+                conn.pragma_query_value(None, "user_version", |row| row.get::<_, i64>(0))
+                    .expect("Failed to open encrypted database: wrong passphrase or not an encrypted database");
+                info!("Opened encrypted database at {}", path);
+                conn
+            }
+        };
+
+        // Set busy timeout to handle concurrent access on systems with strict file locking (e.g., Windows)
+        conn.busy_timeout(std::time::Duration::from_secs(5))
+            .expect("Failed to set busy timeout");
+
+        // WAL turns each commit into a single sequential WAL append+fsync where
+        // delete mode paid two fsyncs plus journal-file churn, and lets future
+        // reader connections proceed while a commit is in flight.
+        // synchronous=FULL is set explicitly (rather than relying on the
+        // default) so a commit remains durable on disk the moment it returns;
+        // with NORMAL the WAL is only synced at checkpoints, which would
+        // silently widen the loss window on power failure.
+        // In-memory databases ignore the request (journal_mode stays "memory").
+        let journal_mode: String = conn
+            .pragma_update_and_check(None, "journal_mode", "WAL", |row| row.get(0))
+            .expect("Failed to query journal_mode");
+        if !matches!(&method, DatastoreMethod::Memory()) && journal_mode != "wal" {
+            warn!("Failed to enable WAL (journal_mode={journal_mode}), continuing without it");
+        }
+        conn.pragma_update(None, "synchronous", "FULL")
+            .expect("Failed to set synchronous=FULL");
+
+        let mut ds = DatastoreInstance::new(&conn, true).unwrap();
+
+        // Load persisted privacy filters before serving inserts. The engine
+        // starts empty; without this, rules saved in a previous process sit
+        // unused until something happens to send RefreshPrivacyFilter.
+        self.reload_privacy_engine(&ds, &conn);
+
+        // Ensure legacy import
+        if self.legacy_import {
+            let transaction = match conn.transaction_with_behavior(TransactionBehavior::Immediate) {
+                Ok(transaction) => transaction,
+                Err(err) => {
+                    panic!("Unable to start immediate transaction on SQLite database! {err}")
+                }
+            };
+            match ds.ensure_legacy_import(&transaction) {
+                Ok(_) => (),
+                Err(err) => error!("Failed to do legacy import: {:?}", err),
+            }
+            match transaction.commit() {
+                Ok(_) => (),
+                Err(err) => {
+                    error!("Failed to commit legacy import transaction: {err}");
+                    // Continue without panicking — legacy import will be retried on
+                    // next startup if the commit didn't persist.
+                }
+            }
+        }
+
+        // Start handling and respond to requests
+        loop {
+            let last_commit_time: DateTime<Utc> = Utc::now();
+            let mut tx: Transaction =
+                match conn.transaction_with_behavior(TransactionBehavior::Immediate) {
+                    Ok(tx) => tx,
+                    Err(err) => {
+                        error!("Unable to start transaction! {:?}", err);
+                        // Wait 1s before retrying
+                        std::thread::sleep(std::time::Duration::from_millis(1000));
+                        continue;
+                    }
+                };
+            // Snapshot BEFORE the request loop. SetKeyValue/DeleteKeyValue
+            // reload the engine from the still-open transaction so a later
+            // insert in the same batch is filtered. If commit fails we restore
+            // this snapshot — not "keep current" (that is the rolled-back
+            // view) and not a durable re-query (a read error would leave the
+            // uncommitted engine in place: fail-open after a rolled-back delete).
+            let privacy_engine_at_tx_start = self.privacy_engine.clone();
+            tx.set_drop_behavior(DropBehavior::Commit);
+
+            self.uncommitted_events = 0;
+            self.commit = false;
+            // ForceCommit and Close promise the caller that their data is
+            // committed, so their acks are held back until the transaction
+            // below has actually committed. Acking first (as before) let a
+            // caller reopen the database and read a pre-commit snapshot —
+            // harmless under the rollback journal's locking, but a real race
+            // in WAL mode where readers never block on the writer.
+            // All other commands are acked immediately: a watcher heartbeat
+            // must not wait up to 15 s for the batch commit.
+            let mut deferred_ack = None;
+            loop {
+                let (request, response_sender) = match self.responder.poll() {
+                    Ok((req, res_sender)) => (req, res_sender),
+                    Err(err) => {
+                        // All references to responder is gone, quit
+                        error!("DB worker quitting, error: {err:?}");
+                        self.quit = true;
+                        break;
+                    }
+                };
+                let ack_after_commit = matches!(request, Command::ForceCommit() | Command::Close());
+                let response = self.handle_request(request, &mut ds, &tx);
+                if ack_after_commit {
+                    // Both commands force a commit, so the loop ends here.
+                    deferred_ack = Some((response_sender, response));
+                    break;
+                }
+                response_sender.respond(response);
+
+                let now: DateTime<Utc> = Utc::now();
+                let commit_interval_passed: bool = (now - last_commit_time) > Duration::seconds(15);
+                if self.commit
+                    || commit_interval_passed
+                    || self.uncommitted_events > 100
+                    || self.quit
+                {
+                    break;
+                };
+            }
+            debug!(
+                "Committing DB! Force commit {}, {} uncommitted events",
+                self.commit, self.uncommitted_events
+            );
+            match tx.commit() {
+                Ok(_) => {
+                    if let Some((sender, response)) = deferred_ack.take() {
+                        sender.respond(response);
+                    }
+                }
+                Err(err) => {
+                    error!(
+                        "Failed to commit datastore transaction ({} events lost): {err}",
+                        self.uncommitted_events
+                    );
+                    // Continue instead of panicking — the worker thread survives this
+                    // transient failure (e.g. SQLITE_FULL on disk full). Note: clients
+                    // already received success responses before the commit, so they won't
+                    // know to retry. Rolled-back events create a gap in the timeline;
+                    // watchers will resume sending heartbeats from current state, but the
+                    // specific batch of events is permanently lost.
+                    //
+                    // Restore the pre-transaction engine. Reloading from the durable
+                    // connection is not enough: if that read fails, keep-on-error would
+                    // preserve the uncommitted engine (empty after a rolled-back delete).
+                    self.privacy_engine = privacy_engine_at_tx_start;
+                    if let Some((sender, _)) = deferred_ack.take() {
+                        sender.respond(Err(DatastoreError::InternalError(format!(
+                            "Failed to commit datastore transaction: {err}"
+                        ))));
+                    }
+                }
+            }
+            if self.quit {
+                break;
+            };
+        }
+        info!("DB Worker thread finished");
+    }
+
+    fn handle_request(
+        &mut self,
+        request: Command,
+        ds: &mut DatastoreInstance,
+        tx: &Transaction,
+    ) -> Result<Response, DatastoreError> {
+        match request {
+            Command::Export(bucket_id, mut file) => {
+                let mut writer = BufWriter::new(&mut file);
+                let name = ds.write_export(tx, bucket_id.as_deref(), &mut writer)?;
+                writer.flush().map_err(|err| {
+                    DatastoreError::InternalError(format!("Failed to flush export: {err}"))
+                })?;
+                drop(writer);
+                Ok(Response::Export(file, name))
+            }
+            Command::CreateBucket(bucket) => match ds.create_bucket(tx, bucket) {
+                Ok(_) => {
+                    self.commit = true;
+                    Ok(Response::Empty())
+                }
+                Err(e) => Err(e),
+            },
+            Command::DeleteBucket(bucketname) => match ds.delete_bucket(tx, &bucketname) {
+                Ok(_) => {
+                    self.commit = true;
+                    Ok(Response::Empty())
+                }
+                Err(e) => Err(e),
+            },
+            Command::GetBucket(bucketname) => match ds.get_bucket(&bucketname) {
+                Ok(b) => Ok(Response::Bucket(b)),
+                Err(e) => Err(e),
+            },
+            Command::GetBuckets() => Ok(Response::BucketMap(ds.get_buckets())),
+            Command::InsertEvents(bucketname, events) => {
+                let filtered = self.privacy_engine.filter_events(&bucketname, events);
+                if filtered.is_empty() {
+                    return Ok(Response::EventList(vec![]));
+                }
+                match ds.insert_events(tx, &bucketname, filtered) {
+                    Ok(events) => {
+                        self.uncommitted_events += events.len();
+                        self.last_heartbeat.insert(bucketname.to_string(), None); // invalidate last_heartbeat cache
+                        Ok(Response::EventList(events))
+                    }
+                    Err(e) => Err(e),
+                }
+            }
+            Command::Heartbeat(bucketname, event, pulsetime) => {
+                // Apply privacy filter to heartbeat
+                let filtered = match self.privacy_engine.filter_event(&bucketname, event.clone()) {
+                    Some(event) => event,
+                    None => {
+                        // Heartbeat dropped by filter — return last cached event so the
+                        // watcher's heartbeat-merge state machine continues correctly.
+                        // Fall back to the incoming event itself if no prior event is cached
+                        // (avoids returning a zero-timestamp default Event).
+                        let last = self
+                            .last_heartbeat
+                            .get(&bucketname)
+                            .and_then(|e| e.clone())
+                            .unwrap_or(event);
+                        return Ok(Response::Event(last));
+                    }
+                };
+                match ds.heartbeat(
+                    tx,
+                    &bucketname,
+                    filtered,
+                    pulsetime,
+                    &mut self.last_heartbeat,
+                ) {
+                    Ok(e) => {
+                        self.uncommitted_events += 1;
+                        Ok(Response::Event(e))
+                    }
+                    Err(e) => Err(e),
+                }
+            }
+            Command::GetEvent(bucketname, event_id) => {
+                match ds.get_event(tx, &bucketname, event_id) {
+                    Ok(el) => Ok(Response::Event(el)),
+                    Err(e) => Err(e),
+                }
+            }
+            Command::GetEvents(bucketname, starttime_opt, endtime_opt, limit_opt, unclipped) => {
+                let result = if unclipped {
+                    ds.get_events_unclipped(tx, &bucketname, starttime_opt, endtime_opt, limit_opt)
+                } else {
+                    ds.get_events(tx, &bucketname, starttime_opt, endtime_opt, limit_opt)
+                };
+                match result {
+                    Ok(el) => Ok(Response::EventList(el)),
+                    Err(e) => Err(e),
+                }
+            }
+            Command::GetEventCount(bucketname, starttime_opt, endtime_opt) => {
+                match ds.get_event_count(tx, &bucketname, starttime_opt, endtime_opt) {
+                    Ok(n) => Ok(Response::Count(n)),
+                    Err(e) => Err(e),
+                }
+            }
+            Command::DeleteEventsById(bucketname, event_ids) => {
+                match ds.delete_events_by_id(tx, &bucketname, event_ids) {
+                    Ok(()) => Ok(Response::Empty()),
+                    Err(e) => Err(e),
+                }
+            }
+            Command::ForceCommit() => {
+                self.commit = true;
+                Ok(Response::Empty())
+            }
+            Command::GetKeyValues(pattern) => match ds.get_key_values(tx, pattern.as_str()) {
+                Ok(result) => Ok(Response::KeyValues(result)),
+                Err(e) => Err(e),
+            },
+            Command::SetKeyValue(key, data) => match ds.insert_key_value(tx, &key, &data) {
+                Ok(()) => {
+                    if key == PRIVACY_FILTERS_KEY {
+                        self.reload_privacy_engine(ds, tx);
+                    }
+                    Ok(Response::Empty())
+                }
+                Err(e) => Err(e),
+            },
+            Command::GetKeyValue(key) => match ds.get_key_value(tx, &key) {
+                Ok(result) => Ok(Response::KeyValue(result)),
+                Err(e) => Err(e),
+            },
+            Command::DeleteKeyValue(key) => match ds.delete_key_value(tx, &key) {
+                Ok(()) => {
+                    if key == PRIVACY_FILTERS_KEY {
+                        self.reload_privacy_engine(ds, tx);
+                    }
+                    Ok(Response::Empty())
+                }
+                Err(e) => Err(e),
+            },
+            Command::RefreshPrivacyFilter() => {
+                self.reload_privacy_engine(ds, tx);
+                Ok(Response::Empty())
+            }
+            Command::RenameBucket(old_id, new_id) => match ds.rename_bucket(tx, &old_id, &new_id) {
+                Ok(()) => {
+                    self.commit = true;
+                    Ok(Response::Empty())
+                }
+                Err(e) => Err(e),
+            },
+            Command::MigrateHostname(new_hostname) => {
+                match ds.migrate_hostname(tx, &new_hostname) {
+                    Ok(count) => {
+                        if count > 0 {
+                            self.commit = true;
+                        }
+                        Ok(Response::Count(count as i64))
+                    }
+                    Err(e) => Err(e),
+                }
+            }
+            Command::MigrateTestBucketNames() => match ds.migrate_test_bucket_names(tx) {
+                Ok(count) => {
+                    if count > 0 {
+                        self.commit = true;
+                    }
+                    Ok(Response::Count(count as i64))
+                }
+                Err(e) => Err(e),
+            },
+            Command::Close() => {
+                self.quit = true;
+                Ok(Response::Empty())
+            }
+        }
+    }
+}
+
+impl Datastore {
+    pub fn new(dbpath: String, legacy_import: bool) -> Self {
+        let method = DatastoreMethod::File(dbpath);
+        Datastore::_new_internal(method, legacy_import)
+    }
+
+    pub fn new_in_memory(legacy_import: bool) -> Self {
+        let method = DatastoreMethod::Memory();
+        Datastore::_new_internal(method, legacy_import)
+    }
+
+    /// Create an encrypted datastore using SQLCipher.
+    ///
+    /// Requires the `encryption` or `encryption-vendored` feature flag.
+    /// Build with: `cargo build --no-default-features --features encryption`
+    #[cfg(any(feature = "encryption", feature = "encryption-vendored"))]
+    pub fn new_encrypted(dbpath: String, key: String, legacy_import: bool) -> Self {
+        let method = DatastoreMethod::FileEncrypted(dbpath, zeroize::Zeroizing::new(key));
+        Datastore::_new_internal(method, legacy_import)
+    }
+
+    fn _new_internal(method: DatastoreMethod, legacy_import: bool) -> Self {
+        let (requester, responder) =
+            mpsc_requests::channel::<Command, Result<Response, DatastoreError>>();
+        let _thread = thread::spawn(move || {
+            let mut di = DatastoreWorker::new(responder, legacy_import);
+            di.work_loop(method);
+        });
+        Datastore { requester }
+    }
+
+    /// Send a command to the worker thread and wait for its response.
+    ///
+    /// Fails with `InternalError` instead of panicking when the worker thread
+    /// is gone (e.g. it panicked on an earlier request), so callers such as
+    /// HTTP endpoints can degrade to a 5xx response instead of crashing the
+    /// request.
+    fn request(&self, cmd: Command) -> Result<Response, DatastoreError> {
+        let receiver = self.requester.request(cmd).map_err(|e| {
+            DatastoreError::InternalError(format!(
+                "Failed to send request, datastore worker is gone: {e:?}"
+            ))
+        })?;
+        receiver.collect().map_err(|e| {
+            DatastoreError::InternalError(format!(
+                "Failed to receive response, datastore worker died while handling request: {e:?}"
+            ))
+        })?
+    }
+
+    pub fn create_bucket(&self, bucket: &Bucket) -> Result<(), DatastoreError> {
+        let cmd = Command::CreateBucket(bucket.clone());
+        _unwrap_empty_response(self.request(cmd)?)
+    }
+
+    pub fn delete_bucket(&self, bucket_id: &str) -> Result<(), DatastoreError> {
+        let cmd = Command::DeleteBucket(bucket_id.to_string());
+        _unwrap_empty_response(self.request(cmd)?)
+    }
+
+    pub fn get_bucket(&self, bucket_id: &str) -> Result<Bucket, DatastoreError> {
+        let cmd = Command::GetBucket(bucket_id.to_string());
+        match self.request(cmd)? {
+            Response::Bucket(b) => Ok(b),
+            _ => panic!("Invalid response"),
+        }
+    }
+
+    pub fn get_buckets(&self) -> Result<HashMap<String, Bucket>, DatastoreError> {
+        let cmd = Command::GetBuckets();
+        match self.request(cmd)? {
+            Response::BucketMap(bm) => Ok(bm),
+            e => Err(DatastoreError::InternalError(format!(
+                "Invalid response: {e:?}"
+            ))),
+        }
+    }
+
+    /// Write a consistent export to an empty temporary file. The file is
+    /// returned only after serialization and flushing succeed. Event rows are
+    /// serialized individually, including this worker's uncommitted writes.
+    pub fn export_to_file(
+        &self,
+        bucket_id: Option<&str>,
+        file: File,
+    ) -> Result<(File, Option<String>), DatastoreError> {
+        match self.request(Command::Export(bucket_id.map(str::to_owned), file))? {
+            Response::Export(file, name) => Ok((file, name)),
+            _ => panic!("Invalid response"),
+        }
+    }
+
+    pub fn insert_events(
+        &self,
+        bucket_id: &str,
+        events: &[Event],
+    ) -> Result<Vec<Event>, DatastoreError> {
+        let cmd = Command::InsertEvents(bucket_id.to_string(), events.to_vec());
+        match self.request(cmd)? {
+            Response::EventList(events) => Ok(events),
+            _ => panic!("Invalid response"),
+        }
+    }
+
+    pub fn heartbeat(
+        &self,
+        bucket_id: &str,
+        heartbeat: Event,
+        pulsetime: f64,
+    ) -> Result<Event, DatastoreError> {
+        let cmd = Command::Heartbeat(bucket_id.to_string(), heartbeat, pulsetime);
+        match self.request(cmd)? {
+            Response::Event(e) => Ok(e),
+            _ => panic!("Invalid response"),
+        }
+    }
+
+    pub fn get_event(&self, bucket_id: &str, event_id: i64) -> Result<Event, DatastoreError> {
+        let cmd = Command::GetEvent(bucket_id.to_string(), event_id);
+        match self.request(cmd)? {
+            Response::Event(el) => Ok(el),
+            _ => panic!("Invalid response"),
+        }
+    }
+
+    pub fn get_events(
+        &self,
+        bucket_id: &str,
+        starttime_opt: Option<DateTime<Utc>>,
+        endtime_opt: Option<DateTime<Utc>>,
+        limit_opt: Option<u64>,
+    ) -> Result<Vec<Event>, DatastoreError> {
+        let cmd = Command::GetEvents(
+            bucket_id.to_string(),
+            starttime_opt,
+            endtime_opt,
+            limit_opt,
+            false,
+        );
+        match self.request(cmd)? {
+            Response::EventList(el) => Ok(el),
+            _ => panic!("Invalid response"),
+        }
+    }
+
+    pub fn get_events_unclipped(
+        &self,
+        bucket_id: &str,
+        starttime_opt: Option<DateTime<Utc>>,
+        endtime_opt: Option<DateTime<Utc>>,
+        limit_opt: Option<u64>,
+    ) -> Result<Vec<Event>, DatastoreError> {
+        let cmd = Command::GetEvents(
+            bucket_id.to_string(),
+            starttime_opt,
+            endtime_opt,
+            limit_opt,
+            true,
+        );
+        match self.request(cmd)? {
+            Response::EventList(el) => Ok(el),
+            _ => panic!("Invalid response"),
+        }
+    }
+
+    pub fn get_event_count(
+        &self,
+        bucket_id: &str,
+        starttime_opt: Option<DateTime<Utc>>,
+        endtime_opt: Option<DateTime<Utc>>,
+    ) -> Result<i64, DatastoreError> {
+        let cmd = Command::GetEventCount(bucket_id.to_string(), starttime_opt, endtime_opt);
+        match self.request(cmd)? {
+            Response::Count(n) => Ok(n),
+            _ => panic!("Invalid response"),
+        }
+    }
+
+    pub fn delete_events_by_id(
+        &self,
+        bucket_id: &str,
+        event_ids: Vec<i64>,
+    ) -> Result<(), DatastoreError> {
+        let cmd = Command::DeleteEventsById(bucket_id.to_string(), event_ids);
+        _unwrap_empty_response(self.request(cmd)?)
+    }
+
+    pub fn force_commit(&self) -> Result<(), DatastoreError> {
+        let cmd = Command::ForceCommit();
+        _unwrap_empty_response(self.request(cmd)?)
+    }
+
+    pub fn get_key_values(&self, pattern: &str) -> Result<HashMap<String, String>, DatastoreError> {
+        let cmd = Command::GetKeyValues(pattern.to_string());
+        match self.request(cmd)? {
+            Response::KeyValues(value) => Ok(value),
+            _ => panic!("Invalid response"),
+        }
+    }
+
+    pub fn get_key_value(&self, key: &str) -> Result<String, DatastoreError> {
+        let cmd = Command::GetKeyValue(key.to_string());
+        match self.request(cmd)? {
+            Response::KeyValue(kv) => Ok(kv),
+            _ => panic!("Invalid response"),
+        }
+    }
+
+    pub fn set_key_value(&self, key: &str, data: &str) -> Result<(), DatastoreError> {
+        let cmd = Command::SetKeyValue(key.to_string(), data.to_string());
+        _unwrap_empty_response(self.request(cmd)?)
+    }
+
+    pub fn delete_key_value(&self, key: &str) -> Result<(), DatastoreError> {
+        let cmd = Command::DeleteKeyValue(key.to_string());
+        _unwrap_empty_response(self.request(cmd)?)
+    }
+
+    pub fn refresh_privacy_filter(&self) -> Result<(), DatastoreError> {
+        _unwrap_empty_response(self.request(Command::RefreshPrivacyFilter())?)
+    }
+
+    /// Renames a bucket from `old_id` to `new_id`.
+    pub fn rename_bucket(&self, old_id: &str, new_id: &str) -> Result<(), DatastoreError> {
+        let cmd = Command::RenameBucket(old_id.to_string(), new_id.to_string());
+        _unwrap_empty_response(self.request(cmd)?)
+    }
+
+    /// Migrates all buckets whose hostname is "unknown" or "Unknown" to `new_hostname`.
+    /// Returns the number of buckets updated.
+    pub fn migrate_hostname(&self, new_hostname: &str) -> Result<usize, DatastoreError> {
+        let cmd = Command::MigrateHostname(new_hostname.to_string());
+        match self.request(cmd)? {
+            Response::Count(n) => Ok(n as usize),
+            _ => Err(DatastoreError::InternalError(
+                "Unexpected response to MigrateHostname command".to_string(),
+            )),
+        }
+    }
+
+    /// Migrates all buckets whose name starts with `aw-watcher-android-test` to use
+    /// `aw-watcher-android` instead (e.g. debug-build buckets from older app versions).
+    /// Returns the number of buckets updated.
+    pub fn migrate_test_bucket_names(&self) -> Result<usize, DatastoreError> {
+        let cmd = Command::MigrateTestBucketNames();
+        match self.request(cmd)? {
+            Response::Count(n) => Ok(n as usize),
+            _ => Err(DatastoreError::InternalError(
+                "Unexpected response to MigrateTestBucketNames command".to_string(),
+            )),
+        }
+    }
+
+    // Should block until worker has stopped
+    pub fn close(&self) {
+        info!("Sending close request to database");
+        match self.request(Command::Close()) {
+            Ok(Response::Empty()) => (),
+            Ok(_) => panic!("Invalid response"),
+            // Worker already gone means there is nothing left to close
+            Err(e) => warn!("Error closing database: {e:?}"),
+        }
+    }
+}