commit da934b693cdf4f286977e3058ccb91e5e0f22afa Author: upstream-mirror Date: Wed Sep 16 00:19:17 2026 +0500 snapshot of https://github.com/ActivityWatch/aw-server-rust.git (master, 2026-09-16; blobs > 131072B omitted) diff --git a/.codecov.yml b/.codecov.yml new file mode 100644 index 0000000..a7f15d9 --- /dev/null +++ b/.codecov.yml @@ -0,0 +1,8 @@ +coverage: + status: + project: + target: auto + threshold: 1% + patch: + target: auto + threshold: 10% diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..871b07b --- /dev/null +++ b/.dockerignore @@ -0,0 +1,8 @@ +target/ +dist/ +aw-webui/dist/ +aw-webui/node_modules/ +**/node_modules/ +.cov/ +coverage-html/ +.serena/ diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..9a83f12 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,51 @@ +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "monthly" + + # Maintain submodule versions + - package-ecosystem: "gitsubmodule" + directory: "/" + schedule: + interval: "monthly" + + # Too spammy, easier to just manually update lockfile very now and then + #- package-ecosystem: cargo + # directory: "/" + # schedule: + # interval: monthly + # open-pull-requests-limit: 10 + # ignore: + # - dependency-name: libc + # versions: + # - 0.2.84 + # - 0.2.86 + # - dependency-name: serde_json + # versions: + # - 1.0.64 + # - dependency-name: reqwest + # versions: + # - 0.11.1 + # - dependency-name: rocket + # versions: + # - 0.4.7 + # - dependency-name: rocket_contrib + # versions: + # - 0.4.7 + # - dependency-name: regex + # versions: + # - 1.4.3 + # - dependency-name: uuid + # versions: + # - 0.8.2 + # - dependency-name: serde + # versions: + # - 1.0.123 + # - dependency-name: multipart + # versions: + # - 0.17.1 + # - dependency-name: log + # versions: + # - 0.4.14 diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..02d69c4 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,136 @@ +name: Build + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + +jobs: + build: + name: ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest, macOS-latest] + + steps: + - uses: actions/checkout@v7 + - name: Set up Rust + id: toolchain + uses: dtolnay/rust-toolchain@stable + - name: Cache cargo build + uses: actions/cache@v6 + if: runner.os != 'macOS' # cache doesn't seem to behave nicely on macOS, see: https://github.com/ActivityWatch/aw-server-rust/issues/180 + env: + cache-name: cargo-build-target + with: + path: target + # key needs to contain cachekey due to https://github.com/ActivityWatch/aw-server-rust/issues/180 + key: ${{ runner.os }}-${{ env.cache-name }}-${{ steps.toolchain.outputs.cachekey }}-${{ hashFiles('**/Cargo.lock') }} + restore-keys: | + ${{ runner.os }}-${{ env.cache-name }}-${{ steps.toolchain.outputs.cachekey }}- + - name: Build + run: cargo build --workspace --verbose + - name: Run tests + run: cargo test --workspace --verbose + - uses: actions/upload-artifact@v7 + with: + # TODO: These binaries are debug builds + name: binaries-${{runner.os}} + path: | + target/*/aw-server + target/*/aw-server.exe + target/*/aw-sync + target/*/aw-sync.exe + + build-android: + name: Android + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - name: Verify Android build script cwd handling + run: bash ./scripts/test-compile-android-cwd.sh + - name: Set up Rust + uses: dtolnay/rust-toolchain@stable + - name: Cache cargo build + uses: actions/cache@v6 + env: + cache-name: cargo-build-target-android + with: + path: target + key: ${{ runner.os }}-${{ env.cache-name }}-${{ hashFiles('**/Cargo.lock') }} + restore-keys: | + ${{ runner.os }}-${{ env.cache-name }}- + - name: Build + run: | + export ANDROID_NDK_HOME= # needed because GitHub Actions sets it by default... + make android + - uses: actions/upload-artifact@v7 + with: + # TODO: These binaries are debug builds + name: binaries-android + path: | + target/*/*/libaw_server.so + + # Code coverage using tarpaulin + # Works better than grcov, but not as many fancy features (no branch coverage, no LLVM) + # See: https://shift.click/blog/github-actions-rust/#code-coverage + build-coverage-tarpaulin: + name: Code coverage + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - name: Set up Rust + uses: dtolnay/rust-toolchain@stable + + # Note: If you need to combine the coverage info of multiple + # feature sets, you need a `.tarpaulin.toml` config file, see + # the link above for those docs. + - name: Install cargo-tarpaulin + run: cargo install cargo-tarpaulin + + - name: Run tests with coverage + run: cargo tarpaulin --out Xml + + # Note: closed-source code needs to provide a token, + # but open source code does not. + - name: Upload to codecov.io + uses: codecov/codecov-action@v7 + + # Code coverage using grcov + #build-coverage-grcov: + # name: Build with coverage + # runs-on: ubuntu-latest + + # steps: + # - uses: actions/checkout@v7 + # - name: Set up Rust nightly + # uses: actions-rs/toolchain@v1 + # with: + # profile: minimal + # toolchain: nightly + # override: true + # - name: Cache cargo build + # uses: actions/cache@v6 + # env: + # cache-name: cargo-build-target-coverage + # with: + # path: target + # key: ${{ runner.os }}-${{ env.cache-name }}-${{ hashFiles('**/Cargo.lock') }} + # restore-keys: | + # ${{ runner.os }}-${{ env.cache-name }}- + # - name: Install llvm-tools + # run: | + # rustup component add llvm-tools-preview + # - name: Download grcov + # run: | + # curl -L https://github.com/mozilla/grcov/releases/latest/download/grcov-x86_64-unknown-linux-gnu.tar.bz2 | tar jxf - + # - name: Run tests with coverage + # run: | + # # Add cwd to path to find grcov + # export PATH=$PATH:. + # make coverage-lcov COVERAGE_CACHE=1 + # - name: Upload coverage files + # run: bash <(curl -s https://codecov.io/bash) -f target/debug/lcov.info; diff --git a/.github/workflows/dependabot-automerge.yml b/.github/workflows/dependabot-automerge.yml new file mode 100644 index 0000000..0c32126 --- /dev/null +++ b/.github/workflows/dependabot-automerge.yml @@ -0,0 +1,61 @@ +name: Dependabot Auto-merge + +# Run in the trusted `workflow_run` context and merge Dependabot PRs directly +# with the repository token instead of depending on an external PAT. + +on: + workflow_run: + types: + - completed + workflows: + # List all required workflow names here. + - Build + +permissions: + contents: write + pull-requests: write + +jobs: + auto_merge: + name: Auto-merge + runs-on: ubuntu-latest + if: >- + github.event.workflow_run.conclusion == 'success' && + github.event.workflow_run.event == 'pull_request' && + github.actor == 'dependabot[bot]' + + steps: + - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9 + with: + script: | + const pr = context.payload.workflow_run.pull_requests?.[0]; + if (!pr) { + core.setFailed("No pull request found in workflow_run payload."); + return; + } + + const { data: pull } = await github.rest.pulls.get({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: pr.number, + }); + + if (pull.state !== "open") { + core.info(`PR #${pull.number} is already ${pull.state}; nothing to do.`); + return; + } + + if (pull.draft) { + core.info(`PR #${pull.number} is a draft; skipping.`); + return; + } + + await github.rest.pulls.merge({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: pull.number, + merge_method: "squash", + sha: context.payload.workflow_run.head_sha, + }); + + core.info(`Merged Dependabot PR #${pull.number}.`); diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 0000000..463cb81 --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,35 @@ +name: Lint + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + +jobs: + format: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v7 + - name: Set up Rust nightly + uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: nightly + components: rustfmt + override: true + - name: Check formatting + run: cargo fmt -- --check + + clippy: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v7 + - name: Set up Rust stable + uses: dtolnay/rust-toolchain@stable + with: + components: clippy + - name: Run clippy + run: cargo clippy --workspace -- -D warnings diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..e68a3e1 --- /dev/null +++ b/.gitignore @@ -0,0 +1,12 @@ +target +dist +.cargo/config +NDK + +**/*.rs.bk +*.zip +*.profraw + +*.sqlite* +*.db +*.db-journal diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..25d96b0 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "aw-webui"] + path = aw-webui + url = https://github.com/ActivityWatch/aw-webui.git diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..c9946cb --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,25 @@ +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v6.0.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-toml + - id: check-yaml + - id: check-merge-conflict + + - repo: local + hooks: + - id: cargo-fmt + name: cargo fmt + entry: cargo fmt --all -- --check + language: system + types: [rust] + pass_filenames: false + + - id: cargo-clippy + name: cargo clippy + entry: cargo clippy --workspace -- -D warnings + language: system + types: [rust] + pass_filenames: false diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..a08dea7 --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,4129 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "addr2line" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfbe277e56a376000877090da837660b4427aad530e3028d44e0bffe4f89a1c1" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "once_cell", + "version_check", + "zerocopy", +] + +[[package]] +name = "aho-corasick" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" +dependencies = [ + "memchr", +] + +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + +[[package]] +name = "android-tzdata" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" + +[[package]] +name = "android_log-sys" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84521a3cf562bc62942e294181d9eef17eb38ceb8c68677bc49f144e4c3d4f8d" + +[[package]] +name = "android_logger" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c494134f746c14dc653a35a4ea5aca24ac368529da5370ecf41fe0341c35772f" +dependencies = [ + "android_log-sys", + "env_logger", + "log", + "once_cell", +] + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "anes" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" + +[[package]] +name = "anstream" +version = "0.6.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "301af1932e46185686725e0fad2f8f2aa7da69dd70bf6ecc44d6b703844a3933" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "862ed96ca487e809f1c8e5a8447f6ee2cf102f846893800b20cebdf541fc6bbd" + +[[package]] +name = "anstyle-parse" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7644824f0aa2c7b9384579234ef10eb7efb6a0deb83f9630a49594dd9c15c2" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c8bdeb6047d8983be085bab0ba1472e6dc604e7041dbf6fcd5e71523014fae9" +dependencies = [ + "windows-sys 0.59.0", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "403f75924867bb1033c59fbf0797484329750cfbe3c4325cd33127941fabc882" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.59.0", +] + +[[package]] +name = "async-stream" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b5a71a6f37880a80d1d7f19efd781e4b5de42c88f0722cc13bcb6cc2cfe8476" +dependencies = [ + "async-stream-impl", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "async-trait" +version = "0.1.88" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e539d3fca749fcee5236ab05e93a52867dd549cc157c8cb7f99595f3cedffdb5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "atomic" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c59bdb34bc650a32731b31bd8f0829cc15d24a708ee31559e0bb34f2bc320cba" + +[[package]] +name = "atomic" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89cbf775b137e9b968e67227ef7f775587cde3fd31b0d8599dbd0f598a48340" +dependencies = [ + "bytemuck", +] + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "autocfg" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" + +[[package]] +name = "aw-client-rust" +version = "0.1.0" +dependencies = [ + "aw-datastore", + "aw-models", + "aw-server", + "chrono", + "dirs", + "fs4", + "gethostname", + "libc", + "log", + "phf", + "reqwest", + "rocket", + "serde", + "serde_json", + "thiserror 1.0.69", + "tokio", + "tokio-test", + "toml", +] + +[[package]] +name = "aw-datastore" +version = "0.1.0" +dependencies = [ + "aw-models", + "aw-transform", + "chrono", + "criterion", + "dirs", + "log", + "mpsc_requests", + "regex", + "rusqlite", + "serde", + "serde_json", + "tempfile", + "zeroize", +] + +[[package]] +name = "aw-models" +version = "0.1.0" +dependencies = [ + "chrono", + "log", + "schemars", + "serde", + "serde_json", +] + +[[package]] +name = "aw-query" +version = "0.1.0" +dependencies = [ + "aw-datastore", + "aw-models", + "aw-transform", + "chrono", + "criterion", + "fancy-regex", + "log", + "plex", + "proptest", + "serde", + "serde_json", +] + +[[package]] +name = "aw-server" +version = "0.14.0" +dependencies = [ + "android_logger", + "aw-client-rust", + "aw-datastore", + "aw-models", + "aw-query", + "aw-transform", + "chrono", + "clap", + "dirs", + "fern", + "gethostname", + "jemallocator", + "jni 0.20.0", + "lazy_static", + "libc", + "log", + "log-panics", + "openssl-sys", + "regex", + "rocket", + "rocket_cors", + "rust-embed", + "sd-notify", + "serde", + "serde_json", + "subtle", + "tempfile", + "toml", + "uuid", +] + +[[package]] +name = "aw-sync" +version = "0.1.0" +dependencies = [ + "android_logger", + "aw-client-rust", + "aw-datastore", + "aw-models", + "aw-server", + "chrono", + "clap", + "ctrlc", + "dirs", + "gethostname", + "jni 0.21.1", + "log", + "openssl", + "reqwest", + "serde", + "serde_json", + "toml", +] + +[[package]] +name = "aw-transform" +version = "0.1.0" +dependencies = [ + "aw-models", + "chrono", + "criterion", + "fancy-regex", + "log", + "lru", + "serde", + "serde_json", + "url", +] + +[[package]] +name = "backtrace" +version = "0.3.75" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6806a6321ec58106fea15becdad98371e28d92ccbc7c8f1b3b6dd724fe8f1002" +dependencies = [ + "addr2line", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", + "windows-targets 0.52.6", +] + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "binascii" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "383d29d513d8764dcdc42ea295d979eb99c3c9f00607b3692cf68a431f7dca72" + +[[package]] +name = "bit-set" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0700ddab506f33b20a03b13996eccd309a48e5ff77d0d95926aa0210fb4e95f1" +dependencies = [ + "bit-vec 0.6.3", +] + +[[package]] +name = "bit-set" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3" +dependencies = [ + "bit-vec 0.8.0", +] + +[[package]] +name = "bit-vec" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb" + +[[package]] +name = "bit-vec" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" + +[[package]] +name = "bitflags" +version = "2.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bumpalo" +version = "3.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43" + +[[package]] +name = "bytemuck" +version = "1.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c76a5792e44e4abe34d3abf15636779261d45a7450612059293d1d2cfc63422" + +[[package]] +name = "bytes" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" + +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + +[[package]] +name = "cc" +version = "1.2.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "deec109607ca693028562ed836a5f1c4b8bd77755c4e132fc5ce11b0b6211ae7" +dependencies = [ + "shlex", +] + +[[package]] +name = "cesu8" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c" + +[[package]] +name = "cfg-if" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9555578bc9e57714c812a1f84e4fc5b4d21fcb063490c624de019f7464c91268" + +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + +[[package]] +name = "chrono" +version = "0.4.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c469d952047f47f91b68d1cba3f10d63c11d73e4636f24f08daf0278abf01c4d" +dependencies = [ + "android-tzdata", + "iana-time-zone", + "js-sys", + "num-traits", + "serde", + "wasm-bindgen", + "windows-link 0.1.3", +] + +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + +[[package]] +name = "clap" +version = "4.5.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed87a9d530bb41a67537289bafcac159cb3ee28460e0a4571123d2a778a6a882" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.5.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64f4f3f3c77c94aff3c7e9aac9a2ca1974a5adf392a8bb751e827d6d127ab966" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.5.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef4f52386a59ca4c860f7393bcf8abd8dfd91ecccc0f774635ff68e92eeef491" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "clap_lex" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b94f61472cee1439c0b966b47e3aca9ae07e45d070759512cd390ea2bebc6675" + +[[package]] +name = "colorchoice" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" + +[[package]] +name = "colored" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "117725a109d387c937a1533ce01b450cbde6b88abceea8473c4d7a85853cda3c" +dependencies = [ + "lazy_static", + "windows-sys 0.59.0", +] + +[[package]] +name = "combine" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" +dependencies = [ + "bytes", + "memchr", +] + +[[package]] +name = "cookie" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ddef33a339a91ea89fb53151bd0a4689cfce27055c291dfa69945475d22c747" +dependencies = [ + "percent-encoding", + "time", + "version_check", +] + +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "criterion" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2b12d017a929603d80db1831cd3a24082f8137ce19c69e6447f54f5fc8d692f" +dependencies = [ + "anes", + "cast", + "ciborium", + "clap", + "criterion-plot", + "is-terminal", + "itertools", + "num-traits", + "once_cell", + "oorandom", + "plotters", + "rayon", + "regex", + "serde", + "serde_derive", + "serde_json", + "tinytemplate", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" +dependencies = [ + "cast", + "itertools", +] + +[[package]] +name = "crossbeam-channel" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "ctrlc" +version = "3.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46f93780a459b7d656ef7f071fe699c4d3d2cb201c4b24d085b6ddc505276e73" +dependencies = [ + "nix", + "windows-sys 0.59.0", +] + +[[package]] +name = "deranged" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" +dependencies = [ + "powerfmt", +] + +[[package]] +name = "devise" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1d90b0c4c777a2cad215e3c7be59ac7c15adf45cf76317009b7d096d46f651d" +dependencies = [ + "devise_codegen", + "devise_core", +] + +[[package]] +name = "devise_codegen" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71b28680d8be17a570a2334922518be6adc3f58ecc880cbb404eaeb8624fd867" +dependencies = [ + "devise_core", + "quote", +] + +[[package]] +name = "devise_core" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b035a542cf7abf01f2e3c4d5a7acbaebfefe120ae4efc7bde3df98186e4b8af7" +dependencies = [ + "bitflags", + "proc-macro2", + "proc-macro2-diagnostics", + "quote", + "syn", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "dirs" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3e8aa94d75141228480295a7d0e7feb620b1a5ad9f12bc40be62411e38cce4e" +dependencies = [ + "dirs-sys", +] + +[[package]] +name = "dirs-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e01a3366d27ee9890022452ee61b2b63a67e6f13f58900b651ff5665f0bb1fab" +dependencies = [ + "libc", + "option-ext", + "redox_users", + "windows-sys 0.59.0", +] + +[[package]] +name = "displaydoc" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "dyn-clone" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" + +[[package]] +name = "either" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" + +[[package]] +name = "encoding_rs" +version = "0.8.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "env_logger" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cd405aab171cb85d6735e5c8d9db038c17d3ca007a4d2c25f337935c3d90580" +dependencies = [ + "log", + "regex", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "778e2ac28f6c47af28e4907f13ffd1e1ddbd400980a9abd7c8df189bf578a5ad" +dependencies = [ + "libc", + "windows-sys 0.59.0", +] + +[[package]] +name = "fallible-iterator" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" + +[[package]] +name = "fallible-streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" + +[[package]] +name = "fancy-regex" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72cf461f865c862bb7dc573f643dd6a2b6842f7c30b07882b56bd148cc2761b8" +dependencies = [ + "bit-set 0.8.0", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "fastrand" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" + +[[package]] +name = "fern" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4316185f709b23713e41e3195f90edef7fb00c3ed4adc79769cf09cc762a3b29" +dependencies = [ + "colored", + "log", +] + +[[package]] +name = "figment" +version = "0.10.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8cb01cd46b0cf372153850f4c6c272d9cbea2da513e07538405148f95bd789f3" +dependencies = [ + "atomic 0.6.1", + "pear", + "serde", + "toml", + "uncased", + "version_check", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + +[[package]] +name = "form_urlencoded" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "fs4" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8640e34b88f7652208ce9e88b1a37a2ae95227d84abec377ccd3c5cfeb141ed4" +dependencies = [ + "rustix", + "windows-sys 0.59.0", +] + +[[package]] +name = "futures" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" + +[[package]] +name = "futures-io" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" + +[[package]] +name = "futures-sink" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" + +[[package]] +name = "futures-task" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" + +[[package]] +name = "futures-util" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "slab", +] + +[[package]] +name = "generator" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cc16584ff22b460a382b7feec54b23d2908d858152e5739a120b949293bd74e" +dependencies = [ + "cc", + "libc", + "log", + "rustversion", + "windows", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "gethostname" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0176e0459c2e4a1fe232f984bca6890e681076abb9934f6cea7c326f3fc47818" +dependencies = [ + "libc", + "windows-targets 0.48.5", +] + +[[package]] +name = "getrandom" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "335ff9f135e4384c8150d6f27c6daed433577f86b4750418338c01a1a2528592" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "wasi 0.11.1+wasi-snapshot-preview1", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "r-efi", + "wasi 0.14.2+wasi-0.2.4", + "wasm-bindgen", +] + +[[package]] +name = "gimli" +version = "0.31.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" + +[[package]] +name = "glob" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2" + +[[package]] +name = "h2" +version = "0.3.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0beca50380b1fc32983fc1cb4587bfa4bb9e78fc259aad4a0032d2080309222d" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http 0.2.12", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "h2" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "171fefbc92fe4a4de27e0698d6a5b392d6a0e333506bc49133760b3bcf948733" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http 1.3.1", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "half" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "459196ed295495a68f7d7fe1d84f6c4b7ff0e21fe3017b2f283c6fac3ad803c9" +dependencies = [ + "cfg-if", + "crunchy", +] + +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +dependencies = [ + "ahash", + "allocator-api2", +] + +[[package]] +name = "hashbrown" +version = "0.15.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5971ac85611da7067dbfcabef3c70ebb5606018acd9e2a3903a0da507521e0d5" + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash", +] + +[[package]] +name = "hashlink" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8094feaf31ff591f651a2664fb9cfd92bba7a60ce3197265e9482ebe753c8f7" +dependencies = [ + "hashbrown 0.14.5", +] + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hermit-abi" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" + +[[package]] +name = "http" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4a85d31aea989eead29a3aaf9e1115a180df8282431156e533de47660892565" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http-body" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" +dependencies = [ + "bytes", + "http 0.2.12", + "pin-project-lite", +] + +[[package]] +name = "http-body" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +dependencies = [ + "bytes", + "http 1.3.1", +] + +[[package]] +name = "http-body-util" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +dependencies = [ + "bytes", + "futures-core", + "http 1.3.1", + "http-body 1.0.1", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "hyper" +version = "0.14.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41dfc780fdec9373c01bae43289ea34c972e40ee3c9f6b3c8801a35f35586ce7" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2 0.3.27", + "http 0.2.12", + "http-body 0.4.6", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "socket2 0.5.10", + "tokio", + "tower-service", + "tracing", + "want", +] + +[[package]] +name = "hyper" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55281c53a1894c864990125767da440a4e630446785086f52523b20033b74498" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "h2 0.4.14", + "http 1.3.1", + "http-body 1.0.1", + "httparse", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" +dependencies = [ + "http 1.3.1", + "hyper 1.10.1", + "hyper-util", + "rustls", + "rustls-native-certs", + "tokio", + "tokio-rustls", + "tower-service", +] + +[[package]] +name = "hyper-tls" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" +dependencies = [ + "bytes", + "http-body-util", + "hyper 1.10.1", + "hyper-util", + "native-tls", + "tokio", + "tokio-native-tls", + "tower-service", +] + +[[package]] +name = "hyper-util" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +dependencies = [ + "base64", + "bytes", + "futures-channel", + "futures-util", + "http 1.3.1", + "http-body 1.0.1", + "hyper 1.10.1", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2 0.6.0", + "system-configuration", + "tokio", + "tower-service", + "tracing", + "windows-registry", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.63" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0c919e5debc312ad217002b8048a17b7d83f80703865bbfcfebb0458b0b27d8" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "icu_collections" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "200072f5d0e3614556f94a9930d5dc3e0662a652823904c3a75dc3b0af7fee47" +dependencies = [ + "displaydoc", + "potential_utf", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cde2700ccaed3872079a65fb1a78f6c0a36c91570f28755dda67bc8f7d9f00a" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "436880e8e18df4d7bbc06d58432329d6458cc84531f7ac5f024e93deadb37979" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00210d6893afc98edb752b664b8890f0ef174c8adbb8d0be9710fa66fbbf72d3" + +[[package]] +name = "icu_properties" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "016c619c1eeb94efb86809b015c58f479963de65bdb6253345c1a1276f22e32b" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "potential_utf", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "298459143998310acd25ffe6810ed544932242d3f07083eee1084d83a71bd632" + +[[package]] +name = "icu_provider" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03c80da27b5f4187909049ee2d72f276f0d9f99a42c306bd0131ecfe04d8e5af" +dependencies = [ + "displaydoc", + "icu_locale_core", + "stable_deref_trait", + "tinystr", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "idna" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "686f825264d630750a544639377bae737628043f20d38bbc029e8f29ea968a7e" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "indexmap" +version = "2.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe4cd85333e22411419a0bcae1297d25e58c9443848b11dc6a86fefe8c78a661" +dependencies = [ + "equivalent", + "hashbrown 0.15.4", + "serde", +] + +[[package]] +name = "inlinable_string" +version = "0.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8fae54786f62fb2918dcfae3d568594e50eb9b5c25bf04371af6fe7516452fb" + +[[package]] +name = "io-uring" +version = "0.7.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d93587f37623a1a17d94ef2bc9ada592f5465fe7732084ab7beefabe5c77c0c4" +dependencies = [ + "bitflags", + "cfg-if", + "libc", +] + +[[package]] +name = "ipnet" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130" + +[[package]] +name = "is-terminal" +version = "0.4.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e04d7f318608d35d4b61ddd75cbdaee86b023ebe2bd5a66ee0915f0bf93095a9" +dependencies = [ + "hermit-abi", + "libc", + "windows-sys 0.59.0", +] + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" + +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" + +[[package]] +name = "jemalloc-sys" +version = "0.5.4+5.3.0-patched" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac6c1946e1cea1788cbfde01c993b52a10e2da07f4bac608228d1bed20bfebf2" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "jemallocator" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0de374a9f8e63150e6f5e8a60cc14c668226d7a347d8aee1a45766e3c4dd3bc" +dependencies = [ + "jemalloc-sys", + "libc", +] + +[[package]] +name = "jni" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "039022cdf4d7b1cf548d31f60ae783138e5fd42013f6271049d7df7afadef96c" +dependencies = [ + "cesu8", + "combine", + "jni-sys", + "log", + "thiserror 1.0.69", + "walkdir", +] + +[[package]] +name = "jni" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a87aa2bb7d2af34197c04845522473242e1aa17c12f4935d5856491a7fb8c97" +dependencies = [ + "cesu8", + "cfg-if", + "combine", + "jni-sys", + "log", + "thiserror 1.0.69", + "walkdir", + "windows-sys 0.45.0", +] + +[[package]] +name = "jni-sys" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8eaf4bc02d17cbdd7ff4c7438cafcdf7fb9a4613313ad11b4f8fefe7d3fa0130" + +[[package]] +name = "js-sys" +version = "0.3.77" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f" +dependencies = [ + "once_cell", + "wasm-bindgen", +] + +[[package]] +name = "lalr" +version = "0.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "106d7548f95adbe3019b4fc4954554d7b72535867aa9ce326d2f766b68958de7" + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "libc" +version = "0.2.174" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1171693293099992e19cddea4e8b849964e9846f4acee11b3948bcc337be8776" + +[[package]] +name = "libredox" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "391290121bad3d37fbddad76d8f5d1c1c314cfc646d143d7e07a3086ddff0ce3" +dependencies = [ + "bitflags", + "libc", +] + +[[package]] +name = "libsqlite3-sys" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf4e226dcd58b4be396f7bd3c20da8fdee2911400705297ba7d2d7cc2c30f716" +dependencies = [ + "cc", + "openssl-sys", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "linux-raw-sys" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd945864f07fe9f5371a27ad7b52a172b4b499999f1d97574c9fa68373937e12" + +[[package]] +name = "litemap" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "241eaef5fd12c88705a01fc1066c48c4b36e0dd4377dcdc7ec3942cea7a69956" + +[[package]] +name = "lock_api" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96936507f153605bddfcda068dd804796c84324ed2510809e5b2a624c81da765" +dependencies = [ + "autocfg", + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94" + +[[package]] +name = "log-panics" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68f9dd8546191c1850ecf67d22f5ff00a935b890d0e84713159a55495cc2ac5f" +dependencies = [ + "backtrace", + "log", +] + +[[package]] +name = "loom" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff50ecb28bb86013e935fb6683ab1f6d3a20016f123c76fd4c27470076ac30f5" +dependencies = [ + "cfg-if", + "generator", + "scoped-tls", + "serde", + "serde_json", + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "lru" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b6180140927ee907000b0aa540091f6ea512ead4447c92b8fc35bc72788a5a6" +dependencies = [ + "hashbrown 0.17.1", +] + +[[package]] +name = "lru-slab" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" + +[[package]] +name = "matchers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" +dependencies = [ + "regex-automata", +] + +[[package]] +name = "memchr" +version = "2.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a282da65faaf38286cf3be983213fcf1d2e2a58700e808f83f4ea9a4804bc0" + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", +] + +[[package]] +name = "mio" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78bed444cc8a2160f01cbcf811ef18cac863ad68ae8ca62092e8db51d51c761c" +dependencies = [ + "libc", + "wasi 0.11.1+wasi-snapshot-preview1", + "windows-sys 0.59.0", +] + +[[package]] +name = "mpsc_requests" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd820d6682abb7a602c084e779aaf8052365bc2d25cd36afe81eee69a75484ae" +dependencies = [ + "crossbeam-channel", +] + +[[package]] +name = "multer" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83e87776546dc87511aa5ee218730c92b666d7264ab6ed41f9d215af9cd5224b" +dependencies = [ + "bytes", + "encoding_rs", + "futures-util", + "http 1.3.1", + "httparse", + "memchr", + "mime", + "spin", + "tokio", + "tokio-util", + "version_check", +] + +[[package]] +name = "native-tls" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87de3442987e9dbec73158d5c715e7ad9072fda936bb03d19d7fa10e00520f0e" +dependencies = [ + "libc", + "log", + "openssl", + "openssl-probe 0.1.6", + "openssl-sys", + "schannel", + "security-framework 2.11.1", + "security-framework-sys", + "tempfile", +] + +[[package]] +name = "nix" +version = "0.30.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74523f3a35e05aba87a1d978330aef40f67b0304ac79c1c00b294c9830543db6" +dependencies = [ + "bitflags", + "cfg-if", + "cfg_aliases", + "libc", +] + +[[package]] +name = "nu-ansi-term" +version = "0.50.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +dependencies = [ + "windows-sys 0.59.0", +] + +[[package]] +name = "num-conv" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6673768db2d862beb9b39a78fdcb1a69439615d5794a1be50caa9bc92c81967" + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "num_cpus" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "object" +version = "0.36.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62948e14d923ea95ea2c7c86c71013138b66525b86bdc08d2dcc262bdb497b87" +dependencies = [ + "memchr", +] + +[[package]] +name = "once_cell" +version = "1.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4895175b425cb1f87721b59f0f286c2092bd4af812243672510e1ac53e2e0ad" + +[[package]] +name = "oorandom" +version = "11.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" + +[[package]] +name = "openssl" +version = "0.10.80" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a45fa2aa886c42762255da344f0a0d313e254066c46aad76f300c3d3da62d967" +dependencies = [ + "bitflags", + "cfg-if", + "foreign-types", + "libc", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "openssl-probe" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" + +[[package]] +name = "openssl-probe" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" + +[[package]] +name = "openssl-src" +version = "300.5.1+3.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "735230c832b28c000e3bc117119e6466a663ec73506bc0a9907ea4187508e42a" +dependencies = [ + "cc", +] + +[[package]] +name = "openssl-sys" +version = "0.9.116" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28a22dc7140cda5f096e5e7724a6962ca81a7f8bfd2979f9b18c11af56318c4" +dependencies = [ + "cc", + "libc", + "openssl-src", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "option-ext" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" + +[[package]] +name = "parking_lot" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70d58bf43669b5795d1576d0641cfb6fbb2057bf629506267a92807158584a13" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc838d2a56b5b1a6c25f55575dfc605fabb63bb2365f6c2353ef9159aa69e4a5" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-targets 0.52.6", +] + +[[package]] +name = "pear" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdeeaa00ce488657faba8ebf44ab9361f9365a97bd39ffb8a60663f57ff4b467" +dependencies = [ + "inlinable_string", + "pear_codegen", + "yansi", +] + +[[package]] +name = "pear_codegen" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bab5b985dc082b345f812b7df84e1bef27e7207b39e448439ba8bd69c93f147" +dependencies = [ + "proc-macro2", + "proc-macro2-diagnostics", + "quote", + "syn", +] + +[[package]] +name = "percent-encoding" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" + +[[package]] +name = "phf" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078" +dependencies = [ + "phf_macros", + "phf_shared", +] + +[[package]] +name = "phf_generator" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d" +dependencies = [ + "phf_shared", + "rand 0.8.6", +] + +[[package]] +name = "phf_macros" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f84ac04429c13a7ff43785d75ad27569f2951ce0ffd30a3321230db2fc727216" +dependencies = [ + "phf_generator", + "phf_shared", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "phf_shared" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5" +dependencies = [ + "siphasher", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "pkg-config" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" + +[[package]] +name = "plex" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d8fc09f81816d9530a29fc1a879c7b4a6b94c3a50bb9d5626a58ae15ca34360" +dependencies = [ + "lalr", + "proc-macro2", + "quote", + "redfa", + "syn", +] + +[[package]] +name = "plotters" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" +dependencies = [ + "num-traits", + "plotters-backend", + "plotters-svg", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "plotters-backend" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" + +[[package]] +name = "plotters-svg" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670" +dependencies = [ + "plotters-backend", +] + +[[package]] +name = "potential_utf" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5a7c30837279ca13e7c867e9e40053bc68740f988cb07f7ca6df43cc734b585" +dependencies = [ + "zerovec", +] + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro2" +version = "1.0.95" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02b3e5e68a3a1a02aad3ec490a98007cbc13c37cbe84a3cd7b8e406d76e7f778" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "proc-macro2-diagnostics" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "version_check", + "yansi", +] + +[[package]] +name = "proptest" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b45fcc2344c680f5025fe57779faef368840d0bd1f42f216291f0dc4ace4744" +dependencies = [ + "bit-set 0.8.0", + "bit-vec 0.8.0", + "bitflags", + "num-traits", + "rand 0.9.4", + "rand_chacha 0.9.0", + "rand_xorshift", + "regex-syntax", + "rusty-fork", + "tempfile", + "unarray", +] + +[[package]] +name = "quick-error" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" + +[[package]] +name = "quinn" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e20a958963c291dc322d98411f541009df2ced7b5a4f2bd52337638cfccf20" +dependencies = [ + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2 0.6.0", + "thiserror 2.0.12", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098" +dependencies = [ + "bytes", + "getrandom 0.3.3", + "lru-slab", + "rand 0.9.4", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror 2.0.12", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "addec6a0dcad8a8d96a771f815f0eaf55f9d1805756410b39f5fa81332574cbd" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2 0.6.0", + "tracing", + "windows-sys 0.59.0", +] + +[[package]] +name = "quote" +version = "1.0.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "rand" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a" +dependencies = [ + "libc", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + +[[package]] +name = "rand" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" +dependencies = [ + "rand_chacha 0.9.0", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.16", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.3", +] + +[[package]] +name = "rand_xorshift" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "513962919efc330f829edb2535844d1b912b0fbe2ca165d613e4e8788bb05a5a" +dependencies = [ + "rand_core 0.9.5", +] + +[[package]] +name = "rayon" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "redfa" +version = "0.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba41b4ee12e29433820b330c8ac41d2f01390be8a8ef8ac59ed2b8edeaa7857e" +dependencies = [ + "bit-set 0.5.3", +] + +[[package]] +name = "redox_syscall" +version = "0.5.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5407465600fb0548f1442edf71dd20683c6ed326200ace4b1ef0763521bb3b77" +dependencies = [ + "bitflags", +] + +[[package]] +name = "redox_users" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4e608c6638b9c18977b00b475ac1f28d14e84b27d8d42f70e0bf1e3dec127ac" +dependencies = [ + "getrandom 0.2.16", + "libredox", + "thiserror 2.0.12", +] + +[[package]] +name = "ref-cast" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a0ae411dbe946a674d89546582cea4ba2bb8defac896622d6496f14c23ba5cf" +dependencies = [ + "ref-cast-impl", +] + +[[package]] +name = "ref-cast-impl" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1165225c21bff1f3bbce98f5a1f889949bc902d3575308cc7b0de30b4f6d27c7" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "regex" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" + +[[package]] +name = "reqwest" +version = "0.12.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +dependencies = [ + "base64", + "bytes", + "encoding_rs", + "futures-channel", + "futures-core", + "futures-util", + "h2 0.4.14", + "http 1.3.1", + "http-body 1.0.1", + "http-body-util", + "hyper 1.10.1", + "hyper-rustls", + "hyper-tls", + "hyper-util", + "js-sys", + "log", + "mime", + "native-tls", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-native-certs", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-native-tls", + "tokio-rustls", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.16", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rocket" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a516907296a31df7dc04310e7043b61d71954d703b603cc6867a026d7e72d73f" +dependencies = [ + "async-stream", + "async-trait", + "atomic 0.5.3", + "binascii", + "bytes", + "either", + "figment", + "futures", + "indexmap", + "log", + "memchr", + "multer", + "num_cpus", + "parking_lot", + "pin-project-lite", + "rand 0.8.6", + "ref-cast", + "rocket_codegen", + "rocket_http", + "serde", + "serde_json", + "state", + "tempfile", + "time", + "tokio", + "tokio-stream", + "tokio-util", + "ubyte", + "version_check", + "yansi", +] + +[[package]] +name = "rocket_codegen" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "575d32d7ec1a9770108c879fc7c47815a80073f96ca07ff9525a94fcede1dd46" +dependencies = [ + "devise", + "glob", + "indexmap", + "proc-macro2", + "quote", + "rocket_http", + "syn", + "unicode-xid", + "version_check", +] + +[[package]] +name = "rocket_cors" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfac3a1df83f8d4fc96aa41dba3b86c786417b7fc0f52ec76295df2ba781aa69" +dependencies = [ + "http 0.2.12", + "log", + "regex", + "rocket", + "serde", + "serde_derive", + "unicase", + "unicase_serde", + "url", +] + +[[package]] +name = "rocket_http" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e274915a20ee3065f611c044bd63c40757396b6dbc057d6046aec27f14f882b9" +dependencies = [ + "cookie", + "either", + "futures", + "http 0.2.12", + "hyper 0.14.32", + "indexmap", + "log", + "memchr", + "pear", + "percent-encoding", + "pin-project-lite", + "ref-cast", + "serde", + "smallvec", + "stable-pattern", + "state", + "time", + "tokio", + "uncased", +] + +[[package]] +name = "rusqlite" +version = "0.30.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a78046161564f5e7cd9008aff3b2990b3850dc8e0349119b98e8f251e099f24d" +dependencies = [ + "bitflags", + "chrono", + "fallible-iterator", + "fallible-streaming-iterator", + "hashlink", + "libsqlite3-sys", + "serde_json", + "smallvec", +] + +[[package]] +name = "rust-embed" +version = "8.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "025908b8682a26ba8d12f6f2d66b987584a4a87bc024abc5bbc12553a8cd178a" +dependencies = [ + "rust-embed-impl", + "rust-embed-utils", + "walkdir", +] + +[[package]] +name = "rust-embed-impl" +version = "8.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6065f1a4392b71819ec1ea1df1120673418bf386f50de1d6f54204d836d4349c" +dependencies = [ + "proc-macro2", + "quote", + "rust-embed-utils", + "shellexpand", + "syn", + "walkdir", +] + +[[package]] +name = "rust-embed-utils" +version = "8.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6cc0c81648b20b70c491ff8cce00c1c3b223bb8ed2b5d41f0e54c6c4c0a3594" +dependencies = [ + "sha2", + "walkdir", +] + +[[package]] +name = "rustc-demangle" +version = "0.1.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56f7d92ca342cea22a06f2121d944b4fd82af56988c270852495420f961d4ace" + +[[package]] +name = "rustc-hash" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" + +[[package]] +name = "rustix" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11181fbabf243db407ef8df94a6ce0b2f9a733bd8be4ad02b4eda9602296cac8" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.59.0", +] + +[[package]] +name = "rustls" +version = "0.23.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b" +dependencies = [ + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-native-certs" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dab5152771c58876a2146916e53e35057e1a4dfa2b9df0f0305b07f611fdea4d" +dependencies = [ + "openssl-probe 0.2.1", + "rustls-pki-types", + "schannel", + "security-framework 3.6.0", +] + +[[package]] +name = "rustls-pki-types" +version = "1.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30a7197ae7eb376e574fe940d068c30fe0462554a3ddbe4eca7838e049c937a9" +dependencies = [ + "web-time", + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "rustversion" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a0d197bd2c9dc6e53b84da9556a69ba4cdfab8619eb41a8bd1cc2027a0f6b1d" + +[[package]] +name = "rusty-fork" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc6bf79ff24e648f6da1f8d1f011e9cac26491b619e6b9280f2b47f1774e6ee2" +dependencies = [ + "fnv", + "quick-error", + "tempfile", + "wait-timeout", +] + +[[package]] +name = "ryu" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "schannel" +version = "0.1.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f29ebaa345f945cec9fbbc532eb307f0fdad8161f281b6369539c8d84876b3d" +dependencies = [ + "windows-sys 0.59.0", +] + +[[package]] +name = "schemars" +version = "0.8.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fbf2ae1b8bc8e02df939598064d22402220cd5bbcca1c76f7d6a310974d5615" +dependencies = [ + "chrono", + "dyn-clone", + "schemars_derive", + "serde", + "serde_json", +] + +[[package]] +name = "schemars_derive" +version = "0.8.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32e265784ad618884abaea0600a9adf15393368d840e0222d101a072f3f7534d" +dependencies = [ + "proc-macro2", + "quote", + "serde_derive_internals", + "syn", +] + +[[package]] +name = "scoped-tls" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "sd-notify" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b943eadf71d8b69e661330cb0e2656e31040acf21ee7708e2c238a0ec6af2bf4" +dependencies = [ + "libc", +] + +[[package]] +name = "security-framework" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" +dependencies = [ + "bitflags", + "core-foundation 0.9.4", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework" +version = "3.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d17b898a6d6948c3a8ee4372c17cb384f90d2e6e912ef00895b14fd7ab54ec38" +dependencies = [ + "bitflags", + "core-foundation 0.10.1", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_derive_internals" +version = "0.29.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.142" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "030fedb782600dcbd6f02d479bf0d817ac3bb40d644745b769d6a96bc3afc5a7" +dependencies = [ + "itoa", + "memchr", + "ryu", + "serde", +] + +[[package]] +name = "serde_spanned" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3" +dependencies = [ + "serde", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "shellexpand" +version = "3.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b1fdf65dd6331831494dd616b30351c38e96e45921a27745cf98490458b90bb" +dependencies = [ + "dirs", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "signal-hook-registry" +version = "1.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9203b8055f63a2a00e2f593bb0510367fe707d7ff1e5c872de2f537b339e5410" +dependencies = [ + "libc", +] + +[[package]] +name = "siphasher" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56199f7ddabf13fe5074ce809e7d3f42b42ae711800501b5b16ea82ad029c39d" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" + +[[package]] +name = "socket2" +version = "0.5.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + +[[package]] +name = "socket2" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "233504af464074f9d066d7b5416c5f9b894a5862a6506e306f7b816cdd6f1807" +dependencies = [ + "libc", + "windows-sys 0.59.0", +] + +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" + +[[package]] +name = "stable-pattern" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4564168c00635f88eaed410d5efa8131afa8d8699a612c80c455a0ba05c21045" +dependencies = [ + "memchr", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" + +[[package]] +name = "state" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b8c4a4445d81357df8b1a650d0d0d6fbbbfe99d064aa5e02f3e4022061476d8" +dependencies = [ + "loom", +] + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "2.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17b6f705963418cdb9927482fa304bc562ece2fdd4f616084c50b7023b435a40" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "system-configuration" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" +dependencies = [ + "bitflags", + "core-foundation 0.9.4", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "tempfile" +version = "3.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8a64e3985349f2441a1a9ef0b853f869006c3855f2cda6862a94d26ebb9d6a1" +dependencies = [ + "fastrand", + "getrandom 0.3.3", + "once_cell", + "rustix", + "windows-sys 0.59.0", +] + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "567b8a2dae586314f7be2a752ec7474332959c6460e02bde30d702a66d488708" +dependencies = [ + "thiserror-impl 2.0.12", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thread_local" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "time" +version = "0.3.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "743bd48c283afc0388f9b8827b976905fb217ad9e647fae3a379a9283c4def2c" +dependencies = [ + "deranged", + "itoa", + "num-conv", + "powerfmt", + "serde_core", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7694e1cfe791f8d31026952abf09c69ca6f6fa4e1a1229e18988f06a04a12dca" + +[[package]] +name = "time-macros" +version = "0.2.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e70e4c5a0e0a8a4823ad65dfe1a6930e4f4d756dcd9dd7939022b5e8c501215" +dependencies = [ + "num-conv", + "time-core", +] + +[[package]] +name = "tinystr" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d4f6d1145dcb577acf783d4e601bc1d76a13337bb54e6233add580b07344c8b" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "tinyvec" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.47.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89e49afdadebb872d3145a5638b59eb0691ea23e46ca484037cfab3b76b95038" +dependencies = [ + "backtrace", + "bytes", + "io-uring", + "libc", + "mio", + "pin-project-lite", + "signal-hook-registry", + "slab", + "socket2 0.6.0", + "tokio-macros", + "windows-sys 0.59.0", +] + +[[package]] +name = "tokio-macros" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-stream" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eca58d7bba4a75707817a2c44174253f9236b2d5fbd055602e9d5c07c139a047" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tokio-test" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2468baabc3311435b55dd935f702f42cd1b8abb7e754fb7dfb16bd36aa88f9f7" +dependencies = [ + "async-stream", + "bytes", + "futures-core", + "tokio", + "tokio-stream", +] + +[[package]] +name = "tokio-util" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66a539a9ad6d5d281510d5bd368c973d636c02dbf8a67300bfb6b950696ad7df" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "toml" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.22.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" +dependencies = [ + "indexmap", + "serde", + "serde_spanned", + "toml_datetime", + "toml_write", + "winnow", +] + +[[package]] +name = "toml_write" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" + +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-http" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" +dependencies = [ + "bitflags", + "bytes", + "futures-util", + "http 1.3.1", + "http-body 1.0.1", + "pin-project-lite", + "tower", + "tower-layer", + "tower-service", + "url", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81383ab64e72a7a8b8e13130c49e3dab29def6d0c7d76a03087b3cf71c5c6903" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d12581f227e93f094d3af2ae690a574abb8a2b9b7a96e7cfe9647b2b617678" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2054a14f5307d601f88daf0553e1cbf472acc4f2c51afab632431cdcd72124d5" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex-automata", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "typenum" +version = "1.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1dccffe3ce07af9386bfd29e80c0ab1a8205a2fc34e4bcd40364df902cfa8f3f" + +[[package]] +name = "ubyte" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f720def6ce1ee2fc44d40ac9ed6d3a59c361c80a75a7aa8e75bb9baed31cf2ea" +dependencies = [ + "serde", +] + +[[package]] +name = "unarray" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" + +[[package]] +name = "uncased" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1b88fcfe09e89d3866a5c11019378088af2d24c3fbd4f0543f96b479ec90697" +dependencies = [ + "serde", + "version_check", +] + +[[package]] +name = "unicase" +version = "2.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75b844d17643ee918803943289730bec8aac480150456169e647ed0b576ba539" + +[[package]] +name = "unicase_serde" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ef53697679d874d69f3160af80bc28de12730a985d57bdf2b47456ccb8b11f1" +dependencies = [ + "serde", + "unicase", +] + +[[package]] +name = "unicode-ident" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" + +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "url" +version = "2.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32f8b686cadd1473f4bd0117a5d28d36b1ade384ea9b5069a1c40aefed7fda60" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", +] + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "uuid" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3cf4199d1e5d15ddd86a694e4d0dffa9c323ce759fea589f00fef9d81cc1931d" +dependencies = [ + "getrandom 0.3.3", + "js-sys", + "serde", + "wasm-bindgen", +] + +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "wait-timeout" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ac3b126d3914f9849036f826e054cbabdc8519970b8998ddaf3b5bd3c65f11" +dependencies = [ + "libc", +] + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasi" +version = "0.14.2+wasi-0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3" +dependencies = [ + "wit-bindgen-rt", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1edc8929d7499fc4e8f0be2262a241556cfc54a0bea223790e71446f2aab1ef5" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f0a0651a5c2bc21487bde11ee802ccaf4c51935d0d3d42a6101f98161700bc6" +dependencies = [ + "bumpalo", + "log", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "555d470ec0bc3bb57890405e5d4322cc9ea83cebb085523ced7be4144dac1e61" +dependencies = [ + "cfg-if", + "js-sys", + "once_cell", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fe63fc6d09ed3792bd0897b314f53de8e16568c2b3f7982f468c0bf9bd0b407" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a05d73b933a847d6cccdda8f838a22ff101ad9bf93e33684f39c1f5f0eece3d" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "web-sys" +version = "0.3.77" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33b6dd2ef9186f1f2072e409e99cd22a975331a6b3591b12c764e0e55c60d5d2" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "winapi-util" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" +dependencies = [ + "windows-sys 0.59.0", +] + +[[package]] +name = "windows" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e686886bc078bc1b0b600cac0147aadb815089b6e4da64016cbd754b6342700f" +dependencies = [ + "windows-targets 0.48.5", +] + +[[package]] +name = "windows-core" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0fdd3ddb90610c7638aa2b3a3ab2904fb9e5cdbecc643ddb3647212781c4ae3" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link 0.1.3", + "windows-result 0.3.4", + "windows-strings 0.4.2", +] + +[[package]] +name = "windows-implement" +version = "0.60.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a47fddd13af08290e67f4acabf4b459f647552718f683a7b415d290ac744a836" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-interface" +version = "0.59.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd9211b69f8dcdfa817bfd14bf1c97c9188afa36f4750130fcdf3f400eca9fa8" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-link" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e6ad25900d524eaabdbbb96d20b4311e1e7ae1699af4fb28c17ae66c80d798a" + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-registry" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02752bf7fbdcce7f2a27a742f798510f3e5ad88dbe84871e5168e2120c3d5720" +dependencies = [ + "windows-link 0.2.1", + "windows-result 0.4.1", + "windows-strings 0.5.1", +] + +[[package]] +name = "windows-result" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56f42bd332cc6c8eac5af113fc0c1fd6a8fd2aa08a0119358686e5160d0586c6" +dependencies = [ + "windows-link 0.1.3", +] + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link 0.2.1", +] + +[[package]] +name = "windows-strings" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56e6c93f3a0c3b36176cb1327a4958a0353d5d166c2a35cb268ace15e91d3b57" +dependencies = [ + "windows-link 0.1.3", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link 0.2.1", +] + +[[package]] +name = "windows-sys" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +dependencies = [ + "windows-targets 0.42.2", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-targets" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" +dependencies = [ + "windows_aarch64_gnullvm 0.42.2", + "windows_aarch64_msvc 0.42.2", + "windows_i686_gnu 0.42.2", + "windows_i686_msvc 0.42.2", + "windows_x86_64_gnu 0.42.2", + "windows_x86_64_gnullvm 0.42.2", + "windows_x86_64_msvc 0.42.2", +] + +[[package]] +name = "windows-targets" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +dependencies = [ + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" + +[[package]] +name = "windows_i686_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" + +[[package]] +name = "windows_i686_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "winnow" +version = "0.7.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3edebf492c8125044983378ecb5766203ad3b4c2f7a922bd7dd207f6d443e95" +dependencies = [ + "memchr", +] + +[[package]] +name = "wit-bindgen-rt" +version = "0.39.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1" +dependencies = [ + "bitflags", +] + +[[package]] +name = "writeable" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea2f10b9bb0928dfb1b42b65e1f9e36f7f54dbdf08457afefb38afcdec4fa2bb" + +[[package]] +name = "yansi" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" +dependencies = [ + "is-terminal", +] + +[[package]] +name = "yoke" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f41bb01b8226ef4bfd589436a297c53d118f65921786300e427be8d487695cc" +dependencies = [ + "serde", + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38da3c9736e16c5d3c8c597a9aaa5d1fa565d0532ae05e27c24aa62fb32c0ab6" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zerocopy" +version = "0.8.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1039dd0d3c310cf05de012d8a39ff557cb0d23087fd44cad61df08fc31907a2f" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ecf5b4cc5364572d7f4c329661bcc82724222973f2cab6f050a4e5c22f75181" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zerofrom" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50cc42e0333e05660c3587f3bf9d0478688e15d870fab3346451ce7f8c9fbea5" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" + +[[package]] +name = "zerotrie" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36f0bbd478583f79edad978b407914f61b2972f5af6fa089686016be8f9af595" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a05eb080e015ba39cc9e23bbe5e7fb04d5fb040350f99f34e338d5fdd294428" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b96237efa0c878c64bd89c436f661be4e46b2f3eff1ebb976f7ef2321d2f58f" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..28d8a7c --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,11 @@ +[workspace] +members = [ + "aw-client-rust", + "aw-datastore", + "aw-models", + "aw-transform", + "aw-server", + "aw-sync", + "aw-query", +] +resolver = "2" diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..a612ad9 --- /dev/null +++ b/LICENSE @@ -0,0 +1,373 @@ +Mozilla Public License Version 2.0 +================================== + +1. Definitions +-------------- + +1.1. "Contributor" + means each individual or legal entity that creates, contributes to + the creation of, or owns Covered Software. + +1.2. "Contributor Version" + means the combination of the Contributions of others (if any) used + by a Contributor and that particular Contributor's Contribution. + +1.3. "Contribution" + means Covered Software of a particular Contributor. + +1.4. "Covered Software" + means Source Code Form to which the initial Contributor has attached + the notice in Exhibit A, the Executable Form of such Source Code + Form, and Modifications of such Source Code Form, in each case + including portions thereof. + +1.5. "Incompatible With Secondary Licenses" + means + + (a) that the initial Contributor has attached the notice described + in Exhibit B to the Covered Software; or + + (b) that the Covered Software was made available under the terms of + version 1.1 or earlier of the License, but not also under the + terms of a Secondary License. + +1.6. "Executable Form" + means any form of the work other than Source Code Form. + +1.7. "Larger Work" + means a work that combines Covered Software with other material, in + a separate file or files, that is not Covered Software. + +1.8. "License" + means this document. + +1.9. "Licensable" + means having the right to grant, to the maximum extent possible, + whether at the time of the initial grant or subsequently, any and + all of the rights conveyed by this License. + +1.10. "Modifications" + means any of the following: + + (a) any file in Source Code Form that results from an addition to, + deletion from, or modification of the contents of Covered + Software; or + + (b) any new file in Source Code Form that contains any Covered + Software. + +1.11. "Patent Claims" of a Contributor + means any patent claim(s), including without limitation, method, + process, and apparatus claims, in any patent Licensable by such + Contributor that would be infringed, but for the grant of the + License, by the making, using, selling, offering for sale, having + made, import, or transfer of either its Contributions or its + Contributor Version. + +1.12. "Secondary License" + means either the GNU General Public License, Version 2.0, the GNU + Lesser General Public License, Version 2.1, the GNU Affero General + Public License, Version 3.0, or any later versions of those + licenses. + +1.13. "Source Code Form" + means the form of the work preferred for making modifications. + +1.14. "You" (or "Your") + means an individual or a legal entity exercising rights under this + License. For legal entities, "You" includes any entity that + controls, is controlled by, or is under common control with You. For + purposes of this definition, "control" means (a) the power, direct + or indirect, to cause the direction or management of such entity, + whether by contract or otherwise, or (b) ownership of more than + fifty percent (50%) of the outstanding shares or beneficial + ownership of such entity. + +2. License Grants and Conditions +-------------------------------- + +2.1. Grants + +Each Contributor hereby grants You a world-wide, royalty-free, +non-exclusive license: + +(a) under intellectual property rights (other than patent or trademark) + Licensable by such Contributor to use, reproduce, make available, + modify, display, perform, distribute, and otherwise exploit its + Contributions, either on an unmodified basis, with Modifications, or + as part of a Larger Work; and + +(b) under Patent Claims of such Contributor to make, use, sell, offer + for sale, have made, import, and otherwise transfer either its + Contributions or its Contributor Version. + +2.2. Effective Date + +The licenses granted in Section 2.1 with respect to any Contribution +become effective for each Contribution on the date the Contributor first +distributes such Contribution. + +2.3. Limitations on Grant Scope + +The licenses granted in this Section 2 are the only rights granted under +this License. No additional rights or licenses will be implied from the +distribution or licensing of Covered Software under this License. +Notwithstanding Section 2.1(b) above, no patent license is granted by a +Contributor: + +(a) for any code that a Contributor has removed from Covered Software; + or + +(b) for infringements caused by: (i) Your and any other third party's + modifications of Covered Software, or (ii) the combination of its + Contributions with other software (except as part of its Contributor + Version); or + +(c) under Patent Claims infringed by Covered Software in the absence of + its Contributions. + +This License does not grant any rights in the trademarks, service marks, +or logos of any Contributor (except as may be necessary to comply with +the notice requirements in Section 3.4). + +2.4. Subsequent Licenses + +No Contributor makes additional grants as a result of Your choice to +distribute the Covered Software under a subsequent version of this +License (see Section 10.2) or under the terms of a Secondary License (if +permitted under the terms of Section 3.3). + +2.5. Representation + +Each Contributor represents that the Contributor believes its +Contributions are its original creation(s) or it has sufficient rights +to grant the rights to its Contributions conveyed by this License. + +2.6. Fair Use + +This License is not intended to limit any rights You have under +applicable copyright doctrines of fair use, fair dealing, or other +equivalents. + +2.7. Conditions + +Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted +in Section 2.1. + +3. Responsibilities +------------------- + +3.1. Distribution of Source Form + +All distribution of Covered Software in Source Code Form, including any +Modifications that You create or to which You contribute, must be under +the terms of this License. You must inform recipients that the Source +Code Form of the Covered Software is governed by the terms of this +License, and how they can obtain a copy of this License. You may not +attempt to alter or restrict the recipients' rights in the Source Code +Form. + +3.2. Distribution of Executable Form + +If You distribute Covered Software in Executable Form then: + +(a) such Covered Software must also be made available in Source Code + Form, as described in Section 3.1, and You must inform recipients of + the Executable Form how they can obtain a copy of such Source Code + Form by reasonable means in a timely manner, at a charge no more + than the cost of distribution to the recipient; and + +(b) You may distribute such Executable Form under the terms of this + License, or sublicense it under different terms, provided that the + license for the Executable Form does not attempt to limit or alter + the recipients' rights in the Source Code Form under this License. + +3.3. Distribution of a Larger Work + +You may create and distribute a Larger Work under terms of Your choice, +provided that You also comply with the requirements of this License for +the Covered Software. If the Larger Work is a combination of Covered +Software with a work governed by one or more Secondary Licenses, and the +Covered Software is not Incompatible With Secondary Licenses, this +License permits You to additionally distribute such Covered Software +under the terms of such Secondary License(s), so that the recipient of +the Larger Work may, at their option, further distribute the Covered +Software under the terms of either this License or such Secondary +License(s). + +3.4. Notices + +You may not remove or alter the substance of any license notices +(including copyright notices, patent notices, disclaimers of warranty, +or limitations of liability) contained within the Source Code Form of +the Covered Software, except that You may alter any license notices to +the extent required to remedy known factual inaccuracies. + +3.5. Application of Additional Terms + +You may choose to offer, and to charge a fee for, warranty, support, +indemnity or liability obligations to one or more recipients of Covered +Software. However, You may do so only on Your own behalf, and not on +behalf of any Contributor. You must make it absolutely clear that any +such warranty, support, indemnity, or liability obligation is offered by +You alone, and You hereby agree to indemnify every Contributor for any +liability incurred by such Contributor as a result of warranty, support, +indemnity or liability terms You offer. You may include additional +disclaimers of warranty and limitations of liability specific to any +jurisdiction. + +4. Inability to Comply Due to Statute or Regulation +--------------------------------------------------- + +If it is impossible for You to comply with any of the terms of this +License with respect to some or all of the Covered Software due to +statute, judicial order, or regulation then You must: (a) comply with +the terms of this License to the maximum extent possible; and (b) +describe the limitations and the code they affect. Such description must +be placed in a text file included with all distributions of the Covered +Software under this License. Except to the extent prohibited by statute +or regulation, such description must be sufficiently detailed for a +recipient of ordinary skill to be able to understand it. + +5. Termination +-------------- + +5.1. The rights granted under this License will terminate automatically +if You fail to comply with any of its terms. However, if You become +compliant, then the rights granted under this License from a particular +Contributor are reinstated (a) provisionally, unless and until such +Contributor explicitly and finally terminates Your grants, and (b) on an +ongoing basis, if such Contributor fails to notify You of the +non-compliance by some reasonable means prior to 60 days after You have +come back into compliance. Moreover, Your grants from a particular +Contributor are reinstated on an ongoing basis if such Contributor +notifies You of the non-compliance by some reasonable means, this is the +first time You have received notice of non-compliance with this License +from such Contributor, and You become compliant prior to 30 days after +Your receipt of the notice. + +5.2. If You initiate litigation against any entity by asserting a patent +infringement claim (excluding declaratory judgment actions, +counter-claims, and cross-claims) alleging that a Contributor Version +directly or indirectly infringes any patent, then the rights granted to +You by any and all Contributors for the Covered Software under Section +2.1 of this License shall terminate. + +5.3. In the event of termination under Sections 5.1 or 5.2 above, all +end user license agreements (excluding distributors and resellers) which +have been validly granted by You or Your distributors under this License +prior to termination shall survive termination. + +************************************************************************ +* * +* 6. Disclaimer of Warranty * +* ------------------------- * +* * +* Covered Software is provided under this License on an "as is" * +* basis, without warranty of any kind, either expressed, implied, or * +* statutory, including, without limitation, warranties that the * +* Covered Software is free of defects, merchantable, fit for a * +* particular purpose or non-infringing. The entire risk as to the * +* quality and performance of the Covered Software is with You. * +* Should any Covered Software prove defective in any respect, You * +* (not any Contributor) assume the cost of any necessary servicing, * +* repair, or correction. This disclaimer of warranty constitutes an * +* essential part of this License. No use of any Covered Software is * +* authorized under this License except under this disclaimer. * +* * +************************************************************************ + +************************************************************************ +* * +* 7. Limitation of Liability * +* -------------------------- * +* * +* Under no circumstances and under no legal theory, whether tort * +* (including negligence), contract, or otherwise, shall any * +* Contributor, or anyone who distributes Covered Software as * +* permitted above, be liable to You for any direct, indirect, * +* special, incidental, or consequential damages of any character * +* including, without limitation, damages for lost profits, loss of * +* goodwill, work stoppage, computer failure or malfunction, or any * +* and all other commercial damages or losses, even if such party * +* shall have been informed of the possibility of such damages. This * +* limitation of liability shall not apply to liability for death or * +* personal injury resulting from such party's negligence to the * +* extent applicable law prohibits such limitation. Some * +* jurisdictions do not allow the exclusion or limitation of * +* incidental or consequential damages, so this exclusion and * +* limitation may not apply to You. * +* * +************************************************************************ + +8. Litigation +------------- + +Any litigation relating to this License may be brought only in the +courts of a jurisdiction where the defendant maintains its principal +place of business and such litigation shall be governed by laws of that +jurisdiction, without reference to its conflict-of-law provisions. +Nothing in this Section shall prevent a party's ability to bring +cross-claims or counter-claims. + +9. Miscellaneous +---------------- + +This License represents the complete agreement concerning the subject +matter hereof. If any provision of this License is held to be +unenforceable, such provision shall be reformed only to the extent +necessary to make it enforceable. Any law or regulation which provides +that the language of a contract shall be construed against the drafter +shall not be used to construe this License against a Contributor. + +10. Versions of the License +--------------------------- + +10.1. New Versions + +Mozilla Foundation is the license steward. Except as provided in Section +10.3, no one other than the license steward has the right to modify or +publish new versions of this License. Each version will be given a +distinguishing version number. + +10.2. Effect of New Versions + +You may distribute the Covered Software under the terms of the version +of the License under which You originally received the Covered Software, +or under the terms of any subsequent version published by the license +steward. + +10.3. Modified Versions + +If you create software not governed by this License, and you want to +create a new license for such software, you may create and use a +modified version of this License if you rename the license and remove +any references to the name of the license steward (except to note that +such modified license differs from this License). + +10.4. Distributing Source Code Form that is Incompatible With Secondary +Licenses + +If You choose to distribute Source Code Form that is Incompatible With +Secondary Licenses under the terms of this version of the License, the +notice described in Exhibit B of this License must be attached. + +Exhibit A - Source Code Form License Notice +------------------------------------------- + + This Source Code Form is subject to the terms of the Mozilla Public + License, v. 2.0. If a copy of the MPL was not distributed with this + file, You can obtain one at http://mozilla.org/MPL/2.0/. + +If it is not possible or desirable to put the notice in a particular +file, then You may include the notice in a location (such as a LICENSE +file in a relevant directory) where a recipient would be likely to look +for such a notice. + +You may add additional accurate notices of copyright ownership. + +Exhibit B - "Incompatible With Secondary Licenses" Notice +--------------------------------------------------------- + + This Source Code Form is "Incompatible With Secondary Licenses", as + defined by the Mozilla Public License, v. 2.0. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..32ee588 --- /dev/null +++ b/Makefile @@ -0,0 +1,110 @@ +.PHONY: all aw-server aw-webui build install package set-version test test-coverage test-coverage-tarpaulin test-coverage-grcov coverage coverage-html coverage-lcov + +all: build +build: aw-server aw-sync + +DESTDIR := +ifeq ($(SUDO_USER),) + PREFIX := $(HOME)/.local +else + PREFIX := /usr/local +endif + + +# Build in release mode by default, unless RELEASE=false +ifeq ($(RELEASE), false) + cargoflag := + targetdir := debug +else + cargoflag := --release + targetdir := release +endif + +aw-server: set-version aw-webui + cargo build $(cargoflag) --bin aw-server + +aw-sync: set-version + cargo build $(cargoflag) --bin aw-sync + +aw-webui: +ifeq ($(SKIP_WEBUI),true) # Skip building webui if SKIP_WEBUI is true + @echo "Skipping building webui" +else + make -C ./aw-webui build +endif + +android: + ./install-ndk.sh + ./compile-android.sh + +test: + cargo test + +fix: + cargo fmt + cargo clippy --fix + +set-version: + @# if GITHUB_REF_TYPE is tag and GITHUB_REF_NAME is not empty, then we are building a release + @# as such, we then need to set the Cargo.toml version to the tag name (with leading 'v' stripped) + @# if tag is on Python-format (short pre-release suffixes), then we need to convert it to Rust-format (long pre-release suffixes) + @# Example: v0.12.0b3 should become 0.12.0-beta.3 + @# Can't use sed with `-i` on macOS due to: https://stackoverflow.com/a/4247319/965332 + @if [ "$(GITHUB_REF_TYPE)" = "tag" ] && [ -n "$(GITHUB_REF_NAME)" ]; then \ + VERSION_SEMVER=$(shell echo $(GITHUB_REF_NAME:v%=%) | sed -E 's/([0-9]+)\.([0-9]+)\.([0-9]+)-?(a|alpha|b|beta|rc)([0-9]+)/\1.\2.\3-\4.\5/; s/-b(.[0-9]+)/-beta\1/; s/-a(.[0-9+])/-alpha\1/'); \ + echo "Building release $(GITHUB_REF_NAME) ($$VERSION_SEMVER), setting version in Cargo.toml"; \ + perl -i -pe "s/^version = .*/version = \"$$VERSION_SEMVER\"/" aw-server/Cargo.toml; \ + fi + +test-coverage-grcov: +ifndef COVERAGE_CACHE + # We need to remove build files in case a non-coverage test has been run + # before without RUST/CARGO flags needed for coverage + rm -rf target/debug +endif + rm -rf **/*.profraw + # Build and test + env RUSTFLAGS="-C instrument-coverage -C link-dead-code -C opt-level=0" \ + LLVM_PROFILE_FILE=".cov/grcov-%p-%m.profraw" \ + cargo test --verbose + +coverage-tarpaulin-html: + cargo tarpaulin -o html --output-dir coverage-html + +GRCOV_PARAMS=$(shell find .cov -name "grcov-*.profraw" -print) --binary-path=./target/debug/aw-server -s . --llvm --branch --ignore-not-existing + +coverage-grcov-html: test-coverage-grcov + grcov ${GRCOV_PARAMS} -t html -o ./target/debug/$@/ + rm -rf **/*.profraw + +coverage-grcov-lcov: test-coverage-grcov + grcov ${GRCOV_PARAMS} -t lcov -o ./target/debug/lcov.info + rm -rf **/*.profraw + +coverage: coverage-tarpaulin-html + +package: + # Clean and prepare target/package folder + rm -rf target/package + mkdir -p target/package + # Copy binaries + cp target/$(targetdir)/aw-server target/package/aw-server-rust + cp target/$(targetdir)/aw-sync target/package/aw-sync + # Copy service file + cp -f aw-server.service target/package/aw-server.service + # Copy everything into `dist/aw-server-rust` + mkdir -p dist + rm -rf dist/aw-server-rust + cp -rf target/package dist/aw-server-rust + +install: + # Install aw-server and aw-sync executables + mkdir -p $(DESTDIR)$(PREFIX)/bin/ + install -m 755 target/$(targetdir)/aw-server $(DESTDIR)$(PREFIX)/bin/aw-server + install -m 755 target/$(targetdir)/aw-sync $(DESTDIR)$(PREFIX)/bin/aw-sync + # Install systemd user service + mkdir -p $(DESTDIR)$(PREFIX)/lib/systemd/user + install -m 644 aw-server.service $(DESTDIR)$(PREFIX)/lib/systemd/user/aw-server.service + +clean: + cargo clean diff --git a/README.md b/README.md new file mode 100644 index 0000000..dedc51c --- /dev/null +++ b/README.md @@ -0,0 +1,125 @@ +aw-server-rust +============== + +[![Build Status](https://github.com/ActivityWatch/aw-server-rust/workflows/Build/badge.svg?branch=master)](https://github.com/ActivityWatch/aw-server-rust/actions?query=workflow%3ABuild+branch%3Amaster) +[![Coverage Status](https://codecov.io/gh/ActivityWatch/aw-server-rust/branch/master/graph/badge.svg)](https://codecov.io/gh/ActivityWatch/aw-server-rust) +[![Dependency Status](https://deps.rs/repo/github/activitywatch/aw-server-rust/status.svg)](https://deps.rs/repo/github/activitywatch/aw-server-rust) + +A reimplementation of aw-server in Rust. + +Features missing compared to the Python implementation of aw-server: + + - API explorer (Swagger/OpenAPI) + +### How to compile + +Build with `cargo`: + +```sh +cargo build --release +``` + +You can also build with make, which will build the web assets as well: + +``` +make build +``` + +Your built executable will be located in `./target/release/aw-server-rust`. If you want to use it with a development version of `aw-qt` you'll want to copy this binary into your `venv`: + +```shell +cp target/release/aw-server ../venv/bin/aw-server-rust +``` + + +### How to run + +If you want to quick-compile for debugging, run cargo run from the project root: + +```sh +cargo run --bin aw-server +``` + +*NOTE:* This will start aw-server-rust in testing mode (on port 5666 instead of port 5600). + +### Docker + +A multi-stage `scripts/Dockerfile` is included. It builds the web UI and the +server, and produces a slim Debian-based image containing only the `aw-server` +binary (the web UI assets are embedded into it at compile time). + +The `aw-webui` submodule must be initialized before building: + +```sh +git submodule update --init --recursive +docker build -f scripts/Dockerfile -t aw-server-rust:local . +``` + +Run it with: + +```sh +docker run -d --name aw-server-rust \ + -p 5600:5600 \ + -v aw-server-rust-data:/home/aw/.local/share/activitywatch \ + aw-server-rust:local aw-server-rust --host 0.0.0.0 +``` + +Notes: + + - The server binds to `127.0.0.1` by default, which is unreachable from outside + the container. Pass `--host 0.0.0.0` (as above) to make it reachable. + - The server runs as the unprivileged user `aw` (uid/gid `10001`). When using a + bind mount instead of a named volume, the host directory must be writable by + that uid (`sudo chown -R 10001:10001 `). + - The database lives in `/home/aw/.local/share/activitywatch/aw-server-rust`. + Mount a volume there to keep it across container recreations. Images built + before the switch to a non-root user stored it under `/root/.local/share`; + to keep existing data, copy it over and fix its ownership, e.g. + `docker run --rm -v aw-server-rust-data:/data alpine sh -c 'chown -R 10001:10001 /data'`. + - The image only ships `aw-server`. The `aw-sync` binary is deliberately not + built, since on Linux it pulls in OpenSSL with the `vendored` feature and + compiles it from source. + - `docker build` produces an image for the architecture of the machine running + it. To target a different one, use + `docker buildx build --platform linux/amd64 -f scripts/Dockerfile ...`. + +### Configuration + +The server reads its configuration from `~/.config/activitywatch/aw-server-rust/config.toml` (or `config-testing.toml` in testing mode). + +Available options: + +```toml +# Address to listen on +#address = "127.0.0.1" + +# Port to listen on (default: 5600, testing: 5666) +#port = 5600 + +# Additional exact CORS origins to allow (e.g. for custom web interfaces) +#cors = ["http://localhost:3000"] + +# Additional regex CORS origins to allow (e.g. for sideloaded browser extensions) +#cors_regex = ["chrome-extension://yourextensionidhere"] +``` + +#### Custom CORS Origins + +By default, the server allows requests from: +- The server's own origin (`http://127.0.0.1:`, `http://localhost:`) +- The official Chrome extension (`chrome-extension://nglaklhklhcoonedhgnpgddginnjdadi`) +- All Firefox extensions (`moz-extension://.*`) + +To allow additional origins (e.g. a sideloaded Chrome extension), add them to your config: + +```toml +# Allow a specific sideloaded Chrome extension +cors_regex = ["chrome-extension://jmdbkmbphoikckgkcnpoojbfeiaoaocl"] + +# Or allow all Chrome extensions (less secure, but convenient for development) +cors_regex = ["chrome-extension://.*"] +``` + +### Syncing + +For details about aw-sync-rust, see the [README](./aw-sync/README.md) in its subdirectory. diff --git a/aw-client-rust/.gitignore b/aw-client-rust/.gitignore new file mode 100644 index 0000000..50281a4 --- /dev/null +++ b/aw-client-rust/.gitignore @@ -0,0 +1,10 @@ +# Generated by Cargo +# will have compiled files and executables +/target/ + +# Remove Cargo.lock from gitignore if creating an executable, leave it for libraries +# More information here http://doc.crates.io/guide.html#cargotoml-vs-cargolock +Cargo.lock + +# These are backup files generated by rustfmt +**/*.rs.bk diff --git a/aw-client-rust/Cargo.toml b/aw-client-rust/Cargo.toml new file mode 100644 index 0000000..6bd4c0d --- /dev/null +++ b/aw-client-rust/Cargo.toml @@ -0,0 +1,27 @@ +[package] +name = "aw-client-rust" +version = "0.1.0" +edition = "2021" +authors = ["Johan Bjäreholt "] + +[dependencies] +reqwest = { version = "0.12", default-features = false, features = ["json", "blocking", "rustls-tls-native-roots"] } +gethostname = "0.4" +serde = { version = "1.0", features = ["derive"] } +phf = { version = "0.11", features = ["macros"] } +serde_json = "1.0" +chrono = { version = "0.4", features = ["serde"] } +aw-models = { path = "../aw-models" } +tokio = { version = "1.28.2", features = ["rt", "net", "time"] } +log = "0.4" +libc = "0.2" +thiserror = "1.0" +dirs = "6.0" +fs4 = { version = "0.13", features = ["sync"] } + +[dev-dependencies] +aw-datastore = { path = "../aw-datastore" } +aw-server = { path = "../aw-server", default-features = false, features=[] } +rocket = "0.5.0-rc.1" +tokio-test = "*" +toml = "0.8" diff --git a/aw-client-rust/README.md b/aw-client-rust/README.md new file mode 100644 index 0000000..f4b78b4 --- /dev/null +++ b/aw-client-rust/README.md @@ -0,0 +1,10 @@ +aw-client-rust +============== + +WIP: aw-client implementation in Rust + +`AwClient::wait_for_start()` is asynchronous: call +`client.wait_for_start().await?` inside a Tokio runtime with networking and timers +enabled. It waits for TCP connectivity, with a ten-second total deadline covering +DNS resolution, connection attempts, and retry delays. It does not check HTTP readiness. +`blocking::AwClient::wait_for_start()?` remains synchronous. diff --git a/aw-client-rust/src/blocking.rs b/aw-client-rust/src/blocking.rs new file mode 100644 index 0000000..e54505a --- /dev/null +++ b/aw-client-rust/src/blocking.rs @@ -0,0 +1,109 @@ +use std::future::Future; +use std::{collections::HashMap, error::Error}; + +use chrono::{DateTime, Utc}; + +use aw_models::{Bucket, Event}; + +use super::AwClient as AsyncAwClient; + +pub struct AwClient { + client: AsyncAwClient, + pub baseurl: reqwest::Url, + pub name: String, + pub hostname: String, +} + +impl std::fmt::Debug for AwClient { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(f, "AwClient(baseurl={:?})", self.client.baseurl) + } +} + +fn block_on(f: F) -> F::Output { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("build shell runtime") + .block_on(f) +} + +macro_rules! proxy_method +{ + ($name:tt, $ret:ty, $($v:ident: $t:ty),*) => { + pub fn $name(&self, $($v: $t),*) -> Result<$ret, reqwest::Error> + { block_on(self.client.$name($($v),*)) } + }; +} + +impl AwClient { + pub fn new(host: &str, port: u16, name: &str) -> Result> { + Self::new_with_api_key(host, port, name, None) + } + + pub fn new_with_api_key( + host: &str, + port: u16, + name: &str, + api_key: Option, + ) -> Result> { + let async_client = AsyncAwClient::new_with_api_key(host, port, name, api_key)?; + + Ok(AwClient { + baseurl: async_client.baseurl.clone(), + name: async_client.name.clone(), + hostname: async_client.hostname.clone(), + client: async_client, + }) + } + + proxy_method!(get_bucket, Bucket, bucketname: &str); + proxy_method!(get_buckets, HashMap,); + proxy_method!(create_bucket, (), bucket: &Bucket); + proxy_method!(create_bucket_simple, (), bucketname: &str, buckettype: &str); + proxy_method!(delete_bucket, (), bucketname: &str); + proxy_method!( + get_events, + Vec, + bucketname: &str, + start: Option>, + stop: Option>, + limit: Option + ); + proxy_method!( + query, + Vec, + query: &str, + timeperiods: Vec<(DateTime, DateTime)> + ); + proxy_method!(insert_event, (), bucketname: &str, event: &Event); + proxy_method!(insert_events, (), bucketname: &str, events: Vec); + proxy_method!( + heartbeat, + (), + bucketname: &str, + event: &Event, + pulsetime: f64 + ); + proxy_method!(delete_event, (), bucketname: &str, event_id: i64); + proxy_method!(get_event_count, i64, bucketname: &str); + proxy_method!(get_info, aw_models::Info,); + proxy_method!(get_setting, serde_json::Value, setting: &str); + proxy_method!(get_settings, aw_models::Settings,); + + pub fn wait_for_start(&self) -> Result<(), Box> { + block_on(self.client.wait_for_start()) + } +} + +#[test] +fn test_wait_for_start_blocking_wrapper() { + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let client = AwClient::new( + "127.0.0.1", + listener.local_addr().unwrap().port(), + "test-wait-for-start-blocking", + ) + .unwrap(); + client.wait_for_start().unwrap(); +} diff --git a/aw-client-rust/src/classes.rs b/aw-client-rust/src/classes.rs new file mode 100644 index 0000000..1bc0989 --- /dev/null +++ b/aw-client-rust/src/classes.rs @@ -0,0 +1,284 @@ +//! Default classes +//! +//! Taken from default classes in aw-webui + +use serde::{Deserialize, Serialize}; +use serde_json; + +pub type CategoryId = Vec; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct CategorySpec { + #[serde(rename = "type")] + pub spec_type: String, + #[serde(default)] + pub regex: String, + #[serde(default)] + pub ignore_case: bool, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ClassSetting { + #[serde(default)] + pub id: Option, + pub name: Vec, + pub rule: CategorySpec, + #[serde(default)] + pub data: Option, +} + +/// Parse categorization classes from a settings JSON value. +/// +/// Accepts the payload the webui writes to `settings.classes`: a JSON array of +/// `{name, rule, ...}` objects. `id` is optional (the webui assigns it in +/// memory). `null`, empty arrays, missing fields, or unparseable JSON fall +/// back to [`default_classes`]. +/// +/// A JSON *string* is treated as double-encoded payload (the localStorage → +/// server settings migration bug, ActivityWatch/activitywatch#1067) and decoded +/// once more before parsing. +pub fn classes_from_settings_json(value: &serde_json::Value) -> Vec<(CategoryId, CategorySpec)> { + let value = decode_maybe_double_encoded(value); + match value { + serde_json::Value::Null => default_classes(), + serde_json::Value::Array(ref arr) if arr.is_empty() => default_classes(), + other => match serde_json::from_value::>(other) { + Ok(classes) if classes.is_empty() => default_classes(), + Ok(classes) => classes.into_iter().map(|c| (c.name, c.rule)).collect(), + Err(e) => { + log::warn!("Failed to parse settings.classes, using defaults: {:?}", e); + default_classes() + } + }, + } +} + +/// Parse `settings.classes` from the raw datastore/HTTP body string. +pub fn classes_from_settings_str(raw: &str) -> Vec<(CategoryId, CategorySpec)> { + match serde_json::from_str::(raw.trim()) { + Ok(v) => classes_from_settings_json(&v), + Err(e) => { + log::warn!( + "settings.classes is not valid JSON, using defaults: {:?}", + e + ); + default_classes() + } + } +} + +fn decode_maybe_double_encoded(value: &serde_json::Value) -> serde_json::Value { + match value { + serde_json::Value::String(s) => serde_json::from_str(s).unwrap_or_else(|_| value.clone()), + _ => value.clone(), + } +} + +/// Returns the default categorization classes +pub fn default_classes() -> Vec<(CategoryId, CategorySpec)> { + vec![ + ( + vec!["Work".to_string()], + CategorySpec { + spec_type: "regex".to_string(), + regex: "Google Docs|libreoffice|ReText".to_string(), + ignore_case: false, + }, + ), + ( + vec!["Work".to_string(), "Programming".to_string()], + CategorySpec { + spec_type: "regex".to_string(), + regex: "GitHub|Stack Overflow|BitBucket|Gitlab|vim|Spyder|kate|Ghidra|Scite" + .to_string(), + ignore_case: false, + }, + ), + ( + vec![ + "Work".to_string(), + "Programming".to_string(), + "ActivityWatch".to_string(), + ], + CategorySpec { + spec_type: "regex".to_string(), + regex: "ActivityWatch|aw-".to_string(), + ignore_case: true, + }, + ), + ( + vec!["Work".to_string(), "Image".to_string()], + CategorySpec { + spec_type: "regex".to_string(), + regex: "Gimp|Inkscape".to_string(), + ignore_case: false, + }, + ), + ( + vec!["Work".to_string(), "Video".to_string()], + CategorySpec { + spec_type: "regex".to_string(), + regex: "Kdenlive".to_string(), + ignore_case: false, + }, + ), + ( + vec!["Work".to_string(), "Audio".to_string()], + CategorySpec { + spec_type: "regex".to_string(), + regex: "Audacity".to_string(), + ignore_case: false, + }, + ), + ( + vec!["Work".to_string(), "3D".to_string()], + CategorySpec { + spec_type: "regex".to_string(), + regex: "Blender".to_string(), + ignore_case: false, + }, + ), + ( + vec!["Media".to_string(), "Games".to_string()], + CategorySpec { + spec_type: "regex".to_string(), + regex: "Minecraft|RimWorld".to_string(), + ignore_case: false, + }, + ), + ( + vec!["Media".to_string(), "Video".to_string()], + CategorySpec { + spec_type: "regex".to_string(), + regex: "YouTube|Plex|VLC".to_string(), + ignore_case: false, + }, + ), + ( + vec!["Media".to_string(), "Social Media".to_string()], + CategorySpec { + spec_type: "regex".to_string(), + regex: "reddit|Facebook|Twitter|Instagram|devRant".to_string(), + ignore_case: true, + }, + ), + ( + vec!["Media".to_string(), "Music".to_string()], + CategorySpec { + spec_type: "regex".to_string(), + regex: "Spotify|Deezer".to_string(), + ignore_case: true, + }, + ), + ( + vec!["Comms".to_string(), "IM".to_string()], + CategorySpec { + spec_type: "regex".to_string(), + regex: "Messenger|Telegram|Signal|WhatsApp|Rambox|Slack|Riot|Discord|Nheko" + .to_string(), + ignore_case: false, + }, + ), + ( + vec!["Comms".to_string(), "Email".to_string()], + CategorySpec { + spec_type: "regex".to_string(), + regex: "Gmail|Thunderbird|mutt|alpine".to_string(), + ignore_case: false, + }, + ), + ] +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + fn work_regex(classes: &[(CategoryId, CategorySpec)]) -> Option<&str> { + classes + .iter() + .find(|(name, _)| name == &vec!["Work".to_string()]) + .map(|(_, spec)| spec.regex.as_str()) + } + + #[test] + fn custom_classes_without_id_are_used() { + let value = json!([ + { + "name": ["Work"], + "rule": {"type": "regex", "regex": "Slack|Notion", "ignore_case": true} + }, + { + "name": ["Uncategorized", "Games"], + "rule": {"type": "regex", "regex": "Steam"} + } + ]); + let classes = classes_from_settings_json(&value); + assert_eq!(classes.len(), 2); + assert_eq!(work_regex(&classes), Some("Slack|Notion")); + assert!(classes[0].1.ignore_case); + assert_eq!( + classes[1].0, + vec!["Uncategorized".to_string(), "Games".to_string()] + ); + assert_eq!(classes[1].1.regex, "Steam"); + } + + #[test] + fn null_and_empty_fall_back_to_defaults() { + let defaults = default_classes(); + assert_eq!( + work_regex(&classes_from_settings_json(&json!(null))), + work_regex(&defaults) + ); + assert_eq!( + work_regex(&classes_from_settings_json(&json!([]))), + work_regex(&defaults) + ); + assert_eq!( + work_regex(&classes_from_settings_str("null")), + work_regex(&defaults) + ); + } + + #[test] + fn invalid_json_falls_back_to_defaults() { + let defaults = default_classes(); + assert_eq!( + work_regex(&classes_from_settings_str("not-json")), + work_regex(&defaults) + ); + assert_eq!( + work_regex(&classes_from_settings_json(&json!({"name": "Work"}))), + work_regex(&defaults) + ); + } + + #[test] + fn double_encoded_string_is_decoded() { + // ActivityWatch/activitywatch#1067: classes stored as a JSON string of JSON. + let inner = json!([ + {"name": ["Work"], "rule": {"type": "regex", "regex": "CustomApp"}} + ]); + let wrapped = serde_json::Value::String(inner.to_string()); + let classes = classes_from_settings_json(&wrapped); + assert_eq!(work_regex(&classes), Some("CustomApp")); + } + + #[test] + fn webui_shaped_payload_with_id_and_data_parses() { + let value = json!([ + { + "id": 0, + "name": ["Media", "Video"], + "rule": {"type": "regex", "regex": "YouTube|Plex", "ignore_case": false}, + "data": {"color": "#F33", "score": 0} + } + ]); + let classes = classes_from_settings_json(&value); + assert_eq!(classes.len(), 1); + assert_eq!(classes[0].0, vec!["Media".to_string(), "Video".to_string()]); + assert_eq!(classes[0].1.regex, "YouTube|Plex"); + } +} diff --git a/aw-client-rust/src/lib.rs b/aw-client-rust/src/lib.rs new file mode 100644 index 0000000..ca35663 --- /dev/null +++ b/aw-client-rust/src/lib.rs @@ -0,0 +1,417 @@ +extern crate aw_models; +extern crate chrono; +extern crate gethostname; +extern crate reqwest; +extern crate serde_json; +extern crate tokio; + +pub mod blocking; +pub mod classes; +pub mod queries; +pub mod single_instance; + +use std::{collections::HashMap, error::Error}; + +use chrono::{DateTime, Utc}; +use reqwest::header::{HeaderMap, HeaderValue, AUTHORIZATION}; +use serde_json::{json, Map}; +use single_instance::SingleInstance; +use std::time::Duration; +use tokio::net::TcpStream; + +pub use aw_models::{Bucket, BucketMetadata, Event}; + +pub struct AwClient { + client: reqwest::Client, + #[allow(dead_code)] + single_instance: SingleInstance, + pub baseurl: reqwest::Url, + pub name: String, + pub hostname: String, +} + +impl std::fmt::Debug for AwClient { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(f, "AwClient(baseurl={:?})", self.baseurl) + } +} + +fn get_hostname() -> String { + gethostname::gethostname().to_string_lossy().to_string() +} + +fn build_client(api_key: Option) -> Result> { + let mut headers = HeaderMap::new(); + if let Some(api_key) = api_key { + let mut header_value = HeaderValue::from_str(&format!("Bearer {api_key}"))?; + header_value.set_sensitive(true); + headers.insert(AUTHORIZATION, header_value); + } + + Ok(reqwest::Client::builder() + .timeout(std::time::Duration::from_secs(120)) + .default_headers(headers) + .build()?) +} + +/// Build the name used for the single-instance lock. +/// +/// `localhost` is normalized to `127.0.0.1` so that a client given either spelling +/// takes the same lock, and so the name matches aw-client-python, which defaults +/// its server hostname to `127.0.0.1`. +fn single_instance_name(name: &str, host: &str, port: u16) -> String { + let host = if host.eq_ignore_ascii_case("localhost") { + "127.0.0.1" + } else { + host + }; + format!("{}-at-{}-on-{}", name, host, port) +} + +impl AwClient { + async fn send_success( + request: reqwest::RequestBuilder, + ) -> Result { + request.send().await?.error_for_status() + } + + pub fn new(host: &str, port: u16, name: &str) -> Result> { + Self::new_with_api_key(host, port, name, None) + } + + pub fn new_with_api_key( + host: &str, + port: u16, + name: &str, + api_key: Option, + ) -> Result> { + let baseurl = reqwest::Url::parse(&format!("http://{}:{}", host, port))?; + let hostname = get_hostname(); + let client = build_client(api_key)?; + let instance_name = single_instance_name(name, host, port); + let single_instance = single_instance::SingleInstance::new(instance_name.as_str())?; + + Ok(AwClient { + client, + single_instance, + baseurl, + name: name.to_string(), + hostname, + }) + } + + pub async fn get_bucket(&self, bucketname: &str) -> Result { + let url = format!("{}api/0/buckets/{}", self.baseurl, bucketname); + let bucket = Self::send_success(self.client.get(url)) + .await? + .json() + .await?; + Ok(bucket) + } + + pub async fn get_buckets(&self) -> Result, reqwest::Error> { + let url = format!("{}api/0/buckets/", self.baseurl); + Self::send_success(self.client.get(url)).await?.json().await + } + + pub async fn create_bucket(&self, bucket: &Bucket) -> Result<(), reqwest::Error> { + let url = format!("{}api/0/buckets/{}", self.baseurl, bucket.id); + Self::send_success(self.client.post(url).json(bucket)).await?; + Ok(()) + } + + pub async fn create_bucket_simple( + &self, + bucketname: &str, + buckettype: &str, + ) -> Result<(), reqwest::Error> { + let bucket = Bucket { + bid: None, + id: bucketname.to_string(), + client: self.name.clone(), + _type: buckettype.to_string(), + hostname: self.hostname.clone(), + data: Map::default(), + metadata: BucketMetadata::default(), + events: None, + created: None, + last_updated: None, + }; + self.create_bucket(&bucket).await + } + + pub async fn delete_bucket(&self, bucketname: &str) -> Result<(), reqwest::Error> { + let url = format!("{}api/0/buckets/{}", self.baseurl, bucketname); + Self::send_success(self.client.delete(url)).await?; + Ok(()) + } + + pub async fn query( + &self, + query: &str, + timeperiods: Vec<(DateTime, DateTime)>, + ) -> Result, reqwest::Error> { + let url = reqwest::Url::parse(format!("{}api/0/query", self.baseurl).as_str()).unwrap(); + + // Format timeperiods as ISO8601 strings, separated by / + let timeperiods_str: Vec = timeperiods + .iter() + .map(|(start, stop)| (start.to_rfc3339(), stop.to_rfc3339())) + .map(|(start, stop)| format!("{}/{}", start, stop)) + .collect(); + + // Result is a sequence, one element per timeperiod + Self::send_success(self.client.post(url).json(&json!({ + "query": query.split('\n').collect::>(), + "timeperiods": timeperiods_str, + }))) + .await? + .json() + .await + } + + pub async fn get_events( + &self, + bucketname: &str, + start: Option>, + stop: Option>, + limit: Option, + ) -> Result, reqwest::Error> { + let mut url = reqwest::Url::parse( + format!("{}api/0/buckets/{}/events", self.baseurl, bucketname).as_str(), + ) + .unwrap(); + + // Must be a better way to build URLs + if let Some(s) = start { + url.query_pairs_mut() + .append_pair("start", s.to_rfc3339().as_str()); + }; + if let Some(s) = stop { + url.query_pairs_mut() + .append_pair("end", s.to_rfc3339().as_str()); + }; + if let Some(s) = limit { + url.query_pairs_mut() + .append_pair("limit", s.to_string().as_str()); + }; + Self::send_success(self.client.get(url)).await?.json().await + } + + pub async fn insert_event( + &self, + bucketname: &str, + event: &Event, + ) -> Result<(), reqwest::Error> { + let url = format!("{}api/0/buckets/{}/events", self.baseurl, bucketname); + let eventlist = vec![event.clone()]; + Self::send_success(self.client.post(url).json(&eventlist)).await?; + Ok(()) + } + + pub async fn insert_events( + &self, + bucketname: &str, + events: Vec, + ) -> Result<(), reqwest::Error> { + let url = format!("{}api/0/buckets/{}/events", self.baseurl, bucketname); + Self::send_success(self.client.post(url).json(&events)).await?; + Ok(()) + } + + pub async fn heartbeat( + &self, + bucketname: &str, + event: &Event, + pulsetime: f64, + ) -> Result<(), reqwest::Error> { + let url = format!( + "{}api/0/buckets/{}/heartbeat?pulsetime={}", + self.baseurl, bucketname, pulsetime + ); + Self::send_success(self.client.post(url).json(&event)).await?; + Ok(()) + } + + pub async fn delete_event( + &self, + bucketname: &str, + event_id: i64, + ) -> Result<(), reqwest::Error> { + let url = format!( + "{}api/0/buckets/{}/events/{}", + self.baseurl, bucketname, event_id + ); + Self::send_success(self.client.delete(url)).await?; + Ok(()) + } + + pub async fn get_event_count(&self, bucketname: &str) -> Result { + let url = format!("{}api/0/buckets/{}/events/count", self.baseurl, bucketname); + let res = Self::send_success(self.client.get(url)) + .await? + .text() + .await?; + let count: i64 = match res.trim().parse() { + Ok(count) => count, + Err(err) => panic!("could not parse get_event_count response: {err:?}"), + }; + Ok(count) + } + + pub async fn get_info(&self) -> Result { + let url = format!("{}api/0/info", self.baseurl); + Self::send_success(self.client.get(url)).await?.json().await + } + + pub async fn get_setting(&self, setting: &str) -> Result { + let url = format!("{}api/0/settings/{}", self.baseurl, setting); + Self::send_success(self.client.get(url)).await?.json().await + } + + pub async fn get_settings(&self) -> Result { + let url = format!("{}api/0/settings", self.baseurl); + Self::send_success(self.client.get(url)).await?.json().await + } + + /// Wait up to ten seconds for a TCP connection, including DNS and retries. + /// Requires a Tokio runtime with networking and time enabled. + pub async fn wait_for_start(&self) -> Result<(), Box> { + wait_for_server(&self.baseurl, Duration::from_secs(10)).await + } +} + +async fn wait_for_server(url: &reqwest::Url, max_wait: Duration) -> Result<(), Box> { + // URL hosts include brackets around IPv6 literals; DNS lookup takes the bare address. + let host = url + .host_str() + .ok_or("Missing server hostname")? + .trim_start_matches('[') + .trim_end_matches(']'); + let port = url.port_or_known_default().ok_or("Missing server port")?; + let attempts = async { + let addresses: Vec<_> = tokio::net::lookup_host((host, port)).await?.collect(); + if addresses.is_empty() { + return Err("Unable to resolve baseurl into socket address".into()); + } + let mut retry_delay = Duration::from_millis(100); + loop { + for address in &addresses { + if matches!( + tokio::time::timeout(retry_delay, TcpStream::connect(address)).await, + Ok(Ok(_)) + ) { + return Ok(()); + } + } + tokio::time::sleep(retry_delay).await; + retry_delay = (retry_delay * 2).min(Duration::from_secs(1)); + } + }; + tokio::time::timeout(max_wait, attempts) + .await + .map_err(|_| -> Box { + format!( + "Local server {host}:{port} not running after {} seconds of retrying", + max_wait.as_secs_f64() + ) + .into() + })? +} + +#[cfg(test)] +mod tests { + use super::single_instance_name; + + #[test] + fn test_wait_for_start_success() { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + runtime.block_on(async { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let url = reqwest::Url::parse(&format!("http://{}/", listener.local_addr().unwrap())) + .unwrap(); + super::wait_for_server(&url, std::time::Duration::from_secs(1)) + .await + .unwrap(); + }); + } + + #[test] + fn test_wait_for_start_retries_without_blocking_executor() { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + runtime.block_on(async { + // Reserve a port without listening, then start the server on this same runtime. + let socket = tokio::net::TcpSocket::new_v4().unwrap(); + socket.bind("127.0.0.1:0".parse().unwrap()).unwrap(); + let url = + reqwest::Url::parse(&format!("http://{}/", socket.local_addr().unwrap())).unwrap(); + let server = tokio::spawn(async move { + tokio::time::sleep(std::time::Duration::from_millis(30)).await; + let listener = socket.listen(8).unwrap(); + listener.accept().await.unwrap(); + }); + super::wait_for_server(&url, std::time::Duration::from_secs(2)) + .await + .unwrap(); + server.await.unwrap(); + }); + } + + #[test] + fn test_wait_for_start_deadline() { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + runtime.block_on(async { + let socket = tokio::net::TcpSocket::new_v4().unwrap(); + socket.bind("127.0.0.1:0".parse().unwrap()).unwrap(); + let url = + reqwest::Url::parse(&format!("http://{}/", socket.local_addr().unwrap())).unwrap(); + let started = std::time::Instant::now(); + let budget = std::time::Duration::from_millis(150); + let error = super::wait_for_server(&url, budget).await.unwrap_err(); + assert!(error.to_string().contains("not running after 0.15 seconds")); + assert!(started.elapsed() >= budget); + assert!(started.elapsed() < std::time::Duration::from_secs(2)); + }); + } + + #[test] + fn test_single_instance_name_normalizes_localhost() { + // Both spellings must take the same lock, otherwise two instances of the + // same watcher can run against the same server. + assert_eq!( + single_instance_name("aw-watcher-afk", "localhost", 5600), + single_instance_name("aw-watcher-afk", "127.0.0.1", 5600), + ); + assert_eq!( + single_instance_name("aw-watcher-afk", "localhost", 5600), + "aw-watcher-afk-at-127.0.0.1-on-5600", + ); + + // Hostnames are case-insensitive, so case variants must take the same + // lock rather than each getting one of their own. + for variant in ["LOCALHOST", "LocalHost", "localHost"] { + assert_eq!( + single_instance_name("aw-watcher-afk", variant, 5600), + "aw-watcher-afk-at-127.0.0.1-on-5600", + "host spelling {variant:?} should normalize" + ); + } + } + + #[test] + fn test_single_instance_name_keeps_other_hosts() { + assert_eq!( + single_instance_name("aw-watcher-afk", "192.168.1.2", 5600), + "aw-watcher-afk-at-192.168.1.2-on-5600", + ); + } +} diff --git a/aw-client-rust/src/queries.rs b/aw-client-rust/src/queries.rs new file mode 100644 index 0000000..99286b4 --- /dev/null +++ b/aw-client-rust/src/queries.rs @@ -0,0 +1,484 @@ +//! Query building utilities for ActivityWatch +//! +//! This module provides functionality to build canonical queries that are compatible +//! with the ActivityWatch query language. It includes support for: +//! +//! - Desktop and Android query parameters +//! - Server-side categorization classes fetching +//! - Browser events integration +//! - AFK filtering +//! +//! ## Server-side Classes +//! +//! The queries can automatically fetch categorization classes from the ActivityWatch server: +//! +//! ```rust +//! use aw_client_rust::queries::{DesktopQueryParams, QueryParams, QueryParamsBase}; +//! +//! let params = DesktopQueryParams { +//! base: QueryParamsBase { +//! bid_browsers: vec![], +//! classes: vec![], // Empty - will fetch from server +//! filter_classes: vec![], +//! filter_afk: true, +//! include_audible: true, +//! }, +//! bid_window: "aw-watcher-window_example".to_string(), +//! bid_afk: "aw-watcher-afk_example".to_string(), +//! always_active_pattern: None, +//! }; +//! +//! // Automatically fetches classes from localhost:5600 +//! let query = QueryParams::Desktop(params.clone()).canonical_events(); +//! +//! ``` + +use crate::classes::{CategoryId, CategorySpec}; +use serde::{Deserialize, Serialize}; + +/// Browser application names mapped by browser type +pub static BROWSER_APPNAMES: phf::Map<&'static str, &'static [&'static str]> = phf::phf_map! { + "chrome" => &[ + // Chrome + "Google Chrome", + "Google-chrome", + "chrome.exe", + "google-chrome-stable", + // Chromium + "Chromium", + "Chromium-browser", + "Chromium-browser-chromium", + "chromium.exe", + // Pre-releases + "Google-chrome-beta", + "Google-chrome-unstable", + // Brave (should this be merged with the brave entry?) + "Brave-browser", + ], + "firefox" => &[ + "Firefox", + "Firefox.exe", + "firefox", + "firefox.exe", + "Firefox Developer Edition", + "firefoxdeveloperedition", + "Firefox-esr", + "Firefox Beta", + "Nightly", + "org.mozilla.firefox", + ], + "opera" => &["opera.exe", "Opera"], + "brave" => &["brave.exe"], + "edge" => &[ + "msedge.exe", // Windows + "Microsoft Edge", // macOS + ], + "vivaldi" => &["Vivaldi-stable", "Vivaldi-snapshot", "vivaldi.exe"], +}; + +pub const DEFAULT_LIMIT: u32 = 100; + +/// Type alias for categorization classes +pub type ClassRule = (CategoryId, CategorySpec); + +/// Base query parameters +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct QueryParamsBase { + #[serde(default)] + pub bid_browsers: Vec, + #[serde(default)] + pub classes: Vec, + #[serde(default)] + pub filter_classes: Vec>, + #[serde(default = "default_true")] + pub filter_afk: bool, + #[serde(default = "default_true")] + pub include_audible: bool, +} + +fn default_true() -> bool { + true +} + +/// Query parameters specific to desktop +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct DesktopQueryParams { + #[serde(flatten)] + pub base: QueryParamsBase, + pub bid_window: String, + pub bid_afk: String, + #[serde(default)] + pub always_active_pattern: Option, +} + +/// Query parameters specific to Android +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AndroidQueryParams { + #[serde(flatten)] + pub base: QueryParamsBase, + pub bid_android: String, +} + +/// Enum to represent different types of query parameters +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(untagged)] +pub enum QueryParams { + Desktop(DesktopQueryParams), + Android(AndroidQueryParams), +} + +impl QueryParams { + /// Build canonical events query string + pub fn canonical_events(&self) -> String { + match self { + QueryParams::Desktop(params) => build_desktop_canonical_events(params), + QueryParams::Android(params) => build_android_canonical_events(params), + } + } +} + +/// Helper function to serialize classes in the format expected by the categorize function +/// This version builds the query string directly without JSON serialization to avoid double-escaping +fn serialize_classes(classes: &[ClassRule]) -> String { + let mut parts = Vec::new(); + + for (category_id, category_spec) in classes { + // Build category array string manually: ["Work", "Programming"] + let category_str = format!( + "[{}]", + category_id + .iter() + .map(|s| format!("\"{}\"", s)) + .collect::>() + .join(", ") + ); + + // Build spec object manually to avoid JSON escaping regex patterns + let mut spec_parts = Vec::new(); + spec_parts.push(format!("\"type\": \"{}\"", category_spec.spec_type)); + + // Only include regex for non-"none" types, and use raw pattern without escaping + if category_spec.spec_type != "none" { + spec_parts.push(format!("\"regex\": \"{}\"", category_spec.regex)); + } + + // Always include ignore_case field + spec_parts.push(format!("\"ignore_case\": {}", category_spec.ignore_case)); + + let spec_str = format!("{{{}}}", spec_parts.join(", ")); + + // Build the tuple [category, spec] + parts.push(format!("[{}, {}]", category_str, spec_str)); + } + + format!("[{}]", parts.join(", ")) +} + +pub fn build_desktop_canonical_events(params: &DesktopQueryParams) -> String { + let mut query = Vec::new(); + + // Fetch window events + query.push(format!( + "events = flood(query_bucket(find_bucket(\"{}\")))", + escape_doublequote(¶ms.bid_window) + )); + + // Fetch not-afk events + if params.base.filter_afk { + let mut not_afk_query = format!( + "not_afk = flood(query_bucket(find_bucket(\"{}\"))); +not_afk = filter_keyvals(not_afk, \"status\", [\"not-afk\"])", + escape_doublequote(¶ms.bid_afk) + ); + + // Add treat_as_active functionality if pattern is provided + if let Some(ref pattern) = params.always_active_pattern { + not_afk_query.push_str(&format!( + "; +not_treat_as_afk = filter_keyvals_regex(events, \"app\", \"{}\"); +not_afk = period_union(not_afk, not_treat_as_afk); +not_treat_as_afk = filter_keyvals_regex(events, \"title\", \"{}\"); +not_afk = period_union(not_afk, not_treat_as_afk)", + escape_doublequote(pattern), + escape_doublequote(pattern) + )); + } + + query.push(not_afk_query); + } + + // Add browser events if any browser buckets specified + if !params.base.bid_browsers.is_empty() { + query.push(build_browser_events(params)); + + if params.base.include_audible { + query.push( + "audible_events = filter_keyvals(browser_events, \"audible\", [true]); +not_afk = period_union(not_afk, audible_events)" + .to_string(), + ); + } + } + + // Filter out window events when user was AFK + if params.base.filter_afk { + query.push("events = filter_period_intersect(events, not_afk)".to_string()); + } + + // Add categorization if classes specified + if !params.base.classes.is_empty() { + query.push(format!( + "events = categorize(events, {});", + serialize_classes(¶ms.base.classes) + )); + } + + // Filter categories if specified + if !params.base.filter_classes.is_empty() { + query.push(format!( + "events = filter_keyvals(events, \"$category\", {})", + serde_json::to_string(¶ms.base.filter_classes).unwrap() + )); + } + + query.join(";\n") +} + +pub fn build_android_canonical_events(params: &AndroidQueryParams) -> String { + let mut query = Vec::new(); + + // Fetch app events + query.push(format!( + "events = flood(query_bucket(find_bucket(\"{}\")))", + escape_doublequote(¶ms.bid_android) + )); + + // Merge events by app + query.push("events = merge_events_by_keys(events, [\"app\"])".to_string()); + + // Add categorization if classes specified + if !params.base.classes.is_empty() { + query.push(format!( + "events = categorize(events, {});", + serialize_classes(¶ms.base.classes) + )); + } + + // Filter categories if specified + if !params.base.filter_classes.is_empty() { + query.push(format!( + "events = filter_keyvals(events, \"$category\", {})", + serde_json::to_string(¶ms.base.filter_classes).unwrap() + )); + } + + query.join(";\n") +} + +pub fn build_browser_events(params: &DesktopQueryParams) -> String { + let mut query = String::from("browser_events = [];"); + + for browser_bucket in ¶ms.base.bid_browsers { + for (browser_name, app_names) in BROWSER_APPNAMES.entries() { + if browser_bucket.contains(browser_name) { + query.push_str(&format!( + " +events_{0} = flood(query_bucket(\"{1}\")); +window_{0} = filter_keyvals(events, \"app\", {2}); +events_{0} = filter_period_intersect(events_{0}, window_{0}); +events_{0} = split_url_events(events_{0}); +browser_events = concat(browser_events, events_{0}); +browser_events = sort_by_timestamp(browser_events)", + browser_name, + escape_doublequote(browser_bucket), + serde_json::to_string(app_names).unwrap() + )); + } + } + } + query +} + +/// Build a full desktop query using default localhost:5600 configuration +pub fn full_desktop_query(params: &DesktopQueryParams) -> String { + let mut query = QueryParams::Desktop(params.clone()).canonical_events(); + + // Add basic event aggregations + query.push_str(&format!( + " + title_events = sort_by_duration(merge_events_by_keys(events, [\"app\", \"title\"])); + app_events = sort_by_duration(merge_events_by_keys(title_events, [\"app\"])); + cat_events = sort_by_duration(merge_events_by_keys(events, [\"$category\"])); + app_events = limit_events(app_events, {}); + title_events = limit_events(title_events, {}); + duration = sum_durations(events); + ", + DEFAULT_LIMIT, DEFAULT_LIMIT + )); + + // Add browser-specific query parts if browser buckets exist + if !params.base.bid_browsers.is_empty() { + query.push_str(&format!( + " + browser_events = split_url_events(browser_events); + browser_urls = merge_events_by_keys(browser_events, [\"url\"]); + browser_urls = sort_by_duration(browser_urls); + browser_urls = limit_events(browser_urls, {}); + browser_domains = merge_events_by_keys(browser_events, [\"$domain\"]); + browser_domains = sort_by_duration(browser_domains); + browser_domains = limit_events(browser_domains, {}); + browser_duration = sum_durations(browser_events); + ", + DEFAULT_LIMIT, DEFAULT_LIMIT + )); + } else { + query.push_str( + " + browser_events = []; + browser_urls = []; + browser_domains = []; + browser_duration = 0; + ", + ); + } + + // Add return statement + query.push_str( + " + RETURN = { + \"events\": events, + \"window\": { + \"app_events\": app_events, + \"title_events\": title_events, + \"cat_events\": cat_events, + \"active_events\": not_afk, + \"duration\": duration + }, + \"browser\": { + \"domains\": browser_domains, + \"urls\": browser_urls, + \"duration\": browser_duration + } + }; + ", + ); + + query +} + +fn escape_doublequote(s: &str) -> String { + s.replace('\"', "\\\"") +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_desktop_query_generation() { + let params = DesktopQueryParams { + base: QueryParamsBase { + bid_browsers: vec![], + classes: vec![], + filter_classes: vec![], + filter_afk: true, + include_audible: true, + }, + bid_window: "aw-watcher-window_".to_string(), + bid_afk: "aw-watcher-afk_".to_string(), + always_active_pattern: None, + }; + + let query = full_desktop_query(¶ms); + assert!(!query.is_empty()); + assert!(query.contains("events = flood")); + } + + #[test] + fn test_classes_serialization() { + let classes = vec![ + ( + vec!["Work".to_string()], + crate::classes::CategorySpec { + spec_type: "regex".to_string(), + regex: "Google Docs".to_string(), + ignore_case: false, + }, + ), + ( + vec!["Work".to_string(), "Programming".to_string()], + crate::classes::CategorySpec { + spec_type: "regex".to_string(), + regex: "GitHub|vim".to_string(), + ignore_case: true, + }, + ), + ]; + + let serialized = serialize_classes(&classes); + + // Should contain the category names and rules in the correct format + assert!(serialized.contains("Work")); + assert!(serialized.contains("Programming")); + assert!(serialized.contains("Google Docs")); + assert!(serialized.contains("GitHub|vim")); + assert!(serialized.contains("\"type\": \"regex\"")); + assert!(serialized.contains("\"ignore_case\": false")); + assert!(serialized.contains("\"ignore_case\": true")); + } + + #[test] + fn test_canonical_events_with_empty_classes() { + let params = DesktopQueryParams { + base: QueryParamsBase { + bid_browsers: vec![], + classes: vec![], + filter_classes: vec![], + filter_afk: true, + include_audible: true, + }, + bid_window: "test-window".to_string(), + bid_afk: "test-afk".to_string(), + always_active_pattern: None, + }; + + let query_params = QueryParams::Desktop(params); + let query = query_params.canonical_events(); + + // Should contain basic query structure + assert!(query.contains("events = flood")); + assert!(query.contains("test-window")); + } + + #[test] + fn test_canonical_events_with_existing_classes() { + let classes = vec![( + vec!["Test".to_string()], + crate::classes::CategorySpec { + spec_type: "regex".to_string(), + regex: "test".to_string(), + ignore_case: false, + }, + )]; + + let params = DesktopQueryParams { + base: QueryParamsBase { + bid_browsers: vec![], + classes, + filter_classes: vec![], + filter_afk: true, + include_audible: true, + }, + bid_window: "test-window".to_string(), + bid_afk: "test-afk".to_string(), + always_active_pattern: None, + }; + + let query_params = QueryParams::Desktop(params); + let query = query_params.canonical_events(); + + // Should contain categorization + assert!(query.contains("events = categorize")); + assert!(query.contains("test")); + } +} diff --git a/aw-client-rust/src/single_instance.rs b/aw-client-rust/src/single_instance.rs new file mode 100644 index 0000000..985da13 --- /dev/null +++ b/aw-client-rust/src/single_instance.rs @@ -0,0 +1,90 @@ +use dirs::cache_dir; +use fs4::fs_std::FileExt; +use log::debug; +use std::fs::{File, OpenOptions}; +use std::io; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; + +#[derive(Debug)] +pub struct SingleInstance { + file: Option, + locked: Arc, +} + +#[derive(Debug, thiserror::Error)] +pub enum SingleInstanceError { + #[error("Another instance is already running")] + AlreadyRunning, + #[error("IO error: {0}")] + Io(#[from] io::Error), + #[error("Failed to create lock directory")] + LockDirCreation, +} + +impl SingleInstance { + pub fn new(client_name: &str) -> Result { + let cache_dir = cache_dir().ok_or(SingleInstanceError::LockDirCreation)?; + let lock_dir = cache_dir.join("activitywatch").join("client_locks"); + std::fs::create_dir_all(&lock_dir).map_err(|_| SingleInstanceError::LockDirCreation)?; + + let lockfile = lock_dir.join(client_name); + debug!("SingleInstance lockfile: {:?}", lockfile); + + #[cfg(windows)] + { + // On Windows, try to create an exclusive file + // Remove existing file if it exists (in case of previous crash) + let _ = std::fs::remove_file(&lockfile); + + match OpenOptions::new() + .write(true) + .create(true) + .create_new(true) + .open(&lockfile) + { + Ok(file) => Ok(SingleInstance { + file: Some(file), + locked: Arc::new(AtomicBool::new(true)), + }), + Err(e) if e.kind() == io::ErrorKind::AlreadyExists => { + // Qualified so the unix build doesn't carry an unused `error` import + log::error!("Another instance is already running"); + Err(SingleInstanceError::AlreadyRunning) + } + Err(e) => Err(SingleInstanceError::Io(e)), + } + } + + #[cfg(unix)] + { + // On Unix-like systems, use flock + match OpenOptions::new() + .write(true) + .create(true) + .truncate(true) + .open(&lockfile) + { + Ok(file) => match file.try_lock_exclusive() { + Ok(true) => Ok(SingleInstance { + file: Some(file), + locked: Arc::new(AtomicBool::new(true)), + }), + Ok(false) => Err(SingleInstanceError::AlreadyRunning), + Err(e) => Err(SingleInstanceError::Io(e)), + }, + Err(e) => Err(SingleInstanceError::Io(e)), + } + } + } +} + +impl Drop for SingleInstance { + fn drop(&mut self) { + if self.locked.load(Ordering::SeqCst) { + //drop the file handle and lock on Unix and Windows + self.file.take(); + self.locked.store(false, Ordering::SeqCst); + } + } +} diff --git a/aw-client-rust/tests/status_errors.rs b/aw-client-rust/tests/status_errors.rs new file mode 100644 index 0000000..1aa8102 --- /dev/null +++ b/aw-client-rust/tests/status_errors.rs @@ -0,0 +1,139 @@ +use std::future::Future; +use std::io::{BufRead, BufReader, Read, Write}; +use std::net::TcpListener; +use std::thread; + +use aw_client_rust::blocking; +use aw_client_rust::AwClient; + +fn block_on(future: F) -> F::Output { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("build test runtime") + .block_on(future) +} + +struct MockResponse { + status_line: &'static str, + content_type: &'static str, + body: &'static str, +} + +/// Drain the HTTP request fully before responding. +/// +/// Parses Content-Length from headers so POST body data (which may arrive +/// in a separate TCP segment) is consumed before the mock writes its +/// response. Without this, reqwest may see a broken pipe on loopback if +/// the response arrives before the body finishes sending. +fn drain_request(stream: &mut impl Read) { + let mut reader = BufReader::new(stream); + let mut content_length = 0_usize; + let mut line = String::new(); + loop { + line.clear(); + let n = reader.read_line(&mut line).expect("read request line"); + if n == 0 { + break; + } + let trimmed = line.trim(); + if trimmed.is_empty() { + // End of headers; body follows if Content-Length > 0 + break; + } + if let Some(val) = trimmed.strip_prefix("Content-Length:") { + content_length = val.trim().parse().unwrap_or(0); + } + } + if content_length > 0 { + let mut body_buf = vec![0_u8; content_length]; + reader + .read_exact(&mut body_buf) + .expect("drain request body"); + } +} + +fn spawn_mock_server(responses: Vec) -> (u16, thread::JoinHandle<()>) { + let listener = TcpListener::bind(("127.0.0.1", 0)).expect("bind mock server"); + let port = listener.local_addr().expect("mock server addr").port(); + let handle = thread::spawn(move || { + for response in responses { + let (mut stream, _) = listener.accept().expect("accept request"); + drain_request(&mut stream); + let body = response.body.as_bytes(); + write!( + stream, + "HTTP/1.1 {}\r\nContent-Length: {}\r\nContent-Type: {}\r\nConnection: close\r\n\r\n{}", + response.status_line, + body.len(), + response.content_type, + response.body + ) + .expect("write response"); + stream.flush().expect("flush response"); + } + }); + (port, handle) +} + +#[test] +fn async_client_rejects_non_success_statuses() { + let (port, handle) = spawn_mock_server(vec![ + MockResponse { + status_line: "500 Internal Server Error", + content_type: "application/json", + body: "{}", + }, + MockResponse { + status_line: "409 Conflict", + content_type: "text/plain", + body: "", + }, + ]); + let client = AwClient::new("127.0.0.1", port, "aw-client-rust-test").expect("create client"); + + let err = block_on(client.get_buckets()).expect_err("500 response must fail"); + assert_eq!( + err.status(), + Some(reqwest::StatusCode::INTERNAL_SERVER_ERROR) + ); + + let err = block_on(client.create_bucket_simple("bucket", "type")) + .expect_err("409 response must fail"); + assert_eq!(err.status(), Some(reqwest::StatusCode::CONFLICT)); + + handle.join().expect("join mock server"); +} + +#[test] +fn blocking_client_rejects_non_success_statuses() { + let (port, handle) = spawn_mock_server(vec![ + MockResponse { + status_line: "500 Internal Server Error", + content_type: "application/json", + body: "{}", + }, + MockResponse { + status_line: "409 Conflict", + content_type: "text/plain", + body: "", + }, + ]); + let client = + blocking::AwClient::new("127.0.0.1", port, "aw-client-rust-test").expect("create client"); + + let err = client + .get_buckets() + .expect_err("500 response must fail for blocking client"); + assert_eq!( + err.status(), + Some(reqwest::StatusCode::INTERNAL_SERVER_ERROR) + ); + + let err = client + .create_bucket_simple("bucket", "type") + .expect_err("409 response must fail for blocking client"); + assert_eq!(err.status(), Some(reqwest::StatusCode::CONFLICT)); + + handle.join().expect("join mock server"); +} diff --git a/aw-client-rust/tests/test.rs b/aw-client-rust/tests/test.rs new file mode 100644 index 0000000..64dd913 --- /dev/null +++ b/aw-client-rust/tests/test.rs @@ -0,0 +1,303 @@ +extern crate aw_client_rust; +extern crate aw_datastore; +extern crate aw_server; +extern crate chrono; +extern crate rocket; +extern crate serde_json; +extern crate tokio_test; + +#[cfg(test)] +mod test { + use aw_client_rust::blocking::AwClient; + use aw_client_rust::Event; + use chrono::{DateTime, Duration, Utc}; + use serde_json::Map; + use std::cell::RefCell; + use std::fs; + use std::net::TcpListener; + use std::path::{Path, PathBuf}; + use std::sync::Mutex; + use std::thread; + use std::time::{SystemTime, UNIX_EPOCH}; + use tokio_test::block_on; + + // Config-reading helpers — only needed in tests; production code passes API keys explicitly. + + #[derive(serde::Deserialize, Default)] + struct LocalAuthConfig { + #[serde(default)] + api_key: Option, + } + + #[derive(serde::Deserialize, Default)] + struct LocalServerConfig { + #[serde(default)] + port: Option, + #[serde(default)] + auth: LocalAuthConfig, + } + + #[derive(Clone, Copy)] + struct ConfigCandidate { + filename: &'static str, + default_port: u16, + } + + fn get_server_config_dir() -> Option { + Some( + dirs::config_dir()? + .join("activitywatch") + .join("aw-server-rust"), + ) + } + + fn load_local_api_key(host: &str, port: u16) -> Option { + if host != "127.0.0.1" && host != "localhost" { + return None; + } + let config_dir = get_server_config_dir()?; + let candidates = [ + ConfigCandidate { + filename: "config.toml", + default_port: 5600, + }, + ConfigCandidate { + filename: "config-testing.toml", + default_port: 5666, + }, + ]; + for candidate in candidates { + let path = config_dir.join(candidate.filename); + let content = match fs::read_to_string(path) { + Ok(content) => content, + Err(_) => continue, + }; + let config: LocalServerConfig = match toml::from_str(&content) { + Ok(config) => config, + Err(_) => continue, + }; + let configured_port = config.port.unwrap_or(candidate.default_port); + if configured_port == port { + return config.auth.api_key.filter(|k| !k.is_empty()); + } + } + None + } + + // A random port, but still not guaranteed to not be bound + // FIXME: Bind to a port that is free for certain and use that for the client instead + static PORT: u16 = 41293; + static ENV_LOCK: Mutex<()> = Mutex::new(()); + + // Keep the listener alive until the server binds — prevents TOCTOU race in reserve_port + thread_local! { + static RESERVED_PORT: RefCell> = const { RefCell::new(None) }; + } + + fn wait_for_server(timeout_s: u32, client: &AwClient) { + for i in 0.. { + match client.get_info() { + Ok(_) => break, + Err(err) => { + if i == timeout_s - 1 { + panic!("Timed out starting aw-server after {timeout_s}s: {err:?}"); + } + } + } + use std::time; + let duration = time::Duration::from_secs(1); + thread::sleep(duration); + } + } + + fn setup_testserver(port: u16, api_key: Option<&str>) -> rocket::Shutdown { + use aw_server::endpoints::AssetResolver; + use aw_server::endpoints::ServerState; + + let state = ServerState { + datastore: aw_datastore::Datastore::new_in_memory(false), + asset_resolver: AssetResolver::new(None), + device_id: "test_id".to_string(), + }; + let aw_config = aw_server::config::AWConfig { + port, + testing: true, + auth: aw_server::config::AWAuthConfig { + api_key: api_key.map(str::to_owned), + }, + ..Default::default() + }; + let server = aw_server::endpoints::build_rocket(state, aw_config); + let server = block_on(server.ignite()).unwrap(); + let shutdown_handler = server.shutdown(); + + thread::spawn(move || { + let _ = block_on(server.launch()).unwrap(); + }); + + shutdown_handler + } + + fn reserve_port() -> u16 { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let port = listener.local_addr().unwrap().port(); + // Keep the listener alive until the server binds — prevents TOCTOU race + RESERVED_PORT.with(|cell| *cell.borrow_mut() = Some(listener)); + port + } + + fn write_server_config(port: u16, api_key: Option<&str>) -> PathBuf { + let unique = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let config_home = std::env::temp_dir().join(format!( + "aw-client-rust-config-{}-{}", + std::process::id(), + unique + )); + let config_dir = config_home.join("activitywatch").join("aw-server-rust"); + fs::create_dir_all(&config_dir).unwrap(); + + let mut content = format!("port = {port}\n"); + if let Some(api_key) = api_key { + content.push_str("\n[auth]\n"); + content.push_str(&format!("api_key = \"{api_key}\"\n")); + } + fs::write(config_dir.join("config.toml"), content).unwrap(); + + config_home + } + + fn with_config_home(config_home: &Path, f: impl FnOnce() -> T) -> T { + let _guard = ENV_LOCK.lock().unwrap(); + let old_value = std::env::var_os("XDG_CONFIG_HOME"); + std::env::set_var("XDG_CONFIG_HOME", config_home); + let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(f)); + if let Some(old_value) = old_value { + std::env::set_var("XDG_CONFIG_HOME", old_value); + } else { + std::env::remove_var("XDG_CONFIG_HOME"); + } + let _ = fs::remove_dir_all(config_home); + result.unwrap() + } + + #[test] + fn test_full() { + let clientname = "aw-client-rust-test"; + + // Hold ENV_LOCK during client creation to prevent parallel-test interference + // with test_reads_api_key_from_matching_server_config (which holds the lock + // via with_config_home for the entire client+server lifetime). + let client: AwClient = { + let _guard = ENV_LOCK.lock().unwrap(); + AwClient::new("127.0.0.1", PORT, clientname).expect("Client creation failed") + }; + + let shutdown_handler = setup_testserver(PORT, None); + + wait_for_server(20, &client); + + let info = client.get_info().unwrap(); + assert!(info.testing); + + let bucketname = format!("aw-client-rust-test_{}", client.hostname); + let buckettype = "test-type"; + client + .create_bucket_simple(&bucketname, buckettype) + .unwrap(); + + let bucket = client.get_bucket(&bucketname).unwrap(); + assert!(bucket.id == bucketname); + println!("{}", bucket.id); + + let buckets = client.get_buckets().unwrap(); + println!("Buckets: {buckets:?}"); + let mut event = Event { + id: None, + timestamp: DateTime::from_utc( + DateTime::parse_from_rfc3339("2017-12-30T01:00:00+00:00") + .unwrap() + .naive_utc(), + Utc, + ), + duration: Duration::seconds(0), + data: Map::new(), + }; + println!("{event:?}"); + client.insert_event(&bucketname, &event).unwrap(); + // Ugly way to create a UTC from timestamp, see https://github.com/chronotope/chrono/issues/263 + event.timestamp = DateTime::from_utc( + DateTime::parse_from_rfc3339("2017-12-30T01:00:01+00:00") + .unwrap() + .naive_utc(), + Utc, + ); + client.heartbeat(&bucketname, &event, 10.0).unwrap(); + + let events = client.get_events(&bucketname, None, None, None).unwrap(); + println!("Events: {events:?}"); + assert!(events[0].duration == Duration::seconds(1)); + + // Query + let query = format!( + "events = query_bucket(\"{}\"); +RETURN = events;", + bucket.id + ); + let start: DateTime = DateTime::parse_from_rfc3339("1996-12-19T00:00:00-08:00") + .unwrap() + .into(); + let end: DateTime = DateTime::parse_from_rfc3339("2020-12-19T00:00:00-08:00") + .unwrap() + .into(); + let timeperiods = (start, end); + let query_result = client.query(&query, vec![timeperiods]).unwrap(); + println!("Query result: {query_result:?}"); + + client + .delete_event(&bucketname, events[0].id.unwrap()) + .unwrap(); + + let count = client.get_event_count(&bucketname).unwrap(); + assert_eq!(count, 0); + + client.delete_bucket(&bucketname).unwrap(); + + shutdown_handler.notify(); + } + + // XDG_CONFIG_HOME is only respected by dirs::config_dir() on Linux. + // On macOS it returns $HOME/Library/Application Support (ignoring XDG_CONFIG_HOME), + // so this test would fail — gate it on Linux only. + #[test] + #[cfg(target_os = "linux")] + fn test_reads_api_key_from_matching_server_config() { + let clientname = "aw-client-rust-auth-test"; + let port = reserve_port(); + let config_home = write_server_config(port, Some("secret123")); + + with_config_home(&config_home, || { + let api_key = load_local_api_key("127.0.0.1", port); + let client: AwClient = + AwClient::new_with_api_key("127.0.0.1", port, clientname, api_key) + .expect("Client creation failed"); + // Drop the reserved listener before Rocket tries to bind the same port. + RESERVED_PORT.with(|cell| *cell.borrow_mut() = None); + let shutdown_handler = setup_testserver(port, Some("secret123")); + + wait_for_server(20, &client); + + let bucketname = format!("aw-client-rust-auth-test_{}", client.hostname); + client + .create_bucket_simple(&bucketname, "test-type") + .unwrap(); + + let bucket = client.get_bucket(&bucketname).unwrap(); + assert_eq!(bucket.id, bucketname); + + shutdown_handler.notify(); + }); + } +} diff --git a/aw-datastore/Cargo.toml b/aw-datastore/Cargo.toml new file mode 100644 index 0000000..b2c2e1e --- /dev/null +++ b/aw-datastore/Cargo.toml @@ -0,0 +1,41 @@ +[package] +name = "aw-datastore" +version = "0.1.0" +authors = ["Johan Bjäreholt "] +edition = "2021" + +[features] +default = ["bundled"] +# Use bundled SQLite (default, no encryption support) +bundled = ["rusqlite/bundled"] +# Use bundled SQLCipher for encrypted databases (mutually exclusive with 'bundled') +# Build with: cargo build --no-default-features --features encryption +# Requires OpenSSL. Use 'encryption-vendored' to vendor OpenSSL as well. +encryption = ["rusqlite/bundled-sqlcipher", "dep:zeroize"] +# Like 'encryption' but also vendors OpenSSL (fully self-contained) +encryption-vendored = ["rusqlite/bundled-sqlcipher-vendored-openssl", "dep:zeroize"] +legacy_import_tests = [] + +[dependencies] +dirs = "6" +serde = "1.0" +serde_json = "1.0" +chrono = { version = "0.4", features = ["serde"] } +rusqlite = { version = "0.30", features = ["chrono", "serde_json"] } +mpsc_requests = "0.3" +log = "0.4" +zeroize = { version = "1", optional = true, features = ["alloc"] } + +aw-models = { path = "../aw-models" } +aw-transform = { path = "../aw-transform" } +regex = "1" + +[dev-dependencies] +criterion = "0.5.1" +tempfile = "3" +# Used by migration tests to construct databases with old schema versions +rusqlite = { version = "0.30", features = ["chrono", "serde_json"] } + +[[bench]] +name = "interval_reads" +harness = false diff --git a/aw-datastore/benches/interval_reads.rs b/aw-datastore/benches/interval_reads.rs new file mode 100644 index 0000000..558d7bd --- /dev/null +++ b/aw-datastore/benches/interval_reads.rs @@ -0,0 +1,80 @@ +use aw_datastore::DatastoreInstance; +use aw_models::{Bucket, BucketMetadata}; +use chrono::{DateTime, Utc}; +use criterion::{criterion_group, criterion_main, BatchSize, Criterion}; +use rusqlite::Connection; + +fn interval_reads(c: &mut Criterion) { + let conn = Connection::open_in_memory().unwrap(); + let mut ds = DatastoreInstance::new(&conn, true).unwrap(); + ds.create_bucket( + &conn, + Bucket { + bid: None, + id: "bench".into(), + _type: "test".into(), + client: "test".into(), + hostname: "test".into(), + created: Some(Utc::now()), + data: Default::default(), + metadata: BucketMetadata::default(), + events: None, + last_updated: None, + }, + ) + .unwrap(); + conn.execute_batch( + "WITH RECURSIVE seq(n) AS (SELECT 1 UNION ALL SELECT n+1 FROM seq WHERE n<1000000) + INSERT INTO events(bucketrow, starttime, endtime, data) + SELECT 1, n*1000000000, n*1000000000+500000000, '{\"app\":\"browser\"}' FROM seq;", + ) + .unwrap(); + let mut ds = DatastoreInstance::new(&conn, true).unwrap(); + let mut group = c.benchmark_group("interval_reads"); + for (name, start, end) in [ + ("recent", 999_000, 1_000_000), + ("middle", 500_000, 501_000), + ("early", 0, 1_000), + ] { + let start = Some(DateTime::from_timestamp(start, 0).unwrap()); + let end = Some(DateTime::from_timestamp(end, 0).unwrap()); + group.bench_function(name, |b| { + b.iter(|| ds.get_events(&conn, "bench", start, end, None).unwrap()) + }); + } + group.bench_function("latest_one", |b| { + b.iter(|| ds.get_events(&conn, "bench", None, None, Some(1)).unwrap()) + }); + let start = Some(DateTime::from_timestamp(999_000, 0).unwrap()); + group.bench_function("recent_count", |b| { + b.iter(|| ds.get_event_count(&conn, "bench", start, None).unwrap()) + }); + group.finish(); +} + +fn index_write_cost(c: &mut Criterion) { + let mut group = c.benchmark_group("index_write_cost"); + for second_index in [false, true] { + group.bench_function(if second_index { "two_indexes" } else { "one_index" }, |b| { + b.iter_batched(|| { + let conn = Connection::open_in_memory().unwrap(); + conn.execute_batch("CREATE TABLE events(id INTEGER PRIMARY KEY AUTOINCREMENT, + bucketrow INTEGER NOT NULL, starttime INTEGER NOT NULL, endtime INTEGER NOT NULL, data TEXT NOT NULL); + CREATE INDEX by_start ON events(bucketrow, starttime DESC, endtime);").unwrap(); + if second_index { + conn.execute_batch("CREATE INDEX by_end ON events(bucketrow, endtime, starttime DESC);").unwrap(); + } + conn + }, |conn| { + conn.execute_batch("WITH RECURSIVE seq(n) AS (SELECT 1 UNION ALL SELECT n+1 FROM seq WHERE n<10000) + INSERT INTO events(bucketrow,starttime,endtime,data) + SELECT 1,n*1000000000,n*1000000000+500000000,'{}' FROM seq;").unwrap(); + conn + }, BatchSize::SmallInput); + }); + } + group.finish(); +} + +criterion_group!(benches, interval_reads, index_write_cost); +criterion_main!(benches); diff --git a/aw-datastore/examples/export_memory.rs b/aw-datastore/examples/export_memory.rs new file mode 100644 index 0000000..fa8be61 --- /dev/null +++ b/aw-datastore/examples/export_memory.rs @@ -0,0 +1,61 @@ +//! Reproducible export-memory comparison using synthetic, in-memory data only. +//! Build with `cargo build -p aw-datastore --example export_memory --release`. +//! Run the resulting executable under a memory profiler with `stream 100000` +//! or `materialized 100000`. No existing database or server is opened. +use aw_datastore::DatastoreInstance; +use aw_models::{Bucket, BucketMetadata, BucketsExport, TryVec}; +use chrono::Utc; +use rusqlite::Connection; +use std::{hint::black_box, time::Instant}; + +fn main() { + let mode = std::env::args().nth(1).unwrap_or_else(|| "stream".into()); + assert!(matches!(mode.as_str(), "stream" | "materialized")); + let count: u32 = std::env::args() + .nth(2) + .unwrap_or_else(|| "100000".into()) + .parse() + .unwrap(); + let conn = Connection::open_in_memory().unwrap(); + let mut ds = DatastoreInstance::new(&conn, true).unwrap(); + ds.create_bucket( + &conn, + Bucket { + bid: None, + id: "synthetic".into(), + _type: "test".into(), + client: "test".into(), + hostname: "test".into(), + created: Some(Utc::now()), + data: Default::default(), + metadata: BucketMetadata::default(), + events: None, + last_updated: None, + }, + ) + .unwrap(); + let payload = serde_json::json!({"app": "browser", "title": "x".repeat(256)}).to_string(); + conn.execute( + "WITH RECURSIVE seq(n) AS (SELECT 1 UNION ALL SELECT n+1 FROM seq WHERE n i32 { + conn.pragma_query_value(None, "user_version", |row| row.get(0)) + .unwrap() +} + +/* + * ### Database version changelog ### + * 0: Uninitialized database + * 1: Initialized database + * 2: Added 'data' field to 'buckets' table + * 3: see: https://github.com/ActivityWatch/aw-server-rust/pull/52 + * 4: Added 'key_value' table for storing key - value pairs + * 5: Replaced single-column events indexes with a composite index + * 6: Added an endtime-first index for recent interval reads + */ +static NEWEST_DB_VERSION: i32 = 6; + +fn _create_tables(conn: &Connection, version: i32) -> bool { + let mut first_init = false; + + if version < 1 { + first_init = true; + _migrate_v0_to_v1(conn); + } + + if version < 2 { + _migrate_v1_to_v2(conn); + } + + if version < 3 { + _migrate_v2_to_v3(conn); + } + + if version < 4 { + _migrate_v3_to_v4(conn); + } + + if version < 5 { + _migrate_v4_to_v5(conn); + } + + if version < 6 { + _migrate_v5_to_v6(conn); + } + + first_init +} + +fn _migrate_v0_to_v1(conn: &Connection) { + /* Set up bucket table */ + conn.execute( + " + CREATE TABLE IF NOT EXISTS buckets ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT UNIQUE NOT NULL, + type TEXT NOT NULL, + client TEXT NOT NULL, + hostname TEXT NOT NULL, + created TEXT NOT NULL + )", + &[] as &[&dyn ToSql], + ) + .expect("Failed to create buckets table"); + + /* Set up index for bucket table */ + conn.execute( + "CREATE INDEX IF NOT EXISTS bucket_id_index ON buckets(id)", + &[] as &[&dyn ToSql], + ) + .expect("Failed to create buckets index"); + + /* Set up events table */ + conn.execute( + " + CREATE TABLE IF NOT EXISTS events ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + bucketrow INTEGER NOT NULL, + starttime INTEGER NOT NULL, + endtime INTEGER NOT NULL, + data TEXT NOT NULL, + FOREIGN KEY (bucketrow) REFERENCES buckets(id) + )", + &[] as &[&dyn ToSql], + ) + .expect("Failed to create events table"); + + /* Set up index for events table */ + conn.execute( + "CREATE INDEX IF NOT EXISTS events_bucketrow_index ON events(bucketrow)", + &[] as &[&dyn ToSql], + ) + .expect("Failed to create events_bucketrow index"); + conn.execute( + "CREATE INDEX IF NOT EXISTS events_starttime_index ON events(starttime)", + &[] as &[&dyn ToSql], + ) + .expect("Failed to create events_starttime index"); + conn.execute( + "CREATE INDEX IF NOT EXISTS events_endtime_index ON events(endtime)", + &[] as &[&dyn ToSql], + ) + .expect("Failed to create events_endtime index"); + + /* Update database version */ + conn.pragma_update(None, "user_version", 1) + .expect("Failed to update database version!"); +} + +fn _migrate_v1_to_v2(conn: &Connection) { + info!("Upgrading database to v2, adding data field to buckets"); + conn.execute( + "ALTER TABLE buckets ADD COLUMN data TEXT DEFAULT '{}';", + &[] as &[&dyn ToSql], + ) + .expect("Failed to upgrade database when adding data field to buckets"); + + conn.pragma_update(None, "user_version", 2) + .expect("Failed to update database version!"); +} + +fn _migrate_v2_to_v3(conn: &Connection) { + // For details about why this migration was necessary, see: https://github.com/ActivityWatch/aw-server-rust/pull/52 + info!("Upgrading database to v3, replacing the broken data field for buckets"); + + // Rename column, marking it as deprecated + match conn.execute( + "ALTER TABLE buckets RENAME COLUMN data TO data_deprecated;", + &[] as &[&dyn ToSql], + ) { + Ok(_) => (), + // This error is okay, it still has the intended effects + Err(rusqlite::Error::ExecuteReturnedResults) => (), + Err(e) => panic!("Unexpected error: {e:?}"), + }; + + // Create new correct column + conn.execute( + "ALTER TABLE buckets ADD COLUMN data TEXT NOT NULL DEFAULT '{}';", + &[] as &[&dyn ToSql], + ) + .expect("Failed to upgrade database when adding new data field to buckets"); + + conn.pragma_update(None, "user_version", 3) + .expect("Failed to update database version!"); +} + +fn _migrate_v3_to_v4(conn: &Connection) { + info!("Upgrading database to v4, adding table for key-value storage"); + conn.execute( + "CREATE TABLE key_value ( + key TEXT PRIMARY KEY, + value TEXT, + last_modified NUMBER NOT NULL + );", + [], + ) + .expect("Failed to upgrade db and add key-value storage table"); + + conn.pragma_update(None, "user_version", 4) + .expect("Failed to update database version!"); +} + +fn _migrate_v4_to_v5(conn: &Connection) { + info!( + "Upgrading database to v5, replacing single-column events indexes with a composite index" + ); + // Every event query filters on bucketrow and a starttime/endtime range, + // ordered by starttime. A composite index serves the seek, the range scan + // and the ORDER BY in one pass (with endtime checked from the index + // without fetching the row), where the single-column indexes could only + // cover one predicate and left the rest as scan + sort. Dropping them + // also makes inserts cheaper (one index to maintain instead of three). + // + // starttime is DESC so a forward scan yields the query's newest-first + // order with equal-timestamp events in rowid (insertion) order, matching + // the ordering callers observed before this index existed. + // + // The drops run before the create so the pages they free are reused to + // build the new index within the same transaction; creating first would + // permanently grow the database file by the new index's size. + conn.execute_batch( + " + BEGIN EXCLUSIVE TRANSACTION; + DROP INDEX IF EXISTS events_bucketrow_index; + DROP INDEX IF EXISTS events_starttime_index; + DROP INDEX IF EXISTS events_endtime_index; + CREATE INDEX IF NOT EXISTS events_bucketrow_starttime_endtime_index + ON events(bucketrow, starttime DESC, endtime); + PRAGMA user_version = 5; + COMMIT; + ", + ) + .expect("Failed to run v5 migration transaction"); +} + +fn _migrate_v5_to_v6(conn: &Connection) { + conn.execute_batch( + "BEGIN EXCLUSIVE TRANSACTION; + CREATE INDEX IF NOT EXISTS events_bucketrow_endtime_starttime_index + ON events(bucketrow, endtime, starttime DESC); + PRAGMA user_version = 6; + COMMIT;", + ) + .expect("Failed to run v6 migration transaction"); +} + +// Both indexes can bound only one of the two interval predicates. Use the +// bucket's time span as a cheap selectivity estimate; this affects performance, +// never which rows qualify. Limited queries retain their ordered starttime scan +// so LIMIT can stop early without sorting all matching rows. +fn prefer_endtime_index(bucket: &Bucket, start: i64, end: i64, limit: Option) -> bool { + if limit.is_some() { + return false; + } + match ( + bucket + .metadata + .start + .and_then(|dt| dt.timestamp_nanos_opt()), + bucket.metadata.end.and_then(|dt| dt.timestamp_nanos_opt()), + ) { + (Some(first), Some(last)) => { + let first = i128::from(first); + let last = i128::from(last); + // Clamp unbounded ranges to the bucket span, so a full-bucket + // export keeps its ordered scan. Require a substantial advantage + // to offset sorting the endtime index's matching rows. + let start_scan = (i128::from(end).min(last) - first).max(0); + let end_scan = (last - i128::from(start).max(first)).max(0); + end_scan * 4 < start_scan + } + _ => false, + } +} + +pub struct DatastoreInstance { + buckets_cache: HashMap, + first_init: bool, + pub db_version: i32, +} + +fn _datetime_from_nanos(ns: i64) -> DateTime { + // Euclidean division so a negative (pre-epoch) timestamp still yields a + // subnanos remainder in [0, 1_000_000_000) instead of a negative value + // that wraps to a bogus u32 and makes `from_timestamp` return None. + let seconds = ns.div_euclid(1_000_000_000); + let subnanos = ns.rem_euclid(1_000_000_000) as u32; + DateTime::from_timestamp(seconds, subnanos).unwrap() +} + +/// Parse an event from a row selected as `id, starttime, endtime, data`. +/// +/// When `clip` is set to `(starttime_filter_ns, endtime_filter_ns)`, the event is +/// clamped to that query range. +pub(crate) fn parse_event_row( + row: &rusqlite::Row, + clip: Option<(i64, i64)>, +) -> rusqlite::Result { + let id = row.get(0)?; + let mut starttime_ns: i64 = row.get(1)?; + let mut endtime_ns: i64 = row.get(2)?; + let data_str: String = row.get(3)?; + + if let Some((starttime_filter_ns, endtime_filter_ns)) = clip { + if starttime_ns < starttime_filter_ns { + starttime_ns = starttime_filter_ns + } + if endtime_ns > endtime_filter_ns { + endtime_ns = endtime_filter_ns + } + } + let duration_ns = endtime_ns - starttime_ns; + + let time_seconds: i64 = starttime_ns / 1_000_000_000; + let time_subnanos: u32 = (starttime_ns % 1_000_000_000) as u32; + let data: serde_json::map::Map = + serde_json::from_str(&data_str).map_err(|err| { + rusqlite::Error::FromSqlConversionFailure(3, rusqlite::types::Type::Text, Box::new(err)) + })?; + + Ok(Event { + id: Some(id), + timestamp: DateTime::from_timestamp(time_seconds, time_subnanos).unwrap(), + duration: Duration::nanoseconds(duration_ns), + data, + }) +} + +/// Legacy-bucket events that overlap no other event in either bucket, so they can +/// be moved into the destination bucket without creating overlapping records. +/// +/// Two events overlap when `a.starttime < b.endtime AND b.starttime < a.endtime` +/// (strict, matching the datastore's overlap semantics elsewhere). Events are +/// read once ordered by `starttime`. A sweep keeps still-open events in a +/// min-heap keyed by `endtime`. A positive-duration event overlaps every +/// still-open event, so those are marked in O(1) via an epoch counter rather +/// than scanning the heap (which would be quadratic for stacked histories). +/// Zero-duration events are the only case that still scans, and they never stay +/// in the open set. The pass is O(n log n). +fn movable_legacy_event_ids( + conn: &Connection, + legacy_bid: Option, + destination_bid: Option, +) -> rusqlite::Result> { + use std::cmp::Reverse; + use std::collections::BinaryHeap; + + struct Span { + id: i64, + starttime: i64, + endtime: i64, + legacy: bool, + } + + let mut stmt = conn.prepare( + "SELECT id, starttime, endtime, bucketrow FROM events + WHERE bucketrow IN (?1, ?2) + ORDER BY starttime ASC, id ASC", + )?; + let spans = stmt + .query_map(params![legacy_bid, destination_bid], |row| { + let bucketrow: Option = row.get(3)?; + Ok(Span { + id: row.get(0)?, + starttime: row.get(1)?, + endtime: row.get(2)?, + legacy: bucketrow == legacy_bid, + }) + })? + .collect::>>()?; + + let mut overlaps = vec![false; spans.len()]; + // Open events: (endtime, index, epoch-at-push). Epoch marks the whole + // concurrent set in O(1) when a later positive-duration event overlaps it. + let mut open: BinaryHeap> = BinaryHeap::new(); + let mut epoch: u32 = 0; + for (i, span) in spans.iter().enumerate() { + // Events that ended at or before this start can never overlap this or + // any later event (later events start no earlier than this one). + while let Some(Reverse((endtime, j, pushed_epoch))) = open.peek().copied() { + if endtime <= span.starttime { + open.pop(); + if pushed_epoch < epoch { + overlaps[j] = true; + } + } else { + break; + } + } + if !open.is_empty() { + if span.endtime > span.starttime { + // Positive duration: every remaining open event `j` has + // `j.starttime <= span.starttime < span.endtime` and + // `span.starttime < j.endtime`, so they all overlap. + overlaps[i] = true; + epoch += 1; + } else { + // Zero-duration: overlaps open events that started strictly + // earlier, but not same-start positive-duration events + // (`j.starttime < span.endtime` fails when starttimes match). + let mut overlapped = false; + for Reverse((_, j, _)) in open.iter() { + if spans[*j].starttime < span.starttime { + overlaps[*j] = true; + overlapped = true; + } + } + if overlapped { + overlaps[i] = true; + } + } + } + open.push(Reverse((span.endtime, i, epoch))); + } + while let Some(Reverse((_, j, pushed_epoch))) = open.pop() { + if pushed_epoch < epoch { + overlaps[j] = true; + } + } + + Ok(spans + .iter() + .zip(overlaps.iter()) + .filter(|(span, overlapping)| span.legacy && !**overlapping) + .map(|(span, _)| span.id) + .collect()) +} + +/// Reassign the given events to `bucket_bid`, in batches so the statement stays +/// well under SQLite's bound-parameter limit. +fn move_events_to_bucket( + conn: &Connection, + bucket_bid: Option, + event_ids: &[i64], +) -> rusqlite::Result<()> { + const BATCH: usize = 500; + for chunk in event_ids.chunks(BATCH) { + let placeholders = (0..chunk.len()) + .map(|i| format!("?{}", i + 2)) + .collect::>() + .join(", "); + let sql = format!("UPDATE events SET bucketrow = ?1 WHERE id IN ({placeholders})"); + let mut stmt = conn.prepare_cached(&sql)?; + let mut values: Vec<&dyn ToSql> = Vec::with_capacity(chunk.len() + 1); + values.push(&bucket_bid); + for id in chunk { + values.push(id); + } + stmt.execute(values.as_slice())?; + } + Ok(()) +} + +impl DatastoreInstance { + pub fn new( + conn: &Connection, + migrate_enabled: bool, + ) -> Result { + let mut first_init = false; + let db_version = _get_db_version(conn); + + if migrate_enabled { + first_init = _create_tables(conn, db_version); + } else if db_version < 0 { + return Err(DatastoreError::Uninitialized( + "Tried to open an uninitialized datastore with migration disabled".to_string(), + )); + } else if db_version != NEWEST_DB_VERSION { + return Err(DatastoreError::OldDbVersion(format!( + "\ + Tried to open an database with an incompatible database version! + Database has version {db_version} while the supported version is {NEWEST_DB_VERSION}" + ))); + } + + let mut ds = DatastoreInstance { + buckets_cache: HashMap::new(), + first_init, + db_version, + }; + ds.get_stored_buckets(conn)?; + Ok(ds) + } + + fn get_stored_buckets(&mut self, conn: &Connection) -> Result<(), DatastoreError> { + let mut stmt = match conn.prepare_cached( + " + SELECT buckets.id, buckets.name, buckets.type, buckets.client, + buckets.hostname, buckets.created, + min(events.starttime), max(events.endtime), + buckets.data + FROM buckets + LEFT OUTER JOIN events ON buckets.id = events.bucketrow + GROUP BY buckets.id + ;", + ) { + Ok(stmt) => stmt, + Err(err) => { + return Err(DatastoreError::InternalError(format!( + "Failed to prepare get_stored_buckets SQL statement: {err}" + ))) + } + }; + let buckets = match stmt.query_map(&[] as &[&dyn ToSql], |row| { + let opt_start_ns: Option = row.get(6)?; + let opt_start = match opt_start_ns { + Some(starttime_ns) => { + let seconds: i64 = starttime_ns / 1_000_000_000; + let subnanos: u32 = (starttime_ns % 1_000_000_000) as u32; + Some(DateTime::from_timestamp(seconds, subnanos).unwrap()) + } + None => None, + }; + + let opt_end_ns: Option = row.get(7)?; + let opt_end = match opt_end_ns { + Some(endtime_ns) => { + let seconds: i64 = endtime_ns / 1_000_000_000; + let subnanos: u32 = (endtime_ns % 1_000_000_000) as u32; + Some(DateTime::from_timestamp(seconds, subnanos).unwrap()) + } + None => None, + }; + + // If data column is not set (possible on old installations), use an empty map as default + let data_str: String = row.get(8)?; + let data_json = match serde_json::from_str(&data_str) { + Ok(data) => data, + Err(e) => { + return Err(rusqlite::Error::InvalidColumnName(format!( + "Failed to parse data to JSON: {e:?}" + ))) + } + }; + + Ok(Bucket { + bid: row.get(0)?, + id: row.get(1)?, + _type: row.get(2)?, + client: row.get(3)?, + hostname: row.get(4)?, + created: row.get(5)?, + data: data_json, + metadata: BucketMetadata { + start: opt_start, + end: opt_end, + }, + events: None, + last_updated: None, + }) + }) { + Ok(buckets) => buckets, + Err(err) => { + return Err(DatastoreError::InternalError(format!( + "Failed to query get_stored_buckets SQL statement: {err:?}" + ))) + } + }; + let mut new_cache = HashMap::new(); + for bucket in buckets { + match bucket { + Ok(b) => { + new_cache.insert(b.id.clone(), b); + } + Err(e) => { + return Err(DatastoreError::InternalError(format!( + "Failed to parse bucket from SQLite, database is corrupt! {e:?}" + ))) + } + } + } + self.buckets_cache = new_cache; + Ok(()) + } + + #[allow(clippy::result_unit_err)] + pub fn ensure_legacy_import(&mut self, conn: &Connection) -> Result { + use super::legacy_import::legacy_import; + if !self.first_init { + Ok(false) + } else { + self.first_init = false; + match legacy_import(self, conn) { + Ok(_) => { + info!("Successfully imported legacy database"); + self.get_stored_buckets(conn).unwrap(); + Ok(true) + } + Err(err) => { + warn!("Failed to import legacy database: {:?}", err); + Err(()) + } + } + } + } + + pub fn create_bucket( + &mut self, + conn: &Connection, + mut bucket: Bucket, + ) -> Result<(), DatastoreError> { + bucket.created = match bucket.created { + Some(created) => Some(created), + None => Some(Utc::now()), + }; + let mut stmt = match conn.prepare_cached( + " + INSERT INTO buckets (name, type, client, hostname, created, data) + VALUES (?1, ?2, ?3, ?4, ?5, ?6)", + ) { + Ok(buckets) => buckets, + Err(err) => { + return Err(DatastoreError::InternalError(format!( + "Failed to prepare create_bucket SQL statement: {err}" + ))) + } + }; + let data = serde_json::to_string(&bucket.data).unwrap(); + let res = stmt.execute([ + &bucket.id, + &bucket._type, + &bucket.client, + &bucket.hostname, + &bucket.created as &dyn ToSql, + &data, + ]); + + match res { + Ok(_) => { + info!("Created bucket {}", bucket.id); + // Get and set rowid + let rowid: i64 = conn.last_insert_rowid(); + bucket.bid = Some(rowid); + // Take out events from struct before caching + let events = bucket.events; + bucket.events = None; + // Cache bucket + self.buckets_cache.insert(bucket.id.clone(), bucket.clone()); + // Insert events + if let Some(events) = events { + self.insert_events(conn, &bucket.id, events.take_inner())?; + bucket.events = None; + } + Ok(()) + } + Err(rusqlite::Error::SqliteFailure(sqlerr, _)) + if sqlerr.code == rusqlite::ErrorCode::ConstraintViolation => + { + Err(DatastoreError::BucketAlreadyExists(bucket.id.to_string())) + } + Err(err) => Err(DatastoreError::InternalError(format!( + "Failed to execute create_bucket SQL statement: {err}" + ))), + } + } + + pub fn delete_bucket( + &mut self, + conn: &Connection, + bucket_id: &str, + ) -> Result<(), DatastoreError> { + let bucket = (self.get_bucket(bucket_id))?; + // Delete all events in bucket + match conn.execute("DELETE FROM events WHERE bucketrow = ?1", [&bucket.bid]) { + Ok(_) => (), + Err(err) => return Err(DatastoreError::InternalError(err.to_string())), + } + // Delete bucket itself + match conn.execute("DELETE FROM buckets WHERE id = ?1", [&bucket.bid]) { + Ok(_) => { + self.buckets_cache.remove(bucket_id); + Ok(()) + } + Err(err) => match err { + rusqlite::Error::SqliteFailure { 0: sqlerr, 1: _ } => match sqlerr.code { + rusqlite::ErrorCode::ConstraintViolation => { + Err(DatastoreError::BucketAlreadyExists(bucket_id.to_string())) + } + _ => Err(DatastoreError::InternalError(err.to_string())), + }, + _ => Err(DatastoreError::InternalError(err.to_string())), + }, + } + } + + pub fn get_bucket(&self, bucket_id: &str) -> Result { + let cached_bucket = self.buckets_cache.get(bucket_id); + match cached_bucket { + Some(bucket) => Ok(bucket.clone()), + None => Err(DatastoreError::NoSuchBucket(bucket_id.to_string())), + } + } + + pub fn get_buckets(&self) -> HashMap { + self.buckets_cache.clone() + } + + /// Serialize an export one event at a time using the caller's connection. + /// Returns the bucket ID for a single-bucket export (for download naming). + pub fn write_export( + &self, + conn: &Connection, + bucket_id: Option<&str>, + writer: impl std::io::Write, + ) -> Result, DatastoreError> { + crate::export::write_export(conn, &self.buckets_cache, bucket_id, writer)?; + Ok(bucket_id.map(str::to_owned).or_else(|| { + (self.buckets_cache.len() == 1) + .then(|| self.buckets_cache.keys().next().unwrap().clone()) + })) + } + + pub fn insert_events( + &mut self, + conn: &Connection, + bucket_id: &str, + mut events: Vec, + ) -> Result, DatastoreError> { + let mut bucket = self.get_bucket(bucket_id)?; + + let mut stmt = match conn.prepare_cached( + " + INSERT OR REPLACE INTO events(bucketrow, id, starttime, endtime, data) + VALUES (?1, ?2, ?3, ?4, ?5)", + ) { + Ok(stmt) => stmt, + Err(err) => { + return Err(DatastoreError::InternalError(format!( + "Failed to prepare insert_events SQL statement: {err}" + ))) + } + }; + for event in &mut events { + let starttime_nanos = event.timestamp.timestamp_nanos_opt().unwrap(); + let duration_nanos = match event.duration.num_nanoseconds() { + Some(nanos) => nanos, + None => { + return Err(DatastoreError::InternalError( + "Failed to convert duration to nanoseconds".to_string(), + )) + } + }; + let endtime_nanos = starttime_nanos + duration_nanos; + let data = serde_json::to_string(&event.data).unwrap(); + let res = stmt.execute([ + &bucket.bid.unwrap(), + &event.id as &dyn ToSql, + &starttime_nanos, + &endtime_nanos, + &data as &dyn ToSql, + ]); + match res { + Ok(_) => { + self.update_endtime(&mut bucket, event); + let rowid = conn.last_insert_rowid(); + event.id = Some(rowid); + } + Err(err) => { + return Err(DatastoreError::InternalError(format!( + "Failed to insert event: {event:?}, {err}" + ))); + } + }; + } + Ok(events) + } + + pub fn delete_events_by_id( + &mut self, + conn: &Connection, + bucket_id: &str, + event_ids: Vec, + ) -> Result<(), DatastoreError> { + let mut bucket = self.get_bucket(bucket_id)?; + let mut stmt = match conn.prepare_cached( + " + DELETE FROM events + WHERE bucketrow = ?1 AND id = ?2", + ) { + Ok(stmt) => stmt, + Err(err) => { + return Err(DatastoreError::InternalError(format!( + "Failed to prepare insert_events SQL statement: {err}" + ))) + } + }; + for id in event_ids { + let res = stmt.execute([&bucket.bid.unwrap(), &id as &dyn ToSql]); + match res { + Ok(_) => {} + Err(err) => { + return Err(DatastoreError::InternalError(format!( + "Failed to delete event with id {id} in bucket {bucket_id}: {err:?}" + ))); + } + }; + } + // Deletion can shrink (or empty) the bucket's time span. prefer_endtime_index + // estimates selectivity from these cached bounds, so refresh them here - + // otherwise a bucket trimmed down after this call keeps the stale, wider + // span and can misjudge which index actually wins. + self.refresh_bucket_bounds(conn, &mut bucket)?; + Ok(()) + } + + /// Recompute a bucket's cached start/end bounds from the events table and + /// update the bucket cache. Used after deletions, which unlike inserts can + /// shrink the bucket's span rather than only extend it. + fn refresh_bucket_bounds( + &mut self, + conn: &Connection, + bucket: &mut Bucket, + ) -> Result<(), DatastoreError> { + let mut stmt = match conn + .prepare_cached("SELECT min(starttime), max(endtime) FROM events WHERE bucketrow = ?1") + { + Ok(stmt) => stmt, + Err(err) => { + return Err(DatastoreError::InternalError(format!( + "Failed to prepare refresh_bucket_bounds SQL statement: {err}" + ))) + } + }; + let (start_ns, end_ns): (Option, Option) = match stmt + .query_row([&bucket.bid.unwrap() as &dyn ToSql], |row| { + Ok((row.get(0)?, row.get(1)?)) + }) { + Ok(bounds) => bounds, + Err(err) => { + return Err(DatastoreError::InternalError(format!( + "Failed to query refresh_bucket_bounds SQL statement: {err:?}" + ))) + } + }; + bucket.metadata.start = start_ns.map(_datetime_from_nanos); + bucket.metadata.end = end_ns.map(_datetime_from_nanos); + self.buckets_cache.insert(bucket.id.clone(), bucket.clone()); + Ok(()) + } + + // TODO: Function for deleting events by timerange with limit + + fn update_endtime(&mut self, bucket: &mut Bucket, event: &Event) { + let mut update = false; + /* Potentially update start */ + match bucket.metadata.start { + None => { + bucket.metadata.start = Some(event.timestamp); + update = true; + } + Some(current_start) => { + if current_start > event.timestamp { + bucket.metadata.start = Some(event.timestamp); + update = true; + } + } + } + /* Potentially update end */ + let event_endtime = event.calculate_endtime(); + match bucket.metadata.end { + None => { + bucket.metadata.end = Some(event_endtime); + update = true; + } + Some(current_end) => { + if current_end < event_endtime { + bucket.metadata.end = Some(event_endtime); + update = true; + } + } + } + /* Update buchets_cache if start or end has been updated */ + if update { + self.buckets_cache.insert(bucket.id.clone(), bucket.clone()); + } + } + + pub fn replace_last_event( + &mut self, + conn: &Connection, + bucket_id: &str, + event_id: i64, + event: &Event, + ) -> Result<(), DatastoreError> { + let mut bucket = self.get_bucket(bucket_id)?; + + // Use event ID directly instead of max(endtime) to avoid mismatch with get_events ordering + let mut stmt = match conn.prepare_cached( + " + UPDATE events + SET starttime = ?2, endtime = ?3, data = ?4 + WHERE bucketrow = ?1 AND id = ?5 + ", + ) { + Ok(stmt) => stmt, + Err(err) => { + return Err(DatastoreError::InternalError(format!( + "Failed to prepare replace_last_event SQL statement: {err}" + ))) + } + }; + let starttime_nanos = event.timestamp.timestamp_nanos_opt().unwrap(); + let duration_nanos = match event.duration.num_nanoseconds() { + Some(nanos) => nanos, + None => { + return Err(DatastoreError::InternalError( + "Failed to convert duration to nanoseconds".to_string(), + )) + } + }; + let endtime_nanos = starttime_nanos + duration_nanos; + let data = serde_json::to_string(&event.data).unwrap(); + match stmt.execute([ + &bucket.bid.unwrap(), + &starttime_nanos, + &endtime_nanos, + &data as &dyn ToSql, + &event_id, + ]) { + Ok(0) => { + return Err(DatastoreError::InternalError(format!( + "replace_last_event matched 0 rows for event_id {event_id} - cache/DB inconsistency" + ))) + } + Ok(_) => self.update_endtime(&mut bucket, event), + Err(err) => { + return Err(DatastoreError::InternalError(format!( + "Failed to execute replace_last_event SQL statement: {err}" + ))) + } + }; + Ok(()) + } + + pub fn heartbeat( + &mut self, + conn: &Connection, + bucket_id: &str, + heartbeat: Event, + pulsetime: f64, + last_heartbeat: &mut HashMap>, + ) -> Result { + self.get_bucket(bucket_id)?; + if !last_heartbeat.contains_key(bucket_id) { + last_heartbeat.insert(bucket_id.to_string(), None); + } + let last_event = match last_heartbeat.remove(bucket_id).unwrap() { + // last heartbeat is in cache + Some(last_event) => last_event, + None => { + // last heartbeat was not in cache, fetch from DB + let mut last_event_vec = self.get_events(conn, bucket_id, None, None, Some(1))?; + match last_event_vec.pop() { + Some(last_event) => last_event, + None => { + // There was no last event, insert and return + let mut inserted = self.insert_events(conn, bucket_id, vec![heartbeat])?; + return Ok(inserted.pop().unwrap()); + } + } + } + }; + let inserted_heartbeat = match aw_transform::heartbeat(&last_event, &heartbeat, pulsetime) { + Some(mut merged_heartbeat) => { + debug!("Merged heartbeat successfully"); + // Use the event ID from last_event to ensure we update the correct row + let event_id = last_event.id.ok_or_else(|| { + DatastoreError::InternalError("last_event has no ID".to_string()) + })?; + self.replace_last_event(conn, bucket_id, event_id, &merged_heartbeat)?; + // Preserve the event ID on the cached heartbeat so subsequent + // heartbeats can look it up for replace_last_event + merged_heartbeat.id = Some(event_id); + merged_heartbeat + } + None => { + debug!("Failed to merge heartbeat"); + // insert_events sets the ID on the events in the vec, so use the + // returned event (with ID) instead of the original heartbeat + let mut inserted = self.insert_events(conn, bucket_id, vec![heartbeat])?; + inserted.pop().unwrap() + } + }; + last_heartbeat.insert(bucket_id.to_string(), Some(inserted_heartbeat.clone())); + Ok(inserted_heartbeat) + } + + pub fn get_event( + &mut self, + conn: &Connection, + bucket_id: &str, + event_id: i64, + ) -> Result { + let bucket = self.get_bucket(bucket_id)?; + + let mut stmt = match conn.prepare_cached( + " + SELECT id, starttime, endtime, data + FROM events + WHERE bucketrow = ?1 + AND id = ?2 + LIMIT 1 + ;", + ) { + Ok(stmt) => stmt, + Err(err) => { + return Err(DatastoreError::InternalError(format!( + "Failed to prepare get_event SQL statement: {err}" + ))) + } + }; + + let row = match stmt.query_row([&bucket.bid.unwrap(), &event_id], |row| { + parse_event_row(row, None) + }) { + Ok(rows) => rows, + Err(err) => { + return Err(DatastoreError::InternalError(format!( + "Failed to map get_event SQL statement: {err}" + ))) + } + }; + + Ok(row) + } + + fn get_events_inner( + &mut self, + conn: &Connection, + bucket_id: &str, + starttime_opt: Option>, + endtime_opt: Option>, + limit_opt: Option, + clip_to_query_range: bool, + ) -> Result, DatastoreError> { + let bucket = self.get_bucket(bucket_id)?; + + let mut list = Vec::new(); + + let starttime_filter_ns: i64 = match starttime_opt { + Some(dt) => dt.timestamp_nanos_opt().unwrap(), + None => 0, + }; + let endtime_filter_ns: i64 = match endtime_opt { + Some(dt) => dt.timestamp_nanos_opt().unwrap(), + None => i64::MAX, + }; + if starttime_filter_ns > endtime_filter_ns { + warn!("Starttime in event query was lower than endtime!"); + return Ok(list); + } + let limit = match limit_opt { + Some(l) => l as i64, + None => -1, + }; + + let sql = + if prefer_endtime_index(&bucket, starttime_filter_ns, endtime_filter_ns, limit_opt) { + "SELECT id, starttime, endtime, data + FROM events INDEXED BY events_bucketrow_endtime_starttime_index + WHERE bucketrow = ?1 AND endtime >= ?2 AND starttime <= ?3 + ORDER BY starttime DESC, endtime ASC, id ASC LIMIT ?4" + } else { + "SELECT id, starttime, endtime, data + FROM events INDEXED BY events_bucketrow_starttime_endtime_index + WHERE bucketrow = ?1 AND endtime >= ?2 AND starttime <= ?3 + ORDER BY starttime DESC, endtime ASC, id ASC LIMIT ?4" + }; + let mut stmt = match conn.prepare_cached(sql) { + Ok(stmt) => stmt, + Err(err) => { + return Err(DatastoreError::InternalError(format!( + "Failed to prepare get_events SQL statement: {err}" + ))) + } + }; + + let rows = match stmt.query_map( + [ + &bucket.bid.unwrap(), + &starttime_filter_ns, + &endtime_filter_ns, + &limit, + ], + |row| { + parse_event_row( + row, + clip_to_query_range.then_some((starttime_filter_ns, endtime_filter_ns)), + ) + }, + ) { + Ok(rows) => rows, + Err(err) => { + return Err(DatastoreError::InternalError(format!( + "Failed to map get_events SQL statement: {err}" + ))) + } + }; + for row in rows { + match row { + Ok(event) => list.push(event), + Err(err) => warn!("Corrupt event in bucket {}: {}", bucket_id, err), + }; + } + + Ok(list) + } + + pub fn get_events( + &mut self, + conn: &Connection, + bucket_id: &str, + starttime_opt: Option>, + endtime_opt: Option>, + limit_opt: Option, + ) -> Result, DatastoreError> { + self.get_events_inner(conn, bucket_id, starttime_opt, endtime_opt, limit_opt, true) + } + + pub fn get_events_unclipped( + &mut self, + conn: &Connection, + bucket_id: &str, + starttime_opt: Option>, + endtime_opt: Option>, + limit_opt: Option, + ) -> Result, DatastoreError> { + self.get_events_inner( + conn, + bucket_id, + starttime_opt, + endtime_opt, + limit_opt, + false, + ) + } + + pub fn get_event_count( + &self, + conn: &Connection, + bucket_id: &str, + starttime_opt: Option>, + endtime_opt: Option>, + ) -> Result { + let bucket = self.get_bucket(bucket_id)?; + + let starttime_filter_ns: i64 = match starttime_opt { + Some(dt) => dt.timestamp_nanos_opt().unwrap(), + None => 0, + }; + let endtime_filter_ns: i64 = match endtime_opt { + Some(dt) => dt.timestamp_nanos_opt().unwrap(), + None => i64::MAX, + }; + if starttime_filter_ns >= endtime_filter_ns { + warn!("Endtime in event query was same or lower than starttime!"); + return Ok(0); + } + + let sql = if prefer_endtime_index(&bucket, starttime_filter_ns, endtime_filter_ns, None) { + "SELECT count(*) FROM events INDEXED BY events_bucketrow_endtime_starttime_index + WHERE bucketrow = ?1 AND endtime >= ?2 AND starttime <= ?3" + } else { + "SELECT count(*) FROM events INDEXED BY events_bucketrow_starttime_endtime_index + WHERE bucketrow = ?1 AND endtime >= ?2 AND starttime <= ?3" + }; + let mut stmt = match conn.prepare_cached(sql) { + Ok(stmt) => stmt, + Err(err) => { + return Err(DatastoreError::InternalError(format!( + "Failed to prepare get_event_count SQL statement: {err}", + ))) + } + }; + + let count = match stmt.query_row( + [ + &bucket.bid.unwrap(), + &starttime_filter_ns, + &endtime_filter_ns, + ], + |row| row.get(0), + ) { + Ok(count) => count, + Err(err) => { + return Err(DatastoreError::InternalError(format!( + "Failed to query get_event_count SQL statement: {err}" + ))) + } + }; + + Ok(count) + } + + pub fn insert_key_value( + &self, + conn: &Connection, + key: &str, + data: &str, + ) -> Result<(), DatastoreError> { + let mut stmt = match conn.prepare_cached( + " + INSERT OR REPLACE INTO key_value(key, value, last_modified) + VALUES (?1, ?2, ?3)", + ) { + Ok(stmt) => stmt, + Err(err) => { + return Err(DatastoreError::InternalError(format!( + "Failed to prepare insert_value SQL statement: {err}" + ))) + } + }; + let timestamp = Utc::now().timestamp(); + #[allow(clippy::expect_fun_call)] + stmt.execute(params![key, data, ×tamp]) + .expect(&format!("Failed to insert key-value pair: {key}")); + Ok(()) + } + + pub fn delete_key_value(&self, conn: &Connection, key: &str) -> Result<(), DatastoreError> { + conn.execute("DELETE FROM key_value WHERE key = ?1", [key]) + .expect("Error deleting value from database"); + Ok(()) + } + + pub fn get_key_value(&self, conn: &Connection, key: &str) -> Result { + let mut stmt = match conn.prepare_cached( + " + SELECT * FROM key_value WHERE KEY = ?1", + ) { + Ok(stmt) => stmt, + Err(err) => { + return Err(DatastoreError::InternalError(format!( + "Failed to prepare get_value SQL statement: {err}" + ))) + } + }; + + match stmt.query_row([key], |row| row.get(1)) { + Ok(result) => Ok(result), + Err(err) => match err { + rusqlite::Error::QueryReturnedNoRows => { + Err(DatastoreError::NoSuchKey(key.to_string())) + } + _ => Err(DatastoreError::InternalError(format!( + "Get value query failed for key {key}" + ))), + }, + } + } + + pub fn get_key_values( + &self, + conn: &Connection, + pattern: &str, + ) -> Result, DatastoreError> { + let mut stmt = + match conn.prepare_cached("SELECT key, value FROM key_value WHERE key LIKE ?") { + Ok(stmt) => stmt, + Err(err) => { + return Err(DatastoreError::InternalError(format!( + "Failed to prepare get_value SQL statement: {err}" + ))) + } + }; + + let mut output = HashMap::::new(); + // Rusqlite's get wants index and item type as parameters. + let result = stmt.query_map([pattern], |row| { + Ok((row.get::(0)?, row.get::(1)?)) + }); + match result { + Ok(settings) => { + for row in settings { + // Unwrap to String or panic on SQL row if type is invalid. Can't happen with a + // properly initialized table. + let (key, value) = row.unwrap(); + // Only return keys starting with "settings.". + if !key.starts_with("settings.") { + continue; + } + output.insert(key, value); + } + Ok(output) + } + Err(err) => match err { + rusqlite::Error::QueryReturnedNoRows => Ok(output), + _ => Err(DatastoreError::InternalError( + "Failed to get settings".to_string(), + )), + }, + } + } + + /// Renames a bucket from `old_id` to `new_id`. + /// Events are left untouched because they reference the integer row ID, not the name. + /// Returns `NoSuchBucket` if `old_id` does not exist, or `BucketAlreadyExists` if + /// `new_id` is already taken. + pub fn rename_bucket( + &mut self, + conn: &Connection, + old_id: &str, + new_id: &str, + ) -> Result<(), DatastoreError> { + if !self.buckets_cache.contains_key(old_id) { + return Err(DatastoreError::NoSuchBucket(old_id.to_string())); + } + if self.buckets_cache.contains_key(new_id) { + return Err(DatastoreError::BucketAlreadyExists(new_id.to_string())); + } + + match conn.execute( + "UPDATE buckets SET name = ?1 WHERE name = ?2", + [new_id, old_id], + ) { + Ok(0) => Err(DatastoreError::NoSuchBucket(old_id.to_string())), + Ok(_) => { + info!("Renamed bucket '{}' to '{}'", old_id, new_id); + // Update the in-memory cache: remove the old entry and re-insert under the new id. + if let Some(mut bucket) = self.buckets_cache.remove(old_id) { + bucket.id = new_id.to_string(); + self.buckets_cache.insert(new_id.to_string(), bucket); + } + Ok(()) + } + Err(err) => Err(DatastoreError::InternalError(format!( + "Failed to rename bucket '{}' to '{}': {err}", + old_id, new_id + ))), + } + } + + /// Migrates all buckets whose hostname is "unknown" or "Unknown" to `new_hostname`. + /// Events are left untouched; only the bucket metadata is updated. + /// Returns the number of buckets that were updated. + pub fn migrate_hostname( + &mut self, + conn: &Connection, + new_hostname: &str, + ) -> Result { + info!( + "Migrating hostname from 'unknown'/'Unknown' to '{}'", + new_hostname + ); + + let updated = match conn.execute( + "UPDATE buckets SET hostname = ?1 WHERE hostname = 'unknown' OR hostname = 'Unknown'", + [new_hostname], + ) { + Ok(n) => n, + Err(err) => { + return Err(DatastoreError::InternalError(format!( + "Failed to migrate hostname: {err}" + ))) + } + }; + + if updated > 0 { + info!("Migrated hostname for {} bucket(s)", updated); + // Refresh the in-memory cache so callers see the new hostnames immediately. + self.get_stored_buckets(conn)?; + } else { + info!("No buckets with hostname 'unknown'/'Unknown' found; nothing to migrate"); + } + + Ok(updated) + } + + /// Migrates all buckets whose name starts with `aw-watcher-android-test` to use + /// `aw-watcher-android` instead. This covers the old production bucket naming + /// convention (e.g. `aw-watcher-android-test_phone` → `aw-watcher-android_phone`). + /// + /// If the destination already exists, disjoint legacy events are moved into it. + /// Events that overlap a destination or another legacy event stay in the legacy + /// bucket so the migration cannot create overlapping activity records. The legacy + /// bucket is deleted only after it is empty. + /// Returns the number of legacy buckets that were fully renamed or merged. + pub fn migrate_test_bucket_names( + &mut self, + conn: &Connection, + ) -> Result { + const OLD_PREFIX: &str = "aw-watcher-android-test"; + const NEW_PREFIX: &str = "aw-watcher-android"; + + info!("Migrating '{OLD_PREFIX}' bucket names to '{NEW_PREFIX}'"); + let legacy_ids: Vec = self + .buckets_cache + .keys() + .filter(|id| id.starts_with(OLD_PREFIX)) + .cloned() + .collect(); + let mut migrated = 0; + let mut cache_dirty = false; + + for old_id in legacy_ids { + let new_id = old_id.replacen(OLD_PREFIX, NEW_PREFIX, 1); + if let Some(new_bucket) = self.buckets_cache.get(&new_id).cloned() { + let old_bucket = self + .buckets_cache + .get(&old_id) + .cloned() + .ok_or_else(|| DatastoreError::NoSuchBucket(old_id.clone()))?; + + // Move only events that do not overlap the destination or another + // legacy event. A single overlapping cutover heartbeat must not strand + // years of disjoint history in the legacy bucket (ActivityWatch/aw-android#243). + // + // Overlaps are found with one sorted scan over both buckets instead of a + // correlated `NOT EXISTS` subquery per legacy event: that subquery had no + // lower bound on `starttime`, so it scanned every earlier event again for + // each row (O(n^2)). With a couple of years of Android history it kept the + // single datastore worker busy for hours, which blanked the web UI and + // produced ANRs in every main-thread datastore call (aw-android#261). + let movable = movable_legacy_event_ids(conn, old_bucket.bid, new_bucket.bid) + .map_err(|err| { + DatastoreError::InternalError(format!( + "Failed to find mergeable events in '{}': {err}", + old_id + )) + })?; + move_events_to_bucket(conn, new_bucket.bid, &movable).map_err(|err| { + DatastoreError::InternalError(format!( + "Failed to merge bucket '{}' into '{}': {err}", + old_id, new_id + )) + })?; + cache_dirty = true; + + let remaining: i64 = conn + .query_row( + "SELECT COUNT(*) FROM events WHERE bucketrow = ?1", + [old_bucket.bid], + |row| row.get(0), + ) + .map_err(|err| { + DatastoreError::InternalError(format!( + "Failed to count leftover events in '{}': {err}", + old_id + )) + })?; + if remaining == 0 { + conn.execute("DELETE FROM buckets WHERE id = ?1", [old_bucket.bid]) + .map_err(|err| { + DatastoreError::InternalError(format!( + "Failed to remove merged bucket '{}': {err}", + old_id + )) + })?; + info!("Merged legacy bucket '{}' into '{}'", old_id, new_id); + migrated += 1; + } else { + warn!( + "Partially merged '{}' into '{}'; {} overlapping event(s) remain in the legacy bucket", + old_id, new_id, remaining + ); + } + } else { + conn.execute( + "UPDATE buckets SET name = ?1 WHERE name = ?2", + [&new_id, &old_id], + ) + .map_err(|err| { + DatastoreError::InternalError(format!( + "Failed to rename bucket '{}' to '{}': {err}", + old_id, new_id + )) + })?; + info!("Renamed legacy bucket '{}' to '{}'", old_id, new_id); + migrated += 1; + cache_dirty = true; + } + } + + if cache_dirty { + self.get_stored_buckets(conn)?; + } + Ok(migrated) + } +} diff --git a/aw-datastore/src/export.rs b/aw-datastore/src/export.rs new file mode 100644 index 0000000..f106cea --- /dev/null +++ b/aw-datastore/src/export.rs @@ -0,0 +1,220 @@ +use std::{collections::HashMap, io::Write}; + +use aw_models::Bucket; +use rusqlite::Connection; +use serde::{ + ser::{Error, SerializeMap, SerializeSeq}, + Serialize, Serializer, +}; + +use crate::DatastoreError; + +struct EventRows<'a> { + conn: &'a Connection, + bucket: &'a Bucket, +} + +impl Serialize for EventRows<'_> { + fn serialize(&self, serializer: S) -> Result { + let mut stmt = self + .conn + .prepare_cached( + "SELECT id, starttime, endtime, data + FROM events INDEXED BY events_bucketrow_starttime_endtime_index + WHERE bucketrow = ?1 AND endtime >= 0 AND starttime <= ?2 + ORDER BY starttime DESC, endtime ASC, id ASC", + ) + .map_err(S::Error::custom)?; + let mut rows = stmt + .query(rusqlite::params![self.bucket.bid.unwrap(), i64::MAX]) + .map_err(S::Error::custom)?; + let mut seq = serializer.serialize_seq(None)?; + while let Some(row) = rows.next().map_err(S::Error::custom)? { + // Match get_events' default range, clipping and corrupt-row policy. + match crate::datastore::parse_event_row(row, Some((0, i64::MAX))) { + Ok(event) => seq.serialize_element(&event)?, + Err(err) => warn!("Corrupt event in bucket {}: {}", self.bucket.id, err), + } + } + seq.end() + } +} + +struct ExportBucket<'a> { + conn: &'a Connection, + bucket: &'a Bucket, +} + +impl Serialize for ExportBucket<'_> { + fn serialize(&self, serializer: S) -> Result { + // Reuse Bucket's field names and serialization rules, replacing only + // events. This allocation contains bucket metadata, never event rows. + let metadata = serde_json::to_value(self.bucket).map_err(S::Error::custom)?; + let metadata = metadata + .as_object() + .ok_or_else(|| S::Error::custom("invalid bucket metadata"))?; + let mut map = serializer.serialize_map(Some(metadata.len()))?; + for (key, value) in metadata { + if key != "events" { + map.serialize_entry(key, value)?; + } + } + map.serialize_entry( + "events", + &EventRows { + conn: self.conn, + bucket: self.bucket, + }, + )?; + map.end() + } +} + +struct ExportBuckets<'a> { + conn: &'a Connection, + buckets: &'a HashMap, + selected: Option<&'a str>, +} + +impl Serialize for ExportBuckets<'_> { + fn serialize(&self, serializer: S) -> Result { + let mut map = serializer.serialize_map(None)?; + for (id, bucket) in self.buckets { + if self.selected.is_none() || self.selected == Some(id.as_str()) { + map.serialize_entry( + id, + &ExportBucket { + conn: self.conn, + bucket, + }, + )?; + } + } + map.end() + } +} + +pub(crate) fn write_export( + conn: &Connection, + buckets: &HashMap, + selected: Option<&str>, + writer: impl Write, +) -> Result<(), DatastoreError> { + if let Some(id) = selected { + if !buckets.contains_key(id) { + return Err(DatastoreError::NoSuchBucket(id.to_owned())); + } + } + #[derive(Serialize)] + struct Export<'a> { + buckets: ExportBuckets<'a>, + } + serde_json::to_writer( + writer, + &Export { + buckets: ExportBuckets { + conn, + buckets, + selected, + }, + }, + ) + .map_err(|err| DatastoreError::InternalError(format!("Failed to write export: {err}"))) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::DatastoreInstance; + use aw_models::{BucketMetadata, BucketsExport, Event, TryVec}; + use chrono::{DateTime, Duration}; + + fn setup() -> (Connection, DatastoreInstance) { + let conn = Connection::open_in_memory().unwrap(); + let mut ds = DatastoreInstance::new(&conn, true).unwrap(); + for id in ["populated", "empty"] { + ds.create_bucket( + &conn, + Bucket { + bid: None, + id: id.into(), + _type: "test".into(), + client: "test".into(), + hostname: "host".into(), + created: None, + data: Default::default(), + metadata: BucketMetadata::default(), + events: None, + last_updated: None, + }, + ) + .unwrap(); + } + let events = [(-10, 20), (5, 20), (5, 10), (5, 10), (30, 0)] + .into_iter() + .map(|(start, duration)| { + Event::new( + DateTime::from_timestamp(start, 0).unwrap(), + Duration::seconds(duration), + serde_json::from_value( + serde_json::json!({"text": "quotes \" and unicode ☀", "nested": [1, true]}), + ) + .unwrap(), + ) + }) + .collect(); + ds.insert_events(&conn, "populated", events).unwrap(); + conn.execute("INSERT INTO events(bucketrow,starttime,endtime,data) VALUES(1,6000000000,7000000000,'invalid json')", []).unwrap(); + (conn, ds) + } + + #[test] + fn streamed_json_matches_materialized_exports() { + let (conn, mut ds) = setup(); + for selected in [None, Some("populated"), Some("empty")] { + let mut buckets = ds.get_buckets(); + buckets.retain(|id, _| selected.is_none() || selected == Some(id.as_str())); + for (id, bucket) in &mut buckets { + bucket.events = Some(TryVec::new( + ds.get_events(&conn, id, None, None, None).unwrap(), + )); + } + let expected = serde_json::to_value(BucketsExport { buckets }).unwrap(); + let mut output = Vec::new(); + ds.write_export(&conn, selected, &mut output).unwrap(); + assert_eq!( + serde_json::from_slice::(&output).unwrap(), + expected + ); + } + } + + #[test] + fn missing_bucket_and_writer_failures_propagate() { + let (conn, ds) = setup(); + let mut output = Vec::new(); + assert!(matches!( + ds.write_export(&conn, Some("missing"), &mut output), + Err(DatastoreError::NoSuchBucket(_)) + )); + assert!(output.is_empty()); + struct FailingWriter(usize); + impl Write for FailingWriter { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + if self.0 == 0 { + return Err(std::io::Error::new(std::io::ErrorKind::Other, "disk full")); + } + let n = bytes.len().min(self.0); + self.0 -= n; + Ok(n) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + assert!(matches!( + ds.write_export(&conn, None, FailingWriter(500)), + Err(DatastoreError::InternalError(_)) + )); + } +} diff --git a/aw-datastore/src/legacy_import.rs b/aw-datastore/src/legacy_import.rs new file mode 100644 index 0000000..a404348 --- /dev/null +++ b/aw-datastore/src/legacy_import.rs @@ -0,0 +1,230 @@ +use crate::datastore::DatastoreInstance; +use rusqlite::Connection; + +#[derive(Debug, Clone)] +pub enum LegacyDatastoreImportError { + #[allow(dead_code)] + SQLPrepareError(String), + #[allow(dead_code)] + SQLMapError(String), +} + +pub fn legacy_import( + new_ds: &mut DatastoreInstance, + new_conn: &Connection, +) -> Result<(), LegacyDatastoreImportError> { + import::legacy_import(new_ds, new_conn) +} + +#[cfg(not(target_os = "android"))] +mod import { + use std::path::PathBuf; + + use rusqlite::types::ToSql; + use rusqlite::Connection; + + use chrono::DateTime; + use chrono::Duration; + use chrono::Utc; + + use aw_models::Bucket; + use aw_models::BucketMetadata; + use aw_models::Event; + + use crate::datastore::DatastoreInstance; + + use super::LegacyDatastoreImportError; + + fn dbfile_path() -> PathBuf { + dirs::data_dir() + .expect("Unable to read user data dir") + .join("activitywatch") + .join("aw-server") + .join("peewee-sqlite.v2.db") + } + + fn get_legacy_buckets(conn: &Connection) -> Result, LegacyDatastoreImportError> { + let mut stmt = match conn + .prepare("SELECT key, id, type, client, hostname, created FROM bucketmodel") + { + Ok(stmt) => stmt, + Err(err) => return Err(LegacyDatastoreImportError::SQLPrepareError(err.to_string())), + }; + let bucket_rows = match stmt.query_map(&[] as &[&dyn ToSql], |row| { + Ok(Bucket { + bid: row.get(0)?, + id: row.get(1)?, + _type: row.get(2)?, + client: row.get(3)?, + hostname: row.get(4)?, + created: row.get(5)?, + data: json_map! {}, + events: None, + last_updated: None, + metadata: BucketMetadata { + start: None, + end: None, + }, + }) + }) { + Ok(buckets) => buckets, + Err(err) => { + return Err(LegacyDatastoreImportError::SQLMapError(format!( + "Failed to query get_legacy_buckets SQL statement: {err:?}" + ))) + } + }; + + let mut buckets = Vec::new(); + for bucket_res in bucket_rows { + match bucket_res { + Ok(bucket) => buckets.push(bucket), + Err(err) => panic!("{err:?}"), + } + } + Ok(buckets) + } + + fn get_legacy_events( + conn: &Connection, + bucket_id: i64, + ) -> Result, LegacyDatastoreImportError> { + let mut stmt = match conn.prepare( + " + SELECT timestamp, duration, datastr + FROM eventmodel + WHERE bucket_id = ?1 + ORDER BY timestamp DESC + ;", + ) { + Ok(stmt) => stmt, + Err(err) => return Err(LegacyDatastoreImportError::SQLPrepareError(err.to_string())), + }; + + let rows = match stmt.query_map([&bucket_id], |row| { + let timestamp_str: String = row.get(0)?; + let duration_float: f64 = row.get(1)?; + let data_str: String = row.get(2)?; + Ok((timestamp_str, duration_float, data_str)) + }) { + Ok(rows) => rows, + Err(err) => { + return Err(LegacyDatastoreImportError::SQLMapError(format!( + "Failed to query get_legacy_events SQL statement: {err:?}" + ))) + } + }; + let mut list = Vec::new(); + for row in rows { + match row { + Ok((timestamp_str, duration_float, data_str)) => { + let timestamp_str = timestamp_str.replace(' ', "T"); + let timestamp = match DateTime::parse_from_rfc3339(×tamp_str) { + Ok(timestamp) => timestamp.with_timezone(&Utc), + Err(err) => panic!("Timestamp string {timestamp_str}: {err:?}"), + }; + + let duration_ns = (duration_float * 1_000_000_000.0) as i64; + + let data: serde_json::map::Map = + match serde_json::from_str(&data_str) { + Ok(data) => data, + Err(err) => { + warn!( + "Unable to parse JSON data in event from bucket {}\n{}\n{}", + bucket_id, err, data_str + ); + continue; + } + }; + + let event = Event { + id: None, + timestamp, + duration: Duration::nanoseconds(duration_ns), + data, + }; + list.push(event) + } + Err(err) => panic!("Corrupt event in bucket {bucket_id}: {err}"), + }; + } + Ok(list) + } + + pub fn legacy_import( + new_ds: &mut DatastoreInstance, + new_conn: &Connection, + ) -> Result<(), LegacyDatastoreImportError> { + let legacy_db_path = dbfile_path(); + if !legacy_db_path.exists() { + info!("Did not find an old database, skipping legacy import"); + return Ok(()); + } + info!("Importing legacy DB"); + let legacy_conn = + Connection::open(legacy_db_path).expect("Unable to open corrupt legacy db file"); + + let buckets = get_legacy_buckets(&legacy_conn)?; + for bucket in &buckets { + println!("Importing legacy bucket: {bucket:?}"); + match new_ds.create_bucket(new_conn, bucket.clone()) { + Ok(_) => (), + Err(err) => panic!("Failed to create bucket '{}': {:?}", bucket.id, err), + }; + let events = get_legacy_events(&legacy_conn, bucket.bid.unwrap())?; + let num_events = events.len(); // Save len before lending events to insert_events + println!("Importing {} events for {}", num_events, bucket.id); + match new_ds.insert_events(new_conn, &bucket.id, events) { + Ok(_) => (), + Err(err) => panic!( + "Failed to insert events to bucket '{}': {:?}", + bucket.id, err + ), + }; + //assert_eq!(new_ds.get_events(new_conn, &bucket.id, None, None, None).unwrap().len(), num_events); + } + assert_eq!(new_ds.get_buckets().len(), buckets.len()); + Ok(()) + } + + /* This test is disabled because it requires manual set-up of a old aw-server database + * Can be run with: + * cargo test --features legacy_import,legacy_import_tests */ + #[test] + #[cfg_attr(not(feature = "legacy_import_tests"), ignore)] + fn test_legacy_import() { + assert!(dbfile_path().exists()); + let mut new_conn = + Connection::open_in_memory().expect("Unable to open corrupt legacy db file"); + let mut ds = DatastoreInstance::new(&mut new_conn, true).unwrap(); + assert!( + ds.ensure_legacy_import(&new_conn).unwrap(), + "Failed to ensure legacy import" + ); + let buckets = ds.get_buckets(); + assert!(!buckets.is_empty()); + let mut num_events = 0; + for (bucket_id, _bucket) in buckets { + let events = ds + .get_events(&new_conn, &bucket_id, None, None, Some(1000)) + .unwrap(); + num_events += events.len(); + } + assert!(num_events > 0); + } +} + +#[cfg(target_os = "android")] +mod import { + use super::LegacyDatastoreImportError; + use crate::datastore::DatastoreInstance; + use rusqlite::Connection; + + pub fn legacy_import( + _new_ds: &mut DatastoreInstance, + _new_conn: &Connection, + ) -> Result<(), LegacyDatastoreImportError> { + Ok(()) + } +} diff --git a/aw-datastore/src/lib.rs b/aw-datastore/src/lib.rs new file mode 100644 index 0000000..a93bf89 --- /dev/null +++ b/aw-datastore/src/lib.rs @@ -0,0 +1,61 @@ +#[macro_use] +extern crate log; + +use std::fmt; + +#[macro_export] +macro_rules! json_map { + { $( $key:literal : $value:expr),* } => {{ + use serde_json::Value; + use serde_json::map::Map; + #[allow(unused_mut)] + let mut map : Map = Map::new(); + $( + map.insert( $key.to_string(), json!($value) ); + )* + map + }}; +} + +mod datastore; +mod export; +mod legacy_import; +mod privacy_filter; +mod worker; + +pub use self::datastore::DatastoreInstance; +pub use self::worker::Datastore; + +#[derive(Clone)] +pub enum DatastoreMethod { + Memory(), + File(String), + /// Encrypted SQLite file using SQLCipher. Only available with the + /// `encryption` or `encryption-vendored` feature flags. + #[cfg(any(feature = "encryption", feature = "encryption-vendored"))] + FileEncrypted(String, zeroize::Zeroizing), // (path, key) +} + +impl fmt::Debug for DatastoreMethod { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + DatastoreMethod::Memory() => write!(f, "Memory()"), + DatastoreMethod::File(p) => write!(f, "File({p:?})"), + #[cfg(any(feature = "encryption", feature = "encryption-vendored"))] + DatastoreMethod::FileEncrypted(p, _) => write!(f, "FileEncrypted({p:?}, )"), + } + } +} + +/* TODO: Implement this as a proper error */ +#[derive(Debug, Clone)] +pub enum DatastoreError { + NoSuchBucket(String), + BucketAlreadyExists(String), + NoSuchKey(String), + MpscError, + InternalError(String), + // Errors specific to when migrate is disabled + Uninitialized(String), + OldDbVersion(String), +} diff --git a/aw-datastore/src/privacy_filter.rs b/aw-datastore/src/privacy_filter.rs new file mode 100644 index 0000000..634e869 --- /dev/null +++ b/aw-datastore/src/privacy_filter.rs @@ -0,0 +1,422 @@ +use std::sync::OnceLock; + +use serde::{Deserialize, Serialize}; +use serde_json::map::Map; +use serde_json::Value; + +use aw_models::Event; + +/// Action to take when a rule matches an event. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(rename_all = "snake_case")] +pub enum PrivacyFilterAction { + /// Drop the entire event from storage + Drop, + /// Redact a specific field's value with a replacement + Redact, +} + +/// A single privacy filter rule. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct PrivacyFilterRule { + /// If true, this rule is active + pub enabled: bool, + /// Only apply to buckets whose ID starts with this prefix (e.g. "aw-watcher-window") + pub bucket_prefix: Option, + /// Dotted path to the event data field to check (e.g. "title") + pub field: Option, + /// Regex pattern to match against the field value + pub pattern: String, + /// What to do when matched + pub action: PrivacyFilterAction, + /// Replacement text for the redact action + pub replacement: Option, + /// Pre-compiled regex, populated lazily on first match. Not serialized. + #[serde(skip)] + regex_cache: OnceLock>, +} + +impl PartialEq for PrivacyFilterRule { + fn eq(&self, other: &Self) -> bool { + self.enabled == other.enabled + && self.bucket_prefix == other.bucket_prefix + && self.field == other.field + && self.pattern == other.pattern + && self.action == other.action + && self.replacement == other.replacement + } +} + +impl PrivacyFilterRule { + /// Check if this rule matches a given event in a given bucket. + pub fn matches(&self, bucket_id: &str, event: &Event) -> bool { + if !self.enabled { + return false; + } + + // Check bucket prefix if specified + if let Some(ref prefix) = self.bucket_prefix { + if !bucket_id.starts_with(prefix) { + return false; + } + } + + // Check field pattern if specified + if let Some(ref field_path) = self.field { + let field_value = resolve_field(&event.data, field_path); + match field_value { + Some(Value::String(s)) => { + // Compile the regex once and cache it for subsequent calls. + let re = self + .regex_cache + .get_or_init(|| regex::Regex::new(&self.pattern).ok()); + re.as_ref().map(|re| re.is_match(s)).unwrap_or(false) + } + Some(_) | None => false, + } + } else { + true + } + } + + /// Apply this rule's action to an event. + /// Returns None if dropped, Some(event) if kept (possibly redacted). + pub fn apply<'a>(&self, event: &'a mut Event) -> Option<&'a mut Event> { + match self.action { + PrivacyFilterAction::Drop => None, + PrivacyFilterAction::Redact => { + if let Some(ref replacement) = self.replacement { + if let Some(ref field_path) = self.field { + set_field( + &mut event.data, + field_path, + Value::String(replacement.clone()), + ); + } + } + Some(event) + } + } + } +} + +/// Engine that holds and applies privacy filter rules. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct PrivacyFilterEngine { + rules: Vec, +} + +impl PrivacyFilterEngine { + pub fn new(rules: Vec) -> Self { + PrivacyFilterEngine { rules } + } + + /// Example rules for common sensitive data patterns. + /// Not applied automatically — use `new()` with these rules to opt in. + #[allow(dead_code)] + pub fn with_defaults() -> Self { + let rules = vec![ + PrivacyFilterRule { + enabled: true, + bucket_prefix: Some("aw-watcher-window".to_string()), + field: Some("title".to_string()), + pattern: r"(?i)(private browsing|incognito)".to_string(), + action: PrivacyFilterAction::Drop, + replacement: None, + regex_cache: OnceLock::new(), + }, + PrivacyFilterRule { + enabled: true, + bucket_prefix: Some("aw-watcher-window".to_string()), + field: Some("title".to_string()), + pattern: r"(?i).*banking.*".to_string(), + action: PrivacyFilterAction::Redact, + replacement: Some("REDACTED".to_string()), + regex_cache: OnceLock::new(), + }, + ]; + PrivacyFilterEngine { rules } + } + + /// Parse rules from a JSON string (as stored in settings). + pub fn from_json(json_str: &str) -> Result { + let rules: Vec = serde_json::from_str(json_str) + .map_err(|e| format!("Failed to parse privacy filter rules: {e}"))?; + for rule in &rules { + if rule.action == PrivacyFilterAction::Redact + && rule.replacement.as_deref().is_none_or(str::is_empty) + { + return Err(format!( + "Redact rule with pattern {:?} is missing `replacement` — add a non-empty replacement string or use action=drop", + rule.pattern + )); + } + if rule.action == PrivacyFilterAction::Redact && rule.field.is_none() { + return Err(format!( + "Redact rule with pattern {:?} is missing `field` — specify which data field to redact (e.g. \"title\")", + rule.pattern + )); + } + if rule.action == PrivacyFilterAction::Drop && rule.field.is_none() { + return Err(format!( + "Drop rule with pattern {:?} is missing `field` — without a field path the pattern is never evaluated and every event in the matching bucket is dropped (specify a dotted field path, e.g. \"title\")", + rule.pattern + )); + } + if let Err(e) = regex::Regex::new(&rule.pattern) { + return Err(format!( + "Rule with pattern {:?} has an invalid regex: {e}", + rule.pattern + )); + } + } + Ok(PrivacyFilterEngine { rules }) + } + + /// Serialize rules to JSON string. + #[allow(dead_code)] + pub fn to_json(&self) -> Result { + serde_json::to_string(&self.rules) + .map_err(|e| format!("Failed to serialize privacy filter rules: {e}")) + } + + /// Filter a single event for a given bucket. + /// Applies all matching rules. Returns None if dropped, Some (possibly redacted) event if kept. + pub fn filter_event(&self, bucket_id: &str, event: Event) -> Option { + let mut event = event; + for rule in &self.rules { + if rule.matches(bucket_id, &event) { + match rule.apply(&mut event) { + Some(_) => {} // Redacted — continue applying other rules + None => return None, // Dropped + } + } + } + Some(event) + } + + /// Filter a batch of events for a given bucket. + pub fn filter_events(&self, bucket_id: &str, events: Vec) -> Vec { + events + .into_iter() + .filter_map(|e| self.filter_event(bucket_id, e)) + .collect() + } +} + +/// Resolve a dotted field path (e.g. "title", "data.url") from a serde_json Map. +fn resolve_field<'a>(data: &'a Map, path: &str) -> Option<&'a Value> { + let parts: Vec<&str> = path.split('.').collect(); + let mut current: &Map = data; + for (i, part) in parts.iter().enumerate() { + let val = current.get(*part)?; + if i == parts.len() - 1 { + return Some(val); + } + match val { + Value::Object(map) => current = map, + _ => return None, + } + } + None +} + +/// Set a field value at a dotted path in a serde_json Map. +fn set_field(data: &mut Map, path: &str, value: Value) { + let parts: Vec<&str> = path.split('.').collect(); + let mut current: &mut Map = data; + for (i, part) in parts.iter().enumerate() { + if i == parts.len() - 1 { + current.insert(part.to_string(), value); + return; + } + current = match current + .entry(part.to_string()) + .or_insert_with(|| Value::Object(Map::new())) + .as_object_mut() + { + Some(m) => m, + None => return, // intermediate segment is not an object — skip silently + }; + } +} + +#[cfg(test)] +mod tests { + use super::*; + use chrono::Utc; + use serde_json::json; + + fn test_event(title: &str) -> Event { + Event { + id: None, + timestamp: Utc::now(), + duration: chrono::Duration::seconds(1), + data: json_map! {"title": json!(title), "app": json!("Firefox")}, + } + } + + #[test] + fn test_drop_incognito_window() { + let engine = PrivacyFilterEngine::with_defaults(); + let event = test_event("Private Browsing - Mozilla Firefox"); + let rule = &engine.rules[0]; + assert!(rule.matches("aw-watcher-window", &event)); + } + + #[test] + fn test_allow_normal_window() { + let engine = PrivacyFilterEngine::with_defaults(); + let event = test_event("GitHub - Mozilla Firefox"); + let rule = &engine.rules[0]; + assert!(!rule.matches("aw-watcher-window", &event)); + } + + #[test] + fn test_redact_banking() { + let engine = PrivacyFilterEngine::with_defaults(); + let mut event = test_event("Online Banking - My Account Balance"); + let rule = &engine.rules[1]; + assert!(rule.matches("aw-watcher-window", &event)); + let result = rule.apply(&mut event); + assert!(result.is_some()); + assert_eq!( + result.unwrap().data.get("title").unwrap().as_str().unwrap(), + "REDACTED" + ); + } + + #[test] + fn test_bucket_scoping() { + let rule = PrivacyFilterRule { + enabled: true, + bucket_prefix: Some("aw-watcher-window".to_string()), + field: Some("title".to_string()), + pattern: r"(?i)(private browsing|incognito)".to_string(), + action: PrivacyFilterAction::Drop, + replacement: None, + regex_cache: OnceLock::new(), + }; + let event = test_event("Private Browsing - Mozilla Firefox"); + assert!(rule.matches("aw-watcher-window", &event)); + assert!(!rule.matches("aw-watcher-afk", &event)); + } + + #[test] + fn test_disabled_rule() { + let rule = PrivacyFilterRule { + enabled: false, + bucket_prefix: Some("aw-watcher-window".to_string()), + field: Some("title".to_string()), + pattern: ".*".to_string(), + action: PrivacyFilterAction::Drop, + replacement: None, + regex_cache: OnceLock::new(), + }; + let event = test_event("Anything at all"); + assert!(!rule.matches("aw-watcher-window", &event)); + } + + #[test] + fn test_invalid_regex_no_panic() { + let rule = PrivacyFilterRule { + enabled: true, + bucket_prefix: Some("aw-watcher-window".to_string()), + field: Some("title".to_string()), + pattern: r"[invalid".to_string(), + action: PrivacyFilterAction::Drop, + replacement: None, + regex_cache: OnceLock::new(), + }; + let event = test_event("test"); + assert!(!rule.matches("aw-watcher-window", &event)); + } + + #[test] + fn test_set_field_no_panic_on_non_object_intermediate() { + let mut data = serde_json::Map::new(); + data.insert( + "title".to_string(), + Value::String("flat string".to_string()), + ); + // "title" is a string, not an object — setting "title.nested" should not panic + set_field( + &mut data, + "title.nested", + Value::String("value".to_string()), + ); + // title should remain unchanged + assert_eq!(data.get("title").unwrap().as_str().unwrap(), "flat string"); + } + + #[test] + fn test_from_json_redact_without_replacement_is_error() { + let json = r#"[{"enabled":true,"pattern":"(?i)secret","action":"redact","field":"title"}]"#; + let result = PrivacyFilterEngine::from_json(json); + assert!( + result.is_err(), + "Redact rule without replacement must fail from_json" + ); + assert!(result.unwrap_err().contains("replacement")); + } + + #[test] + fn test_from_json_redact_with_empty_replacement_is_error() { + let json = r#"[{"enabled":true,"pattern":"(?i)secret","action":"redact","field":"title","replacement":""}]"#; + let result = PrivacyFilterEngine::from_json(json); + assert!( + result.is_err(), + "Redact rule with empty replacement must fail from_json" + ); + assert!(result.unwrap_err().contains("replacement")); + } + + #[test] + fn test_from_json_redact_without_field_is_error() { + let json = r#"[{"enabled":true,"pattern":"(?i)secret","action":"redact","replacement":"REDACTED"}]"#; + let result = PrivacyFilterEngine::from_json(json); + assert!( + result.is_err(), + "Redact rule without field must fail from_json" + ); + assert!(result.unwrap_err().contains("field")); + } + + #[test] + fn test_from_json_invalid_regex_is_error() { + let json = r#"[{"enabled":true,"pattern":"[unclosed","action":"drop","field":"title"}]"#; + let result = PrivacyFilterEngine::from_json(json); + assert!( + result.is_err(), + "Rule with invalid regex must fail from_json" + ); + } + + #[test] + fn test_from_json_drop_without_field_is_error() { + let json = r#"[{"enabled":true,"pattern":"(?i)incognito","action":"drop"}]"#; + let result = PrivacyFilterEngine::from_json(json); + assert!( + result.is_err(), + "Drop rule without field must fail from_json" + ); + assert!(result.unwrap_err().contains("field")); + } + + #[test] + fn test_drop_action() { + let rule = PrivacyFilterRule { + enabled: true, + bucket_prefix: None, + field: Some("title".to_string()), + pattern: ".*".to_string(), + action: PrivacyFilterAction::Drop, + replacement: None, + regex_cache: OnceLock::new(), + }; + let mut event = test_event("anything"); + assert!(rule.matches("any-bucket", &event)); + let result = rule.apply(&mut event); + assert!(result.is_none(), "Drop action should return None"); + } +} diff --git a/aw-datastore/src/worker.rs b/aw-datastore/src/worker.rs new file mode 100644 index 0000000..0a3a4b1 --- /dev/null +++ b/aw-datastore/src/worker.rs @@ -0,0 +1,768 @@ +use std::collections::HashMap; +use std::fmt; +use std::thread; +use std::{ + fs::File, + io::{BufWriter, Write}, +}; + +use chrono::DateTime; +use chrono::Duration; +use chrono::Utc; + +use rusqlite::Connection; +use rusqlite::DropBehavior; +use rusqlite::Transaction; +use rusqlite::TransactionBehavior; + +use aw_models::Bucket; +use aw_models::Event; + +use crate::privacy_filter::PrivacyFilterEngine; +use crate::DatastoreError; +use crate::DatastoreInstance; +use crate::DatastoreMethod; + +type RequestSender = mpsc_requests::RequestSender>; +type RequestReceiver = mpsc_requests::RequestReceiver>; + +#[derive(Clone)] +pub struct Datastore { + requester: RequestSender, +} + +impl fmt::Debug for Datastore { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "Datastore()") + } +} + +/* + * TODO: + * - Allow read requests to go straight through a read-only db connection instead of requesting the + * worker thread for better performance? + * TODO: Add an separate "Import" request which does an import with an transaction + */ + +#[allow(clippy::large_enum_variant)] +#[derive(Debug)] +pub enum Response { + Export(File, Option), + Empty(), + Bucket(Bucket), + BucketMap(HashMap), + Event(Event), + EventList(Vec), + Count(i64), + KeyValue(String), + KeyValues(HashMap), +} + +#[allow(clippy::large_enum_variant)] +#[derive(Debug)] +pub enum Command { + Export(Option, File), + CreateBucket(Bucket), + DeleteBucket(String), + GetBucket(String), + GetBuckets(), + InsertEvents(String, Vec), + Heartbeat(String, Event, f64), + GetEvent(String, i64), + GetEvents( + String, + Option>, + Option>, + Option, + bool, + ), + GetEventCount(String, Option>, Option>), + DeleteEventsById(String, Vec), + ForceCommit(), + GetKeyValues(String), + GetKeyValue(String), + SetKeyValue(String, String), + DeleteKeyValue(String), + RefreshPrivacyFilter(), + RenameBucket(String, String), + MigrateHostname(String), + MigrateTestBucketNames(), + Close(), +} + +/// Key the webui writes via POST /0/settings/privacy_filters. +/// The worker's in-memory PrivacyFilterEngine is the only thing that actually +/// filters inserts/heartbeats, so every write/delete of this key (and startup) +/// must reload the engine. RefreshPrivacyFilter exists for explicit reloads. +const PRIVACY_FILTERS_KEY: &str = "settings.privacy_filters"; + +fn _unwrap_empty_response(response: Response) -> Result<(), DatastoreError> { + match response { + Response::Empty() => Ok(()), + _ => panic!("Invalid response"), + } +} + +struct DatastoreWorker { + responder: RequestReceiver, + legacy_import: bool, + quit: bool, + uncommitted_events: usize, + commit: bool, + last_heartbeat: HashMap>, + privacy_engine: PrivacyFilterEngine, +} + +impl DatastoreWorker { + pub fn new( + responder: mpsc_requests::RequestReceiver>, + legacy_import: bool, + ) -> Self { + DatastoreWorker { + responder, + legacy_import, + quit: false, + uncommitted_events: 0, + commit: false, + last_heartbeat: HashMap::new(), + privacy_engine: PrivacyFilterEngine::new(vec![]), + } + } + + /// Replace the in-memory engine from `settings.privacy_filters`. + /// Only an absent key clears the engine, so deleting the setting actually + /// disables filtering. Parse errors and query errors keep the previous + /// engine: unfiltering on a bad save or a transient database error would + /// silently store the events these rules exist to keep out. + fn reload_privacy_engine(&mut self, ds: &DatastoreInstance, conn: &Connection) { + match ds.get_key_value(conn, PRIVACY_FILTERS_KEY) { + Ok(json_str) => match PrivacyFilterEngine::from_json(&json_str) { + Ok(engine) => self.privacy_engine = engine, + Err(e) => warn!("Failed to parse privacy_filters setting: {e}"), + }, + Err(DatastoreError::NoSuchKey(_)) => { + self.privacy_engine = PrivacyFilterEngine::new(vec![]); + } + Err(e) => warn!("Failed to load privacy_filters setting: {e:?}"), + } + } + + fn work_loop(&mut self, method: DatastoreMethod) { + // Open SQLite connection + let mut conn = match &method { + DatastoreMethod::Memory() => { + Connection::open_in_memory().expect("Failed to create in-memory datastore") + } + DatastoreMethod::File(path) => { + Connection::open(path).expect("Failed to create datastore") + } + #[cfg(any(feature = "encryption", feature = "encryption-vendored"))] + DatastoreMethod::FileEncrypted(path, key) => { + let conn = Connection::open(path).expect("Failed to create encrypted datastore"); + conn.pragma_update(None, "key", key.as_str()) + .expect("Failed to set SQLCipher encryption key"); + // PRAGMA key always succeeds even with a wrong passphrase; the + // first real SQL query is what fails. Read user_version immediately + // to surface an incorrect key as a clear error rather than an + // opaque panic later. + conn.pragma_query_value(None, "user_version", |row| row.get::<_, i64>(0)) + .expect("Failed to open encrypted database: wrong passphrase or not an encrypted database"); + info!("Opened encrypted database at {}", path); + conn + } + }; + + // Set busy timeout to handle concurrent access on systems with strict file locking (e.g., Windows) + conn.busy_timeout(std::time::Duration::from_secs(5)) + .expect("Failed to set busy timeout"); + + // WAL turns each commit into a single sequential WAL append+fsync where + // delete mode paid two fsyncs plus journal-file churn, and lets future + // reader connections proceed while a commit is in flight. + // synchronous=FULL is set explicitly (rather than relying on the + // default) so a commit remains durable on disk the moment it returns; + // with NORMAL the WAL is only synced at checkpoints, which would + // silently widen the loss window on power failure. + // In-memory databases ignore the request (journal_mode stays "memory"). + let journal_mode: String = conn + .pragma_update_and_check(None, "journal_mode", "WAL", |row| row.get(0)) + .expect("Failed to query journal_mode"); + if !matches!(&method, DatastoreMethod::Memory()) && journal_mode != "wal" { + warn!("Failed to enable WAL (journal_mode={journal_mode}), continuing without it"); + } + conn.pragma_update(None, "synchronous", "FULL") + .expect("Failed to set synchronous=FULL"); + + let mut ds = DatastoreInstance::new(&conn, true).unwrap(); + + // Load persisted privacy filters before serving inserts. The engine + // starts empty; without this, rules saved in a previous process sit + // unused until something happens to send RefreshPrivacyFilter. + self.reload_privacy_engine(&ds, &conn); + + // Ensure legacy import + if self.legacy_import { + let transaction = match conn.transaction_with_behavior(TransactionBehavior::Immediate) { + Ok(transaction) => transaction, + Err(err) => { + panic!("Unable to start immediate transaction on SQLite database! {err}") + } + }; + match ds.ensure_legacy_import(&transaction) { + Ok(_) => (), + Err(err) => error!("Failed to do legacy import: {:?}", err), + } + match transaction.commit() { + Ok(_) => (), + Err(err) => { + error!("Failed to commit legacy import transaction: {err}"); + // Continue without panicking — legacy import will be retried on + // next startup if the commit didn't persist. + } + } + } + + // Start handling and respond to requests + loop { + let last_commit_time: DateTime = Utc::now(); + let mut tx: Transaction = + match conn.transaction_with_behavior(TransactionBehavior::Immediate) { + Ok(tx) => tx, + Err(err) => { + error!("Unable to start transaction! {:?}", err); + // Wait 1s before retrying + std::thread::sleep(std::time::Duration::from_millis(1000)); + continue; + } + }; + // Snapshot BEFORE the request loop. SetKeyValue/DeleteKeyValue + // reload the engine from the still-open transaction so a later + // insert in the same batch is filtered. If commit fails we restore + // this snapshot — not "keep current" (that is the rolled-back + // view) and not a durable re-query (a read error would leave the + // uncommitted engine in place: fail-open after a rolled-back delete). + let privacy_engine_at_tx_start = self.privacy_engine.clone(); + tx.set_drop_behavior(DropBehavior::Commit); + + self.uncommitted_events = 0; + self.commit = false; + // ForceCommit and Close promise the caller that their data is + // committed, so their acks are held back until the transaction + // below has actually committed. Acking first (as before) let a + // caller reopen the database and read a pre-commit snapshot — + // harmless under the rollback journal's locking, but a real race + // in WAL mode where readers never block on the writer. + // All other commands are acked immediately: a watcher heartbeat + // must not wait up to 15 s for the batch commit. + let mut deferred_ack = None; + loop { + let (request, response_sender) = match self.responder.poll() { + Ok((req, res_sender)) => (req, res_sender), + Err(err) => { + // All references to responder is gone, quit + error!("DB worker quitting, error: {err:?}"); + self.quit = true; + break; + } + }; + let ack_after_commit = matches!(request, Command::ForceCommit() | Command::Close()); + let response = self.handle_request(request, &mut ds, &tx); + if ack_after_commit { + // Both commands force a commit, so the loop ends here. + deferred_ack = Some((response_sender, response)); + break; + } + response_sender.respond(response); + + let now: DateTime = Utc::now(); + let commit_interval_passed: bool = (now - last_commit_time) > Duration::seconds(15); + if self.commit + || commit_interval_passed + || self.uncommitted_events > 100 + || self.quit + { + break; + }; + } + debug!( + "Committing DB! Force commit {}, {} uncommitted events", + self.commit, self.uncommitted_events + ); + match tx.commit() { + Ok(_) => { + if let Some((sender, response)) = deferred_ack.take() { + sender.respond(response); + } + } + Err(err) => { + error!( + "Failed to commit datastore transaction ({} events lost): {err}", + self.uncommitted_events + ); + // Continue instead of panicking — the worker thread survives this + // transient failure (e.g. SQLITE_FULL on disk full). Note: clients + // already received success responses before the commit, so they won't + // know to retry. Rolled-back events create a gap in the timeline; + // watchers will resume sending heartbeats from current state, but the + // specific batch of events is permanently lost. + // + // Restore the pre-transaction engine. Reloading from the durable + // connection is not enough: if that read fails, keep-on-error would + // preserve the uncommitted engine (empty after a rolled-back delete). + self.privacy_engine = privacy_engine_at_tx_start; + if let Some((sender, _)) = deferred_ack.take() { + sender.respond(Err(DatastoreError::InternalError(format!( + "Failed to commit datastore transaction: {err}" + )))); + } + } + } + if self.quit { + break; + }; + } + info!("DB Worker thread finished"); + } + + fn handle_request( + &mut self, + request: Command, + ds: &mut DatastoreInstance, + tx: &Transaction, + ) -> Result { + match request { + Command::Export(bucket_id, mut file) => { + let mut writer = BufWriter::new(&mut file); + let name = ds.write_export(tx, bucket_id.as_deref(), &mut writer)?; + writer.flush().map_err(|err| { + DatastoreError::InternalError(format!("Failed to flush export: {err}")) + })?; + drop(writer); + Ok(Response::Export(file, name)) + } + Command::CreateBucket(bucket) => match ds.create_bucket(tx, bucket) { + Ok(_) => { + self.commit = true; + Ok(Response::Empty()) + } + Err(e) => Err(e), + }, + Command::DeleteBucket(bucketname) => match ds.delete_bucket(tx, &bucketname) { + Ok(_) => { + self.commit = true; + Ok(Response::Empty()) + } + Err(e) => Err(e), + }, + Command::GetBucket(bucketname) => match ds.get_bucket(&bucketname) { + Ok(b) => Ok(Response::Bucket(b)), + Err(e) => Err(e), + }, + Command::GetBuckets() => Ok(Response::BucketMap(ds.get_buckets())), + Command::InsertEvents(bucketname, events) => { + let filtered = self.privacy_engine.filter_events(&bucketname, events); + if filtered.is_empty() { + return Ok(Response::EventList(vec![])); + } + match ds.insert_events(tx, &bucketname, filtered) { + Ok(events) => { + self.uncommitted_events += events.len(); + self.last_heartbeat.insert(bucketname.to_string(), None); // invalidate last_heartbeat cache + Ok(Response::EventList(events)) + } + Err(e) => Err(e), + } + } + Command::Heartbeat(bucketname, event, pulsetime) => { + // Apply privacy filter to heartbeat + let filtered = match self.privacy_engine.filter_event(&bucketname, event.clone()) { + Some(event) => event, + None => { + // Heartbeat dropped by filter — return last cached event so the + // watcher's heartbeat-merge state machine continues correctly. + // Fall back to the incoming event itself if no prior event is cached + // (avoids returning a zero-timestamp default Event). + let last = self + .last_heartbeat + .get(&bucketname) + .and_then(|e| e.clone()) + .unwrap_or(event); + return Ok(Response::Event(last)); + } + }; + match ds.heartbeat( + tx, + &bucketname, + filtered, + pulsetime, + &mut self.last_heartbeat, + ) { + Ok(e) => { + self.uncommitted_events += 1; + Ok(Response::Event(e)) + } + Err(e) => Err(e), + } + } + Command::GetEvent(bucketname, event_id) => { + match ds.get_event(tx, &bucketname, event_id) { + Ok(el) => Ok(Response::Event(el)), + Err(e) => Err(e), + } + } + Command::GetEvents(bucketname, starttime_opt, endtime_opt, limit_opt, unclipped) => { + let result = if unclipped { + ds.get_events_unclipped(tx, &bucketname, starttime_opt, endtime_opt, limit_opt) + } else { + ds.get_events(tx, &bucketname, starttime_opt, endtime_opt, limit_opt) + }; + match result { + Ok(el) => Ok(Response::EventList(el)), + Err(e) => Err(e), + } + } + Command::GetEventCount(bucketname, starttime_opt, endtime_opt) => { + match ds.get_event_count(tx, &bucketname, starttime_opt, endtime_opt) { + Ok(n) => Ok(Response::Count(n)), + Err(e) => Err(e), + } + } + Command::DeleteEventsById(bucketname, event_ids) => { + match ds.delete_events_by_id(tx, &bucketname, event_ids) { + Ok(()) => Ok(Response::Empty()), + Err(e) => Err(e), + } + } + Command::ForceCommit() => { + self.commit = true; + Ok(Response::Empty()) + } + Command::GetKeyValues(pattern) => match ds.get_key_values(tx, pattern.as_str()) { + Ok(result) => Ok(Response::KeyValues(result)), + Err(e) => Err(e), + }, + Command::SetKeyValue(key, data) => match ds.insert_key_value(tx, &key, &data) { + Ok(()) => { + if key == PRIVACY_FILTERS_KEY { + self.reload_privacy_engine(ds, tx); + } + Ok(Response::Empty()) + } + Err(e) => Err(e), + }, + Command::GetKeyValue(key) => match ds.get_key_value(tx, &key) { + Ok(result) => Ok(Response::KeyValue(result)), + Err(e) => Err(e), + }, + Command::DeleteKeyValue(key) => match ds.delete_key_value(tx, &key) { + Ok(()) => { + if key == PRIVACY_FILTERS_KEY { + self.reload_privacy_engine(ds, tx); + } + Ok(Response::Empty()) + } + Err(e) => Err(e), + }, + Command::RefreshPrivacyFilter() => { + self.reload_privacy_engine(ds, tx); + Ok(Response::Empty()) + } + Command::RenameBucket(old_id, new_id) => match ds.rename_bucket(tx, &old_id, &new_id) { + Ok(()) => { + self.commit = true; + Ok(Response::Empty()) + } + Err(e) => Err(e), + }, + Command::MigrateHostname(new_hostname) => { + match ds.migrate_hostname(tx, &new_hostname) { + Ok(count) => { + if count > 0 { + self.commit = true; + } + Ok(Response::Count(count as i64)) + } + Err(e) => Err(e), + } + } + Command::MigrateTestBucketNames() => match ds.migrate_test_bucket_names(tx) { + Ok(count) => { + if count > 0 { + self.commit = true; + } + Ok(Response::Count(count as i64)) + } + Err(e) => Err(e), + }, + Command::Close() => { + self.quit = true; + Ok(Response::Empty()) + } + } + } +} + +impl Datastore { + pub fn new(dbpath: String, legacy_import: bool) -> Self { + let method = DatastoreMethod::File(dbpath); + Datastore::_new_internal(method, legacy_import) + } + + pub fn new_in_memory(legacy_import: bool) -> Self { + let method = DatastoreMethod::Memory(); + Datastore::_new_internal(method, legacy_import) + } + + /// Create an encrypted datastore using SQLCipher. + /// + /// Requires the `encryption` or `encryption-vendored` feature flag. + /// Build with: `cargo build --no-default-features --features encryption` + #[cfg(any(feature = "encryption", feature = "encryption-vendored"))] + pub fn new_encrypted(dbpath: String, key: String, legacy_import: bool) -> Self { + let method = DatastoreMethod::FileEncrypted(dbpath, zeroize::Zeroizing::new(key)); + Datastore::_new_internal(method, legacy_import) + } + + fn _new_internal(method: DatastoreMethod, legacy_import: bool) -> Self { + let (requester, responder) = + mpsc_requests::channel::>(); + let _thread = thread::spawn(move || { + let mut di = DatastoreWorker::new(responder, legacy_import); + di.work_loop(method); + }); + Datastore { requester } + } + + /// Send a command to the worker thread and wait for its response. + /// + /// Fails with `InternalError` instead of panicking when the worker thread + /// is gone (e.g. it panicked on an earlier request), so callers such as + /// HTTP endpoints can degrade to a 5xx response instead of crashing the + /// request. + fn request(&self, cmd: Command) -> Result { + let receiver = self.requester.request(cmd).map_err(|e| { + DatastoreError::InternalError(format!( + "Failed to send request, datastore worker is gone: {e:?}" + )) + })?; + receiver.collect().map_err(|e| { + DatastoreError::InternalError(format!( + "Failed to receive response, datastore worker died while handling request: {e:?}" + )) + })? + } + + pub fn create_bucket(&self, bucket: &Bucket) -> Result<(), DatastoreError> { + let cmd = Command::CreateBucket(bucket.clone()); + _unwrap_empty_response(self.request(cmd)?) + } + + pub fn delete_bucket(&self, bucket_id: &str) -> Result<(), DatastoreError> { + let cmd = Command::DeleteBucket(bucket_id.to_string()); + _unwrap_empty_response(self.request(cmd)?) + } + + pub fn get_bucket(&self, bucket_id: &str) -> Result { + let cmd = Command::GetBucket(bucket_id.to_string()); + match self.request(cmd)? { + Response::Bucket(b) => Ok(b), + _ => panic!("Invalid response"), + } + } + + pub fn get_buckets(&self) -> Result, DatastoreError> { + let cmd = Command::GetBuckets(); + match self.request(cmd)? { + Response::BucketMap(bm) => Ok(bm), + e => Err(DatastoreError::InternalError(format!( + "Invalid response: {e:?}" + ))), + } + } + + /// Write a consistent export to an empty temporary file. The file is + /// returned only after serialization and flushing succeed. Event rows are + /// serialized individually, including this worker's uncommitted writes. + pub fn export_to_file( + &self, + bucket_id: Option<&str>, + file: File, + ) -> Result<(File, Option), DatastoreError> { + match self.request(Command::Export(bucket_id.map(str::to_owned), file))? { + Response::Export(file, name) => Ok((file, name)), + _ => panic!("Invalid response"), + } + } + + pub fn insert_events( + &self, + bucket_id: &str, + events: &[Event], + ) -> Result, DatastoreError> { + let cmd = Command::InsertEvents(bucket_id.to_string(), events.to_vec()); + match self.request(cmd)? { + Response::EventList(events) => Ok(events), + _ => panic!("Invalid response"), + } + } + + pub fn heartbeat( + &self, + bucket_id: &str, + heartbeat: Event, + pulsetime: f64, + ) -> Result { + let cmd = Command::Heartbeat(bucket_id.to_string(), heartbeat, pulsetime); + match self.request(cmd)? { + Response::Event(e) => Ok(e), + _ => panic!("Invalid response"), + } + } + + pub fn get_event(&self, bucket_id: &str, event_id: i64) -> Result { + let cmd = Command::GetEvent(bucket_id.to_string(), event_id); + match self.request(cmd)? { + Response::Event(el) => Ok(el), + _ => panic!("Invalid response"), + } + } + + pub fn get_events( + &self, + bucket_id: &str, + starttime_opt: Option>, + endtime_opt: Option>, + limit_opt: Option, + ) -> Result, DatastoreError> { + let cmd = Command::GetEvents( + bucket_id.to_string(), + starttime_opt, + endtime_opt, + limit_opt, + false, + ); + match self.request(cmd)? { + Response::EventList(el) => Ok(el), + _ => panic!("Invalid response"), + } + } + + pub fn get_events_unclipped( + &self, + bucket_id: &str, + starttime_opt: Option>, + endtime_opt: Option>, + limit_opt: Option, + ) -> Result, DatastoreError> { + let cmd = Command::GetEvents( + bucket_id.to_string(), + starttime_opt, + endtime_opt, + limit_opt, + true, + ); + match self.request(cmd)? { + Response::EventList(el) => Ok(el), + _ => panic!("Invalid response"), + } + } + + pub fn get_event_count( + &self, + bucket_id: &str, + starttime_opt: Option>, + endtime_opt: Option>, + ) -> Result { + let cmd = Command::GetEventCount(bucket_id.to_string(), starttime_opt, endtime_opt); + match self.request(cmd)? { + Response::Count(n) => Ok(n), + _ => panic!("Invalid response"), + } + } + + pub fn delete_events_by_id( + &self, + bucket_id: &str, + event_ids: Vec, + ) -> Result<(), DatastoreError> { + let cmd = Command::DeleteEventsById(bucket_id.to_string(), event_ids); + _unwrap_empty_response(self.request(cmd)?) + } + + pub fn force_commit(&self) -> Result<(), DatastoreError> { + let cmd = Command::ForceCommit(); + _unwrap_empty_response(self.request(cmd)?) + } + + pub fn get_key_values(&self, pattern: &str) -> Result, DatastoreError> { + let cmd = Command::GetKeyValues(pattern.to_string()); + match self.request(cmd)? { + Response::KeyValues(value) => Ok(value), + _ => panic!("Invalid response"), + } + } + + pub fn get_key_value(&self, key: &str) -> Result { + let cmd = Command::GetKeyValue(key.to_string()); + match self.request(cmd)? { + Response::KeyValue(kv) => Ok(kv), + _ => panic!("Invalid response"), + } + } + + pub fn set_key_value(&self, key: &str, data: &str) -> Result<(), DatastoreError> { + let cmd = Command::SetKeyValue(key.to_string(), data.to_string()); + _unwrap_empty_response(self.request(cmd)?) + } + + pub fn delete_key_value(&self, key: &str) -> Result<(), DatastoreError> { + let cmd = Command::DeleteKeyValue(key.to_string()); + _unwrap_empty_response(self.request(cmd)?) + } + + pub fn refresh_privacy_filter(&self) -> Result<(), DatastoreError> { + _unwrap_empty_response(self.request(Command::RefreshPrivacyFilter())?) + } + + /// Renames a bucket from `old_id` to `new_id`. + pub fn rename_bucket(&self, old_id: &str, new_id: &str) -> Result<(), DatastoreError> { + let cmd = Command::RenameBucket(old_id.to_string(), new_id.to_string()); + _unwrap_empty_response(self.request(cmd)?) + } + + /// Migrates all buckets whose hostname is "unknown" or "Unknown" to `new_hostname`. + /// Returns the number of buckets updated. + pub fn migrate_hostname(&self, new_hostname: &str) -> Result { + let cmd = Command::MigrateHostname(new_hostname.to_string()); + match self.request(cmd)? { + Response::Count(n) => Ok(n as usize), + _ => Err(DatastoreError::InternalError( + "Unexpected response to MigrateHostname command".to_string(), + )), + } + } + + /// Migrates all buckets whose name starts with `aw-watcher-android-test` to use + /// `aw-watcher-android` instead (e.g. debug-build buckets from older app versions). + /// Returns the number of buckets updated. + pub fn migrate_test_bucket_names(&self) -> Result { + let cmd = Command::MigrateTestBucketNames(); + match self.request(cmd)? { + Response::Count(n) => Ok(n as usize), + _ => Err(DatastoreError::InternalError( + "Unexpected response to MigrateTestBucketNames command".to_string(), + )), + } + } + + // Should block until worker has stopped + pub fn close(&self) { + info!("Sending close request to database"); + match self.request(Command::Close()) { + Ok(Response::Empty()) => (), + Ok(_) => panic!("Invalid response"), + // Worker already gone means there is nothing left to close + Err(e) => warn!("Error closing database: {e:?}"), + } + } +} diff --git a/aw-datastore/tests/datastore.rs b/aw-datastore/tests/datastore.rs new file mode 100644 index 0000000..f4d836c --- /dev/null +++ b/aw-datastore/tests/datastore.rs @@ -0,0 +1,1127 @@ +#[macro_use] +extern crate log; +extern crate chrono; +#[macro_use] +extern crate aw_datastore; +extern crate serde_json; + +extern crate dirs; + +#[cfg(test)] +mod datastore_tests { + use chrono::DateTime; + use chrono::Duration; + use chrono::Utc; + use serde_json::json; + + use aw_datastore::Datastore; + + use aw_models::Bucket; + use aw_models::BucketMetadata; + use aw_models::Event; + + fn test_bucket() -> Bucket { + Bucket { + bid: None, + id: "testid".to_string(), + _type: "testtype".to_string(), + client: "testclient".to_string(), + hostname: "testhost".to_string(), + created: None, + data: json_map! {}, + metadata: BucketMetadata::default(), + events: None, + last_updated: None, + } + } + + fn create_test_bucket(ds: &Datastore) -> Bucket { + let bucket = test_bucket(); + ds.create_bucket(&bucket).unwrap(); + bucket + } + + fn create_named_test_bucket(ds: &Datastore, id: &str) -> Bucket { + let mut bucket = test_bucket(); + bucket.id = id.to_string(); + ds.create_bucket(&bucket).unwrap(); + bucket + } + + fn test_event(timestamp: chrono::DateTime, duration: Duration) -> Event { + Event { + id: None, + timestamp, + duration, + data: json_map! {"key": json!("value")}, + } + } + + #[test] + fn interval_access_paths_preserve_overlaps_ties_and_clipping() { + let ds = Datastore::new_in_memory(false); + let bucket = create_test_bucket(&ds); + let epoch = chrono::DateTime::from_timestamp(1_000_000, 0).unwrap(); + let events: Vec<_> = [ + (0, 1), + (1, 999), + (900, 5), + (900, 1), + (900, 1), + (901, 0), + (1000, 1), + ] + .into_iter() + .map(|(start, duration)| { + test_event( + epoch + Duration::seconds(start), + Duration::seconds(duration), + ) + }) + .collect(); + ds.insert_events(&bucket.id, &events).unwrap(); + for (start, end) in [(0, 10), (500, 510), (900, 905), (1100, 1200)] { + let start = Some(epoch + Duration::seconds(start)); + let end = Some(epoch + Duration::seconds(end)); + // Limited reads always use the original starttime-first index; + // unrestricted reads may use the endtime-first path. + let original = ds.get_events(&bucket.id, start, end, Some(100)).unwrap(); + let selective = ds.get_events(&bucket.id, start, end, None).unwrap(); + assert_eq!( + serde_json::to_value(&original).unwrap(), + serde_json::to_value(&selective).unwrap() + ); + assert_eq!( + ds.get_event_count(&bucket.id, start, end).unwrap(), + original.len() as i64 + ); + for limit in [0, 1, 3] { + let limited = ds.get_events(&bucket.id, start, end, Some(limit)).unwrap(); + assert_eq!( + serde_json::to_value(limited).unwrap(), + serde_json::to_value(&original[..original.len().min(limit as usize)]).unwrap() + ); + } + } + } + + #[test] + fn test_migrate_test_bucket_names_renames_bucket_and_preserves_events() { + let ds = Datastore::new_in_memory(false); + let old_id = "aw-watcher-android-test_phone"; + let new_id = "aw-watcher-android_phone"; + create_named_test_bucket(&ds, old_id); + let event = test_event(Utc::now(), Duration::seconds(30)); + ds.insert_events(old_id, std::slice::from_ref(&event)) + .unwrap(); + + assert_eq!(ds.migrate_test_bucket_names().unwrap(), 1); + assert!(!ds.get_buckets().unwrap().contains_key(old_id)); + assert!(ds.get_buckets().unwrap().contains_key(new_id)); + assert_eq!(ds.get_events(new_id, None, None, None).unwrap().len(), 1); + } + + #[test] + fn test_migrate_test_bucket_names_merges_non_overlapping_buckets() { + let ds = Datastore::new_in_memory(false); + let old_id = "aw-watcher-android-test_phone"; + let new_id = "aw-watcher-android_phone"; + create_named_test_bucket(&ds, old_id); + create_named_test_bucket(&ds, new_id); + let now = Utc::now(); + ds.insert_events( + old_id, + &[test_event(now - Duration::hours(2), Duration::minutes(30))], + ) + .unwrap(); + ds.insert_events(new_id, &[test_event(now, Duration::minutes(30))]) + .unwrap(); + + assert_eq!(ds.migrate_test_bucket_names().unwrap(), 1); + assert!(!ds.get_buckets().unwrap().contains_key(old_id)); + assert_eq!(ds.get_events(new_id, None, None, None).unwrap().len(), 2); + } + + #[test] + fn test_migrate_test_bucket_names_keeps_overlapping_buckets_separate() { + let ds = Datastore::new_in_memory(false); + let old_id = "aw-watcher-android-test_phone"; + let new_id = "aw-watcher-android_phone"; + create_named_test_bucket(&ds, old_id); + create_named_test_bucket(&ds, new_id); + let now = Utc::now(); + ds.insert_events(old_id, &[test_event(now, Duration::minutes(30))]) + .unwrap(); + ds.insert_events( + new_id, + &[test_event( + now + Duration::minutes(15), + Duration::minutes(30), + )], + ) + .unwrap(); + + assert_eq!(ds.migrate_test_bucket_names().unwrap(), 0); + assert!(ds.get_buckets().unwrap().contains_key(old_id)); + assert_eq!(ds.get_events(old_id, None, None, None).unwrap().len(), 1); + assert_eq!(ds.get_events(new_id, None, None, None).unwrap().len(), 1); + } + + #[test] + fn test_migrate_test_bucket_names_moves_disjoint_events_when_some_overlap() { + let ds = Datastore::new_in_memory(false); + let old_id = "aw-watcher-android-test_phone"; + let new_id = "aw-watcher-android_phone"; + create_named_test_bucket(&ds, old_id); + create_named_test_bucket(&ds, new_id); + let now = Utc::now(); + ds.insert_events( + old_id, + &[ + test_event(now - Duration::hours(2), Duration::minutes(30)), + test_event(now, Duration::minutes(30)), + ], + ) + .unwrap(); + ds.insert_events( + new_id, + &[test_event( + now + Duration::minutes(15), + Duration::minutes(30), + )], + ) + .unwrap(); + + assert_eq!(ds.migrate_test_bucket_names().unwrap(), 0); + assert!(ds.get_buckets().unwrap().contains_key(old_id)); + assert_eq!(ds.get_events(old_id, None, None, None).unwrap().len(), 1); + assert_eq!(ds.get_events(new_id, None, None, None).unwrap().len(), 2); + } + + #[test] + fn test_migrate_test_bucket_names_keeps_overlapping_legacy_events_together() { + let ds = Datastore::new_in_memory(false); + let old_id = "aw-watcher-android-test_phone"; + let new_id = "aw-watcher-android_phone"; + create_named_test_bucket(&ds, old_id); + create_named_test_bucket(&ds, new_id); + let now = Utc::now(); + ds.insert_events( + old_id, + &[ + test_event(now - Duration::hours(3), Duration::minutes(20)), + test_event(now, Duration::minutes(30)), + test_event(now + Duration::minutes(15), Duration::minutes(30)), + ], + ) + .unwrap(); + ds.insert_events( + new_id, + &[test_event(now + Duration::hours(2), Duration::minutes(20))], + ) + .unwrap(); + + assert_eq!(ds.migrate_test_bucket_names().unwrap(), 0); + assert!(ds.get_buckets().unwrap().contains_key(old_id)); + assert_eq!(ds.get_events(old_id, None, None, None).unwrap().len(), 2); + assert_eq!(ds.get_events(new_id, None, None, None).unwrap().len(), 2); + } + + #[test] + fn test_migrate_test_bucket_names_zero_duration_event_at_shared_start_stays() { + // Strict overlap semantics: a zero-duration legacy event that starts exactly + // where a destination event starts is inside that event's open interval. + let ds = Datastore::new_in_memory(false); + let old_id = "aw-watcher-android-test_phone"; + let new_id = "aw-watcher-android_phone"; + create_named_test_bucket(&ds, old_id); + create_named_test_bucket(&ds, new_id); + let now = Utc::now(); + ds.insert_events( + old_id, + &[test_event(now + Duration::minutes(1), Duration::zero())], + ) + .unwrap(); + ds.insert_events(new_id, &[test_event(now, Duration::minutes(30))]) + .unwrap(); + + assert_eq!(ds.migrate_test_bucket_names().unwrap(), 0); + assert_eq!(ds.get_events(old_id, None, None, None).unwrap().len(), 1); + assert_eq!(ds.get_events(new_id, None, None, None).unwrap().len(), 1); + } + + #[test] + fn test_migrate_test_bucket_names_merges_large_history_quickly() { + // Regression for ActivityWatch/aw-android#261: the merge used a correlated + // overlap subquery per legacy event (O(n^2)), which took hours on a phone + // with a couple of years of history and wedged the datastore worker. The + // sorted sweep must handle six-figure histories in seconds. + let ds = Datastore::new_in_memory(false); + let old_id = "aw-watcher-android-test_phone"; + let new_id = "aw-watcher-android_phone"; + create_named_test_bucket(&ds, old_id); + create_named_test_bucket(&ds, new_id); + let legacy_count = 100_000; + let destination_count = 10_000; + let start = Utc::now() - Duration::days(400); + let step = Duration::minutes(5); + let legacy: Vec = (0..legacy_count) + .map(|i| test_event(start + step * i, step - Duration::seconds(1))) + .collect(); + for chunk in legacy.chunks(10_000) { + ds.insert_events(old_id, chunk).unwrap(); + } + let cutover = start + step * legacy_count; + // One overlapping cutover event must stay behind while the rest merges. + ds.insert_events( + old_id, + &[test_event( + cutover - Duration::minutes(2), + Duration::minutes(4), + )], + ) + .unwrap(); + let destination: Vec = (0..destination_count) + .map(|i| test_event(cutover + step * i, step - Duration::seconds(1))) + .collect(); + ds.insert_events(new_id, &destination).unwrap(); + + let started = std::time::Instant::now(); + assert_eq!(ds.migrate_test_bucket_names().unwrap(), 0); + let elapsed = started.elapsed(); + assert!( + elapsed < std::time::Duration::from_secs(30), + "merge took {elapsed:?}; the overlap scan regressed to quadratic time" + ); + assert_eq!( + ds.get_event_count(old_id, None, None).unwrap(), + 2, + "only the overlapping cutover pair stays in the legacy bucket" + ); + // Everything except the legacy event that the cutover overlaps moved over. + assert_eq!( + ds.get_event_count(new_id, None, None).unwrap(), + (legacy_count - 1 + destination_count) as i64, + ); + } + + #[test] + fn test_migrate_test_bucket_names_heavy_overlap_is_linear() { + // Stacked histories must not reintroduce a quadratic open-set scan. + // Each event starts 1s later and outlives every later start, so the + // open heap grows to n; scanning it per event would be ~n². + let ds = Datastore::new_in_memory(false); + let old_id = "aw-watcher-android-test_phone"; + let new_id = "aw-watcher-android_phone"; + create_named_test_bucket(&ds, old_id); + create_named_test_bucket(&ds, new_id); + let n = 20_000; + let start = Utc::now(); + let span = Duration::seconds(n as i64); + let legacy: Vec = (0..n) + .map(|i| test_event(start + Duration::seconds(i as i64), span)) + .collect(); + for chunk in legacy.chunks(5_000) { + ds.insert_events(old_id, chunk).unwrap(); + } + + let started = std::time::Instant::now(); + assert_eq!(ds.migrate_test_bucket_names().unwrap(), 0); + let elapsed = started.elapsed(); + assert!( + elapsed < std::time::Duration::from_secs(10), + "heavy-overlap merge took {elapsed:?}; open-set scan is quadratic" + ); + assert_eq!( + ds.get_event_count(old_id, None, None).unwrap(), + n as i64, + "every stacked event overlaps another, so none should move" + ); + assert_eq!(ds.get_event_count(new_id, None, None).unwrap(), 0); + } + + #[test] + fn test_migrate_test_bucket_names_merges_interleaved_non_overlapping_events() { + let ds = Datastore::new_in_memory(false); + let old_id = "aw-watcher-android-test_phone"; + let new_id = "aw-watcher-android_phone"; + create_named_test_bucket(&ds, old_id); + create_named_test_bucket(&ds, new_id); + let now = Utc::now(); + ds.insert_events( + old_id, + &[ + test_event(now - Duration::hours(2), Duration::minutes(20)), + test_event(now, Duration::minutes(20)), + ], + ) + .unwrap(); + ds.insert_events( + new_id, + &[test_event(now - Duration::hours(1), Duration::minutes(20))], + ) + .unwrap(); + + assert_eq!(ds.migrate_test_bucket_names().unwrap(), 1); + assert!(!ds.get_buckets().unwrap().contains_key(old_id)); + assert_eq!(ds.get_events(new_id, None, None, None).unwrap().len(), 3); + } + + #[test] + fn test_key_values_empty_result() { + let ds = Datastore::new_in_memory(false); + + let values = ds.get_key_values("settings.%").unwrap(); + + assert!(values.is_empty()); + } + + #[test] + fn test_bucket_create_delete() { + // Setup datastore + let ds = Datastore::new_in_memory(false); + let bucket = create_test_bucket(&ds); + + // Fetch bucket + let bucket_fetched = ds.get_bucket(&bucket.id).unwrap(); + assert_eq!(bucket_fetched.id, bucket.id); + assert_eq!(bucket_fetched._type, bucket._type); + assert_eq!(bucket_fetched.client, bucket.client); + assert_eq!(bucket_fetched.hostname, bucket.hostname); + assert_eq!(bucket_fetched.metadata.end, None); + + match bucket_fetched.created { + None => panic!("Expected 'None' in bucket to be replaced with current time"), + Some(created) => { + let now = Utc::now(); + assert!(created <= now); + assert!(created > now - Duration::seconds(10)); + } + }; + + // Fetch all buckets + let fetched_buckets = ds.get_buckets().unwrap(); + assert!(fetched_buckets.contains_key(&bucket.id)); + assert_eq!(fetched_buckets[&bucket.id].id, bucket.id); + assert_eq!(fetched_buckets[&bucket.id]._type, bucket._type); + assert_eq!(fetched_buckets[&bucket.id].client, bucket.client); + assert_eq!(fetched_buckets[&bucket.id].hostname, bucket.hostname); + assert_eq!( + fetched_buckets[&bucket.id].metadata.start, + bucket.metadata.start + ); + assert_eq!( + fetched_buckets[&bucket.id].metadata.end, + bucket.metadata.end + ); + + match fetched_buckets[&bucket.id].created { + None => panic!("Expected 'None' in bucket to be replaced with current time"), + Some(created) => { + let now = Utc::now(); + assert!(created <= now); + assert!(created > now - Duration::seconds(10)); + } + }; + + // Delete bucket + match ds.delete_bucket(&bucket.id) { + Ok(_) => info!("bucket successfully deleted"), + Err(e) => panic!("{e:?}"), + } + match ds.get_bucket(&bucket.id) { + Ok(_) => { + panic!("Expected datastore to delete bucket but bucket seems to still be available") + } + Err(_e) => (), + } + } + + #[test] + fn test_events_get_single() { + // Setup datastore + let ds = Datastore::new_in_memory(false); + let bucket = create_test_bucket(&ds); + + // Insert event + let e1 = Event { + id: None, + timestamp: Utc::now(), + duration: Duration::seconds(0), + data: json_map! {"key": json!("value")}, + }; + let mut e2 = e1.clone(); + e2.timestamp += Duration::nanoseconds(1); + + let event_list = [e1, e2]; + ds.insert_events(&bucket.id, &event_list).unwrap(); + + let events = ds.get_events(&bucket.id, None, None, None).unwrap(); + let first_event = events.first().unwrap(); + let first_event_id = first_event.id.unwrap(); + + let fetched_event = ds.get_event(&bucket.id, first_event_id).unwrap(); + // TODO: Check entire events to ensure integrity + assert_eq!(fetched_event.id.unwrap(), first_event_id); + } + + #[test] + fn test_events_get_filters() { + // Setup datastore + let ds = Datastore::new_in_memory(false); + let bucket = create_test_bucket(&ds); + + // Insert event + let e1 = Event { + id: None, + timestamp: Utc::now(), + duration: Duration::seconds(0), + data: json_map! {"key": json!("value")}, + }; + let mut e2 = e1.clone(); + e2.timestamp += Duration::nanoseconds(1); + + let event_list = [e1.clone(), e2.clone()]; + + ds.insert_events(&bucket.id, &event_list).unwrap(); + + // Get all events + let fetched_events_all = ds.get_events(&bucket.id, None, None, None).unwrap(); + let expected_fetched_events = [&e2, &e1]; + assert_eq!(fetched_events_all.len(), 2); + for i in 0..fetched_events_all.len() { + let expected = &expected_fetched_events[i]; + let new = &fetched_events_all[i]; + assert_eq!(new.timestamp, expected.timestamp); + assert_eq!(new.duration, expected.duration); + assert_eq!(new.data, expected.data); + } + + info!("Get events with limit filter"); + let fetched_events_limit = ds.get_events(&bucket.id, None, None, Some(1)).unwrap(); + assert_eq!(fetched_events_limit.len(), 1); + assert_eq!(fetched_events_limit[0].timestamp, e2.timestamp); + assert_eq!(fetched_events_limit[0].duration, e2.duration); + assert_eq!(fetched_events_limit[0].data, e2.data); + + info!("Get events with starttime filter"); + let fetched_events_start = ds + .get_events(&bucket.id, Some(e2.timestamp), None, None) + .unwrap(); + assert_eq!(fetched_events_start.len(), 1); + assert_eq!(fetched_events_start[0].timestamp, e2.timestamp); + assert_eq!(fetched_events_start[0].duration, e2.duration); + assert_eq!(fetched_events_start[0].data, e2.data); + + info!("Get events with endtime filter"); + let fetched_events_start = ds + .get_events(&bucket.id, None, Some(e1.timestamp), None) + .unwrap(); + assert_eq!(fetched_events_start.len(), 1); + assert_eq!(fetched_events_start[0].timestamp, e1.timestamp); + assert_eq!(fetched_events_start[0].duration, e1.duration); + assert_eq!(fetched_events_start[0].data, e1.data); + + // Get eventcount + let event_count = ds.get_event_count(&bucket.id, None, None).unwrap(); + assert_eq!(event_count, 2); + } + + /// Tests that events that cover a timeperiod get included when that timeperiod is queried. + #[test] + fn test_get_events_filters_cover() { + // TODO: Also test event-cutoff, although perhaps that happens in the transforms/queries? + + // Setup datastore + let ds = Datastore::new_in_memory(false); + let bucket = create_test_bucket(&ds); + + let now = Utc::now(); + + // Insert event + let e1 = Event { + id: None, + timestamp: Utc::now(), + duration: Duration::seconds(100), + data: json_map! {"key": json!("value")}, + }; + + let event_list = [e1]; + ds.insert_events(&bucket.id, &event_list).unwrap(); + + info!("Get event that covers queried timeperiod"); + let query_start = now + Duration::seconds(1); + let query_end = query_start + Duration::seconds(1); + let fetched_events_limit = ds + .get_events(&bucket.id, Some(query_start), Some(query_end), Some(1)) + .unwrap(); + assert_eq!(fetched_events_limit.len(), 1); + + // Get eventcount + let event_count = ds + .get_event_count(&bucket.id, Some(query_start), Some(query_end)) + .unwrap(); + assert_eq!(event_count, 1); + } + + #[test] + fn test_events_delete() { + // Setup datastore + let ds = Datastore::new_in_memory(false); + let bucket = create_test_bucket(&ds); + + // Insert event + let e1 = Event { + id: None, + timestamp: Utc::now(), + duration: Duration::seconds(0), + data: json_map! {"key": json!("value")}, + }; + let mut e2 = e1.clone(); + e2.timestamp += Duration::seconds(1); + + let event_list = [e1.clone(), e2.clone()]; + + ds.insert_events(&bucket.id, &event_list).unwrap(); + + // Get all events + let fetched_events_all = ds.get_events(&bucket.id, None, None, None).unwrap(); + let expected_fetched_events = [&e2, &e1]; + assert_eq!(fetched_events_all.len(), 2); + for i in 0..fetched_events_all.len() { + let expected = &expected_fetched_events[i]; + let new = &fetched_events_all[i]; + assert_eq!(new.timestamp, expected.timestamp); + assert_eq!(new.duration, expected.duration); + assert_eq!(new.data, expected.data); + } + let e1 = &fetched_events_all[0]; + let e2 = &fetched_events_all[1]; + + // Delete one event + ds.delete_events_by_id(&bucket.id, vec![e1.id.unwrap()]) + .unwrap(); + + // Get all events + let fetched_events_all = ds.get_events(&bucket.id, None, None, None).unwrap(); + let expected_fetched_events = [e2]; + assert_eq!(fetched_events_all.len(), 1); + for i in 0..fetched_events_all.len() { + let expected = &expected_fetched_events[i]; + let new = &fetched_events_all[i]; + assert_eq!(new.id, expected.id); + assert_eq!(new.timestamp, expected.timestamp); + assert_eq!(new.duration, expected.duration); + assert_eq!(new.data, expected.data); + } + } + + #[test] + fn test_events_delete_refreshes_bounds_with_pre_epoch_subsecond_event() { + // Regression test: refreshing bucket bounds after a delete must not + // panic when a remaining event has a pre-epoch timestamp with a + // subsecond component (negative nanoseconds, not a multiple of 1s). + let ds = Datastore::new_in_memory(false); + let bucket = create_test_bucket(&ds); + + // 1969-12-31T23:59:55.250Z: negative seconds, with a fractional part. + let pre_epoch = DateTime::from_timestamp(-5, 250_000_000).unwrap(); + let e_pre_epoch = Event { + id: None, + timestamp: pre_epoch, + duration: Duration::seconds(0), + data: json_map! {"key": json!("pre-epoch")}, + }; + let e_other = Event { + id: None, + timestamp: Utc::now(), + duration: Duration::seconds(0), + data: json_map! {"key": json!("other")}, + }; + + // get_events with an unbounded (None) starttime defaults to the + // epoch, which would silently exclude the pre-epoch event - so fetch + // the inserted events' assigned ids directly instead. + let event_list = [e_pre_epoch.clone(), e_other.clone()]; + let inserted = ds.insert_events(&bucket.id, &event_list).unwrap(); + let inserted_other = inserted + .iter() + .find(|e| e.timestamp == e_other.timestamp) + .unwrap(); + + // Deleting the other event forces refresh_bucket_bounds to recompute + // start/end from the remaining pre-epoch event; this must not panic. + ds.delete_events_by_id(&bucket.id, vec![inserted_other.id.unwrap()]) + .unwrap(); + + let bucket_fetched = ds.get_bucket(&bucket.id).unwrap(); + assert_eq!(bucket_fetched.metadata.start, Some(pre_epoch)); + assert_eq!( + bucket_fetched.metadata.end, + Some(e_pre_epoch.calculate_endtime()) + ); + } + + #[test] + fn test_bucket_metadata_start_end() { + // Setup datastore + let ds = Datastore::new_in_memory(false); + let bucket = create_test_bucket(&ds); + + // Insert event + let e1 = Event { + id: None, + timestamp: Utc::now(), + duration: Duration::seconds(0), + data: json_map! {"key": json!("value")}, + }; + let mut e2 = e1.clone(); + e2.timestamp += Duration::nanoseconds(1); + + let event_list = [e1.clone(), e2.clone()]; + + ds.insert_events(&bucket.id, &event_list).unwrap(); + + // Validate correct start and end in bucket + let bucket_fetched = ds.get_bucket(&bucket.id).unwrap(); + assert_eq!(bucket_fetched.metadata.start, Some(e1.timestamp)); + assert_eq!(bucket_fetched.metadata.end, Some(e2.calculate_endtime())); + } + + #[test] + fn test_event_heartbeat() { + // Setup datastore + let ds = Datastore::new_in_memory(false); + let bucket = create_test_bucket(&ds); + + // Insert event + let e1 = Event { + id: None, + timestamp: Utc::now(), + duration: Duration::seconds(0), + data: json_map! {"key": json!("value")}, + }; + let mut e2 = e1.clone(); + e2.timestamp += Duration::seconds(1); + + let mut e_diff_data = e2.clone(); + e_diff_data.timestamp += Duration::seconds(1); + e_diff_data.data = json_map! {"key": json!("other value")}; + + // First event + ds.heartbeat(&bucket.id, e1.clone(), 10.0).unwrap(); + let fetched_events = ds.get_events(&bucket.id, None, None, None).unwrap(); + assert_eq!(fetched_events.len(), 1); + assert_eq!(fetched_events[0].timestamp, e1.timestamp); + assert_eq!(fetched_events[0].duration, e1.duration); + assert_eq!(fetched_events[0].data, e1.data); + let e1 = &fetched_events[0]; + + // Heartbeat match + ds.heartbeat(&bucket.id, e2, 10.0).unwrap(); + let fetched_events = ds.get_events(&bucket.id, None, None, None).unwrap(); + assert_eq!(fetched_events.len(), 1); + assert_eq!(fetched_events[0].timestamp, e1.timestamp); + assert_eq!(fetched_events[0].duration, Duration::seconds(1)); + assert_eq!(fetched_events[0].data, e1.data); + assert_eq!(fetched_events[0].id, e1.id); + let e2 = &fetched_events[0]; + + // Heartbeat diff + ds.heartbeat(&bucket.id, e_diff_data.clone(), 10.0).unwrap(); + let fetched_events = ds.get_events(&bucket.id, None, None, None).unwrap(); + assert_eq!(fetched_events.len(), 2); + assert_eq!(fetched_events[0].timestamp, e_diff_data.timestamp); + assert_eq!(fetched_events[0].duration, e_diff_data.duration); + assert_eq!(fetched_events[0].data, e_diff_data.data); + assert_ne!(fetched_events[0].id, e2.id); + } + + #[test] + fn test_event_heartbeat_consecutive_merges() { + // Regression test for https://github.com/ActivityWatch/aw-server-rust/issues/559 + // After insert_heartbeat merges two events, the merged event stored in cache + // must retain the DB event ID so subsequent heartbeats can find it for + // replace_last_event. Without the ID, every other heartbeat would fail. + let ds = Datastore::new_in_memory(false); + let bucket = create_test_bucket(&ds); + + let now = Utc::now(); + let pulsetime = 10.0; + + // Send 5 consecutive heartbeats with the same data, 1 second apart. + // They should all merge into a single event with increasing duration. + for i in 0..5 { + let e = Event { + id: None, + timestamp: now + Duration::seconds(i), + duration: Duration::seconds(0), + data: json_map! {"key": json!("value")}, + }; + ds.heartbeat(&bucket.id, e, pulsetime).unwrap(); + } + + let events = ds.get_events(&bucket.id, None, None, None).unwrap(); + assert_eq!( + events.len(), + 1, + "all heartbeats should merge into one event" + ); + assert_eq!(events[0].timestamp, now); + assert_eq!(events[0].duration, Duration::seconds(4)); + assert!(events[0].id.is_some(), "event should have a DB id"); + } + + #[test] + fn test_event_replace() { + // Setup datastore + let ds = Datastore::new_in_memory(false); + let bucket = create_test_bucket(&ds); + + // Insert event + let e = Event { + id: None, + timestamp: Utc::now(), + duration: Duration::seconds(0), + data: json_map! {"key": json!("value")}, + }; + let mut e1 = e.clone(); + e1.data = json_map! {"key": json!("value1")}; + let mut e2 = e.clone(); + e2.data = json_map! {"key": json!("value2")}; + let mut e3 = e; + e3.data = json_map! {"key": json!("value3")}; + + let events_init = &[e1, e2, e3]; + let events_ret = ds.insert_events(&bucket.id, events_init).unwrap(); + // Validate return from insert + assert_eq!(events_ret[0].id, Some(1)); + assert_eq!(events_ret[1].id, Some(2)); + assert_eq!(events_ret[2].id, Some(3)); + assert_eq!(events_ret.len(), 3); + assert_eq!(events_ret[0], events_init[0]); + assert_eq!(events_ret[1], events_init[1]); + assert_eq!(events_ret[2], events_init[2]); + + let events_init = events_ret; + + // Insert e2 with identical data and id (which means a replace) + { + let events_ret = ds + .insert_events(&bucket.id, &[events_init[1].clone()]) + .unwrap(); + assert_eq!(events_ret.len(), 1); + assert_eq!(events_ret[0], events_init[1]); + let fetched_events = ds.get_events(&bucket.id, None, None, None).unwrap(); + assert_eq!(fetched_events, events_init); + } + + // Make new event with same id but different data + { + let mut e2 = events_init[1].clone(); + e2.data = json_map! {"key": json!("value2_modified")}; + let events_ret = ds.insert_events(&bucket.id, &[e2.clone()]).unwrap(); + assert_eq!(events_ret.len(), 1); + assert_eq!(events_ret[0], e2); + let fetched_events = ds.get_events(&bucket.id, None, None, None).unwrap(); + assert_eq!(fetched_events.len(), 3); + assert_eq!(fetched_events[1], e2); + assert_eq!(fetched_events[0].id, Some(1)); + assert_eq!(fetched_events[1].id, Some(2)); + assert_eq!(fetched_events[2].id, Some(3)); + } + } + + #[test] + fn test_migration_v4_to_v5() { + let test_dir = tempfile::tempdir().unwrap(); + let db_path = test_dir.path().join("datastore-unittest-migration-v4.db"); + let db_path_str = db_path.to_str().unwrap().to_string(); + + if db_path.exists() { + std::fs::remove_file(&db_path) + .expect("Failed to remove datastore-unittest-migration-v4.db file"); + } + + // Construct a database with the v4 schema (single-column indexes) + { + let conn = rusqlite::Connection::open(&db_path).unwrap(); + conn.execute_batch( + r#" + CREATE TABLE buckets ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT UNIQUE NOT NULL, + type TEXT NOT NULL, + client TEXT NOT NULL, + hostname TEXT NOT NULL, + created TEXT NOT NULL, + data TEXT NOT NULL DEFAULT '{}' + ); + CREATE INDEX bucket_id_index ON buckets(id); + CREATE TABLE events ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + bucketrow INTEGER NOT NULL, + starttime INTEGER NOT NULL, + endtime INTEGER NOT NULL, + data TEXT NOT NULL, + FOREIGN KEY (bucketrow) REFERENCES buckets(id) + ); + CREATE INDEX events_bucketrow_index ON events(bucketrow); + CREATE INDEX events_starttime_index ON events(starttime); + CREATE INDEX events_endtime_index ON events(endtime); + CREATE TABLE key_value ( + key TEXT PRIMARY KEY, + value TEXT, + last_modified NUMBER NOT NULL + ); + INSERT INTO buckets (name, type, client, hostname, created, data) + VALUES ('testid', 'testtype', 'testclient', 'testhost', + '2024-01-01T00:00:00+00:00', '{}'); + INSERT INTO events (bucketrow, starttime, endtime, data) + VALUES (1, 1000000000, 2000000000, '{"key": "value"}'); + PRAGMA user_version = 4; + "#, + ) + .unwrap(); + } + + // Opening the datastore migrates to the newest version + { + let ds = Datastore::new(db_path_str, false); + let events = ds.get_events("testid", None, None, None).unwrap(); + assert_eq!(events.len(), 1); + assert_eq!(events[0].data, json_map! {"key": json!("value")}); + ds.close(); + } + + // Verify version bump and index replacement + { + let conn = rusqlite::Connection::open(&db_path).unwrap(); + let version: i32 = conn + .pragma_query_value(None, "user_version", |row| row.get(0)) + .unwrap(); + assert_eq!(version, 6); + let old_indexes: i64 = conn + .query_row( + "SELECT count(*) FROM sqlite_master WHERE type = 'index' AND name IN + ('events_bucketrow_index', 'events_starttime_index', 'events_endtime_index')", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(old_indexes, 0, "single-column indexes should be dropped"); + let new_index: i64 = conn + .query_row( + "SELECT count(*) FROM sqlite_master WHERE type = 'index' + AND name = 'events_bucketrow_starttime_endtime_index'", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(new_index, 1, "composite index should exist"); + let end_index: i64 = conn + .query_row( + "SELECT count(*) FROM sqlite_master WHERE type = 'index' + AND name = 'events_bucketrow_endtime_starttime_index'", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(end_index, 1); + } + + std::fs::remove_file(&db_path) + .expect("Failed to remove datastore-unittest-migration-v4.db file"); + } + + #[test] + fn test_datastore_reload() { + // Create tmp datastore path + let test_dir = tempfile::tempdir().unwrap(); + let db_path = test_dir.path().join("datastore-unittest.db"); + let db_path_str = db_path.to_str().unwrap().to_string(); + + if db_path.exists() { + std::fs::remove_file(db_path.clone()) + .expect("Failed to remove datastore-unittest.db file"); + } + + let empty_bucket = test_bucket(); + let mut populated_bucket = empty_bucket.clone(); + populated_bucket.id = "testid2".to_string(); + let e1 = Event { + id: None, + timestamp: Utc::now(), + duration: Duration::seconds(0), + data: json_map! {"key": json!("value")}, + }; + { + // Initialize database and create buckets + let ds = Datastore::new(db_path_str.clone(), false); + ds.create_bucket(&empty_bucket).unwrap(); + ds.create_bucket(&populated_bucket).unwrap(); + // Insert event + ds.insert_events(&populated_bucket.id, std::slice::from_ref(&e1)) + .unwrap(); + + // Check that all cached bucket data is correct + let buckets = ds.get_buckets().unwrap(); + assert_eq!(buckets[&empty_bucket.id].metadata.start, None); + assert_eq!(buckets[&empty_bucket.id].metadata.end, None); + assert_eq!( + buckets[&populated_bucket.id].metadata.start, + Some(e1.calculate_endtime()) + ); + assert_eq!( + buckets[&populated_bucket.id].metadata.end, + Some(e1.timestamp) + ); + ds.force_commit().unwrap(); + } + { + // Load database again + let ds = Datastore::new(db_path_str, false); + // Check that all bucket data is correct after reload + let buckets = ds.get_buckets().unwrap(); + assert_eq!(buckets[&empty_bucket.id].metadata.start, None); + assert_eq!(buckets[&empty_bucket.id].metadata.end, None); + assert_eq!( + buckets[&populated_bucket.id].metadata.start, + Some(e1.timestamp) + ); + assert_eq!( + buckets[&populated_bucket.id].metadata.end, + Some(e1.calculate_endtime()) + ); + } + } + + /// Test that an encrypted datastore can be created, written to, and reopened with the same key + /// with data intact. + #[test] + #[cfg(any(feature = "encryption", feature = "encryption-vendored"))] + fn test_encrypted_datastore_roundtrip() { + use std::fs; + let test_dir = tempfile::tempdir().unwrap(); + let dir = test_dir.path(); + let db_path = dir.join("test-encrypted.db").to_str().unwrap().to_string(); + // Clean up from previous runs + let _ = fs::remove_file(&db_path); + + let key = "s3cr3t-p@ssw0rd".to_string(); + + // Create and populate encrypted datastore + { + let ds = Datastore::new_encrypted(db_path.clone(), key.clone(), false); + let bucket = create_test_bucket(&ds); + let e = Event { + id: None, + timestamp: Utc::now(), + duration: Duration::seconds(1), + data: json_map! { "app": "test-encrypted" }, + }; + let inserted = ds.insert_events(&bucket.id, &[e]).unwrap(); + assert_eq!(inserted.len(), 1); + ds.force_commit().unwrap(); + ds.close(); + } + + // Reopen with correct key — data must survive the roundtrip + { + let ds = Datastore::new_encrypted(db_path.clone(), key.clone(), false); + let events = ds + .get_events("testid", None, None, None) + .expect("should read events from encrypted DB after reopen"); + assert_eq!(events.len(), 1, "expected 1 event after encrypted reopen"); + assert_eq!(events[0].data["app"], "test-encrypted"); + ds.close(); + } + + let _ = fs::remove_file(&db_path); + } + + fn privacy_event(title: &str) -> Event { + Event { + id: None, + timestamp: Utc::now(), + duration: Duration::seconds(1), + data: json_map! {"title": json!(title), "app": json!("Firefox")}, + } + } + + /// Saving privacy_filters used to persist the JSON and never load it into + /// the worker's engine (ActivityWatch/aw-server-rust#659). A drop rule + /// must actually drop a matching insert without an explicit refresh call. + #[test] + fn test_privacy_filter_applies_after_setting_saved() { + let ds = Datastore::new_in_memory(false); + let bucket = create_test_bucket(&ds); + + let drop_secret = + r#"[{"enabled":true,"field":"title","pattern":"(?i)secret","action":"drop"}]"#; + ds.set_key_value("settings.privacy_filters", drop_secret) + .unwrap(); + + ds.insert_events( + &bucket.id, + &[privacy_event("my secret file"), privacy_event("readme.md")], + ) + .unwrap(); + + let events = ds.get_events(&bucket.id, None, None, None).unwrap(); + assert_eq!( + events.len(), + 1, + "drop rule should discard the matching event" + ); + assert_eq!(events[0].data.get("title").unwrap(), "readme.md"); + + ds.delete_key_value("settings.privacy_filters").unwrap(); + ds.insert_events(&bucket.id, &[privacy_event("another secret")]) + .unwrap(); + let events = ds.get_events(&bucket.id, None, None, None).unwrap(); + assert_eq!( + events.len(), + 2, + "deleting the setting should stop filtering" + ); + } + + /// Rules must load at worker startup, not only after a later SetKeyValue. + #[test] + fn test_privacy_filter_survives_datastore_reload() { + let test_dir = tempfile::tempdir().unwrap(); + let db_path = test_dir + .path() + .join("datastore-unittest-privacy-filters.db"); + let db_path_str = db_path.to_str().unwrap().to_string(); + + let drop_secret = + r#"[{"enabled":true,"field":"title","pattern":"(?i)secret","action":"drop"}]"#; + { + let ds = Datastore::new(db_path_str.clone(), false); + create_test_bucket(&ds); + ds.set_key_value("settings.privacy_filters", drop_secret) + .unwrap(); + ds.force_commit().unwrap(); + ds.close(); + } + { + let ds = Datastore::new(db_path_str, false); + ds.insert_events("testid", &[privacy_event("top secret notes")]) + .unwrap(); + let events = ds.get_events("testid", None, None, None).unwrap(); + assert!( + events.is_empty(), + "persisted drop rule must apply after reopen, got {events:?}" + ); + ds.close(); + } + + // Windows can still hold the SQLite handle after close() while the + // worker thread unwinds (ERROR_SHARING_VIOLATION). Cleanup is best-effort. + let _ = std::fs::remove_file(&db_path); + let _ = std::fs::remove_file(db_path.with_extension("db-wal")); + let _ = std::fs::remove_file(db_path.with_extension("db-shm")); + } +} diff --git a/aw-models/Cargo.toml b/aw-models/Cargo.toml new file mode 100644 index 0000000..b1e84bb --- /dev/null +++ b/aw-models/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "aw-models" +version = "0.1.0" +authors = ["Johan Bjäreholt "] +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[lib] +name = "aw_models" +crate-type = ["lib", "cdylib"] +path = "src/lib.rs" + + +[dependencies] +chrono = { version = "0.4", features = ["serde"] } +log = "0.4" +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" +schemars = { version = "0.8", features = ["chrono"] } diff --git a/aw-models/examples/schema.rs b/aw-models/examples/schema.rs new file mode 100644 index 0000000..1e0326e --- /dev/null +++ b/aw-models/examples/schema.rs @@ -0,0 +1,8 @@ +extern crate aw_models; + +use schemars::schema_for; + +fn main() { + let schema = schema_for!(aw_models::Bucket); + println!("{}", serde_json::to_string_pretty(&schema).unwrap()); +} diff --git a/aw-models/src/bucket.rs b/aw-models/src/bucket.rs new file mode 100644 index 0000000..e608006 --- /dev/null +++ b/aw-models/src/bucket.rs @@ -0,0 +1,60 @@ +use chrono::DateTime; +use chrono::Utc; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use serde_json::map::Map; +use serde_json::value::Value; +use std::collections::HashMap; + +use crate::Event; +use crate::TryVec; + +#[derive(Serialize, Deserialize, JsonSchema, Clone, Debug)] +pub struct Bucket { + #[serde(skip)] + pub bid: Option, + #[serde(default)] + pub id: String, + #[serde(rename = "type")] /* type is a reserved Rust keyword */ pub _type: String, + pub client: String, + pub hostname: String, + pub created: Option>, + #[serde(default)] + pub data: Map, + #[serde(default, skip_deserializing)] + pub metadata: BucketMetadata, + // Events should only be "Some" during import/export + // It's using a TryVec to discard only the events which were failed to be serialized so only a + // few events are being dropped during import instead of failing the whole import + pub events: Option>, + pub last_updated: Option>, // TODO: Should probably be moved into metadata field +} + +#[derive(Serialize, Deserialize, JsonSchema, Clone, Debug, Default)] +pub struct BucketMetadata { + #[serde(default)] + pub start: Option>, + pub end: Option>, +} + +#[derive(Serialize, Deserialize, JsonSchema, Clone)] +pub struct BucketsExport { + pub buckets: HashMap, +} + +#[test] +fn test_bucket() { + let b = Bucket { + bid: None, + id: "id".to_string(), + _type: "type".to_string(), + client: "client".to_string(), + hostname: "hostname".into(), + created: None, + data: json_map! {}, + metadata: BucketMetadata::default(), + events: None, + last_updated: None, + }; + debug!("bucket: {:?}", b); +} diff --git a/aw-models/src/duration.rs b/aw-models/src/duration.rs new file mode 100644 index 0000000..558cf48 --- /dev/null +++ b/aw-models/src/duration.rs @@ -0,0 +1,19 @@ +use serde::{Deserialize, Serialize}; + +// Max duration of a i64 nanosecond is 2562047.7880152157 hours +// ((2**64)/2)/1000000000/60/60 + +fn get_nanos(duration: &chrono::Duration) -> f64 { + (duration.num_nanoseconds().unwrap() as f64) / 1_000_000_000.0 +} + +#[derive(Serialize, Deserialize)] +#[serde(remote = "chrono::Duration")] +pub struct DurationSerialization(#[serde(getter = "get_nanos")] f64); + +// Provide a conversion to construct the remote type. +impl From for chrono::Duration { + fn from(def: DurationSerialization) -> chrono::Duration { + chrono::Duration::nanoseconds((def.0 * 1_000_000_000.0) as i64) + } +} diff --git a/aw-models/src/event.rs b/aw-models/src/event.rs new file mode 100644 index 0000000..b7e6ed2 --- /dev/null +++ b/aw-models/src/event.rs @@ -0,0 +1,101 @@ +use chrono::DateTime; +use chrono::Duration; +use chrono::Utc; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use serde_json::Map; +use serde_json::Value; + +use crate::duration::DurationSerialization; +use crate::TimeInterval; + +#[derive(Serialize, Deserialize, JsonSchema, Clone, Debug)] +pub struct Event { + /// An unique id for this event. + /// Will be assigned once the event has reached the servers datastore. + /// + /// **WARNING:** If you set the ID and insert the event to the server it will replace the previous + /// event with that ID. Only do this if you are completely sure what you are doing. + pub id: Option, + /// An rfc3339 timestamp which represents the start of the event + pub timestamp: DateTime, + /// Duration of the event as a floating point number in seconds. + /// Appended to the timestamp it can represent the end of the event + /// Maximum precision is nanoseconds. + #[serde(with = "DurationSerialization", default = "default_duration")] + #[schemars(with = "f64")] + pub duration: Duration, + /// Can contain any arbitrary JSON data that represents the value of the event. + /// All events in a bucket should follow the format of it's respective bucket-type. + pub data: Map, +} + +impl Event { + pub fn new(timestamp: DateTime, duration: Duration, data: Map) -> Self { + Event { + id: None, + timestamp, + duration, + data, + } + } + pub fn calculate_endtime(&self) -> DateTime { + self.timestamp + chrono::Duration::nanoseconds(self.duration.num_nanoseconds().unwrap()) + } + pub fn interval(&self) -> TimeInterval { + TimeInterval::new(self.timestamp, self.calculate_endtime()) + } +} + +impl PartialEq for Event { + fn eq(&self, other: &Event) -> bool { + !(self.timestamp != other.timestamp + || self.duration != other.duration + || self.data != other.data) + } +} + +impl Default for Event { + fn default() -> Self { + Event { + id: None, + timestamp: Utc::now(), + duration: Duration::seconds(0), + data: serde_json::Map::new(), + } + } +} + +fn default_duration() -> Duration { + Duration::seconds(0) +} + +#[test] +fn test_event() { + use serde_json::json; + + let e = Event { + id: None, + timestamp: Utc::now(), + duration: Duration::seconds(1), + data: json_map! {"test": json!(1)}, + }; + debug!("event: {:?}", e); +} + +#[test] +fn test_event_default_duration() { + // Watchers may omit duration when posting an event, in which case it + // defaults to zero rather than failing to deserialize. + let e: Event = + serde_json::from_str(r#"{"timestamp": "2000-01-01T00:00:00Z", "data": {"test": 1}}"#) + .unwrap(); + assert_eq!(e.duration, Duration::seconds(0)); + + // An explicit duration is still honoured. + let e: Event = serde_json::from_str( + r#"{"timestamp": "2000-01-01T00:00:00Z", "duration": 1.5, "data": {}}"#, + ) + .unwrap(); + assert_eq!(e.duration, Duration::milliseconds(1500)); +} diff --git a/aw-models/src/export.rs b/aw-models/src/export.rs new file mode 100644 index 0000000..e378a9e --- /dev/null +++ b/aw-models/src/export.rs @@ -0,0 +1,4 @@ +#[derive(Serialize, Deserialize, JsonSchema, Clone)] +pub struct BucketsExport { + pub buckets: HashMap, +} diff --git a/aw-models/src/info.rs b/aw-models/src/info.rs new file mode 100644 index 0000000..64f0971 --- /dev/null +++ b/aw-models/src/info.rs @@ -0,0 +1,19 @@ +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +#[derive(Serialize, Deserialize, JsonSchema)] +pub struct Info { + pub hostname: String, + pub version: String, + pub testing: bool, + /// Name of the running instance profile ("default" unless `--profile` was + /// given). Lets clients tell concurrent instances apart. Defaults on + /// deserialization so a new client can still talk to an older server. + #[serde(default = "default_profile")] + pub profile: String, + pub device_id: String, +} + +fn default_profile() -> String { + "default".to_string() +} diff --git a/aw-models/src/lib.rs b/aw-models/src/lib.rs new file mode 100644 index 0000000..4639fa7 --- /dev/null +++ b/aw-models/src/lib.rs @@ -0,0 +1,39 @@ +#[macro_use] +extern crate log; + +// TODO: Move me to an appropriate place +#[cfg(test)] // Only macro use for tests +macro_rules! json_map { + { $( $key:literal : $value:expr),* } => {{ + use serde_json::{Value}; + use serde_json::map::Map; + #[allow(unused_mut)] + let mut map : Map = Map::new(); + $( + map.insert( $key.to_string(), json!($value) ); + )* + map + }}; +} + +mod bucket; +mod duration; +mod event; +mod info; +mod query; +mod settings; +mod timeinterval; +mod tryvec; + +pub use self::bucket::Bucket; +pub use self::bucket::BucketMetadata; +pub use self::bucket::BucketsExport; +pub use self::event::Event; +pub use self::info::Info; +pub use self::query::Query; +pub use self::settings::Settings; +pub use self::settings::{ + Class, ClassData, ClassRule, NewReleaseCheckData, UserSatisfactionPollData, View, ViewElement, +}; +pub use self::timeinterval::TimeInterval; +pub use self::tryvec::TryVec; diff --git a/aw-models/src/query.rs b/aw-models/src/query.rs new file mode 100644 index 0000000..4e3e744 --- /dev/null +++ b/aw-models/src/query.rs @@ -0,0 +1,38 @@ +use serde::{Deserialize, Serialize}; + +use crate::TimeInterval; + +#[derive(Serialize, Deserialize, Clone, Debug)] +pub struct Query { + //#[serde(with = "DurationSerialization")] + pub timeperiods: Vec, + pub query: Vec, +} + +#[test] +fn test_query_serialization_roundtrip() { + for payload in [ + serde_json::json!({ + "timeperiods": [ + "2000-01-01T00:00:00+00:00/2000-01-02T00:00:00+00:00", + "2000-01-02T00:00:00.123456789+00:00/2000-01-03T00:00:00+00:00" + ], + "query": ["events = query_bucket(\"test\");\nRETURN = events;", "RETURN = [];" ] + }), + serde_json::json!({"timeperiods": [], "query": []}), + ] { + let query: Query = serde_json::from_value(payload.clone()).unwrap(); + let encoded = serde_json::to_string(&query).unwrap(); + assert_eq!( + serde_json::from_str::(&encoded).unwrap(), + payload + ); + let decoded: Query = serde_json::from_str(&encoded).unwrap(); + assert_eq!(decoded.query, query.query); + assert_eq!(decoded.timeperiods.len(), query.timeperiods.len()); + for (actual, expected) in decoded.timeperiods.iter().zip(&query.timeperiods) { + assert_eq!(actual.start(), expected.start()); + assert_eq!(actual.end(), expected.end()); + } + } +} diff --git a/aw-models/src/settings.rs b/aw-models/src/settings.rs new file mode 100644 index 0000000..57cbc47 --- /dev/null +++ b/aw-models/src/settings.rs @@ -0,0 +1,76 @@ +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +#[derive(Serialize, Deserialize, JsonSchema, Clone)] +pub struct NewReleaseCheckData { + pub how_often_to_check: i64, + pub is_enabled: bool, + pub next_check_time: String, + pub times_checked: i32, +} + +#[derive(Serialize, Deserialize, JsonSchema, Clone)] +pub struct UserSatisfactionPollData { + pub is_enabled: bool, + pub next_poll_time: String, + pub times_poll_is_shown: i32, +} + +#[derive(Serialize, Deserialize, JsonSchema, Clone)] +pub struct ViewElement { + pub size: i32, + #[serde(rename = "type")] + pub element_type: String, +} + +#[derive(Serialize, Deserialize, JsonSchema, Clone)] +pub struct View { + pub elements: Vec, + pub id: String, + pub name: String, +} + +#[derive(Serialize, Deserialize, JsonSchema, Clone)] +pub struct ClassData { + #[serde(skip_serializing_if = "Option::is_none")] + pub color: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub score: Option, +} + +#[derive(Serialize, Deserialize, JsonSchema, Clone)] +pub struct ClassRule { + #[serde(skip_serializing_if = "Option::is_none")] + pub ignore_case: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub regex: Option, + #[serde(rename = "type")] + pub rule_type: String, +} + +#[derive(Serialize, Deserialize, JsonSchema, Clone)] +pub struct Class { + #[serde(skip_serializing_if = "Option::is_none")] + pub data: Option, + pub id: i32, + pub name: Vec, + pub rule: ClassRule, +} + +#[derive(Serialize, Deserialize, JsonSchema)] +pub struct Settings { + pub landing_page: String, + pub start_of_day: String, + pub always_active_pattern: String, + pub new_release_check_data: NewReleaseCheckData, + pub start_of_week: String, + pub use_color_fallback: bool, + pub user_satisfaction_poll_data: UserSatisfactionPollData, + pub request_timeout: i32, + pub devmode: bool, + pub theme: String, + pub views: Vec, + pub duration_default: i32, + pub classes: Vec, + pub initial_timestamp: String, +} diff --git a/aw-models/src/timeinterval.rs b/aw-models/src/timeinterval.rs new file mode 100644 index 0000000..8e06450 --- /dev/null +++ b/aw-models/src/timeinterval.rs @@ -0,0 +1,211 @@ +use crate::Event; +use std::cmp::{max, min}; +use std::fmt; + +use serde::de::{self, Deserialize, Deserializer, Visitor}; +use serde::{Serialize, Serializer}; + +use chrono::DateTime; +use chrono::Duration; +use chrono::Utc; + +#[derive(Clone, Debug)] +pub struct TimeInterval { + start: DateTime, + end: DateTime, +} + +#[derive(Debug)] +pub enum TimeIntervalError { + ParseError(), +} + +/// Python versions of many of these functions can be found at https://github.com/ErikBjare/timeslot +impl TimeInterval { + pub fn new(start: DateTime, end: DateTime) -> TimeInterval { + TimeInterval { start, end } + } + + pub fn new_from_string(period: &str) -> Result { + let splits = period.split('/').collect::>(); + if splits.len() != 2 { + return Err(TimeIntervalError::ParseError()); + } + let start = match DateTime::parse_from_rfc3339(splits[0]) { + Ok(dt) => dt.with_timezone(&Utc), + Err(_e) => return Err(TimeIntervalError::ParseError()), + }; + let end = match DateTime::parse_from_rfc3339(splits[1]) { + Ok(dt) => dt.with_timezone(&Utc), + Err(_e) => return Err(TimeIntervalError::ParseError()), + }; + + Ok(TimeInterval::new(start, end)) + } + + pub fn start(&self) -> &DateTime { + &self.start + } + + pub fn end(&self) -> &DateTime { + &self.end + } + + pub fn duration(&self) -> Duration { + self.end - self.start + } + + /// If intervals are separated by a non-zero gap, return the gap as a new TimeInterval, else None + pub fn gap(&self, other: &TimeInterval) -> Option { + if self.end < other.start { + Some(TimeInterval::new(self.end, other.start)) + } else if other.end < self.start { + Some(TimeInterval::new(other.end, self.start)) + } else { + None + } + } + + /// Joins two intervals together if they don't have a gap, else None + pub fn union(&self, other: &TimeInterval) -> Option { + match self.gap(other) { + Some(_) => None, + None => Some(TimeInterval::new( + min(self.start, other.start), + max(self.end, other.end), + )), + } + } + + /// The intersection of two intervals + pub fn intersection(&self, other: &TimeInterval) -> Option { + let last_start = max(self.start, other.start); + let first_end = min(self.end, other.end); + if last_start < first_end { + Some(TimeInterval::new(last_start, first_end)) + } else { + None + } + } + + /// A boolean whether the two intervals intersect + pub fn intersects(&self, other: &TimeInterval) -> bool { + self.intersection(other).is_some() + } +} + +impl From<&Event> for TimeInterval { + fn from(e: &Event) -> TimeInterval { + TimeInterval::new(e.timestamp, e.timestamp + e.duration) + } +} + +impl fmt::Display for TimeInterval { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "{}/{}", self.start.to_rfc3339(), self.end.to_rfc3339()) + } +} + +struct TimeIntervalVisitor; +use serde::de::Unexpected; + +impl<'de> Visitor<'de> for TimeIntervalVisitor { + type Value = TimeInterval; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("an string in ISO timeinterval format (such as 2000-01-01T00:00:00+01:00/2001-02-02T01:01:01+01:00)") + } + + fn visit_str(self, value: &str) -> Result + where + E: de::Error, + { + match TimeInterval::new_from_string(value) { + Ok(ti) => Ok(ti), + Err(e) => { + warn!("{:?}", e); + Err(de::Error::invalid_value(Unexpected::Str(value), &self)) + } + } + } +} + +impl<'de> Deserialize<'de> for TimeInterval { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + deserializer.deserialize_str(TimeIntervalVisitor) + } +} + +impl Serialize for TimeInterval { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + serializer.collect_str(self) + } +} + +#[test] +fn test_timeinterval_serialization() { + for (input, expected) in [ + ( + "2000-01-01T00:00:00Z/2000-01-02T00:00:00Z", + "2000-01-01T00:00:00+00:00/2000-01-02T00:00:00+00:00", + ), + ( + "2000-01-01T03:00:00.123456789+03:00/2000-01-01T01:00:00.987654321-02:00", + "2000-01-01T00:00:00.123456789+00:00/2000-01-01T03:00:00.987654321+00:00", + ), + ( + "2000-01-01T00:00:00Z/2000-01-01T00:00:00Z", + "2000-01-01T00:00:00+00:00/2000-01-01T00:00:00+00:00", + ), + ] { + let interval = TimeInterval::new_from_string(input).unwrap(); + let encoded = serde_json::to_value(&interval).unwrap(); + assert_eq!(encoded, serde_json::json!(expected)); + let decoded: TimeInterval = serde_json::from_value(encoded).unwrap(); + assert_eq!(decoded.start(), interval.start()); + assert_eq!(decoded.end(), interval.end()); + } +} + +#[test] +fn test_timeinterval() { + use std::str::FromStr; + + let start = DateTime::from_str("2000-01-01T00:00:00Z").unwrap(); + let end = DateTime::from_str("2000-01-02T00:00:00Z").unwrap(); + let period_str = "2000-01-01T00:00:00+00:00/2000-01-02T00:00:00+00:00"; + let duration = end - start; + let tp = TimeInterval::new(start, end); + assert_eq!(tp.start(), &start); + assert_eq!(tp.end(), &end); + assert_eq!(tp.duration(), duration); + assert_eq!(tp.to_string(), period_str); + + let tp = TimeInterval::new_from_string(period_str).unwrap(); + assert_eq!(tp.start(), &start); + assert_eq!(tp.end(), &end); + assert_eq!(tp.duration(), duration); + assert_eq!(tp.to_string(), period_str); +} + +#[test] +fn test_timeinterval_intersection() { + use std::str::FromStr; + + // Check that two exactly adjacent events don't intersect + let tp1 = TimeInterval::new( + DateTime::from_str("2000-01-01T00:00:00Z").unwrap(), + DateTime::from_str("2000-01-01T00:01:00Z").unwrap(), + ); + let tp2 = TimeInterval::new( + DateTime::from_str("2000-01-01T00:01:00Z").unwrap(), + DateTime::from_str("2000-01-01T00:02:00Z").unwrap(), + ); + assert!(!tp1.intersects(&tp2)); +} diff --git a/aw-models/src/tryvec.rs b/aw-models/src/tryvec.rs new file mode 100644 index 0000000..4b3bf1a --- /dev/null +++ b/aw-models/src/tryvec.rs @@ -0,0 +1,208 @@ +use core::marker::PhantomData; +use schemars::{gen::SchemaGenerator, schema::Schema, JsonSchema}; +use serde::de::{DeserializeOwned, SeqAccess, Visitor}; +use serde::ser::SerializeSeq; +use serde::{Deserialize, Deserializer, Serialize, Serializer}; +use serde_json::Value; +use std::fmt; +use std::fmt::Debug; + +#[derive(Debug, Clone)] +pub enum TryParse { + Parsed(T), + #[allow(dead_code)] + Unparsed(Value), + NotPresent, +} + +impl JsonSchema for TryParse { + fn schema_name() -> String { + format!("Try<{}>", std::any::type_name::()) + } + + fn json_schema(gen: &mut SchemaGenerator) -> Schema { + gen.subschema_for::() + } +} + +impl<'de, T: DeserializeOwned + JsonSchema> Deserialize<'de> for TryParse { + fn deserialize>(deserializer: D) -> Result { + match Option::::deserialize(deserializer)? { + None => Ok(TryParse::NotPresent), + Some(value) => match T::deserialize(&value) { + Ok(t) => Ok(TryParse::Parsed(t)), + Err(_) => Ok(TryParse::Unparsed(value)), + }, + } + } +} + +#[derive(Debug, Clone, JsonSchema)] +#[serde(transparent)] +pub struct TryVec { + inner: Vec>, +} + +impl TryVec { + pub fn new(mut vec: Vec) -> Self { + let mut vec_marked: Vec> = Vec::new(); + for item in vec.drain(..) { + vec_marked.push(TryParse::Parsed(item)); + } + TryVec { inner: vec_marked } + } + + pub fn new_empty() -> Self { + TryVec { inner: Vec::new() } + } + + pub fn take_inner(self) -> Vec { + let mut vec: Vec = Vec::new(); + for item in self.inner { + match item { + TryParse::Parsed(i) => vec.push(i), + _ => continue, + }; + } + vec + } +} + +impl Serialize for TryVec +where + T: Serialize, +{ + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + let mut seq = serializer.serialize_seq(Some(self.inner.len()))?; + for element in &self.inner { + match element { + TryParse::Parsed(t) => seq.serialize_element(t)?, + _ => continue, + }; + } + seq.end() + } +} + +struct TryVecVisitor { + marker: PhantomData TryVec>, +} + +impl TryVecVisitor { + fn new() -> Self { + TryVecVisitor { + marker: PhantomData, + } + } +} + +impl<'de, T> Visitor<'de> for TryVecVisitor +where + T: DeserializeOwned + JsonSchema, +{ + type Value = TryVec; + + // Format a message stating what data this Visitor expects to receive. + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("a sequence") + } + + fn visit_seq(self, mut access: M) -> Result + where + M: SeqAccess<'de>, + { + let mut vec = Vec::new(); + + loop { + let res = match access.next_element() { + Ok(val) => val, + Err(err) => { + println!("Failed to parse event because '{err}', the event will be discarded"); + continue; + } + }; + match res { + Some(item) => vec.push(item), + None => break, + }; + } + + Ok(TryVec { inner: vec }) + } +} + +impl<'de, T: JsonSchema> Deserialize<'de> for TryVec +where + T: DeserializeOwned, +{ + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + deserializer.deserialize_seq(TryVecVisitor::new()) + } +} + +#[cfg(test)] +mod test { + use schemars::JsonSchema; + use serde::{Deserialize, Serialize}; + + use super::TryVec; + + #[derive(Deserialize, Serialize, JsonSchema, Debug)] + struct TestEvent { + data: String, + } + + fn assert_serialized_deserialized_eq(data: &str, eq: &str) { + let deserialized = serde_json::from_str::>(data).unwrap(); + let serialized = serde_json::to_string(&deserialized).unwrap(); + assert_eq!(serialized, eq); + } + + #[test] + fn test_serialize_deserialize() { + println!("test empty array"); + assert_serialized_deserialized_eq(r#"[]"#, r#"[]"#); + + println!("test one valid event"); + assert_serialized_deserialized_eq(r#"[{"data":"test"}]"#, r#"[{"data":"test"}]"#); + + println!("test invalid type int, skip event"); + assert_serialized_deserialized_eq(r#"[{ "data": 1 }]"#, r#"[]"#); + + println!("test invalid type dict, skip event"); + assert_serialized_deserialized_eq(r#"[{"data":{}}]"#, r#"[]"#); + + println!("test invalid type arr, skip event"); + assert_serialized_deserialized_eq(r#"[{"data":[]}]"#, r#"[]"#); + + println!("test multiple valid events"); + assert_serialized_deserialized_eq( + r#"[{"data":"test"},{"data":"test2"},{"data":"test3"}]"#, + r#"[{"data":"test"},{"data":"test2"},{"data":"test3"}]"#, + ); + + println!("test invalid event in middle of sequence, skip one event"); + assert_serialized_deserialized_eq( + r#"[{"data":"test"},{"data":2},{"data":"test3"}]"#, + r#"[{"data":"test"},{"data":"test3"}]"#, + ); + + println!("test utf-16 character"); + assert_serialized_deserialized_eq(r#"[{"data":"\ud835\udc47"}]"#, r#"[{"data":"𝑇"}]"#); + + println!("test invalid utf-8/16, skip event"); + assert_serialized_deserialized_eq(r#"[{"data":"\ud835"}]"#, r#"[]"#); + } + + #[test] + fn test_methods() { + let tryvec = TryVec::::new_empty(); + assert_eq!(tryvec.take_inner().len(), Vec::::new().len()); + } +} diff --git a/aw-query/Cargo.toml b/aw-query/Cargo.toml new file mode 100644 index 0000000..fdba896 --- /dev/null +++ b/aw-query/Cargo.toml @@ -0,0 +1,26 @@ +[package] +name = "aw-query" +version = "0.1.0" +authors = ["Johan Bjäreholt "] +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" +chrono = { version = "0.4", features = ["serde"] } +plex = "0.3.0" +log = "0.4" +fancy-regex = "0.17.0" +aw-datastore = { path = "../aw-datastore" } +aw-models = { path = "../aw-models" } +aw-transform = { path = "../aw-transform" } + +[dev-dependencies] +criterion = "0.5.1" +proptest = "1.4" + +[[bench]] +name = "benchmark" +harness = false diff --git a/aw-query/benches/benchmark.rs b/aw-query/benches/benchmark.rs new file mode 100644 index 0000000..3c0d344 --- /dev/null +++ b/aw-query/benches/benchmark.rs @@ -0,0 +1,127 @@ +use criterion::{criterion_group, criterion_main}; + +// TODO: Move me to an appropriate place +#[macro_export] +macro_rules! json_map { + { $( $key:literal : $value:expr),* } => {{ + use serde_json::Value; + use serde_json::map::Map; + #[allow(unused_mut)] + let mut map : Map = Map::new(); + $( + map.insert( $key.to_string(), json!($value) ); + )* + map + }}; +} + +#[cfg(test)] +mod query_benchmarks { + use chrono::Duration; + use criterion::Criterion; + use serde_json::json; + use serde_json::Map; + use serde_json::Value; + + use aw_datastore::Datastore; + use aw_models::Bucket; + use aw_models::BucketMetadata; + use aw_models::Event; + use aw_models::TimeInterval; + + static BUCKETNAME: &str = "testbucket"; + static TIME_INTERVAL: &str = "1980-01-01T00:00:00Z/2080-01-02T00:00:00Z"; + + fn setup_datastore() -> Datastore { + Datastore::new_in_memory(false) + } + + fn create_bucket(ds: &Datastore, bucketname: String) { + let bucket = Bucket { + bid: None, + id: bucketname, + _type: "testtype".to_string(), + client: "testclient".to_string(), + hostname: "testhost".to_string(), + created: Some(chrono::Utc::now()), + data: json_map! {}, + metadata: BucketMetadata::default(), + events: None, + last_updated: None, + }; + ds.create_bucket(&bucket).unwrap(); + } + + fn insert_events(ds: &Datastore, bucketname: &str, num_events: i64) { + let mut possible_data = Vec::>::new(); + for i in 0..20 { + possible_data.push(json_map! {"number": i}); + } + let mut event_list = Vec::new(); + for i in 0..num_events { + let e = Event { + id: None, + timestamp: chrono::Utc::now() + Duration::seconds(i), + duration: Duration::seconds(10), + data: possible_data[i as usize % 20].clone(), + }; + event_list.push(e); + } + ds.insert_events(bucketname, &event_list).unwrap(); + } + + pub fn bench_assign(c: &mut Criterion) { + let ds = setup_datastore(); + let interval = TimeInterval::new_from_string(TIME_INTERVAL).unwrap(); + c.bench_function("bench assign", |b| { + b.iter(|| { + let code = String::from("return a=1;"); + match aw_query::query(&code, &interval, &ds).unwrap() { + aw_query::DataType::None() => (), + ref data => panic!("Wrong datatype, {data:?}"), + }; + }) + }); + } + + pub fn bench_many_events(c: &mut Criterion) { + let ds = setup_datastore(); + create_bucket(&ds, BUCKETNAME.to_string()); + insert_events(&ds, BUCKETNAME, 5000); + + let interval = TimeInterval::new_from_string(TIME_INTERVAL).unwrap(); + c.bench_function("bench many events", |b| { + b.iter(|| { + let code = String::from( + " + events = query_bucket(\"testbucket\"); + return events; + ", + ); + aw_query::query(&code, &interval, &ds).unwrap(); + }) + }); + } + + pub fn bench_read_only_references(c: &mut Criterion) { + let ds = setup_datastore(); + create_bucket(&ds, BUCKETNAME.to_string()); + insert_events(&ds, BUCKETNAME, 50_000); + let interval = TimeInterval::new_from_string(TIME_INTERVAL).unwrap(); + let code = format!( + "events = query_bucket(\"testbucket\"); total = 0; {} return total;", + "total = total + sum_durations(events);".repeat(20) + ); + c.bench_function("read-only references 50000 events", |b| { + b.iter(|| aw_query::query(&code, &interval, &ds).unwrap()) + }); + } +} + +criterion_group!( + benches, + query_benchmarks::bench_assign, + query_benchmarks::bench_many_events, + query_benchmarks::bench_read_only_references +); +criterion_main!(benches); diff --git a/aw-query/src/ast.rs b/aw-query/src/ast.rs new file mode 100644 index 0000000..0a45d78 --- /dev/null +++ b/aw-query/src/ast.rs @@ -0,0 +1,38 @@ +use crate::lexer::Span; + +use std::collections::HashMap; + +#[derive(Debug)] +pub struct Program { + pub stmts: Vec, +} + +#[derive(Debug, Clone)] +pub struct Expr { + #[allow(dead_code)] + pub span: Span, + pub node: Expr_, +} + +#[derive(Debug, Clone)] +pub enum Expr_ { + Add(Box, Box), + Sub(Box, Box), + Mul(Box, Box), + Div(Box, Box), + Mod(Box, Box), + + Equal(Box, Box), + + Var(String), + Assign(String, Box), + Function(String, Box), + If(Vec<(Box, Vec)>), + Return(Box), + + Bool(bool), + Number(f64), + String(String), + List(Vec), + Dict(HashMap), +} diff --git a/aw-query/src/datatype.rs b/aw-query/src/datatype.rs new file mode 100644 index 0000000..85a0e8f --- /dev/null +++ b/aw-query/src/datatype.rs @@ -0,0 +1,479 @@ +use std::collections::HashMap; +use std::fmt; + +use super::functions; +use super::QueryError; +use aw_models::Event; +use aw_transform::classify::{CategoryRule, RegexRule, Rule}; + +use serde::{Serialize, Serializer}; +use serde_json::value::Value; +use serde_json::Number; + +// TODO: greater/less comparisons + +#[derive(Clone, Serialize)] +#[serde(untagged)] +#[non_exhaustive] +pub enum DataType { + None(), + Bool(bool), + Number(f64), + String(String), + Event(Event), + List(Vec), + Dict(HashMap), + #[serde(serialize_with = "serialize_function")] + Function(String, functions::QueryFn), + #[serde(serialize_with = "serialize_function")] + ReadOnlyFunction(String, functions::ReadOnlyQueryFn), +} + +#[allow(clippy::trivially_copy_pass_by_ref)] +fn serialize_function(_element: &str, _fun: &F, _serializer: S) -> Result +where + S: Serializer, +{ + panic!("Query function was unevaluated and was attempted to be serialized, panic!"); + //element.id.serialize(serializer) +} + +// Needed because of a limitation in rust where you cannot derive(Debug) on a +// enum which has a fn with reference parameters which our QueryFn has +// https://stackoverflow.com/questions/53380040/function-pointer-with-a-reference-argument-cannot-derive-debug +impl fmt::Debug for DataType { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + DataType::None() => write!(f, "None()"), + DataType::Bool(b) => write!(f, "Bool({b})"), + DataType::Number(n) => write!(f, "Number({n})"), + DataType::String(s) => write!(f, "String({s})"), + DataType::Event(e) => write!(f, "Event({e:?})"), + DataType::List(l) => write!(f, "List({l:?})"), + DataType::Dict(d) => write!(f, "Dict({d:?})"), + DataType::Function(name, _fun) => write!(f, "Function({name})"), + DataType::ReadOnlyFunction(name, _fun) => write!(f, "Function({name})"), + } + } +} + +/* Like eq, but raises an error when comparing between different types. + * Should be used as often as possible */ +impl DataType { + pub fn query_eq(&self, other: &DataType) -> Result { + match (self, other) { + // TODO: Comparisons of bool == num, bool == str + // None is equal to itself, matching the PartialEq impl below. + (DataType::None(), DataType::None()) => Ok(true), + (DataType::Bool(b1), DataType::Bool(b2)) => Ok(b1 == b2), + (DataType::Number(n1), DataType::Number(n2)) => Ok(n1 == n2), + (DataType::String(s1), DataType::String(s2)) => Ok(s1 == s2), + (DataType::Event(e1), DataType::Event(e2)) => Ok(e1 == e2), + (DataType::List(l1), DataType::List(l2)) => Ok(l1 == l2), + (DataType::Dict(d1), DataType::Dict(d2)) => Ok(d1 == d2), + // We do not care about comparing functions + _ => Err(QueryError::InvalidType(format!( + "Cannot compare values of different types {self:?} and {other:?}" + ))), + } + } +} + +/* Required for query_eq when comparing two dicts */ +impl PartialEq for DataType { + fn eq(&self, other: &DataType) -> bool { + match (self, other) { + (DataType::None(), DataType::None()) => true, + // TODO: Comparisons of bool == num, bool == str + (DataType::Bool(b1), DataType::Bool(b2)) => b1 == b2, + (DataType::Number(n1), DataType::Number(n2)) => n1 == n2, + (DataType::String(s1), DataType::String(s2)) => s1 == s2, + (DataType::Event(e1), DataType::Event(e2)) => e1 == e2, + (DataType::List(l1), DataType::List(l2)) => l1 == l2, + (DataType::Dict(d1), DataType::Dict(d2)) => d1 == d2, + // We do not care about comparing functions + _ => false, + } + } +} + +/* Conversions out of DataType. + * + * The by-value impls move the contained data out and are what query functions + * should use, since functions own their argument vector. The by-ref impls + * delegate via a clone and only exist for callers that cannot give up + * ownership — going through them deep-copies every contained event. */ + +impl TryFrom for Vec { + type Error = QueryError; + fn try_from(value: DataType) -> Result { + match value { + DataType::List(s) => Ok(s), + invalid_type => Err(QueryError::InvalidFunctionParameters(format!( + "Expected function parameter of type List, got {invalid_type:?}" + ))), + } + } +} + +impl TryFrom<&DataType> for Vec { + type Error = QueryError; + fn try_from(value: &DataType) -> Result { + value.clone().try_into() + } +} + +impl TryFrom for String { + type Error = QueryError; + fn try_from(value: DataType) -> Result { + match value { + DataType::String(s) => Ok(s), + invalid_type => Err(QueryError::InvalidFunctionParameters(format!( + "Expected function parameter of type String, list contains {invalid_type:?}" + ))), + } + } +} + +impl TryFrom<&DataType> for String { + type Error = QueryError; + fn try_from(value: &DataType) -> Result { + value.clone().try_into() + } +} + +impl TryFrom for Vec { + type Error = QueryError; + fn try_from(value: DataType) -> Result { + let tagged_strings: Vec = value.try_into()?; + let mut strings = Vec::with_capacity(tagged_strings.len()); + for string in tagged_strings { + strings.push(string.try_into()?); + } + Ok(strings) + } +} + +impl TryFrom<&DataType> for Vec { + type Error = QueryError; + fn try_from(value: &DataType) -> Result { + value.clone().try_into() + } +} + +impl TryFrom for Vec { + type Error = QueryError; + fn try_from(value: DataType) -> Result { + let tagged_events: Vec = value.try_into()?; + let mut events = Vec::with_capacity(tagged_events.len()); + for event in tagged_events { + match event { + DataType::Event(e) => events.push(e), + invalid_type => { + return Err(QueryError::InvalidFunctionParameters(format!( + "Expected function parameter of type List of Events, list contains {invalid_type:?}" + ))) + } + } + } + Ok(events) + } +} + +impl TryFrom<&DataType> for Vec { + type Error = QueryError; + fn try_from(value: &DataType) -> Result { + value.clone().try_into() + } +} + +impl TryFrom<&DataType> for Vec<(String, Rule)> { + type Error = QueryError; + fn try_from(value: &DataType) -> Result { + value.clone().try_into() + } +} + +impl TryFrom for Vec<(String, Rule)> { + type Error = QueryError; + fn try_from(value: DataType) -> Result { + let tagged_lists: Vec = value.try_into()?; + let mut lists: Vec<(String, Rule)> = Vec::with_capacity(tagged_lists.len()); + for list in tagged_lists { + match list { + DataType::List(ref l) => { + let tag: String = match l.first() { + Some(tag) => tag.try_into()?, + None => return Err(QueryError::InvalidFunctionParameters( + format!("Expected function parameter of type list of (tag, rule) tuples, list contains {l:?}"))) + }; + let rule: Rule = match l.get(1) { + Some(rule) => rule.try_into()?, + None => return Err(QueryError::InvalidFunctionParameters( + format!("Expected function parameter of type list of (tag, rule) tuples, list contains {l:?}"))) + }; + lists.push((tag, rule)); + } + ref invalid_type => { + return Err(QueryError::InvalidFunctionParameters(format!( + "Expected function parameter of type list of (tag, rule) tuples, got {invalid_type:?}" + ))) + } + } + } + Ok(lists) + } +} + +impl TryFrom<&DataType> for Vec { + type Error = QueryError; + fn try_from(value: &DataType) -> Result { + value.clone().try_into() + } +} + +fn parse_optional_priority(rule: &DataType) -> Result, QueryError> { + let obj = match rule { + DataType::Dict(dict) => dict, + _ => return Ok(None), + }; + let val = match obj.get("priority").or_else(|| obj.get("weight")) { + Some(v) => v, + None => return Ok(None), + }; + match val { + DataType::Number(n) => { + if !n.is_finite() { + return Err(QueryError::InvalidFunctionParameters( + "priority/weight must be a finite integer".to_string(), + )); + } + let as_int = *n as i64; + if (as_int as f64) != *n { + return Err(QueryError::InvalidFunctionParameters( + "priority/weight must be an integer".to_string(), + )); + } + Ok(Some(as_int)) + } + _ => Err(QueryError::InvalidFunctionParameters( + "priority/weight must be an integer".to_string(), + )), + } +} + +impl TryFrom for Vec { + type Error = QueryError; + fn try_from(value: DataType) -> Result { + let tagged_lists: Vec = value.try_into()?; + let mut lists: Vec = Vec::with_capacity(tagged_lists.len()); + for list in tagged_lists { + match list { + DataType::List(ref l) => { + let category: Vec = match l.first() { + Some(category) => category.try_into()?, + None => return Err(QueryError::InvalidFunctionParameters( + format!("Expected function parameter of type list of (category, rule) tuples, list contains {l:?}"))) + }; + let rule_data = match l.get(1) { + Some(rule) => rule, + None => return Err(QueryError::InvalidFunctionParameters( + format!("Expected function parameter of type list of (category, rule) tuples, list contains {l:?}"))) + }; + let priority = parse_optional_priority(rule_data)?; + let rule: Rule = rule_data.try_into()?; + lists.push(CategoryRule { + category, + rule, + priority, + }); + } + ref invalid_type => { + return Err(QueryError::InvalidFunctionParameters(format!( + "Expected function parameter of type list of (category, rule) tuples, got {invalid_type:?}" + ))) + } + } + } + Ok(lists) + } +} + +impl TryFrom<&DataType> for f64 { + type Error = QueryError; + fn try_from(value: &DataType) -> Result { + match value { + DataType::Number(f) => Ok(*f), + ref invalid_type => Err(QueryError::InvalidFunctionParameters(format!( + "Expected function parameter of type Number, got {invalid_type:?}" + ))), + } + } +} + +impl TryFrom for f64 { + type Error = QueryError; + fn try_from(value: DataType) -> Result { + (&value).try_into() + } +} + +impl TryFrom<&DataType> for usize { + type Error = QueryError; + fn try_from(value: &DataType) -> Result { + let f: f64 = value.try_into()?; + Ok(f as usize) + } +} + +impl TryFrom for usize { + type Error = QueryError; + fn try_from(value: DataType) -> Result { + (&value).try_into() + } +} + +impl TryFrom for Value { + type Error = QueryError; + fn try_from(value: DataType) -> Result { + match value { + DataType::None() => Ok(Value::Null), + DataType::Bool(b) => Ok(Value::Bool(b)), + DataType::Number(n) => Ok(Value::Number(Number::from_f64(n).unwrap())), + DataType::String(s) => Ok(Value::String(s)), + DataType::List(l) => { + let mut values: Vec = Vec::with_capacity(l.len()); + for value in l { + values.push(value.try_into()?); + } + Ok(Value::Array(values)) + } + invalid_type => Err(QueryError::InvalidFunctionParameters(format!( + "Query2 support for parsing values is limited, does not support parsing {invalid_type:?}" + ))), + } + } +} + +impl TryFrom<&DataType> for Value { + type Error = QueryError; + fn try_from(value: &DataType) -> Result { + value.clone().try_into() + } +} + +impl TryFrom for Vec { + type Error = QueryError; + fn try_from(value: DataType) -> Result { + let tagged_values: Vec = value.try_into()?; + let mut values: Vec = Vec::with_capacity(tagged_values.len()); + for value in tagged_values { + values.push(value.try_into()?); + } + Ok(values) + } +} + +impl TryFrom<&DataType> for Vec { + type Error = QueryError; + fn try_from(value: &DataType) -> Result { + value.clone().try_into() + } +} + +impl TryFrom<&DataType> for Rule { + type Error = QueryError; + + fn try_from(data: &DataType) -> Result { + let obj = match data { + DataType::Dict(dict) => dict, + _ => { + return Err(QueryError::InvalidFunctionParameters(format!( + "Expected rule dict, got {data:?}" + ))) + } + }; + let rtype_val = match obj.get("type") { + Some(rtype) => rtype, + None => { + return Err(QueryError::InvalidFunctionParameters( + "rule does not have a type".to_string(), + )) + } + }; + let rtype = match rtype_val { + DataType::String(s) => s, + _ => { + return Err(QueryError::InvalidFunctionParameters( + "rule type is not a string".to_string(), + )) + } + }; + if rtype == "none" { + Ok(Self::None) + } else if rtype == "regex" { + let regex_val = match obj.get("regex") { + Some(regex_val) => regex_val, + None => { + return Err(QueryError::InvalidFunctionParameters( + "regex rule is missing the 'regex' field".to_string(), + )) + } + }; + let regex_str = match regex_val { + DataType::String(s) => s, + _ => { + return Err(QueryError::InvalidFunctionParameters( + "the regex field of the regex rule is not a string".to_string(), + )) + } + }; + let ignore_case_val = match obj.get("ignore_case") { + Some(case_val) => case_val, + None => &DataType::Bool(false), + }; + let ignore_case = match ignore_case_val { + DataType::Bool(b) => b, + _ => { + return Err(QueryError::InvalidFunctionParameters( + "the ignore_case field of the regex rule is not a bool".to_string(), + )) + } + }; + let select_keys = match obj.get("select_keys") { + Some(DataType::List(keys)) => { + let mut select_keys = Vec::with_capacity(keys.len()); + for key in keys { + match key { + DataType::String(key) => select_keys.push(key.clone()), + _ => return Err(QueryError::InvalidFunctionParameters( + "the select_keys field of the regex rule must contain only strings" + .to_string(), + )), + } + } + Some(select_keys) + } + Some(_) => { + return Err(QueryError::InvalidFunctionParameters( + "the select_keys field of the regex rule is not a list".to_string(), + )) + } + None => None, + }; + let regex_rule = match RegexRule::new(regex_str, *ignore_case, select_keys) { + Ok(regex_rule) => regex_rule, + Err(err) => { + return Err(QueryError::RegexCompileError(format!( + "Failed to compile regex string '{regex_str}': '{err:?}" + ))) + } + }; + Ok(Self::Regex(regex_rule)) + } else { + Err(QueryError::InvalidFunctionParameters(format!( + "Unknown rule type '{rtype}'" + ))) + } + } +} diff --git a/aw-query/src/functions.rs b/aw-query/src/functions.rs new file mode 100644 index 0000000..2460550 --- /dev/null +++ b/aw-query/src/functions.rs @@ -0,0 +1,651 @@ +use crate::DataType; +use crate::QueryError; +use crate::VarEnv; +use aw_datastore::Datastore; + +pub type QueryFn = + fn(args: Vec, env: &VarEnv, ds: &Datastore) -> Result; + +pub type ReadOnlyQueryFn = + fn(args: &[&DataType], env: &VarEnv, ds: &Datastore) -> Result; + +pub fn fill_env(env: &mut VarEnv) { + env.insert( + "print".to_string(), + DataType::ReadOnlyFunction("print".to_string(), qfunctions::print), + ); + env.insert( + "query_bucket".to_string(), + DataType::Function("query_bucket".to_string(), qfunctions::query_bucket), + ); + env.insert( + "query_bucket_names".to_string(), + DataType::Function( + "query_bucket_names".to_string(), + qfunctions::query_bucket_names, + ), + ); + env.insert( + "sort_by_duration".to_string(), + DataType::Function("sort_by_duration".to_string(), qfunctions::sort_by_duration), + ); + env.insert( + "sort_by_timestamp".to_string(), + DataType::Function( + "sort_by_timestamp".to_string(), + qfunctions::sort_by_timestamp, + ), + ); + env.insert( + "sum_durations".to_string(), + DataType::ReadOnlyFunction("sum_durations".to_string(), qfunctions::sum_durations), + ); + env.insert( + "limit_events".to_string(), + DataType::Function("limit_events".to_string(), qfunctions::limit_events), + ); + env.insert( + "contains".to_string(), + DataType::ReadOnlyFunction("contains".to_string(), qfunctions::contains), + ); + env.insert( + "flood".to_string(), + DataType::Function("flood".to_string(), qfunctions::flood), + ); + env.insert( + "find_bucket".to_string(), + DataType::Function("find_bucket".to_string(), qfunctions::find_bucket), + ); + env.insert( + "merge_events_by_keys".to_string(), + DataType::Function( + "merge_events_by_keys".to_string(), + qfunctions::merge_events_by_keys, + ), + ); + env.insert( + "chunk_events_by_key".to_string(), + DataType::Function( + "chunk_events_by_key".to_string(), + qfunctions::chunk_events_by_key, + ), + ); + env.insert( + "exclude_keyvals".to_string(), + DataType::Function("exclude_keyvals".to_string(), qfunctions::exclude_keyvals), + ); + env.insert( + "filter_keyvals".to_string(), + DataType::Function("filter_keyvals".to_string(), qfunctions::filter_keyvals), + ); + env.insert( + "filter_keyvals_regex".to_string(), + DataType::Function( + "filter_keyvals_regex".to_string(), + qfunctions::filter_keyvals_regex, + ), + ); + env.insert( + "filter_period_intersect".to_string(), + DataType::Function( + "filter_period_intersect".to_string(), + qfunctions::filter_period_intersect, + ), + ); + env.insert( + "split_url_events".to_string(), + DataType::Function("split_url_events".to_string(), qfunctions::split_url_events), + ); + env.insert( + "concat".to_string(), + DataType::Function("concat".to_string(), qfunctions::concat), + ); + env.insert( + "categorize".to_string(), + DataType::Function("categorize".into(), qfunctions::categorize), + ); + env.insert( + "tag".to_string(), + DataType::Function("tag".into(), qfunctions::tag), + ); + env.insert( + "period_union".to_string(), + DataType::Function("period_union".into(), qfunctions::period_union), + ); + env.insert( + "union_no_overlap".to_string(), + DataType::Function("union_no_overlap".into(), qfunctions::union_no_overlap), + ); +} + +mod qfunctions { + use aw_datastore::Datastore; + use aw_models::Event; + use aw_transform::classify::{CategoryRule, Rule}; + + use super::validate; + use crate::DataType; + use crate::QueryError; + use crate::VarEnv; + + pub fn print( + args: &[&DataType], + _env: &VarEnv, + _ds: &Datastore, + ) -> Result { + for arg in args { + info!("{:?}", arg); + } + Ok(DataType::None()) + } + + pub fn query_bucket( + args: Vec, + env: &VarEnv, + ds: &Datastore, + ) -> Result { + // Typecheck + validate::args_length(&args, 1)?; + + let bucket_id: String = args.into_iter().next().unwrap().try_into()?; + let interval = validate::get_timeinterval(env)?; + + let events = match ds.get_events( + bucket_id.as_str(), + Some(*interval.start()), + Some(*interval.end()), + None, + ) { + Ok(events) => events, + Err(aw_datastore::DatastoreError::NoSuchBucket(_)) => { + return Err(QueryError::BucketNotFound(format!( + "Failed to find bucket '{bucket_id}'" + ))) + } + Err(e) => { + return Err(QueryError::BucketQueryError(format!( + "Failed to query bucket: {e:?}" + ))) + } + }; + let mut ret = Vec::new(); + for event in events { + ret.push(DataType::Event(event)); + } + Ok(DataType::List(ret)) + } + + pub fn query_bucket_names( + args: Vec, + _env: &VarEnv, + ds: &Datastore, + ) -> Result { + validate::args_length(&args, 0)?; + let mut bucketnames: Vec = Vec::new(); + let buckets = match ds.get_buckets() { + Ok(buckets) => buckets, + Err(e) => { + return Err(QueryError::BucketQueryError(format!( + "Failed to query bucket names: {e:?}" + ))) + } + }; + for bucketname in buckets.keys() { + bucketnames.push(DataType::String(bucketname.to_string())); + } + Ok(DataType::List(bucketnames)) + } + + pub fn find_bucket( + args: Vec, + _env: &VarEnv, + ds: &Datastore, + ) -> Result { + validate::args_length(&args, 1).or_else(|_| validate::args_length(&args, 2))?; + + let mut args = args.into_iter(); + let bucket_filter: String = args.next().unwrap().try_into()?; + let hostname_filter: Option = match args.next() { + Some(arg) => Some(arg.try_into()?), + None => None, + }; + + let buckets = match ds.get_buckets() { + Ok(buckets) => buckets, + Err(e) => { + return Err(QueryError::BucketQueryError(format!( + "Failed to query bucket names: {e:?}" + ))) + } + }; + let bucketname = match aw_transform::find_bucket( + &bucket_filter, + &hostname_filter, + buckets.values(), + ) { + Some(bucketname) => bucketname, + None => { + return Err(QueryError::BucketNotFound(match hostname_filter { + None => { + format!("Failed to find bucket matching filter '{bucket_filter}'") + } + Some(hostname_filter) => format!( + "Failed to find bucket matching filter '{bucket_filter}' and hostname '{hostname_filter}'" + ), + })); + } + }; + Ok(DataType::String(bucketname)) + } + + pub fn contains( + args: &[&DataType], + _env: &VarEnv, + _ds: &Datastore, + ) -> Result { + // typecheck + validate::args_length(args, 2)?; + match args[0] { + DataType::List(list) => Ok(DataType::Bool(list.contains(args[1]))), + DataType::Dict(ref dict) => { + let s = match &args[1] { + DataType::String(s) => s.as_str(), + _ => { + return Err(QueryError::InvalidFunctionParameters(format!( + "function contains got second argument {:?}, expected type String", + args[0] + ))) + } + }; + Ok(DataType::Bool(dict.contains_key(s))) + } + _ => Err(QueryError::InvalidFunctionParameters(format!( + "function contains got first argument {:?}, expected type List or Dict", + args[0] + ))), + } + } + + pub fn flood( + args: Vec, + _env: &VarEnv, + _ds: &Datastore, + ) -> Result { + // typecheck + validate::args_length(&args, 1)?; + let events: Vec = args.into_iter().next().unwrap().try_into()?; + // Run flood + let mut flooded_events = aw_transform::flood(events, chrono::Duration::seconds(5)); + // Put events back into DataType::Event container + let mut tagged_flooded_events = Vec::new(); + for event in flooded_events.drain(..) { + tagged_flooded_events.push(DataType::Event(event)); + } + Ok(DataType::List(tagged_flooded_events)) + } + + pub fn categorize( + args: Vec, + _env: &VarEnv, + _ds: &Datastore, + ) -> Result { + // typecheck + validate::args_length(&args, 2)?; + let mut args = args.into_iter(); + let events: Vec = args.next().unwrap().try_into()?; + let rules: Vec = args.next().unwrap().try_into()?; + // Run categorize + let mut flooded_events = aw_transform::classify::categorize(events, &rules); + // Put events back into DataType::Event container + let mut tagged_flooded_events = Vec::new(); + for event in flooded_events.drain(..) { + tagged_flooded_events.push(DataType::Event(event)); + } + Ok(DataType::List(tagged_flooded_events)) + } + + pub fn tag( + args: Vec, + _env: &VarEnv, + _ds: &Datastore, + ) -> Result { + // typecheck + validate::args_length(&args, 2)?; + let mut args = args.into_iter(); + let events: Vec = args.next().unwrap().try_into()?; + let rules: Vec<(String, Rule)> = args.next().unwrap().try_into()?; + // Run categorize + let mut flooded_events = aw_transform::classify::tag(events, &rules); + // Put events back into DataType::Event container + let mut tagged_flooded_events = Vec::new(); + for event in flooded_events.drain(..) { + tagged_flooded_events.push(DataType::Event(event)); + } + Ok(DataType::List(tagged_flooded_events)) + } + + pub fn sort_by_duration( + args: Vec, + _env: &VarEnv, + _ds: &Datastore, + ) -> Result { + // typecheck + validate::args_length(&args, 1)?; + let events: Vec = args.into_iter().next().unwrap().try_into()?; + + // Sort by duration + let mut sorted_events = aw_transform::sort_by_duration(events); + // Put events back into DataType::Event container + let mut tagged_sorted_events = Vec::new(); + for event in sorted_events.drain(..) { + tagged_sorted_events.push(DataType::Event(event)); + } + Ok(DataType::List(tagged_sorted_events)) + } + + pub fn limit_events( + args: Vec, + _env: &VarEnv, + _ds: &Datastore, + ) -> Result { + // typecheck + validate::args_length(&args, 2)?; + let mut args = args.into_iter(); + let mut events: Vec = args.next().unwrap().try_into()?; + let mut limit: usize = args.next().unwrap().try_into()?; + + if events.len() < limit { + limit = events.len() + } + let mut limited_tagged_events = Vec::new(); + for event in events.drain(0..limit) { + limited_tagged_events.push(DataType::Event(event)); + } + Ok(DataType::List(limited_tagged_events)) + } + + pub fn sort_by_timestamp( + args: Vec, + _env: &VarEnv, + _ds: &Datastore, + ) -> Result { + // typecheck + validate::args_length(&args, 1)?; + let events: Vec = args.into_iter().next().unwrap().try_into()?; + + // Sort by duration + let mut sorted_events = aw_transform::sort_by_timestamp(events); + // Put events back into DataType::Event container + let mut tagged_sorted_events = Vec::new(); + for event in sorted_events.drain(..) { + tagged_sorted_events.push(DataType::Event(event)); + } + Ok(DataType::List(tagged_sorted_events)) + } + + pub fn sum_durations( + args: &[&DataType], + _env: &VarEnv, + _ds: &Datastore, + ) -> Result { + // typecheck + validate::args_length(args, 1)?; + let events = match args[0] { + DataType::List(events) => events, + invalid_type => { + return Err(QueryError::InvalidFunctionParameters(format!( + "Expected function parameter of type List, got {invalid_type:?}" + ))) + } + }; + let mut sum_durations = chrono::Duration::zero(); + for event in events { + match event { + DataType::Event(event) => sum_durations += event.duration, + invalid_type => return Err(QueryError::InvalidFunctionParameters(format!( + "Expected function parameter of type List of Events, list contains {invalid_type:?}" + ))), + } + } + Ok(DataType::Number( + (sum_durations.num_milliseconds() as f64) / 1000.0, + )) + } + + pub fn merge_events_by_keys( + args: Vec, + _env: &VarEnv, + _ds: &Datastore, + ) -> Result { + // typecheck + validate::args_length(&args, 2)?; + let mut args = args.into_iter(); + let events: Vec = args.next().unwrap().try_into()?; + let keys: Vec = args.next().unwrap().try_into()?; + + let mut merged_events = aw_transform::merge_events_by_keys(events, keys); + let mut merged_tagged_events = Vec::new(); + for event in merged_events.drain(..) { + merged_tagged_events.push(DataType::Event(event)); + } + Ok(DataType::List(merged_tagged_events)) + } + + pub fn chunk_events_by_key( + args: Vec, + _env: &VarEnv, + _ds: &Datastore, + ) -> Result { + // typecheck + validate::args_length(&args, 2)?; + let mut args = args.into_iter(); + let events: Vec = args.next().unwrap().try_into()?; + let key: String = args.next().unwrap().try_into()?; + + let mut merged_events = aw_transform::chunk_events_by_key(events, &key); + let mut merged_tagged_events = Vec::new(); + for event in merged_events.drain(..) { + merged_tagged_events.push(DataType::Event(event)); + } + Ok(DataType::List(merged_tagged_events)) + } + + pub fn filter_keyvals( + args: Vec, + _env: &VarEnv, + _ds: &Datastore, + ) -> Result { + // typecheck + validate::args_length(&args, 3)?; + let mut args = args.into_iter(); + let events = args.next().unwrap().try_into()?; + let key: String = args.next().unwrap().try_into()?; + let vals: Vec<_> = args.next().unwrap().try_into()?; + + let mut filtered_events = aw_transform::filter_keyvals(events, &key, &vals); + let mut filtered_tagged_events = Vec::new(); + for event in filtered_events.drain(..) { + filtered_tagged_events.push(DataType::Event(event)); + } + Ok(DataType::List(filtered_tagged_events)) + } + + use fancy_regex::RegexBuilder; + + pub fn filter_keyvals_regex( + args: Vec, + _env: &VarEnv, + _ds: &Datastore, + ) -> Result { + // typecheck + validate::args_length(&args, 3)?; + let mut args = args.into_iter(); + let events = args.next().unwrap().try_into()?; + let key: String = args.next().unwrap().try_into()?; + let regex_str: String = args.next().unwrap().try_into()?; + let regex = match RegexBuilder::new(®ex_str).build() { + Ok(regex) => regex, + Err(e) => { + return Err(QueryError::RegexCompileError(format!( + "Failed to compile regex string '{regex_str}': {e}" + ))) + } + }; + + let mut filtered_events = aw_transform::filter_keyvals_regex(events, &key, ®ex); + let mut filtered_tagged_events = Vec::new(); + for event in filtered_events.drain(..) { + filtered_tagged_events.push(DataType::Event(event)); + } + Ok(DataType::List(filtered_tagged_events)) + } + + pub fn exclude_keyvals( + args: Vec, + _env: &VarEnv, + _ds: &Datastore, + ) -> Result { + // typecheck + validate::args_length(&args, 3)?; + let mut args = args.into_iter(); + let events = args.next().unwrap().try_into()?; + let key: String = args.next().unwrap().try_into()?; + let vals: Vec<_> = args.next().unwrap().try_into()?; + + let mut filtered_events = aw_transform::exclude_keyvals(events, &key, &vals); + let mut filtered_tagged_events = Vec::new(); + for event in filtered_events.drain(..) { + filtered_tagged_events.push(DataType::Event(event)); + } + Ok(DataType::List(filtered_tagged_events)) + } + + pub fn filter_period_intersect( + args: Vec, + _env: &VarEnv, + _ds: &Datastore, + ) -> Result { + // typecheck + validate::args_length(&args, 2)?; + let mut args = args.into_iter(); + let events: Vec = args.next().unwrap().try_into()?; + let filter_events: Vec = args.next().unwrap().try_into()?; + + let mut filtered_events = aw_transform::filter_period_intersect(events, filter_events); + let mut filtered_tagged_events = Vec::new(); + for event in filtered_events.drain(..) { + filtered_tagged_events.push(DataType::Event(event)); + } + Ok(DataType::List(filtered_tagged_events)) + } + + pub fn split_url_events( + args: Vec, + _env: &VarEnv, + _ds: &Datastore, + ) -> Result { + // typecheck + validate::args_length(&args, 1)?; + let mut events: Vec = args.into_iter().next().unwrap().try_into()?; + + let mut tagged_split_url_events = Vec::new(); + for mut event in events.drain(..) { + aw_transform::split_url_event(&mut event); + tagged_split_url_events.push(DataType::Event(event)); + } + Ok(DataType::List(tagged_split_url_events)) + } + + pub fn concat( + args: Vec, + _env: &VarEnv, + _ds: &Datastore, + ) -> Result { + let mut event_list = Vec::new(); + for arg in args { + let mut events: Vec = arg.try_into()?; + for event in events.drain(..) { + event_list.push(DataType::Event(event)); + } + } + Ok(DataType::List(event_list)) + } + + pub fn period_union( + args: Vec, + _env: &VarEnv, + _ds: &Datastore, + ) -> Result { + // typecheck + validate::args_length(&args, 2)?; + let mut args = args.into_iter(); + let events1: Vec = args.next().unwrap().try_into()?; + let events2: Vec = args.next().unwrap().try_into()?; + + let mut result = aw_transform::period_union(&events1, &events2); + let mut result_tagged = Vec::new(); + for event in result.drain(..) { + result_tagged.push(DataType::Event(event)); + } + Ok(DataType::List(result_tagged)) + } + + pub fn union_no_overlap( + args: Vec, + _env: &VarEnv, + _ds: &Datastore, + ) -> Result { + // typecheck + validate::args_length(&args, 2)?; + let mut args = args.into_iter(); + let events1: Vec = args.next().unwrap().try_into()?; + let events2: Vec = args.next().unwrap().try_into()?; + + let mut result = aw_transform::union_no_overlap(events1, events2); + let mut result_tagged = Vec::new(); + for event in result.drain(..) { + result_tagged.push(DataType::Event(event)); + } + Ok(DataType::List(result_tagged)) + } +} + +mod validate { + use crate::{DataType, QueryError, VarEnv}; + use aw_models::TimeInterval; + + pub fn args_length(args: &[T], len: usize) -> Result<(), QueryError> { + if args.len() != len { + return Err(QueryError::InvalidFunctionParameters(format!( + "Expected {} parameters in function, got {}", + len, + args.len() + ))); + } + Ok(()) + } + + pub fn get_timeinterval(env: &VarEnv) -> Result { + let interval_str = match env.get("TIMEINTERVAL") { + Some(data_ti) => match data_ti { + DataType::String(ti_str) => ti_str, + _ => { + return Err(QueryError::TimeIntervalError( + "TIMEINTERVAL is not of type string!".to_string(), + )) + } + }, + None => { + return Err(QueryError::TimeIntervalError( + "TIMEINTERVAL not defined!".to_string(), + )) + } + }; + match TimeInterval::new_from_string(interval_str) { + Ok(ti) => Ok(ti), + Err(_e) => Err(QueryError::TimeIntervalError(format!( + "Failed to parse TIMEINTERVAL: {interval_str}" + ))), + } + } +} diff --git a/aw-query/src/grammar.rs b/aw-query/src/grammar.rs new file mode 100644 index 0000000..e69de29 diff --git a/aw-query/src/interpret.rs b/aw-query/src/interpret.rs new file mode 100644 index 0000000..7867753 --- /dev/null +++ b/aw-query/src/interpret.rs @@ -0,0 +1,403 @@ +use std::{borrow::Cow, collections::HashMap}; + +use crate::functions; + +use aw_datastore::Datastore; +use aw_models::TimeInterval; + +use crate::ast::*; +use crate::DataType; +use crate::QueryError; + +pub type VarEnv = HashMap; + +fn init_env(ti: &TimeInterval) -> VarEnv { + let mut env = HashMap::new(); + env.insert("TIMEINTERVAL".to_string(), DataType::String(ti.to_string())); + functions::fill_env(&mut env); + env +} + +pub fn interpret_prog( + p: Program, + ti: &TimeInterval, + ds: &Datastore, +) -> Result { + let mut env = init_env(ti); + for expr in p.stmts { + interpret_expr(&mut env, ds, expr)?; + } + match env.remove("RETURN") { + Some(ret) => Ok(ret), + None => Err(QueryError::EmptyQuery()), + } +} + +fn interpret_expr( + env: &mut HashMap, + ds: &Datastore, + expr: Expr, +) -> Result { + use crate::ast::Expr_::*; + match expr.node { + Add(a, b) => { + let a_res = interpret_expr(env, ds, *a)?; + let b_res = interpret_expr(env, ds, *b)?; + let res = match a_res { + DataType::Number(n1) => match b_res { + DataType::Number(n2) => DataType::Number(n1 + n2), + _ => { + return Err(QueryError::InvalidType( + "Cannot use + on something that is not a number with a number!" + .to_string(), + )) + } + }, + DataType::List(mut l1) => match b_res { + DataType::List(mut l2) => { + l1.append(&mut l2); + DataType::List(l1) + } + _ => { + return Err(QueryError::InvalidType( + "Cannot use + on something that is not a list with a list!".to_string(), + )) + } + }, + DataType::String(s1) => match b_res { + DataType::String(s2) => { + let mut new_string = s1; + new_string.push_str(&s2); + DataType::String(new_string) + } + _ => { + return Err(QueryError::InvalidType( + "Cannot use + on something that is not a list with a list!".to_string(), + )) + } + }, + _ => { + return Err(QueryError::InvalidType( + "Cannot use + on something that is not a number, list or string!" + .to_string(), + )) + } + }; + Ok(res) + } + Sub(a, b) => { + let a_res = interpret_expr(env, ds, *a)?; + let b_res = interpret_expr(env, ds, *b)?; + let a_num = match a_res { + DataType::Number(n) => n, + _ => { + return Err(QueryError::InvalidType( + "Cannot sub something that is not a number!".to_string(), + )) + } + }; + let b_num = match b_res { + DataType::Number(n) => n, + _ => { + return Err(QueryError::InvalidType( + "Cannot sub something that is not a number!".to_string(), + )) + } + }; + Ok(DataType::Number(a_num - b_num)) + } + Mul(a, b) => { + let a_res = interpret_expr(env, ds, *a)?; + let b_res = interpret_expr(env, ds, *b)?; + let a_num = match a_res { + DataType::Number(n) => n, + _ => { + return Err(QueryError::InvalidType( + "Cannot sub something that is not a number!".to_string(), + )) + } + }; + let b_num = match b_res { + DataType::Number(n) => n, + _ => { + return Err(QueryError::InvalidType( + "Cannot sub something that is not a number!".to_string(), + )) + } + }; + Ok(DataType::Number(a_num * b_num)) + } + Div(a, b) => { + let a_res = interpret_expr(env, ds, *a)?; + let b_res = interpret_expr(env, ds, *b)?; + let a_num = match a_res { + DataType::Number(n) => n, + _ => { + return Err(QueryError::InvalidType( + "Cannot sub something that is not a number!".to_string(), + )) + } + }; + let b_num = match b_res { + DataType::Number(n) => n, + _ => { + return Err(QueryError::InvalidType( + "Cannot sub something that is not a number!".to_string(), + )) + } + }; + if b_num == 0.0 { + return Err(QueryError::MathError( + "Tried to divide by zero!".to_string(), + )); + } + Ok(DataType::Number(a_num / b_num)) + } + Mod(a, b) => { + let a_res = interpret_expr(env, ds, *a)?; + let b_res = interpret_expr(env, ds, *b)?; + let a_num = match a_res { + DataType::Number(n) => n, + _ => { + return Err(QueryError::InvalidType( + "Cannot sub something that is not a number!".to_string(), + )) + } + }; + let b_num = match b_res { + DataType::Number(n) => n, + _ => { + return Err(QueryError::InvalidType( + "Cannot sub something that is not a number!".to_string(), + )) + } + }; + Ok(DataType::Number(a_num % b_num)) + } + Equal(lhs, rhs) => { + let lhs_res = interpret_expr(env, ds, *lhs)?; + let rhs_res = interpret_expr(env, ds, *rhs)?; + Ok(DataType::Bool(lhs_res.query_eq(&rhs_res)?)) + } + Assign(var, b) => { + let val = interpret_expr(env, ds, *b)?; + env.insert(var, val); + Ok(DataType::None()) + } + // This clone is the one necessary copy per variable reference: the env + // must retain the value since it may be referenced again, while + // function arguments and transforms need owned events. + Var(var) => match env.get(&var) { + Some(v) => Ok(v.clone()), + None => Err(QueryError::VariableNotDefined(var.to_string())), + }, + Bool(lit) => Ok(DataType::Bool(lit)), + Number(lit) => Ok(DataType::Number(lit)), + String(litstr) => Ok(DataType::String(litstr)), + Return(e) => { + let val = interpret_expr(env, ds, *e)?; + // TODO: Once RETURN is deprecated we can fix this + env.insert("RETURN".to_string(), val); + Ok(DataType::None()) + } + If(ifs) => { + for (cond, block) in ifs { + let c = interpret_expr(env, ds, *cond)?; + if c.query_eq(&DataType::Bool(true))? { + for expr in block { + interpret_expr(env, ds, expr)?; + } + break; + } + } + Ok(DataType::None()) + } + Function(fname, e) => { + // Borrow variables only when every argument is an atom. Evaluating + // arbitrary arguments may assign to the environment, including the + // function binding itself, so those retain the owned evaluation path. + if let Expr_::List(exprs) = &e.node { + if exprs.iter().all(|expr| { + matches!( + expr.node, + Expr_::Var(_) | Expr_::Bool(_) | Expr_::Number(_) | Expr_::String(_) + ) + }) { + if let Some(DataType::ReadOnlyFunction(_, fun)) = env.get(&fname) { + let args: Result>, QueryError> = exprs + .iter() + .map(|expr| { + Ok(match &expr.node { + Expr_::Var(name) => { + Cow::Borrowed(env.get(name).ok_or_else(|| { + QueryError::VariableNotDefined(name.clone()) + })?) + } + Expr_::Bool(value) => Cow::Owned(DataType::Bool(*value)), + Expr_::Number(value) => Cow::Owned(DataType::Number(*value)), + Expr_::String(value) => { + Cow::Owned(DataType::String(value.clone())) + } + _ => unreachable!(), + }) + }) + .collect(); + let args = args?; + let refs: Vec<_> = args.iter().map(|arg| arg.as_ref()).collect(); + return fun(&refs, env, ds); + } + } + } + let args = match interpret_expr(env, ds, *e)? { + DataType::List(l) => l, + _ => unreachable!(), + }; + let var = match env.get(&fname[..]) { + Some(v) => v, + None => return Err(QueryError::VariableNotDefined(fname.clone())), + }; + match var { + DataType::Function(_, fun) => fun(args, env, ds), + DataType::ReadOnlyFunction(_, fun) => { + let refs: Vec<_> = args.iter().collect(); + fun(&refs, env, ds) + } + _ => Err(QueryError::InvalidType(fname.to_string())), + } + } + List(list) => { + let mut l = Vec::new(); + for entry in list { + let res = interpret_expr(env, ds, entry)?; + l.push(res); + } + Ok(DataType::List(l)) + } + Dict(d) => { + let mut dict = HashMap::new(); + for (key, val_uninterpreted) in d { + let val = interpret_expr(env, ds, val_uninterpreted)?; + dict.insert(key, val); + } + Ok(DataType::Dict(dict)) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn run(code: &str, env: &mut VarEnv, ds: &Datastore) -> DataType { + let program = crate::parser::parse(crate::lexer::Lexer::new(code)).unwrap(); + for expr in program.stmts { + interpret_expr(env, ds, expr).unwrap(); + } + env.remove("RETURN").unwrap_or(DataType::None()) + } + + #[test] + fn read_only_arguments_borrow_the_original_value_and_support_aliases() { + fn check(args: &[&DataType], env: &VarEnv, _: &Datastore) -> Result { + assert!(std::ptr::eq(args[0], env.get("events").unwrap())); + Ok(DataType::Bool(true)) + } + let ds = Datastore::new_in_memory(false); + let mut env = VarEnv::new(); + env.insert( + "check".into(), + DataType::ReadOnlyFunction("check".into(), check), + ); + env.insert( + "events".into(), + DataType::List(vec![DataType::Event(aw_models::Event::default())]), + ); + assert_eq!( + run("alias = check; return alias(events);", &mut env, &ds), + DataType::Bool(true) + ); + assert!(env.contains_key("events")); + } + + #[test] + fn side_effecting_arguments_keep_snapshots_and_resolve_the_function_after_evaluation() { + fn check(args: &[&DataType], env: &VarEnv, _: &Datastore) -> Result { + assert_eq!(args[0], &DataType::List(vec![DataType::Number(1.0)])); + assert_eq!(env["items"], DataType::List(vec![DataType::Number(2.0)])); + Ok(DataType::Bool(true)) + } + let ds = Datastore::new_in_memory(false); + let mut env = VarEnv::new(); + env.insert( + "check".into(), + DataType::ReadOnlyFunction("check".into(), check), + ); + // Assignment nodes are supported by the interpreter, although the + // current parser only emits them at statement level. + let expr = |node| Expr { + span: crate::lexer::Span { + lo: 0, + hi: 0, + line: 1, + }, + node, + }; + for rebind in [false, true] { + env.insert("items".into(), DataType::List(vec![DataType::Number(1.0)])); + env.insert( + "f".into(), + if rebind { + DataType::Number(0.0) + } else { + env["check"].clone() + }, + ); + let mut args = vec![ + expr(Expr_::Var("items".into())), + expr(Expr_::Assign( + "items".into(), + Box::new(expr(Expr_::List(vec![expr(Expr_::Number(2.0))]))), + )), + ]; + if rebind { + args.push(expr(Expr_::Assign( + "f".into(), + Box::new(expr(Expr_::Var("check".into()))), + ))); + } + let call = expr(Expr_::Function( + "f".into(), + Box::new(expr(Expr_::List(args))), + )); + assert_eq!( + interpret_expr(&mut env, &ds, call).unwrap(), + DataType::Bool(true) + ); + } + } + + #[test] + fn read_only_builtins_keep_values_available_and_accept_computed_arguments() { + let ds = Datastore::new_in_memory(false); + let mut env = VarEnv::new(); + functions::fill_env(&mut env); + let mut event = aw_models::Event::default(); + event.duration = chrono::Duration::seconds(3); + env.insert( + "events".into(), + DataType::List(vec![DataType::Event(event)]), + ); + assert_eq!(run("total = sum_durations; a = total(events); b = total(events + events); return a + b;", &mut env, &ds), DataType::Number(9.0)); + assert_eq!( + run( + "values = [1, 2]; f = contains; a = f(values, 2); return [a, values];", + &mut env, + &ds + ), + DataType::List(vec![ + DataType::Bool(true), + DataType::List(vec![DataType::Number(1.0), DataType::Number(2.0)]) + ]) + ); + } +} diff --git a/aw-query/src/lexer.rs b/aw-query/src/lexer.rs new file mode 100644 index 0000000..36d8cc9 --- /dev/null +++ b/aw-query/src/lexer.rs @@ -0,0 +1,144 @@ +use plex::lexer; + +#[derive(Debug, Clone)] +pub enum Token { + Ident(String), + + If, + ElseIf, + Else, + Return, + + Bool(bool), + Number(f64), + String(String), + Plus, + Minus, + Star, + Slash, + Percent, + Equals, + Assign, + LParen, + RParen, + LBracket, + RBracket, + LBrace, + RBrace, + Comma, + Colon, + Semi, + + Whitespace, + Newline, + Comment, +} + +lexer! { + fn next_token(text: 'a) -> (Token, &'a str); + + r#"[ \t\r]+"# => (Token::Whitespace, text), + r#"\n"# => (Token::Newline, text), + // Python-style comments (# ...) + r#"#[^\n]*"# => (Token::Comment, text), + + r#"if"# => (Token::If, text), + r#"elif"# => (Token::ElseIf, text), + r#"else"# => (Token::Else, text), + r#"return"# => (Token::Return, text), + + r#"true"# => (Token::Bool(true), text), + r#"false"# => (Token::Bool(false), text), + // TODO: Deprecate/Remove? + r#"True"# => (Token::Bool(true), text), + r#"False"# => (Token::Bool(false), text), + + r#"\"([^\"]|(\\\"))*\""# => ( + Token::String(text.to_owned()[1..text.len()-1].replace("\\\"", "\"")), + text + ), + r#"[0-9]+[\.]?[0-9]*"# => { + let tok = match text.parse() { + Ok(n) => Token::Number(n), + Err(e) => panic!("Integer {text} is out of range: {e}"), + }; + (tok, text) + } + + r#"[a-zA-Z_][a-zA-Z0-9_]*"# => (Token::Ident(text.to_owned()), text), + + r#"=="# => (Token::Equals, text), + r#"="# => (Token::Assign, text), + r#"\+"# => (Token::Plus, text), + r#"-"# => (Token::Minus, text), + r#"\*"# => (Token::Star, text), + r#"/"# => (Token::Slash, text), + r#"%"# => (Token::Percent, text), + r#"\("# => (Token::LParen, text), + r#"\)"# => (Token::RParen, text), + r#"\["# => (Token::LBracket, text), + r#"\]"# => (Token::RBracket, text), + r#"\{"# => (Token::LBrace, text), + r#"\}"# => (Token::RBrace, text), + r#","# => (Token::Comma, text), + r#":"# => (Token::Colon, text), + r#";"# => (Token::Semi, text), +} + +pub struct Lexer<'a> { + original: &'a str, + remaining: &'a str, + line: usize, +} + +impl<'a> Lexer<'a> { + pub fn new(s: &'a str) -> Lexer<'a> { + Lexer { + original: s, + remaining: s, + line: 1, + } + } +} + +#[derive(Debug, Clone, Copy)] +pub struct Span { + pub lo: usize, + pub hi: usize, + pub line: usize, +} + +fn span_in(s: &str, t: &str, l: usize) -> Span { + let lo = s.as_ptr() as usize - t.as_ptr() as usize; + Span { + lo, + hi: lo + s.len(), + line: l, + } +} + +impl<'a> Iterator for Lexer<'a> { + type Item = (Token, Span); + fn next(&mut self) -> Option<(Token, Span)> { + loop { + let tok = if let Some((tok, new_remaining)) = next_token(self.remaining) { + self.remaining = new_remaining; + tok + } else { + return None; + }; + match tok { + (Token::Whitespace, _) | (Token::Comment, _) => { + continue; + } + (Token::Newline, _) => { + self.line += 1; + continue; + } + (tok, span) => { + return Some((tok, span_in(span, self.original, self.line))); + } + } + } + } +} diff --git a/aw-query/src/lib.rs b/aw-query/src/lib.rs new file mode 100644 index 0000000..dc8e1c0 --- /dev/null +++ b/aw-query/src/lib.rs @@ -0,0 +1,65 @@ +#[macro_use] +extern crate log; +extern crate serde; +extern crate serde_json; + +use std::fmt; + +use aw_models::TimeInterval; + +use aw_datastore::Datastore; + +pub mod datatype; + +mod ast; +mod functions; +mod interpret; +mod lexer; +#[allow( + clippy::match_single_binding, + clippy::redundant_closure_call, + unused_braces +)] +mod parser; + +pub use crate::datatype::DataType; +pub use crate::interpret::VarEnv; + +// TODO: add line numbers to errors +// (works during lexing, but not during parsing I believe) + +#[derive(Debug)] +pub enum QueryError { + // Parser + ParsingError(String), + + // Execution + EmptyQuery(), + VariableNotDefined(String), + MathError(String), + InvalidType(String), + InvalidFunctionParameters(String), + TimeIntervalError(String), + BucketNotFound(String), + BucketQueryError(String), + RegexCompileError(String), +} + +impl fmt::Display for QueryError { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "{self:?}") + } +} + +pub fn query(code: &str, ti: &TimeInterval, ds: &Datastore) -> Result { + let lexer = lexer::Lexer::new(code); + let program = match parser::parse(lexer) { + Ok(p) => p, + Err(e) => { + // TODO: Improve parsing error message + warn!("ParsingError: {:?}", e); + return Err(QueryError::ParsingError(format!("{e:?}"))); + } + }; + interpret::interpret_prog(program, ti, ds) +} diff --git a/aw-query/src/parser.rs b/aw-query/src/parser.rs new file mode 100644 index 0000000..f4d6e77 --- /dev/null +++ b/aw-query/src/parser.rs @@ -0,0 +1,272 @@ +#![allow(clippy::ptr_arg)] +#![allow(clippy::vec_init_then_push)] + +use crate::ast::*; +use crate::lexer::Token::*; +use crate::lexer::*; +use plex::parser; + +use std::collections::HashMap; + +fn merge_if_vecs(lhs: Expr_, rhs: Expr_) -> Expr_ { + let mut ifs = match lhs { + Expr_::If(_ifs) => _ifs, + _ => unreachable!(), + }; + match rhs { + Expr_::If(mut _ifs) => ifs.append(&mut _ifs), + _ => unreachable!(), + }; + Expr_::If(ifs) +} + +parser! { + fn parse_(Token, Span); + + // combine two spans + (a, b) { + Span { + lo: a.lo, + hi: b.hi, + line: a.line, + } + } + + program: Program { + statements[s] => Program { stmts: s } + } + + statements: Vec { + => vec![], + statements[mut st] statement[x] => { + st.push(x); + st + }, + statements[st] Semi => st, + } + + statement: Expr { + ifs[x] => x, + ret[x] Semi => x, + } + + ifs: Expr { + _if[l_ifs] => l_ifs, + _elif[l_ifs] => l_ifs, + _else[l_ifs] => l_ifs, + } + + _cond_block: Expr { + binop[cond] LBrace statements[block] RBrace => Expr { + span: span!(), + node: { + let mut ifs = Vec::new(); + ifs.push((Box::new(cond), block)); + Expr_::If(ifs) + } + }, + } + + _if: Expr { + If _cond_block[x] => x + } + + _elif: Expr { + // Else if + _if[l_ifs] ElseIf _cond_block[l_preceding_ifs] => Expr { + span: span!(), + node: merge_if_vecs(l_ifs.node, l_preceding_ifs.node), + }, + // Else if else if + _elif[l_ifs] ElseIf _cond_block[l_preceding_ifs] => Expr { + span: span!(), + node: merge_if_vecs(l_ifs.node, l_preceding_ifs.node), + }, + } + + _else: Expr { + // if else + _if[l_ifs] Else LBrace statements[l_else_block] RBrace => Expr { + span: span!(), + node: { + let mut l_new = match l_ifs.node { + Expr_::If(l_ifs) => l_ifs, + _ => unreachable!(), + }; + let true_expr = Expr { span: span!(), node: Expr_::Bool(true) }; + l_new.push((Box::new(true_expr), l_else_block)); + Expr_::If(l_new) + } + }, + // else if else + _elif[l_ifs] Else LBrace statements[l_else_block] RBrace => Expr { + span: span!(), + node: { + let mut l_new = match l_ifs.node { + Expr_::If(l_ifs) => l_ifs, + _ => unreachable!(), + }; + let true_expr = Expr { span: span!(), node: Expr_::Bool(true) }; + l_new.push((Box::new(true_expr), l_else_block)); + Expr_::If(l_new) + } + }, + } + + ret: Expr { + Return assign[a] => Expr { + span: span!(), + node: Expr_::Return(Box::new(a)), + }, + assign[x] => x, + } + + assign: Expr { + Ident(var) Assign binop[rhs] => Expr { + span: span!(), + node: Expr_::Assign(var, Box::new(rhs)), + }, + binop[x] => x + } + + binop: Expr { + binop[lhs] Plus func[rhs] => Expr { + span: span!(), + node: Expr_::Add(Box::new(lhs), Box::new(rhs)), + }, + binop[lhs] Minus func[rhs] => Expr { + span: span!(), + node: Expr_::Sub(Box::new(lhs), Box::new(rhs)), + }, + binop[lhs] Star func[rhs] => Expr { + span: span!(), + node: Expr_::Mul(Box::new(lhs), Box::new(rhs)), + }, + binop[lhs] Slash func[rhs] => Expr { + span: span!(), + node: Expr_::Div(Box::new(lhs), Box::new(rhs)), + }, + binop[lhs] Percent func[rhs] => Expr { + span: span!(), + node: Expr_::Mod(Box::new(lhs), Box::new(rhs)), + }, + binop[lhs] Equals func[rhs] => Expr { + span: span!(), + node: Expr_::Equal(Box::new(lhs), Box::new(rhs)), + }, + func[x] => x + } + + func: Expr { + Ident(fname) LParen _inner_list[l] RParen => Expr { + span: span!(), + node: Expr_::Function(fname, Box::new(l)), + }, + Ident(fname) LParen RParen => Expr { + span: span!(), + node: { + let empty_expr_list = Expr { + span: span!(), + node: Expr_::List(Vec::new()) + }; + Expr_::Function(fname, Box::new(empty_expr_list)) + }, + }, + object[o] => o, + } + + object: Expr { + LBrace dict[d] RBrace => d, + LBrace RBrace => Expr { + span: span!(), + node: { + Expr_::Dict(HashMap::new()) + } + }, + list[l] => l + } + + list: Expr { + LBracket _inner_list[l] RBracket => l, + LBracket RBracket => Expr { + span: span!(), + node: { + Expr_::List(Vec::new()) + } + }, + atom[a] => a + } + + _inner_list: Expr { + binop[o] => Expr { + span: span!(), + node: { + let mut list = Vec::new(); + list.push(o); + Expr_::List(list) + } + }, + _inner_list[l] Comma binop[o] => Expr { + span: span!(), + node: { + match l.node { + Expr_::List(mut l) => { + l.push(o); + Expr_::List(l) + }, + _ => unreachable!(), + } + } + }, + } + + dict: Expr { + String(k) Colon binop[v] => Expr { + span: span!(), + node: { + let mut dict = HashMap::new(); + dict.insert(k, v); + Expr_::Dict(dict) + } + }, + dict[d] Comma String(k) Colon binop[v] => Expr { + span: span!(), + node: { + match d.node { + Expr_::Dict(mut d) => { + d.insert(k, v); + Expr_::Dict(d) + }, + _ => unreachable!(), + } + } + }, + } + + atom: Expr { + // round brackets to destructure tokens + Ident(v) => Expr { + span: span!(), + node: Expr_::Var(v), + }, + Bool(b) => Expr { + span: span!(), + node: Expr_::Bool(b), + }, + Number(i) => Expr { + span: span!(), + node: Expr_::Number(i), + }, + String(s) => Expr { + span: span!(), + node: Expr_::String(s), + }, + LParen binop[x] RParen => x + } +} + +pub type ParseError = (Option<(Token, Span)>, &'static str); + +pub fn parse>(i: I) -> Result { + parse_(i) +} diff --git a/aw-query/tests/query.rs b/aw-query/tests/query.rs new file mode 100644 index 0000000..bec7c99 --- /dev/null +++ b/aw-query/tests/query.rs @@ -0,0 +1,778 @@ +extern crate chrono; +extern crate serde_json; + +extern crate aw_query; + +// TODO: Move me to an appropriate place +#[macro_export] +macro_rules! json_map { + { $( $key:literal : $value:expr),* } => {{ + use serde_json::Value; + use serde_json::map::Map; + #[allow(unused_mut)] + let mut map : Map = Map::new(); + $( + map.insert( $key.to_string(), json!($value) ); + )* + map + }}; +} + +#[cfg(test)] +mod query_tests { + + use chrono::Duration; + use serde_json::json; + use std::convert::TryFrom; + + use aw_query::DataType; + use aw_query::QueryError; + + use aw_datastore::Datastore; + + use aw_models::Bucket; + use aw_models::BucketMetadata; + use aw_models::Event; + use aw_models::TimeInterval; + + static TIME_INTERVAL: &str = "1980-01-01T00:00:00Z/2080-01-02T00:00:00Z"; + static BUCKET_ID: &str = "testid"; + + fn setup_datastore_empty() -> Datastore { + Datastore::new_in_memory(false) + } + + fn setup_datastore_with_bucket() -> Datastore { + let ds = setup_datastore_empty(); + // Create bucket + let bucket = Bucket { + bid: None, + id: BUCKET_ID.to_string(), + _type: "testtype".to_string(), + client: "testclient".to_string(), + hostname: "testhost".to_string(), + created: Some(chrono::Utc::now()), + data: json_map! {}, + metadata: BucketMetadata::default(), + events: None, + last_updated: None, + }; + ds.create_bucket(&bucket).unwrap(); + ds + } + + fn setup_datastore_populated() -> Datastore { + let ds = setup_datastore_with_bucket(); + // Insert events + let e1 = Event { + id: None, + timestamp: chrono::Utc::now(), + duration: Duration::seconds(0), + data: json_map! {"key": json!("value")}, + }; + let mut e2 = e1.clone(); + e2.timestamp = chrono::Utc::now(); + let mut e_replace = e2.clone(); + e_replace.data = json_map! {"key": json!("value2")}; + e_replace.duration = Duration::seconds(2); + + let event_list = vec![e1, e2]; + + ds.insert_events(BUCKET_ID, &event_list).unwrap(); + + ds + } + + macro_rules! assert_err_type { + ($v:expr, $p:pat) => { + match $v { + Ok(_) => panic!("Expected an error, got {:?}", $v), + Err(e) => match e { + $p => (), + _ => panic!("Expected an error of another type, got {:?}", e), + }, + } + }; + } + + #[test] + fn test_bool() { + let ds = setup_datastore_empty(); + let interval = TimeInterval::new_from_string(TIME_INTERVAL).unwrap(); + + let code = String::from("True;False;a=True;return True;"); + match aw_query::query(&code, &interval, &ds).unwrap() { + aw_query::DataType::Bool(b) => assert!(b), + ref data => panic!("Wrong datatype, {data:?}"), + }; + } + + #[test] + fn test_number() { + let ds = setup_datastore_empty(); + let interval = TimeInterval::new_from_string(TIME_INTERVAL).unwrap(); + + let code = String::from("1;1.;return 1.1;"); + match aw_query::query(&code, &interval, &ds).unwrap() { + aw_query::DataType::Number(n) => assert_eq!(n, 1.1), + ref data => panic!("Wrong datatype, {data:?}"), + }; + } + + #[test] + fn test_equals() { + let ds = setup_datastore_empty(); + let interval = TimeInterval::new_from_string(TIME_INTERVAL).unwrap(); + + // number comparison true + let code = String::from("return 1==1;"); + match aw_query::query(&code, &interval, &ds).unwrap() { + aw_query::DataType::Bool(b) => assert!(b), + ref data => panic!("Wrong datatype, {data:?}"), + }; + + // number comparison false + let code = String::from("return 2==1;"); + match aw_query::query(&code, &interval, &ds).unwrap() { + aw_query::DataType::Bool(b) => assert!(!b), + ref data => panic!("Wrong datatype, {data:?}"), + }; + + // string comparison true + let code = String::from(r#"return "a"=="a";"#); + match aw_query::query(&code, &interval, &ds).unwrap() { + aw_query::DataType::Bool(b) => assert!(b), + ref data => panic!("Wrong datatype, {data:?}"), + }; + + // string comparison false + let code = String::from(r#"return "a"=="b";"#); + match aw_query::query(&code, &interval, &ds).unwrap() { + aw_query::DataType::Bool(b) => assert!(!b), + ref data => panic!("Wrong datatype, {data:?}"), + }; + + // bool comparison true + let code = String::from("return True==True;"); + match aw_query::query(&code, &interval, &ds).unwrap() { + aw_query::DataType::Bool(b) => assert!(b), + ref data => panic!("Wrong datatype, {data:?}"), + }; + + // bool comparison false + let code = String::from("return False==True;"); + match aw_query::query(&code, &interval, &ds).unwrap() { + aw_query::DataType::Bool(b) => assert!(!b), + ref data => panic!("Wrong datatype, {data:?}"), + }; + + // different types comparison (should raise an error) + let code = String::from("return True==1;"); + let res = aw_query::query(&code, &interval, &ds); + assert_err_type!(res, QueryError::InvalidType(_)); + + // None comparison, consistent with the PartialEq impl on DataType. + // print() returns None, so this compares two None values. + let code = String::from(r#"a=print("a"); b=print("b"); return a==b;"#); + match aw_query::query(&code, &interval, &ds).unwrap() { + aw_query::DataType::Bool(b) => assert!(b), + ref data => panic!("Wrong datatype, {data:?}"), + }; + } + + #[test] + fn test_return() { + let ds = setup_datastore_empty(); + let interval = TimeInterval::new_from_string(TIME_INTERVAL).unwrap(); + + let code = String::from("return 1;"); + match aw_query::query(&code, &interval, &ds).unwrap() { + aw_query::DataType::Number(n) => assert_eq!(n, 1.0), + ref data => panic!("Wrong datatype, {data:?}"), + }; + + // Test with old RETURN method + let code = String::from("RETURN=1;"); + match aw_query::query(&code, &interval, &ds).unwrap() { + aw_query::DataType::Number(n) => assert_eq!(n, 1.0), + ref data => panic!("Wrong datatype, {data:?}"), + }; + + let code = String::from("return 1+1;"); + match aw_query::query(&code, &interval, &ds).unwrap() { + aw_query::DataType::Number(n) => assert_eq!(n, 2.0), + ref data => panic!("Wrong datatype, {data:?}"), + }; + } + + #[test] + fn test_if() { + let ds = setup_datastore_empty(); + let interval = TimeInterval::new_from_string(TIME_INTERVAL).unwrap(); + + // Test hardcoded True + let code = String::from( + " + n=1; + if True { n=2; } + return n;", + ); + match aw_query::query(&code, &interval, &ds).unwrap() { + aw_query::DataType::Number(n) => assert_eq!(n, 2.0), + ref data => panic!("Wrong datatype, {data:?}"), + }; + + // Test hardcoded False + let code = String::from( + " + n=1; + if False { n=2; } + return n;", + ); + match aw_query::query(&code, &interval, &ds).unwrap() { + aw_query::DataType::Number(n) => assert_eq!(n, 1.0), + ref data => panic!("Wrong datatype, {data:?}"), + }; + + // Test expression True + let code = String::from( + " + a=True; n=1; + if a { n=2; } + return n;", + ); + match aw_query::query(&code, &interval, &ds).unwrap() { + aw_query::DataType::Number(n) => assert_eq!(n, 2.0), + ref data => panic!("Wrong datatype, {data:?}"), + }; + + // Test expression False + let code = String::from( + " + a=False; n=1; + if a { n=2; } + return n;", + ); + match aw_query::query(&code, &interval, &ds).unwrap() { + aw_query::DataType::Number(n) => assert_eq!(n, 1.0), + ref data => panic!("Wrong datatype, {data:?}"), + }; + + // Test if else + let code = String::from( + " + a=False; n=1; + if a { } + else { n=3; } + return n;", + ); + match aw_query::query(&code, &interval, &ds).unwrap() { + aw_query::DataType::Number(n) => assert_eq!(n, 3.0), + ref data => panic!("Wrong datatype, {data:?}"), + }; + + // Test if else if + let code = String::from( + " + a=False; b=True; n=1; + if a { n=2; } + elif b { n=3; } + return n;", + ); + match aw_query::query(&code, &interval, &ds).unwrap() { + aw_query::DataType::Number(n) => assert_eq!(n, 3.0), + ref data => panic!("Wrong datatype, {data:?}"), + }; + + // Test if else if else + let code = String::from( + " + a=False; b=True; n=1; + if a { n=2; } + elif a { n=3; } + else { n=4; } + return n;", + ); + match aw_query::query(&code, &interval, &ds).unwrap() { + aw_query::DataType::Number(n) => assert_eq!(n, 4.0), + ref data => panic!("Wrong datatype, {data:?}"), + }; + + // Test if inside if + let code = String::from( + " + a=True; n=1; + if a { if a { n = 2; } } + return n;", + ); + match aw_query::query(&code, &interval, &ds).unwrap() { + aw_query::DataType::Number(n) => assert_eq!(n, 2.0), + ref data => panic!("Wrong datatype, {data:?}"), + }; + } + + #[test] + fn test_function() { + let ds = setup_datastore_empty(); + let interval = TimeInterval::new_from_string(TIME_INTERVAL).unwrap(); + + let code = String::from("return print(1);"); + aw_query::query(&code, &interval, &ds).unwrap(); + + let code = String::from("return print(1, 2);"); + aw_query::query(&code, &interval, &ds).unwrap(); + + let code = String::from("return no_such_function(1);"); + match aw_query::query(&code, &interval, &ds) { + Ok(ok) => panic!("Expected QueryError, got {ok:?}"), + Err(e) => match e { + QueryError::VariableNotDefined(qe) => assert_eq!(qe, "no_such_function"), + qe => panic!("Expected QueryError::VariableNotDefined, got {qe:?}"), + }, + } + + let code = String::from("invalid_type=1; return invalid_type(1);"); + match aw_query::query(&code, &interval, &ds) { + Ok(ok) => panic!("Expected QueryError, got {ok:?}"), + Err(e) => match e { + QueryError::InvalidType(qe) => assert_eq!(qe, "invalid_type"), + qe => panic!("Expected QueryError::VariableNotDefined, got {qe:?}"), + }, + } + } + + #[test] + fn test_all_functions() { + let ds = setup_datastore_populated(); + let interval = TimeInterval::new_from_string(TIME_INTERVAL).unwrap(); + + let code = String::from("return query_bucket(\"testid\");"); + aw_query::query(&code, &interval, &ds).unwrap(); + + let code = format!( + r#" + events = query_bucket(find_bucket("{}", "testhost")); + events = flood(events); + events = sort_by_duration(events); + events = limit_events(events, 10000); + events = sort_by_timestamp(events); + events = concat(events, query_bucket("{}")); + events = categorize(events, [[["test"], {{ "type": "regex", "regex": "value$" }}], [["test", "testing"], {{ "type": "regex", "regex": "value$" }}]]); + events = tag(events, [["testtag", {{ "type": "regex", "regex": "test$" }}], ["another testtag", {{ "type": "regex", "regex": "test-pat$" }}]]); + total_duration = sum_durations(events); + bucketnames = query_bucket_names(); + print("test", "test2"); + url_events = split_url_events (events); + filtered_events = filter_period_intersect(events, events); + filtered_events = filter_keyvals(events, "$category", [["Uncategorized"]]); + filtered_events = filter_keyvals_regex(events, "key", "regex"); + chunked_events = chunk_events_by_key(events, "key"); + merged_events = merge_events_by_keys(events, ["key"]); + return merged_events;"#, + "testid", "testid" + ); + match aw_query::query(&code, &interval, &ds).unwrap() { + aw_query::DataType::List(l) => l, + ref data => panic!("Wrong datatype, {data:?}"), + }; + // TODO: assert_eq result + } + + #[test] + fn test_categorize() { + let ds = setup_datastore_populated(); + let interval = TimeInterval::new_from_string(TIME_INTERVAL).unwrap(); + + let code = format!( + r#" + events = query_bucket("{}"); + events = categorize(events, [[["Test", "Subtest"], {{ "type": "regex", "regex": "^value$" }}]]); + return events;"#, + "testid" + ); + let result: DataType = aw_query::query(&code, &interval, &ds).unwrap(); + let events: Vec = Vec::try_from(&result).unwrap(); + + let event = events.first().unwrap(); + let cats = event.data.get("$category").unwrap(); + assert_eq!(cats, &serde_json::json!(vec!["Test", "Subtest"])); + } + + #[test] + fn test_categorize_priority_overrides_depth() { + let ds = setup_datastore_populated(); + let interval = TimeInterval::new_from_string(TIME_INTERVAL).unwrap(); + + // Without priority, the deeper rule wins (existing contract). + let code = format!( + r#" + events = query_bucket("{}"); + events = categorize(events, [[["A"], {{ "type": "regex", "regex": "^value$" }}], [["B", "B1"], {{ "type": "regex", "regex": "^value$" }}]]); + return events;"#, + "testid" + ); + let result: DataType = aw_query::query(&code, &interval, &ds).unwrap(); + let events: Vec = Vec::try_from(&result).unwrap(); + assert_eq!( + events.first().unwrap().data.get("$category").unwrap(), + &serde_json::json!(vec!["B", "B1"]) + ); + + // With an explicit priority on A, organizational depth no longer wins. + let code = format!( + r#" + events = query_bucket("{}"); + events = categorize(events, [[["A"], {{ "type": "regex", "regex": "^value$", "priority": 25 }}], [["B", "B1"], {{ "type": "regex", "regex": "^value$" }}]]); + return events;"#, + "testid" + ); + let result: DataType = aw_query::query(&code, &interval, &ds).unwrap(); + let events: Vec = Vec::try_from(&result).unwrap(); + assert_eq!( + events.first().unwrap().data.get("$category").unwrap(), + &serde_json::json!(vec!["A"]) + ); + + // `weight` is accepted as an alias for `priority`. + let code = format!( + r#" + events = query_bucket("{}"); + events = categorize(events, [[["A"], {{ "type": "regex", "regex": "^value$", "weight": 25 }}], [["B", "B1"], {{ "type": "regex", "regex": "^value$" }}]]); + return events;"#, + "testid" + ); + let result: DataType = aw_query::query(&code, &interval, &ds).unwrap(); + let events: Vec = Vec::try_from(&result).unwrap(); + assert_eq!( + events.first().unwrap().data.get("$category").unwrap(), + &serde_json::json!(vec!["A"]) + ); + } + + #[test] + fn test_tag() { + let ds = setup_datastore_populated(); + let interval = TimeInterval::new_from_string(TIME_INTERVAL).unwrap(); + + let code = format!( + r#" + events = query_bucket("{}"); + events = tag(events, [["testtag", {{ "type": "regex", "regex": "value$" }}], ["another testtag", {{ "type": "regex", "regex": "value$" }}]]); + return events;"#, + "testid" + ); + let result: DataType = aw_query::query(&code, &interval, &ds).unwrap(); + let events: Vec = Vec::try_from(&result).unwrap(); + + let event = events.first().unwrap(); + let tags = event.data.get("$tags").unwrap().as_array().unwrap(); + assert_eq!(tags.len(), 2); + } + + #[test] + fn test_tag_select_keys() { + let ds = setup_datastore_with_bucket(); + let interval = TimeInterval::new_from_string(TIME_INTERVAL).unwrap(); + + let event = Event { + id: None, + timestamp: chrono::Utc::now(), + duration: Duration::seconds(0), + data: json_map! { + "app": json!("terminal"), + "title": json!("just a test"), + "pid": json!(123) + }, + }; + ds.insert_events(BUCKET_ID, &[event]).unwrap(); + + let code = format!( + r#" + events = query_bucket("{}"); + events = tag(events, [["title-match", {{ "type": "regex", "regex": "test$", "select_keys": ["title"] }}], ["app-match", {{ "type": "regex", "regex": "test$", "select_keys": ["app"] }}], ["pid-match", {{ "type": "regex", "regex": "123", "select_keys": ["pid"] }}], ["missing-match", {{ "type": "regex", "regex": "test", "select_keys": ["missing"] }}]]); + return events;"#, + BUCKET_ID + ); + let result: DataType = aw_query::query(&code, &interval, &ds).unwrap(); + let events: Vec = Vec::try_from(&result).unwrap(); + + let event = events.first().unwrap(); + let tags = event.data.get("$tags").unwrap(); + assert_eq!(tags, &serde_json::json!(vec!["title-match"])); + } + + #[test] + fn test_rule_parsing() { + let ds = setup_datastore_populated(); + let interval = TimeInterval::new_from_string(TIME_INTERVAL).unwrap(); + + // Test rule where rule is not a dict + let code = r#" + events = []; + events = tag(events, ["test", false]); + return events;"#; + let res = aw_query::query(code, &interval, &ds); + assert_err_type!(res, QueryError::InvalidFunctionParameters(_)); + + // Test rule without type + let code = r#" + events = []; + events = tag(events, [["testtag", { }]]); + return events;"#; + let res = aw_query::query(code, &interval, &ds); + assert_err_type!(res, QueryError::InvalidFunctionParameters(_)); + + // Test invalid rule type + let code = r#" + events = []; + events = tag(events, [["testtag", { "type": false }]]); + return events;"#; + let res = aw_query::query(code, &interval, &ds); + assert_err_type!(res, QueryError::InvalidFunctionParameters(_)); + + // Test invalid rule name + let code = r#" + events = []; + events = tag(events, [["testtag", { "type": "rgex" }]]); + return events;"#; + let res = aw_query::query(code, &interval, &ds); + assert_err_type!(res, QueryError::InvalidFunctionParameters(_)); + + // Test "none" rule type + let code = r#" + events = []; + events = tag(events, [["testtag", { "type": "none" }]]); + return events;"#; + aw_query::query(code, &interval, &ds).unwrap(); + + // Test regex rule where regex field has wrong type + let code = r#" + events = []; + events = tag(events, [["testtag", { "type": "regex", "regex": true }]]); + return events;"#; + let res = aw_query::query(code, &interval, &ds); + assert_err_type!(res, QueryError::InvalidFunctionParameters(_)); + + // Test regex rule where regex field is not set + let code = r#" + events = []; + events = tag(events, [["testtag", { "type": "regex" }]]); + return events;"#; + let res = aw_query::query(code, &interval, &ds); + assert_err_type!(res, QueryError::InvalidFunctionParameters(_)); + + // Test regex rule with ignore_case field + let code = r#" + events = []; + events = tag(events, [["testtag", { "type": "regex", "regex": "test", "ignore_case": false }]]); + return events;"#; + aw_query::query(code, &interval, &ds).unwrap(); + + // Test regex rule with valid select_keys field + let code = r#" + events = []; + events = tag(events, [["testtag", { "type": "regex", "regex": "test", "select_keys": ["key"] }]]); + return events;"#; + aw_query::query(code, &interval, &ds).unwrap(); + + // Test regex rule where ignore_case field is of invalid type + let code = r#" + events = []; + events = tag(events, [["testtag", { "type": "regex", "regex": "test", "ignore_case": "" }]]); + return events;"#; + let res = aw_query::query(code, &interval, &ds); + assert_err_type!(res, QueryError::InvalidFunctionParameters(_)); + + // Test regex rule where select_keys field is of invalid type + let code = r#" + events = []; + events = tag(events, [["testtag", { "type": "regex", "regex": "test", "select_keys": "key" }]]); + return events;"#; + let res = aw_query::query(code, &interval, &ds); + assert_err_type!(res, QueryError::InvalidFunctionParameters(_)); + + // Test regex rule where select_keys contains non-string values + let code = r#" + events = []; + events = tag(events, [["testtag", { "type": "regex", "regex": "test", "select_keys": ["key", false] }]]); + return events;"#; + let res = aw_query::query(code, &interval, &ds); + assert_err_type!(res, QueryError::InvalidFunctionParameters(_)); + + // Test regex rule where uncompilable regex is supplied + let code = r#" + events = []; + events = tag(events, [["testtag", { "type": "regex", "regex": "!#¤%&/(=" }]]); + return events;"#; + let res = aw_query::query(code, &interval, &ds); + assert_err_type!(res, QueryError::RegexCompileError(_)); + + // Test categorize rule with non-integer priority + let code = r#" + events = []; + events = categorize(events, [[["test"], { "type": "regex", "regex": "test", "priority": 1.5 }]]); + return events;"#; + let res = aw_query::query(code, &interval, &ds); + assert_err_type!(res, QueryError::InvalidFunctionParameters(_)); + + // Test categorize rule with non-numeric priority + let code = r#" + events = []; + events = categorize(events, [[["test"], { "type": "regex", "regex": "test", "priority": "high" }]]); + return events;"#; + let res = aw_query::query(code, &interval, &ds); + assert_err_type!(res, QueryError::InvalidFunctionParameters(_)); + } + + #[test] + fn test_string() { + let ds = setup_datastore_empty(); + let interval = TimeInterval::new_from_string(TIME_INTERVAL).unwrap(); + + let code = String::from("return \"test \\\" with escaped quote\";"); + match aw_query::query(&code, &interval, &ds).unwrap() { + aw_query::DataType::String(s) => assert_eq!(s, "test \" with escaped quote"), + _ => panic!("Wrong datatype"), + } + } + + #[test] + fn test_list() { + let ds = setup_datastore_empty(); + let interval = TimeInterval::new_from_string(TIME_INTERVAL).unwrap(); + + let code = String::from("return [];"); + aw_query::query(&code, &interval, &ds).unwrap(); + + let code = String::from("return [1];"); + aw_query::query(&code, &interval, &ds).unwrap(); + + let code = String::from("return [1+1];"); + aw_query::query(&code, &interval, &ds).unwrap(); + + let code = String::from("return [1,1];"); + aw_query::query(&code, &interval, &ds).unwrap(); + + let code = String::from("return [1,1+2];"); + aw_query::query(&code, &interval, &ds).unwrap(); + + let code = String::from("return [1,1+1,1+2+3,4/3,[1+2]];"); + aw_query::query(&code, &interval, &ds).unwrap(); + } + + #[test] + fn test_comment() { + let ds = setup_datastore_empty(); + let interval = TimeInterval::new_from_string(TIME_INTERVAL).unwrap(); + + let code = String::from("return 1;# testing 123"); + aw_query::query(&code, &interval, &ds).unwrap(); + } + + #[test] + fn test_dict() { + let ds = setup_datastore_empty(); + let interval = TimeInterval::new_from_string(TIME_INTERVAL).unwrap(); + + let code = String::from("return {};"); + aw_query::query(&code, &interval, &ds).unwrap(); + + let code = String::from("return {\"test\": 2};"); + aw_query::query(&code, &interval, &ds).unwrap(); + + let code = String::from("return {\"test\": 2, \"test2\": \"teststr\"};"); + aw_query::query(&code, &interval, &ds).unwrap(); + + let code = String::from("return {\"test\": {\"test\": \"test\"}};"); + aw_query::query(&code, &interval, &ds).unwrap(); + } + + #[test] + fn test_concat() { + let ds = setup_datastore_empty(); + let interval = TimeInterval::new_from_string(TIME_INTERVAL).unwrap(); + + // Append lists + let code = String::from("return [1]+[2];"); + let res = aw_query::query(&code, &interval, &ds).unwrap(); + let v = vec![DataType::Number(1.0), DataType::Number(2.0)]; + assert_eq!(res, DataType::List(v)); + + // Append strings + let code = String::from(r#"return "a"+"b";"#); + let res = aw_query::query(&code, &interval, &ds).unwrap(); + assert_eq!(res, DataType::String("ab".to_string())); + } + + #[test] + fn test_contains() { + let ds = setup_datastore_empty(); + let interval = TimeInterval::new_from_string(TIME_INTERVAL).unwrap(); + + // test list true + let code = String::from(r#"a = ["b", "a"]; return contains(a, "a");"#); + let res = aw_query::query(&code, &interval, &ds).unwrap(); + assert_eq!(res, DataType::Bool(true)); + + // test list false + let code = String::from(r#"a = ["b", "a"]; return contains(a, "c");"#); + let res = aw_query::query(&code, &interval, &ds).unwrap(); + assert_eq!(res, DataType::Bool(false)); + + // test dict true + let code = String::from(r#"a = {"a": 1}; return contains(a, "a");"#); + let res = aw_query::query(&code, &interval, &ds).unwrap(); + assert_eq!(res, DataType::Bool(true)); + + // test dict false + let code = String::from(r#"a = {"b": 1}; return contains(a, "a");"#); + let res = aw_query::query(&code, &interval, &ds).unwrap(); + assert_eq!(res, DataType::Bool(false)); + } + + #[test] + fn test_math() { + let ds = setup_datastore_empty(); + let interval = TimeInterval::new_from_string(TIME_INTERVAL).unwrap(); + + let code = String::from("return 1+1;"); + match aw_query::query(&code, &interval, &ds).unwrap() { + DataType::Number(n) => assert_eq!(n, 2.0), + num => panic!("Expected number, got {num:?}"), + }; + + let code = String::from("return 1-1;"); + match aw_query::query(&code, &interval, &ds).unwrap() { + DataType::Number(n) => assert_eq!(n, 0.0), + num => panic!("Expected number, got {num:?}"), + }; + + let code = String::from("return 3*5;"); + match aw_query::query(&code, &interval, &ds).unwrap() { + DataType::Number(n) => assert_eq!(n, 15.0), + num => panic!("Expected number, got {num:?}"), + }; + + let code = String::from("return 4/2;"); + match aw_query::query(&code, &interval, &ds).unwrap() { + DataType::Number(n) => assert_eq!(n, 2.0), + num => panic!("Expected number, got {num:?}"), + }; + + let code = String::from("return 1/0;"); + let res = aw_query::query(&code, &interval, &ds); + assert_err_type!(res, QueryError::MathError(_)); + + let code = String::from("return 2.5%1;"); + match aw_query::query(&code, &interval, &ds).unwrap() { + DataType::Number(n) => assert_eq!(n, 0.5), + num => panic!("Expected number, got {num:?}"), + }; + + let code = String::from("return 1+1+0+1;"); + match aw_query::query(&code, &interval, &ds).unwrap() { + DataType::Number(n) => assert_eq!(n, 3.0), + num => panic!("Expected number, got {num:?}"), + }; + } +} diff --git a/aw-query/tests/regex_fuzz.rs b/aw-query/tests/regex_fuzz.rs new file mode 100644 index 0000000..855a496 --- /dev/null +++ b/aw-query/tests/regex_fuzz.rs @@ -0,0 +1,229 @@ +//! Property-based fuzz tests for regex compilation in aw-query. +//! +//! These tests verify that invalid regex patterns fed into the query language +//! (via categorize, tag, or filter_keyvals_regex) are rejected with +//! QueryError::RegexCompileError rather than panicking or returning a +//! server-side 500-equivalent error. + +use aw_datastore::Datastore; +use aw_models::{Bucket, BucketMetadata, TimeInterval}; +use aw_query::QueryError; +use proptest::prelude::*; +use serde_json::json; + +static TIME_INTERVAL: &str = "1980-01-01T00:00:00Z/2080-01-02T00:00:00Z"; + +fn setup_datastore_with_bucket() -> Datastore { + let ds = Datastore::new_in_memory(false); + let bucket = Bucket { + bid: None, + id: "testid".to_string(), + _type: "testtype".to_string(), + client: "testclient".to_string(), + hostname: "testhost".to_string(), + created: Some(chrono::Utc::now()), + data: json!({}).as_object().unwrap().clone(), + metadata: BucketMetadata::default(), + events: None, + last_updated: None, + }; + ds.create_bucket(&bucket).unwrap(); + ds +} + +fn setup_interval() -> TimeInterval { + TimeInterval::new_from_string(TIME_INTERVAL).unwrap() +} + +/// Strategy that generates regex strings known to be invalid for fancy_regex. +fn invalid_regex_strategy() -> impl Strategy { + prop_oneof![ + // Unclosed groups and character classes + Just("(".to_string()), + Just("[".to_string()), + Just("(?P impl Strategy { + proptest::collection::vec(any::(), 0..64).prop_map(|v| v.into_iter().collect()) +} + +/// Build a query that uses categorize() with the given regex pattern. +/// +/// Only quotes are escaped; backslashes are passed through as-is because +/// the aw-query lexer only handles `\"` escapes, not `\\`. +fn categorize_query(pattern: &str) -> String { + format!( + r#" + events = query_bucket("testid"); + events = categorize(events, [[["TestCat"], {{ "type": "regex", "regex": "{}" }}]]); + return events; + "#, + pattern.replace('"', "\\\"") + ) +} + +/// Build a query that uses tag() with the given regex pattern. +fn tag_query(pattern: &str) -> String { + format!( + r#" + events = query_bucket("testid"); + events = tag(events, [["testtag", {{ "type": "regex", "regex": "{}" }}]]); + return events; + "#, + pattern.replace('"', "\\\"") + ) +} + +/// Build a query that uses filter_keyvals_regex() with the given regex pattern. +fn filter_keyvals_regex_query(pattern: &str) -> String { + format!( + r#" + events = query_bucket("testid"); + events = filter_keyvals_regex(events, "key", "{}"); + return events; + "#, + pattern.replace('"', "\\\"") + ) +} + +proptest! { + /// Any invalid regex pattern passed to categorize() must produce + /// QueryError::RegexCompileError, never panic. + #[test] + fn categorize_rejects_invalid_regex(pattern in invalid_regex_strategy()) { + let ds = setup_datastore_with_bucket(); + let interval = setup_interval(); + let code = categorize_query(&pattern); + + let result = aw_query::query(&code, &interval, &ds); + prop_assert!( + matches!(result, Err(QueryError::RegexCompileError(_))), + "Expected RegexCompileError for pattern {:?}, got {:?}", + pattern, result + ); + } + + /// Any invalid regex pattern passed to tag() must produce + /// QueryError::RegexCompileError, never panic. + #[test] + fn tag_rejects_invalid_regex(pattern in invalid_regex_strategy()) { + let ds = setup_datastore_with_bucket(); + let interval = setup_interval(); + let code = tag_query(&pattern); + + let result = aw_query::query(&code, &interval, &ds); + prop_assert!( + matches!(result, Err(QueryError::RegexCompileError(_))), + "Expected RegexCompileError for pattern {:?}, got {:?}", + pattern, result + ); + } + + /// Any invalid regex pattern passed to filter_keyvals_regex() must produce + /// QueryError::RegexCompileError, never panic. + #[test] + fn filter_keyvals_regex_rejects_invalid_regex(pattern in invalid_regex_strategy()) { + let ds = setup_datastore_with_bucket(); + let interval = setup_interval(); + let code = filter_keyvals_regex_query(&pattern); + + let result = aw_query::query(&code, &interval, &ds); + prop_assert!( + matches!(result, Err(QueryError::RegexCompileError(_))), + "Expected RegexCompileError for pattern {:?}, got {:?}", + pattern, result + ); + } + + /// Arbitrary strings passed to categorize() must never panic. + /// The result may be Ok or Err, but it must not crash. + #[test] + fn categorize_never_panics(pattern in arbitrary_regex_strategy()) { + let ds = setup_datastore_with_bucket(); + let interval = setup_interval(); + let code = categorize_query(&pattern); + + // We only assert that this does not panic; the result itself is allowed + // to be either Ok or Err (some patterns are valid regex). + let _result = aw_query::query(&code, &interval, &ds); + } + + /// Arbitrary strings passed to tag() must never panic. + #[test] + fn tag_never_panics(pattern in arbitrary_regex_strategy()) { + let ds = setup_datastore_with_bucket(); + let interval = setup_interval(); + let code = tag_query(&pattern); + + let _result = aw_query::query(&code, &interval, &ds); + } + + /// Arbitrary strings passed to filter_keyvals_regex() must never panic. + #[test] + fn filter_keyvals_regex_never_panics(pattern in arbitrary_regex_strategy()) { + let ds = setup_datastore_with_bucket(); + let interval = setup_interval(); + let code = filter_keyvals_regex_query(&pattern); + + let _result = aw_query::query(&code, &interval, &ds); + } +} + +/// Regression test: the specific pattern that caused ActivityWatch#1340 +/// (`++` in category pattern). In aw-server-rust this is a valid possessive +/// quantifier, so it should NOT produce RegexCompileError. The original bug +/// only affected the Python aw-server. +#[test] +fn regression_notepad_plus_plus_is_valid_in_rust() { + let ds = setup_datastore_with_bucket(); + let interval = setup_interval(); + let code = categorize_query("Notepad++"); + + let result = aw_query::query(&code, &interval, &ds); + assert!( + result.is_ok(), + "Expected 'Notepad++' to be accepted as valid possessive quantifier, got {:?}", + result + ); +} + +/// Regression test: the literal-escaped form of the #1340 pattern must also work. +#[test] +fn regression_notepad_escaped_plus_is_valid() { + let ds = setup_datastore_with_bucket(); + let interval = setup_interval(); + let code = categorize_query(r"Notepad\+\+"); + + let result = aw_query::query(&code, &interval, &ds); + assert!( + result.is_ok(), + "Expected 'Notepad\\+\\+' to be accepted, got {:?}", + result + ); +} diff --git a/aw-server.service b/aw-server.service new file mode 100644 index 0000000..24ab545 --- /dev/null +++ b/aw-server.service @@ -0,0 +1,25 @@ +########################## +# aw-server-rust.service # +########################## +# +# 1. Build and package aw-server-rust and aw-webui with "make" +# 2. Install aw-server-rust with "sudo make install" +# 3. Run 'systemctl --user daemon-reload' to make systemd load in the new service file +# 4. Run 'systemctl --user start aw-server-rust.service' to start aw-server-rust +# 5. (Optional) Run 'systemctl --user enable aw-server-rust.service' to always start aw-server-rust when you log in +# +# Now aw-server-rust should be running, you can now start whatever +# ActivityWatch watchers you desire and can find the WebUI at +# http://localhost:5600 +# + +[Service] +Type=notify +ExecStart=aw-server + +[Unit] +Description=ActivityWatch Server (Rust implementation) +After=network.target + +[Install] +WantedBy=default.target diff --git a/aw-server/Cargo.toml b/aw-server/Cargo.toml new file mode 100644 index 0000000..9426c24 --- /dev/null +++ b/aw-server/Cargo.toml @@ -0,0 +1,61 @@ +[package] +name = "aw-server" +version = "0.14.0" +authors = ["Johan Bjäreholt ", "Erik Bjäreholt "] +edition = "2021" + +[lib] +name = "aw_server" +crate-type = ["lib", "cdylib"] +path = "src/lib.rs" + +[[bin]] +name = "aw-server" +path = "src/main.rs" + +[features] +default = ["bundled"] +# Use bundled SQLite (default, no encryption support) +bundled = ["aw-datastore/bundled"] +# Enable SQLCipher encryption support (requires OpenSSL) +# Build with: cargo build --no-default-features --features encryption +encryption = ["aw-datastore/encryption"] +# Enable SQLCipher encryption with vendored OpenSSL (fully self-contained) +encryption-vendored = ["aw-datastore/encryption-vendored"] + +[dependencies] +rocket = { version = "0.5.0", features = ["json"] } +rocket_cors = { version = "0.6.0" } +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" +chrono = { version = "0.4", features = ["serde"] } +dirs = "6" +lazy_static = "1.4" +log = "0.4" +fern = { version = "0.7", features = ["colored"] } +toml = "0.8" +regex = "1" +gethostname = "0.4" +uuid = { version = "1.3", features = ["serde", "v4"] } +clap = { version = "4.1", features = ["derive", "cargo"] } +log-panics = { version = "2", features = ["with-backtrace"]} +subtle = "2" +tempfile = "3" +rust-embed = { version = "8.0.0", features = ["interpolate-folder-path", "debug-embed"] } +aw-datastore = { path = "../aw-datastore", default-features = false } +aw-models = { path = "../aw-models" } +aw-transform = { path = "../aw-transform" } +aw-query = { path = "../aw-query" } + +[target.'cfg(target_os="linux")'.dependencies] +sd-notify = "0.4.2" + +[target.'cfg(all(target_os="linux", target_arch="x86"))'.dependencies] +jemallocator = "0.5.0" + +[target.'cfg(target_os="android")'.dependencies] +jni = { version = "0.20", default-features = false } +libc = "0.2" +android_logger = "0.13" +openssl-sys = { version = "0.9.82", features = ["vendored"]} +aw-client-rust = { path = "../aw-client-rust" } diff --git a/aw-server/build.rs b/aw-server/build.rs new file mode 100644 index 0000000..7146eee --- /dev/null +++ b/aw-server/build.rs @@ -0,0 +1,30 @@ +use std::error::Error; + +fn main() -> Result<(), Box> { + let webui_var = std::env::var("AW_WEBUI_DIR"); + let path = if let Ok(var_path) = &webui_var { + std::path::Path::new(var_path) + } else { + let path = std::path::Path::new("../aw-webui/dist"); + // ensure folder exists, since macro requires it + std::fs::create_dir_all(path)?; + println!("cargo:rustc-env=AW_WEBUI_DIR={}", path.display()); + path + }; + + let path_index = path.join("index.html"); + if !path_index.exists() { + println!( + "cargo:warning=`{}` is not built, compiling without webui", + path.display() + ); + } + + // Rebuild if the webui directory changes + println!("cargo:rerun-if-env-changed=AW_WEBUI_DIR"); + if let Ok(dir) = webui_var { + println!("cargo:rerun-if-changed={dir}"); + } + + Ok(()) +} diff --git a/aw-server/src/android/mod.rs b/aw-server/src/android/mod.rs new file mode 100644 index 0000000..da512b9 --- /dev/null +++ b/aw-server/src/android/mod.rs @@ -0,0 +1,536 @@ +// Based On the following guide from Mozilla: +// https://mozilla.github.io/firefox-browser-architecture/experiments/2017-09-21-rust-on-android.html + +extern crate android_logger; + +use std::ffi::{CStr, CString}; +use std::os::raw::c_char; + +use crate::device_id; +use crate::dirs; + +use android_logger::Config; +use rocket::serde::json::json; + +#[no_mangle] +pub extern "C" fn rust_greeting(to: *const c_char) -> *mut c_char { + let c_str = unsafe { CStr::from_ptr(to) }; + let recipient = match c_str.to_str() { + Err(_) => "there", + Ok(string) => string, + }; + + CString::new("Hello ".to_owned() + recipient + " (from Rust!)") + .unwrap() + .into_raw() +} + +#[cfg(target_os = "android")] +#[allow(non_snake_case)] +pub mod android { + extern crate jni; + + use self::jni::objects::{JClass, JString}; + use self::jni::sys::{jdouble, jint, jstring}; + use self::jni::JNIEnv; + use super::*; + + use crate::panic_guard::catch_panic; + + use std::path::PathBuf; + + use crate::endpoints; + use crate::endpoints::ServerState; + use aw_client_rust::classes::{classes_from_settings_str, default_classes}; + use aw_client_rust::queries::{ + build_android_canonical_events, AndroidQueryParams, QueryParamsBase, + }; + use aw_datastore::Datastore; + use aw_models::{Bucket, Event, TimeInterval}; + + static mut DATASTORE: Option = None; + + unsafe fn openDatastore() -> Datastore { + match DATASTORE { + Some(ref ds) => ds.clone(), + None => { + let db_dir = dirs::db_path("default") + .expect("Failed to get db path") + .to_str() + .unwrap() + .to_string(); + DATASTORE = Some(Datastore::new(db_dir, false)); + openDatastore() + } + } + } + + #[no_mangle] + pub unsafe extern "C" fn Java_net_activitywatch_android_RustInterface_greeting( + env: JNIEnv, + _: JClass, + java_pattern: JString, + ) -> jstring { + jni_guard(env, "greeting", || { + // Our Java companion code might pass-in "world" as a string, hence the name. + let world = rust_greeting( + env.get_string(java_pattern) + .expect("invalid pattern string") + .as_ptr(), + ); + // Retake pointer so that we can use it below and allow memory to be freed when it goes out of scope. + let world_ptr = CString::from_raw(world); + let output = env + .new_string(world_ptr.to_str().unwrap()) + .expect("Couldn't create java string!"); + + output.into_raw() + }) + } + + unsafe fn jstring_to_string(env: &JNIEnv, string: JString) -> String { + let jstr = env.get_string(string).expect("Failed to get Java string"); + jstr.into() + } + + unsafe fn string_to_jstring(env: &JNIEnv, string: String) -> jstring { + env.new_string(string) + .expect("Couldn't create java string") + .into_raw() + } + + unsafe fn create_error_object(env: &JNIEnv, msg: String) -> jstring { + let obj = json!({ "error": &msg }); + string_to_jstring(&env, obj.to_string()) + } + + /// Run a `jstring`-returning JNI entry point with panics caught. + /// + /// Since Rust 1.81 a panic that unwinds out of an `extern "C"` function + /// aborts the process, so any `unwrap()`/`expect()` reached from one of + /// these natives kills the app with `SIGABRT` instead of failing the call. + /// Those are the `libaw_server.so` → `SIGABRT` clusters in + /// ActivityWatch/aw-android#267. Catching the unwind here returns the same + /// `{"error": "…"}` object the callers already produce for ordinary + /// failures, which `RustInterface` parses as a normal `JSONObject`. + /// + /// `log_panics::init()` (installed in `initialize`) still logs the panic and + /// its backtrace to logcat before the unwind is stopped, so nothing is + /// hidden. + unsafe fn jni_guard(env: JNIEnv, name: &str, f: F) -> jstring + where + F: FnOnce() -> jstring, + { + match catch_panic(name, f) { + Ok(result) => result, + Err(msg) => { + error!("{}", msg); + // A panic mid-JNI-call can leave a pending Java exception, which + // would make the NewStringUTF below fail. + let _ = env.exception_clear(); + // Building the error object allocates a Java string, which can + // itself fail; never let that second failure unwind out of the + // `extern "C"` frame. A null return surfaces as a Java-level + // NullPointerException, which is recoverable, unlike SIGABRT. + catch_panic(name, || create_error_object(&env, msg)) + .unwrap_or_else(|_| std::ptr::null_mut()) + } + } + } + + /// Run a `void` JNI entry point with panics caught. There is no return value + /// to carry an error, so the panic is logged and swallowed. + unsafe fn jni_guard_void(name: &str, f: F) + where + F: FnOnce(), + { + if let Err(msg) = catch_panic(name, f) { + error!("{}", msg); + } + } + + #[no_mangle] + pub unsafe extern "C" fn Java_net_activitywatch_android_RustInterface_startServer( + env: JNIEnv, + _: JClass, + ) { + jni_guard_void("startServer", || { + info!("Starting server..."); + start_server(); + info!("Server exited"); + }); + } + + #[rocket::main] + async fn start_server() { + info!("Building server state..."); + + // FIXME: Why is unsafe needed here? Can we get rid of it? + unsafe { + let server_state: ServerState = endpoints::ServerState { + datastore: openDatastore(), + asset_resolver: endpoints::AssetResolver::new(None), + device_id: device_id::get_device_id(), + }; + info!("Using server_state:: device_id: {}", server_state.device_id); + + let mut server_config = crate::config::create_config("default", None); + server_config.port = 5600; + + endpoints::build_rocket(server_state, server_config) + .launch() + .await; + } + } + + static mut INITIALIZED: bool = false; + + #[no_mangle] + pub unsafe extern "C" fn Java_net_activitywatch_android_RustInterface_initialize( + env: JNIEnv, + _: JClass, + ) { + jni_guard_void("initialize", || { + if !INITIALIZED { + android_logger::init_once( + Config::default() + .with_max_level(log::LevelFilter::Info) // limit log level + .with_tag("aw-server-rust"), // logs will show under mytag tag + //.with_filter( // configure messages for specific crate + // FilterBuilder::new() + // .parse("debug,hello::crate=error") + // .build()) + ); + // Default panic hook writes to stderr, which Android discards + // (ActivityWatch/aw-android#220). log_panics routes them through + // android_logger so they appear in logcat. + log_panics::init(); + info!("Initializing aw-server-rust..."); + debug!("Redirected aw-server-rust stdout/stderr to logcat"); + } else { + info!("Already initialized"); + } + INITIALIZED = true; + + // Without this it might not work due to weird error probably arising from Rust optimizing away the JNIEnv: + // JNI DETECTED ERROR IN APPLICATION: use of deleted weak global reference + string_to_jstring(&env, "test".to_string()); + }); + } + + #[no_mangle] + pub unsafe extern "C" fn Java_net_activitywatch_android_RustInterface_setDataDir( + env: JNIEnv, + _: JClass, + java_dir: JString, + ) { + jni_guard_void("setDataDir", || { + let path = &jstring_to_string(&env, java_dir); + debug!("Setting android data dir as {}", path); + dirs::set_android_data_dir(path); + }); + } + + /// Report the Android app's release version from `/api/0/info` instead of + /// the aw-server-rust package version, which is the version of a component + /// rather than of the app the user installed. + #[no_mangle] + pub unsafe extern "C" fn Java_net_activitywatch_android_RustInterface_setVersionOverride( + env: JNIEnv, + _: JClass, + java_version: JString, + ) { + jni_guard_void("setVersionOverride", || { + let version = &jstring_to_string(&env, java_version); + debug!("Setting reported version to {}", version); + crate::version::set_version_override(version); + }); + } + + #[no_mangle] + pub unsafe extern "C" fn Java_net_activitywatch_android_RustInterface_getBuckets( + env: JNIEnv, + _: JClass, + ) -> jstring { + jni_guard(env, "getBuckets", || { + // Return an error object instead of unwrapping: if the datastore worker + // is gone (it panicked, e.g. the database could not be opened), the + // request fails with SendError/RecvError and a panic here unwinds across + // the JNI boundary on whatever thread called getBuckets — usually main. + match openDatastore().get_buckets() { + Ok(buckets) => string_to_jstring(&env, json!(buckets).to_string()), + Err(e) => create_error_object(&env, format!("Failed to get buckets: {e:?}")), + } + }) + } + + #[no_mangle] + pub unsafe extern "C" fn Java_net_activitywatch_android_RustInterface_createBucket( + env: JNIEnv, + _: JClass, + java_bucket: JString, + ) -> jstring { + jni_guard(env, "createBucket", || { + let bucket = jstring_to_string(&env, java_bucket); + let bucket_json: Bucket = match serde_json::from_str(&bucket) { + Ok(json) => json, + Err(err) => return create_error_object(&env, err.to_string()), + }; + match openDatastore().create_bucket(&bucket_json) { + Ok(()) => string_to_jstring(&env, "Bucket successfully created".to_string()), + Err(e) => create_error_object( + &env, + format!("Something went wrong when trying to create bucket: {:?}", e), + ), + } + }) + } + + #[no_mangle] + pub unsafe extern "C" fn Java_net_activitywatch_android_RustInterface_heartbeat( + env: JNIEnv, + _: JClass, + java_bucket_id: JString, + java_event: JString, + java_pulsetime: jdouble, + ) -> jstring { + jni_guard(env, "heartbeat", || { + let bucket_id = jstring_to_string(&env, java_bucket_id); + let event = jstring_to_string(&env, java_event); + let pulsetime = java_pulsetime as f64; + let event_json: Event = match serde_json::from_str(&event) { + Ok(json) => json, + Err(err) => return create_error_object(&env, err.to_string()), + }; + match openDatastore().heartbeat(&bucket_id, event_json, pulsetime) { + Ok(_) => string_to_jstring(&env, "Heartbeat successfully received".to_string()), + Err(e) => create_error_object( + &env, + format!( + "Something went wrong when trying to send heartbeat: {:?}", + e + ), + ), + } + }) + } + + #[no_mangle] + pub unsafe extern "C" fn Java_net_activitywatch_android_RustInterface_getEvents( + env: JNIEnv, + _: JClass, + java_bucket_id: JString, + java_limit: jint, + ) -> jstring { + jni_guard(env, "getEvents", || { + let bucket_id = jstring_to_string(&env, java_bucket_id); + let limit = java_limit as u64; + match openDatastore().get_events(&bucket_id, None, None, Some(limit)) { + Ok(events) => string_to_jstring(&env, json!(events).to_string()), + Err(e) => create_error_object( + &env, + format!("Something went wrong when trying to get events: {:?}", e), + ), + } + }) + } + + #[no_mangle] + pub unsafe extern "C" fn Java_net_activitywatch_android_RustInterface_migrateHostname( + env: JNIEnv, + _: JClass, + hostname: JString, + ) -> jstring { + jni_guard(env, "migrateHostname", || { + let hostname = jstring_to_string(&env, hostname); + if hostname.is_empty() { + return create_error_object(&env, "hostname must not be empty".to_string()); + } + match openDatastore().migrate_hostname(&hostname) { + Ok(count) => { + string_to_jstring(&env, format!("Migrated hostname for {} bucket(s)", count)) + } + Err(e) => create_error_object(&env, format!("Failed to migrate hostname: {:?}", e)), + } + }) + } + + #[no_mangle] + pub unsafe extern "C" fn Java_net_activitywatch_android_RustInterface_migrateAndroidBucketName( + env: JNIEnv, + _: JClass, + ) -> jstring { + jni_guard(env, "migrateAndroidBucketName", || { + match openDatastore().rename_bucket("aw-android-test", "aw-android") { + Ok(()) => string_to_jstring( + &env, + "Renamed bucket 'aw-android-test' to 'aw-android'".to_string(), + ), + Err(e) => create_error_object(&env, format!("Failed to rename bucket: {:?}", e)), + } + }) + } + + #[no_mangle] + pub unsafe extern "C" fn Java_net_activitywatch_android_RustInterface_migrateWatcherAndroidBucketNames( + env: JNIEnv, + _: JClass, + ) -> jstring { + jni_guard( + env, + "migrateWatcherAndroidBucketNames", + || match openDatastore().migrate_test_bucket_names() { + Ok(count) => string_to_jstring( + &env, + format!("Migrated {} 'aw-watcher-android-test' bucket(s)", count), + ), + Err(e) => create_error_object( + &env, + format!("Failed to migrate watcher bucket names: {:?}", e), + ), + }, + ) + } + + /// Return a raw settings JSON value (the datastore body, matching GET /api/0/settings/). + /// Missing or invalid keys return the JSON literal `null`. + /// + /// Widget/worker code must use this instead of unauthenticated HTTP: Android + /// enables API-key auth by default, so GET /api/0/settings/... from the + /// widget process 401s and silently falls back to defaults. + #[no_mangle] + pub unsafe extern "C" fn Java_net_activitywatch_android_RustInterface_getSetting( + env: JNIEnv, + _: JClass, + java_key: JString, + ) -> jstring { + jni_guard(env, "getSetting", || { + let key = jstring_to_string(&env, java_key); + // Match GET /api/0/settings/: dots are valid (nested-looking + // keys like "foo.bar" store as settings.foo.bar). Reject empty keys + // and path/NUL bytes so JNI cannot smuggle a lookup the HTTP router + // would never pass through. + if key.is_empty() || key.contains('/') || key.contains('\\') || key.contains('\0') { + return string_to_jstring(&env, "null".to_string()); + } + let setting_key = match crate::endpoints::settings_datastore_key(&key) { + Ok(k) => k, + Err(_) => return string_to_jstring(&env, "null".to_string()), + }; + match openDatastore().get_key_value(&setting_key) { + Ok(value) => string_to_jstring(&env, value), + Err(_) => string_to_jstring(&env, "null".to_string()), + } + }) + } + + #[no_mangle] + pub unsafe extern "C" fn Java_net_activitywatch_android_RustInterface_query( + env: JNIEnv, + _: JClass, + java_query: JString, + java_timeperiods: JString, + ) -> jstring { + jni_guard(env, "query", || { + let query_code = jstring_to_string(&env, java_query); + let timeperiods_str = jstring_to_string(&env, java_timeperiods); + let timeperiods: Vec = match serde_json::from_str(&timeperiods_str) { + Ok(json) => json, + Err(err) => return create_error_object(&env, err.to_string()), + }; + + let datastore = openDatastore(); + let mut results = Vec::new(); + + for interval in &timeperiods { + let result = match aw_query::query(&query_code, interval, &datastore) { + Ok(data) => data, + Err(e) => { + return create_error_object( + &env, + format!("Something went wrong when trying to query: {:?}", e), + ) + } + }; + results.push(result); + } + + string_to_jstring(&env, json!(results).to_string()) + }) + } + + #[no_mangle] + pub unsafe extern "C" fn Java_net_activitywatch_android_RustInterface_androidQuery( + env: JNIEnv, + _: JClass, + java_timeperiods: JString, + ) -> jstring { + jni_guard(env, "androidQuery", || { + let timeperiods_str = jstring_to_string(&env, java_timeperiods); + + let timeperiods: Vec = match serde_json::from_str(&timeperiods_str) { + Ok(json) => json, + Err(err) => return create_error_object(&env, err.to_string()), + }; + + // Hardcoded bucket ID + let bid_android = "aw-watcher-android".to_string(); + + // Read classes from the datastore directly. Do NOT fetch them over HTTP: + // Android enables API-key auth by default, and androidQuery runs from the + // widget process which does not send a Bearer token. The previous + // AwClient GET /api/0/settings/classes path 401'd (or failed if the + // HTTP server wasn't up) and silently fell back to default_classes(), + // which is why the homescreen widget disagreed with the Activity view + // on per-category time while totals still matched. + // See ActivityWatch/aw-android#142. + let datastore = openDatastore(); + let classes = match datastore.get_key_value("settings.classes") { + Ok(raw) => { + info!("Loaded classes from datastore settings.classes"); + classes_from_settings_str(&raw) + } + Err(_) => { + info!("settings.classes unset or unreadable, using default classes"); + default_classes() + } + }; + + // Build canonical Android query + let params = AndroidQueryParams { + base: QueryParamsBase { + bid_browsers: Vec::new(), + classes, + filter_classes: Vec::new(), + filter_afk: true, + include_audible: true, + }, + bid_android, + }; + let query_code = format!( + r#"{} + duration = sum_durations(events); + cat_events = sort_by_duration(merge_events_by_keys(events, ["$category"])); + RETURN = {{"events": events, "duration": duration, "cat_events": cat_events}};"#, + build_android_canonical_events(¶ms) + ); + + let mut results = Vec::new(); + + for interval in &timeperiods { + let result = match aw_query::query(&query_code, interval, &datastore) { + Ok(data) => data, + Err(e) => { + return create_error_object( + &env, + format!("Something went wrong when trying to query: {:?}", e), + ) + } + }; + results.push(result); + } + + string_to_jstring(&env, json!(results).to_string()) + }) + } +} diff --git a/aw-server/src/config.rs b/aw-server/src/config.rs new file mode 100644 index 0000000..e32d82e --- /dev/null +++ b/aw-server/src/config.rs @@ -0,0 +1,278 @@ +use std::fs::{self, File}; +use std::io::{Read, Write}; +use std::path::{Path, PathBuf}; +use std::sync::OnceLock; + +use rocket::config::Config; +use rocket::data::{Limits, ToByteUnit}; +use rocket::log::LogLevel; +use serde::{Deserialize, Serialize}; + +use crate::dirs; + +static PROFILE: OnceLock = OnceLock::new(); + +/// Return the current profile name (default: "default"). +pub fn get_profile() -> &'static str { + PROFILE.get().map(|s| s.as_str()).unwrap_or("default") +} + +/// Set the profile. Idempotent for the same value; panics if called with +/// a conflicting value (would silently redirect to a different datastore). +pub fn set_profile(profile: String) { + // Use the atomic OnceLock::set result to avoid a TOCTOU race: two callers + // with different values could both see get()==None before either sets it, + // causing the loser to silently proceed under the wrong profile. + match PROFILE.set(profile.clone()) { + Ok(()) => {} // we set it atomically, done + Err(_) => { + // Already set — verify no conflict (duplicate or concurrent same-value call is fine) + let existing = PROFILE + .get() + .expect("set returned Err but get returned None"); + if existing != &profile { + panic!( + "set_profile called with conflicting value: existing={existing:?}, new={profile:?}" + ); + } + // same value — idempotent, no action needed + } + } +} + +/// True when running in the legacy "testing" profile or in debug builds. +pub fn is_testing() -> bool { + get_profile() == "testing" +} + +/// Backwards-compat shim: callers that still use set_testing(bool) convert +/// true → profile "testing" and false → profile "default". +pub fn set_testing(testing: bool) { + let profile = if testing { "testing" } else { "default" }; + set_profile(profile.to_string()); +} + +/// Authentication configuration, serialised as `[auth]` in config.toml. +#[derive(Serialize, Deserialize, Default)] +pub struct AWAuthConfig { + /// Optional API key for Bearer-token authentication. + /// When set, all `/api/*` endpoints except `/api/0/info` require: + /// Authorization: Bearer + /// Leave unset (default) to disable authentication. + #[serde(default)] + pub api_key: Option, +} + +#[derive(Serialize, Deserialize)] +pub struct AWConfig { + #[serde(default = "default_address")] + pub address: String, + + #[serde(default = "default_port")] + pub port: u16, + + #[serde(skip, default = "default_testing")] + pub testing: bool, // This is not written to the config file (serde(skip)) + + #[serde(default = "default_cors")] + pub cors: Vec, + + #[serde(default = "default_cors")] + pub cors_regex: Vec, + + // Authentication settings — serialised as [auth] section. + #[serde(default)] + pub auth: AWAuthConfig, + + // A mapping of watcher names to paths where the + // custom visualizations are located. + #[serde(default = "default_custom_static")] + pub custom_static: std::collections::HashMap, +} + +impl Default for AWConfig { + fn default() -> AWConfig { + AWConfig { + address: default_address(), + port: default_port(), + testing: default_testing(), + auth: AWAuthConfig::default(), + cors: default_cors(), + cors_regex: default_cors(), + custom_static: default_custom_static(), + } + } +} + +impl AWConfig { + pub fn to_rocket_config(&self) -> rocket::Config { + let mut config; + if self.testing { + config = Config::debug_default(); + config.log_level = LogLevel::Debug; + } else { + config = Config::release_default() + }; + + // Needed for bucket imports + let limits = Limits::default() + .limit("json", 1000u64.megabytes()) + .limit("data-form", 1000u64.megabytes()); + + config.address = self.address.parse().unwrap(); + config.port = self.port; + config.limits = limits; + + config + } +} + +fn default_address() -> String { + "127.0.0.1".to_string() +} + +fn default_cors() -> Vec { + Vec::::new() +} + +fn default_testing() -> bool { + is_testing() +} + +fn default_port() -> u16 { + if is_testing() { + 5666 + } else { + 5600 + } +} + +fn default_custom_static() -> std::collections::HashMap { + std::collections::HashMap::new() +} + +/// Config filename for a profile. +/// +/// Isolated profile roots (including new-style `activitywatch-testing/`) use +/// bare `config.toml` — the directory already isolates. Suffixed +/// `config-testing.toml` remains only in the legacy shared-root layout so +/// existing testing config is not orphaned. +pub fn config_filename(profile: &str) -> String { + crate::dirs::config_filename(profile) +} + +fn get_config_path(profile: &str, config_override: Option<&Path>) -> PathBuf { + if let Some(config_path) = config_override { + return config_path.to_path_buf(); + } + + let mut config_path = dirs::get_config_dir().unwrap(); + config_path.push(config_filename(profile)); + config_path +} + +pub fn create_config(profile: &str, config_override: Option<&Path>) -> AWConfig { + set_profile(profile.to_string()); + let config_path = get_config_path(profile, config_override); + if let Some(parent) = config_path.parent() { + fs::create_dir_all(parent).expect("Unable to create config dir"); + } + + /* If there is no config file, create a new config file with default values but every value is + * commented out by default in case we would change a default value at some point in the future */ + if !config_path.is_file() { + debug!("Writing default commented out config at {:?}", config_path); + let mut wfile = File::create(config_path.clone()).expect("Unable to create config file"); + let default_config = AWConfig::default(); + let default_config_str = + toml::to_string(&default_config).expect("Failed to convert default config to string"); + let mut default_config_str_commented = String::new(); + default_config_str_commented.push_str("### DEFAULT SETTINGS ###\n"); + for line in default_config_str.lines() { + default_config_str_commented.push_str(&format!("#{line}\n")); + } + wfile + .write_all(&default_config_str_commented.into_bytes()) + .expect("Failed to write config to file"); + wfile.sync_all().expect("Unable to sync config file"); + } + + debug!("Reading config at {:?}", config_path); + let mut rfile = File::open(config_path).expect("Failed to open config file for reading"); + let mut content = String::new(); + rfile + .read_to_string(&mut content) + .expect("Failed to read config as a string"); + let aw_config: AWConfig = toml::from_str(&content).expect("Failed to parse config file"); + + aw_config +} + +#[cfg(test)] +mod tests { + use super::create_config; + use std::fs; + use std::path::PathBuf; + use std::sync::Mutex; + use uuid::Uuid; + + static TEST_LOCK: Mutex<()> = Mutex::new(()); + + struct TestConfigPath { + root: PathBuf, + config_path: PathBuf, + } + + impl TestConfigPath { + fn new(name: &str) -> Self { + let root = std::env::temp_dir() + .join("aw-server-config-tests") + .join(format!("{name}-{}", Uuid::new_v4())); + let config_path = root.join("config.toml"); + + Self { root, config_path } + } + } + + impl Drop for TestConfigPath { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.root); + } + } + + #[test] + fn config_filename_isolated_named_profiles_are_bare() { + // default and named isolated roots are always config.toml. Suffixed + // config-testing.toml is filesystem-dependent (legacy shared root) and + // covered in dirs.rs against fake roots. + assert_eq!(super::config_filename("default"), "config.toml"); + assert_eq!(super::config_filename("research"), "config.toml"); + assert_eq!(super::config_filename("my-profile"), "config.toml"); + } + + #[test] + fn create_config_uses_override_path() { + // create_config sets the PROFILE OnceLock, so these tests must not overlap. + let _lock = TEST_LOCK.lock().unwrap(); + let paths = TestConfigPath::new("override"); + fs::create_dir_all(paths.config_path.parent().unwrap()).unwrap(); + fs::write(&paths.config_path, "address = \"0.0.0.0\"\nport = 5611\n").unwrap(); + + let config = create_config("default", Some(paths.config_path.as_path())); + + assert_eq!(config.address, "0.0.0.0"); + assert_eq!(config.port, 5611); + } + + #[test] + fn create_config_creates_missing_override_file() { + let _lock = TEST_LOCK.lock().unwrap(); + let paths = TestConfigPath::new("missing"); + + let config = create_config("default", Some(paths.config_path.as_path())); + + assert!(paths.config_path.is_file()); + assert_eq!(config.address, "127.0.0.1"); + assert_eq!(config.port, 5600); + } +} diff --git a/aw-server/src/device_id.rs b/aw-server/src/device_id.rs new file mode 100644 index 0000000..2b024b1 --- /dev/null +++ b/aw-server/src/device_id.rs @@ -0,0 +1,19 @@ +use std::fs; + +use uuid::Uuid; + +use crate::dirs; + +/// Retrieves the device ID, if none exists it generates one (using UUID v4) +pub fn get_device_id() -> String { + // Device IDs are mutable app data, so they live next to the datastore rather than config. + let mut path = dirs::get_data_dir().unwrap(); + path.push("device_id"); + if path.exists() { + fs::read_to_string(path).unwrap() + } else { + let uuid = Uuid::new_v4().as_hyphenated().to_string(); + fs::write(path, &uuid).unwrap(); + uuid + } +} diff --git a/aw-server/src/dirs.rs b/aw-server/src/dirs.rs new file mode 100644 index 0000000..d677bda --- /dev/null +++ b/aw-server/src/dirs.rs @@ -0,0 +1,583 @@ +// The public dir helpers use `Result<_, ()>` and are part of the crate API. +#![allow(clippy::result_unit_err)] + +use std::fs; +use std::path::{Path, PathBuf}; + +#[cfg(target_os = "android")] +use std::sync::Mutex; + +#[cfg(target_os = "android")] +lazy_static! { + static ref ANDROID_DATA_DIR: Mutex = Mutex::new(PathBuf::from( + "/data/user/0/net.activitywatch.android/files" + )); +} + +const DEFAULT_APPNAME: &str = "activitywatch"; +const TESTING_PROFILE: &str = "testing"; +const TESTING_APPNAME: &str = "activitywatch-testing"; + +/// Filenames that mark a machine as still using the pre-profile shared-root +/// testing layout (ActivityWatch/activitywatch#1399). Keep this list specific: +/// a false positive would pin a fresh install to the legacy layout forever. +/// Identical to the python list in aw-core so both sides agree on disk state. +const LEGACY_TESTING_FILENAME_MARKERS: &[&str] = &[ + "peewee-sqlite-testing", + "sqlite-testing", + "settings-testing", + "config-testing", + "-testing.db", + "-testing.toml", + "-testing.json", + "_testing_", +]; + +/// Platform "appname" root for the current profile. +/// +/// Named profiles use a sibling root (`activitywatch-research`, …). `default` +/// keeps the bare `activitywatch` root. `testing` follows the new-root-plus +/// legacy-fallback rule from ActivityWatch/activitywatch#1399 — see +/// [`using_legacy_testing_root`]. +#[cfg(not(target_os = "android"))] +pub fn appname() -> String { + appname_for(crate::config::get_profile()) +} + +/// Platform appname for a given profile, observing on-disk state for `testing`. +#[cfg(not(target_os = "android"))] +pub fn appname_for(profile: &str) -> String { + match platform_roots() { + Some((data, config, cache)) => appname_for_in(profile, &data, &config, &cache), + None => appname_for_in(profile, Path::new(""), Path::new(""), Path::new("")), + } +} + +/// Pure appname resolution against explicit XDG-style parent dirs (testable). +pub fn appname_for_in(profile: &str, data: &Path, config: &Path, cache: &Path) -> String { + if profile.is_empty() || profile == "default" { + return DEFAULT_APPNAME.to_string(); + } + if profile == TESTING_PROFILE && using_legacy_testing_root_in(profile, data, config, cache) { + return DEFAULT_APPNAME.to_string(); + } + format!("{DEFAULT_APPNAME}-{profile}") +} + +fn platform_roots() -> Option<(PathBuf, PathBuf, PathBuf)> { + Some((dirs::data_dir()?, dirs::config_dir()?, dirs::cache_dir()?)) +} + +fn is_legacy_testing_filename(name: &str) -> bool { + let lower = name.to_lowercase(); + LEGACY_TESTING_FILENAME_MARKERS + .iter() + .any(|marker| lower.contains(marker)) +} + +fn dir_has_legacy_testing_file(dir: &Path) -> bool { + let Ok(entries) = fs::read_dir(dir) else { + return false; + }; + entries.flatten().any(|entry| { + entry.file_type().map(|t| t.is_file()).unwrap_or(false) + && is_legacy_testing_filename(&entry.file_name().to_string_lossy()) + }) +} + +/// Walk `activitywatch/` plus one extra level (`activitywatch/aw-server-rust/`). +fn legacy_testing_artifacts_in_app_root(app_root: &Path) -> bool { + if !app_root.is_dir() { + return false; + } + if dir_has_legacy_testing_file(app_root) { + return true; + } + let Ok(entries) = fs::read_dir(app_root) else { + return false; + }; + entries.flatten().any(|entry| { + let path = entry.path(); + path.is_dir() && dir_has_legacy_testing_file(&path) + }) +} + +fn new_testing_root_exists_in(data: &Path, config: &Path, cache: &Path) -> bool { + [data, config, cache] + .iter() + .any(|root| root.join(TESTING_APPNAME).is_dir()) +} + +fn legacy_testing_artifacts_exist_in(data: &Path, config: &Path, cache: &Path) -> bool { + [data, config, cache] + .iter() + .any(|root| legacy_testing_artifacts_in_app_root(&root.join(DEFAULT_APPNAME))) +} + +/// Testing-root resolution against explicit parent dirs (testable). +/// +/// Rule (ActivityWatch/activitywatch#1399), identical on python and rust: +/// +/// 1. If `activitywatch-testing/` already exists: use it (new layout). +/// 2. Else if legacy testing artifacts exist in the bare `activitywatch/` +/// root: stay in legacy mode (old paths, old filenames). +/// 3. Else (fresh setup): create and use `activitywatch-testing/`. +pub fn using_legacy_testing_root_in( + profile: &str, + data: &Path, + config: &Path, + cache: &Path, +) -> bool { + if profile != TESTING_PROFILE { + return false; + } + if new_testing_root_exists_in(data, config, cache) { + return false; + } + legacy_testing_artifacts_exist_in(data, config, cache) +} + +/// Whether `profile=testing` should stay on the shared `activitywatch` root. +pub fn using_legacy_testing_root(profile: &str) -> bool { + match platform_roots() { + Some((data, config, cache)) => { + using_legacy_testing_root_in(profile, &data, &config, &cache) + } + None => false, + } +} + +/// `"-testing"` only when testing data still shares the default root. +/// +/// Isolated profile roots (including new-style `activitywatch-testing/`) use +/// bare filenames: the directory already isolates. Suffixed names remain only +/// in legacy mode so existing `sqlite-testing.db` files keep working. +pub fn legacy_testing_suffix(profile: &str) -> &'static str { + if using_legacy_testing_root(profile) { + "-testing" + } else { + "" + } +} + +fn db_filename_for_legacy(legacy: bool) -> &'static str { + if legacy { + "sqlite-testing.db" + } else { + "sqlite.db" + } +} + +fn config_filename_for_legacy(legacy: bool) -> &'static str { + if legacy { + "config-testing.toml" + } else { + "config.toml" + } +} + +/// Database filename for a profile. Isolated roots use `sqlite.db`; legacy +/// testing keeps `sqlite-testing.db`. +pub fn db_filename(profile: &str) -> String { + db_filename_for_legacy(using_legacy_testing_root(profile)).to_string() +} + +/// Config filename for a profile. Isolated roots use `config.toml`; legacy +/// testing keeps `config-testing.toml`. +pub fn config_filename(profile: &str) -> String { + config_filename_for_legacy(using_legacy_testing_root(profile)).to_string() +} + +#[cfg(not(target_os = "android"))] +pub fn get_config_dir() -> Result { + let dir = dirs::config_dir() + .ok_or(())? + .join(appname()) + .join("aw-server-rust"); + fs::create_dir_all(&dir).expect("Unable to create config dir"); + Ok(dir) +} + +#[cfg(target_os = "android")] +pub fn get_config_dir() -> Result { + Ok(ANDROID_DATA_DIR.lock().unwrap().to_path_buf()) +} + +#[cfg(not(target_os = "android"))] +pub fn get_data_dir() -> Result { + let dir = dirs::data_dir() + .ok_or(())? + .join(appname()) + .join("aw-server-rust"); + fs::create_dir_all(&dir).expect("Unable to create data dir"); + Ok(dir) +} + +#[cfg(target_os = "android")] +pub fn get_data_dir() -> Result { + return Ok(ANDROID_DATA_DIR.lock().unwrap().to_path_buf()); +} + +#[cfg(not(target_os = "android"))] +pub fn get_cache_dir() -> Result { + let dir = dirs::cache_dir() + .ok_or(())? + .join(appname()) + .join("aw-server-rust"); + fs::create_dir_all(&dir).expect("Unable to create cache dir"); + Ok(dir) +} + +#[cfg(target_os = "android")] +pub fn get_cache_dir() -> Result { + panic!("not implemented on Android"); +} + +#[cfg(not(target_os = "android"))] +pub fn get_log_dir(module: &str) -> Result { + let dir = get_user_log_dir()?.join(module); + fs::create_dir_all(&dir).expect("Unable to create log dir"); + Ok(dir) +} + +/// Returns the platform-appropriate log directory for ActivityWatch. +/// +/// Replicates the behavior of the old `appdirs::user_log_dir("activitywatch")`: +/// - Linux: ~/.cache/activitywatch/log/ +/// - macOS: ~/Library/Logs/activitywatch/ +/// - Windows: {LOCALAPPDATA}\activitywatch\Logs\ +#[cfg(target_os = "linux")] +fn get_user_log_dir() -> Result { + Ok(dirs::cache_dir().ok_or(())?.join(appname()).join("log")) +} + +#[cfg(target_os = "macos")] +fn get_user_log_dir() -> Result { + Ok(dirs::home_dir() + .ok_or(())? + .join("Library") + .join("Logs") + .join(appname())) +} + +#[cfg(target_os = "windows")] +fn get_user_log_dir() -> Result { + Ok(dirs::data_local_dir() + .ok_or(())? + .join(appname()) + .join("Logs")) +} + +#[cfg(target_os = "android")] +pub fn get_log_dir(module: &str) -> Result { + panic!("not implemented on Android"); +} + +/// Validate a profile name: lowercase alphanumerics plus `-` and `_`, max 32 +/// chars, must start with a letter or digit. Returns Err(message) on failure. +pub fn validate_profile(name: &str) -> Result<(), String> { + if name.is_empty() { + return Err("profile name must not be empty".into()); + } + if name.len() > 32 { + return Err(format!( + "profile name too long ({} chars, max 32)", + name.len() + )); + } + let first = name.chars().next().unwrap(); + if !first.is_ascii_alphanumeric() { + return Err(format!( + "profile name must start with a letter or digit, got '{first}'" + )); + } + for c in name.chars() { + if !c.is_ascii_alphanumeric() && c != '-' && c != '_' { + return Err(format!("invalid character '{c}' in profile name")); + } + } + if name != name.to_lowercase() { + return Err("profile name must be lowercase".into()); + } + Ok(()) +} + +/// Data dir for an explicit profile (does not depend on the process-global +/// `OnceLock`). Creates the directory. Used by `db_path` so a caller asking +/// for `research` cannot land in the default root just because `set_profile` +/// has not run yet. +#[cfg(not(target_os = "android"))] +fn get_data_dir_for(profile: &str) -> Result { + let dir = dirs::data_dir() + .ok_or(())? + .join(appname_for(profile)) + .join("aw-server-rust"); + fs::create_dir_all(&dir).expect("Unable to create data dir"); + Ok(dir) +} + +#[cfg(target_os = "android")] +fn get_data_dir_for(_profile: &str) -> Result { + get_data_dir() +} + +pub fn db_path(profile: &str) -> Result { + let mut db_path = get_data_dir_for(profile)?; + db_path.push(db_filename(profile)); + Ok(db_path) +} + +#[cfg(target_os = "android")] +pub fn set_android_data_dir(path: &str) { + let mut android_data_dir = ANDROID_DATA_DIR.lock().unwrap(); + *android_data_dir = PathBuf::from(path); +} + +#[cfg(test)] +fn fake_roots() -> (PathBuf, PathBuf, PathBuf, PathBuf) { + let root = std::env::temp_dir() + .join("aw-testing-root-fallback") + .join(uuid::Uuid::new_v4().to_string()); + let data = root.join("data"); + let config = root.join("config"); + let cache = root.join("cache"); + fs::create_dir_all(&data).unwrap(); + fs::create_dir_all(&config).unwrap(); + fs::create_dir_all(&cache).unwrap(); + (root, data, config, cache) +} + +#[cfg(test)] +fn plant_legacy_testing_db(data: &Path) -> PathBuf { + let aw_server = data.join("activitywatch").join("aw-server-rust"); + fs::create_dir_all(&aw_server).unwrap(); + let marker = aw_server.join("sqlite-testing.db"); + fs::write(&marker, b"").unwrap(); + marker +} + +#[cfg(not(target_os = "android"))] +#[test] +fn test_appname_root_isolation() { + let (_root, data, config, cache) = fake_roots(); + assert_eq!( + appname_for_in("default", &data, &config, &cache), + "activitywatch" + ); + // Fresh setup: testing uses the isolated sibling root. + assert_eq!( + appname_for_in("testing", &data, &config, &cache), + "activitywatch-testing" + ); + assert_eq!( + appname_for_in("research", &data, &config, &cache), + "activitywatch-research" + ); + assert_eq!( + appname_for_in("my-profile", &data, &config, &cache), + "activitywatch-my-profile" + ); + let _ = fs::remove_dir_all(_root); +} + +#[test] +fn test_testing_root_fresh_setup_uses_new_root() { + let (_root, data, config, cache) = fake_roots(); + assert!(!using_legacy_testing_root_in( + "testing", &data, &config, &cache + )); + assert_eq!( + appname_for_in("testing", &data, &config, &cache), + "activitywatch-testing" + ); + let _ = fs::remove_dir_all(_root); +} + +#[test] +fn test_testing_root_legacy_artifacts_keep_shared_root() { + let (_root, data, config, cache) = fake_roots(); + plant_legacy_testing_db(&data); + assert!(using_legacy_testing_root_in( + "testing", &data, &config, &cache + )); + assert_eq!( + appname_for_in("testing", &data, &config, &cache), + "activitywatch" + ); + assert!(!data.join("activitywatch-testing").exists()); + let _ = fs::remove_dir_all(_root); +} + +#[test] +fn test_testing_root_new_root_wins_over_legacy_artifacts() { + let (_root, data, config, cache) = fake_roots(); + plant_legacy_testing_db(&data); + fs::create_dir_all(data.join("activitywatch-testing")).unwrap(); + assert!(!using_legacy_testing_root_in( + "testing", &data, &config, &cache + )); + assert_eq!( + appname_for_in("testing", &data, &config, &cache), + "activitywatch-testing" + ); + let _ = fs::remove_dir_all(_root); +} + +#[test] +fn test_config_testing_toml_is_a_legacy_marker() { + let (_root, data, config, cache) = fake_roots(); + let cfg = config.join("activitywatch"); + fs::create_dir_all(&cfg).unwrap(); + fs::write(cfg.join("config-testing.toml"), b"").unwrap(); + assert!(using_legacy_testing_root_in( + "testing", &data, &config, &cache + )); + assert_eq!( + appname_for_in("testing", &data, &config, &cache), + "activitywatch" + ); + let _ = fs::remove_dir_all(_root); +} + +#[test] +fn test_named_profile_never_uses_legacy_root() { + let (_root, data, config, cache) = fake_roots(); + plant_legacy_testing_db(&data); + assert!(!using_legacy_testing_root_in( + "research", &data, &config, &cache + )); + assert_eq!( + appname_for_in("research", &data, &config, &cache), + "activitywatch-research" + ); + let _ = fs::remove_dir_all(_root); +} + +#[cfg(test)] +fn db_filename_in(profile: &str, data: &Path, config: &Path, cache: &Path) -> &'static str { + db_filename_for_legacy(using_legacy_testing_root_in(profile, data, config, cache)) +} + +#[cfg(test)] +fn config_filename_in(profile: &str, data: &Path, config: &Path, cache: &Path) -> &'static str { + config_filename_for_legacy(using_legacy_testing_root_in(profile, data, config, cache)) +} + +#[test] +fn test_filenames_bare_except_legacy_testing() { + assert_eq!(db_filename_for_legacy(false), "sqlite.db"); + assert_eq!(db_filename_for_legacy(true), "sqlite-testing.db"); + assert_eq!(config_filename_for_legacy(false), "config.toml"); + assert_eq!(config_filename_for_legacy(true), "config-testing.toml"); +} + +#[test] +fn test_filenames_follow_disk_state_rule() { + let (_root, data, config, cache) = fake_roots(); + assert_eq!( + db_filename_in("testing", &data, &config, &cache), + "sqlite.db" + ); + assert_eq!( + config_filename_in("testing", &data, &config, &cache), + "config.toml" + ); + assert_eq!( + db_filename_in("research", &data, &config, &cache), + "sqlite.db" + ); + + plant_legacy_testing_db(&data); + assert_eq!( + db_filename_in("testing", &data, &config, &cache), + "sqlite-testing.db" + ); + assert_eq!( + config_filename_in("testing", &data, &config, &cache), + "config-testing.toml" + ); + // Named profiles stay bare even when legacy testing artifacts exist. + assert_eq!( + db_filename_in("research", &data, &config, &cache), + "sqlite.db" + ); + assert_eq!( + config_filename_in("research", &data, &config, &cache), + "config.toml" + ); + + fs::create_dir_all(data.join("activitywatch-testing")).unwrap(); + assert_eq!( + db_filename_in("testing", &data, &config, &cache), + "sqlite.db" + ); + assert_eq!( + config_filename_in("testing", &data, &config, &cache), + "config.toml" + ); + let _ = fs::remove_dir_all(_root); +} + +#[test] +fn test_validate_profile() { + assert!(validate_profile("default").is_ok()); + assert!(validate_profile("testing").is_ok()); + assert!(validate_profile("research").is_ok()); + assert!(validate_profile("my-profile").is_ok()); + assert!(validate_profile("profile_1").is_ok()); + + assert!(validate_profile("").is_err()); + assert!( + validate_profile("Research").is_err(), + "uppercase should be rejected" + ); + assert!(validate_profile("-bad").is_err(), "must start with alnum"); + assert!(validate_profile("bad name").is_err(), "spaces not allowed"); + assert!( + validate_profile("a/b").is_err(), + "path separator not allowed" + ); + assert!(validate_profile(&"a".repeat(33)).is_err(), "too long"); +} + +#[test] +fn test_get_dirs() { + #[cfg(target_os = "android")] + set_android_data_dir("/test"); + + get_cache_dir().unwrap(); + get_log_dir("aw-server-rust").unwrap(); + // Do not call db_path("testing"): on a fresh CI machine that would create + // ~/.local/share/activitywatch-testing and pin later tests to the new root. + db_path("default").unwrap(); +} + +#[test] +#[cfg(not(target_os = "android"))] +fn test_log_dir_has_log_component() { + let log_dir = get_log_dir("aw-server-rust").unwrap(); + let path_str = log_dir.to_string_lossy(); + + // The log path must contain a log-specific subdirectory, not just the cache dir. + // This guards against the regression from PR #562 where /log was dropped. + #[cfg(target_os = "linux")] + assert!( + path_str.contains("activitywatch/log/"), + "Linux log path should contain activitywatch/log/, got: {}", + path_str + ); + + #[cfg(target_os = "macos")] + assert!( + path_str.contains("Library/Logs/activitywatch"), + "macOS log path should use Library/Logs, got: {}", + path_str + ); + + #[cfg(target_os = "windows")] + assert!( + path_str.contains("activitywatch\\Logs\\") || path_str.contains("activitywatch/Logs/"), + "Windows log path should contain activitywatch/Logs, got: {}", + path_str + ); +} diff --git a/aw-server/src/endpoints/apikey.rs b/aw-server/src/endpoints/apikey.rs new file mode 100644 index 0000000..e1234cf --- /dev/null +++ b/aw-server/src/endpoints/apikey.rs @@ -0,0 +1,409 @@ +//! API key authentication via Bearer token. +//! +//! When `api_key` is set under `[auth]` in the config, all API endpoints except +//! `/api/0/info` require an `Authorization: Bearer ` header. Requests +//! missing or presenting an invalid key receive a 401 Unauthorized response. +//! +//! By default `api_key` is `None`, meaning authentication is disabled. +//! To enable, add to `config.toml`: +//! +//! ```toml +//! [auth] +//! api_key = "your-secret-key-here" +//! ``` +//! +//! Exempt paths (always public): +//! - `GET /api/0/info` — health/version endpoint used by clients and the webui +//! +//! CORS preflight requests (OPTIONS) are also passed through unconditionally so +//! the browser can obtain allowed headers before sending the actual request. +//! +//! # Path matching +//! +//! The gate matches on `request.uri().path().segments()` — the same +//! percent-decoded, empty-skipping segment view Rocket's router uses to select +//! a handler (see `rocket::router::collider::paths_match`). Comparing the raw +//! path string instead lets an attacker desynchronize the gate from the router +//! (e.g. `/%61pi/0/buckets/` or `//api/0/buckets/`), skipping auth on a request +//! the router still dispatches to a real `/api/` handler. + +use subtle::ConstantTimeEq; + +use rocket::fairing::Fairing; +use rocket::http::uri::Origin; +use rocket::http::{Method, Status}; +use rocket::route::Outcome; +use rocket::{Data, Request, Rocket, Route}; + +use crate::config::AWConfig; +use crate::endpoints::HttpErrorJson; + +static FAIRING_ROUTE_BASE: &str = "/apikey_fairing"; + +/// Path segments gated by API key authentication. +const API_SEGMENT: &str = "api"; + +/// Paths that are always accessible without authentication, as decoded segments. +const PUBLIC_PATHS: &[&[&str]] = &[&["api", "0", "info"]]; + +pub struct ApiKeyCheck { + api_key: Option, +} + +impl ApiKeyCheck { + pub fn new(config: &AWConfig) -> ApiKeyCheck { + let api_key = match &config.auth.api_key { + Some(k) if k.is_empty() => { + warn!("api_key is set to an empty string — authentication is disabled. Set a non-empty key to enable auth."); + None + } + other => other.clone(), + }; + ApiKeyCheck { api_key } + } +} + +/// Route handler that returns 401 Unauthorized for failed auth checks. +#[derive(Clone)] +struct FairingErrorRoute {} + +#[rocket::async_trait] +impl rocket::route::Handler for FairingErrorRoute { + async fn handle<'r>( + &self, + request: &'r Request<'_>, + _: rocket::Data<'r>, + ) -> rocket::route::Outcome<'r> { + let err = HttpErrorJson::new( + Status::Unauthorized, + "Missing or invalid API key. Set 'Authorization: Bearer ' header.".to_string(), + ); + Outcome::from(request, err) + } +} + +fn fairing_route() -> Route { + Route::ranked(1, Method::Get, "/", FairingErrorRoute {}) +} + +fn redirect_unauthorized(request: &mut Request) { + let uri = FAIRING_ROUTE_BASE.to_string(); + let origin = Origin::parse_owned(uri).unwrap(); + request.set_method(Method::Get); + request.set_uri(origin); +} + +#[rocket::async_trait] +impl Fairing for ApiKeyCheck { + fn info(&self) -> rocket::fairing::Info { + rocket::fairing::Info { + name: "ApiKeyCheck", + kind: rocket::fairing::Kind::Ignite | rocket::fairing::Kind::Request, + } + } + + async fn on_ignite(&self, rocket: Rocket) -> rocket::fairing::Result { + match &self.api_key { + Some(_) => Ok(rocket.mount(FAIRING_ROUTE_BASE, vec![fairing_route()])), + None => { + debug!("API key authentication is disabled"); + Ok(rocket) + } + } + } + + async fn on_request(&self, request: &mut Request<'_>, _: &mut Data<'_>) { + let api_key = match &self.api_key { + None => return, // auth disabled + Some(k) => k, + }; + + // Always allow OPTIONS (CORS preflight) + if request.method() == Method::Options { + return; + } + + // Match on the same decoded segment view the router uses to dispatch, + // so the gate cannot be desynchronized from the handler it protects. + let segments: Vec<&str> = request.uri().path().segments().collect(); + + // Only gate API endpoints — static web UI assets are not under /api/ + if segments.first() != Some(&API_SEGMENT) { + return; + } + + // Always allow public API paths (e.g. /api/0/info for health checks) + if PUBLIC_PATHS.contains(&segments.as_slice()) { + return; + } + + // Validate Authorization: Bearer + // Use constant-time comparison to prevent timing attacks. + let auth_header = request.headers().get_one("Authorization"); + let valid = match auth_header { + Some(value) => { + if let Some(token) = value.strip_prefix("Bearer ") { + token.as_bytes().ct_eq(api_key.as_bytes()).into() + } else { + false + } + } + None => false, + }; + + if !valid { + debug!("API key check failed for {}", request.uri()); + redirect_unauthorized(request); + } + } +} + +#[cfg(test)] +mod tests { + + use rocket::http::{ContentType, Header, Status}; + use rocket::Rocket; + + use crate::config::AWConfig; + use crate::endpoints; + + fn setup_testserver(api_key: Option) -> Rocket { + let state = endpoints::ServerState { + datastore: aw_datastore::Datastore::new_in_memory(false), + asset_resolver: endpoints::AssetResolver::new(None), + device_id: "test_id".to_string(), + }; + let mut aw_config = AWConfig::default(); + aw_config.auth.api_key = api_key; + endpoints::build_rocket(state, aw_config) + } + + #[test] + fn test_no_api_key_configured() { + // When no api_key is set, all endpoints are accessible without auth. + let server = setup_testserver(None); + let client = rocket::local::blocking::Client::tracked(server).expect("valid instance"); + + let res = client + .get("/api/0/info") + .header(ContentType::JSON) + .header(Header::new("Host", "localhost:5600")) + .dispatch(); + assert_eq!(res.status(), Status::Ok); + + let res = client + .get("/api/0/buckets/") + .header(ContentType::JSON) + .header(Header::new("Host", "localhost:5600")) + .dispatch(); + assert_eq!(res.status(), Status::Ok); + } + + #[test] + fn test_api_key_required() { + // With api_key set, requests without a key should be rejected. + let server = setup_testserver(Some("secret123".to_string())); + let client = rocket::local::blocking::Client::tracked(server).expect("valid instance"); + + // /api/0/info is always public + let res = client + .get("/api/0/info") + .header(ContentType::JSON) + .header(Header::new("Host", "localhost:5600")) + .dispatch(); + assert_eq!(res.status(), Status::Ok); + + // Other endpoints require auth + let res = client + .get("/api/0/buckets/") + .header(ContentType::JSON) + .header(Header::new("Host", "localhost:5600")) + .dispatch(); + assert_eq!(res.status(), Status::Unauthorized); + + // Double slash should also require auth + let res = client + .get("//api/0/buckets/") + .header(ContentType::JSON) + .header(Header::new("Host", "localhost:5600")) + .dispatch(); + assert_eq!(res.status(), Status::Unauthorized); + } + + /// Percent-encoded paths must not bypass the gate. + /// + /// Rocket's router decodes the path before matching, so a fairing that + /// compares the *raw* path string can be desynchronized from the handler + /// it protects: `/%61pi/0/buckets/` reaches the `/api/0/buckets/` handler + /// while a raw `starts_with("/api/")` check sees `/%61pi/...` and skips auth. + #[test] + fn test_api_key_percent_encoded_paths_require_auth() { + let server = setup_testserver(Some("secret123".to_string())); + let client = rocket::local::blocking::Client::tracked(server).expect("valid instance"); + + // Each of these decodes to the real /api/0/buckets/ route, so each must + // be rejected by the auth gate specifically — asserting "not 200" would + // also pass on a 404, hiding a route miss as an auth success. + for path in [ + "/%61pi/0/buckets/", // lowercase hex, first char + "/ap%69/0/buckets/", // encoded char mid-segment + "/%61%70%69/0/buckets/", // fully encoded segment + "//%61pi/0/buckets/", // encoding combined with the #588 double-slash trick + ] { + let res = client + .get(path) + .header(ContentType::JSON) + .header(Header::new("Host", "localhost:5600")) + .dispatch(); + assert_eq!( + res.status(), + Status::Unauthorized, + "expected auth rejection for encoded path {path}" + ); + } + + // Sanity check on the above: these paths really do reach the buckets + // handler once a valid key is supplied, so the 401s are the gate + // rejecting them rather than the router failing to match. + for path in ["/%61pi/0/buckets/", "//%61pi/0/buckets/"] { + let res = client + .get(path) + .header(ContentType::JSON) + .header(Header::new("Host", "localhost:5600")) + .header(Header::new("Authorization", "Bearer secret123")) + .dispatch(); + assert_eq!( + res.status(), + Status::Ok, + "encoded path {path} does not reach the real handler" + ); + } + + // Case is significant: /API/ is not a registered route at all, so this + // 404s at the router rather than exercising the gate. Asserted + // explicitly so a future case-insensitive router change would show up + // here instead of silently passing a "not 200" check. + let res = client + .get("/%41PI/0/buckets/") + .header(ContentType::JSON) + .header(Header::new("Host", "localhost:5600")) + .dispatch(); + assert_eq!(res.status(), Status::NotFound); + } + + /// Writes must be gated too — a bypass that only blocked GETs would still + /// allow bucket creation / event insertion. + #[test] + fn test_api_key_percent_encoded_write_requires_auth() { + let server = setup_testserver(Some("secret123".to_string())); + let client = rocket::local::blocking::Client::tracked(server).expect("valid instance"); + + let res = client + .post("/%61pi/0/buckets/authbypass-poc") + .header(ContentType::JSON) + .header(Header::new("Host", "localhost:5600")) + .body(r#"{"type":"test","client":"authbypass-poc","hostname":"poc-host"}"#) + .dispatch(); + assert_eq!( + res.status(), + Status::Unauthorized, + "write bypassed via encoded path" + ); + + // And the bucket must not exist afterwards — 404 from an authenticated + // lookup, not merely "some non-200 status". + let res = client + .get("/api/0/buckets/authbypass-poc") + .header(ContentType::JSON) + .header(Header::new("Host", "localhost:5600")) + .header(Header::new("Authorization", "Bearer secret123")) + .dispatch(); + assert_eq!( + res.status(), + Status::NotFound, + "bucket was created without auth" + ); + } + + /// The public-path exemption is matched on decoded segments, so encoded + /// spellings of `/api/0/info` stay public (parity with the router) while + /// nothing else slips through the exemption. + #[test] + fn test_public_path_matched_on_decoded_segments() { + let server = setup_testserver(Some("secret123".to_string())); + let client = rocket::local::blocking::Client::tracked(server).expect("valid instance"); + + let res = client + .get("/api/0/%69nfo") + .header(ContentType::JSON) + .header(Header::new("Host", "localhost:5600")) + .dispatch(); + assert_eq!(res.status(), Status::Ok); + + // A longer path that merely starts with the public prefix is not public: + // the exemption is an exact segment match, so this hits the gate. + let res = client + .get("/api/0/info/../buckets/") + .header(ContentType::JSON) + .header(Header::new("Host", "localhost:5600")) + .dispatch(); + assert_eq!(res.status(), Status::Unauthorized); + } + + #[test] + fn test_api_key_valid() { + let server = setup_testserver(Some("secret123".to_string())); + let client = rocket::local::blocking::Client::tracked(server).expect("valid instance"); + + let res = client + .get("/api/0/buckets/") + .header(ContentType::JSON) + .header(Header::new("Host", "localhost:5600")) + .header(Header::new("Authorization", "Bearer secret123")) + .dispatch(); + assert_eq!(res.status(), Status::Ok); + } + + #[test] + fn test_api_key_invalid() { + let server = setup_testserver(Some("secret123".to_string())); + let client = rocket::local::blocking::Client::tracked(server).expect("valid instance"); + + let res = client + .get("/api/0/buckets/") + .header(ContentType::JSON) + .header(Header::new("Host", "localhost:5600")) + .header(Header::new("Authorization", "Bearer wrongkey")) + .dispatch(); + assert_eq!(res.status(), Status::Unauthorized); + } + + #[test] + fn test_api_key_wrong_scheme() { + // Must be Bearer, not Basic or bare key + let server = setup_testserver(Some("secret123".to_string())); + let client = rocket::local::blocking::Client::tracked(server).expect("valid instance"); + + let res = client + .get("/api/0/buckets/") + .header(ContentType::JSON) + .header(Header::new("Host", "localhost:5600")) + .header(Header::new("Authorization", "Basic secret123")) + .dispatch(); + assert_eq!(res.status(), Status::Unauthorized); + } + + #[test] + fn test_empty_api_key_disables_auth() { + // An empty string key should be treated as disabled (no auth required). + let server = setup_testserver(Some("".to_string())); + let client = rocket::local::blocking::Client::tracked(server).expect("valid instance"); + + let res = client + .get("/api/0/buckets/") + .header(ContentType::JSON) + .header(Header::new("Host", "localhost:5600")) + .dispatch(); + assert_eq!(res.status(), Status::Ok); + } +} diff --git a/aw-server/src/endpoints/bucket.rs b/aw-server/src/endpoints/bucket.rs new file mode 100644 index 0000000..10e67e6 --- /dev/null +++ b/aw-server/src/endpoints/bucket.rs @@ -0,0 +1,233 @@ +use std::collections::HashMap; + +use gethostname::gethostname; +use rocket::serde::json::Json; + +use chrono::DateTime; +use chrono::Utc; + +use aw_datastore::DatastoreError; +use aw_models::Bucket; +use aw_models::Event; + +use rocket::http::Status; +use rocket::State; + +use crate::endpoints::util::BucketsExportRocket; +use crate::endpoints::{HttpErrorJson, ServerState}; + +#[get("/")] +pub fn buckets_get( + state: &State, +) -> Result>, HttpErrorJson> { + let datastore = &state.datastore; + match datastore.get_buckets() { + Ok(bucketlist) => Ok(Json(bucketlist)), + Err(err) => Err(err.into()), + } +} + +#[get("/")] +pub fn bucket_get( + bucket_id: &str, + state: &State, +) -> Result, HttpErrorJson> { + let datastore = &state.datastore; + match datastore.get_bucket(bucket_id) { + Ok(bucket) => Ok(Json(bucket)), + Err(e) => Err(e.into()), + } +} + +/// Create a new bucket +/// +/// If hostname is "!local", the hostname and device_id will be set from the server info. +/// This is useful for watchers which are known/assumed to run locally but might not know their hostname (like aw-watcher-web). +#[post("/", data = "", format = "application/json")] +pub fn bucket_new( + bucket_id: &str, + message: Json, + state: &State, +) -> Result<(), HttpErrorJson> { + let mut bucket = message.into_inner(); + if bucket.id != bucket_id { + bucket.id = bucket_id.to_string(); + } + // Reject client-supplied hostnames containing whitespace. Such hostnames come from + // misconfigured clients and produce buckets that are awkward to address and query. + // + // Two deliberate exemptions keep this from breaking working setups: + // - The "!local" sentinel is resolved server-side below and is not validated here, + // so a machine whose own hostname contains a space is unaffected. + // - Buckets that already exist are not validated, so watchers that idempotently + // re-create a pre-existing bucket on startup keep getting the established + // "already exists" response instead of a new hard error. + if bucket.hostname != "!local" && bucket.hostname.contains(char::is_whitespace) { + // Distinguish "bucket genuinely absent" from "datastore unavailable": + // only the former should trigger the whitespace rejection (400). An + // InternalError means the datastore worker is down and must become 500, + // not be silently swallowed and misreported as invalid client input. + match state.datastore.get_bucket(&bucket.id) { + Ok(_) => {} // existing bucket — skip validation; create_bucket returns 304 + Err(DatastoreError::NoSuchBucket(_)) => { + // Format the hostname with Debug to escape control characters, so + // untrusted request data cannot inject newlines into the log. + let err_msg = format!( + "Invalid hostname {:?}: hostname may not contain whitespace", + bucket.hostname + ); + warn!("{}", err_msg); + return Err(HttpErrorJson::new(Status::BadRequest, err_msg)); + } + Err(e) => return Err(e.into()), // datastore failure → 500 + } + } + if bucket.hostname == "!local" { + bucket.hostname = gethostname() + .into_string() + .unwrap_or_else(|_| "unknown".to_string()); + bucket + .data + .insert("device_id".to_string(), state.device_id.clone().into()); + } + let datastore = &state.datastore; + let ret = datastore.create_bucket(&bucket); + match ret { + Ok(_) => Ok(()), + Err(err) => Err(err.into()), + } +} + +#[get("//events?&&")] +pub fn bucket_events_get( + bucket_id: &str, + start: Option, + end: Option, + limit: Option, + state: &State, +) -> Result>, HttpErrorJson> { + let starttime: Option> = match start { + Some(dt_str) => match DateTime::parse_from_rfc3339(&dt_str) { + Ok(dt) => Some(dt.with_timezone(&Utc)), + Err(e) => { + let err_msg = format!( + "Failed to parse starttime, datetime needs to be in rfc3339 format: {e}" + ); + warn!("{}", err_msg); + return Err(HttpErrorJson::new(Status::BadRequest, err_msg)); + } + }, + None => None, + }; + let endtime: Option> = match end { + Some(dt_str) => match DateTime::parse_from_rfc3339(&dt_str) { + Ok(dt) => Some(dt.with_timezone(&Utc)), + Err(e) => { + let err_msg = + format!("Failed to parse endtime, datetime needs to be in rfc3339 format: {e}"); + warn!("{}", err_msg); + return Err(HttpErrorJson::new(Status::BadRequest, err_msg)); + } + }, + None => None, + }; + let datastore = &state.datastore; + let res = datastore.get_events(bucket_id, starttime, endtime, limit); + match res { + Ok(events) => Ok(Json(events)), + Err(err) => Err(err.into()), + } +} + +// Needs unused parameter, otherwise there'll be a route collision +// See: https://api.rocket.rs/master/rocket/struct.Route.html#resolving-collisions +#[get("//events/?<_unused..>")] +pub fn bucket_events_get_single( + bucket_id: &str, + event_id: i64, + _unused: Option, + state: &State, +) -> Result, HttpErrorJson> { + let datastore = &state.datastore; + let res = datastore.get_event(bucket_id, event_id); + match res { + Ok(events) => Ok(Json(events)), + Err(err) => Err(err.into()), + } +} + +#[post("//events", data = "", format = "application/json")] +pub fn bucket_events_create( + bucket_id: &str, + events: Json>, + state: &State, +) -> Result>, HttpErrorJson> { + let datastore = &state.datastore; + let res = datastore.insert_events(bucket_id, &events); + match res { + Ok(events) => Ok(Json(events)), + Err(err) => Err(err.into()), + } +} + +#[post( + "//heartbeat?", + data = "", + format = "application/json" +)] +pub fn bucket_events_heartbeat( + bucket_id: &str, + heartbeat_json: Json, + pulsetime: f64, + state: &State, +) -> Result, HttpErrorJson> { + let heartbeat = heartbeat_json.into_inner(); + let datastore = &state.datastore; + match datastore.heartbeat(bucket_id, heartbeat, pulsetime) { + Ok(e) => Ok(Json(e)), + Err(err) => Err(err.into()), + } +} + +#[get("//events/count")] +pub fn bucket_event_count( + bucket_id: &str, + state: &State, +) -> Result, HttpErrorJson> { + let datastore = &state.datastore; + let res = datastore.get_event_count(bucket_id, None, None); + match res { + Ok(eventcount) => Ok(Json(eventcount as u64)), + Err(err) => Err(err.into()), + } +} + +#[delete("//events/")] +pub fn bucket_events_delete_by_id( + bucket_id: &str, + event_id: i64, + state: &State, +) -> Result<(), HttpErrorJson> { + let datastore = &state.datastore; + match datastore.delete_events_by_id(bucket_id, vec![event_id]) { + Ok(_) => Ok(()), + Err(err) => Err(err.into()), + } +} + +#[get("//export")] +pub fn bucket_export( + bucket_id: &str, + state: &State, +) -> Result { + BucketsExportRocket::new(&state.datastore, Some(bucket_id)) +} + +#[delete("/")] +pub fn bucket_delete(bucket_id: &str, state: &State) -> Result<(), HttpErrorJson> { + let datastore = &state.datastore; + match datastore.delete_bucket(bucket_id) { + Ok(_) => Ok(()), + Err(err) => Err(err.into()), + } +} diff --git a/aw-server/src/endpoints/cors.rs b/aw-server/src/endpoints/cors.rs new file mode 100644 index 0000000..ab59958 --- /dev/null +++ b/aw-server/src/endpoints/cors.rs @@ -0,0 +1,47 @@ +use rocket::http::Method; +use rocket_cors::{AllowedHeaders, AllowedOrigins}; + +use crate::config::AWConfig; + +pub fn cors(config: &AWConfig) -> rocket_cors::Cors { + let root_url = format!("http://127.0.0.1:{}", config.port); + let root_url_localhost = format!("http://localhost:{}", config.port); + let mut allowed_exact_origins = vec![root_url, root_url_localhost]; + allowed_exact_origins.extend(config.cors.clone()); + + if config.testing { + allowed_exact_origins.push("http://127.0.0.1:27180".to_string()); + allowed_exact_origins.push("http://localhost:27180".to_string()); + } + let mut allowed_regex_origins = vec![ + "chrome-extension://nglaklhklhcoonedhgnpgddginnjdadi".to_string(), + // Every version of a mozilla extension has its own ID to avoid fingerprinting, so we + // unfortunately have to allow all extensions to have access to aw-server. + // The endpoints this wildcard actually reaches are narrowed to the ones + // aw-watcher-web needs by the ExtensionCorsScope fairing — see + // endpoints/extension_cors.rs. + "moz-extension://.*".to_string(), + ]; + allowed_regex_origins.extend(config.cors_regex.clone()); + if config.testing { + allowed_regex_origins.push("chrome-extension://.*".to_string()); + } + + let allowed_origins = AllowedOrigins::some(&allowed_exact_origins, &allowed_regex_origins); + let allowed_methods = vec![Method::Get, Method::Post, Method::Delete] + .into_iter() + .map(From::from) + .collect(); + let allowed_headers = AllowedHeaders::all(); // TODO: is this unsafe? + + // You can also deserialize this + rocket_cors::CorsOptions { + allowed_origins, + allowed_methods, + allowed_headers, + allow_credentials: false, + ..Default::default() + } + .to_cors() + .expect("Failed to set up CORS") +} diff --git a/aw-server/src/endpoints/export.rs b/aw-server/src/endpoints/export.rs new file mode 100644 index 0000000..4441aff --- /dev/null +++ b/aw-server/src/endpoints/export.rs @@ -0,0 +1,9 @@ +use rocket::State; + +use crate::endpoints::util::BucketsExportRocket; +use crate::endpoints::{HttpErrorJson, ServerState}; + +#[get("/")] +pub fn buckets_export(state: &State) -> Result { + BucketsExportRocket::new(&state.datastore, None) +} diff --git a/aw-server/src/endpoints/extension_cors.rs b/aw-server/src/endpoints/extension_cors.rs new file mode 100644 index 0000000..4bbbf7a --- /dev/null +++ b/aw-server/src/endpoints/extension_cors.rs @@ -0,0 +1,521 @@ +//! Endpoint scoping for the blanket `moz-extension://` CORS origin. +//! +//! Firefox gives every install of every extension its own random origin, so +//! there is no way to allowlist *only* `aw-watcher-web` up front — hence the +//! `moz-extension://.*` wildcard in [`crate::endpoints::cors`]. The cost is +//! that any installed extension can talk to the local API, with no host +//! permission and therefore no install-time prompt naming ActivityWatch. +//! +//! This fairing narrows that blanket trust to the endpoints `aw-watcher-web` +//! actually uses: +//! +//! - `GET /api/0/info` — hostname/version detection +//! - `POST /api/0/buckets/aw-watcher-web-` — ensure its web-watcher bucket +//! - `POST /api/0/buckets/aw-watcher-web-/heartbeat` — heartbeats +//! +//! # Accepted limitation +//! +//! The bucket prefix is a coarse scope, not an origin-to-bucket ownership +//! boundary. Any wildcard-matched Firefox extension can still create or send +//! heartbeats to another existing `aw-watcher-web-*` bucket. This preserves +//! the official watcher's zero-configuration flow while blocking writes to +//! other watcher types; eliminating web-bucket poisoning requires pairing an +//! extension origin with its bucket in a future protocol. +//! +//! Everything else (`/api/0/export`, `/api/0/import`, `/api/0/query`, +//! `/api/0/settings`, event reads, bucket deletion, ...) is closed to +//! wildcard-matched extension origins. That removes the data-exfiltration half +//! of the problem — app names, window titles, tab URLs, AFK data — without any +//! client change. +//! +//! Origins the *user* allowed explicitly via `cors_regex` in `config.toml` +//! are exempt: an explicit allowlist entry is a deliberate opt-in, unlike the +//! built-in wildcard. (The exact `cors` list cannot express an extension +//! origin — rocket_cors rejects `moz-extension://` there as an opaque origin — +//! so `cors_regex` is the only opt-in path for these.) +//! +//! # Enforcement +//! +//! Two stages, because CORS alone only hides *responses*: +//! +//! - `on_request` rejects disallowed requests with 403 before they reach a +//! handler. Without this, a "simple" cross-origin `POST` (e.g. +//! `text/plain`) is sent by the browser and still executes server-side even +//! though the response is unreadable. +//! - `on_response` strips the `Access-Control-*` headers so preflights for +//! disallowed endpoints fail in the browser. +//! +//! # Path matching +//! +//! Like [`crate::endpoints::apikey`], matching is done on +//! `request.uri().path().segments()` — the same percent-decoded, +//! empty-skipping view Rocket's router dispatches on. Matching the raw path +//! string would let `/%61pi/0/export` desynchronize this gate from the handler +//! it protects (the bug class of #588 and #636). + +use regex::RegexSet; + +use rocket::fairing::Fairing; +use rocket::http::uri::Origin; +use rocket::http::{Method, Status}; +use rocket::route::Outcome; +use rocket::{Data, Request, Response, Rocket, Route}; + +use crate::config::AWConfig; +use crate::endpoints::HttpErrorJson; + +static FAIRING_ROUTE_BASE: &str = "/extension_cors_fairing"; + +/// Scheme of the origins covered by the built-in wildcard. +const EXTENSION_ORIGIN_SCHEME: &str = "moz-extension://"; + +/// CORS response headers to strip when an extension origin is out of scope. +const CORS_HEADERS: &[&str] = &[ + "Access-Control-Allow-Origin", + "Access-Control-Allow-Methods", + "Access-Control-Allow-Headers", + "Access-Control-Allow-Credentials", + "Access-Control-Expose-Headers", + "Access-Control-Max-Age", +]; + +pub struct ExtensionCorsScope { + /// Regexes from `[server] cors_regex`, matched unanchored like rocket_cors. + /// The exact `cors` list is not consulted: rocket_cors cannot hold an + /// extension origin there, so it can never allow one. + user_regex_origins: Option, +} + +impl ExtensionCorsScope { + pub fn new(config: &AWConfig) -> ExtensionCorsScope { + let user_regex_origins = if config.cors_regex.is_empty() { + None + } else { + match RegexSet::new(&config.cors_regex) { + Ok(set) => Some(set), + Err(e) => { + // rocket_cors will fail on the same input; don't also panic here. + warn!("Invalid cors_regex in config, ignoring for extension scoping: {e}"); + None + } + } + }; + ExtensionCorsScope { user_regex_origins } + } + + /// True if the user explicitly allowed this origin in their config. + fn user_allowed(&self, origin: &str) -> bool { + match &self.user_regex_origins { + Some(set) => set.is_match(origin), + None => false, + } + } + + /// True if this request must be blocked for the given origin. + fn is_blocked(&self, request: &Request) -> bool { + let origin = match request.headers().get_one("Origin") { + // No Origin header: not a browser cross-origin request (native + // watchers, curl). CORS is not the relevant control there. + None => return false, + Some(origin) => origin, + }; + + if !is_extension_origin(origin) || self.user_allowed(origin) { + return false; + } + + let method = match effective_method(request) { + // A preflight without Access-Control-Request-Method tells us + // nothing about the real request; fail closed. + None => return true, + Some(method) => method, + }; + let segments: Vec<&str> = request.uri().path().segments().collect(); + !is_watcher_endpoint(method, &segments) + } +} + +/// Origins matched by the built-in `moz-extension://.*` wildcard. +fn is_extension_origin(origin: &str) -> bool { + origin + .to_ascii_lowercase() + .starts_with(EXTENSION_ORIGIN_SCHEME) +} + +/// The method the browser will actually use: for a preflight that is the +/// `Access-Control-Request-Method` header, not `OPTIONS`. +fn effective_method(request: &Request) -> Option { + if request.method() == Method::Options { + request + .headers() + .get_one("Access-Control-Request-Method") + .and_then(|m| m.parse::().ok()) + } else { + Some(request.method()) + } +} + +/// The endpoints `aw-watcher-web` needs, as decoded path segments. +/// +/// The prefix blocks writes to other watcher types, but deliberately does not +/// bind an extension origin to one `aw-watcher-web-*` bucket. See the accepted +/// limitation in the module documentation. +fn is_watcher_endpoint(method: Method, segments: &[&str]) -> bool { + match (method, segments) { + (Method::Get, ["api", "0", "info"]) => true, + (Method::Post, ["api", "0", "buckets", bucket_id]) + | (Method::Post, ["api", "0", "buckets", bucket_id, "heartbeat"]) => { + bucket_id.starts_with("aw-watcher-web-") + } + _ => false, + } +} + +/// Route handler returning 403 for out-of-scope extension requests. +#[derive(Clone)] +struct FairingErrorRoute {} + +#[rocket::async_trait] +impl rocket::route::Handler for FairingErrorRoute { + async fn handle<'r>(&self, request: &'r Request<'_>, _: Data<'r>) -> Outcome<'r> { + let err = HttpErrorJson::new( + Status::Forbidden, + "This endpoint is not available to browser extension origins. \ + Add the origin to 'cors_regex' in config.toml to allow it." + .to_string(), + ); + Outcome::from(request, err) + } +} + +fn fairing_route() -> Route { + Route::ranked(1, Method::Get, "/", FairingErrorRoute {}) +} + +fn redirect_forbidden(request: &mut Request) { + let uri = FAIRING_ROUTE_BASE.to_string(); + let origin = Origin::parse_owned(uri).unwrap(); + request.set_method(Method::Get); + request.set_uri(origin); +} + +#[rocket::async_trait] +impl Fairing for ExtensionCorsScope { + fn info(&self) -> rocket::fairing::Info { + rocket::fairing::Info { + name: "ExtensionCorsScope", + kind: rocket::fairing::Kind::Ignite + | rocket::fairing::Kind::Request + | rocket::fairing::Kind::Response, + } + } + + async fn on_ignite(&self, rocket: Rocket) -> rocket::fairing::Result { + Ok(rocket.mount(FAIRING_ROUTE_BASE, vec![fairing_route()])) + } + + async fn on_request(&self, request: &mut Request<'_>, _: &mut Data<'_>) { + // Preflights are answered by rocket_cors; on_response strips the + // CORS headers so the browser rejects the preflight itself. + if request.method() == Method::Options { + return; + } + if self.is_blocked(request) { + debug!( + "Blocking extension origin {:?} from {}", + request.headers().get_one("Origin"), + request.uri() + ); + redirect_forbidden(request); + } + } + + async fn on_response<'r>(&self, request: &'r Request<'_>, response: &mut Response<'r>) { + if self.is_blocked(request) { + for header in CORS_HEADERS { + response.remove_header(header); + } + } + } +} + +#[cfg(test)] +mod tests { + use rocket::http::{ContentType, Header, Status}; + use rocket::local::blocking::Client; + use rocket::Rocket; + + use crate::config::AWConfig; + use crate::endpoints; + + const EXT_ORIGIN: &str = "moz-extension://3f2b1c9d-dead-beef-cafe-000000000000"; + const WEBUI_ORIGIN: &str = "http://127.0.0.1:5600"; + + fn setup_testserver(cors_regex: Vec) -> Rocket { + let state = endpoints::ServerState { + datastore: aw_datastore::Datastore::new_in_memory(false), + asset_resolver: endpoints::AssetResolver::new(None), + device_id: "test_id".to_string(), + }; + let mut aw_config = AWConfig::default(); + // Pin the port: the default is derived from a mutable global that other + // tests flip via create_config(), which would otherwise make the + // allowed webui origin depend on test execution order. + aw_config.port = 5600; + aw_config.cors_regex = cors_regex; + endpoints::build_rocket(state, aw_config) + } + + fn client() -> Client { + Client::tracked(setup_testserver(vec![])).expect("valid instance") + } + + fn allow_origin(res: &rocket::local::blocking::LocalResponse) -> Option { + res.headers() + .get_one("Access-Control-Allow-Origin") + .map(str::to_string) + } + + /// The reported exfiltration path: an extension reading the full export. + #[test] + fn test_extension_origin_blocked_from_export() { + let client = client(); + let res = client + .get("/api/0/export") + .header(Header::new("Host", "localhost:5600")) + .header(Header::new("Origin", EXT_ORIGIN)) + .dispatch(); + assert_eq!(res.status(), Status::Forbidden); + assert_eq!(allow_origin(&res), None, "export readable by any extension"); + } + + /// The other half of the report: writing events through /api/0/import. + #[test] + fn test_extension_origin_blocked_from_import_preflight() { + let client = client(); + let res = client + .options("/api/0/import") + .header(Header::new("Host", "localhost:5600")) + .header(Header::new("Origin", EXT_ORIGIN)) + .header(Header::new("Access-Control-Request-Method", "POST")) + .header(Header::new( + "Access-Control-Request-Headers", + "content-type", + )) + .dispatch(); + assert_eq!(allow_origin(&res), None, "import preflight still allowed"); + assert_eq!(res.headers().get_one("Access-Control-Allow-Methods"), None); + } + + /// Everything aw-watcher-web needs must keep working, unchanged. + #[test] + fn test_watcher_endpoints_still_allowed() { + let client = client(); + + let res = client + .get("/api/0/info") + .header(Header::new("Host", "localhost:5600")) + .header(Header::new("Origin", EXT_ORIGIN)) + .dispatch(); + assert_eq!(res.status(), Status::Ok); + assert_eq!(allow_origin(&res).as_deref(), Some(EXT_ORIGIN)); + + let res = client + .post("/api/0/buckets/aw-watcher-web-firefox_testhost") + .header(ContentType::JSON) + .header(Header::new("Host", "localhost:5600")) + .header(Header::new("Origin", EXT_ORIGIN)) + .body(r#"{"type":"web.tab.current","client":"aw-client-web","hostname":"testhost"}"#) + .dispatch(); + assert_ne!(res.status(), Status::Forbidden, "ensureBucket blocked"); + assert_eq!(allow_origin(&res).as_deref(), Some(EXT_ORIGIN)); + + let res = client + .post("/api/0/buckets/aw-watcher-web-firefox_testhost/heartbeat?pulsetime=60") + .header(ContentType::JSON) + .header(Header::new("Host", "localhost:5600")) + .header(Header::new("Origin", EXT_ORIGIN)) + .body( + r#"{"timestamp":"2026-07-28T09:00:00+00:00","duration":0, + "data":{"url":"https://example.com","title":"Example","audible":false, + "incognito":false,"tabCount":1}}"#, + ) + .dispatch(); + assert_eq!(res.status(), Status::Ok, "heartbeat blocked"); + assert_eq!(allow_origin(&res).as_deref(), Some(EXT_ORIGIN)); + } + + /// A wildcard-matched extension must not create or write to buckets owned + /// by other watchers. The URL bucket ID is the downstream authorization + /// boundary, so this check must happen before the handler runs. + #[test] + fn test_extension_cannot_write_other_watcher_bucket() { + let client = client(); + + let res = client + .post("/api/0/buckets/aw-watcher-window_testhost") + .header(ContentType::JSON) + .header(Header::new("Host", "localhost:5600")) + .header(Header::new("Origin", EXT_ORIGIN)) + .body(r#"{"type":"currentwindow","client":"aw-client","hostname":"testhost"}"#) + .dispatch(); + assert_eq!(res.status(), Status::Forbidden, "foreign bucket created"); + assert_eq!(allow_origin(&res), None); + + let res = client + .post("/api/0/buckets/aw-watcher-window_testhost/heartbeat?pulsetime=60") + .header(ContentType::JSON) + .header(Header::new("Host", "localhost:5600")) + .header(Header::new("Origin", EXT_ORIGIN)) + .body( + r#"{"timestamp":"2026-07-28T09:00:00+00:00","duration":0, + "data":{"app":"attacker","title":"injected"}}"#, + ) + .dispatch(); + assert_eq!(res.status(), Status::Forbidden, "foreign heartbeat written"); + assert_eq!(allow_origin(&res), None); + } + + /// Preflight for a watcher endpoint must still succeed — aw-client-js + /// sends JSON, which is always preflighted. + #[test] + fn test_watcher_preflight_still_allowed() { + let client = client(); + let res = client + .options("/api/0/buckets/aw-watcher-web-firefox_testhost/heartbeat") + .header(Header::new("Host", "localhost:5600")) + .header(Header::new("Origin", EXT_ORIGIN)) + .header(Header::new("Access-Control-Request-Method", "POST")) + .header(Header::new( + "Access-Control-Request-Headers", + "content-type", + )) + .dispatch(); + assert_eq!(allow_origin(&res).as_deref(), Some(EXT_ORIGIN)); + } + + /// Scoping is per (method, path): the bucket path is writable for create, + /// but must not become a deletion or read channel. + #[test] + fn test_bucket_path_scoped_by_method() { + let client = client(); + + let res = client + .delete("/api/0/buckets/aw-watcher-web-firefox_testhost") + .header(Header::new("Host", "localhost:5600")) + .header(Header::new("Origin", EXT_ORIGIN)) + .dispatch(); + assert_eq!(res.status(), Status::Forbidden, "bucket deletable"); + + let res = client + .get("/api/0/buckets/aw-watcher-web-firefox_testhost/events") + .header(Header::new("Host", "localhost:5600")) + .header(Header::new("Origin", EXT_ORIGIN)) + .dispatch(); + assert_eq!(res.status(), Status::Forbidden, "events readable"); + } + + /// Same desync bug class as #588/#636: matching the raw path string would + /// let an encoded spelling reach the handler while skipping this gate. + #[test] + fn test_encoded_paths_do_not_bypass_scope() { + let client = client(); + for path in [ + "/%61pi/0/export", // encoded 'a' in /api/ + "/api/0/%65xport", // encoded 'e' in /export + "//api/0/export", // #588 double-slash trick + "/%61pi/0/%65xport", // both + "/api/0/info/../query", // dot-segment + ] { + let res = client + .get(path) + .header(Header::new("Host", "localhost:5600")) + .header(Header::new("Origin", EXT_ORIGIN)) + .dispatch(); + assert_eq!( + res.status(), + Status::Forbidden, + "extension scope bypassed via {path}" + ); + assert_eq!(allow_origin(&res), None, "CORS header leaked for {path}"); + } + + // Sanity check: these paths really do reach the export handler without + // an extension Origin, so the 403s above are this gate rejecting them + // rather than the router failing to match. + for path in ["/%61pi/0/export", "//api/0/export", "/api/0/%65xport"] { + let res = client + .get(path) + .header(Header::new("Host", "localhost:5600")) + .dispatch(); + assert_eq!( + res.status(), + Status::Ok, + "{path} does not reach the handler" + ); + } + } + + /// Non-extension origins are untouched — the webui is same-origin and + /// still needs full access. + #[test] + fn test_webui_origin_unaffected() { + let client = client(); + let res = client + .get("/api/0/export") + .header(Header::new("Host", "localhost:5600")) + .header(Header::new("Origin", WEBUI_ORIGIN)) + .dispatch(); + assert_eq!(res.status(), Status::Ok); + assert_eq!(allow_origin(&res).as_deref(), Some(WEBUI_ORIGIN)); + } + + /// Native clients send no Origin header and must be unaffected. + #[test] + fn test_no_origin_unaffected() { + let client = client(); + let res = client + .get("/api/0/export") + .header(Header::new("Host", "localhost:5600")) + .dispatch(); + assert_eq!(res.status(), Status::Ok); + } + + /// An explicit `cors_regex` entry is a deliberate opt-in and keeps full + /// access — but only for the origins it actually matches. + #[test] + fn test_user_regex_allowlist_scoped_to_match() { + let server = setup_testserver(vec![format!("^{EXT_ORIGIN}$")]); + let client = Client::tracked(server).expect("valid instance"); + + let res = client + .get("/api/0/export") + .header(Header::new("Host", "localhost:5600")) + .header(Header::new("Origin", EXT_ORIGIN)) + .dispatch(); + assert_eq!(res.status(), Status::Ok); + + // A different extension is still scoped. + let res = client + .get("/api/0/export") + .header(Header::new("Host", "localhost:5600")) + .header(Header::new( + "Origin", + "moz-extension://00000000-0000-0000-0000-999999999999", + )) + .dispatch(); + assert_eq!(res.status(), Status::Forbidden); + } + + /// A preflight that hides its real method must not pass. + #[test] + fn test_preflight_without_request_method_is_blocked() { + let client = client(); + let res = client + .options("/api/0/buckets/some-bucket") + .header(Header::new("Host", "localhost:5600")) + .header(Header::new("Origin", EXT_ORIGIN)) + .dispatch(); + assert_eq!(allow_origin(&res), None); + } +} diff --git a/aw-server/src/endpoints/hostcheck.rs b/aw-server/src/endpoints/hostcheck.rs new file mode 100644 index 0000000..e5cd947 --- /dev/null +++ b/aw-server/src/endpoints/hostcheck.rs @@ -0,0 +1,195 @@ +//! Host header check needs to be performed to protect against DNS poisoning +//! attacks[1]. +//! +//! Uses a Request Fairing to intercept the request before it's handled. +//! If the Host header is not valid, the request will be rerouted to a +//! BadRequest +//! +//! [1]: https://github.com/ActivityWatch/activitywatch/security/advisories/GHSA-v9fg-6g9j-h4x4 +use rocket::fairing::Fairing; +use rocket::http::uri::Origin; +use rocket::http::{Method, Status}; +use rocket::route::Outcome; +use rocket::{Data, Request, Rocket, Route}; + +use crate::config::AWConfig; +use crate::endpoints::HttpErrorJson; + +static FAIRING_ROUTE_BASE: &str = "/checkheader_fairing"; + +pub struct HostCheck { + validate: bool, +} + +impl HostCheck { + pub fn new(config: &AWConfig) -> HostCheck { + // We only validate requests if the server binds a local address + let validate = config.address == "127.0.0.1" || config.address == "localhost"; + HostCheck { validate } + } +} + +/// Create a `Handler` for Fairing error handling +#[derive(Clone)] +struct FairingErrorRoute {} + +#[rocket::async_trait] +impl rocket::route::Handler for FairingErrorRoute { + async fn handle<'r>( + &self, + request: &'r Request<'_>, + _: rocket::Data<'r>, + ) -> rocket::route::Outcome<'r> { + let err = HttpErrorJson::new(Status::BadRequest, "Host header is invalid".to_string()); + Outcome::from(request, err) + } +} + +/// Create a new `Route` for Fairing handling +fn fairing_route() -> Route { + Route::ranked(1, Method::Get, "/", FairingErrorRoute {}) +} + +fn redirect_bad_request(request: &mut Request) { + let uri = FAIRING_ROUTE_BASE.to_string(); + let origin = Origin::parse_owned(uri).unwrap(); + request.set_method(Method::Get); + request.set_uri(origin); +} + +#[rocket::async_trait] +impl Fairing for HostCheck { + fn info(&self) -> rocket::fairing::Info { + rocket::fairing::Info { + name: "HostCheck", + kind: rocket::fairing::Kind::Ignite | rocket::fairing::Kind::Request, + } + } + + async fn on_ignite(&self, rocket: Rocket) -> rocket::fairing::Result { + match self.validate { + true => Ok(rocket.mount(FAIRING_ROUTE_BASE, vec![fairing_route()])), + false => { + warn!("Host header validation is turned off, this is a security risk"); + Ok(rocket) + } + } + } + + async fn on_request(&self, request: &mut Request<'_>, _: &mut Data<'_>) { + if !self.validate { + // host header check is disabled + return; + } + + // Fetch header + let hostheader_opt = request.headers().get_one("host"); + if hostheader_opt.is_none() { + info!("Missing 'Host' header, denying request"); + redirect_bad_request(request); + return; + } + + // Parse hostname from host header + // hostname contains port, which we don't care about and filter out + let hostheader = hostheader_opt.unwrap(); + let host_opt = hostheader.split(':').next(); + if host_opt.is_none() { + info!("Host header '{}' not allowed, denying request", hostheader); + redirect_bad_request(request); + return; + } + + // Deny requests to hosts that are not localhost + let valid_hosts: Vec<&str> = vec!["127.0.0.1", "localhost"]; + let host = host_opt.unwrap(); + if !valid_hosts.contains(&host) { + info!("Host header '{}' not allowed, denying request", hostheader); + redirect_bad_request(request); + } + + // host header is verified, proceed with request + } +} + +#[cfg(test)] +mod tests { + + use rocket::http::{ContentType, Header, Status}; + use rocket::Rocket; + + use crate::config::AWConfig; + use crate::endpoints; + + fn setup_testserver(address: String) -> Rocket { + let state = endpoints::ServerState { + datastore: aw_datastore::Datastore::new_in_memory(false), + asset_resolver: endpoints::AssetResolver::new(None), + device_id: "test_id".to_string(), + }; + let aw_config = AWConfig { + address, + ..AWConfig::default() + }; + endpoints::build_rocket(state, aw_config) + } + + #[test] + fn test_public_address() { + let server = setup_testserver("0.0.0.0".to_string()); + let client = rocket::local::blocking::Client::tracked(server).expect("valid instance"); + + // When a public address is used, request should always pass, regardless + // if the Host header is missing + let res = client + .get("/api/0/info") + .header(ContentType::JSON) + .dispatch(); + assert_eq!(res.status(), Status::Ok); + } + + #[test] + fn test_localhost_address() { + let server = setup_testserver("127.0.0.1".to_string()); + let client = rocket::local::blocking::Client::tracked(server).expect("valid instance"); + + // If Host header is missing we should get a BadRequest + let res = client + .get("/api/0/info") + .header(ContentType::JSON) + .dispatch(); + assert_eq!(res.status(), Status::BadRequest); + + // If Host header is not 127.0.0.1 or localhost we should get BadRequest + let res = client + .get("/api/0/info") + .header(ContentType::JSON) + .header(Header::new("Host", "192.168.0.1:1234")) + .dispatch(); + assert_eq!(res.status(), Status::BadRequest); + + // If Host header is 127.0.0.1:5600 we should get OK + let res = client + .get("/api/0/info") + .header(ContentType::JSON) + .header(Header::new("Host", "127.0.0.1:5600")) + .dispatch(); + assert_eq!(res.status(), Status::Ok); + + // If Host header is localhost:5600 we should get OK + let res = client + .get("/api/0/info") + .header(ContentType::JSON) + .header(Header::new("Host", "localhost:5600")) + .dispatch(); + assert_eq!(res.status(), Status::Ok); + + // If Host header is missing port, we should still get OK + let res = client + .get("/api/0/info") + .header(ContentType::JSON) + .header(Header::new("Host", "localhost")) + .dispatch(); + assert_eq!(res.status(), Status::Ok); + } +} diff --git a/aw-server/src/endpoints/import.rs b/aw-server/src/endpoints/import.rs new file mode 100644 index 0000000..d6d0ca2 --- /dev/null +++ b/aw-server/src/endpoints/import.rs @@ -0,0 +1,146 @@ +use rocket::form::Form; +use rocket::http::Status; +use rocket::serde::json::Json; +use rocket::State; + +use std::collections::{BTreeMap, HashSet}; + +use aw_models::{BucketsExport, Event}; + +use aw_datastore::{Datastore, DatastoreError}; + +use crate::endpoints::{HttpErrorJson, ServerState}; + +/// Computes a dedup identity tuple for an event. +/// +/// Uses canonical JSON serialization (sorted keys via `BTreeMap`) so that +/// events with identical key-value pairs but different insertion order +/// (e.g., from different clients) are correctly identified as duplicates. +fn event_identity( + event: &Event, +) -> Result<(chrono::DateTime, i64, String), HttpErrorJson> { + let duration_ns = event.duration.num_nanoseconds().ok_or_else(|| { + HttpErrorJson::new( + Status::InternalServerError, + "Failed to encode event duration for dedup".to_string(), + ) + })?; + // Sort keys before serializing for canonical, order-independent dedup. + // This prevents missed duplicates when events from different clients + // serialize the same data with different key orderings. + let sorted: BTreeMap<_, _> = event.data.iter().collect(); + let data_json = serde_json::to_string(&sorted).map_err(|e| { + HttpErrorJson::new( + Status::InternalServerError, + format!("Failed to encode event data for dedup: {e}"), + ) + })?; + Ok((event.timestamp, duration_ns, data_json)) +} + +fn import(datastore: &Datastore, import: BucketsExport) -> Result<(), HttpErrorJson> { + for (_bucketname, mut bucket) in import.buckets { + match datastore.create_bucket(&bucket) { + Ok(_) => (), + Err(DatastoreError::BucketAlreadyExists(_)) => { + // Bucket already exists — merge events, skipping duplicates + info!("Bucket '{}' already exists, merging events", bucket.id); + if let Some(events) = bucket.events.take() { + let events_vec = events.take_inner(); + if !events_vec.is_empty() { + // Determine time range of events to import + let start = events_vec.iter().map(|e| e.timestamp).min().unwrap(); + let end = events_vec + .iter() + .map(|e| e.calculate_endtime()) + .max() + .unwrap(); + + // Fetch existing events in that range to detect duplicates. + // Events without an explicit ID would otherwise be inserted as new rows + // via AUTOINCREMENT, silently creating duplicates on re-import. + // + // **Memory note**: This loads all events in the import time range into + // memory for O(1) dedup lookups. Typical Android re-imports involve a + // few thousand events (~1-2 MB), which is well within server bounds. + // Pathological cases (years of data) could be mitigated with pagination + // or a bloom filter if OOM issues arise in practice. + let existing = datastore + .get_events_unclipped(&bucket.id, Some(start), Some(end), None) + .map_err(|e| { + HttpErrorJson::new( + Status::InternalServerError, + format!( + "Failed to fetch existing events for dedup in '{}': {e:?}", + bucket.id + ), + ) + })?; + + let existing_identities: HashSet<_> = existing + .iter() + .map(event_identity) + .collect::>()?; + + // Filter out events already present (matched by timestamp, duration, data) + let new_events: Vec<_> = events_vec + .into_iter() + .map(|event| Ok((event_identity(&event)?, event))) + .collect::, HttpErrorJson>>()? + .into_iter() + .filter_map(|(identity, event)| { + (!existing_identities.contains(&identity)).then_some(event) + }) + .collect(); + + if !new_events.is_empty() { + if let Err(e) = datastore.insert_events(&bucket.id, &new_events) { + let err_msg = format!( + "Failed to merge events into existing bucket '{}': {e:?}", + bucket.id + ); + warn!("{}", err_msg); + return Err(HttpErrorJson::new( + Status::InternalServerError, + err_msg, + )); + } + } + } + } + } + Err(e) => { + let err_msg = format!("Failed to import bucket: {e:?}"); + warn!("{}", err_msg); + return Err(HttpErrorJson::new(Status::InternalServerError, err_msg)); + } + } + } + Ok(()) +} + +#[post("/", data = "", format = "application/json")] +pub fn bucket_import_json( + state: &State, + json_data: Json, +) -> Result<(), HttpErrorJson> { + import(&state.datastore, json_data.into_inner()) +} + +#[derive(FromForm)] +pub struct ImportForm { + // FIXME: In the web-ui the name of this field is buckets.json, but "." is not allowed in field + // names in Rocket and just simply "buckets" seems to work apparently but not sure why. + // FIXME: In aw-server python it will import all fields rather just the one named + // "buckets.json", that should probably be done here as well. + #[field(name = "buckets")] + import: Json, +} + +#[post("/", data = "
", format = "multipart/form-data")] +pub fn bucket_import_form( + state: &State, + form: Form, +) -> Result<(), HttpErrorJson> { + import(&state.datastore, form.into_inner().import.into_inner()) +} diff --git a/aw-server/src/endpoints/mod.rs b/aw-server/src/endpoints/mod.rs new file mode 100644 index 0000000..276677e --- /dev/null +++ b/aw-server/src/endpoints/mod.rs @@ -0,0 +1,236 @@ +use rust_embed::RustEmbed; +use std::ffi::OsStr; +use std::path::{Path, PathBuf}; + +use gethostname::gethostname; +use rocket::fs::FileServer; +use rocket::http::ContentType; +use rocket::serde::json::Json; +use rocket::State; + +use crate::config::AWConfig; + +use aw_datastore::Datastore; +use aw_models::Info; + +#[derive(RustEmbed)] +#[folder = "$AW_WEBUI_DIR"] +struct EmbeddedAssets; + +pub struct AssetResolver { + asset_path: Option, +} + +impl AssetResolver { + pub fn new(asset_path: Option) -> Self { + Self { asset_path } + } + + fn resolve(&self, file_path: &str) -> Option> { + if let Some(asset_path) = &self.asset_path { + let content = std::fs::read(asset_path.join(file_path)); + if let Ok(data) = content { + return Some(data); + } + } + Some(EmbeddedAssets::get(file_path)?.data.to_vec()) + } +} + +// The Datastore is just a cheap handle to the DB worker thread (a crossbeam +// channel sender), which serializes all DB access internally. No mutex is +// needed here — wrapping it in one would serialize all HTTP requests, letting +// a slow query block every heartbeat. +pub struct ServerState { + pub datastore: Datastore, + pub asset_resolver: AssetResolver, + pub device_id: String, +} + +#[macro_use] +mod util; +mod apikey; +mod bucket; +mod cors; +mod export; +mod extension_cors; +mod hostcheck; +mod import; +mod query; +mod settings; + +#[cfg(target_os = "android")] +pub(crate) use settings::settings_datastore_key; +pub use util::HttpErrorJson; + +#[get("/")] +fn root_index(state: &State) -> Option<(ContentType, Vec)> { + get_file("index.html".into(), state) +} + +#[get("/css/")] +fn root_css(file: PathBuf, state: &State) -> Option<(ContentType, Vec)> { + get_file(Path::new("css").join(file), state) +} + +#[get("/fonts/")] +fn root_fonts(file: PathBuf, state: &State) -> Option<(ContentType, Vec)> { + get_file(Path::new("fonts").join(file), state) +} + +#[get("/js/")] +fn root_js(file: PathBuf, state: &State) -> Option<(ContentType, Vec)> { + get_file(Path::new("js").join(file), state) +} + +#[get("/static/")] +fn root_static(file: PathBuf, state: &State) -> Option<(ContentType, Vec)> { + get_file(Path::new("static").join(file), state) +} + +#[get("/favicon.ico")] +fn root_favicon(state: &State) -> Option<(ContentType, Vec)> { + get_file("favicon.ico".into(), state) +} + +#[get("/dark.css")] +fn root_dark(state: &State) -> Option<(ContentType, Vec)> { + get_file("dark.css".into(), state) +} + +#[get("/logo.png")] +fn root_logo(state: &State) -> Option<(ContentType, Vec)> { + get_file("logo.png".into(), state) +} + +#[get("/manifest.json")] +fn root_manifest(state: &State) -> Option<(ContentType, Vec)> { + get_file("manifest.json".into(), state) +} + +#[get("/")] +fn server_info(config: &State, state: &State) -> Json { + #[allow(clippy::or_fun_call)] + let hostname = gethostname().into_string().unwrap_or("unknown".to_string()); + + Json(Info { + hostname, + version: crate::version::version_string(), + testing: config.testing, + profile: crate::config::get_profile().to_string(), + device_id: state.device_id.clone(), + }) +} + +fn get_file(file: PathBuf, state: &State) -> Option<(ContentType, Vec)> { + let asset = state.asset_resolver.resolve(&file.display().to_string())?; + + let content_type = file + .extension() + .and_then(OsStr::to_str) + .and_then(ContentType::from_extension) + .unwrap_or(ContentType::Bytes); + + Some((content_type, asset)) +} + +pub fn build_rocket(server_state: ServerState, config: AWConfig) -> rocket::Rocket { + info!( + "Starting aw-server-rust at {}:{}", + config.address, config.port + ); + let cors = cors::cors(&config); + let extension_cors = extension_cors::ExtensionCorsScope::new(&config); + let hostcheck = hostcheck::HostCheck::new(&config); + let apikey = apikey::ApiKeyCheck::new(&config); + let custom_static = config.custom_static.clone(); + + let mut rocket = rocket::custom(config.to_rocket_config()) + .attach(cors.clone()) + // Attached before the other request fairings so a blocked extension + // request is rewritten to the 403 route before they inspect the path. + .attach(extension_cors) + .attach(hostcheck) + .attach(apikey) + .manage(cors) + .manage(server_state) + .manage(config) + .mount( + "/", + routes![ + root_index, + root_favicon, + root_fonts, + root_css, + root_js, + root_static, + // custom static files + root_dark, + root_logo, + root_manifest + ], + ) + .mount("/api/0/info", routes![server_info]) + .mount( + "/api/0/buckets", + routes![ + bucket::bucket_new, + bucket::bucket_delete, + bucket::buckets_get, + bucket::bucket_get, + bucket::bucket_events_get, + bucket::bucket_events_create, + bucket::bucket_events_heartbeat, + bucket::bucket_event_count, + bucket::bucket_events_get_single, + bucket::bucket_events_delete_by_id, + bucket::bucket_export + ], + ) + .mount("/api/0/query", routes![query::query]) + .mount( + "/api/0/import", + routes![import::bucket_import_json, import::bucket_import_form], + ) + .mount("/api/0/export", routes![export::buckets_export]) + .mount( + "/api/0/settings", + routes![ + settings::setting_get, + settings::setting_set, + settings::setting_delete, + settings::settings_get, + ], + ) + .mount("/", rocket_cors::catch_all_options_routes()); + + // for each custom static directory, mount it at the given name + for (name, dir) in custom_static { + info!( + "Serving /pages/{} custom static directory from {}", + name, dir + ); + rocket = rocket.mount(&format!("/pages/{name}"), FileServer::from(dir)); + } + rocket +} + +mod tests { + #[test] + fn test_filesystem_resolver() { + let resolver = super::AssetResolver::new(Some(".".into())); + + let content = resolver.resolve("Cargo.toml").unwrap(); + + assert!(String::from_utf8(content).unwrap().contains("aw-server")); + } + + #[test] + fn test_resolver_without_asset() { + let resolver = super::AssetResolver::new(Some(".".into())); + + let content = resolver.resolve("Cargo.json"); + + assert!(content.is_none()); + } +} diff --git a/aw-server/src/endpoints/query.rs b/aw-server/src/endpoints/query.rs new file mode 100644 index 0000000..bbee854 --- /dev/null +++ b/aw-server/src/endpoints/query.rs @@ -0,0 +1,65 @@ +use rocket::http::Status; +use rocket::serde::json::{json, Json, Value}; +use rocket::State; + +use aw_models::Query; +use aw_query::QueryError; + +use crate::endpoints::{HttpErrorJson, ServerState}; + +fn query_error_status(e: &QueryError) -> Status { + match e { + // BucketQueryError also wraps datastore failures, which are server + // errors rather than malformed client queries. + QueryError::BucketQueryError(_) => Status::InternalServerError, + QueryError::ParsingError(_) + | QueryError::EmptyQuery() + | QueryError::BucketNotFound(_) + | QueryError::VariableNotDefined(_) + | QueryError::MathError(_) + | QueryError::InvalidType(_) + | QueryError::InvalidFunctionParameters(_) + | QueryError::TimeIntervalError(_) + | QueryError::RegexCompileError(_) => Status::BadRequest, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn query_error_status_distinguishes_client_and_server_errors() { + assert_eq!( + query_error_status(&QueryError::RegexCompileError("invalid regex".into())), + Status::BadRequest + ); + assert_eq!( + query_error_status(&QueryError::BucketNotFound("missing bucket".into())), + Status::BadRequest + ); + assert_eq!( + query_error_status(&QueryError::BucketQueryError("datastore failure".into())), + Status::InternalServerError + ); + } +} + +#[post("/", data = "", format = "application/json")] +pub fn query(query_req: Json, state: &State) -> Result { + let query_code = query_req.0.query.join("\n"); + let intervals = &query_req.0.timeperiods; + let mut results = Vec::new(); + let datastore = &state.datastore; + for interval in intervals { + let result = match aw_query::query(&query_code, interval, datastore) { + Ok(data) => data, + Err(e) => { + warn!("Query failed: {:?}", e); + return Err(HttpErrorJson::new(query_error_status(&e), e.to_string())); + } + }; + results.push(result); + } + Ok(json!(results)) +} diff --git a/aw-server/src/endpoints/settings.rs b/aw-server/src/endpoints/settings.rs new file mode 100644 index 0000000..0520911 --- /dev/null +++ b/aw-server/src/endpoints/settings.rs @@ -0,0 +1,156 @@ +use crate::endpoints::ServerState; +use rocket::http::Status; +use rocket::serde::json::Json; +use rocket::State; +use std::collections::HashMap; + +use aw_datastore::DatastoreError; + +use crate::endpoints::HttpErrorJson; + +/// Map a settings API key to the datastore key (`settings.`). +/// +/// Dots are allowed: `GET /api/0/settings/foo.bar` stores and retrieves +/// `settings.foo.bar`. Keys of length >= 128 are rejected (same rule as the +/// HTTP handler). Android JNI `getSetting` uses this helper so native callers +/// cannot silently miss a value the HTTP API would return. +pub(crate) fn settings_datastore_key(key: &str) -> Result { + if key.len() >= 128 { + Err("Too long key") + } else { + Ok(format!("settings.{}", key)) + } +} + +fn parse_key(key: String) -> Result { + settings_datastore_key(&key) + .map_err(|msg| HttpErrorJson::new(Status::BadRequest, msg.to_string())) +} + +#[cfg(test)] +mod key_tests { + use super::settings_datastore_key; + + #[test] + fn simple_keys_are_namespaced() { + assert_eq!( + settings_datastore_key("classes").unwrap(), + "settings.classes" + ); + assert_eq!( + settings_datastore_key("startOfDay").unwrap(), + "settings.startOfDay" + ); + } + + #[test] + fn dotted_keys_are_namespaced_not_rejected() { + assert_eq!( + settings_datastore_key("foo.bar").unwrap(), + "settings.foo.bar" + ); + assert_eq!(settings_datastore_key("a.b.c").unwrap(), "settings.a.b.c"); + } + + #[test] + fn overly_long_keys_are_rejected() { + assert!(settings_datastore_key(&"a".repeat(128)).is_err()); + assert_eq!( + settings_datastore_key(&"a".repeat(127)).unwrap(), + format!("settings.{}", "a".repeat(127)) + ); + } +} + +#[get("/")] +pub fn settings_get( + state: &State, +) -> Result>, HttpErrorJson> { + let datastore = &state.datastore; + let queryresults = match datastore.get_key_values("settings.%") { + Ok(result) => Ok(result), + Err(err) => Err(err.into()), + }; + + match queryresults { + Ok(settings) => { + // strip 'settings.' prefix from keys + let mut map: HashMap = HashMap::new(); + for (key, value) in settings.iter() { + map.insert( + key.strip_prefix("settings.").unwrap_or(key).to_string(), + serde_json::from_str(value.clone().as_str()).unwrap(), + ); + } + Ok(Json(map)) + } + Err(err) => Err(err), + } +} + +#[get("/")] +pub fn setting_get( + state: &State, + key: String, +) -> Result, HttpErrorJson> { + let setting_key = parse_key(key)?; + let datastore = &state.datastore; + + match datastore.get_key_value(&setting_key) { + Ok(value) => Ok(Json(serde_json::from_str(&value).unwrap())), + Err(DatastoreError::NoSuchKey(_)) => Ok(Json(serde_json::from_str("null").unwrap())), + Err(err) => Err(err.into()), + } +} + +#[post("/", data = "", format = "application/json")] +pub fn setting_set( + state: &State, + key: String, + value: Json, +) -> Result { + let setting_key = parse_key(key)?; + let value_str = match serde_json::to_string(&value.0) { + Ok(value) => value, + Err(err) => { + return Err(HttpErrorJson::new( + Status::BadRequest, + format!("Invalid JSON: {}", err), + )) + } + }; + + let datastore = &state.datastore; + let result = datastore.set_key_value(&setting_key, &value_str); + + match result { + Ok(_) => { + // Worker also reloads on SetKeyValue of this key; this second + // RefreshPrivacyFilter is belt-and-suspenders so the HTTP path + // still works if a future writer bypasses that hook. + if setting_key == "settings.privacy_filters" { + let _ = datastore.refresh_privacy_filter(); + } + Ok(Status::Created) + } + Err(err) => Err(err.into()), + } +} + +#[delete("/")] +pub fn setting_delete(state: &State, key: String) -> Result<(), HttpErrorJson> { + let setting_key = parse_key(key)?; + + let datastore = &state.datastore; + let result = datastore.delete_key_value(&setting_key); + + match result { + Ok(_) => { + if setting_key == "settings.privacy_filters" { + let _ = datastore.refresh_privacy_filter(); + } + Ok(()) + } + Err(err) => Err(err.into()), + } +} diff --git a/aw-server/src/endpoints/util.rs b/aw-server/src/endpoints/util.rs new file mode 100644 index 0000000..8fce458 --- /dev/null +++ b/aw-server/src/endpoints/util.rs @@ -0,0 +1,115 @@ +use std::io::{Cursor, Seek, SeekFrom}; + +use rocket::http::ContentType; +use rocket::http::Header; +use rocket::http::Status; +use rocket::request::Request; +use rocket::response::{self, Responder, Response}; +use serde::Serialize; + +#[derive(Serialize, Debug)] +pub struct HttpErrorJson { + #[serde(skip_serializing)] + status: Status, + message: String, +} + +impl HttpErrorJson { + pub fn new(status: Status, err: String) -> HttpErrorJson { + HttpErrorJson { + status, + message: err, + } + } +} + +impl<'r> Responder<'r, 'static> for HttpErrorJson { + fn respond_to(self, _: &Request) -> response::Result<'static> { + let body = serde_json::to_string(&self).map_err(|err| { + error!("Failed to serialize error response: {err}"); + Status::InternalServerError + })?; + Response::build() + .status(self.status) + .sized_body(body.len(), Cursor::new(body)) + .header(ContentType::new("application", "json")) + .ok() + } +} + +pub struct BucketsExportRocket { + file: std::fs::File, + filename: String, +} + +impl BucketsExportRocket { + pub fn new( + datastore: &aw_datastore::Datastore, + bucket_id: Option<&str>, + ) -> Result { + let io_error = |err: std::io::Error| { + error!("Failed to prepare export file: {err}"); + HttpErrorJson::new( + Status::InternalServerError, + "Failed to prepare export file".into(), + ) + }; + // tempfile creates a private file and removes it when the response is + // dropped. Spooling preserves HTTP errors even if serialization or disk + // writes fail, while keeping event buffering bounded. + let file = tempfile::tempfile().map_err(io_error)?; + let (mut file, name) = datastore.export_to_file(bucket_id, file)?; + file.seek(SeekFrom::Start(0)).map_err(io_error)?; + let filename = match name { + Some(id) => format!("attachment; filename=aw-bucket-export_{id}.json"), + None => "attachment; filename=aw-buckets-export.json".into(), + }; + Ok(Self { file, filename }) + } +} + +impl<'r> Responder<'r, 'static> for BucketsExportRocket { + fn respond_to(self, _: &Request) -> response::Result<'static> { + Response::build() + .status(Status::Ok) + .header(Header::new("Content-Disposition", self.filename)) + .header(ContentType::JSON) + .streamed_body(rocket::tokio::fs::File::from_std(self.file)) + .ok() + } +} + +use aw_datastore::DatastoreError; + +impl From for HttpErrorJson { + fn from(val: DatastoreError) -> Self { + match val { + DatastoreError::NoSuchBucket(bucket_id) => HttpErrorJson::new( + Status::NotFound, + format!("The requested bucket '{bucket_id}' does not exist"), + ), + DatastoreError::BucketAlreadyExists(bucket_id) => HttpErrorJson::new( + Status::NotModified, + format!("Bucket '{bucket_id}' already exists"), + ), + DatastoreError::NoSuchKey(key) => HttpErrorJson::new( + Status::NotFound, + format!("The requested key(s) '{key}' do not exist"), + ), + DatastoreError::MpscError => HttpErrorJson::new( + Status::InternalServerError, + "Unexpected Mpsc error!".to_string(), + ), + DatastoreError::InternalError(msg) => { + HttpErrorJson::new(Status::InternalServerError, msg) + } + // When upgrade is disabled + DatastoreError::Uninitialized(msg) => { + HttpErrorJson::new(Status::InternalServerError, msg) + } + DatastoreError::OldDbVersion(msg) => { + HttpErrorJson::new(Status::InternalServerError, msg) + } + } + } +} diff --git a/aw-server/src/lib.rs b/aw-server/src/lib.rs new file mode 100644 index 0000000..eb32370 --- /dev/null +++ b/aw-server/src/lib.rs @@ -0,0 +1,36 @@ +#[macro_use] +extern crate rocket; +extern crate rocket_cors; + +extern crate serde; +extern crate serde_json; + +extern crate chrono; + +#[cfg(target_os = "android")] +#[macro_use] +extern crate lazy_static; + +#[macro_use] +extern crate log; +extern crate fern; + +extern crate toml; + +#[macro_use] +pub mod macros; +pub mod config; +pub mod device_id; +pub mod dirs; +pub mod endpoints; +pub mod logging; +pub mod panic_guard; +pub mod version; + +#[cfg(target_os = "android")] +pub mod android; + +extern crate aw_datastore; +extern crate aw_models; +extern crate aw_query; +extern crate aw_transform; diff --git a/aw-server/src/logging.rs b/aw-server/src/logging.rs new file mode 100644 index 0000000..10a6aee --- /dev/null +++ b/aw-server/src/logging.rs @@ -0,0 +1,153 @@ +use std::fs; +use std::path::PathBuf; + +use fern::colors::{Color, ColoredLevelConfig}; + +use crate::dirs; + +/// Render a single log line. +/// +/// `level` is taken as a `Display` so the caller decides whether it is colored: +/// stdout passes a colorized level, the logfile passes the plain one. +fn format_log_line( + level: &dyn std::fmt::Display, + target: &str, + message: &std::fmt::Arguments, +) -> String { + format!( + "[{}][{}][{}]: {}", + chrono::Local::now().format("%Y-%m-%d %H:%M:%S"), + level, + target, + message, + ) +} + +pub fn setup_logger(module: &str, profile: &str, verbose: bool) -> Result<(), fern::InitError> { + let testing = profile == "testing"; + let mut logfile_path: PathBuf = + dirs::get_log_dir(module).expect("Unable to get log dir to store logs in"); + fs::create_dir_all(logfile_path.clone()).expect("Unable to create folder for logs"); + // Isolated profile roots (including new-style testing) use a bare filename + // — the directory already isolates. The `-testing` infix stays only in + // the legacy shared-root layout so existing log files keep matching. + let filename = if dirs::legacy_testing_suffix(profile).is_empty() { + format!("{}_%Y-%m-%dT%H-%M-%S%z.log", module) + } else { + format!("{}-testing_%Y-%m-%dT%H-%M-%S%z.log", module) + }; + + logfile_path.push(chrono::Local::now().format(&filename).to_string()); + + log_panics::init(); + + let colors = ColoredLevelConfig::new() + .debug(Color::White) + .info(Color::Green) + .warn(Color::Yellow) + .error(Color::Red); + + let default_log_level = if testing || verbose { + log::LevelFilter::Debug + } else { + log::LevelFilter::Info + }; + + let log_level = std::env::var("LOG_LEVEL").map_or(default_log_level, |level| { + match level.to_lowercase().as_str() { + "trace" => log::LevelFilter::Trace, + "debug" => log::LevelFilter::Debug, + "info" => log::LevelFilter::Info, + "warn" => log::LevelFilter::Warn, + "error" => log::LevelFilter::Error, + _ => default_log_level, + } + }); + + let mut dispatch = fern::Dispatch::new().level(log_level); + // Set some Rocket messages to debug level + + let is_debug = matches!(log_level, log::LevelFilter::Trace | log::LevelFilter::Debug); + if !is_debug { + dispatch = dispatch + .level_for("rocket", log::LevelFilter::Warn) + // rocket_cors has a lot of unhelpful info messages that spam the log on every request + // https://github.com/ActivityWatch/activitywatch/issues/975 + .level_for("rocket_cors", log::LevelFilter::Warn) + .level_for("_", log::LevelFilter::Warn) // Rocket requests + .level_for("launch_", log::LevelFilter::Warn); // Rocket config info + } + + dispatch + // Colored output to stdout + .chain( + fern::Dispatch::new() + .format(move |out, message, record| { + out.finish(format_args!( + "{}", + format_log_line(&colors.color(record.level()), record.target(), message) + )) + }) + .chain(std::io::stdout()), + ) + // Uncolored output to logfile, so the file doesn't contain ANSI escapes + .chain( + fern::Dispatch::new() + .format(|out, message, record| { + out.finish(format_args!( + "{}", + format_log_line(&record.level(), record.target(), message) + )) + }) + .chain(fern::log_file(logfile_path)?), + ) + .apply()?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::{format_log_line, setup_logger}; + use fern::colors::{Color, ColoredLevelConfig}; + + /* disable this test. + * This is due to it failing in GitHub actions, claiming that the logger + * has been initialized twice which is not allowed */ + #[ignore] + #[test] + fn test_setup_logger() { + setup_logger("aw-server-rust", "testing", true).unwrap(); + } + + /* The formatting is tested directly rather than through setup_logger, which + * installs the global logger and so can only ever run once per process. */ + #[test] + fn test_format_log_line_is_plain_for_logfile() { + let line = format_log_line(&log::Level::Info, "aw_server::test", &format_args!("hello")); + + assert!( + !line.contains('\u{1b}'), + "logfile lines must not contain ANSI escapes, got {line:?}" + ); + assert!( + line.ends_with("[INFO][aw_server::test]: hello"), + "unexpected log line: {line:?}" + ); + } + + #[test] + fn test_format_log_line_is_colored_for_stdout() { + let colors = ColoredLevelConfig::new().info(Color::Green); + let line = format_log_line( + &colors.color(log::Level::Info), + "aw_server::test", + &format_args!("hello"), + ); + + assert!( + line.contains('\u{1b}'), + "stdout lines should keep their color, got {line:?}" + ); + assert!(line.ends_with("[aw_server::test]: hello")); + } +} diff --git a/aw-server/src/macros.rs b/aw-server/src/macros.rs new file mode 100644 index 0000000..c626754 --- /dev/null +++ b/aw-server/src/macros.rs @@ -0,0 +1,13 @@ +#[macro_export] +macro_rules! json_map { + { $( $key:literal : $value:expr),* } => {{ + use serde_json::Value; + use serde_json::map::Map; + #[allow(unused_mut)] + let mut map : Map = Map::new(); + $( + map.insert( $key.to_string(), json!($value) ); + )* + map + }}; +} diff --git a/aw-server/src/main.rs b/aw-server/src/main.rs new file mode 100644 index 0000000..f703f09 --- /dev/null +++ b/aw-server/src/main.rs @@ -0,0 +1,233 @@ +#[macro_use] +extern crate log; + +use std::env; +use std::path::PathBuf; + +use clap::crate_version; +use clap::Parser; + +use aw_server::*; + +#[cfg(target_os = "linux")] +use sd_notify::NotifyState; +#[cfg(all(target_os = "linux", target_arch = "x86"))] +extern crate jemallocator; +#[cfg(all(target_os = "linux", target_arch = "x86"))] +#[global_allocator] +static ALLOC: jemallocator::Jemalloc = jemallocator::Jemalloc; + +/// Rust server for ActivityWatch +#[derive(Parser)] +#[clap(version = crate_version!(), author = "Johan Bjäreholt, Erik Bjäreholt, et al.")] +struct Opts { + /// Named instance profile (e.g. "default", "testing", "research"). + /// Drives the isolated directory root (and legacy testing-filename fallback). + /// Defaults to "testing" in debug builds, "default" in release builds. + #[clap(long)] + profile: Option, + + /// Run in testing mode (alias for --profile testing) + #[clap(long)] + testing: bool, + + /// Verbose output + #[clap(long)] + verbose: bool, + + /// Address to listen to + #[clap(long)] + host: Option, + + /// Port to listen on + #[clap(long)] + port: Option, + + /// Path to database override + /// Also implies --no-legacy-import if no db found + #[clap(long)] + dbpath: Option, + + /// Path to config file override + #[clap(short = 'c', long = "config")] + config: Option, + + /// Path to webui override + #[clap(long)] + webpath: Option, + + /// Mapping of custom static paths to serve, in the format: watcher1=/path,watcher2=/path2 + #[clap(long)] + custom_static: Option, + + /// Device ID override + #[clap(long)] + device_id: Option, + + /// Don't import from aw-server-python if no aw-server-rust db found + #[clap(long)] + no_legacy_import: bool, + + /// Encryption key for the database (requires 'encryption' feature). + /// Can also be set via the AW_DB_PASSWORD environment variable. + /// WARNING: passing a password on the command line may expose it in process listings. + #[clap(long, env = "AW_DB_PASSWORD")] + #[cfg(any(feature = "encryption", feature = "encryption-vendored"))] + db_password: Option, +} + +#[rocket::main] +#[allow(clippy::result_large_err)] +async fn main() -> Result<(), rocket::Error> { + let opts: Opts = Opts::parse(); + + // Clear sensitive env vars immediately after parse, before setup_logger can start + // background threads (std::env::remove_var is not thread-safe on all platforms). + #[cfg(any(feature = "encryption", feature = "encryption-vendored"))] + std::env::remove_var("AW_DB_PASSWORD"); + + // Resolve profile: --profile wins; --testing is an alias for "testing"; + // debug builds default to "testing" (preserving prior behaviour). + let profile: String = match opts.profile { + Some(p) => { + if opts.testing && p != "testing" { + warn!( + "--testing and --profile {} both given; --profile wins, using '{}'", + p, p + ); + } + p + } + None => match std::env::var("AW_PROFILE") { + // aw-qt exports AW_PROFILE for the modules it spawns, so a profile + // set on the launcher propagates without every module growing a flag. + Ok(p) if !p.is_empty() => p, + _ => { + if opts.testing || cfg!(debug_assertions) { + "testing".to_string() + } else { + "default".to_string() + } + } + }, + }; + + dirs::validate_profile(&profile).unwrap_or_else(|e| panic!("Invalid profile name: {e}")); + // set_profile must run before setup_logger: get_log_dir reads the + // process-global profile via appname(), and named profiles otherwise + // land in the shared cache dir (ActivityWatch/activitywatch#1399). + config::set_profile(profile.clone()); + + logging::setup_logger("aw-server-rust", &profile, opts.verbose) + .expect("Failed to setup logging"); + + if profile != "default" { + info!("Running server with profile '{}'", profile); + } + + let mut config = config::create_config(&profile, opts.config.as_deref()); + + // set host if overridden + if let Some(host) = opts.host { + config.address = host; + } + + // set port if overridden + if let Some(port) = opts.port { + config.port = port.parse().unwrap(); + } + + // set custom_static if overridden, transform into map + if let Some(custom_static_str) = opts.custom_static { + let custom_static_map: std::collections::HashMap = custom_static_str + .split(',') + .map(|s| { + let mut split = s.split('='); + let key = split.next().unwrap().to_string(); + let value = split.next().unwrap().to_string(); + (key, value) + }) + .collect(); + config.custom_static.extend(custom_static_map); + + // validate paths, log error if invalid + // remove invalid paths + for (name, path) in config.custom_static.clone().iter() { + if !std::path::Path::new(path).exists() { + error!("custom_static path for {} does not exist ({})", name, path); + config.custom_static.remove(name); + } + } + } + + // Set db path if overridden + let db_path: String = if let Some(dbpath) = opts.dbpath.clone() { + dbpath + } else { + dirs::db_path(&profile) + .expect("Failed to get db path") + .to_str() + .unwrap() + .to_string() + }; + info!("Using DB at path {:?}", db_path); + + let asset_path = opts.webpath.map(PathBuf::from); + info!("Using aw-webui assets at path {:?}", asset_path); + + // Only use legacy import if opts.dbpath is not set + let legacy_import = !opts.no_legacy_import && opts.dbpath.is_none(); + if opts.dbpath.is_some() { + info!("Since custom dbpath is set, --no-legacy-import is implied"); + } + + let device_id: String = if let Some(id) = opts.device_id { + id + } else { + device_id::get_device_id() + }; + + #[cfg(any(feature = "encryption", feature = "encryption-vendored"))] + let datastore = match opts.db_password { + Some(key) if key.is_empty() => { + // SQLCipher silently treats PRAGMA key '' as no encryption — reject early so + // callers don't end up with a plaintext database while believing it is encrypted. + panic!("--db-password / AW_DB_PASSWORD must not be empty; aborting to prevent silent plaintext storage"); + } + Some(key) => { + info!("Using encrypted database (SQLCipher)"); + aw_datastore::Datastore::new_encrypted(db_path, key, legacy_import) + } + None => aw_datastore::Datastore::new(db_path, legacy_import), + }; + #[cfg(not(any(feature = "encryption", feature = "encryption-vendored")))] + { + if std::env::var("AW_DB_PASSWORD").is_ok() { + panic!( + "AW_DB_PASSWORD is set but this binary was not compiled with encryption support. \ + Refusing to start with an unencrypted database when the user requested encryption. \ + Rebuild with the 'encryption' or 'encryption-vendored' feature, or unset \ + AW_DB_PASSWORD to use an unencrypted database." + ); + } + } + #[cfg(not(any(feature = "encryption", feature = "encryption-vendored")))] + let datastore = aw_datastore::Datastore::new(db_path, legacy_import); + + let server_state = endpoints::ServerState { + // Even if legacy_import is set to true it is disabled on Android so + // it will not happen there + datastore, + asset_resolver: endpoints::AssetResolver::new(asset_path), + device_id, + }; + + let _rocket = endpoints::build_rocket(server_state, config) + .ignite() + .await?; + #[cfg(target_os = "linux")] + let _ = sd_notify::notify(true, &[NotifyState::Ready]); + _rocket.launch().await?; + + Ok(()) +} diff --git a/aw-server/src/panic_guard.rs b/aw-server/src/panic_guard.rs new file mode 100644 index 0000000..db8b60a --- /dev/null +++ b/aw-server/src/panic_guard.rs @@ -0,0 +1,106 @@ +//! Catching Rust panics before they cross an FFI boundary. +//! +//! Since Rust 1.81 a panic that unwinds out of an `extern "C"` function aborts +//! the process. On Android that means every panic inside a +//! `Java_net_activitywatch_android_…` JNI entry point kills the whole app with +//! `SIGABRT` instead of surfacing as a recoverable error — which is what the +//! `libaw_server.so`/`libaw_sync.so` crash clusters in +//! ActivityWatch/aw-android#267 and the sync aborts in +//! ActivityWatch/aw-android#220 are. +//! +//! The JNI-facing wrappers in [`crate::android`] are thin: they call +//! [`catch_panic`] and turn an `Err` into the same `{"error": "…"}` object the +//! Kotlin side already parses. The logic here is platform-independent so it can +//! be unit tested on the host, where the JNI modules do not even compile. + +use std::any::Any; +use std::panic::{catch_unwind, AssertUnwindSafe}; + +/// Best-effort human-readable message for a panic payload. +/// +/// `panic!` payloads are `&'static str` for literal messages and `String` for +/// formatted ones; anything else (`panic_any`) is reported by type only. +pub fn panic_message(payload: &(dyn Any + Send)) -> String { + if let Some(s) = payload.downcast_ref::<&'static str>() { + (*s).to_string() + } else if let Some(s) = payload.downcast_ref::() { + s.clone() + } else { + "Box".to_string() + } +} + +/// Run `f`, converting a panic into an `Err` describing it. +/// +/// The panic hook still runs first (and on Android still logs to logcat), so +/// this does not hide the panic — it only stops the unwind before it reaches +/// the `extern "C"` frame that would abort the process. +/// +/// `JNIEnv` is not `UnwindSafe`, and neither is the `&mut` reborrow the JNI +/// wrappers capture, so [`AssertUnwindSafe`] is applied here rather than at +/// every call site. That is sound for this use: on the error path the caller +/// only allocates a Java string from the env, and any Rust state the panic left +/// inconsistent (the datastore connection) is behind a channel to a separate +/// worker thread rather than shared through the closure. +pub fn catch_panic T>(name: &str, f: F) -> Result { + catch_unwind(AssertUnwindSafe(f)) + .map_err(|payload| format!("panic in {}: {}", name, panic_message(payload.as_ref()))) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn passes_through_the_return_value_when_no_panic() { + assert_eq!(catch_panic("ok", || 42), Ok(42)); + } + + #[test] + fn reports_str_payloads() { + let err = catch_panic("entry", || panic!("boom")).unwrap_err(); + assert_eq!(err, "panic in entry: boom"); + } + + #[test] + fn reports_formatted_string_payloads() { + let err = catch_panic("entry", || panic!("bad bucket {}", 7)).unwrap_err(); + assert_eq!(err, "panic in entry: bad bucket 7"); + } + + #[test] + fn reports_unwrap_and_expect_payloads() { + // This is the shape the sync path actually panics in: an `unwrap()` or + // `expect()` on a datastore error deep inside the JNI call. + fn missing() -> Option { + None + } + let err = catch_panic("entry", || missing().expect("no value")).unwrap_err(); + assert!(err.starts_with("panic in entry: no value"), "{err}"); + + fn failing() -> Result { + Err("io".to_string()) + } + let err = catch_panic("entry", || failing().unwrap()).unwrap_err(); + assert!(err.contains("io"), "{err}"); + } + + #[test] + fn reports_non_string_payloads_by_placeholder() { + let err = catch_panic("entry", || std::panic::panic_any(7u8)).unwrap_err(); + assert_eq!(err, "panic in entry: Box"); + } + + #[test] + fn allows_closures_capturing_mutable_state() { + // Mirrors how the JNI wrappers capture a `&mut JNIEnv` reborrow: the + // borrow must end when catch_panic returns so the error path can use it. + let mut env = String::new(); + let result = catch_panic("entry", || { + env.push_str("touched"); + panic!("after touching env"); + }); + assert!(result.is_err()); + assert_eq!(env, "touched"); + } +} diff --git a/aw-server/src/version.rs b/aw-server/src/version.rs new file mode 100644 index 0000000..b32d347 --- /dev/null +++ b/aw-server/src/version.rs @@ -0,0 +1,62 @@ +//! Version string reported by `GET /api/0/info`. +//! +//! By default this is the aw-server-rust package version. That is wrong for +//! builds that embed aw-server-rust as a component of a larger product: on +//! Android the webui footer showed `v0.14.0 (rust)` (the Cargo.toml version) +//! while the installed app was `v0.14.0b2`. +//! +//! Such builds call [`set_version_override`] at startup to report their own +//! release version instead. + +use std::sync::RwLock; + +static VERSION_OVERRIDE: RwLock> = RwLock::new(None); + +/// Report `version` from `/api/0/info` instead of the package version. +/// +/// The string is used verbatim, so the caller controls the exact format +/// (including any `v` prefix). +pub fn set_version_override(version: &str) { + let mut guard = VERSION_OVERRIDE.write().unwrap(); + *guard = Some(version.to_string()); +} + +/// The version string to report from `/api/0/info`. +pub fn version_string() -> String { + if let Some(version) = VERSION_OVERRIDE.read().unwrap().as_ref() { + return version.clone(); + } + const VERSION: Option<&str> = option_env!("CARGO_PKG_VERSION"); + format!("v{} (rust)", VERSION.unwrap_or("(unknown)")) +} + +#[cfg(test)] +mod tests { + use super::*; + + struct VersionGuard; + impl Drop for VersionGuard { + fn drop(&mut self) { + *VERSION_OVERRIDE.write().unwrap() = None; + } + } + + // These share process-global state, so they run as one test. + #[test] + fn override_replaces_package_version() { + let _guard = VersionGuard; // resets VERSION_OVERRIDE on exit, even on panic + + assert!( + version_string().ends_with(" (rust)"), + "default should report the package version, got {:?}", + version_string() + ); + + set_version_override("v0.14.0b2"); + assert_eq!(version_string(), "v0.14.0b2"); + + // Used verbatim: the caller owns the format. + set_version_override("1.2.3-custom"); + assert_eq!(version_string(), "1.2.3-custom"); + } +} diff --git a/aw-server/tests/api.rs b/aw-server/tests/api.rs new file mode 100644 index 0000000..99690f0 --- /dev/null +++ b/aw-server/tests/api.rs @@ -0,0 +1,914 @@ +extern crate log; + +extern crate rocket; + +extern crate aw_datastore; +extern crate aw_server; + +#[cfg(test)] +mod api_tests { + use std::collections::HashMap; + + use rocket::http::{ContentType, Header, Status}; + use serde_json::{json, Value}; + + use aw_server::config; + use aw_server::endpoints; + + use aw_models::{Bucket, BucketsExport}; + use rocket::local::blocking::Client; + + fn setup_testserver() -> rocket::Rocket { + let state = endpoints::ServerState { + datastore: aw_datastore::Datastore::new_in_memory(false), + asset_resolver: endpoints::AssetResolver::new(None), + device_id: "test_id".to_string(), + }; + let aw_config = config::AWConfig::default(); + endpoints::build_rocket(state, aw_config) + } + + #[test] + fn export_includes_pending_writes_and_preserves_headers_and_missing_bucket_errors() { + let server = setup_testserver(); + let datastore = server + .state::() + .unwrap() + .datastore + .clone(); + let bucket: Bucket = serde_json::from_value(json!({ + "id": "live", "type": "test", "client": "test", "hostname": "test" + })) + .unwrap(); + datastore.create_bucket(&bucket).unwrap(); + let event = aw_models::Event::default(); + let inserted = datastore.insert_events("live", &[event]).unwrap(); + // No force_commit: exports must see writes acknowledged by the worker. + let client = Client::untracked(server).unwrap(); + for path in ["/api/0/export", "/api/0/buckets/live/export"] { + let response = client + .get(path) + .header(Header::new("Host", "127.0.0.1:5600")) + .dispatch(); + assert_eq!(response.status(), Status::Ok); + assert_eq!(response.content_type(), Some(ContentType::JSON)); + assert_eq!( + response.headers().get_one("Content-Disposition"), + Some("attachment; filename=aw-bucket-export_live.json") + ); + let body: Value = serde_json::from_str(&response.into_string().unwrap()).unwrap(); + assert_eq!( + body["buckets"]["live"]["events"], + serde_json::to_value(&inserted).unwrap() + ); + } + let response = client + .get("/api/0/buckets/missing/export") + .header(Header::new("Host", "127.0.0.1:5600")) + .dispatch(); + assert_eq!(response.status(), Status::NotFound); + let body: Value = serde_json::from_str(&response.into_string().unwrap()).unwrap(); + assert!(body["message"].as_str().unwrap().contains("does not exist")); + } + + #[test] + fn test_bucket() { + let server = setup_testserver(); + let client = Client::untracked(server).expect("valid instance"); + + // Get empty list of buckets + let mut res = client + .get("/api/0/buckets/") + .header(ContentType::JSON) + .header(Header::new("Host", "127.0.0.1:5600")) + .dispatch(); + assert_eq!(res.status(), rocket::http::Status::Ok); + let buckets: HashMap = + serde_json::from_str(&res.into_string().unwrap()).unwrap(); + assert_eq!(buckets.len(), 0); + + // Try to fetch non-existing bucket + res = client + .get("/api/0/buckets/id") + .header(ContentType::JSON) + .header(Header::new("Host", "127.0.0.1:5600")) + .dispatch(); + assert_eq!(res.status(), rocket::http::Status::NotFound); + + // Create bucket + res = client + .post("/api/0/buckets/id") + .header(ContentType::JSON) + .header(Header::new("Host", "127.0.0.1:5600")) + .body( + r#"{ + "id": "id", + "type": "type", + "client": "client", + "hostname": "hostname" + }"#, + ) + .dispatch(); + assert_eq!(res.status(), rocket::http::Status::Ok); + + // Try to create bucket which already exists + res = client + .post("/api/0/buckets/id") + .header(ContentType::JSON) + .header(Header::new("Host", "127.0.0.1:5600")) + .body( + r#"{ + "id": "id", + "type": "type", + "client": "client", + "hostname": "hostname" + }"#, + ) + .dispatch(); + assert_eq!(res.status(), rocket::http::Status::NotModified); + assert_eq!( + res.into_string().unwrap(), + r#"{"message":"Bucket 'id' already exists"}"# + ); + + // Get list of buckets (1 bucket) + res = client + .get("/api/0/buckets/") + .header(ContentType::JSON) + .header(Header::new("Host", "127.0.0.1:5600")) + .dispatch(); + assert_eq!(res.status(), rocket::http::Status::Ok); + let buckets: HashMap = + serde_json::from_str(&res.into_string().unwrap()).unwrap(); + assert_eq!(buckets.len(), 1); + let bucket = buckets.get("id").unwrap(); + assert_eq!(bucket.id, "id"); + assert_eq!(bucket._type, "type"); + assert_eq!(bucket.client, "client"); + assert_eq!(bucket.hostname, "hostname"); + assert!(bucket.events.is_none()); + assert_eq!(bucket.metadata.start, None); + assert_eq!(bucket.metadata.end, None); + + // Get newly created bucket + res = client + .get("/api/0/buckets/id") + .header(ContentType::JSON) + .header(Header::new("Host", "127.0.0.1:5600")) + .dispatch(); + assert_eq!(res.status(), rocket::http::Status::Ok); + // Validate output + let bucket: Bucket = serde_json::from_str(&res.into_string().unwrap()).unwrap(); + assert_eq!(bucket.id, "id"); + assert_eq!(bucket._type, "type"); + assert_eq!(bucket.client, "client"); + assert_eq!(bucket.hostname, "hostname"); + assert!(bucket.events.is_none()); + assert_eq!(bucket.metadata.start, None); + assert_eq!(bucket.metadata.end, None); + + // Get non-existing bucket + res = client + .get("/api/0/buckets/invalid_bucket") + .header(ContentType::JSON) + .header(Header::new("Host", "127.0.0.1:5600")) + .dispatch(); + assert_eq!(res.status(), rocket::http::Status::NotFound); + + // Delete bucket + res = client + .delete("/api/0/buckets/id") + .header(ContentType::JSON) + .header(Header::new("Host", "127.0.0.1:5600")) + .dispatch(); + assert_eq!(res.status(), rocket::http::Status::Ok); + + // Try (and fail) to get deleted bucket + res = client + .get("/api/0/buckets/id") + .header(ContentType::JSON) + .header(Header::new("Host", "127.0.0.1:5600")) + .dispatch(); + assert_eq!(res.status(), rocket::http::Status::NotFound); + + // Get empty list of buckets + let res = client + .get("/api/0/buckets/") + .header(ContentType::JSON) + .header(Header::new("Host", "127.0.0.1:5600")) + .dispatch(); + assert_eq!(res.status(), rocket::http::Status::Ok); + let buckets: HashMap = + serde_json::from_str(&res.into_string().unwrap()).unwrap(); + assert_eq!(buckets.len(), 0); + } + + #[test] + fn test_bucket_hostname_validation() { + let server = setup_testserver(); + let client = Client::untracked(server).expect("valid instance"); + + // A client-supplied hostname containing whitespace is rejected + let res = client + .post("/api/0/buckets/spacey") + .header(ContentType::JSON) + .header(Header::new("Host", "127.0.0.1:5600")) + .body( + r#"{ + "id": "spacey", + "type": "type", + "client": "client", + "hostname": "my phone" + }"#, + ) + .dispatch(); + assert_eq!(res.status(), rocket::http::Status::BadRequest); + assert_eq!( + res.into_string().unwrap(), + r#"{"message":"Invalid hostname \"my phone\": hostname may not contain whitespace"}"# + ); + + // The rejected bucket was not created + let res = client + .get("/api/0/buckets/spacey") + .header(ContentType::JSON) + .header(Header::new("Host", "127.0.0.1:5600")) + .dispatch(); + assert_eq!(res.status(), rocket::http::Status::NotFound); + + // An already-existing bucket keeps the established "already exists" response + // even if the re-create payload carries a whitespace hostname. Watchers + // idempotently re-create their bucket on startup, so a legacy bucket stored + // under a spacey hostname must not start hard-failing every boot. + let res = client + .post("/api/0/buckets/existing") + .header(ContentType::JSON) + .header(Header::new("Host", "127.0.0.1:5600")) + .body( + r#"{ + "id": "existing", + "type": "type", + "client": "client", + "hostname": "hostname" + }"#, + ) + .dispatch(); + assert_eq!(res.status(), rocket::http::Status::Ok); + + let res = client + .post("/api/0/buckets/existing") + .header(ContentType::JSON) + .header(Header::new("Host", "127.0.0.1:5600")) + .body( + r#"{ + "id": "existing", + "type": "type", + "client": "client", + "hostname": "my phone" + }"#, + ) + .dispatch(); + assert_eq!(res.status(), rocket::http::Status::NotModified); + + // The "!local" sentinel bypasses validation (resolved server-side) + let res = client + .post("/api/0/buckets/localbucket") + .header(ContentType::JSON) + .header(Header::new("Host", "127.0.0.1:5600")) + .body( + r#"{ + "id": "localbucket", + "type": "type", + "client": "client", + "hostname": "!local" + }"#, + ) + .dispatch(); + assert_eq!(res.status(), rocket::http::Status::Ok); + } + #[test] + fn test_events() { + let server = setup_testserver(); + let client = Client::untracked(server).expect("valid instance"); + + // Create bucket + let res = client + .post("/api/0/buckets/id") + .header(ContentType::JSON) + .header(Header::new("Host", "127.0.0.1:5600")) + .body( + r#"{ + "id": "id", + "type": "type", + "client": "client", + "hostname": "hostname" + }"#, + ) + .dispatch(); + assert_eq!(res.status(), rocket::http::Status::Ok); + + // Insert a single event + let res = client + .post("/api/0/buckets/id/events") + .header(ContentType::JSON) + .header(Header::new("Host", "127.0.0.1:5600")) + .body( + r#"[{ + "timestamp": "2018-01-01T01:01:01Z", + "duration": 1.0, + "data": {} + }]"#, + ) + .dispatch(); + assert_eq!(res.status(), rocket::http::Status::Ok); + assert_eq!( + res.into_string().unwrap(), + r#"[{"id":1,"timestamp":"2018-01-01T01:01:01Z","duration":1.0,"data":{}}]"# + ); + + // Get inserted event + let res = client + .get("/api/0/buckets/id/events") + .header(ContentType::JSON) + .header(Header::new("Host", "127.0.0.1:5600")) + .dispatch(); + assert_eq!(res.status(), rocket::http::Status::Ok); + assert_eq!( + res.into_string().unwrap(), + r#"[{"id":1,"timestamp":"2018-01-01T01:01:01Z","duration":1.0,"data":{}}]"# + ); + + // Heartbeat + let res = client + .post("/api/0/buckets/id/heartbeat?pulsetime=2") + .header(ContentType::JSON) + .header(Header::new("Host", "127.0.0.1:5600")) + .body( + r#"{ + "timestamp": "2018-01-01T01:01:02Z", + "duration": 1.0, + "data": {} + }"#, + ) + .dispatch(); + assert_eq!(res.status(), rocket::http::Status::Ok); + // After the fix for #559, the heartbeat response includes the DB event ID + assert_eq!( + res.into_string().unwrap(), + r#"{"id":1,"timestamp":"2018-01-01T01:01:01Z","duration":2.0,"data":{}}"# + ); + + // Get heartbeat event + let res = client + .get("/api/0/buckets/id/events") + .header(ContentType::JSON) + .header(Header::new("Host", "127.0.0.1:5600")) + .dispatch(); + assert_eq!(res.status(), rocket::http::Status::Ok); + assert_eq!( + res.into_string().unwrap(), + r#"[{"id":1,"timestamp":"2018-01-01T01:01:01Z","duration":2.0,"data":{}}]"# + ); + + // Delete event + client + .delete("/api/0/buckets/id/events/1") + .header(Header::new("Host", "127.0.0.1:5600")) + .dispatch(); + + // Get eventcount + let res = client + .get("/api/0/buckets/id/events/count") + .header(ContentType::JSON) + .header(Header::new("Host", "127.0.0.1:5600")) + .dispatch(); + assert_eq!(res.status(), rocket::http::Status::Ok); + assert_eq!(res.into_string().unwrap(), "0"); + + // Delete bucket + let res = client + .delete("/api/0/buckets/id") + .header(ContentType::JSON) + .header(Header::new("Host", "127.0.0.1:5600")) + .dispatch(); + assert_eq!(res.status(), rocket::http::Status::Ok); + } + + #[test] + fn test_import_export() { + let server = setup_testserver(); + let client = Client::untracked(server).expect("valid instance"); + + // Import bucket + let res = client + .post("/api/0/import") + .header(ContentType::JSON) + .header(Header::new("Host", "127.0.0.1:5600")) + .body( + r#"{"buckets": + {"id1": { + "id": "id1", + "type": "type", + "client": "client", + "hostname": "hostname", + "events": [{ + "timestamp":"2000-01-01T00:00:00Z", + "duration":1.0, + "data": {} + }] + }}}"#, + ) + .dispatch(); + assert_eq!(res.status(), rocket::http::Status::Ok); + + // Import already existing bucket with a new event — should merge instead of fail + let res = client + .post("/api/0/import") + .header(ContentType::JSON) + .header(Header::new("Host", "127.0.0.1:5600")) + .body( + r#"{"buckets": + {"id1": { + "id": "id1", + "type": "type", + "client": "client", + "hostname": "hostname", + "events": [{ + "timestamp":"2000-01-02T00:00:00Z", + "duration":1.0, + "data": {} + }] + }}}"#, + ) + .dispatch(); + assert_eq!(res.status(), rocket::http::Status::Ok); + + // Verify events were merged — bucket should now have 2 events + let res = client + .get("/api/0/buckets/id1/events") + .header(ContentType::JSON) + .header(Header::new("Host", "127.0.0.1:5600")) + .dispatch(); + assert_eq!(res.status(), rocket::http::Status::Ok); + let events: serde_json::Value = serde_json::from_str(&res.into_string().unwrap()).unwrap(); + assert_eq!(events.as_array().unwrap().len(), 2); + + // Re-import the first event again — should be idempotent (no duplicate created) + let res = client + .post("/api/0/import") + .header(ContentType::JSON) + .header(Header::new("Host", "127.0.0.1:5600")) + .body( + r#"{"buckets": + {"id1": { + "id": "id1", + "type": "type", + "client": "client", + "hostname": "hostname", + "events": [{ + "timestamp":"2000-01-01T00:00:00Z", + "duration":1.0, + "data": {} + }] + }}}"#, + ) + .dispatch(); + assert_eq!(res.status(), rocket::http::Status::Ok); + + // Count should still be 2, not 3 — re-import is idempotent + let res = client + .get("/api/0/buckets/id1/events") + .header(ContentType::JSON) + .header(Header::new("Host", "127.0.0.1:5600")) + .dispatch(); + assert_eq!(res.status(), rocket::http::Status::Ok); + let events: serde_json::Value = serde_json::from_str(&res.into_string().unwrap()).unwrap(); + assert_eq!( + events.as_array().unwrap().len(), + 2, + "Re-importing the same event should be idempotent" + ); + + // Import a narrower event fully contained within an existing longer event. + // This should be preserved as a distinct event, not dropped by clipped dedup. + let res = client + .post("/api/0/import") + .header(ContentType::JSON) + .header(Header::new("Host", "127.0.0.1:5600")) + .body( + r#"{"buckets": + {"id1": { + "id": "id1", + "type": "type", + "client": "client", + "hostname": "hostname", + "events": [{ + "timestamp":"2000-01-01T00:00:30Z", + "duration":30.0, + "data": {} + }] + }}}"#, + ) + .dispatch(); + assert_eq!(res.status(), rocket::http::Status::Ok); + + let res = client + .get("/api/0/buckets/id1/events") + .header(ContentType::JSON) + .header(Header::new("Host", "127.0.0.1:5600")) + .dispatch(); + assert_eq!(res.status(), rocket::http::Status::Ok); + let events: serde_json::Value = serde_json::from_str(&res.into_string().unwrap()).unwrap(); + assert_eq!( + events.as_array().unwrap().len(), + 3, + "Contained event should not be dropped by clipped dedup" + ); + + // Export single created bucket + let res = client + .get("/api/0/buckets/id1/export") + .header(ContentType::JSON) + .header(Header::new("Host", "127.0.0.1:5600")) + .dispatch(); + assert_eq!(res.status(), rocket::http::Status::Ok); + let export: BucketsExport = serde_json::from_str(&res.into_string().unwrap()).unwrap(); + + // Delete bucket so we can import it again + let res = client + .delete("/api/0/buckets/id1") + .header(ContentType::JSON) + .header(Header::new("Host", "127.0.0.1:5600")) + .dispatch(); + assert_eq!(res.status(), rocket::http::Status::Ok); + + // Import exported bucket again but with multipart/form-data + let boundary_start = b"--a\r\n"; + let disposition = b"Content-Disposition: form-data; name=\"buckets.json\"\r\n"; + let content_type = b"Content-Type: application/json\r\n"; + let data_start = b"\r\n"; + let content = serde_json::to_vec(&export).unwrap(); + let boundary_end = b"\r\n--a--"; + let sum = [ + &boundary_start[..], + &disposition[..], + &content_type[..], + &data_start[..], + &content[..], + &boundary_end[..], + ] + .concat(); + + let res = client + .post("/api/0/import") + .header(Header::new( + "Content-Type", + "multipart/form-data; boundary=a", + )) + .header(Header::new("Host", "127.0.0.1:5600")) + .body(&sum[..]) + .dispatch(); + assert_eq!(res.status(), rocket::http::Status::Ok); + + // Get created bucket + let res = client + .get("/api/0/buckets/id1") + .header(ContentType::JSON) + .header(Header::new("Host", "127.0.0.1:5600")) + .dispatch(); + assert_eq!(res.status(), rocket::http::Status::Ok); + println!("{:?}", res.into_string()); + + // Export all buckets + let res = client + .get("/api/0/export") + .header(ContentType::JSON) + .header(Header::new("Host", "127.0.0.1:5600")) + .dispatch(); + assert_eq!(res.status(), rocket::http::Status::Ok); + let export: BucketsExport = serde_json::from_str(&res.into_string().unwrap()).unwrap(); + let mut buckets = export.buckets; + assert_eq!(buckets.len(), 1); + let b = buckets.remove("id1").unwrap(); + assert_eq!( + b.events.unwrap().take_inner().len(), + 3, + "Export should preserve the contained event added during merge testing" + ); + + assert_eq!(buckets.len(), 0); + } + + #[test] + fn test_query() { + let server = setup_testserver(); + let client = Client::untracked(server).expect("valid instance"); + + // Minimal query + let res = client + .post("/api/0/query") + .header(ContentType::JSON) + .header(Header::new("Host", "127.0.0.1:5600")) + .body( + r#"{ + "timeperiods": ["2000-01-01T00:00:00Z/2020-01-01T00:00:00Z"], + "query": ["return 1;"] + }"#, + ) + .dispatch(); + assert_eq!(res.status(), rocket::http::Status::Ok); + assert_eq!(res.into_string().unwrap(), r#"[1.0]"#); + + // Create bucket to query later + let res = client + .post("/api/0/buckets/id") + .header(ContentType::JSON) + .header(Header::new("Host", "127.0.0.1:5600")) + .body( + r#"{ + "id": "id", + "type": "type", + "client": "client", + "hostname": "hostname" + }"#, + ) + .dispatch(); + assert_eq!(res.status(), rocket::http::Status::Ok); + + // Insert a event to query later + let res = client + .post("/api/0/buckets/id/events") + .header(ContentType::JSON) + .header(Header::new("Host", "127.0.0.1:5600")) + .body( + r#"[{ + "timestamp": "2018-01-01T01:01:01Z", + "duration": 1.0, + "data": {} + }]"#, + ) + .dispatch(); + assert_eq!(res.status(), rocket::http::Status::Ok); + + // Query events + let res = client + .post("/api/0/query") + .header(ContentType::JSON) + .header(Header::new("Host", "127.0.0.1:5600")) + .body( + r#"{ + "timeperiods": ["2000-01-01T00:00:00Z/2020-01-01T00:00:00Z"], + "query": ["return query_bucket(\"id\");"] + }"#, + ) + .dispatch(); + assert_eq!(res.status(), rocket::http::Status::Ok); + assert_eq!( + res.into_string().unwrap(), + r#"[[{"data":{},"duration":1.0,"id":1,"timestamp":"2018-01-01T01:01:01Z"}]]"# + ); + + // Test error + let res = client + .post("/api/0/query") + .header(ContentType::JSON) + .header(Header::new("Host", "127.0.0.1:5600")) + .body( + r#"{ + "timeperiods": ["2000-01-01T00:00:00Z/2020-01-01T00:00:00Z"], + "query": [""] + }"#, + ) + .dispatch(); + assert_eq!(res.status(), rocket::http::Status::BadRequest); + assert_eq!(res.into_string().unwrap(), r#"{"message":"EmptyQuery"}"#); + } + + fn set_setting_request(client: &Client, key: &str, value: &Value) -> Status { + let body = serde_json::to_string(value).unwrap(); + let res = client + .post(format!("/api/0/settings/{}", key)) + .header(ContentType::JSON) + .header(Header::new("Host", "127.0.0.1:5600")) + .body(body) + .dispatch(); + res.status() + } + + #[test] + fn test_illegally_long_key() { + let server = setup_testserver(); + let client = Client::untracked(server).expect("valid instance"); + + // Test getting not found (getting nonexistent key) + let key = "thisisaverylongkthisisaverylongkthisisaverylongkthisisaverylongkthisisaverylongkthisisaverylongkthisisaverylongkthisisaverylongk"; + let value = json!("test_value"); + let res = set_setting_request(&client, key, &value); + assert_eq!(res, rocket::http::Status::BadRequest); + } + + #[test] + fn test_setting_setting() { + let server = setup_testserver(); + let client = Client::untracked(server).expect("valid instance"); + + // Test value creation + let key = "test_key"; + let value = json!("test_value"); + let response_status = set_setting_request(&client, key, &value); + assert_eq!(response_status, rocket::http::Status::Created); + } + + #[test] + fn test_get_unset_setting() { + let server = setup_testserver(); + let client = Client::untracked(server).expect("valid instance"); + + // Test getting not found (getting nonexistent key) + let key = "non_existent_key"; + let res = client + .get(format!("/api/0/settings/{}", key)) + .header(Header::new("Host", "127.0.0.1:5600")) + .dispatch(); + assert_eq!(res.status(), rocket::http::Status::Ok); + assert_eq!(res.into_string().unwrap(), "null") + } + + #[test] + fn test_get_settings() { + let server = setup_testserver(); + let client = Client::untracked(server).expect("valid instance"); + + let key1 = "test_key"; + let key2 = "test_key_2"; + let value = json!("test_value"); + let response1_status = set_setting_request(&client, key1, &value); + assert_eq!(response1_status, rocket::http::Status::Created); + let response2_status = set_setting_request(&client, key2, &value); + assert_eq!(response2_status, rocket::http::Status::Created); + + let res = client + .get("/api/0/settings") + .header(Header::new("Host", "127.0.0.1:5600")) + .dispatch(); + + assert_eq!(res.status(), rocket::http::Status::Ok); + + let deserialized: Value = serde_json::from_str(&res.into_string().unwrap()).unwrap(); + let expected: Value = + serde_json::from_str(r#"{"test_key_2":"test_value","test_key":"test_value"}"#).unwrap(); + assert_eq!(deserialized, expected); + } + + #[test] + fn test_get_setting() { + let server = setup_testserver(); + let client = Client::untracked(server).expect("valid instance"); + + let key = "test_key"; + let value = json!("test_value"); + let response_status = set_setting_request(&client, key, &value); + assert_eq!(response_status, rocket::http::Status::Created); + + // Test getting + let res = client + .get(format!("/api/0/settings/{}", key)) + .header(Header::new("Host", "127.0.0.1:5600")) + .dispatch(); + assert_eq!(res.status(), rocket::http::Status::Ok); + let deserialized: Value = serde_json::from_str(&res.into_string().unwrap()).unwrap(); + assert_eq!(deserialized, value); + } + + #[test] + fn test_get_setting_dotted_key() { + // HTTP accepts dotted keys; JNI getSetting must use the same grammar + // (ActivityWatch/aw-server-rust#653 Greptile: don't reject '.'). + let server = setup_testserver(); + let client = Client::untracked(server).expect("valid instance"); + + let key = "nested.like.key"; + let value = json!("dotted_value"); + assert_eq!( + set_setting_request(&client, key, &value), + rocket::http::Status::Created + ); + + let res = client + .get(format!("/api/0/settings/{}", key)) + .header(Header::new("Host", "127.0.0.1:5600")) + .dispatch(); + assert_eq!(res.status(), rocket::http::Status::Ok); + let deserialized: Value = serde_json::from_str(&res.into_string().unwrap()).unwrap(); + assert_eq!(deserialized, value); + } + + #[test] + fn test_get_setting_list() { + let server = setup_testserver(); + let client = Client::untracked(server).expect("valid instance"); + + // Test array + let key = "test_key_array"; + let value = json!([1, 2, 3]); + let response_status = set_setting_request(&client, key, &value); + assert_eq!(response_status, rocket::http::Status::Created); + + let res = client + .get(format!("/api/0/settings/{}", key)) + .header(Header::new("Host", "127.0.0.1:5600")) + .dispatch(); + assert_eq!(res.status(), rocket::http::Status::Ok); + let deserialized: Value = serde_json::from_str(&res.into_string().unwrap()).unwrap(); + assert_eq!(deserialized, value); + } + + #[test] + fn test_get_setting_dict() { + let server = setup_testserver(); + let client = Client::untracked(server).expect("valid instance"); + + // Test dict + let key = "test_key_dict"; + let value = json!({"key": "value", "another_key": "another value"}); + let response_status = set_setting_request(&client, key, &value); + assert_eq!(response_status, rocket::http::Status::Created); + + let res = client + .get("/api/0/settings/test_key_dict") + .header(Header::new("Host", "127.0.0.1:5600")) + .dispatch(); + assert_eq!(res.status(), rocket::http::Status::Ok); + let deserialized: Value = serde_json::from_str(&res.into_string().unwrap()).unwrap(); + assert_eq!(deserialized, value); + } + + #[test] + fn test_set_setting() { + let server = setup_testserver(); + let client = Client::untracked(server).expect("valid instance"); + + let key = "test_key"; + let value1 = json!("test_value"); + let value2 = json!("changed_test_value"); + let post_1_status = set_setting_request(&client, key, &value1); + assert_eq!(post_1_status, rocket::http::Status::Created); + + let res = client + .get(format!("/api/0/settings/{}", key)) + .header(Header::new("Host", "127.0.0.1:5600")) + .dispatch(); + assert_eq!(res.status(), rocket::http::Status::Ok); + let deserialized: Value = serde_json::from_str(&res.into_string().unwrap()).unwrap(); + assert_eq!(deserialized, value1); + + let post_2_status = set_setting_request(&client, key, &value2); + assert_eq!(post_2_status, rocket::http::Status::Created); + + let res = client + .get("/api/0/settings/test_key") + .header(Header::new("Host", "127.0.0.1:5600")) + .dispatch(); + assert_eq!(res.status(), rocket::http::Status::Ok); + + let new_deserialized: Value = serde_json::from_str(&res.into_string().unwrap()).unwrap(); + assert_eq!(new_deserialized, value2); + } + + #[test] + fn test_delete_setting() { + let server = setup_testserver(); + let client = Client::untracked(server).expect("valid instance"); + + let value = json!("test_value"); + let response_status = set_setting_request(&client, "test_key", &value); + assert_eq!(response_status, rocket::http::Status::Created); + + // Test deleting + let res = client + .delete("/api/0/settings/test_key") + .header(Header::new("Host", "127.0.0.1:5600")) + .dispatch(); + assert_eq!(res.status(), rocket::http::Status::Ok); + + let res = client + .get("/api/0/settings/test_key") + .header(Header::new("Host", "127.0.0.1:5600")) + .dispatch(); + assert_eq!(res.status(), rocket::http::Status::Ok); + assert_eq!(res.into_string().unwrap(), "null"); + } + + #[test] + fn test_cors_catching() { + let server = setup_testserver(); + let client = Client::untracked(server).expect("valid instance"); + + let res = client + .options("/api/0/buckets/") + .header(ContentType::JSON) + .header(Header::new("Host", "127.0.0.1:5600")) + .dispatch(); + assert_eq!(res.status(), rocket::http::Status::Ok); + } +} diff --git a/aw-server/tests/macros.rs b/aw-server/tests/macros.rs new file mode 100644 index 0000000..c1d4970 --- /dev/null +++ b/aw-server/tests/macros.rs @@ -0,0 +1,15 @@ +#[macro_use] +extern crate serde_json; + +#[macro_use] +extern crate aw_server; + +#[cfg(test)] +mod macros_tests { + #[test] + fn test_json_map() { + json_map! {}; + json_map! {"a": json!(1)}; + json_map! {"a": json!(1), "b": json!(2)}; + } +} diff --git a/aw-sync/Cargo.toml b/aw-sync/Cargo.toml new file mode 100644 index 0000000..74d825d --- /dev/null +++ b/aw-sync/Cargo.toml @@ -0,0 +1,44 @@ +[package] +name = "aw-sync" +version = "0.1.0" +authors = ["Erik Bjäreholt "] +edition = "2021" + +[lib] +name = "aw_sync" +path = "src/lib.rs" +crate-type = ["cdylib", "rlib"] + +[[bin]] +name = "aw-sync" +path = "src/main.rs" + +[dependencies] +log = "0.4" +toml = "0.8" +chrono = { version = "0.4", features = ["serde"] } +serde = "1.0" +serde_json = "1.0" +reqwest = { version = "0.12", features = ["json", "blocking"] } +dirs = "6" +gethostname = "0.4.3" + +# CLI-only dependencies (optional) +clap = { version = "4.1", features = ["derive"], optional = true } +ctrlc = { version = "3.4.5", optional = true } + +aw-server = { path = "../aw-server" } +aw-models = { path = "../aw-models" } +aw-datastore = { path = "../aw-datastore" } +aw-client-rust = { path = "../aw-client-rust" } + +[target.'cfg(target_os="linux")'.dependencies] +openssl = { version = "0.10.80", features = ["vendored"] } # https://github.com/ActivityWatch/aw-server-rust/issues/478 + +[target.'cfg(target_os="android")'.dependencies] +jni = { version = "0.21", default-features = false } +android_logger = "0.13" + +[features] +default = ["cli"] +cli = ["clap", "ctrlc"] diff --git a/aw-sync/README.md b/aw-sync/README.md new file mode 100644 index 0000000..e1d1907 --- /dev/null +++ b/aw-sync/README.md @@ -0,0 +1,142 @@ +aw-sync-rust +============ + +Synchronization for ActivityWatch. + +Works by syncing local buckets with a special folder, which in turn should be synchronized by rsync/Syncthing/Dropbox/GDrive/etc. + +The latest beta versions of ActivityWatch ship with the `aw-sync` binary, but it's not enabled by default. You can start it from aw-qt or the command line, but due to the early state of development might not have the best UX. Please report issues and submit PRs! + +Was originally prototyped as a PR to aw-server: https://github.com/ActivityWatch/aw-server/pull/50 + + +## Usage + +This will start a daemon which pulls and pushes events with the sync directory (`~/ActivityWatchSync` by default) every 5 minutes: + +```sh +# Basic sync daemon (syncs all buckets every 5 minutes) +aw-sync + +# Same as above +aw-sync daemon + +# Sync daemon with specific buckets only +aw-sync daemon --buckets "aw-watcher-window,aw-watcher-afk" --start-date "2024-01-01" + +# Sync daemon in push-only or pull-only mode +aw-sync daemon --mode push +aw-sync daemon --mode pull + +# Sync all buckets once and exit +aw-sync sync --start-date "2024-01-01" +``` + +For more options, see `aw-sync --help`. Some notable options: +- `--config`: Read the local server port and API key from a custom aw-server config file. Pass the same path to both aw-server and aw-sync when using aw-server's `--config` override. +- `--buckets`: Specify which buckets to sync (comma-separated). By default, all buckets are synced. + - Use `--buckets "bucket1,bucket2"` to sync specific buckets + - Not specifying this option syncs all buckets by default +- `--start-date`: Only sync events after this date (YYYY-MM-DD) +- `--sync-db`: Specify a specific database file in the sync directory +- `--mode`: Choose sync mode: "push", "pull", or "both" (default: "both") + - For `aw-sync sync`, passing `--mode` (with no other options) is enough to opt into the per-bucket sync path where it's respected; without it, or any of `--buckets`/`--start-date`/`--sync-db`, `aw-sync sync` falls back to the legacy host-based mode, which always does both a pull and a push + +### Setting up sync + +Once you have aw-sync running, you need to set up syncing with the sync directory using your preferred syncing tool. + +The default sync directory is `~/ActivityWatchSync`, but you can change it using the `--sync-dir` option or by setting the `AW_SYNC_DIR` environment variable. + +### Running from source + +If you want to run it from source, in the root of the repository run: + +```sh +cargo run --bin aw-sync +``` +For more options, see `cargo run --bin aw-sync -- --help`. + +## FAQ + +### When is it ready? + +It works today, but there is still testing and polishing to be done before it's "click and play". + +### Why do it this way? + +By essentially only offering a "sync with folder" feature, we give the user a lot of freedom to choose how to store and sync their data. + +We also avoid having to implement complex features such as conflict resolution, by enforcing that each device only writes to files in the sync folder they "own", and other devices may not modify them. + +### What are the limitations? + +- It only syncs afk and window buckets by default (since bucket IDs need to be unique) + - It will work a lot better once proper `hostname -> device ID` migration is complete. +- It doesn't sync settings +- It doesn't support Android, yet. +- It mirrors events to all devices, + - If you have a lot of devices you'll get a lot of duplicates, taking up a lot of space and potentially impacting performance. +- It doesn't support modifying/deleting events, yet. + +--- + +## Advanced usage + +### Syncing real data to a testing instance + +If you want to try sync, you can do so by following these steps. + +We will use a separate testing instance of aw-server(-rust) to store and view the synced data from the sync directory. This is to avoid testing against & potentially pollute production instances in write-scenarios. We will sync all devices with the sync folder, and then sync the sync folder into our testing instance to view. + +To test syncing real events to a sync folder which can then be pulled from. +We will use some helper scripts to do the following: + +1. `./test-sync-push.sh` + - Creates a sync directory **for you to set up sync** with Syncthing/Dropbox/Gdrive/rclone/whatever + - By default `~/ActivityWatchSync` + - Creates a datastore for the current host in the sync folder + - Sync all local buckets of interest (window & afk buckets, by default) to the sync dir + +2. `./test-server.sh` + - Starts a testing server **on port 5667** using a temporary directory as datastore (`/tmp/...`) + +3. `./test-sync-pull.sh` + - Imports all the events from sync folder into the testing server on port 5667 + +4. You should now have all events synced to a local testing instance! + - You can browse [127.0.0.1:5667](http://127.0.0.1:5667) to view testing instance, where you'll see events from synced all hosts. + - You can now set up syncing for `~/ActivityWatchSync` on more devices, and on each one use the script `./test-sync.sh` to push their events into the sync folder, then run `./test-import-sync.sh` on the device where you have the testing instance to update the data there. + +5. To view data from all devices at once, go into [127.0.0.1:5667/#/settings](127.0.0.1:5667/#/settings) and check the "Use multidevice query" checkbox (near the bottom, under "developer settings"). + - You can now navigate back to the activity view for any device, where you should see data from multiple devices being included in (most of) the visualizations. + +In the end, You should get something like this: https://twitter.com/ErikBjare/status/1519399784234246147 + +### Testing with fake data + +#### Pushing to the sync directory + +First start the sending aw-server instance. For example: + +```sh +PORT=5667 +cargo run --bin aw-server -- --testing --port $PORT --dbpath test-$PORT.sqlite --device-id $PORT --no-legacy-import +``` + +You can create some test data by opening `http://localhost:5667/#/stopwatch` and creating a few events. + +Then run `cargo run --bin aw-sync-rust -- --port 5667` to sync your instance's buckets with the target directory. + +#### Pulling from the sync directory + +Now to sync things back from the sync directory into another instance. + +First, lets start another instance: + +```sh +PORT=5668 +cargo run --bin aw-server -- --testing --port $PORT --dbpath test-$PORT.sqlite --device-id $PORT --no-legacy-import +``` + +Now run `cargo run --bin aw-sync -- --port 5668` to pull buckets from the sync dir (retrieving events from the 5667 instance) and push buckets from the 5668 instance to the sync dir. diff --git a/aw-sync/src/accessmethod.rs b/aw-sync/src/accessmethod.rs new file mode 100644 index 0000000..be01d78 --- /dev/null +++ b/aw-sync/src/accessmethod.rs @@ -0,0 +1,114 @@ +use std::collections::HashMap; + +use aw_client_rust::blocking::AwClient; +use chrono::{DateTime, Utc}; +use reqwest::StatusCode; + +use aw_datastore::{Datastore, DatastoreError}; +use aw_models::{Bucket, Event}; + +// This trait should be implemented by both AwClient and Datastore, unifying them under a single API +pub trait AccessMethod: std::fmt::Debug { + fn get_buckets(&self) -> Result, String>; + fn get_bucket(&self, bucket_id: &str) -> Result; + fn create_bucket(&self, bucket: &Bucket) -> Result<(), DatastoreError>; + fn get_events( + &self, + bucket_id: &str, + start: Option>, + end: Option>, + limit: Option, + ) -> Result, String>; + fn insert_events(&self, bucket_id: &str, events: Vec) -> Result<(), String>; + fn get_event_count(&self, bucket_id: &str) -> Result; + fn heartbeat(&self, bucket_id: &str, event: Event, duration: f64) -> Result<(), String>; +} + +/// Every method here returns a `Result`, so a datastore failure must be reported +/// through it rather than unwrapped. +/// +/// These used to `unwrap()`, which on Android turns any datastore error (a +/// dead worker thread, a locked or corrupt database, a disappearing sync file) +/// into a panic that unwinds out of the JNI `extern "C"` frame and aborts the +/// whole app — see ActivityWatch/aw-android#220. +impl AccessMethod for Datastore { + fn get_buckets(&self) -> Result, String> { + Datastore::get_buckets(self).map_err(|e| format!("{e:?}")) + } + fn get_bucket(&self, bucket_id: &str) -> Result { + Datastore::get_bucket(self, bucket_id) + } + fn create_bucket(&self, bucket: &Bucket) -> Result<(), DatastoreError> { + Datastore::create_bucket(self, bucket)?; + self.force_commit()?; + Ok(()) + } + fn get_events( + &self, + bucket_id: &str, + start: Option>, + end: Option>, + limit: Option, + ) -> Result, String> { + Datastore::get_events(self, bucket_id, start, end, limit).map_err(|e| format!("{e:?}")) + } + fn heartbeat(&self, bucket_id: &str, event: Event, duration: f64) -> Result<(), String> { + Datastore::heartbeat(self, bucket_id, event, duration).map_err(|e| format!("{e:?}"))?; + self.force_commit().map_err(|e| format!("{e:?}"))?; + Ok(()) + } + fn insert_events(&self, bucket_id: &str, events: Vec) -> Result<(), String> { + Datastore::insert_events(self, bucket_id, &events[..]).map_err(|e| format!("{e:?}"))?; + self.force_commit().map_err(|e| format!("{e:?}"))?; + Ok(()) + } + fn get_event_count(&self, bucket_id: &str) -> Result { + Datastore::get_event_count(self, bucket_id, None, None).map_err(|e| format!("{e:?}")) + } +} + +impl AccessMethod for AwClient { + fn get_buckets(&self) -> Result, String> { + AwClient::get_buckets(self).map_err(|e| e.to_string()) + } + fn get_bucket(&self, bucket_id: &str) -> Result { + let bucket = AwClient::get_bucket(self, bucket_id); + match bucket { + Ok(bucket) => Ok(bucket), + Err(e) => { + warn!("{:?}", e); + let code = e.status().unwrap_or(StatusCode::INTERNAL_SERVER_ERROR); + if code == StatusCode::NOT_FOUND { + Err(DatastoreError::NoSuchBucket(bucket_id.into())) + } else { + Err(DatastoreError::InternalError(format!( + "Unexpected error fetching bucket {bucket_id}: {e}" + ))) + } + } + } + } + fn get_events( + &self, + bucket_id: &str, + start: Option>, + end: Option>, + limit: Option, + ) -> Result, String> { + AwClient::get_events(self, bucket_id, start, end, limit).map_err(|e| e.to_string()) + } + fn insert_events(&self, bucket_id: &str, events: Vec) -> Result<(), String> { + AwClient::insert_events(self, bucket_id, events).map_err(|e| e.to_string()) + } + fn get_event_count(&self, bucket_id: &str) -> Result { + AwClient::get_event_count(self, bucket_id).map_err(|e| e.to_string()) + } + fn create_bucket(&self, bucket: &Bucket) -> Result<(), DatastoreError> { + AwClient::create_bucket(self, bucket) + .map_err(|e| DatastoreError::InternalError(e.to_string()))?; + Ok(()) + } + fn heartbeat(&self, bucket_id: &str, event: Event, duration: f64) -> Result<(), String> { + AwClient::heartbeat(self, bucket_id, &event, duration).map_err(|e| format!("{e:?}")) + } +} diff --git a/aw-sync/src/android.rs b/aw-sync/src/android.rs new file mode 100644 index 0000000..c0b27d5 --- /dev/null +++ b/aw-sync/src/android.rs @@ -0,0 +1,400 @@ +use std::ffi::CString; +use std::os::raw::{c_char, c_int, c_void}; +use std::panic; +use std::path::Path; +use std::sync::Once; + +use aw_client_rust::blocking::AwClient; +use aw_server::panic_guard::catch_panic; +use jni::objects::{JClass, JString}; +use jni::sys::{jint, jstring, JNI_VERSION_1_6}; +use jni::JNIEnv; +use serde_json::json; + +use crate::{pull, pull_all, push_with_hostname}; + +const ANDROID_LOG_FATAL: c_int = 7; +const ANDROID_LOG_TAG: &str = "aw-sync"; +/// logcat truncates a single `__android_log_write` payload around 4 KiB. +const MAX_LOG_BYTES: usize = 4000; + +#[link(name = "log")] +extern "C" { + fn __android_log_write(prio: c_int, tag: *const c_char, text: *const c_char) -> c_int; +} + +/// Write `msg` to logcat, splitting on the 4 KiB limit and stripping NULs so +/// `CString::new` cannot fail. Never panics. +fn android_log_fatal(msg: &str) { + let Ok(tag) = CString::new(ANDROID_LOG_TAG) else { + return; + }; + let sanitized = msg.replace('\0', "\\0"); + let mut rest = sanitized.as_str(); + while !rest.is_empty() { + let mut idx = rest.len().min(MAX_LOG_BYTES); + while idx > 0 && !rest.is_char_boundary(idx) { + idx -= 1; + } + if idx == 0 { + break; + } + if let Ok(text) = CString::new(&rest[..idx]) { + unsafe { + __android_log_write(ANDROID_LOG_FATAL, tag.as_ptr(), text.as_ptr()); + } + } + rest = &rest[idx..]; + } +} + +/// Install a panic hook that writes to logcat via `android_log`. +/// +/// The default Rust hook writes to stderr, which Android discards, so a panic +/// in this `.so` currently surfaces only as `SIGABRT` with no message +/// (ActivityWatch/aw-android#220). +fn install_panic_hook() { + panic::set_hook(Box::new(|info| { + let payload = if let Some(s) = info.payload().downcast_ref::<&'static str>() { + *s + } else if let Some(s) = info.payload().downcast_ref::() { + s.as_str() + } else { + "Box" + }; + let location = match info.location() { + Some(loc) => format!("{}:{}:{}", loc.file(), loc.line(), loc.column()), + None => "unknown location".to_string(), + }; + let thread = std::thread::current(); + let thread_name = thread.name().unwrap_or(""); + let header = format!("Rust panic in thread '{thread_name}' at {location}: {payload}"); + + android_log_fatal(&header); + error!("{}", header); + + let backtrace = + panic::catch_unwind(|| format!("{}", std::backtrace::Backtrace::force_capture())); + if let Ok(bt) = backtrace { + android_log_fatal("Backtrace:"); + for line in bt.lines() { + android_log_fatal(line); + } + } + })); +} + +/// Route `log` macros to logcat and install the panic hook. Safe to call more than once. +fn init_android_logging() { + static INIT: Once = Once::new(); + INIT.call_once(|| { + android_logger::init_once( + android_logger::Config::default() + .with_max_level(log::LevelFilter::Info) + .with_tag(ANDROID_LOG_TAG), + ); + install_panic_hook(); + info!("aw-sync: android_logger and panic hook installed"); + }); +} + +/// Called automatically when `System.loadLibrary("aw_sync")` loads this `.so`, +/// so panics are diagnosable even if a JNI entry point is never reached. +#[no_mangle] +pub extern "system" fn JNI_OnLoad(_vm: *mut jni::sys::JavaVM, _reserved: *mut c_void) -> jint { + // Guarded like the other entry points: this is the one that runs before any + // logging exists, so an unwind out of it would abort the app during + // `System.loadLibrary("aw_sync")` with nothing in logcat at all. + let _ = catch_panic("JNI_OnLoad", init_android_logging); + JNI_VERSION_1_6 +} + +/// Helper function to convert Rust string to Java string. +/// +/// Must not panic: it is also used on the guard's error path, where a second +/// unwind would reach the `extern "C"` frame and abort after all. A null return +/// surfaces in Kotlin as a `NullPointerException` at the call site, which +/// `SyncInterface.performSyncAsync` already catches — recoverable, unlike +/// `SIGABRT`. +fn rust_string_to_jstring(env: &JNIEnv, s: String) -> jstring { + match env.new_string(s) { + Ok(output) => output.into_raw(), + Err(e) => { + error!("Couldn't create java string: {}", e); + std::ptr::null_mut() + } + } +} + +/// The response shape `SyncInterface.performSyncAsync` parses: it reads +/// `success` and then either `message` or `error`. +fn sync_error_json(msg: &str) -> String { + json!({ + "success": false, + "error": msg + }) + .to_string() +} + +/// Run a `jstring`-returning JNI entry point with panics caught. +/// +/// A panic that unwinds out of an `extern "C"` function aborts the process +/// (Rust >= 1.81), which is how a plain `unwrap()` on the sync path takes the +/// whole app down — see ActivityWatch/aw-android#220 and #267. Catching it here +/// turns that abort into the ordinary `{"success": false, "error": ...}` object +/// the Kotlin caller already handles. +/// +/// The panic hook installed by `init_android_logging` still runs first, so the +/// message and backtrace reach logcat before the unwind is stopped. +fn jni_guard(env: &mut JNIEnv, name: &str, f: F) -> jstring +where + F: FnOnce(&mut JNIEnv) -> jstring, +{ + match catch_panic(name, || { + init_android_logging(); + f(&mut *env) + }) { + Ok(result) => result, + Err(msg) => { + error!("{}", msg); + android_log_fatal(&msg); + // A panic in the middle of a JNI call can leave a pending Java + // exception, which would make the following NewStringUTF fail. + let _ = env.exception_clear(); + rust_string_to_jstring(env, sync_error_json(&msg)) + } + } +} + +/// Run a `void` JNI entry point with panics caught. There is no return value to +/// carry an error, so the panic is logged and swallowed. +fn jni_guard_void(name: &str, f: F) +where + F: FnOnce(), +{ + if let Err(msg) = catch_panic(name, || { + init_android_logging(); + f(); + }) { + error!("{}", msg); + android_log_fatal(&msg); + } +} + +/// Point this library's `ANDROID_DATA_DIR` at the app filesDir. +/// +/// `libaw_sync.so` and `libaw_server.so` are separate cdylibs, so +/// `RustInterface.setDataDir` does not update the copy compiled into this +/// `.so`. SyncInterface.kt already sets `XDG_DATA_HOME=$filesDir/data`, which +/// is the path that works for debug (`applicationIdSuffix ".debug"`) and +/// work-profile installs — the hardcoded default is only the release user-0 +/// path. +fn apply_android_data_dir_from_env() { + let Ok(xdg_data) = std::env::var("XDG_DATA_HOME") else { + return; + }; + let Some(files_dir) = crate::dirs::files_dir_from_xdg_data_home(Path::new(&xdg_data)) else { + warn!( + "XDG_DATA_HOME={} is not $filesDir/data; leaving android data dir unchanged", + xdg_data + ); + return; + }; + let path = files_dir.to_string_lossy(); + info!("android data dir from XDG_DATA_HOME: {}", path); + aw_server::dirs::set_android_data_dir(&path); +} + +/// Mirror of `RustInterface.setDataDir`. Prefer this explicit path; the XDG +/// fallback in `get_client` covers current Kotlin that does not call it. +#[no_mangle] +pub extern "C" fn Java_net_activitywatch_android_SyncInterface_setDataDir( + mut env: JNIEnv, + _class: JClass, + java_dir: JString, +) { + jni_guard_void("setDataDir", || match env.get_string(&java_dir) { + Ok(s) => { + let path: String = s.into(); + info!("Setting android data dir as {}", path); + aw_server::dirs::set_android_data_dir(&path); + } + Err(e) => { + error!("setDataDir: failed to read path: {}", e); + } + }); +} + +/// Helper function to get AwClient from port. +/// +/// Android enables API-key auth whenever `config.toml` has `[auth].api_key`. +/// The desktop CLI path (`main.rs`) already forwards that key; this JNI path +/// used `AwClient::new()` and 401'd on `GET /api/0/buckets` (aw-android#247). +fn get_client(port: i32) -> Result { + apply_android_data_dir_from_env(); + let host = "127.0.0.1"; + let api_key = match crate::util::get_server_config(false, None) { + Ok(cfg) => { + if cfg.api_key.is_some() { + info!("using API key from config.toml for local client"); + } + cfg.api_key + } + Err(e) => { + warn!("failed to read server config for API key: {}", e); + None + } + }; + AwClient::new_with_api_key(host, port as u16, "aw-sync-android", api_key) + .map_err(|e| format!("Failed to create client: {}", e)) +} + +/// Render a sync result as the JSON object the Kotlin side parses. +fn sync_result_to_jstring(env: &JNIEnv, name: &str, result: Result) -> jstring { + match result { + Ok(msg) => rust_string_to_jstring(env, msg), + Err(e) => { + error!("{} error: {}", name, e); + rust_string_to_jstring(env, sync_error_json(&e)) + } + } +} + +/// Pull sync data from all hosts in the sync directory +#[no_mangle] +pub extern "C" fn Java_net_activitywatch_android_SyncInterface_syncPullAll( + mut env: JNIEnv, + _class: JClass, + port: i32, + // syncPullAll iterates every host found in the sync directory, so the + // hostname the Kotlin side passes is unused. Kept for signature parity with + // the other SyncInterface natives. + _hostname: JString, +) -> jstring { + jni_guard(&mut env, "syncPullAll", |env| { + let result: Result = (|| { + let client = get_client(port)?; + pull_all(&client).map_err(|e| format!("Sync pull failed: {}", e))?; + Ok(json!({ + "success": true, + "message": "Successfully pulled from all hosts" + }) + .to_string()) + })(); + sync_result_to_jstring(env, "syncPullAll", result) + }) +} + +/// Pull sync data from a specific host +#[no_mangle] +pub extern "C" fn Java_net_activitywatch_android_SyncInterface_syncPull( + mut env: JNIEnv, + _class: JClass, + port: i32, + hostname: JString, +) -> jstring { + jni_guard(&mut env, "syncPull", |env| { + let result: Result = (|| { + let client = get_client(port)?; + let hostname_str: String = env + .get_string(&hostname) + .map_err(|e| format!("Failed to get hostname string: {}", e))? + .into(); + + pull(&hostname_str, &client).map_err(|e| format!("Sync pull failed: {}", e))?; + + Ok(json!({ + "success": true, + "message": format!("Successfully pulled from host: {}", hostname_str) + }) + .to_string()) + })(); + sync_result_to_jstring(env, "syncPull", result) + }) +} + +/// Push local sync data to the sync directory +#[no_mangle] +pub extern "C" fn Java_net_activitywatch_android_SyncInterface_syncPush( + mut env: JNIEnv, + _class: JClass, + port: i32, + hostname: JString, +) -> jstring { + jni_guard(&mut env, "syncPush", |env| { + let result: Result = (|| { + let hostname_str: String = env + .get_string(&hostname) + .map_err(|e| format!("Failed to get hostname: {}", e))? + .into(); + let client = get_client(port)?; + push_with_hostname(&client, &hostname_str) + .map_err(|e| format!("Sync push failed: {}", e))?; + Ok(json!({ + "success": true, + "message": "Successfully pushed local data" + }) + .to_string()) + })(); + sync_result_to_jstring(env, "syncPush", result) + }) +} + +/// Perform full sync (pull from all hosts, then push local data) +#[no_mangle] +pub extern "C" fn Java_net_activitywatch_android_SyncInterface_syncBoth( + mut env: JNIEnv, + _class: JClass, + port: i32, + hostname: JString, +) -> jstring { + jni_guard(&mut env, "syncBoth", |env| { + let result: Result = (|| { + let hostname_str: String = env + .get_string(&hostname) + .map_err(|e| format!("Failed to get hostname: {}", e))? + .into(); + let client = get_client(port)?; + + pull_all(&client).map_err(|e| format!("Pull phase failed: {}", e))?; + + push_with_hostname(&client, &hostname_str) + .map_err(|e| format!("Push phase failed: {}", e))?; + + Ok(json!({ + "success": true, + "message": "Successfully completed full sync" + }) + .to_string()) + })(); + sync_result_to_jstring(env, "syncBoth", result) + }) +} + +/// Get the sync directory path +#[no_mangle] +pub extern "C" fn Java_net_activitywatch_android_SyncInterface_getSyncDir( + mut env: JNIEnv, + _class: JClass, +) -> jstring { + jni_guard( + &mut env, + "getSyncDir", + |env| match crate::dirs::get_sync_dir() { + Ok(path) => { + let path_str = path.to_string_lossy().to_string(); + let response = json!({ + "success": true, + "path": path_str + }) + .to_string(); + rust_string_to_jstring(env, response) + } + Err(e) => { + let msg = format!("Failed to get sync dir: {}", e); + error!("getSyncDir error: {}", msg); + rust_string_to_jstring(env, sync_error_json(&msg)) + } + }, + ) +} diff --git a/aw-sync/src/dirs.rs b/aw-sync/src/dirs.rs new file mode 100644 index 0000000..d8291c1 --- /dev/null +++ b/aw-sync/src/dirs.rs @@ -0,0 +1,215 @@ +use dirs::home_dir; +use std::error::Error; +use std::fs; +#[cfg(any(target_os = "android", test))] +use std::path::Path; +use std::path::PathBuf; + +/// Resolve the instance profile. +/// `--profile` wins, then `AW_PROFILE`, then `--testing` → `"testing"`, else `"default"`. +#[allow(dead_code)] // used by the aw-sync binary; the lib copy is unused +pub fn resolve_profile( + cli_profile: Option<&str>, + testing: bool, + env_profile: Option<&str>, +) -> String { + if let Some(p) = cli_profile { + if !p.is_empty() { + return p.to_string(); + } + } + if let Some(p) = env_profile { + if !p.is_empty() { + return p.to_string(); + } + } + if testing { + "testing".to_string() + } else { + "default".to_string() + } +} + +/// aw-sync's own config dir: `{appname}/aw-sync`. +/// +/// Uses the same profile appname as aw-server so a named profile (e.g. +/// `research`) does not share prod's sync config. `testing` follows the +/// same new-root-plus-legacy-fallback rule as aw-server. +// TODO: add proper config support +#[cfg(not(target_os = "android"))] +#[allow(dead_code)] +pub fn get_config_dir() -> Result> { + let dir = sync_config_dir(&aw_server::dirs::appname())?; + fs::create_dir_all(&dir)?; + Ok(dir) +} + +/// Path construction only — does not create directories (so tests stay off-disk). +#[cfg(not(target_os = "android"))] +fn sync_config_dir(appname: &str) -> Result> { + Ok(dirs::config_dir() + .ok_or("Unable to read user config dir")? + .join(appname) + .join("aw-sync")) +} + +/// Path to the embedded/local aw-server config. On Android this is +/// `filesDir/config.toml` (same file ConfigManager and the server use). +#[allow(dead_code)] +pub fn get_server_config_path(testing: bool) -> Result { + let profile = aw_server::config::get_profile(); + // If set_profile hasn't run yet, honour the legacy testing bool so + // `--testing` still finds the testing config (legacy or new root). + let effective = if profile == "default" && testing { + "testing" + } else { + profile + }; + let filename = aw_server::config::config_filename(effective); + // Desktop: honour profile isolation (`activitywatch-/aw-server-rust`). + // Android: stay on filesDir — sibling XDG roots do not exist there, and + // master already reads this path for the embedded server's api_key (#666). + #[cfg(not(target_os = "android"))] + { + let dir = dirs::config_dir() + .ok_or(())? + .join(aw_server::dirs::appname_for(effective)) + .join("aw-server-rust"); + Ok(dir.join(filename)) + } + #[cfg(target_os = "android")] + { + let dir = aw_server::dirs::get_config_dir()?; + Ok(dir.join(filename)) + } +} + +pub fn get_sync_dir() -> Result> { + // if AW_SYNC_DIR is set, use that + if let Ok(dir) = std::env::var("AW_SYNC_DIR") { + return Ok(PathBuf::from(dir)); + } + let home_dir = home_dir().ok_or("Unable to read home_dir")?; + Ok(home_dir.join("ActivityWatchSync")) +} + +/// SyncInterface.kt sets `XDG_DATA_HOME=$filesDir/data` before `loadLibrary`. +/// `libaw_sync.so` is a separate cdylib from `libaw_server.so`, so +/// `RustInterface.setDataDir` does not update this library's `ANDROID_DATA_DIR`. +/// Parent of that env var is the app filesDir (release, `.debug`, work profile). +#[cfg(any(target_os = "android", test))] +pub(crate) fn files_dir_from_xdg_data_home(xdg_data_home: &Path) -> Option { + if xdg_data_home.file_name() != Some(std::ffi::OsStr::new("data")) { + return None; + } + let parent = xdg_data_home.parent()?; + if parent.as_os_str().is_empty() || parent == Path::new("/") { + return None; + } + Some(parent.to_path_buf()) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::path::Path; + + #[test] + fn resolve_profile_cli_wins_over_env_and_testing() { + assert_eq!( + resolve_profile(Some("research"), true, Some("testing")), + "research" + ); + } + + #[test] + fn resolve_profile_env_wins_over_testing() { + assert_eq!(resolve_profile(None, true, Some("research")), "research"); + } + + #[test] + fn resolve_profile_testing_alias_and_default() { + assert_eq!(resolve_profile(None, true, None), "testing"); + assert_eq!(resolve_profile(None, false, None), "default"); + assert_eq!(resolve_profile(Some(""), false, Some("")), "default"); + } + + #[cfg(not(target_os = "android"))] + #[test] + fn sync_config_dir_is_isolated_per_named_profile() { + use aw_server::dirs::appname_for_in; + use std::fs; + let root = std::env::temp_dir().join(format!( + "aw-sync-profile-tests-{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + let data = root.join("data"); + let config = root.join("config"); + let cache = root.join("cache"); + fs::create_dir_all(&data).unwrap(); + fs::create_dir_all(&config).unwrap(); + fs::create_dir_all(&cache).unwrap(); + + let default_app = appname_for_in("default", &data, &config, &cache); + let testing_app = appname_for_in("testing", &data, &config, &cache); + let research_app = appname_for_in("research", &data, &config, &cache); + + assert_eq!(default_app, "activitywatch"); + // Fresh setup: testing is a sibling root, not the shared one. + assert_eq!(testing_app, "activitywatch-testing"); + assert_eq!(research_app, "activitywatch-research"); + assert_ne!(testing_app, default_app); + assert_ne!(research_app, default_app); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn server_config_filename_default_and_research_are_bare() { + // Isolated roots use config.toml. Suffixed config-testing.toml is + // only for the legacy shared-root layout (dirs.rs tests). + assert_eq!(aw_server::config::config_filename("default"), "config.toml"); + assert_eq!( + aw_server::config::config_filename("research"), + "config.toml" + ); + } + + #[cfg(not(target_os = "android"))] + #[test] + fn server_config_path_default_is_config_toml() { + let production = get_server_config_path(false).unwrap(); + assert!( + production.ends_with("config.toml"), + "default should read config.toml, got {production:?}" + ); + } + + #[test] + fn xdg_data_home_parent_is_files_dir() { + let debug = Path::new("/data/user/0/net.activitywatch.android.debug/files/data"); + assert_eq!( + files_dir_from_xdg_data_home(debug).as_deref(), + Some(Path::new( + "/data/user/0/net.activitywatch.android.debug/files" + )) + ); + + let work_profile = Path::new("/data/user/10/net.activitywatch.android/files/data"); + assert_eq!( + files_dir_from_xdg_data_home(work_profile).as_deref(), + Some(Path::new("/data/user/10/net.activitywatch.android/files")) + ); + } + + #[test] + fn rejects_non_data_leaf_and_root() { + assert!(files_dir_from_xdg_data_home(Path::new("/data")).is_none()); + assert!(files_dir_from_xdg_data_home(Path::new("/")).is_none()); + assert!(files_dir_from_xdg_data_home(Path::new("data")).is_none()); + assert!(files_dir_from_xdg_data_home(Path::new("/tmp/config")).is_none()); + } +} diff --git a/aw-sync/src/lib.rs b/aw-sync/src/lib.rs new file mode 100644 index 0000000..31bae33 --- /dev/null +++ b/aw-sync/src/lib.rs @@ -0,0 +1,24 @@ +#[macro_use] +extern crate log; +extern crate chrono; +extern crate serde; +extern crate serde_json; + +mod sync; +pub use sync::create_datastore; +pub use sync::sync_datastores; +pub use sync::sync_run; +pub use sync::SyncSpec; + +mod sync_wrapper; +pub use sync_wrapper::{pull, pull_all}; +pub use sync_wrapper::{push, push_with_hostname}; + +mod accessmethod; +pub use accessmethod::AccessMethod; + +mod dirs; +mod util; + +#[cfg(target_os = "android")] +pub mod android; diff --git a/aw-sync/src/main.rs b/aw-sync/src/main.rs new file mode 100644 index 0000000..604b723 --- /dev/null +++ b/aw-sync/src/main.rs @@ -0,0 +1,351 @@ +#![cfg(feature = "cli")] + +// What needs to be done: +// - [x] Setup local sync bucket +// - [x] Import local buckets and sync events from aw-server (either through API or through creating a read-only Datastore) +// - [x] Import buckets and sync events from remotes +// - [x] Add CLI arguments +// - [x] For which local server to use +// - [x] For which sync dir to use +// - [x] Date to start syncing from + +#[macro_use] +extern crate log; +extern crate chrono; +extern crate serde; +extern crate serde_json; + +use chrono::{DateTime, Utc}; +use clap::{Parser, Subcommand}; +use std::error::Error; +use std::path::PathBuf; +use std::sync::mpsc::{channel, RecvTimeoutError}; +use std::time::Duration; + +use aw_client_rust::blocking::AwClient; + +mod accessmethod; +mod dirs; +mod sync; +mod sync_wrapper; +mod util; + +#[derive(Parser)] +#[clap(version = "0.1", author = "Erik Bjäreholt")] +struct Opts { + #[clap(subcommand)] + command: Option, + + /// Host of instance to connect to. + #[clap(long, default_value = "127.0.0.1")] + host: String, + + /// Port of instance to connect to. + #[clap(long)] + port: Option, + + /// Path to the local aw-server config file. + #[clap(short = 'c', long = "config")] + config: Option, + + /// Named instance profile (e.g. "default", "testing", "research"). + /// `--testing` is an alias for `--profile testing`. Also reads AW_PROFILE, + /// which aw-qt exports for the modules it spawns. + #[clap(long)] + profile: Option, + + /// Convenience option for using the default testing host and port. + /// Alias for `--profile testing`. + #[clap(long)] + testing: bool, + + /// Full path to sync directory. + /// If not specified, use AW_SYNC_DIR env var, or default to ~/ActivityWatchSync + #[clap(long)] + sync_dir: Option, + + /// Enable debug logging. + #[clap(long)] + verbose: bool, +} + +#[derive(Subcommand)] +enum Commands { + /// Daemon subcommand + /// Starts aw-sync as a daemon, which will sync every 5 minutes. + Daemon { + /// Date to start syncing from. + /// If not specified, start from beginning. + /// Format: YYYY-MM-DD + #[clap(long, value_parser=parse_start_date)] + start_date: Option>, + + /// Specify buckets to sync using a comma-separated list. + /// By default, all buckets are synced. + #[clap(long, value_parser=parse_list)] + buckets: Option>, + + /// Mode to sync in. Can be "push", "pull", or "both". + /// Defaults to "both". + #[clap(long, default_value = "both")] + mode: sync::SyncMode, + + /// Full path to sync db file + /// Useful for syncing buckets from a specific db file in the sync directory. + /// Must be a valid absolute path to a file in the sync directory. + #[clap(long)] + sync_db: Option, + }, + + /// Sync subcommand + /// + /// Syncs data between local aw-server and sync directory. + /// First pulls remote buckets from the sync directory to the local aw-server. + /// Then pushes local buckets from the aw-server to the local sync directory. + Sync { + /// Host(s) to pull from, comma separated. Will pull from all hosts if not specified. + #[clap(long, value_parser=parse_list)] + host: Option>, + + /// Date to start syncing from. + /// If not specified, start from beginning. + /// Format: YYYY-MM-DD + #[clap(long, value_parser=parse_start_date)] + start_date: Option>, + + /// Specify buckets to sync using a comma-separated list. + /// By default, all buckets are synced. + #[clap(long, value_parser=parse_list)] + buckets: Option>, + + /// Mode to sync in. Can be "push", "pull", or "both". + /// Defaults to "both". Implies advanced sync mode (see below), + /// since the simple host-based sync mode always does both. + #[clap(long)] + mode: Option, + + /// Full path to sync db file + /// Useful for syncing buckets from a specific db file in the sync directory. + /// Must be a valid absolute path to a file in the sync directory. + #[clap(long)] + sync_db: Option, + }, + /// List buckets and their sync status. + List {}, +} + +fn parse_start_date(arg: &str) -> Result, chrono::ParseError> { + chrono::NaiveDate::parse_from_str(arg, "%Y-%m-%d") + .map(|nd| nd.and_time(chrono::NaiveTime::MIN).and_utc()) +} + +fn parse_list(arg: &str) -> Result, clap::Error> { + // If the argument is empty or just whitespace, return an empty Vec + // This handles the case when --buckets is used without a value + if arg.trim().is_empty() { + return Ok(vec![]); + } + + // Otherwise, split by comma, trim each part, and filter out empty strings + Ok(arg + .split(',') + .map(|s| s.trim().to_string()) + .filter(|s| !s.is_empty()) + .collect()) +} + +fn main() -> Result<(), Box> { + let opts: Opts = Opts::parse(); + let verbose = opts.verbose; + + info!("Started aw-sync..."); + + let env_profile = std::env::var("AW_PROFILE").ok(); + let profile = dirs::resolve_profile( + opts.profile.as_deref(), + opts.testing, + env_profile.as_deref(), + ); + aw_server::dirs::validate_profile(&profile) + .unwrap_or_else(|e| panic!("Invalid profile name: {e}")); + aw_server::config::set_profile(profile.clone()); + + aw_server::logging::setup_logger("aw-sync", &profile, verbose)?; + + if opts.testing && opts.profile.as_deref().is_some_and(|p| p != "testing") { + warn!("--testing and --profile {profile} both given; --profile wins, using '{profile}'"); + } + if profile != "default" { + info!("Running aw-sync with profile '{profile}'"); + } + + // if sync_dir, set env var + if let Some(sync_dir) = opts.sync_dir { + if !sync_dir.is_absolute() { + Err("Sync dir must be absolute")? + } + + info!("Using sync dir: {}", sync_dir.display()); + std::env::set_var("AW_SYNC_DIR", sync_dir); + } + + // Named profiles (and `--profile testing` without `--testing`) must use the + // profile's own server config / port default, not the CLI testing bool. + let testing = profile == "testing"; + let server_config = if util::is_loopback_host(&opts.host) { + util::get_server_config(testing, opts.config.as_deref())? + } else { + util::ServerConfig::default_for(testing) + }; + let port = opts.port.unwrap_or(server_config.port); + + let api_key = server_config.api_key; + if api_key.is_some() { + info!("Using API key from server config for authentication"); + } + let url_host = util::host_for_url(&opts.host); + let client = AwClient::new_with_api_key(&url_host, port, "aw-sync", api_key)?; + + // if opts.command is None, then we're using the default subcommand (Daemon) + match opts.command.unwrap_or(Commands::Daemon { + start_date: None, + buckets: None, + mode: sync::SyncMode::Both, + sync_db: None, + }) { + // Start daemon + Commands::Daemon { + start_date, + buckets, + mode, + sync_db, + } => { + info!("Starting daemon..."); + + let effective_buckets = buckets; + + daemon(&client, start_date, effective_buckets, sync_db, mode)?; + } + // Perform sync + Commands::Sync { + host, + start_date, + buckets, + mode, + sync_db, + } => { + let effective_buckets = buckets; + + // If advanced options are provided, use advanced sync mode + if start_date.is_some() + || effective_buckets.is_some() + || sync_db.is_some() + || mode.is_some() + { + let sync_dir = dirs::get_sync_dir()?; + if let Some(db_path) = &sync_db { + info!("Using sync db: {}", db_path.display()); + + if !db_path.is_absolute() { + Err("Sync db path must be absolute")? + } + if !db_path.starts_with(&sync_dir) { + Err("Sync db path must be in sync directory")? + } + } + + let sync_spec = sync::SyncSpec { + path: sync_dir, + path_db: sync_db, + buckets: effective_buckets, + start: start_date, + }; + + sync::sync_run(&client, &sync_spec, mode.unwrap_or(sync::SyncMode::Both))? + } else { + // Simple host-based sync mode (backwards compatibility) + // Pull + match host { + Some(hosts) => { + for host in hosts.iter() { + info!("Pulling from host: {}", host); + sync_wrapper::pull(host, &client)?; + } + } + None => { + info!("Pulling from all hosts"); + sync_wrapper::pull_all(&client)?; + } + } + + // Push + info!("Pushing local data"); + sync_wrapper::push(&client)? + } + } + + // List all buckets + Commands::List {} => sync::list_buckets(&client)?, + } + + // Needed to give the datastores some time to commit before program is shut down. + // 100ms isn't actually needed, seemed to work fine with as little as 10ms, but I'd rather give + // it some wiggleroom. + std::thread::sleep(std::time::Duration::from_millis(100)); + + Ok(()) +} + +fn daemon( + client: &AwClient, + start_date: Option>, + buckets: Option>, + sync_db: Option, + mode: sync::SyncMode, +) -> Result<(), Box> { + let (tx, rx) = channel(); + + ctrlc::set_handler(move || { + let _ = tx.send(()); + })?; + + let sync_dir = dirs::get_sync_dir()?; + if let Some(db_path) = &sync_db { + info!("Using sync db: {}", db_path.display()); + + if !db_path.is_absolute() { + Err("Sync db path must be absolute")? + } + if !db_path.starts_with(&sync_dir) { + Err("Sync db path must be in sync directory")? + } + } + + let sync_spec = sync::SyncSpec { + path: sync_dir, + buckets, + path_db: sync_db, + start: start_date, + }; + + loop { + if let Err(e) = sync::sync_run(client, &sync_spec, mode) { + error!("Error during sync cycle: {}", e); + return Err(e); + } + + info!("Sync pass done, sleeping for 5 minutes"); + + match rx.recv_timeout(Duration::from_secs(300)) { + Ok(_) | Err(RecvTimeoutError::Disconnected) => { + info!("Termination signal received, shutting down."); + break; + } + Err(RecvTimeoutError::Timeout) => { + // Continue the loop if the timeout occurs + } + } + } + + Ok(()) +} diff --git a/aw-sync/src/sync.rs b/aw-sync/src/sync.rs new file mode 100644 index 0000000..35a9cef --- /dev/null +++ b/aw-sync/src/sync.rs @@ -0,0 +1,568 @@ +/// Basic syncing for ActivityWatch +/// Based on: https://github.com/ActivityWatch/aw-server/pull/50 +/// +/// This does not handle any direct peer interaction/connections/networking, it works as a "bring your own folder synchronizer". +/// +/// It manages a sync-folder by syncing the aw-server datastore with a copy/staging datastore in the folder (one for each host). +/// The sync folder is then synced with remotes using Syncthing/Dropbox/whatever. +extern crate chrono; +extern crate reqwest; +extern crate serde_json; + +use std::error::Error; +use std::fs; +use std::path::{Path, PathBuf}; + +use aw_client_rust::blocking::AwClient; +use chrono::{DateTime, Duration, Utc}; + +use aw_datastore::{Datastore, DatastoreError}; +use aw_models::{Bucket, Event}; + +#[cfg(feature = "cli")] +use clap::ValueEnum; + +use crate::accessmethod::AccessMethod; + +#[derive(PartialEq, Eq, Copy, Clone)] +#[cfg_attr(feature = "cli", derive(ValueEnum))] +pub enum SyncMode { + Push, + Pull, + Both, +} + +#[derive(Debug)] +pub struct SyncSpec { + /// Path of sync folder + pub path: PathBuf, + /// Path of sync db + /// If None, will use all + pub path_db: Option, + /// Bucket IDs to sync + pub buckets: Option>, + /// Start of time range to sync + pub start: Option>, +} + +impl Default for SyncSpec { + fn default() -> Self { + // TODO: Better default path + let path = Path::new("/tmp/aw-sync").to_path_buf(); + SyncSpec { + path, + path_db: None, + buckets: None, + start: None, + } + } +} + +/// Performs a single sync pass +pub fn sync_run( + client: &AwClient, + sync_spec: &SyncSpec, + mode: SyncMode, +) -> Result<(), Box> { + let info = client.get_info()?; + + // FIXME: Here it is assumed that the device_id for the local server is the one used by + // aw-server-rust, which is not necessarily true (aw-server-python has seperate device_id). + // Therefore, this may sometimes fail to pick up the correct local datastore. + let device_id = info.device_id.as_str(); + + // FIXME: Bad device_id assumption? + let ds_localremote = setup_local_remote(sync_spec.path.as_path(), device_id)?; + let remote_dbfiles = crate::util::find_remotes_nonlocal( + sync_spec.path.as_path(), + device_id, + sync_spec.path_db.as_ref(), + )?; + + // Log if remotes found + // TODO: Only log remotes of interest + if !remote_dbfiles.is_empty() { + info!( + "Found {} remote db files: {:?}", + remote_dbfiles.len(), + remote_dbfiles + ); + } + + // TODO: Check for compatible remote db version before opening + let ds_remotes: Vec = remote_dbfiles + .iter() + .map(|p| p.as_path()) + .map(create_datastore) + .collect::, _>>()?; + + if !ds_remotes.is_empty() { + info!( + "Found {} remote datastores: {:?}", + ds_remotes.len(), + ds_remotes + ); + } + + // Pull + if mode == SyncMode::Pull || mode == SyncMode::Both { + info!("Pulling..."); + for ds_from in &ds_remotes { + sync_datastores(ds_from, client, false, None, sync_spec)?; + } + } + + // Push local server buckets to sync folder + if mode == SyncMode::Push || mode == SyncMode::Both { + info!("Pushing..."); + sync_datastores(client, &ds_localremote, true, Some(device_id), sync_spec)?; + } + + // Close open database connections + for ds_from in &ds_remotes { + ds_from.close(); + } + ds_localremote.close(); + + // Dropping also works to close the database connections, weirdly enough. + // Probably because once the database is dropped, the thread will stop, + // and then the Connection will be dropped, which closes the connection. + std::mem::drop(ds_remotes); + std::mem::drop(ds_localremote); + + // NOTE: Will fail if db connections not closed (as it will open them again) + //list_buckets(&client, sync_spec.path.as_path()); + + Ok(()) +} + +#[allow(dead_code)] +pub fn list_buckets(client: &AwClient) -> Result<(), Box> { + let sync_directory = crate::dirs::get_sync_dir().map_err(|_| "Could not get sync dir")?; + let sync_directory = sync_directory.as_path(); + let info = client.get_info()?; + + // FIXME: Incorrect device_id assumption? + let device_id = info.device_id.as_str(); + let ds_localremote = setup_local_remote(sync_directory, device_id)?; + + let remote_dbfiles = crate::util::find_remotes_nonlocal(sync_directory, device_id, None)?; + info!("Found remotes: {:?}", remote_dbfiles); + + // TODO: Check for compatible remote db version before opening + let ds_remotes: Vec = remote_dbfiles + .iter() + .map(|p| p.as_path()) + .map(create_datastore) + .collect::, _>>()?; + + log_buckets(client)?; + log_buckets(&ds_localremote)?; + for ds_from in &ds_remotes { + log_buckets(ds_from)?; + } + + Ok(()) +} + +fn setup_local_remote(path: &Path, device_id: &str) -> Result> { + // FIXME: Don't run twice if already exists + fs::create_dir_all(path)?; + + let remotedir = path.join(device_id); + fs::create_dir_all(&remotedir)?; + + let dbfile = remotedir.join("test.db"); + + // Print a message if dbfile doesn't already exist + if !dbfile.exists() { + info!("Creating new database file: {}", dbfile.display()); + } + + let ds_localremote = create_datastore(&dbfile)?; + Ok(ds_localremote) +} + +/// Open (or create) the sqlite datastore at `path`. +/// +/// `Datastore::new` takes a `String`, so a non-UTF-8 path cannot be passed +/// through faithfully. Report that as an error rather than unwrapping (a panic +/// here aborts the app on Android, aw-android#220) and rather than lossily +/// converting it, which would silently open a *different* file than the caller +/// asked for. +pub fn create_datastore(path: &Path) -> Result { + let pathstr = path + .to_str() + .ok_or_else(|| format!("Sync database path is not valid UTF-8: {}", path.display()))?; + Ok(Datastore::new(pathstr.to_string(), false)) +} + +/// Returns the sync-destination bucket for a given bucket, creates it if it doesn't exist. +/// +/// Returns an error rather than panicking on a datastore failure or on bucket +/// metadata of an unexpected shape: this runs inside a JNI call on Android, +/// where an unwind out of the `extern "C"` frame aborts the app +/// (ActivityWatch/aw-android#220). +fn get_or_create_sync_bucket( + bucket_from: &Bucket, + ds_to: &dyn AccessMethod, + is_push: bool, +) -> Result { + // On pull/import: derive the origin from $aw.sync.origin metadata (preferred) or the + // hostname (legacy fallback for buckets that predate the metadata field). On push-staging + // the bucket keeps its original ID and we do NOT stamp $aw.sync.origin — staging copies + // should not look like synced-from-remote buckets. + let (new_id, sync_origin) = if is_push { + (bucket_from.id.clone(), None) + } else { + // `split` always yields at least one item, so this cannot be None. + let orig_bucketid = bucket_from + .id + .split("-synced-from-") + .next() + .unwrap_or(bucket_from.id.as_str()); + let origin = match bucket_from.data.get("$aw.sync.origin") { + Some(value) => value.as_str().map(str::to_string).ok_or_else(|| { + format!( + "Bucket '{}' has a non-string $aw.sync.origin: {}", + bucket_from.id, value + ) + })?, + None => bucket_from.hostname.clone(), + }; + ( + format!("{orig_bucketid}-synced-from-{origin}"), + Some(origin), + ) + }; + + match ds_to.get_bucket(new_id.as_str()) { + Ok(bucket) => Ok(bucket), + Err(DatastoreError::NoSuchBucket(_)) => { + let mut bucket_new = bucket_from.clone(); + bucket_new.id = new_id.clone(); + // Only stamp $aw.sync.origin on pull/import. The derived origin already handles + // the legacy case: hostname is used when the source bucket has no metadata field. + if let Some(origin) = sync_origin { + bucket_new + .data + .insert("$aw.sync.origin".to_string(), serde_json::json!(origin)); + } else { + // Push path: strip any stale $aw.sync.origin that bucket_from may carry + // (e.g. if it was previously imported by a pull). Staging copies must + // never look like synced-from-remote buckets. + bucket_new.data.remove("$aw.sync.origin"); + } + ds_to + .create_bucket(&bucket_new) + .map_err(|e| format!("Failed to create bucket '{new_id}': {e:?}"))?; + ds_to + .get_bucket(new_id.as_str()) + .map_err(|e| format!("Failed to read back bucket '{new_id}': {e:?}")) + } + Err(e) => Err(format!("Failed to get bucket '{new_id}': {e:?}")), + } +} + +/// Number of events fetched per page in the chunked-fetch loop in `sync_one`. +/// Reduced in tests so multi-page paths can be exercised with a small event count. +#[cfg(not(test))] +const BATCH_SIZE: usize = 5000; +#[cfg(test)] +const BATCH_SIZE: usize = 5; + +/// Whether a bucket holds data synced from another host, rather than data +/// collected on this host. +/// +/// The `-synced-from-` ID suffix is the marker, matching how +/// `get_or_create_sync_bucket` builds and parses these IDs. Note that +/// `$aw.sync.origin` cannot be used here: it is written on push-staging as well +/// as on import (see the FIXME on `sync_datastores`), so it is set on a host's +/// own exported buckets too and would make every bucket look second-hand. +/// +/// `-synced-from-` is a **reserved token** in aw-sync's ID grammar, not merely a +/// convention: `get_or_create_sync_bucket` splits on it to recover the original +/// ID, so a first-hand bucket whose own ID contained it would already have that +/// ID truncated on import, independently of this check. Treating it as a marker +/// therefore adds no new failure mode. Issue #649 tracks moving provenance to +/// bucket metadata, which removes the dependency on the ID string entirely. +fn is_synced_bucket(bucket: &Bucket) -> bool { + bucket.id.contains("-synced-from-") +} + +/// Syncs all buckets from `ds_from` to `ds_to` with `-synced` appended to the ID of the destination bucket. +/// +/// Buckets that were themselves synced from another host are skipped in both +/// directions, so data is only ever exchanged first-hand. +/// +/// is_push: a bool indicating if we're pushing local buckets to the sync dir +/// (as opposed to pulling from remotes) +/// src_did: source device ID +/// +/// Returns an error instead of panicking: this is the step the abort in +/// ActivityWatch/aw-android#220 happens in (the sync directory and its +/// datastore are already created by the time the process dies), and on Android +/// it runs inside a JNI `extern "C"` frame where an unwind is an abort rather +/// than a caught exception. +pub fn sync_datastores( + ds_from: &dyn AccessMethod, + ds_to: &dyn AccessMethod, + is_push: bool, + src_did: Option<&str>, + sync_spec: &SyncSpec, +) -> Result<(), String> { + // FIXME: "-synced" should only be appended when synced to the local database, not to the + // staging area for local buckets. + info!("Syncing {:?} to {:?}", ds_from, ds_to); + + let mut buckets_from: Vec = ds_from + .get_buckets() + .map_err(|e| format!("Failed to list buckets in {ds_from:?}: {e}"))? + .iter_mut() + // Never sync a bucket that is itself a copy synced from another host. + // A host must only ever offer data it collected itself. Without this, + // HOSTA's buckets reach HOSTB, are re-exported by HOSTB's next push, and + // come back to HOSTA as `_HOSTA-synced-from-HOSTA` — a duplicate + // of the local bucket, so /timeline renders every event twice. + // See https://github.com/orgs/ActivityWatch/discussions/1373 + .filter(|tup| { + if is_synced_bucket(tup.1) { + debug!(" - Skipping already-synced bucket '{}'", tup.1.id); + false + } else { + true + } + }) + // Only filter buckets if specific bucket IDs are provided + .filter(|tup| { + let bucket = &tup.1; + if let Some(buckets) = &sync_spec.buckets { + // If "*" is in the buckets list or no buckets specified, sync all buckets + if buckets.iter().any(|b_id| b_id == "*") || buckets.is_empty() { + true + } else { + buckets.iter().any(|b_id| b_id == &bucket.id) + } + } else { + // By default, sync all buckets + true + } + }) + .map(|tup| { + // TODO: Refuse to sync buckets without hostname/device ID set, or if set to 'unknown' + if tup.1.hostname == "unknown" { + // Only the push path carries a source device ID to substitute. + // On pull there is none, and continuing would give the bucket a + // `-synced-from-unknown` destination ID shared by every remote + // with that bucket ID, mixing events from unrelated devices. + // Refuse the sync instead (the previous code unwrapped the None + // here, which on Android aborts the whole app). + let did = src_did.ok_or_else(|| { + format!( + "Bucket '{}' has an unknown hostname/device ID and there is no source \ + device ID to substitute; refusing to sync it without provenance", + tup.1.id + ) + })?; + warn!(" ! Bucket hostname/device ID was invalid, setting to device ID/hostname"); + tup.1.hostname = did.to_string(); + } + Ok(tup.1.clone()) + }) + .collect::, String>>()?; + + // Log warning for buckets requested but not found + if let Some(buckets) = &sync_spec.buckets { + for b_id in buckets { + if !buckets_from.iter().any(|b| b.id == *b_id) { + error!(" ! Bucket \"{}\" not found in source datastore", b_id); + } + } + } + + // Sync buckets in order of most recently updated + buckets_from.sort_by_key(|b| b.metadata.end); + + for bucket_from in buckets_from { + let bucket_to = get_or_create_sync_bucket(&bucket_from, ds_to, is_push)?; + sync_one(ds_from, ds_to, bucket_from, bucket_to, sync_spec)?; + } + + Ok(()) +} + +/// Syncs a single bucket from one datastore to another +fn sync_one( + ds_from: &dyn AccessMethod, + ds_to: &dyn AccessMethod, + bucket_from: Bucket, + bucket_to: Bucket, + sync_spec: &SyncSpec, +) -> Result<(), String> { + let eventcount_to_old = ds_to.get_event_count(bucket_to.id.as_str())?; + info!(" ⟳ Syncing bucket '{}'", bucket_to.id); + + // Sync events + // FIXME: This should use bucket_to.metadata.end, but it doesn't because it doesn't work + // for empty buckets (Should be None, is Some(unknown_time)) + // let resume_sync_at = bucket_to.metadata.end; + let most_recent_events = ds_to.get_events(bucket_to.id.as_str(), None, None, Some(1))?; + // If the destination bucket already has events, resume from where it left off. + // Otherwise (first sync of this bucket), fall back to sync_spec.start, if specified. + let resume_sync_at = most_recent_events + .first() + .map(|e| e.timestamp + e.duration) + .or(sync_spec.start); + + if let Some(resume_time) = resume_sync_at { + info!(" + Resuming at {:?}", resume_time); + } else { + info!(" + Starting from beginning"); + } + + // Fetch events in bounded chunks to avoid OOM on devices with limited RAM (e.g. Android). + // get_events returns events in descending order (newest first), so we paginate backwards + // using the `end` parameter. Each chunk is written to `ds_to` as soon as it is fetched, + // so peak memory is O(BATCH_SIZE), not O(total events in the bucket). + // + // Each chunk is reversed before writing so events are inserted oldest-first (matching the + // insertion order in the source DB). This preserves consistent ID assignment across source + // and destination, which the sync tests rely on. + // + // Heartbeat semantics at the resume boundary: we use heartbeat() for the globally-oldest + // new event ONLY in the single-page case (pages_written == 0 when is_last_fetch fires). + // In that case dest's "last event" is still the pre-sync resume-boundary row, so + // heartbeat() can correctly merge an adjacent new event into it. + // + // In the multi-page case (pages_written > 0), newer pages have already been inserted and + // dest's "last event" is no longer the resume boundary — heartbeat() would compare against + // the wrong row and skip the merge anyway. Inserting the oldest event directly is correct. + let mut fetch_end: Option> = None; + let mut events_sent = 0usize; + let mut pages_written = 0u32; + + loop { + let raw = ds_from.get_events( + bucket_from.id.as_str(), + resume_sync_at, + fetch_end, + Some(BATCH_SIZE as u64), + )?; + + if raw.is_empty() { + break; + } + + // Fewer events than requested means there's nothing older left to fetch. + let is_last_fetch = raw.len() < BATCH_SIZE; + + let mut chunk: Vec = raw + .into_iter() + .map(|mut e| { + // Unset ID on events, as they are not globally unique + e.id = None; + e + }) + .collect(); + + if !is_last_fetch { + // chunk is in DESC order (newest first); chunk.last() = oldest in this (full) page. + // Naively setting the next `end` to `oldest.timestamp - 1ns` silently drops events + // if the page happens to end mid-run of same-timestamp events: anything else at + // that exact timestamp would fall outside the next page's range. Guard against that + // by dropping ALL trailing events at `boundary_ts` from this page and leaving them + // for the next fetch (whose `end = boundary_ts` is inclusive, so it re-fetches + // the whole tied run at once). + // + // Note: we must drop the boundary event itself, not just its duplicates. Keeping + // one copy in this chunk while also setting `fetch_end = Some(boundary_ts)` (inclusive) + // would cause that event to be fetched again next page, producing a duplicate row. + // `raw` was non-empty and `chunk` is a 1:1 map of it, so both ends exist. + let (Some(oldest), Some(newest)) = (chunk.last(), chunk.first()) else { + return Err(format!( + "Empty event page while syncing bucket '{}'", + bucket_from.id + )); + }; + let boundary_ts = oldest.timestamp; + if newest.timestamp != boundary_ts { + // Safe to pop all boundary_ts events: the `if` guard ensures at least one + // earlier event (with a different timestamp) remains in the chunk. + while chunk.last().is_some_and(|e| e.timestamp == boundary_ts) { + chunk.pop(); + } + fetch_end = Some(boundary_ts); + } else { + // Pathological case: every event in this full page shares the exact same + // timestamp, so we can't tell where the tied run ends without an unbounded + // query. This can't occur with AW's event model in practice (activity records + // span seconds+) — accept the page as-is rather than looping forever. + fetch_end = Some(boundary_ts - Duration::nanoseconds(1)); + } + + // Reverse to ASC order (oldest first) before inserting. + chunk.reverse(); + events_sent += chunk.len(); + pages_written += 1; + for batch in chunk.chunks(BATCH_SIZE) { + print!("({}/…)\r", events_sent); + ds_to.insert_events(bucket_to.id.as_str(), batch.to_vec())?; + } + } else { + // Last (oldest) page: process oldest-first to preserve ID ordering. + chunk.reverse(); // chunk is now ASC (oldest first) + + // Use heartbeat() for the oldest event only in the single-page case: + // dest's "last event" is still the pre-sync resume-boundary row, so heartbeat() + // can correctly merge an adjacent new event into it (delta=0.0 → exact adjacency). + // In multi-page syncs, newer pages are already in dest, so heartbeat() would + // compare against the wrong row — insert directly instead. + if !chunk.is_empty() && pages_written == 0 { + let oldest = chunk.remove(0); + ds_to.heartbeat(bucket_to.id.as_str(), oldest, 0.0)?; + events_sent += 1; + } + + // Insert the remaining events from the last page in ASC order. + if !chunk.is_empty() { + events_sent += chunk.len(); + for batch in chunk.chunks(BATCH_SIZE) { + print!("({}/…)\r", events_sent); + ds_to.insert_events(bucket_to.id.as_str(), batch.to_vec())?; + } + } + + break; + } + } + + let eventcount_to_new = ds_to.get_event_count(bucket_to.id.as_str())?; + let new_events_count = eventcount_to_new - eventcount_to_old; + if new_events_count < 0 { + return Err(format!( + "Event count of bucket '{}' shrank during sync ({eventcount_to_old} -> {eventcount_to_new})", + bucket_to.id + )); + } + if new_events_count > 0 { + info!(" = Synced {} new events", new_events_count); + } else { + info!(" ✓ Already up to date!"); + } + + Ok(()) +} + +fn log_buckets(ds: &dyn AccessMethod) -> Result<(), String> { + // Logs all buckets and some metadata for a given datastore + let buckets = ds.get_buckets()?; + info!("Buckets in {:?}:", ds); + for bucket in buckets.values() { + info!(" - {}", bucket.id.as_str()); + info!( + " eventcount: {:?}", + ds.get_event_count(bucket.id.as_str())? + ); + } + Ok(()) +} diff --git a/aw-sync/src/sync_wrapper.rs b/aw-sync/src/sync_wrapper.rs new file mode 100644 index 0000000..47067a5 --- /dev/null +++ b/aw-sync/src/sync_wrapper.rs @@ -0,0 +1,77 @@ +use std::error::Error; +use std::fs; + +use crate::sync::{sync_run, SyncMode, SyncSpec}; +use aw_client_rust::blocking::AwClient; + +pub fn pull_all(client: &AwClient) -> Result<(), Box> { + let hostnames = crate::util::get_remotes()?; + for host in hostnames { + pull(&host, client)? + } + Ok(()) +} + +pub fn pull(host: &str, client: &AwClient) -> Result<(), Box> { + client.wait_for_start()?; + + // Path to the sync folder + // Sync folder is structured ./{hostname}/{device_id}/test.db + let sync_root_dir = crate::dirs::get_sync_dir().map_err(|_| "Could not get sync dir")?; + let sync_dir = sync_root_dir.join(host); + let dbs = fs::read_dir(&sync_dir)? + .filter_map(Result::ok) + .filter(|entry| entry.path().is_dir()) + .map(|entry| fs::read_dir(entry.path())) + .filter_map(Result::ok) + .flatten() + .filter_map(Result::ok) + .filter(|entry| { + entry.path().is_file() + && entry.path().extension().and_then(|os_str| os_str.to_str()) == Some("db") + }) + .collect::>(); + + // if more than one db, warn and use the largest one + if dbs.len() > 1 { + warn!( + "More than one db found in sync folder for host, choosing largest db {:?}", + dbs + ); + } + + let db = dbs + .into_iter() + .max_by_key(|entry| entry.metadata().map(|m| m.len()).unwrap_or(0)) + .ok_or_else(|| format!("No db found in sync folder {:?}", sync_dir))?; + + let sync_spec = SyncSpec { + path: sync_dir.clone(), + path_db: Some(db.path().clone()), + buckets: None, // Sync all buckets by default + start: None, + }; + sync_run(client, &sync_spec, SyncMode::Pull)?; + + Ok(()) +} + +pub fn push(client: &AwClient) -> Result<(), Box> { + push_with_hostname(client, &client.hostname) +} + +pub fn push_with_hostname(client: &AwClient, hostname: &str) -> Result<(), Box> { + let sync_dir = crate::dirs::get_sync_dir() + .map_err(|_| "Could not get sync dir")? + .join(hostname); + + let sync_spec = SyncSpec { + path: sync_dir, + path_db: None, + buckets: None, // Sync all buckets by default + start: None, + }; + sync_run(client, &sync_spec, SyncMode::Push)?; + + Ok(()) +} diff --git a/aw-sync/src/util.rs b/aw-sync/src/util.rs new file mode 100644 index 0000000..a026123 --- /dev/null +++ b/aw-sync/src/util.rs @@ -0,0 +1,262 @@ +use std::error::Error; +use std::ffi::OsStr; +use std::fs; +use std::fs::File; +use std::io::Read; +use std::net::IpAddr; +use std::path::{Path, PathBuf}; + +// Only used by the binary (main.rs) and the Android entrypoint, so these are +// dead code in the plain library build. +#[allow(dead_code)] +pub struct ServerConfig { + pub port: u16, + pub api_key: Option, +} + +#[allow(dead_code)] +impl ServerConfig { + pub fn default_for(testing: bool) -> Self { + Self { + port: if testing { 5666 } else { 5600 }, + api_key: None, + } + } +} + +/// Returns the settings aw-sync needs from the selected aw-server config. +/// +/// Also used on Android: the embedded server writes `config.toml` under +/// `filesDir`, and `get_client()` in `android.rs` must send the same +/// `[auth].api_key` or `/api/0/buckets` returns 401 (aw-android#247). +#[allow(dead_code)] +pub fn get_server_config( + testing: bool, + config_override: Option<&Path>, +) -> Result> { + let path = match config_override { + Some(path) => path.to_path_buf(), + None => crate::dirs::get_server_config_path(testing) + .map_err(|_| "Could not get aw-server config path")?, + }; + let default = ServerConfig::default_for(testing); + if !path.exists() { + return Ok(default); + } + + let mut contents = String::new(); + File::open(path)?.read_to_string(&mut contents)?; + let value: toml::Value = toml::from_str(&contents)?; + let port = value + .get("port") + .and_then(|v| v.as_integer()) + .and_then(|v| u16::try_from(v).ok()) + .unwrap_or(default.port); + let api_key = value + .get("auth") + .and_then(|a| a.get("api_key")) + .and_then(|v| v.as_str()) + .filter(|s| !s.is_empty()) + .map(String::from); + + Ok(ServerConfig { port, api_key }) +} + +/// Local config must never be read for a caller-selected remote target. +#[cfg(not(target_os = "android"))] +#[allow(dead_code)] +pub fn is_loopback_host(host: &str) -> bool { + host.eq_ignore_ascii_case("localhost") + || host + .parse::() + .is_ok_and(|address| address.is_loopback()) +} + +/// Add URL brackets around bare IPv6 literals. +#[cfg(not(target_os = "android"))] +#[allow(dead_code)] +pub fn host_for_url(host: &str) -> String { + match host.parse::() { + Ok(IpAddr::V6(_)) => format!("[{host}]"), + _ => host.to_string(), + } +} + +#[cfg(all(test, not(target_os = "android")))] +mod tests { + use super::{get_server_config, host_for_url, is_loopback_host}; + use std::fs; + use std::time::{SystemTime, UNIX_EPOCH}; + + #[test] + fn reads_port_and_api_key_from_config_override() { + let config_path = std::env::temp_dir().join(format!( + "aw-sync-config-{}-{}.toml", + std::process::id(), + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + fs::write( + &config_path, + "port = 5611\n[auth]\napi_key = \"custom-key\"\n", + ) + .unwrap(); + + let config = get_server_config(false, Some(&config_path)).unwrap(); + + fs::remove_file(config_path).unwrap(); + assert_eq!(config.port, 5611); + assert_eq!(config.api_key.as_deref(), Some("custom-key")); + } + + #[test] + fn missing_config_override_uses_defaults() { + let config_path = std::env::temp_dir().join(format!( + "missing-aw-sync-config-{}.toml", + std::process::id() + )); + let _ = fs::remove_file(&config_path); + + let production = get_server_config(false, Some(&config_path)).unwrap(); + let testing = get_server_config(true, Some(&config_path)).unwrap(); + + assert_eq!(production.port, 5600); + assert!(production.api_key.is_none()); + assert_eq!(testing.port, 5666); + assert!(testing.api_key.is_none()); + } + + #[test] + fn commented_or_empty_api_key_is_absent() { + let config_path = std::env::temp_dir().join(format!( + "aw-sync-config-commented-{}-{}.toml", + std::process::id(), + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + fs::write( + &config_path, + "port = 5600\n[auth]\n#api_key = \"secret\"\napi_key = \"\"\n", + ) + .unwrap(); + + let config = get_server_config(false, Some(&config_path)).unwrap(); + fs::remove_file(config_path).unwrap(); + assert!(config.api_key.is_none()); + } + + #[test] + fn recognizes_only_loopback_hosts() { + for host in ["127.0.0.1", "127.0.0.2", "::1", "localhost", "LOCALHOST"] { + assert!(is_loopback_host(host)); + } + for host in ["example.com", "192.0.2.1", "localhost.example.com"] { + assert!(!is_loopback_host(host)); + } + } + + #[test] + fn brackets_bare_ipv6_hosts_for_urls() { + assert_eq!(host_for_url("::1"), "[::1]"); + assert_eq!(host_for_url("2001:db8::1"), "[2001:db8::1]"); + assert_eq!(host_for_url("127.0.0.1"), "127.0.0.1"); + assert_eq!(host_for_url("localhost"), "localhost"); + } +} + +/// Check if a directory contains a .db file +fn contains_db_file(dir: &std::path::Path) -> bool { + fs::read_dir(dir) + .ok() + .map(|entries| { + entries.filter_map(Result::ok).any(|entry| { + entry + .path() + .extension() + .map(|ext| ext == "db") + .unwrap_or(false) + }) + }) + .unwrap_or(false) +} + +/// Check if a directory contains a subdirectory that contains a .db file +fn contains_subdir_with_db_file(dir: &std::path::Path) -> bool { + fs::read_dir(dir) + .ok() + .map(|entries| { + entries + .filter_map(Result::ok) + .any(|entry| entry.path().is_dir() && contains_db_file(&entry.path())) + }) + .unwrap_or(false) +} + +/// Return all remotes in the sync folder +/// Only returns folders that match ./{host}/{device_id}/*.db +// TODO: share logic with find_remotes and find_remotes_nonlocal +pub fn get_remotes() -> Result, Box> { + let sync_root_dir = crate::dirs::get_sync_dir()?; + fs::create_dir_all(&sync_root_dir)?; + let hostnames = fs::read_dir(sync_root_dir)? + .filter_map(Result::ok) + .filter(|entry| entry.path().is_dir() && contains_subdir_with_db_file(&entry.path())) + .filter_map(|entry| { + entry + .path() + .file_name() + .and_then(|os_str| os_str.to_str().map(String::from)) + }) + .collect(); + info!("Found remotes: {:?}", hostnames); + Ok(hostnames) +} + +/// Returns a list of all remote dbs +/// +/// I/O errors are propagated rather than unwrapped (a panic here aborts the app +/// on Android, ActivityWatch/aw-android#220) and rather than skipped: silently +/// dropping a host directory we failed to read would report a successful sync +/// that quietly omitted that host's data. +fn find_remotes(sync_directory: &Path) -> std::io::Result> { + let mut dbs = Vec::new(); + for entry in fs::read_dir(sync_directory)? { + let hostdir = entry?.path(); + if !hostdir.is_dir() { + continue; + } + for entry in fs::read_dir(&hostdir)? { + let path = entry?.path(); + if path.extension().unwrap_or_else(|| OsStr::new("")) == "db" { + dbs.push(path); + } + } + } + Ok(dbs) +} + +/// Returns a list of all remotes, excluding local ones +pub fn find_remotes_nonlocal( + sync_directory: &Path, + device_id: &str, + sync_db: Option<&PathBuf>, +) -> std::io::Result> { + let remotes_all = find_remotes(sync_directory)?; + Ok(remotes_all + .into_iter() + // Filter out own remote + .filter(|path| !path.to_string_lossy().contains(device_id)) + // If sync_db is Some, return only remotes in that path + .filter(|path| { + if let Some(sync_db) = sync_db { + path.starts_with(sync_db) + } else { + true + } + }) + .collect()) +} diff --git a/aw-sync/test-server.sh b/aw-sync/test-server.sh new file mode 100755 index 0000000..4f162b7 --- /dev/null +++ b/aw-sync/test-server.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +# port and db used for testing instance + +# If port already set, use that, otherwise, use 5667 +PORT=${PORT:-5667} + +DBPATH=/tmp/aw-server-rust-sync-testing/ +mkdir -p $DBPATH + +# Set up an isolated ActivityWatch instance +pushd .. +cargo run --bin aw-server -- --testing --port $PORT --dbpath $DBPATH/data.db --no-legacy-import --verbose diff --git a/aw-sync/test-sync-pull.sh b/aw-sync/test-sync-pull.sh new file mode 100755 index 0000000..851f821 --- /dev/null +++ b/aw-sync/test-sync-pull.sh @@ -0,0 +1,78 @@ +#!/bin/bash + +# exit on fail +set -e + +# get script path +SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )" +pushd $SCRIPTPATH + +# port used for testing instance +PORT=5667 + +# if server already running on port 5667, don't start again +if [ "$(lsof -i:$PORT -sTCP:LISTEN -t)" ]; then + echo "ActivityWatch server already running on port $PORT, using that." +else + # Set up an isolated ActivityWatch instance + ./test-server.sh $PORT & + SERVER_PID=$! +fi + + +sleep 1; +SYNCROOTDIR="$HOME/ActivityWatchSync" + + +function sync_host() { + host=$1 + SYNCDIR="$SYNCROOTDIR/$host" + dbs=$(find $SYNCDIR -name "*.db") + for db in $dbs; do + # workaround to avoid trying to sync empty database files (size 45056) + if [ "$(stat -c%s $db)" -lt 50000 ]; then + continue + fi + + AWSYNC_ARGS="--port $PORT" + AWSYNC_ARGS_ADV="--sync-dir $SYNCDIR --sync-db $db" + BUCKETS="aw-watcher-window_$host,aw-watcher-afk_$host" + + echo "Syncing $db to $host" + cargo run --bin aw-sync -- $AWSYNC_ARGS sync-advanced $AWSYNC_ARGS_ADV --mode pull --buckets $BUCKETS + # TODO: If there are no buckets from the expected host, emit a warning at the end. + # (push-script should not have created them to begin with) + done +} + +host=$1 + +# if no host given, sync all, otherwise sync only the given host +if [ -z "$host" ]; then + echo "Syncing all hosts" + sleep 0.5 + # For each host in the sync directory, pull the data from each database file using aw-sync + # Use `find` to get all directories in the sync directory + hostnames=$(find $SYNCROOTDIR -maxdepth 1 -type d -exec basename {} \;) + # filter out "erb-m2.local" + hostnames=$(echo $hostnames | tr ' ' '\n' | grep -v "erb-m2.local") + # filter out folder not containing subfolders with .db files + for host in $hostnames; do + if [ ! "$(find $SYNCROOTDIR/$host -name "*.db")" ]; then + hostnames=$(echo $hostnames | tr ' ' '\n' | grep -v $host) + fi + done + # Sync each host, file-by-file + for host in $hostnames; do + sync_host $host + done +else + echo "Syncing host $1" + sleep 0.5 + sync_host $host +fi + +# kill aw-server-rust (if started by us) +if [ "$SERVER_PID" ]; then + kill $SERVER_PID +fi diff --git a/aw-sync/test-sync-push.sh b/aw-sync/test-sync-push.sh new file mode 100755 index 0000000..3ba4b17 --- /dev/null +++ b/aw-sync/test-sync-push.sh @@ -0,0 +1,32 @@ +#!/bin/bash +# Helper script meant to be used to test aw-sync +# Example of a single-entry for cronjobs and the like + +# exit on fail +set -e + +# on Linux, use `hostnamectl`, on macOS, use `hostname` +if [ -x "$(command -v hostnamectl)" ]; then + HOSTNAME=$(hostnamectl --static) +else + HOSTNAME=$(hostname) +fi + +# TODO: Fetch in a cross-platform way (from aw-client command output?) +AWSERVERCONF=~/.config/activitywatch/aw-server/aw-server.toml + +# trim everything in file AWSERVERCONF before '[server-testing]' section +# grep for the aw-server port in aw-server.toml +# if config doesn't exist, assume 5600 +if [ -f "$AWSERVERCONF" ]; then + PORT=$(sed '/\[server-testing\]/,/\[.*\]/{//!d}' $AWSERVERCONF | grep -oP 'port = "\K[0-9]+') +else + PORT=5600 +fi + +SYNCDIR="$HOME/ActivityWatchSync/$HOSTNAME" +AWSYNC_ARGS="--port $PORT" +AWSYNC_ARGS_ADV="--sync-dir $SYNCDIR" + +# NOTE: Only sync window and AFK buckets, for now +cargo run --bin aw-sync --release -- $AWSYNC_ARGS sync-advanced $AWSYNC_ARGS_ADV --mode push --buckets aw-watcher-window_$HOSTNAME,aw-watcher-afk_$HOSTNAME diff --git a/aw-sync/tests/sync.rs b/aw-sync/tests/sync.rs new file mode 100644 index 0000000..9d24695 --- /dev/null +++ b/aw-sync/tests/sync.rs @@ -0,0 +1,544 @@ +#[macro_use] +extern crate log; +extern crate aw_sync; + +#[cfg(test)] +mod sync_tests { + use std::collections::HashMap; + use std::path::Path; + + use chrono::{DateTime, Duration, Utc}; + + use aw_datastore::{Datastore, DatastoreError}; + use aw_models::{Bucket, Event}; + use aw_sync::{create_datastore, AccessMethod, SyncSpec}; + + struct TestState { + ds_src: Datastore, + ds_dest: Datastore, + } + + fn init_teststate() -> TestState { + TestState { + ds_src: Datastore::new_in_memory(false), + ds_dest: Datastore::new_in_memory(false), + } + } + + fn create_bucket(ds: &Datastore, n: i32) -> String { + // Create a bucket + let bucket_id = format!("bucket-{n}"); + let bucket_jsonstr = format!( + r#"{{ + "id": "{bucket_id}", + "type": "test", + "hostname": "device-{n}", + "client": "test" + }}"# + ); + let bucket: Bucket = serde_json::from_str(&bucket_jsonstr).unwrap(); + match ds.create_bucket(&bucket) { + Ok(()) => (), + Err(e) => match e { + DatastoreError::BucketAlreadyExists(_) => { + debug!("bucket already exists, skipping"); + } + e => panic!("woops! {e:?}"), + }, + }; + bucket_id + } + + fn create_event(data_str: &str) -> Event { + // A workaround needed because otherwise events might get same timestamp if + // call is repeated quickly on platforms with a low-precision clock. + std::thread::sleep(std::time::Duration::from_millis(5)); + + let timestamp: DateTime = Utc::now(); + let event_jsonstr = format!( + r#"{{ + "timestamp": "{}", + "duration": 0, + "data": {{"test": {} }} + }}"#, + timestamp.to_rfc3339(), + data_str + ); + serde_json::from_str(&event_jsonstr).unwrap() + } + + fn create_events(ds: &Datastore, bucket_id: &str, n: i64) { + let events: Vec = (0..n) + .map(|i| create_event(format!("{i}").as_str())) + .collect::>(); + + ds.insert_events(bucket_id, &events[..]).unwrap(); + ds.force_commit().unwrap(); + } + + fn get_all_buckets(datastores: Vec<&Datastore>) -> Vec<(&Datastore, Bucket)> { + let mut all_buckets: Vec<(&Datastore, Bucket)> = Vec::new(); + for ds in datastores { + let buckets = ds.get_buckets().unwrap(); + for bucket in buckets.values() { + all_buckets.push((ds, bucket.clone())); + } + } + all_buckets + } + + fn get_all_buckets_map(datastores: Vec<&Datastore>) -> HashMap { + let all_buckets = get_all_buckets(datastores); + all_buckets + .iter() + .cloned() + .map(|(ds, b)| (b.id.clone(), (ds, b))) + .collect() + } + + /// A datastore failure must be *returned*, never panicked. + /// + /// On Android the sync step runs inside a JNI `extern "C"` frame, where an + /// unwinding panic aborts the whole app process rather than surfacing as an + /// exception — the SIGABRT in ActivityWatch/aw-android#220. `sync_datastores` + /// used to `unwrap()` every datastore call, so any failure here was fatal. + #[test] + fn test_unusable_datastore_returns_error_instead_of_panicking() { + let state = init_teststate(); + create_bucket(&state.ds_src, 0); + + // A datastore whose database file cannot be opened: its worker is gone, + // so every request fails. + let ds_broken = create_datastore(Path::new( + "/nonexistent-directory-for-aw-sync-tests/test.db", + )) + .expect("path is valid UTF-8"); + + let result = aw_sync::sync_datastores( + &state.ds_src, + &ds_broken, + true, + Some("device-0"), + &SyncSpec::default(), + ); + assert!( + result.is_err(), + "an unusable destination datastore must return Err, got {result:?}" + ); + } + + /// Bucket metadata of an unexpected shape must also be an error, not a panic: + /// `$aw.sync.origin` is read from data written by another host, so it is not + /// under this host's control. + #[test] + fn test_non_string_sync_origin_returns_error_instead_of_panicking() { + let state = init_teststate(); + let bucket: Bucket = serde_json::from_str( + r#"{ + "id": "bucket-weird", + "type": "test", + "hostname": "device-0", + "client": "test", + "data": {"$aw.sync.origin": 42} + }"#, + ) + .unwrap(); + state.ds_src.create_bucket(&bucket).unwrap(); + + let result = aw_sync::sync_datastores( + &state.ds_src, + &state.ds_dest, + false, // pull: this is the path that reads $aw.sync.origin + None, + &SyncSpec::default(), + ); + let err = result.expect_err("a non-string $aw.sync.origin must return Err"); + assert!( + err.contains("$aw.sync.origin"), + "error should name the offending field, got: {err}" + ); + } + + /// A pulled bucket whose hostname is the "unknown" sentinel has no provenance, + /// and pull passes no source device ID to substitute. Continuing would map + /// every such bucket from every remote onto one `-synced-from-unknown` + /// destination, mixing events from unrelated devices — so refuse the sync. + /// (The old code unwrapped the `None` here, i.e. aborted the app on Android.) + #[test] + fn test_unknown_hostname_on_pull_returns_error() { + let state = init_teststate(); + let bucket: Bucket = serde_json::from_str( + r#"{ + "id": "bucket-unknown-host", + "type": "test", + "hostname": "unknown", + "client": "test" + }"#, + ) + .unwrap(); + state.ds_src.create_bucket(&bucket).unwrap(); + + let result = aw_sync::sync_datastores( + &state.ds_src, + &state.ds_dest, + false, // pull: no source device ID is passed + None, + &SyncSpec::default(), + ); + let err = result.expect_err("an unknown hostname on pull must return Err"); + assert!( + err.contains("bucket-unknown-host"), + "error should name the bucket, got: {err}" + ); + + // On push the source device ID is known, so the same bucket syncs fine. + aw_sync::sync_datastores( + &state.ds_src, + &state.ds_dest, + true, + Some("device-0"), + &SyncSpec::default(), + ) + .unwrap(); + } + + #[test] + fn test_buckets_created() { + // TODO: Split up this test + let state = init_teststate(); + create_bucket(&state.ds_src, 0); + + aw_sync::sync_datastores( + &state.ds_src, + &state.ds_dest, + false, + None, + &SyncSpec::default(), + ) + .unwrap(); + + let buckets_src: HashMap = state.ds_src.get_buckets().unwrap(); + let buckets_dest: HashMap = state.ds_dest.get_buckets().unwrap(); + assert!(buckets_src.len() == buckets_dest.len()); + } + + /// On pull (is_push=false), the destination bucket should have $aw.sync.origin set to + /// the source bucket's hostname. On push (is_push=true), $aw.sync.origin must NOT be + /// written to the staging copy. + #[test] + fn test_sync_origin_metadata() { + let state = init_teststate(); + let bucket_id = create_bucket(&state.ds_src, 0); + + // --- push: staging copy must have no $aw.sync.origin --- + aw_sync::sync_datastores( + &state.ds_src, + &state.ds_dest, + true, // is_push + Some("device-0"), + &SyncSpec::default(), + ) + .unwrap(); + + let push_buckets = state.ds_dest.get_buckets().unwrap(); + let push_bucket = push_buckets.get(&bucket_id).expect("push bucket not found"); + assert!( + !push_bucket.data.contains_key("$aw.sync.origin"), + "push-staging bucket must not have $aw.sync.origin, got: {:?}", + push_bucket.data + ); + + // --- pull: imported bucket must have $aw.sync.origin set to the hostname --- + let ds_pull_dest = Datastore::new_in_memory(false); + aw_sync::sync_datastores( + &state.ds_src, + &ds_pull_dest, + false, // is_push (pull) + None, + &SyncSpec::default(), + ) + .unwrap(); + + let pull_buckets = ds_pull_dest.get_buckets().unwrap(); + let pull_bucket_id = format!("{bucket_id}-synced-from-device-0"); + let pull_bucket = pull_buckets + .get(&pull_bucket_id) + .expect("pull bucket not found"); + let origin = pull_bucket + .data + .get("$aw.sync.origin") + .expect("pull bucket must have $aw.sync.origin") + .as_str() + .expect("$aw.sync.origin must be a string"); + assert_eq!( + origin, "device-0", + "$aw.sync.origin should match source hostname" + ); + } + + /// When bucket_from was previously pulled (carries $aw.sync.origin in its data), pushing it + /// to staging must strip the stale field so downstream devices don't trust a wrong origin. + #[test] + fn test_push_strips_stale_sync_origin() { + let state = init_teststate(); + + // Create a source bucket that already has $aw.sync.origin (simulating a + // previously-pulled bucket being pushed back to staging). + let bucket_id = "bucket-with-origin".to_string(); + let bucket: Bucket = serde_json::from_value(serde_json::json!({ + "id": bucket_id, + "type": "test", + "hostname": "device-0", + "client": "test", + "data": { "$aw.sync.origin": "some-other-host" } + })) + .unwrap(); + state.ds_src.create_bucket(&bucket).unwrap(); + + aw_sync::sync_datastores( + &state.ds_src, + &state.ds_dest, + true, // is_push + Some("device-0"), + &SyncSpec::default(), + ) + .unwrap(); + + let push_buckets = state.ds_dest.get_buckets().unwrap(); + let push_bucket = push_buckets.get(&bucket_id).expect("push bucket not found"); + assert!( + !push_bucket.data.contains_key("$aw.sync.origin"), + "push-staging bucket must not carry stale $aw.sync.origin from a previously-pulled source, got: {:?}", + push_bucket.data + ); + } + + fn check_synced_buckets_equal_to_src(all_buckets_map: &HashMap) { + for (ds, bucket) in all_buckets_map.values() { + if bucket.id.contains("-synced") { + let bucket_src_id = bucket.id.split("-synced-").next().unwrap(); + let (ds_src, bucket_src) = all_buckets_map.get(bucket_src_id).unwrap(); + let events_synced = ds.get_events(bucket.id.as_str(), None, None, None).unwrap(); + let events_src = ds_src + .get_events(bucket_src.id.as_str(), None, None, None) + .unwrap(); + println!("{events_synced:?}"); + println!("{events_src:?}"); + assert!(events_synced == events_src); + } + } + } + + #[test] + fn test_one_updated_event() { + // This tests the syncing of one single event that is then updated by a heartbeat after the + // first sync pass. + let state = init_teststate(); + + let bucket_id = create_bucket(&state.ds_src, 0); + state + .ds_src + .heartbeat(bucket_id.as_str(), create_event("1"), 1.0) + .unwrap(); + + aw_sync::sync_datastores( + &state.ds_src, + &state.ds_dest, + false, + None, + &SyncSpec::default(), + ) + .unwrap(); + + let all_datastores: Vec<&Datastore> = [&state.ds_src, &state.ds_dest].to_vec(); + let all_buckets_map = get_all_buckets_map(all_datastores); + + // Check that all synced buckets are identical to source bucket + check_synced_buckets_equal_to_src(&all_buckets_map); + + // Add some more events + state + .ds_src + .heartbeat(bucket_id.as_str(), create_event("1"), 1.0) + .unwrap(); + aw_sync::sync_datastores( + &state.ds_src, + &state.ds_dest, + false, + None, + &SyncSpec::default(), + ) + .unwrap(); + + // Check again that new events were indeed synced + check_synced_buckets_equal_to_src(&all_buckets_map); + } + + #[test] + fn test_events() { + let state = init_teststate(); + + let bucket_id = create_bucket(&state.ds_src, 0); + create_events(&state.ds_src, bucket_id.as_str(), 10); + + aw_sync::sync_datastores( + &state.ds_src, + &state.ds_dest, + false, + None, + &SyncSpec::default(), + ) + .unwrap(); + + let all_datastores: Vec<&Datastore> = [&state.ds_src, &state.ds_dest].to_vec(); + let all_buckets_map = get_all_buckets_map(all_datastores); + + // Check that all synced buckets are identical to source bucket + check_synced_buckets_equal_to_src(&all_buckets_map); + + // Add some more events + create_events(&state.ds_src, bucket_id.as_str(), 10); + aw_sync::sync_datastores( + &state.ds_src, + &state.ds_dest, + false, + None, + &SyncSpec::default(), + ) + .unwrap(); + + // Check again that new events were indeed synced + check_synced_buckets_equal_to_src(&all_buckets_map); + } + + #[test] + fn test_sync_resume_after_partial_sync() { + // Verify that resuming an interrupted sync picks up where it left off. + // This exercises the resume_sync_at path in the chunked-fetch loop. + let state = init_teststate(); + + let bucket_id = create_bucket(&state.ds_src, 0); + create_events(&state.ds_src, bucket_id.as_str(), 15); + + // First sync pass + aw_sync::sync_datastores( + &state.ds_src, + &state.ds_dest, + false, + None, + &SyncSpec::default(), + ) + .unwrap(); + + let all_datastores_1: Vec<&Datastore> = [&state.ds_src, &state.ds_dest].to_vec(); + check_synced_buckets_equal_to_src(&get_all_buckets_map(all_datastores_1)); + + // Add more events and sync again — verifies resume logic + create_events(&state.ds_src, bucket_id.as_str(), 15); + aw_sync::sync_datastores( + &state.ds_src, + &state.ds_dest, + false, + None, + &SyncSpec::default(), + ) + .unwrap(); + + let all_datastores_2: Vec<&Datastore> = [&state.ds_src, &state.ds_dest].to_vec(); + check_synced_buckets_equal_to_src(&get_all_buckets_map(all_datastores_2)); + } + + #[test] + fn test_sync_multipage_no_duplicates() { + // Verify that multi-page syncs (more events than BATCH_SIZE) produce no duplicates. + // In tests, BATCH_SIZE=5, so 12 events triggers 3 pages (5+5+2). + // This exercises the boundary-event cursor logic that previously left one copy of + // the page-boundary event in the current chunk AND re-fetched it on the next page. + let state = init_teststate(); + + let bucket_id = create_bucket(&state.ds_src, 0); + create_events(&state.ds_src, bucket_id.as_str(), 12); + + // Initial sync + aw_sync::sync_datastores( + &state.ds_src, + &state.ds_dest, + false, + None, + &SyncSpec::default(), + ) + .unwrap(); + + let all_datastores_1: Vec<&Datastore> = [&state.ds_src, &state.ds_dest].to_vec(); + check_synced_buckets_equal_to_src(&get_all_buckets_map(all_datastores_1)); + + // Resume sync: add another 12 events (3 more pages) and sync again. + // Exercises multi-page resume path where heartbeat order matters. + create_events(&state.ds_src, bucket_id.as_str(), 12); + aw_sync::sync_datastores( + &state.ds_src, + &state.ds_dest, + false, + None, + &SyncSpec::default(), + ) + .unwrap(); + + let all_datastores_2: Vec<&Datastore> = [&state.ds_src, &state.ds_dest].to_vec(); + check_synced_buckets_equal_to_src(&get_all_buckets_map(all_datastores_2)); + } + + // TODO: Find a way to reuse this (previously used in an integration test) + fn setup_test(sync_directory: &Path) -> std::io::Result> { + let mut datastores: Vec = Vec::new(); + for n in 0..2 { + let dspath = sync_directory.join(format!("test-remote-{n}.db")); + let ds_ = create_datastore(&dspath).expect("test db path is valid UTF-8"); + let ds = &ds_ as &dyn AccessMethod; + + // Create a bucket + // NOTE: Created with duplicate name to make sure it still works under such conditions + let bucket_jsonstr = format!( + r#"{{ + "id": "bucket", + "type": "test", + "hostname": "device-{n}", + "client": "test" + }}"# + ); + let bucket: Bucket = serde_json::from_str(&bucket_jsonstr)?; + match ds.create_bucket(&bucket) { + Ok(()) => (), + Err(e) => match e { + DatastoreError::BucketAlreadyExists(_) => { + debug!("bucket already exists, skipping"); + } + e => panic!("woops! {e:?}"), + }, + }; + + // Insert some testing events into the bucket + let events: Vec = (0..3) + .map(|i| { + let timestamp: DateTime = Utc::now() + Duration::milliseconds(i * 10); + let event_jsonstr = format!( + r#"{{ + "timestamp": "{}", + "duration": 0, + "data": {{"test": {} }} + }}"#, + timestamp.to_rfc3339(), + i + ); + serde_json::from_str(&event_jsonstr).unwrap() + }) + .collect::>(); + + ds.insert_events(bucket.id.as_str(), events).unwrap(); + //let new_eventcount = ds.get_event_count(bucket.id.as_str(), None, None).unwrap(); + //info!("Eventcount: {:?} ({} new)", new_eventcount, events.len()); + datastores.push(ds_); + } + Ok(datastores) + } +} diff --git a/aw-sync/tests/sync_roundtrip.rs b/aw-sync/tests/sync_roundtrip.rs new file mode 100644 index 0000000..e9477bf --- /dev/null +++ b/aw-sync/tests/sync_roundtrip.rs @@ -0,0 +1,164 @@ +/// Regression test for the "synced-from-" duplication reported in +/// https://github.com/orgs/ActivityWatch/discussions/1373 +/// +/// aw-sync already refuses to import its *own* export (`find_remotes_nonlocal` +/// filters the local device_id out of the remote list). That guard only covers +/// the direct path. It does not cover the case where a host's data is laundered +/// through a *peer*: the peer imports HOSTA's buckets, then re-exports them as +/// part of its own push, and HOSTA imports them back as +/// `_HOSTA-synced-from-HOSTA` sitting next to the real local bucket. +/// +/// Both copies then render in /timeline, so every event is shown twice. +use std::path::PathBuf; +use std::sync::atomic::{AtomicU64, Ordering}; + +use aw_datastore::Datastore; +use aw_models::{Bucket, BucketMetadata}; +use aw_sync::{sync_datastores, AccessMethod, SyncSpec}; + +// Tests in this binary run in parallel. Use a monotonic counter to guarantee +// each datastore gets a unique path even when two tests start within the same +// clock tick (Windows/macOS SystemTime resolution can be coarser than 1 ns, +// causing path collisions: "duplicate column name" on macOS, "database is +// locked" on Windows). +static DB_COUNTER: AtomicU64 = AtomicU64::new(0); + +fn tmp_db(name: &str) -> PathBuf { + let mut p = std::env::temp_dir(); + // Counter: within-run uniqueness. Timestamp: cross-run uniqueness when + // the PID is reused and the counter resets to 0. Same as #655. + let ts = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos(); + p.push(format!( + "aw-sync-roundtrip-{}-{}-{}-{}.db", + std::process::id(), + name, + DB_COUNTER.fetch_add(1, Ordering::Relaxed), + ts, + )); + p +} + +fn datastore(name: &str) -> Datastore { + Datastore::new(tmp_db(name).to_str().unwrap().to_string(), false) +} + +fn bucket(id: &str, hostname: &str) -> Bucket { + Bucket { + bid: None, + id: id.to_string(), + _type: "currentwindow".to_string(), + client: "aw-watcher-window".to_string(), + hostname: hostname.to_string(), + created: None, + data: serde_json::Map::new(), + metadata: BucketMetadata::default(), + events: None, + last_updated: None, + } +} + +fn bucket_ids(ds: &dyn AccessMethod) -> Vec { + let mut ids: Vec = ds.get_buckets().unwrap().keys().cloned().collect(); + ids.sort(); + ids +} + +/// Two hosts sharing a sync folder. Returns (a_local, b_export) after a full +/// push/pull round: HOSTA pushes, HOSTB pulls, HOSTB pushes, HOSTA pulls. +fn round_trip() -> (Datastore, Datastore) { + let spec = SyncSpec::default(); + + // HOSTA: local server datastore + the db it pushes into the sync folder. + let a_local = datastore("a-local"); + let a_export = datastore("a-export"); + // HOSTB: same. + let b_local = datastore("b-local"); + let b_export = datastore("b-export"); + + // HOSTA collects some data locally. + a_local + .create_bucket(&bucket("aw-watcher-window_HOSTA", "HOSTA")) + .unwrap(); + + // 1. HOSTA pushes to its own folder in the sync dir. + sync_datastores(&a_local, &a_export, true, Some("device-A"), &spec).unwrap(); + + // 2. HOSTB pulls HOSTA's export. This copy is correct and expected. + sync_datastores(&a_export, &b_local, false, None, &spec).unwrap(); + assert!( + bucket_ids(&b_local).contains(&"aw-watcher-window_HOSTA-synced-from-HOSTA".to_string()), + "precondition: HOSTB should hold HOSTA's data as a synced-from-HOSTA bucket, got {:?}", + bucket_ids(&b_local) + ); + + // 3. HOSTB pushes its own data to the sync folder. + sync_datastores(&b_local, &b_export, true, Some("device-B"), &spec).unwrap(); + + // 4. HOSTA pulls HOSTB's export. + sync_datastores(&b_export, &a_local, false, None, &spec).unwrap(); + + (a_local, b_export) +} + +/// The fix location: a push must export only buckets that originate on this +/// host. Re-exporting buckets pulled from a peer is what lets data round-trip. +#[test] +fn test_push_does_not_reexport_synced_buckets() { + let (_a_local, b_export) = round_trip(); + assert_eq!( + bucket_ids(&b_export), + Vec::::new(), + "HOSTB's export must not contain buckets it merely synced from HOSTA" + ); +} + +/// The reported symptom: HOSTA must never end up with a `-synced-from-HOSTA` +/// copy of its own bucket, which renders every event twice in /timeline. +#[test] +fn test_own_data_does_not_return_via_peer() { + let (a_local, _b_export) = round_trip(); + assert_eq!( + bucket_ids(&a_local), + vec!["aw-watcher-window_HOSTA".to_string()], + "HOSTA must not gain a synced-from-HOSTA copy of its own bucket" + ); +} + +/// `-synced-from-` is a reserved token in aw-sync's ID grammar, not just a +/// naming convention. `get_or_create_sync_bucket` splits on it to recover the +/// original bucket ID, so a first-hand bucket whose own ID contains it is +/// already mangled on import regardless of provenance filtering. +/// +/// This pins that pre-existing behaviour so the reservation is explicit: such a +/// bucket is treated as a synced copy. Issue #649 tracks moving provenance into +/// bucket metadata, which removes the dependency on the ID string. +#[test] +fn test_synced_from_is_a_reserved_id_token() { + let spec = SyncSpec::default(); + let local = datastore("reserved-local"); + let export = datastore("reserved-export"); + + // A bucket the sync layer would mangle anyway: the ID grammar reserves the + // token, so this is not a supported first-hand bucket name. + local + .create_bucket(&bucket( + "aw-watcher-window_HOSTA-synced-from-HOSTB", + "HOSTA", + )) + .unwrap(); + // A normally-named bucket alongside it, to prove the filter is not blanket. + local + .create_bucket(&bucket("aw-watcher-afk_HOSTA", "HOSTA")) + .unwrap(); + + sync_datastores(&local, &export, true, Some("device-A"), &spec).unwrap(); + + assert_eq!( + bucket_ids(&export), + vec!["aw-watcher-afk_HOSTA".to_string()], + "the reserved token marks a bucket as synced; unmarked buckets still export" + ); +} diff --git a/aw-transform/Cargo.toml b/aw-transform/Cargo.toml new file mode 100644 index 0000000..3d77a29 --- /dev/null +++ b/aw-transform/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "aw-transform" +version = "0.1.0" +authors = ["Johan Bjäreholt "] +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +serde = "1.0" +url = ">=2.2" +serde_json = "1.0" +fancy-regex = "0.17.0" +lru = "0.18.1" +log = "0.4" +chrono = { version = "0.4", features = ["serde"] } +aw-models = { path = "../aw-models" } + +[dev-dependencies] +criterion = "0.5.1" + +[[bench]] +name = "bench" +harness = false diff --git a/aw-transform/benches/bench.rs b/aw-transform/benches/bench.rs new file mode 100644 index 0000000..79bcbe2 --- /dev/null +++ b/aw-transform/benches/bench.rs @@ -0,0 +1,91 @@ +use chrono::Duration; +use criterion::{criterion_group, criterion_main, BatchSize, BenchmarkId, Criterion}; +use serde_json::json; +use serde_json::Map; +use serde_json::Value; + +use aw_models::Event; +use aw_transform::*; + +// TODO: Move me to an appropriate place +#[macro_export] +macro_rules! json_map { + { $( $key:literal : $value:expr),* } => {{ + use serde_json::Value; + use serde_json::map::Map; + #[allow(unused_mut)] + let mut map : Map = Map::new(); + $( + map.insert( $key.to_string(), json!($value) ); + )* + map + }}; +} + +fn create_events(num_events: i64) -> Vec { + let mut possible_data = Vec::>::new(); + for i in 0..20 { + possible_data.push(json_map! {"number": i}); + } + let mut event_list = Vec::new(); + for i in 0..num_events { + let e = Event { + id: None, + timestamp: chrono::Utc::now() + Duration::seconds(i), + duration: Duration::seconds(10), + data: possible_data[i as usize % 20].clone(), + }; + event_list.push(e); + } + event_list +} + +fn bench_filter_period_intersect(c: &mut Criterion) { + let events2 = create_events(1000); + c.bench_function("1000 events", |b| { + b.iter(|| { + let events1 = create_events(1000); + filter_period_intersect(events1, events2.clone()); + }) + }); +} + +fn bench_union_no_overlap(c: &mut Criterion) { + let mut group = c.benchmark_group("union_no_overlap"); + for n in [1_000, 8_000, 32_000] { + let now = chrono::Utc::now(); + let first: Vec<_> = (0..n) + .map(|i| { + Event::new( + now + Duration::seconds(i * 4 + 1), + Duration::seconds(1), + json_map! {"app": "foreground"}, + ) + }) + .collect(); + let second: Vec<_> = (0..n) + .map(|i| { + Event::new( + now + Duration::seconds(i * 4), + Duration::seconds(3), + json_map! {"app": "background"}, + ) + }) + .collect(); + group.bench_function(BenchmarkId::new("overlapping", n), |b| { + b.iter_batched( + || (first.clone(), second.clone()), + |(a, b)| union_no_overlap(a, b), + BatchSize::LargeInput, + ); + }); + } + group.finish(); +} + +criterion_group!( + benches, + bench_filter_period_intersect, + bench_union_no_overlap +); +criterion_main!(benches); diff --git a/aw-transform/src/chunk.rs b/aw-transform/src/chunk.rs new file mode 100644 index 0000000..2b8d258 --- /dev/null +++ b/aw-transform/src/chunk.rs @@ -0,0 +1,80 @@ +use aw_models::Event; + +/// Chunks together events with the same key +/// +/// NOTE: In most cases you should use merge_events_by_keys instead, this +/// transform is mostly just for backwards compatibility with older versions +/// of aw-webui +/// NOTE: Does not support sub-chunking which aw-server-python supports +/// Without sub-chunking it is pretty much the same as merge_events_by_key +/// +/// # Example +/// ```ignore +/// key: a +/// input: +/// { duration: 1.0, data: { "a": 1, "b": 1 } } +/// { duration: 1.0, data: { "a": 1, "b": 2 } } +/// { duration: 1.0, data: { "a": 2, "b": 1 } } +/// output: +/// { duration: 2.0, data: { "a": 1 } } +/// { duration: 1.0, data: { "a": 2 } } +/// ``` +pub fn chunk_events_by_key(events: Vec, key: &str) -> Vec { + let mut chunked_events: Vec = Vec::new(); + for event in events { + if chunked_events.is_empty() && event.data.get(key).is_some() { + // TODO: Add sub-chunks + chunked_events.push(event); + } else { + let val = match event.data.get(key) { + None => continue, + Some(v) => v, + }; + let mut last_event = chunked_events.pop().unwrap(); + let last_val = last_event.data.get(key).unwrap().clone(); + if &last_val == val { + // TODO: Add sub-chunks + last_event.duration += event.duration; + } + chunked_events.push(last_event); + if &last_val != val { + // TODO: Add sub-chunks + chunked_events.push(event); + } + } + } + chunked_events +} + +#[cfg(test)] +mod tests { + use std::str::FromStr; + + use chrono::DateTime; + use chrono::Duration; + use serde_json::json; + + use aw_models::Event; + + use super::chunk_events_by_key; + + #[test] + fn test_chunk_events_by_key() { + let e1 = Event { + id: None, + timestamp: DateTime::from_str("2000-01-01T00:00:01Z").unwrap(), + duration: Duration::seconds(1), + data: json_map! {"test": json!(1)}, + }; + let mut e2 = e1.clone(); + e2.data = json_map! {"test2": json!(1)}; + let e3 = e1.clone(); + let mut e4 = e1.clone(); + e4.data = json_map! {"test": json!(2)}; + + let res = chunk_events_by_key(vec![e1, e2, e3, e4], "test"); + assert_eq!(res.len(), 2); + assert_eq!(res[0].duration, Duration::seconds(2)); + assert_eq!(res[1].duration, Duration::seconds(1)); + } +} diff --git a/aw-transform/src/classify.rs b/aw-transform/src/classify.rs new file mode 100644 index 0000000..8b24430 --- /dev/null +++ b/aw-transform/src/classify.rs @@ -0,0 +1,642 @@ +/// Transforms for classifying (tagging and categorizing) events. +/// +/// Based on code in aw_research: https://github.com/ActivityWatch/aw-research/blob/master/aw_research/classify.py +use aw_models::Event; +use fancy_regex::Regex; +use lru::LruCache; +use std::collections::HashMap; +use std::num::NonZeroUsize; +use std::sync::{Arc, Mutex, OnceLock}; + +const REGEX_CACHE_CAPACITY: usize = 512; + +static REGEX_CACHE: OnceLock>>> = OnceLock::new(); + +pub enum Rule { + None, + Regex(RegexRule), +} + +impl RuleTrait for Rule { + fn matches(&self, event: &Event) -> bool { + match self { + Rule::None => false, + Rule::Regex(rule) => rule.matches(event), + } + } +} + +trait RuleTrait { + fn matches(&self, event: &Event) -> bool; +} + +pub struct RegexRule { + regex: Arc, + select_keys: Option>, +} + +impl RegexRule { + pub fn new( + regex_str: &str, + ignore_case: bool, + select_keys: Option>, + ) -> Result { + // Validate that select_keys is not an empty list, which would silently never match. + if let Some(ref keys) = select_keys { + if keys.is_empty() { + return Err(fancy_regex::Error::ParseError( + 0, + fancy_regex::ParseError::GeneralParseError( + "select_keys must not be empty".to_string(), + ), + )); + } + } + + // can't use `RegexBuilder::case_insensitive` because it's not supported by fancy_regex, + // so we need to prefix with `(?i)` to make it case insensitive. + let full_regex_str = if ignore_case { + format!("(?i){regex_str}") + } else { + regex_str.to_string() + }; + + let cache = REGEX_CACHE.get_or_init(|| { + Mutex::new(LruCache::new( + NonZeroUsize::new(REGEX_CACHE_CAPACITY).unwrap(), + )) + }); + let mut cache = cache.lock().unwrap(); + + let regex = if let Some(re) = cache.get(&full_regex_str) { + re.clone() + } else { + let re = Arc::new(Regex::new(&full_regex_str)?); + cache.put(full_regex_str.clone(), re.clone()); + re + }; + + Ok(RegexRule { regex, select_keys }) + } + + fn value_matches(&self, value: &serde_json::Value) -> bool { + match value.as_str() { + Some(value) => self.regex.is_match(value).unwrap_or(false), + None => false, + } + } +} + +/// This struct defines the rules for classification. +/// For now it just needs to contain the regex to match with, but in the future it might contain a +/// glob-pattern, or other options for classifying. +/// It's puropse is to make the API easy to extend in the future without having to break backwards +/// compatibility (or have to maintain "old" query2 functions). +impl RuleTrait for RegexRule { + fn matches(&self, event: &Event) -> bool { + match &self.select_keys { + Some(select_keys) => select_keys + .iter() + .filter_map(|key| event.data.get(key)) + .any(|val| self.value_matches(val)), + None => event.data.values().any(|val| self.value_matches(val)), + } + } +} + +impl From for Rule { + fn from(re: Regex) -> Self { + Rule::Regex(RegexRule { + regex: Arc::new(re), + select_keys: None, + }) + } +} + +/// A category matching rule passed to [`categorize`]. +/// +/// `priority` is an optional integer ranking score. When set, it is used +/// instead of the depth-derived default to pick among matching rules (higher +/// wins). When `None`, ranking falls back to `depth * 10` so existing configs +/// keep their current ordering, while explicit values can slot between levels +/// (depth 1 → 10, depth 2 → 20). +pub struct CategoryRule { + pub category: Vec, + pub rule: Rule, + pub priority: Option, +} + +impl CategoryRule { + pub fn new(category: Vec, rule: Rule) -> Self { + Self { + category, + rule, + priority: None, + } + } + + pub fn with_priority(mut self, priority: i64) -> Self { + self.priority = Some(priority); + self + } +} + +impl From<(Vec, Rule)> for CategoryRule { + fn from((category, rule): (Vec, Rule)) -> Self { + Self::new(category, rule) + } +} + +/// Categorizes a list of events +/// +/// An event can only have one category, although the category may have a hierarchy, +/// for instance: "Work -> ActivityWatch -> aw-server-rust" +/// If multiple categories match, the highest-ranking one is chosen. +/// Ranking is the optional integer `priority` on the rule when present, +/// otherwise `depth * 10` ("the deepest one will be chosen", with room to +/// slot values between levels). Equal ranks keep the later match, matching +/// the previous depth-only `>=` comparison. +/// +/// Performance: builds an in-memory cache keyed on the event's data JSON so that +/// events with identical data (same app/title — very common in practice) are only +/// matched against the rule set once. On a month's data with 50k+ events but only +/// a few hundred distinct app/title pairs this reduces regex work by >99%. +pub fn categorize(mut events: Vec, rules: &[CategoryRule]) -> Vec { + // Cache: serialized event data → assigned category + let mut category_cache: HashMap> = HashMap::new(); + let mut classified_events = Vec::with_capacity(events.len()); + for mut event in events.drain(..) { + // Key on the full event data. serde_json::Map preserves insertion order, so + // events with the same fields in the same order produce the same key — which + // is the normal case for heartbeat-based watchers. + let cache_key = serde_json::to_string(&event.data).unwrap_or_default(); + let category = category_cache + .entry(cache_key) + .or_insert_with(|| _pick_category(&event, rules)) + .clone(); + event + .data + .insert("$category".into(), serde_json::json!(category)); + classified_events.push(event); + } + classified_events +} + +fn _pick_category(event: &Event, rules: &[CategoryRule]) -> Vec { + let mut category: Vec = vec!["Uncategorized".into()]; + // Uncategorized loses to any non-empty match, including a match with a very + // low explicit priority. i64::MIN is only used as this sentinel. + // Empty category paths are skipped so they cannot replace the fallback + // (old depth comparison: len 0 does not beat Uncategorized's len 1). + let mut rank = i64::MIN; + for class in rules { + if class.category.is_empty() { + continue; + } + if class.rule.matches(event) { + let item_rank = _effective_rank(&class.category, class.priority); + if item_rank >= rank { + category = class.category.clone(); + rank = item_rank; + } + } + } + category +} + +/// Tags a list of events +/// +/// An event can have many tags (as opposed to only one category) which will be put into the `$tags` key of +/// the event data object. +pub fn tag(mut events: Vec, rules: &[(String, Rule)]) -> Vec { + let mut events_tagged = Vec::new(); + for event in events.drain(..) { + events_tagged.push(tag_one(event, rules)); + } + events_tagged +} + +fn tag_one(mut event: Event, rules: &[(String, Rule)]) -> Event { + let mut tags: Vec = Vec::new(); + for (cls, rule) in rules { + if rule.matches(&event) { + tags.push(cls.clone()); + } + } + tags.sort_unstable(); + tags.dedup(); + event.data.insert("$tags".into(), serde_json::json!(tags)); + event +} + +fn _effective_rank(category: &[String], priority: Option) -> i64 { + // Integer-only. Default is depth * 10 so explicit priorities can slot + // between nesting levels (depth 1 → 10, depth 2 → 20). Relative order of + // unprioritized rules is unchanged. + // https://github.com/ActivityWatch/aw-server-rust/pull/663#issuecomment-5481349757 + priority.unwrap_or((category.len() as i64) * 10) +} + +#[test] +fn test_rule() { + let mut e_match = Event::default(); + e_match + .data + .insert("test".into(), serde_json::json!("just a test")); + + let mut e_no_match = Event::default(); + e_no_match + .data + .insert("nonono".into(), serde_json::json!("no match!")); + + let rule_from_regex = Rule::from(Regex::new("test").unwrap()); + let rule_from_new = Rule::Regex(RegexRule::new("test", false, None).unwrap()); + let rule_none = Rule::None; + assert!(rule_from_regex.matches(&e_match)); + assert!(rule_from_new.matches(&e_match)); + assert!(!rule_from_regex.matches(&e_no_match)); + assert!(!rule_from_new.matches(&e_no_match)); + + assert!(!rule_none.matches(&e_match)); +} + +#[test] +fn test_rule_lookahead() { + // Originally requested by a user here, to match aw-server-python: https://canary.discord.com/channels/755040852727955476/755334543891759194/994291987878522961 + let mut e_match = Event::default(); + e_match + .data + .insert("test".into(), serde_json::json!("testing lookahead")); + + let rule_from_regex = Rule::from(Regex::new("testing (?!lookahead)").unwrap()); + assert!(!rule_from_regex.matches(&e_match)); +} + +#[test] +fn test_rule_select_keys() { + let mut event = Event::default(); + event + .data + .insert("app".into(), serde_json::json!("terminal")); + event + .data + .insert("title".into(), serde_json::json!("just a test")); + event.data.insert("pid".into(), serde_json::json!(123)); + + let title_only = + Rule::Regex(RegexRule::new("test", false, Some(vec!["title".into()])).unwrap()); + let app_only = Rule::Regex(RegexRule::new("test", false, Some(vec!["app".into()])).unwrap()); + let missing_key = + Rule::Regex(RegexRule::new("test", false, Some(vec!["missing".into()])).unwrap()); + let non_string_key = + Rule::Regex(RegexRule::new("123", false, Some(vec!["pid".into()])).unwrap()); + + assert!(title_only.matches(&event)); + assert!(!app_only.matches(&event)); + assert!(!missing_key.matches(&event)); + assert!(!non_string_key.matches(&event)); +} + +#[test] +fn test_rule_select_keys_empty_list() { + // An empty select_keys list should return an error rather than + // silently producing a rule that never matches anything. + let result = RegexRule::new("test", false, Some(vec![])); + assert!(result.is_err()); +} +#[test] +fn test_categorize() { + let mut e = Event::default(); + e.data + .insert("test".into(), serde_json::json!("just a test")); + + let mut events = vec![e]; + let rules: Vec = vec![ + CategoryRule::new( + vec!["Test".into()], + Rule::from(Regex::new(r"test").unwrap()), + ), + CategoryRule::new( + vec!["Test".into(), "Subtest".into()], + Rule::from(Regex::new(r"test").unwrap()), + ), + CategoryRule::new( + vec!["Other".into()], + Rule::from(Regex::new(r"nonmatching").unwrap()), + ), + ]; + events = categorize(events, &rules); + + assert_eq!(events.len(), 1); + assert_eq!( + events.first().unwrap().data.get("$category").unwrap(), + &serde_json::json!(vec!["Test", "Subtest"]) + ); +} + +#[test] +fn test_categorize_uncategorized() { + // Checks that the category correctly becomes uncategorized when no category matches + let mut e = Event::default(); + e.data + .insert("test".into(), serde_json::json!("just a test")); + + let mut events = vec![e]; + let rules: Vec = vec![CategoryRule::new( + vec!["Non-matching".into(), "test".into()], + Rule::from(Regex::new(r"not going to match").unwrap()), + )]; + events = categorize(events, &rules); + + assert_eq!(events.len(), 1); + assert_eq!( + events.first().unwrap().data.get("$category").unwrap(), + &serde_json::json!(vec!["Uncategorized"]) + ); +} + +#[cfg(test)] +fn event_with_data(value: &str) -> Event { + let mut e = Event::default(); + e.data.insert("test".into(), serde_json::json!(value)); + e +} + +#[cfg(test)] +fn category_of(events: &[Event]) -> &serde_json::Value { + events.first().unwrap().data.get("$category").unwrap() +} + +#[test] +fn test_categorize_depth_wins_without_priority() { + // Reported scenario from ActivityWatch/aw-server-rust#597: a deeper nested + // match still beats a shallower match when neither rule sets priority. + // Category A (depth 1) and Category B → B1 (depth 2) both match. + let events = categorize( + vec![event_with_data("just a test")], + &[ + CategoryRule::new(vec!["A".into()], Rule::from(Regex::new(r"test").unwrap())), + CategoryRule::new( + vec!["B".into(), "B1".into()], + Rule::from(Regex::new(r"test").unwrap()), + ), + ], + ); + assert_eq!(category_of(&events), &serde_json::json!(vec!["B", "B1"])); +} + +#[test] +fn test_categorize_explicit_priority_overrides_depth() { + // The same #597 tree, but A is given a higher priority than B1's default + // (depth 2 → 20). Organizational nesting no longer forces B1 to win. + let events = categorize( + vec![event_with_data("just a test")], + &[ + CategoryRule::new(vec!["A".into()], Rule::from(Regex::new(r"test").unwrap())) + .with_priority(25), + CategoryRule::new( + vec!["B".into(), "B1".into()], + Rule::from(Regex::new(r"test").unwrap()), + ), + ], + ); + assert_eq!(category_of(&events), &serde_json::json!(vec!["A"])); +} + +#[test] +fn test_categorize_inter_level_priority() { + // depth * 10 leaves integers between levels: 15 beats a depth-1 default + // (10) but loses to a depth-2 default (20). + let between = categorize( + vec![event_with_data("just a test")], + &[ + CategoryRule::new(vec!["A".into()], Rule::from(Regex::new(r"test").unwrap())), + CategoryRule::new(vec!["A2".into()], Rule::from(Regex::new(r"test").unwrap())) + .with_priority(15), + ], + ); + assert_eq!(category_of(&between), &serde_json::json!(vec!["A2"])); + + let still_loses_to_deeper = categorize( + vec![event_with_data("just a test")], + &[ + CategoryRule::new(vec!["A2".into()], Rule::from(Regex::new(r"test").unwrap())) + .with_priority(15), + CategoryRule::new( + vec!["B".into(), "B1".into()], + Rule::from(Regex::new(r"test").unwrap()), + ), + ], + ); + assert_eq!( + category_of(&still_loses_to_deeper), + &serde_json::json!(vec!["B", "B1"]) + ); +} + +#[test] +fn test_categorize_lower_priority_loses_to_default_depth() { + // A deep rule can also be demoted below a shallow rule's default + // (depth * 10) by setting an explicit lower priority on the deep rule. + let events = categorize( + vec![event_with_data("just a test")], + &[ + CategoryRule::new(vec!["A".into()], Rule::from(Regex::new(r"test").unwrap())), + CategoryRule::new( + vec!["B".into(), "B1".into()], + Rule::from(Regex::new(r"test").unwrap()), + ) + .with_priority(0), + ], + ); + assert_eq!(category_of(&events), &serde_json::json!(vec!["A"])); +} + +#[test] +fn test_categorize_equal_priority_keeps_later_match() { + // Preserve the historical `>=` later-wins rule when ranks tie. + let events = categorize( + vec![event_with_data("just a test")], + &[ + CategoryRule::new( + vec!["First".into()], + Rule::from(Regex::new(r"test").unwrap()), + ) + .with_priority(5), + CategoryRule::new( + vec!["Second".into()], + Rule::from(Regex::new(r"test").unwrap()), + ) + .with_priority(5), + ], + ); + assert_eq!(category_of(&events), &serde_json::json!(vec!["Second"])); +} + +#[test] +fn test_categorize_negative_priority_still_beats_uncategorized() { + let events = categorize( + vec![event_with_data("just a test")], + &[ + CategoryRule::new(vec!["Low".into()], Rule::from(Regex::new(r"test").unwrap())) + .with_priority(-100), + ], + ); + assert_eq!(category_of(&events), &serde_json::json!(vec!["Low"])); +} + +#[test] +fn test_categorize_empty_category_keeps_uncategorized() { + // Empty path used to lose to Uncategorized (depth 0 < 1). The MIN-rank + // sentinel must not let it replace the fallback. + let events = categorize( + vec![event_with_data("just a test")], + &[CategoryRule::new( + vec![], + Rule::from(Regex::new(r"test").unwrap()), + )], + ); + assert_eq!( + category_of(&events), + &serde_json::json!(vec!["Uncategorized"]) + ); +} + +#[test] +fn test_categorize_cache_correctness() { + // Verifies that the deduplication cache produces the same result as + // per-event categorization when many events share the same data. + let mut base = Event::default(); + base.data.insert("app".into(), serde_json::json!("firefox")); + base.data + .insert("title".into(), serde_json::json!("GitHub")); + + let mut other = Event::default(); + other + .data + .insert("app".into(), serde_json::json!("terminal")); + other.data.insert("title".into(), serde_json::json!("bash")); + + // 50 events with same data, then 1 different event, then 50 more same + let mut events: Vec = std::iter::repeat(base.clone()) + .take(50) + .chain(std::iter::once(other.clone())) + .chain(std::iter::repeat(base.clone()).take(50)) + .collect(); + + let rules: Vec = vec![ + CategoryRule::new( + vec!["Browser".into()], + Rule::Regex(RegexRule::new("firefox", true, Some(vec!["app".into()])).unwrap()), + ), + CategoryRule::new( + vec!["Terminal".into()], + Rule::Regex(RegexRule::new("terminal", true, Some(vec!["app".into()])).unwrap()), + ), + ]; + + events = categorize(events, &rules); + + assert_eq!(events.len(), 101); + // All firefox events → Browser + for e in events.iter().take(50) { + assert_eq!( + e.data.get("$category").unwrap(), + &serde_json::json!(vec!["Browser"]) + ); + } + // The single terminal event → Terminal + assert_eq!( + events[50].data.get("$category").unwrap(), + &serde_json::json!(vec!["Terminal"]) + ); + // Remaining firefox events → Browser (cache hit path) + for e in events.iter().skip(51) { + assert_eq!( + e.data.get("$category").unwrap(), + &serde_json::json!(vec!["Browser"]) + ); + } +} + +#[test] +fn test_tag() { + let mut e = Event::default(); + e.data + .insert("test".into(), serde_json::json!("just a test")); + + let mut events = vec![e]; + let rules: Vec<(String, Rule)> = vec![ + ("test".into(), Rule::from(Regex::new(r"test").unwrap())), + ("test-2".into(), Rule::from(Regex::new(r"test").unwrap())), + ( + "nomatch".into(), + Rule::from(Regex::new(r"nomatch").unwrap()), + ), + ]; + events = tag(events, &rules); + + assert_eq!(events.len(), 1); + + let event = events.first().unwrap(); + let tags = event.data.get("$tags").unwrap(); + assert_eq!(tags, &serde_json::json!(vec!["test", "test-2"])); +} + +/// Verify that syntactically invalid regex patterns are rejected by RegexRule::new() +/// rather than panicking or silently succeeding. +/// +/// Note: fancy_regex supports possessive quantifiers (e.g. `++`, `**`) which standard +/// Python `re` does not. The original ActivityWatch#1340 bug (`Notepad++` → 500 error) +/// only affected the Python aw-server; in aw-server-rust `Notepad++` is a valid +/// possessive quantifier and is accepted. Users wanting a literal `+` must escape it: +/// `Notepad\+\+`. +#[test] +fn test_invalid_regex_patterns_are_rejected() { + let invalid_patterns = [ + "***", // no target for first `*` quantifier + "???", // no target for first `?` quantifier + "[unclosed", // unclosed character class + "(", // unclosed group + "(?P, key: &str, vals: &[Value]) -> Vec { + let mut filtered_events = Vec::new(); + for event in events.drain(..) { + if let Some(v) = event.data.get(key) { + for val in vals { + if val == v { + filtered_events.push(event); + break; + } + } + } + } + filtered_events +} + +/// Drops events not matching the regex on the value for a specified key +/// Will only match if the value is a string +/// +/// # Example +/// ```ignore +/// key: a +/// regex: "[A-Z]+" +/// input: [a:"HELLO"][a:"hello"][a:3][b:"HELLO"] +/// output: [a:"HELLO"] +/// ``` +pub fn filter_keyvals_regex(mut events: Vec, key: &str, regex: &Regex) -> Vec { + let mut filtered_events = Vec::new(); + + for event in events.drain(..) { + if let Some(value) = event.data.get(key).and_then(|v| v.as_str()) { + match regex.is_match(value) { + Ok(true) => filtered_events.push(event), + Ok(false) => (), + Err(err) => warn!("Failed to run regex: {}", err), + }; + } + } + filtered_events +} + +/// Drops events matching the specified key and value(s). Opposite of filter_keyvals. +/// +/// # Example +/// ```ignore +/// key: a +/// vals: [1,2] +/// input: [a:1][a:2][a:3][b:4] +/// output: [a:3][b:4] +/// ``` +pub fn exclude_keyvals(mut events: Vec, key: &str, vals: &[Value]) -> Vec { + let mut filtered_events = Vec::new(); + 'events: for event in events.drain(..) { + if let Some(v) = event.data.get(key) { + for val in vals { + if val == v { + continue 'events; + } + } + } + filtered_events.push(event); + } + filtered_events +} + +#[cfg(test)] +mod tests { + use std::str::FromStr; + + use chrono::{DateTime, Duration}; + use fancy_regex::RegexBuilder; + use serde_json::json; + + use aw_models::Event; + + use super::{exclude_keyvals, filter_keyvals, filter_keyvals_regex}; + + #[test] + fn test_filter_keyvals() { + let e1 = Event { + id: None, + timestamp: DateTime::from_str("2000-01-01T00:00:00Z").unwrap(), + duration: Duration::seconds(1), + data: json_map! {"test": json!(1)}, + }; + let mut e2 = e1.clone(); + e2.data = json_map! {"test": json!(1), "test2": json!(1)}; + let mut e3 = e1.clone(); + e3.data = json_map! {"test2": json!(2)}; + let res = filter_keyvals(vec![e1.clone(), e2.clone(), e3], "test", &[json!(1)]); + assert_eq!(vec![e1, e2], res); + } + + #[test] + fn test_filter_keyvals_regex() { + let e1 = Event { + id: None, + timestamp: DateTime::from_str("2000-01-01T00:00:00Z").unwrap(), + duration: Duration::seconds(1), + data: json_map! {"key1": json!("value1")}, + }; + let mut e2 = e1.clone(); + e2.data = json_map! {"key1": json!("value2")}; + let mut e3 = e1.clone(); + e3.data = json_map! {"key2": json!("value3")}; + + let events = vec![e1.clone(), e2.clone(), e3.clone()]; + + let regex_value = RegexBuilder::new("value").build().unwrap(); + let regex_value1 = RegexBuilder::new("value1").build().unwrap(); + + let res = filter_keyvals_regex(events.clone(), "key1", ®ex_value); + assert_eq!(vec![e1.clone(), e2], res); + let res = filter_keyvals_regex(events.clone(), "key1", ®ex_value1); + assert_eq!(vec![e1], res); + let res = filter_keyvals_regex(events.clone(), "key2", ®ex_value); + assert_eq!(vec![e3], res); + let res = filter_keyvals_regex(events.clone(), "key2", ®ex_value1); + assert_eq!(0, res.len()); + let res = filter_keyvals_regex(events, "key3", ®ex_value); + assert_eq!(0, res.len()); + } + + #[test] + fn test_filter_keyvals_regex_non_string_data_value() { + let e1 = Event { + id: None, + timestamp: DateTime::from_str("2000-01-01T00:00:00Z").unwrap(), + duration: Duration::seconds(1), + data: json_map! {"key1": json!(100)}, + }; + let events = vec![e1.clone()]; + let regex_value = RegexBuilder::new("value").build().unwrap(); + let res = filter_keyvals_regex(events.clone(), "key1", ®ex_value); + assert!(res.is_empty()); + } + + #[test] + fn test_exclude_keyvals() { + let e1 = Event { + id: None, + timestamp: DateTime::from_str("2000-01-01T00:00:00Z").unwrap(), + duration: Duration::seconds(1), + data: json_map! {"test": json!(1)}, + }; + let mut e2 = e1.clone(); + e2.data = json_map! {"test": json!(1), "test2": json!(2)}; + let mut e3 = e1.clone(); + e3.data = json_map! {"test": json!(2)}; + let res = exclude_keyvals(vec![e1.clone(), e2.clone(), e3], "test", &[json!(2)]); + assert_eq!(vec![e1, e2], res); + } +} diff --git a/aw-transform/src/filter_period.rs b/aw-transform/src/filter_period.rs new file mode 100644 index 0000000..b69f656 --- /dev/null +++ b/aw-transform/src/filter_period.rs @@ -0,0 +1,158 @@ +use aw_models::Event; +use chrono::Duration; + +use crate::sort_by_timestamp; + +/// Removes events not intersecting with the provided filter_events +/// +/// Usually used to filter buckets unaware if the user is making any activity with an bucket which +/// is aware if the user is at the computer or not. +/// For example the events from aw-watcher-window should be called with filter_period_intersect +/// with the "not-afk" events from aw-watcher-afk to give events with durations of only when the +/// user is at the computer. +/// +/// # Example +/// ```ignore +/// events: [a ][b ] +/// filter_events: [ ] [ ] +/// output: [a ] [a ][b ] +/// ``` +pub fn filter_period_intersect(events: Vec, filter_events: Vec) -> Vec { + if events.is_empty() || filter_events.is_empty() { + return Vec::new(); + } + + let mut filtered_events = Vec::new(); + let events = sort_by_timestamp(events); + let filter_events = sort_by_timestamp(filter_events); + + let mut events_iter = events.into_iter(); + let mut filter_events_iter = filter_events.into_iter(); + let mut cur_event = events_iter.next().unwrap(); + let mut cur_filter_event = filter_events_iter.next().unwrap(); + + loop { + let event_endtime = cur_event.calculate_endtime(); + let filter_endtime = cur_filter_event.calculate_endtime(); + if cur_event.duration == Duration::seconds(0) || event_endtime <= cur_filter_event.timestamp + { + match events_iter.next() { + Some(e) => { + cur_event = e; + continue; + } + None => return filtered_events, + } + } + if cur_event.timestamp >= cur_filter_event.calculate_endtime() { + match filter_events_iter.next() { + Some(e) => { + cur_filter_event = e; + continue; + } + None => return filtered_events, + } + } + + let mut e = cur_event.clone(); + e.timestamp = std::cmp::max(e.timestamp, cur_filter_event.timestamp); + let endtime = std::cmp::min(event_endtime, filter_endtime); + e.duration = endtime - e.timestamp; + + // trim current event + let old_timestamp = cur_event.timestamp; + cur_event.timestamp = e.timestamp + e.duration; + cur_event.duration = old_timestamp + cur_event.duration - cur_event.timestamp; + + filtered_events.push(e); + } +} + +#[cfg(test)] +mod tests { + use std::str::FromStr; + + use chrono::DateTime; + use chrono::Duration; + use chrono::Utc; + use serde_json::json; + + use aw_models::Event; + + use super::filter_period_intersect; + + #[test] + fn test_filter_period_intersect() { + let e1 = Event { + id: None, + timestamp: DateTime::from_str("2000-01-01T00:00:01Z").unwrap(), + duration: Duration::seconds(1), + data: json_map! {"test": json!(1)}, + }; + let mut e2 = e1.clone(); + e2.timestamp = DateTime::from_str("2000-01-01T00:00:02Z").unwrap(); + let mut e3 = e1.clone(); + e3.timestamp = DateTime::from_str("2000-01-01T00:00:03Z").unwrap(); + let mut e4 = e1.clone(); + e4.timestamp = DateTime::from_str("2000-01-01T00:00:04Z").unwrap(); + let mut e5 = e1.clone(); + e5.timestamp = DateTime::from_str("2000-01-01T00:00:05Z").unwrap(); + + let filter_event = Event { + id: None, + timestamp: DateTime::from_str("2000-01-01T00:00:02.5Z").unwrap(), + duration: Duration::seconds(2), + data: json_map! {"test": json!(1)}, + }; + + let filtered_events = + filter_period_intersect(vec![e1, e2, e3, e4, e5], vec![filter_event.clone()]); + assert_eq!(filtered_events.len(), 3); + assert_eq!(filtered_events[0].duration, Duration::milliseconds(500)); + assert_eq!(filtered_events[1].duration, Duration::milliseconds(1000)); + assert_eq!(filtered_events[2].duration, Duration::milliseconds(500)); + + let dt: DateTime = DateTime::from_str("2000-01-01T00:00:02.500Z").unwrap(); + assert_eq!(filtered_events[0].timestamp, dt); + let dt: DateTime = DateTime::from_str("2000-01-01T00:00:03.000Z").unwrap(); + assert_eq!(filtered_events[1].timestamp, dt); + let dt: DateTime = DateTime::from_str("2000-01-01T00:00:04.000Z").unwrap(); + assert_eq!(filtered_events[2].timestamp, dt); + + let timestamp_01s = DateTime::from_str("2000-01-01T00:00:01Z").unwrap(); + let e = Event { + id: None, + timestamp: timestamp_01s, + duration: Duration::seconds(1), + data: json_map! {"test": json!(1)}, + }; + let mut f2 = filter_event.clone(); + f2.timestamp = DateTime::from_str("2000-01-01T00:00:00Z").unwrap(); + f2.duration = Duration::milliseconds(1500); + let res = filter_period_intersect(vec![e.clone()], vec![f2]); + assert_eq!(res[0].timestamp, timestamp_01s); + assert_eq!(res[0].duration, Duration::milliseconds(500)); + + let timestamp_01_5s = DateTime::from_str("2000-01-01T00:00:01.5Z").unwrap(); + let mut f3 = filter_event.clone(); + f3.timestamp = timestamp_01_5s; + f3.duration = Duration::milliseconds(1000); + let res = filter_period_intersect(vec![e.clone()], vec![f3]); + assert_eq!(res[0].timestamp, timestamp_01_5s); + assert_eq!(res[0].duration, Duration::milliseconds(500)); + + let mut f4 = filter_event.clone(); + f4.timestamp = DateTime::from_str("2000-01-01T00:00:01.5Z").unwrap(); + f4.duration = Duration::milliseconds(100); + let res = filter_period_intersect(vec![e.clone()], vec![f4]); + assert_eq!(res[0].timestamp, timestamp_01_5s); + assert_eq!(res[0].duration, Duration::milliseconds(100)); + + let mut f5 = filter_event.clone(); + f5.timestamp = DateTime::from_str("2000-01-01T00:00:00Z").unwrap(); + f5.duration = Duration::seconds(10); + let res = filter_period_intersect(vec![e.clone()], vec![f5]); + assert_eq!(res[0].timestamp, timestamp_01s); + assert_eq!(res[0].duration, Duration::milliseconds(1000)); + } +} diff --git a/aw-transform/src/find_bucket.rs b/aw-transform/src/find_bucket.rs new file mode 100644 index 0000000..a86941d --- /dev/null +++ b/aw-transform/src/find_bucket.rs @@ -0,0 +1,78 @@ +use aw_models::Bucket; + +/// Finds the first bucket which starts with the specified string, and optionally matches a +/// hostname. +pub fn find_bucket<'a>( + bucket_filter: &str, + hostname_filter: &Option, + buckets: impl IntoIterator, +) -> Option { + let mut prefix_match = None; + for bucket in buckets { + if bucket.id.starts_with(bucket_filter) { + let hostname_matches = match hostname_filter { + Some(hostname) => hostname == &bucket.hostname, + None => true, + }; + + if hostname_matches { + if bucket.id == bucket_filter { + // Exact match, return immediately + return Some(bucket.id.to_string()); + } + if prefix_match.is_none() { + // Save the first prefix match in case we don't find an exact match + prefix_match = Some(bucket.id.to_string()); + } + } + } + } + prefix_match +} + +#[cfg(test)] +mod tests { + use super::find_bucket; + use aw_models::Bucket; + use aw_models::BucketMetadata; + + #[test] + fn test_find_bucket() { + let expected_bucketname = "aw-datastore-test_test-host".to_string(); + let expected_hostname = "testhost".to_string(); + let b1 = Bucket { + bid: None, + id: "no match".to_string(), + _type: "type".to_string(), + hostname: expected_hostname, + client: "testclient".to_string(), + created: None, + data: json_map! {}, + metadata: BucketMetadata::default(), + events: None, + last_updated: None, + }; + let mut b2 = b1.clone(); + b2.id = expected_bucketname.clone(); + let b3 = b1.clone(); + + let buckets = vec![b1.clone(), b2, b3.clone()]; + + // Check that it correctly finds bucket + let res = find_bucket("aw-datastore-test", &Some("testhost".to_string()), &buckets); + assert_eq!(res, Some(expected_bucketname)); + + // Check that it doesn't find a bucket for an unavailable hostname + let res = find_bucket( + "aw-datastore-test", + &Some("unavailablehost".to_string()), + &buckets, + ); + assert_eq!(res, None); + + // Check that it doesn't find a bucket for any hostname + let buckets = vec![b1, b3]; + let res = find_bucket("aw-datastore-test", &None, &buckets); + assert_eq!(res, None); + } +} diff --git a/aw-transform/src/flood.rs b/aw-transform/src/flood.rs new file mode 100644 index 0000000..7d6e7ba --- /dev/null +++ b/aw-transform/src/flood.rs @@ -0,0 +1,389 @@ +use aw_models::Event; + +use crate::sort_by_timestamp; + +/// Floods event to the nearest neighbouring event if within the specified pulsetime. +/// +/// Also merges events if they have the same data and are within the pulsetime. +/// +/// "Flooding" actually performs two similar, but slightly different tasks, which we can describe as: +/// - Flooding: removes small gaps between events, by extending nearby events to fill the space. +/// - Merging: merging adjacent or overlapping events that have the same data. +/// - This is sometimes done in conjunction with flooding, if events with nearby events have the same data. +/// +/// Python implementation: +/// https://github.com/ActivityWatch/aw-core/blob/master/aw_transform/flood.py +/// Python re-implementation using generators (different spec): +/// https://github.com/ErikBjare/copilot-testing/blob/master/playground/flooding.py +/// +/// # Example +/// +/// Example with forward-fill: +/// +/// ```ignore +/// pulsetime: 1 second (one space) +/// input: [a] [b] [c] +/// output: [a ][b ][c] +/// ``` +/// +/// Example with forward-fill and event merging: +/// +/// ```ignore +/// pulsetime: 1 second (one space) +/// input: [a] [a] [b ][b] +/// output: [a ] [b ] +/// ``` +pub fn flood(events: Vec, pulsetime: chrono::Duration) -> Vec { + let mut new_events = Vec::new(); + let mut events_sorted = sort_by_timestamp(events); + let mut e1_iter = events_sorted.drain(..).peekable(); + + let mut gap_prev: Option = None; + let mut retry_e: Option = None; + + // If negative gaps are smaller than this, prune them to become zero + let negative_gap_trim_thres = chrono::Duration::milliseconds(100); + + let mut warned_negative_gap_safe = false; + let mut warned_negative_gap_unsafe = false; + + while let Some(mut e1) = match retry_e { + Some(e) => { + retry_e = None; + Some(e) + } + None => e1_iter.next(), + } { + if let Some(gap) = gap_prev { + e1.timestamp -= gap / 2; + e1.duration += gap / 2; + gap_prev = None; + } + let e2 = match e1_iter.peek() { + Some(e) => e, + None => { + new_events.push(e1); + break; + } + }; + + // Ensure that ordering is preserved, at least in tests + debug_assert!(e1.timestamp <= e2.timestamp); + + let gap = e2.timestamp - e1.calculate_endtime(); + + // We split the program flow into 2 parts: positive and negative gaps + // First we check negative gaps (if events overlap) + if gap < chrono::Duration::seconds(0) { + // Python implementation: + // + // if gap < timedelta(0) and e1.data == e2.data: + // start = min(e1.timestamp, e2.timestamp) + // end = max(e1.timestamp + e1.duration, e2.timestamp + e2.duration) + // e1.timestamp, e1.duration = start, (end - start) + // e2.timestamp, e2.duration = end, timedelta(0) + // if not warned_about_negative_gap_safe: + // logger.warning( + // "Gap was of negative duration but could be safely merged ({}s). This message will only show once per batch.".format( + // gap.total_seconds() + // ) + // ) + // warned_about_negative_gap_safe = True + + // If data is same, we can safely merge them. + if e1.data == e2.data { + // Gap was negative and could be safely merged + if !warned_negative_gap_safe { + warn!("Gap was of negative duration ({}s), but could be safely merged. This error will only show once per batch.", gap); + warned_negative_gap_safe = true; + } + let start = std::cmp::min(e1.timestamp, e2.timestamp); + let end = std::cmp::max(e1.calculate_endtime(), e2.calculate_endtime()); // e2 isn't guaranteed to end last + e1.timestamp = start; + e1.duration = end - start; + // Drop next event since it is merged/flooded into e1 + e1_iter.next(); + // Retry this event again to give it a change to merge e1 + // with 'e3' + retry_e = Some(e1); + continue; + // If data differs and gap is negative beyond the trim thres, throw a warning + } else if gap < -negative_gap_trim_thres && !warned_negative_gap_unsafe { + // Events with negative gap but differing data cannot be merged safely + warn!("Gap was of negative duration and could NOT be safely merged ({}s). This warning will only show once per batch.", gap); + warned_negative_gap_unsafe = true; + } + + // By now, we've ensured gap is non-negative, now we want to know if the gap is smaller + // than pulsetime, in which case we should fill it. + } else if gap < pulsetime { + // Python implementation: + // + // elif gap < -negative_gap_trim_thres and not warned_about_negative_gap_unsafe: + // # Events with negative gap but differing data cannot be merged safely + // logger.warning( + // "Gap was of negative duration and could NOT be safely merged ({}s). This warning will only show once per batch.".format( + // gap.total_seconds() + // ) + // ) + // warned_about_negative_gap_unsafe = True + + // Ensure that gap is actually non-negative here, at least in tests + debug_assert!(gap >= chrono::Duration::seconds(0)); + + // If data is the same, we should merge them. + if e1.data == e2.data { + // Choose the longest event and set the endtime to it + let start = std::cmp::min(e1.timestamp, e2.timestamp); // isn't e1 guaranteed to start? + let end = std::cmp::max(e1.calculate_endtime(), e2.calculate_endtime()); // e2 isn't guaranteed to end last + e1.timestamp = start; + e1.duration = end - start; + // Drop next event since it is merged/flooded into e1 + e1_iter.next(); + // Retry this event again to give it a change to merge e1 + // with 'e3' + retry_e = Some(e1); + // Since we are retrying on this event we don't want to push it + // to the new_events vec + continue; + } else { + // Extend e1 to middle of the gap. + e1.duration += gap / 2; + + // Make sure next event (e2) is gets extended before it's processed + gap_prev = Some(gap); + } + } + // else: nothing to do, events not near each other + + new_events.push(e1); + } + new_events +} + +#[cfg(test)] +mod tests { + use std::str::FromStr; + + use chrono::DateTime; + use chrono::Duration; + use serde_json::json; + + use aw_models::Event; + + use super::flood; + + #[test] + fn test_flood_merge() { + // Test merging of events with the same data + let e1 = Event { + id: None, + timestamp: DateTime::from_str("2000-01-01T00:00:00Z").unwrap(), + duration: Duration::seconds(1), + data: json_map! {"test": json!(1)}, + }; + let e2 = Event { + id: None, + timestamp: DateTime::from_str("2000-01-01T00:00:03Z").unwrap(), + duration: Duration::seconds(1), + data: json_map! {"test": json!(1)}, + }; + let e_expected = Event { + id: None, + timestamp: DateTime::from_str("2000-01-01T00:00:00Z").unwrap(), + duration: Duration::seconds(4), + data: json_map! {"test": json!(1)}, + }; + let res = flood(vec![e1, e2], Duration::seconds(5)); + assert_eq!(1, res.len()); + assert_eq!(&res[0], &e_expected); + } + + #[test] + fn test_flood_meet_in_middle() { + // Test flood gap between two different events which should meet in the middle + let e1 = Event { + id: None, + timestamp: DateTime::from_str("2000-01-01T00:00:00Z").unwrap(), + duration: Duration::seconds(1), + data: json_map! {"test": json!(1)}, + }; + let e2 = Event { + id: None, + timestamp: DateTime::from_str("2000-01-01T00:00:03Z").unwrap(), + duration: Duration::seconds(1), + data: json_map! {"test": json!(2)}, + }; + let e1_expected = Event { + id: None, + timestamp: DateTime::from_str("2000-01-01T00:00:00Z").unwrap(), + duration: Duration::seconds(2), + data: json_map! {"test": json!(1)}, + }; + let e2_expected = Event { + id: None, + timestamp: DateTime::from_str("2000-01-01T00:00:02Z").unwrap(), + duration: Duration::seconds(2), + data: json_map! {"test": json!(2)}, + }; + let res = flood(vec![e1, e2], Duration::seconds(5)); + assert_eq!(2, res.len()); + assert_eq!(&res[0], &e1_expected); + assert_eq!(&res[1], &e2_expected); + } + + #[test] + fn test_flood_partial_overlap() { + // Tests flooding an identical event contained within another event + let e1 = Event { + id: None, + timestamp: DateTime::from_str("2000-01-01T00:00:00Z").unwrap(), + duration: Duration::seconds(10), + data: json_map! {"type": "a"}, + }; + let e2 = Event { + id: None, + timestamp: DateTime::from_str("2000-01-01T00:00:05Z").unwrap(), + duration: Duration::seconds(10), + data: json_map! {"type": "a"}, + }; + let e1_expected = Event { + id: None, + timestamp: DateTime::from_str("2000-01-01T00:00:00Z").unwrap(), + duration: Duration::seconds(15), + data: json_map! {"type": "a"}, + }; + let res = flood(vec![e1, e2], Duration::seconds(5)); + assert_eq!(1, res.len()); + assert_eq!(&res[0], &e1_expected); + } + + #[test] + fn test_flood_containing() { + // Tests flooding an identical event contained within another event + let e1 = Event { + id: None, + timestamp: DateTime::from_str("2000-01-01T00:00:00Z").unwrap(), + duration: Duration::seconds(10), + data: json_map! {"type": "a"}, + }; + let e2 = Event { + id: None, + timestamp: DateTime::from_str("2000-01-01T00:00:01Z").unwrap(), + duration: Duration::seconds(5), + data: json_map! {"type": "a"}, + }; + let res = flood(vec![e1.clone(), e2], Duration::seconds(5)); + assert_eq!(1, res.len()); + assert_eq!(&res[0], &e1); + } + + #[test] + fn test_flood_containing_diff() { + // Tests flooding an event with different data contained within another event + // Events should pass unmodified. + let e1 = Event { + id: None, + timestamp: DateTime::from_str("2000-01-01T00:00:00Z").unwrap(), + duration: Duration::seconds(10), + data: json_map! {"type": "a"}, + }; + let e2 = Event { + id: None, + timestamp: DateTime::from_str("2000-01-01T00:00:01Z").unwrap(), + duration: Duration::seconds(5), + data: json_map! {"type": "b"}, + }; + let res = flood(vec![e1.clone(), e2.clone()], Duration::seconds(5)); + assert_eq!(2, res.len()); + assert_eq!(&res[0], &e1); + assert_eq!(&res[1], &e2); + } + + #[test] + fn test_flood_same_timestamp() { + // e1, stay same + // e2, base merge (longest duration, this should be the duration selected) + // e3, merge with e2 + // e4, stay same + let e1 = Event { + id: None, + timestamp: DateTime::from_str("2000-01-01T00:00:00Z").unwrap(), + duration: Duration::seconds(1), + data: json_map! {"status": "afk"}, + }; + let e2 = Event { + id: None, + timestamp: DateTime::from_str("2000-01-01T00:00:01Z").unwrap(), + duration: Duration::seconds(5), + data: json_map! {"status": "not-afk"}, + }; + let e3 = Event { + id: None, + timestamp: DateTime::from_str("2000-01-01T00:00:01Z").unwrap(), + duration: Duration::seconds(1), + data: json_map! {"status": "not-afk"}, + }; + let e4 = Event { + id: None, + timestamp: DateTime::from_str("2000-01-01T00:00:06Z").unwrap(), + duration: Duration::seconds(1), + data: json_map! {"status": "afk"}, + }; + let res = flood( + vec![e1.clone(), e2.clone(), e3, e4.clone()], + Duration::seconds(5), + ); + assert_eq!(3, res.len()); + assert_eq!(&res[0], &e1); + assert_eq!(&res[1], &e2); + assert_eq!(&res[2], &e4); + } + + #[test] + fn test_flood_same_timestamp_duplicates() { + // e1, stay same + // e2, base merge + // e3, merge with e2 + // e4, merge with e2 (longest duration, this should be the duration selected) + // e5, stay same + let e1 = Event { + id: None, + timestamp: DateTime::from_str("2000-01-01T00:00:00Z").unwrap(), + duration: Duration::seconds(1), + data: json_map! {"status": "afk"}, + }; + let e2 = Event { + id: None, + timestamp: DateTime::from_str("2000-01-01T00:00:01Z").unwrap(), + duration: Duration::seconds(5), + data: json_map! {"status": "not-afk"}, + }; + let e3 = Event { + id: None, + timestamp: DateTime::from_str("2000-01-01T00:00:01Z").unwrap(), + duration: Duration::seconds(1), + data: json_map! {"status": "not-afk"}, + }; + let e4 = Event { + id: None, + timestamp: DateTime::from_str("2000-01-01T00:00:01Z").unwrap(), + duration: Duration::seconds(10), + data: json_map! {"status": "not-afk"}, + }; + let e5 = Event { + id: None, + timestamp: DateTime::from_str("2000-01-01T00:00:11Z").unwrap(), + duration: Duration::seconds(1), + data: json_map! {"status": "afk"}, + }; + let res = flood( + vec![e1.clone(), e2, e3, e4.clone(), e5.clone()], + Duration::seconds(5), + ); + assert_eq!(3, res.len()); + assert_eq!(&res[0], &e1); + assert_eq!(&res[1], &e4); + assert_eq!(&res[2], &e5); + } +} diff --git a/aw-transform/src/heartbeat.rs b/aw-transform/src/heartbeat.rs new file mode 100644 index 0000000..c517383 --- /dev/null +++ b/aw-transform/src/heartbeat.rs @@ -0,0 +1,183 @@ +use aw_models::Event; + +/// Returns a merged event if two events have the same data and are within the pulsetime +/// +/// # Example +/// +/// ```ignore +/// pulsetime: 1 second (one space) +/// input: [a] [a] [a][b] +/// output: [a ] [a][b] +/// ``` +pub fn heartbeat(last_event: &Event, heartbeat: &Event, pulsetime: f64) -> Option { + // Verify that data is the same + if heartbeat.data != last_event.data { + debug!("Can't merge, data is different"); + return None; + } + + let last_event_endtime = last_event.calculate_endtime(); + let heartbeat_endtime = heartbeat.calculate_endtime(); + + // Verify that timestamps intersect (including pulsetime) + let pulsetime_ns: i64 = (pulsetime * 1_000_000_000.0).round() as i64; + let last_endtime_allowed = last_event_endtime + chrono::Duration::nanoseconds(pulsetime_ns); + if last_event.timestamp > heartbeat.timestamp { + debug!("Can't merge, last event timestamp is after heartbeat timestamp"); + return None; + } + if heartbeat.timestamp > last_endtime_allowed { + debug!("Can't merge, heartbeat timestamp is after last event endtime"); + return None; + } + + let starttime = if heartbeat.timestamp < last_event.timestamp { + &heartbeat.timestamp + } else { + &last_event.timestamp + }; + + let endtime = if last_event_endtime > heartbeat_endtime { + &last_event_endtime + } else { + &heartbeat_endtime + }; + + let duration = endtime.signed_duration_since(*starttime); + if duration.num_nanoseconds().unwrap() < 0 { + debug!("Merging heartbeats would result in a negative duration, refusing to merge!"); + return None; + } + + // Success, return successful heartbeat last_event + Some(Event { + id: None, + timestamp: *starttime, + duration, + data: last_event.data.clone(), + }) +} + +#[cfg(test)] +mod tests { + use chrono::Duration; + use chrono::Utc; + use serde_json::json; + + use aw_models::Event; + + use super::heartbeat; + + #[test] + fn test_heartbeat_pulsetime() { + let now = Utc::now(); + let event1 = Event { + id: None, + timestamp: now, + duration: Duration::seconds(1), + data: json_map! {"test": json!(1)}, + }; + let heartbeat1 = Event { + id: None, + timestamp: now + Duration::seconds(2), + duration: Duration::seconds(1), + data: json_map! {"test": json!(1)}, + }; + + // Merge result + let res_merge = heartbeat(&event1, &heartbeat1, 2.0).unwrap(); + assert_eq!(res_merge.timestamp, now); + assert_eq!(res_merge.duration, Duration::seconds(3)); + assert_eq!(res_merge.data, event1.data); + + // No merge result + let res_no_merge = heartbeat(&event1, &heartbeat1, 0.0); + assert!(res_no_merge.is_none()); + + // TODO: needs more tests! + } + + #[test] + fn test_heartbeat_long_pulse_merge() { + let now = Utc::now(); + let event = Event { + id: None, + timestamp: now, + duration: Duration::seconds(1), + data: json_map! {"test": json!(1)}, + }; + let long_pulse_event = Event { + id: None, + // note that no duration is sent, which is how aw-client works + duration: Duration::seconds(0), + timestamp: now + Duration::seconds(120), + data: json_map! {"test": json!(1)}, + }; + + // Merge result + let res_merge = heartbeat(&event, &long_pulse_event, 120.0).unwrap(); + assert_eq!(res_merge.timestamp, now); + assert_eq!(res_merge.data, event.data); + assert_eq!(res_merge.duration, Duration::seconds(120)); + + // No merge result when pulsetime is less than the timestamp delta between heartbeats + let res_no_merge = heartbeat(&event, &long_pulse_event, 60.0); + assert!(res_no_merge.is_none()); + } + + #[test] + fn test_heartbeat_data() { + let now = Utc::now(); + let event = Event { + id: None, + timestamp: now, + duration: Duration::seconds(0), + data: json_map! {"test": json!(1)}, + }; + let heartbeat_same_data = Event { + id: None, + timestamp: now, + duration: Duration::seconds(1), + data: json_map! {"test": json!(1)}, + }; + + // Data is same, should merge + let res_merge = heartbeat(&event, &heartbeat_same_data, 1.0); + assert!(res_merge.is_some()); + + let heartbeat_different_data = Event { + id: None, + timestamp: now, + duration: Duration::seconds(1), + data: json_map! {"test": json!(2)}, + }; + // Data is different, should not merge + let res_merge = heartbeat(&event, &heartbeat_different_data, 1.0); + assert!(res_merge.is_none()); + } + + #[test] + fn test_heartbeat_same_timestamp() { + let now = Utc::now(); + let event = Event { + id: None, + timestamp: now, + duration: Duration::seconds(0), + data: json_map! {"test": json!(1)}, + }; + let heartbeat_same_data = Event { + id: None, + timestamp: now, + duration: Duration::seconds(1), + data: json_map! {"test": json!(1)}, + }; + + // Should merge + let res_merge = heartbeat(&event, &heartbeat_same_data, 1.0).unwrap(); + assert_eq!(Duration::seconds(1), res_merge.duration); + + // Order shouldn't matter, should merge anyway + let res_merge = heartbeat(&heartbeat_same_data, &event, 1.0).unwrap(); + assert_eq!(Duration::seconds(1), res_merge.duration); + } +} diff --git a/aw-transform/src/lib.rs b/aw-transform/src/lib.rs new file mode 100644 index 0000000..d6447c6 --- /dev/null +++ b/aw-transform/src/lib.rs @@ -0,0 +1,52 @@ +#[macro_use] +extern crate log; + +// TODO: Move this to some more suitable place +#[macro_export] +macro_rules! json_map { + { $( $key:literal : $value:expr),* } => {{ + use serde_json::Value; + use serde_json::map::Map; + #[allow(unused_mut)] + let mut map : Map = Map::new(); + $( + map.insert( $key.to_string(), json!($value) ); + )* + map + }}; +} + +pub mod classify; + +mod heartbeat; +pub use heartbeat::heartbeat; + +mod find_bucket; +pub use find_bucket::find_bucket; + +mod flood; +pub use flood::flood; + +mod merge; +pub use merge::merge_events_by_keys; + +mod chunk; +pub use chunk::chunk_events_by_key; + +mod sort; +pub use sort::{sort_by_duration, sort_by_timestamp}; + +mod filter_keyvals; +pub use filter_keyvals::{exclude_keyvals, filter_keyvals, filter_keyvals_regex}; + +mod filter_period; +pub use filter_period::filter_period_intersect; + +mod split_url; +pub use split_url::split_url_event; + +mod period_union; +pub use period_union::period_union; + +mod union_no_overlap; +pub use union_no_overlap::union_no_overlap; diff --git a/aw-transform/src/merge.rs b/aw-transform/src/merge.rs new file mode 100644 index 0000000..f2034dd --- /dev/null +++ b/aw-transform/src/merge.rs @@ -0,0 +1,141 @@ +use std::collections::HashMap; + +use aw_models::Event; + +/// Merge events with the same values at the specified keys +/// +/// Doesn't care about if events are neighbouring or not, this transform merges +/// all events with the same key. +/// The timestamp will be the timestamp of the first event with a specific key value +/// +/// # Example 1 +/// A simple example only using one key +/// +/// ```ignore +/// keys: ["a"] +/// input: +/// { duration: 1.0, data: { "a": 1 } } +/// { duration: 1.0, data: { "a": 1 } } +/// { duration: 1.0, data: { "a": 2 } } +/// { duration: 1.0, data: { "b": 1 } } +/// { duration: 1.0, data: { "a": 1 } } +/// output: +/// { duration: 3.0, data: { "a": 1 } } +/// { duration: 1.0, data: { "a": 2 } } +/// { duration: 1.0, data: { "b": 1 } } +/// ``` +/// +/// # Example 2 +/// A more complex example only using two keys +/// ```ignore +/// keys: ["a", "b"] +/// input: +/// { duration: 1.0, data: { "a": 1, "b": 1 } } +/// { duration: 1.0, data: { "a": 2, "b": 2 } } +/// { duration: 1.0, data: { "a": 1, "b": 1 } } +/// { duration: 1.0, data: { "a": 1, "b": 2 } } +/// output: +/// { duration: 2.0, data: { "a": 1, "b": 1 } } +/// { duration: 1.0, data: { "a": 2, "b": 2 } } +/// { duration: 1.0, data: { "a": 1, "b": 2 } } +/// ``` +#[allow(clippy::map_entry)] +pub fn merge_events_by_keys(events: Vec, keys: Vec) -> Vec { + if keys.is_empty() { + return vec![]; + } + let mut merged_events_map: HashMap = HashMap::new(); + 'event: for event in events { + let mut key_values = Vec::new(); + for key in &keys { + match event.data.get(key) { + Some(v) => key_values.push(v.to_string()), + None => continue 'event, + } + } + let summed_key = key_values.join("."); + if merged_events_map.contains_key(&summed_key) { + let merged_event = merged_events_map.get_mut(&summed_key).unwrap(); + merged_event.duration += event.duration; + } else { + let mut data = HashMap::new(); + for key in &keys { + data.insert(key.clone(), event.data.get(key).unwrap()); + } + let merged_event = Event { + id: None, + timestamp: event.timestamp, + duration: event.duration, + data: event.data.clone(), + }; + merged_events_map.insert(summed_key, merged_event); + } + } + let mut merged_events_list = Vec::new(); + for (_key, event) in merged_events_map.drain() { + merged_events_list.push(event); + } + merged_events_list +} + +#[cfg(test)] +mod tests { + use std::str::FromStr; + + use chrono::DateTime; + use chrono::Duration; + use serde_json::json; + + use aw_models::Event; + + use crate::sort_by_timestamp; + + use super::merge_events_by_keys; + + #[test] + fn test_merge_events_by_key() { + let e1 = Event { + id: None, + timestamp: DateTime::from_str("2000-01-01T00:00:00Z").unwrap(), + duration: Duration::seconds(1), + data: json_map! {"test": json!(1)}, + }; + let e2 = Event { + id: None, + timestamp: DateTime::from_str("2000-01-01T00:00:01Z").unwrap(), + duration: Duration::seconds(3), + data: json_map! {"test2": json!(3)}, + }; + let e3 = Event { + id: None, + timestamp: DateTime::from_str("2000-01-01T00:00:02Z").unwrap(), + duration: Duration::seconds(7), + data: json_map! {"test": json!(6)}, + }; + let e4 = Event { + id: None, + timestamp: DateTime::from_str("2000-01-01T00:00:03Z").unwrap(), + duration: Duration::seconds(9), + data: json_map! {"test": json!(1)}, + }; + let in_events = vec![e1, e2, e3, e4]; + let res1 = merge_events_by_keys(in_events, vec!["test".to_string()]); + // Needed, otherwise the order is undeterministic + let res2 = sort_by_timestamp(res1); + let expected = vec![ + Event { + id: None, + timestamp: DateTime::from_str("2000-01-01T00:00:00Z").unwrap(), + duration: Duration::seconds(10), + data: json_map! {"test": json!(1)}, + }, + Event { + id: None, + timestamp: DateTime::from_str("2000-01-01T00:00:02Z").unwrap(), + duration: Duration::seconds(7), + data: json_map! {"test": json!(6)}, + }, + ]; + assert_eq!(&res2, &expected); + } +} diff --git a/aw-transform/src/period_union.rs b/aw-transform/src/period_union.rs new file mode 100644 index 0000000..28cdda6 --- /dev/null +++ b/aw-transform/src/period_union.rs @@ -0,0 +1,139 @@ +use super::sort::sort_by_timestamp; +use aw_models::Event; +use std::collections::VecDeque; + +/// Takes a list of two events and returns a new list of events covering the union +/// of the timeperiods contained in the eventlists with no overlapping events. +/// +/// aw-core implementation: https://github.com/ActivityWatch/aw-core/blob/b11fbe08a0405dec01380493f7b3261163cc6878/aw_transform/filter_period_intersect.py#L92 +/// +/// WARNING: This function strips all data from events as it cannot keep it consistent. +/// +/// +/// # Example +/// ```ignore +/// events1 | ------- --------- | +/// events2 | ------ --- -- ---- | +/// result | ----------- -- --------- | +/// ``` +pub fn period_union(events1: &[Event], events2: &[Event]) -> Vec { + let mut sorted_events: VecDeque = VecDeque::new(); + sorted_events.extend(sort_by_timestamp([events1, events2].concat())); + + let mut events_union = Vec::new(); + + if !sorted_events.is_empty() { + events_union.push(sorted_events.pop_front().unwrap()) + } + + for e in sorted_events { + let last_event = events_union.last().unwrap(); + + let e_p = e.interval(); + let le_p = last_event.interval(); + + match e_p.union(&le_p) { + Some(new_period) => { + // If no gap and could be unioned, modify last event + let mut e_mod = events_union.pop().unwrap(); + e_mod.duration = new_period.duration(); + events_union.push(e_mod); + } + None => { + // If gap and could not be unioned, push event + events_union.push(e); + } + } + } + + events_union + .drain(..) + .map(|mut e| { + e.data = json_map! {}; + e + }) + .collect() +} + +#[cfg(test)] +mod tests { + use std::str::FromStr; + + use chrono::DateTime; + use chrono::Duration; + use chrono::Utc; + use serde_json::json; + + use aw_models::Event; + + use super::period_union; + + #[test] + fn test_period_union_empty() { + let e_result = period_union(&[], &[]); + assert_eq!(e_result.len(), 0); + } + + #[test] + fn test_period_union() { + let e1 = Event { + id: None, + timestamp: DateTime::from_str("2000-01-01T00:00:01Z").unwrap(), + duration: Duration::seconds(1), + data: json_map! {"test": json!(1)}, + }; + + let mut e2 = e1.clone(); + e2.timestamp = DateTime::from_str("2000-01-01T00:00:02Z").unwrap(); + + let e_result = period_union(&[e1], &[e2]); + assert_eq!(e_result.len(), 1); + + let dt: DateTime = DateTime::from_str("2000-01-01T00:00:01.000Z").unwrap(); + assert_eq!(e_result[0].timestamp, dt); + assert_eq!(e_result[0].duration, Duration::milliseconds(2000)); + } + + /// Make sure nothing gets done when nothing to union (gaps present) + #[test] + fn test_period_union_nop() { + let e1 = Event { + id: None, + timestamp: DateTime::from_str("2000-01-01T00:00:01Z").unwrap(), + duration: Duration::seconds(1), + data: json_map! {"test": json!(1)}, + }; + + let mut e2 = e1.clone(); + e2.timestamp = DateTime::from_str("2000-01-01T00:00:03Z").unwrap(); + + let e_result = period_union(&[e1], &[e2]); + assert_eq!(e_result.len(), 2); + } + + #[test] + fn test_period_union_2nd_empty() { + let e1 = Event { + id: None, + timestamp: DateTime::from_str("2000-01-01T00:00:01Z").unwrap(), + duration: Duration::seconds(1), + data: json_map! {"test": json!(1)}, + }; + + let e_result = period_union(&[e1], &[]); + assert_eq!(e_result.len(), 1); + } + + #[test] + fn test_period_union_1st_empty() { + let e1 = Event { + id: None, + timestamp: DateTime::from_str("2000-01-01T00:00:01Z").unwrap(), + duration: Duration::seconds(1), + data: json_map! {"test": json!(1)}, + }; + + let e_result = period_union(&[], &[e1]); + assert_eq!(e_result.len(), 1); + } +} diff --git a/aw-transform/src/sort.rs b/aw-transform/src/sort.rs new file mode 100644 index 0000000..ee6d323 --- /dev/null +++ b/aw-transform/src/sort.rs @@ -0,0 +1,62 @@ +use aw_models::Event; + +/// Sort a list of events by timestamp +pub fn sort_by_timestamp(mut events: Vec) -> Vec { + events.sort_by_key(|e1| e1.timestamp); + events +} + +/// Sort a list of events by duration with the highest duration first +pub fn sort_by_duration(mut events: Vec) -> Vec { + events.sort_by_key(|e| std::cmp::Reverse(e.duration)); + events +} + +#[cfg(test)] +mod tests { + use std::str::FromStr; + + use chrono::DateTime; + use chrono::Duration; + use serde_json::json; + + use aw_models::Event; + + use super::{sort_by_duration, sort_by_timestamp}; + + #[test] + fn test_sort_by_timestamp() { + let e1 = Event { + id: None, + timestamp: DateTime::from_str("2000-01-01T00:00:00Z").unwrap(), + duration: Duration::seconds(1), + data: json_map! {"test": json!(1)}, + }; + let e2 = Event { + id: None, + timestamp: DateTime::from_str("2000-01-01T00:00:03Z").unwrap(), + duration: Duration::seconds(1), + data: json_map! {"test": json!(1)}, + }; + let res = sort_by_timestamp(vec![e2.clone(), e1.clone()]); + assert_eq!(res, vec![e1, e2]); + } + + #[test] + fn test_sort_by_duration() { + let e1 = Event { + id: None, + timestamp: DateTime::from_str("2000-01-01T00:00:00Z").unwrap(), + duration: Duration::seconds(2), + data: json_map! {"test": json!(1)}, + }; + let e2 = Event { + id: None, + timestamp: DateTime::from_str("2000-01-01T00:00:03Z").unwrap(), + duration: Duration::seconds(1), + data: json_map! {"test": json!(1)}, + }; + let res = sort_by_duration(vec![e2.clone(), e1.clone()]); + assert_eq!(res, vec![e1, e2]); + } +} diff --git a/aw-transform/src/split_url.rs b/aw-transform/src/split_url.rs new file mode 100644 index 0000000..8eab4bc --- /dev/null +++ b/aw-transform/src/split_url.rs @@ -0,0 +1,126 @@ +use aw_models::Event; +use serde_json::value::Value; + +/// Adds $protocol, $domain, $path and $params keys for events with an "url" key +/// +/// But it only adds the generated field if it exists, for example if a url does not have a path +/// the path value will not be set at all. +/// +/// # Example +/// ```ignore +/// input: { +/// "data": { +/// "url": "http://google.com/test" +/// } +/// } +/// output: { +/// "data": { +/// "$domain": "google.com", +/// "$path": "/test", +/// "$protocol": "http" +/// } +/// } +/// ``` +pub fn split_url_event(event: &mut Event) { + use url::Url; + + let uri_str = match event.data.get("url") { + Some(Value::String(s)) => s.clone(), + _ => return, + }; + let uri = match Url::parse(&uri_str) { + Ok(uri) => uri, + Err(_) => return, + }; + // Protocol + let protocol = uri.scheme().to_string(); + event + .data + .insert("$protocol".to_string(), Value::String(protocol)); + // Domain + // For URLs without a host (e.g. file://, about:), fall back to the scheme + // so they don't all cluster as an empty string in "Top Browser Domains". + let domain = match uri.host_str() { + Some(domain) => domain.trim_start_matches("www.").to_string(), + None => uri.scheme().to_string(), + }; + event + .data + .insert("$domain".to_string(), Value::String(domain)); + + // Path + let path = uri.path().to_string(); + event.data.insert("$path".to_string(), Value::String(path)); + + // Params + let params = match uri.query() { + Some(query) => query.to_string(), + None => "".to_string(), + }; + event + .data + .insert("$params".to_string(), Value::String(params)); + + // TODO: aw-server-python also has options and identifier +} + +#[cfg(test)] +mod tests { + use std::str::FromStr; + + use chrono::DateTime; + use chrono::Duration; + use serde_json::json; + + use aw_models::Event; + + use super::split_url_event; + + #[test] + fn test_split_url_events() { + let mut e1 = Event { + id: None, + timestamp: DateTime::from_str("2000-01-01T00:00:01Z").unwrap(), + duration: Duration::seconds(1), + data: json_map! {"url": "http://www.google.com/path?query=1"}, + }; + split_url_event(&mut e1); + assert_eq!( + e1.data, + json_map! { + "url": json!("http://www.google.com/path?query=1"), + "$protocol": json!("http"), + "$domain": json!("google.com"), + "$path": json!("/path"), + "$params": json!("query=1") + } + ); + } + + #[test] + fn test_split_url_file_scheme() { + let mut e = Event { + id: None, + timestamp: DateTime::from_str("2000-01-01T00:00:01Z").unwrap(), + duration: Duration::seconds(1), + data: json_map! {"url": "file:///home/user/document.pdf"}, + }; + split_url_event(&mut e); + assert_eq!(e.data.get("$protocol"), Some(&json!("file"))); + assert_eq!(e.data.get("$domain"), Some(&json!("file"))); + assert_eq!(e.data.get("$path"), Some(&json!("/home/user/document.pdf"))); + } + + #[test] + fn test_split_url_about_scheme() { + let mut e = Event { + id: None, + timestamp: DateTime::from_str("2000-01-01T00:00:01Z").unwrap(), + duration: Duration::seconds(1), + data: json_map! {"url": "about:blank"}, + }; + split_url_event(&mut e); + assert_eq!(e.data.get("$protocol"), Some(&json!("about"))); + assert_eq!(e.data.get("$domain"), Some(&json!("about"))); + } +} diff --git a/aw-transform/src/union_no_overlap.rs b/aw-transform/src/union_no_overlap.rs new file mode 100644 index 0000000..06486f4 --- /dev/null +++ b/aw-transform/src/union_no_overlap.rs @@ -0,0 +1,297 @@ +use aw_models::{Event, TimeInterval}; +use chrono::{DateTime, Utc}; + +/// Merges two eventlists and removes overlap, the first eventlist will have precedence +/// +/// aw-core implementation: https://github.com/ActivityWatch/aw-core/blob/master/aw_transform/union_no_overlap.py +/// +/// # Example +/// ```ignore +/// events1 | xxx xx xxx | +/// events1 | ---- ------ -- | +/// result | xxx-- xx ----xxx -- | +/// ``` +pub fn union_no_overlap(events1: Vec, events2: Vec) -> Vec { + let mut events_union: Vec = Vec::new(); + let mut events1 = events1.into_iter().peekable(); + let mut events2 = events2.into_iter(); + // Keep a split remainder here instead of inserting it ahead of the entire + // unprocessed suffix. Each step consumes an input or emits a fragment. + let mut pending = events2.next(); + while let (Some(e1), Some(e2)) = (events1.peek(), pending.as_ref()) { + let e1_p: TimeInterval = e1.into(); + let e2_p: TimeInterval = e2.into(); + + if e1_p.intersects(&e2_p) { + if e1.timestamp <= e2.timestamp { + let end = e1.timestamp + e1.duration; + let e2_end = e2.timestamp + e2.duration; + if end > e2_end { + // This foreground event may cover more background events. + // Keep it pending until all of those have been removed. + pending = events2.next(); + continue; + } + if e2.timestamp < end && end < e2_end { + let remainder = pending.as_mut().unwrap(); + remainder.timestamp = end; + remainder.duration = e2_end - end; + remainder.id = None; + } else { + pending = events2.next(); + } + events_union.push(events1.next().unwrap()); + } else { + let (prefix, remainder) = split_event(pending.take().unwrap(), e1.timestamp); + events_union.push(prefix); + pending = remainder.or_else(|| events2.next()); + } + } else if e1.timestamp <= e2.timestamp { + events_union.push(events1.next().unwrap()); + } else { + events_union.push(pending.take().unwrap()); + pending = events2.next(); + } + } + + // Now we just need to add any remaining events + events_union.extend(events1); + events_union.extend(pending); + events_union.extend(events2); + + events_union +} + +fn split_event(mut e: Event, timestamp: DateTime) -> (Event, Option) { + if e.timestamp < timestamp && timestamp < e.timestamp + e.duration { + let e1 = Event::new(e.timestamp, timestamp - e.timestamp, e.data.clone()); + e.duration -= timestamp - e.timestamp; + e.timestamp = timestamp; + e.id = None; + (e1, Some(e)) + } else { + (e, None) + } +} + +// Some tests +#[cfg(test)] +mod tests { + use super::*; + use chrono::Duration; + + #[test] + fn union_matches_discrete_coverage_for_all_small_inputs() { + let now = Utc::now(); + let events = |mask: u32, source: &str| { + let mut result = Vec::new(); + let mut start = 0; + while start < 6 { + if mask & (1 << start) == 0 { + start += 1; + continue; + } + let mut end = start + 1; + while end < 6 && mask & (1 << end) != 0 { + end += 1; + } + let mut data = serde_json::Map::new(); + data.insert("source".into(), serde_json::json!(source)); + result.push(Event::new( + now + Duration::seconds(start), + Duration::seconds(end - start), + data, + )); + start = end; + } + result + }; + for a in 0..64 { + for b in 0..64 { + let result = union_no_overlap(events(a, "a"), events(b, "b")); + for slot in 0..6 { + let time = now + Duration::seconds(slot); + let covering: Vec<_> = result + .iter() + .filter(|e| e.timestamp <= time && time < e.timestamp + e.duration) + .collect(); + let expected = if a & (1 << slot) != 0 { + Some("a") + } else if b & (1 << slot) != 0 { + Some("b") + } else { + None + }; + assert_eq!( + covering.len(), + usize::from(expected.is_some()), + "a={a}, b={b}, slot={slot}" + ); + if let Some(source) = expected { + assert_eq!(covering[0].data["source"], source); + } + } + assert!(result + .windows(2) + .all(|pair| pair[0].timestamp + pair[0].duration <= pair[1].timestamp)); + } + } + } + + #[test] + fn repeated_splits_preserve_precedence_payloads_and_ids() { + let now = Utc::now(); + let mut background = Event::new(now, Duration::seconds(12), serde_json::Map::new()); + background.id = Some(99); + background + .data + .insert("source".into(), serde_json::json!("background")); + let foreground: Vec<_> = [2, 5, 8] + .into_iter() + .map(|start| { + let mut e = Event::new( + now + Duration::seconds(start), + Duration::seconds(1), + serde_json::Map::new(), + ); + e.id = Some(start); + e.data + .insert("source".into(), serde_json::json!("foreground")); + e + }) + .collect(); + let result = union_no_overlap(foreground, vec![background]); + let expected = [ + (0, 2, None), + (2, 1, Some(2)), + (3, 2, None), + (5, 1, Some(5)), + (6, 2, None), + (8, 1, Some(8)), + (9, 3, None), + ]; + assert_eq!(result.len(), expected.len()); + for (event, (start, duration, id)) in result.iter().zip(expected) { + assert_eq!(event.timestamp, now + Duration::seconds(start)); + assert_eq!(event.duration, Duration::seconds(duration)); + assert_eq!(event.id, id); + assert_eq!( + event.data["source"], + if id.is_some() { + "foreground" + } else { + "background" + } + ); + } + } + + #[test] + fn empty_disjoint_and_zero_duration_events_keep_ids() { + let now = Utc::now(); + let mut event = Event::new(now, Duration::zero(), serde_json::Map::new()); + event.id = Some(12); + assert_eq!( + union_no_overlap(vec![], vec![event.clone()])[0].id, + Some(12) + ); + assert_eq!( + union_no_overlap(vec![event.clone()], vec![])[0].id, + Some(12) + ); + let mut later = event.clone(); + later.timestamp += Duration::seconds(1); + later.id = Some(13); + let result = union_no_overlap(vec![event], vec![later]); + assert_eq!( + result.iter().map(|e| e.id).collect::>(), + vec![Some(12), Some(13)] + ); + } + + #[test] + fn test_split_event() { + let now = Utc::now(); + let td1h = Duration::hours(1); + let e = Event { + id: None, + timestamp: now, + duration: Duration::hours(2), + data: serde_json::Map::new(), + }; + let (e1, e2_opt) = split_event(e.clone(), now + td1h); + assert_eq!(e1.timestamp, now); + assert_eq!(e1.duration, td1h); + + let e2 = e2_opt.unwrap(); + assert_eq!(e2.timestamp, now + td1h); + assert_eq!(e2.duration, td1h); + + // Now a test which does not lead to a split + let (e1, e2_opt) = split_event(e, now); + assert_eq!(e1.timestamp, now); + assert_eq!(e1.duration, Duration::hours(2)); + assert!(e2_opt.is_none()); + } + + #[test] + fn test_union_no_overlap() { + // A test without any actual overlap + let now = Utc::now(); + let td1h = Duration::hours(1); + let e1 = Event::new(now, td1h, serde_json::Map::new()); + let e2 = Event::new(now + td1h, td1h, serde_json::Map::new()); + let events1 = vec![e1.clone()]; + let events2 = vec![e2.clone()]; + let events_union = union_no_overlap(events1, events2); + + assert_eq!(events_union.len(), 2); + assert_eq!(events_union[0].timestamp, now); + assert_eq!(events_union[0].duration, td1h); + assert_eq!(events_union[1].timestamp, now + td1h); + assert_eq!(events_union[1].duration, td1h); + + // Now do in reverse order + let events1 = vec![e2]; + let events2 = vec![e1]; + let events_union = union_no_overlap(events1, events2); + + // Resulting order should be the same, since there is no overlap. + assert_eq!(events_union.len(), 2); + assert_eq!(events_union[0].timestamp, now); + assert_eq!(events_union[0].duration, td1h); + assert_eq!(events_union[1].timestamp, now + td1h); + assert_eq!(events_union[1].duration, td1h); + } + + #[test] + fn test_union_no_overlap_with_overlap() { + // A test where the events overlap + let now = Utc::now(); + let td1h = Duration::hours(1); + let e1 = Event::new(now, td1h, serde_json::Map::new()); + let e2 = Event::new(now, Duration::hours(2), serde_json::Map::new()); + let events1 = vec![e1]; + let events2 = vec![e2]; + let events_union = union_no_overlap(events1, events2); + + assert_eq!(events_union.len(), 2); + assert_eq!(events_union[0].timestamp, now); + assert_eq!(events_union[0].duration, td1h); + assert_eq!(events_union[1].timestamp, now + td1h); + assert_eq!(events_union[1].duration, td1h); + + // Now test the case where e2 starts before e1 + let e1 = Event::new(now + td1h, td1h, serde_json::Map::new()); + let e2 = Event::new(now, Duration::hours(2), serde_json::Map::new()); + let events1 = vec![e1]; + let events2 = vec![e2]; + let events_union = union_no_overlap(events1, events2); + + assert_eq!(events_union.len(), 2); + assert_eq!(events_union[0].timestamp, now); + assert_eq!(events_union[0].duration, td1h); + assert_eq!(events_union[1].timestamp, now + td1h); + assert_eq!(events_union[1].duration, td1h); + } +} diff --git a/compile-android.sh b/compile-android.sh new file mode 100755 index 0000000..f497373 --- /dev/null +++ b/compile-android.sh @@ -0,0 +1,120 @@ +#!/bin/bash + +set -e +SCRIPT_DIR="$( + unset CDPATH + cd -- "$(dirname -- "${BASH_SOURCE[0]}")" + pwd +)" +cd "$SCRIPT_DIR" + +platform="$(uname -s | tr '[:upper:]' '[:lower:]')" + +# if args, use them to select targets (x86_64, arm64, etc) +if [ $# -gt 0 ]; then + targets="$*" +else + # otherwise, default to all targets + targets="arm64 x86_64 x86 arm" +fi + +ORIG_PATH="$PATH" +ORIG_RUSTFLAGS="$RUSTFLAGS" +release_args=() + +if [ -z "$ANDROID_NDK_HOME" ]; then + if [ -d "$SCRIPT_DIR"/NDK ]; then + echo "Found NDK folder in root, using." + ANDROID_NDK_HOME="$SCRIPT_DIR"/NDK + else + # NOTE: I had some issues with this and cargo that magically resolved themselves when I made the path absolute. + echo "Environment variable ANDROID_NDK_HOME not set, please set to location of Android NDK." + exit 1 + fi +fi +export ANDROID_NDK_HOME + +if [ "$RELEASE" = "true" ]; then + echo "Building in release mode... (slow)"; + release_args+=(--release) +else + echo "Building in debug mode... (fast)" + RELEASE=false; +fi + +# Workaround for "error: unable to find library -lgcc" +# See: https://stackoverflow.com/questions/68873570/how-do-i-fix-ld-error-unable-to-find-library-lgcc-when-cross-compiling-rust +find "${ANDROID_NDK_HOME}" -name "libunwind.a" -execdir bash -c 'echo "INPUT(-lunwind)" > libgcc.a' \; + +for archtargetstr in \ + 'arm64 aarch64-linux-android' \ + 'x86_64 x86_64-linux-android' \ + 'x86 i686-linux-android' \ + 'arm armv7-linux-androideabi' \ +; do + arch="$(echo "$archtargetstr" | cut -d " " -f 1)" + target="$(echo "$archtargetstr" | cut -d " " -f 2)" + target_underscore="${target//-/_}" + + echo ARCH "$arch" + echo TARGET "$target" + selected=false + for requested_arch in $targets; do + if [ "$requested_arch" = "$arch" ]; then + selected=true + break + fi + done + if [ "$selected" != "true" ]; then + echo "Skipping $arch..." + continue + fi + + NDK_ARCH_DIR="$ANDROID_NDK_HOME/toolchains/llvm/prebuilt/$platform-x86_64/bin" + if [ ! -d "$NDK_ARCH_DIR" ]; then + echo "Couldn't find directory $NDK_ARCH_DIR" + exit 1 + fi + + echo "Building for $arch..." + + export PATH="$NDK_ARCH_DIR:$ORIG_PATH" + export RUSTFLAGS="$ORIG_RUSTFLAGS" + # Need to set AR for target since NDK 21+: + # https://github.com/rust-lang/cc-rs/issues/636#issuecomment-1075352495 + cc="$NDK_ARCH_DIR/${target}-clang" + if [ "$arch" = "arm" ]; then + cc="$NDK_ARCH_DIR/arm-linux-androideabi-clang" + fi + + declare -x "AR_${target_underscore}"="$NDK_ARCH_DIR/llvm-ar" + declare -x "CC_${target_underscore}"="$cc" + declare -x "RANLIB_${target_underscore}"="$NDK_ARCH_DIR/llvm-ranlib" + + # Needed for runtime error: https://github.com/termux/termux-packages/issues/8029 + # java.lang.UnsatisfiedLinkError: dlopen failed: cannot locate symbol "__extenddftf2" + libgcc_path="$("$cc" -print-libgcc-file-name)" + export RUSTFLAGS+=" -C link-arg=$libgcc_path" + # Align to 16KB for Android 15 compatibility + export RUSTFLAGS+=" -C link-arg=-z -C link-arg=max-page-size=16384" + echo RUSTFLAGS="$RUSTFLAGS" + + # check that they exist + for var in AR_${target_underscore} CC_${target_underscore} RANLIB_${target_underscore}; do + if [ ! -f "${!var}" ]; then + echo "Couldn't find ${!var} set for variable $var" + exit 1 + fi + done + + # People suggest to use this, but ime it needs all the same workarounds anyway :shrug: + #cargo ndk build -p aw-server --target $target --lib $($RELEASE && echo '--release') + + # Build aw-server + echo "Building aw-server for $arch..." + cargo build -p aw-server --target "$target" --lib "${release_args[@]}" + + # Build aw-sync (without cli feature for Android) + echo "Building aw-sync for $arch..." + cargo build -p aw-sync --target "$target" --lib --no-default-features "${release_args[@]}" +done diff --git a/install-ndk.sh b/install-ndk.sh new file mode 100755 index 0000000..6460c3f --- /dev/null +++ b/install-ndk.sh @@ -0,0 +1,66 @@ +#!/bin/bash +# Based on https://mozilla.github.io/firefox-browser-architecture/experiments/2017-09-21-rust-on-android.html +# Depended on by aw-android/scripts/setup-rust-with-ndk.sh + +set -e; + +NDK_VERSION=r25c + +script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +project_path="$(readlink -f "$script_dir/.")" + +platform="$(uname -s | tr '[:upper:]' '[:lower:]')" + +if [ -z "$ANDROID_NDK_HOME" ]; then + if [ -d `pwd`/"NDK" ]; then + echo "Found NDK folder in root, using." + else + echo 'ANDROID_NDK_HOME not set, downloading NDK...'; + # Download Linux NDK or macOS NDK, depending on OS + wget --no-verbose -O android-ndk.zip https://dl.google.com/android/repository/android-ndk-$NDK_VERSION-$platform.zip; + unzip -q -d NDK android-ndk.zip; + ls NDK; + mv NDK/*/* NDK/; + fi + ANDROID_NDK_HOME=`pwd`/NDK; +fi + +# Needed since dependency 'ring' doesn't respect .cargo/config +echo "Setting up toolchain binary symlinks..." +NDK_TOOLCHAIN_BIN=$ANDROID_NDK_HOME/toolchains/llvm/prebuilt/$platform-x86_64/bin +for arch in \ + 'aarch64' \ + 'x86_64' \ + 'i686' \ +; do + ln -s -f $NDK_TOOLCHAIN_BIN/$arch-linux-android26-clang $NDK_TOOLCHAIN_BIN/$arch-linux-android-clang +done + +# This has a slightly different path from the ones above +ln -s -f $NDK_TOOLCHAIN_BIN/armv7a-linux-androideabi26-clang $NDK_TOOLCHAIN_BIN/armv7a-linux-androideabi-clang +ln -s -f $NDK_TOOLCHAIN_BIN/armv7a-linux-androideabi26-clang $NDK_TOOLCHAIN_BIN/arm-linux-androideabi-clang + +# Add to Rust +echo "Setting up Rust toolchains..." +rustup target add aarch64-linux-android armv7-linux-androideabi i686-linux-android x86_64-linux-android; + +# Creates cargo config +echo "Creating cargo config..." +mkdir -p $project_path/.cargo +echo " +[target.aarch64-linux-android] +ar = '$ANDROID_NDK_HOME/toolchains/llvm/prebuilt/$platform-x86_64/bin/llvm-ar' +linker = '$ANDROID_NDK_HOME/toolchains/llvm/prebuilt/$platform-x86_64/bin/aarch64-linux-android26-clang' + +[target.armv7-linux-androideabi] +ar = '$ANDROID_NDK_HOME/toolchains/llvm/prebuilt/$platform-x86_64/bin/llvm-ar' +linker = '$ANDROID_NDK_HOME/toolchains/llvm/prebuilt/$platform-x86_64/bin/armv7a-linux-androideabi-clang' + +[target.i686-linux-android] +ar = '$ANDROID_NDK_HOME/toolchains/llvm/prebuilt/$platform-x86_64/bin/llvm-ar' +linker = '$ANDROID_NDK_HOME/toolchains/llvm/prebuilt/$platform-x86_64/bin/i686-linux-android26-clang' + +[target.x86_64-linux-android] +ar = '$ANDROID_NDK_HOME/toolchains/llvm/prebuilt/$platform-x86_64/bin/llvm-ar' +linker = '$ANDROID_NDK_HOME/toolchains/llvm/prebuilt/$platform-x86_64/bin/x86_64-linux-android26-clang' +" > $project_path/.cargo/config diff --git a/scripts/Dockerfile b/scripts/Dockerfile new file mode 100644 index 0000000..35558af --- /dev/null +++ b/scripts/Dockerfile @@ -0,0 +1,53 @@ +# Build stage: aw-webui (node) + aw-server (rust, with the webui embedded via rust-embed) +# Node is taken from the official image instead of the NodeSource install script, +# so no remotely fetched script is executed as root in the stage that produces +# the artifacts shipped in the final image. +FROM node:22-bookworm-slim AS node + +FROM rust:1-bookworm AS builder + +RUN apt-get update -qq -y && \ + apt-get install -qq -y --no-install-recommends \ + build-essential pkg-config libssl-dev ca-certificates git make && \ + rm -rf /var/lib/apt/lists/* + +COPY --from=node /usr/local/bin/node /usr/local/bin/node +COPY --from=node /usr/local/lib/node_modules /usr/local/lib/node_modules +RUN ln -s ../lib/node_modules/npm/bin/npm-cli.js /usr/local/bin/npm && \ + ln -s ../lib/node_modules/npm/bin/npx-cli.js /usr/local/bin/npx + +WORKDIR /app/aw-server-rust +COPY . . + +# `make aw-server` builds the webui (aw-webui/dist) before cargo, so that +# aw-server's build.rs can embed the assets into the binary. +# We avoid the `build` target, which also builds aw-sync: that binary pulls in +# openssl with the "vendored" feature (builds OpenSSL from source) and is not +# used by this image. +RUN make aw-server + +FROM debian:bookworm-slim + +RUN apt-get update -qq -y && \ + apt-get install -qq -y --no-install-recommends \ + libssl3 && \ + rm -rf /var/lib/apt/lists/* + +RUN groupadd --gid 10001 aw && \ + useradd --uid 10001 --gid 10001 --home-dir /home/aw --create-home \ + --shell /usr/sbin/nologin aw && \ + mkdir -p /home/aw/.local/share/activitywatch /home/aw/.config/activitywatch && \ + chown -R aw:aw /home/aw + +COPY --from=builder /app/aw-server-rust/target/release/aw-server /usr/local/bin/aw-server-rust +COPY --from=builder /app/aw-server-rust/aw-webui/dist /usr/local/share/aw-webui + +USER aw +# Set explicitly so the datastore path does not depend on the runtime resolving +# the home directory from /etc/passwd. +ENV HOME=/home/aw +WORKDIR /home/aw + +EXPOSE 5600 + +CMD ["aw-server-rust"] diff --git a/scripts/create-cargo-config.sh b/scripts/create-cargo-config.sh new file mode 100755 index 0000000..42f4bf7 --- /dev/null +++ b/scripts/create-cargo-config.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +if [ -z $ANDROID_NDK_HOME ]; then + echo '$ANDROID_NDK_HOME not set'; + exit 1; +fi + +tee ~/test-cargo-config.toml <<< " +[target.aarch64-linux-android] +ar = '$NDK_HOME/arm64/bin/aarch64-linux-android-ar' +linker = '$NDK_HOME/arm64/bin/aarch64-linux-android-clang' + +[target.armv7-linux-androideabi] +ar = '$NDK_HOME/arm/bin/arm-linux-androideabi-ar' +linker = '$NDK_HOME/arm/bin/arm-linux-androideabi-clang' + +[target.i686-linux-android] +ar = '$NDK_HOME/x86/bin/i686-linux-android-ar' +linker = '$NDK_HOME/x86/bin/i686-linux-android-clang' +" diff --git a/scripts/test-compile-android-cwd.sh b/scripts/test-compile-android-cwd.sh new file mode 100755 index 0000000..0b8e39b --- /dev/null +++ b/scripts/test-compile-android-cwd.sh @@ -0,0 +1,101 @@ +#!/bin/bash + +set -euo pipefail + +REPO_ROOT="$( + unset CDPATH + cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." + pwd +)" +TEST_DIR="$(mktemp -d)" +trap 'rm -rf "$TEST_DIR"' EXIT + +platform="$(uname -s | tr '[:upper:]' '[:lower:]')" +NDK_BIN="$TEST_DIR/ndk/toolchains/llvm/prebuilt/${platform}-x86_64/bin" +mkdir -p "$NDK_BIN" "$TEST_DIR/ndk/sysroot/lib" "$TEST_DIR/bin" +touch "$TEST_DIR/ndk/sysroot/lib/libunwind.a" "$TEST_DIR/libgcc.a" + +cat > "$NDK_BIN/llvm-ar" <<'EOF' +#!/bin/bash +exit 0 +EOF + +cat > "$NDK_BIN/llvm-ranlib" <<'EOF' +#!/bin/bash +exit 0 +EOF + +write_clang_stub() { + cat > "$NDK_BIN/$1" < "$TEST_DIR/bin/cargo" <> "$TEST_DIR/cargo_pwds.txt" +printf '%s\n' "\$*" >> "$TEST_DIR/cargo_args.txt" +exit 0 +EOF + +chmod +x \ + "$NDK_BIN/llvm-ar" \ + "$NDK_BIN/llvm-ranlib" \ + "$NDK_BIN/aarch64-linux-android-clang" \ + "$NDK_BIN/arm-linux-androideabi-clang" \ + "$NDK_BIN/i686-linux-android-clang" \ + "$NDK_BIN/x86_64-linux-android-clang" \ + "$TEST_DIR/bin/cargo" + +run_compile() { + cd /tmp + PATH="$TEST_DIR/bin:$PATH" \ + ANDROID_NDK_HOME="$TEST_DIR/ndk" \ + RELEASE=false \ + "$REPO_ROOT/compile-android.sh" "$@" >/dev/null +} + +assert_cargo_invocations() { + if [ ! -s "$TEST_DIR/cargo_pwds.txt" ]; then + echo "Expected cargo to be invoked" + exit 1 + fi + + if [ "$(wc -l < "$TEST_DIR/cargo_pwds.txt")" -ne "$1" ]; then + echo "Expected exactly $1 cargo invocations" + cat "$TEST_DIR/cargo_args.txt" + exit 1 + fi + + if grep -Fxv "$REPO_ROOT" "$TEST_DIR/cargo_pwds.txt" >/dev/null; then + echo "cargo ran outside repo root:" + cat "$TEST_DIR/cargo_pwds.txt" + exit 1 + fi +} + +run_compile arm64 +assert_cargo_invocations 2 +grep -F -- "-p aw-server" "$TEST_DIR/cargo_args.txt" >/dev/null +grep -F -- "-p aw-sync" "$TEST_DIR/cargo_args.txt" >/dev/null +grep -F -- "--target aarch64-linux-android" "$TEST_DIR/cargo_args.txt" >/dev/null +if grep -F -- "--target armv7-linux-androideabi" "$TEST_DIR/cargo_args.txt" >/dev/null; then + echo "arm64 should not match the arm target" + exit 1 +fi + +: > "$TEST_DIR/cargo_pwds.txt" +: > "$TEST_DIR/cargo_args.txt" + +run_compile arm +assert_cargo_invocations 2 +grep -F -- "--target armv7-linux-androideabi" "$TEST_DIR/cargo_args.txt" >/dev/null diff --git a/todo.md b/todo.md new file mode 100644 index 0000000..3a4524f --- /dev/null +++ b/todo.md @@ -0,0 +1,4 @@ +bucket already exists test +get_keys_starting no rows test +event model default duration test +make a log message in logging.rs test