snapshot of https://github.com/ActivityWatch/aw-core.git (master, 2026-09-16; blobs > 131072B omitted)

.coveragerc

@@ -0,0 +1,23 @@
+# .coveragerc to control coverage.py
+[run]
+branch = True
+omit =
+    aw_datastore/benchmark.py
+
+[report]
+# Regexes for lines to exclude from consideration
+exclude_lines =
+    # Have to re-enable the standard pragma
+    pragma: no cover
+
+    # Don't complain about missing debug-only code:
+    def __repr__
+    if self\.debug
+
+    # Don't complain if tests don't hit defensive assertion code:
+    raise AssertionError
+    raise NotImplementedError
+
+    # Don't complain if non-runnable code isn't run:
+    if 0:
+    if __name__ == .__main__.:

.github/workflows/build.yml

@@ -0,0 +1,44 @@
+name: Build
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  build:
+    name: Test on ${{ matrix.os }}, py-${{ matrix.python_version }}
+    runs-on: ${{ matrix.os }}
+    env:
+      RELEASE: false
+    strategy:
+      matrix:
+        os: [ubuntu-latest, windows-latest, macOS-latest]
+        python_version: [3.8]
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        submodules: 'recursive'
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: ${{ matrix.python_version }}
+    - name: Install dependencies
+      shell: bash
+      run: |
+        pip install poetry==1.3.2  # due to: https://github.com/python-poetry/poetry/issues/7611
+        python -m venv venv
+        source venv/bin/activate || source venv/Scripts/activate
+        make build
+    - name: Run tests
+      shell: bash
+      run: |
+        source venv/bin/activate || source venv/Scripts/activate
+        make test
+        make typecheck
+    - name: Report coverage
+      shell: bash
+      run: |
+        # Allow failing, since sometimes codecov is grumpy and we just get "no healthy upstream"
+        bash <(curl -s https://codecov.io/bash) || true

.github/workflows/codeql.yml

@@ -0,0 +1,41 @@
+name: "CodeQL"
+
+on:
+  push:
+    branches: [ "master" ]
+  pull_request:
+    branches: [ "master" ]
+  schedule:
+    - cron: "49 7 * * 2"
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ python ]
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@v2
+        with:
+          languages: ${{ matrix.language }}
+          queries: +security-and-quality
+
+      - name: Autobuild
+        uses: github/codeql-action/autobuild@v2
+
+      - name: Perform CodeQL Analysis
+        uses: github/codeql-action/analyze@v2
+        with:
+          category: "/language:${{ matrix.language }}"

.github/workflows/lint.yml

@@ -0,0 +1,15 @@
+name: Lint
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+      - uses: jpetrucciani/ruff-check@main

.gitignore

@@ -0,0 +1,10 @@
+__pycache__
+*.pyc
+*.swp
+*.egg-info
+build
+dist
+.coverage
+venv*
+.*cache
+coverage_html

.pre-commit-config.yaml

@@ -0,0 +1,16 @@
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v6.0.0
+    hooks:
+      - id: trailing-whitespace
+      - id: end-of-file-fixer
+      - id: check-toml
+      - id: check-yaml
+      - id: check-merge-conflict
+
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.15.4
+    hooks:
+      - id: ruff
+        args: [--fix]
+      - id: ruff-format

LICENSE.txt

@@ -0,0 +1,373 @@
+Mozilla Public License Version 2.0
+==================================
+
+1. Definitions
+--------------
+
+1.1. "Contributor"
+    means each individual or legal entity that creates, contributes to
+    the creation of, or owns Covered Software.
+
+1.2. "Contributor Version"
+    means the combination of the Contributions of others (if any) used
+    by a Contributor and that particular Contributor's Contribution.
+
+1.3. "Contribution"
+    means Covered Software of a particular Contributor.
+
+1.4. "Covered Software"
+    means Source Code Form to which the initial Contributor has attached
+    the notice in Exhibit A, the Executable Form of such Source Code
+    Form, and Modifications of such Source Code Form, in each case
+    including portions thereof.
+
+1.5. "Incompatible With Secondary Licenses"
+    means
+
+    (a) that the initial Contributor has attached the notice described
+        in Exhibit B to the Covered Software; or
+
+    (b) that the Covered Software was made available under the terms of
+        version 1.1 or earlier of the License, but not also under the
+        terms of a Secondary License.
+
+1.6. "Executable Form"
+    means any form of the work other than Source Code Form.
+
+1.7. "Larger Work"
+    means a work that combines Covered Software with other material, in
+    a separate file or files, that is not Covered Software.
+
+1.8. "License"
+    means this document.
+
+1.9. "Licensable"
+    means having the right to grant, to the maximum extent possible,
+    whether at the time of the initial grant or subsequently, any and
+    all of the rights conveyed by this License.
+
+1.10. "Modifications"
+    means any of the following:
+
+    (a) any file in Source Code Form that results from an addition to,
+        deletion from, or modification of the contents of Covered
+        Software; or
+
+    (b) any new file in Source Code Form that contains any Covered
+        Software.
+
+1.11. "Patent Claims" of a Contributor
+    means any patent claim(s), including without limitation, method,
+    process, and apparatus claims, in any patent Licensable by such
+    Contributor that would be infringed, but for the grant of the
+    License, by the making, using, selling, offering for sale, having
+    made, import, or transfer of either its Contributions or its
+    Contributor Version.
+
+1.12. "Secondary License"
+    means either the GNU General Public License, Version 2.0, the GNU
+    Lesser General Public License, Version 2.1, the GNU Affero General
+    Public License, Version 3.0, or any later versions of those
+    licenses.
+
+1.13. "Source Code Form"
+    means the form of the work preferred for making modifications.
+
+1.14. "You" (or "Your")
+    means an individual or a legal entity exercising rights under this
+    License. For legal entities, "You" includes any entity that
+    controls, is controlled by, or is under common control with You. For
+    purposes of this definition, "control" means (a) the power, direct
+    or indirect, to cause the direction or management of such entity,
+    whether by contract or otherwise, or (b) ownership of more than
+    fifty percent (50%) of the outstanding shares or beneficial
+    ownership of such entity.
+
+2. License Grants and Conditions
+--------------------------------
+
+2.1. Grants
+
+Each Contributor hereby grants You a world-wide, royalty-free,
+non-exclusive license:
+
+(a) under intellectual property rights (other than patent or trademark)
+    Licensable by such Contributor to use, reproduce, make available,
+    modify, display, perform, distribute, and otherwise exploit its
+    Contributions, either on an unmodified basis, with Modifications, or
+    as part of a Larger Work; and
+
+(b) under Patent Claims of such Contributor to make, use, sell, offer
+    for sale, have made, import, and otherwise transfer either its
+    Contributions or its Contributor Version.
+
+2.2. Effective Date
+
+The licenses granted in Section 2.1 with respect to any Contribution
+become effective for each Contribution on the date the Contributor first
+distributes such Contribution.
+
+2.3. Limitations on Grant Scope
+
+The licenses granted in this Section 2 are the only rights granted under
+this License. No additional rights or licenses will be implied from the
+distribution or licensing of Covered Software under this License.
+Notwithstanding Section 2.1(b) above, no patent license is granted by a
+Contributor:
+
+(a) for any code that a Contributor has removed from Covered Software;
+    or
+
+(b) for infringements caused by: (i) Your and any other third party's
+    modifications of Covered Software, or (ii) the combination of its
+    Contributions with other software (except as part of its Contributor
+    Version); or
+
+(c) under Patent Claims infringed by Covered Software in the absence of
+    its Contributions.
+
+This License does not grant any rights in the trademarks, service marks,
+or logos of any Contributor (except as may be necessary to comply with
+the notice requirements in Section 3.4).
+
+2.4. Subsequent Licenses
+
+No Contributor makes additional grants as a result of Your choice to
+distribute the Covered Software under a subsequent version of this
+License (see Section 10.2) or under the terms of a Secondary License (if
+permitted under the terms of Section 3.3).
+
+2.5. Representation
+
+Each Contributor represents that the Contributor believes its
+Contributions are its original creation(s) or it has sufficient rights
+to grant the rights to its Contributions conveyed by this License.
+
+2.6. Fair Use
+
+This License is not intended to limit any rights You have under
+applicable copyright doctrines of fair use, fair dealing, or other
+equivalents.
+
+2.7. Conditions
+
+Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted
+in Section 2.1.
+
+3. Responsibilities
+-------------------
+
+3.1. Distribution of Source Form
+
+All distribution of Covered Software in Source Code Form, including any
+Modifications that You create or to which You contribute, must be under
+the terms of this License. You must inform recipients that the Source
+Code Form of the Covered Software is governed by the terms of this
+License, and how they can obtain a copy of this License. You may not
+attempt to alter or restrict the recipients' rights in the Source Code
+Form.
+
+3.2. Distribution of Executable Form
+
+If You distribute Covered Software in Executable Form then:
+
+(a) such Covered Software must also be made available in Source Code
+    Form, as described in Section 3.1, and You must inform recipients of
+    the Executable Form how they can obtain a copy of such Source Code
+    Form by reasonable means in a timely manner, at a charge no more
+    than the cost of distribution to the recipient; and
+
+(b) You may distribute such Executable Form under the terms of this
+    License, or sublicense it under different terms, provided that the
+    license for the Executable Form does not attempt to limit or alter
+    the recipients' rights in the Source Code Form under this License.
+
+3.3. Distribution of a Larger Work
+
+You may create and distribute a Larger Work under terms of Your choice,
+provided that You also comply with the requirements of this License for
+the Covered Software. If the Larger Work is a combination of Covered
+Software with a work governed by one or more Secondary Licenses, and the
+Covered Software is not Incompatible With Secondary Licenses, this
+License permits You to additionally distribute such Covered Software
+under the terms of such Secondary License(s), so that the recipient of
+the Larger Work may, at their option, further distribute the Covered
+Software under the terms of either this License or such Secondary
+License(s).
+
+3.4. Notices
+
+You may not remove or alter the substance of any license notices
+(including copyright notices, patent notices, disclaimers of warranty,
+or limitations of liability) contained within the Source Code Form of
+the Covered Software, except that You may alter any license notices to
+the extent required to remedy known factual inaccuracies.
+
+3.5. Application of Additional Terms
+
+You may choose to offer, and to charge a fee for, warranty, support,
+indemnity or liability obligations to one or more recipients of Covered
+Software. However, You may do so only on Your own behalf, and not on
+behalf of any Contributor. You must make it absolutely clear that any
+such warranty, support, indemnity, or liability obligation is offered by
+You alone, and You hereby agree to indemnify every Contributor for any
+liability incurred by such Contributor as a result of warranty, support,
+indemnity or liability terms You offer. You may include additional
+disclaimers of warranty and limitations of liability specific to any
+jurisdiction.
+
+4. Inability to Comply Due to Statute or Regulation
+---------------------------------------------------
+
+If it is impossible for You to comply with any of the terms of this
+License with respect to some or all of the Covered Software due to
+statute, judicial order, or regulation then You must: (a) comply with
+the terms of this License to the maximum extent possible; and (b)
+describe the limitations and the code they affect. Such description must
+be placed in a text file included with all distributions of the Covered
+Software under this License. Except to the extent prohibited by statute
+or regulation, such description must be sufficiently detailed for a
+recipient of ordinary skill to be able to understand it.
+
+5. Termination
+--------------
+
+5.1. The rights granted under this License will terminate automatically
+if You fail to comply with any of its terms. However, if You become
+compliant, then the rights granted under this License from a particular
+Contributor are reinstated (a) provisionally, unless and until such
+Contributor explicitly and finally terminates Your grants, and (b) on an
+ongoing basis, if such Contributor fails to notify You of the
+non-compliance by some reasonable means prior to 60 days after You have
+come back into compliance. Moreover, Your grants from a particular
+Contributor are reinstated on an ongoing basis if such Contributor
+notifies You of the non-compliance by some reasonable means, this is the
+first time You have received notice of non-compliance with this License
+from such Contributor, and You become compliant prior to 30 days after
+Your receipt of the notice.
+
+5.2. If You initiate litigation against any entity by asserting a patent
+infringement claim (excluding declaratory judgment actions,
+counter-claims, and cross-claims) alleging that a Contributor Version
+directly or indirectly infringes any patent, then the rights granted to
+You by any and all Contributors for the Covered Software under Section
+2.1 of this License shall terminate.
+
+5.3. In the event of termination under Sections 5.1 or 5.2 above, all
+end user license agreements (excluding distributors and resellers) which
+have been validly granted by You or Your distributors under this License
+prior to termination shall survive termination.
+
+************************************************************************
+*                                                                      *
+*  6. Disclaimer of Warranty                                           *
+*  -------------------------                                           *
+*                                                                      *
+*  Covered Software is provided under this License on an "as is"       *
+*  basis, without warranty of any kind, either expressed, implied, or  *
+*  statutory, including, without limitation, warranties that the       *
+*  Covered Software is free of defects, merchantable, fit for a        *
+*  particular purpose or non-infringing. The entire risk as to the     *
+*  quality and performance of the Covered Software is with You.        *
+*  Should any Covered Software prove defective in any respect, You     *
+*  (not any Contributor) assume the cost of any necessary servicing,   *
+*  repair, or correction. This disclaimer of warranty constitutes an   *
+*  essential part of this License. No use of any Covered Software is   *
+*  authorized under this License except under this disclaimer.         *
+*                                                                      *
+************************************************************************
+
+************************************************************************
+*                                                                      *
+*  7. Limitation of Liability                                          *
+*  --------------------------                                          *
+*                                                                      *
+*  Under no circumstances and under no legal theory, whether tort      *
+*  (including negligence), contract, or otherwise, shall any           *
+*  Contributor, or anyone who distributes Covered Software as          *
+*  permitted above, be liable to You for any direct, indirect,         *
+*  special, incidental, or consequential damages of any character      *
+*  including, without limitation, damages for lost profits, loss of    *
+*  goodwill, work stoppage, computer failure or malfunction, or any    *
+*  and all other commercial damages or losses, even if such party      *
+*  shall have been informed of the possibility of such damages. This   *
+*  limitation of liability shall not apply to liability for death or   *
+*  personal injury resulting from such party's negligence to the       *
+*  extent applicable law prohibits such limitation. Some               *
+*  jurisdictions do not allow the exclusion or limitation of           *
+*  incidental or consequential damages, so this exclusion and          *
+*  limitation may not apply to You.                                    *
+*                                                                      *
+************************************************************************
+
+8. Litigation
+-------------
+
+Any litigation relating to this License may be brought only in the
+courts of a jurisdiction where the defendant maintains its principal
+place of business and such litigation shall be governed by laws of that
+jurisdiction, without reference to its conflict-of-law provisions.
+Nothing in this Section shall prevent a party's ability to bring
+cross-claims or counter-claims.
+
+9. Miscellaneous
+----------------
+
+This License represents the complete agreement concerning the subject
+matter hereof. If any provision of this License is held to be
+unenforceable, such provision shall be reformed only to the extent
+necessary to make it enforceable. Any law or regulation which provides
+that the language of a contract shall be construed against the drafter
+shall not be used to construe this License against a Contributor.
+
+10. Versions of the License
+---------------------------
+
+10.1. New Versions
+
+Mozilla Foundation is the license steward. Except as provided in Section
+10.3, no one other than the license steward has the right to modify or
+publish new versions of this License. Each version will be given a
+distinguishing version number.
+
+10.2. Effect of New Versions
+
+You may distribute the Covered Software under the terms of the version
+of the License under which You originally received the Covered Software,
+or under the terms of any subsequent version published by the license
+steward.
+
+10.3. Modified Versions
+
+If you create software not governed by this License, and you want to
+create a new license for such software, you may create and use a
+modified version of this License if you rename the license and remove
+any references to the name of the license steward (except to note that
+such modified license differs from this License).
+
+10.4. Distributing Source Code Form that is Incompatible With Secondary
+Licenses
+
+If You choose to distribute Source Code Form that is Incompatible With
+Secondary Licenses under the terms of this version of the License, the
+notice described in Exhibit B of this License must be attached.
+
+Exhibit A - Source Code Form License Notice
+-------------------------------------------
+
+  This Source Code Form is subject to the terms of the Mozilla Public
+  License, v. 2.0. If a copy of the MPL was not distributed with this
+  file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+If it is not possible or desirable to put the notice in a particular
+file, then You may include the notice in a location (such as a LICENSE
+file in a relevant directory) where a recipient would be likely to look
+for such a notice.
+
+You may add additional accurate notices of copyright ownership.
+
+Exhibit B - "Incompatible With Secondary Licenses" Notice
+---------------------------------------------------------
+
+  This Source Code Form is "Incompatible With Secondary Licenses", as
+  defined by the Mozilla Public License, v. 2.0.

Makefile

@@ -0,0 +1,35 @@
+.PHONY: build test benchmark typecheck typecheck-strict clean
+
+build:
+	poetry install
+
+test:
+	python -m pytest tests -v --cov=aw_core --cov=aw_datastore --cov=aw_transform --cov=aw_query
+
+.coverage:
+	make test
+
+coverage_html: .coverage
+	python -m coverage html -d coverage_html
+
+benchmark:
+	python -m aw_datastore.benchmark
+
+typecheck:
+	export MYPYPATH=./stubs; python -m mypy aw_core aw_datastore aw_transform aw_query --show-traceback --ignore-missing-imports --follow-imports=skip
+
+typecheck-strict:
+	export MYPYPATH=./stubs; python -m mypy aw_core aw_datastore aw_transform aw_query --strict-optional --check-untyped-defs; echo "Not a failing step"
+
+lint:
+	ruff check .
+
+lint-fix:
+	ruff check --fix .
+
+format:
+	ruff format .
+
+clean:
+	rm -rf build dist
+	rm -rf aw_core/__pycache__ aw_datastore/__pycache__

README.md

@@ -0,0 +1,35 @@
+aw-core
+=======
+
+[![GitHub Actions badge](https://github.com/ActivityWatch/aw-core/workflows/Build/badge.svg)](https://github.com/ActivityWatch/aw-core/actions)
+[![Code coverage](https://codecov.io/gh/ActivityWatch/aw-core/branch/master/graph/badge.svg)](https://codecov.io/gh/ActivityWatch/aw-core)
+[![PyPI](https://img.shields.io/pypi/v/aw-core)](https://pypi.org/project/aw-core/)
+[![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
+[![Typechecking: Mypy](http://www.mypy-lang.org/static/mypy_badge.svg)](http://mypy-lang.org/)
+
+
+Core library for ActivityWatch.
+
+
+## Modules
+
+ - `aw_core`, contains basic datatypes and utilities, such as the `Event` class, helpers for configuration and logging, as well as schemas for buckets, events, and exports.
+ - `aw_datastore`, contains the datastore classes used by aw-server-python.
+ - `aw_transform`, all event-transforms used in queries.
+ - `aw_query`, the query-language used by ActivityWatch.
+
+## Logging
+
+Run python with `LOG_LEVEL=debug` to use change the log level across all AW components
+
+## How to install
+
+To install the latest git version directly from github without cloning, run
+`pip install git+https://github.com/ActivityWatch/aw-core.git`
+
+To install from a cloned version, cd into the directory and run
+`poetry install` to install inside an virtualenv. If you want to install it
+system-wide it can be installed with `pip install .`, but that has the issue
+that it might not get the exact version of the dependencies due to not reading
+the poetry.lock file.
+

aw_cli/__main__.py

@@ -0,0 +1,107 @@
+"""
+The idea behind this `aw` or `aw-cli` wrapper script is to act as a collection of helper tools,
+and perhaps even as a way to list and run ActivityWatch modules on a system (a bit like aw-qt, but without the GUI).
+"""
+
+from pathlib import Path
+from datetime import datetime
+import os
+import subprocess
+
+import click
+
+from aw_cli.log import find_oldest_log, print_log, LOGLEVELS
+from typing import Optional
+
+
+def _resolve_profile(profile: Optional[str], testing: bool) -> str:
+    """Return the active profile name, resolving --testing as an alias for 'testing'."""
+    if profile:
+        return profile
+    if testing:
+        return "testing"
+    return ""
+
+
+@click.group()
+@click.option(
+    "--profile",
+    default=None,
+    help="Named instance profile (e.g. 'testing', 'research').",
+)
+@click.option("--testing", is_flag=True, help="Alias for --profile testing.")
+def main(profile: Optional[str] = None, testing: bool = False):
+    active = _resolve_profile(profile, testing)
+    if active:
+        os.environ["AW_PROFILE"] = active
+    # If no explicit flag, preserve any inherited AW_PROFILE from the environment.
+
+
+@main.command()
+@click.pass_context
+def qt(ctx):
+    profile = ctx.parent.params.get("profile")
+    testing = ctx.parent.params.get("testing")
+    active = _resolve_profile(profile, testing)
+    args = ["aw-qt"]
+    if active:
+        args += ["--profile", active]
+    return subprocess.call(args)
+
+
+@main.command()
+def directories():
+    # Print all directories (respects AW_PROFILE set by the group callback)
+    from aw_core.dirs import get_data_dir, get_config_dir, get_cache_dir, get_log_dir
+
+    print("Directory paths used")
+    print(" - config: ", get_config_dir(None))
+    print(" - data:   ", get_data_dir(None))
+    print(" - logs:   ", get_log_dir(None))
+    print(" - cache:  ", get_cache_dir(None))
+
+
+@main.command()
+@click.pass_context
+@click.argument("module_name", type=str, required=False)
+@click.option(
+    "--since",
+    type=click.DateTime(formats=["%Y-%m-%d"]),
+    help="Only show logs since this date",
+)
+@click.option(
+    "--level",
+    type=click.Choice(LOGLEVELS),
+    help="Only show logs of this level, or higher.",
+)
+def logs(
+    ctx,
+    module_name: Optional[str] = None,
+    since: Optional[datetime] = None,
+    level: Optional[str] = None,
+):
+    from aw_core.dirs import get_log_dir
+
+    # AW_PROFILE was set by the group callback, so get_log_dir returns the profile-specific dir.
+    logdir: Path = Path(get_log_dir(None))
+
+    if module_name:
+        _print_newest_log(logdir / module_name, since, level)
+    else:
+        for subdir in sorted(logdir.iterdir()):
+            if subdir.is_dir():
+                _print_newest_log(subdir, since, level)
+
+
+def _print_newest_log(
+    path: Path, since: Optional[datetime], level: Optional[str]
+) -> None:
+    logfile = find_oldest_log(path)
+    if logfile:
+        print_log(logfile, since, level)
+    else:
+        print(f"No logfile found in {path}")
+
+
+if __name__ == "__main__":
+    main()

aw_cli/log.py

@@ -0,0 +1,57 @@
+from pathlib import Path
+from datetime import datetime
+from typing import Optional
+
+
+LOGLEVELS = ["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]
+
+
+def print_log(
+    path: Path, since: Optional[datetime] = None, level: Optional[str] = None
+):
+    if not path.is_file():
+        return
+
+    show_levels = LOGLEVELS[LOGLEVELS.index(level) :] if level else None
+
+    lines_printed = 0
+    with path.open("r") as f:
+        lines = f.readlines()
+        print(f"Logs for module {path.parent.name} ({path.name}, {len(lines)} lines)")
+        for line in lines:
+            if since:
+                try:
+                    linedate = datetime.strptime(line.split(" ")[0], "%Y-%m-%d")
+                except ValueError:
+                    # Could not parse the date, so skip this line
+                    # NOTE: Just because the date could not be parsed, doesn't mean there isn't meaningful info there.
+                    #       Would be better to find the first line after the cutoff, and then just print everything past that.
+                    continue
+                # Skip lines before the date
+                if linedate < since:
+                    continue
+            if level:
+                if not any(level in line for level in show_levels):
+                    continue
+            print(line, end="")
+            lines_printed += 1
+
+    print(f"  (Filtered {lines_printed}/{len(lines)} lines)")
+
+
+def find_oldest_log(path: Path) -> Optional[Path]:
+    """Return the most-recently-modified log file under *path*.
+
+    Since aw-core #149 each profile runs in its own platform directory
+    (``activitywatch-<profile>``), so no filename-based profile filtering is
+    needed here — every ``.log`` file in *path* belongs to the active profile.
+    """
+    if not path.is_dir():
+        return None
+
+    logfiles = [f for f in path.iterdir() if f.is_file() and f.name.endswith(".log")]
+    if not logfiles:
+        return None
+
+    logfiles.sort(key=lambda f: f.stat().st_mtime)
+    return logfiles[-1]

aw_core/__about__.py

@@ -0,0 +1,23 @@
+# Inspired by:
+# https://github.com/pypa/pipfile/blob/master/pipfile/__about__.py
+
+
+__all__ = [
+    "__title__",
+    "__summary__",
+    "__uri__",
+    "__author__",
+    "__email__",
+    "__license__",
+    "__copyright__",
+]
+
+__title__ = "aw-core"
+__summary__ = "Core library for ActivityWatch"
+__uri__ = "https://github.com/ActivityWatch/aw-core"
+
+__author__ = "Erik Bjäreholt, Johan Bjäreholt"
+__email__ = "erik@bjareho.lt, johan@bjareho.lt"
+
+__license__ = "MPL2"
+__copyright__ = f"Copyright 2017 {__author__}"

aw_core/__init__.py

@@ -0,0 +1,29 @@
+# ignore: F401
+
+from . import __about__
+
+from . import decorators
+from . import util
+
+from . import dirs
+from . import config
+from . import log
+
+from . import models
+from .models import Event
+
+from . import schema
+
+__all__ = [
+    "__about__",
+    # Classes
+    "Event",
+    # Modules
+    "decorators",
+    "util",
+    "dirs",
+    "config",
+    "log",
+    "models",
+    "schema",
+]

aw_core/config.py

@@ -0,0 +1,116 @@
+import logging
+import os
+from typing import Union
+
+import tomlkit
+from deprecation import deprecated
+
+from aw_core import dirs
+
+logger = logging.getLogger(__name__)
+
+
+def _merge(a: dict, b: dict, path=None):
+    """
+    Recursively merges b into a, with b taking precedence.
+
+    From: https://stackoverflow.com/a/7205107/965332
+    """
+    if path is None:
+        path = []
+    for key in b:
+        if key in a:
+            if isinstance(a[key], dict) and isinstance(b[key], dict):
+                _merge(a[key], b[key], path + [str(key)])
+            elif a[key] == b[key]:
+                pass  # same leaf value
+            else:
+                a[key] = b[key]
+        else:
+            a[key] = b[key]
+    return a
+
+
+def _comment_out_toml(s: str):
+    # Only comment out keys, not headers or empty lines
+    return "\n".join(
+        [
+            "#" + line if line.strip() and not line.strip().startswith("[") else line
+            for line in s.split("\n")
+        ]
+    )
+
+
+def load_config_toml(
+    appname: str, default_config: str
+) -> Union[dict, tomlkit.container.Container]:
+    config_dir = dirs.get_config_dir(appname)
+    config_file_path = os.path.join(config_dir, f"{appname}.toml")
+
+    # Run early to ensure input is valid toml before writing
+    default_config_toml = tomlkit.parse(default_config)
+
+    # Override defaults from existing config file
+    if os.path.isfile(config_file_path):
+        with open(config_file_path) as f:
+            config = f.read()
+        config_toml = tomlkit.parse(config)
+    else:
+        # If file doesn't exist, write with commented-out default config
+        with open(config_file_path, "w") as f:
+            f.write(_comment_out_toml(default_config))
+        config_toml = dict()
+
+    config = _merge(default_config_toml, config_toml)
+
+    return config
+
+
+def save_config_toml(appname: str, config: str) -> None:
+    # Check that passed config string is valid toml
+    assert tomlkit.parse(config)
+
+    config_dir = dirs.get_config_dir(appname)
+    config_file_path = os.path.join(config_dir, f"{appname}.toml")
+
+    with open(config_file_path, "w") as f:
+        f.write(config)
+
+
+@deprecated(
+    details="Use the load_config_toml function instead",
+    deprecated_in="0.5.3",
+)
+def load_config(appname, default_config):
+    """
+    Take the defaults, and if a config file exists, use the settings specified
+    there as overrides for their respective defaults.
+    """
+    config = default_config
+
+    config_dir = dirs.get_config_dir(appname)
+    config_file_path = os.path.join(config_dir, f"{appname}.toml")
+
+    # Override defaults from existing config file
+    if os.path.isfile(config_file_path):
+        with open(config_file_path) as f:
+            config.read_file(f)
+
+    # Overwrite current config file (necessary in case new default would be added)
+    save_config(appname, config)
+
+    return config
+
+
+@deprecated(
+    details="Use the save_config_toml function instead",
+    deprecated_in="0.5.3",
+)
+def save_config(appname, config):
+    config_dir = dirs.get_config_dir(appname)
+    config_file_path = os.path.join(config_dir, f"{appname}.ini")
+    with open(config_file_path, "w") as f:
+        config.write(f)
+        # Flush and fsync to lower risk of corrupted files
+        f.flush()
+        os.fsync(f.fileno())

aw_core/decorators.py

@@ -0,0 +1,51 @@
+import functools
+import logging
+import time
+import warnings
+
+
+def deprecated(f):  # pragma: no cover
+    """
+    This is a decorator which can be used to mark functions
+    as deprecated. It will result in a warning being emitted
+    when the function is used.
+
+    Taken from: http://stackoverflow.com/a/30253848/965332
+    """
+    # Warn only once per deprecated function
+    warned_for = False
+
+    @functools.wraps(f)
+    def g(*args, **kwargs):
+        # TODO: Use logging module instead?
+        nonlocal warned_for
+        if not warned_for:
+            warnings.simplefilter("always", DeprecationWarning)  # turn off filter
+            warnings.warn(
+                f"Call to deprecated function {f.__name__}, "
+                "this warning will only show once per function.",
+                category=DeprecationWarning,
+                stacklevel=2,
+            )
+            warnings.simplefilter("default", DeprecationWarning)  # reset filter
+            warned_for = True
+        return f(*args, **kwargs)
+
+    return g
+
+
+def restart_on_exception(f, delay=1, exception=Exception):  # pragma: no cover
+    @functools.wraps(f)
+    def g(*args, **kwargs):
+        while True:
+            try:
+                f(*args, **kwargs)
+            except exception as e:
+                # TODO: Use warnings module instead?
+                logging.error(f"{f.__name__} crashed due to exception, restarting.")
+                logging.error(e)
+                time.sleep(
+                    delay
+                )  # To prevent extremely fast restarts in case of bad state.
+
+    return g

aw_core/dirs.py

@@ -0,0 +1,179 @@
+import os
+import sys
+from functools import wraps
+from typing import Callable, Optional
+
+import platformdirs
+
+GetDirFunc = Callable[[Optional[str]], str]
+
+_DEFAULT_APPNAME = "activitywatch"
+_TESTING_PROFILE = "testing"
+_TESTING_APPNAME = f"{_DEFAULT_APPNAME}-{_TESTING_PROFILE}"
+
+# Filenames that mark a machine as still using the pre-profile shared-root
+# testing layout (ActivityWatch/activitywatch#1399). Keep this list specific:
+# a false positive would pin a fresh install to the legacy layout forever.
+_LEGACY_TESTING_FILENAME_MARKERS = (
+    "peewee-sqlite-testing",
+    "sqlite-testing",
+    "settings-testing",
+    "config-testing",
+    "-testing.db",
+    "-testing.toml",
+    "-testing.json",
+    "_testing_",
+)
+
+
+def get_profile() -> str:
+    """Return the active ``AW_PROFILE`` value, or ``""`` for the default profile."""
+    return os.environ.get("AW_PROFILE", "") or ""
+
+
+def _new_testing_root_exists() -> bool:
+    """True if any platform dir for ``activitywatch-testing`` already exists.
+
+    Must not create directories: existence is the signal that a previous run
+    already adopted the isolated testing root.
+    """
+    for getter in (
+        platformdirs.user_data_dir,
+        platformdirs.user_config_dir,
+        platformdirs.user_cache_dir,
+    ):
+        if os.path.isdir(getter(_TESTING_APPNAME)):
+            return True
+    return False
+
+
+def _is_legacy_testing_filename(name: str) -> bool:
+    lower = name.lower()
+    return any(marker in lower for marker in _LEGACY_TESTING_FILENAME_MARKERS)
+
+
+def _legacy_testing_artifacts_exist() -> bool:
+    """True if testing data still lives under the shared ``activitywatch`` root."""
+    roots = (
+        platformdirs.user_data_dir(_DEFAULT_APPNAME),
+        platformdirs.user_config_dir(_DEFAULT_APPNAME),
+        platformdirs.user_cache_dir(_DEFAULT_APPNAME),
+    )
+    for root in roots:
+        if not os.path.isdir(root):
+            continue
+        for dirpath, dirnames, filenames in os.walk(root):
+            for filename in filenames:
+                if _is_legacy_testing_filename(filename):
+                    return True
+            # Descend one level (activitywatch/aw-server/...) not further.
+            if os.path.relpath(dirpath, root) != ".":
+                dirnames.clear()
+    return False
+
+
+def using_legacy_testing_root() -> bool:
+    """Whether ``AW_PROFILE=testing`` should stay on the shared ``activitywatch`` root.
+
+    Resolution rule (ActivityWatch/activitywatch#1399), identical on python and rust:
+
+    1. If ``activitywatch-testing/`` already exists: use it (new layout).
+    2. Else if legacy testing artifacts exist in the bare ``activitywatch/``
+       root: stay in legacy mode (old paths, old filenames).
+    3. Else (fresh setup): create and use ``activitywatch-testing/``.
+    """
+    if get_profile() != _TESTING_PROFILE:
+        return False
+    if _new_testing_root_exists():
+        return False
+    return _legacy_testing_artifacts_exist()
+
+
+def legacy_testing_suffix(testing: bool) -> str:
+    """Return ``"-testing"`` only when testing data shares the default root.
+
+    Isolated profile roots (including new-style ``activitywatch-testing/``) use
+    bare filenames: the directory already isolates. Suffixed names remain only
+    in legacy mode so existing ``peewee-sqlite-testing.v2.db`` files keep working.
+    """
+    if not testing:
+        return ""
+    profile = get_profile()
+    if profile == _TESTING_PROFILE and not using_legacy_testing_root():
+        return ""
+    if profile and profile != _TESTING_PROFILE:
+        # Named isolated profile — directory already isolates.
+        return ""
+    return "-testing"
+
+
+def _get_appname() -> str:
+    """Return the platformdirs appname, optionally suffixed by the active profile.
+
+    If the ``AW_PROFILE`` environment variable is set to a non-empty string the
+    appname becomes ``activitywatch-<profile>`` so that *all* platform
+    directories (data, config, cache, log) are completely separate from the
+    default profile.  An unset or empty ``AW_PROFILE`` returns the bare
+    ``"activitywatch"`` name, which is identical to the pre-profile behaviour.
+
+    The ``testing`` profile is special: see :func:`using_legacy_testing_root`.
+    Existing testing data in the shared root is not orphaned; fresh setups and
+    machines that already have ``activitywatch-testing/`` use the isolated root.
+
+    This is the single authoritative place where profile isolation is applied.
+    Every module that uses :func:`get_data_dir`, :func:`get_config_dir`,
+    :func:`get_cache_dir` or :func:`get_log_dir` automatically inherits the
+    correct root for the running profile; no ``profile=`` parameter needs to be
+    threaded through the call chain.
+    """
+    profile = get_profile()
+    if not profile:
+        return _DEFAULT_APPNAME
+    if profile == _TESTING_PROFILE and using_legacy_testing_root():
+        return _DEFAULT_APPNAME
+    return f"{_DEFAULT_APPNAME}-{profile}"
+
+
+def ensure_path_exists(path: str) -> None:
+    if not os.path.exists(path):
+        os.makedirs(path)
+
+
+def _ensure_returned_path_exists(f: GetDirFunc) -> GetDirFunc:
+    @wraps(f)
+    def wrapper(subpath: Optional[str] = None) -> str:
+        path = f(subpath)
+        ensure_path_exists(path)
+        return path
+
+    return wrapper
+
+
+@_ensure_returned_path_exists
+def get_data_dir(module_name: Optional[str] = None) -> str:
+    data_dir = platformdirs.user_data_dir(_get_appname())
+    return os.path.join(data_dir, module_name) if module_name else data_dir
+
+
+@_ensure_returned_path_exists
+def get_cache_dir(module_name: Optional[str] = None) -> str:
+    cache_dir = platformdirs.user_cache_dir(_get_appname())
+    return os.path.join(cache_dir, module_name) if module_name else cache_dir
+
+
+@_ensure_returned_path_exists
+def get_config_dir(module_name: Optional[str] = None) -> str:
+    config_dir = platformdirs.user_config_dir(_get_appname())
+    return os.path.join(config_dir, module_name) if module_name else config_dir
+
+
+@_ensure_returned_path_exists
+def get_log_dir(module_name: Optional[str] = None) -> str:  # pragma: no cover
+    # on Linux/Unix, platformdirs changed to using XDG_STATE_HOME instead of XDG_DATA_HOME for log_dir in v2.6
+    # we want to keep using XDG_DATA_HOME for backwards compatibility
+    # https://github.com/ActivityWatch/aw-core/pull/122#issuecomment-1768020335
+    if sys.platform.startswith("linux"):
+        log_dir = platformdirs.user_cache_path(_get_appname()) / "log"
+    else:
+        log_dir = platformdirs.user_log_dir(_get_appname())
+    return os.path.join(log_dir, module_name) if module_name else log_dir

aw_core/log.py

@@ -0,0 +1,129 @@
+import logging
+import os
+import sys
+from datetime import datetime
+from logging.handlers import RotatingFileHandler
+from typing import List, Optional
+
+from . import dirs
+from .decorators import deprecated
+
+# NOTE: Will be removed in a future version since it's not compatible
+#       with running a multi-service process.
+# TODO: prefix with `_`
+log_file_path = None
+
+
+@deprecated
+def get_log_file_path() -> Optional[str]:  # pragma: no cover
+    """DEPRECATED: Use get_latest_log_file instead."""
+    return log_file_path
+
+
+def setup_logging(
+    name: str,
+    testing=False,
+    verbose=False,
+    log_stderr=True,
+    log_file=False,
+):  # pragma: no cover
+    root_logger = logging.getLogger()
+    root_logger.setLevel(logging.DEBUG if verbose else logging.INFO)
+    root_logger.handlers = []
+
+    # run with LOG_LEVEL=DEBUG to customize log level across all AW components
+    log_level = os.environ.get("LOG_LEVEL")
+    if log_level:
+        if hasattr(logging, log_level.upper()):
+            root_logger.setLevel(getattr(logging, log_level.upper()))
+        else:
+            root_logger.warning(
+                f"No logging level called {log_level} (as specified in env var)"
+            )
+
+    if log_stderr:
+        root_logger.addHandler(_create_stderr_handler())
+    if log_file:
+        root_logger.addHandler(_create_file_handler(name, testing=testing))
+
+    def excepthook(type_, value, traceback):
+        root_logger.exception("Unhandled exception", exc_info=(type_, value, traceback))
+        # call the default excepthook if log_stderr isn't true
+        # (otherwise it'll just get duplicated)
+        if not log_stderr:
+            sys.__excepthook__(type_, value, traceback)
+
+    sys.excepthook = excepthook
+
+
+def _get_latest_log_files(name, testing=False) -> List[str]:  # pragma: no cover
+    """
+    Returns a list with the paths of all available logfiles for `name`,
+    sorted by latest first.
+    """
+    log_dir = dirs.get_log_dir(name)
+    files = filter(lambda filename: name in filename, os.listdir(log_dir))
+    # Isolated testing roots already separate logs by directory; the
+    # filename filter is only needed in the shared-root legacy layout.
+    isolated_testing = testing and not dirs.legacy_testing_suffix(testing)
+    if not isolated_testing:
+        files = filter(
+            lambda filename: (
+                "testing" in filename if testing else "testing" not in filename
+            ),
+            files,
+        )
+    return [os.path.join(log_dir, filename) for filename in sorted(files, reverse=True)]
+
+
+def get_latest_log_file(name, testing=False) -> Optional[str]:  # pragma: no cover
+    """
+    Returns the filename of the last logfile with ``name``.
+    Useful when you want to read the logfile of another ActivityWatch service.
+    """
+    last_logs = _get_latest_log_files(name, testing=testing)
+    return last_logs[0] if last_logs else None
+
+
+def _create_stderr_handler() -> logging.Handler:  # pragma: no cover
+    stderr_handler = logging.StreamHandler(stream=sys.stderr)
+    stderr_handler.setFormatter(_create_human_formatter())
+
+    return stderr_handler
+
+
+def _create_file_handler(
+    name, testing=False, log_json=False
+) -> logging.Handler:  # pragma: no cover
+    log_dir = dirs.get_log_dir(name)
+
+    # Set logfile path and name
+    global log_file_path
+
+    # Should result in something like:
+    # $LOG_DIR/aw-server_testing_2017-01-05T00:21:39.log
+    file_ext = ".log.json" if log_json else ".log"
+    now_str = str(datetime.now().replace(microsecond=0).isoformat()).replace(":", "-")
+    testing_prefix = (
+        "testing_" if testing and dirs.legacy_testing_suffix(testing) else ""
+    )
+    log_name = name + "_" + testing_prefix + now_str + file_ext
+    log_file_path = os.path.join(log_dir, log_name)
+
+    # Create rotating logfile handler, max 10MB per file, 3 files max
+    # Prevents logfile from growing too large, like in:
+    #  - https://github.com/ActivityWatch/activitywatch/issues/815#issue-1423555466
+    #  - https://github.com/ActivityWatch/activitywatch/issues/756#issuecomment-1266662861
+    fh = RotatingFileHandler(
+        log_file_path, mode="a", maxBytes=10 * 1024 * 1024, backupCount=3
+    )
+    fh.setFormatter(_create_human_formatter())
+
+    return fh
+
+
+def _create_human_formatter() -> logging.Formatter:  # pragma: no cover
+    return logging.Formatter(
+        "%(asctime)s [%(levelname)-5s]: %(message)s  (%(name)s:%(lineno)s)",
+        "%Y-%m-%d %H:%M:%S",
+    )

aw_core/models.py

@@ -0,0 +1,142 @@
+import json
+import logging
+import numbers
+import typing
+from datetime import datetime, timedelta, timezone
+from typing import (
+    Any,
+    Dict,
+    Optional,
+    Union,
+)
+
+import iso8601
+
+logger = logging.getLogger(__name__)
+
+
+Number = Union[int, float]
+Id = Optional[Union[int, str]]
+ConvertibleTimestamp = Union[datetime, str]
+Duration = Union[timedelta, Number]
+Data = Dict[str, Any]
+
+
+def _timestamp_parse(ts_in: ConvertibleTimestamp) -> datetime:
+    """
+    Takes something representing a timestamp and
+    returns a timestamp in the representation we want.
+    """
+    ts = iso8601.parse_date(ts_in) if isinstance(ts_in, str) else ts_in
+    # Set resolution to milliseconds instead of microseconds
+    # (Fixes incompability with software based on unix time, for example mongodb)
+    ts = ts.replace(microsecond=int(ts.microsecond / 1000) * 1000)
+    # Add timezone if not set
+    if not ts.tzinfo:
+        # Needed? All timestamps should be iso8601 so ought to always contain timezone.
+        # Yes, because it is optional in iso8601
+        logger.warning(f"timestamp without timezone found, using UTC: {ts}")
+        ts = ts.replace(tzinfo=timezone.utc)
+    return ts
+
+
+class Event(dict):
+    """
+    Used to represents an event.
+    """
+
+    def __init__(
+        self,
+        id: Optional[Id] = None,
+        timestamp: Optional[ConvertibleTimestamp] = None,
+        duration: Duration = 0,
+        data: Optional[Data] = None,
+    ) -> None:
+        self.id = id
+        if timestamp is None:
+            logger.warning(
+                "Event initializer did not receive a timestamp argument, "
+                "using now as timestamp"
+            )
+            # FIXME: The typing.cast here was required for mypy to shut up, weird...
+            self.timestamp = datetime.now(typing.cast(timezone, timezone.utc))
+        else:
+            # The conversion needs to be explicit here for mypy to pick it up
+            # (lacks support for properties)
+            self.timestamp = _timestamp_parse(timestamp)
+        self.duration = duration  # type: ignore
+        self.data = data or {}
+
+    def __eq__(self, other: object) -> bool:
+        if isinstance(other, Event):
+            return (
+                self.timestamp == other.timestamp
+                and self.duration == other.duration
+                and self.data == other.data
+            )
+        else:
+            raise TypeError(
+                f"operator not supported between instances of '{type(self)}' and '{type(other)}'"
+            )
+
+    def __lt__(self, other: object) -> bool:
+        if isinstance(other, Event):
+            return self.timestamp < other.timestamp
+        else:
+            raise TypeError(
+                f"operator not supported between instances of '{type(self)}' and '{type(other)}'"
+            )
+
+    def to_json_dict(self) -> dict:
+        """Useful when sending data over the wire.
+        Any mongodb interop should not use do this as it accepts datetimes."""
+        json_data = self.copy()
+        json_data["timestamp"] = self.timestamp.astimezone(timezone.utc).isoformat()
+        json_data["duration"] = self.duration.total_seconds()
+        return json_data
+
+    def to_json_str(self) -> str:
+        data = self.to_json_dict()
+        return json.dumps(data)
+
+    def _hasprop(self, propname: str) -> bool:
+        """Badly named, but basically checks if the underlying
+        dict has a prop, and if it is a non-empty list"""
+        return propname in self and self[propname] is not None
+
+    @property
+    def id(self) -> Id:
+        return self["id"] if self._hasprop("id") else None
+
+    @id.setter
+    def id(self, id: Id) -> None:
+        self["id"] = id
+
+    @property
+    def data(self) -> dict:
+        return self["data"] if self._hasprop("data") else {}
+
+    @data.setter
+    def data(self, data: dict) -> None:
+        self["data"] = data
+
+    @property
+    def timestamp(self) -> datetime:
+        return self["timestamp"]
+
+    @timestamp.setter
+    def timestamp(self, timestamp: ConvertibleTimestamp) -> None:
+        self["timestamp"] = _timestamp_parse(timestamp).astimezone(timezone.utc)
+
+    @property
+    def duration(self) -> timedelta:
+        return self["duration"] if self._hasprop("duration") else timedelta(0)
+
+    @duration.setter
+    def duration(self, duration: Duration) -> None:
+        if isinstance(duration, timedelta):
+            self["duration"] = duration
+        elif isinstance(duration, numbers.Real):
+            self["duration"] = timedelta(seconds=duration)  # type: ignore
+        else:
+            raise TypeError(f"Couldn't parse duration of invalid type {type(duration)}")

aw_core/schema.py

@@ -0,0 +1,20 @@
+import os
+import json
+
+
+def _this_dir() -> str:
+    return os.path.dirname(os.path.abspath(__file__))
+
+
+def _schema_dir() -> str:
+    return os.path.join(os.path.dirname(_this_dir()), "aw_core", "schemas")
+
+
+def get_json_schema(name: str) -> dict:
+    with open(os.path.join(_schema_dir(), name + ".json")) as f:
+        data = json.load(f)
+    return data
+
+
+if __name__ == "__main__":
+    print(get_json_schema("event"))

aw_core/schemas/bucket.json

@@ -0,0 +1,45 @@
+{
+    "$id": "https://activitywatch.net/schemas/bucket.v0.json",
+	"$schema": "http://json-schema.org/draft-04/schema#",
+	"title": "Bucket",
+	"description": "The Bucket model that is used in ActivityWatch",
+	"type": "object",
+	"required": ["id", "type", "client", "hostname"],
+	"properties": {
+		"id": {
+            "description": "The unique id for the bucket",
+            "type": "string"
+		},
+		"name": {
+            "description": "The readable and renameable name for the bucket",
+            "type": "string"
+		},
+		"type": {
+            "description": "The event type",
+            "type": "string"
+		},
+		"client": {
+            "description": "The name of the client that is reporting to the bucket",
+            "type": "string"
+		},
+		"hostname": {
+            "description": "The hostname of the machine on which the client is running",
+            "type": "string"
+		},
+		"created": {
+            "description": "The creation datetime of the bucket",
+            "type": "string",
+            "format": "date-time"
+		},
+        "data": {
+            "description": "",
+            "type": "object"
+        },
+        "events": {
+            "type": "array",
+            "items": {
+                "$ref": "#/definitions/Event"
+            }
+        }
+    }
+}

aw_core/schemas/event.json

@@ -0,0 +1,19 @@
+{
+	"$schema": "http://json-schema.org/draft-04/schema#",
+	"title": "Event",
+	"description": "The Event model that is used in ActivityWatch",
+	"type": "object",
+	"required": ["timestamp"],
+	"properties": {
+		"timestamp": {
+            "type": "string",
+            "format": "date-time"
+		},
+		"duration": {
+            "type": "number"
+		},
+		"data": {
+			"type": "object"
+		}
+	}
+}

aw_core/schemas/export.json

@@ -0,0 +1,16 @@
+{
+    "$id": "https://activitywatch.net/schemas/export.v0.json",
+	"$schema": "http://json-schema.org/draft-04/schema#",
+	"title": "Export",
+	"description": "The Export model that is used by ActivityWatch",
+	"type": "object",
+	"required": [],
+	"properties": {
+        "buckets": {
+            "type": "array",
+            "items": {
+                "$ref": "#/definitions/Bucket"
+            }
+        }
+    }
+}

aw_core/util.py

@@ -0,0 +1,24 @@
+import sys
+from typing import Tuple
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+class VersionException(Exception): ...
+
+
+def _version_info_tuple() -> Tuple[int, int, int]:  # pragma: no cover
+    return (sys.version_info.major, sys.version_info.minor, sys.version_info.micro)
+
+
+def assert_version(required_version: Tuple[int, ...] = (3, 5)):  # pragma: no cover
+    actual_version = _version_info_tuple()
+    if actual_version <= required_version:
+        raise VersionException(
+            (
+                "Python version {} not supported, you need to upgrade your Python"
+                + " version to at least {}."
+            ).format(required_version)
+        )
+    logger.debug(f"Python version: {_version_info_tuple()}")

aw_datastore/__init__.py

@@ -0,0 +1,19 @@
+from typing import Callable, Dict
+
+from . import storages
+from .datastore import Datastore
+from .migration import check_for_migration
+
+
+def get_storage_methods() -> Dict[str, Callable[..., storages.AbstractStorage]]:
+    from .storages import MemoryStorage, PeeweeStorage, SqliteStorage
+
+    methods: Dict[str, Callable[..., storages.AbstractStorage]] = {
+        PeeweeStorage.sid: PeeweeStorage,
+        MemoryStorage.sid: MemoryStorage,
+        SqliteStorage.sid: SqliteStorage,
+    }
+    return methods
+
+
+__all__ = ["Datastore", "get_storage_methods", "check_for_migration"]

aw_datastore/benchmark.py

@@ -0,0 +1,97 @@
+#!/usr/bin/env python3
+import sys
+from contextlib import contextmanager
+from datetime import datetime, timedelta, timezone
+from typing import Callable
+
+from aw_core.models import Event
+
+from aw_datastore import get_storage_methods
+from aw_datastore.storages import AbstractStorage
+
+td1s = timedelta(seconds=1)
+
+
+def create_test_events(n):
+    now = datetime.now(timezone.utc) - timedelta(days=1000)
+
+    events = []
+    for i in range(n):
+        events.append(
+            Event(timestamp=now + i * td1s, duration=td1s, data={"label": "asd"})
+        )
+
+    return events
+
+
+@contextmanager
+def temporary_bucket(ds):
+    bucket_id = "test_bucket"
+    try:
+        ds.delete_bucket(bucket_id)
+    except Exception:
+        pass
+    bucket = ds.create_bucket(bucket_id, "testingtype", "test-client", "testing-box")
+    yield bucket
+    ds.delete_bucket(bucket_id)
+
+
+def benchmark(storage: Callable[..., AbstractStorage]):
+    raise NotImplementedError(
+        "No longer implemented as ttt/takethetime dependency is removed"
+    )
+
+    # if storage.__name__ == "PeeweeStorage":
+    #     ds = Datastore(storage, testing=True, filepath="test.db")
+    # else:
+    #     ds = Datastore(storage, testing=True)
+
+    # num_single_events = 50
+    # num_replace_events = 50
+    # num_bulk_events = 20_000
+    # num_events = num_single_events + num_replace_events + num_bulk_events + 1
+    # num_final_events = num_single_events + num_bulk_events + 1
+
+    # events = create_test_events(num_events)
+    # single_events = events[:num_single_events]
+    # replace_events = events[num_single_events : num_single_events + num_replace_events]
+    # bulk_events = events[num_single_events + num_replace_events : -1]
+
+    # print(storage.__name__)
+
+    # with temporary_bucket(ds) as bucket:
+    #     with ttt(" sum"):
+    #         with ttt(f" single insert {num_single_events} events"):
+    #             for event in single_events:
+    #                 bucket.insert(event)
+
+    #         with ttt(f" bulk insert {num_bulk_events} events"):
+    #             bucket.insert(bulk_events)
+
+    #         with ttt(f" replace last {num_replace_events}"):
+    #             for e in replace_events:
+    #                 bucket.replace_last(e)
+
+    #         with ttt(" insert 1 event"):
+    #             bucket.insert(events[-1])
+
+    #         with ttt(" get one"):
+    #             events_tmp = bucket.get(limit=1)
+
+    #         with ttt(" get all"):
+    #             events_tmp = bucket.get(limit=-1)
+    #             assert len(events_tmp) == num_final_events
+
+    #         with ttt(" get range"):
+    #             events_tmp = bucket.get(
+    #                 limit=-1,
+    #                 starttime=events[1].timestamp + 0.01 * td1s,
+    #                 endtime=events[-1].timestamp + events[-1].duration,
+    #             )
+    #             assert len(events_tmp) == num_final_events - 1
+
+
+if __name__ == "__main__":
+    for storage in get_storage_methods().values():
+        if len(sys.argv) <= 1 or storage.__name__ in sys.argv:
+            benchmark(storage)

aw_datastore/datastore.py

@@ -0,0 +1,189 @@
+import logging
+from datetime import datetime, timedelta, timezone
+from typing import (
+    Callable,
+    Dict,
+    List,
+    Optional,
+    Union,
+)
+
+from aw_core.models import Event
+
+from .storages import AbstractStorage
+
+logger = logging.getLogger(__name__)
+
+
+class Datastore:
+    def __init__(
+        self,
+        storage_strategy: Callable[..., AbstractStorage],
+        testing=False,
+        **kwargs,
+    ) -> None:
+        self.logger = logger.getChild("Datastore")
+        self.bucket_instances: Dict[str, Bucket] = dict()
+
+        self.storage_strategy = storage_strategy(testing=testing, **kwargs)
+
+    def __repr__(self):
+        return f"<Datastore object using {self.storage_strategy.__class__.__name__}>"
+
+    def __getitem__(self, bucket_id: str) -> "Bucket":
+        # If this bucket doesn't have a initialized object, create it
+        if bucket_id not in self.bucket_instances:
+            # If the bucket exists in the database, create an object representation of it
+            if bucket_id in self.buckets():
+                bucket = Bucket(self, bucket_id)
+                self.bucket_instances[bucket_id] = bucket
+            else:
+                self.logger.error(
+                    f"Cannot create a Bucket object for {bucket_id} because it doesn't exist in the database"
+                )
+                raise KeyError
+
+        return self.bucket_instances[bucket_id]
+
+    def create_bucket(
+        self,
+        bucket_id: str,
+        type: str,
+        client: str,
+        hostname: str,
+        created: Optional[datetime] = None,
+        name: Optional[str] = None,
+        data: Optional[dict] = None,
+    ) -> "Bucket":
+        created = created or datetime.now(timezone.utc)
+        self.logger.info(f"Creating bucket '{bucket_id}'")
+        self.storage_strategy.create_bucket(
+            bucket_id, type, client, hostname, created.isoformat(), name=name, data=data
+        )
+        return self[bucket_id]
+
+    def update_bucket(self, bucket_id: str, **kwargs):
+        self.logger.info(f"Updating bucket '{bucket_id}'")
+        return self.storage_strategy.update_bucket(bucket_id, **kwargs)
+
+    def delete_bucket(self, bucket_id: str):
+        self.logger.info(f"Deleting bucket '{bucket_id}'")
+        if bucket_id in self.bucket_instances:
+            del self.bucket_instances[bucket_id]
+        return self.storage_strategy.delete_bucket(bucket_id)
+
+    def buckets(self):
+        return self.storage_strategy.buckets()
+
+
+class Bucket:
+    def __init__(self, datastore: Datastore, bucket_id: str) -> None:
+        self.logger = logger.getChild("Bucket")
+        self.ds = datastore
+        self.bucket_id = bucket_id
+
+    def metadata(self) -> dict:
+        return self.ds.storage_strategy.get_metadata(self.bucket_id)
+
+    def get(
+        self,
+        limit: int = -1,
+        starttime: Optional[datetime] = None,
+        endtime: Optional[datetime] = None,
+    ) -> List[Event]:
+        """Returns events sorted in descending order by timestamp"""
+        # Resolution is rounded down since not all datastores like microsecond precision
+        if starttime:
+            starttime = starttime.replace(
+                microsecond=1000 * int(starttime.microsecond / 1000)
+            )
+        if endtime:
+            # Rounding up here in order to ensure events aren't missed
+            # second_offset and microseconds modulo required since replace() only takes microseconds up to 999999 (doesn't handle overflow)
+            milliseconds = 1 + int(endtime.microsecond / 1000)
+            second_offset = int(milliseconds / 1000)  # usually 0, rarely 1
+            microseconds = (
+                (1000 * milliseconds) % 1000000
+            )  # will likely just be 1000 * milliseconds, if it overflows it would become zero
+            endtime = endtime.replace(microsecond=microseconds) + timedelta(
+                seconds=second_offset
+            )
+
+        return self.ds.storage_strategy.get_events(
+            self.bucket_id, limit, starttime, endtime
+        )
+
+    def get_by_id(self, event_id) -> Optional[Event]:
+        """Will return the event with the provided ID, or None if not found."""
+        return self.ds.storage_strategy.get_event(self.bucket_id, event_id)
+
+    def get_eventcount(
+        self, starttime: Optional[datetime] = None, endtime: Optional[datetime] = None
+    ) -> int:
+        return self.ds.storage_strategy.get_eventcount(
+            self.bucket_id, starttime, endtime
+        )
+
+    def insert(self, events: Union[Event, List[Event]]) -> Optional[Event]:
+        """
+        Inserts one or several events.
+        If a single event is inserted, return the event with its id assigned.
+        If several events are inserted, returns None. (This is due to there being no efficient way of getting ids out when doing bulk inserts with some datastores such as peewee/SQLite)
+        """
+
+        # NOTE: Should we keep the timestamp checking?
+        warn_older_event = False
+
+        # Get last event for timestamp check after insert
+        if warn_older_event:
+            last_event_list = self.get(1)
+            last_event = None
+            if last_event_list:
+                last_event = last_event_list[0]
+
+        now = datetime.now(tz=timezone.utc)
+
+        inserted: Optional[Event] = None
+
+        # Call insert
+        if isinstance(events, Event):
+            oldest_event: Optional[Event] = events
+            if events.timestamp + events.duration > now:
+                self.logger.warning(
+                    f"Event inserted into bucket {self.bucket_id} reaches into the future. Current UTC time: {str(now)}. Event data: {str(events)}"
+                )
+            inserted = self.ds.storage_strategy.insert_one(self.bucket_id, events)
+            # assert inserted
+        elif isinstance(events, list):
+            if events:
+                oldest_event = sorted(events, key=lambda k: k["timestamp"])[0]
+            else:  # pragma: no cover
+                oldest_event = None
+            for event in events:
+                if event.timestamp + event.duration > now:
+                    self.logger.warning(
+                        f"Event inserted into bucket {self.bucket_id} reaches into the future. Current UTC time: {str(now)}. Event data: {str(event)}"
+                    )
+            self.ds.storage_strategy.insert_many(self.bucket_id, events)
+        else:
+            raise TypeError
+
+        # Warn if timestamp is older than last event
+        if warn_older_event and last_event and oldest_event:
+            if oldest_event.timestamp < last_event.timestamp:  # pragma: no cover
+                self.logger.warning(
+                    f"""Inserting event that has a older timestamp than previous event!
+Previous: {last_event}
+Inserted: {oldest_event}"""
+                )
+
+        return inserted
+
+    def delete(self, event_id):
+        return self.ds.storage_strategy.delete(self.bucket_id, event_id)
+
+    def replace_last(self, event):
+        return self.ds.storage_strategy.replace_last(self.bucket_id, event)
+
+    def replace(self, event_id, event):
+        return self.ds.storage_strategy.replace(self.bucket_id, event_id, event)

aw_datastore/migration.py

@@ -0,0 +1,62 @@
+import logging
+import os
+from typing import List, Optional
+
+from aw_core.dirs import get_data_dir, legacy_testing_suffix
+
+from .storages import AbstractStorage
+
+logger = logging.getLogger(__name__)
+
+
+def detect_db_files(
+    data_dir: str, datastore_name: Optional[str] = None, version=None
+) -> List[str]:
+    db_files = [filename for filename in os.listdir(data_dir)]
+    if datastore_name:
+        db_files = [
+            filename
+            for filename in db_files
+            if filename.split(".")[0] == datastore_name
+        ]
+    if version:
+        db_files = [
+            filename for filename in db_files if filename.split(".")[1] == f"v{version}"
+        ]
+    return db_files
+
+
+def check_for_migration(datastore: AbstractStorage):
+    data_dir = get_data_dir("aw-server")
+
+    if datastore.sid == "sqlite":
+        peewee_type = "peewee-sqlite"
+        peewee_name = peewee_type + legacy_testing_suffix(datastore.testing)
+        # Migrate from peewee v2
+        peewee_db_v2 = detect_db_files(data_dir, peewee_name, 2)
+        if len(peewee_db_v2) > 0:
+            peewee_v2_to_sqlite_v1(datastore)
+
+
+def peewee_v2_to_sqlite_v1(datastore):
+    logger.info("Migrating database from peewee v2 to sqlite v1")
+    from .storages import PeeweeStorage
+
+    pw_db = PeeweeStorage(datastore.testing)
+    # Fetch buckets and events
+    buckets = pw_db.buckets()
+    # Insert buckets and events to new db
+    for bucket_id in buckets:
+        logger.info(f"Migrating bucket {bucket_id}")
+        bucket = buckets[bucket_id]
+        datastore.create_bucket(
+            bucket["id"],
+            bucket["type"],
+            bucket["client"],
+            bucket["hostname"],
+            bucket["created"],
+            bucket["name"],
+        )
+        bucket_events = pw_db.get_events(bucket_id, -1)
+        datastore.insert_many(bucket_id, bucket_events)
+    logger.info("Migration of peewee v2 to sqlite v1 finished")

aw_datastore/storages/__init__.py

@@ -0,0 +1,15 @@
+import logging as _logging
+
+logger: _logging.Logger = _logging.getLogger(__name__)
+
+from .abstract import AbstractStorage
+from .memory import MemoryStorage
+from .peewee import PeeweeStorage
+from .sqlite import SqliteStorage
+
+__all__ = [
+    "AbstractStorage",
+    "MemoryStorage",
+    "PeeweeStorage",
+    "SqliteStorage",
+]

aw_datastore/storages/abstract.py

@@ -0,0 +1,101 @@
+from abc import ABCMeta, abstractmethod
+from datetime import datetime
+from typing import Dict, List, Optional
+
+from aw_core.models import Event
+
+
+class AbstractStorage(metaclass=ABCMeta):
+    """
+    Interface for storage methods.
+    """
+
+    sid = "Storage id not set, fix me"
+
+    @abstractmethod
+    def __init__(self, testing: bool) -> None:
+        self.testing = True
+        raise NotImplementedError
+
+    @abstractmethod
+    def buckets(self) -> Dict[str, dict]:
+        raise NotImplementedError
+
+    @abstractmethod
+    def create_bucket(
+        self,
+        bucket_id: str,
+        type_id: str,
+        client: str,
+        hostname: str,
+        created: str,
+        name: Optional[str] = None,
+        data: Optional[dict] = None,
+    ) -> None:
+        raise NotImplementedError
+
+    @abstractmethod
+    def update_bucket(
+        self,
+        bucket_id: str,
+        type_id: Optional[str] = None,
+        client: Optional[str] = None,
+        hostname: Optional[str] = None,
+        name: Optional[str] = None,
+        data: Optional[dict] = None,
+    ) -> None:
+        raise NotImplementedError
+
+    @abstractmethod
+    def delete_bucket(self, bucket_id: str) -> None:
+        raise NotImplementedError
+
+    @abstractmethod
+    def get_metadata(self, bucket_id: str) -> dict:
+        raise NotImplementedError
+
+    @abstractmethod
+    def get_event(
+        self,
+        bucket_id: str,
+        event_id: int,
+    ) -> Optional[Event]:
+        raise NotImplementedError
+
+    @abstractmethod
+    def get_events(
+        self,
+        bucket_id: str,
+        limit: int,
+        starttime: Optional[datetime] = None,
+        endtime: Optional[datetime] = None,
+    ) -> List[Event]:
+        raise NotImplementedError
+
+    def get_eventcount(
+        self,
+        bucket_id: str,
+        starttime: Optional[datetime] = None,
+        endtime: Optional[datetime] = None,
+    ) -> int:
+        raise NotImplementedError
+
+    @abstractmethod
+    def insert_one(self, bucket_id: str, event: Event) -> Event:
+        raise NotImplementedError
+
+    def insert_many(self, bucket_id: str, events: List[Event]) -> None:
+        for event in events:
+            self.insert_one(bucket_id, event)
+
+    @abstractmethod
+    def delete(self, bucket_id: str, event_id: int) -> bool:
+        raise NotImplementedError
+
+    @abstractmethod
+    def replace(self, bucket_id: str, event_id: int, event: Event) -> bool:
+        raise NotImplementedError
+
+    @abstractmethod
+    def replace_last(self, bucket_id: str, event: Event) -> None:
+        raise NotImplementedError

aw_datastore/storages/memory.py

@@ -0,0 +1,187 @@
+import copy
+import sys
+from datetime import datetime
+from typing import Dict, List, Optional
+
+from aw_core.models import Event
+
+from . import logger
+from .abstract import AbstractStorage
+
+
+class MemoryStorage(AbstractStorage):
+    """For storage of data in-memory, useful primarily in testing"""
+
+    sid = "memory"
+
+    def __init__(self, testing: bool) -> None:
+        self.logger = logger.getChild(self.sid)
+        # self.logger.warning("Using in-memory storage, any events stored will not be persistent and will be lost when server is shut down. Use the --storage parameter to set a different storage method.")
+        self.db: Dict[str, List[Event]] = {}
+        self._metadata: Dict[str, dict] = dict()
+
+    def create_bucket(
+        self,
+        bucket_id,
+        type_id,
+        client,
+        hostname,
+        created,
+        name=None,
+        data=None,
+    ) -> None:
+        if not name:
+            name = bucket_id
+        self._metadata[bucket_id] = {
+            "id": bucket_id,
+            "name": name,
+            "type": type_id,
+            "client": client,
+            "hostname": hostname,
+            "created": created,
+            "data": data or {},
+        }
+        self.db[bucket_id] = []
+
+    def update_bucket(
+        self,
+        bucket_id: str,
+        type_id: Optional[str] = None,
+        client: Optional[str] = None,
+        hostname: Optional[str] = None,
+        name: Optional[str] = None,
+        data: Optional[dict] = None,
+    ) -> None:
+        if bucket_id in self._metadata:
+            if type_id:
+                self._metadata[bucket_id]["type"] = type_id
+            if client:
+                self._metadata[bucket_id]["client"] = client
+            if hostname:
+                self._metadata[bucket_id]["hostname"] = hostname
+            if name:
+                self._metadata[bucket_id]["name"] = name
+            if data:
+                self._metadata[bucket_id]["data"] = data
+        else:
+            raise ValueError("Bucket did not exist, could not update")
+
+    def delete_bucket(self, bucket_id: str) -> None:
+        if bucket_id in self.db:
+            del self.db[bucket_id]
+        if bucket_id in self._metadata:
+            del self._metadata[bucket_id]
+        else:
+            raise ValueError("Bucket did not exist, could not delete")
+
+    def buckets(self):
+        buckets = dict()
+        for bucket_id in self.db:
+            buckets[bucket_id] = self.get_metadata(bucket_id)
+        return buckets
+
+    def get_event(
+        self,
+        bucket_id: str,
+        event_id: int,
+    ) -> Optional[Event]:
+        event = self._get_event(bucket_id, event_id)
+        return copy.deepcopy(event)
+
+    def get_events(
+        self,
+        bucket: str,
+        limit: int,
+        starttime: Optional[datetime] = None,
+        endtime: Optional[datetime] = None,
+    ) -> List[Event]:
+        events = self.db[bucket]
+
+        # Sort by timestamp
+        events = sorted(events, key=lambda k: k["timestamp"])[::-1]
+
+        # Filter by date
+        if starttime:
+            events = [e for e in events if starttime <= (e.timestamp + e.duration)]
+        if endtime:
+            events = [e for e in events if e.timestamp <= endtime]
+
+        # Limit
+        if limit == 0:
+            return []
+        elif limit < 0:
+            limit = sys.maxsize
+        events = events[:limit]
+        # Return
+        return copy.deepcopy(events)
+
+    def get_eventcount(
+        self,
+        bucket: str,
+        starttime: Optional[datetime] = None,
+        endtime: Optional[datetime] = None,
+    ) -> int:
+        return len(
+            [
+                e
+                for e in self.db[bucket]
+                if (not starttime or starttime <= e.timestamp)
+                and (not endtime or e.timestamp <= endtime)
+            ]
+        )
+
+    def get_metadata(self, bucket_id: str):
+        if bucket_id in self._metadata:
+            return self._metadata[bucket_id]
+        else:
+            raise ValueError("Bucket did not exist, could not get metadata")
+
+    def insert_one(self, bucket: str, event: Event) -> Event:
+        if event.id is not None:
+            self.replace(bucket, event.id, event)
+        else:
+            # We need to copy the event to avoid setting the ID on the passed event
+            event = copy.copy(event)
+            if self.db[bucket]:
+                event.id = max(int(e.id or 0) for e in self.db[bucket]) + 1
+            else:
+                event.id = 0
+            self.db[bucket].append(event)
+        return event
+
+    def delete(self, bucket_id, event_id):
+        for idx in (
+            idx
+            for idx, event in reversed(list(enumerate(self.db[bucket_id])))
+            if event.id == event_id
+        ):
+            self.db[bucket_id].pop(idx)
+            return True
+        return False
+
+    def _get_event(self, bucket_id, event_id) -> Optional[Event]:
+        events = [
+            event
+            for idx, event in reversed(list(enumerate(self.db[bucket_id])))
+            if event.id == event_id
+        ]
+        if len(events) < 1:
+            return None
+        else:
+            return events[0]
+
+    def replace(self, bucket_id, event_id, event):
+        for idx in (
+            idx
+            for idx, event in reversed(list(enumerate(self.db[bucket_id])))
+            if event.id == event_id
+        ):
+            # We need to copy the event to avoid setting the ID on the passed event
+            event = copy.copy(event)
+            event.id = event_id
+            self.db[bucket_id][idx] = event
+
+    def replace_last(self, bucket_id, event):
+        # NOTE: This does not actually get the most recent event, only the last inserted
+        last = sorted(self.db[bucket_id], key=lambda e: e.timestamp)[-1]
+        self.replace(bucket_id, last.id, event)

aw_datastore/storages/peewee.py

@@ -0,0 +1,451 @@
+import json
+import logging
+import os
+from datetime import datetime, timedelta, timezone
+from typing import (
+    Any,
+    Dict,
+    List,
+    Optional,
+)
+
+import iso8601
+from aw_core.dirs import get_data_dir, legacy_testing_suffix
+from aw_core.models import Event
+from playhouse.migrate import SqliteMigrator, migrate
+
+try:
+    from playhouse.sqlite_ext import SqliteExtDatabase
+except ImportError:
+    # peewee 4 is missing SqliteExtDatabase
+    from peewee import SqliteDatabase as SqliteExtDatabase
+
+import peewee
+from peewee import (
+    AutoField,
+    CharField,
+    DateTimeField,
+    DecimalField,
+    ForeignKeyField,
+    IntegerField,
+    Model,
+)
+
+from .abstract import AbstractStorage
+
+logger = logging.getLogger(__name__)
+
+# Prevent debug output from propagating
+peewee_logger = logging.getLogger("peewee")
+peewee_logger.setLevel(logging.INFO)
+
+# Init'd later in the PeeweeStorage constructor.
+#   See: http://docs.peewee-orm.com/en/latest/peewee/database.html#run-time-database-configuration
+# Another option would be to use peewee's Proxy.
+#   See: http://docs.peewee-orm.com/en/latest/peewee/database.html#dynamic-db
+_db = SqliteExtDatabase(None)
+
+
+LATEST_VERSION = 2
+
+
+def auto_migrate(path: str) -> None:
+    db = SqliteExtDatabase(path)
+    migrator = SqliteMigrator(db)
+
+    # check if bucketmodel has datastr field
+    info = db.execute_sql("PRAGMA table_info(bucketmodel)")
+    has_datastr = any(row[1] == "datastr" for row in info)
+
+    if not has_datastr:
+        datastr_field = CharField(default="{}")
+        with db.atomic():
+            migrate(migrator.add_column("bucketmodel", "datastr", datastr_field))
+
+    # Replace the single-column event indexes with a composite index.
+    # Every event query filters on bucket_id and orders by timestamp (or
+    # narrows to a timestamp range), so the composite index serves the seek,
+    # the range and the ORDER BY in one pass. With only single-column indexes
+    # SQLite reads all of a bucket's rows and sorts them in a temp B-tree,
+    # even for the LIMIT 1 last-event query issued on every heartbeat merge.
+    # Dropping the old indexes also makes inserts cheaper.
+    indexes = {row[1] for row in db.execute_sql("PRAGMA index_list(eventmodel)")}
+    if "eventmodel_bucket_id_timestamp" not in indexes:
+        logger.info(
+            "Migrating event indexes, this can take a few minutes on large databases..."
+        )
+        # Drops run before the create so the pages they free are reused to
+        # build the new index within the same transaction; creating first
+        # would permanently grow the database file by the new index's size.
+        with db.atomic():
+            db.execute_sql("DROP INDEX IF EXISTS eventmodel_bucket_id")
+            db.execute_sql("DROP INDEX IF EXISTS eventmodel_timestamp")
+            db.execute_sql(
+                "CREATE INDEX eventmodel_bucket_id_timestamp"
+                " ON eventmodel (bucket_id, timestamp)"
+            )
+
+    db.close()
+
+
+def chunks(ls, n):
+    """Yield successive n-sized chunks from ls.
+    From: https://stackoverflow.com/a/312464/965332"""
+    for i in range(0, len(ls), n):
+        yield ls[i : i + n]
+
+
+def dt_plus_duration(dt, duration):
+    # See peewee docs on datemath: https://docs.peewee-orm.com/en/latest/peewee/hacks.html#date-math
+    return peewee.fn.strftime(
+        "%Y-%m-%d %H:%M:%f+00:00",
+        (peewee.fn.julianday(dt) - 2440587.5) * 86400.0 + duration,
+        "unixepoch",
+    )
+
+
+class BaseModel(Model):
+    class Meta:
+        database = _db
+
+
+class BucketModel(BaseModel):
+    key = IntegerField(primary_key=True)
+    id = CharField(unique=True)
+    created = DateTimeField(default=lambda: datetime.now(timezone.utc))
+    name = CharField(null=True)
+    type = CharField()
+    client = CharField()
+    hostname = CharField()
+    datastr = CharField(null=True)  # JSON-encoded object
+
+    def json(self):
+        return {
+            "id": self.id,
+            "created": iso8601.parse_date(self.created)
+            .astimezone(timezone.utc)
+            .isoformat(),
+            "name": self.name,
+            "type": self.type,
+            "client": self.client,
+            "hostname": self.hostname,
+            "data": json.loads(self.datastr) if self.datastr else {},
+        }
+
+
+class EventModel(BaseModel):
+    # Events are covered by the composite (bucket_id, timestamp) index created
+    # in auto_migrate(); it is kept out of the model so create_table doesn't
+    # build it before auto_migrate has dropped the old single-column indexes.
+    id = AutoField()
+    bucket = ForeignKeyField(BucketModel, backref="events", index=False)
+    timestamp = DateTimeField(default=lambda: datetime.now(timezone.utc))
+    duration = DecimalField()
+    datastr = CharField()
+
+    @classmethod
+    def from_event(cls, bucket_key, event: Event):
+        return cls(
+            bucket=bucket_key,
+            id=event.id,
+            timestamp=event.timestamp,
+            duration=event.duration.total_seconds(),
+            datastr=json.dumps(event.data),
+        )
+
+    def json(self):
+        return {
+            "id": self.id,
+            "timestamp": self.timestamp,
+            "duration": float(self.duration),
+            "data": json.loads(self.datastr),
+        }
+
+
+class PeeweeStorage(AbstractStorage):
+    sid = "peewee"
+
+    def __init__(self, testing: bool = True, filepath: Optional[str] = None) -> None:
+        data_dir = get_data_dir("aw-server")
+
+        if not filepath:
+            filename = (
+                "peewee-sqlite"
+                + legacy_testing_suffix(testing)
+                + f".v{LATEST_VERSION}"
+                + ".db"
+            )
+            filepath = os.path.join(data_dir, filename)
+        self.db = _db
+        self.db.init(
+            filepath,
+            pragmas={
+                "journal_mode": "wal",
+            },
+        )
+        logger.info(f"Using database file: {filepath}")
+        self.db.connect()
+
+        self.bucket_keys: Dict[str, int] = {}
+        BucketModel.create_table(safe=True)
+        EventModel.create_table(safe=True)
+
+        # Migrate database if needed, requires closing the connection first
+        self.db.close()
+        auto_migrate(filepath)
+        self.db.connect()
+
+        # Update bucket keys
+        self.update_bucket_keys()
+
+    def update_bucket_keys(self) -> None:
+        buckets = BucketModel.select()
+        self.bucket_keys = {bucket.id: bucket.key for bucket in buckets}
+
+    def buckets(self) -> Dict[str, Dict[str, Any]]:
+        return {bucket.id: bucket.json() for bucket in BucketModel.select()}
+
+    def create_bucket(
+        self,
+        bucket_id: str,
+        type_id: str,
+        client: str,
+        hostname: str,
+        created: str,
+        name: Optional[str] = None,
+        data: Optional[Dict[str, Any]] = None,
+    ):
+        BucketModel.create(
+            id=bucket_id,
+            type=type_id,
+            client=client,
+            hostname=hostname,
+            created=created,
+            name=name,
+            datastr=json.dumps(data or {}),
+        )
+        self.update_bucket_keys()
+
+    def update_bucket(
+        self,
+        bucket_id: str,
+        type_id: Optional[str] = None,
+        client: Optional[str] = None,
+        hostname: Optional[str] = None,
+        name: Optional[str] = None,
+        data: Optional[dict] = None,
+    ) -> None:
+        if bucket_id in self.bucket_keys:
+            bucket = BucketModel.get(BucketModel.key == self.bucket_keys[bucket_id])
+
+            if type_id is not None:
+                bucket.type = type_id
+            if client is not None:
+                bucket.client = client
+            if hostname is not None:
+                bucket.hostname = hostname
+            if name is not None:
+                bucket.name = name
+            if data is not None:
+                bucket.datastr = json.dumps(data)  # Encoding data dictionary to JSON
+
+            bucket.save()
+        else:
+            raise ValueError("Bucket did not exist, could not update")
+
+    def delete_bucket(self, bucket_id: str) -> None:
+        if bucket_id in self.bucket_keys:
+            EventModel.delete().where(
+                EventModel.bucket == self.bucket_keys[bucket_id]
+            ).execute()
+            BucketModel.delete().where(
+                BucketModel.key == self.bucket_keys[bucket_id]
+            ).execute()
+            self.update_bucket_keys()
+        else:
+            raise ValueError("Bucket did not exist, could not delete")
+
+    def get_metadata(self, bucket_id: str):
+        if bucket_id in self.bucket_keys:
+            bucket = BucketModel.get(
+                BucketModel.key == self.bucket_keys[bucket_id]
+            ).json()
+            return bucket
+        else:
+            raise ValueError("Bucket did not exist, could not get metadata")
+
+    def insert_one(self, bucket_id: str, event: Event) -> Event:
+        e = EventModel.from_event(self.bucket_keys[bucket_id], event)
+        e.save()
+        event.id = e.id
+        return event
+
+    def insert_many(self, bucket_id, events: List[Event]) -> None:
+        # NOTE: Events need to be handled differently depending on
+        #       if they're upserts or inserts (have id's or not).
+
+        # These events are updates which need to be applied one by one
+        events_updates = [e for e in events if e.id is not None]
+        for e in events_updates:
+            self.insert_one(bucket_id, e)
+
+        # These events can be inserted with insert_many
+        events_dictlist = [
+            {
+                "bucket": self.bucket_keys[bucket_id],
+                "timestamp": event.timestamp,
+                "duration": event.duration.total_seconds(),
+                "datastr": json.dumps(event.data),
+            }
+            for event in events
+            if event.id is None
+        ]
+
+        # Chunking into lists of length 100 is needed here due to SQLITE_MAX_COMPOUND_SELECT
+        # and SQLITE_LIMIT_VARIABLE_NUMBER under Windows.
+        # See: https://github.com/coleifer/peewee/issues/948
+        for chunk in chunks(events_dictlist, 100):
+            EventModel.insert_many(chunk).execute()
+
+    def _get_event(self, bucket_id, event_id) -> Optional[EventModel]:
+        try:
+            return (
+                EventModel.select()
+                .where(EventModel.id == event_id)
+                .where(EventModel.bucket == self.bucket_keys[bucket_id])
+                .get()
+            )
+        except peewee.DoesNotExist:
+            return None
+
+    def _get_last(self, bucket_id) -> EventModel:
+        return (
+            EventModel.select()
+            .where(EventModel.bucket == self.bucket_keys[bucket_id])
+            .order_by(EventModel.timestamp.desc())
+            .get()
+        )
+
+    def replace_last(self, bucket_id, event):
+        e = self._get_last(bucket_id)
+        e.timestamp = event.timestamp
+        e.duration = event.duration.total_seconds()
+        e.datastr = json.dumps(event.data)
+        e.save()
+        event.id = e.id
+        return event
+
+    def delete(self, bucket_id, event_id):
+        return (
+            EventModel.delete()
+            .where(EventModel.id == event_id)
+            .where(EventModel.bucket == self.bucket_keys[bucket_id])
+            .execute()
+        )
+
+    def replace(self, bucket_id, event_id, event):
+        e = self._get_event(bucket_id, event_id)
+        e.timestamp = event.timestamp
+        e.duration = event.duration.total_seconds()
+        e.datastr = json.dumps(event.data)
+        e.save()
+        event.id = e.id
+        return event
+
+    def get_event(
+        self,
+        bucket_id: str,
+        event_id: int,
+    ) -> Optional[Event]:
+        """
+        Fetch a single event from a bucket.
+        """
+        res = self._get_event(bucket_id, event_id)
+        return Event(**EventModel.json(res)) if res else None
+
+    def get_events(
+        self,
+        bucket_id: str,
+        limit: int,
+        starttime: Optional[datetime] = None,
+        endtime: Optional[datetime] = None,
+    ):
+        """
+        Fetch events from a certain bucket, optionally from a given range of time.
+
+        Example raw query:
+
+            SELECT strftime(
+              "%Y-%m-%d %H:%M:%f+00:00",
+              ((julianday(timestamp) - 2440587.5) * 86400),
+              'unixepoch'
+            )
+            FROM eventmodel
+            WHERE eventmodel.timestamp > '2021-06-20'
+            LIMIT 10;
+
+        """
+        if limit == 0:
+            return []
+        q = (
+            EventModel.select()
+            .where(EventModel.bucket == self.bucket_keys[bucket_id])
+            .order_by(EventModel.timestamp.desc())
+            .limit(limit)
+        )
+
+        q = self._where_range(q, starttime, endtime)
+
+        res = q.execute()
+        events = [Event(**e) for e in list(map(EventModel.json, res))]
+
+        # Trim events that are out of range (as done in aw-server-rust)
+        # TODO: Do the same for the other storage methods
+        for e in events:
+            if starttime:
+                if e.timestamp < starttime:
+                    e_end = e.timestamp + e.duration
+                    e.timestamp = starttime
+                    e.duration = e_end - e.timestamp
+            if endtime:
+                if e.timestamp + e.duration > endtime:
+                    e.duration = endtime - e.timestamp
+
+        return events
+
+    def get_eventcount(
+        self,
+        bucket_id: str,
+        starttime: Optional[datetime] = None,
+        endtime: Optional[datetime] = None,
+    ) -> int:
+        q = EventModel.select().where(EventModel.bucket == self.bucket_keys[bucket_id])
+        q = self._where_range(q, starttime, endtime)
+        return q.count()
+
+    def _where_range(
+        self,
+        q,
+        starttime: Optional[datetime] = None,
+        endtime: Optional[datetime] = None,
+    ):
+        # Important to normalize datetimes to UTC, otherwise any UTC offset will be ignored
+        if starttime:
+            starttime = starttime.astimezone(timezone.utc)
+        if endtime:
+            endtime = endtime.astimezone(timezone.utc)
+
+        if starttime:
+            # Faster WHERE to speed up slow query below, leads to ~2-3x speedup
+            # We'll assume events aren't >24h
+            q = q.where(starttime - timedelta(hours=24) <= EventModel.timestamp)
+
+            # This can be slow on large databases...
+            # Tried creating various indexes and using SQLite's unlikely() function, but it had no effect
+            q = q.where(
+                starttime <= dt_plus_duration(EventModel.timestamp, EventModel.duration)
+            )
+        if endtime:
+            q = q.where(EventModel.timestamp <= endtime)
+
+        return q

aw_datastore/storages/sqlite.py

@@ -0,0 +1,378 @@
+import json
+import logging
+import os
+import sqlite3
+from datetime import datetime, timedelta, timezone
+from typing import Iterable, List, Optional
+
+from aw_core.dirs import get_data_dir, legacy_testing_suffix
+from aw_core.models import Event
+
+from .abstract import AbstractStorage
+
+logger = logging.getLogger(__name__)
+
+LATEST_VERSION = 1
+
+# The max integer value in SQLite is signed 8 Bytes / 64 bits
+MAX_TIMESTAMP = 2**63 - 1
+
+CREATE_BUCKETS_TABLE = """
+    CREATE TABLE IF NOT EXISTS buckets (
+        rowid INTEGER PRIMARY KEY AUTOINCREMENT,
+        id TEXT UNIQUE NOT NULL,
+        name TEXT,
+        type TEXT NOT NULL,
+        client TEXT NOT NULL,
+        hostname TEXT NOT NULL,
+        created TEXT NOT NULL,
+        datastr TEXT NOT NULL
+    )
+"""
+
+CREATE_EVENTS_TABLE = """
+    CREATE TABLE IF NOT EXISTS events (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        bucketrow INTEGER NOT NULL,
+        starttime INTEGER NOT NULL,
+        endtime INTEGER NOT NULL,
+        datastr TEXT NOT NULL,
+        FOREIGN KEY (bucketrow) REFERENCES buckets(rowid)
+    )
+"""
+
+INDEX_BUCKETS_TABLE_ID = """
+    CREATE INDEX IF NOT EXISTS event_index_id ON events(id);
+"""
+
+INDEX_EVENTS_TABLE_STARTTIME = """
+    CREATE INDEX IF NOT EXISTS event_index_starttime ON events(bucketrow, starttime);
+"""
+INDEX_EVENTS_TABLE_ENDTIME = """
+    CREATE INDEX IF NOT EXISTS event_index_endtime ON events(bucketrow, endtime);
+"""
+
+
+def _rows_to_events(rows: Iterable) -> List[Event]:
+    events = []
+    for row in rows:
+        eid = row[0]
+        starttime = datetime.fromtimestamp(row[1] / 1000000, timezone.utc)
+        endtime = datetime.fromtimestamp(row[2] / 1000000, timezone.utc)
+        duration = endtime - starttime
+        data = json.loads(row[3])
+        events.append(Event(id=eid, timestamp=starttime, duration=duration, data=data))
+    return events
+
+
+class SqliteStorage(AbstractStorage):
+    sid = "sqlite"
+
+    def __init__(
+        self, testing, filepath: Optional[str] = None, enable_lazy_commit=True
+    ) -> None:
+        self.testing = testing
+        self.enable_lazy_commit = enable_lazy_commit
+
+        # Ignore the migration check if custom filepath is set
+        ignore_migration_check = filepath is not None
+
+        ds_name = self.sid + legacy_testing_suffix(testing)
+        if not filepath:
+            data_dir = get_data_dir("aw-server")
+            filename = ds_name + f".v{LATEST_VERSION}" + ".db"
+            filepath = os.path.join(data_dir, filename)
+
+        new_db_file = not os.path.exists(filepath)
+        self.conn = sqlite3.connect(filepath)
+        logger.info(f"Using database file: {filepath}")
+
+        # Create tables
+        self.conn.execute(CREATE_BUCKETS_TABLE)
+        self.conn.execute(CREATE_EVENTS_TABLE)
+        self.conn.execute(INDEX_BUCKETS_TABLE_ID)
+        self.conn.execute(INDEX_EVENTS_TABLE_STARTTIME)
+        self.conn.execute(INDEX_EVENTS_TABLE_ENDTIME)
+        self.conn.execute("PRAGMA journal_mode=WAL;")
+        self.commit()
+
+        if new_db_file and not ignore_migration_check:
+            logger.info("Created new SQlite db file")
+
+            from aw_datastore import check_for_migration  # fmt: skip
+
+            check_for_migration(self)
+
+        self.last_commit = datetime.now()
+        self.num_uncommitted_statements = 0
+
+    def commit(self):
+        """
+        Useful for debugging and trying to lower the amount of
+        unnecessary commits
+        """
+        self.conn.commit()
+        self.last_commit = datetime.now()
+        self.num_uncommitted_statements = 0
+
+    def conditional_commit(self, num_statements):
+        """
+        Only commit transactions if:
+         - We have a lot of statements in our transaction
+         - Was a while ago since last commit
+        This is because sqlite is very slow with small inserts, this
+        is a way to batch them together and lower CPU+disk usage
+        """
+        if self.enable_lazy_commit:
+            self.num_uncommitted_statements += num_statements
+            if self.num_uncommitted_statements > 50:
+                self.commit()
+            if (self.last_commit - datetime.now()) > timedelta(seconds=10):
+                self.commit()
+        else:
+            self.commit()
+
+    def buckets(self):
+        buckets = {}
+        c = self.conn.cursor()
+        for row in c.execute(
+            "SELECT id, name, type, client, hostname, created, datastr FROM buckets"
+        ):
+            buckets[row[0]] = {
+                "id": row[0],
+                "name": row[1],
+                "type": row[2],
+                "client": row[3],
+                "hostname": row[4],
+                "created": row[5],
+                "data": json.loads(row[6] or "{}"),
+            }
+        return buckets
+
+    def create_bucket(
+        self,
+        bucket_id: str,
+        type_id: str,
+        client: str,
+        hostname: str,
+        created: str,
+        name: Optional[str] = None,
+        data: Optional[dict] = None,
+    ):
+        self.conn.execute(
+            "INSERT INTO buckets(id, name, type, client, hostname, created, datastr) "
+            + "VALUES (?, ?, ?, ?, ?, ?, ?)",
+            [
+                bucket_id,
+                name,
+                type_id,
+                client,
+                hostname,
+                created,
+                json.dumps(data or {}),
+            ],
+        )
+        self.commit()
+        return self.get_metadata(bucket_id)
+
+    def update_bucket(
+        self,
+        bucket_id: str,
+        type_id: Optional[str] = None,
+        client: Optional[str] = None,
+        hostname: Optional[str] = None,
+        name: Optional[str] = None,
+        data: Optional[dict] = None,
+    ):
+        update_values = [
+            ("type", type_id),
+            ("client", client),
+            ("hostname", hostname),
+            ("name", name),
+            ("datastr", json.dumps(data) if data is not None else None),
+        ]
+        updates, values = zip(*[(k, v) for k, v in update_values if v is not None])
+        if not updates:
+            raise ValueError("At least one field must be updated.")
+
+        sql = (
+            "UPDATE buckets SET "
+            + ", ".join(f"{u} = ?" for u in updates)
+            + " WHERE id = ?"
+        )
+        self.conn.execute(sql, (*values, bucket_id))
+        self.commit()
+        return self.get_metadata(bucket_id)
+
+    def delete_bucket(self, bucket_id: str):
+        self.conn.execute(
+            "DELETE FROM events WHERE bucketrow IN (SELECT rowid FROM buckets WHERE id = ?)",
+            [bucket_id],
+        )
+        cursor = self.conn.execute("DELETE FROM buckets WHERE id = ?", [bucket_id])
+        self.commit()
+        if cursor.rowcount != 1:
+            raise ValueError("Bucket did not exist, could not delete")
+
+    def get_metadata(self, bucket_id: str):
+        c = self.conn.cursor()
+        res = c.execute(
+            "SELECT id, name, type, client, hostname, created, datastr FROM buckets WHERE id = ?",
+            [bucket_id],
+        )
+        row = res.fetchone()
+        if row is not None:
+            return {
+                "id": row[0],
+                "name": row[1],
+                "type": row[2],
+                "client": row[3],
+                "hostname": row[4],
+                "created": row[5],
+                "data": json.loads(row[6] or "{}"),
+            }
+        else:
+            raise ValueError("Bucket did not exist, could not get metadata")
+
+    def insert_one(self, bucket_id: str, event: Event) -> Event:
+        c = self.conn.cursor()
+        starttime = event.timestamp.timestamp() * 1000000
+        endtime = starttime + (event.duration.total_seconds() * 1000000)
+        datastr = json.dumps(event.data)
+        c.execute(
+            "INSERT INTO events(bucketrow, starttime, endtime, datastr) "
+            + "VALUES ((SELECT rowid FROM buckets WHERE id = ?), ?, ?, ?)",
+            [bucket_id, starttime, endtime, datastr],
+        )
+        event.id = c.lastrowid
+        self.conditional_commit(1)
+        return event
+
+    def insert_many(self, bucket_id, events: List[Event]) -> None:
+        # FIXME: Is this true not only for peewee but sqlite aswell?
+        # Chunking into lists of length 100 is needed here due to SQLITE_MAX_COMPOUND_SELECT
+        # and SQLITE_LIMIT_VARIABLE_NUMBER under Windows.
+        # See: https://github.com/coleifer/peewee/issues/948
+
+        # First, upsert events with id's set
+        events_upsert = [e for e in events if e.id is not None]
+        for e in events_upsert:
+            self.replace(bucket_id, e.id, e)
+
+        # Then insert events without id's set
+        events_insert = [e for e in events if e.id is None]
+        event_rows = []
+        for event in events_insert:
+            starttime = event.timestamp.timestamp() * 1000000
+            endtime = starttime + (event.duration.total_seconds() * 1000000)
+            datastr = json.dumps(event.data)
+            event_rows.append((bucket_id, starttime, endtime, datastr))
+        query = (
+            "INSERT INTO events(bucketrow, starttime, endtime, datastr) "
+            + "VALUES ((SELECT rowid FROM buckets WHERE id = ?), ?, ?, ?)"
+        )
+        self.conn.executemany(query, event_rows)
+        self.conditional_commit(len(event_rows))
+
+    def replace_last(self, bucket_id, event):
+        starttime = event.timestamp.timestamp() * 1000000
+        endtime = starttime + (event.duration.total_seconds() * 1000000)
+        datastr = json.dumps(event.data)
+        query = """UPDATE events
+                   SET starttime = ?, endtime = ?, datastr = ?
+                   WHERE id = (
+                        SELECT id FROM events WHERE endtime =
+                            (SELECT max(endtime) FROM events WHERE bucketrow =
+                                (SELECT rowid FROM buckets WHERE id = ?) LIMIT 1))"""
+        self.conn.execute(query, [starttime, endtime, datastr, bucket_id])
+        self.conditional_commit(1)
+        return True
+
+    def delete(self, bucket_id, event_id):
+        query = (
+            "DELETE FROM events "
+            + "WHERE id = ? AND bucketrow = (SELECT b.rowid FROM buckets b WHERE b.id = ?)"
+        )
+        cursor = self.conn.execute(query, [event_id, bucket_id])
+        return cursor.rowcount == 1
+
+    def replace(self, bucket_id, event_id, event) -> bool:
+        starttime = event.timestamp.timestamp() * 1000000
+        endtime = starttime + (event.duration.total_seconds() * 1000000)
+        datastr = json.dumps(event.data)
+        query = """UPDATE events
+                     SET bucketrow = (SELECT rowid FROM buckets WHERE id = ?),
+                         starttime = ?,
+                         endtime = ?,
+                         datastr = ?
+                     WHERE id = ?"""
+        self.conn.execute(query, [bucket_id, starttime, endtime, datastr, event_id])
+        self.conditional_commit(1)
+        return True
+
+    def get_event(
+        self,
+        bucket_id: str,
+        event_id: int,
+    ) -> Optional[Event]:
+        self.commit()
+        c = self.conn.cursor()
+        query = """
+            SELECT id, starttime, endtime, datastr
+            FROM events
+            WHERE bucketrow = (SELECT rowid FROM buckets WHERE id = ?) AND id = ?
+            LIMIT 1
+        """
+        rows = c.execute(query, [bucket_id, event_id])
+        events = _rows_to_events(rows)
+        if events:
+            return events[0]
+        else:
+            return None
+
+    def get_events(
+        self,
+        bucket_id: str,
+        limit: int,
+        starttime: Optional[datetime] = None,
+        endtime: Optional[datetime] = None,
+    ):
+        if limit == 0:
+            return []
+        elif limit < 0:
+            limit = -1
+        self.commit()
+        c = self.conn.cursor()
+        starttime_i = starttime.timestamp() * 1000000 if starttime else 0
+        endtime_i = endtime.timestamp() * 1000000 if endtime else MAX_TIMESTAMP
+        query = """
+            SELECT id, starttime, endtime, datastr
+            FROM events
+            WHERE bucketrow = (SELECT rowid FROM buckets WHERE id = ?)
+            AND endtime >= ? AND starttime <= ?
+            ORDER BY endtime DESC LIMIT ?
+        """
+        rows = c.execute(query, [bucket_id, starttime_i, endtime_i, limit])
+        events = _rows_to_events(rows)
+        return events
+
+    def get_eventcount(
+        self,
+        bucket_id: str,
+        starttime: Optional[datetime] = None,
+        endtime: Optional[datetime] = None,
+    ):
+        self.commit()
+        c = self.conn.cursor()
+        starttime_i = starttime.timestamp() * 1000000 if starttime else 0
+        endtime_i = endtime.timestamp() * 1000000 if endtime else MAX_TIMESTAMP
+        query = (
+            "SELECT count(*) "
+            + "FROM events "
+            + "WHERE bucketrow = (SELECT rowid FROM buckets WHERE id = ?) "
+            + "AND endtime >= ? AND starttime <= ?"
+        )
+        rows = c.execute(query, [bucket_id, starttime_i, endtime_i])
+        row = rows.fetchone()
+        eventcount = row[0]
+        return eventcount

aw_query/__init__.py

@@ -0,0 +1,3 @@
+from .query2 import query
+
+__all__ = ["query"]

aw_query/exceptions.py

@@ -0,0 +1,14 @@
+class QueryException(Exception):
+    pass
+
+
+class QueryFunctionException(QueryException):
+    pass
+
+
+class QueryParseException(QueryException):
+    pass
+
+
+class QueryInterpretException(QueryException):
+    pass

aw_query/functions.py

@@ -0,0 +1,350 @@
+from datetime import timedelta
+from functools import wraps
+from inspect import signature
+from typing import (
+    Any,
+    Callable,
+    Dict,
+    List,
+    Optional,
+)
+
+import iso8601
+from aw_core.models import Event
+from aw_datastore import Datastore
+from aw_transform import (
+    Rule,
+    categorize,
+    chunk_events_by_key,
+    concat,
+    filter_keyvals,
+    filter_keyvals_regex,
+    filter_period_intersect,
+    flood,
+    limit_events,
+    merge_events_by_keys,
+    merge_subwatcher_fields,
+    period_union,
+    simplify_string,
+    sort_by_duration,
+    sort_by_timestamp,
+    split_url_events,
+    sum_durations,
+    tag,
+    union_no_overlap,
+)
+
+from .exceptions import QueryFunctionException
+
+
+def _verify_bucket_exists(datastore, bucketname):
+    if bucketname in datastore.buckets():
+        return
+    else:
+        raise QueryFunctionException(f"There's no bucket named '{bucketname}'")
+
+
+def _verify_variable_is_type(variable, t):
+    if not isinstance(variable, t):
+        raise QueryFunctionException(
+            f"Variable '{variable}' passed to function call is of invalid type. Expected {t} but was {type(variable)}"
+        )
+
+
+# TODO: proper type checking (typecheck-decorator in pypi?)
+
+
+TNamespace = Dict[str, Any]
+TQueryFunction = Callable[..., Any]
+
+
+"""
+    Declarations
+"""
+functions: Dict[str, TQueryFunction] = {}
+
+
+def q2_function(transform_func=None):
+    """
+    Decorator used to register query functions.
+
+    Automatically adds mock arguments for Datastore and TNamespace
+    if not in function signature.
+    """
+
+    def h(f):
+        sig = signature(f)
+        # If function lacks docstring, use docstring from underlying function in aw_transform
+        if transform_func and transform_func.__doc__ and not f.__doc__:
+            f.__doc__ = f".. note:: Documentation automatically copied from underlying function `aw_transform.{transform_func.__name__}`\n\n{transform_func.__doc__}"
+
+        @wraps(f)
+        def g(datastore: Datastore, namespace: TNamespace, *args, **kwargs):
+            # Remove datastore and namespace argument for functions that don't need it
+            args = (datastore, namespace, *args)
+            if TNamespace not in (sig.parameters[p].annotation for p in sig.parameters):
+                args = (args[0], *args[2:])
+            if Datastore not in (sig.parameters[p].annotation for p in sig.parameters):
+                args = args[1:]
+            return f(*args, **kwargs)
+
+        fname = f.__name__
+        if fname[:3] == "q2_":
+            fname = fname[3:]
+        functions[fname] = g
+        return g
+
+    return h
+
+
+def q2_typecheck(f):
+    """Decorator that typechecks using `_verify_variable_is_type`"""
+    sig = signature(f)
+
+    @wraps(f)
+    def g(*args, **kwargs):
+        # FIXME: If the first argument passed to a query2 function is a straight [] then the second argument disappears from the argument list for unknown reasons, which breaks things
+        for i, p in enumerate(sig.parameters):
+            param = sig.parameters[p]
+
+            # print(f"Checking that param ({param}) was {param.annotation}, value: {args[i]}")
+            # FIXME: Won't check keyword arguments
+            if (
+                param.annotation in [list, str, int, float]
+                and param.default == param.empty
+            ):
+                _verify_variable_is_type(args[i], param.annotation)
+
+        return f(*args, **kwargs)
+
+    return g
+
+
+"""
+    Getting buckets
+"""
+
+
+@q2_function()
+@q2_typecheck
+def q2_find_bucket(
+    datastore: Datastore, filter_str: str, hostname: Optional[str] = None
+):
+    """Find bucket by using a filter_str (to avoid hardcoding bucket names)"""
+    for bucket in datastore.buckets():
+        if filter_str in bucket:
+            bucket_metadata = datastore[bucket].metadata()
+            if hostname:
+                if bucket_metadata["hostname"] == hostname:
+                    return bucket
+            else:
+                return bucket
+    raise QueryFunctionException(
+        f"Unable to find bucket matching '{filter_str}' (hostname filter set to '{hostname}')"
+    )
+
+
+"""
+    Data gathering functions
+"""
+
+
+@q2_function()
+@q2_typecheck
+def q2_query_bucket(
+    datastore: Datastore, namespace: TNamespace, bucketname: str
+) -> List[Event]:
+    _verify_bucket_exists(datastore, bucketname)
+    try:
+        starttime = iso8601.parse_date(namespace["STARTTIME"])
+        endtime = iso8601.parse_date(namespace["ENDTIME"])
+    except iso8601.ParseError:
+        raise QueryFunctionException(
+            "Unable to parse starttime/endtime for query_bucket"
+        ) from None
+    return datastore[bucketname].get(starttime=starttime, endtime=endtime)
+
+
+@q2_function()
+@q2_typecheck
+def q2_query_bucket_eventcount(
+    datastore: Datastore, namespace: TNamespace, bucketname: str
+) -> int:
+    _verify_bucket_exists(datastore, bucketname)
+    starttime = iso8601.parse_date(namespace["STARTTIME"])
+    endtime = iso8601.parse_date(namespace["ENDTIME"])
+    return datastore[bucketname].get_eventcount(starttime=starttime, endtime=endtime)
+
+
+"""
+    Filtering functions
+"""
+
+
+@q2_function(filter_keyvals)
+@q2_typecheck
+def q2_filter_keyvals(events: list, key: str, vals: list) -> List[Event]:
+    return filter_keyvals(events, key, vals, False)
+
+
+@q2_function(filter_keyvals)
+@q2_typecheck
+def q2_exclude_keyvals(events: list, key: str, vals: list) -> List[Event]:
+    return filter_keyvals(events, key, vals, True)
+
+
+@q2_function(filter_keyvals_regex)
+@q2_typecheck
+def q2_filter_keyvals_regex(events: list, key: str, regex: str) -> List[Event]:
+    return filter_keyvals_regex(events, key, regex)
+
+
+@q2_function(filter_period_intersect)
+@q2_typecheck
+def q2_filter_period_intersect(events: list, filterevents: list) -> List[Event]:
+    return filter_period_intersect(events, filterevents)
+
+
+@q2_function(period_union)
+@q2_typecheck
+def q2_period_union(events1: list, events2: list) -> List[Event]:
+    return period_union(events1, events2)
+
+
+@q2_function(limit_events)
+@q2_typecheck
+def q2_limit_events(events: list, count: int) -> List[Event]:
+    return limit_events(events, count)
+
+
+"""
+    Merge functions
+"""
+
+
+@q2_function(merge_events_by_keys)
+@q2_typecheck
+def q2_merge_events_by_keys(events: list, keys: list) -> List[Event]:
+    return merge_events_by_keys(events, keys)
+
+
+@q2_function(merge_subwatcher_fields)
+@q2_typecheck
+def q2_merge_subwatcher_fields(
+    base_events: list, subwatcher_events: list, keys: list, conflict: str = "base_wins"
+) -> List[Event]:
+    try:
+        return merge_subwatcher_fields(base_events, subwatcher_events, keys, conflict)
+    except ValueError as exc:
+        raise QueryFunctionException(str(exc)) from None
+
+
+@q2_function(chunk_events_by_key)
+@q2_typecheck
+def q2_chunk_events_by_key(events: list, key: str) -> List[Event]:
+    return chunk_events_by_key(events, key)
+
+
+"""
+    Sort functions
+"""
+
+
+@q2_function(sort_by_timestamp)
+@q2_typecheck
+def q2_sort_by_timestamp(events: list) -> List[Event]:
+    return sort_by_timestamp(events)
+
+
+@q2_function(sort_by_duration)
+@q2_typecheck
+def q2_sort_by_duration(events: list) -> List[Event]:
+    return sort_by_duration(events)
+
+
+"""
+    Summarizing functions
+"""
+
+
+@q2_function(sum_durations)
+@q2_typecheck
+def q2_sum_durations(events: list) -> timedelta:
+    return sum_durations(events)
+
+
+@q2_function(concat)
+@q2_typecheck
+def q2_concat(events1: list, events2: list) -> List[Event]:
+    return concat(events1, events2)
+
+
+@q2_function(union_no_overlap)
+@q2_typecheck
+def q2_union_no_overlap(events1: list, events2: list) -> List[Event]:
+    return union_no_overlap(events1, events2)
+
+
+"""
+    Flood functions
+"""
+
+
+@q2_function(flood)
+@q2_typecheck
+def q2_flood(events: list, pulsetime: float = 5) -> List[Event]:
+    return flood(events, pulsetime)
+
+
+"""
+    Watcher specific functions
+"""
+
+
+@q2_function(split_url_events)
+@q2_typecheck
+def q2_split_url_events(events: list) -> List[Event]:
+    return split_url_events(events)
+
+
+@q2_function(simplify_string)
+@q2_typecheck
+def q2_simplify_window_titles(events: list, key: str) -> List[Event]:
+    return simplify_string(events, key=key)
+
+
+"""
+    Test functions
+"""
+
+
+@q2_function()
+@q2_typecheck
+def q2_nop():
+    """No operation function for unittesting"""
+    return 1
+
+
+"""
+    Classify
+"""
+
+
+@q2_function(categorize)
+@q2_typecheck
+def q2_categorize(events: list, classes: list):
+    try:
+        classes = [(_cls, Rule(rule_dict)) for _cls, rule_dict in classes]
+    except ValueError as exc:
+        raise QueryFunctionException(str(exc)) from None
+    return categorize(events, classes)
+
+
+@q2_function(tag)
+@q2_typecheck
+def q2_tag(events: list, classes: list):
+    try:
+        classes = [(_cls, Rule(rule_dict)) for _cls, rule_dict in classes]
+    except ValueError as exc:
+        raise QueryFunctionException(str(exc)) from None
+    return tag(events, classes)

aw_query/query2.py

@@ -0,0 +1,448 @@
+import logging
+from abc import abstractmethod
+from datetime import datetime
+from typing import (
+    Any,
+    Dict,
+    List,
+    Optional,
+    Sequence,
+    Tuple,
+    Type,
+)
+
+from aw_datastore import Datastore
+
+from .exceptions import QueryInterpretException, QueryParseException
+from .functions import functions
+
+logger = logging.getLogger(__name__)
+
+
+class QToken:
+    @abstractmethod
+    def interpret(self, datastore: Datastore, namespace: dict):
+        raise NotImplementedError
+
+    @staticmethod
+    @abstractmethod
+    def parse(string: str, namespace: dict):
+        raise NotImplementedError
+
+    @staticmethod
+    @abstractmethod
+    def check(string: str) -> Tuple[Optional[str], str]:
+        raise NotImplementedError
+
+
+class QInteger(QToken):
+    def __init__(self, value) -> None:
+        self.value = value
+
+    def interpret(self, datastore: Datastore, namespace: dict):
+        return self.value
+
+    @staticmethod
+    def parse(string: str, namespace: dict) -> QToken:
+        return QInteger(int(string))
+
+    @staticmethod
+    def check(string: str):
+        token = ""
+        for char in string:
+            if char.isdigit():
+                token += char
+            else:
+                break
+        return token, string[len(token) :]
+
+
+class QVariable(QToken):
+    def __init__(self, name, value) -> None:
+        self.name = name
+        self.value = value
+
+    def interpret(self, datastore: Datastore, namespace: dict):
+        if self.name not in namespace:
+            raise QueryInterpretException(
+                f"Tried to reference variable '{self.name}' which is not defined"
+            )
+        namespace[self.name] = self.value
+        return self.value
+
+    @staticmethod
+    def parse(string: str, namespace: dict) -> QToken:
+        val = None
+        if string in namespace:
+            val = namespace[string]
+        return QVariable(string, val)
+
+    @staticmethod
+    def check(string: str):
+        token = ""
+        for i, char in enumerate(string):
+            if char.isalpha() or char == "_":
+                token += char
+            elif i != 0 and char.isdigit():
+                token += char
+            else:
+                break
+        return token, string[len(token) :]
+
+
+class QString(QToken):
+    def __init__(self, value):
+        self.value = value
+
+    def interpret(self, datastore: Datastore, namespace: dict):
+        return self.value
+
+    @staticmethod
+    def parse(string: str, namespace: dict) -> "QString":
+        quotes_type = string[0]
+        string = string.replace("\\" + quotes_type, quotes_type)
+        string = string[1:-1]
+        return QString(string)
+
+    @staticmethod
+    def check(string: str):
+        token = ""
+        quotes_type = string[0]
+        if quotes_type != '"' and quotes_type != "'":
+            return token, string
+        token += quotes_type
+        prev_char = None
+        for char in string[1:]:
+            token += char
+            if (
+                char == quotes_type and prev_char != "\\"
+            ):  # escape quote_type with backslash
+                break
+            prev_char = char
+        if token[-1] != quotes_type or len(token) < 2:
+            # Unclosed string?
+            raise QueryParseException("Failed to parse string")
+        return token, string[len(token) :]
+
+
+class QFunction(QToken):
+    def __init__(self, name, args):
+        self.name = name
+        self.args = args
+
+    def interpret(self, datastore: Datastore, namespace: dict):
+        if self.name not in functions:
+            raise QueryInterpretException(
+                f"Tried to call function '{self.name}' which doesn't exist"
+            )
+        call_args = [datastore, namespace]
+        for arg in self.args:
+            call_args.append(arg.interpret(datastore, namespace))
+        # logger.debug("Arguments for functioncall to {} is {}".format(self.name, call_args))
+        try:
+            result = functions[self.name](*call_args)  # type: ignore
+        except TypeError:
+            raise QueryInterpretException(
+                f"Tried to call function {self.name} with invalid amount of arguments"
+            ) from None
+        return result
+
+    @staticmethod
+    def parse(string: str, namespace: dict) -> QToken:
+        arg_start = 0
+        arg_end = len(string) - 1
+        # Find opening bracket
+        for char in string:
+            if char == "(":
+                break
+            arg_start = arg_start + 1
+        # Parse name
+        name = string[:arg_start]
+        # Parse arguments
+        args = []
+        args_str = string[arg_start + 1 : arg_end]
+        while args_str:
+            (arg_t, arg), args_str = _parse_token(args_str, namespace)
+            comma = args_str.find(",")
+            if comma != -1:
+                args_str = args_str[comma + 1 :]
+            args.append(arg_t.parse(arg, namespace))
+        return QFunction(name, args)
+
+    @staticmethod
+    def check(string: str):
+        i = 0
+        # Find opening bracket
+        found = False
+        for char in string:
+            if char.isalpha() or char == "_":
+                i = i + 1
+            elif i != 0 and char.isdigit():
+                i = i + 1
+            elif char == "(":
+                i = i + 1
+                found = True
+                break
+            else:
+                break
+        if not found:
+            return None, string
+        to_consume = 1
+        single_quote = False
+        double_quote = False
+        prev_char = None
+        for char in string[i:]:
+            i = i + 1
+            if char == "'" and prev_char != "\\" and not double_quote:
+                single_quote = not single_quote
+            elif char == '"' and prev_char != "\\" and not single_quote:
+                double_quote = not double_quote
+            elif single_quote or double_quote:
+                pass
+            elif i != 0 and char.isdigit():
+                pass
+            elif char == "(":
+                to_consume += 1
+            elif char == ")":
+                to_consume -= 1
+            if to_consume == 0:
+                break
+            prev_char = char
+        if to_consume != 0:
+            return None, string
+        return string[:i], string[i + 1 :]
+
+
+class QDict(QToken):
+    def __init__(self, value: dict) -> None:
+        self.value = value
+
+    def interpret(self, datastore: Datastore, namespace: dict):
+        expanded_dict = {}
+        for key, value in self.value.items():
+            expanded_dict[key] = value.interpret(datastore, namespace)
+        return expanded_dict
+
+    @staticmethod
+    def parse(string: str, namespace: dict) -> QToken:
+        entries_str = string[1:-1]
+        d: Dict[str, QToken] = {}
+        while len(entries_str) > 0:
+            entries_str = entries_str.strip()
+            if len(d) > 0 and entries_str[0] == ",":
+                entries_str = entries_str[1:]
+            # parse key
+            (key_t, key_str), entries_str = _parse_token(entries_str, namespace)
+            if key_t != QString:
+                raise QueryParseException("Key in dict is not a str")
+            key = QString.parse(key_str, {}).value
+            entries_str = entries_str.strip()
+            # Remove :
+            if entries_str[0] != ":":
+                raise QueryParseException("Key in dict is not followed by a :")
+            entries_str = entries_str[1:]
+            # parse val
+            (val_t, val_str), entries_str = _parse_token(entries_str, namespace)
+            if not val_t:
+                raise QueryParseException("Dict expected a value, got nothing")
+            val = val_t.parse(val_str, namespace)
+            # set
+            d[key] = val
+        return QDict(d)
+
+    @staticmethod
+    def check(string: str):
+        if string[0] != "{":
+            return None, string
+        # Find closing bracket
+        i = 1
+        to_consume = 1
+        single_quote = False
+        double_quote = False
+        prev_char = None
+        for char in string[i:]:
+            i += 1
+            if char == "'" and prev_char != "\\" and not double_quote:
+                single_quote = not single_quote
+            elif char == '"' and prev_char != "\\" and not single_quote:
+                double_quote = not double_quote
+            elif single_quote or double_quote:
+                pass
+            elif char == "}":
+                to_consume = to_consume - 1
+            elif char == "{":
+                to_consume = to_consume + 1
+            if to_consume == 0:
+                break
+            prev_char = char
+        return string[:i], string[i + 1 :]
+
+
+class QList(QToken):
+    def __init__(self, value: list) -> None:
+        self.value = value
+
+    def interpret(self, datastore: Datastore, namespace: dict):
+        expanded_list = []
+        for value in self.value:
+            expanded_list.append(value.interpret(datastore, namespace))
+        return expanded_list
+
+    @staticmethod
+    def parse(string: str, namespace: dict) -> QToken:
+        entries_str = string[1:-1]
+        ls: List[QToken] = []
+        while len(entries_str) > 0:
+            entries_str = entries_str.strip()
+            if len(ls) > 0 and entries_str[0] == ",":
+                entries_str = entries_str[1:]
+            # parse
+            (val_t, val_str), entries_str = _parse_token(entries_str, namespace)
+            if not val_t:
+                raise QueryParseException("List expected a value, got nothing")
+            val = val_t.parse(val_str, namespace)
+            # set
+            ls.append(val)
+        return QList(ls)
+
+    @staticmethod
+    def check(string: str):
+        if string[0] != "[":
+            return None, string
+        # Find closing bracket
+        i = 1
+        to_consume = 1
+        single_quote = False
+        double_quote = False
+        prev_char = None
+        for char in string[i:]:
+            i += 1
+            if char == "'" and prev_char != "\\" and not double_quote:
+                single_quote = not single_quote
+            elif char == '"' and prev_char != "\\" and not single_quote:
+                double_quote = not double_quote
+            elif double_quote or single_quote:
+                pass
+            elif char == "]":
+                to_consume = to_consume - 1
+            elif char == "[":
+                to_consume = to_consume + 1
+            if to_consume == 0:
+                break
+            prev_char = char
+        return string[:i], string[i + 1 :]
+
+
+qtypes: Sequence[Type[QToken]] = [QString, QInteger, QFunction, QDict, QList, QVariable]
+
+
+def _parse_token(string: str, namespace: dict) -> Tuple[Tuple[Any, str], str]:
+    # TODO: The whole parsing thing is shoddily written, needs a rewrite from ground-up
+    if not isinstance(string, str):
+        raise QueryParseException(
+            "Reached unreachable, cannot parse something that isn't a string"
+        )
+    if len(string) == 0:
+        return (None, ""), string
+    string = string.strip()
+    token = None
+    t = None  # Declare so we can return it
+    for t in qtypes:
+        token, string = t.check(string)
+        if token:
+            break
+    if not token:
+        raise QueryParseException(f"Syntax error: {string}")
+    return (t, token), string
+
+
+def create_namespace() -> dict:
+    namespace = {
+        "True": True,
+        "False": False,
+        "true": True,
+        "false": False,
+    }
+    return namespace
+
+
+def parse(line, namespace):
+    separator_i = line.find("=")
+    var_str = line[:separator_i]
+    val_str = line[separator_i + 1 :]
+    if not val_str:
+        # TODO: Proper message
+        raise QueryParseException("Nothing to assign")
+    (var_t, var), var_str = _parse_token(var_str, namespace)
+    var_str = var_str.strip()
+    if var_str:  # Didn't consume whole var string
+        raise QueryParseException("Invalid syntax for assignment variable")
+    if var_t is not QVariable:
+        raise QueryParseException("Cannot assign to a non-variable")
+    (val_t, val), var_str = _parse_token(val_str, namespace)
+    if var_str:  # Didn't consume whole val string
+        raise QueryParseException("Invalid syntax for value to assign")
+    # Parse token
+    var = var_t.parse(var, namespace)
+    val = val_t.parse(val, namespace)
+    return var, val
+
+
+def interpret(var, val, namespace, datastore):
+    namespace[var.name] = val.interpret(datastore, namespace)
+    # logger.debug("Set {} to {}".format(var.name, namespace[var.name]))
+
+
+def get_return(namespace):
+    if "RETURN" not in namespace:
+        raise QueryParseException(
+            "Query doesn't assign the RETURN variable, nothing to respond"
+        )
+    return namespace["RETURN"]
+
+
+def _split_query_statements(query: str) -> List[str]:
+    """Split query into statements on semicolons, ignoring semicolons inside string literals."""
+    statements = []
+    current = ""
+    in_single_quote = False
+    in_double_quote = False
+    prev_char = None
+
+    for char in query:
+        if char == "'" and prev_char != "\\" and not in_double_quote:
+            in_single_quote = not in_single_quote
+            current += char
+        elif char == '"' and prev_char != "\\" and not in_single_quote:
+            in_double_quote = not in_double_quote
+            current += char
+        elif char == ";" and not in_single_quote and not in_double_quote:
+            statements.append(current)
+            current = ""
+        else:
+            current += char
+        prev_char = char
+
+    if current:
+        statements.append(current)
+    return statements
+
+
+def query(
+    name: str, query: str, starttime: datetime, endtime: datetime, datastore: Datastore
+) -> Any:
+    namespace = create_namespace()
+    namespace["NAME"] = name
+    namespace["STARTTIME"] = starttime.isoformat()
+    namespace["ENDTIME"] = endtime.isoformat()
+
+    query_stmts = _split_query_statements(query)
+    for statement in query_stmts:
+        statement = statement.strip()
+        if statement:
+            logger.debug("Parsing: " + statement)
+            var, val = parse(statement, namespace)
+            interpret(var, val, namespace, datastore)
+
+    result = get_return(namespace)
+    return result

aw_transform/__init__.py

@@ -0,0 +1,44 @@
+from .filter_keyvals import filter_keyvals, filter_keyvals_regex
+from .filter_period_intersect import filter_period_intersect, period_union, union
+from .heartbeats import heartbeat_merge, heartbeat_reduce
+from .merge_events_by_keys import merge_events_by_keys
+from .chunk_events_by_key import chunk_events_by_key
+from .sort_by import (
+    sort_by_timestamp,
+    sort_by_duration,
+    sum_durations,
+    concat,
+    limit_events,
+)
+from .split_url_events import split_url_events
+from .merge_subwatcher_fields import merge_subwatcher_fields
+from .simplify import simplify_string
+from .flood import flood
+from .classify import categorize, tag, Rule
+from .union_no_overlap import union_no_overlap
+
+__all__ = [
+    "flood",
+    "concat",
+    "categorize",
+    "tag",
+    "Rule",
+    "period_union",
+    "filter_period_intersect",
+    "union",
+    "union_no_overlap",
+    "concat",
+    "sum_durations",
+    "sort_by_timestamp",
+    "sort_by_duration",
+    "heartbeat_reduce",
+    "heartbeat_merge",
+    "merge_events_by_keys",
+    "merge_subwatcher_fields",
+    "chunk_events_by_key",
+    "limit_events",
+    "filter_keyvals",
+    "filter_keyvals_regex",
+    "split_url_events",
+    "simplify_string",
+]

aw_transform/chunk_events_by_key.py

@@ -0,0 +1,39 @@
+import logging
+from datetime import timedelta
+from typing import List
+
+from aw_core.models import Event
+
+logger = logging.getLogger(__name__)
+
+
+def chunk_events_by_key(
+    events: List[Event], key: str, pulsetime: float = 5.0
+) -> List[Event]:
+    """
+    "Chunks" adjacent events together which have the same value for a key, and stores the
+    original events in the :code:`subevents` key of the new event.
+    """
+    chunked_events: List[Event] = []
+    for event in events:
+        if key not in event.data:
+            break
+        timediff = timedelta(seconds=999999999)  # FIXME: ugly but works
+        if len(chunked_events) > 0:
+            timediff = event.timestamp - (events[-1].timestamp + events[-1].duration)
+        if (
+            len(chunked_events) > 0
+            and chunked_events[-1].data[key] == event.data[key]
+            and timediff < timedelta(seconds=pulsetime)
+        ):
+            chunked_event = chunked_events[-1]
+            chunked_event.duration += event.duration
+            chunked_event.data["subevents"].append(event)
+        else:
+            data = {key: event.data[key], "subevents": [event]}
+            chunked_event = Event(
+                timestamp=event.timestamp, duration=event.duration, data=data
+            )
+            chunked_events.append(chunked_event)
+
+    return chunked_events

aw_transform/classify.py

@@ -0,0 +1,122 @@
+import json
+from typing import Pattern, List, Iterable, Tuple, Dict, Optional, Any
+import re
+
+from aw_core import Event
+
+
+Tag = str
+Category = List[str]
+
+
+def _parse_optional_priority(rules: Dict[str, Any]) -> Optional[int]:
+    if "priority" in rules:
+        val = rules["priority"]
+    elif "weight" in rules:
+        val = rules["weight"]
+    else:
+        return None
+    # bool is a subclass of int
+    if isinstance(val, bool) or not isinstance(val, int):
+        raise ValueError("priority/weight must be an integer")
+    return val
+
+
+class Rule:
+    regex: Optional[Pattern]
+    select_keys: Optional[List[str]]
+    ignore_case: bool
+    priority: Optional[int]
+
+    def __init__(self, rules: Dict[str, Any]) -> None:
+        self.select_keys = rules.get("select_keys", None)
+        self.ignore_case = rules.get("ignore_case", False)
+        self.priority = _parse_optional_priority(rules)
+
+        # NOTE: Also checks that the regex isn't an empty string (which would erroneously match everything)
+        regex_str = rules.get("regex", None)
+        self.regex = (
+            re.compile(
+                regex_str, (re.IGNORECASE if self.ignore_case else 0) | re.UNICODE
+            )
+            if regex_str
+            else None
+        )
+
+    def match(self, e: Event) -> bool:
+        if self.select_keys:
+            values = [e.data.get(key, None) for key in self.select_keys]
+        else:
+            values = list(e.data.values())
+        if self.regex:
+            for val in values:
+                if isinstance(val, str) and self.regex.search(val):
+                    return True
+        return False
+
+
+def categorize(
+    events: List[Event], classes: List[Tuple[Category, Rule]]
+) -> List[Event]:
+    cache: Dict[str, Category] = {}
+    for e in events:
+        try:
+            key = json.dumps(e.data, sort_keys=True)
+        except TypeError:
+            key = str(id(e.data))
+        if key not in cache:
+            cache[key] = _pick_category(
+                [(_cls, rule) for _cls, rule in classes if rule.match(e)]
+            )
+        e.data["$category"] = list(cache[key])
+    return events
+
+
+def _categorize_one(e: Event, classes: List[Tuple[Category, Rule]]) -> Event:
+    e.data["$category"] = _pick_category(
+        [(_cls, rule) for _cls, rule in classes if rule.match(e)]
+    )
+    return e
+
+
+def tag(events: List[Event], classes: List[Tuple[Tag, Rule]]) -> List[Event]:
+    cache: Dict[str, List[Tag]] = {}
+    for e in events:
+        try:
+            key = json.dumps(e.data, sort_keys=True)
+        except TypeError:
+            key = str(id(e.data))
+        if key not in cache:
+            cache[key] = [_cls for _cls, rule in classes if rule.match(e)]
+        e.data["$tags"] = list(cache[key])
+    return events
+
+
+def _tag_one(e: Event, classes: List[Tuple[Tag, Rule]]) -> Event:
+    e.data["$tags"] = [_cls for _cls, rule in classes if rule.match(e)]
+    return e
+
+
+def _effective_rank(category: Category, rule: Rule) -> int:
+    # Integer-only. Default is depth * 10 so explicit priorities can slot
+    # between nesting levels (depth 1 → 10, depth 2 → 20). Relative order of
+    # unprioritized rules is unchanged.
+    if rule.priority is not None:
+        return rule.priority
+    return len(category) * 10
+
+
+def _pick_category(matches: Iterable[Tuple[Category, Rule]]) -> Category:
+    category: Category = ["Uncategorized"]
+    rank: Optional[int] = None
+    for cat, rule in matches:
+        if not cat:
+            continue
+        item_rank = _effective_rank(cat, rule)
+        # None means no match yet, so any non-empty category wins — including
+        # an explicit priority below a signed 64-bit floor. Equal ranks keep
+        # the later match (same contract as the old depth-only `>=`).
+        if rank is None or item_rank >= rank:
+            category = cat
+            rank = item_rank
+    return category

aw_transform/filter_keyvals.py

@@ -0,0 +1,28 @@
+import logging
+from typing import List
+import re
+
+from aw_core.models import Event
+
+logger = logging.getLogger(__name__)
+
+
+def filter_keyvals(
+    events: List[Event], key: str, vals: List[str], exclude=False
+) -> List[Event]:
+    def predicate(event):
+        return key in event.data and event.data[key] in vals
+
+    if exclude:
+        return [e for e in events if not predicate(e)]
+    else:
+        return [e for e in events if predicate(e)]
+
+
+def filter_keyvals_regex(events: List[Event], key: str, regex: str) -> List[Event]:
+    r = re.compile(regex)
+
+    def predicate(event):
+        return key in event.data and bool(r.findall(event.data[key]))
+
+    return [e for e in events if predicate(e)]

aw_transform/filter_period_intersect.py

@@ -0,0 +1,173 @@
+import logging
+from typing import List, Iterable, Tuple
+from copy import deepcopy
+
+from aw_core import Event
+from timeslot import Timeslot
+
+logger = logging.getLogger(__name__)
+
+
+def _get_event_period(event: Event) -> Timeslot:
+    start = event.timestamp
+    end = start + event.duration
+    return Timeslot(start, end)
+
+
+def _replace_event_period(event: Event, period: Timeslot) -> Event:
+    e = deepcopy(event)
+    e.timestamp = period.start
+    e.duration = period.duration
+    return e
+
+
+def _intersecting_eventpairs(
+    events1: List[Event], events2: List[Event]
+) -> Iterable[Tuple[Event, Event, Timeslot]]:
+    """A generator that yields each overlapping pair of events from two eventlists along with a Timeslot of the intersection"""
+    events1.sort(key=lambda e: e.timestamp)
+    events2.sort(key=lambda e: e.timestamp)
+    e1_i = 0
+    e2_i = 0
+    while e1_i < len(events1) and e2_i < len(events2):
+        e1 = events1[e1_i]
+        e2 = events2[e2_i]
+        e1_p = _get_event_period(e1)
+        e2_p = _get_event_period(e2)
+
+        ip = e1_p.intersection(e2_p)
+        if ip:
+            # If events intersected, yield events
+            yield (e1, e2, ip)
+            if e1_p.end <= e2_p.end:
+                e1_i += 1
+            else:
+                e2_i += 1
+        else:
+            # No intersection, check if event is before/after filterevent
+            if e1_p.end <= e2_p.start:
+                # Event ended before filter event started
+                e1_i += 1
+            elif e2_p.end <= e1_p.start:
+                # Event started after filter event ended
+                e2_i += 1
+            else:
+                logger.error("Should be unreachable, skipping period")
+                e1_i += 1
+                e2_i += 1
+
+
+def filter_period_intersect(
+    events: List[Event], filterevents: List[Event]
+) -> List[Event]:
+    """
+    Filters away all events or time periods of events in which a
+    filterevent does not have an intersecting time period.
+
+    Useful for example when you want to filter away events or
+    part of events during which a user was AFK.
+
+    Usage:
+      windowevents_notafk = filter_period_intersect(windowevents, notafkevents)
+
+    Example:
+      .. code-block:: none
+
+        events1   |   =======        ======== |
+        events2   | ------  ---  ---   ----   |
+        result    |   ====  =          ====   |
+
+    A JavaScript version used to exist in aw-webui but was removed in `this PR <https://github.com/ActivityWatch/aw-webui/pull/48>`_.
+    """
+
+    events = sorted(events)
+    filterevents = sorted(filterevents)
+
+    return [
+        _replace_event_period(e1, ip)
+        for (e1, _, ip) in _intersecting_eventpairs(events, filterevents)
+    ]
+
+
+def period_union(events1: List[Event], events2: List[Event]) -> List[Event]:
+    """
+    Takes a list of two events and returns a new list of events covering the union
+    of the timeperiods contained in the eventlists with no overlapping events.
+
+    .. warning:: This function strips all data from events as it cannot keep it consistent.
+
+    Example:
+      .. code-block:: none
+
+        events1   |   -------       --------- |
+        events2   | ------  ---  --    ----   |
+        result    | -----------  -- --------- |
+    """
+    events = sorted(events1 + events2)
+    merged_events = []
+    if events:
+        merged_events.append(events.pop(0))
+    for e in events:
+        last_event = merged_events[-1]
+
+        e_p = _get_event_period(e)
+        le_p = _get_event_period(last_event)
+
+        if not e_p.gap(le_p):
+            new_period = e_p.union(le_p)
+            merged_events[-1] = _replace_event_period(last_event, new_period)
+        else:
+            merged_events.append(e)
+    for event in merged_events:
+        # Clear data
+        event.data = {}
+    return merged_events
+
+
+def union(events1: List[Event], events2: List[Event]) -> List[Event]:
+    """
+    Concatenates and sorts union of 2 event lists and removes duplicates.
+
+    Example:
+      Merges events from a backup-bucket with events from a "living" bucket.
+
+      .. code-block:: python
+
+        events = union(events_backup, events_living)
+    """
+
+    events1 = sorted(events1, key=lambda e: (e.timestamp, e.duration))
+    events2 = sorted(events2, key=lambda e: (e.timestamp, e.duration))
+    events_union = []
+
+    e1_i = 0
+    e2_i = 0
+    while e1_i < len(events1) and e2_i < len(events2):
+        e1 = events1[e1_i]
+        e2 = events2[e2_i]
+
+        if e1 == e2:
+            events_union.append(e1)
+            e1_i += 1
+            e2_i += 1
+        else:
+            if e1.timestamp < e2.timestamp:
+                events_union.append(e1)
+                e1_i += 1
+            elif e1.timestamp > e2.timestamp:
+                events_union.append(e2)
+                e2_i += 1
+            elif e1.duration < e2.duration:
+                events_union.append(e1)
+                e1_i += 1
+            else:
+                events_union.append(e2)
+                e2_i += 1
+
+    if e1_i < len(events1):
+        events_union.extend(events1[e1_i:])
+
+    if e2_i < len(events2):
+        events_union.extend(events2[e2_i:])
+
+    return events_union

aw_transform/flood.py

@@ -0,0 +1,121 @@
+import logging
+from datetime import timedelta
+from copy import deepcopy
+from typing import List
+
+from aw_core.models import Event
+
+logger = logging.getLogger(__name__)
+
+
+def flood(events: List[Event], pulsetime: float = 5) -> List[Event]:
+    """Fill short gaps between events and merge nearby equal-data events.
+
+    Events are ordered by timestamp and duration. Gaps no larger than
+    ``pulsetime`` are filled. Equal-data neighbours merge across the gap; when
+    their data differs, both events extend to the midpoint. Splitting an
+    uncertain interval evenly is independent of the surrounding event lengths
+    and matches the Rust implementation.
+
+    Overlapping equal-data events merge. For overlapping differing-data events,
+    final normalization gives the later event precedence so the result never
+    double-counts time.
+
+    See https://github.com/ActivityWatch/activitywatch/issues/124 for the data
+    collection uncertainty that flooding is intended to handle.
+    """
+    # Originally written in aw-research: https://github.com/ActivityWatch/aw-analysis/blob/7da1f2cd8552f866f643501de633d74cdecab168/aw_analysis/flood.py
+    # NOTE: This algorithm has a lot of smaller details that need to be
+    #       carefully considered by anyone wishing to edit it, see:
+    #        - https://github.com/ActivityWatch/aw-core/pull/73
+
+    events = deepcopy(events)
+    # Sort by (timestamp, duration) so shorter same-timestamp events come first.
+    # This matches aw-server-rust's sort_by_timestamp behavior and ensures
+    # a well-defined processing order when events share the same timestamp.
+    events = sorted(events, key=lambda e: (e.timestamp, e.duration))
+
+    # If negative gaps are smaller than this, prune them to become zero
+    negative_gap_trim_thres = timedelta(seconds=0.1)
+
+    warned_about_negative_gap_safe = False
+    warned_about_negative_gap_unsafe = False
+
+    for e1, e2 in zip(events[:-1], events[1:]):
+        gap = e2.timestamp - (e1.timestamp + e1.duration)
+
+        if not gap:
+            continue
+
+        # Sanity check in case events overlap
+        if gap < timedelta(0) and e1.data == e2.data:
+            # Events with negative gap but same data can safely be merged
+            start = min(e1.timestamp, e2.timestamp)
+            end = max(e1.timestamp + e1.duration, e2.timestamp + e2.duration)
+            e1.timestamp, e1.duration = start, (end - start)
+            e2.timestamp, e2.duration = end, timedelta(0)
+            if not warned_about_negative_gap_safe:
+                logger.warning(
+                    f"Gap was of negative duration but could be safely merged ({gap.total_seconds()}s). This message will only show once per batch."
+                )
+                warned_about_negative_gap_safe = True
+        elif gap < -negative_gap_trim_thres and not warned_about_negative_gap_unsafe:
+            # Events with negative gap but differing data cannot be merged safely
+            logger.warning(
+                f"Gap was of negative duration and could NOT be safely merged ({gap.total_seconds()}s). This warning will only show once per batch."
+            )
+            warned_about_negative_gap_unsafe = True
+            # logger.warning("Event 1 (id {}): {} {}".format(e1.id, e1.timestamp, e1.duration))
+            # logger.warning("Event 2 (id {}): {} {}".format(e2.id, e2.timestamp, e2.duration))
+        elif -negative_gap_trim_thres < gap <= timedelta(seconds=pulsetime):
+            e2_end = e2.timestamp + e2.duration
+
+            if e1.data == e2.data:
+                # Preserve the longer neighbour's extent while merging across
+                # the gap, matching the existing semantics for equal data.
+                if e1.duration >= e2.duration:
+                    e1.duration = e2_end - e1.timestamp
+                    e2.timestamp = e2_end
+                    e2.duration = timedelta(0)
+                else:
+                    e2.timestamp = e1.timestamp
+                    e2.duration = e2_end - e2.timestamp
+                    e1.duration = timedelta(0)
+            else:
+                # The gap is an interval of uncertainty: without evidence that
+                # either neighbour owns more of it, split it at the midpoint.
+                midpoint = e1.timestamp + e1.duration + gap / 2
+                e1.duration = midpoint - e1.timestamp
+                e2.timestamp = midpoint
+                e2.duration = e2_end - midpoint
+
+    # Pairwise flooding can mutate an event after its previous pair has already
+    # been processed. Normalize the final stream so downstream consumers never
+    # double-count overlapping time. For differing data, the later event wins.
+    normalized: List[Event] = []
+    for event in (e for e in events if e.duration > timedelta(0)):
+        while (
+            normalized
+            and normalized[-1].timestamp + normalized[-1].duration > event.timestamp
+        ):
+            previous = normalized[-1]
+
+            if previous.data == event.data:
+                end = max(
+                    previous.timestamp + previous.duration,
+                    event.timestamp + event.duration,
+                )
+                previous.duration = end - previous.timestamp
+                break
+
+            previous.duration = event.timestamp - previous.timestamp
+            if previous.duration <= timedelta(0):
+                normalized.pop()
+                continue
+
+            normalized.append(event)
+            break
+        else:
+            normalized.append(event)
+
+    return normalized

aw_transform/heartbeats.py

@@ -0,0 +1,56 @@
+import logging
+from datetime import timedelta
+from typing import List, Optional
+
+from aw_core.models import Event
+
+logger = logging.getLogger(__name__)
+
+
+def heartbeat_reduce(events: List[Event], pulsetime: float) -> List[Event]:
+    """Merges consecutive events together according to the rules of `heartbeat_merge`."""
+    reduced = []
+    if events:
+        reduced.append(events.pop(0))
+    for heartbeat in events:
+        merged = heartbeat_merge(reduced[-1], heartbeat, pulsetime)
+        if merged is not None:
+            # Heartbeat was merged
+            reduced[-1] = merged
+        else:
+            # Heartbeat was not merged
+            reduced.append(heartbeat)
+    return reduced
+
+
+def heartbeat_merge(
+    last_event: Event, heartbeat: Event, pulsetime: float
+) -> Optional[Event]:
+    """
+    Merges two events if they have identical data
+    and the heartbeat timestamp is within the pulsetime window.
+    """
+    if last_event.data == heartbeat.data:
+        # Seconds between end of last_event and start of heartbeat
+        pulseperiod_end = (
+            last_event.timestamp + last_event.duration + timedelta(seconds=pulsetime)
+        )
+        within_pulsetime_window = (
+            last_event.timestamp <= heartbeat.timestamp <= pulseperiod_end
+        )
+
+        if within_pulsetime_window:
+            # Seconds between end of last_event and start of timestamp
+            new_duration = (
+                heartbeat.timestamp - last_event.timestamp
+            ) + heartbeat.duration
+            if last_event.duration < timedelta(0):
+                logger.warning(
+                    "Merging heartbeats would result in a negative duration, refusing to merge."
+                )
+            else:
+                # Taking the max of durations ensures heartbeats that end before the last event don't shorten it
+                last_event.duration = max((last_event.duration, new_duration))
+                return last_event
+
+    return None

aw_transform/merge_events_by_keys.py

@@ -0,0 +1,40 @@
+import logging
+from typing import List, Dict, Tuple
+
+from aw_core.models import Event
+
+logger = logging.getLogger(__name__)
+
+
+def merge_events_by_keys(events, keys) -> List[Event]:
+    """
+    Sums the duration of all events which share a value for a key and returns a new event for each value.
+
+    .. note: The result will be a list of events without timestamp since they are merged.
+    """
+    # Call recursively until all keys are consumed
+    if len(keys) < 1:
+        return events
+    merged_events: Dict[Tuple, Event] = {}
+    for event in events:
+        composite_key: Tuple = ()
+        for key in keys:
+            if key in event.data:
+                val = event["data"][key]
+                # Needed for when the value is a list, such as for categories
+                if isinstance(val, list):
+                    val = tuple(val)
+                composite_key = composite_key + (val,)
+        if composite_key not in merged_events:
+            merged_events[composite_key] = Event(
+                timestamp=event.timestamp, duration=event.duration, data={}
+            )
+            for key in keys:
+                if key in event.data:
+                    merged_events[composite_key].data[key] = event.data[key]
+        else:
+            merged_events[composite_key].duration += event.duration
+    result = []
+    for key in merged_events:
+        result.append(Event(**merged_events[key]))
+    return result

aw_transform/merge_subwatcher_fields.py

@@ -0,0 +1,150 @@
+import logging
+from copy import deepcopy
+from typing import List, Optional, Tuple
+
+from aw_core.models import Event
+from timeslot import Timeslot
+
+from .filter_period_intersect import _get_event_period
+
+logger = logging.getLogger(__name__)
+
+
+def merge_subwatcher_fields(
+    base_events: List[Event],
+    subwatcher_events: List[Event],
+    keys: List[str],
+    conflict: str = "base_wins",
+) -> List[Event]:
+    """
+    Split each event in *base_events* on overlapping subwatcher boundaries and
+    copy the named *keys* from the matching subwatcher event into each segment's
+    ``data`` dict.
+
+    Unlike the ``concat`` workaround, this does not fabricate extra duration.
+    App/title/duration aggregations stay correct because the output still covers
+    exactly the same total time as *base_events*; only the segmentation changes
+    where subwatcher fields actually change.
+
+    This is the backend primitive that lets every client (webui, native UIs,
+    exporters) categorize by subwatcher fields (browser ``url``/``$domain``;
+    editor ``project``/``file``/``language``) without bespoke per-watcher
+    client-side code.
+
+    Args:
+        base_events: The canonical window/afk-filtered stream to enrich.
+        subwatcher_events: Events from a subwatcher bucket (e.g. aw-watcher-vim,
+            aw-watcher-web).  Should already be clipped via
+            ``filter_period_intersect`` before passing here.
+        keys: Which keys to copy from the subwatcher event into the base event.
+            Keys already present in the base event are left untouched when
+            ``conflict="base_wins"`` (default).
+        conflict: ``"base_wins"`` (default) — base event's existing keys are
+            never overwritten; subwatcher fields are purely additive.
+            ``"sub_wins"`` — subwatcher fields overwrite base fields.
+
+    Returns:
+        A new list of base event segments with subwatcher fields injected.
+        Events in *base_events* that have no overlapping subwatcher event are
+        returned with their original data unchanged.
+
+    Example::
+
+        window_events = query_bucket(bid_window)
+        editor_events = flood(query_bucket(bid_editor))
+        editor_events = filter_period_intersect(editor_events, window_events)
+        window_events = merge_subwatcher_fields(
+            window_events, editor_events, ["project", "file", "language"]
+        )
+        # Now categorize(window_events, ...) can match on "project"/"file"
+
+    Note on overlap:
+        Base events are split at the clipped subwatcher boundaries so each
+        output segment only carries the subwatcher fields that were actually
+        present during that slice of time. If multiple subwatcher events cover
+        the same slice, the most recent one wins so a later transition does not
+        get smeared backward by an older, longer pulse.
+    """
+    if conflict not in ("base_wins", "sub_wins"):
+        raise ValueError(
+            f"conflict must be 'base_wins' or 'sub_wins', got {conflict!r}"
+        )
+    if not subwatcher_events or not keys:
+        return [deepcopy(e) for e in base_events]
+
+    # Build a sorted copy so we can do a linear scan
+    sub_sorted = sorted(subwatcher_events, key=lambda e: e.timestamp)
+
+    result: List[Event] = []
+    for base in base_events:
+        base_period = _get_event_period(base)
+        overlapping: List[Tuple[Event, Timeslot]] = []
+        boundaries = {base_period.start, base_period.end}
+
+        for sub in sub_sorted:
+            sub_period = _get_event_period(sub)
+            # Once sub starts after base ends we can stop
+            if sub_period.start >= base_period.end:
+                break
+            # Skip sub events that end before base starts
+            if sub_period.end <= base_period.start:
+                continue
+            ip = base_period.intersection(sub_period)
+            if ip:
+                overlapping.append((sub, sub_period))
+                boundaries.add(ip.start)
+                boundaries.add(ip.end)
+
+        if not overlapping:
+            result.append(deepcopy(base))
+            continue
+
+        boundary_points = sorted(boundaries)
+        base_segments: List[Event] = []
+        for start, end in zip(boundary_points, boundary_points[1:]):
+            segment_period = Timeslot(start, end)
+            best_sub: Optional[Event] = None
+            best_sub_period: Optional[Timeslot] = None
+
+            for sub, sub_period in overlapping:
+                if not segment_period.intersection(sub_period):
+                    continue
+
+                # Later subwatcher events should supersede older overlapping
+                # pulses on the shared slice instead of letting stale data linger.
+                if (
+                    best_sub is None
+                    or sub.timestamp > best_sub.timestamp
+                    or (
+                        sub.timestamp == best_sub.timestamp
+                        and best_sub_period is not None
+                        and sub_period.end > best_sub_period.end
+                    )
+                ):
+                    best_sub = sub
+                    best_sub_period = sub_period
+
+            enriched = deepcopy(base)
+            enriched.timestamp = start
+            enriched.duration = end - start
+
+            if best_sub is not None:
+                for key in keys:
+                    if key in best_sub.data:
+                        if conflict == "base_wins" and key in enriched.data:
+                            continue
+                        enriched.data[key] = deepcopy(best_sub.data[key])
+
+            if (
+                base_segments
+                and base_segments[-1].timestamp + base_segments[-1].duration
+                == enriched.timestamp
+                and base_segments[-1].data == enriched.data
+            ):
+                base_segments[-1].duration += enriched.duration
+            else:
+                base_segments.append(enriched)
+
+        result.extend(base_segments)
+
+    return result

aw_transform/simplify.py

@@ -0,0 +1,30 @@
+import re
+from copy import deepcopy
+from typing import List
+
+from aw_core import Event
+
+
+def simplify_string(events: List[Event], key: str = "title") -> List[Event]:
+    events = deepcopy(events)
+
+    re_leadingdot = re.compile(r"^(●|\*)\s*")
+    re_parensprefix = re.compile(r"^\([0-9]+\)\s*")
+    re_fps = re.compile(r"FPS:\s+[0-9\.]+")
+
+    for e in events:
+        # Remove prefixes that are numbers within parenthesis
+        # Example: "(2) Facebook" -> "Facebook"
+        # Example: "(1) YouTube" -> "YouTube"
+        e.data[key] = re_parensprefix.sub("", e.data[key])
+
+        # Things generally specific to window events with the "app" key
+        if key == "title" and "app" in e["data"]:
+            # Remove FPS display in window title
+            # Example: "Cemu - FPS: 59.2 - ..." -> "Cemu - FPS: ... - ..."
+            e.data[key] = re_fps.sub("FPS: ...", e.data[key])
+
+            # For VSCode (uses ●), gedit (uses *), et al
+            # See: https://github.com/ActivityWatch/aw-watcher-window/issues/32
+            e.data[key] = re_leadingdot.sub("", e.data[key])
+    return events

aw_transform/sort_by.py

@@ -0,0 +1,32 @@
+import logging
+from datetime import timedelta
+from typing import List
+from aw_core.models import Event
+
+logger = logging.getLogger(__name__)
+
+
+def sort_by_timestamp(events) -> List[Event]:
+    """Sorts a list of events by timestamp"""
+    return sorted(events, key=lambda e: e.timestamp)
+
+
+def sort_by_duration(events) -> List[Event]:
+    """Sorts a list of events by duration"""
+    return sorted(events, key=lambda e: e.duration, reverse=True)
+
+
+def limit_events(events, count) -> List[Event]:
+    """Returns the ``count`` first events in the list of events"""
+    return events[:count]
+
+
+def sum_durations(events) -> timedelta:
+    """Sums the durations for the given events"""
+    return timedelta(seconds=(sum(event.duration.total_seconds() for event in events)))
+
+
+def concat(events1, events2) -> List[Event]:
+    """Concatenates two lists of events"""
+    events = events1 + events2
+    return events

aw_transform/split_url_events.py

@@ -0,0 +1,32 @@
+import logging
+from typing import List
+
+from urllib.parse import urlparse
+
+from aw_core.models import Event
+
+logger = logging.getLogger(__name__)
+
+
+def split_url_events(events: List[Event]) -> List[Event]:
+    for event in events:
+        if "url" in event.data:
+            url = event.data["url"]
+            parsed_url = urlparse(url)
+            event.data["$protocol"] = parsed_url.scheme
+            netloc = parsed_url.netloc
+            if netloc:
+                domain = netloc[4:] if netloc[:4] == "www." else netloc
+            elif parsed_url.scheme:
+                # For URLs without a domain (e.g. file://, about:),
+                # use the scheme as domain so they don't all cluster as empty.
+                domain = parsed_url.scheme
+            else:
+                domain = ""
+            event.data["$domain"] = domain
+            event.data["$path"] = parsed_url.path
+            event.data["$params"] = parsed_url.params
+            event.data["$options"] = parsed_url.query
+            event.data["$identifier"] = parsed_url.fragment
+            # TODO: Parse user, port etc aswell
+    return events

aw_transform/union_no_overlap.py

@@ -0,0 +1,85 @@
+"""
+Originally from aw-research
+"""
+
+from copy import deepcopy
+from typing import List, Tuple, Optional
+from datetime import datetime, timedelta, timezone
+
+from timeslot import Timeslot
+
+from aw_core import Event
+
+
+def _split_event(e: Event, dt: datetime) -> Tuple[Event, Optional[Event]]:
+    if e.timestamp < dt < e.timestamp + e.duration:
+        e1 = deepcopy(e)
+        e2 = deepcopy(e)
+        e1.duration = dt - e.timestamp
+        e2.timestamp = dt
+        e2.duration = (e.timestamp + e.duration) - dt
+        return (e1, e2)
+    else:
+        return (e, None)
+
+
+def test_split_event():
+    now = datetime(2018, 1, 1, 0, 0).astimezone(timezone.utc)
+    td1h = timedelta(hours=1)
+    e = Event(timestamp=now, duration=2 * td1h, data={})
+    e1, e2 = _split_event(e, now + td1h)
+    assert e1.timestamp == now
+    assert e1.duration == td1h
+    assert e2
+    assert e2.timestamp == now + td1h
+    assert e2.duration == td1h
+
+
+def union_no_overlap(events1: List[Event], events2: List[Event]) -> List[Event]:
+    """Merges two eventlists and removes overlap, the first eventlist will have precedence
+
+    Example:
+      events1  | xxx    xx     xxx     |
+      events1  |  ----     ------   -- |
+      result   | xxx--  xx ----xxx  -- |
+    """
+    events1 = deepcopy(events1)
+    events2 = deepcopy(events2)
+
+    # I looked a lot at aw_transform.union when I wrote this
+    events_union = []
+    e1_i = 0
+    e2_i = 0
+    while e1_i < len(events1) and e2_i < len(events2):
+        e1 = events1[e1_i]
+        e2 = events2[e2_i]
+        e1_p = Timeslot(e1.timestamp, e1.timestamp + e1.duration)
+        e2_p = Timeslot(e2.timestamp, e2.timestamp + e2.duration)
+
+        if e1_p.intersects(e2_p):
+            if e1.timestamp <= e2.timestamp:
+                events_union.append(e1)
+                e1_i += 1
+
+                # If e2 continues after e1, we need to split up the event so we only get the part that comes after
+                _, e2_next = _split_event(e2, e1.timestamp + e1.duration)
+                if e2_next:
+                    events2[e2_i] = e2_next
+                else:
+                    e2_i += 1
+            else:
+                e2_next, e2_next2 = _split_event(e2, e1.timestamp)
+                events_union.append(e2_next)
+                e2_i += 1
+                if e2_next2:
+                    events2.insert(e2_i, e2_next2)
+        else:
+            if e1.timestamp <= e2.timestamp:
+                events_union.append(e1)
+                e1_i += 1
+            else:
+                events_union.append(e2)
+                e2_i += 1
+    events_union += events1[e1_i:]
+    events_union += events2[e2_i:]
+    return events_union

poetry.lock

@@ -0,0 +1,696 @@
+# This file is automatically @generated by Poetry 2.3.2 and should not be changed by hand.
+
+[[package]]
+name = "attrs"
+version = "24.2.0"
+description = "Classes Without Boilerplate"
+optional = false
+python-versions = ">=3.7"
+groups = ["main"]
+files = [
+    {file = "attrs-24.2.0-py3-none-any.whl", hash = "sha256:81921eb96de3191c8258c199618104dd27ac608d9366f5e35d011eae1867ede2"},
+    {file = "attrs-24.2.0.tar.gz", hash = "sha256:5cfb1b9148b5b086569baec03f20d7b6bf3bcacc9a42bebf87ffaaca362f6346"},
+]
+
+[package.extras]
+benchmark = ["cloudpickle ; platform_python_implementation == \"CPython\"", "hypothesis", "mypy (>=1.11.1) ; platform_python_implementation == \"CPython\" and python_version >= \"3.9\"", "pympler", "pytest (>=4.3.0)", "pytest-codspeed", "pytest-mypy-plugins ; platform_python_implementation == \"CPython\" and python_version >= \"3.9\" and python_version < \"3.13\"", "pytest-xdist[psutil]"]
+cov = ["cloudpickle ; platform_python_implementation == \"CPython\"", "coverage[toml] (>=5.3)", "hypothesis", "mypy (>=1.11.1) ; platform_python_implementation == \"CPython\" and python_version >= \"3.9\"", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins ; platform_python_implementation == \"CPython\" and python_version >= \"3.9\" and python_version < \"3.13\"", "pytest-xdist[psutil]"]
+dev = ["cloudpickle ; platform_python_implementation == \"CPython\"", "hypothesis", "mypy (>=1.11.1) ; platform_python_implementation == \"CPython\" and python_version >= \"3.9\"", "pre-commit", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins ; platform_python_implementation == \"CPython\" and python_version >= \"3.9\" and python_version < \"3.13\"", "pytest-xdist[psutil]"]
+docs = ["cogapp", "furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier (<24.7)"]
+tests = ["cloudpickle ; platform_python_implementation == \"CPython\"", "hypothesis", "mypy (>=1.11.1) ; platform_python_implementation == \"CPython\" and python_version >= \"3.9\"", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins ; platform_python_implementation == \"CPython\" and python_version >= \"3.9\" and python_version < \"3.13\"", "pytest-xdist[psutil]"]
+tests-mypy = ["mypy (>=1.11.1) ; platform_python_implementation == \"CPython\" and python_version >= \"3.9\"", "pytest-mypy-plugins ; platform_python_implementation == \"CPython\" and python_version >= \"3.9\" and python_version < \"3.13\""]
+
+[[package]]
+name = "click"
+version = "8.1.8"
+description = "Composable command line interface toolkit"
+optional = false
+python-versions = ">=3.7"
+groups = ["main"]
+files = [
+    {file = "click-8.1.8-py3-none-any.whl", hash = "sha256:63c132bbbed01578a06712a2d1f497bb62d9c1c0d329b7903a866228027263b2"},
+    {file = "click-8.1.8.tar.gz", hash = "sha256:ed53c9d8990d83c2a27deae68e4ee337473f6330c040a31d4225c9574d16096a"},
+]
+
+[package.dependencies]
+colorama = {version = "*", markers = "platform_system == \"Windows\""}
+
+[[package]]
+name = "colorama"
+version = "0.4.6"
+description = "Cross-platform colored terminal text."
+optional = false
+python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7"
+groups = ["main", "dev"]
+files = [
+    {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"},
+    {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"},
+]
+markers = {main = "platform_system == \"Windows\"", dev = "sys_platform == \"win32\""}
+
+[[package]]
+name = "coverage"
+version = "7.6.1"
+description = "Code coverage measurement for Python"
+optional = false
+python-versions = ">=3.8"
+groups = ["dev"]
+files = [
+    {file = "coverage-7.6.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b06079abebbc0e89e6163b8e8f0e16270124c154dc6e4a47b413dd538859af16"},
+    {file = "coverage-7.6.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:cf4b19715bccd7ee27b6b120e7e9dd56037b9c0681dcc1adc9ba9db3d417fa36"},
+    {file = "coverage-7.6.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e61c0abb4c85b095a784ef23fdd4aede7a2628478e7baba7c5e3deba61070a02"},
+    {file = "coverage-7.6.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fd21f6ae3f08b41004dfb433fa895d858f3f5979e7762d052b12aef444e29afc"},
+    {file = "coverage-7.6.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f59d57baca39b32db42b83b2a7ba6f47ad9c394ec2076b084c3f029b7afca23"},
+    {file = "coverage-7.6.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:a1ac0ae2b8bd743b88ed0502544847c3053d7171a3cff9228af618a068ed9c34"},
+    {file = "coverage-7.6.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:e6a08c0be454c3b3beb105c0596ebdc2371fab6bb90c0c0297f4e58fd7e1012c"},
+    {file = "coverage-7.6.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:f5796e664fe802da4f57a168c85359a8fbf3eab5e55cd4e4569fbacecc903959"},
+    {file = "coverage-7.6.1-cp310-cp310-win32.whl", hash = "sha256:7bb65125fcbef8d989fa1dd0e8a060999497629ca5b0efbca209588a73356232"},
+    {file = "coverage-7.6.1-cp310-cp310-win_amd64.whl", hash = "sha256:3115a95daa9bdba70aea750db7b96b37259a81a709223c8448fa97727d546fe0"},
+    {file = "coverage-7.6.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:7dea0889685db8550f839fa202744652e87c60015029ce3f60e006f8c4462c93"},
+    {file = "coverage-7.6.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ed37bd3c3b063412f7620464a9ac1314d33100329f39799255fb8d3027da50d3"},
+    {file = "coverage-7.6.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d85f5e9a5f8b73e2350097c3756ef7e785f55bd71205defa0bfdaf96c31616ff"},
+    {file = "coverage-7.6.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9bc572be474cafb617672c43fe989d6e48d3c83af02ce8de73fff1c6bb3c198d"},
+    {file = "coverage-7.6.1-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0c0420b573964c760df9e9e86d1a9a622d0d27f417e1a949a8a66dd7bcee7bc6"},
+    {file = "coverage-7.6.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1f4aa8219db826ce6be7099d559f8ec311549bfc4046f7f9fe9b5cea5c581c56"},
+    {file = "coverage-7.6.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:fc5a77d0c516700ebad189b587de289a20a78324bc54baee03dd486f0855d234"},
+    {file = "coverage-7.6.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:b48f312cca9621272ae49008c7f613337c53fadca647d6384cc129d2996d1133"},
+    {file = "coverage-7.6.1-cp311-cp311-win32.whl", hash = "sha256:1125ca0e5fd475cbbba3bb67ae20bd2c23a98fac4e32412883f9bcbaa81c314c"},
+    {file = "coverage-7.6.1-cp311-cp311-win_amd64.whl", hash = "sha256:8ae539519c4c040c5ffd0632784e21b2f03fc1340752af711f33e5be83a9d6c6"},
+    {file = "coverage-7.6.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:95cae0efeb032af8458fc27d191f85d1717b1d4e49f7cb226cf526ff28179778"},
+    {file = "coverage-7.6.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:5621a9175cf9d0b0c84c2ef2b12e9f5f5071357c4d2ea6ca1cf01814f45d2391"},
+    {file = "coverage-7.6.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:260933720fdcd75340e7dbe9060655aff3af1f0c5d20f46b57f262ab6c86a5e8"},
+    {file = "coverage-7.6.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:07e2ca0ad381b91350c0ed49d52699b625aab2b44b65e1b4e02fa9df0e92ad2d"},
+    {file = "coverage-7.6.1-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c44fee9975f04b33331cb8eb272827111efc8930cfd582e0320613263ca849ca"},
+    {file = "coverage-7.6.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:877abb17e6339d96bf08e7a622d05095e72b71f8afd8a9fefc82cf30ed944163"},
+    {file = "coverage-7.6.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:3e0cadcf6733c09154b461f1ca72d5416635e5e4ec4e536192180d34ec160f8a"},
+    {file = "coverage-7.6.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c3c02d12f837d9683e5ab2f3d9844dc57655b92c74e286c262e0fc54213c216d"},
+    {file = "coverage-7.6.1-cp312-cp312-win32.whl", hash = "sha256:e05882b70b87a18d937ca6768ff33cc3f72847cbc4de4491c8e73880766718e5"},
+    {file = "coverage-7.6.1-cp312-cp312-win_amd64.whl", hash = "sha256:b5d7b556859dd85f3a541db6a4e0167b86e7273e1cdc973e5b175166bb634fdb"},
+    {file = "coverage-7.6.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:a4acd025ecc06185ba2b801f2de85546e0b8ac787cf9d3b06e7e2a69f925b106"},
+    {file = "coverage-7.6.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a6d3adcf24b624a7b778533480e32434a39ad8fa30c315208f6d3e5542aeb6e9"},
+    {file = "coverage-7.6.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d0c212c49b6c10e6951362f7c6df3329f04c2b1c28499563d4035d964ab8e08c"},
+    {file = "coverage-7.6.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6e81d7a3e58882450ec4186ca59a3f20a5d4440f25b1cff6f0902ad890e6748a"},
+    {file = "coverage-7.6.1-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:78b260de9790fd81e69401c2dc8b17da47c8038176a79092a89cb2b7d945d060"},
+    {file = "coverage-7.6.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a78d169acd38300060b28d600344a803628c3fd585c912cacc9ea8790fe96862"},
+    {file = "coverage-7.6.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:2c09f4ce52cb99dd7505cd0fc8e0e37c77b87f46bc9c1eb03fe3bc9991085388"},
+    {file = "coverage-7.6.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:6878ef48d4227aace338d88c48738a4258213cd7b74fd9a3d4d7582bb1d8a155"},
+    {file = "coverage-7.6.1-cp313-cp313-win32.whl", hash = "sha256:44df346d5215a8c0e360307d46ffaabe0f5d3502c8a1cefd700b34baf31d411a"},
+    {file = "coverage-7.6.1-cp313-cp313-win_amd64.whl", hash = "sha256:8284cf8c0dd272a247bc154eb6c95548722dce90d098c17a883ed36e67cdb129"},
+    {file = "coverage-7.6.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:d3296782ca4eab572a1a4eca686d8bfb00226300dcefdf43faa25b5242ab8a3e"},
+    {file = "coverage-7.6.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:502753043567491d3ff6d08629270127e0c31d4184c4c8d98f92c26f65019962"},
+    {file = "coverage-7.6.1-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6a89ecca80709d4076b95f89f308544ec8f7b4727e8a547913a35f16717856cb"},
+    {file = "coverage-7.6.1-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a318d68e92e80af8b00fa99609796fdbcdfef3629c77c6283566c6f02c6d6704"},
+    {file = "coverage-7.6.1-cp313-cp313t-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:13b0a73a0896988f053e4fbb7de6d93388e6dd292b0d87ee51d106f2c11b465b"},
+    {file = "coverage-7.6.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:4421712dbfc5562150f7554f13dde997a2e932a6b5f352edcce948a815efee6f"},
+    {file = "coverage-7.6.1-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:166811d20dfea725e2e4baa71fffd6c968a958577848d2131f39b60043400223"},
+    {file = "coverage-7.6.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:225667980479a17db1048cb2bf8bfb39b8e5be8f164b8f6628b64f78a72cf9d3"},
+    {file = "coverage-7.6.1-cp313-cp313t-win32.whl", hash = "sha256:170d444ab405852903b7d04ea9ae9b98f98ab6d7e63e1115e82620807519797f"},
+    {file = "coverage-7.6.1-cp313-cp313t-win_amd64.whl", hash = "sha256:b9f222de8cded79c49bf184bdbc06630d4c58eec9459b939b4a690c82ed05657"},
+    {file = "coverage-7.6.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6db04803b6c7291985a761004e9060b2bca08da6d04f26a7f2294b8623a0c1a0"},
+    {file = "coverage-7.6.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:f1adfc8ac319e1a348af294106bc6a8458a0f1633cc62a1446aebc30c5fa186a"},
+    {file = "coverage-7.6.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a95324a9de9650a729239daea117df21f4b9868ce32e63f8b650ebe6cef5595b"},
+    {file = "coverage-7.6.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b43c03669dc4618ec25270b06ecd3ee4fa94c7f9b3c14bae6571ca00ef98b0d3"},
+    {file = "coverage-7.6.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8929543a7192c13d177b770008bc4e8119f2e1f881d563fc6b6305d2d0ebe9de"},
+    {file = "coverage-7.6.1-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:a09ece4a69cf399510c8ab25e0950d9cf2b42f7b3cb0374f95d2e2ff594478a6"},
+    {file = "coverage-7.6.1-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:9054a0754de38d9dbd01a46621636689124d666bad1936d76c0341f7d71bf569"},
+    {file = "coverage-7.6.1-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:0dbde0f4aa9a16fa4d754356a8f2e36296ff4d83994b2c9d8398aa32f222f989"},
+    {file = "coverage-7.6.1-cp38-cp38-win32.whl", hash = "sha256:da511e6ad4f7323ee5702e6633085fb76c2f893aaf8ce4c51a0ba4fc07580ea7"},
+    {file = "coverage-7.6.1-cp38-cp38-win_amd64.whl", hash = "sha256:3f1156e3e8f2872197af3840d8ad307a9dd18e615dc64d9ee41696f287c57ad8"},
+    {file = "coverage-7.6.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:abd5fd0db5f4dc9289408aaf34908072f805ff7792632250dcb36dc591d24255"},
+    {file = "coverage-7.6.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:547f45fa1a93154bd82050a7f3cddbc1a7a4dd2a9bf5cb7d06f4ae29fe94eaf8"},
+    {file = "coverage-7.6.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:645786266c8f18a931b65bfcefdbf6952dd0dea98feee39bd188607a9d307ed2"},
+    {file = "coverage-7.6.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9e0b2df163b8ed01d515807af24f63de04bebcecbd6c3bfeff88385789fdf75a"},
+    {file = "coverage-7.6.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:609b06f178fe8e9f89ef676532760ec0b4deea15e9969bf754b37f7c40326dbc"},
+    {file = "coverage-7.6.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:702855feff378050ae4f741045e19a32d57d19f3e0676d589df0575008ea5004"},
+    {file = "coverage-7.6.1-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:2bdb062ea438f22d99cba0d7829c2ef0af1d768d1e4a4f528087224c90b132cb"},
+    {file = "coverage-7.6.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:9c56863d44bd1c4fe2abb8a4d6f5371d197f1ac0ebdee542f07f35895fc07f36"},
+    {file = "coverage-7.6.1-cp39-cp39-win32.whl", hash = "sha256:6e2cd258d7d927d09493c8df1ce9174ad01b381d4729a9d8d4e38670ca24774c"},
+    {file = "coverage-7.6.1-cp39-cp39-win_amd64.whl", hash = "sha256:06a737c882bd26d0d6ee7269b20b12f14a8704807a01056c80bb881a4b2ce6ca"},
+    {file = "coverage-7.6.1-pp38.pp39.pp310-none-any.whl", hash = "sha256:e9a6e0eb86070e8ccaedfbd9d38fec54864f3125ab95419970575b42af7541df"},
+    {file = "coverage-7.6.1.tar.gz", hash = "sha256:953510dfb7b12ab69d20135a0662397f077c59b1e6379a768e97c59d852ee51d"},
+]
+
+[package.dependencies]
+tomli = {version = "*", optional = true, markers = "python_full_version <= \"3.11.0a6\" and extra == \"toml\""}
+
+[package.extras]
+toml = ["tomli ; python_full_version <= \"3.11.0a6\""]
+
+[[package]]
+name = "deprecation"
+version = "2.1.0"
+description = "A library to handle automated deprecations"
+optional = false
+python-versions = "*"
+groups = ["main"]
+files = [
+    {file = "deprecation-2.1.0-py2.py3-none-any.whl", hash = "sha256:a10811591210e1fb0e768a8c25517cabeabcba6f0bf96564f8ff45189f90b14a"},
+    {file = "deprecation-2.1.0.tar.gz", hash = "sha256:72b3bde64e5d778694b0cf68178aed03d15e15477116add3fb773e581f9518ff"},
+]
+
+[package.dependencies]
+packaging = "*"
+
+[[package]]
+name = "exceptiongroup"
+version = "1.2.2"
+description = "Backport of PEP 654 (exception groups)"
+optional = false
+python-versions = ">=3.7"
+groups = ["dev"]
+markers = "python_version < \"3.11\""
+files = [
+    {file = "exceptiongroup-1.2.2-py3-none-any.whl", hash = "sha256:3111b9d131c238bec2f8f516e123e14ba243563fb135d3fe885990585aa7795b"},
+    {file = "exceptiongroup-1.2.2.tar.gz", hash = "sha256:47c2edf7c6738fafb49fd34290706d1a1a2f4d1c6df275526b62cbb4aa5393cc"},
+]
+
+[package.extras]
+test = ["pytest (>=6)"]
+
+[[package]]
+name = "importlib-resources"
+version = "6.4.5"
+description = "Read resources from Python packages"
+optional = false
+python-versions = ">=3.8"
+groups = ["main"]
+markers = "python_version == \"3.8\""
+files = [
+    {file = "importlib_resources-6.4.5-py3-none-any.whl", hash = "sha256:ac29d5f956f01d5e4bb63102a5a19957f1b9175e45649977264a1416783bb717"},
+    {file = "importlib_resources-6.4.5.tar.gz", hash = "sha256:980862a1d16c9e147a59603677fa2aa5fd82b87f223b6cb870695bcfce830065"},
+]
+
+[package.dependencies]
+zipp = {version = ">=3.1.0", markers = "python_version < \"3.10\""}
+
+[package.extras]
+check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1) ; sys_platform != \"cygwin\""]
+cover = ["pytest-cov"]
+doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"]
+enabler = ["pytest-enabler (>=2.2)"]
+test = ["jaraco.test (>=5.4)", "pytest (>=6,!=8.1.*)", "zipp (>=3.17)"]
+type = ["pytest-mypy"]
+
+[[package]]
+name = "iniconfig"
+version = "2.0.0"
+description = "brain-dead simple config-ini parsing"
+optional = false
+python-versions = ">=3.7"
+groups = ["dev"]
+files = [
+    {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"},
+    {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"},
+]
+
+[[package]]
+name = "iso8601"
+version = "2.1.0"
+description = "Simple module to parse ISO 8601 dates"
+optional = false
+python-versions = ">=3.7,<4.0"
+groups = ["main"]
+files = [
+    {file = "iso8601-2.1.0-py3-none-any.whl", hash = "sha256:aac4145c4dcb66ad8b648a02830f5e2ff6c24af20f4f482689be402db2429242"},
+    {file = "iso8601-2.1.0.tar.gz", hash = "sha256:6b1d3829ee8921c4301998c909f7829fa9ed3cbdac0d3b16af2d743aed1ba8df"},
+]
+
+[[package]]
+name = "jsonschema"
+version = "4.23.0"
+description = "An implementation of JSON Schema validation for Python"
+optional = false
+python-versions = ">=3.8"
+groups = ["main"]
+files = [
+    {file = "jsonschema-4.23.0-py3-none-any.whl", hash = "sha256:fbadb6f8b144a8f8cf9f0b89ba94501d143e50411a1278633f56a7acf7fd5566"},
+    {file = "jsonschema-4.23.0.tar.gz", hash = "sha256:d71497fef26351a33265337fa77ffeb82423f3ea21283cd9467bb03999266bc4"},
+]
+
+[package.dependencies]
+attrs = ">=22.2.0"
+importlib-resources = {version = ">=1.4.0", markers = "python_version < \"3.9\""}
+jsonschema-specifications = ">=2023.3.6"
+pkgutil-resolve-name = {version = ">=1.3.10", markers = "python_version < \"3.9\""}
+referencing = ">=0.28.4"
+rpds-py = ">=0.7.1"
+
+[package.extras]
+format = ["fqdn", "idna", "isoduration", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3987", "uri-template", "webcolors (>=1.11)"]
+format-nongpl = ["fqdn", "idna", "isoduration", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3986-validator (>0.1.0)", "uri-template", "webcolors (>=24.6.0)"]
+
+[[package]]
+name = "jsonschema-specifications"
+version = "2023.12.1"
+description = "The JSON Schema meta-schemas and vocabularies, exposed as a Registry"
+optional = false
+python-versions = ">=3.8"
+groups = ["main"]
+files = [
+    {file = "jsonschema_specifications-2023.12.1-py3-none-any.whl", hash = "sha256:87e4fdf3a94858b8a2ba2778d9ba57d8a9cafca7c7489c46ba0d30a8bc6a9c3c"},
+    {file = "jsonschema_specifications-2023.12.1.tar.gz", hash = "sha256:48a76787b3e70f5ed53f1160d2b81f586e4ca6d1548c5de7085d1682674764cc"},
+]
+
+[package.dependencies]
+importlib-resources = {version = ">=1.4.0", markers = "python_version < \"3.9\""}
+referencing = ">=0.31.0"
+
+[[package]]
+name = "mypy"
+version = "1.11.2"
+description = "Optional static typing for Python"
+optional = false
+python-versions = ">=3.8"
+groups = ["dev"]
+files = [
+    {file = "mypy-1.11.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d42a6dd818ffce7be66cce644f1dff482f1d97c53ca70908dff0b9ddc120b77a"},
+    {file = "mypy-1.11.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:801780c56d1cdb896eacd5619a83e427ce436d86a3bdf9112527f24a66618fef"},
+    {file = "mypy-1.11.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:41ea707d036a5307ac674ea172875f40c9d55c5394f888b168033177fce47383"},
+    {file = "mypy-1.11.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:6e658bd2d20565ea86da7d91331b0eed6d2eee22dc031579e6297f3e12c758c8"},
+    {file = "mypy-1.11.2-cp310-cp310-win_amd64.whl", hash = "sha256:478db5f5036817fe45adb7332d927daa62417159d49783041338921dcf646fc7"},
+    {file = "mypy-1.11.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:75746e06d5fa1e91bfd5432448d00d34593b52e7e91a187d981d08d1f33d4385"},
+    {file = "mypy-1.11.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a976775ab2256aadc6add633d44f100a2517d2388906ec4f13231fafbb0eccca"},
+    {file = "mypy-1.11.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cd953f221ac1379050a8a646585a29574488974f79d8082cedef62744f0a0104"},
+    {file = "mypy-1.11.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:57555a7715c0a34421013144a33d280e73c08df70f3a18a552938587ce9274f4"},
+    {file = "mypy-1.11.2-cp311-cp311-win_amd64.whl", hash = "sha256:36383a4fcbad95f2657642a07ba22ff797de26277158f1cc7bd234821468b1b6"},
+    {file = "mypy-1.11.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:e8960dbbbf36906c5c0b7f4fbf2f0c7ffb20f4898e6a879fcf56a41a08b0d318"},
+    {file = "mypy-1.11.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:06d26c277962f3fb50e13044674aa10553981ae514288cb7d0a738f495550b36"},
+    {file = "mypy-1.11.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6e7184632d89d677973a14d00ae4d03214c8bc301ceefcdaf5c474866814c987"},
+    {file = "mypy-1.11.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:3a66169b92452f72117e2da3a576087025449018afc2d8e9bfe5ffab865709ca"},
+    {file = "mypy-1.11.2-cp312-cp312-win_amd64.whl", hash = "sha256:969ea3ef09617aff826885a22ece0ddef69d95852cdad2f60c8bb06bf1f71f70"},
+    {file = "mypy-1.11.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:37c7fa6121c1cdfcaac97ce3d3b5588e847aa79b580c1e922bb5d5d2902df19b"},
+    {file = "mypy-1.11.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:4a8a53bc3ffbd161b5b2a4fff2f0f1e23a33b0168f1c0778ec70e1a3d66deb86"},
+    {file = "mypy-1.11.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2ff93107f01968ed834f4256bc1fc4475e2fecf6c661260066a985b52741ddce"},
+    {file = "mypy-1.11.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:edb91dded4df17eae4537668b23f0ff6baf3707683734b6a818d5b9d0c0c31a1"},
+    {file = "mypy-1.11.2-cp38-cp38-win_amd64.whl", hash = "sha256:ee23de8530d99b6db0573c4ef4bd8f39a2a6f9b60655bf7a1357e585a3486f2b"},
+    {file = "mypy-1.11.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:801ca29f43d5acce85f8e999b1e431fb479cb02d0e11deb7d2abb56bdaf24fd6"},
+    {file = "mypy-1.11.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:af8d155170fcf87a2afb55b35dc1a0ac21df4431e7d96717621962e4b9192e70"},
+    {file = "mypy-1.11.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f7821776e5c4286b6a13138cc935e2e9b6fde05e081bdebf5cdb2bb97c9df81d"},
+    {file = "mypy-1.11.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:539c570477a96a4e6fb718b8d5c3e0c0eba1f485df13f86d2970c91f0673148d"},
+    {file = "mypy-1.11.2-cp39-cp39-win_amd64.whl", hash = "sha256:3f14cd3d386ac4d05c5a39a51b84387403dadbd936e17cb35882134d4f8f0d24"},
+    {file = "mypy-1.11.2-py3-none-any.whl", hash = "sha256:b499bc07dbdcd3de92b0a8b29fdf592c111276f6a12fe29c30f6c417dd546d12"},
+    {file = "mypy-1.11.2.tar.gz", hash = "sha256:7f9993ad3e0ffdc95c2a14b66dee63729f021968bff8ad911867579c65d13a79"},
+]
+
+[package.dependencies]
+mypy-extensions = ">=1.0.0"
+tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""}
+typing-extensions = ">=4.6.0"
+
+[package.extras]
+dmypy = ["psutil (>=4.0)"]
+install-types = ["pip"]
+mypyc = ["setuptools (>=50)"]
+reports = ["lxml"]
+
+[[package]]
+name = "mypy-extensions"
+version = "1.0.0"
+description = "Type system extensions for programs checked with the mypy type checker."
+optional = false
+python-versions = ">=3.5"
+groups = ["dev"]
+files = [
+    {file = "mypy_extensions-1.0.0-py3-none-any.whl", hash = "sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d"},
+    {file = "mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782"},
+]
+
+[[package]]
+name = "packaging"
+version = "24.1"
+description = "Core utilities for Python packages"
+optional = false
+python-versions = ">=3.8"
+groups = ["main", "dev"]
+files = [
+    {file = "packaging-24.1-py3-none-any.whl", hash = "sha256:5b8f2217dbdbd2f7f384c41c628544e6d52f2d0f53c6d0c3ea61aa5d1d7ff124"},
+    {file = "packaging-24.1.tar.gz", hash = "sha256:026ed72c8ed3fcce5bf8950572258698927fd1dbda10a5e981cdf0ac37f4f002"},
+]
+
+[[package]]
+name = "peewee"
+version = "3.17.6"
+description = "a little orm"
+optional = false
+python-versions = "*"
+groups = ["main"]
+files = [
+    {file = "peewee-3.17.6.tar.gz", hash = "sha256:cea5592c6f4da1592b7cff8eaf655be6648a1f5857469e30037bf920c03fb8fb"},
+]
+
+[[package]]
+name = "pkgutil-resolve-name"
+version = "1.3.10"
+description = "Resolve a name to an object."
+optional = false
+python-versions = ">=3.6"
+groups = ["main"]
+markers = "python_version == \"3.8\""
+files = [
+    {file = "pkgutil_resolve_name-1.3.10-py3-none-any.whl", hash = "sha256:ca27cc078d25c5ad71a9de0a7a330146c4e014c2462d9af19c6b828280649c5e"},
+    {file = "pkgutil_resolve_name-1.3.10.tar.gz", hash = "sha256:357d6c9e6a755653cfd78893817c0853af365dd51ec97f3d358a819373bbd174"},
+]
+
+[[package]]
+name = "platformdirs"
+version = "3.10.0"
+description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"."
+optional = false
+python-versions = ">=3.7"
+groups = ["main"]
+files = [
+    {file = "platformdirs-3.10.0-py3-none-any.whl", hash = "sha256:d7c24979f292f916dc9cbf8648319032f551ea8c49a4c9bf2fb556a02070ec1d"},
+    {file = "platformdirs-3.10.0.tar.gz", hash = "sha256:b45696dab2d7cc691a3226759c0d3b00c47c8b6e293d96f6436f733303f77f6d"},
+]
+
+[package.extras]
+docs = ["furo (>=2023.7.26)", "proselint (>=0.13)", "sphinx (>=7.1.1)", "sphinx-autodoc-typehints (>=1.24)"]
+test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4)", "pytest-cov (>=4.1)", "pytest-mock (>=3.11.1)"]
+
+[[package]]
+name = "pluggy"
+version = "1.5.0"
+description = "plugin and hook calling mechanisms for python"
+optional = false
+python-versions = ">=3.8"
+groups = ["dev"]
+files = [
+    {file = "pluggy-1.5.0-py3-none-any.whl", hash = "sha256:44e1ad92c8ca002de6377e165f3e0f1be63266ab4d554740532335b9d75ea669"},
+    {file = "pluggy-1.5.0.tar.gz", hash = "sha256:2cffa88e94fdc978c4c574f15f9e59b7f4201d439195c3715ca9e2486f1d0cf1"},
+]
+
+[package.extras]
+dev = ["pre-commit", "tox"]
+testing = ["pytest", "pytest-benchmark"]
+
+[[package]]
+name = "pytest"
+version = "8.3.3"
+description = "pytest: simple powerful testing with Python"
+optional = false
+python-versions = ">=3.8"
+groups = ["dev"]
+files = [
+    {file = "pytest-8.3.3-py3-none-any.whl", hash = "sha256:a6853c7375b2663155079443d2e45de913a911a11d669df02a50814944db57b2"},
+    {file = "pytest-8.3.3.tar.gz", hash = "sha256:70b98107bd648308a7952b06e6ca9a50bc660be218d53c257cc1fc94fda10181"},
+]
+
+[package.dependencies]
+colorama = {version = "*", markers = "sys_platform == \"win32\""}
+exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""}
+iniconfig = "*"
+packaging = "*"
+pluggy = ">=1.5,<2"
+tomli = {version = ">=1", markers = "python_version < \"3.11\""}
+
+[package.extras]
+dev = ["argcomplete", "attrs (>=19.2)", "hypothesis (>=3.56)", "mock", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"]
+
+[[package]]
+name = "pytest-cov"
+version = "5.0.0"
+description = "Pytest plugin for measuring coverage."
+optional = false
+python-versions = ">=3.8"
+groups = ["dev"]
+files = [
+    {file = "pytest-cov-5.0.0.tar.gz", hash = "sha256:5837b58e9f6ebd335b0f8060eecce69b662415b16dc503883a02f45dfeb14857"},
+    {file = "pytest_cov-5.0.0-py3-none-any.whl", hash = "sha256:4f0764a1219df53214206bf1feea4633c3b558a2925c8b59f144f682861ce652"},
+]
+
+[package.dependencies]
+coverage = {version = ">=5.2.1", extras = ["toml"]}
+pytest = ">=4.6"
+
+[package.extras]
+testing = ["fields", "hunter", "process-tests", "pytest-xdist", "virtualenv"]
+
+[[package]]
+name = "referencing"
+version = "0.35.1"
+description = "JSON Referencing + Python"
+optional = false
+python-versions = ">=3.8"
+groups = ["main"]
+files = [
+    {file = "referencing-0.35.1-py3-none-any.whl", hash = "sha256:eda6d3234d62814d1c64e305c1331c9a3a6132da475ab6382eaa997b21ee75de"},
+    {file = "referencing-0.35.1.tar.gz", hash = "sha256:25b42124a6c8b632a425174f24087783efb348a6f1e0008e63cd4466fedf703c"},
+]
+
+[package.dependencies]
+attrs = ">=22.2.0"
+rpds-py = ">=0.7.0"
+
+[[package]]
+name = "rfc3339-validator"
+version = "0.1.4"
+description = "A pure python RFC3339 validator"
+optional = false
+python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
+groups = ["main"]
+files = [
+    {file = "rfc3339_validator-0.1.4-py2.py3-none-any.whl", hash = "sha256:24f6ec1eda14ef823da9e36ec7113124b39c04d50a4d3d3a3c2859577e7791fa"},
+    {file = "rfc3339_validator-0.1.4.tar.gz", hash = "sha256:138a2abdf93304ad60530167e51d2dfb9549521a836871b88d7f4695d0022f6b"},
+]
+
+[package.dependencies]
+six = "*"
+
+[[package]]
+name = "rpds-py"
+version = "0.20.0"
+description = "Python bindings to Rust's persistent data structures (rpds)"
+optional = false
+python-versions = ">=3.8"
+groups = ["main"]
+files = [
+    {file = "rpds_py-0.20.0-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:3ad0fda1635f8439cde85c700f964b23ed5fc2d28016b32b9ee5fe30da5c84e2"},
+    {file = "rpds_py-0.20.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9bb4a0d90fdb03437c109a17eade42dfbf6190408f29b2744114d11586611d6f"},
+    {file = "rpds_py-0.20.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c6377e647bbfd0a0b159fe557f2c6c602c159fc752fa316572f012fc0bf67150"},
+    {file = "rpds_py-0.20.0-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:eb851b7df9dda52dc1415ebee12362047ce771fc36914586b2e9fcbd7d293b3e"},
+    {file = "rpds_py-0.20.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1e0f80b739e5a8f54837be5d5c924483996b603d5502bfff79bf33da06164ee2"},
+    {file = "rpds_py-0.20.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5a8c94dad2e45324fc74dce25e1645d4d14df9a4e54a30fa0ae8bad9a63928e3"},
+    {file = "rpds_py-0.20.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f8e604fe73ba048c06085beaf51147eaec7df856824bfe7b98657cf436623daf"},
+    {file = "rpds_py-0.20.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:df3de6b7726b52966edf29663e57306b23ef775faf0ac01a3e9f4012a24a4140"},
+    {file = "rpds_py-0.20.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:cf258ede5bc22a45c8e726b29835b9303c285ab46fc7c3a4cc770736b5304c9f"},
+    {file = "rpds_py-0.20.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:55fea87029cded5df854ca7e192ec7bdb7ecd1d9a3f63d5c4eb09148acf4a7ce"},
+    {file = "rpds_py-0.20.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:ae94bd0b2f02c28e199e9bc51485d0c5601f58780636185660f86bf80c89af94"},
+    {file = "rpds_py-0.20.0-cp310-none-win32.whl", hash = "sha256:28527c685f237c05445efec62426d285e47a58fb05ba0090a4340b73ecda6dee"},
+    {file = "rpds_py-0.20.0-cp310-none-win_amd64.whl", hash = "sha256:238a2d5b1cad28cdc6ed15faf93a998336eb041c4e440dd7f902528b8891b399"},
+    {file = "rpds_py-0.20.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:ac2f4f7a98934c2ed6505aead07b979e6f999389f16b714448fb39bbaa86a489"},
+    {file = "rpds_py-0.20.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:220002c1b846db9afd83371d08d239fdc865e8f8c5795bbaec20916a76db3318"},
+    {file = "rpds_py-0.20.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8d7919548df3f25374a1f5d01fbcd38dacab338ef5f33e044744b5c36729c8db"},
+    {file = "rpds_py-0.20.0-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:758406267907b3781beee0f0edfe4a179fbd97c0be2e9b1154d7f0a1279cf8e5"},
+    {file = "rpds_py-0.20.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3d61339e9f84a3f0767b1995adfb171a0d00a1185192718a17af6e124728e0f5"},
+    {file = "rpds_py-0.20.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1259c7b3705ac0a0bd38197565a5d603218591d3f6cee6e614e380b6ba61c6f6"},
+    {file = "rpds_py-0.20.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5c1dc0f53856b9cc9a0ccca0a7cc61d3d20a7088201c0937f3f4048c1718a209"},
+    {file = "rpds_py-0.20.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:7e60cb630f674a31f0368ed32b2a6b4331b8350d67de53c0359992444b116dd3"},
+    {file = "rpds_py-0.20.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:dbe982f38565bb50cb7fb061ebf762c2f254ca3d8c20d4006878766e84266272"},
+    {file = "rpds_py-0.20.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:514b3293b64187172bc77c8fb0cdae26981618021053b30d8371c3a902d4d5ad"},
+    {file = "rpds_py-0.20.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:d0a26ffe9d4dd35e4dfdd1e71f46401cff0181c75ac174711ccff0459135fa58"},
+    {file = "rpds_py-0.20.0-cp311-none-win32.whl", hash = "sha256:89c19a494bf3ad08c1da49445cc5d13d8fefc265f48ee7e7556839acdacf69d0"},
+    {file = "rpds_py-0.20.0-cp311-none-win_amd64.whl", hash = "sha256:c638144ce971df84650d3ed0096e2ae7af8e62ecbbb7b201c8935c370df00a2c"},
+    {file = "rpds_py-0.20.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:a84ab91cbe7aab97f7446652d0ed37d35b68a465aeef8fc41932a9d7eee2c1a6"},
+    {file = "rpds_py-0.20.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:56e27147a5a4c2c21633ff8475d185734c0e4befd1c989b5b95a5d0db699b21b"},
+    {file = "rpds_py-0.20.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2580b0c34583b85efec8c5c5ec9edf2dfe817330cc882ee972ae650e7b5ef739"},
+    {file = "rpds_py-0.20.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b80d4a7900cf6b66bb9cee5c352b2d708e29e5a37fe9bf784fa97fc11504bf6c"},
+    {file = "rpds_py-0.20.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:50eccbf054e62a7b2209b28dc7a22d6254860209d6753e6b78cfaeb0075d7bee"},
+    {file = "rpds_py-0.20.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:49a8063ea4296b3a7e81a5dfb8f7b2d73f0b1c20c2af401fb0cdf22e14711a96"},
+    {file = "rpds_py-0.20.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ea438162a9fcbee3ecf36c23e6c68237479f89f962f82dae83dc15feeceb37e4"},
+    {file = "rpds_py-0.20.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:18d7585c463087bddcfa74c2ba267339f14f2515158ac4db30b1f9cbdb62c8ef"},
+    {file = "rpds_py-0.20.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:d4c7d1a051eeb39f5c9547e82ea27cbcc28338482242e3e0b7768033cb083821"},
+    {file = "rpds_py-0.20.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:e4df1e3b3bec320790f699890d41c59d250f6beda159ea3c44c3f5bac1976940"},
+    {file = "rpds_py-0.20.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:2cf126d33a91ee6eedc7f3197b53e87a2acdac63602c0f03a02dd69e4b138174"},
+    {file = "rpds_py-0.20.0-cp312-none-win32.whl", hash = "sha256:8bc7690f7caee50b04a79bf017a8d020c1f48c2a1077ffe172abec59870f1139"},
+    {file = "rpds_py-0.20.0-cp312-none-win_amd64.whl", hash = "sha256:0e13e6952ef264c40587d510ad676a988df19adea20444c2b295e536457bc585"},
+    {file = "rpds_py-0.20.0-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:aa9a0521aeca7d4941499a73ad7d4f8ffa3d1affc50b9ea11d992cd7eff18a29"},
+    {file = "rpds_py-0.20.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:4a1f1d51eccb7e6c32ae89243cb352389228ea62f89cd80823ea7dd1b98e0b91"},
+    {file = "rpds_py-0.20.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8a86a9b96070674fc88b6f9f71a97d2c1d3e5165574615d1f9168ecba4cecb24"},
+    {file = "rpds_py-0.20.0-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:6c8ef2ebf76df43f5750b46851ed1cdf8f109d7787ca40035fe19fbdc1acc5a7"},
+    {file = "rpds_py-0.20.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b74b25f024b421d5859d156750ea9a65651793d51b76a2e9238c05c9d5f203a9"},
+    {file = "rpds_py-0.20.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:57eb94a8c16ab08fef6404301c38318e2c5a32216bf5de453e2714c964c125c8"},
+    {file = "rpds_py-0.20.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e1940dae14e715e2e02dfd5b0f64a52e8374a517a1e531ad9412319dc3ac7879"},
+    {file = "rpds_py-0.20.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d20277fd62e1b992a50c43f13fbe13277a31f8c9f70d59759c88f644d66c619f"},
+    {file = "rpds_py-0.20.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:06db23d43f26478303e954c34c75182356ca9aa7797d22c5345b16871ab9c45c"},
+    {file = "rpds_py-0.20.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:b2a5db5397d82fa847e4c624b0c98fe59d2d9b7cf0ce6de09e4d2e80f8f5b3f2"},
+    {file = "rpds_py-0.20.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:5a35df9f5548fd79cb2f52d27182108c3e6641a4feb0f39067911bf2adaa3e57"},
+    {file = "rpds_py-0.20.0-cp313-none-win32.whl", hash = "sha256:fd2d84f40633bc475ef2d5490b9c19543fbf18596dcb1b291e3a12ea5d722f7a"},
+    {file = "rpds_py-0.20.0-cp313-none-win_amd64.whl", hash = "sha256:9bc2d153989e3216b0559251b0c260cfd168ec78b1fac33dd485750a228db5a2"},
+    {file = "rpds_py-0.20.0-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:f2fbf7db2012d4876fb0d66b5b9ba6591197b0f165db8d99371d976546472a24"},
+    {file = "rpds_py-0.20.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:1e5f3cd7397c8f86c8cc72d5a791071431c108edd79872cdd96e00abd8497d29"},
+    {file = "rpds_py-0.20.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ce9845054c13696f7af7f2b353e6b4f676dab1b4b215d7fe5e05c6f8bb06f965"},
+    {file = "rpds_py-0.20.0-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c3e130fd0ec56cb76eb49ef52faead8ff09d13f4527e9b0c400307ff72b408e1"},
+    {file = "rpds_py-0.20.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4b16aa0107ecb512b568244ef461f27697164d9a68d8b35090e9b0c1c8b27752"},
+    {file = "rpds_py-0.20.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:aa7f429242aae2947246587d2964fad750b79e8c233a2367f71b554e9447949c"},
+    {file = "rpds_py-0.20.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:af0fc424a5842a11e28956e69395fbbeab2c97c42253169d87e90aac2886d751"},
+    {file = "rpds_py-0.20.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:b8c00a3b1e70c1d3891f0db1b05292747f0dbcfb49c43f9244d04c70fbc40eb8"},
+    {file = "rpds_py-0.20.0-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:40ce74fc86ee4645d0a225498d091d8bc61f39b709ebef8204cb8b5a464d3c0e"},
+    {file = "rpds_py-0.20.0-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:4fe84294c7019456e56d93e8ababdad5a329cd25975be749c3f5f558abb48253"},
+    {file = "rpds_py-0.20.0-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:338ca4539aad4ce70a656e5187a3a31c5204f261aef9f6ab50e50bcdffaf050a"},
+    {file = "rpds_py-0.20.0-cp38-none-win32.whl", hash = "sha256:54b43a2b07db18314669092bb2de584524d1ef414588780261e31e85846c26a5"},
+    {file = "rpds_py-0.20.0-cp38-none-win_amd64.whl", hash = "sha256:a1862d2d7ce1674cffa6d186d53ca95c6e17ed2b06b3f4c476173565c862d232"},
+    {file = "rpds_py-0.20.0-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:3fde368e9140312b6e8b6c09fb9f8c8c2f00999d1823403ae90cc00480221b22"},
+    {file = "rpds_py-0.20.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:9824fb430c9cf9af743cf7aaf6707bf14323fb51ee74425c380f4c846ea70789"},
+    {file = "rpds_py-0.20.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:11ef6ce74616342888b69878d45e9f779b95d4bd48b382a229fe624a409b72c5"},
+    {file = "rpds_py-0.20.0-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c52d3f2f82b763a24ef52f5d24358553e8403ce05f893b5347098014f2d9eff2"},
+    {file = "rpds_py-0.20.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9d35cef91e59ebbeaa45214861874bc6f19eb35de96db73e467a8358d701a96c"},
+    {file = "rpds_py-0.20.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d72278a30111e5b5525c1dd96120d9e958464316f55adb030433ea905866f4de"},
+    {file = "rpds_py-0.20.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b4c29cbbba378759ac5786730d1c3cb4ec6f8ababf5c42a9ce303dc4b3d08cda"},
+    {file = "rpds_py-0.20.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:6632f2d04f15d1bd6fe0eedd3b86d9061b836ddca4c03d5cf5c7e9e6b7c14580"},
+    {file = "rpds_py-0.20.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:d0b67d87bb45ed1cd020e8fbf2307d449b68abc45402fe1a4ac9e46c3c8b192b"},
+    {file = "rpds_py-0.20.0-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:ec31a99ca63bf3cd7f1a5ac9fe95c5e2d060d3c768a09bc1d16e235840861420"},
+    {file = "rpds_py-0.20.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:22e6c9976e38f4d8c4a63bd8a8edac5307dffd3ee7e6026d97f3cc3a2dc02a0b"},
+    {file = "rpds_py-0.20.0-cp39-none-win32.whl", hash = "sha256:569b3ea770c2717b730b61998b6c54996adee3cef69fc28d444f3e7920313cf7"},
+    {file = "rpds_py-0.20.0-cp39-none-win_amd64.whl", hash = "sha256:e6900ecdd50ce0facf703f7a00df12374b74bbc8ad9fe0f6559947fb20f82364"},
+    {file = "rpds_py-0.20.0-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:617c7357272c67696fd052811e352ac54ed1d9b49ab370261a80d3b6ce385045"},
+    {file = "rpds_py-0.20.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:9426133526f69fcaba6e42146b4e12d6bc6c839b8b555097020e2b78ce908dcc"},
+    {file = "rpds_py-0.20.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:deb62214c42a261cb3eb04d474f7155279c1a8a8c30ac89b7dcb1721d92c3c02"},
+    {file = "rpds_py-0.20.0-pp310-pypy310_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:fcaeb7b57f1a1e071ebd748984359fef83ecb026325b9d4ca847c95bc7311c92"},
+    {file = "rpds_py-0.20.0-pp310-pypy310_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d454b8749b4bd70dd0a79f428731ee263fa6995f83ccb8bada706e8d1d3ff89d"},
+    {file = "rpds_py-0.20.0-pp310-pypy310_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d807dc2051abe041b6649681dce568f8e10668e3c1c6543ebae58f2d7e617855"},
+    {file = "rpds_py-0.20.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c3c20f0ddeb6e29126d45f89206b8291352b8c5b44384e78a6499d68b52ae511"},
+    {file = "rpds_py-0.20.0-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:b7f19250ceef892adf27f0399b9e5afad019288e9be756d6919cb58892129f51"},
+    {file = "rpds_py-0.20.0-pp310-pypy310_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:4f1ed4749a08379555cebf4650453f14452eaa9c43d0a95c49db50c18b7da075"},
+    {file = "rpds_py-0.20.0-pp310-pypy310_pp73-musllinux_1_2_i686.whl", hash = "sha256:dcedf0b42bcb4cfff4101d7771a10532415a6106062f005ab97d1d0ab5681c60"},
+    {file = "rpds_py-0.20.0-pp310-pypy310_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:39ed0d010457a78f54090fafb5d108501b5aa5604cc22408fc1c0c77eac14344"},
+    {file = "rpds_py-0.20.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:bb273176be34a746bdac0b0d7e4e2c467323d13640b736c4c477881a3220a989"},
+    {file = "rpds_py-0.20.0-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:f918a1a130a6dfe1d7fe0f105064141342e7dd1611f2e6a21cd2f5c8cb1cfb3e"},
+    {file = "rpds_py-0.20.0-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:f60012a73aa396be721558caa3a6fd49b3dd0033d1675c6d59c4502e870fcf0c"},
+    {file = "rpds_py-0.20.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3d2b1ad682a3dfda2a4e8ad8572f3100f95fad98cb99faf37ff0ddfe9cbf9d03"},
+    {file = "rpds_py-0.20.0-pp39-pypy39_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:614fdafe9f5f19c63ea02817fa4861c606a59a604a77c8cdef5aa01d28b97921"},
+    {file = "rpds_py-0.20.0-pp39-pypy39_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fa518bcd7600c584bf42e6617ee8132869e877db2f76bcdc281ec6a4113a53ab"},
+    {file = "rpds_py-0.20.0-pp39-pypy39_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f0475242f447cc6cb8a9dd486d68b2ef7fbee84427124c232bff5f63b1fe11e5"},
+    {file = "rpds_py-0.20.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f90a4cd061914a60bd51c68bcb4357086991bd0bb93d8aa66a6da7701370708f"},
+    {file = "rpds_py-0.20.0-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:def7400461c3a3f26e49078302e1c1b38f6752342c77e3cf72ce91ca69fb1bc1"},
+    {file = "rpds_py-0.20.0-pp39-pypy39_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:65794e4048ee837494aea3c21a28ad5fc080994dfba5b036cf84de37f7ad5074"},
+    {file = "rpds_py-0.20.0-pp39-pypy39_pp73-musllinux_1_2_i686.whl", hash = "sha256:faefcc78f53a88f3076b7f8be0a8f8d35133a3ecf7f3770895c25f8813460f08"},
+    {file = "rpds_py-0.20.0-pp39-pypy39_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:5b4f105deeffa28bbcdff6c49b34e74903139afa690e35d2d9e3c2c2fba18cec"},
+    {file = "rpds_py-0.20.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:fdfc3a892927458d98f3d55428ae46b921d1f7543b89382fdb483f5640daaec8"},
+    {file = "rpds_py-0.20.0.tar.gz", hash = "sha256:d72a210824facfdaf8768cf2d7ca25a042c30320b3020de2fa04640920d4e121"},
+]
+
+[[package]]
+name = "ruff"
+version = "0.6.8"
+description = "An extremely fast Python linter and code formatter, written in Rust."
+optional = false
+python-versions = ">=3.7"
+groups = ["dev"]
+files = [
+    {file = "ruff-0.6.8-py3-none-linux_armv6l.whl", hash = "sha256:77944bca110ff0a43b768f05a529fecd0706aac7bcce36d7f1eeb4cbfca5f0f2"},
+    {file = "ruff-0.6.8-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:27b87e1801e786cd6ede4ada3faa5e254ce774de835e6723fd94551464c56b8c"},
+    {file = "ruff-0.6.8-py3-none-macosx_11_0_arm64.whl", hash = "sha256:cd48f945da2a6334f1793d7f701725a76ba93bf3d73c36f6b21fb04d5338dcf5"},
+    {file = "ruff-0.6.8-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:677e03c00f37c66cea033274295a983c7c546edea5043d0c798833adf4cf4c6f"},
+    {file = "ruff-0.6.8-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:9f1476236b3eacfacfc0f66aa9e6cd39f2a624cb73ea99189556015f27c0bdeb"},
+    {file = "ruff-0.6.8-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6f5a2f17c7d32991169195d52a04c95b256378bbf0de8cb98478351eb70d526f"},
+    {file = "ruff-0.6.8-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:5fd0d4b7b1457c49e435ee1e437900ced9b35cb8dc5178921dfb7d98d65a08d0"},
+    {file = "ruff-0.6.8-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f8034b19b993e9601f2ddf2c517451e17a6ab5cdb1c13fdff50c1442a7171d87"},
+    {file = "ruff-0.6.8-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6cfb227b932ba8ef6e56c9f875d987973cd5e35bc5d05f5abf045af78ad8e098"},
+    {file = "ruff-0.6.8-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6ef0411eccfc3909269fed47c61ffebdcb84a04504bafa6b6df9b85c27e813b0"},
+    {file = "ruff-0.6.8-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:007dee844738c3d2e6c24ab5bc7d43c99ba3e1943bd2d95d598582e9c1b27750"},
+    {file = "ruff-0.6.8-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:ce60058d3cdd8490e5e5471ef086b3f1e90ab872b548814e35930e21d848c9ce"},
+    {file = "ruff-0.6.8-py3-none-musllinux_1_2_i686.whl", hash = "sha256:1085c455d1b3fdb8021ad534379c60353b81ba079712bce7a900e834859182fa"},
+    {file = "ruff-0.6.8-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:70edf6a93b19481affd287d696d9e311388d808671bc209fb8907b46a8c3af44"},
+    {file = "ruff-0.6.8-py3-none-win32.whl", hash = "sha256:792213f7be25316f9b46b854df80a77e0da87ec66691e8f012f887b4a671ab5a"},
+    {file = "ruff-0.6.8-py3-none-win_amd64.whl", hash = "sha256:ec0517dc0f37cad14a5319ba7bba6e7e339d03fbf967a6d69b0907d61be7a263"},
+    {file = "ruff-0.6.8-py3-none-win_arm64.whl", hash = "sha256:8d3bb2e3fbb9875172119021a13eed38849e762499e3cfde9588e4b4d70968dc"},
+    {file = "ruff-0.6.8.tar.gz", hash = "sha256:a5bf44b1aa0adaf6d9d20f86162b34f7c593bfedabc51239953e446aefc8ce18"},
+]
+
+[[package]]
+name = "six"
+version = "1.16.0"
+description = "Python 2 and 3 compatibility utilities"
+optional = false
+python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*"
+groups = ["main"]
+files = [
+    {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"},
+    {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"},
+]
+
+[[package]]
+name = "strict-rfc3339"
+version = "0.7"
+description = "Strict, simple, lightweight RFC3339 functions"
+optional = false
+python-versions = "*"
+groups = ["main"]
+files = [
+    {file = "strict-rfc3339-0.7.tar.gz", hash = "sha256:5cad17bedfc3af57b399db0fed32771f18fc54bbd917e85546088607ac5e1277"},
+]
+
+[[package]]
+name = "timeslot"
+version = "0.1.2"
+description = "Data type for representing time slots with a start and end."
+optional = false
+python-versions = ">=3.6,<4.0"
+groups = ["main"]
+files = [
+    {file = "timeslot-0.1.2-py3-none-any.whl", hash = "sha256:2f8efaec7b0a4c1e56a92ec05533219332dd9d8b577539077664c233996911b5"},
+    {file = "timeslot-0.1.2.tar.gz", hash = "sha256:a2ac998657e3f3b9ca928757b4906add2c05390c5fc14ed792bb9028d08547b1"},
+]
+
+[[package]]
+name = "tomli"
+version = "2.0.1"
+description = "A lil' TOML parser"
+optional = false
+python-versions = ">=3.7"
+groups = ["dev"]
+markers = "python_full_version <= \"3.11.0a6\""
+files = [
+    {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"},
+    {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"},
+]
+
+[[package]]
+name = "tomlkit"
+version = "0.13.2"
+description = "Style preserving TOML library"
+optional = false
+python-versions = ">=3.8"
+groups = ["main"]
+files = [
+    {file = "tomlkit-0.13.2-py3-none-any.whl", hash = "sha256:7a974427f6e119197f670fbbbeae7bef749a6c14e793db934baefc1b5f03efde"},
+    {file = "tomlkit-0.13.2.tar.gz", hash = "sha256:fff5fe59a87295b278abd31bec92c15d9bc4a06885ab12bcea52c71119392e79"},
+]
+
+[[package]]
+name = "typing-extensions"
+version = "4.12.2"
+description = "Backported and Experimental Type Hints for Python 3.8+"
+optional = false
+python-versions = ">=3.8"
+groups = ["dev"]
+files = [
+    {file = "typing_extensions-4.12.2-py3-none-any.whl", hash = "sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d"},
+    {file = "typing_extensions-4.12.2.tar.gz", hash = "sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8"},
+]
+
+[[package]]
+name = "zipp"
+version = "3.20.2"
+description = "Backport of pathlib-compatible object wrapper for zip files"
+optional = false
+python-versions = ">=3.8"
+groups = ["main"]
+markers = "python_version == \"3.8\""
+files = [
+    {file = "zipp-3.20.2-py3-none-any.whl", hash = "sha256:a817ac80d6cf4b23bf7f2828b7cabf326f15a001bea8b1f9b49631780ba28350"},
+    {file = "zipp-3.20.2.tar.gz", hash = "sha256:bc9eb26f4506fda01b81bcde0ca78103b6e62f991b381fec825435c836edbc29"},
+]
+
+[package.extras]
+check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1) ; sys_platform != \"cygwin\""]
+cover = ["pytest-cov"]
+doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"]
+enabler = ["pytest-enabler (>=2.2)"]
+test = ["big-O", "importlib-resources ; python_version < \"3.9\"", "jaraco.functools", "jaraco.itertools", "jaraco.test", "more-itertools", "pytest (>=6,!=8.1.*)", "pytest-ignore-flaky"]
+type = ["pytest-mypy"]
+
+[metadata]
+lock-version = "2.1"
+python-versions = "^3.8"
+content-hash = "9ac93f2d6bac28473f4b5a4dabda037554159497caa7ae113ca25ea79f763cbe"

pyproject.toml

@@ -0,0 +1,53 @@
+[tool.poetry]
+name = "aw-core"
+version = "0.5.18"
+description = "Core library for ActivityWatch"
+authors = ["Erik Bjäreholt <erik@bjareho.lt>", "Johan Bjäreholt <johan@bjareho.lt>"]
+license = "MPL-2.0"
+readme = "README.md"
+homepage = "https://activitywatch.net/"
+repository = "https://github.com/ActivityWatch/aw-core/"
+documentation = "https://docs.activitywatch.net/"
+packages = [
+    { include = "aw_cli" },
+    { include = "aw_core" },
+    { include = "aw_datastore" },
+    { include = "aw_transform" },
+    { include = "aw_query" },
+]
+
+[tool.poetry.scripts]
+aw-cli = "aw_cli.__main__:main"
+
+[tool.poetry.dependencies]
+python = "^3.8"
+jsonschema = "^4.3"
+peewee = ">=3.0.0,<5.0.0"
+platformdirs = "3.10"  # pinned due to sometimes breaking changes in minor versions: https://github.com/ActivityWatch/aw-core/pull/122#issuecomment-1768020335
+iso8601 = "*"
+rfc3339-validator = "^0.1.4"  # needed for the date-type format in jsonschema
+strict-rfc3339 = "^0.7"
+tomlkit = "*"
+deprecation = "*"
+timeslot = "*"
+click = "*"
+
+[tool.poetry.group.dev.dependencies]
+mypy = "*"
+ruff = "*"
+pytest = "*"
+pytest-cov = "*"
+
+[tool.ruff]
+target-version = "py38"
+
+[tool.ruff.lint]
+select = ["E4", "E7", "E9", "F", "B", "UP"]
+ignore = ["E402", "E501", "B905"]
+
+[tool.lint.ruff]
+ignore = ["E402", "E501"]
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"

scripts/install_python.ps1

@@ -0,0 +1,233 @@
+# Sample script to install Python and pip under Windows
+# Authors: Olivier Grisel, Jonathan Helmus, Kyle Kastner, and Alex Willmer
+# License: CC0 1.0 Universal: http://creativecommons.org/publicdomain/zero/1.0/
+#
+# Find the latest version of this script at:
+# https://github.com/ogrisel/python-appveyor-demo/blob/master/appveyor/install.ps1
+
+
+$MINICONDA_URL = "http://repo.continuum.io/miniconda/"
+$BASE_URL = "https://www.python.org/ftp/python/"
+$GET_PIP_URL = "https://bootstrap.pypa.io/get-pip.py"
+$GET_PIP_PATH = "C:\get-pip.py"
+
+$PYTHON_PRERELEASE_REGEX = @"
+(?x)
+(?<major>\d+)
+\.
+(?<minor>\d+)
+\.
+(?<micro>\d+)
+(?<prerelease>[a-z]{1,2}\d+)
+"@
+
+
+function Download ($filename, $url) {
+    $webclient = New-Object System.Net.WebClient
+
+    $basedir = $pwd.Path + "\"
+    $filepath = $basedir + $filename
+    if (Test-Path $filename) {
+        Write-Host "Reusing" $filepath
+        return $filepath
+    }
+
+    # Download and retry up to 3 times in case of network transient errors.
+    Write-Host "Downloading" $filename "from" $url
+    $retry_attempts = 2
+    for ($i = 0; $i -lt $retry_attempts; $i++) {
+        try {
+            $webclient.DownloadFile($url, $filepath)
+            break
+        }
+        Catch [Exception]{
+            Start-Sleep 1
+        }
+    }
+    if (Test-Path $filepath) {
+        Write-Host "File saved at" $filepath
+    } else {
+        # Retry once to get the error message if any at the last try
+        $webclient.DownloadFile($url, $filepath)
+    }
+    return $filepath
+}
+
+
+function ParsePythonVersion ($python_version) {
+    if ($python_version -match $PYTHON_PRERELEASE_REGEX) {
+        return ([int]$matches.major, [int]$matches.minor, [int]$matches.micro,
+                $matches.prerelease)
+    }
+    $version_obj = [version]$python_version
+    return ($version_obj.major, $version_obj.minor, $version_obj.build, "")
+}
+
+
+function DownloadPython ($python_version, $platform_suffix) {
+    $major, $minor, $micro, $prerelease = ParsePythonVersion $python_version
+
+    if (($major -le 2 -and $micro -eq 0) `
+        -or ($major -eq 3 -and $minor -le 2 -and $micro -eq 0) `
+        ) {
+        $dir = "$major.$minor"
+        $python_version = "$major.$minor$prerelease"
+    } else {
+        $dir = "$major.$minor.$micro"
+    }
+
+    if ($prerelease) {
+        if (($major -le 2) `
+            -or ($major -eq 3 -and $minor -eq 1) `
+            -or ($major -eq 3 -and $minor -eq 2) `
+            -or ($major -eq 3 -and $minor -eq 3) `
+            ) {
+            $dir = "$dir/prev"
+        }
+    }
+
+    if (($major -le 2) -or ($major -le 3 -and $minor -le 4)) {
+        $ext = "msi"
+        if ($platform_suffix) {
+            $platform_suffix = ".$platform_suffix"
+        }
+    } else {
+        $ext = "exe"
+        if ($platform_suffix) {
+            $platform_suffix = "-$platform_suffix"
+        }
+    }
+
+    $filename = "python-$python_version$platform_suffix.$ext"
+    $url = "$BASE_URL$dir/$filename"
+    $filepath = Download $filename $url
+    return $filepath
+}
+
+
+function InstallPython ($python_version, $architecture, $python_home) {
+    Write-Host "Installing Python" $python_version "for" $architecture "bit architecture to" $python_home
+    if (Test-Path $python_home) {
+        Write-Host $python_home "already exists, skipping."
+        return $false
+    }
+    if ($architecture -eq "32") {
+        $platform_suffix = ""
+    } else {
+        $platform_suffix = "amd64"
+    }
+    $installer_path = DownloadPython $python_version $platform_suffix
+    $installer_ext = [System.IO.Path]::GetExtension($installer_path)
+    Write-Host "Installing $installer_path to $python_home"
+    $install_log = $python_home + ".log"
+    if ($installer_ext -eq '.msi') {
+        InstallPythonMSI $installer_path $python_home $install_log
+    } else {
+        InstallPythonEXE $installer_path $python_home $install_log
+    }
+    if (Test-Path $python_home) {
+        Write-Host "Python $python_version ($architecture) installation complete"
+    } else {
+        Write-Host "Failed to install Python in $python_home"
+        Get-Content -Path $install_log
+        Exit 1
+    }
+}
+
+
+function InstallPythonEXE ($exepath, $python_home, $install_log) {
+    $install_args = "/quiet InstallAllUsers=1 TargetDir=$python_home"
+    RunCommand $exepath $install_args
+}
+
+
+function InstallPythonMSI ($msipath, $python_home, $install_log) {
+    $install_args = "/qn /log $install_log /i $msipath TARGETDIR=$python_home"
+    $uninstall_args = "/qn /x $msipath"
+    RunCommand "msiexec.exe" $install_args
+    if (-not(Test-Path $python_home)) {
+        Write-Host "Python seems to be installed else-where, reinstalling."
+        RunCommand "msiexec.exe" $uninstall_args
+        RunCommand "msiexec.exe" $install_args
+    }
+}
+
+function RunCommand ($command, $command_args) {
+    Write-Host $command $command_args
+    Start-Process -FilePath $command -ArgumentList $command_args -Wait -Passthru
+}
+
+
+function InstallPip ($python_home) {
+    $pip_path = $python_home + "\Scripts\pip.exe"
+    $python_path = $python_home + "\python.exe"
+    if (-not(Test-Path $pip_path)) {
+        Write-Host "Installing pip..."
+        $webclient = New-Object System.Net.WebClient
+        $webclient.DownloadFile($GET_PIP_URL, $GET_PIP_PATH)
+        Write-Host "Executing:" $python_path $GET_PIP_PATH
+        & $python_path $GET_PIP_PATH
+    } else {
+        Write-Host "pip already installed."
+    }
+}
+
+
+function DownloadMiniconda ($python_version, $platform_suffix) {
+    if ($python_version -eq "3.4") {
+        $filename = "Miniconda3-3.5.5-Windows-" + $platform_suffix + ".exe"
+    } else {
+        $filename = "Miniconda-3.5.5-Windows-" + $platform_suffix + ".exe"
+    }
+    $url = $MINICONDA_URL + $filename
+    $filepath = Download $filename $url
+    return $filepath
+}
+
+
+function InstallMiniconda ($python_version, $architecture, $python_home) {
+    Write-Host "Installing Python" $python_version "for" $architecture "bit architecture to" $python_home
+    if (Test-Path $python_home) {
+        Write-Host $python_home "already exists, skipping."
+        return $false
+    }
+    if ($architecture -eq "32") {
+        $platform_suffix = "x86"
+    } else {
+        $platform_suffix = "x86_64"
+    }
+    $filepath = DownloadMiniconda $python_version $platform_suffix
+    Write-Host "Installing" $filepath "to" $python_home
+    $install_log = $python_home + ".log"
+    $args = "/S /D=$python_home"
+    Write-Host $filepath $args
+    Start-Process -FilePath $filepath -ArgumentList $args -Wait -Passthru
+    if (Test-Path $python_home) {
+        Write-Host "Python $python_version ($architecture) installation complete"
+    } else {
+        Write-Host "Failed to install Python in $python_home"
+        Get-Content -Path $install_log
+        Exit 1
+    }
+}
+
+
+function InstallMinicondaPip ($python_home) {
+    $pip_path = $python_home + "\Scripts\pip.exe"
+    $conda_path = $python_home + "\Scripts\conda.exe"
+    if (-not(Test-Path $pip_path)) {
+        Write-Host "Installing pip..."
+        $args = "install --yes pip"
+        Write-Host $conda_path $args
+        Start-Process -FilePath "$conda_path" -ArgumentList $args -Wait -Passthru
+    } else {
+        Write-Host "pip already installed."
+    }
+}
+
+function main () {
+    InstallPython $env:PYTHON_VERSION $env:PYTHON_ARCH $env:PYTHON
+    InstallPip $env:PYTHON
+}
+
+main

scripts/run_with_env.cmd

@@ -0,0 +1,87 @@
+:: To build extensions for 64 bit Python 3, we need to configure environment
+:: variables to use the MSVC 2010 C++ compilers from GRMSDKX_EN_DVD.iso of:
+:: MS Windows SDK for Windows 7 and .NET Framework 4 (SDK v7.1)
+::
+:: To build extensions for 64 bit Python 2, we need to configure environment
+:: variables to use the MSVC 2008 C++ compilers from GRMSDKX_EN_DVD.iso of:
+:: MS Windows SDK for Windows 7 and .NET Framework 3.5 (SDK v7.0)
+::
+:: 32 bit builds, and 64-bit builds for 3.5 and beyond, do not require specific
+:: environment configurations.
+::
+:: Note: this script needs to be run with the /E:ON and /V:ON flags for the
+:: cmd interpreter, at least for (SDK v7.0)
+::
+:: More details at:
+:: https://github.com/cython/cython/wiki/64BitCythonExtensionsOnWindows
+:: http://stackoverflow.com/a/13751649/163740
+::
+:: Author: Olivier Grisel
+:: License: CC0 1.0 Universal: http://creativecommons.org/publicdomain/zero/1.0/
+::
+:: Notes about batch files for Python people:
+::
+:: Quotes in values are literally part of the values:
+::      SET FOO="bar"
+:: FOO is now five characters long: " b a r "
+:: If you don't want quotes, don't include them on the right-hand side.
+::
+:: The CALL lines at the end of this file look redundant, but if you move them
+:: outside of the IF clauses, they do not run properly in the SET_SDK_64==Y
+:: case, I don't know why.
+@ECHO OFF
+SET COMMAND_TO_RUN=%*
+SET WIN_SDK_ROOT=C:\Program Files\Microsoft SDKs\Windows
+SET WIN_WDK=c:\Program Files (x86)\Windows Kits\10\Include\wdf
+
+:: Extract the major and minor versions, and allow for the minor version to be
+:: more than 9.  This requires the version number to have two dots in it.
+SET MAJOR_PYTHON_VERSION=%PYTHON_VERSION:~0,1%
+IF "%PYTHON_VERSION:~3,1%" == "." (
+    SET MINOR_PYTHON_VERSION=%PYTHON_VERSION:~2,1%
+) ELSE (
+    SET MINOR_PYTHON_VERSION=%PYTHON_VERSION:~2,2%
+)
+
+:: Based on the Python version, determine what SDK version to use, and whether
+:: to set the SDK for 64-bit.
+IF %MAJOR_PYTHON_VERSION% == 2 (
+    SET WINDOWS_SDK_VERSION="v7.0"
+    SET SET_SDK_64=Y
+) ELSE (
+    IF %MAJOR_PYTHON_VERSION% == 3 (
+        SET WINDOWS_SDK_VERSION="v7.1"
+        IF %MINOR_PYTHON_VERSION% LEQ 4 (
+            SET SET_SDK_64=Y
+        ) ELSE (
+            SET SET_SDK_64=N
+            IF EXIST "%WIN_WDK%" (
+                :: See: https://connect.microsoft.com/VisualStudio/feedback/details/1610302/
+                REN "%WIN_WDK%" 0wdf
+            )
+        )
+    ) ELSE (
+        ECHO Unsupported Python version: "%MAJOR_PYTHON_VERSION%"
+        EXIT 1
+    )
+)
+
+IF %PYTHON_ARCH% == 64 (
+    IF %SET_SDK_64% == Y (
+        ECHO Configuring Windows SDK %WINDOWS_SDK_VERSION% for Python %MAJOR_PYTHON_VERSION% on a 64 bit architecture
+        SET DISTUTILS_USE_SDK=1
+        SET MSSdk=1
+        "%WIN_SDK_ROOT%\%WINDOWS_SDK_VERSION%\Setup\WindowsSdkVer.exe" -q -version:%WINDOWS_SDK_VERSION%
+        "%WIN_SDK_ROOT%\%WINDOWS_SDK_VERSION%\Bin\SetEnv.cmd" /x64 /release
+        ECHO Executing: %COMMAND_TO_RUN%
+        call %COMMAND_TO_RUN% || EXIT 1
+    ) ELSE (
+        ECHO Using default MSVC build environment for 64 bit architecture
+        ECHO Executing: %COMMAND_TO_RUN%
+        call %COMMAND_TO_RUN% || EXIT 1
+    )
+) ELSE (
+    ECHO Using default MSVC build environment for 32 bit architecture
+    ECHO Executing: %COMMAND_TO_RUN%
+    call %COMMAND_TO_RUN% || EXIT 1
+)

stubs/appdirs.pyi

@@ -0,0 +1,4 @@
+def user_data_dir(a1: str, a2: str = "") -> str: ...
+def user_cache_dir(a1: str, a2: str = "") -> str: ...
+def user_config_dir(a1: str, a2: str = "") -> str: ...
+def user_log_dir(a1: str, a2: str = "") -> str: ...

stubs/iso8601.pyi

@@ -0,0 +1,6 @@
+from datetime import datetime
+
+def parse_date(x: str) -> datetime: ...  # function
+
+class ParseError(Exception):
+    pass

tests/__init__.py

@@ -0,0 +1 @@
+from . import context  # noqa: F401

tests/context.py

@@ -0,0 +1,6 @@
+import os
+import sys
+
+# This will cause imports to use the packages in the
+# source folder instead of those installed on the system.
+sys.path.insert(0, os.path.abspath(".."))

tests/test_cli_log.py

@@ -0,0 +1,64 @@
+"""Tests for aw_cli.log.find_oldest_log after profile isolation (#149).
+
+With per-profile appname dirs, every .log file in a module dir belongs to the
+active profile — no filename-based filtering is needed.
+"""
+
+import time
+from pathlib import Path
+
+import pytest
+
+from aw_cli.log import find_oldest_log
+
+from . import context  # noqa: F401
+
+
+@pytest.fixture()
+def log_dir(tmp_path: Path) -> Path:
+    return tmp_path / "aw-server"
+
+
+def test_returns_none_for_missing_dir(tmp_path: Path) -> None:
+    assert find_oldest_log(tmp_path / "nonexistent") is None
+
+
+def test_returns_none_for_empty_dir(log_dir: Path) -> None:
+    log_dir.mkdir()
+    assert find_oldest_log(log_dir) is None
+
+
+def test_returns_single_log(log_dir: Path) -> None:
+    log_dir.mkdir()
+    f = log_dir / "aw-server_2026-01-01.log"
+    f.write_text("line\n")
+    assert find_oldest_log(log_dir) == f
+
+
+def test_returns_newest_by_mtime(log_dir: Path) -> None:
+    log_dir.mkdir()
+    old = log_dir / "aw-server_old.log"
+    new = log_dir / "aw-server_new.log"
+    old.write_text("old\n")
+    time.sleep(0.01)
+    new.write_text("new\n")
+    assert find_oldest_log(log_dir) == new
+
+
+def test_ignores_non_log_files(log_dir: Path) -> None:
+    log_dir.mkdir()
+    (log_dir / "notes.txt").write_text("not a log")
+    assert find_oldest_log(log_dir) is None
+
+
+def test_no_profile_filtering_in_filenames(log_dir: Path) -> None:
+    """All .log files in the dir belong to the active profile — no name filter."""
+    log_dir.mkdir()
+    default_log = log_dir / "aw-server_2026-01-01.log"
+    default_log.write_text("line\n")
+    # A file named with 'testing' in its name is still returned;
+    # profile selection happens at the directory level, not the filename level.
+    testing_named = log_dir / "aw-server_testing_2026-01-02.log"
+    time.sleep(0.01)
+    testing_named.write_text("testing-named\n")
+    assert find_oldest_log(log_dir) == testing_named

tests/test_config.py

@@ -0,0 +1,102 @@
+import shutil
+from configparser import ConfigParser
+from pathlib import Path
+
+import deprecation
+import pytest
+from aw_core import dirs
+from aw_core.config import (
+    load_config,
+    load_config_toml,
+    save_config,
+    save_config_toml,
+)
+
+appname = "aw-core-test"
+section = "section"
+config_dir = dirs.get_config_dir(appname)
+
+default_config_str = f"""# A default config file, with comments!
+[{section}]
+somestring = "Hello World!"    # A comment
+somevalue = 12.3               # Another comment
+somearray = ["asd", 123]"""
+
+
+@pytest.fixture(autouse=True)
+def clean_config():
+    # Remove test config file if it already exists
+    shutil.rmtree(config_dir, ignore_errors=True)
+
+    # Rerun get_config dir to create config directory
+    dirs.get_config_dir(appname)
+
+    yield
+
+    # Remove test config file if it already exists
+    shutil.rmtree(config_dir)
+
+
+def test_create():
+    appname = "aw-core-test"
+    config_dir = dirs.get_config_dir(appname)
+    assert Path(config_dir).exists()
+
+
+def test_config_defaults():
+    # Load non-existing config (will create a out-commented default config file)
+    config = load_config_toml(appname, default_config_str)
+
+    # Check that load_config used defaults
+    assert config[section]["somestring"] == "Hello World!"
+    assert config[section]["somevalue"] == 12.3
+    assert config[section]["somearray"] == ["asd", 123]
+
+
+def test_config_no_defaults():
+    # Write defaults to file
+    save_config_toml(appname, default_config_str)
+
+    # Load written defaults without defaults
+    config = load_config_toml(appname, "")
+    assert config[section]["somestring"] == "Hello World!"
+    assert config[section]["somevalue"] == 12.3
+    assert config[section]["somearray"] == ["asd", 123]
+
+
+def test_config_override():
+    # Create a minimal config file with one overridden value
+    config_str = """[section]
+somevalue = 1000.1"""
+    save_config_toml(appname, config_str)
+
+    # Open non-default config file and verify that values are correct
+    config = load_config_toml(appname, default_config_str)
+    assert config[section]["somevalue"] == 1000.1
+
+
+@deprecation.fail_if_not_removed
+def test_config_ini():
+    # Create default config
+    default_config = ConfigParser()
+    default_config[section] = {"somestring": "Hello World!", "somevalue": 12.3}  # type: ignore
+
+    # Load non-existing config (will create a default config file)
+    config = load_config(appname, default_config)
+
+    # Check that current config file is same as default config file
+    assert config[section]["somestring"] == default_config[section]["somestring"]
+    assert config[section].getfloat("somevalue") == default_config[section].getfloat(
+        "somevalue"
+    )
+
+    # Modify and save config file
+    config[section]["somevalue"] = "1000.1"
+    save_config(appname, config)
+
+    # Open non-default config file and verify that values are correct
+    new_config = load_config(appname, default_config)
+    assert new_config[section]["somestring"] == config[section]["somestring"]
+    assert new_config[section].getfloat("somevalue") == config[section].getfloat(
+        "somevalue"
+    )

tests/test_datastore.py

@@ -0,0 +1,586 @@
+import logging
+import random
+from datetime import datetime, timedelta, timezone
+
+import iso8601
+import pytest
+from aw_core.models import Event
+from aw_datastore import get_storage_methods
+from aw_datastore.storages import PeeweeStorage
+
+from . import context  # noqa: F401
+from .utils import param_datastore_objects, param_testing_buckets_cm
+
+logging.basicConfig(level=logging.DEBUG)
+
+# Useful when you just want some placeholder time in your events, saves typing
+now = datetime.now(tz=timezone.utc)
+td1s = timedelta(seconds=1)
+td1d = timedelta(days=1)
+
+
+def test_get_storage_methods():
+    assert get_storage_methods()
+
+
+@pytest.mark.parametrize("datastore", param_datastore_objects())
+def test_get_buckets(datastore):
+    """
+    Tests fetching buckets
+    """
+    buckets = datastore.buckets()
+    for bucket in buckets.values():
+        assert bucket["id"] in buckets
+        assert bucket["data"] == {}
+
+
+@pytest.mark.parametrize("datastore", param_datastore_objects())
+def test_create_bucket(datastore):
+    name = "A label/name for a test bucket"
+    bid = "test-identifier"
+    try:
+        bucket = datastore.create_bucket(
+            bucket_id=bid,
+            type="testtype",
+            client="testclient",
+            hostname="testhost",
+            name=name,
+            created=now,
+            data={"key": "value"},
+        )
+        metadata = bucket.metadata()
+        assert bid == metadata["id"]
+        assert name == metadata["name"]
+        assert "testtype" == metadata["type"]
+        assert "testclient" == metadata["client"]
+        assert "testhost" == metadata["hostname"]
+        assert now == iso8601.parse_date(metadata["created"])
+        assert bid in datastore.buckets()
+        assert {"key": "value"} == metadata["data"]
+    finally:
+        datastore.delete_bucket(bid)
+    assert bid not in datastore.buckets()
+
+
+@pytest.mark.parametrize("datastore", param_datastore_objects())
+def test_update_bucket(datastore):
+    bid = "test-" + str(random.randint(0, 1000000))
+    try:
+        datastore.create_bucket(
+            bucket_id=bid, type="test", client="test", hostname="test", name="test"
+        )
+        datastore.update_bucket(bid, name="new name")
+        assert datastore[bid].metadata()["name"] == "new name"
+        datastore.update_bucket(bid, data={"key": "value"})
+        assert datastore[bid].metadata()["data"] == {"key": "value"}
+    finally:
+        datastore.delete_bucket(bid)
+
+
+@pytest.mark.parametrize("datastore", param_datastore_objects())
+def test_delete_bucket(datastore):
+    bid = "test-" + str(random.randint(0, 1000000))
+    datastore.create_bucket(
+        bucket_id=bid, type="test", client="test", hostname="test", name="test"
+    )
+    datastore.delete_bucket(bid)
+    assert bid not in datastore.buckets()
+    with pytest.raises(ValueError):
+        datastore.delete_bucket(bid)
+
+
+@pytest.mark.parametrize("datastore", param_datastore_objects())
+def test_nonexistent_bucket(datastore):
+    """
+    Tests that a KeyError is raised if you request a non-existent bucket
+    """
+    with pytest.raises(KeyError):
+        datastore["I-do-not-exist"]
+
+
+@pytest.mark.parametrize("bucket_cm", param_testing_buckets_cm())
+def test_insert_one(bucket_cm):
+    """
+    Tests inserting one event into a bucket
+    """
+    with bucket_cm as bucket:
+        n_events = len(bucket.get())
+        event = Event(timestamp=now, duration=td1s, data={"key": "val"})
+        bucket.insert(event)
+        fetched_events = bucket.get()
+        assert n_events + 1 == len(fetched_events)
+        assert isinstance(fetched_events[0], Event)
+        assert event == fetched_events[0]
+        logging.info(event)
+        logging.info(fetched_events[0].to_json_str())
+
+
+@pytest.mark.parametrize("bucket_cm", param_testing_buckets_cm())
+def test_empty_bucket(bucket_cm):
+    """
+    Ensures empty buckets are empty
+    """
+    with bucket_cm as bucket:
+        assert 0 == len(bucket.get())
+
+
+@pytest.mark.parametrize("bucket_cm", param_testing_buckets_cm())
+def test_insert_many(bucket_cm):
+    """
+    Tests that you can insert many events at the same time to a bucket
+    """
+    num_events = 5000
+    with bucket_cm as bucket:
+        events = num_events * [Event(timestamp=now, duration=td1s, data={"key": "val"})]
+        bucket.insert(events)
+        fetched_events = bucket.get(limit=-1)
+        assert num_events == len(fetched_events)
+        for e, fe in zip(events, fetched_events):
+            assert e == fe
+
+
+@pytest.mark.parametrize("bucket_cm", param_testing_buckets_cm())
+def test_insert_many_upsert(bucket_cm):
+    """
+    Tests that you can update/upsert many events at the same time to a bucket
+    """
+    num_events = 10
+    with bucket_cm as bucket:
+        events = num_events * [Event(timestamp=now, duration=td1s, data={"key": "val"})]
+        # insert events to get IDs assigned
+        bucket.insert(events)
+
+        events = bucket.get(limit=-1)
+        assert num_events == len(events)
+        for e in events:
+            assert e.id is not None
+            e.data["key"] = "new val"
+
+        # Upsert the events
+        bucket.insert(events)
+
+        events = bucket.get(limit=-1)
+        assert num_events == len(events)
+        for e in events:
+            assert e.data["key"] == "new val"
+
+
+@pytest.mark.parametrize("bucket_cm", param_testing_buckets_cm())
+def test_delete(bucket_cm):
+    """
+    Tests deleting single events
+    """
+    num_events = 10
+    with bucket_cm as bucket:
+        events = num_events * [Event(timestamp=now, duration=td1s, data={"key": "val"})]
+        bucket.insert(events)
+
+        fetched_events = bucket.get(limit=-1)
+        print(fetched_events[0])
+        assert num_events == len(fetched_events)
+
+        # Test deleting event
+        assert bucket.delete(fetched_events[0]["id"])
+
+        # Test deleting non-existent event
+        # FIXME: Doesn't work due to lazy evaluation in SqliteDatastore
+        # assert not bucket.delete(fetched_events[0]["id"])
+
+        fetched_events = bucket.get(limit=-1)
+        assert num_events - 1 == len(fetched_events)
+
+
+@pytest.mark.parametrize("bucket_cm", param_testing_buckets_cm())
+def test_insert_badtype(bucket_cm):
+    """
+    Tests that you cannot insert non-event types into a bucket
+    """
+    with bucket_cm as bucket:
+        bucket_len = len(bucket.get())
+        badevent = 1
+        with pytest.raises(TypeError):
+            bucket.insert(badevent)
+        assert bucket_len == len(bucket.get())
+
+
+@pytest.mark.parametrize("bucket_cm", param_testing_buckets_cm())
+def test_get_ordered(bucket_cm):
+    """
+    Makes sure that received events are ordered
+    """
+    with bucket_cm as bucket:
+        eventcount = 10
+        events = []
+        for i in range(10):
+            events.append(Event(timestamp=now + i * td1s, duration=td1s))
+        random.shuffle(events)
+        print(events)
+        bucket.insert(events)
+        fetched_events = bucket.get(-1)
+        for i in range(eventcount - 1):
+            print("1:" + fetched_events[i].to_json_str())
+            print("2:" + fetched_events[i + 1].to_json_str())
+            assert fetched_events[i].timestamp > fetched_events[i + 1].timestamp
+
+
+@pytest.mark.parametrize("bucket_cm", param_testing_buckets_cm())
+def test_get_event_with_timezone(bucket_cm):
+    """Tries to retrieve an event using a timezone aware datetime."""
+    hour = timedelta(hours=1)
+    td_offset = 2 * hour
+    tz = timezone(td_offset)
+
+    dt_utc = datetime(2017, 10, 27, hour=0, minute=5, tzinfo=timezone.utc)
+    dt_with_tz = dt_utc.replace(tzinfo=tz)
+
+    with bucket_cm as bucket:
+        bucket.insert(Event(timestamp=dt_with_tz))
+        fetched_events = bucket.get(
+            starttime=dt_with_tz - hour, endtime=dt_with_tz + hour
+        )
+        assert len(fetched_events) == 1
+
+        fetched_events = bucket.get(
+            starttime=dt_utc - td_offset - hour, endtime=dt_utc - td_offset + hour
+        )
+        assert len(fetched_events) == 1
+
+
+@pytest.mark.parametrize("bucket_cm", param_testing_buckets_cm())
+def test_get_datefilter_simple(bucket_cm):
+    with bucket_cm as bucket:
+        eventcount = 3
+        events = [
+            Event(timestamp=now + i * td1s, duration=td1s) for i in range(eventcount)
+        ]
+        bucket.insert(events)
+
+        # Get first event, but expect only half the event to match the interval
+        fetched_events = bucket.get(
+            -1,
+            starttime=now - 0.5 * td1s,
+            endtime=now + 0.5 * td1s,
+        )
+        assert 1 == len(fetched_events)
+
+        # Get first two events, but expect only half of each to match the interval
+        fetched_events = bucket.get(
+            -1,
+            starttime=now + 0.5 * td1s,
+            endtime=now + 1.5 * td1s,
+        )
+        assert 2 == len(fetched_events)
+
+        # Get last event, but expect only half to match the interval
+        fetched_events = bucket.get(
+            -1,
+            starttime=now + 2.5 * td1s,
+            endtime=now + 3.5 * td1s,
+        )
+        assert 1 == len(fetched_events)
+
+        # Check approx precision
+        fetched_events = bucket.get(
+            -1,
+            starttime=now - 0.01 * td1s,
+            endtime=now + 0.01 * td1s,
+        )
+        assert 1 == len(fetched_events)
+
+        # Check precision of start
+        fetched_events = bucket.get(
+            -1,
+            starttime=now,
+            endtime=now,
+        )
+        assert 1 == len(fetched_events)
+
+        # Check approx precision of end
+        fetched_events = bucket.get(
+            -1,
+            starttime=now + 2.99 * td1s,
+            endtime=now + 3.01 * td1s,
+        )
+        assert 1 == len(fetched_events)
+
+
+@pytest.mark.parametrize("bucket_cm", param_testing_buckets_cm())
+def test_get_event_by_id(bucket_cm):
+    """Test that we can retrieve single events by their IDs"""
+    with bucket_cm as bucket:
+        eventcount = 2
+        # Create 1-day long events
+        events = [
+            Event(timestamp=now + i * td1d, duration=td1d) for i in range(eventcount)
+        ]
+        bucket.insert(events)
+
+        # Retrieve stored events
+        events = bucket.get()
+        for e in events:
+            # Query them one-by-one
+            event = bucket.get_by_id(e.id)
+            assert e == event
+
+
+@pytest.mark.parametrize("bucket_cm", param_testing_buckets_cm())
+def test_get_event_by_id_notfound(bucket_cm):
+    """Test fetching an ID that does not exist"""
+    with bucket_cm as bucket:
+        assert bucket.get_by_id(1337 * 10**6) is None
+
+
+@pytest.mark.parametrize("bucket_cm", param_testing_buckets_cm())
+def test_get_event_trimming(bucket_cm):
+    """Test that event trimming works correctly (when querying events that intersect with the query range)"""
+    # TODO: Trimming should be possible to disable
+    # (needed in raw data view, among other places where event editing is permitted)
+
+    with bucket_cm as bucket:
+        if not isinstance(bucket.ds.storage_strategy, PeeweeStorage):
+            pytest.skip("Trimming not supported for datastore")
+
+        eventcount = 2
+        # Create 1-day long events
+        events = [
+            Event(timestamp=now + i * td1d, duration=td1d) for i in range(eventcount)
+        ]
+        bucket.insert(events)
+
+        # Result should contain half of each event
+        fetched_events = bucket.get(
+            -1,
+            starttime=now + td1d / 2,
+            endtime=now + 1.5 * td1d,
+        )
+        assert 2 == len(fetched_events)
+        total_duration = sum((e.duration for e in fetched_events), timedelta())
+        assert td1d == timedelta(seconds=round(total_duration.total_seconds()))
+
+
+@pytest.mark.parametrize("bucket_cm", param_testing_buckets_cm())
+def test_get_datefilter_start(bucket_cm):
+    """
+    Tests the datetimefilter when fetching events
+    """
+    with bucket_cm as bucket:
+        eventcount = 10
+        events = [
+            Event(timestamp=now + i * td1s, duration=td1s) for i in range(eventcount)
+        ]
+        bucket.insert(events)
+
+        # Starttime
+        for i in range(eventcount):
+            fetched_events = bucket.get(-1, starttime=events[i].timestamp + 0.01 * td1s)
+            assert eventcount - i == len(fetched_events)
+
+
+@pytest.mark.parametrize("bucket_cm", param_testing_buckets_cm())
+def test_get_datefilter_end(bucket_cm):
+    """
+    Tests the datetimefilter when fetching events
+    """
+    with bucket_cm as bucket:
+        eventcount = 10
+        events = [
+            Event(timestamp=now + i * td1s, duration=td1s) for i in range(eventcount)
+        ]
+        bucket.insert(events)
+
+        # Endtime
+        for i in range(eventcount):
+            fetched_events = bucket.get(-1, endtime=events[i].timestamp - 0.01 * td1s)
+            assert i == len(fetched_events)
+
+
+@pytest.mark.parametrize("bucket_cm", param_testing_buckets_cm())
+def test_get_datefilter_both(bucket_cm):
+    """
+    Tests the datetimefilter when fetching events
+    """
+    with bucket_cm as bucket:
+        eventcount = 10
+        events = [
+            Event(timestamp=now + i * td1s, duration=td1s) for i in range(eventcount)
+        ]
+        bucket.insert(events)
+
+        # Both
+        for i in range(eventcount):
+            for j in range(i + 1, eventcount):
+                fetched_events = bucket.get(
+                    starttime=events[i].timestamp + timedelta(seconds=0.01),
+                    endtime=events[j].timestamp
+                    + events[j].duration
+                    - timedelta(seconds=0.01),
+                )
+                assert j - i + 1 == len(fetched_events)
+
+
+@pytest.mark.parametrize("bucket_cm", param_testing_buckets_cm())
+def test_insert_invalid(bucket_cm):
+    with bucket_cm as bucket:
+        event = "not a real event"
+        with pytest.raises(TypeError):
+            bucket.insert(event)
+
+
+@pytest.mark.parametrize("bucket_cm", param_testing_buckets_cm())
+def test_replace(bucket_cm):
+    """
+    Tests the replace event event in bucket functionality
+    """
+    with bucket_cm as bucket:
+        # Create two events
+        e1 = bucket.insert(Event(data={"label": "test1"}, timestamp=now))
+        assert e1
+        assert e1.id is not None
+        e2 = bucket.insert(
+            Event(data={"label": "test2"}, timestamp=now + timedelta(seconds=1))
+        )
+        assert e2
+        assert e2.id is not None
+
+        e1.data["label"] = "test1-replaced"
+        bucket.replace(e1.id, e1)
+
+        bucket.insert(
+            Event(data={"label": "test3"}, timestamp=now + timedelta(seconds=2))
+        )
+
+        e2.data["label"] = "test2-replaced"
+        bucket.replace(e2.id, e2)
+
+        # Assert length
+        assert 3 == len(bucket.get(-1))
+        assert bucket.get(-1)[0]["data"]["label"] == "test3"
+        assert bucket.get(-1)[1]["data"]["label"] == "test2-replaced"
+        assert bucket.get(-1)[2]["data"]["label"] == "test1-replaced"
+
+
+@pytest.mark.parametrize("bucket_cm", param_testing_buckets_cm())
+def test_replace_last(bucket_cm):
+    """
+    Tests the replace last event in bucket functionality (simple)
+    """
+    with bucket_cm as bucket:
+        # Create two events
+        bucket.insert(Event(data={"label": "test1"}, timestamp=now))
+        bucket.insert(
+            Event(data={"label": "test2"}, timestamp=now + timedelta(seconds=1))
+        )
+        # Create second event to replace with the second one
+        bucket.replace_last(
+            Event(
+                data={"label": "test2-replaced"}, timestamp=now + timedelta(seconds=1)
+            )
+        )
+        bucket.insert(
+            Event(data={"label": "test3"}, timestamp=now + timedelta(seconds=2))
+        )
+        # Assert data
+        result = bucket.get(-1)
+        assert 3 == len(result)
+        assert result[1]["data"]["label"] == "test2-replaced"
+
+
+@pytest.mark.parametrize("bucket_cm", param_testing_buckets_cm())
+def test_replace_last_complex(bucket_cm):
+    """
+    Tests the replace last event in bucket functionality (complex)
+    """
+    with bucket_cm as bucket:
+        # Create first event
+        event1 = Event(data={"label": "test1"}, timestamp=now, duration=timedelta(1))
+        bucket.insert(event1)
+        eventcount = len(bucket.get(-1))
+        # Create second event to replace with the first one
+        event2 = Event(
+            data={"label": "test2"},
+            duration=timedelta(0),
+            timestamp=now + timedelta(seconds=1),
+        )
+        bucket.replace_last(event2)
+        # Assert length and content
+        result = bucket.get(-1)
+        assert eventcount == len(result)
+        assert event2 == result[0]
+
+
+@pytest.mark.parametrize("bucket_cm", param_testing_buckets_cm())
+def test_get_last(bucket_cm):
+    """
+    Tests setting the result limit when fetching events
+    """
+    now = datetime.now()
+    second = timedelta(seconds=1)
+    with bucket_cm as bucket:
+        events = [
+            Event(data={"label": "test"}, timestamp=ts, duration=timedelta(0))
+            for ts in [now + second, now + second * 2, now + second * 3]
+        ]
+
+        for event in events:
+            bucket.insert(event)
+
+        assert bucket.get(limit=1)[0] == events[-1]
+        for event in bucket.get(limit=5):
+            print(event.timestamp, event.data["label"])
+
+
+@pytest.mark.parametrize("bucket_cm", param_testing_buckets_cm())
+def test_limit(bucket_cm):
+    """
+    Tests setting the result limit when fetching events
+    """
+    with bucket_cm as bucket:
+        for _ in range(5):
+            bucket.insert(Event(timestamp=now))
+
+        assert 0 == len(bucket.get(limit=0))
+        assert 1 == len(bucket.get(limit=1))
+        assert 3 == len(bucket.get(limit=3))
+        assert 5 == len(bucket.get(limit=5))
+        assert 5 == len(bucket.get(limit=-1))
+
+
+@pytest.mark.parametrize("bucket_cm", param_testing_buckets_cm())
+def test_get_metadata(bucket_cm):
+    """
+    Tests the get_metadata function
+    """
+    with bucket_cm as bucket:
+        print(bucket.ds.storage_strategy)
+        metadata = bucket.metadata()
+        print(metadata)
+        assert "created" in metadata
+        assert "client" in metadata
+        assert "hostname" in metadata
+        assert "id" in metadata
+        assert "name" in metadata
+        assert "type" in metadata
+    with pytest.raises(ValueError):
+        bucket.metadata()
+
+
+@pytest.mark.parametrize("bucket_cm", param_testing_buckets_cm())
+def test_get_eventcount(bucket_cm):
+    """
+    Tests the get_eventcount function
+    """
+    with bucket_cm as bucket:
+        print(bucket.ds.storage_strategy)
+        assert bucket.get_eventcount() == 0
+        for _ in range(5):
+            bucket.insert(Event(timestamp=now))
+        assert bucket.get_eventcount() == 5
+        # TODO: Test with timestamps and start/endtime filtering
+
+        bucket.insert(Event(timestamp=now + timedelta(seconds=5)))
+        assert (
+            bucket.get_eventcount(starttime=now - timedelta(seconds=1), endtime=now)
+            == 5
+        )
+        assert bucket.get_eventcount(endtime=now + timedelta(seconds=1)) == 5
+        assert bucket.get_eventcount(starttime=now + timedelta(seconds=1)) == 1

tests/test_dirs.py

@@ -0,0 +1,201 @@
+"""Tests for profile isolation via AW_PROFILE in aw_core.dirs.
+
+Three profiles (default / testing / research) must yield completely disjoint
+directory roots so that an ActivityWatch research build cannot read from or
+write to a participant's personal datastore.
+
+The ``testing`` profile additionally follows the new-root-plus-legacy-fallback
+rule from ActivityWatch/activitywatch#1399.
+"""
+
+import os
+from pathlib import Path
+from unittest.mock import patch
+
+import pytest
+
+from aw_core.dirs import (
+    _get_appname,
+    get_cache_dir,
+    get_config_dir,
+    get_data_dir,
+    legacy_testing_suffix,
+    using_legacy_testing_root,
+)
+
+from . import context  # noqa: F401
+
+
+@pytest.fixture
+def fake_dirs(tmp_path, monkeypatch):
+    """Point platformdirs at a tmp tree so tests never touch the real home."""
+    data = tmp_path / "data"
+    config = tmp_path / "config"
+    cache = tmp_path / "cache"
+
+    def _join(root: Path, appname: str) -> str:
+        return str(root / appname)
+
+    monkeypatch.setattr(
+        "aw_core.dirs.platformdirs.user_data_dir",
+        lambda appname, *a, **k: _join(data, appname),
+    )
+    monkeypatch.setattr(
+        "aw_core.dirs.platformdirs.user_config_dir",
+        lambda appname, *a, **k: _join(config, appname),
+    )
+    monkeypatch.setattr(
+        "aw_core.dirs.platformdirs.user_cache_dir",
+        lambda appname, *a, **k: _join(cache, appname),
+    )
+    monkeypatch.setattr(
+        "aw_core.dirs.platformdirs.user_cache_path",
+        lambda appname, *a, **k: cache / appname,
+    )
+    monkeypatch.setattr(
+        "aw_core.dirs.platformdirs.user_log_dir",
+        lambda appname, *a, **k: str(cache / appname / "log"),
+    )
+    monkeypatch.delenv("AW_PROFILE", raising=False)
+    return {"data": data, "config": config, "cache": cache, "root": tmp_path}
+
+
+def _plant_legacy_testing_db(fake_dirs) -> Path:
+    aw_server = fake_dirs["data"] / "activitywatch" / "aw-server"
+    aw_server.mkdir(parents=True)
+    marker = aw_server / "peewee-sqlite-testing.v2.db"
+    marker.write_text("")
+    return marker
+
+
+# ---------------------------------------------------------------------------
+# _get_appname
+# ---------------------------------------------------------------------------
+
+
+class TestGetAppname:
+    def test_unset_returns_bare_name(self, fake_dirs):
+        """AW_PROFILE absent → bare 'activitywatch', identical to legacy."""
+        with patch.dict(os.environ, {"AW_PROFILE": ""}):
+            assert _get_appname() == "activitywatch"
+
+    def test_empty_string_returns_bare_name(self, fake_dirs):
+        """AW_PROFILE='' is treated the same as unset."""
+        with patch.dict(os.environ, {"AW_PROFILE": ""}):
+            assert _get_appname() == "activitywatch"
+
+    def test_testing_profile_fresh_uses_new_root(self, fake_dirs):
+        with patch.dict(os.environ, {"AW_PROFILE": "testing"}):
+            assert _get_appname() == "activitywatch-testing"
+
+    def test_research_profile(self, fake_dirs):
+        with patch.dict(os.environ, {"AW_PROFILE": "research"}):
+            assert _get_appname() == "activitywatch-research"
+
+    def test_arbitrary_profile(self, fake_dirs):
+        with patch.dict(os.environ, {"AW_PROFILE": "myproject"}):
+            assert _get_appname() == "activitywatch-myproject"
+
+
+# ---------------------------------------------------------------------------
+# testing-root fallback (ActivityWatch/activitywatch#1399)
+# ---------------------------------------------------------------------------
+
+
+class TestTestingRootFallback:
+    def test_fresh_setup_uses_new_root(self, fake_dirs):
+        with patch.dict(os.environ, {"AW_PROFILE": "testing"}):
+            assert using_legacy_testing_root() is False
+            assert _get_appname() == "activitywatch-testing"
+            assert legacy_testing_suffix(True) == ""
+            data = get_data_dir()
+            assert "activitywatch-testing" in data
+            assert (fake_dirs["data"] / "activitywatch-testing").is_dir()
+
+    def test_legacy_artifacts_keep_shared_root(self, fake_dirs):
+        _plant_legacy_testing_db(fake_dirs)
+        with patch.dict(os.environ, {"AW_PROFILE": "testing"}):
+            assert using_legacy_testing_root() is True
+            assert _get_appname() == "activitywatch"
+            assert legacy_testing_suffix(True) == "-testing"
+            data = get_data_dir()
+            assert "activitywatch-testing" not in data
+            assert data.endswith("activitywatch") or data.endswith("activitywatch/")
+            assert not (fake_dirs["data"] / "activitywatch-testing").exists()
+
+    def test_new_root_wins_over_legacy_artifacts(self, fake_dirs):
+        _plant_legacy_testing_db(fake_dirs)
+        (fake_dirs["data"] / "activitywatch-testing").mkdir()
+        with patch.dict(os.environ, {"AW_PROFILE": "testing"}):
+            assert using_legacy_testing_root() is False
+            assert _get_appname() == "activitywatch-testing"
+            assert legacy_testing_suffix(True) == ""
+
+    def test_config_testing_toml_is_a_legacy_marker(self, fake_dirs):
+        cfg = fake_dirs["config"] / "activitywatch"
+        cfg.mkdir(parents=True)
+        (cfg / "config-testing.toml").write_text("")
+        with patch.dict(os.environ, {"AW_PROFILE": "testing"}):
+            assert using_legacy_testing_root() is True
+            assert _get_appname() == "activitywatch"
+
+    def test_testing_without_aw_profile_keeps_filename_suffix(self, fake_dirs):
+        """``testing=True`` with no profile still shares the default root."""
+        assert legacy_testing_suffix(True) == "-testing"
+        assert legacy_testing_suffix(False) == ""
+
+    def test_named_profile_never_suffixes_filenames(self, fake_dirs):
+        with patch.dict(os.environ, {"AW_PROFILE": "research"}):
+            assert legacy_testing_suffix(True) == ""
+            assert legacy_testing_suffix(False) == ""
+
+
+# ---------------------------------------------------------------------------
+# Directory isolation
+# ---------------------------------------------------------------------------
+
+
+def _data_dir(profile: str) -> str:
+    """Return get_data_dir() under the given profile (empty string = default)."""
+    with patch.dict(os.environ, {"AW_PROFILE": profile}):
+        return get_data_dir()
+
+
+class TestDirsIsolation:
+    """Three profiles must yield fully disjoint directory roots."""
+
+    def test_default_has_no_profile_suffix(self, fake_dirs):
+        d = _data_dir("")
+        assert "activitywatch" in d
+        # The bare appname must not contain a dash after "activitywatch"
+        assert "activitywatch-" not in d
+
+    def test_testing_suffix_present(self, fake_dirs):
+        assert "activitywatch-testing" in _data_dir("testing")
+
+    def test_research_suffix_present(self, fake_dirs):
+        assert "activitywatch-research" in _data_dir("research")
+
+    def test_three_profiles_are_disjoint(self, fake_dirs):
+        """default, testing, research all produce distinct, non-nested paths."""
+        default = _data_dir("")
+        testing = _data_dir("testing")
+        research = _data_dir("research")
+
+        dirs = {default, testing, research}
+        assert len(dirs) == 3, f"Profiles are not disjoint: {dirs}"
+
+        for a in dirs:
+            for b in dirs:
+                if a != b:
+                    assert not a.startswith(b + os.sep), f"{a!r} is a subpath of {b!r}"
+
+    def test_config_dir_isolated(self, fake_dirs):
+        with patch.dict(os.environ, {"AW_PROFILE": "research"}):
+            cfg = get_config_dir()
+        assert "activitywatch-research" in cfg
+
+    def test_cache_dir_isolated(self, fake_dirs):
+        with patch.dict(os.environ, {"AW_PROFILE": "testing"}):
+            cache = get_cache_dir()
+        assert "activitywatch-testing" in cache

tests/test_event.py

@@ -0,0 +1,41 @@
+from datetime import datetime, timedelta, timezone
+import json
+
+import pytest
+
+from aw_core.models import Event
+
+valid_timestamp = "1937-01-01T12:00:27.87+00:20"
+
+now = datetime.now(timezone.utc)
+td1s = timedelta(seconds=1)
+
+
+def test_create() -> None:
+    Event(timestamp=now, duration=timedelta(hours=13, minutes=37), data={"key": "val"})
+    Event(
+        timestamp=valid_timestamp,
+        duration=timedelta(hours=13, minutes=37),
+        data={"key": "val"},
+    )
+
+
+def test_json_serialization() -> None:
+    e = Event(
+        timestamp=now, duration=timedelta(hours=13, minutes=37), data={"key": "val"}
+    )
+    assert e == Event(**json.loads(e.to_json_str()))
+
+
+def test_set_invalid_duration() -> None:
+    e = Event()
+    with pytest.raises(TypeError):
+        e.duration = "12"  # type: ignore
+
+
+def test_sort() -> None:
+    e1 = Event(timestamp=now)
+    e2 = Event(timestamp=now + td1s)
+    e_sorted = sorted([e2, e1])
+    assert e_sorted[0] == e1
+    assert e_sorted[1] == e2

tests/test_flood.py

@@ -0,0 +1,167 @@
+from datetime import datetime, timedelta, timezone
+
+from aw_core.models import Event
+from aw_transform import flood
+
+
+now = datetime.now(tz=timezone.utc)
+td1s = timedelta(seconds=1)
+
+
+def test_flood_differing_data_meet_at_gap_midpoint():
+    events = [
+        Event(timestamp=now, duration=10, data={"a": 0}),
+        Event(timestamp=now + 14 * td1s, duration=5, data={"b": 1}),
+    ]
+
+    flooded = flood(events)
+
+    assert flooded == [
+        Event(timestamp=now, duration=12, data={"a": 0}),
+        Event(timestamp=now + 12 * td1s, duration=7, data={"b": 1}),
+    ]
+
+
+def test_flood_forward_merge():
+    events = [
+        Event(timestamp=now, duration=10),
+        Event(timestamp=now + 15 * td1s, duration=5),
+    ]
+    flooded = flood(events)
+    assert len(flooded) == 1
+    assert flooded[0].duration == timedelta(seconds=20)
+
+
+def test_flood_backward_merge():
+    events = [
+        Event(timestamp=now, duration=5),
+        Event(timestamp=now + 10 * td1s, duration=10),
+    ]
+    flooded = flood(events)
+    assert len(flooded) == 1
+    assert flooded[0].duration == timedelta(seconds=20)
+
+
+def test_flood_negative_gap_same_data():
+    events = [
+        Event(timestamp=now, duration=100, data={"a": 0}),
+        Event(timestamp=now, duration=5, data={"a": 0}),
+    ]
+    flooded = flood(events)
+    total_duration = sum((e.duration for e in flooded), timedelta(0))
+    assert len(flooded) == 1
+    assert total_duration == timedelta(seconds=100)
+
+
+def test_flood_negative_gap_differing_data():
+    events = [
+        Event(timestamp=now, duration=5, data={"a": 0}),
+        Event(timestamp=now, duration=100, data={"b": 1}),
+    ]
+    flooded = flood(events)
+    assert flooded == [events[1]]
+
+
+def test_flood_zero_duration_chain_does_not_leave_overlaps():
+    events = [
+        Event(timestamp=now, duration=0, data={"title": "first"}),
+        Event(timestamp=now, duration=1, data={"title": "first"}),
+        Event(timestamp=now, duration=1, data={"title": "second"}),
+    ]
+
+    flooded = flood(events)
+
+    assert flooded == [events[2]]
+    assert all(
+        previous.timestamp + previous.duration <= current.timestamp
+        for previous, current in zip(flooded, flooded[1:])
+    )
+
+
+def test_flood_normalization_preserves_non_overlapping_tail():
+    events = [
+        Event(timestamp=now, duration=10, data={"title": "first"}),
+        Event(timestamp=now + 5 * td1s, duration=2, data={"title": "second"}),
+        Event(timestamp=now + 12 * td1s, duration=1, data={"title": "third"}),
+    ]
+
+    flooded = flood(events)
+
+    assert flooded == [
+        Event(timestamp=now, duration=5, data={"title": "first"}),
+        Event(timestamp=now + 5 * td1s, duration=4.5, data={"title": "second"}),
+        Event(timestamp=now + 9.5 * td1s, duration=3.5, data={"title": "third"}),
+    ]
+
+
+def test_flood_normalization_merges_same_data_after_zero_duration_event():
+    events = [
+        Event(timestamp=now, duration=0, data={"title": "zero"}),
+        Event(timestamp=now, duration=10, data={"title": "same"}),
+        Event(timestamp=now + 5 * td1s, duration=10, data={"title": "same"}),
+    ]
+
+    flooded = flood(events)
+
+    assert flooded == [Event(timestamp=now, duration=15, data={"title": "same"})]
+
+
+def test_flood_negative_small_gap_differing_data():
+    events = [
+        Event(timestamp=now, duration=100, data={"b": 1}),
+        Event(timestamp=now + 99.99 * td1s, duration=100, data={"a": 0}),
+    ]
+    flooded = flood(events)
+    duration = sum((e.duration for e in flooded), timedelta(0))
+    assert duration == timedelta(seconds=100 + 99.99)
+
+
+def test_flood_with_custom_pulsetime():
+    # Events with a 30s gap between them (simulating 30s poll_time)
+    events = [
+        Event(timestamp=now, duration=5, data={"a": 0}),
+        Event(timestamp=now + 35 * td1s, duration=5, data={"b": 1}),
+    ]
+
+    # Default pulsetime=5: gap (30s) > pulsetime, so no flooding
+    flooded_default = flood(events)
+    assert len(flooded_default) == 2
+    total_duration_default = sum((e.duration for e in flooded_default), timedelta(0))
+    assert total_duration_default == timedelta(seconds=10)
+
+    # pulsetime=31: gap (30s) <= pulsetime, so both events extend to midpoint
+    flooded_custom = flood(events, pulsetime=31)
+    assert flooded_custom == [
+        Event(timestamp=now, duration=20, data={"a": 0}),
+        Event(timestamp=now + 20 * td1s, duration=20, data={"b": 1}),
+    ]
+
+
+def test_flood_idempotent():
+    """flood() applied repeatedly should produce the same result."""
+    events = [
+        # slight overlap, same data
+        Event(timestamp=now, duration=10, data={"a": 0}),
+        Event(timestamp=now + 9 * td1s, duration=5, data={"a": 0}),
+        # different data, no overlap
+        Event(timestamp=now + 15 * td1s, duration=5, data={"b": 0}),
+    ]
+    flood_first = flood(events, pulsetime=0)
+    flooded = flood_first
+    for _ in range(2):
+        flooded = flood(flooded, pulsetime=0)
+        assert flood_first == flooded
+
+    assert sum((e.duration for e in flooded), timedelta(0)) == 19 * td1s
+
+
+def test_flood_unsafe_gap():
+    """Overlapping events with differing data must not double-count time."""
+    events = [
+        Event(timestamp=now, duration=10, data={"a": 0}),
+        Event(timestamp=now + 9 * td1s, duration=5, data={"b": 0}),
+    ]
+    flooded = flood(events, pulsetime=0)
+
+    # Total duration must not exceed the span covered by the two events
+    assert sum((e.duration for e in flooded), timedelta(0)) == 14 * td1s

tests/test_heartbeat.py

@@ -0,0 +1,84 @@
+from datetime import datetime, timedelta
+
+from aw_core.models import Event
+from aw_transform import heartbeat_merge, heartbeat_reduce
+
+
+def test_heartbeat_merge():
+    """Events should merge"""
+    now = datetime.now()
+    td_1s = timedelta(seconds=1)
+
+    last_event, heartbeat = Event(timestamp=now), Event(timestamp=now + td_1s)
+    merged = heartbeat_merge(last_event, heartbeat, pulsetime=2)
+    assert merged is not None
+
+
+def test_heartbeat_merge_fail():
+    """Merge should not happen"""
+    now = datetime.now()
+    td_1s = timedelta(seconds=1)
+
+    # timestamp of heartbeat more than pulsetime away
+    last_event, heartbeat = (
+        Event(timestamp=now, data={"label": "test"}),
+        Event(timestamp=now + 3 * td_1s, data={"label": "test"}),
+    )
+    merged = heartbeat_merge(last_event, heartbeat, pulsetime=2)
+    assert merged is None
+
+    # labels not identical
+    last_event, heartbeat = (
+        Event(timestamp=now, data={"label": "test"}),
+        Event(timestamp=now + td_1s, data={"label": "test2"}),
+    )
+    merged = heartbeat_merge(last_event, heartbeat, pulsetime=2)
+    assert merged is None
+
+
+def test_heartbeat_reduce():
+    """Events should reduce"""
+    now = datetime.now()
+    td_1s = timedelta(seconds=1)
+
+    # Check that empty list works
+    assert not heartbeat_reduce([], pulsetime=1)
+
+    events = [
+        Event(timestamp=now, data={"label": "test"}),
+        Event(timestamp=now + td_1s, data={"label": "test"}),
+    ]
+    reduced_events = heartbeat_reduce(events, pulsetime=2)
+    assert len(reduced_events) == 1
+
+
+def test_heartbeat_same_timestamp():
+    now = datetime.now()
+    td_1s = timedelta(seconds=1)
+    td_0s = timedelta(seconds=0)
+
+    e1 = Event(timestamp=now, duration=td_1s, data={"label": "test"})
+    e2 = Event(timestamp=now, duration=td_0s, data={"label": "test"})
+
+    # Should merge
+    res = heartbeat_reduce(list([e1, e2]), pulsetime=5)
+    assert len(res) == 1
+    assert res[0].duration == td_1s
+
+    # Order shouldn't matter, should merge
+    res = heartbeat_reduce([e2, e1], pulsetime=5)
+    assert len(res) == 1
+    assert res[0].duration == td_1s
+
+
+def test_heartbeat_reduce_fail():
+    """Events should not reduce"""
+    now = datetime.now()
+    td_1s = timedelta(seconds=1)
+
+    events = [
+        Event(timestamp=now, data={"label": "test"}),
+        Event(timestamp=now + 3 * td_1s, data={"label": "test"}),
+    ]
+    reduced_events = heartbeat_reduce(events, pulsetime=2)
+    assert len(reduced_events) == 2

tests/test_peewee.py

@@ -0,0 +1,66 @@
+"""
+Peewee-storage-specific tests, mainly for the schema/index migration in
+auto_migrate() and for guarding against query-plan regressions.
+"""
+
+import logging
+
+from aw_datastore.storages import PeeweeStorage
+from aw_datastore.storages import peewee as peewee_storage
+
+from . import context  # noqa: F401
+
+logging.basicConfig(level=logging.DEBUG)
+
+
+def _indexes(db) -> set:
+    return {row[1] for row in db.execute_sql("PRAGMA index_list(eventmodel)")}
+
+
+def test_index_migration():
+    """auto_migrate() replaces the old single-column indexes with the composite one."""
+    storage = PeeweeStorage(testing=True)
+    db = storage.db
+    path = db.database
+
+    assert "eventmodel_bucket_id_timestamp" in _indexes(db)
+
+    # Recreate the pre-migration state: single-column indexes, no composite
+    db.execute_sql("DROP INDEX eventmodel_bucket_id_timestamp")
+    db.execute_sql("CREATE INDEX eventmodel_bucket_id ON eventmodel (bucket_id)")
+    db.execute_sql("CREATE INDEX eventmodel_timestamp ON eventmodel (timestamp)")
+
+    peewee_storage.auto_migrate(path)
+
+    indexes = _indexes(db)
+    assert "eventmodel_bucket_id_timestamp" in indexes
+    assert "eventmodel_bucket_id" not in indexes
+    assert "eventmodel_timestamp" not in indexes
+
+
+def test_query_plan_uses_composite_index():
+    """
+    The last-event query (issued by replace_last on every merged heartbeat) and
+    range queries must be served by the composite index in a single pass.
+
+    Without it, SQLite reads *all* of a bucket's rows and sorts them in a temp
+    B-tree even for LIMIT 1, which on multi-million-event buckets meant seconds
+    of CPU/IO per heartbeat (and terabytes read per day).
+    """
+    storage = PeeweeStorage(testing=True)
+    db = storage.db
+
+    queries = [
+        # replace_last/_get_last
+        "SELECT * FROM eventmodel WHERE bucket_id = 1"
+        " ORDER BY timestamp DESC LIMIT 1",
+        # get_events with a time range
+        "SELECT * FROM eventmodel WHERE bucket_id = 1"
+        " AND timestamp >= '2026-01-01' AND timestamp <= '2026-01-02'"
+        " ORDER BY timestamp DESC LIMIT 100",
+    ]
+    for query in queries:
+        rows = db.execute_sql(f"EXPLAIN QUERY PLAN {query}").fetchall()
+        plan = "\n".join(str(row) for row in rows)
+        assert "eventmodel_bucket_id_timestamp" in plan, plan
+        assert "TEMP B-TREE" not in plan, plan

tests/test_query2.py

@@ -0,0 +1,715 @@
+from datetime import datetime, timedelta, timezone
+from typing import Any, Dict
+
+import iso8601
+import pytest
+from aw_core.models import Event
+from aw_datastore.storages import MemoryStorage
+from aw_query import query
+from aw_query.exceptions import (
+    QueryFunctionException,
+    QueryInterpretException,
+    QueryParseException,
+)
+from aw_query.query2 import (
+    QDict,
+    QFunction,
+    QInteger,
+    QList,
+    QString,
+    QVariable,
+    _parse_token,
+    _split_query_statements,
+)
+
+from .utils import param_datastore_objects
+
+
+class MockDatastore(MemoryStorage):
+    pass
+
+
+mock_ds = MockDatastore(testing=True)
+
+
+def test_query2_test_token_parsing():
+    ns: Dict[str, Any] = {}
+    (t, token), trash = _parse_token("123", ns)
+    assert token == "123"
+    assert t == QInteger
+    (t, token), trash = _parse_token('"test"', ns)
+    assert token == '"test"'
+    assert t == QString
+    (t, token), trash = _parse_token("'test'", ns)
+    assert token == "'test'"
+    assert t == QString
+    (t, token), trash = _parse_token("'te\\'st'", ns)
+    assert token == "'te\\'st'"
+    assert t == QString
+    (t, token), trash = _parse_token('"te\\"st"', ns)
+    assert token == '"te\\"st"'
+    assert t == QString
+    (t, token), trash = _parse_token("test0xDEADBEEF", ns)
+    assert token == "test0xDEADBEEF"
+    assert t == QVariable
+    (t, token), trash = _parse_token("test1337(')')", ns)
+    assert token == "test1337(')')"
+    assert t == QFunction
+    (t, token), trash = _parse_token("test1337('test\\'test',\"test\\\"test\")", ns)
+    assert token == "test1337('test\\'test',\"test\\\"test\")"
+    assert t == QFunction
+    (t, token), trash = _parse_token("[1, 'a', {}]", ns)
+    assert token == "[1, 'a', {}]"
+    assert t == QList
+    (t, token), trash = _parse_token("{'a': 1, 'b}': 2}", ns)
+    assert token == "{'a': 1, 'b}': 2}"
+    assert t == QDict
+
+    assert _parse_token("", ns) == ((None, ""), "")
+
+    with pytest.raises(QueryParseException):
+        _parse_token(None, ns)  # type: ignore
+
+    with pytest.raises(QueryParseException):
+        _parse_token('"', ns)
+
+    with pytest.raises(QueryParseException):
+        _parse_token("#", ns)
+
+
+def test_dict():
+    ds = mock_ds
+    ns: Dict[str, Any] = {}
+    d_str = "{'a': {'a': {'a': 1}}, 'b': {'b\\'\"': ':'}}"
+    d = QDict.parse(d_str, ns)
+    expected_res = {"a": {"a": {"a": 1}}, "b": {"b'\"": ":"}}
+    assert expected_res == d.interpret(ds, ns)
+
+    # Key in dict is not a string
+    with pytest.raises(QueryParseException):
+        d_str = "{b: 1}"
+        d = QDict.parse(d_str, ns)
+
+    # Key in dict without a value
+    with pytest.raises(QueryParseException):
+        d_str = "{'test': }"
+        d = QDict.parse(d_str, ns)
+
+    # Char following key string is not a :
+    with pytest.raises(QueryParseException):
+        d_str = "{'test'p 1}"
+        d = QDict.parse(d_str, ns)
+
+    with pytest.raises(QueryParseException):
+        d_str = "{'test': #}"
+        d = QDict.parse(d_str, ns)
+
+    # Semicolon without key
+    with pytest.raises(QueryParseException):
+        d_str = "{:}"
+        d = QDict.parse(d_str, ns)
+
+    # Trailing comma
+    with pytest.raises(QueryParseException):
+        d_str = "{'test':1,}"
+        d = QDict.parse(d_str, ns)
+
+
+def test_list():
+    ds = mock_ds
+    ns: Dict[str, Any] = {}
+    l_str = "[1,2,[[3],4],5]"
+    ls = QList.parse(l_str, ns)
+    expected_res = [1, 2, [[3], 4], 5]
+    assert expected_res == ls.interpret(ds, ns)
+
+    l_str = "['\\'',\"\\\"\"]"
+    ls = QList.parse(l_str, ns)
+    expected_res = ["'", '"']
+    assert expected_res == ls.interpret(ds, ns)
+
+    l_str = "[]"
+    ls = QList.parse(l_str, ns)
+    expected_res = []
+    assert expected_res == ls.interpret(ds, ns)
+
+    # Comma without pre/post value
+    with pytest.raises(QueryParseException):
+        l_str = "[,]"
+        ls = QList.parse(l_str, ns)
+
+    # Comma without post value
+    with pytest.raises(QueryParseException):
+        l_str = "[1,]"
+        ls = QList.parse(l_str, ns)
+
+    # Comma without pre value
+    with pytest.raises(QueryParseException):
+        l_str = "[,2]"
+        ls = QList.parse(l_str, ns)
+
+
+def test_query2_bogus_query():
+    ds = mock_ds
+    qname = "test"
+    qstartdate = datetime.now(tz=timezone.utc)
+    qenddate = qstartdate
+
+    # Nothing to assign
+    with pytest.raises(QueryParseException):
+        example_query = "a="
+        query(qname, example_query, qstartdate, qenddate, ds)
+
+    # Assign to non-variable
+    with pytest.raises(QueryParseException):
+        example_query = "1 = 2"
+        query(qname, example_query, qstartdate, qenddate, ds)
+
+    # Unclosed function
+    with pytest.raises(QueryParseException):
+        example_query = "a = unclosed_function(var1"
+        query(qname, example_query, qstartdate, qenddate, ds)
+
+    # Two tokens in assignment
+    with pytest.raises(QueryParseException):
+        example_query = "asd nop() = 2"
+        query(qname, example_query, qstartdate, qenddate, ds)
+
+    # Unclosed string
+    with pytest.raises(QueryParseException):
+        example_query = 'asd = "something is wrong with me'
+        query(qname, example_query, qstartdate, qenddate, ds)
+
+    # Two tokens in value
+    with pytest.raises(QueryParseException):
+        example_query = "asd = asd1 asd2"
+        query(qname, example_query, qstartdate, qenddate, ds)
+
+
+def test_query2_query_function_calling():
+    ds = mock_ds
+    qname = "asd"
+    starttime = iso8601.parse_date("1970-01-01")
+    endtime = iso8601.parse_date("1970-01-02")
+
+    # Function which doesn't exist
+    with pytest.raises(QueryInterpretException):
+        example_query = "RETURN = asd();"
+        query(qname, example_query, starttime, endtime, ds)
+
+    # Function which does exist with invalid arguments
+    with pytest.raises(QueryInterpretException):
+        example_query = "RETURN = nop(badarg);"
+        query(qname, example_query, starttime, endtime, ds)
+
+    # Function which does exist with valid arguments
+    example_query = "RETURN = nop();"
+    query(qname, example_query, starttime, endtime, ds)
+
+
+def test_query2_return_value():
+    ds = mock_ds
+    qname = "asd"
+    starttime = iso8601.parse_date("1970-01-01")
+    endtime = iso8601.parse_date("1970-01-02")
+    example_query = "RETURN = 1;"
+    result = query(qname, example_query, starttime, endtime, ds)
+    assert result == 1
+
+    example_query = "RETURN = 'testing 123'"
+    result = query(qname, example_query, starttime, endtime, ds)
+    assert result == "testing 123"
+
+    example_query = "RETURN = {'a': 1}"
+    result = query(qname, example_query, starttime, endtime, ds)
+    assert result == {"a": 1}
+
+    # Nothing to return
+    with pytest.raises(QueryParseException):
+        example_query = "a=1"
+        result = query(qname, example_query, starttime, endtime, ds)
+
+
+def test_query2_semicolon_in_string():
+    """Test that semicolons inside string literals don't break query parsing (issue #145)."""
+    ds = mock_ds
+    qname = "asd"
+    starttime = iso8601.parse_date("1970-01-01")
+    endtime = iso8601.parse_date("1970-01-02")
+
+    # Semicolon in double-quoted string
+    example_query = 'RETURN = "hello;world"'
+    result = query(qname, example_query, starttime, endtime, ds)
+    assert result == "hello;world"
+
+    # Semicolon in single-quoted string
+    example_query = "RETURN = 'hello;world'"
+    result = query(qname, example_query, starttime, endtime, ds)
+    assert result == "hello;world"
+
+    # Multiple statements where one value contains a semicolon in a string
+    example_query = 'a = "foo;bar"; RETURN = a'
+    result = query(qname, example_query, starttime, endtime, ds)
+    assert result == "foo;bar"
+
+
+def test_split_query_statements():
+    """Unit test for _split_query_statements."""
+    # Basic split
+    assert _split_query_statements("a=1;b=2") == ["a=1", "b=2"]
+    # Semicolon inside double quotes should NOT split
+    assert _split_query_statements('a="x;y";b=2') == ['a="x;y"', "b=2"]
+    # Semicolon inside single quotes should NOT split
+    assert _split_query_statements("a='x;y';b=2") == ["a='x;y'", "b=2"]
+    # Trailing semicolon — empty trailing part is omitted
+    assert _split_query_statements("a=1;") == ["a=1"]
+
+
+def test_query2_multiline():
+    ds = mock_ds
+    qname = "asd"
+    starttime = iso8601.parse_date("1970-01-01")
+    endtime = iso8601.parse_date("1970-01-02")
+    example_query = """
+my_multiline_string = "a
+b";
+RETURN = my_multiline_string;
+    """
+    result = query(qname, example_query, starttime, endtime, ds)
+    assert result == "a\nb"
+
+
+def test_query2_function_invalid_types():
+    """Tests the q2_typecheck decorator"""
+    ds = mock_ds
+    qname = "asd"
+    starttime = iso8601.parse_date("1970-01-01")
+    endtime = iso8601.parse_date("1970-01-02")
+
+    # int instead of str
+    example_query = """
+        events = [];
+        RETURN = filter_keyvals(events, 666, ["invalid_val"]);
+    """
+    with pytest.raises(QueryFunctionException):
+        query(qname, example_query, starttime, endtime, ds)
+
+    # str instead of list
+    example_query = """
+        events = [];
+        RETURN = filter_keyvals(events, "2", "invalid_val");
+    """
+    with pytest.raises(QueryFunctionException):
+        query(qname, example_query, starttime, endtime, ds)
+
+    # FIXME: For unknown reasons, query2 drops the second argument
+    #        when the first argument is a bare []
+    """
+    example_query = '''
+        RETURN = filter_keyvals([], "2", "invalid_val");
+    '''
+    with pytest.raises(QueryFunctionException) as e:
+        result = query(qname, example_query, starttime, endtime, None)
+    """
+
+
+def test_query2_function_invalid_argument_count():
+    ds = mock_ds
+    qname = "asd"
+    starttime = iso8601.parse_date("1970-01-01")
+    endtime = iso8601.parse_date("1970-01-02")
+    example_query = "RETURN=nop(nop())"
+    with pytest.raises(QueryInterpretException):
+        query(qname, example_query, starttime, endtime, ds)
+
+
+def test_query2_merge_subwatcher_fields_invalid_conflict():
+    ds = mock_ds
+    qname = "asd"
+    starttime = iso8601.parse_date("1970-01-01")
+    endtime = iso8601.parse_date("1970-01-02")
+    example_query = """
+        base = [];
+        sub = [];
+        RETURN = merge_subwatcher_fields(base, sub, ["project"], "invalid");
+    """
+
+    with pytest.raises(QueryFunctionException, match="conflict must be"):
+        query(qname, example_query, starttime, endtime, ds)
+
+
+def test_query2_categorize_invalid_priority():
+    ds = mock_ds
+    qname = "asd"
+    starttime = iso8601.parse_date("1970-01-01")
+    endtime = iso8601.parse_date("1970-01-02")
+    example_query = """
+        events = [];
+        RETURN = categorize(events, [[["test"], {"regex": "test", "priority": "high"}]]);
+    """
+    with pytest.raises(QueryFunctionException, match="integer"):
+        query(qname, example_query, starttime, endtime, ds)
+
+
+@pytest.mark.parametrize("datastore", param_datastore_objects())
+def test_query2_function_in_function(datastore):
+    qname = "asd"
+    bid = "test_bucket"
+    starttime = iso8601.parse_date("1970-01-01")
+    endtime = iso8601.parse_date("1970-01-02")
+    example_query = f"""
+    RETURN=limit_events(query_bucket("{bid}"), 1);
+    """
+    try:
+        # Setup buckets
+        bucket1 = datastore.create_bucket(
+            bucket_id=bid, type="test", client="test", hostname="test", name="test"
+        )
+        # Prepare buckets
+        e1 = Event(data={}, timestamp=starttime, duration=timedelta(seconds=1))
+        bucket1.insert(e1)
+        result = query(qname, example_query, starttime, endtime, datastore)
+        assert 1 == len(result)
+    finally:
+        datastore.delete_bucket(bid)
+
+
+@pytest.mark.parametrize("datastore", param_datastore_objects())
+def test_query2_query_functions(datastore):
+    """
+    Just test calling all functions just to see something isn't completely broken
+    In many cases the functions doesn't change the result at all, so it's not a test
+    for testing the validity of the data the functions transform
+    """
+    bid = "test_'bucket"
+    qname = "test"
+    starttime = iso8601.parse_date("1970")
+    endtime = starttime + timedelta(hours=1)
+
+    example_query = """
+    bid = "{bid}";
+    events = query_bucket("{bid}");
+    events2 = query_bucket('{bid_escaped}');
+    events2 = filter_keyvals(events2, "label", ["test1"]);
+    events2 = exclude_keyvals(events2, "label", ["test2"]);
+    events = filter_period_intersect(events, events2);
+    events = filter_keyvals_regex(events, "label", ".*");
+    events = limit_events(events, 1);
+    events = merge_events_by_keys(events, ["label"]);
+    events = chunk_events_by_key(events, "label");
+    events = split_url_events(events);
+    events = sort_by_timestamp(events);
+    events = sort_by_duration(events);
+    events = categorize(events, [[["test", "subtest"], {{"regex": "test1"}}]]);
+    duration = sum_durations(events);
+    eventcount = query_bucket_eventcount(bid);
+    asd = nop();
+    RETURN = {{"events": events, "eventcount": eventcount}};
+    """.format(bid=bid, bid_escaped=bid.replace("'", "\\'"))
+    try:
+        bucket = datastore.create_bucket(
+            bucket_id=bid, type="test", client="test", hostname="test", name="asd"
+        )
+        bucket.insert(
+            Event(
+                data={"label": "test1"},
+                timestamp=starttime,
+                duration=timedelta(seconds=1),
+            )
+        )
+        result = query(qname, example_query, starttime, endtime, datastore)
+        assert result["eventcount"] == 1
+        assert len(result["events"]) == 1
+        assert result["events"][0].data["label"] == "test1"
+        assert result["events"][0].data["$category"] == ["test", "subtest"]
+    finally:
+        datastore.delete_bucket(bid)
+
+
+@pytest.mark.parametrize("datastore", param_datastore_objects())
+def test_query2_basic_query(datastore):
+    name = "A label/name for a test bucket"
+    bid1 = "bucket1"
+    bid2 = "bucket2"
+    qname = "test_query_basic"
+    starttime = iso8601.parse_date("1970")
+    endtime = starttime + timedelta(hours=1)
+
+    example_query = f"""
+    bid1 = "{bid1}";
+    bid2 = "{bid2}";
+    events = query_bucket(bid1);
+    intersect_events = query_bucket(bid2);
+    RETURN = filter_period_intersect(events, intersect_events);
+    """
+
+    try:
+        # Setup buckets
+        bucket1 = datastore.create_bucket(
+            bucket_id=bid1, type="test", client="test", hostname="test", name=name
+        )
+        bucket2 = datastore.create_bucket(
+            bucket_id=bid2, type="test", client="test", hostname="test", name=name
+        )
+        # Prepare buckets
+        e1 = Event(
+            data={"label": "test1"}, timestamp=starttime, duration=timedelta(seconds=1)
+        )
+        e2 = Event(
+            data={"label": "test2"},
+            timestamp=starttime + timedelta(seconds=2),
+            duration=timedelta(seconds=1),
+        )
+        et = Event(
+            data={"label": "intersect-label"},
+            timestamp=starttime,
+            duration=timedelta(seconds=1),
+        )
+        bucket1.insert(e1)
+        bucket1.insert(e2)
+        bucket2.insert(et)
+        # Query
+        result = query(qname, example_query, starttime, endtime, datastore)
+        # Assert
+        assert len(result) == 1
+        assert result[0]["data"]["label"] == "test1"
+    finally:
+        datastore.delete_bucket(bid1)
+        datastore.delete_bucket(bid2)
+
+
+@pytest.mark.parametrize("datastore", param_datastore_objects())
+def test_query2_test_merged_keys(datastore):
+    name = "A label/name for a test bucket"
+    bid = "bucket1"
+    qname = "test_query_merged_keys"
+    starttime = iso8601.parse_date("2080")
+    endtime = starttime + timedelta(hours=1)
+
+    example_query = f"""
+    bid1 = "{bid}";
+    events = query_bucket(bid1);
+    events = merge_events_by_keys(events, ["label1", "label2"]);
+    events = sort_by_duration(events);
+    eventcount = query_bucket_eventcount(bid1);
+    RETURN = {{"events": events, "eventcount": eventcount}};
+    """
+    try:
+        # Setup buckets
+        bucket1 = datastore.create_bucket(
+            bucket_id=bid, type="test", client="test", hostname="test", name=name
+        )
+        # Prepare buckets
+        e1 = Event(
+            data={"label1": "test1", "label2": "test1"},
+            timestamp=starttime,
+            duration=timedelta(seconds=1),
+        )
+        e2 = Event(
+            data={"label1": "test1", "label2": "test1"},
+            timestamp=starttime + timedelta(seconds=1),
+            duration=timedelta(seconds=1),
+        )
+        e3 = Event(
+            data={"label1": "test1", "label2": "test2"},
+            timestamp=starttime + timedelta(seconds=2),
+            duration=timedelta(seconds=1),
+        )
+        bucket1.insert(e3)
+        bucket1.insert(e1)
+        bucket1.insert(e2)
+        # Query
+        result = query(qname, example_query, starttime, endtime, datastore)
+        # Assert
+        print(result)
+        assert len(result["events"]) == 2
+        assert result["eventcount"] == 3
+        assert result["events"][0]["data"]["label1"] == "test1"
+        assert result["events"][0]["data"]["label2"] == "test1"
+        assert result["events"][0]["duration"] == timedelta(seconds=2)
+        assert result["events"][1]["data"]["label1"] == "test1"
+        assert result["events"][1]["data"]["label2"] == "test2"
+        assert result["events"][1]["duration"] == timedelta(seconds=1)
+    finally:
+        datastore.delete_bucket(bid)
+
+
+@pytest.mark.parametrize("datastore", param_datastore_objects())
+def test_query2_fancy_query(datastore):
+    """
+    Tests:
+     - find_bucket
+     - simplify_window_titles
+    """
+    name = "A label/name for a test bucket"
+    bid1 = "bucket-the-one"
+    qname = "test_query_basic_fancy"
+    starttime = iso8601.parse_date("1970")
+    endtime = starttime + timedelta(hours=1)
+
+    example_query = f"""
+    bid = find_bucket("{bid1[:10]}");
+    events = query_bucket(bid);
+    RETURN = simplify_window_titles(events, "title");
+    """
+
+    try:
+        # Setup buckets
+        bucket_main = datastore.create_bucket(
+            bucket_id=bid1, type="test", client="test", hostname="test", name=name
+        )
+        # Prepare buckets
+        e1 = Event(
+            data={"title": "(2) YouTube"},
+            timestamp=starttime,
+            duration=timedelta(seconds=1),
+        )
+        bucket_main.insert(e1)
+        # Query
+        result = query(qname, example_query, starttime, endtime, datastore)
+        # Assert
+        assert result[0]["data"]["title"] == "YouTube"
+    finally:
+        datastore.delete_bucket(bid1)
+
+
+@pytest.mark.parametrize("datastore", param_datastore_objects())
+def test_query2_query_categorize(datastore):
+    bid = "test_bucket"
+    qname = "test"
+    starttime = iso8601.parse_date("1970")
+    endtime = starttime + timedelta(hours=1)
+
+    example_query = rf"""
+    events = query_bucket("{bid}");
+    events = sort_by_timestamp(events);
+    events = categorize(events, [
+                [["test"], {{"regex": "test"}}],
+                [["test", "subtest"], {{"regex": "test\w"}}]
+            ]);
+    events_by_cat = merge_events_by_keys(events, ["$category"]);
+    RETURN = {{"events": events, "events_by_cat": events_by_cat}};
+    """
+    try:
+        bucket = datastore.create_bucket(
+            bucket_id=bid, type="test", client="test", hostname="test", name="asd"
+        )
+        events = [
+            Event(
+                data={"label": "test"},
+                timestamp=starttime,
+                duration=timedelta(seconds=1),
+            ),
+            Event(
+                data={"label": "testwithmoredetail"},
+                timestamp=starttime + timedelta(seconds=1),
+                duration=timedelta(seconds=1),
+            ),
+            Event(
+                data={"label": "testwithmoredetail"},
+                timestamp=starttime + timedelta(seconds=2),
+                duration=timedelta(seconds=1),
+            ),
+        ]
+        bucket.insert(events)
+        result = query(qname, example_query, starttime, endtime, datastore)
+        print(result)
+        assert len(result["events"]) == 3
+        assert result["events"][0].data["label"] == "test"
+        assert result["events"][0].data["$category"] == ["test"]
+        assert result["events"][1].data["$category"] == ["test", "subtest"]
+
+        assert len(result["events_by_cat"]) == 2
+        assert result["events_by_cat"][0].data["$category"] == ["test"]
+        assert result["events_by_cat"][1].data["$category"] == ["test", "subtest"]
+        assert result["events_by_cat"][1].duration == timedelta(seconds=2)
+    finally:
+        datastore.delete_bucket(bid)
+
+
+@pytest.mark.parametrize("datastore", param_datastore_objects())
+def test_query2_query_categorize_priority(datastore):
+    bid = "test_bucket_priority"
+    qname = "test"
+    starttime = iso8601.parse_date("1970")
+    endtime = starttime + timedelta(hours=1)
+
+    example_query = rf"""
+    events = query_bucket("{bid}");
+    events = categorize(events, [
+                [["A"], {{"regex": "test", "priority": 25}}],
+                [["B", "B1"], {{"regex": "test"}}]
+            ]);
+    RETURN = events;
+    """
+    try:
+        bucket = datastore.create_bucket(
+            bucket_id=bid, type="test", client="test", hostname="test", name="asd"
+        )
+        bucket.insert(
+            Event(
+                data={"label": "test"},
+                timestamp=starttime,
+                duration=timedelta(seconds=1),
+            )
+        )
+        result = query(qname, example_query, starttime, endtime, datastore)
+        assert len(result) == 1
+        assert result[0].data["$category"] == ["A"]
+    finally:
+        datastore.delete_bucket(bid)
+
+
+@pytest.mark.parametrize("datastore", param_datastore_objects())
+def test_query2_flood_with_pulsetime(datastore):
+    """Test that flood() accepts an optional pulsetime argument (fixes WorkReport query bug)."""
+    bid = "test_flood_bucket"
+    qname = "test_flood"
+    starttime = iso8601.parse_date("1970")
+    endtime = starttime + timedelta(hours=1)
+
+    try:
+        bucket = datastore.create_bucket(
+            bucket_id=bid,
+            type="test",
+            client="test",
+            hostname="test",
+            name="flood_test",
+        )
+        # Two events with a 60-second gap — flood with pulsetime=300 should merge them
+        bucket.insert(
+            Event(
+                data={"app": "Firefox"},
+                timestamp=starttime,
+                duration=timedelta(seconds=10),
+            )
+        )
+        bucket.insert(
+            Event(
+                data={"app": "Firefox"},
+                timestamp=starttime + timedelta(seconds=70),
+                duration=timedelta(seconds=10),
+            )
+        )
+
+        # flood with default pulsetime (5s) — gap is 60s, events should NOT merge
+        result_default = query(
+            qname,
+            f'events = query_bucket("{bid}"); events = flood(events); RETURN = events;',
+            starttime,
+            endtime,
+            datastore,
+        )
+        assert len(result_default) == 2
+
+        # flood with pulsetime=300 — gap is 60s < 300s, events SHOULD merge
+        result_pulsetime = query(
+            qname,
+            f'events = query_bucket("{bid}"); events = flood(events, 300); RETURN = events;',
+            starttime,
+            endtime,
+            datastore,
+        )
+        assert len(result_pulsetime) == 1
+    finally:
+        datastore.delete_bucket(bid)

tests/test_schemas.py

@@ -0,0 +1,60 @@
+import unittest
+
+from jsonschema import validate as _validate, FormatChecker
+from jsonschema.exceptions import ValidationError
+
+from aw_core import schema
+from aw_core.models import Event
+
+# TODO: Include date-time format
+# https://python-jsonschema.readthedocs.io/en/latest/validate/#jsonschema.FormatChecker
+
+# The default FormatChecker, uses the date-time checker
+fc = FormatChecker(["date-time"])
+
+valid_timestamp = "1937-01-01T12:00:27.87+00:20"
+
+
+event_schema = schema.get_json_schema("event")
+
+
+class EventSchemaTest(unittest.TestCase):
+    def setUp(self):
+        self.schema = event_schema
+
+    def validate(self, obj):
+        _validate(obj, self.schema, format_checker=fc)
+
+    def test_event(self):
+        event = Event(timestamp=valid_timestamp, data={"label": "test"})
+        self.validate(event.to_json_dict())
+
+    def test_data(self):
+        self.validate(
+            {"timestamp": valid_timestamp, "data": {"label": "test", "number": 1.1}}
+        )
+
+    def test_timestamp(self):
+        self.validate({"timestamp": valid_timestamp})
+
+    def test_timestamp_invalid_string(self):
+        with self.assertRaises(ValidationError):
+            self.validate({"timestamp": ""})
+        with self.assertRaises(ValidationError):
+            self.validate({"timestamp": "123"})
+
+    def test_timestamp_invalid_number(self):
+        with self.assertRaises(ValidationError):
+            self.validate({"timestamp": 2})
+
+    def test_duration(self):
+        self.validate({"timestamp": valid_timestamp, "duration": 1000})
+        self.validate({"timestamp": valid_timestamp, "duration": 3.13})
+
+    def test_duration_invalid_string(self):
+        with self.assertRaises(ValidationError):
+            self.validate({"timestamp": valid_timestamp, "duration": "not a number"})
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_transforms.py

@@ -0,0 +1,830 @@
+from pprint import pprint
+from datetime import datetime, timedelta, timezone
+
+import pytest
+from aw_core.models import Event
+from aw_transform import (
+    filter_period_intersect,
+    filter_keyvals_regex,
+    filter_keyvals,
+    merge_subwatcher_fields,
+    period_union,
+    sort_by_timestamp,
+    sort_by_duration,
+    sum_durations,
+    merge_events_by_keys,
+    chunk_events_by_key,
+    split_url_events,
+    simplify_string,
+    union,
+    union_no_overlap,
+    categorize,
+    tag,
+    Rule,
+)
+from aw_transform.filter_period_intersect import _intersecting_eventpairs
+
+
+def test_simplify_string():
+    events = [
+        Event(data={"label": "(99) Facebook"}),
+        Event(data={"label": "(14) YouTube"}),
+    ]
+    assert simplify_string(events, "label")[0].data["label"] == "Facebook"
+    assert simplify_string(events, "label")[1].data["label"] == "YouTube"
+
+    events = [Event(data={"app": "Cemu.exe", "title": "Cemu - FPS: 133.7 - BotW"})]
+    assert simplify_string(events, "title")[0].data["title"] == "Cemu - FPS: ... - BotW"
+
+    events = [
+        Event(data={"app": "VSCode.exe", "title": "● report.md - Visual Studio Code"})
+    ]
+    assert (
+        simplify_string(events, "title")[0].data["title"]
+        == "report.md - Visual Studio Code"
+    )
+
+    events = [Event(data={"app": "Gedit", "title": "*test.md - gedit"})]
+    assert simplify_string(events, "title")[0].data["title"] == "test.md - gedit"
+
+
+def test_filter_keyval():
+    labels = ["aa", "cc"]
+    events = [
+        Event(data={"label": "aa"}),
+        Event(data={"label": "bb"}),
+        Event(data={"label": "cc"}),
+    ]
+    included_events = filter_keyvals(events, "label", labels)
+    excluded_events = filter_keyvals(events, "label", labels, exclude=True)
+    assert len(included_events) == 2
+    assert len(excluded_events) == 1
+
+
+def test_filter_keyval_regex():
+    events = [
+        Event(data={"label": "aa"}),
+        Event(data={"label": "bb"}),
+        Event(data={"label": "cc"}),
+    ]
+    events_re = filter_keyvals_regex(events, "label", "aa|cc")
+    assert len(events_re) == 2
+
+
+def test_filter_keyval_regex_keyerror():
+    events = [
+        Event(data={"label": "aa"}),
+        Event(),
+        Event(data={"label": "cc"}),
+    ]
+    events_re = filter_keyvals_regex(events, "label", "aa|cc")
+    assert len(events_re) == 2
+
+
+def test_intersecting_eventpairs():
+    td1h = timedelta(hours=1)
+    now = datetime.now()
+
+    # Test with two identical lists
+    e1 = [
+        Event(timestamp=now, duration=td1h),
+        Event(timestamp=now + td1h, duration=td1h),
+    ]
+    e2 = [
+        Event(timestamp=now, duration=td1h),
+        Event(timestamp=now + td1h, duration=td1h),
+    ]
+    intersecting = list(_intersecting_eventpairs(e1, e2))
+    assert len(intersecting) == 2
+
+    # Test with events in first list being in between events of second list
+    e1 = [
+        Event(timestamp=now + td1h, duration=td1h),
+    ]
+    e2 = [
+        Event(timestamp=now, duration=td1h),
+        Event(timestamp=now + 2 * td1h, duration=td1h),
+    ]
+    intersecting = list(_intersecting_eventpairs(e1, e2))
+    assert not intersecting
+
+    # Test with event in first list being identical to middle event in second list
+    e1 = [
+        Event(timestamp=now + td1h, duration=td1h),
+    ]
+    e2 = [
+        Event(timestamp=now, duration=td1h),
+        Event(timestamp=now + 1 * td1h, duration=td1h),
+        Event(timestamp=now + 2 * td1h, duration=td1h),
+    ]
+    intersecting = list(_intersecting_eventpairs(e1, e2))
+    assert len(intersecting) == 1
+
+    # Test same as before, but reversed
+    e1 = list(reversed(e1))
+    e2 = list(reversed(e2))
+    intersecting = list(_intersecting_eventpairs(e1, e2))
+    assert len(intersecting) == 1
+
+
+def test_filter_period_intersect():
+    td1h = timedelta(hours=1)
+    td30min = timedelta(minutes=30)
+    now = datetime.now()
+
+    # Filter 1h event with another 1h event at a 30min offset
+    to_filter = [Event(timestamp=now, duration=td1h)]
+    filter_with = [Event(timestamp=now + td30min, duration=td1h)]
+    filtered_events = filter_period_intersect(to_filter, filter_with)
+    assert filtered_events[0].duration == td30min
+
+    # Filter 2x 30min events with a 15min gap with another 45min event in between intersecting both
+    to_filter = [
+        Event(timestamp=now, duration=td30min),
+        Event(timestamp=now + timedelta(minutes=45), duration=td30min),
+    ]
+    filter_with = [
+        Event(timestamp=now + timedelta(minutes=15), duration=timedelta(minutes=45))
+    ]
+    filtered_events = filter_period_intersect(to_filter, filter_with)
+    assert len(filtered_events) == 2
+    assert filtered_events[0].duration == timedelta(minutes=15)
+    assert filtered_events[1].duration == timedelta(minutes=15)
+
+    # Same as previous intersection, but reversing filter and to_filter events
+    to_filter = [
+        Event(timestamp=now + timedelta(minutes=15), duration=timedelta(minutes=45))
+    ]
+    filter_with = [
+        Event(timestamp=now, duration=td30min),
+        Event(timestamp=now + timedelta(minutes=45), duration=td30min),
+    ]
+    filtered_events = filter_period_intersect(to_filter, filter_with)
+    assert len(filtered_events) == 2
+    assert filtered_events[0].duration == timedelta(minutes=15)
+    assert filtered_events[1].duration == timedelta(minutes=15)
+
+
+def test_period_union():
+    now = datetime.now(timezone.utc)
+
+    # Events overlapping
+    events1 = [Event(timestamp=now, duration=timedelta(seconds=10))]
+    events2 = [
+        Event(timestamp=now + timedelta(seconds=9), duration=timedelta(seconds=10))
+    ]
+    unioned_events = period_union(events1, events2)
+    assert len(unioned_events) == 1
+
+    # Events adjacent but not overlapping
+    events1 = [Event(timestamp=now, duration=timedelta(seconds=10))]
+    events2 = [
+        Event(timestamp=now + timedelta(seconds=10), duration=timedelta(seconds=10))
+    ]
+    unioned_events = period_union(events1, events2)
+    assert len(unioned_events) == 1
+
+    # Events not overlapping or adjacent
+    events1 = [Event(timestamp=now, duration=timedelta(seconds=10))]
+    events2 = [
+        Event(timestamp=now + timedelta(seconds=11), duration=timedelta(seconds=10))
+    ]
+    unioned_events = period_union(events1, events2)
+    assert len(unioned_events) == 2
+
+
+def test_sort_by_timestamp():
+    now = datetime.now(timezone.utc)
+    events = []
+    events.append(
+        Event(timestamp=now + timedelta(seconds=2), duration=timedelta(seconds=1))
+    )
+    events.append(
+        Event(timestamp=now + timedelta(seconds=1), duration=timedelta(seconds=2))
+    )
+    events_sorted = sort_by_timestamp(events)
+    assert events_sorted == events[::-1]
+
+
+def test_sort_by_duration():
+    now = datetime.now(timezone.utc)
+    events = []
+    events.append(
+        Event(timestamp=now + timedelta(seconds=2), duration=timedelta(seconds=1))
+    )
+    events.append(
+        Event(timestamp=now + timedelta(seconds=1), duration=timedelta(seconds=2))
+    )
+    events_sorted = sort_by_duration(events)
+    assert events_sorted == events[::-1]
+
+
+def test_sum_durations():
+    now = datetime.now(timezone.utc)
+    events = []
+    for i in range(10):
+        events.append(
+            Event(timestamp=now + timedelta(seconds=i), duration=timedelta(seconds=1))
+        )
+    result = sum_durations(events)
+    assert result == timedelta(seconds=10)
+
+
+def test_merge_events_by_keys_1():
+    now = datetime.now(timezone.utc)
+    events = []
+    e1_data = {"label": "a"}
+    e2_data = {"label": "b"}
+    e1 = Event(data=e1_data, timestamp=now, duration=timedelta(seconds=1))
+    e2 = Event(data=e2_data, timestamp=now, duration=timedelta(seconds=1))
+    events = events + [e1] * 10
+    events = events + [e2] * 5
+
+    # Check that an empty key list has no effect
+    assert merge_events_by_keys(events, []) == events
+
+    # Check that trying to merge on unavailable key has no effect
+    assert len(merge_events_by_keys(events, ["unknown"])) == 1
+
+    result = merge_events_by_keys(events, ["label"])
+    result = sort_by_duration(result)
+    print(result)
+    print(len(result))
+    assert len(result) == 2
+    assert result[0].duration == timedelta(seconds=10)
+    assert result[1].duration == timedelta(seconds=5)
+
+
+def test_merge_events_by_keys_2():
+    now = datetime.now(timezone.utc)
+    events = []
+    e1_data = {"k1": "a", "k2": "a"}
+    e2_data = {"k1": "a", "k2": "c"}
+    e3_data = {"k1": "b", "k2": "a"}
+    e1 = Event(data=e1_data, timestamp=now, duration=timedelta(seconds=1))
+    e2 = Event(data=e2_data, timestamp=now, duration=timedelta(seconds=1))
+    e3 = Event(data=e3_data, timestamp=now, duration=timedelta(seconds=1))
+    events = events + [e1] * 10
+    events = events + [e2] * 9
+    events = events + [e3] * 8
+    result = merge_events_by_keys(events, ["k1", "k2"])
+    result = sort_by_duration(result)
+    print(result)
+    print(len(result))
+    assert len(result) == 3
+    assert result[0].data == e1_data
+    assert result[0].duration == timedelta(seconds=10)
+    assert result[1].data == e2_data
+    assert result[1].duration == timedelta(seconds=9)
+    assert result[2].data == e3_data
+    assert result[2].duration == timedelta(seconds=8)
+
+
+def test_chunk_events_by_key():
+    now = datetime.now(timezone.utc)
+    events = []
+    e1_data = {"label1": "1a", "label2": "2a"}
+    e2_data = {"label1": "1a", "label2": "2b"}
+    e3_data = {"label1": "1b", "label2": "2b"}
+    e1 = Event(data=e1_data, timestamp=now, duration=timedelta(seconds=1))
+    e2 = Event(data=e2_data, timestamp=now, duration=timedelta(seconds=1))
+    e3 = Event(data=e3_data, timestamp=now, duration=timedelta(seconds=1))
+    events = [e1, e2, e3]
+    result = chunk_events_by_key(events, "label1")
+    print(len(result))
+    pprint(result)
+    assert len(result) == 2
+    # Check root label
+    assert result[0].data["label1"] == "1a"
+    assert result[1].data["label1"] == "1b"
+    # Check timestamp
+    assert result[0].timestamp == e1.timestamp
+    assert result[1].timestamp == e3.timestamp
+    # Check duration
+    assert result[0].duration == e1.duration + e2.duration
+    assert result[1].duration == e3.duration
+    # Check subevents
+    assert result[0].data["subevents"][0] == e1
+    assert result[0].data["subevents"][1] == e2
+    assert result[1].data["subevents"][0] == e3
+
+
+def test_url_parse_event():
+    now = datetime.now(timezone.utc)
+    e = Event(
+        data={"url": "http://asd.com/test/?a=1"},
+        timestamp=now,
+        duration=timedelta(seconds=1),
+    )
+    result = split_url_events([e])
+    print(result)
+    assert result[0].data["$protocol"] == "http"
+    assert result[0].data["$domain"] == "asd.com"
+    assert result[0].data["$path"] == "/test/"
+    assert result[0].data["$params"] == ""
+    assert result[0].data["$options"] == "a=1"
+    assert result[0].data["$identifier"] == ""
+
+    e2 = Event(
+        data={"url": "https://www.asd.asd.com/test/test2/meh;meh2?asd=2&asdf=3#id"},
+        timestamp=now,
+        duration=timedelta(seconds=1),
+    )
+    result = split_url_events([e2])
+    print(result)
+    assert result[0].data["$protocol"] == "https"
+    assert result[0].data["$domain"] == "asd.asd.com"
+    assert result[0].data["$path"] == "/test/test2/meh"
+    assert result[0].data["$params"] == "meh2"
+    assert result[0].data["$options"] == "asd=2&asdf=3"
+    assert result[0].data["$identifier"] == "id"
+
+    e3 = Event(
+        data={"url": "file:///home/johan/myfile.txt"},
+        timestamp=now,
+        duration=timedelta(seconds=1),
+    )
+    result = split_url_events([e3])
+    print(result)
+    assert result[0].data["$protocol"] == "file"
+    assert result[0].data["$domain"] == "file"
+    assert result[0].data["$path"] == "/home/johan/myfile.txt"
+    assert result[0].data["$params"] == ""
+    assert result[0].data["$options"] == ""
+    assert result[0].data["$identifier"] == ""
+
+    # Test about: URLs
+    e4 = Event(
+        data={"url": "about:blank"},
+        timestamp=now,
+        duration=timedelta(seconds=1),
+    )
+    result = split_url_events([e4])
+    assert result[0].data["$protocol"] == "about"
+    assert result[0].data["$domain"] == "about"
+
+
+def test_union():
+    now = datetime.now(timezone.utc)
+
+    e1 = Event(timestamp=now - timedelta(seconds=20), duration=timedelta(seconds=5))
+    e2 = Event(timestamp=now - timedelta(seconds=10), duration=timedelta(seconds=5))
+    e3 = Event(timestamp=now, duration=timedelta(seconds=1))
+    e4 = Event(timestamp=now + timedelta(seconds=20), duration=timedelta(seconds=1))
+
+    # union separate event lists with duplicates
+    events_union = union([e1, e2, e4], [e2, e3])
+    assert events_union == [e1, e2, e3, e4]
+
+    e1 = Event(timestamp=now - timedelta(seconds=20), duration=timedelta(seconds=5))
+    e2 = Event(timestamp=now - timedelta(seconds=10), duration=timedelta(seconds=5))
+    e3 = Event(timestamp=now - timedelta(seconds=10), duration=timedelta(seconds=10))
+    e4 = Event(timestamp=now - timedelta(seconds=5), duration=timedelta(seconds=5))
+    e5 = Event(timestamp=now, duration=timedelta(seconds=10))
+
+    # union event lists with intersecting duplicates
+    events_union = union([e3, e2, e5], [e1, e3, e4, e5])
+    assert events_union == [e1, e2, e3, e4, e5]
+
+    e1 = Event(timestamp=now - timedelta(seconds=30), duration=timedelta(seconds=15))
+    e2 = Event(timestamp=now, duration=timedelta(seconds=3))
+    e3 = Event(timestamp=now, duration=timedelta(seconds=5))
+    e4 = Event(timestamp=now, duration=timedelta(seconds=10))
+
+    # union event lists with same timestamp but different duration duplicates
+    events_union = union([e1, e2, e4], [e3, e2, e1])
+    assert events_union == [e1, e2, e3, e4]
+
+
+def test_categorize():
+    now = datetime.now(timezone.utc)
+
+    classes = [
+        (["Test"], Rule({"regex": "^just"})),
+        (["Test", "Subtest"], Rule({"regex": "subtest$"})),
+        (["Test", "Ignorecase"], Rule({"regex": "ignorecase", "ignore_case": True})),
+    ]
+    events = [
+        Event(timestamp=now, duration=0, data={"key": "just a test"}),
+        Event(timestamp=now, duration=0, data={"key": "just a subtest"}),
+        Event(timestamp=now, duration=0, data={"key": "just a IGNORECASE test"}),
+        Event(timestamp=now, duration=0, data={}),
+    ]
+    events = categorize(events, classes)
+
+    assert events[0].data["$category"] == ["Test"]
+    assert events[1].data["$category"] == ["Test", "Subtest"]
+    assert events[2].data["$category"] == ["Test", "Ignorecase"]
+    assert events[3].data["$category"] == ["Uncategorized"]
+
+
+def _event(value: str = "just a test") -> Event:
+    return Event(timestamp=datetime.now(timezone.utc), duration=0, data={"key": value})
+
+
+def test_categorize_depth_wins_without_priority():
+    events = categorize(
+        [_event()],
+        [
+            (["A"], Rule({"regex": "test"})),
+            (["B", "B1"], Rule({"regex": "test"})),
+        ],
+    )
+    assert events[0].data["$category"] == ["B", "B1"]
+
+
+def test_categorize_explicit_priority_overrides_depth():
+    # Default for B1 is depth 2 → 20; 25 beats it.
+    events = categorize(
+        [_event()],
+        [
+            (["A"], Rule({"regex": "test", "priority": 25})),
+            (["B", "B1"], Rule({"regex": "test"})),
+        ],
+    )
+    assert events[0].data["$category"] == ["A"]
+
+
+def test_categorize_weight_alias():
+    events = categorize(
+        [_event()],
+        [
+            (["A"], Rule({"regex": "test", "weight": 25})),
+            (["B", "B1"], Rule({"regex": "test"})),
+        ],
+    )
+    assert events[0].data["$category"] == ["A"]
+
+
+def test_categorize_inter_level_priority():
+    between = categorize(
+        [_event()],
+        [
+            (["A"], Rule({"regex": "test"})),
+            (["A2"], Rule({"regex": "test", "priority": 15})),
+        ],
+    )
+    assert between[0].data["$category"] == ["A2"]
+
+    still_loses_to_deeper = categorize(
+        [_event()],
+        [
+            (["A2"], Rule({"regex": "test", "priority": 15})),
+            (["B", "B1"], Rule({"regex": "test"})),
+        ],
+    )
+    assert still_loses_to_deeper[0].data["$category"] == ["B", "B1"]
+
+
+def test_categorize_lower_priority_loses_to_default_depth():
+    events = categorize(
+        [_event()],
+        [
+            (["A"], Rule({"regex": "test"})),
+            (["B", "B1"], Rule({"regex": "test", "priority": 0})),
+        ],
+    )
+    assert events[0].data["$category"] == ["A"]
+
+
+def test_categorize_equal_priority_keeps_later_match():
+    events = categorize(
+        [_event()],
+        [
+            (["First"], Rule({"regex": "test", "priority": 5})),
+            (["Second"], Rule({"regex": "test", "priority": 5})),
+        ],
+    )
+    assert events[0].data["$category"] == ["Second"]
+
+
+def test_categorize_negative_priority_still_beats_uncategorized():
+    events = categorize(
+        [_event()],
+        [(["Low"], Rule({"regex": "test", "priority": -100}))],
+    )
+    assert events[0].data["$category"] == ["Low"]
+
+
+def test_categorize_priority_below_i64_min_still_beats_uncategorized():
+    # Python ints are unbounded; a signed-64-bit fallback sentinel would
+    # incorrectly keep Uncategorized for values below -(2**63).
+    events = categorize(
+        [_event()],
+        [(["Low"], Rule({"regex": "test", "priority": -(2**63) - 1}))],
+    )
+    assert events[0].data["$category"] == ["Low"]
+
+
+def test_categorize_empty_category_keeps_uncategorized():
+    events = categorize(
+        [_event()],
+        [([], Rule({"regex": "test"}))],
+    )
+    assert events[0].data["$category"] == ["Uncategorized"]
+
+
+def test_rule_invalid_priority():
+    with pytest.raises(ValueError, match="integer"):
+        Rule({"regex": "test", "priority": 1.5})
+    with pytest.raises(ValueError, match="integer"):
+        Rule({"regex": "test", "priority": "high"})
+    with pytest.raises(ValueError, match="integer"):
+        Rule({"regex": "test", "priority": True})
+
+
+def test_categorize_cache_correctness():
+    """Cache reuses category for identical data; distinct data gets its own category."""
+    now = datetime.now(timezone.utc)
+
+    classes = [
+        (["Browser"], Rule({"regex": "Firefox"})),
+        (["Editor"], Rule({"regex": "vim"})),
+    ]
+    firefox_data = {"app": "Firefox", "title": "Home"}
+    vim_data = {"app": "vim", "title": "classify.py"}
+
+    # 50 Firefox events, 1 vim event, 50 more Firefox events
+    events = (
+        [Event(timestamp=now, duration=0, data=dict(firefox_data)) for _ in range(50)]
+        + [Event(timestamp=now, duration=0, data=dict(vim_data))]
+        + [Event(timestamp=now, duration=0, data=dict(firefox_data)) for _ in range(50)]
+    )
+    result = categorize(events, classes)
+
+    for e in result[:50]:
+        assert e.data["$category"] == ["Browser"]
+    assert result[50].data["$category"] == ["Editor"]
+    for e in result[51:]:
+        assert e.data["$category"] == ["Browser"]
+
+    # Mutating one event's category must not affect others sharing the same data fingerprint
+    result[0].data["$category"].append("MUTATED")
+    assert result[1].data["$category"] == ["Browser"]
+
+
+def test_tags():
+    now = datetime.now(timezone.utc)
+
+    classes = [
+        ("Test", Rule({"regex": "value$"})),
+        ("Test", Rule({"regex": "^just"})),
+    ]
+    events = [
+        Event(timestamp=now, duration=0, data={"key": "just a test value"}),
+        Event(timestamp=now, duration=0, data={}),
+    ]
+    events = tag(events, classes)
+
+    assert len(events[0].data["$tags"]) == 2
+    assert len(events[1].data["$tags"]) == 0
+
+
+def test_union_no_overlap():
+    from pprint import pprint
+
+    now = datetime(2018, 1, 1, 0, 0)
+    td1h = timedelta(hours=1)
+    events1 = [
+        Event(timestamp=now + 2 * i * td1h, duration=td1h, data={"test": 1})
+        for i in range(3)
+    ]
+    events2 = [
+        Event(timestamp=now + (2 * i + 0.5) * td1h, duration=td1h, data={"test": 2})
+        for i in range(3)
+    ]
+
+    events_union = union_no_overlap(events1, events2)
+    # pprint(events_union)
+    dur = sum((e.duration for e in events_union), timedelta(0))
+    assert dur == timedelta(hours=4, minutes=30)
+    assert sorted(events_union, key=lambda e: e.timestamp)
+
+    events_union = union_no_overlap(events2, events1)
+    # pprint(events_union)
+    dur = sum((e.duration for e in events_union), timedelta(0))
+    assert dur == timedelta(hours=4, minutes=30)
+    assert sorted(events_union, key=lambda e: e.timestamp)
+
+    events1 = [
+        Event(timestamp=now + (2 * i) * td1h, duration=td1h, data={"test": 1})
+        for i in range(3)
+    ]
+    events2 = [Event(timestamp=now, duration=5 * td1h, data={"test": 2})]
+    events_union = union_no_overlap(events1, events2)
+    pprint(events_union)
+    dur = sum((e.duration for e in events_union), timedelta(0))
+    assert dur == timedelta(hours=5, minutes=0)
+    assert sorted(events_union, key=lambda e: e.timestamp)
+
+
+def test_merge_subwatcher_fields_basic():
+    """Fully overlapping subwatcher fields are injected without changing duration."""
+    now = datetime(2024, 1, 1, 12, 0, tzinfo=timezone.utc)
+    td1h = timedelta(hours=1)
+
+    base = [
+        Event(
+            timestamp=now,
+            duration=td1h,
+            data={"app": "vim", "title": "file.py"},
+        )
+    ]
+    sub = [
+        Event(
+            timestamp=now,
+            duration=td1h,
+            data={"project": "myproject", "file": "file.py", "language": "python"},
+        )
+    ]
+    result = merge_subwatcher_fields(base, sub, ["project", "file", "language"])
+
+    assert len(result) == 1
+    # Original base fields preserved
+    assert result[0].data["app"] == "vim"
+    assert result[0].data["title"] == "file.py"
+    # Subwatcher fields injected
+    assert result[0].data["project"] == "myproject"
+    assert result[0].data["language"] == "python"
+    # Exact overlap means no extra segmentation
+    assert result[0].timestamp == now
+    assert result[0].duration == td1h
+
+
+def test_merge_subwatcher_fields_partial_overlap_splits_base():
+    """Partial overlap only enriches the covered slice of the base event."""
+    now = datetime(2024, 1, 1, 12, 0, tzinfo=timezone.utc)
+    td15m = timedelta(minutes=15)
+    td30m = timedelta(minutes=30)
+    td1h = timedelta(hours=1)
+
+    base = [Event(timestamp=now, duration=td1h, data={"app": "vim"})]
+    sub = [
+        Event(
+            timestamp=now + td15m,
+            duration=td30m,
+            data={"project": "myproject"},
+        )
+    ]
+
+    result = merge_subwatcher_fields(base, sub, ["project"])
+
+    assert len(result) == 3
+    assert [event.duration for event in result] == [td15m, td30m, td15m]
+    assert [event.timestamp for event in result] == [
+        now,
+        now + td15m,
+        now + td15m + td30m,
+    ]
+    assert "project" not in result[0].data
+    assert result[1].data["project"] == "myproject"
+    assert "project" not in result[2].data
+    assert sum_durations(result) == td1h
+
+
+def test_merge_subwatcher_fields_overlapping_subwatchers_prefer_latest_start():
+    """Newer overlapping subwatcher events take over from their own start time."""
+    now = datetime(2024, 1, 1, 12, 0, tzinfo=timezone.utc)
+    td20m = timedelta(minutes=20)
+    td40m = timedelta(minutes=40)
+    td1h = timedelta(hours=1)
+
+    base = [Event(timestamp=now, duration=td1h, data={"app": "vim"})]
+    sub = [
+        Event(timestamp=now, duration=td40m, data={"project": "alpha"}),
+        Event(timestamp=now + td20m, duration=td40m, data={"project": "beta"}),
+    ]
+
+    result = merge_subwatcher_fields(base, sub, ["project"])
+
+    assert len(result) == 2
+    assert [event.timestamp for event in result] == [now, now + td20m]
+    assert [event.duration for event in result] == [td20m, td40m]
+    assert [event.data.get("project") for event in result] == ["alpha", "beta"]
+
+    by_project = {
+        event.data.get("project"): event.duration
+        for event in merge_events_by_keys(result, ["project"])
+    }
+    assert by_project["alpha"] == td20m
+    assert by_project["beta"] == td40m
+    assert sum_durations(result) == td1h
+
+
+def test_merge_subwatcher_fields_no_overlap():
+    """Base events with no overlapping subwatcher event are returned unchanged."""
+    now = datetime(2024, 1, 1, 12, 0, tzinfo=timezone.utc)
+    td1h = timedelta(hours=1)
+
+    base = [Event(timestamp=now, duration=td1h, data={"app": "vim"})]
+    # Subwatcher event is entirely after the base event
+    sub = [
+        Event(
+            timestamp=now + 2 * td1h,
+            duration=td1h,
+            data={"project": "other"},
+        )
+    ]
+    result = merge_subwatcher_fields(base, sub, ["project"])
+
+    assert len(result) == 1
+    assert "project" not in result[0].data
+    assert result[0].data["app"] == "vim"
+
+
+def test_merge_subwatcher_fields_base_wins_conflict():
+    """With conflict='base_wins' (default), existing base keys are not overwritten."""
+    now = datetime(2024, 1, 1, 12, 0, tzinfo=timezone.utc)
+    td1h = timedelta(hours=1)
+
+    base = [Event(timestamp=now, duration=td1h, data={"app": "vim", "file": "base.py"})]
+    sub = [Event(timestamp=now, duration=td1h, data={"file": "sub.py", "project": "p"})]
+
+    result = merge_subwatcher_fields(
+        base, sub, ["file", "project"], conflict="base_wins"
+    )
+    # base's "file" must not be overwritten
+    assert result[0].data["file"] == "base.py"
+    # "project" not in base → injected from sub
+    assert result[0].data["project"] == "p"
+
+
+def test_merge_subwatcher_fields_sub_wins_conflict():
+    """With conflict='sub_wins', subwatcher fields overwrite base fields."""
+    now = datetime(2024, 1, 1, 12, 0, tzinfo=timezone.utc)
+    td1h = timedelta(hours=1)
+
+    base = [Event(timestamp=now, duration=td1h, data={"app": "vim", "file": "base.py"})]
+    sub = [Event(timestamp=now, duration=td1h, data={"file": "sub.py"})]
+
+    result = merge_subwatcher_fields(base, sub, ["file"], conflict="sub_wins")
+    assert result[0].data["file"] == "sub.py"
+
+
+def test_merge_subwatcher_fields_multiple_subsegments_preserve_duration():
+    """Repeated subwatcher values aggregate to their true covered duration."""
+    now = datetime(2024, 1, 1, 12, 0, tzinfo=timezone.utc)
+    td15m = timedelta(minutes=15)
+    td1h = timedelta(hours=1)
+
+    base = [Event(timestamp=now, duration=td1h, data={"app": "vim"})]
+    sub = [
+        Event(timestamp=now, duration=td15m, data={"project": "alpha"}),
+        Event(timestamp=now + td15m, duration=td15m, data={"project": "beta"}),
+        Event(
+            timestamp=now + 2 * td15m,
+            duration=td15m,
+            data={"project": "alpha"},
+        ),
+    ]
+
+    result = merge_subwatcher_fields(base, sub, ["project"])
+
+    by_project = {
+        event.data.get("project"): event.duration
+        for event in merge_events_by_keys(result, ["project"])
+    }
+    by_app = {
+        event.data.get("app"): event.duration
+        for event in merge_events_by_keys(result, ["app"])
+    }
+
+    assert by_project["alpha"] == 2 * td15m
+    assert by_project["beta"] == td15m
+    assert by_project[None] == td15m
+    assert by_app["vim"] == td1h
+    assert sum_durations(result) == td1h
+
+
+def test_merge_subwatcher_fields_empty_inputs():
+    """Empty sub or keys returns a defensive copy of base (not the same list)."""
+    now = datetime(2024, 1, 1, 12, 0, tzinfo=timezone.utc)
+    td1h = timedelta(hours=1)
+    base = [Event(timestamp=now, duration=td1h, data={"app": "vim"})]
+
+    # Empty subwatcher list — returns a new list, data unchanged
+    result = merge_subwatcher_fields(base, [], ["project"])
+    assert result[0].data == {"app": "vim"}
+    assert result is not base
+
+    # Empty keys list
+    sub = [Event(timestamp=now, duration=td1h, data={"project": "p"})]
+    result = merge_subwatcher_fields(base, sub, [])
+    assert "project" not in result[0].data
+    assert result is not base
+
+    # Both empty
+    result = merge_subwatcher_fields(base, [], [])
+    assert result[0].data == {"app": "vim"}
+    assert result is not base
+
+
+def test_merge_subwatcher_fields_invalid_conflict():
+    """Invalid conflict value raises ValueError immediately."""
+    now = datetime(2024, 1, 1, 12, 0, tzinfo=timezone.utc)
+    td1h = timedelta(hours=1)
+    base = [Event(timestamp=now, duration=td1h, data={"app": "vim"})]
+    sub = [Event(timestamp=now, duration=td1h, data={"project": "p"})]
+
+    with pytest.raises(ValueError, match="conflict must be"):
+        merge_subwatcher_fields(base, sub, ["project"], conflict="invalid")

tests/utils.py

@@ -0,0 +1,52 @@
+import logging
+import random
+from datetime import datetime, timezone
+
+from aw_datastore import Datastore, get_storage_methods
+
+logging.basicConfig(level=logging.DEBUG)
+
+# Useful when you just want some placeholder time in your events, saves typing
+now = datetime.now(timezone.utc)
+
+
+class TempTestBucket:
+    """Context manager for creating a test bucket"""
+
+    def __init__(self, datastore):
+        self.ds = datastore
+        self.bucket_id = f"test-{random.randint(0, 10**4)}"
+
+    def __enter__(self):
+        self.ds.create_bucket(
+            bucket_id=self.bucket_id,
+            type="testtype",
+            client="testclient",
+            hostname="testhost",
+            name="testname",
+        )
+        return self.ds[self.bucket_id]
+
+    def __exit__(self, *_):
+        self.ds.delete_bucket(bucket_id=self.bucket_id)
+
+    def __repr__(self):
+        return f"<TempTestBucket using {self.ds.storage_strategy.__class__.__name__}>"
+
+
+_storage_methods = get_storage_methods()
+
+
+def param_datastore_objects():
+    return [
+        Datastore(storage_strategy=strategy, testing=True)
+        for name, strategy in _storage_methods.items()
+    ]
+
+
+def param_testing_buckets_cm():
+    datastores = [
+        Datastore(storage_strategy=strategy, testing=True)
+        for name, strategy in _storage_methods.items()
+    ]
+    return [TempTestBucket(ds) for ds in datastores]