From 8757f2fd5e2fb46c47e0f695fae5e52efc767617 Mon Sep 17 00:00:00 2001 From: upstream-mirror Date: Wed, 16 Sep 2026 00:17:07 +0500 Subject: [PATCH] snapshot of https://github.com/ActivityWatch/aw-core.git (master, 2026-09-16; blobs > 131072B omitted) --- .coveragerc | 23 + .github/workflows/build.yml | 44 ++ .github/workflows/codeql.yml | 41 ++ .github/workflows/lint.yml | 15 + .gitignore | 10 + .pre-commit-config.yaml | 16 + LICENSE.txt | 373 +++++++++++ Makefile | 35 + README.md | 35 + aw_cli/__main__.py | 107 +++ aw_cli/log.py | 57 ++ aw_core/__about__.py | 23 + aw_core/__init__.py | 29 + aw_core/config.py | 116 ++++ aw_core/decorators.py | 51 ++ aw_core/dirs.py | 179 +++++ aw_core/log.py | 129 ++++ aw_core/models.py | 142 ++++ aw_core/py.typed | 0 aw_core/schema.py | 20 + aw_core/schemas/bucket.json | 45 ++ aw_core/schemas/event.json | 19 + aw_core/schemas/export.json | 16 + aw_core/util.py | 24 + aw_datastore/__init__.py | 19 + aw_datastore/benchmark.py | 97 +++ aw_datastore/datastore.py | 189 ++++++ aw_datastore/migration.py | 62 ++ aw_datastore/py.typed | 0 aw_datastore/storages/__init__.py | 15 + aw_datastore/storages/abstract.py | 101 +++ aw_datastore/storages/memory.py | 187 ++++++ aw_datastore/storages/peewee.py | 451 +++++++++++++ aw_datastore/storages/sqlite.py | 378 +++++++++++ aw_query/__init__.py | 3 + aw_query/exceptions.py | 14 + aw_query/functions.py | 350 ++++++++++ aw_query/py.typed | 0 aw_query/query2.py | 448 +++++++++++++ aw_transform/__init__.py | 44 ++ aw_transform/chunk_events_by_key.py | 39 ++ aw_transform/classify.py | 122 ++++ aw_transform/filter_keyvals.py | 28 + aw_transform/filter_period_intersect.py | 173 +++++ aw_transform/flood.py | 121 ++++ aw_transform/heartbeats.py | 56 ++ aw_transform/merge_events_by_keys.py | 40 ++ aw_transform/merge_subwatcher_fields.py | 150 +++++ aw_transform/py.typed | 0 aw_transform/simplify.py | 30 + aw_transform/sort_by.py | 32 + aw_transform/split_url_events.py | 32 + aw_transform/union_no_overlap.py | 85 +++ poetry.lock | 696 ++++++++++++++++++++ pyproject.toml | 53 ++ scripts/install_python.ps1 | 233 +++++++ scripts/run_with_env.cmd | 87 +++ stubs/appdirs.pyi | 4 + stubs/iso8601.pyi | 6 + tests/__init__.py | 1 + tests/context.py | 6 + tests/test_cli_log.py | 64 ++ tests/test_config.py | 102 +++ tests/test_datastore.py | 586 +++++++++++++++++ tests/test_dirs.py | 201 ++++++ tests/test_event.py | 41 ++ tests/test_flood.py | 167 +++++ tests/test_heartbeat.py | 84 +++ tests/test_peewee.py | 66 ++ tests/test_query2.py | 715 ++++++++++++++++++++ tests/test_schemas.py | 60 ++ tests/test_transforms.py | 830 ++++++++++++++++++++++++ tests/utils.py | 52 ++ 73 files changed, 8869 insertions(+) create mode 100644 .coveragerc create mode 100644 .github/workflows/build.yml create mode 100644 .github/workflows/codeql.yml create mode 100644 .github/workflows/lint.yml create mode 100644 .gitignore create mode 100644 .pre-commit-config.yaml create mode 100644 LICENSE.txt create mode 100644 Makefile create mode 100644 README.md create mode 100644 aw_cli/__main__.py create mode 100644 aw_cli/log.py create mode 100644 aw_core/__about__.py create mode 100644 aw_core/__init__.py create mode 100644 aw_core/config.py create mode 100644 aw_core/decorators.py create mode 100644 aw_core/dirs.py create mode 100644 aw_core/log.py create mode 100644 aw_core/models.py create mode 100644 aw_core/py.typed create mode 100644 aw_core/schema.py create mode 100644 aw_core/schemas/bucket.json create mode 100644 aw_core/schemas/event.json create mode 100644 aw_core/schemas/export.json create mode 100644 aw_core/util.py create mode 100644 aw_datastore/__init__.py create mode 100755 aw_datastore/benchmark.py create mode 100644 aw_datastore/datastore.py create mode 100644 aw_datastore/migration.py create mode 100644 aw_datastore/py.typed create mode 100644 aw_datastore/storages/__init__.py create mode 100644 aw_datastore/storages/abstract.py create mode 100644 aw_datastore/storages/memory.py create mode 100644 aw_datastore/storages/peewee.py create mode 100644 aw_datastore/storages/sqlite.py create mode 100644 aw_query/__init__.py create mode 100644 aw_query/exceptions.py create mode 100644 aw_query/functions.py create mode 100644 aw_query/py.typed create mode 100644 aw_query/query2.py create mode 100644 aw_transform/__init__.py create mode 100644 aw_transform/chunk_events_by_key.py create mode 100644 aw_transform/classify.py create mode 100644 aw_transform/filter_keyvals.py create mode 100644 aw_transform/filter_period_intersect.py create mode 100644 aw_transform/flood.py create mode 100644 aw_transform/heartbeats.py create mode 100644 aw_transform/merge_events_by_keys.py create mode 100644 aw_transform/merge_subwatcher_fields.py create mode 100644 aw_transform/py.typed create mode 100644 aw_transform/simplify.py create mode 100644 aw_transform/sort_by.py create mode 100644 aw_transform/split_url_events.py create mode 100644 aw_transform/union_no_overlap.py create mode 100644 poetry.lock create mode 100644 pyproject.toml create mode 100644 scripts/install_python.ps1 create mode 100644 scripts/run_with_env.cmd create mode 100644 stubs/appdirs.pyi create mode 100644 stubs/iso8601.pyi create mode 100644 tests/__init__.py create mode 100644 tests/context.py create mode 100644 tests/test_cli_log.py create mode 100644 tests/test_config.py create mode 100644 tests/test_datastore.py create mode 100644 tests/test_dirs.py create mode 100644 tests/test_event.py create mode 100644 tests/test_flood.py create mode 100644 tests/test_heartbeat.py create mode 100644 tests/test_peewee.py create mode 100644 tests/test_query2.py create mode 100644 tests/test_schemas.py create mode 100644 tests/test_transforms.py create mode 100644 tests/utils.py diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 0000000..9fa2001 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,23 @@ +# .coveragerc to control coverage.py +[run] +branch = True +omit = + aw_datastore/benchmark.py + +[report] +# Regexes for lines to exclude from consideration +exclude_lines = + # Have to re-enable the standard pragma + pragma: no cover + + # Don't complain about missing debug-only code: + def __repr__ + if self\.debug + + # Don't complain if tests don't hit defensive assertion code: + raise AssertionError + raise NotImplementedError + + # Don't complain if non-runnable code isn't run: + if 0: + if __name__ == .__main__.: diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..4b74ae5 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,44 @@ +name: Build + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + +jobs: + build: + name: Test on ${{ matrix.os }}, py-${{ matrix.python_version }} + runs-on: ${{ matrix.os }} + env: + RELEASE: false + strategy: + matrix: + os: [ubuntu-latest, windows-latest, macOS-latest] + python_version: [3.8] + steps: + - uses: actions/checkout@v3 + with: + submodules: 'recursive' + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python_version }} + - name: Install dependencies + shell: bash + run: | + pip install poetry==1.3.2 # due to: https://github.com/python-poetry/poetry/issues/7611 + python -m venv venv + source venv/bin/activate || source venv/Scripts/activate + make build + - name: Run tests + shell: bash + run: | + source venv/bin/activate || source venv/Scripts/activate + make test + make typecheck + - name: Report coverage + shell: bash + run: | + # Allow failing, since sometimes codecov is grumpy and we just get "no healthy upstream" + bash <(curl -s https://codecov.io/bash) || true diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 0000000..99a4c44 --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,41 @@ +name: "CodeQL" + +on: + push: + branches: [ "master" ] + pull_request: + branches: [ "master" ] + schedule: + - cron: "49 7 * * 2" + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + security-events: write + + strategy: + fail-fast: false + matrix: + language: [ python ] + + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Initialize CodeQL + uses: github/codeql-action/init@v2 + with: + languages: ${{ matrix.language }} + queries: +security-and-quality + + - name: Autobuild + uses: github/codeql-action/autobuild@v2 + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v2 + with: + category: "/language:${{ matrix.language }}" diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 0000000..4c4a1c4 --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,15 @@ +name: Lint + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + +jobs: + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + - uses: jpetrucciani/ruff-check@main diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..aced738 --- /dev/null +++ b/.gitignore @@ -0,0 +1,10 @@ +__pycache__ +*.pyc +*.swp +*.egg-info +build +dist +.coverage +venv* +.*cache +coverage_html diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..1193eab --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,16 @@ +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v6.0.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-toml + - id: check-yaml + - id: check-merge-conflict + + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.15.4 + hooks: + - id: ruff + args: [--fix] + - id: ruff-format diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 0000000..d0a1fa1 --- /dev/null +++ b/LICENSE.txt @@ -0,0 +1,373 @@ +Mozilla Public License Version 2.0 +================================== + +1. Definitions +-------------- + +1.1. "Contributor" + means each individual or legal entity that creates, contributes to + the creation of, or owns Covered Software. + +1.2. "Contributor Version" + means the combination of the Contributions of others (if any) used + by a Contributor and that particular Contributor's Contribution. + +1.3. "Contribution" + means Covered Software of a particular Contributor. + +1.4. "Covered Software" + means Source Code Form to which the initial Contributor has attached + the notice in Exhibit A, the Executable Form of such Source Code + Form, and Modifications of such Source Code Form, in each case + including portions thereof. + +1.5. "Incompatible With Secondary Licenses" + means + + (a) that the initial Contributor has attached the notice described + in Exhibit B to the Covered Software; or + + (b) that the Covered Software was made available under the terms of + version 1.1 or earlier of the License, but not also under the + terms of a Secondary License. + +1.6. "Executable Form" + means any form of the work other than Source Code Form. + +1.7. "Larger Work" + means a work that combines Covered Software with other material, in + a separate file or files, that is not Covered Software. + +1.8. "License" + means this document. + +1.9. "Licensable" + means having the right to grant, to the maximum extent possible, + whether at the time of the initial grant or subsequently, any and + all of the rights conveyed by this License. + +1.10. "Modifications" + means any of the following: + + (a) any file in Source Code Form that results from an addition to, + deletion from, or modification of the contents of Covered + Software; or + + (b) any new file in Source Code Form that contains any Covered + Software. + +1.11. "Patent Claims" of a Contributor + means any patent claim(s), including without limitation, method, + process, and apparatus claims, in any patent Licensable by such + Contributor that would be infringed, but for the grant of the + License, by the making, using, selling, offering for sale, having + made, import, or transfer of either its Contributions or its + Contributor Version. + +1.12. "Secondary License" + means either the GNU General Public License, Version 2.0, the GNU + Lesser General Public License, Version 2.1, the GNU Affero General + Public License, Version 3.0, or any later versions of those + licenses. + +1.13. "Source Code Form" + means the form of the work preferred for making modifications. + +1.14. "You" (or "Your") + means an individual or a legal entity exercising rights under this + License. For legal entities, "You" includes any entity that + controls, is controlled by, or is under common control with You. For + purposes of this definition, "control" means (a) the power, direct + or indirect, to cause the direction or management of such entity, + whether by contract or otherwise, or (b) ownership of more than + fifty percent (50%) of the outstanding shares or beneficial + ownership of such entity. + +2. License Grants and Conditions +-------------------------------- + +2.1. Grants + +Each Contributor hereby grants You a world-wide, royalty-free, +non-exclusive license: + +(a) under intellectual property rights (other than patent or trademark) + Licensable by such Contributor to use, reproduce, make available, + modify, display, perform, distribute, and otherwise exploit its + Contributions, either on an unmodified basis, with Modifications, or + as part of a Larger Work; and + +(b) under Patent Claims of such Contributor to make, use, sell, offer + for sale, have made, import, and otherwise transfer either its + Contributions or its Contributor Version. + +2.2. Effective Date + +The licenses granted in Section 2.1 with respect to any Contribution +become effective for each Contribution on the date the Contributor first +distributes such Contribution. + +2.3. Limitations on Grant Scope + +The licenses granted in this Section 2 are the only rights granted under +this License. No additional rights or licenses will be implied from the +distribution or licensing of Covered Software under this License. +Notwithstanding Section 2.1(b) above, no patent license is granted by a +Contributor: + +(a) for any code that a Contributor has removed from Covered Software; + or + +(b) for infringements caused by: (i) Your and any other third party's + modifications of Covered Software, or (ii) the combination of its + Contributions with other software (except as part of its Contributor + Version); or + +(c) under Patent Claims infringed by Covered Software in the absence of + its Contributions. + +This License does not grant any rights in the trademarks, service marks, +or logos of any Contributor (except as may be necessary to comply with +the notice requirements in Section 3.4). + +2.4. Subsequent Licenses + +No Contributor makes additional grants as a result of Your choice to +distribute the Covered Software under a subsequent version of this +License (see Section 10.2) or under the terms of a Secondary License (if +permitted under the terms of Section 3.3). + +2.5. Representation + +Each Contributor represents that the Contributor believes its +Contributions are its original creation(s) or it has sufficient rights +to grant the rights to its Contributions conveyed by this License. + +2.6. Fair Use + +This License is not intended to limit any rights You have under +applicable copyright doctrines of fair use, fair dealing, or other +equivalents. + +2.7. Conditions + +Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted +in Section 2.1. + +3. Responsibilities +------------------- + +3.1. Distribution of Source Form + +All distribution of Covered Software in Source Code Form, including any +Modifications that You create or to which You contribute, must be under +the terms of this License. You must inform recipients that the Source +Code Form of the Covered Software is governed by the terms of this +License, and how they can obtain a copy of this License. You may not +attempt to alter or restrict the recipients' rights in the Source Code +Form. + +3.2. Distribution of Executable Form + +If You distribute Covered Software in Executable Form then: + +(a) such Covered Software must also be made available in Source Code + Form, as described in Section 3.1, and You must inform recipients of + the Executable Form how they can obtain a copy of such Source Code + Form by reasonable means in a timely manner, at a charge no more + than the cost of distribution to the recipient; and + +(b) You may distribute such Executable Form under the terms of this + License, or sublicense it under different terms, provided that the + license for the Executable Form does not attempt to limit or alter + the recipients' rights in the Source Code Form under this License. + +3.3. Distribution of a Larger Work + +You may create and distribute a Larger Work under terms of Your choice, +provided that You also comply with the requirements of this License for +the Covered Software. If the Larger Work is a combination of Covered +Software with a work governed by one or more Secondary Licenses, and the +Covered Software is not Incompatible With Secondary Licenses, this +License permits You to additionally distribute such Covered Software +under the terms of such Secondary License(s), so that the recipient of +the Larger Work may, at their option, further distribute the Covered +Software under the terms of either this License or such Secondary +License(s). + +3.4. Notices + +You may not remove or alter the substance of any license notices +(including copyright notices, patent notices, disclaimers of warranty, +or limitations of liability) contained within the Source Code Form of +the Covered Software, except that You may alter any license notices to +the extent required to remedy known factual inaccuracies. + +3.5. Application of Additional Terms + +You may choose to offer, and to charge a fee for, warranty, support, +indemnity or liability obligations to one or more recipients of Covered +Software. However, You may do so only on Your own behalf, and not on +behalf of any Contributor. You must make it absolutely clear that any +such warranty, support, indemnity, or liability obligation is offered by +You alone, and You hereby agree to indemnify every Contributor for any +liability incurred by such Contributor as a result of warranty, support, +indemnity or liability terms You offer. You may include additional +disclaimers of warranty and limitations of liability specific to any +jurisdiction. + +4. Inability to Comply Due to Statute or Regulation +--------------------------------------------------- + +If it is impossible for You to comply with any of the terms of this +License with respect to some or all of the Covered Software due to +statute, judicial order, or regulation then You must: (a) comply with +the terms of this License to the maximum extent possible; and (b) +describe the limitations and the code they affect. Such description must +be placed in a text file included with all distributions of the Covered +Software under this License. Except to the extent prohibited by statute +or regulation, such description must be sufficiently detailed for a +recipient of ordinary skill to be able to understand it. + +5. Termination +-------------- + +5.1. The rights granted under this License will terminate automatically +if You fail to comply with any of its terms. However, if You become +compliant, then the rights granted under this License from a particular +Contributor are reinstated (a) provisionally, unless and until such +Contributor explicitly and finally terminates Your grants, and (b) on an +ongoing basis, if such Contributor fails to notify You of the +non-compliance by some reasonable means prior to 60 days after You have +come back into compliance. Moreover, Your grants from a particular +Contributor are reinstated on an ongoing basis if such Contributor +notifies You of the non-compliance by some reasonable means, this is the +first time You have received notice of non-compliance with this License +from such Contributor, and You become compliant prior to 30 days after +Your receipt of the notice. + +5.2. If You initiate litigation against any entity by asserting a patent +infringement claim (excluding declaratory judgment actions, +counter-claims, and cross-claims) alleging that a Contributor Version +directly or indirectly infringes any patent, then the rights granted to +You by any and all Contributors for the Covered Software under Section +2.1 of this License shall terminate. + +5.3. In the event of termination under Sections 5.1 or 5.2 above, all +end user license agreements (excluding distributors and resellers) which +have been validly granted by You or Your distributors under this License +prior to termination shall survive termination. + +************************************************************************ +* * +* 6. Disclaimer of Warranty * +* ------------------------- * +* * +* Covered Software is provided under this License on an "as is" * +* basis, without warranty of any kind, either expressed, implied, or * +* statutory, including, without limitation, warranties that the * +* Covered Software is free of defects, merchantable, fit for a * +* particular purpose or non-infringing. The entire risk as to the * +* quality and performance of the Covered Software is with You. * +* Should any Covered Software prove defective in any respect, You * +* (not any Contributor) assume the cost of any necessary servicing, * +* repair, or correction. This disclaimer of warranty constitutes an * +* essential part of this License. No use of any Covered Software is * +* authorized under this License except under this disclaimer. * +* * +************************************************************************ + +************************************************************************ +* * +* 7. Limitation of Liability * +* -------------------------- * +* * +* Under no circumstances and under no legal theory, whether tort * +* (including negligence), contract, or otherwise, shall any * +* Contributor, or anyone who distributes Covered Software as * +* permitted above, be liable to You for any direct, indirect, * +* special, incidental, or consequential damages of any character * +* including, without limitation, damages for lost profits, loss of * +* goodwill, work stoppage, computer failure or malfunction, or any * +* and all other commercial damages or losses, even if such party * +* shall have been informed of the possibility of such damages. This * +* limitation of liability shall not apply to liability for death or * +* personal injury resulting from such party's negligence to the * +* extent applicable law prohibits such limitation. Some * +* jurisdictions do not allow the exclusion or limitation of * +* incidental or consequential damages, so this exclusion and * +* limitation may not apply to You. * +* * +************************************************************************ + +8. Litigation +------------- + +Any litigation relating to this License may be brought only in the +courts of a jurisdiction where the defendant maintains its principal +place of business and such litigation shall be governed by laws of that +jurisdiction, without reference to its conflict-of-law provisions. +Nothing in this Section shall prevent a party's ability to bring +cross-claims or counter-claims. + +9. Miscellaneous +---------------- + +This License represents the complete agreement concerning the subject +matter hereof. If any provision of this License is held to be +unenforceable, such provision shall be reformed only to the extent +necessary to make it enforceable. Any law or regulation which provides +that the language of a contract shall be construed against the drafter +shall not be used to construe this License against a Contributor. + +10. Versions of the License +--------------------------- + +10.1. New Versions + +Mozilla Foundation is the license steward. Except as provided in Section +10.3, no one other than the license steward has the right to modify or +publish new versions of this License. Each version will be given a +distinguishing version number. + +10.2. Effect of New Versions + +You may distribute the Covered Software under the terms of the version +of the License under which You originally received the Covered Software, +or under the terms of any subsequent version published by the license +steward. + +10.3. Modified Versions + +If you create software not governed by this License, and you want to +create a new license for such software, you may create and use a +modified version of this License if you rename the license and remove +any references to the name of the license steward (except to note that +such modified license differs from this License). + +10.4. Distributing Source Code Form that is Incompatible With Secondary +Licenses + +If You choose to distribute Source Code Form that is Incompatible With +Secondary Licenses under the terms of this version of the License, the +notice described in Exhibit B of this License must be attached. + +Exhibit A - Source Code Form License Notice +------------------------------------------- + + This Source Code Form is subject to the terms of the Mozilla Public + License, v. 2.0. If a copy of the MPL was not distributed with this + file, You can obtain one at https://mozilla.org/MPL/2.0/. + +If it is not possible or desirable to put the notice in a particular +file, then You may include the notice in a location (such as a LICENSE +file in a relevant directory) where a recipient would be likely to look +for such a notice. + +You may add additional accurate notices of copyright ownership. + +Exhibit B - "Incompatible With Secondary Licenses" Notice +--------------------------------------------------------- + + This Source Code Form is "Incompatible With Secondary Licenses", as + defined by the Mozilla Public License, v. 2.0. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..b13329d --- /dev/null +++ b/Makefile @@ -0,0 +1,35 @@ +.PHONY: build test benchmark typecheck typecheck-strict clean + +build: + poetry install + +test: + python -m pytest tests -v --cov=aw_core --cov=aw_datastore --cov=aw_transform --cov=aw_query + +.coverage: + make test + +coverage_html: .coverage + python -m coverage html -d coverage_html + +benchmark: + python -m aw_datastore.benchmark + +typecheck: + export MYPYPATH=./stubs; python -m mypy aw_core aw_datastore aw_transform aw_query --show-traceback --ignore-missing-imports --follow-imports=skip + +typecheck-strict: + export MYPYPATH=./stubs; python -m mypy aw_core aw_datastore aw_transform aw_query --strict-optional --check-untyped-defs; echo "Not a failing step" + +lint: + ruff check . + +lint-fix: + ruff check --fix . + +format: + ruff format . + +clean: + rm -rf build dist + rm -rf aw_core/__pycache__ aw_datastore/__pycache__ diff --git a/README.md b/README.md new file mode 100644 index 0000000..001b72f --- /dev/null +++ b/README.md @@ -0,0 +1,35 @@ +aw-core +======= + +[![GitHub Actions badge](https://github.com/ActivityWatch/aw-core/workflows/Build/badge.svg)](https://github.com/ActivityWatch/aw-core/actions) +[![Code coverage](https://codecov.io/gh/ActivityWatch/aw-core/branch/master/graph/badge.svg)](https://codecov.io/gh/ActivityWatch/aw-core) +[![PyPI](https://img.shields.io/pypi/v/aw-core)](https://pypi.org/project/aw-core/) +[![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) +[![Typechecking: Mypy](http://www.mypy-lang.org/static/mypy_badge.svg)](http://mypy-lang.org/) + + +Core library for ActivityWatch. + + +## Modules + + - `aw_core`, contains basic datatypes and utilities, such as the `Event` class, helpers for configuration and logging, as well as schemas for buckets, events, and exports. + - `aw_datastore`, contains the datastore classes used by aw-server-python. + - `aw_transform`, all event-transforms used in queries. + - `aw_query`, the query-language used by ActivityWatch. + +## Logging + +Run python with `LOG_LEVEL=debug` to use change the log level across all AW components + +## How to install + +To install the latest git version directly from github without cloning, run +`pip install git+https://github.com/ActivityWatch/aw-core.git` + +To install from a cloned version, cd into the directory and run +`poetry install` to install inside an virtualenv. If you want to install it +system-wide it can be installed with `pip install .`, but that has the issue +that it might not get the exact version of the dependencies due to not reading +the poetry.lock file. + diff --git a/aw_cli/__main__.py b/aw_cli/__main__.py new file mode 100644 index 0000000..acea0ea --- /dev/null +++ b/aw_cli/__main__.py @@ -0,0 +1,107 @@ +""" +The idea behind this `aw` or `aw-cli` wrapper script is to act as a collection of helper tools, +and perhaps even as a way to list and run ActivityWatch modules on a system (a bit like aw-qt, but without the GUI). +""" + +from pathlib import Path +from datetime import datetime +import os +import subprocess + +import click + +from aw_cli.log import find_oldest_log, print_log, LOGLEVELS +from typing import Optional + + +def _resolve_profile(profile: Optional[str], testing: bool) -> str: + """Return the active profile name, resolving --testing as an alias for 'testing'.""" + if profile: + return profile + if testing: + return "testing" + return "" + + +@click.group() +@click.option( + "--profile", + default=None, + help="Named instance profile (e.g. 'testing', 'research').", +) +@click.option("--testing", is_flag=True, help="Alias for --profile testing.") +def main(profile: Optional[str] = None, testing: bool = False): + active = _resolve_profile(profile, testing) + if active: + os.environ["AW_PROFILE"] = active + # If no explicit flag, preserve any inherited AW_PROFILE from the environment. + + +@main.command() +@click.pass_context +def qt(ctx): + profile = ctx.parent.params.get("profile") + testing = ctx.parent.params.get("testing") + active = _resolve_profile(profile, testing) + args = ["aw-qt"] + if active: + args += ["--profile", active] + return subprocess.call(args) + + +@main.command() +def directories(): + # Print all directories (respects AW_PROFILE set by the group callback) + from aw_core.dirs import get_data_dir, get_config_dir, get_cache_dir, get_log_dir + + print("Directory paths used") + print(" - config: ", get_config_dir(None)) + print(" - data: ", get_data_dir(None)) + print(" - logs: ", get_log_dir(None)) + print(" - cache: ", get_cache_dir(None)) + + +@main.command() +@click.pass_context +@click.argument("module_name", type=str, required=False) +@click.option( + "--since", + type=click.DateTime(formats=["%Y-%m-%d"]), + help="Only show logs since this date", +) +@click.option( + "--level", + type=click.Choice(LOGLEVELS), + help="Only show logs of this level, or higher.", +) +def logs( + ctx, + module_name: Optional[str] = None, + since: Optional[datetime] = None, + level: Optional[str] = None, +): + from aw_core.dirs import get_log_dir + + # AW_PROFILE was set by the group callback, so get_log_dir returns the profile-specific dir. + logdir: Path = Path(get_log_dir(None)) + + if module_name: + _print_newest_log(logdir / module_name, since, level) + else: + for subdir in sorted(logdir.iterdir()): + if subdir.is_dir(): + _print_newest_log(subdir, since, level) + + +def _print_newest_log( + path: Path, since: Optional[datetime], level: Optional[str] +) -> None: + logfile = find_oldest_log(path) + if logfile: + print_log(logfile, since, level) + else: + print(f"No logfile found in {path}") + + +if __name__ == "__main__": + main() diff --git a/aw_cli/log.py b/aw_cli/log.py new file mode 100644 index 0000000..ee8deb9 --- /dev/null +++ b/aw_cli/log.py @@ -0,0 +1,57 @@ +from pathlib import Path +from datetime import datetime +from typing import Optional + + +LOGLEVELS = ["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"] + + +def print_log( + path: Path, since: Optional[datetime] = None, level: Optional[str] = None +): + if not path.is_file(): + return + + show_levels = LOGLEVELS[LOGLEVELS.index(level) :] if level else None + + lines_printed = 0 + with path.open("r") as f: + lines = f.readlines() + print(f"Logs for module {path.parent.name} ({path.name}, {len(lines)} lines)") + for line in lines: + if since: + try: + linedate = datetime.strptime(line.split(" ")[0], "%Y-%m-%d") + except ValueError: + # Could not parse the date, so skip this line + # NOTE: Just because the date could not be parsed, doesn't mean there isn't meaningful info there. + # Would be better to find the first line after the cutoff, and then just print everything past that. + continue + # Skip lines before the date + if linedate < since: + continue + if level: + if not any(level in line for level in show_levels): + continue + print(line, end="") + lines_printed += 1 + + print(f" (Filtered {lines_printed}/{len(lines)} lines)") + + +def find_oldest_log(path: Path) -> Optional[Path]: + """Return the most-recently-modified log file under *path*. + + Since aw-core #149 each profile runs in its own platform directory + (``activitywatch-``), so no filename-based profile filtering is + needed here — every ``.log`` file in *path* belongs to the active profile. + """ + if not path.is_dir(): + return None + + logfiles = [f for f in path.iterdir() if f.is_file() and f.name.endswith(".log")] + if not logfiles: + return None + + logfiles.sort(key=lambda f: f.stat().st_mtime) + return logfiles[-1] diff --git a/aw_core/__about__.py b/aw_core/__about__.py new file mode 100644 index 0000000..9a9f58f --- /dev/null +++ b/aw_core/__about__.py @@ -0,0 +1,23 @@ +# Inspired by: +# https://github.com/pypa/pipfile/blob/master/pipfile/__about__.py + + +__all__ = [ + "__title__", + "__summary__", + "__uri__", + "__author__", + "__email__", + "__license__", + "__copyright__", +] + +__title__ = "aw-core" +__summary__ = "Core library for ActivityWatch" +__uri__ = "https://github.com/ActivityWatch/aw-core" + +__author__ = "Erik Bjäreholt, Johan Bjäreholt" +__email__ = "erik@bjareho.lt, johan@bjareho.lt" + +__license__ = "MPL2" +__copyright__ = f"Copyright 2017 {__author__}" diff --git a/aw_core/__init__.py b/aw_core/__init__.py new file mode 100644 index 0000000..0faa8ea --- /dev/null +++ b/aw_core/__init__.py @@ -0,0 +1,29 @@ +# ignore: F401 + +from . import __about__ + +from . import decorators +from . import util + +from . import dirs +from . import config +from . import log + +from . import models +from .models import Event + +from . import schema + +__all__ = [ + "__about__", + # Classes + "Event", + # Modules + "decorators", + "util", + "dirs", + "config", + "log", + "models", + "schema", +] diff --git a/aw_core/config.py b/aw_core/config.py new file mode 100644 index 0000000..c9b6ef3 --- /dev/null +++ b/aw_core/config.py @@ -0,0 +1,116 @@ +import logging +import os +from typing import Union + +import tomlkit +from deprecation import deprecated + +from aw_core import dirs + +logger = logging.getLogger(__name__) + + +def _merge(a: dict, b: dict, path=None): + """ + Recursively merges b into a, with b taking precedence. + + From: https://stackoverflow.com/a/7205107/965332 + """ + if path is None: + path = [] + for key in b: + if key in a: + if isinstance(a[key], dict) and isinstance(b[key], dict): + _merge(a[key], b[key], path + [str(key)]) + elif a[key] == b[key]: + pass # same leaf value + else: + a[key] = b[key] + else: + a[key] = b[key] + return a + + +def _comment_out_toml(s: str): + # Only comment out keys, not headers or empty lines + return "\n".join( + [ + "#" + line if line.strip() and not line.strip().startswith("[") else line + for line in s.split("\n") + ] + ) + + +def load_config_toml( + appname: str, default_config: str +) -> Union[dict, tomlkit.container.Container]: + config_dir = dirs.get_config_dir(appname) + config_file_path = os.path.join(config_dir, f"{appname}.toml") + + # Run early to ensure input is valid toml before writing + default_config_toml = tomlkit.parse(default_config) + + # Override defaults from existing config file + if os.path.isfile(config_file_path): + with open(config_file_path) as f: + config = f.read() + config_toml = tomlkit.parse(config) + else: + # If file doesn't exist, write with commented-out default config + with open(config_file_path, "w") as f: + f.write(_comment_out_toml(default_config)) + config_toml = dict() + + config = _merge(default_config_toml, config_toml) + + return config + + +def save_config_toml(appname: str, config: str) -> None: + # Check that passed config string is valid toml + assert tomlkit.parse(config) + + config_dir = dirs.get_config_dir(appname) + config_file_path = os.path.join(config_dir, f"{appname}.toml") + + with open(config_file_path, "w") as f: + f.write(config) + + +@deprecated( + details="Use the load_config_toml function instead", + deprecated_in="0.5.3", +) +def load_config(appname, default_config): + """ + Take the defaults, and if a config file exists, use the settings specified + there as overrides for their respective defaults. + """ + config = default_config + + config_dir = dirs.get_config_dir(appname) + config_file_path = os.path.join(config_dir, f"{appname}.toml") + + # Override defaults from existing config file + if os.path.isfile(config_file_path): + with open(config_file_path) as f: + config.read_file(f) + + # Overwrite current config file (necessary in case new default would be added) + save_config(appname, config) + + return config + + +@deprecated( + details="Use the save_config_toml function instead", + deprecated_in="0.5.3", +) +def save_config(appname, config): + config_dir = dirs.get_config_dir(appname) + config_file_path = os.path.join(config_dir, f"{appname}.ini") + with open(config_file_path, "w") as f: + config.write(f) + # Flush and fsync to lower risk of corrupted files + f.flush() + os.fsync(f.fileno()) diff --git a/aw_core/decorators.py b/aw_core/decorators.py new file mode 100644 index 0000000..f7df776 --- /dev/null +++ b/aw_core/decorators.py @@ -0,0 +1,51 @@ +import functools +import logging +import time +import warnings + + +def deprecated(f): # pragma: no cover + """ + This is a decorator which can be used to mark functions + as deprecated. It will result in a warning being emitted + when the function is used. + + Taken from: http://stackoverflow.com/a/30253848/965332 + """ + # Warn only once per deprecated function + warned_for = False + + @functools.wraps(f) + def g(*args, **kwargs): + # TODO: Use logging module instead? + nonlocal warned_for + if not warned_for: + warnings.simplefilter("always", DeprecationWarning) # turn off filter + warnings.warn( + f"Call to deprecated function {f.__name__}, " + "this warning will only show once per function.", + category=DeprecationWarning, + stacklevel=2, + ) + warnings.simplefilter("default", DeprecationWarning) # reset filter + warned_for = True + return f(*args, **kwargs) + + return g + + +def restart_on_exception(f, delay=1, exception=Exception): # pragma: no cover + @functools.wraps(f) + def g(*args, **kwargs): + while True: + try: + f(*args, **kwargs) + except exception as e: + # TODO: Use warnings module instead? + logging.error(f"{f.__name__} crashed due to exception, restarting.") + logging.error(e) + time.sleep( + delay + ) # To prevent extremely fast restarts in case of bad state. + + return g diff --git a/aw_core/dirs.py b/aw_core/dirs.py new file mode 100644 index 0000000..2527d7f --- /dev/null +++ b/aw_core/dirs.py @@ -0,0 +1,179 @@ +import os +import sys +from functools import wraps +from typing import Callable, Optional + +import platformdirs + +GetDirFunc = Callable[[Optional[str]], str] + +_DEFAULT_APPNAME = "activitywatch" +_TESTING_PROFILE = "testing" +_TESTING_APPNAME = f"{_DEFAULT_APPNAME}-{_TESTING_PROFILE}" + +# Filenames that mark a machine as still using the pre-profile shared-root +# testing layout (ActivityWatch/activitywatch#1399). Keep this list specific: +# a false positive would pin a fresh install to the legacy layout forever. +_LEGACY_TESTING_FILENAME_MARKERS = ( + "peewee-sqlite-testing", + "sqlite-testing", + "settings-testing", + "config-testing", + "-testing.db", + "-testing.toml", + "-testing.json", + "_testing_", +) + + +def get_profile() -> str: + """Return the active ``AW_PROFILE`` value, or ``""`` for the default profile.""" + return os.environ.get("AW_PROFILE", "") or "" + + +def _new_testing_root_exists() -> bool: + """True if any platform dir for ``activitywatch-testing`` already exists. + + Must not create directories: existence is the signal that a previous run + already adopted the isolated testing root. + """ + for getter in ( + platformdirs.user_data_dir, + platformdirs.user_config_dir, + platformdirs.user_cache_dir, + ): + if os.path.isdir(getter(_TESTING_APPNAME)): + return True + return False + + +def _is_legacy_testing_filename(name: str) -> bool: + lower = name.lower() + return any(marker in lower for marker in _LEGACY_TESTING_FILENAME_MARKERS) + + +def _legacy_testing_artifacts_exist() -> bool: + """True if testing data still lives under the shared ``activitywatch`` root.""" + roots = ( + platformdirs.user_data_dir(_DEFAULT_APPNAME), + platformdirs.user_config_dir(_DEFAULT_APPNAME), + platformdirs.user_cache_dir(_DEFAULT_APPNAME), + ) + for root in roots: + if not os.path.isdir(root): + continue + for dirpath, dirnames, filenames in os.walk(root): + for filename in filenames: + if _is_legacy_testing_filename(filename): + return True + # Descend one level (activitywatch/aw-server/...) not further. + if os.path.relpath(dirpath, root) != ".": + dirnames.clear() + return False + + +def using_legacy_testing_root() -> bool: + """Whether ``AW_PROFILE=testing`` should stay on the shared ``activitywatch`` root. + + Resolution rule (ActivityWatch/activitywatch#1399), identical on python and rust: + + 1. If ``activitywatch-testing/`` already exists: use it (new layout). + 2. Else if legacy testing artifacts exist in the bare ``activitywatch/`` + root: stay in legacy mode (old paths, old filenames). + 3. Else (fresh setup): create and use ``activitywatch-testing/``. + """ + if get_profile() != _TESTING_PROFILE: + return False + if _new_testing_root_exists(): + return False + return _legacy_testing_artifacts_exist() + + +def legacy_testing_suffix(testing: bool) -> str: + """Return ``"-testing"`` only when testing data shares the default root. + + Isolated profile roots (including new-style ``activitywatch-testing/``) use + bare filenames: the directory already isolates. Suffixed names remain only + in legacy mode so existing ``peewee-sqlite-testing.v2.db`` files keep working. + """ + if not testing: + return "" + profile = get_profile() + if profile == _TESTING_PROFILE and not using_legacy_testing_root(): + return "" + if profile and profile != _TESTING_PROFILE: + # Named isolated profile — directory already isolates. + return "" + return "-testing" + + +def _get_appname() -> str: + """Return the platformdirs appname, optionally suffixed by the active profile. + + If the ``AW_PROFILE`` environment variable is set to a non-empty string the + appname becomes ``activitywatch-`` so that *all* platform + directories (data, config, cache, log) are completely separate from the + default profile. An unset or empty ``AW_PROFILE`` returns the bare + ``"activitywatch"`` name, which is identical to the pre-profile behaviour. + + The ``testing`` profile is special: see :func:`using_legacy_testing_root`. + Existing testing data in the shared root is not orphaned; fresh setups and + machines that already have ``activitywatch-testing/`` use the isolated root. + + This is the single authoritative place where profile isolation is applied. + Every module that uses :func:`get_data_dir`, :func:`get_config_dir`, + :func:`get_cache_dir` or :func:`get_log_dir` automatically inherits the + correct root for the running profile; no ``profile=`` parameter needs to be + threaded through the call chain. + """ + profile = get_profile() + if not profile: + return _DEFAULT_APPNAME + if profile == _TESTING_PROFILE and using_legacy_testing_root(): + return _DEFAULT_APPNAME + return f"{_DEFAULT_APPNAME}-{profile}" + + +def ensure_path_exists(path: str) -> None: + if not os.path.exists(path): + os.makedirs(path) + + +def _ensure_returned_path_exists(f: GetDirFunc) -> GetDirFunc: + @wraps(f) + def wrapper(subpath: Optional[str] = None) -> str: + path = f(subpath) + ensure_path_exists(path) + return path + + return wrapper + + +@_ensure_returned_path_exists +def get_data_dir(module_name: Optional[str] = None) -> str: + data_dir = platformdirs.user_data_dir(_get_appname()) + return os.path.join(data_dir, module_name) if module_name else data_dir + + +@_ensure_returned_path_exists +def get_cache_dir(module_name: Optional[str] = None) -> str: + cache_dir = platformdirs.user_cache_dir(_get_appname()) + return os.path.join(cache_dir, module_name) if module_name else cache_dir + + +@_ensure_returned_path_exists +def get_config_dir(module_name: Optional[str] = None) -> str: + config_dir = platformdirs.user_config_dir(_get_appname()) + return os.path.join(config_dir, module_name) if module_name else config_dir + + +@_ensure_returned_path_exists +def get_log_dir(module_name: Optional[str] = None) -> str: # pragma: no cover + # on Linux/Unix, platformdirs changed to using XDG_STATE_HOME instead of XDG_DATA_HOME for log_dir in v2.6 + # we want to keep using XDG_DATA_HOME for backwards compatibility + # https://github.com/ActivityWatch/aw-core/pull/122#issuecomment-1768020335 + if sys.platform.startswith("linux"): + log_dir = platformdirs.user_cache_path(_get_appname()) / "log" + else: + log_dir = platformdirs.user_log_dir(_get_appname()) + return os.path.join(log_dir, module_name) if module_name else log_dir diff --git a/aw_core/log.py b/aw_core/log.py new file mode 100644 index 0000000..a0fd551 --- /dev/null +++ b/aw_core/log.py @@ -0,0 +1,129 @@ +import logging +import os +import sys +from datetime import datetime +from logging.handlers import RotatingFileHandler +from typing import List, Optional + +from . import dirs +from .decorators import deprecated + +# NOTE: Will be removed in a future version since it's not compatible +# with running a multi-service process. +# TODO: prefix with `_` +log_file_path = None + + +@deprecated +def get_log_file_path() -> Optional[str]: # pragma: no cover + """DEPRECATED: Use get_latest_log_file instead.""" + return log_file_path + + +def setup_logging( + name: str, + testing=False, + verbose=False, + log_stderr=True, + log_file=False, +): # pragma: no cover + root_logger = logging.getLogger() + root_logger.setLevel(logging.DEBUG if verbose else logging.INFO) + root_logger.handlers = [] + + # run with LOG_LEVEL=DEBUG to customize log level across all AW components + log_level = os.environ.get("LOG_LEVEL") + if log_level: + if hasattr(logging, log_level.upper()): + root_logger.setLevel(getattr(logging, log_level.upper())) + else: + root_logger.warning( + f"No logging level called {log_level} (as specified in env var)" + ) + + if log_stderr: + root_logger.addHandler(_create_stderr_handler()) + if log_file: + root_logger.addHandler(_create_file_handler(name, testing=testing)) + + def excepthook(type_, value, traceback): + root_logger.exception("Unhandled exception", exc_info=(type_, value, traceback)) + # call the default excepthook if log_stderr isn't true + # (otherwise it'll just get duplicated) + if not log_stderr: + sys.__excepthook__(type_, value, traceback) + + sys.excepthook = excepthook + + +def _get_latest_log_files(name, testing=False) -> List[str]: # pragma: no cover + """ + Returns a list with the paths of all available logfiles for `name`, + sorted by latest first. + """ + log_dir = dirs.get_log_dir(name) + files = filter(lambda filename: name in filename, os.listdir(log_dir)) + # Isolated testing roots already separate logs by directory; the + # filename filter is only needed in the shared-root legacy layout. + isolated_testing = testing and not dirs.legacy_testing_suffix(testing) + if not isolated_testing: + files = filter( + lambda filename: ( + "testing" in filename if testing else "testing" not in filename + ), + files, + ) + return [os.path.join(log_dir, filename) for filename in sorted(files, reverse=True)] + + +def get_latest_log_file(name, testing=False) -> Optional[str]: # pragma: no cover + """ + Returns the filename of the last logfile with ``name``. + Useful when you want to read the logfile of another ActivityWatch service. + """ + last_logs = _get_latest_log_files(name, testing=testing) + return last_logs[0] if last_logs else None + + +def _create_stderr_handler() -> logging.Handler: # pragma: no cover + stderr_handler = logging.StreamHandler(stream=sys.stderr) + stderr_handler.setFormatter(_create_human_formatter()) + + return stderr_handler + + +def _create_file_handler( + name, testing=False, log_json=False +) -> logging.Handler: # pragma: no cover + log_dir = dirs.get_log_dir(name) + + # Set logfile path and name + global log_file_path + + # Should result in something like: + # $LOG_DIR/aw-server_testing_2017-01-05T00:21:39.log + file_ext = ".log.json" if log_json else ".log" + now_str = str(datetime.now().replace(microsecond=0).isoformat()).replace(":", "-") + testing_prefix = ( + "testing_" if testing and dirs.legacy_testing_suffix(testing) else "" + ) + log_name = name + "_" + testing_prefix + now_str + file_ext + log_file_path = os.path.join(log_dir, log_name) + + # Create rotating logfile handler, max 10MB per file, 3 files max + # Prevents logfile from growing too large, like in: + # - https://github.com/ActivityWatch/activitywatch/issues/815#issue-1423555466 + # - https://github.com/ActivityWatch/activitywatch/issues/756#issuecomment-1266662861 + fh = RotatingFileHandler( + log_file_path, mode="a", maxBytes=10 * 1024 * 1024, backupCount=3 + ) + fh.setFormatter(_create_human_formatter()) + + return fh + + +def _create_human_formatter() -> logging.Formatter: # pragma: no cover + return logging.Formatter( + "%(asctime)s [%(levelname)-5s]: %(message)s (%(name)s:%(lineno)s)", + "%Y-%m-%d %H:%M:%S", + ) diff --git a/aw_core/models.py b/aw_core/models.py new file mode 100644 index 0000000..3904fb1 --- /dev/null +++ b/aw_core/models.py @@ -0,0 +1,142 @@ +import json +import logging +import numbers +import typing +from datetime import datetime, timedelta, timezone +from typing import ( + Any, + Dict, + Optional, + Union, +) + +import iso8601 + +logger = logging.getLogger(__name__) + + +Number = Union[int, float] +Id = Optional[Union[int, str]] +ConvertibleTimestamp = Union[datetime, str] +Duration = Union[timedelta, Number] +Data = Dict[str, Any] + + +def _timestamp_parse(ts_in: ConvertibleTimestamp) -> datetime: + """ + Takes something representing a timestamp and + returns a timestamp in the representation we want. + """ + ts = iso8601.parse_date(ts_in) if isinstance(ts_in, str) else ts_in + # Set resolution to milliseconds instead of microseconds + # (Fixes incompability with software based on unix time, for example mongodb) + ts = ts.replace(microsecond=int(ts.microsecond / 1000) * 1000) + # Add timezone if not set + if not ts.tzinfo: + # Needed? All timestamps should be iso8601 so ought to always contain timezone. + # Yes, because it is optional in iso8601 + logger.warning(f"timestamp without timezone found, using UTC: {ts}") + ts = ts.replace(tzinfo=timezone.utc) + return ts + + +class Event(dict): + """ + Used to represents an event. + """ + + def __init__( + self, + id: Optional[Id] = None, + timestamp: Optional[ConvertibleTimestamp] = None, + duration: Duration = 0, + data: Optional[Data] = None, + ) -> None: + self.id = id + if timestamp is None: + logger.warning( + "Event initializer did not receive a timestamp argument, " + "using now as timestamp" + ) + # FIXME: The typing.cast here was required for mypy to shut up, weird... + self.timestamp = datetime.now(typing.cast(timezone, timezone.utc)) + else: + # The conversion needs to be explicit here for mypy to pick it up + # (lacks support for properties) + self.timestamp = _timestamp_parse(timestamp) + self.duration = duration # type: ignore + self.data = data or {} + + def __eq__(self, other: object) -> bool: + if isinstance(other, Event): + return ( + self.timestamp == other.timestamp + and self.duration == other.duration + and self.data == other.data + ) + else: + raise TypeError( + f"operator not supported between instances of '{type(self)}' and '{type(other)}'" + ) + + def __lt__(self, other: object) -> bool: + if isinstance(other, Event): + return self.timestamp < other.timestamp + else: + raise TypeError( + f"operator not supported between instances of '{type(self)}' and '{type(other)}'" + ) + + def to_json_dict(self) -> dict: + """Useful when sending data over the wire. + Any mongodb interop should not use do this as it accepts datetimes.""" + json_data = self.copy() + json_data["timestamp"] = self.timestamp.astimezone(timezone.utc).isoformat() + json_data["duration"] = self.duration.total_seconds() + return json_data + + def to_json_str(self) -> str: + data = self.to_json_dict() + return json.dumps(data) + + def _hasprop(self, propname: str) -> bool: + """Badly named, but basically checks if the underlying + dict has a prop, and if it is a non-empty list""" + return propname in self and self[propname] is not None + + @property + def id(self) -> Id: + return self["id"] if self._hasprop("id") else None + + @id.setter + def id(self, id: Id) -> None: + self["id"] = id + + @property + def data(self) -> dict: + return self["data"] if self._hasprop("data") else {} + + @data.setter + def data(self, data: dict) -> None: + self["data"] = data + + @property + def timestamp(self) -> datetime: + return self["timestamp"] + + @timestamp.setter + def timestamp(self, timestamp: ConvertibleTimestamp) -> None: + self["timestamp"] = _timestamp_parse(timestamp).astimezone(timezone.utc) + + @property + def duration(self) -> timedelta: + return self["duration"] if self._hasprop("duration") else timedelta(0) + + @duration.setter + def duration(self, duration: Duration) -> None: + if isinstance(duration, timedelta): + self["duration"] = duration + elif isinstance(duration, numbers.Real): + self["duration"] = timedelta(seconds=duration) # type: ignore + else: + raise TypeError(f"Couldn't parse duration of invalid type {type(duration)}") diff --git a/aw_core/py.typed b/aw_core/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/aw_core/schema.py b/aw_core/schema.py new file mode 100644 index 0000000..4c87bee --- /dev/null +++ b/aw_core/schema.py @@ -0,0 +1,20 @@ +import os +import json + + +def _this_dir() -> str: + return os.path.dirname(os.path.abspath(__file__)) + + +def _schema_dir() -> str: + return os.path.join(os.path.dirname(_this_dir()), "aw_core", "schemas") + + +def get_json_schema(name: str) -> dict: + with open(os.path.join(_schema_dir(), name + ".json")) as f: + data = json.load(f) + return data + + +if __name__ == "__main__": + print(get_json_schema("event")) diff --git a/aw_core/schemas/bucket.json b/aw_core/schemas/bucket.json new file mode 100644 index 0000000..b46e2c8 --- /dev/null +++ b/aw_core/schemas/bucket.json @@ -0,0 +1,45 @@ +{ + "$id": "https://activitywatch.net/schemas/bucket.v0.json", + "$schema": "http://json-schema.org/draft-04/schema#", + "title": "Bucket", + "description": "The Bucket model that is used in ActivityWatch", + "type": "object", + "required": ["id", "type", "client", "hostname"], + "properties": { + "id": { + "description": "The unique id for the bucket", + "type": "string" + }, + "name": { + "description": "The readable and renameable name for the bucket", + "type": "string" + }, + "type": { + "description": "The event type", + "type": "string" + }, + "client": { + "description": "The name of the client that is reporting to the bucket", + "type": "string" + }, + "hostname": { + "description": "The hostname of the machine on which the client is running", + "type": "string" + }, + "created": { + "description": "The creation datetime of the bucket", + "type": "string", + "format": "date-time" + }, + "data": { + "description": "", + "type": "object" + }, + "events": { + "type": "array", + "items": { + "$ref": "#/definitions/Event" + } + } + } +} diff --git a/aw_core/schemas/event.json b/aw_core/schemas/event.json new file mode 100644 index 0000000..7539d4d --- /dev/null +++ b/aw_core/schemas/event.json @@ -0,0 +1,19 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "title": "Event", + "description": "The Event model that is used in ActivityWatch", + "type": "object", + "required": ["timestamp"], + "properties": { + "timestamp": { + "type": "string", + "format": "date-time" + }, + "duration": { + "type": "number" + }, + "data": { + "type": "object" + } + } +} diff --git a/aw_core/schemas/export.json b/aw_core/schemas/export.json new file mode 100644 index 0000000..4073934 --- /dev/null +++ b/aw_core/schemas/export.json @@ -0,0 +1,16 @@ +{ + "$id": "https://activitywatch.net/schemas/export.v0.json", + "$schema": "http://json-schema.org/draft-04/schema#", + "title": "Export", + "description": "The Export model that is used by ActivityWatch", + "type": "object", + "required": [], + "properties": { + "buckets": { + "type": "array", + "items": { + "$ref": "#/definitions/Bucket" + } + } + } +} diff --git a/aw_core/util.py b/aw_core/util.py new file mode 100644 index 0000000..9efe5d8 --- /dev/null +++ b/aw_core/util.py @@ -0,0 +1,24 @@ +import sys +from typing import Tuple +import logging + +logger = logging.getLogger(__name__) + + +class VersionException(Exception): ... + + +def _version_info_tuple() -> Tuple[int, int, int]: # pragma: no cover + return (sys.version_info.major, sys.version_info.minor, sys.version_info.micro) + + +def assert_version(required_version: Tuple[int, ...] = (3, 5)): # pragma: no cover + actual_version = _version_info_tuple() + if actual_version <= required_version: + raise VersionException( + ( + "Python version {} not supported, you need to upgrade your Python" + + " version to at least {}." + ).format(required_version) + ) + logger.debug(f"Python version: {_version_info_tuple()}") diff --git a/aw_datastore/__init__.py b/aw_datastore/__init__.py new file mode 100644 index 0000000..53795f9 --- /dev/null +++ b/aw_datastore/__init__.py @@ -0,0 +1,19 @@ +from typing import Callable, Dict + +from . import storages +from .datastore import Datastore +from .migration import check_for_migration + + +def get_storage_methods() -> Dict[str, Callable[..., storages.AbstractStorage]]: + from .storages import MemoryStorage, PeeweeStorage, SqliteStorage + + methods: Dict[str, Callable[..., storages.AbstractStorage]] = { + PeeweeStorage.sid: PeeweeStorage, + MemoryStorage.sid: MemoryStorage, + SqliteStorage.sid: SqliteStorage, + } + return methods + + +__all__ = ["Datastore", "get_storage_methods", "check_for_migration"] diff --git a/aw_datastore/benchmark.py b/aw_datastore/benchmark.py new file mode 100755 index 0000000..fd8daa5 --- /dev/null +++ b/aw_datastore/benchmark.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 +import sys +from contextlib import contextmanager +from datetime import datetime, timedelta, timezone +from typing import Callable + +from aw_core.models import Event + +from aw_datastore import get_storage_methods +from aw_datastore.storages import AbstractStorage + +td1s = timedelta(seconds=1) + + +def create_test_events(n): + now = datetime.now(timezone.utc) - timedelta(days=1000) + + events = [] + for i in range(n): + events.append( + Event(timestamp=now + i * td1s, duration=td1s, data={"label": "asd"}) + ) + + return events + + +@contextmanager +def temporary_bucket(ds): + bucket_id = "test_bucket" + try: + ds.delete_bucket(bucket_id) + except Exception: + pass + bucket = ds.create_bucket(bucket_id, "testingtype", "test-client", "testing-box") + yield bucket + ds.delete_bucket(bucket_id) + + +def benchmark(storage: Callable[..., AbstractStorage]): + raise NotImplementedError( + "No longer implemented as ttt/takethetime dependency is removed" + ) + + # if storage.__name__ == "PeeweeStorage": + # ds = Datastore(storage, testing=True, filepath="test.db") + # else: + # ds = Datastore(storage, testing=True) + + # num_single_events = 50 + # num_replace_events = 50 + # num_bulk_events = 20_000 + # num_events = num_single_events + num_replace_events + num_bulk_events + 1 + # num_final_events = num_single_events + num_bulk_events + 1 + + # events = create_test_events(num_events) + # single_events = events[:num_single_events] + # replace_events = events[num_single_events : num_single_events + num_replace_events] + # bulk_events = events[num_single_events + num_replace_events : -1] + + # print(storage.__name__) + + # with temporary_bucket(ds) as bucket: + # with ttt(" sum"): + # with ttt(f" single insert {num_single_events} events"): + # for event in single_events: + # bucket.insert(event) + + # with ttt(f" bulk insert {num_bulk_events} events"): + # bucket.insert(bulk_events) + + # with ttt(f" replace last {num_replace_events}"): + # for e in replace_events: + # bucket.replace_last(e) + + # with ttt(" insert 1 event"): + # bucket.insert(events[-1]) + + # with ttt(" get one"): + # events_tmp = bucket.get(limit=1) + + # with ttt(" get all"): + # events_tmp = bucket.get(limit=-1) + # assert len(events_tmp) == num_final_events + + # with ttt(" get range"): + # events_tmp = bucket.get( + # limit=-1, + # starttime=events[1].timestamp + 0.01 * td1s, + # endtime=events[-1].timestamp + events[-1].duration, + # ) + # assert len(events_tmp) == num_final_events - 1 + + +if __name__ == "__main__": + for storage in get_storage_methods().values(): + if len(sys.argv) <= 1 or storage.__name__ in sys.argv: + benchmark(storage) diff --git a/aw_datastore/datastore.py b/aw_datastore/datastore.py new file mode 100644 index 0000000..3cb0909 --- /dev/null +++ b/aw_datastore/datastore.py @@ -0,0 +1,189 @@ +import logging +from datetime import datetime, timedelta, timezone +from typing import ( + Callable, + Dict, + List, + Optional, + Union, +) + +from aw_core.models import Event + +from .storages import AbstractStorage + +logger = logging.getLogger(__name__) + + +class Datastore: + def __init__( + self, + storage_strategy: Callable[..., AbstractStorage], + testing=False, + **kwargs, + ) -> None: + self.logger = logger.getChild("Datastore") + self.bucket_instances: Dict[str, Bucket] = dict() + + self.storage_strategy = storage_strategy(testing=testing, **kwargs) + + def __repr__(self): + return f"" + + def __getitem__(self, bucket_id: str) -> "Bucket": + # If this bucket doesn't have a initialized object, create it + if bucket_id not in self.bucket_instances: + # If the bucket exists in the database, create an object representation of it + if bucket_id in self.buckets(): + bucket = Bucket(self, bucket_id) + self.bucket_instances[bucket_id] = bucket + else: + self.logger.error( + f"Cannot create a Bucket object for {bucket_id} because it doesn't exist in the database" + ) + raise KeyError + + return self.bucket_instances[bucket_id] + + def create_bucket( + self, + bucket_id: str, + type: str, + client: str, + hostname: str, + created: Optional[datetime] = None, + name: Optional[str] = None, + data: Optional[dict] = None, + ) -> "Bucket": + created = created or datetime.now(timezone.utc) + self.logger.info(f"Creating bucket '{bucket_id}'") + self.storage_strategy.create_bucket( + bucket_id, type, client, hostname, created.isoformat(), name=name, data=data + ) + return self[bucket_id] + + def update_bucket(self, bucket_id: str, **kwargs): + self.logger.info(f"Updating bucket '{bucket_id}'") + return self.storage_strategy.update_bucket(bucket_id, **kwargs) + + def delete_bucket(self, bucket_id: str): + self.logger.info(f"Deleting bucket '{bucket_id}'") + if bucket_id in self.bucket_instances: + del self.bucket_instances[bucket_id] + return self.storage_strategy.delete_bucket(bucket_id) + + def buckets(self): + return self.storage_strategy.buckets() + + +class Bucket: + def __init__(self, datastore: Datastore, bucket_id: str) -> None: + self.logger = logger.getChild("Bucket") + self.ds = datastore + self.bucket_id = bucket_id + + def metadata(self) -> dict: + return self.ds.storage_strategy.get_metadata(self.bucket_id) + + def get( + self, + limit: int = -1, + starttime: Optional[datetime] = None, + endtime: Optional[datetime] = None, + ) -> List[Event]: + """Returns events sorted in descending order by timestamp""" + # Resolution is rounded down since not all datastores like microsecond precision + if starttime: + starttime = starttime.replace( + microsecond=1000 * int(starttime.microsecond / 1000) + ) + if endtime: + # Rounding up here in order to ensure events aren't missed + # second_offset and microseconds modulo required since replace() only takes microseconds up to 999999 (doesn't handle overflow) + milliseconds = 1 + int(endtime.microsecond / 1000) + second_offset = int(milliseconds / 1000) # usually 0, rarely 1 + microseconds = ( + (1000 * milliseconds) % 1000000 + ) # will likely just be 1000 * milliseconds, if it overflows it would become zero + endtime = endtime.replace(microsecond=microseconds) + timedelta( + seconds=second_offset + ) + + return self.ds.storage_strategy.get_events( + self.bucket_id, limit, starttime, endtime + ) + + def get_by_id(self, event_id) -> Optional[Event]: + """Will return the event with the provided ID, or None if not found.""" + return self.ds.storage_strategy.get_event(self.bucket_id, event_id) + + def get_eventcount( + self, starttime: Optional[datetime] = None, endtime: Optional[datetime] = None + ) -> int: + return self.ds.storage_strategy.get_eventcount( + self.bucket_id, starttime, endtime + ) + + def insert(self, events: Union[Event, List[Event]]) -> Optional[Event]: + """ + Inserts one or several events. + If a single event is inserted, return the event with its id assigned. + If several events are inserted, returns None. (This is due to there being no efficient way of getting ids out when doing bulk inserts with some datastores such as peewee/SQLite) + """ + + # NOTE: Should we keep the timestamp checking? + warn_older_event = False + + # Get last event for timestamp check after insert + if warn_older_event: + last_event_list = self.get(1) + last_event = None + if last_event_list: + last_event = last_event_list[0] + + now = datetime.now(tz=timezone.utc) + + inserted: Optional[Event] = None + + # Call insert + if isinstance(events, Event): + oldest_event: Optional[Event] = events + if events.timestamp + events.duration > now: + self.logger.warning( + f"Event inserted into bucket {self.bucket_id} reaches into the future. Current UTC time: {str(now)}. Event data: {str(events)}" + ) + inserted = self.ds.storage_strategy.insert_one(self.bucket_id, events) + # assert inserted + elif isinstance(events, list): + if events: + oldest_event = sorted(events, key=lambda k: k["timestamp"])[0] + else: # pragma: no cover + oldest_event = None + for event in events: + if event.timestamp + event.duration > now: + self.logger.warning( + f"Event inserted into bucket {self.bucket_id} reaches into the future. Current UTC time: {str(now)}. Event data: {str(event)}" + ) + self.ds.storage_strategy.insert_many(self.bucket_id, events) + else: + raise TypeError + + # Warn if timestamp is older than last event + if warn_older_event and last_event and oldest_event: + if oldest_event.timestamp < last_event.timestamp: # pragma: no cover + self.logger.warning( + f"""Inserting event that has a older timestamp than previous event! +Previous: {last_event} +Inserted: {oldest_event}""" + ) + + return inserted + + def delete(self, event_id): + return self.ds.storage_strategy.delete(self.bucket_id, event_id) + + def replace_last(self, event): + return self.ds.storage_strategy.replace_last(self.bucket_id, event) + + def replace(self, event_id, event): + return self.ds.storage_strategy.replace(self.bucket_id, event_id, event) diff --git a/aw_datastore/migration.py b/aw_datastore/migration.py new file mode 100644 index 0000000..887b7b4 --- /dev/null +++ b/aw_datastore/migration.py @@ -0,0 +1,62 @@ +import logging +import os +from typing import List, Optional + +from aw_core.dirs import get_data_dir, legacy_testing_suffix + +from .storages import AbstractStorage + +logger = logging.getLogger(__name__) + + +def detect_db_files( + data_dir: str, datastore_name: Optional[str] = None, version=None +) -> List[str]: + db_files = [filename for filename in os.listdir(data_dir)] + if datastore_name: + db_files = [ + filename + for filename in db_files + if filename.split(".")[0] == datastore_name + ] + if version: + db_files = [ + filename for filename in db_files if filename.split(".")[1] == f"v{version}" + ] + return db_files + + +def check_for_migration(datastore: AbstractStorage): + data_dir = get_data_dir("aw-server") + + if datastore.sid == "sqlite": + peewee_type = "peewee-sqlite" + peewee_name = peewee_type + legacy_testing_suffix(datastore.testing) + # Migrate from peewee v2 + peewee_db_v2 = detect_db_files(data_dir, peewee_name, 2) + if len(peewee_db_v2) > 0: + peewee_v2_to_sqlite_v1(datastore) + + +def peewee_v2_to_sqlite_v1(datastore): + logger.info("Migrating database from peewee v2 to sqlite v1") + from .storages import PeeweeStorage + + pw_db = PeeweeStorage(datastore.testing) + # Fetch buckets and events + buckets = pw_db.buckets() + # Insert buckets and events to new db + for bucket_id in buckets: + logger.info(f"Migrating bucket {bucket_id}") + bucket = buckets[bucket_id] + datastore.create_bucket( + bucket["id"], + bucket["type"], + bucket["client"], + bucket["hostname"], + bucket["created"], + bucket["name"], + ) + bucket_events = pw_db.get_events(bucket_id, -1) + datastore.insert_many(bucket_id, bucket_events) + logger.info("Migration of peewee v2 to sqlite v1 finished") diff --git a/aw_datastore/py.typed b/aw_datastore/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/aw_datastore/storages/__init__.py b/aw_datastore/storages/__init__.py new file mode 100644 index 0000000..420c08d --- /dev/null +++ b/aw_datastore/storages/__init__.py @@ -0,0 +1,15 @@ +import logging as _logging + +logger: _logging.Logger = _logging.getLogger(__name__) + +from .abstract import AbstractStorage +from .memory import MemoryStorage +from .peewee import PeeweeStorage +from .sqlite import SqliteStorage + +__all__ = [ + "AbstractStorage", + "MemoryStorage", + "PeeweeStorage", + "SqliteStorage", +] diff --git a/aw_datastore/storages/abstract.py b/aw_datastore/storages/abstract.py new file mode 100644 index 0000000..b6bb49c --- /dev/null +++ b/aw_datastore/storages/abstract.py @@ -0,0 +1,101 @@ +from abc import ABCMeta, abstractmethod +from datetime import datetime +from typing import Dict, List, Optional + +from aw_core.models import Event + + +class AbstractStorage(metaclass=ABCMeta): + """ + Interface for storage methods. + """ + + sid = "Storage id not set, fix me" + + @abstractmethod + def __init__(self, testing: bool) -> None: + self.testing = True + raise NotImplementedError + + @abstractmethod + def buckets(self) -> Dict[str, dict]: + raise NotImplementedError + + @abstractmethod + def create_bucket( + self, + bucket_id: str, + type_id: str, + client: str, + hostname: str, + created: str, + name: Optional[str] = None, + data: Optional[dict] = None, + ) -> None: + raise NotImplementedError + + @abstractmethod + def update_bucket( + self, + bucket_id: str, + type_id: Optional[str] = None, + client: Optional[str] = None, + hostname: Optional[str] = None, + name: Optional[str] = None, + data: Optional[dict] = None, + ) -> None: + raise NotImplementedError + + @abstractmethod + def delete_bucket(self, bucket_id: str) -> None: + raise NotImplementedError + + @abstractmethod + def get_metadata(self, bucket_id: str) -> dict: + raise NotImplementedError + + @abstractmethod + def get_event( + self, + bucket_id: str, + event_id: int, + ) -> Optional[Event]: + raise NotImplementedError + + @abstractmethod + def get_events( + self, + bucket_id: str, + limit: int, + starttime: Optional[datetime] = None, + endtime: Optional[datetime] = None, + ) -> List[Event]: + raise NotImplementedError + + def get_eventcount( + self, + bucket_id: str, + starttime: Optional[datetime] = None, + endtime: Optional[datetime] = None, + ) -> int: + raise NotImplementedError + + @abstractmethod + def insert_one(self, bucket_id: str, event: Event) -> Event: + raise NotImplementedError + + def insert_many(self, bucket_id: str, events: List[Event]) -> None: + for event in events: + self.insert_one(bucket_id, event) + + @abstractmethod + def delete(self, bucket_id: str, event_id: int) -> bool: + raise NotImplementedError + + @abstractmethod + def replace(self, bucket_id: str, event_id: int, event: Event) -> bool: + raise NotImplementedError + + @abstractmethod + def replace_last(self, bucket_id: str, event: Event) -> None: + raise NotImplementedError diff --git a/aw_datastore/storages/memory.py b/aw_datastore/storages/memory.py new file mode 100644 index 0000000..32e3938 --- /dev/null +++ b/aw_datastore/storages/memory.py @@ -0,0 +1,187 @@ +import copy +import sys +from datetime import datetime +from typing import Dict, List, Optional + +from aw_core.models import Event + +from . import logger +from .abstract import AbstractStorage + + +class MemoryStorage(AbstractStorage): + """For storage of data in-memory, useful primarily in testing""" + + sid = "memory" + + def __init__(self, testing: bool) -> None: + self.logger = logger.getChild(self.sid) + # self.logger.warning("Using in-memory storage, any events stored will not be persistent and will be lost when server is shut down. Use the --storage parameter to set a different storage method.") + self.db: Dict[str, List[Event]] = {} + self._metadata: Dict[str, dict] = dict() + + def create_bucket( + self, + bucket_id, + type_id, + client, + hostname, + created, + name=None, + data=None, + ) -> None: + if not name: + name = bucket_id + self._metadata[bucket_id] = { + "id": bucket_id, + "name": name, + "type": type_id, + "client": client, + "hostname": hostname, + "created": created, + "data": data or {}, + } + self.db[bucket_id] = [] + + def update_bucket( + self, + bucket_id: str, + type_id: Optional[str] = None, + client: Optional[str] = None, + hostname: Optional[str] = None, + name: Optional[str] = None, + data: Optional[dict] = None, + ) -> None: + if bucket_id in self._metadata: + if type_id: + self._metadata[bucket_id]["type"] = type_id + if client: + self._metadata[bucket_id]["client"] = client + if hostname: + self._metadata[bucket_id]["hostname"] = hostname + if name: + self._metadata[bucket_id]["name"] = name + if data: + self._metadata[bucket_id]["data"] = data + else: + raise ValueError("Bucket did not exist, could not update") + + def delete_bucket(self, bucket_id: str) -> None: + if bucket_id in self.db: + del self.db[bucket_id] + if bucket_id in self._metadata: + del self._metadata[bucket_id] + else: + raise ValueError("Bucket did not exist, could not delete") + + def buckets(self): + buckets = dict() + for bucket_id in self.db: + buckets[bucket_id] = self.get_metadata(bucket_id) + return buckets + + def get_event( + self, + bucket_id: str, + event_id: int, + ) -> Optional[Event]: + event = self._get_event(bucket_id, event_id) + return copy.deepcopy(event) + + def get_events( + self, + bucket: str, + limit: int, + starttime: Optional[datetime] = None, + endtime: Optional[datetime] = None, + ) -> List[Event]: + events = self.db[bucket] + + # Sort by timestamp + events = sorted(events, key=lambda k: k["timestamp"])[::-1] + + # Filter by date + if starttime: + events = [e for e in events if starttime <= (e.timestamp + e.duration)] + if endtime: + events = [e for e in events if e.timestamp <= endtime] + + # Limit + if limit == 0: + return [] + elif limit < 0: + limit = sys.maxsize + events = events[:limit] + # Return + return copy.deepcopy(events) + + def get_eventcount( + self, + bucket: str, + starttime: Optional[datetime] = None, + endtime: Optional[datetime] = None, + ) -> int: + return len( + [ + e + for e in self.db[bucket] + if (not starttime or starttime <= e.timestamp) + and (not endtime or e.timestamp <= endtime) + ] + ) + + def get_metadata(self, bucket_id: str): + if bucket_id in self._metadata: + return self._metadata[bucket_id] + else: + raise ValueError("Bucket did not exist, could not get metadata") + + def insert_one(self, bucket: str, event: Event) -> Event: + if event.id is not None: + self.replace(bucket, event.id, event) + else: + # We need to copy the event to avoid setting the ID on the passed event + event = copy.copy(event) + if self.db[bucket]: + event.id = max(int(e.id or 0) for e in self.db[bucket]) + 1 + else: + event.id = 0 + self.db[bucket].append(event) + return event + + def delete(self, bucket_id, event_id): + for idx in ( + idx + for idx, event in reversed(list(enumerate(self.db[bucket_id]))) + if event.id == event_id + ): + self.db[bucket_id].pop(idx) + return True + return False + + def _get_event(self, bucket_id, event_id) -> Optional[Event]: + events = [ + event + for idx, event in reversed(list(enumerate(self.db[bucket_id]))) + if event.id == event_id + ] + if len(events) < 1: + return None + else: + return events[0] + + def replace(self, bucket_id, event_id, event): + for idx in ( + idx + for idx, event in reversed(list(enumerate(self.db[bucket_id]))) + if event.id == event_id + ): + # We need to copy the event to avoid setting the ID on the passed event + event = copy.copy(event) + event.id = event_id + self.db[bucket_id][idx] = event + + def replace_last(self, bucket_id, event): + # NOTE: This does not actually get the most recent event, only the last inserted + last = sorted(self.db[bucket_id], key=lambda e: e.timestamp)[-1] + self.replace(bucket_id, last.id, event) diff --git a/aw_datastore/storages/peewee.py b/aw_datastore/storages/peewee.py new file mode 100644 index 0000000..d114b9c --- /dev/null +++ b/aw_datastore/storages/peewee.py @@ -0,0 +1,451 @@ +import json +import logging +import os +from datetime import datetime, timedelta, timezone +from typing import ( + Any, + Dict, + List, + Optional, +) + +import iso8601 +from aw_core.dirs import get_data_dir, legacy_testing_suffix +from aw_core.models import Event +from playhouse.migrate import SqliteMigrator, migrate + +try: + from playhouse.sqlite_ext import SqliteExtDatabase +except ImportError: + # peewee 4 is missing SqliteExtDatabase + from peewee import SqliteDatabase as SqliteExtDatabase + +import peewee +from peewee import ( + AutoField, + CharField, + DateTimeField, + DecimalField, + ForeignKeyField, + IntegerField, + Model, +) + +from .abstract import AbstractStorage + +logger = logging.getLogger(__name__) + +# Prevent debug output from propagating +peewee_logger = logging.getLogger("peewee") +peewee_logger.setLevel(logging.INFO) + +# Init'd later in the PeeweeStorage constructor. +# See: http://docs.peewee-orm.com/en/latest/peewee/database.html#run-time-database-configuration +# Another option would be to use peewee's Proxy. +# See: http://docs.peewee-orm.com/en/latest/peewee/database.html#dynamic-db +_db = SqliteExtDatabase(None) + + +LATEST_VERSION = 2 + + +def auto_migrate(path: str) -> None: + db = SqliteExtDatabase(path) + migrator = SqliteMigrator(db) + + # check if bucketmodel has datastr field + info = db.execute_sql("PRAGMA table_info(bucketmodel)") + has_datastr = any(row[1] == "datastr" for row in info) + + if not has_datastr: + datastr_field = CharField(default="{}") + with db.atomic(): + migrate(migrator.add_column("bucketmodel", "datastr", datastr_field)) + + # Replace the single-column event indexes with a composite index. + # Every event query filters on bucket_id and orders by timestamp (or + # narrows to a timestamp range), so the composite index serves the seek, + # the range and the ORDER BY in one pass. With only single-column indexes + # SQLite reads all of a bucket's rows and sorts them in a temp B-tree, + # even for the LIMIT 1 last-event query issued on every heartbeat merge. + # Dropping the old indexes also makes inserts cheaper. + indexes = {row[1] for row in db.execute_sql("PRAGMA index_list(eventmodel)")} + if "eventmodel_bucket_id_timestamp" not in indexes: + logger.info( + "Migrating event indexes, this can take a few minutes on large databases..." + ) + # Drops run before the create so the pages they free are reused to + # build the new index within the same transaction; creating first + # would permanently grow the database file by the new index's size. + with db.atomic(): + db.execute_sql("DROP INDEX IF EXISTS eventmodel_bucket_id") + db.execute_sql("DROP INDEX IF EXISTS eventmodel_timestamp") + db.execute_sql( + "CREATE INDEX eventmodel_bucket_id_timestamp" + " ON eventmodel (bucket_id, timestamp)" + ) + + db.close() + + +def chunks(ls, n): + """Yield successive n-sized chunks from ls. + From: https://stackoverflow.com/a/312464/965332""" + for i in range(0, len(ls), n): + yield ls[i : i + n] + + +def dt_plus_duration(dt, duration): + # See peewee docs on datemath: https://docs.peewee-orm.com/en/latest/peewee/hacks.html#date-math + return peewee.fn.strftime( + "%Y-%m-%d %H:%M:%f+00:00", + (peewee.fn.julianday(dt) - 2440587.5) * 86400.0 + duration, + "unixepoch", + ) + + +class BaseModel(Model): + class Meta: + database = _db + + +class BucketModel(BaseModel): + key = IntegerField(primary_key=True) + id = CharField(unique=True) + created = DateTimeField(default=lambda: datetime.now(timezone.utc)) + name = CharField(null=True) + type = CharField() + client = CharField() + hostname = CharField() + datastr = CharField(null=True) # JSON-encoded object + + def json(self): + return { + "id": self.id, + "created": iso8601.parse_date(self.created) + .astimezone(timezone.utc) + .isoformat(), + "name": self.name, + "type": self.type, + "client": self.client, + "hostname": self.hostname, + "data": json.loads(self.datastr) if self.datastr else {}, + } + + +class EventModel(BaseModel): + # Events are covered by the composite (bucket_id, timestamp) index created + # in auto_migrate(); it is kept out of the model so create_table doesn't + # build it before auto_migrate has dropped the old single-column indexes. + id = AutoField() + bucket = ForeignKeyField(BucketModel, backref="events", index=False) + timestamp = DateTimeField(default=lambda: datetime.now(timezone.utc)) + duration = DecimalField() + datastr = CharField() + + @classmethod + def from_event(cls, bucket_key, event: Event): + return cls( + bucket=bucket_key, + id=event.id, + timestamp=event.timestamp, + duration=event.duration.total_seconds(), + datastr=json.dumps(event.data), + ) + + def json(self): + return { + "id": self.id, + "timestamp": self.timestamp, + "duration": float(self.duration), + "data": json.loads(self.datastr), + } + + +class PeeweeStorage(AbstractStorage): + sid = "peewee" + + def __init__(self, testing: bool = True, filepath: Optional[str] = None) -> None: + data_dir = get_data_dir("aw-server") + + if not filepath: + filename = ( + "peewee-sqlite" + + legacy_testing_suffix(testing) + + f".v{LATEST_VERSION}" + + ".db" + ) + filepath = os.path.join(data_dir, filename) + self.db = _db + self.db.init( + filepath, + pragmas={ + "journal_mode": "wal", + }, + ) + logger.info(f"Using database file: {filepath}") + self.db.connect() + + self.bucket_keys: Dict[str, int] = {} + BucketModel.create_table(safe=True) + EventModel.create_table(safe=True) + + # Migrate database if needed, requires closing the connection first + self.db.close() + auto_migrate(filepath) + self.db.connect() + + # Update bucket keys + self.update_bucket_keys() + + def update_bucket_keys(self) -> None: + buckets = BucketModel.select() + self.bucket_keys = {bucket.id: bucket.key for bucket in buckets} + + def buckets(self) -> Dict[str, Dict[str, Any]]: + return {bucket.id: bucket.json() for bucket in BucketModel.select()} + + def create_bucket( + self, + bucket_id: str, + type_id: str, + client: str, + hostname: str, + created: str, + name: Optional[str] = None, + data: Optional[Dict[str, Any]] = None, + ): + BucketModel.create( + id=bucket_id, + type=type_id, + client=client, + hostname=hostname, + created=created, + name=name, + datastr=json.dumps(data or {}), + ) + self.update_bucket_keys() + + def update_bucket( + self, + bucket_id: str, + type_id: Optional[str] = None, + client: Optional[str] = None, + hostname: Optional[str] = None, + name: Optional[str] = None, + data: Optional[dict] = None, + ) -> None: + if bucket_id in self.bucket_keys: + bucket = BucketModel.get(BucketModel.key == self.bucket_keys[bucket_id]) + + if type_id is not None: + bucket.type = type_id + if client is not None: + bucket.client = client + if hostname is not None: + bucket.hostname = hostname + if name is not None: + bucket.name = name + if data is not None: + bucket.datastr = json.dumps(data) # Encoding data dictionary to JSON + + bucket.save() + else: + raise ValueError("Bucket did not exist, could not update") + + def delete_bucket(self, bucket_id: str) -> None: + if bucket_id in self.bucket_keys: + EventModel.delete().where( + EventModel.bucket == self.bucket_keys[bucket_id] + ).execute() + BucketModel.delete().where( + BucketModel.key == self.bucket_keys[bucket_id] + ).execute() + self.update_bucket_keys() + else: + raise ValueError("Bucket did not exist, could not delete") + + def get_metadata(self, bucket_id: str): + if bucket_id in self.bucket_keys: + bucket = BucketModel.get( + BucketModel.key == self.bucket_keys[bucket_id] + ).json() + return bucket + else: + raise ValueError("Bucket did not exist, could not get metadata") + + def insert_one(self, bucket_id: str, event: Event) -> Event: + e = EventModel.from_event(self.bucket_keys[bucket_id], event) + e.save() + event.id = e.id + return event + + def insert_many(self, bucket_id, events: List[Event]) -> None: + # NOTE: Events need to be handled differently depending on + # if they're upserts or inserts (have id's or not). + + # These events are updates which need to be applied one by one + events_updates = [e for e in events if e.id is not None] + for e in events_updates: + self.insert_one(bucket_id, e) + + # These events can be inserted with insert_many + events_dictlist = [ + { + "bucket": self.bucket_keys[bucket_id], + "timestamp": event.timestamp, + "duration": event.duration.total_seconds(), + "datastr": json.dumps(event.data), + } + for event in events + if event.id is None + ] + + # Chunking into lists of length 100 is needed here due to SQLITE_MAX_COMPOUND_SELECT + # and SQLITE_LIMIT_VARIABLE_NUMBER under Windows. + # See: https://github.com/coleifer/peewee/issues/948 + for chunk in chunks(events_dictlist, 100): + EventModel.insert_many(chunk).execute() + + def _get_event(self, bucket_id, event_id) -> Optional[EventModel]: + try: + return ( + EventModel.select() + .where(EventModel.id == event_id) + .where(EventModel.bucket == self.bucket_keys[bucket_id]) + .get() + ) + except peewee.DoesNotExist: + return None + + def _get_last(self, bucket_id) -> EventModel: + return ( + EventModel.select() + .where(EventModel.bucket == self.bucket_keys[bucket_id]) + .order_by(EventModel.timestamp.desc()) + .get() + ) + + def replace_last(self, bucket_id, event): + e = self._get_last(bucket_id) + e.timestamp = event.timestamp + e.duration = event.duration.total_seconds() + e.datastr = json.dumps(event.data) + e.save() + event.id = e.id + return event + + def delete(self, bucket_id, event_id): + return ( + EventModel.delete() + .where(EventModel.id == event_id) + .where(EventModel.bucket == self.bucket_keys[bucket_id]) + .execute() + ) + + def replace(self, bucket_id, event_id, event): + e = self._get_event(bucket_id, event_id) + e.timestamp = event.timestamp + e.duration = event.duration.total_seconds() + e.datastr = json.dumps(event.data) + e.save() + event.id = e.id + return event + + def get_event( + self, + bucket_id: str, + event_id: int, + ) -> Optional[Event]: + """ + Fetch a single event from a bucket. + """ + res = self._get_event(bucket_id, event_id) + return Event(**EventModel.json(res)) if res else None + + def get_events( + self, + bucket_id: str, + limit: int, + starttime: Optional[datetime] = None, + endtime: Optional[datetime] = None, + ): + """ + Fetch events from a certain bucket, optionally from a given range of time. + + Example raw query: + + SELECT strftime( + "%Y-%m-%d %H:%M:%f+00:00", + ((julianday(timestamp) - 2440587.5) * 86400), + 'unixepoch' + ) + FROM eventmodel + WHERE eventmodel.timestamp > '2021-06-20' + LIMIT 10; + + """ + if limit == 0: + return [] + q = ( + EventModel.select() + .where(EventModel.bucket == self.bucket_keys[bucket_id]) + .order_by(EventModel.timestamp.desc()) + .limit(limit) + ) + + q = self._where_range(q, starttime, endtime) + + res = q.execute() + events = [Event(**e) for e in list(map(EventModel.json, res))] + + # Trim events that are out of range (as done in aw-server-rust) + # TODO: Do the same for the other storage methods + for e in events: + if starttime: + if e.timestamp < starttime: + e_end = e.timestamp + e.duration + e.timestamp = starttime + e.duration = e_end - e.timestamp + if endtime: + if e.timestamp + e.duration > endtime: + e.duration = endtime - e.timestamp + + return events + + def get_eventcount( + self, + bucket_id: str, + starttime: Optional[datetime] = None, + endtime: Optional[datetime] = None, + ) -> int: + q = EventModel.select().where(EventModel.bucket == self.bucket_keys[bucket_id]) + q = self._where_range(q, starttime, endtime) + return q.count() + + def _where_range( + self, + q, + starttime: Optional[datetime] = None, + endtime: Optional[datetime] = None, + ): + # Important to normalize datetimes to UTC, otherwise any UTC offset will be ignored + if starttime: + starttime = starttime.astimezone(timezone.utc) + if endtime: + endtime = endtime.astimezone(timezone.utc) + + if starttime: + # Faster WHERE to speed up slow query below, leads to ~2-3x speedup + # We'll assume events aren't >24h + q = q.where(starttime - timedelta(hours=24) <= EventModel.timestamp) + + # This can be slow on large databases... + # Tried creating various indexes and using SQLite's unlikely() function, but it had no effect + q = q.where( + starttime <= dt_plus_duration(EventModel.timestamp, EventModel.duration) + ) + if endtime: + q = q.where(EventModel.timestamp <= endtime) + + return q diff --git a/aw_datastore/storages/sqlite.py b/aw_datastore/storages/sqlite.py new file mode 100644 index 0000000..f139697 --- /dev/null +++ b/aw_datastore/storages/sqlite.py @@ -0,0 +1,378 @@ +import json +import logging +import os +import sqlite3 +from datetime import datetime, timedelta, timezone +from typing import Iterable, List, Optional + +from aw_core.dirs import get_data_dir, legacy_testing_suffix +from aw_core.models import Event + +from .abstract import AbstractStorage + +logger = logging.getLogger(__name__) + +LATEST_VERSION = 1 + +# The max integer value in SQLite is signed 8 Bytes / 64 bits +MAX_TIMESTAMP = 2**63 - 1 + +CREATE_BUCKETS_TABLE = """ + CREATE TABLE IF NOT EXISTS buckets ( + rowid INTEGER PRIMARY KEY AUTOINCREMENT, + id TEXT UNIQUE NOT NULL, + name TEXT, + type TEXT NOT NULL, + client TEXT NOT NULL, + hostname TEXT NOT NULL, + created TEXT NOT NULL, + datastr TEXT NOT NULL + ) +""" + +CREATE_EVENTS_TABLE = """ + CREATE TABLE IF NOT EXISTS events ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + bucketrow INTEGER NOT NULL, + starttime INTEGER NOT NULL, + endtime INTEGER NOT NULL, + datastr TEXT NOT NULL, + FOREIGN KEY (bucketrow) REFERENCES buckets(rowid) + ) +""" + +INDEX_BUCKETS_TABLE_ID = """ + CREATE INDEX IF NOT EXISTS event_index_id ON events(id); +""" + +INDEX_EVENTS_TABLE_STARTTIME = """ + CREATE INDEX IF NOT EXISTS event_index_starttime ON events(bucketrow, starttime); +""" +INDEX_EVENTS_TABLE_ENDTIME = """ + CREATE INDEX IF NOT EXISTS event_index_endtime ON events(bucketrow, endtime); +""" + + +def _rows_to_events(rows: Iterable) -> List[Event]: + events = [] + for row in rows: + eid = row[0] + starttime = datetime.fromtimestamp(row[1] / 1000000, timezone.utc) + endtime = datetime.fromtimestamp(row[2] / 1000000, timezone.utc) + duration = endtime - starttime + data = json.loads(row[3]) + events.append(Event(id=eid, timestamp=starttime, duration=duration, data=data)) + return events + + +class SqliteStorage(AbstractStorage): + sid = "sqlite" + + def __init__( + self, testing, filepath: Optional[str] = None, enable_lazy_commit=True + ) -> None: + self.testing = testing + self.enable_lazy_commit = enable_lazy_commit + + # Ignore the migration check if custom filepath is set + ignore_migration_check = filepath is not None + + ds_name = self.sid + legacy_testing_suffix(testing) + if not filepath: + data_dir = get_data_dir("aw-server") + filename = ds_name + f".v{LATEST_VERSION}" + ".db" + filepath = os.path.join(data_dir, filename) + + new_db_file = not os.path.exists(filepath) + self.conn = sqlite3.connect(filepath) + logger.info(f"Using database file: {filepath}") + + # Create tables + self.conn.execute(CREATE_BUCKETS_TABLE) + self.conn.execute(CREATE_EVENTS_TABLE) + self.conn.execute(INDEX_BUCKETS_TABLE_ID) + self.conn.execute(INDEX_EVENTS_TABLE_STARTTIME) + self.conn.execute(INDEX_EVENTS_TABLE_ENDTIME) + self.conn.execute("PRAGMA journal_mode=WAL;") + self.commit() + + if new_db_file and not ignore_migration_check: + logger.info("Created new SQlite db file") + + from aw_datastore import check_for_migration # fmt: skip + + check_for_migration(self) + + self.last_commit = datetime.now() + self.num_uncommitted_statements = 0 + + def commit(self): + """ + Useful for debugging and trying to lower the amount of + unnecessary commits + """ + self.conn.commit() + self.last_commit = datetime.now() + self.num_uncommitted_statements = 0 + + def conditional_commit(self, num_statements): + """ + Only commit transactions if: + - We have a lot of statements in our transaction + - Was a while ago since last commit + This is because sqlite is very slow with small inserts, this + is a way to batch them together and lower CPU+disk usage + """ + if self.enable_lazy_commit: + self.num_uncommitted_statements += num_statements + if self.num_uncommitted_statements > 50: + self.commit() + if (self.last_commit - datetime.now()) > timedelta(seconds=10): + self.commit() + else: + self.commit() + + def buckets(self): + buckets = {} + c = self.conn.cursor() + for row in c.execute( + "SELECT id, name, type, client, hostname, created, datastr FROM buckets" + ): + buckets[row[0]] = { + "id": row[0], + "name": row[1], + "type": row[2], + "client": row[3], + "hostname": row[4], + "created": row[5], + "data": json.loads(row[6] or "{}"), + } + return buckets + + def create_bucket( + self, + bucket_id: str, + type_id: str, + client: str, + hostname: str, + created: str, + name: Optional[str] = None, + data: Optional[dict] = None, + ): + self.conn.execute( + "INSERT INTO buckets(id, name, type, client, hostname, created, datastr) " + + "VALUES (?, ?, ?, ?, ?, ?, ?)", + [ + bucket_id, + name, + type_id, + client, + hostname, + created, + json.dumps(data or {}), + ], + ) + self.commit() + return self.get_metadata(bucket_id) + + def update_bucket( + self, + bucket_id: str, + type_id: Optional[str] = None, + client: Optional[str] = None, + hostname: Optional[str] = None, + name: Optional[str] = None, + data: Optional[dict] = None, + ): + update_values = [ + ("type", type_id), + ("client", client), + ("hostname", hostname), + ("name", name), + ("datastr", json.dumps(data) if data is not None else None), + ] + updates, values = zip(*[(k, v) for k, v in update_values if v is not None]) + if not updates: + raise ValueError("At least one field must be updated.") + + sql = ( + "UPDATE buckets SET " + + ", ".join(f"{u} = ?" for u in updates) + + " WHERE id = ?" + ) + self.conn.execute(sql, (*values, bucket_id)) + self.commit() + return self.get_metadata(bucket_id) + + def delete_bucket(self, bucket_id: str): + self.conn.execute( + "DELETE FROM events WHERE bucketrow IN (SELECT rowid FROM buckets WHERE id = ?)", + [bucket_id], + ) + cursor = self.conn.execute("DELETE FROM buckets WHERE id = ?", [bucket_id]) + self.commit() + if cursor.rowcount != 1: + raise ValueError("Bucket did not exist, could not delete") + + def get_metadata(self, bucket_id: str): + c = self.conn.cursor() + res = c.execute( + "SELECT id, name, type, client, hostname, created, datastr FROM buckets WHERE id = ?", + [bucket_id], + ) + row = res.fetchone() + if row is not None: + return { + "id": row[0], + "name": row[1], + "type": row[2], + "client": row[3], + "hostname": row[4], + "created": row[5], + "data": json.loads(row[6] or "{}"), + } + else: + raise ValueError("Bucket did not exist, could not get metadata") + + def insert_one(self, bucket_id: str, event: Event) -> Event: + c = self.conn.cursor() + starttime = event.timestamp.timestamp() * 1000000 + endtime = starttime + (event.duration.total_seconds() * 1000000) + datastr = json.dumps(event.data) + c.execute( + "INSERT INTO events(bucketrow, starttime, endtime, datastr) " + + "VALUES ((SELECT rowid FROM buckets WHERE id = ?), ?, ?, ?)", + [bucket_id, starttime, endtime, datastr], + ) + event.id = c.lastrowid + self.conditional_commit(1) + return event + + def insert_many(self, bucket_id, events: List[Event]) -> None: + # FIXME: Is this true not only for peewee but sqlite aswell? + # Chunking into lists of length 100 is needed here due to SQLITE_MAX_COMPOUND_SELECT + # and SQLITE_LIMIT_VARIABLE_NUMBER under Windows. + # See: https://github.com/coleifer/peewee/issues/948 + + # First, upsert events with id's set + events_upsert = [e for e in events if e.id is not None] + for e in events_upsert: + self.replace(bucket_id, e.id, e) + + # Then insert events without id's set + events_insert = [e for e in events if e.id is None] + event_rows = [] + for event in events_insert: + starttime = event.timestamp.timestamp() * 1000000 + endtime = starttime + (event.duration.total_seconds() * 1000000) + datastr = json.dumps(event.data) + event_rows.append((bucket_id, starttime, endtime, datastr)) + query = ( + "INSERT INTO events(bucketrow, starttime, endtime, datastr) " + + "VALUES ((SELECT rowid FROM buckets WHERE id = ?), ?, ?, ?)" + ) + self.conn.executemany(query, event_rows) + self.conditional_commit(len(event_rows)) + + def replace_last(self, bucket_id, event): + starttime = event.timestamp.timestamp() * 1000000 + endtime = starttime + (event.duration.total_seconds() * 1000000) + datastr = json.dumps(event.data) + query = """UPDATE events + SET starttime = ?, endtime = ?, datastr = ? + WHERE id = ( + SELECT id FROM events WHERE endtime = + (SELECT max(endtime) FROM events WHERE bucketrow = + (SELECT rowid FROM buckets WHERE id = ?) LIMIT 1))""" + self.conn.execute(query, [starttime, endtime, datastr, bucket_id]) + self.conditional_commit(1) + return True + + def delete(self, bucket_id, event_id): + query = ( + "DELETE FROM events " + + "WHERE id = ? AND bucketrow = (SELECT b.rowid FROM buckets b WHERE b.id = ?)" + ) + cursor = self.conn.execute(query, [event_id, bucket_id]) + return cursor.rowcount == 1 + + def replace(self, bucket_id, event_id, event) -> bool: + starttime = event.timestamp.timestamp() * 1000000 + endtime = starttime + (event.duration.total_seconds() * 1000000) + datastr = json.dumps(event.data) + query = """UPDATE events + SET bucketrow = (SELECT rowid FROM buckets WHERE id = ?), + starttime = ?, + endtime = ?, + datastr = ? + WHERE id = ?""" + self.conn.execute(query, [bucket_id, starttime, endtime, datastr, event_id]) + self.conditional_commit(1) + return True + + def get_event( + self, + bucket_id: str, + event_id: int, + ) -> Optional[Event]: + self.commit() + c = self.conn.cursor() + query = """ + SELECT id, starttime, endtime, datastr + FROM events + WHERE bucketrow = (SELECT rowid FROM buckets WHERE id = ?) AND id = ? + LIMIT 1 + """ + rows = c.execute(query, [bucket_id, event_id]) + events = _rows_to_events(rows) + if events: + return events[0] + else: + return None + + def get_events( + self, + bucket_id: str, + limit: int, + starttime: Optional[datetime] = None, + endtime: Optional[datetime] = None, + ): + if limit == 0: + return [] + elif limit < 0: + limit = -1 + self.commit() + c = self.conn.cursor() + starttime_i = starttime.timestamp() * 1000000 if starttime else 0 + endtime_i = endtime.timestamp() * 1000000 if endtime else MAX_TIMESTAMP + query = """ + SELECT id, starttime, endtime, datastr + FROM events + WHERE bucketrow = (SELECT rowid FROM buckets WHERE id = ?) + AND endtime >= ? AND starttime <= ? + ORDER BY endtime DESC LIMIT ? + """ + rows = c.execute(query, [bucket_id, starttime_i, endtime_i, limit]) + events = _rows_to_events(rows) + return events + + def get_eventcount( + self, + bucket_id: str, + starttime: Optional[datetime] = None, + endtime: Optional[datetime] = None, + ): + self.commit() + c = self.conn.cursor() + starttime_i = starttime.timestamp() * 1000000 if starttime else 0 + endtime_i = endtime.timestamp() * 1000000 if endtime else MAX_TIMESTAMP + query = ( + "SELECT count(*) " + + "FROM events " + + "WHERE bucketrow = (SELECT rowid FROM buckets WHERE id = ?) " + + "AND endtime >= ? AND starttime <= ?" + ) + rows = c.execute(query, [bucket_id, starttime_i, endtime_i]) + row = rows.fetchone() + eventcount = row[0] + return eventcount diff --git a/aw_query/__init__.py b/aw_query/__init__.py new file mode 100644 index 0000000..4d42d80 --- /dev/null +++ b/aw_query/__init__.py @@ -0,0 +1,3 @@ +from .query2 import query + +__all__ = ["query"] diff --git a/aw_query/exceptions.py b/aw_query/exceptions.py new file mode 100644 index 0000000..aeb6e25 --- /dev/null +++ b/aw_query/exceptions.py @@ -0,0 +1,14 @@ +class QueryException(Exception): + pass + + +class QueryFunctionException(QueryException): + pass + + +class QueryParseException(QueryException): + pass + + +class QueryInterpretException(QueryException): + pass diff --git a/aw_query/functions.py b/aw_query/functions.py new file mode 100644 index 0000000..00a85fc --- /dev/null +++ b/aw_query/functions.py @@ -0,0 +1,350 @@ +from datetime import timedelta +from functools import wraps +from inspect import signature +from typing import ( + Any, + Callable, + Dict, + List, + Optional, +) + +import iso8601 +from aw_core.models import Event +from aw_datastore import Datastore +from aw_transform import ( + Rule, + categorize, + chunk_events_by_key, + concat, + filter_keyvals, + filter_keyvals_regex, + filter_period_intersect, + flood, + limit_events, + merge_events_by_keys, + merge_subwatcher_fields, + period_union, + simplify_string, + sort_by_duration, + sort_by_timestamp, + split_url_events, + sum_durations, + tag, + union_no_overlap, +) + +from .exceptions import QueryFunctionException + + +def _verify_bucket_exists(datastore, bucketname): + if bucketname in datastore.buckets(): + return + else: + raise QueryFunctionException(f"There's no bucket named '{bucketname}'") + + +def _verify_variable_is_type(variable, t): + if not isinstance(variable, t): + raise QueryFunctionException( + f"Variable '{variable}' passed to function call is of invalid type. Expected {t} but was {type(variable)}" + ) + + +# TODO: proper type checking (typecheck-decorator in pypi?) + + +TNamespace = Dict[str, Any] +TQueryFunction = Callable[..., Any] + + +""" + Declarations +""" +functions: Dict[str, TQueryFunction] = {} + + +def q2_function(transform_func=None): + """ + Decorator used to register query functions. + + Automatically adds mock arguments for Datastore and TNamespace + if not in function signature. + """ + + def h(f): + sig = signature(f) + # If function lacks docstring, use docstring from underlying function in aw_transform + if transform_func and transform_func.__doc__ and not f.__doc__: + f.__doc__ = f".. note:: Documentation automatically copied from underlying function `aw_transform.{transform_func.__name__}`\n\n{transform_func.__doc__}" + + @wraps(f) + def g(datastore: Datastore, namespace: TNamespace, *args, **kwargs): + # Remove datastore and namespace argument for functions that don't need it + args = (datastore, namespace, *args) + if TNamespace not in (sig.parameters[p].annotation for p in sig.parameters): + args = (args[0], *args[2:]) + if Datastore not in (sig.parameters[p].annotation for p in sig.parameters): + args = args[1:] + return f(*args, **kwargs) + + fname = f.__name__ + if fname[:3] == "q2_": + fname = fname[3:] + functions[fname] = g + return g + + return h + + +def q2_typecheck(f): + """Decorator that typechecks using `_verify_variable_is_type`""" + sig = signature(f) + + @wraps(f) + def g(*args, **kwargs): + # FIXME: If the first argument passed to a query2 function is a straight [] then the second argument disappears from the argument list for unknown reasons, which breaks things + for i, p in enumerate(sig.parameters): + param = sig.parameters[p] + + # print(f"Checking that param ({param}) was {param.annotation}, value: {args[i]}") + # FIXME: Won't check keyword arguments + if ( + param.annotation in [list, str, int, float] + and param.default == param.empty + ): + _verify_variable_is_type(args[i], param.annotation) + + return f(*args, **kwargs) + + return g + + +""" + Getting buckets +""" + + +@q2_function() +@q2_typecheck +def q2_find_bucket( + datastore: Datastore, filter_str: str, hostname: Optional[str] = None +): + """Find bucket by using a filter_str (to avoid hardcoding bucket names)""" + for bucket in datastore.buckets(): + if filter_str in bucket: + bucket_metadata = datastore[bucket].metadata() + if hostname: + if bucket_metadata["hostname"] == hostname: + return bucket + else: + return bucket + raise QueryFunctionException( + f"Unable to find bucket matching '{filter_str}' (hostname filter set to '{hostname}')" + ) + + +""" + Data gathering functions +""" + + +@q2_function() +@q2_typecheck +def q2_query_bucket( + datastore: Datastore, namespace: TNamespace, bucketname: str +) -> List[Event]: + _verify_bucket_exists(datastore, bucketname) + try: + starttime = iso8601.parse_date(namespace["STARTTIME"]) + endtime = iso8601.parse_date(namespace["ENDTIME"]) + except iso8601.ParseError: + raise QueryFunctionException( + "Unable to parse starttime/endtime for query_bucket" + ) from None + return datastore[bucketname].get(starttime=starttime, endtime=endtime) + + +@q2_function() +@q2_typecheck +def q2_query_bucket_eventcount( + datastore: Datastore, namespace: TNamespace, bucketname: str +) -> int: + _verify_bucket_exists(datastore, bucketname) + starttime = iso8601.parse_date(namespace["STARTTIME"]) + endtime = iso8601.parse_date(namespace["ENDTIME"]) + return datastore[bucketname].get_eventcount(starttime=starttime, endtime=endtime) + + +""" + Filtering functions +""" + + +@q2_function(filter_keyvals) +@q2_typecheck +def q2_filter_keyvals(events: list, key: str, vals: list) -> List[Event]: + return filter_keyvals(events, key, vals, False) + + +@q2_function(filter_keyvals) +@q2_typecheck +def q2_exclude_keyvals(events: list, key: str, vals: list) -> List[Event]: + return filter_keyvals(events, key, vals, True) + + +@q2_function(filter_keyvals_regex) +@q2_typecheck +def q2_filter_keyvals_regex(events: list, key: str, regex: str) -> List[Event]: + return filter_keyvals_regex(events, key, regex) + + +@q2_function(filter_period_intersect) +@q2_typecheck +def q2_filter_period_intersect(events: list, filterevents: list) -> List[Event]: + return filter_period_intersect(events, filterevents) + + +@q2_function(period_union) +@q2_typecheck +def q2_period_union(events1: list, events2: list) -> List[Event]: + return period_union(events1, events2) + + +@q2_function(limit_events) +@q2_typecheck +def q2_limit_events(events: list, count: int) -> List[Event]: + return limit_events(events, count) + + +""" + Merge functions +""" + + +@q2_function(merge_events_by_keys) +@q2_typecheck +def q2_merge_events_by_keys(events: list, keys: list) -> List[Event]: + return merge_events_by_keys(events, keys) + + +@q2_function(merge_subwatcher_fields) +@q2_typecheck +def q2_merge_subwatcher_fields( + base_events: list, subwatcher_events: list, keys: list, conflict: str = "base_wins" +) -> List[Event]: + try: + return merge_subwatcher_fields(base_events, subwatcher_events, keys, conflict) + except ValueError as exc: + raise QueryFunctionException(str(exc)) from None + + +@q2_function(chunk_events_by_key) +@q2_typecheck +def q2_chunk_events_by_key(events: list, key: str) -> List[Event]: + return chunk_events_by_key(events, key) + + +""" + Sort functions +""" + + +@q2_function(sort_by_timestamp) +@q2_typecheck +def q2_sort_by_timestamp(events: list) -> List[Event]: + return sort_by_timestamp(events) + + +@q2_function(sort_by_duration) +@q2_typecheck +def q2_sort_by_duration(events: list) -> List[Event]: + return sort_by_duration(events) + + +""" + Summarizing functions +""" + + +@q2_function(sum_durations) +@q2_typecheck +def q2_sum_durations(events: list) -> timedelta: + return sum_durations(events) + + +@q2_function(concat) +@q2_typecheck +def q2_concat(events1: list, events2: list) -> List[Event]: + return concat(events1, events2) + + +@q2_function(union_no_overlap) +@q2_typecheck +def q2_union_no_overlap(events1: list, events2: list) -> List[Event]: + return union_no_overlap(events1, events2) + + +""" + Flood functions +""" + + +@q2_function(flood) +@q2_typecheck +def q2_flood(events: list, pulsetime: float = 5) -> List[Event]: + return flood(events, pulsetime) + + +""" + Watcher specific functions +""" + + +@q2_function(split_url_events) +@q2_typecheck +def q2_split_url_events(events: list) -> List[Event]: + return split_url_events(events) + + +@q2_function(simplify_string) +@q2_typecheck +def q2_simplify_window_titles(events: list, key: str) -> List[Event]: + return simplify_string(events, key=key) + + +""" + Test functions +""" + + +@q2_function() +@q2_typecheck +def q2_nop(): + """No operation function for unittesting""" + return 1 + + +""" + Classify +""" + + +@q2_function(categorize) +@q2_typecheck +def q2_categorize(events: list, classes: list): + try: + classes = [(_cls, Rule(rule_dict)) for _cls, rule_dict in classes] + except ValueError as exc: + raise QueryFunctionException(str(exc)) from None + return categorize(events, classes) + + +@q2_function(tag) +@q2_typecheck +def q2_tag(events: list, classes: list): + try: + classes = [(_cls, Rule(rule_dict)) for _cls, rule_dict in classes] + except ValueError as exc: + raise QueryFunctionException(str(exc)) from None + return tag(events, classes) diff --git a/aw_query/py.typed b/aw_query/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/aw_query/query2.py b/aw_query/query2.py new file mode 100644 index 0000000..87f8edb --- /dev/null +++ b/aw_query/query2.py @@ -0,0 +1,448 @@ +import logging +from abc import abstractmethod +from datetime import datetime +from typing import ( + Any, + Dict, + List, + Optional, + Sequence, + Tuple, + Type, +) + +from aw_datastore import Datastore + +from .exceptions import QueryInterpretException, QueryParseException +from .functions import functions + +logger = logging.getLogger(__name__) + + +class QToken: + @abstractmethod + def interpret(self, datastore: Datastore, namespace: dict): + raise NotImplementedError + + @staticmethod + @abstractmethod + def parse(string: str, namespace: dict): + raise NotImplementedError + + @staticmethod + @abstractmethod + def check(string: str) -> Tuple[Optional[str], str]: + raise NotImplementedError + + +class QInteger(QToken): + def __init__(self, value) -> None: + self.value = value + + def interpret(self, datastore: Datastore, namespace: dict): + return self.value + + @staticmethod + def parse(string: str, namespace: dict) -> QToken: + return QInteger(int(string)) + + @staticmethod + def check(string: str): + token = "" + for char in string: + if char.isdigit(): + token += char + else: + break + return token, string[len(token) :] + + +class QVariable(QToken): + def __init__(self, name, value) -> None: + self.name = name + self.value = value + + def interpret(self, datastore: Datastore, namespace: dict): + if self.name not in namespace: + raise QueryInterpretException( + f"Tried to reference variable '{self.name}' which is not defined" + ) + namespace[self.name] = self.value + return self.value + + @staticmethod + def parse(string: str, namespace: dict) -> QToken: + val = None + if string in namespace: + val = namespace[string] + return QVariable(string, val) + + @staticmethod + def check(string: str): + token = "" + for i, char in enumerate(string): + if char.isalpha() or char == "_": + token += char + elif i != 0 and char.isdigit(): + token += char + else: + break + return token, string[len(token) :] + + +class QString(QToken): + def __init__(self, value): + self.value = value + + def interpret(self, datastore: Datastore, namespace: dict): + return self.value + + @staticmethod + def parse(string: str, namespace: dict) -> "QString": + quotes_type = string[0] + string = string.replace("\\" + quotes_type, quotes_type) + string = string[1:-1] + return QString(string) + + @staticmethod + def check(string: str): + token = "" + quotes_type = string[0] + if quotes_type != '"' and quotes_type != "'": + return token, string + token += quotes_type + prev_char = None + for char in string[1:]: + token += char + if ( + char == quotes_type and prev_char != "\\" + ): # escape quote_type with backslash + break + prev_char = char + if token[-1] != quotes_type or len(token) < 2: + # Unclosed string? + raise QueryParseException("Failed to parse string") + return token, string[len(token) :] + + +class QFunction(QToken): + def __init__(self, name, args): + self.name = name + self.args = args + + def interpret(self, datastore: Datastore, namespace: dict): + if self.name not in functions: + raise QueryInterpretException( + f"Tried to call function '{self.name}' which doesn't exist" + ) + call_args = [datastore, namespace] + for arg in self.args: + call_args.append(arg.interpret(datastore, namespace)) + # logger.debug("Arguments for functioncall to {} is {}".format(self.name, call_args)) + try: + result = functions[self.name](*call_args) # type: ignore + except TypeError: + raise QueryInterpretException( + f"Tried to call function {self.name} with invalid amount of arguments" + ) from None + return result + + @staticmethod + def parse(string: str, namespace: dict) -> QToken: + arg_start = 0 + arg_end = len(string) - 1 + # Find opening bracket + for char in string: + if char == "(": + break + arg_start = arg_start + 1 + # Parse name + name = string[:arg_start] + # Parse arguments + args = [] + args_str = string[arg_start + 1 : arg_end] + while args_str: + (arg_t, arg), args_str = _parse_token(args_str, namespace) + comma = args_str.find(",") + if comma != -1: + args_str = args_str[comma + 1 :] + args.append(arg_t.parse(arg, namespace)) + return QFunction(name, args) + + @staticmethod + def check(string: str): + i = 0 + # Find opening bracket + found = False + for char in string: + if char.isalpha() or char == "_": + i = i + 1 + elif i != 0 and char.isdigit(): + i = i + 1 + elif char == "(": + i = i + 1 + found = True + break + else: + break + if not found: + return None, string + to_consume = 1 + single_quote = False + double_quote = False + prev_char = None + for char in string[i:]: + i = i + 1 + if char == "'" and prev_char != "\\" and not double_quote: + single_quote = not single_quote + elif char == '"' and prev_char != "\\" and not single_quote: + double_quote = not double_quote + elif single_quote or double_quote: + pass + elif i != 0 and char.isdigit(): + pass + elif char == "(": + to_consume += 1 + elif char == ")": + to_consume -= 1 + if to_consume == 0: + break + prev_char = char + if to_consume != 0: + return None, string + return string[:i], string[i + 1 :] + + +class QDict(QToken): + def __init__(self, value: dict) -> None: + self.value = value + + def interpret(self, datastore: Datastore, namespace: dict): + expanded_dict = {} + for key, value in self.value.items(): + expanded_dict[key] = value.interpret(datastore, namespace) + return expanded_dict + + @staticmethod + def parse(string: str, namespace: dict) -> QToken: + entries_str = string[1:-1] + d: Dict[str, QToken] = {} + while len(entries_str) > 0: + entries_str = entries_str.strip() + if len(d) > 0 and entries_str[0] == ",": + entries_str = entries_str[1:] + # parse key + (key_t, key_str), entries_str = _parse_token(entries_str, namespace) + if key_t != QString: + raise QueryParseException("Key in dict is not a str") + key = QString.parse(key_str, {}).value + entries_str = entries_str.strip() + # Remove : + if entries_str[0] != ":": + raise QueryParseException("Key in dict is not followed by a :") + entries_str = entries_str[1:] + # parse val + (val_t, val_str), entries_str = _parse_token(entries_str, namespace) + if not val_t: + raise QueryParseException("Dict expected a value, got nothing") + val = val_t.parse(val_str, namespace) + # set + d[key] = val + return QDict(d) + + @staticmethod + def check(string: str): + if string[0] != "{": + return None, string + # Find closing bracket + i = 1 + to_consume = 1 + single_quote = False + double_quote = False + prev_char = None + for char in string[i:]: + i += 1 + if char == "'" and prev_char != "\\" and not double_quote: + single_quote = not single_quote + elif char == '"' and prev_char != "\\" and not single_quote: + double_quote = not double_quote + elif single_quote or double_quote: + pass + elif char == "}": + to_consume = to_consume - 1 + elif char == "{": + to_consume = to_consume + 1 + if to_consume == 0: + break + prev_char = char + return string[:i], string[i + 1 :] + + +class QList(QToken): + def __init__(self, value: list) -> None: + self.value = value + + def interpret(self, datastore: Datastore, namespace: dict): + expanded_list = [] + for value in self.value: + expanded_list.append(value.interpret(datastore, namespace)) + return expanded_list + + @staticmethod + def parse(string: str, namespace: dict) -> QToken: + entries_str = string[1:-1] + ls: List[QToken] = [] + while len(entries_str) > 0: + entries_str = entries_str.strip() + if len(ls) > 0 and entries_str[0] == ",": + entries_str = entries_str[1:] + # parse + (val_t, val_str), entries_str = _parse_token(entries_str, namespace) + if not val_t: + raise QueryParseException("List expected a value, got nothing") + val = val_t.parse(val_str, namespace) + # set + ls.append(val) + return QList(ls) + + @staticmethod + def check(string: str): + if string[0] != "[": + return None, string + # Find closing bracket + i = 1 + to_consume = 1 + single_quote = False + double_quote = False + prev_char = None + for char in string[i:]: + i += 1 + if char == "'" and prev_char != "\\" and not double_quote: + single_quote = not single_quote + elif char == '"' and prev_char != "\\" and not single_quote: + double_quote = not double_quote + elif double_quote or single_quote: + pass + elif char == "]": + to_consume = to_consume - 1 + elif char == "[": + to_consume = to_consume + 1 + if to_consume == 0: + break + prev_char = char + return string[:i], string[i + 1 :] + + +qtypes: Sequence[Type[QToken]] = [QString, QInteger, QFunction, QDict, QList, QVariable] + + +def _parse_token(string: str, namespace: dict) -> Tuple[Tuple[Any, str], str]: + # TODO: The whole parsing thing is shoddily written, needs a rewrite from ground-up + if not isinstance(string, str): + raise QueryParseException( + "Reached unreachable, cannot parse something that isn't a string" + ) + if len(string) == 0: + return (None, ""), string + string = string.strip() + token = None + t = None # Declare so we can return it + for t in qtypes: + token, string = t.check(string) + if token: + break + if not token: + raise QueryParseException(f"Syntax error: {string}") + return (t, token), string + + +def create_namespace() -> dict: + namespace = { + "True": True, + "False": False, + "true": True, + "false": False, + } + return namespace + + +def parse(line, namespace): + separator_i = line.find("=") + var_str = line[:separator_i] + val_str = line[separator_i + 1 :] + if not val_str: + # TODO: Proper message + raise QueryParseException("Nothing to assign") + (var_t, var), var_str = _parse_token(var_str, namespace) + var_str = var_str.strip() + if var_str: # Didn't consume whole var string + raise QueryParseException("Invalid syntax for assignment variable") + if var_t is not QVariable: + raise QueryParseException("Cannot assign to a non-variable") + (val_t, val), var_str = _parse_token(val_str, namespace) + if var_str: # Didn't consume whole val string + raise QueryParseException("Invalid syntax for value to assign") + # Parse token + var = var_t.parse(var, namespace) + val = val_t.parse(val, namespace) + return var, val + + +def interpret(var, val, namespace, datastore): + namespace[var.name] = val.interpret(datastore, namespace) + # logger.debug("Set {} to {}".format(var.name, namespace[var.name])) + + +def get_return(namespace): + if "RETURN" not in namespace: + raise QueryParseException( + "Query doesn't assign the RETURN variable, nothing to respond" + ) + return namespace["RETURN"] + + +def _split_query_statements(query: str) -> List[str]: + """Split query into statements on semicolons, ignoring semicolons inside string literals.""" + statements = [] + current = "" + in_single_quote = False + in_double_quote = False + prev_char = None + + for char in query: + if char == "'" and prev_char != "\\" and not in_double_quote: + in_single_quote = not in_single_quote + current += char + elif char == '"' and prev_char != "\\" and not in_single_quote: + in_double_quote = not in_double_quote + current += char + elif char == ";" and not in_single_quote and not in_double_quote: + statements.append(current) + current = "" + else: + current += char + prev_char = char + + if current: + statements.append(current) + return statements + + +def query( + name: str, query: str, starttime: datetime, endtime: datetime, datastore: Datastore +) -> Any: + namespace = create_namespace() + namespace["NAME"] = name + namespace["STARTTIME"] = starttime.isoformat() + namespace["ENDTIME"] = endtime.isoformat() + + query_stmts = _split_query_statements(query) + for statement in query_stmts: + statement = statement.strip() + if statement: + logger.debug("Parsing: " + statement) + var, val = parse(statement, namespace) + interpret(var, val, namespace, datastore) + + result = get_return(namespace) + return result diff --git a/aw_transform/__init__.py b/aw_transform/__init__.py new file mode 100644 index 0000000..d17e0fe --- /dev/null +++ b/aw_transform/__init__.py @@ -0,0 +1,44 @@ +from .filter_keyvals import filter_keyvals, filter_keyvals_regex +from .filter_period_intersect import filter_period_intersect, period_union, union +from .heartbeats import heartbeat_merge, heartbeat_reduce +from .merge_events_by_keys import merge_events_by_keys +from .chunk_events_by_key import chunk_events_by_key +from .sort_by import ( + sort_by_timestamp, + sort_by_duration, + sum_durations, + concat, + limit_events, +) +from .split_url_events import split_url_events +from .merge_subwatcher_fields import merge_subwatcher_fields +from .simplify import simplify_string +from .flood import flood +from .classify import categorize, tag, Rule +from .union_no_overlap import union_no_overlap + +__all__ = [ + "flood", + "concat", + "categorize", + "tag", + "Rule", + "period_union", + "filter_period_intersect", + "union", + "union_no_overlap", + "concat", + "sum_durations", + "sort_by_timestamp", + "sort_by_duration", + "heartbeat_reduce", + "heartbeat_merge", + "merge_events_by_keys", + "merge_subwatcher_fields", + "chunk_events_by_key", + "limit_events", + "filter_keyvals", + "filter_keyvals_regex", + "split_url_events", + "simplify_string", +] diff --git a/aw_transform/chunk_events_by_key.py b/aw_transform/chunk_events_by_key.py new file mode 100644 index 0000000..a99993b --- /dev/null +++ b/aw_transform/chunk_events_by_key.py @@ -0,0 +1,39 @@ +import logging +from datetime import timedelta +from typing import List + +from aw_core.models import Event + +logger = logging.getLogger(__name__) + + +def chunk_events_by_key( + events: List[Event], key: str, pulsetime: float = 5.0 +) -> List[Event]: + """ + "Chunks" adjacent events together which have the same value for a key, and stores the + original events in the :code:`subevents` key of the new event. + """ + chunked_events: List[Event] = [] + for event in events: + if key not in event.data: + break + timediff = timedelta(seconds=999999999) # FIXME: ugly but works + if len(chunked_events) > 0: + timediff = event.timestamp - (events[-1].timestamp + events[-1].duration) + if ( + len(chunked_events) > 0 + and chunked_events[-1].data[key] == event.data[key] + and timediff < timedelta(seconds=pulsetime) + ): + chunked_event = chunked_events[-1] + chunked_event.duration += event.duration + chunked_event.data["subevents"].append(event) + else: + data = {key: event.data[key], "subevents": [event]} + chunked_event = Event( + timestamp=event.timestamp, duration=event.duration, data=data + ) + chunked_events.append(chunked_event) + + return chunked_events diff --git a/aw_transform/classify.py b/aw_transform/classify.py new file mode 100644 index 0000000..35a33b8 --- /dev/null +++ b/aw_transform/classify.py @@ -0,0 +1,122 @@ +import json +from typing import Pattern, List, Iterable, Tuple, Dict, Optional, Any +import re + +from aw_core import Event + + +Tag = str +Category = List[str] + + +def _parse_optional_priority(rules: Dict[str, Any]) -> Optional[int]: + if "priority" in rules: + val = rules["priority"] + elif "weight" in rules: + val = rules["weight"] + else: + return None + # bool is a subclass of int + if isinstance(val, bool) or not isinstance(val, int): + raise ValueError("priority/weight must be an integer") + return val + + +class Rule: + regex: Optional[Pattern] + select_keys: Optional[List[str]] + ignore_case: bool + priority: Optional[int] + + def __init__(self, rules: Dict[str, Any]) -> None: + self.select_keys = rules.get("select_keys", None) + self.ignore_case = rules.get("ignore_case", False) + self.priority = _parse_optional_priority(rules) + + # NOTE: Also checks that the regex isn't an empty string (which would erroneously match everything) + regex_str = rules.get("regex", None) + self.regex = ( + re.compile( + regex_str, (re.IGNORECASE if self.ignore_case else 0) | re.UNICODE + ) + if regex_str + else None + ) + + def match(self, e: Event) -> bool: + if self.select_keys: + values = [e.data.get(key, None) for key in self.select_keys] + else: + values = list(e.data.values()) + if self.regex: + for val in values: + if isinstance(val, str) and self.regex.search(val): + return True + return False + + +def categorize( + events: List[Event], classes: List[Tuple[Category, Rule]] +) -> List[Event]: + cache: Dict[str, Category] = {} + for e in events: + try: + key = json.dumps(e.data, sort_keys=True) + except TypeError: + key = str(id(e.data)) + if key not in cache: + cache[key] = _pick_category( + [(_cls, rule) for _cls, rule in classes if rule.match(e)] + ) + e.data["$category"] = list(cache[key]) + return events + + +def _categorize_one(e: Event, classes: List[Tuple[Category, Rule]]) -> Event: + e.data["$category"] = _pick_category( + [(_cls, rule) for _cls, rule in classes if rule.match(e)] + ) + return e + + +def tag(events: List[Event], classes: List[Tuple[Tag, Rule]]) -> List[Event]: + cache: Dict[str, List[Tag]] = {} + for e in events: + try: + key = json.dumps(e.data, sort_keys=True) + except TypeError: + key = str(id(e.data)) + if key not in cache: + cache[key] = [_cls for _cls, rule in classes if rule.match(e)] + e.data["$tags"] = list(cache[key]) + return events + + +def _tag_one(e: Event, classes: List[Tuple[Tag, Rule]]) -> Event: + e.data["$tags"] = [_cls for _cls, rule in classes if rule.match(e)] + return e + + +def _effective_rank(category: Category, rule: Rule) -> int: + # Integer-only. Default is depth * 10 so explicit priorities can slot + # between nesting levels (depth 1 → 10, depth 2 → 20). Relative order of + # unprioritized rules is unchanged. + if rule.priority is not None: + return rule.priority + return len(category) * 10 + + +def _pick_category(matches: Iterable[Tuple[Category, Rule]]) -> Category: + category: Category = ["Uncategorized"] + rank: Optional[int] = None + for cat, rule in matches: + if not cat: + continue + item_rank = _effective_rank(cat, rule) + # None means no match yet, so any non-empty category wins — including + # an explicit priority below a signed 64-bit floor. Equal ranks keep + # the later match (same contract as the old depth-only `>=`). + if rank is None or item_rank >= rank: + category = cat + rank = item_rank + return category diff --git a/aw_transform/filter_keyvals.py b/aw_transform/filter_keyvals.py new file mode 100644 index 0000000..349e121 --- /dev/null +++ b/aw_transform/filter_keyvals.py @@ -0,0 +1,28 @@ +import logging +from typing import List +import re + +from aw_core.models import Event + +logger = logging.getLogger(__name__) + + +def filter_keyvals( + events: List[Event], key: str, vals: List[str], exclude=False +) -> List[Event]: + def predicate(event): + return key in event.data and event.data[key] in vals + + if exclude: + return [e for e in events if not predicate(e)] + else: + return [e for e in events if predicate(e)] + + +def filter_keyvals_regex(events: List[Event], key: str, regex: str) -> List[Event]: + r = re.compile(regex) + + def predicate(event): + return key in event.data and bool(r.findall(event.data[key])) + + return [e for e in events if predicate(e)] diff --git a/aw_transform/filter_period_intersect.py b/aw_transform/filter_period_intersect.py new file mode 100644 index 0000000..6e3c799 --- /dev/null +++ b/aw_transform/filter_period_intersect.py @@ -0,0 +1,173 @@ +import logging +from typing import List, Iterable, Tuple +from copy import deepcopy + +from aw_core import Event +from timeslot import Timeslot + +logger = logging.getLogger(__name__) + + +def _get_event_period(event: Event) -> Timeslot: + start = event.timestamp + end = start + event.duration + return Timeslot(start, end) + + +def _replace_event_period(event: Event, period: Timeslot) -> Event: + e = deepcopy(event) + e.timestamp = period.start + e.duration = period.duration + return e + + +def _intersecting_eventpairs( + events1: List[Event], events2: List[Event] +) -> Iterable[Tuple[Event, Event, Timeslot]]: + """A generator that yields each overlapping pair of events from two eventlists along with a Timeslot of the intersection""" + events1.sort(key=lambda e: e.timestamp) + events2.sort(key=lambda e: e.timestamp) + e1_i = 0 + e2_i = 0 + while e1_i < len(events1) and e2_i < len(events2): + e1 = events1[e1_i] + e2 = events2[e2_i] + e1_p = _get_event_period(e1) + e2_p = _get_event_period(e2) + + ip = e1_p.intersection(e2_p) + if ip: + # If events intersected, yield events + yield (e1, e2, ip) + if e1_p.end <= e2_p.end: + e1_i += 1 + else: + e2_i += 1 + else: + # No intersection, check if event is before/after filterevent + if e1_p.end <= e2_p.start: + # Event ended before filter event started + e1_i += 1 + elif e2_p.end <= e1_p.start: + # Event started after filter event ended + e2_i += 1 + else: + logger.error("Should be unreachable, skipping period") + e1_i += 1 + e2_i += 1 + + +def filter_period_intersect( + events: List[Event], filterevents: List[Event] +) -> List[Event]: + """ + Filters away all events or time periods of events in which a + filterevent does not have an intersecting time period. + + Useful for example when you want to filter away events or + part of events during which a user was AFK. + + Usage: + windowevents_notafk = filter_period_intersect(windowevents, notafkevents) + + Example: + .. code-block:: none + + events1 | ======= ======== | + events2 | ------ --- --- ---- | + result | ==== = ==== | + + A JavaScript version used to exist in aw-webui but was removed in `this PR `_. + """ + + events = sorted(events) + filterevents = sorted(filterevents) + + return [ + _replace_event_period(e1, ip) + for (e1, _, ip) in _intersecting_eventpairs(events, filterevents) + ] + + +def period_union(events1: List[Event], events2: List[Event]) -> List[Event]: + """ + Takes a list of two events and returns a new list of events covering the union + of the timeperiods contained in the eventlists with no overlapping events. + + .. warning:: This function strips all data from events as it cannot keep it consistent. + + Example: + .. code-block:: none + + events1 | ------- --------- | + events2 | ------ --- -- ---- | + result | ----------- -- --------- | + """ + events = sorted(events1 + events2) + merged_events = [] + if events: + merged_events.append(events.pop(0)) + for e in events: + last_event = merged_events[-1] + + e_p = _get_event_period(e) + le_p = _get_event_period(last_event) + + if not e_p.gap(le_p): + new_period = e_p.union(le_p) + merged_events[-1] = _replace_event_period(last_event, new_period) + else: + merged_events.append(e) + for event in merged_events: + # Clear data + event.data = {} + return merged_events + + +def union(events1: List[Event], events2: List[Event]) -> List[Event]: + """ + Concatenates and sorts union of 2 event lists and removes duplicates. + + Example: + Merges events from a backup-bucket with events from a "living" bucket. + + .. code-block:: python + + events = union(events_backup, events_living) + """ + + events1 = sorted(events1, key=lambda e: (e.timestamp, e.duration)) + events2 = sorted(events2, key=lambda e: (e.timestamp, e.duration)) + events_union = [] + + e1_i = 0 + e2_i = 0 + while e1_i < len(events1) and e2_i < len(events2): + e1 = events1[e1_i] + e2 = events2[e2_i] + + if e1 == e2: + events_union.append(e1) + e1_i += 1 + e2_i += 1 + else: + if e1.timestamp < e2.timestamp: + events_union.append(e1) + e1_i += 1 + elif e1.timestamp > e2.timestamp: + events_union.append(e2) + e2_i += 1 + elif e1.duration < e2.duration: + events_union.append(e1) + e1_i += 1 + else: + events_union.append(e2) + e2_i += 1 + + if e1_i < len(events1): + events_union.extend(events1[e1_i:]) + + if e2_i < len(events2): + events_union.extend(events2[e2_i:]) + + return events_union diff --git a/aw_transform/flood.py b/aw_transform/flood.py new file mode 100644 index 0000000..48b1021 --- /dev/null +++ b/aw_transform/flood.py @@ -0,0 +1,121 @@ +import logging +from datetime import timedelta +from copy import deepcopy +from typing import List + +from aw_core.models import Event + +logger = logging.getLogger(__name__) + + +def flood(events: List[Event], pulsetime: float = 5) -> List[Event]: + """Fill short gaps between events and merge nearby equal-data events. + + Events are ordered by timestamp and duration. Gaps no larger than + ``pulsetime`` are filled. Equal-data neighbours merge across the gap; when + their data differs, both events extend to the midpoint. Splitting an + uncertain interval evenly is independent of the surrounding event lengths + and matches the Rust implementation. + + Overlapping equal-data events merge. For overlapping differing-data events, + final normalization gives the later event precedence so the result never + double-counts time. + + See https://github.com/ActivityWatch/activitywatch/issues/124 for the data + collection uncertainty that flooding is intended to handle. + """ + # Originally written in aw-research: https://github.com/ActivityWatch/aw-analysis/blob/7da1f2cd8552f866f643501de633d74cdecab168/aw_analysis/flood.py + # NOTE: This algorithm has a lot of smaller details that need to be + # carefully considered by anyone wishing to edit it, see: + # - https://github.com/ActivityWatch/aw-core/pull/73 + + events = deepcopy(events) + # Sort by (timestamp, duration) so shorter same-timestamp events come first. + # This matches aw-server-rust's sort_by_timestamp behavior and ensures + # a well-defined processing order when events share the same timestamp. + events = sorted(events, key=lambda e: (e.timestamp, e.duration)) + + # If negative gaps are smaller than this, prune them to become zero + negative_gap_trim_thres = timedelta(seconds=0.1) + + warned_about_negative_gap_safe = False + warned_about_negative_gap_unsafe = False + + for e1, e2 in zip(events[:-1], events[1:]): + gap = e2.timestamp - (e1.timestamp + e1.duration) + + if not gap: + continue + + # Sanity check in case events overlap + if gap < timedelta(0) and e1.data == e2.data: + # Events with negative gap but same data can safely be merged + start = min(e1.timestamp, e2.timestamp) + end = max(e1.timestamp + e1.duration, e2.timestamp + e2.duration) + e1.timestamp, e1.duration = start, (end - start) + e2.timestamp, e2.duration = end, timedelta(0) + if not warned_about_negative_gap_safe: + logger.warning( + f"Gap was of negative duration but could be safely merged ({gap.total_seconds()}s). This message will only show once per batch." + ) + warned_about_negative_gap_safe = True + elif gap < -negative_gap_trim_thres and not warned_about_negative_gap_unsafe: + # Events with negative gap but differing data cannot be merged safely + logger.warning( + f"Gap was of negative duration and could NOT be safely merged ({gap.total_seconds()}s). This warning will only show once per batch." + ) + warned_about_negative_gap_unsafe = True + # logger.warning("Event 1 (id {}): {} {}".format(e1.id, e1.timestamp, e1.duration)) + # logger.warning("Event 2 (id {}): {} {}".format(e2.id, e2.timestamp, e2.duration)) + elif -negative_gap_trim_thres < gap <= timedelta(seconds=pulsetime): + e2_end = e2.timestamp + e2.duration + + if e1.data == e2.data: + # Preserve the longer neighbour's extent while merging across + # the gap, matching the existing semantics for equal data. + if e1.duration >= e2.duration: + e1.duration = e2_end - e1.timestamp + e2.timestamp = e2_end + e2.duration = timedelta(0) + else: + e2.timestamp = e1.timestamp + e2.duration = e2_end - e2.timestamp + e1.duration = timedelta(0) + else: + # The gap is an interval of uncertainty: without evidence that + # either neighbour owns more of it, split it at the midpoint. + midpoint = e1.timestamp + e1.duration + gap / 2 + e1.duration = midpoint - e1.timestamp + e2.timestamp = midpoint + e2.duration = e2_end - midpoint + + # Pairwise flooding can mutate an event after its previous pair has already + # been processed. Normalize the final stream so downstream consumers never + # double-count overlapping time. For differing data, the later event wins. + normalized: List[Event] = [] + for event in (e for e in events if e.duration > timedelta(0)): + while ( + normalized + and normalized[-1].timestamp + normalized[-1].duration > event.timestamp + ): + previous = normalized[-1] + + if previous.data == event.data: + end = max( + previous.timestamp + previous.duration, + event.timestamp + event.duration, + ) + previous.duration = end - previous.timestamp + break + + previous.duration = event.timestamp - previous.timestamp + if previous.duration <= timedelta(0): + normalized.pop() + continue + + normalized.append(event) + break + else: + normalized.append(event) + + return normalized diff --git a/aw_transform/heartbeats.py b/aw_transform/heartbeats.py new file mode 100644 index 0000000..7f12fb1 --- /dev/null +++ b/aw_transform/heartbeats.py @@ -0,0 +1,56 @@ +import logging +from datetime import timedelta +from typing import List, Optional + +from aw_core.models import Event + +logger = logging.getLogger(__name__) + + +def heartbeat_reduce(events: List[Event], pulsetime: float) -> List[Event]: + """Merges consecutive events together according to the rules of `heartbeat_merge`.""" + reduced = [] + if events: + reduced.append(events.pop(0)) + for heartbeat in events: + merged = heartbeat_merge(reduced[-1], heartbeat, pulsetime) + if merged is not None: + # Heartbeat was merged + reduced[-1] = merged + else: + # Heartbeat was not merged + reduced.append(heartbeat) + return reduced + + +def heartbeat_merge( + last_event: Event, heartbeat: Event, pulsetime: float +) -> Optional[Event]: + """ + Merges two events if they have identical data + and the heartbeat timestamp is within the pulsetime window. + """ + if last_event.data == heartbeat.data: + # Seconds between end of last_event and start of heartbeat + pulseperiod_end = ( + last_event.timestamp + last_event.duration + timedelta(seconds=pulsetime) + ) + within_pulsetime_window = ( + last_event.timestamp <= heartbeat.timestamp <= pulseperiod_end + ) + + if within_pulsetime_window: + # Seconds between end of last_event and start of timestamp + new_duration = ( + heartbeat.timestamp - last_event.timestamp + ) + heartbeat.duration + if last_event.duration < timedelta(0): + logger.warning( + "Merging heartbeats would result in a negative duration, refusing to merge." + ) + else: + # Taking the max of durations ensures heartbeats that end before the last event don't shorten it + last_event.duration = max((last_event.duration, new_duration)) + return last_event + + return None diff --git a/aw_transform/merge_events_by_keys.py b/aw_transform/merge_events_by_keys.py new file mode 100644 index 0000000..42bccf9 --- /dev/null +++ b/aw_transform/merge_events_by_keys.py @@ -0,0 +1,40 @@ +import logging +from typing import List, Dict, Tuple + +from aw_core.models import Event + +logger = logging.getLogger(__name__) + + +def merge_events_by_keys(events, keys) -> List[Event]: + """ + Sums the duration of all events which share a value for a key and returns a new event for each value. + + .. note: The result will be a list of events without timestamp since they are merged. + """ + # Call recursively until all keys are consumed + if len(keys) < 1: + return events + merged_events: Dict[Tuple, Event] = {} + for event in events: + composite_key: Tuple = () + for key in keys: + if key in event.data: + val = event["data"][key] + # Needed for when the value is a list, such as for categories + if isinstance(val, list): + val = tuple(val) + composite_key = composite_key + (val,) + if composite_key not in merged_events: + merged_events[composite_key] = Event( + timestamp=event.timestamp, duration=event.duration, data={} + ) + for key in keys: + if key in event.data: + merged_events[composite_key].data[key] = event.data[key] + else: + merged_events[composite_key].duration += event.duration + result = [] + for key in merged_events: + result.append(Event(**merged_events[key])) + return result diff --git a/aw_transform/merge_subwatcher_fields.py b/aw_transform/merge_subwatcher_fields.py new file mode 100644 index 0000000..2f86040 --- /dev/null +++ b/aw_transform/merge_subwatcher_fields.py @@ -0,0 +1,150 @@ +import logging +from copy import deepcopy +from typing import List, Optional, Tuple + +from aw_core.models import Event +from timeslot import Timeslot + +from .filter_period_intersect import _get_event_period + +logger = logging.getLogger(__name__) + + +def merge_subwatcher_fields( + base_events: List[Event], + subwatcher_events: List[Event], + keys: List[str], + conflict: str = "base_wins", +) -> List[Event]: + """ + Split each event in *base_events* on overlapping subwatcher boundaries and + copy the named *keys* from the matching subwatcher event into each segment's + ``data`` dict. + + Unlike the ``concat`` workaround, this does not fabricate extra duration. + App/title/duration aggregations stay correct because the output still covers + exactly the same total time as *base_events*; only the segmentation changes + where subwatcher fields actually change. + + This is the backend primitive that lets every client (webui, native UIs, + exporters) categorize by subwatcher fields (browser ``url``/``$domain``; + editor ``project``/``file``/``language``) without bespoke per-watcher + client-side code. + + Args: + base_events: The canonical window/afk-filtered stream to enrich. + subwatcher_events: Events from a subwatcher bucket (e.g. aw-watcher-vim, + aw-watcher-web). Should already be clipped via + ``filter_period_intersect`` before passing here. + keys: Which keys to copy from the subwatcher event into the base event. + Keys already present in the base event are left untouched when + ``conflict="base_wins"`` (default). + conflict: ``"base_wins"`` (default) — base event's existing keys are + never overwritten; subwatcher fields are purely additive. + ``"sub_wins"`` — subwatcher fields overwrite base fields. + + Returns: + A new list of base event segments with subwatcher fields injected. + Events in *base_events* that have no overlapping subwatcher event are + returned with their original data unchanged. + + Example:: + + window_events = query_bucket(bid_window) + editor_events = flood(query_bucket(bid_editor)) + editor_events = filter_period_intersect(editor_events, window_events) + window_events = merge_subwatcher_fields( + window_events, editor_events, ["project", "file", "language"] + ) + # Now categorize(window_events, ...) can match on "project"/"file" + + Note on overlap: + Base events are split at the clipped subwatcher boundaries so each + output segment only carries the subwatcher fields that were actually + present during that slice of time. If multiple subwatcher events cover + the same slice, the most recent one wins so a later transition does not + get smeared backward by an older, longer pulse. + """ + if conflict not in ("base_wins", "sub_wins"): + raise ValueError( + f"conflict must be 'base_wins' or 'sub_wins', got {conflict!r}" + ) + if not subwatcher_events or not keys: + return [deepcopy(e) for e in base_events] + + # Build a sorted copy so we can do a linear scan + sub_sorted = sorted(subwatcher_events, key=lambda e: e.timestamp) + + result: List[Event] = [] + for base in base_events: + base_period = _get_event_period(base) + overlapping: List[Tuple[Event, Timeslot]] = [] + boundaries = {base_period.start, base_period.end} + + for sub in sub_sorted: + sub_period = _get_event_period(sub) + # Once sub starts after base ends we can stop + if sub_period.start >= base_period.end: + break + # Skip sub events that end before base starts + if sub_period.end <= base_period.start: + continue + ip = base_period.intersection(sub_period) + if ip: + overlapping.append((sub, sub_period)) + boundaries.add(ip.start) + boundaries.add(ip.end) + + if not overlapping: + result.append(deepcopy(base)) + continue + + boundary_points = sorted(boundaries) + base_segments: List[Event] = [] + for start, end in zip(boundary_points, boundary_points[1:]): + segment_period = Timeslot(start, end) + best_sub: Optional[Event] = None + best_sub_period: Optional[Timeslot] = None + + for sub, sub_period in overlapping: + if not segment_period.intersection(sub_period): + continue + + # Later subwatcher events should supersede older overlapping + # pulses on the shared slice instead of letting stale data linger. + if ( + best_sub is None + or sub.timestamp > best_sub.timestamp + or ( + sub.timestamp == best_sub.timestamp + and best_sub_period is not None + and sub_period.end > best_sub_period.end + ) + ): + best_sub = sub + best_sub_period = sub_period + + enriched = deepcopy(base) + enriched.timestamp = start + enriched.duration = end - start + + if best_sub is not None: + for key in keys: + if key in best_sub.data: + if conflict == "base_wins" and key in enriched.data: + continue + enriched.data[key] = deepcopy(best_sub.data[key]) + + if ( + base_segments + and base_segments[-1].timestamp + base_segments[-1].duration + == enriched.timestamp + and base_segments[-1].data == enriched.data + ): + base_segments[-1].duration += enriched.duration + else: + base_segments.append(enriched) + + result.extend(base_segments) + + return result diff --git a/aw_transform/py.typed b/aw_transform/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/aw_transform/simplify.py b/aw_transform/simplify.py new file mode 100644 index 0000000..7204eeb --- /dev/null +++ b/aw_transform/simplify.py @@ -0,0 +1,30 @@ +import re +from copy import deepcopy +from typing import List + +from aw_core import Event + + +def simplify_string(events: List[Event], key: str = "title") -> List[Event]: + events = deepcopy(events) + + re_leadingdot = re.compile(r"^(●|\*)\s*") + re_parensprefix = re.compile(r"^\([0-9]+\)\s*") + re_fps = re.compile(r"FPS:\s+[0-9\.]+") + + for e in events: + # Remove prefixes that are numbers within parenthesis + # Example: "(2) Facebook" -> "Facebook" + # Example: "(1) YouTube" -> "YouTube" + e.data[key] = re_parensprefix.sub("", e.data[key]) + + # Things generally specific to window events with the "app" key + if key == "title" and "app" in e["data"]: + # Remove FPS display in window title + # Example: "Cemu - FPS: 59.2 - ..." -> "Cemu - FPS: ... - ..." + e.data[key] = re_fps.sub("FPS: ...", e.data[key]) + + # For VSCode (uses ●), gedit (uses *), et al + # See: https://github.com/ActivityWatch/aw-watcher-window/issues/32 + e.data[key] = re_leadingdot.sub("", e.data[key]) + return events diff --git a/aw_transform/sort_by.py b/aw_transform/sort_by.py new file mode 100644 index 0000000..5f7b691 --- /dev/null +++ b/aw_transform/sort_by.py @@ -0,0 +1,32 @@ +import logging +from datetime import timedelta +from typing import List +from aw_core.models import Event + +logger = logging.getLogger(__name__) + + +def sort_by_timestamp(events) -> List[Event]: + """Sorts a list of events by timestamp""" + return sorted(events, key=lambda e: e.timestamp) + + +def sort_by_duration(events) -> List[Event]: + """Sorts a list of events by duration""" + return sorted(events, key=lambda e: e.duration, reverse=True) + + +def limit_events(events, count) -> List[Event]: + """Returns the ``count`` first events in the list of events""" + return events[:count] + + +def sum_durations(events) -> timedelta: + """Sums the durations for the given events""" + return timedelta(seconds=(sum(event.duration.total_seconds() for event in events))) + + +def concat(events1, events2) -> List[Event]: + """Concatenates two lists of events""" + events = events1 + events2 + return events diff --git a/aw_transform/split_url_events.py b/aw_transform/split_url_events.py new file mode 100644 index 0000000..8a4466f --- /dev/null +++ b/aw_transform/split_url_events.py @@ -0,0 +1,32 @@ +import logging +from typing import List + +from urllib.parse import urlparse + +from aw_core.models import Event + +logger = logging.getLogger(__name__) + + +def split_url_events(events: List[Event]) -> List[Event]: + for event in events: + if "url" in event.data: + url = event.data["url"] + parsed_url = urlparse(url) + event.data["$protocol"] = parsed_url.scheme + netloc = parsed_url.netloc + if netloc: + domain = netloc[4:] if netloc[:4] == "www." else netloc + elif parsed_url.scheme: + # For URLs without a domain (e.g. file://, about:), + # use the scheme as domain so they don't all cluster as empty. + domain = parsed_url.scheme + else: + domain = "" + event.data["$domain"] = domain + event.data["$path"] = parsed_url.path + event.data["$params"] = parsed_url.params + event.data["$options"] = parsed_url.query + event.data["$identifier"] = parsed_url.fragment + # TODO: Parse user, port etc aswell + return events diff --git a/aw_transform/union_no_overlap.py b/aw_transform/union_no_overlap.py new file mode 100644 index 0000000..b1e4723 --- /dev/null +++ b/aw_transform/union_no_overlap.py @@ -0,0 +1,85 @@ +""" +Originally from aw-research +""" + +from copy import deepcopy +from typing import List, Tuple, Optional +from datetime import datetime, timedelta, timezone + +from timeslot import Timeslot + +from aw_core import Event + + +def _split_event(e: Event, dt: datetime) -> Tuple[Event, Optional[Event]]: + if e.timestamp < dt < e.timestamp + e.duration: + e1 = deepcopy(e) + e2 = deepcopy(e) + e1.duration = dt - e.timestamp + e2.timestamp = dt + e2.duration = (e.timestamp + e.duration) - dt + return (e1, e2) + else: + return (e, None) + + +def test_split_event(): + now = datetime(2018, 1, 1, 0, 0).astimezone(timezone.utc) + td1h = timedelta(hours=1) + e = Event(timestamp=now, duration=2 * td1h, data={}) + e1, e2 = _split_event(e, now + td1h) + assert e1.timestamp == now + assert e1.duration == td1h + assert e2 + assert e2.timestamp == now + td1h + assert e2.duration == td1h + + +def union_no_overlap(events1: List[Event], events2: List[Event]) -> List[Event]: + """Merges two eventlists and removes overlap, the first eventlist will have precedence + + Example: + events1 | xxx xx xxx | + events1 | ---- ------ -- | + result | xxx-- xx ----xxx -- | + """ + events1 = deepcopy(events1) + events2 = deepcopy(events2) + + # I looked a lot at aw_transform.union when I wrote this + events_union = [] + e1_i = 0 + e2_i = 0 + while e1_i < len(events1) and e2_i < len(events2): + e1 = events1[e1_i] + e2 = events2[e2_i] + e1_p = Timeslot(e1.timestamp, e1.timestamp + e1.duration) + e2_p = Timeslot(e2.timestamp, e2.timestamp + e2.duration) + + if e1_p.intersects(e2_p): + if e1.timestamp <= e2.timestamp: + events_union.append(e1) + e1_i += 1 + + # If e2 continues after e1, we need to split up the event so we only get the part that comes after + _, e2_next = _split_event(e2, e1.timestamp + e1.duration) + if e2_next: + events2[e2_i] = e2_next + else: + e2_i += 1 + else: + e2_next, e2_next2 = _split_event(e2, e1.timestamp) + events_union.append(e2_next) + e2_i += 1 + if e2_next2: + events2.insert(e2_i, e2_next2) + else: + if e1.timestamp <= e2.timestamp: + events_union.append(e1) + e1_i += 1 + else: + events_union.append(e2) + e2_i += 1 + events_union += events1[e1_i:] + events_union += events2[e2_i:] + return events_union diff --git a/poetry.lock b/poetry.lock new file mode 100644 index 0000000..56bac17 --- /dev/null +++ b/poetry.lock @@ -0,0 +1,696 @@ +# This file is automatically @generated by Poetry 2.3.2 and should not be changed by hand. + +[[package]] +name = "attrs" +version = "24.2.0" +description = "Classes Without Boilerplate" +optional = false +python-versions = ">=3.7" +groups = ["main"] +files = [ + {file = "attrs-24.2.0-py3-none-any.whl", hash = "sha256:81921eb96de3191c8258c199618104dd27ac608d9366f5e35d011eae1867ede2"}, + {file = "attrs-24.2.0.tar.gz", hash = "sha256:5cfb1b9148b5b086569baec03f20d7b6bf3bcacc9a42bebf87ffaaca362f6346"}, +] + +[package.extras] +benchmark = ["cloudpickle ; platform_python_implementation == \"CPython\"", "hypothesis", "mypy (>=1.11.1) ; platform_python_implementation == \"CPython\" and python_version >= \"3.9\"", "pympler", "pytest (>=4.3.0)", "pytest-codspeed", "pytest-mypy-plugins ; platform_python_implementation == \"CPython\" and python_version >= \"3.9\" and python_version < \"3.13\"", "pytest-xdist[psutil]"] +cov = ["cloudpickle ; platform_python_implementation == \"CPython\"", "coverage[toml] (>=5.3)", "hypothesis", "mypy (>=1.11.1) ; platform_python_implementation == \"CPython\" and python_version >= \"3.9\"", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins ; platform_python_implementation == \"CPython\" and python_version >= \"3.9\" and python_version < \"3.13\"", "pytest-xdist[psutil]"] +dev = ["cloudpickle ; platform_python_implementation == \"CPython\"", "hypothesis", "mypy (>=1.11.1) ; platform_python_implementation == \"CPython\" and python_version >= \"3.9\"", "pre-commit", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins ; platform_python_implementation == \"CPython\" and python_version >= \"3.9\" and python_version < \"3.13\"", "pytest-xdist[psutil]"] +docs = ["cogapp", "furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier (<24.7)"] +tests = ["cloudpickle ; platform_python_implementation == \"CPython\"", "hypothesis", "mypy (>=1.11.1) ; platform_python_implementation == \"CPython\" and python_version >= \"3.9\"", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins ; platform_python_implementation == \"CPython\" and python_version >= \"3.9\" and python_version < \"3.13\"", "pytest-xdist[psutil]"] +tests-mypy = ["mypy (>=1.11.1) ; platform_python_implementation == \"CPython\" and python_version >= \"3.9\"", "pytest-mypy-plugins ; platform_python_implementation == \"CPython\" and python_version >= \"3.9\" and python_version < \"3.13\""] + +[[package]] +name = "click" +version = "8.1.8" +description = "Composable command line interface toolkit" +optional = false +python-versions = ">=3.7" +groups = ["main"] +files = [ + {file = "click-8.1.8-py3-none-any.whl", hash = "sha256:63c132bbbed01578a06712a2d1f497bb62d9c1c0d329b7903a866228027263b2"}, + {file = "click-8.1.8.tar.gz", hash = "sha256:ed53c9d8990d83c2a27deae68e4ee337473f6330c040a31d4225c9574d16096a"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "platform_system == \"Windows\""} + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +groups = ["main", "dev"] +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] +markers = {main = "platform_system == \"Windows\"", dev = "sys_platform == \"win32\""} + +[[package]] +name = "coverage" +version = "7.6.1" +description = "Code coverage measurement for Python" +optional = false +python-versions = ">=3.8" +groups = ["dev"] +files = [ + {file = "coverage-7.6.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b06079abebbc0e89e6163b8e8f0e16270124c154dc6e4a47b413dd538859af16"}, + {file = "coverage-7.6.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:cf4b19715bccd7ee27b6b120e7e9dd56037b9c0681dcc1adc9ba9db3d417fa36"}, + {file = "coverage-7.6.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e61c0abb4c85b095a784ef23fdd4aede7a2628478e7baba7c5e3deba61070a02"}, + {file = "coverage-7.6.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fd21f6ae3f08b41004dfb433fa895d858f3f5979e7762d052b12aef444e29afc"}, + {file = "coverage-7.6.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f59d57baca39b32db42b83b2a7ba6f47ad9c394ec2076b084c3f029b7afca23"}, + {file = "coverage-7.6.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:a1ac0ae2b8bd743b88ed0502544847c3053d7171a3cff9228af618a068ed9c34"}, + {file = "coverage-7.6.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:e6a08c0be454c3b3beb105c0596ebdc2371fab6bb90c0c0297f4e58fd7e1012c"}, + {file = "coverage-7.6.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:f5796e664fe802da4f57a168c85359a8fbf3eab5e55cd4e4569fbacecc903959"}, + {file = "coverage-7.6.1-cp310-cp310-win32.whl", hash = "sha256:7bb65125fcbef8d989fa1dd0e8a060999497629ca5b0efbca209588a73356232"}, + {file = "coverage-7.6.1-cp310-cp310-win_amd64.whl", hash = "sha256:3115a95daa9bdba70aea750db7b96b37259a81a709223c8448fa97727d546fe0"}, + {file = "coverage-7.6.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:7dea0889685db8550f839fa202744652e87c60015029ce3f60e006f8c4462c93"}, + {file = "coverage-7.6.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ed37bd3c3b063412f7620464a9ac1314d33100329f39799255fb8d3027da50d3"}, + {file = "coverage-7.6.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d85f5e9a5f8b73e2350097c3756ef7e785f55bd71205defa0bfdaf96c31616ff"}, + {file = "coverage-7.6.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9bc572be474cafb617672c43fe989d6e48d3c83af02ce8de73fff1c6bb3c198d"}, + {file = "coverage-7.6.1-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0c0420b573964c760df9e9e86d1a9a622d0d27f417e1a949a8a66dd7bcee7bc6"}, + {file = "coverage-7.6.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1f4aa8219db826ce6be7099d559f8ec311549bfc4046f7f9fe9b5cea5c581c56"}, + {file = "coverage-7.6.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:fc5a77d0c516700ebad189b587de289a20a78324bc54baee03dd486f0855d234"}, + {file = "coverage-7.6.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:b48f312cca9621272ae49008c7f613337c53fadca647d6384cc129d2996d1133"}, + {file = "coverage-7.6.1-cp311-cp311-win32.whl", hash = "sha256:1125ca0e5fd475cbbba3bb67ae20bd2c23a98fac4e32412883f9bcbaa81c314c"}, + {file = "coverage-7.6.1-cp311-cp311-win_amd64.whl", hash = "sha256:8ae539519c4c040c5ffd0632784e21b2f03fc1340752af711f33e5be83a9d6c6"}, + {file = "coverage-7.6.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:95cae0efeb032af8458fc27d191f85d1717b1d4e49f7cb226cf526ff28179778"}, + {file = "coverage-7.6.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:5621a9175cf9d0b0c84c2ef2b12e9f5f5071357c4d2ea6ca1cf01814f45d2391"}, + {file = "coverage-7.6.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:260933720fdcd75340e7dbe9060655aff3af1f0c5d20f46b57f262ab6c86a5e8"}, + {file = "coverage-7.6.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:07e2ca0ad381b91350c0ed49d52699b625aab2b44b65e1b4e02fa9df0e92ad2d"}, + {file = "coverage-7.6.1-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c44fee9975f04b33331cb8eb272827111efc8930cfd582e0320613263ca849ca"}, + {file = "coverage-7.6.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:877abb17e6339d96bf08e7a622d05095e72b71f8afd8a9fefc82cf30ed944163"}, + {file = "coverage-7.6.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:3e0cadcf6733c09154b461f1ca72d5416635e5e4ec4e536192180d34ec160f8a"}, + {file = "coverage-7.6.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c3c02d12f837d9683e5ab2f3d9844dc57655b92c74e286c262e0fc54213c216d"}, + {file = "coverage-7.6.1-cp312-cp312-win32.whl", hash = "sha256:e05882b70b87a18d937ca6768ff33cc3f72847cbc4de4491c8e73880766718e5"}, + {file = "coverage-7.6.1-cp312-cp312-win_amd64.whl", hash = "sha256:b5d7b556859dd85f3a541db6a4e0167b86e7273e1cdc973e5b175166bb634fdb"}, + {file = "coverage-7.6.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:a4acd025ecc06185ba2b801f2de85546e0b8ac787cf9d3b06e7e2a69f925b106"}, + {file = "coverage-7.6.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a6d3adcf24b624a7b778533480e32434a39ad8fa30c315208f6d3e5542aeb6e9"}, + {file = "coverage-7.6.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d0c212c49b6c10e6951362f7c6df3329f04c2b1c28499563d4035d964ab8e08c"}, + {file = "coverage-7.6.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6e81d7a3e58882450ec4186ca59a3f20a5d4440f25b1cff6f0902ad890e6748a"}, + {file = "coverage-7.6.1-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:78b260de9790fd81e69401c2dc8b17da47c8038176a79092a89cb2b7d945d060"}, + {file = "coverage-7.6.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a78d169acd38300060b28d600344a803628c3fd585c912cacc9ea8790fe96862"}, + {file = "coverage-7.6.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:2c09f4ce52cb99dd7505cd0fc8e0e37c77b87f46bc9c1eb03fe3bc9991085388"}, + {file = "coverage-7.6.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:6878ef48d4227aace338d88c48738a4258213cd7b74fd9a3d4d7582bb1d8a155"}, + {file = "coverage-7.6.1-cp313-cp313-win32.whl", hash = "sha256:44df346d5215a8c0e360307d46ffaabe0f5d3502c8a1cefd700b34baf31d411a"}, + {file = "coverage-7.6.1-cp313-cp313-win_amd64.whl", hash = "sha256:8284cf8c0dd272a247bc154eb6c95548722dce90d098c17a883ed36e67cdb129"}, + {file = "coverage-7.6.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:d3296782ca4eab572a1a4eca686d8bfb00226300dcefdf43faa25b5242ab8a3e"}, + {file = "coverage-7.6.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:502753043567491d3ff6d08629270127e0c31d4184c4c8d98f92c26f65019962"}, + {file = "coverage-7.6.1-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6a89ecca80709d4076b95f89f308544ec8f7b4727e8a547913a35f16717856cb"}, + {file = "coverage-7.6.1-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a318d68e92e80af8b00fa99609796fdbcdfef3629c77c6283566c6f02c6d6704"}, + {file = "coverage-7.6.1-cp313-cp313t-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:13b0a73a0896988f053e4fbb7de6d93388e6dd292b0d87ee51d106f2c11b465b"}, + {file = "coverage-7.6.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:4421712dbfc5562150f7554f13dde997a2e932a6b5f352edcce948a815efee6f"}, + {file = "coverage-7.6.1-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:166811d20dfea725e2e4baa71fffd6c968a958577848d2131f39b60043400223"}, + {file = "coverage-7.6.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:225667980479a17db1048cb2bf8bfb39b8e5be8f164b8f6628b64f78a72cf9d3"}, + {file = "coverage-7.6.1-cp313-cp313t-win32.whl", hash = "sha256:170d444ab405852903b7d04ea9ae9b98f98ab6d7e63e1115e82620807519797f"}, + {file = "coverage-7.6.1-cp313-cp313t-win_amd64.whl", hash = "sha256:b9f222de8cded79c49bf184bdbc06630d4c58eec9459b939b4a690c82ed05657"}, + {file = "coverage-7.6.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6db04803b6c7291985a761004e9060b2bca08da6d04f26a7f2294b8623a0c1a0"}, + {file = "coverage-7.6.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:f1adfc8ac319e1a348af294106bc6a8458a0f1633cc62a1446aebc30c5fa186a"}, + {file = "coverage-7.6.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a95324a9de9650a729239daea117df21f4b9868ce32e63f8b650ebe6cef5595b"}, + {file = "coverage-7.6.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b43c03669dc4618ec25270b06ecd3ee4fa94c7f9b3c14bae6571ca00ef98b0d3"}, + {file = "coverage-7.6.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8929543a7192c13d177b770008bc4e8119f2e1f881d563fc6b6305d2d0ebe9de"}, + {file = "coverage-7.6.1-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:a09ece4a69cf399510c8ab25e0950d9cf2b42f7b3cb0374f95d2e2ff594478a6"}, + {file = "coverage-7.6.1-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:9054a0754de38d9dbd01a46621636689124d666bad1936d76c0341f7d71bf569"}, + {file = "coverage-7.6.1-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:0dbde0f4aa9a16fa4d754356a8f2e36296ff4d83994b2c9d8398aa32f222f989"}, + {file = "coverage-7.6.1-cp38-cp38-win32.whl", hash = "sha256:da511e6ad4f7323ee5702e6633085fb76c2f893aaf8ce4c51a0ba4fc07580ea7"}, + {file = "coverage-7.6.1-cp38-cp38-win_amd64.whl", hash = "sha256:3f1156e3e8f2872197af3840d8ad307a9dd18e615dc64d9ee41696f287c57ad8"}, + {file = "coverage-7.6.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:abd5fd0db5f4dc9289408aaf34908072f805ff7792632250dcb36dc591d24255"}, + {file = "coverage-7.6.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:547f45fa1a93154bd82050a7f3cddbc1a7a4dd2a9bf5cb7d06f4ae29fe94eaf8"}, + {file = "coverage-7.6.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:645786266c8f18a931b65bfcefdbf6952dd0dea98feee39bd188607a9d307ed2"}, + {file = "coverage-7.6.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9e0b2df163b8ed01d515807af24f63de04bebcecbd6c3bfeff88385789fdf75a"}, + {file = "coverage-7.6.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:609b06f178fe8e9f89ef676532760ec0b4deea15e9969bf754b37f7c40326dbc"}, + {file = "coverage-7.6.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:702855feff378050ae4f741045e19a32d57d19f3e0676d589df0575008ea5004"}, + {file = "coverage-7.6.1-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:2bdb062ea438f22d99cba0d7829c2ef0af1d768d1e4a4f528087224c90b132cb"}, + {file = "coverage-7.6.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:9c56863d44bd1c4fe2abb8a4d6f5371d197f1ac0ebdee542f07f35895fc07f36"}, + {file = "coverage-7.6.1-cp39-cp39-win32.whl", hash = "sha256:6e2cd258d7d927d09493c8df1ce9174ad01b381d4729a9d8d4e38670ca24774c"}, + {file = "coverage-7.6.1-cp39-cp39-win_amd64.whl", hash = "sha256:06a737c882bd26d0d6ee7269b20b12f14a8704807a01056c80bb881a4b2ce6ca"}, + {file = "coverage-7.6.1-pp38.pp39.pp310-none-any.whl", hash = "sha256:e9a6e0eb86070e8ccaedfbd9d38fec54864f3125ab95419970575b42af7541df"}, + {file = "coverage-7.6.1.tar.gz", hash = "sha256:953510dfb7b12ab69d20135a0662397f077c59b1e6379a768e97c59d852ee51d"}, +] + +[package.dependencies] +tomli = {version = "*", optional = true, markers = "python_full_version <= \"3.11.0a6\" and extra == \"toml\""} + +[package.extras] +toml = ["tomli ; python_full_version <= \"3.11.0a6\""] + +[[package]] +name = "deprecation" +version = "2.1.0" +description = "A library to handle automated deprecations" +optional = false +python-versions = "*" +groups = ["main"] +files = [ + {file = "deprecation-2.1.0-py2.py3-none-any.whl", hash = "sha256:a10811591210e1fb0e768a8c25517cabeabcba6f0bf96564f8ff45189f90b14a"}, + {file = "deprecation-2.1.0.tar.gz", hash = "sha256:72b3bde64e5d778694b0cf68178aed03d15e15477116add3fb773e581f9518ff"}, +] + +[package.dependencies] +packaging = "*" + +[[package]] +name = "exceptiongroup" +version = "1.2.2" +description = "Backport of PEP 654 (exception groups)" +optional = false +python-versions = ">=3.7" +groups = ["dev"] +markers = "python_version < \"3.11\"" +files = [ + {file = "exceptiongroup-1.2.2-py3-none-any.whl", hash = "sha256:3111b9d131c238bec2f8f516e123e14ba243563fb135d3fe885990585aa7795b"}, + {file = "exceptiongroup-1.2.2.tar.gz", hash = "sha256:47c2edf7c6738fafb49fd34290706d1a1a2f4d1c6df275526b62cbb4aa5393cc"}, +] + +[package.extras] +test = ["pytest (>=6)"] + +[[package]] +name = "importlib-resources" +version = "6.4.5" +description = "Read resources from Python packages" +optional = false +python-versions = ">=3.8" +groups = ["main"] +markers = "python_version == \"3.8\"" +files = [ + {file = "importlib_resources-6.4.5-py3-none-any.whl", hash = "sha256:ac29d5f956f01d5e4bb63102a5a19957f1b9175e45649977264a1416783bb717"}, + {file = "importlib_resources-6.4.5.tar.gz", hash = "sha256:980862a1d16c9e147a59603677fa2aa5fd82b87f223b6cb870695bcfce830065"}, +] + +[package.dependencies] +zipp = {version = ">=3.1.0", markers = "python_version < \"3.10\""} + +[package.extras] +check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1) ; sys_platform != \"cygwin\""] +cover = ["pytest-cov"] +doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] +enabler = ["pytest-enabler (>=2.2)"] +test = ["jaraco.test (>=5.4)", "pytest (>=6,!=8.1.*)", "zipp (>=3.17)"] +type = ["pytest-mypy"] + +[[package]] +name = "iniconfig" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" +optional = false +python-versions = ">=3.7" +groups = ["dev"] +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] + +[[package]] +name = "iso8601" +version = "2.1.0" +description = "Simple module to parse ISO 8601 dates" +optional = false +python-versions = ">=3.7,<4.0" +groups = ["main"] +files = [ + {file = "iso8601-2.1.0-py3-none-any.whl", hash = "sha256:aac4145c4dcb66ad8b648a02830f5e2ff6c24af20f4f482689be402db2429242"}, + {file = "iso8601-2.1.0.tar.gz", hash = "sha256:6b1d3829ee8921c4301998c909f7829fa9ed3cbdac0d3b16af2d743aed1ba8df"}, +] + +[[package]] +name = "jsonschema" +version = "4.23.0" +description = "An implementation of JSON Schema validation for Python" +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "jsonschema-4.23.0-py3-none-any.whl", hash = "sha256:fbadb6f8b144a8f8cf9f0b89ba94501d143e50411a1278633f56a7acf7fd5566"}, + {file = "jsonschema-4.23.0.tar.gz", hash = "sha256:d71497fef26351a33265337fa77ffeb82423f3ea21283cd9467bb03999266bc4"}, +] + +[package.dependencies] +attrs = ">=22.2.0" +importlib-resources = {version = ">=1.4.0", markers = "python_version < \"3.9\""} +jsonschema-specifications = ">=2023.3.6" +pkgutil-resolve-name = {version = ">=1.3.10", markers = "python_version < \"3.9\""} +referencing = ">=0.28.4" +rpds-py = ">=0.7.1" + +[package.extras] +format = ["fqdn", "idna", "isoduration", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3987", "uri-template", "webcolors (>=1.11)"] +format-nongpl = ["fqdn", "idna", "isoduration", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3986-validator (>0.1.0)", "uri-template", "webcolors (>=24.6.0)"] + +[[package]] +name = "jsonschema-specifications" +version = "2023.12.1" +description = "The JSON Schema meta-schemas and vocabularies, exposed as a Registry" +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "jsonschema_specifications-2023.12.1-py3-none-any.whl", hash = "sha256:87e4fdf3a94858b8a2ba2778d9ba57d8a9cafca7c7489c46ba0d30a8bc6a9c3c"}, + {file = "jsonschema_specifications-2023.12.1.tar.gz", hash = "sha256:48a76787b3e70f5ed53f1160d2b81f586e4ca6d1548c5de7085d1682674764cc"}, +] + +[package.dependencies] +importlib-resources = {version = ">=1.4.0", markers = "python_version < \"3.9\""} +referencing = ">=0.31.0" + +[[package]] +name = "mypy" +version = "1.11.2" +description = "Optional static typing for Python" +optional = false +python-versions = ">=3.8" +groups = ["dev"] +files = [ + {file = "mypy-1.11.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d42a6dd818ffce7be66cce644f1dff482f1d97c53ca70908dff0b9ddc120b77a"}, + {file = "mypy-1.11.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:801780c56d1cdb896eacd5619a83e427ce436d86a3bdf9112527f24a66618fef"}, + {file = "mypy-1.11.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:41ea707d036a5307ac674ea172875f40c9d55c5394f888b168033177fce47383"}, + {file = "mypy-1.11.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:6e658bd2d20565ea86da7d91331b0eed6d2eee22dc031579e6297f3e12c758c8"}, + {file = "mypy-1.11.2-cp310-cp310-win_amd64.whl", hash = "sha256:478db5f5036817fe45adb7332d927daa62417159d49783041338921dcf646fc7"}, + {file = "mypy-1.11.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:75746e06d5fa1e91bfd5432448d00d34593b52e7e91a187d981d08d1f33d4385"}, + {file = "mypy-1.11.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a976775ab2256aadc6add633d44f100a2517d2388906ec4f13231fafbb0eccca"}, + {file = "mypy-1.11.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cd953f221ac1379050a8a646585a29574488974f79d8082cedef62744f0a0104"}, + {file = "mypy-1.11.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:57555a7715c0a34421013144a33d280e73c08df70f3a18a552938587ce9274f4"}, + {file = "mypy-1.11.2-cp311-cp311-win_amd64.whl", hash = "sha256:36383a4fcbad95f2657642a07ba22ff797de26277158f1cc7bd234821468b1b6"}, + {file = "mypy-1.11.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:e8960dbbbf36906c5c0b7f4fbf2f0c7ffb20f4898e6a879fcf56a41a08b0d318"}, + {file = "mypy-1.11.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:06d26c277962f3fb50e13044674aa10553981ae514288cb7d0a738f495550b36"}, + {file = "mypy-1.11.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6e7184632d89d677973a14d00ae4d03214c8bc301ceefcdaf5c474866814c987"}, + {file = "mypy-1.11.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:3a66169b92452f72117e2da3a576087025449018afc2d8e9bfe5ffab865709ca"}, + {file = "mypy-1.11.2-cp312-cp312-win_amd64.whl", hash = "sha256:969ea3ef09617aff826885a22ece0ddef69d95852cdad2f60c8bb06bf1f71f70"}, + {file = "mypy-1.11.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:37c7fa6121c1cdfcaac97ce3d3b5588e847aa79b580c1e922bb5d5d2902df19b"}, + {file = "mypy-1.11.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:4a8a53bc3ffbd161b5b2a4fff2f0f1e23a33b0168f1c0778ec70e1a3d66deb86"}, + {file = "mypy-1.11.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2ff93107f01968ed834f4256bc1fc4475e2fecf6c661260066a985b52741ddce"}, + {file = "mypy-1.11.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:edb91dded4df17eae4537668b23f0ff6baf3707683734b6a818d5b9d0c0c31a1"}, + {file = "mypy-1.11.2-cp38-cp38-win_amd64.whl", hash = "sha256:ee23de8530d99b6db0573c4ef4bd8f39a2a6f9b60655bf7a1357e585a3486f2b"}, + {file = "mypy-1.11.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:801ca29f43d5acce85f8e999b1e431fb479cb02d0e11deb7d2abb56bdaf24fd6"}, + {file = "mypy-1.11.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:af8d155170fcf87a2afb55b35dc1a0ac21df4431e7d96717621962e4b9192e70"}, + {file = "mypy-1.11.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f7821776e5c4286b6a13138cc935e2e9b6fde05e081bdebf5cdb2bb97c9df81d"}, + {file = "mypy-1.11.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:539c570477a96a4e6fb718b8d5c3e0c0eba1f485df13f86d2970c91f0673148d"}, + {file = "mypy-1.11.2-cp39-cp39-win_amd64.whl", hash = "sha256:3f14cd3d386ac4d05c5a39a51b84387403dadbd936e17cb35882134d4f8f0d24"}, + {file = "mypy-1.11.2-py3-none-any.whl", hash = "sha256:b499bc07dbdcd3de92b0a8b29fdf592c111276f6a12fe29c30f6c417dd546d12"}, + {file = "mypy-1.11.2.tar.gz", hash = "sha256:7f9993ad3e0ffdc95c2a14b66dee63729f021968bff8ad911867579c65d13a79"}, +] + +[package.dependencies] +mypy-extensions = ">=1.0.0" +tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} +typing-extensions = ">=4.6.0" + +[package.extras] +dmypy = ["psutil (>=4.0)"] +install-types = ["pip"] +mypyc = ["setuptools (>=50)"] +reports = ["lxml"] + +[[package]] +name = "mypy-extensions" +version = "1.0.0" +description = "Type system extensions for programs checked with the mypy type checker." +optional = false +python-versions = ">=3.5" +groups = ["dev"] +files = [ + {file = "mypy_extensions-1.0.0-py3-none-any.whl", hash = "sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d"}, + {file = "mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782"}, +] + +[[package]] +name = "packaging" +version = "24.1" +description = "Core utilities for Python packages" +optional = false +python-versions = ">=3.8" +groups = ["main", "dev"] +files = [ + {file = "packaging-24.1-py3-none-any.whl", hash = "sha256:5b8f2217dbdbd2f7f384c41c628544e6d52f2d0f53c6d0c3ea61aa5d1d7ff124"}, + {file = "packaging-24.1.tar.gz", hash = "sha256:026ed72c8ed3fcce5bf8950572258698927fd1dbda10a5e981cdf0ac37f4f002"}, +] + +[[package]] +name = "peewee" +version = "3.17.6" +description = "a little orm" +optional = false +python-versions = "*" +groups = ["main"] +files = [ + {file = "peewee-3.17.6.tar.gz", hash = "sha256:cea5592c6f4da1592b7cff8eaf655be6648a1f5857469e30037bf920c03fb8fb"}, +] + +[[package]] +name = "pkgutil-resolve-name" +version = "1.3.10" +description = "Resolve a name to an object." +optional = false +python-versions = ">=3.6" +groups = ["main"] +markers = "python_version == \"3.8\"" +files = [ + {file = "pkgutil_resolve_name-1.3.10-py3-none-any.whl", hash = "sha256:ca27cc078d25c5ad71a9de0a7a330146c4e014c2462d9af19c6b828280649c5e"}, + {file = "pkgutil_resolve_name-1.3.10.tar.gz", hash = "sha256:357d6c9e6a755653cfd78893817c0853af365dd51ec97f3d358a819373bbd174"}, +] + +[[package]] +name = "platformdirs" +version = "3.10.0" +description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +optional = false +python-versions = ">=3.7" +groups = ["main"] +files = [ + {file = "platformdirs-3.10.0-py3-none-any.whl", hash = "sha256:d7c24979f292f916dc9cbf8648319032f551ea8c49a4c9bf2fb556a02070ec1d"}, + {file = "platformdirs-3.10.0.tar.gz", hash = "sha256:b45696dab2d7cc691a3226759c0d3b00c47c8b6e293d96f6436f733303f77f6d"}, +] + +[package.extras] +docs = ["furo (>=2023.7.26)", "proselint (>=0.13)", "sphinx (>=7.1.1)", "sphinx-autodoc-typehints (>=1.24)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4)", "pytest-cov (>=4.1)", "pytest-mock (>=3.11.1)"] + +[[package]] +name = "pluggy" +version = "1.5.0" +description = "plugin and hook calling mechanisms for python" +optional = false +python-versions = ">=3.8" +groups = ["dev"] +files = [ + {file = "pluggy-1.5.0-py3-none-any.whl", hash = "sha256:44e1ad92c8ca002de6377e165f3e0f1be63266ab4d554740532335b9d75ea669"}, + {file = "pluggy-1.5.0.tar.gz", hash = "sha256:2cffa88e94fdc978c4c574f15f9e59b7f4201d439195c3715ca9e2486f1d0cf1"}, +] + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] + +[[package]] +name = "pytest" +version = "8.3.3" +description = "pytest: simple powerful testing with Python" +optional = false +python-versions = ">=3.8" +groups = ["dev"] +files = [ + {file = "pytest-8.3.3-py3-none-any.whl", hash = "sha256:a6853c7375b2663155079443d2e45de913a911a11d669df02a50814944db57b2"}, + {file = "pytest-8.3.3.tar.gz", hash = "sha256:70b98107bd648308a7952b06e6ca9a50bc660be218d53c257cc1fc94fda10181"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "sys_platform == \"win32\""} +exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=1.5,<2" +tomli = {version = ">=1", markers = "python_version < \"3.11\""} + +[package.extras] +dev = ["argcomplete", "attrs (>=19.2)", "hypothesis (>=3.56)", "mock", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] + +[[package]] +name = "pytest-cov" +version = "5.0.0" +description = "Pytest plugin for measuring coverage." +optional = false +python-versions = ">=3.8" +groups = ["dev"] +files = [ + {file = "pytest-cov-5.0.0.tar.gz", hash = "sha256:5837b58e9f6ebd335b0f8060eecce69b662415b16dc503883a02f45dfeb14857"}, + {file = "pytest_cov-5.0.0-py3-none-any.whl", hash = "sha256:4f0764a1219df53214206bf1feea4633c3b558a2925c8b59f144f682861ce652"}, +] + +[package.dependencies] +coverage = {version = ">=5.2.1", extras = ["toml"]} +pytest = ">=4.6" + +[package.extras] +testing = ["fields", "hunter", "process-tests", "pytest-xdist", "virtualenv"] + +[[package]] +name = "referencing" +version = "0.35.1" +description = "JSON Referencing + Python" +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "referencing-0.35.1-py3-none-any.whl", hash = "sha256:eda6d3234d62814d1c64e305c1331c9a3a6132da475ab6382eaa997b21ee75de"}, + {file = "referencing-0.35.1.tar.gz", hash = "sha256:25b42124a6c8b632a425174f24087783efb348a6f1e0008e63cd4466fedf703c"}, +] + +[package.dependencies] +attrs = ">=22.2.0" +rpds-py = ">=0.7.0" + +[[package]] +name = "rfc3339-validator" +version = "0.1.4" +description = "A pure python RFC3339 validator" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +groups = ["main"] +files = [ + {file = "rfc3339_validator-0.1.4-py2.py3-none-any.whl", hash = "sha256:24f6ec1eda14ef823da9e36ec7113124b39c04d50a4d3d3a3c2859577e7791fa"}, + {file = "rfc3339_validator-0.1.4.tar.gz", hash = "sha256:138a2abdf93304ad60530167e51d2dfb9549521a836871b88d7f4695d0022f6b"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "rpds-py" +version = "0.20.0" +description = "Python bindings to Rust's persistent data structures (rpds)" +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "rpds_py-0.20.0-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:3ad0fda1635f8439cde85c700f964b23ed5fc2d28016b32b9ee5fe30da5c84e2"}, + {file = "rpds_py-0.20.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9bb4a0d90fdb03437c109a17eade42dfbf6190408f29b2744114d11586611d6f"}, + {file = "rpds_py-0.20.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c6377e647bbfd0a0b159fe557f2c6c602c159fc752fa316572f012fc0bf67150"}, + {file = "rpds_py-0.20.0-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:eb851b7df9dda52dc1415ebee12362047ce771fc36914586b2e9fcbd7d293b3e"}, + {file = "rpds_py-0.20.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1e0f80b739e5a8f54837be5d5c924483996b603d5502bfff79bf33da06164ee2"}, + {file = "rpds_py-0.20.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5a8c94dad2e45324fc74dce25e1645d4d14df9a4e54a30fa0ae8bad9a63928e3"}, + {file = "rpds_py-0.20.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f8e604fe73ba048c06085beaf51147eaec7df856824bfe7b98657cf436623daf"}, + {file = "rpds_py-0.20.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:df3de6b7726b52966edf29663e57306b23ef775faf0ac01a3e9f4012a24a4140"}, + {file = "rpds_py-0.20.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:cf258ede5bc22a45c8e726b29835b9303c285ab46fc7c3a4cc770736b5304c9f"}, + {file = "rpds_py-0.20.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:55fea87029cded5df854ca7e192ec7bdb7ecd1d9a3f63d5c4eb09148acf4a7ce"}, + {file = "rpds_py-0.20.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:ae94bd0b2f02c28e199e9bc51485d0c5601f58780636185660f86bf80c89af94"}, + {file = "rpds_py-0.20.0-cp310-none-win32.whl", hash = "sha256:28527c685f237c05445efec62426d285e47a58fb05ba0090a4340b73ecda6dee"}, + {file = "rpds_py-0.20.0-cp310-none-win_amd64.whl", hash = "sha256:238a2d5b1cad28cdc6ed15faf93a998336eb041c4e440dd7f902528b8891b399"}, + {file = "rpds_py-0.20.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:ac2f4f7a98934c2ed6505aead07b979e6f999389f16b714448fb39bbaa86a489"}, + {file = "rpds_py-0.20.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:220002c1b846db9afd83371d08d239fdc865e8f8c5795bbaec20916a76db3318"}, + {file = "rpds_py-0.20.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8d7919548df3f25374a1f5d01fbcd38dacab338ef5f33e044744b5c36729c8db"}, + {file = "rpds_py-0.20.0-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:758406267907b3781beee0f0edfe4a179fbd97c0be2e9b1154d7f0a1279cf8e5"}, + {file = "rpds_py-0.20.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3d61339e9f84a3f0767b1995adfb171a0d00a1185192718a17af6e124728e0f5"}, + {file = "rpds_py-0.20.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1259c7b3705ac0a0bd38197565a5d603218591d3f6cee6e614e380b6ba61c6f6"}, + {file = "rpds_py-0.20.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5c1dc0f53856b9cc9a0ccca0a7cc61d3d20a7088201c0937f3f4048c1718a209"}, + {file = "rpds_py-0.20.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:7e60cb630f674a31f0368ed32b2a6b4331b8350d67de53c0359992444b116dd3"}, + {file = "rpds_py-0.20.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:dbe982f38565bb50cb7fb061ebf762c2f254ca3d8c20d4006878766e84266272"}, + {file = "rpds_py-0.20.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:514b3293b64187172bc77c8fb0cdae26981618021053b30d8371c3a902d4d5ad"}, + {file = "rpds_py-0.20.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:d0a26ffe9d4dd35e4dfdd1e71f46401cff0181c75ac174711ccff0459135fa58"}, + {file = "rpds_py-0.20.0-cp311-none-win32.whl", hash = "sha256:89c19a494bf3ad08c1da49445cc5d13d8fefc265f48ee7e7556839acdacf69d0"}, + {file = "rpds_py-0.20.0-cp311-none-win_amd64.whl", hash = "sha256:c638144ce971df84650d3ed0096e2ae7af8e62ecbbb7b201c8935c370df00a2c"}, + {file = "rpds_py-0.20.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:a84ab91cbe7aab97f7446652d0ed37d35b68a465aeef8fc41932a9d7eee2c1a6"}, + {file = "rpds_py-0.20.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:56e27147a5a4c2c21633ff8475d185734c0e4befd1c989b5b95a5d0db699b21b"}, + {file = "rpds_py-0.20.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2580b0c34583b85efec8c5c5ec9edf2dfe817330cc882ee972ae650e7b5ef739"}, + {file = "rpds_py-0.20.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b80d4a7900cf6b66bb9cee5c352b2d708e29e5a37fe9bf784fa97fc11504bf6c"}, + {file = "rpds_py-0.20.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:50eccbf054e62a7b2209b28dc7a22d6254860209d6753e6b78cfaeb0075d7bee"}, + {file = "rpds_py-0.20.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:49a8063ea4296b3a7e81a5dfb8f7b2d73f0b1c20c2af401fb0cdf22e14711a96"}, + {file = "rpds_py-0.20.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ea438162a9fcbee3ecf36c23e6c68237479f89f962f82dae83dc15feeceb37e4"}, + {file = "rpds_py-0.20.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:18d7585c463087bddcfa74c2ba267339f14f2515158ac4db30b1f9cbdb62c8ef"}, + {file = "rpds_py-0.20.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:d4c7d1a051eeb39f5c9547e82ea27cbcc28338482242e3e0b7768033cb083821"}, + {file = "rpds_py-0.20.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:e4df1e3b3bec320790f699890d41c59d250f6beda159ea3c44c3f5bac1976940"}, + {file = "rpds_py-0.20.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:2cf126d33a91ee6eedc7f3197b53e87a2acdac63602c0f03a02dd69e4b138174"}, + {file = "rpds_py-0.20.0-cp312-none-win32.whl", hash = "sha256:8bc7690f7caee50b04a79bf017a8d020c1f48c2a1077ffe172abec59870f1139"}, + {file = "rpds_py-0.20.0-cp312-none-win_amd64.whl", hash = "sha256:0e13e6952ef264c40587d510ad676a988df19adea20444c2b295e536457bc585"}, + {file = "rpds_py-0.20.0-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:aa9a0521aeca7d4941499a73ad7d4f8ffa3d1affc50b9ea11d992cd7eff18a29"}, + {file = "rpds_py-0.20.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:4a1f1d51eccb7e6c32ae89243cb352389228ea62f89cd80823ea7dd1b98e0b91"}, + {file = "rpds_py-0.20.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8a86a9b96070674fc88b6f9f71a97d2c1d3e5165574615d1f9168ecba4cecb24"}, + {file = "rpds_py-0.20.0-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:6c8ef2ebf76df43f5750b46851ed1cdf8f109d7787ca40035fe19fbdc1acc5a7"}, + {file = "rpds_py-0.20.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b74b25f024b421d5859d156750ea9a65651793d51b76a2e9238c05c9d5f203a9"}, + {file = "rpds_py-0.20.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:57eb94a8c16ab08fef6404301c38318e2c5a32216bf5de453e2714c964c125c8"}, + {file = "rpds_py-0.20.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e1940dae14e715e2e02dfd5b0f64a52e8374a517a1e531ad9412319dc3ac7879"}, + {file = "rpds_py-0.20.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d20277fd62e1b992a50c43f13fbe13277a31f8c9f70d59759c88f644d66c619f"}, + {file = "rpds_py-0.20.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:06db23d43f26478303e954c34c75182356ca9aa7797d22c5345b16871ab9c45c"}, + {file = "rpds_py-0.20.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:b2a5db5397d82fa847e4c624b0c98fe59d2d9b7cf0ce6de09e4d2e80f8f5b3f2"}, + {file = "rpds_py-0.20.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:5a35df9f5548fd79cb2f52d27182108c3e6641a4feb0f39067911bf2adaa3e57"}, + {file = "rpds_py-0.20.0-cp313-none-win32.whl", hash = "sha256:fd2d84f40633bc475ef2d5490b9c19543fbf18596dcb1b291e3a12ea5d722f7a"}, + {file = "rpds_py-0.20.0-cp313-none-win_amd64.whl", hash = "sha256:9bc2d153989e3216b0559251b0c260cfd168ec78b1fac33dd485750a228db5a2"}, + {file = "rpds_py-0.20.0-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:f2fbf7db2012d4876fb0d66b5b9ba6591197b0f165db8d99371d976546472a24"}, + {file = "rpds_py-0.20.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:1e5f3cd7397c8f86c8cc72d5a791071431c108edd79872cdd96e00abd8497d29"}, + {file = "rpds_py-0.20.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ce9845054c13696f7af7f2b353e6b4f676dab1b4b215d7fe5e05c6f8bb06f965"}, + {file = "rpds_py-0.20.0-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c3e130fd0ec56cb76eb49ef52faead8ff09d13f4527e9b0c400307ff72b408e1"}, + {file = "rpds_py-0.20.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4b16aa0107ecb512b568244ef461f27697164d9a68d8b35090e9b0c1c8b27752"}, + {file = "rpds_py-0.20.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:aa7f429242aae2947246587d2964fad750b79e8c233a2367f71b554e9447949c"}, + {file = "rpds_py-0.20.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:af0fc424a5842a11e28956e69395fbbeab2c97c42253169d87e90aac2886d751"}, + {file = "rpds_py-0.20.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:b8c00a3b1e70c1d3891f0db1b05292747f0dbcfb49c43f9244d04c70fbc40eb8"}, + {file = "rpds_py-0.20.0-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:40ce74fc86ee4645d0a225498d091d8bc61f39b709ebef8204cb8b5a464d3c0e"}, + {file = "rpds_py-0.20.0-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:4fe84294c7019456e56d93e8ababdad5a329cd25975be749c3f5f558abb48253"}, + {file = "rpds_py-0.20.0-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:338ca4539aad4ce70a656e5187a3a31c5204f261aef9f6ab50e50bcdffaf050a"}, + {file = "rpds_py-0.20.0-cp38-none-win32.whl", hash = "sha256:54b43a2b07db18314669092bb2de584524d1ef414588780261e31e85846c26a5"}, + {file = "rpds_py-0.20.0-cp38-none-win_amd64.whl", hash = "sha256:a1862d2d7ce1674cffa6d186d53ca95c6e17ed2b06b3f4c476173565c862d232"}, + {file = "rpds_py-0.20.0-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:3fde368e9140312b6e8b6c09fb9f8c8c2f00999d1823403ae90cc00480221b22"}, + {file = "rpds_py-0.20.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:9824fb430c9cf9af743cf7aaf6707bf14323fb51ee74425c380f4c846ea70789"}, + {file = "rpds_py-0.20.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:11ef6ce74616342888b69878d45e9f779b95d4bd48b382a229fe624a409b72c5"}, + {file = "rpds_py-0.20.0-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c52d3f2f82b763a24ef52f5d24358553e8403ce05f893b5347098014f2d9eff2"}, + {file = "rpds_py-0.20.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9d35cef91e59ebbeaa45214861874bc6f19eb35de96db73e467a8358d701a96c"}, + {file = "rpds_py-0.20.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d72278a30111e5b5525c1dd96120d9e958464316f55adb030433ea905866f4de"}, + {file = "rpds_py-0.20.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b4c29cbbba378759ac5786730d1c3cb4ec6f8ababf5c42a9ce303dc4b3d08cda"}, + {file = "rpds_py-0.20.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:6632f2d04f15d1bd6fe0eedd3b86d9061b836ddca4c03d5cf5c7e9e6b7c14580"}, + {file = "rpds_py-0.20.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:d0b67d87bb45ed1cd020e8fbf2307d449b68abc45402fe1a4ac9e46c3c8b192b"}, + {file = "rpds_py-0.20.0-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:ec31a99ca63bf3cd7f1a5ac9fe95c5e2d060d3c768a09bc1d16e235840861420"}, + {file = "rpds_py-0.20.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:22e6c9976e38f4d8c4a63bd8a8edac5307dffd3ee7e6026d97f3cc3a2dc02a0b"}, + {file = "rpds_py-0.20.0-cp39-none-win32.whl", hash = "sha256:569b3ea770c2717b730b61998b6c54996adee3cef69fc28d444f3e7920313cf7"}, + {file = "rpds_py-0.20.0-cp39-none-win_amd64.whl", hash = "sha256:e6900ecdd50ce0facf703f7a00df12374b74bbc8ad9fe0f6559947fb20f82364"}, + {file = "rpds_py-0.20.0-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:617c7357272c67696fd052811e352ac54ed1d9b49ab370261a80d3b6ce385045"}, + {file = "rpds_py-0.20.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:9426133526f69fcaba6e42146b4e12d6bc6c839b8b555097020e2b78ce908dcc"}, + {file = "rpds_py-0.20.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:deb62214c42a261cb3eb04d474f7155279c1a8a8c30ac89b7dcb1721d92c3c02"}, + {file = "rpds_py-0.20.0-pp310-pypy310_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:fcaeb7b57f1a1e071ebd748984359fef83ecb026325b9d4ca847c95bc7311c92"}, + {file = "rpds_py-0.20.0-pp310-pypy310_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d454b8749b4bd70dd0a79f428731ee263fa6995f83ccb8bada706e8d1d3ff89d"}, + {file = "rpds_py-0.20.0-pp310-pypy310_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d807dc2051abe041b6649681dce568f8e10668e3c1c6543ebae58f2d7e617855"}, + {file = "rpds_py-0.20.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c3c20f0ddeb6e29126d45f89206b8291352b8c5b44384e78a6499d68b52ae511"}, + {file = "rpds_py-0.20.0-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:b7f19250ceef892adf27f0399b9e5afad019288e9be756d6919cb58892129f51"}, + {file = "rpds_py-0.20.0-pp310-pypy310_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:4f1ed4749a08379555cebf4650453f14452eaa9c43d0a95c49db50c18b7da075"}, + {file = "rpds_py-0.20.0-pp310-pypy310_pp73-musllinux_1_2_i686.whl", hash = "sha256:dcedf0b42bcb4cfff4101d7771a10532415a6106062f005ab97d1d0ab5681c60"}, + {file = "rpds_py-0.20.0-pp310-pypy310_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:39ed0d010457a78f54090fafb5d108501b5aa5604cc22408fc1c0c77eac14344"}, + {file = "rpds_py-0.20.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:bb273176be34a746bdac0b0d7e4e2c467323d13640b736c4c477881a3220a989"}, + {file = "rpds_py-0.20.0-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:f918a1a130a6dfe1d7fe0f105064141342e7dd1611f2e6a21cd2f5c8cb1cfb3e"}, + {file = "rpds_py-0.20.0-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:f60012a73aa396be721558caa3a6fd49b3dd0033d1675c6d59c4502e870fcf0c"}, + {file = "rpds_py-0.20.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3d2b1ad682a3dfda2a4e8ad8572f3100f95fad98cb99faf37ff0ddfe9cbf9d03"}, + {file = "rpds_py-0.20.0-pp39-pypy39_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:614fdafe9f5f19c63ea02817fa4861c606a59a604a77c8cdef5aa01d28b97921"}, + {file = "rpds_py-0.20.0-pp39-pypy39_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fa518bcd7600c584bf42e6617ee8132869e877db2f76bcdc281ec6a4113a53ab"}, + {file = "rpds_py-0.20.0-pp39-pypy39_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f0475242f447cc6cb8a9dd486d68b2ef7fbee84427124c232bff5f63b1fe11e5"}, + {file = "rpds_py-0.20.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f90a4cd061914a60bd51c68bcb4357086991bd0bb93d8aa66a6da7701370708f"}, + {file = "rpds_py-0.20.0-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:def7400461c3a3f26e49078302e1c1b38f6752342c77e3cf72ce91ca69fb1bc1"}, + {file = "rpds_py-0.20.0-pp39-pypy39_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:65794e4048ee837494aea3c21a28ad5fc080994dfba5b036cf84de37f7ad5074"}, + {file = "rpds_py-0.20.0-pp39-pypy39_pp73-musllinux_1_2_i686.whl", hash = "sha256:faefcc78f53a88f3076b7f8be0a8f8d35133a3ecf7f3770895c25f8813460f08"}, + {file = "rpds_py-0.20.0-pp39-pypy39_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:5b4f105deeffa28bbcdff6c49b34e74903139afa690e35d2d9e3c2c2fba18cec"}, + {file = "rpds_py-0.20.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:fdfc3a892927458d98f3d55428ae46b921d1f7543b89382fdb483f5640daaec8"}, + {file = "rpds_py-0.20.0.tar.gz", hash = "sha256:d72a210824facfdaf8768cf2d7ca25a042c30320b3020de2fa04640920d4e121"}, +] + +[[package]] +name = "ruff" +version = "0.6.8" +description = "An extremely fast Python linter and code formatter, written in Rust." +optional = false +python-versions = ">=3.7" +groups = ["dev"] +files = [ + {file = "ruff-0.6.8-py3-none-linux_armv6l.whl", hash = "sha256:77944bca110ff0a43b768f05a529fecd0706aac7bcce36d7f1eeb4cbfca5f0f2"}, + {file = "ruff-0.6.8-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:27b87e1801e786cd6ede4ada3faa5e254ce774de835e6723fd94551464c56b8c"}, + {file = "ruff-0.6.8-py3-none-macosx_11_0_arm64.whl", hash = "sha256:cd48f945da2a6334f1793d7f701725a76ba93bf3d73c36f6b21fb04d5338dcf5"}, + {file = "ruff-0.6.8-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:677e03c00f37c66cea033274295a983c7c546edea5043d0c798833adf4cf4c6f"}, + {file = "ruff-0.6.8-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:9f1476236b3eacfacfc0f66aa9e6cd39f2a624cb73ea99189556015f27c0bdeb"}, + {file = "ruff-0.6.8-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6f5a2f17c7d32991169195d52a04c95b256378bbf0de8cb98478351eb70d526f"}, + {file = "ruff-0.6.8-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:5fd0d4b7b1457c49e435ee1e437900ced9b35cb8dc5178921dfb7d98d65a08d0"}, + {file = "ruff-0.6.8-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f8034b19b993e9601f2ddf2c517451e17a6ab5cdb1c13fdff50c1442a7171d87"}, + {file = "ruff-0.6.8-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6cfb227b932ba8ef6e56c9f875d987973cd5e35bc5d05f5abf045af78ad8e098"}, + {file = "ruff-0.6.8-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6ef0411eccfc3909269fed47c61ffebdcb84a04504bafa6b6df9b85c27e813b0"}, + {file = "ruff-0.6.8-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:007dee844738c3d2e6c24ab5bc7d43c99ba3e1943bd2d95d598582e9c1b27750"}, + {file = "ruff-0.6.8-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:ce60058d3cdd8490e5e5471ef086b3f1e90ab872b548814e35930e21d848c9ce"}, + {file = "ruff-0.6.8-py3-none-musllinux_1_2_i686.whl", hash = "sha256:1085c455d1b3fdb8021ad534379c60353b81ba079712bce7a900e834859182fa"}, + {file = "ruff-0.6.8-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:70edf6a93b19481affd287d696d9e311388d808671bc209fb8907b46a8c3af44"}, + {file = "ruff-0.6.8-py3-none-win32.whl", hash = "sha256:792213f7be25316f9b46b854df80a77e0da87ec66691e8f012f887b4a671ab5a"}, + {file = "ruff-0.6.8-py3-none-win_amd64.whl", hash = "sha256:ec0517dc0f37cad14a5319ba7bba6e7e339d03fbf967a6d69b0907d61be7a263"}, + {file = "ruff-0.6.8-py3-none-win_arm64.whl", hash = "sha256:8d3bb2e3fbb9875172119021a13eed38849e762499e3cfde9588e4b4d70968dc"}, + {file = "ruff-0.6.8.tar.gz", hash = "sha256:a5bf44b1aa0adaf6d9d20f86162b34f7c593bfedabc51239953e446aefc8ce18"}, +] + +[[package]] +name = "six" +version = "1.16.0" +description = "Python 2 and 3 compatibility utilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +groups = ["main"] +files = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] + +[[package]] +name = "strict-rfc3339" +version = "0.7" +description = "Strict, simple, lightweight RFC3339 functions" +optional = false +python-versions = "*" +groups = ["main"] +files = [ + {file = "strict-rfc3339-0.7.tar.gz", hash = "sha256:5cad17bedfc3af57b399db0fed32771f18fc54bbd917e85546088607ac5e1277"}, +] + +[[package]] +name = "timeslot" +version = "0.1.2" +description = "Data type for representing time slots with a start and end." +optional = false +python-versions = ">=3.6,<4.0" +groups = ["main"] +files = [ + {file = "timeslot-0.1.2-py3-none-any.whl", hash = "sha256:2f8efaec7b0a4c1e56a92ec05533219332dd9d8b577539077664c233996911b5"}, + {file = "timeslot-0.1.2.tar.gz", hash = "sha256:a2ac998657e3f3b9ca928757b4906add2c05390c5fc14ed792bb9028d08547b1"}, +] + +[[package]] +name = "tomli" +version = "2.0.1" +description = "A lil' TOML parser" +optional = false +python-versions = ">=3.7" +groups = ["dev"] +markers = "python_full_version <= \"3.11.0a6\"" +files = [ + {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, + {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, +] + +[[package]] +name = "tomlkit" +version = "0.13.2" +description = "Style preserving TOML library" +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "tomlkit-0.13.2-py3-none-any.whl", hash = "sha256:7a974427f6e119197f670fbbbeae7bef749a6c14e793db934baefc1b5f03efde"}, + {file = "tomlkit-0.13.2.tar.gz", hash = "sha256:fff5fe59a87295b278abd31bec92c15d9bc4a06885ab12bcea52c71119392e79"}, +] + +[[package]] +name = "typing-extensions" +version = "4.12.2" +description = "Backported and Experimental Type Hints for Python 3.8+" +optional = false +python-versions = ">=3.8" +groups = ["dev"] +files = [ + {file = "typing_extensions-4.12.2-py3-none-any.whl", hash = "sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d"}, + {file = "typing_extensions-4.12.2.tar.gz", hash = "sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8"}, +] + +[[package]] +name = "zipp" +version = "3.20.2" +description = "Backport of pathlib-compatible object wrapper for zip files" +optional = false +python-versions = ">=3.8" +groups = ["main"] +markers = "python_version == \"3.8\"" +files = [ + {file = "zipp-3.20.2-py3-none-any.whl", hash = "sha256:a817ac80d6cf4b23bf7f2828b7cabf326f15a001bea8b1f9b49631780ba28350"}, + {file = "zipp-3.20.2.tar.gz", hash = "sha256:bc9eb26f4506fda01b81bcde0ca78103b6e62f991b381fec825435c836edbc29"}, +] + +[package.extras] +check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1) ; sys_platform != \"cygwin\""] +cover = ["pytest-cov"] +doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] +enabler = ["pytest-enabler (>=2.2)"] +test = ["big-O", "importlib-resources ; python_version < \"3.9\"", "jaraco.functools", "jaraco.itertools", "jaraco.test", "more-itertools", "pytest (>=6,!=8.1.*)", "pytest-ignore-flaky"] +type = ["pytest-mypy"] + +[metadata] +lock-version = "2.1" +python-versions = "^3.8" +content-hash = "9ac93f2d6bac28473f4b5a4dabda037554159497caa7ae113ca25ea79f763cbe" diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..1293a5f --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,53 @@ +[tool.poetry] +name = "aw-core" +version = "0.5.18" +description = "Core library for ActivityWatch" +authors = ["Erik Bjäreholt ", "Johan Bjäreholt "] +license = "MPL-2.0" +readme = "README.md" +homepage = "https://activitywatch.net/" +repository = "https://github.com/ActivityWatch/aw-core/" +documentation = "https://docs.activitywatch.net/" +packages = [ + { include = "aw_cli" }, + { include = "aw_core" }, + { include = "aw_datastore" }, + { include = "aw_transform" }, + { include = "aw_query" }, +] + +[tool.poetry.scripts] +aw-cli = "aw_cli.__main__:main" + +[tool.poetry.dependencies] +python = "^3.8" +jsonschema = "^4.3" +peewee = ">=3.0.0,<5.0.0" +platformdirs = "3.10" # pinned due to sometimes breaking changes in minor versions: https://github.com/ActivityWatch/aw-core/pull/122#issuecomment-1768020335 +iso8601 = "*" +rfc3339-validator = "^0.1.4" # needed for the date-type format in jsonschema +strict-rfc3339 = "^0.7" +tomlkit = "*" +deprecation = "*" +timeslot = "*" +click = "*" + +[tool.poetry.group.dev.dependencies] +mypy = "*" +ruff = "*" +pytest = "*" +pytest-cov = "*" + +[tool.ruff] +target-version = "py38" + +[tool.ruff.lint] +select = ["E4", "E7", "E9", "F", "B", "UP"] +ignore = ["E402", "E501", "B905"] + +[tool.lint.ruff] +ignore = ["E402", "E501"] + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" diff --git a/scripts/install_python.ps1 b/scripts/install_python.ps1 new file mode 100644 index 0000000..a17235f --- /dev/null +++ b/scripts/install_python.ps1 @@ -0,0 +1,233 @@ +# Sample script to install Python and pip under Windows +# Authors: Olivier Grisel, Jonathan Helmus, Kyle Kastner, and Alex Willmer +# License: CC0 1.0 Universal: http://creativecommons.org/publicdomain/zero/1.0/ +# +# Find the latest version of this script at: +# https://github.com/ogrisel/python-appveyor-demo/blob/master/appveyor/install.ps1 + + +$MINICONDA_URL = "http://repo.continuum.io/miniconda/" +$BASE_URL = "https://www.python.org/ftp/python/" +$GET_PIP_URL = "https://bootstrap.pypa.io/get-pip.py" +$GET_PIP_PATH = "C:\get-pip.py" + +$PYTHON_PRERELEASE_REGEX = @" +(?x) +(?\d+) +\. +(?\d+) +\. +(?\d+) +(?[a-z]{1,2}\d+) +"@ + + +function Download ($filename, $url) { + $webclient = New-Object System.Net.WebClient + + $basedir = $pwd.Path + "\" + $filepath = $basedir + $filename + if (Test-Path $filename) { + Write-Host "Reusing" $filepath + return $filepath + } + + # Download and retry up to 3 times in case of network transient errors. + Write-Host "Downloading" $filename "from" $url + $retry_attempts = 2 + for ($i = 0; $i -lt $retry_attempts; $i++) { + try { + $webclient.DownloadFile($url, $filepath) + break + } + Catch [Exception]{ + Start-Sleep 1 + } + } + if (Test-Path $filepath) { + Write-Host "File saved at" $filepath + } else { + # Retry once to get the error message if any at the last try + $webclient.DownloadFile($url, $filepath) + } + return $filepath +} + + +function ParsePythonVersion ($python_version) { + if ($python_version -match $PYTHON_PRERELEASE_REGEX) { + return ([int]$matches.major, [int]$matches.minor, [int]$matches.micro, + $matches.prerelease) + } + $version_obj = [version]$python_version + return ($version_obj.major, $version_obj.minor, $version_obj.build, "") +} + + +function DownloadPython ($python_version, $platform_suffix) { + $major, $minor, $micro, $prerelease = ParsePythonVersion $python_version + + if (($major -le 2 -and $micro -eq 0) ` + -or ($major -eq 3 -and $minor -le 2 -and $micro -eq 0) ` + ) { + $dir = "$major.$minor" + $python_version = "$major.$minor$prerelease" + } else { + $dir = "$major.$minor.$micro" + } + + if ($prerelease) { + if (($major -le 2) ` + -or ($major -eq 3 -and $minor -eq 1) ` + -or ($major -eq 3 -and $minor -eq 2) ` + -or ($major -eq 3 -and $minor -eq 3) ` + ) { + $dir = "$dir/prev" + } + } + + if (($major -le 2) -or ($major -le 3 -and $minor -le 4)) { + $ext = "msi" + if ($platform_suffix) { + $platform_suffix = ".$platform_suffix" + } + } else { + $ext = "exe" + if ($platform_suffix) { + $platform_suffix = "-$platform_suffix" + } + } + + $filename = "python-$python_version$platform_suffix.$ext" + $url = "$BASE_URL$dir/$filename" + $filepath = Download $filename $url + return $filepath +} + + +function InstallPython ($python_version, $architecture, $python_home) { + Write-Host "Installing Python" $python_version "for" $architecture "bit architecture to" $python_home + if (Test-Path $python_home) { + Write-Host $python_home "already exists, skipping." + return $false + } + if ($architecture -eq "32") { + $platform_suffix = "" + } else { + $platform_suffix = "amd64" + } + $installer_path = DownloadPython $python_version $platform_suffix + $installer_ext = [System.IO.Path]::GetExtension($installer_path) + Write-Host "Installing $installer_path to $python_home" + $install_log = $python_home + ".log" + if ($installer_ext -eq '.msi') { + InstallPythonMSI $installer_path $python_home $install_log + } else { + InstallPythonEXE $installer_path $python_home $install_log + } + if (Test-Path $python_home) { + Write-Host "Python $python_version ($architecture) installation complete" + } else { + Write-Host "Failed to install Python in $python_home" + Get-Content -Path $install_log + Exit 1 + } +} + + +function InstallPythonEXE ($exepath, $python_home, $install_log) { + $install_args = "/quiet InstallAllUsers=1 TargetDir=$python_home" + RunCommand $exepath $install_args +} + + +function InstallPythonMSI ($msipath, $python_home, $install_log) { + $install_args = "/qn /log $install_log /i $msipath TARGETDIR=$python_home" + $uninstall_args = "/qn /x $msipath" + RunCommand "msiexec.exe" $install_args + if (-not(Test-Path $python_home)) { + Write-Host "Python seems to be installed else-where, reinstalling." + RunCommand "msiexec.exe" $uninstall_args + RunCommand "msiexec.exe" $install_args + } +} + +function RunCommand ($command, $command_args) { + Write-Host $command $command_args + Start-Process -FilePath $command -ArgumentList $command_args -Wait -Passthru +} + + +function InstallPip ($python_home) { + $pip_path = $python_home + "\Scripts\pip.exe" + $python_path = $python_home + "\python.exe" + if (-not(Test-Path $pip_path)) { + Write-Host "Installing pip..." + $webclient = New-Object System.Net.WebClient + $webclient.DownloadFile($GET_PIP_URL, $GET_PIP_PATH) + Write-Host "Executing:" $python_path $GET_PIP_PATH + & $python_path $GET_PIP_PATH + } else { + Write-Host "pip already installed." + } +} + + +function DownloadMiniconda ($python_version, $platform_suffix) { + if ($python_version -eq "3.4") { + $filename = "Miniconda3-3.5.5-Windows-" + $platform_suffix + ".exe" + } else { + $filename = "Miniconda-3.5.5-Windows-" + $platform_suffix + ".exe" + } + $url = $MINICONDA_URL + $filename + $filepath = Download $filename $url + return $filepath +} + + +function InstallMiniconda ($python_version, $architecture, $python_home) { + Write-Host "Installing Python" $python_version "for" $architecture "bit architecture to" $python_home + if (Test-Path $python_home) { + Write-Host $python_home "already exists, skipping." + return $false + } + if ($architecture -eq "32") { + $platform_suffix = "x86" + } else { + $platform_suffix = "x86_64" + } + $filepath = DownloadMiniconda $python_version $platform_suffix + Write-Host "Installing" $filepath "to" $python_home + $install_log = $python_home + ".log" + $args = "/S /D=$python_home" + Write-Host $filepath $args + Start-Process -FilePath $filepath -ArgumentList $args -Wait -Passthru + if (Test-Path $python_home) { + Write-Host "Python $python_version ($architecture) installation complete" + } else { + Write-Host "Failed to install Python in $python_home" + Get-Content -Path $install_log + Exit 1 + } +} + + +function InstallMinicondaPip ($python_home) { + $pip_path = $python_home + "\Scripts\pip.exe" + $conda_path = $python_home + "\Scripts\conda.exe" + if (-not(Test-Path $pip_path)) { + Write-Host "Installing pip..." + $args = "install --yes pip" + Write-Host $conda_path $args + Start-Process -FilePath "$conda_path" -ArgumentList $args -Wait -Passthru + } else { + Write-Host "pip already installed." + } +} + +function main () { + InstallPython $env:PYTHON_VERSION $env:PYTHON_ARCH $env:PYTHON + InstallPip $env:PYTHON +} + +main diff --git a/scripts/run_with_env.cmd b/scripts/run_with_env.cmd new file mode 100644 index 0000000..3a25095 --- /dev/null +++ b/scripts/run_with_env.cmd @@ -0,0 +1,87 @@ +:: To build extensions for 64 bit Python 3, we need to configure environment +:: variables to use the MSVC 2010 C++ compilers from GRMSDKX_EN_DVD.iso of: +:: MS Windows SDK for Windows 7 and .NET Framework 4 (SDK v7.1) +:: +:: To build extensions for 64 bit Python 2, we need to configure environment +:: variables to use the MSVC 2008 C++ compilers from GRMSDKX_EN_DVD.iso of: +:: MS Windows SDK for Windows 7 and .NET Framework 3.5 (SDK v7.0) +:: +:: 32 bit builds, and 64-bit builds for 3.5 and beyond, do not require specific +:: environment configurations. +:: +:: Note: this script needs to be run with the /E:ON and /V:ON flags for the +:: cmd interpreter, at least for (SDK v7.0) +:: +:: More details at: +:: https://github.com/cython/cython/wiki/64BitCythonExtensionsOnWindows +:: http://stackoverflow.com/a/13751649/163740 +:: +:: Author: Olivier Grisel +:: License: CC0 1.0 Universal: http://creativecommons.org/publicdomain/zero/1.0/ +:: +:: Notes about batch files for Python people: +:: +:: Quotes in values are literally part of the values: +:: SET FOO="bar" +:: FOO is now five characters long: " b a r " +:: If you don't want quotes, don't include them on the right-hand side. +:: +:: The CALL lines at the end of this file look redundant, but if you move them +:: outside of the IF clauses, they do not run properly in the SET_SDK_64==Y +:: case, I don't know why. +@ECHO OFF +SET COMMAND_TO_RUN=%* +SET WIN_SDK_ROOT=C:\Program Files\Microsoft SDKs\Windows +SET WIN_WDK=c:\Program Files (x86)\Windows Kits\10\Include\wdf + +:: Extract the major and minor versions, and allow for the minor version to be +:: more than 9. This requires the version number to have two dots in it. +SET MAJOR_PYTHON_VERSION=%PYTHON_VERSION:~0,1% +IF "%PYTHON_VERSION:~3,1%" == "." ( + SET MINOR_PYTHON_VERSION=%PYTHON_VERSION:~2,1% +) ELSE ( + SET MINOR_PYTHON_VERSION=%PYTHON_VERSION:~2,2% +) + +:: Based on the Python version, determine what SDK version to use, and whether +:: to set the SDK for 64-bit. +IF %MAJOR_PYTHON_VERSION% == 2 ( + SET WINDOWS_SDK_VERSION="v7.0" + SET SET_SDK_64=Y +) ELSE ( + IF %MAJOR_PYTHON_VERSION% == 3 ( + SET WINDOWS_SDK_VERSION="v7.1" + IF %MINOR_PYTHON_VERSION% LEQ 4 ( + SET SET_SDK_64=Y + ) ELSE ( + SET SET_SDK_64=N + IF EXIST "%WIN_WDK%" ( + :: See: https://connect.microsoft.com/VisualStudio/feedback/details/1610302/ + REN "%WIN_WDK%" 0wdf + ) + ) + ) ELSE ( + ECHO Unsupported Python version: "%MAJOR_PYTHON_VERSION%" + EXIT 1 + ) +) + +IF %PYTHON_ARCH% == 64 ( + IF %SET_SDK_64% == Y ( + ECHO Configuring Windows SDK %WINDOWS_SDK_VERSION% for Python %MAJOR_PYTHON_VERSION% on a 64 bit architecture + SET DISTUTILS_USE_SDK=1 + SET MSSdk=1 + "%WIN_SDK_ROOT%\%WINDOWS_SDK_VERSION%\Setup\WindowsSdkVer.exe" -q -version:%WINDOWS_SDK_VERSION% + "%WIN_SDK_ROOT%\%WINDOWS_SDK_VERSION%\Bin\SetEnv.cmd" /x64 /release + ECHO Executing: %COMMAND_TO_RUN% + call %COMMAND_TO_RUN% || EXIT 1 + ) ELSE ( + ECHO Using default MSVC build environment for 64 bit architecture + ECHO Executing: %COMMAND_TO_RUN% + call %COMMAND_TO_RUN% || EXIT 1 + ) +) ELSE ( + ECHO Using default MSVC build environment for 32 bit architecture + ECHO Executing: %COMMAND_TO_RUN% + call %COMMAND_TO_RUN% || EXIT 1 +) diff --git a/stubs/appdirs.pyi b/stubs/appdirs.pyi new file mode 100644 index 0000000..c9b1c6d --- /dev/null +++ b/stubs/appdirs.pyi @@ -0,0 +1,4 @@ +def user_data_dir(a1: str, a2: str = "") -> str: ... +def user_cache_dir(a1: str, a2: str = "") -> str: ... +def user_config_dir(a1: str, a2: str = "") -> str: ... +def user_log_dir(a1: str, a2: str = "") -> str: ... diff --git a/stubs/iso8601.pyi b/stubs/iso8601.pyi new file mode 100644 index 0000000..ed0cbc4 --- /dev/null +++ b/stubs/iso8601.pyi @@ -0,0 +1,6 @@ +from datetime import datetime + +def parse_date(x: str) -> datetime: ... # function + +class ParseError(Exception): + pass diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..6c11436 --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1 @@ +from . import context # noqa: F401 diff --git a/tests/context.py b/tests/context.py new file mode 100644 index 0000000..1efc4db --- /dev/null +++ b/tests/context.py @@ -0,0 +1,6 @@ +import os +import sys + +# This will cause imports to use the packages in the +# source folder instead of those installed on the system. +sys.path.insert(0, os.path.abspath("..")) diff --git a/tests/test_cli_log.py b/tests/test_cli_log.py new file mode 100644 index 0000000..9ee5b76 --- /dev/null +++ b/tests/test_cli_log.py @@ -0,0 +1,64 @@ +"""Tests for aw_cli.log.find_oldest_log after profile isolation (#149). + +With per-profile appname dirs, every .log file in a module dir belongs to the +active profile — no filename-based filtering is needed. +""" + +import time +from pathlib import Path + +import pytest + +from aw_cli.log import find_oldest_log + +from . import context # noqa: F401 + + +@pytest.fixture() +def log_dir(tmp_path: Path) -> Path: + return tmp_path / "aw-server" + + +def test_returns_none_for_missing_dir(tmp_path: Path) -> None: + assert find_oldest_log(tmp_path / "nonexistent") is None + + +def test_returns_none_for_empty_dir(log_dir: Path) -> None: + log_dir.mkdir() + assert find_oldest_log(log_dir) is None + + +def test_returns_single_log(log_dir: Path) -> None: + log_dir.mkdir() + f = log_dir / "aw-server_2026-01-01.log" + f.write_text("line\n") + assert find_oldest_log(log_dir) == f + + +def test_returns_newest_by_mtime(log_dir: Path) -> None: + log_dir.mkdir() + old = log_dir / "aw-server_old.log" + new = log_dir / "aw-server_new.log" + old.write_text("old\n") + time.sleep(0.01) + new.write_text("new\n") + assert find_oldest_log(log_dir) == new + + +def test_ignores_non_log_files(log_dir: Path) -> None: + log_dir.mkdir() + (log_dir / "notes.txt").write_text("not a log") + assert find_oldest_log(log_dir) is None + + +def test_no_profile_filtering_in_filenames(log_dir: Path) -> None: + """All .log files in the dir belong to the active profile — no name filter.""" + log_dir.mkdir() + default_log = log_dir / "aw-server_2026-01-01.log" + default_log.write_text("line\n") + # A file named with 'testing' in its name is still returned; + # profile selection happens at the directory level, not the filename level. + testing_named = log_dir / "aw-server_testing_2026-01-02.log" + time.sleep(0.01) + testing_named.write_text("testing-named\n") + assert find_oldest_log(log_dir) == testing_named diff --git a/tests/test_config.py b/tests/test_config.py new file mode 100644 index 0000000..0bc4ad4 --- /dev/null +++ b/tests/test_config.py @@ -0,0 +1,102 @@ +import shutil +from configparser import ConfigParser +from pathlib import Path + +import deprecation +import pytest +from aw_core import dirs +from aw_core.config import ( + load_config, + load_config_toml, + save_config, + save_config_toml, +) + +appname = "aw-core-test" +section = "section" +config_dir = dirs.get_config_dir(appname) + +default_config_str = f"""# A default config file, with comments! +[{section}] +somestring = "Hello World!" # A comment +somevalue = 12.3 # Another comment +somearray = ["asd", 123]""" + + +@pytest.fixture(autouse=True) +def clean_config(): + # Remove test config file if it already exists + shutil.rmtree(config_dir, ignore_errors=True) + + # Rerun get_config dir to create config directory + dirs.get_config_dir(appname) + + yield + + # Remove test config file if it already exists + shutil.rmtree(config_dir) + + +def test_create(): + appname = "aw-core-test" + config_dir = dirs.get_config_dir(appname) + assert Path(config_dir).exists() + + +def test_config_defaults(): + # Load non-existing config (will create a out-commented default config file) + config = load_config_toml(appname, default_config_str) + + # Check that load_config used defaults + assert config[section]["somestring"] == "Hello World!" + assert config[section]["somevalue"] == 12.3 + assert config[section]["somearray"] == ["asd", 123] + + +def test_config_no_defaults(): + # Write defaults to file + save_config_toml(appname, default_config_str) + + # Load written defaults without defaults + config = load_config_toml(appname, "") + assert config[section]["somestring"] == "Hello World!" + assert config[section]["somevalue"] == 12.3 + assert config[section]["somearray"] == ["asd", 123] + + +def test_config_override(): + # Create a minimal config file with one overridden value + config_str = """[section] +somevalue = 1000.1""" + save_config_toml(appname, config_str) + + # Open non-default config file and verify that values are correct + config = load_config_toml(appname, default_config_str) + assert config[section]["somevalue"] == 1000.1 + + +@deprecation.fail_if_not_removed +def test_config_ini(): + # Create default config + default_config = ConfigParser() + default_config[section] = {"somestring": "Hello World!", "somevalue": 12.3} # type: ignore + + # Load non-existing config (will create a default config file) + config = load_config(appname, default_config) + + # Check that current config file is same as default config file + assert config[section]["somestring"] == default_config[section]["somestring"] + assert config[section].getfloat("somevalue") == default_config[section].getfloat( + "somevalue" + ) + + # Modify and save config file + config[section]["somevalue"] = "1000.1" + save_config(appname, config) + + # Open non-default config file and verify that values are correct + new_config = load_config(appname, default_config) + assert new_config[section]["somestring"] == config[section]["somestring"] + assert new_config[section].getfloat("somevalue") == config[section].getfloat( + "somevalue" + ) diff --git a/tests/test_datastore.py b/tests/test_datastore.py new file mode 100644 index 0000000..e8b21f6 --- /dev/null +++ b/tests/test_datastore.py @@ -0,0 +1,586 @@ +import logging +import random +from datetime import datetime, timedelta, timezone + +import iso8601 +import pytest +from aw_core.models import Event +from aw_datastore import get_storage_methods +from aw_datastore.storages import PeeweeStorage + +from . import context # noqa: F401 +from .utils import param_datastore_objects, param_testing_buckets_cm + +logging.basicConfig(level=logging.DEBUG) + +# Useful when you just want some placeholder time in your events, saves typing +now = datetime.now(tz=timezone.utc) +td1s = timedelta(seconds=1) +td1d = timedelta(days=1) + + +def test_get_storage_methods(): + assert get_storage_methods() + + +@pytest.mark.parametrize("datastore", param_datastore_objects()) +def test_get_buckets(datastore): + """ + Tests fetching buckets + """ + buckets = datastore.buckets() + for bucket in buckets.values(): + assert bucket["id"] in buckets + assert bucket["data"] == {} + + +@pytest.mark.parametrize("datastore", param_datastore_objects()) +def test_create_bucket(datastore): + name = "A label/name for a test bucket" + bid = "test-identifier" + try: + bucket = datastore.create_bucket( + bucket_id=bid, + type="testtype", + client="testclient", + hostname="testhost", + name=name, + created=now, + data={"key": "value"}, + ) + metadata = bucket.metadata() + assert bid == metadata["id"] + assert name == metadata["name"] + assert "testtype" == metadata["type"] + assert "testclient" == metadata["client"] + assert "testhost" == metadata["hostname"] + assert now == iso8601.parse_date(metadata["created"]) + assert bid in datastore.buckets() + assert {"key": "value"} == metadata["data"] + finally: + datastore.delete_bucket(bid) + assert bid not in datastore.buckets() + + +@pytest.mark.parametrize("datastore", param_datastore_objects()) +def test_update_bucket(datastore): + bid = "test-" + str(random.randint(0, 1000000)) + try: + datastore.create_bucket( + bucket_id=bid, type="test", client="test", hostname="test", name="test" + ) + datastore.update_bucket(bid, name="new name") + assert datastore[bid].metadata()["name"] == "new name" + datastore.update_bucket(bid, data={"key": "value"}) + assert datastore[bid].metadata()["data"] == {"key": "value"} + finally: + datastore.delete_bucket(bid) + + +@pytest.mark.parametrize("datastore", param_datastore_objects()) +def test_delete_bucket(datastore): + bid = "test-" + str(random.randint(0, 1000000)) + datastore.create_bucket( + bucket_id=bid, type="test", client="test", hostname="test", name="test" + ) + datastore.delete_bucket(bid) + assert bid not in datastore.buckets() + with pytest.raises(ValueError): + datastore.delete_bucket(bid) + + +@pytest.mark.parametrize("datastore", param_datastore_objects()) +def test_nonexistent_bucket(datastore): + """ + Tests that a KeyError is raised if you request a non-existent bucket + """ + with pytest.raises(KeyError): + datastore["I-do-not-exist"] + + +@pytest.mark.parametrize("bucket_cm", param_testing_buckets_cm()) +def test_insert_one(bucket_cm): + """ + Tests inserting one event into a bucket + """ + with bucket_cm as bucket: + n_events = len(bucket.get()) + event = Event(timestamp=now, duration=td1s, data={"key": "val"}) + bucket.insert(event) + fetched_events = bucket.get() + assert n_events + 1 == len(fetched_events) + assert isinstance(fetched_events[0], Event) + assert event == fetched_events[0] + logging.info(event) + logging.info(fetched_events[0].to_json_str()) + + +@pytest.mark.parametrize("bucket_cm", param_testing_buckets_cm()) +def test_empty_bucket(bucket_cm): + """ + Ensures empty buckets are empty + """ + with bucket_cm as bucket: + assert 0 == len(bucket.get()) + + +@pytest.mark.parametrize("bucket_cm", param_testing_buckets_cm()) +def test_insert_many(bucket_cm): + """ + Tests that you can insert many events at the same time to a bucket + """ + num_events = 5000 + with bucket_cm as bucket: + events = num_events * [Event(timestamp=now, duration=td1s, data={"key": "val"})] + bucket.insert(events) + fetched_events = bucket.get(limit=-1) + assert num_events == len(fetched_events) + for e, fe in zip(events, fetched_events): + assert e == fe + + +@pytest.mark.parametrize("bucket_cm", param_testing_buckets_cm()) +def test_insert_many_upsert(bucket_cm): + """ + Tests that you can update/upsert many events at the same time to a bucket + """ + num_events = 10 + with bucket_cm as bucket: + events = num_events * [Event(timestamp=now, duration=td1s, data={"key": "val"})] + # insert events to get IDs assigned + bucket.insert(events) + + events = bucket.get(limit=-1) + assert num_events == len(events) + for e in events: + assert e.id is not None + e.data["key"] = "new val" + + # Upsert the events + bucket.insert(events) + + events = bucket.get(limit=-1) + assert num_events == len(events) + for e in events: + assert e.data["key"] == "new val" + + +@pytest.mark.parametrize("bucket_cm", param_testing_buckets_cm()) +def test_delete(bucket_cm): + """ + Tests deleting single events + """ + num_events = 10 + with bucket_cm as bucket: + events = num_events * [Event(timestamp=now, duration=td1s, data={"key": "val"})] + bucket.insert(events) + + fetched_events = bucket.get(limit=-1) + print(fetched_events[0]) + assert num_events == len(fetched_events) + + # Test deleting event + assert bucket.delete(fetched_events[0]["id"]) + + # Test deleting non-existent event + # FIXME: Doesn't work due to lazy evaluation in SqliteDatastore + # assert not bucket.delete(fetched_events[0]["id"]) + + fetched_events = bucket.get(limit=-1) + assert num_events - 1 == len(fetched_events) + + +@pytest.mark.parametrize("bucket_cm", param_testing_buckets_cm()) +def test_insert_badtype(bucket_cm): + """ + Tests that you cannot insert non-event types into a bucket + """ + with bucket_cm as bucket: + bucket_len = len(bucket.get()) + badevent = 1 + with pytest.raises(TypeError): + bucket.insert(badevent) + assert bucket_len == len(bucket.get()) + + +@pytest.mark.parametrize("bucket_cm", param_testing_buckets_cm()) +def test_get_ordered(bucket_cm): + """ + Makes sure that received events are ordered + """ + with bucket_cm as bucket: + eventcount = 10 + events = [] + for i in range(10): + events.append(Event(timestamp=now + i * td1s, duration=td1s)) + random.shuffle(events) + print(events) + bucket.insert(events) + fetched_events = bucket.get(-1) + for i in range(eventcount - 1): + print("1:" + fetched_events[i].to_json_str()) + print("2:" + fetched_events[i + 1].to_json_str()) + assert fetched_events[i].timestamp > fetched_events[i + 1].timestamp + + +@pytest.mark.parametrize("bucket_cm", param_testing_buckets_cm()) +def test_get_event_with_timezone(bucket_cm): + """Tries to retrieve an event using a timezone aware datetime.""" + hour = timedelta(hours=1) + td_offset = 2 * hour + tz = timezone(td_offset) + + dt_utc = datetime(2017, 10, 27, hour=0, minute=5, tzinfo=timezone.utc) + dt_with_tz = dt_utc.replace(tzinfo=tz) + + with bucket_cm as bucket: + bucket.insert(Event(timestamp=dt_with_tz)) + fetched_events = bucket.get( + starttime=dt_with_tz - hour, endtime=dt_with_tz + hour + ) + assert len(fetched_events) == 1 + + fetched_events = bucket.get( + starttime=dt_utc - td_offset - hour, endtime=dt_utc - td_offset + hour + ) + assert len(fetched_events) == 1 + + +@pytest.mark.parametrize("bucket_cm", param_testing_buckets_cm()) +def test_get_datefilter_simple(bucket_cm): + with bucket_cm as bucket: + eventcount = 3 + events = [ + Event(timestamp=now + i * td1s, duration=td1s) for i in range(eventcount) + ] + bucket.insert(events) + + # Get first event, but expect only half the event to match the interval + fetched_events = bucket.get( + -1, + starttime=now - 0.5 * td1s, + endtime=now + 0.5 * td1s, + ) + assert 1 == len(fetched_events) + + # Get first two events, but expect only half of each to match the interval + fetched_events = bucket.get( + -1, + starttime=now + 0.5 * td1s, + endtime=now + 1.5 * td1s, + ) + assert 2 == len(fetched_events) + + # Get last event, but expect only half to match the interval + fetched_events = bucket.get( + -1, + starttime=now + 2.5 * td1s, + endtime=now + 3.5 * td1s, + ) + assert 1 == len(fetched_events) + + # Check approx precision + fetched_events = bucket.get( + -1, + starttime=now - 0.01 * td1s, + endtime=now + 0.01 * td1s, + ) + assert 1 == len(fetched_events) + + # Check precision of start + fetched_events = bucket.get( + -1, + starttime=now, + endtime=now, + ) + assert 1 == len(fetched_events) + + # Check approx precision of end + fetched_events = bucket.get( + -1, + starttime=now + 2.99 * td1s, + endtime=now + 3.01 * td1s, + ) + assert 1 == len(fetched_events) + + +@pytest.mark.parametrize("bucket_cm", param_testing_buckets_cm()) +def test_get_event_by_id(bucket_cm): + """Test that we can retrieve single events by their IDs""" + with bucket_cm as bucket: + eventcount = 2 + # Create 1-day long events + events = [ + Event(timestamp=now + i * td1d, duration=td1d) for i in range(eventcount) + ] + bucket.insert(events) + + # Retrieve stored events + events = bucket.get() + for e in events: + # Query them one-by-one + event = bucket.get_by_id(e.id) + assert e == event + + +@pytest.mark.parametrize("bucket_cm", param_testing_buckets_cm()) +def test_get_event_by_id_notfound(bucket_cm): + """Test fetching an ID that does not exist""" + with bucket_cm as bucket: + assert bucket.get_by_id(1337 * 10**6) is None + + +@pytest.mark.parametrize("bucket_cm", param_testing_buckets_cm()) +def test_get_event_trimming(bucket_cm): + """Test that event trimming works correctly (when querying events that intersect with the query range)""" + # TODO: Trimming should be possible to disable + # (needed in raw data view, among other places where event editing is permitted) + + with bucket_cm as bucket: + if not isinstance(bucket.ds.storage_strategy, PeeweeStorage): + pytest.skip("Trimming not supported for datastore") + + eventcount = 2 + # Create 1-day long events + events = [ + Event(timestamp=now + i * td1d, duration=td1d) for i in range(eventcount) + ] + bucket.insert(events) + + # Result should contain half of each event + fetched_events = bucket.get( + -1, + starttime=now + td1d / 2, + endtime=now + 1.5 * td1d, + ) + assert 2 == len(fetched_events) + total_duration = sum((e.duration for e in fetched_events), timedelta()) + assert td1d == timedelta(seconds=round(total_duration.total_seconds())) + + +@pytest.mark.parametrize("bucket_cm", param_testing_buckets_cm()) +def test_get_datefilter_start(bucket_cm): + """ + Tests the datetimefilter when fetching events + """ + with bucket_cm as bucket: + eventcount = 10 + events = [ + Event(timestamp=now + i * td1s, duration=td1s) for i in range(eventcount) + ] + bucket.insert(events) + + # Starttime + for i in range(eventcount): + fetched_events = bucket.get(-1, starttime=events[i].timestamp + 0.01 * td1s) + assert eventcount - i == len(fetched_events) + + +@pytest.mark.parametrize("bucket_cm", param_testing_buckets_cm()) +def test_get_datefilter_end(bucket_cm): + """ + Tests the datetimefilter when fetching events + """ + with bucket_cm as bucket: + eventcount = 10 + events = [ + Event(timestamp=now + i * td1s, duration=td1s) for i in range(eventcount) + ] + bucket.insert(events) + + # Endtime + for i in range(eventcount): + fetched_events = bucket.get(-1, endtime=events[i].timestamp - 0.01 * td1s) + assert i == len(fetched_events) + + +@pytest.mark.parametrize("bucket_cm", param_testing_buckets_cm()) +def test_get_datefilter_both(bucket_cm): + """ + Tests the datetimefilter when fetching events + """ + with bucket_cm as bucket: + eventcount = 10 + events = [ + Event(timestamp=now + i * td1s, duration=td1s) for i in range(eventcount) + ] + bucket.insert(events) + + # Both + for i in range(eventcount): + for j in range(i + 1, eventcount): + fetched_events = bucket.get( + starttime=events[i].timestamp + timedelta(seconds=0.01), + endtime=events[j].timestamp + + events[j].duration + - timedelta(seconds=0.01), + ) + assert j - i + 1 == len(fetched_events) + + +@pytest.mark.parametrize("bucket_cm", param_testing_buckets_cm()) +def test_insert_invalid(bucket_cm): + with bucket_cm as bucket: + event = "not a real event" + with pytest.raises(TypeError): + bucket.insert(event) + + +@pytest.mark.parametrize("bucket_cm", param_testing_buckets_cm()) +def test_replace(bucket_cm): + """ + Tests the replace event event in bucket functionality + """ + with bucket_cm as bucket: + # Create two events + e1 = bucket.insert(Event(data={"label": "test1"}, timestamp=now)) + assert e1 + assert e1.id is not None + e2 = bucket.insert( + Event(data={"label": "test2"}, timestamp=now + timedelta(seconds=1)) + ) + assert e2 + assert e2.id is not None + + e1.data["label"] = "test1-replaced" + bucket.replace(e1.id, e1) + + bucket.insert( + Event(data={"label": "test3"}, timestamp=now + timedelta(seconds=2)) + ) + + e2.data["label"] = "test2-replaced" + bucket.replace(e2.id, e2) + + # Assert length + assert 3 == len(bucket.get(-1)) + assert bucket.get(-1)[0]["data"]["label"] == "test3" + assert bucket.get(-1)[1]["data"]["label"] == "test2-replaced" + assert bucket.get(-1)[2]["data"]["label"] == "test1-replaced" + + +@pytest.mark.parametrize("bucket_cm", param_testing_buckets_cm()) +def test_replace_last(bucket_cm): + """ + Tests the replace last event in bucket functionality (simple) + """ + with bucket_cm as bucket: + # Create two events + bucket.insert(Event(data={"label": "test1"}, timestamp=now)) + bucket.insert( + Event(data={"label": "test2"}, timestamp=now + timedelta(seconds=1)) + ) + # Create second event to replace with the second one + bucket.replace_last( + Event( + data={"label": "test2-replaced"}, timestamp=now + timedelta(seconds=1) + ) + ) + bucket.insert( + Event(data={"label": "test3"}, timestamp=now + timedelta(seconds=2)) + ) + # Assert data + result = bucket.get(-1) + assert 3 == len(result) + assert result[1]["data"]["label"] == "test2-replaced" + + +@pytest.mark.parametrize("bucket_cm", param_testing_buckets_cm()) +def test_replace_last_complex(bucket_cm): + """ + Tests the replace last event in bucket functionality (complex) + """ + with bucket_cm as bucket: + # Create first event + event1 = Event(data={"label": "test1"}, timestamp=now, duration=timedelta(1)) + bucket.insert(event1) + eventcount = len(bucket.get(-1)) + # Create second event to replace with the first one + event2 = Event( + data={"label": "test2"}, + duration=timedelta(0), + timestamp=now + timedelta(seconds=1), + ) + bucket.replace_last(event2) + # Assert length and content + result = bucket.get(-1) + assert eventcount == len(result) + assert event2 == result[0] + + +@pytest.mark.parametrize("bucket_cm", param_testing_buckets_cm()) +def test_get_last(bucket_cm): + """ + Tests setting the result limit when fetching events + """ + now = datetime.now() + second = timedelta(seconds=1) + with bucket_cm as bucket: + events = [ + Event(data={"label": "test"}, timestamp=ts, duration=timedelta(0)) + for ts in [now + second, now + second * 2, now + second * 3] + ] + + for event in events: + bucket.insert(event) + + assert bucket.get(limit=1)[0] == events[-1] + for event in bucket.get(limit=5): + print(event.timestamp, event.data["label"]) + + +@pytest.mark.parametrize("bucket_cm", param_testing_buckets_cm()) +def test_limit(bucket_cm): + """ + Tests setting the result limit when fetching events + """ + with bucket_cm as bucket: + for _ in range(5): + bucket.insert(Event(timestamp=now)) + + assert 0 == len(bucket.get(limit=0)) + assert 1 == len(bucket.get(limit=1)) + assert 3 == len(bucket.get(limit=3)) + assert 5 == len(bucket.get(limit=5)) + assert 5 == len(bucket.get(limit=-1)) + + +@pytest.mark.parametrize("bucket_cm", param_testing_buckets_cm()) +def test_get_metadata(bucket_cm): + """ + Tests the get_metadata function + """ + with bucket_cm as bucket: + print(bucket.ds.storage_strategy) + metadata = bucket.metadata() + print(metadata) + assert "created" in metadata + assert "client" in metadata + assert "hostname" in metadata + assert "id" in metadata + assert "name" in metadata + assert "type" in metadata + with pytest.raises(ValueError): + bucket.metadata() + + +@pytest.mark.parametrize("bucket_cm", param_testing_buckets_cm()) +def test_get_eventcount(bucket_cm): + """ + Tests the get_eventcount function + """ + with bucket_cm as bucket: + print(bucket.ds.storage_strategy) + assert bucket.get_eventcount() == 0 + for _ in range(5): + bucket.insert(Event(timestamp=now)) + assert bucket.get_eventcount() == 5 + # TODO: Test with timestamps and start/endtime filtering + + bucket.insert(Event(timestamp=now + timedelta(seconds=5))) + assert ( + bucket.get_eventcount(starttime=now - timedelta(seconds=1), endtime=now) + == 5 + ) + assert bucket.get_eventcount(endtime=now + timedelta(seconds=1)) == 5 + assert bucket.get_eventcount(starttime=now + timedelta(seconds=1)) == 1 diff --git a/tests/test_dirs.py b/tests/test_dirs.py new file mode 100644 index 0000000..482bb04 --- /dev/null +++ b/tests/test_dirs.py @@ -0,0 +1,201 @@ +"""Tests for profile isolation via AW_PROFILE in aw_core.dirs. + +Three profiles (default / testing / research) must yield completely disjoint +directory roots so that an ActivityWatch research build cannot read from or +write to a participant's personal datastore. + +The ``testing`` profile additionally follows the new-root-plus-legacy-fallback +rule from ActivityWatch/activitywatch#1399. +""" + +import os +from pathlib import Path +from unittest.mock import patch + +import pytest + +from aw_core.dirs import ( + _get_appname, + get_cache_dir, + get_config_dir, + get_data_dir, + legacy_testing_suffix, + using_legacy_testing_root, +) + +from . import context # noqa: F401 + + +@pytest.fixture +def fake_dirs(tmp_path, monkeypatch): + """Point platformdirs at a tmp tree so tests never touch the real home.""" + data = tmp_path / "data" + config = tmp_path / "config" + cache = tmp_path / "cache" + + def _join(root: Path, appname: str) -> str: + return str(root / appname) + + monkeypatch.setattr( + "aw_core.dirs.platformdirs.user_data_dir", + lambda appname, *a, **k: _join(data, appname), + ) + monkeypatch.setattr( + "aw_core.dirs.platformdirs.user_config_dir", + lambda appname, *a, **k: _join(config, appname), + ) + monkeypatch.setattr( + "aw_core.dirs.platformdirs.user_cache_dir", + lambda appname, *a, **k: _join(cache, appname), + ) + monkeypatch.setattr( + "aw_core.dirs.platformdirs.user_cache_path", + lambda appname, *a, **k: cache / appname, + ) + monkeypatch.setattr( + "aw_core.dirs.platformdirs.user_log_dir", + lambda appname, *a, **k: str(cache / appname / "log"), + ) + monkeypatch.delenv("AW_PROFILE", raising=False) + return {"data": data, "config": config, "cache": cache, "root": tmp_path} + + +def _plant_legacy_testing_db(fake_dirs) -> Path: + aw_server = fake_dirs["data"] / "activitywatch" / "aw-server" + aw_server.mkdir(parents=True) + marker = aw_server / "peewee-sqlite-testing.v2.db" + marker.write_text("") + return marker + + +# --------------------------------------------------------------------------- +# _get_appname +# --------------------------------------------------------------------------- + + +class TestGetAppname: + def test_unset_returns_bare_name(self, fake_dirs): + """AW_PROFILE absent → bare 'activitywatch', identical to legacy.""" + with patch.dict(os.environ, {"AW_PROFILE": ""}): + assert _get_appname() == "activitywatch" + + def test_empty_string_returns_bare_name(self, fake_dirs): + """AW_PROFILE='' is treated the same as unset.""" + with patch.dict(os.environ, {"AW_PROFILE": ""}): + assert _get_appname() == "activitywatch" + + def test_testing_profile_fresh_uses_new_root(self, fake_dirs): + with patch.dict(os.environ, {"AW_PROFILE": "testing"}): + assert _get_appname() == "activitywatch-testing" + + def test_research_profile(self, fake_dirs): + with patch.dict(os.environ, {"AW_PROFILE": "research"}): + assert _get_appname() == "activitywatch-research" + + def test_arbitrary_profile(self, fake_dirs): + with patch.dict(os.environ, {"AW_PROFILE": "myproject"}): + assert _get_appname() == "activitywatch-myproject" + + +# --------------------------------------------------------------------------- +# testing-root fallback (ActivityWatch/activitywatch#1399) +# --------------------------------------------------------------------------- + + +class TestTestingRootFallback: + def test_fresh_setup_uses_new_root(self, fake_dirs): + with patch.dict(os.environ, {"AW_PROFILE": "testing"}): + assert using_legacy_testing_root() is False + assert _get_appname() == "activitywatch-testing" + assert legacy_testing_suffix(True) == "" + data = get_data_dir() + assert "activitywatch-testing" in data + assert (fake_dirs["data"] / "activitywatch-testing").is_dir() + + def test_legacy_artifacts_keep_shared_root(self, fake_dirs): + _plant_legacy_testing_db(fake_dirs) + with patch.dict(os.environ, {"AW_PROFILE": "testing"}): + assert using_legacy_testing_root() is True + assert _get_appname() == "activitywatch" + assert legacy_testing_suffix(True) == "-testing" + data = get_data_dir() + assert "activitywatch-testing" not in data + assert data.endswith("activitywatch") or data.endswith("activitywatch/") + assert not (fake_dirs["data"] / "activitywatch-testing").exists() + + def test_new_root_wins_over_legacy_artifacts(self, fake_dirs): + _plant_legacy_testing_db(fake_dirs) + (fake_dirs["data"] / "activitywatch-testing").mkdir() + with patch.dict(os.environ, {"AW_PROFILE": "testing"}): + assert using_legacy_testing_root() is False + assert _get_appname() == "activitywatch-testing" + assert legacy_testing_suffix(True) == "" + + def test_config_testing_toml_is_a_legacy_marker(self, fake_dirs): + cfg = fake_dirs["config"] / "activitywatch" + cfg.mkdir(parents=True) + (cfg / "config-testing.toml").write_text("") + with patch.dict(os.environ, {"AW_PROFILE": "testing"}): + assert using_legacy_testing_root() is True + assert _get_appname() == "activitywatch" + + def test_testing_without_aw_profile_keeps_filename_suffix(self, fake_dirs): + """``testing=True`` with no profile still shares the default root.""" + assert legacy_testing_suffix(True) == "-testing" + assert legacy_testing_suffix(False) == "" + + def test_named_profile_never_suffixes_filenames(self, fake_dirs): + with patch.dict(os.environ, {"AW_PROFILE": "research"}): + assert legacy_testing_suffix(True) == "" + assert legacy_testing_suffix(False) == "" + + +# --------------------------------------------------------------------------- +# Directory isolation +# --------------------------------------------------------------------------- + + +def _data_dir(profile: str) -> str: + """Return get_data_dir() under the given profile (empty string = default).""" + with patch.dict(os.environ, {"AW_PROFILE": profile}): + return get_data_dir() + + +class TestDirsIsolation: + """Three profiles must yield fully disjoint directory roots.""" + + def test_default_has_no_profile_suffix(self, fake_dirs): + d = _data_dir("") + assert "activitywatch" in d + # The bare appname must not contain a dash after "activitywatch" + assert "activitywatch-" not in d + + def test_testing_suffix_present(self, fake_dirs): + assert "activitywatch-testing" in _data_dir("testing") + + def test_research_suffix_present(self, fake_dirs): + assert "activitywatch-research" in _data_dir("research") + + def test_three_profiles_are_disjoint(self, fake_dirs): + """default, testing, research all produce distinct, non-nested paths.""" + default = _data_dir("") + testing = _data_dir("testing") + research = _data_dir("research") + + dirs = {default, testing, research} + assert len(dirs) == 3, f"Profiles are not disjoint: {dirs}" + + for a in dirs: + for b in dirs: + if a != b: + assert not a.startswith(b + os.sep), f"{a!r} is a subpath of {b!r}" + + def test_config_dir_isolated(self, fake_dirs): + with patch.dict(os.environ, {"AW_PROFILE": "research"}): + cfg = get_config_dir() + assert "activitywatch-research" in cfg + + def test_cache_dir_isolated(self, fake_dirs): + with patch.dict(os.environ, {"AW_PROFILE": "testing"}): + cache = get_cache_dir() + assert "activitywatch-testing" in cache diff --git a/tests/test_event.py b/tests/test_event.py new file mode 100644 index 0000000..4d99e53 --- /dev/null +++ b/tests/test_event.py @@ -0,0 +1,41 @@ +from datetime import datetime, timedelta, timezone +import json + +import pytest + +from aw_core.models import Event + +valid_timestamp = "1937-01-01T12:00:27.87+00:20" + +now = datetime.now(timezone.utc) +td1s = timedelta(seconds=1) + + +def test_create() -> None: + Event(timestamp=now, duration=timedelta(hours=13, minutes=37), data={"key": "val"}) + Event( + timestamp=valid_timestamp, + duration=timedelta(hours=13, minutes=37), + data={"key": "val"}, + ) + + +def test_json_serialization() -> None: + e = Event( + timestamp=now, duration=timedelta(hours=13, minutes=37), data={"key": "val"} + ) + assert e == Event(**json.loads(e.to_json_str())) + + +def test_set_invalid_duration() -> None: + e = Event() + with pytest.raises(TypeError): + e.duration = "12" # type: ignore + + +def test_sort() -> None: + e1 = Event(timestamp=now) + e2 = Event(timestamp=now + td1s) + e_sorted = sorted([e2, e1]) + assert e_sorted[0] == e1 + assert e_sorted[1] == e2 diff --git a/tests/test_flood.py b/tests/test_flood.py new file mode 100644 index 0000000..cf59cf8 --- /dev/null +++ b/tests/test_flood.py @@ -0,0 +1,167 @@ +from datetime import datetime, timedelta, timezone + +from aw_core.models import Event +from aw_transform import flood + + +now = datetime.now(tz=timezone.utc) +td1s = timedelta(seconds=1) + + +def test_flood_differing_data_meet_at_gap_midpoint(): + events = [ + Event(timestamp=now, duration=10, data={"a": 0}), + Event(timestamp=now + 14 * td1s, duration=5, data={"b": 1}), + ] + + flooded = flood(events) + + assert flooded == [ + Event(timestamp=now, duration=12, data={"a": 0}), + Event(timestamp=now + 12 * td1s, duration=7, data={"b": 1}), + ] + + +def test_flood_forward_merge(): + events = [ + Event(timestamp=now, duration=10), + Event(timestamp=now + 15 * td1s, duration=5), + ] + flooded = flood(events) + assert len(flooded) == 1 + assert flooded[0].duration == timedelta(seconds=20) + + +def test_flood_backward_merge(): + events = [ + Event(timestamp=now, duration=5), + Event(timestamp=now + 10 * td1s, duration=10), + ] + flooded = flood(events) + assert len(flooded) == 1 + assert flooded[0].duration == timedelta(seconds=20) + + +def test_flood_negative_gap_same_data(): + events = [ + Event(timestamp=now, duration=100, data={"a": 0}), + Event(timestamp=now, duration=5, data={"a": 0}), + ] + flooded = flood(events) + total_duration = sum((e.duration for e in flooded), timedelta(0)) + assert len(flooded) == 1 + assert total_duration == timedelta(seconds=100) + + +def test_flood_negative_gap_differing_data(): + events = [ + Event(timestamp=now, duration=5, data={"a": 0}), + Event(timestamp=now, duration=100, data={"b": 1}), + ] + flooded = flood(events) + assert flooded == [events[1]] + + +def test_flood_zero_duration_chain_does_not_leave_overlaps(): + events = [ + Event(timestamp=now, duration=0, data={"title": "first"}), + Event(timestamp=now, duration=1, data={"title": "first"}), + Event(timestamp=now, duration=1, data={"title": "second"}), + ] + + flooded = flood(events) + + assert flooded == [events[2]] + assert all( + previous.timestamp + previous.duration <= current.timestamp + for previous, current in zip(flooded, flooded[1:]) + ) + + +def test_flood_normalization_preserves_non_overlapping_tail(): + events = [ + Event(timestamp=now, duration=10, data={"title": "first"}), + Event(timestamp=now + 5 * td1s, duration=2, data={"title": "second"}), + Event(timestamp=now + 12 * td1s, duration=1, data={"title": "third"}), + ] + + flooded = flood(events) + + assert flooded == [ + Event(timestamp=now, duration=5, data={"title": "first"}), + Event(timestamp=now + 5 * td1s, duration=4.5, data={"title": "second"}), + Event(timestamp=now + 9.5 * td1s, duration=3.5, data={"title": "third"}), + ] + + +def test_flood_normalization_merges_same_data_after_zero_duration_event(): + events = [ + Event(timestamp=now, duration=0, data={"title": "zero"}), + Event(timestamp=now, duration=10, data={"title": "same"}), + Event(timestamp=now + 5 * td1s, duration=10, data={"title": "same"}), + ] + + flooded = flood(events) + + assert flooded == [Event(timestamp=now, duration=15, data={"title": "same"})] + + +def test_flood_negative_small_gap_differing_data(): + events = [ + Event(timestamp=now, duration=100, data={"b": 1}), + Event(timestamp=now + 99.99 * td1s, duration=100, data={"a": 0}), + ] + flooded = flood(events) + duration = sum((e.duration for e in flooded), timedelta(0)) + assert duration == timedelta(seconds=100 + 99.99) + + +def test_flood_with_custom_pulsetime(): + # Events with a 30s gap between them (simulating 30s poll_time) + events = [ + Event(timestamp=now, duration=5, data={"a": 0}), + Event(timestamp=now + 35 * td1s, duration=5, data={"b": 1}), + ] + + # Default pulsetime=5: gap (30s) > pulsetime, so no flooding + flooded_default = flood(events) + assert len(flooded_default) == 2 + total_duration_default = sum((e.duration for e in flooded_default), timedelta(0)) + assert total_duration_default == timedelta(seconds=10) + + # pulsetime=31: gap (30s) <= pulsetime, so both events extend to midpoint + flooded_custom = flood(events, pulsetime=31) + assert flooded_custom == [ + Event(timestamp=now, duration=20, data={"a": 0}), + Event(timestamp=now + 20 * td1s, duration=20, data={"b": 1}), + ] + + +def test_flood_idempotent(): + """flood() applied repeatedly should produce the same result.""" + events = [ + # slight overlap, same data + Event(timestamp=now, duration=10, data={"a": 0}), + Event(timestamp=now + 9 * td1s, duration=5, data={"a": 0}), + # different data, no overlap + Event(timestamp=now + 15 * td1s, duration=5, data={"b": 0}), + ] + flood_first = flood(events, pulsetime=0) + flooded = flood_first + for _ in range(2): + flooded = flood(flooded, pulsetime=0) + assert flood_first == flooded + + assert sum((e.duration for e in flooded), timedelta(0)) == 19 * td1s + + +def test_flood_unsafe_gap(): + """Overlapping events with differing data must not double-count time.""" + events = [ + Event(timestamp=now, duration=10, data={"a": 0}), + Event(timestamp=now + 9 * td1s, duration=5, data={"b": 0}), + ] + flooded = flood(events, pulsetime=0) + + # Total duration must not exceed the span covered by the two events + assert sum((e.duration for e in flooded), timedelta(0)) == 14 * td1s diff --git a/tests/test_heartbeat.py b/tests/test_heartbeat.py new file mode 100644 index 0000000..8112f2a --- /dev/null +++ b/tests/test_heartbeat.py @@ -0,0 +1,84 @@ +from datetime import datetime, timedelta + +from aw_core.models import Event +from aw_transform import heartbeat_merge, heartbeat_reduce + + +def test_heartbeat_merge(): + """Events should merge""" + now = datetime.now() + td_1s = timedelta(seconds=1) + + last_event, heartbeat = Event(timestamp=now), Event(timestamp=now + td_1s) + merged = heartbeat_merge(last_event, heartbeat, pulsetime=2) + assert merged is not None + + +def test_heartbeat_merge_fail(): + """Merge should not happen""" + now = datetime.now() + td_1s = timedelta(seconds=1) + + # timestamp of heartbeat more than pulsetime away + last_event, heartbeat = ( + Event(timestamp=now, data={"label": "test"}), + Event(timestamp=now + 3 * td_1s, data={"label": "test"}), + ) + merged = heartbeat_merge(last_event, heartbeat, pulsetime=2) + assert merged is None + + # labels not identical + last_event, heartbeat = ( + Event(timestamp=now, data={"label": "test"}), + Event(timestamp=now + td_1s, data={"label": "test2"}), + ) + merged = heartbeat_merge(last_event, heartbeat, pulsetime=2) + assert merged is None + + +def test_heartbeat_reduce(): + """Events should reduce""" + now = datetime.now() + td_1s = timedelta(seconds=1) + + # Check that empty list works + assert not heartbeat_reduce([], pulsetime=1) + + events = [ + Event(timestamp=now, data={"label": "test"}), + Event(timestamp=now + td_1s, data={"label": "test"}), + ] + reduced_events = heartbeat_reduce(events, pulsetime=2) + assert len(reduced_events) == 1 + + +def test_heartbeat_same_timestamp(): + now = datetime.now() + td_1s = timedelta(seconds=1) + td_0s = timedelta(seconds=0) + + e1 = Event(timestamp=now, duration=td_1s, data={"label": "test"}) + e2 = Event(timestamp=now, duration=td_0s, data={"label": "test"}) + + # Should merge + res = heartbeat_reduce(list([e1, e2]), pulsetime=5) + assert len(res) == 1 + assert res[0].duration == td_1s + + # Order shouldn't matter, should merge + res = heartbeat_reduce([e2, e1], pulsetime=5) + assert len(res) == 1 + assert res[0].duration == td_1s + + +def test_heartbeat_reduce_fail(): + """Events should not reduce""" + now = datetime.now() + td_1s = timedelta(seconds=1) + + events = [ + Event(timestamp=now, data={"label": "test"}), + Event(timestamp=now + 3 * td_1s, data={"label": "test"}), + ] + reduced_events = heartbeat_reduce(events, pulsetime=2) + assert len(reduced_events) == 2 diff --git a/tests/test_peewee.py b/tests/test_peewee.py new file mode 100644 index 0000000..a382870 --- /dev/null +++ b/tests/test_peewee.py @@ -0,0 +1,66 @@ +""" +Peewee-storage-specific tests, mainly for the schema/index migration in +auto_migrate() and for guarding against query-plan regressions. +""" + +import logging + +from aw_datastore.storages import PeeweeStorage +from aw_datastore.storages import peewee as peewee_storage + +from . import context # noqa: F401 + +logging.basicConfig(level=logging.DEBUG) + + +def _indexes(db) -> set: + return {row[1] for row in db.execute_sql("PRAGMA index_list(eventmodel)")} + + +def test_index_migration(): + """auto_migrate() replaces the old single-column indexes with the composite one.""" + storage = PeeweeStorage(testing=True) + db = storage.db + path = db.database + + assert "eventmodel_bucket_id_timestamp" in _indexes(db) + + # Recreate the pre-migration state: single-column indexes, no composite + db.execute_sql("DROP INDEX eventmodel_bucket_id_timestamp") + db.execute_sql("CREATE INDEX eventmodel_bucket_id ON eventmodel (bucket_id)") + db.execute_sql("CREATE INDEX eventmodel_timestamp ON eventmodel (timestamp)") + + peewee_storage.auto_migrate(path) + + indexes = _indexes(db) + assert "eventmodel_bucket_id_timestamp" in indexes + assert "eventmodel_bucket_id" not in indexes + assert "eventmodel_timestamp" not in indexes + + +def test_query_plan_uses_composite_index(): + """ + The last-event query (issued by replace_last on every merged heartbeat) and + range queries must be served by the composite index in a single pass. + + Without it, SQLite reads *all* of a bucket's rows and sorts them in a temp + B-tree even for LIMIT 1, which on multi-million-event buckets meant seconds + of CPU/IO per heartbeat (and terabytes read per day). + """ + storage = PeeweeStorage(testing=True) + db = storage.db + + queries = [ + # replace_last/_get_last + "SELECT * FROM eventmodel WHERE bucket_id = 1" + " ORDER BY timestamp DESC LIMIT 1", + # get_events with a time range + "SELECT * FROM eventmodel WHERE bucket_id = 1" + " AND timestamp >= '2026-01-01' AND timestamp <= '2026-01-02'" + " ORDER BY timestamp DESC LIMIT 100", + ] + for query in queries: + rows = db.execute_sql(f"EXPLAIN QUERY PLAN {query}").fetchall() + plan = "\n".join(str(row) for row in rows) + assert "eventmodel_bucket_id_timestamp" in plan, plan + assert "TEMP B-TREE" not in plan, plan diff --git a/tests/test_query2.py b/tests/test_query2.py new file mode 100644 index 0000000..f7e864d --- /dev/null +++ b/tests/test_query2.py @@ -0,0 +1,715 @@ +from datetime import datetime, timedelta, timezone +from typing import Any, Dict + +import iso8601 +import pytest +from aw_core.models import Event +from aw_datastore.storages import MemoryStorage +from aw_query import query +from aw_query.exceptions import ( + QueryFunctionException, + QueryInterpretException, + QueryParseException, +) +from aw_query.query2 import ( + QDict, + QFunction, + QInteger, + QList, + QString, + QVariable, + _parse_token, + _split_query_statements, +) + +from .utils import param_datastore_objects + + +class MockDatastore(MemoryStorage): + pass + + +mock_ds = MockDatastore(testing=True) + + +def test_query2_test_token_parsing(): + ns: Dict[str, Any] = {} + (t, token), trash = _parse_token("123", ns) + assert token == "123" + assert t == QInteger + (t, token), trash = _parse_token('"test"', ns) + assert token == '"test"' + assert t == QString + (t, token), trash = _parse_token("'test'", ns) + assert token == "'test'" + assert t == QString + (t, token), trash = _parse_token("'te\\'st'", ns) + assert token == "'te\\'st'" + assert t == QString + (t, token), trash = _parse_token('"te\\"st"', ns) + assert token == '"te\\"st"' + assert t == QString + (t, token), trash = _parse_token("test0xDEADBEEF", ns) + assert token == "test0xDEADBEEF" + assert t == QVariable + (t, token), trash = _parse_token("test1337(')')", ns) + assert token == "test1337(')')" + assert t == QFunction + (t, token), trash = _parse_token("test1337('test\\'test',\"test\\\"test\")", ns) + assert token == "test1337('test\\'test',\"test\\\"test\")" + assert t == QFunction + (t, token), trash = _parse_token("[1, 'a', {}]", ns) + assert token == "[1, 'a', {}]" + assert t == QList + (t, token), trash = _parse_token("{'a': 1, 'b}': 2}", ns) + assert token == "{'a': 1, 'b}': 2}" + assert t == QDict + + assert _parse_token("", ns) == ((None, ""), "") + + with pytest.raises(QueryParseException): + _parse_token(None, ns) # type: ignore + + with pytest.raises(QueryParseException): + _parse_token('"', ns) + + with pytest.raises(QueryParseException): + _parse_token("#", ns) + + +def test_dict(): + ds = mock_ds + ns: Dict[str, Any] = {} + d_str = "{'a': {'a': {'a': 1}}, 'b': {'b\\'\"': ':'}}" + d = QDict.parse(d_str, ns) + expected_res = {"a": {"a": {"a": 1}}, "b": {"b'\"": ":"}} + assert expected_res == d.interpret(ds, ns) + + # Key in dict is not a string + with pytest.raises(QueryParseException): + d_str = "{b: 1}" + d = QDict.parse(d_str, ns) + + # Key in dict without a value + with pytest.raises(QueryParseException): + d_str = "{'test': }" + d = QDict.parse(d_str, ns) + + # Char following key string is not a : + with pytest.raises(QueryParseException): + d_str = "{'test'p 1}" + d = QDict.parse(d_str, ns) + + with pytest.raises(QueryParseException): + d_str = "{'test': #}" + d = QDict.parse(d_str, ns) + + # Semicolon without key + with pytest.raises(QueryParseException): + d_str = "{:}" + d = QDict.parse(d_str, ns) + + # Trailing comma + with pytest.raises(QueryParseException): + d_str = "{'test':1,}" + d = QDict.parse(d_str, ns) + + +def test_list(): + ds = mock_ds + ns: Dict[str, Any] = {} + l_str = "[1,2,[[3],4],5]" + ls = QList.parse(l_str, ns) + expected_res = [1, 2, [[3], 4], 5] + assert expected_res == ls.interpret(ds, ns) + + l_str = "['\\'',\"\\\"\"]" + ls = QList.parse(l_str, ns) + expected_res = ["'", '"'] + assert expected_res == ls.interpret(ds, ns) + + l_str = "[]" + ls = QList.parse(l_str, ns) + expected_res = [] + assert expected_res == ls.interpret(ds, ns) + + # Comma without pre/post value + with pytest.raises(QueryParseException): + l_str = "[,]" + ls = QList.parse(l_str, ns) + + # Comma without post value + with pytest.raises(QueryParseException): + l_str = "[1,]" + ls = QList.parse(l_str, ns) + + # Comma without pre value + with pytest.raises(QueryParseException): + l_str = "[,2]" + ls = QList.parse(l_str, ns) + + +def test_query2_bogus_query(): + ds = mock_ds + qname = "test" + qstartdate = datetime.now(tz=timezone.utc) + qenddate = qstartdate + + # Nothing to assign + with pytest.raises(QueryParseException): + example_query = "a=" + query(qname, example_query, qstartdate, qenddate, ds) + + # Assign to non-variable + with pytest.raises(QueryParseException): + example_query = "1 = 2" + query(qname, example_query, qstartdate, qenddate, ds) + + # Unclosed function + with pytest.raises(QueryParseException): + example_query = "a = unclosed_function(var1" + query(qname, example_query, qstartdate, qenddate, ds) + + # Two tokens in assignment + with pytest.raises(QueryParseException): + example_query = "asd nop() = 2" + query(qname, example_query, qstartdate, qenddate, ds) + + # Unclosed string + with pytest.raises(QueryParseException): + example_query = 'asd = "something is wrong with me' + query(qname, example_query, qstartdate, qenddate, ds) + + # Two tokens in value + with pytest.raises(QueryParseException): + example_query = "asd = asd1 asd2" + query(qname, example_query, qstartdate, qenddate, ds) + + +def test_query2_query_function_calling(): + ds = mock_ds + qname = "asd" + starttime = iso8601.parse_date("1970-01-01") + endtime = iso8601.parse_date("1970-01-02") + + # Function which doesn't exist + with pytest.raises(QueryInterpretException): + example_query = "RETURN = asd();" + query(qname, example_query, starttime, endtime, ds) + + # Function which does exist with invalid arguments + with pytest.raises(QueryInterpretException): + example_query = "RETURN = nop(badarg);" + query(qname, example_query, starttime, endtime, ds) + + # Function which does exist with valid arguments + example_query = "RETURN = nop();" + query(qname, example_query, starttime, endtime, ds) + + +def test_query2_return_value(): + ds = mock_ds + qname = "asd" + starttime = iso8601.parse_date("1970-01-01") + endtime = iso8601.parse_date("1970-01-02") + example_query = "RETURN = 1;" + result = query(qname, example_query, starttime, endtime, ds) + assert result == 1 + + example_query = "RETURN = 'testing 123'" + result = query(qname, example_query, starttime, endtime, ds) + assert result == "testing 123" + + example_query = "RETURN = {'a': 1}" + result = query(qname, example_query, starttime, endtime, ds) + assert result == {"a": 1} + + # Nothing to return + with pytest.raises(QueryParseException): + example_query = "a=1" + result = query(qname, example_query, starttime, endtime, ds) + + +def test_query2_semicolon_in_string(): + """Test that semicolons inside string literals don't break query parsing (issue #145).""" + ds = mock_ds + qname = "asd" + starttime = iso8601.parse_date("1970-01-01") + endtime = iso8601.parse_date("1970-01-02") + + # Semicolon in double-quoted string + example_query = 'RETURN = "hello;world"' + result = query(qname, example_query, starttime, endtime, ds) + assert result == "hello;world" + + # Semicolon in single-quoted string + example_query = "RETURN = 'hello;world'" + result = query(qname, example_query, starttime, endtime, ds) + assert result == "hello;world" + + # Multiple statements where one value contains a semicolon in a string + example_query = 'a = "foo;bar"; RETURN = a' + result = query(qname, example_query, starttime, endtime, ds) + assert result == "foo;bar" + + +def test_split_query_statements(): + """Unit test for _split_query_statements.""" + # Basic split + assert _split_query_statements("a=1;b=2") == ["a=1", "b=2"] + # Semicolon inside double quotes should NOT split + assert _split_query_statements('a="x;y";b=2') == ['a="x;y"', "b=2"] + # Semicolon inside single quotes should NOT split + assert _split_query_statements("a='x;y';b=2") == ["a='x;y'", "b=2"] + # Trailing semicolon — empty trailing part is omitted + assert _split_query_statements("a=1;") == ["a=1"] + + +def test_query2_multiline(): + ds = mock_ds + qname = "asd" + starttime = iso8601.parse_date("1970-01-01") + endtime = iso8601.parse_date("1970-01-02") + example_query = """ +my_multiline_string = "a +b"; +RETURN = my_multiline_string; + """ + result = query(qname, example_query, starttime, endtime, ds) + assert result == "a\nb" + + +def test_query2_function_invalid_types(): + """Tests the q2_typecheck decorator""" + ds = mock_ds + qname = "asd" + starttime = iso8601.parse_date("1970-01-01") + endtime = iso8601.parse_date("1970-01-02") + + # int instead of str + example_query = """ + events = []; + RETURN = filter_keyvals(events, 666, ["invalid_val"]); + """ + with pytest.raises(QueryFunctionException): + query(qname, example_query, starttime, endtime, ds) + + # str instead of list + example_query = """ + events = []; + RETURN = filter_keyvals(events, "2", "invalid_val"); + """ + with pytest.raises(QueryFunctionException): + query(qname, example_query, starttime, endtime, ds) + + # FIXME: For unknown reasons, query2 drops the second argument + # when the first argument is a bare [] + """ + example_query = ''' + RETURN = filter_keyvals([], "2", "invalid_val"); + ''' + with pytest.raises(QueryFunctionException) as e: + result = query(qname, example_query, starttime, endtime, None) + """ + + +def test_query2_function_invalid_argument_count(): + ds = mock_ds + qname = "asd" + starttime = iso8601.parse_date("1970-01-01") + endtime = iso8601.parse_date("1970-01-02") + example_query = "RETURN=nop(nop())" + with pytest.raises(QueryInterpretException): + query(qname, example_query, starttime, endtime, ds) + + +def test_query2_merge_subwatcher_fields_invalid_conflict(): + ds = mock_ds + qname = "asd" + starttime = iso8601.parse_date("1970-01-01") + endtime = iso8601.parse_date("1970-01-02") + example_query = """ + base = []; + sub = []; + RETURN = merge_subwatcher_fields(base, sub, ["project"], "invalid"); + """ + + with pytest.raises(QueryFunctionException, match="conflict must be"): + query(qname, example_query, starttime, endtime, ds) + + +def test_query2_categorize_invalid_priority(): + ds = mock_ds + qname = "asd" + starttime = iso8601.parse_date("1970-01-01") + endtime = iso8601.parse_date("1970-01-02") + example_query = """ + events = []; + RETURN = categorize(events, [[["test"], {"regex": "test", "priority": "high"}]]); + """ + with pytest.raises(QueryFunctionException, match="integer"): + query(qname, example_query, starttime, endtime, ds) + + +@pytest.mark.parametrize("datastore", param_datastore_objects()) +def test_query2_function_in_function(datastore): + qname = "asd" + bid = "test_bucket" + starttime = iso8601.parse_date("1970-01-01") + endtime = iso8601.parse_date("1970-01-02") + example_query = f""" + RETURN=limit_events(query_bucket("{bid}"), 1); + """ + try: + # Setup buckets + bucket1 = datastore.create_bucket( + bucket_id=bid, type="test", client="test", hostname="test", name="test" + ) + # Prepare buckets + e1 = Event(data={}, timestamp=starttime, duration=timedelta(seconds=1)) + bucket1.insert(e1) + result = query(qname, example_query, starttime, endtime, datastore) + assert 1 == len(result) + finally: + datastore.delete_bucket(bid) + + +@pytest.mark.parametrize("datastore", param_datastore_objects()) +def test_query2_query_functions(datastore): + """ + Just test calling all functions just to see something isn't completely broken + In many cases the functions doesn't change the result at all, so it's not a test + for testing the validity of the data the functions transform + """ + bid = "test_'bucket" + qname = "test" + starttime = iso8601.parse_date("1970") + endtime = starttime + timedelta(hours=1) + + example_query = """ + bid = "{bid}"; + events = query_bucket("{bid}"); + events2 = query_bucket('{bid_escaped}'); + events2 = filter_keyvals(events2, "label", ["test1"]); + events2 = exclude_keyvals(events2, "label", ["test2"]); + events = filter_period_intersect(events, events2); + events = filter_keyvals_regex(events, "label", ".*"); + events = limit_events(events, 1); + events = merge_events_by_keys(events, ["label"]); + events = chunk_events_by_key(events, "label"); + events = split_url_events(events); + events = sort_by_timestamp(events); + events = sort_by_duration(events); + events = categorize(events, [[["test", "subtest"], {{"regex": "test1"}}]]); + duration = sum_durations(events); + eventcount = query_bucket_eventcount(bid); + asd = nop(); + RETURN = {{"events": events, "eventcount": eventcount}}; + """.format(bid=bid, bid_escaped=bid.replace("'", "\\'")) + try: + bucket = datastore.create_bucket( + bucket_id=bid, type="test", client="test", hostname="test", name="asd" + ) + bucket.insert( + Event( + data={"label": "test1"}, + timestamp=starttime, + duration=timedelta(seconds=1), + ) + ) + result = query(qname, example_query, starttime, endtime, datastore) + assert result["eventcount"] == 1 + assert len(result["events"]) == 1 + assert result["events"][0].data["label"] == "test1" + assert result["events"][0].data["$category"] == ["test", "subtest"] + finally: + datastore.delete_bucket(bid) + + +@pytest.mark.parametrize("datastore", param_datastore_objects()) +def test_query2_basic_query(datastore): + name = "A label/name for a test bucket" + bid1 = "bucket1" + bid2 = "bucket2" + qname = "test_query_basic" + starttime = iso8601.parse_date("1970") + endtime = starttime + timedelta(hours=1) + + example_query = f""" + bid1 = "{bid1}"; + bid2 = "{bid2}"; + events = query_bucket(bid1); + intersect_events = query_bucket(bid2); + RETURN = filter_period_intersect(events, intersect_events); + """ + + try: + # Setup buckets + bucket1 = datastore.create_bucket( + bucket_id=bid1, type="test", client="test", hostname="test", name=name + ) + bucket2 = datastore.create_bucket( + bucket_id=bid2, type="test", client="test", hostname="test", name=name + ) + # Prepare buckets + e1 = Event( + data={"label": "test1"}, timestamp=starttime, duration=timedelta(seconds=1) + ) + e2 = Event( + data={"label": "test2"}, + timestamp=starttime + timedelta(seconds=2), + duration=timedelta(seconds=1), + ) + et = Event( + data={"label": "intersect-label"}, + timestamp=starttime, + duration=timedelta(seconds=1), + ) + bucket1.insert(e1) + bucket1.insert(e2) + bucket2.insert(et) + # Query + result = query(qname, example_query, starttime, endtime, datastore) + # Assert + assert len(result) == 1 + assert result[0]["data"]["label"] == "test1" + finally: + datastore.delete_bucket(bid1) + datastore.delete_bucket(bid2) + + +@pytest.mark.parametrize("datastore", param_datastore_objects()) +def test_query2_test_merged_keys(datastore): + name = "A label/name for a test bucket" + bid = "bucket1" + qname = "test_query_merged_keys" + starttime = iso8601.parse_date("2080") + endtime = starttime + timedelta(hours=1) + + example_query = f""" + bid1 = "{bid}"; + events = query_bucket(bid1); + events = merge_events_by_keys(events, ["label1", "label2"]); + events = sort_by_duration(events); + eventcount = query_bucket_eventcount(bid1); + RETURN = {{"events": events, "eventcount": eventcount}}; + """ + try: + # Setup buckets + bucket1 = datastore.create_bucket( + bucket_id=bid, type="test", client="test", hostname="test", name=name + ) + # Prepare buckets + e1 = Event( + data={"label1": "test1", "label2": "test1"}, + timestamp=starttime, + duration=timedelta(seconds=1), + ) + e2 = Event( + data={"label1": "test1", "label2": "test1"}, + timestamp=starttime + timedelta(seconds=1), + duration=timedelta(seconds=1), + ) + e3 = Event( + data={"label1": "test1", "label2": "test2"}, + timestamp=starttime + timedelta(seconds=2), + duration=timedelta(seconds=1), + ) + bucket1.insert(e3) + bucket1.insert(e1) + bucket1.insert(e2) + # Query + result = query(qname, example_query, starttime, endtime, datastore) + # Assert + print(result) + assert len(result["events"]) == 2 + assert result["eventcount"] == 3 + assert result["events"][0]["data"]["label1"] == "test1" + assert result["events"][0]["data"]["label2"] == "test1" + assert result["events"][0]["duration"] == timedelta(seconds=2) + assert result["events"][1]["data"]["label1"] == "test1" + assert result["events"][1]["data"]["label2"] == "test2" + assert result["events"][1]["duration"] == timedelta(seconds=1) + finally: + datastore.delete_bucket(bid) + + +@pytest.mark.parametrize("datastore", param_datastore_objects()) +def test_query2_fancy_query(datastore): + """ + Tests: + - find_bucket + - simplify_window_titles + """ + name = "A label/name for a test bucket" + bid1 = "bucket-the-one" + qname = "test_query_basic_fancy" + starttime = iso8601.parse_date("1970") + endtime = starttime + timedelta(hours=1) + + example_query = f""" + bid = find_bucket("{bid1[:10]}"); + events = query_bucket(bid); + RETURN = simplify_window_titles(events, "title"); + """ + + try: + # Setup buckets + bucket_main = datastore.create_bucket( + bucket_id=bid1, type="test", client="test", hostname="test", name=name + ) + # Prepare buckets + e1 = Event( + data={"title": "(2) YouTube"}, + timestamp=starttime, + duration=timedelta(seconds=1), + ) + bucket_main.insert(e1) + # Query + result = query(qname, example_query, starttime, endtime, datastore) + # Assert + assert result[0]["data"]["title"] == "YouTube" + finally: + datastore.delete_bucket(bid1) + + +@pytest.mark.parametrize("datastore", param_datastore_objects()) +def test_query2_query_categorize(datastore): + bid = "test_bucket" + qname = "test" + starttime = iso8601.parse_date("1970") + endtime = starttime + timedelta(hours=1) + + example_query = rf""" + events = query_bucket("{bid}"); + events = sort_by_timestamp(events); + events = categorize(events, [ + [["test"], {{"regex": "test"}}], + [["test", "subtest"], {{"regex": "test\w"}}] + ]); + events_by_cat = merge_events_by_keys(events, ["$category"]); + RETURN = {{"events": events, "events_by_cat": events_by_cat}}; + """ + try: + bucket = datastore.create_bucket( + bucket_id=bid, type="test", client="test", hostname="test", name="asd" + ) + events = [ + Event( + data={"label": "test"}, + timestamp=starttime, + duration=timedelta(seconds=1), + ), + Event( + data={"label": "testwithmoredetail"}, + timestamp=starttime + timedelta(seconds=1), + duration=timedelta(seconds=1), + ), + Event( + data={"label": "testwithmoredetail"}, + timestamp=starttime + timedelta(seconds=2), + duration=timedelta(seconds=1), + ), + ] + bucket.insert(events) + result = query(qname, example_query, starttime, endtime, datastore) + print(result) + assert len(result["events"]) == 3 + assert result["events"][0].data["label"] == "test" + assert result["events"][0].data["$category"] == ["test"] + assert result["events"][1].data["$category"] == ["test", "subtest"] + + assert len(result["events_by_cat"]) == 2 + assert result["events_by_cat"][0].data["$category"] == ["test"] + assert result["events_by_cat"][1].data["$category"] == ["test", "subtest"] + assert result["events_by_cat"][1].duration == timedelta(seconds=2) + finally: + datastore.delete_bucket(bid) + + +@pytest.mark.parametrize("datastore", param_datastore_objects()) +def test_query2_query_categorize_priority(datastore): + bid = "test_bucket_priority" + qname = "test" + starttime = iso8601.parse_date("1970") + endtime = starttime + timedelta(hours=1) + + example_query = rf""" + events = query_bucket("{bid}"); + events = categorize(events, [ + [["A"], {{"regex": "test", "priority": 25}}], + [["B", "B1"], {{"regex": "test"}}] + ]); + RETURN = events; + """ + try: + bucket = datastore.create_bucket( + bucket_id=bid, type="test", client="test", hostname="test", name="asd" + ) + bucket.insert( + Event( + data={"label": "test"}, + timestamp=starttime, + duration=timedelta(seconds=1), + ) + ) + result = query(qname, example_query, starttime, endtime, datastore) + assert len(result) == 1 + assert result[0].data["$category"] == ["A"] + finally: + datastore.delete_bucket(bid) + + +@pytest.mark.parametrize("datastore", param_datastore_objects()) +def test_query2_flood_with_pulsetime(datastore): + """Test that flood() accepts an optional pulsetime argument (fixes WorkReport query bug).""" + bid = "test_flood_bucket" + qname = "test_flood" + starttime = iso8601.parse_date("1970") + endtime = starttime + timedelta(hours=1) + + try: + bucket = datastore.create_bucket( + bucket_id=bid, + type="test", + client="test", + hostname="test", + name="flood_test", + ) + # Two events with a 60-second gap — flood with pulsetime=300 should merge them + bucket.insert( + Event( + data={"app": "Firefox"}, + timestamp=starttime, + duration=timedelta(seconds=10), + ) + ) + bucket.insert( + Event( + data={"app": "Firefox"}, + timestamp=starttime + timedelta(seconds=70), + duration=timedelta(seconds=10), + ) + ) + + # flood with default pulsetime (5s) — gap is 60s, events should NOT merge + result_default = query( + qname, + f'events = query_bucket("{bid}"); events = flood(events); RETURN = events;', + starttime, + endtime, + datastore, + ) + assert len(result_default) == 2 + + # flood with pulsetime=300 — gap is 60s < 300s, events SHOULD merge + result_pulsetime = query( + qname, + f'events = query_bucket("{bid}"); events = flood(events, 300); RETURN = events;', + starttime, + endtime, + datastore, + ) + assert len(result_pulsetime) == 1 + finally: + datastore.delete_bucket(bid) diff --git a/tests/test_schemas.py b/tests/test_schemas.py new file mode 100644 index 0000000..558cf42 --- /dev/null +++ b/tests/test_schemas.py @@ -0,0 +1,60 @@ +import unittest + +from jsonschema import validate as _validate, FormatChecker +from jsonschema.exceptions import ValidationError + +from aw_core import schema +from aw_core.models import Event + +# TODO: Include date-time format +# https://python-jsonschema.readthedocs.io/en/latest/validate/#jsonschema.FormatChecker + +# The default FormatChecker, uses the date-time checker +fc = FormatChecker(["date-time"]) + +valid_timestamp = "1937-01-01T12:00:27.87+00:20" + + +event_schema = schema.get_json_schema("event") + + +class EventSchemaTest(unittest.TestCase): + def setUp(self): + self.schema = event_schema + + def validate(self, obj): + _validate(obj, self.schema, format_checker=fc) + + def test_event(self): + event = Event(timestamp=valid_timestamp, data={"label": "test"}) + self.validate(event.to_json_dict()) + + def test_data(self): + self.validate( + {"timestamp": valid_timestamp, "data": {"label": "test", "number": 1.1}} + ) + + def test_timestamp(self): + self.validate({"timestamp": valid_timestamp}) + + def test_timestamp_invalid_string(self): + with self.assertRaises(ValidationError): + self.validate({"timestamp": ""}) + with self.assertRaises(ValidationError): + self.validate({"timestamp": "123"}) + + def test_timestamp_invalid_number(self): + with self.assertRaises(ValidationError): + self.validate({"timestamp": 2}) + + def test_duration(self): + self.validate({"timestamp": valid_timestamp, "duration": 1000}) + self.validate({"timestamp": valid_timestamp, "duration": 3.13}) + + def test_duration_invalid_string(self): + with self.assertRaises(ValidationError): + self.validate({"timestamp": valid_timestamp, "duration": "not a number"}) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_transforms.py b/tests/test_transforms.py new file mode 100644 index 0000000..c30e97d --- /dev/null +++ b/tests/test_transforms.py @@ -0,0 +1,830 @@ +from pprint import pprint +from datetime import datetime, timedelta, timezone + +import pytest +from aw_core.models import Event +from aw_transform import ( + filter_period_intersect, + filter_keyvals_regex, + filter_keyvals, + merge_subwatcher_fields, + period_union, + sort_by_timestamp, + sort_by_duration, + sum_durations, + merge_events_by_keys, + chunk_events_by_key, + split_url_events, + simplify_string, + union, + union_no_overlap, + categorize, + tag, + Rule, +) +from aw_transform.filter_period_intersect import _intersecting_eventpairs + + +def test_simplify_string(): + events = [ + Event(data={"label": "(99) Facebook"}), + Event(data={"label": "(14) YouTube"}), + ] + assert simplify_string(events, "label")[0].data["label"] == "Facebook" + assert simplify_string(events, "label")[1].data["label"] == "YouTube" + + events = [Event(data={"app": "Cemu.exe", "title": "Cemu - FPS: 133.7 - BotW"})] + assert simplify_string(events, "title")[0].data["title"] == "Cemu - FPS: ... - BotW" + + events = [ + Event(data={"app": "VSCode.exe", "title": "● report.md - Visual Studio Code"}) + ] + assert ( + simplify_string(events, "title")[0].data["title"] + == "report.md - Visual Studio Code" + ) + + events = [Event(data={"app": "Gedit", "title": "*test.md - gedit"})] + assert simplify_string(events, "title")[0].data["title"] == "test.md - gedit" + + +def test_filter_keyval(): + labels = ["aa", "cc"] + events = [ + Event(data={"label": "aa"}), + Event(data={"label": "bb"}), + Event(data={"label": "cc"}), + ] + included_events = filter_keyvals(events, "label", labels) + excluded_events = filter_keyvals(events, "label", labels, exclude=True) + assert len(included_events) == 2 + assert len(excluded_events) == 1 + + +def test_filter_keyval_regex(): + events = [ + Event(data={"label": "aa"}), + Event(data={"label": "bb"}), + Event(data={"label": "cc"}), + ] + events_re = filter_keyvals_regex(events, "label", "aa|cc") + assert len(events_re) == 2 + + +def test_filter_keyval_regex_keyerror(): + events = [ + Event(data={"label": "aa"}), + Event(), + Event(data={"label": "cc"}), + ] + events_re = filter_keyvals_regex(events, "label", "aa|cc") + assert len(events_re) == 2 + + +def test_intersecting_eventpairs(): + td1h = timedelta(hours=1) + now = datetime.now() + + # Test with two identical lists + e1 = [ + Event(timestamp=now, duration=td1h), + Event(timestamp=now + td1h, duration=td1h), + ] + e2 = [ + Event(timestamp=now, duration=td1h), + Event(timestamp=now + td1h, duration=td1h), + ] + intersecting = list(_intersecting_eventpairs(e1, e2)) + assert len(intersecting) == 2 + + # Test with events in first list being in between events of second list + e1 = [ + Event(timestamp=now + td1h, duration=td1h), + ] + e2 = [ + Event(timestamp=now, duration=td1h), + Event(timestamp=now + 2 * td1h, duration=td1h), + ] + intersecting = list(_intersecting_eventpairs(e1, e2)) + assert not intersecting + + # Test with event in first list being identical to middle event in second list + e1 = [ + Event(timestamp=now + td1h, duration=td1h), + ] + e2 = [ + Event(timestamp=now, duration=td1h), + Event(timestamp=now + 1 * td1h, duration=td1h), + Event(timestamp=now + 2 * td1h, duration=td1h), + ] + intersecting = list(_intersecting_eventpairs(e1, e2)) + assert len(intersecting) == 1 + + # Test same as before, but reversed + e1 = list(reversed(e1)) + e2 = list(reversed(e2)) + intersecting = list(_intersecting_eventpairs(e1, e2)) + assert len(intersecting) == 1 + + +def test_filter_period_intersect(): + td1h = timedelta(hours=1) + td30min = timedelta(minutes=30) + now = datetime.now() + + # Filter 1h event with another 1h event at a 30min offset + to_filter = [Event(timestamp=now, duration=td1h)] + filter_with = [Event(timestamp=now + td30min, duration=td1h)] + filtered_events = filter_period_intersect(to_filter, filter_with) + assert filtered_events[0].duration == td30min + + # Filter 2x 30min events with a 15min gap with another 45min event in between intersecting both + to_filter = [ + Event(timestamp=now, duration=td30min), + Event(timestamp=now + timedelta(minutes=45), duration=td30min), + ] + filter_with = [ + Event(timestamp=now + timedelta(minutes=15), duration=timedelta(minutes=45)) + ] + filtered_events = filter_period_intersect(to_filter, filter_with) + assert len(filtered_events) == 2 + assert filtered_events[0].duration == timedelta(minutes=15) + assert filtered_events[1].duration == timedelta(minutes=15) + + # Same as previous intersection, but reversing filter and to_filter events + to_filter = [ + Event(timestamp=now + timedelta(minutes=15), duration=timedelta(minutes=45)) + ] + filter_with = [ + Event(timestamp=now, duration=td30min), + Event(timestamp=now + timedelta(minutes=45), duration=td30min), + ] + filtered_events = filter_period_intersect(to_filter, filter_with) + assert len(filtered_events) == 2 + assert filtered_events[0].duration == timedelta(minutes=15) + assert filtered_events[1].duration == timedelta(minutes=15) + + +def test_period_union(): + now = datetime.now(timezone.utc) + + # Events overlapping + events1 = [Event(timestamp=now, duration=timedelta(seconds=10))] + events2 = [ + Event(timestamp=now + timedelta(seconds=9), duration=timedelta(seconds=10)) + ] + unioned_events = period_union(events1, events2) + assert len(unioned_events) == 1 + + # Events adjacent but not overlapping + events1 = [Event(timestamp=now, duration=timedelta(seconds=10))] + events2 = [ + Event(timestamp=now + timedelta(seconds=10), duration=timedelta(seconds=10)) + ] + unioned_events = period_union(events1, events2) + assert len(unioned_events) == 1 + + # Events not overlapping or adjacent + events1 = [Event(timestamp=now, duration=timedelta(seconds=10))] + events2 = [ + Event(timestamp=now + timedelta(seconds=11), duration=timedelta(seconds=10)) + ] + unioned_events = period_union(events1, events2) + assert len(unioned_events) == 2 + + +def test_sort_by_timestamp(): + now = datetime.now(timezone.utc) + events = [] + events.append( + Event(timestamp=now + timedelta(seconds=2), duration=timedelta(seconds=1)) + ) + events.append( + Event(timestamp=now + timedelta(seconds=1), duration=timedelta(seconds=2)) + ) + events_sorted = sort_by_timestamp(events) + assert events_sorted == events[::-1] + + +def test_sort_by_duration(): + now = datetime.now(timezone.utc) + events = [] + events.append( + Event(timestamp=now + timedelta(seconds=2), duration=timedelta(seconds=1)) + ) + events.append( + Event(timestamp=now + timedelta(seconds=1), duration=timedelta(seconds=2)) + ) + events_sorted = sort_by_duration(events) + assert events_sorted == events[::-1] + + +def test_sum_durations(): + now = datetime.now(timezone.utc) + events = [] + for i in range(10): + events.append( + Event(timestamp=now + timedelta(seconds=i), duration=timedelta(seconds=1)) + ) + result = sum_durations(events) + assert result == timedelta(seconds=10) + + +def test_merge_events_by_keys_1(): + now = datetime.now(timezone.utc) + events = [] + e1_data = {"label": "a"} + e2_data = {"label": "b"} + e1 = Event(data=e1_data, timestamp=now, duration=timedelta(seconds=1)) + e2 = Event(data=e2_data, timestamp=now, duration=timedelta(seconds=1)) + events = events + [e1] * 10 + events = events + [e2] * 5 + + # Check that an empty key list has no effect + assert merge_events_by_keys(events, []) == events + + # Check that trying to merge on unavailable key has no effect + assert len(merge_events_by_keys(events, ["unknown"])) == 1 + + result = merge_events_by_keys(events, ["label"]) + result = sort_by_duration(result) + print(result) + print(len(result)) + assert len(result) == 2 + assert result[0].duration == timedelta(seconds=10) + assert result[1].duration == timedelta(seconds=5) + + +def test_merge_events_by_keys_2(): + now = datetime.now(timezone.utc) + events = [] + e1_data = {"k1": "a", "k2": "a"} + e2_data = {"k1": "a", "k2": "c"} + e3_data = {"k1": "b", "k2": "a"} + e1 = Event(data=e1_data, timestamp=now, duration=timedelta(seconds=1)) + e2 = Event(data=e2_data, timestamp=now, duration=timedelta(seconds=1)) + e3 = Event(data=e3_data, timestamp=now, duration=timedelta(seconds=1)) + events = events + [e1] * 10 + events = events + [e2] * 9 + events = events + [e3] * 8 + result = merge_events_by_keys(events, ["k1", "k2"]) + result = sort_by_duration(result) + print(result) + print(len(result)) + assert len(result) == 3 + assert result[0].data == e1_data + assert result[0].duration == timedelta(seconds=10) + assert result[1].data == e2_data + assert result[1].duration == timedelta(seconds=9) + assert result[2].data == e3_data + assert result[2].duration == timedelta(seconds=8) + + +def test_chunk_events_by_key(): + now = datetime.now(timezone.utc) + events = [] + e1_data = {"label1": "1a", "label2": "2a"} + e2_data = {"label1": "1a", "label2": "2b"} + e3_data = {"label1": "1b", "label2": "2b"} + e1 = Event(data=e1_data, timestamp=now, duration=timedelta(seconds=1)) + e2 = Event(data=e2_data, timestamp=now, duration=timedelta(seconds=1)) + e3 = Event(data=e3_data, timestamp=now, duration=timedelta(seconds=1)) + events = [e1, e2, e3] + result = chunk_events_by_key(events, "label1") + print(len(result)) + pprint(result) + assert len(result) == 2 + # Check root label + assert result[0].data["label1"] == "1a" + assert result[1].data["label1"] == "1b" + # Check timestamp + assert result[0].timestamp == e1.timestamp + assert result[1].timestamp == e3.timestamp + # Check duration + assert result[0].duration == e1.duration + e2.duration + assert result[1].duration == e3.duration + # Check subevents + assert result[0].data["subevents"][0] == e1 + assert result[0].data["subevents"][1] == e2 + assert result[1].data["subevents"][0] == e3 + + +def test_url_parse_event(): + now = datetime.now(timezone.utc) + e = Event( + data={"url": "http://asd.com/test/?a=1"}, + timestamp=now, + duration=timedelta(seconds=1), + ) + result = split_url_events([e]) + print(result) + assert result[0].data["$protocol"] == "http" + assert result[0].data["$domain"] == "asd.com" + assert result[0].data["$path"] == "/test/" + assert result[0].data["$params"] == "" + assert result[0].data["$options"] == "a=1" + assert result[0].data["$identifier"] == "" + + e2 = Event( + data={"url": "https://www.asd.asd.com/test/test2/meh;meh2?asd=2&asdf=3#id"}, + timestamp=now, + duration=timedelta(seconds=1), + ) + result = split_url_events([e2]) + print(result) + assert result[0].data["$protocol"] == "https" + assert result[0].data["$domain"] == "asd.asd.com" + assert result[0].data["$path"] == "/test/test2/meh" + assert result[0].data["$params"] == "meh2" + assert result[0].data["$options"] == "asd=2&asdf=3" + assert result[0].data["$identifier"] == "id" + + e3 = Event( + data={"url": "file:///home/johan/myfile.txt"}, + timestamp=now, + duration=timedelta(seconds=1), + ) + result = split_url_events([e3]) + print(result) + assert result[0].data["$protocol"] == "file" + assert result[0].data["$domain"] == "file" + assert result[0].data["$path"] == "/home/johan/myfile.txt" + assert result[0].data["$params"] == "" + assert result[0].data["$options"] == "" + assert result[0].data["$identifier"] == "" + + # Test about: URLs + e4 = Event( + data={"url": "about:blank"}, + timestamp=now, + duration=timedelta(seconds=1), + ) + result = split_url_events([e4]) + assert result[0].data["$protocol"] == "about" + assert result[0].data["$domain"] == "about" + + +def test_union(): + now = datetime.now(timezone.utc) + + e1 = Event(timestamp=now - timedelta(seconds=20), duration=timedelta(seconds=5)) + e2 = Event(timestamp=now - timedelta(seconds=10), duration=timedelta(seconds=5)) + e3 = Event(timestamp=now, duration=timedelta(seconds=1)) + e4 = Event(timestamp=now + timedelta(seconds=20), duration=timedelta(seconds=1)) + + # union separate event lists with duplicates + events_union = union([e1, e2, e4], [e2, e3]) + assert events_union == [e1, e2, e3, e4] + + e1 = Event(timestamp=now - timedelta(seconds=20), duration=timedelta(seconds=5)) + e2 = Event(timestamp=now - timedelta(seconds=10), duration=timedelta(seconds=5)) + e3 = Event(timestamp=now - timedelta(seconds=10), duration=timedelta(seconds=10)) + e4 = Event(timestamp=now - timedelta(seconds=5), duration=timedelta(seconds=5)) + e5 = Event(timestamp=now, duration=timedelta(seconds=10)) + + # union event lists with intersecting duplicates + events_union = union([e3, e2, e5], [e1, e3, e4, e5]) + assert events_union == [e1, e2, e3, e4, e5] + + e1 = Event(timestamp=now - timedelta(seconds=30), duration=timedelta(seconds=15)) + e2 = Event(timestamp=now, duration=timedelta(seconds=3)) + e3 = Event(timestamp=now, duration=timedelta(seconds=5)) + e4 = Event(timestamp=now, duration=timedelta(seconds=10)) + + # union event lists with same timestamp but different duration duplicates + events_union = union([e1, e2, e4], [e3, e2, e1]) + assert events_union == [e1, e2, e3, e4] + + +def test_categorize(): + now = datetime.now(timezone.utc) + + classes = [ + (["Test"], Rule({"regex": "^just"})), + (["Test", "Subtest"], Rule({"regex": "subtest$"})), + (["Test", "Ignorecase"], Rule({"regex": "ignorecase", "ignore_case": True})), + ] + events = [ + Event(timestamp=now, duration=0, data={"key": "just a test"}), + Event(timestamp=now, duration=0, data={"key": "just a subtest"}), + Event(timestamp=now, duration=0, data={"key": "just a IGNORECASE test"}), + Event(timestamp=now, duration=0, data={}), + ] + events = categorize(events, classes) + + assert events[0].data["$category"] == ["Test"] + assert events[1].data["$category"] == ["Test", "Subtest"] + assert events[2].data["$category"] == ["Test", "Ignorecase"] + assert events[3].data["$category"] == ["Uncategorized"] + + +def _event(value: str = "just a test") -> Event: + return Event(timestamp=datetime.now(timezone.utc), duration=0, data={"key": value}) + + +def test_categorize_depth_wins_without_priority(): + events = categorize( + [_event()], + [ + (["A"], Rule({"regex": "test"})), + (["B", "B1"], Rule({"regex": "test"})), + ], + ) + assert events[0].data["$category"] == ["B", "B1"] + + +def test_categorize_explicit_priority_overrides_depth(): + # Default for B1 is depth 2 → 20; 25 beats it. + events = categorize( + [_event()], + [ + (["A"], Rule({"regex": "test", "priority": 25})), + (["B", "B1"], Rule({"regex": "test"})), + ], + ) + assert events[0].data["$category"] == ["A"] + + +def test_categorize_weight_alias(): + events = categorize( + [_event()], + [ + (["A"], Rule({"regex": "test", "weight": 25})), + (["B", "B1"], Rule({"regex": "test"})), + ], + ) + assert events[0].data["$category"] == ["A"] + + +def test_categorize_inter_level_priority(): + between = categorize( + [_event()], + [ + (["A"], Rule({"regex": "test"})), + (["A2"], Rule({"regex": "test", "priority": 15})), + ], + ) + assert between[0].data["$category"] == ["A2"] + + still_loses_to_deeper = categorize( + [_event()], + [ + (["A2"], Rule({"regex": "test", "priority": 15})), + (["B", "B1"], Rule({"regex": "test"})), + ], + ) + assert still_loses_to_deeper[0].data["$category"] == ["B", "B1"] + + +def test_categorize_lower_priority_loses_to_default_depth(): + events = categorize( + [_event()], + [ + (["A"], Rule({"regex": "test"})), + (["B", "B1"], Rule({"regex": "test", "priority": 0})), + ], + ) + assert events[0].data["$category"] == ["A"] + + +def test_categorize_equal_priority_keeps_later_match(): + events = categorize( + [_event()], + [ + (["First"], Rule({"regex": "test", "priority": 5})), + (["Second"], Rule({"regex": "test", "priority": 5})), + ], + ) + assert events[0].data["$category"] == ["Second"] + + +def test_categorize_negative_priority_still_beats_uncategorized(): + events = categorize( + [_event()], + [(["Low"], Rule({"regex": "test", "priority": -100}))], + ) + assert events[0].data["$category"] == ["Low"] + + +def test_categorize_priority_below_i64_min_still_beats_uncategorized(): + # Python ints are unbounded; a signed-64-bit fallback sentinel would + # incorrectly keep Uncategorized for values below -(2**63). + events = categorize( + [_event()], + [(["Low"], Rule({"regex": "test", "priority": -(2**63) - 1}))], + ) + assert events[0].data["$category"] == ["Low"] + + +def test_categorize_empty_category_keeps_uncategorized(): + events = categorize( + [_event()], + [([], Rule({"regex": "test"}))], + ) + assert events[0].data["$category"] == ["Uncategorized"] + + +def test_rule_invalid_priority(): + with pytest.raises(ValueError, match="integer"): + Rule({"regex": "test", "priority": 1.5}) + with pytest.raises(ValueError, match="integer"): + Rule({"regex": "test", "priority": "high"}) + with pytest.raises(ValueError, match="integer"): + Rule({"regex": "test", "priority": True}) + + +def test_categorize_cache_correctness(): + """Cache reuses category for identical data; distinct data gets its own category.""" + now = datetime.now(timezone.utc) + + classes = [ + (["Browser"], Rule({"regex": "Firefox"})), + (["Editor"], Rule({"regex": "vim"})), + ] + firefox_data = {"app": "Firefox", "title": "Home"} + vim_data = {"app": "vim", "title": "classify.py"} + + # 50 Firefox events, 1 vim event, 50 more Firefox events + events = ( + [Event(timestamp=now, duration=0, data=dict(firefox_data)) for _ in range(50)] + + [Event(timestamp=now, duration=0, data=dict(vim_data))] + + [Event(timestamp=now, duration=0, data=dict(firefox_data)) for _ in range(50)] + ) + result = categorize(events, classes) + + for e in result[:50]: + assert e.data["$category"] == ["Browser"] + assert result[50].data["$category"] == ["Editor"] + for e in result[51:]: + assert e.data["$category"] == ["Browser"] + + # Mutating one event's category must not affect others sharing the same data fingerprint + result[0].data["$category"].append("MUTATED") + assert result[1].data["$category"] == ["Browser"] + + +def test_tags(): + now = datetime.now(timezone.utc) + + classes = [ + ("Test", Rule({"regex": "value$"})), + ("Test", Rule({"regex": "^just"})), + ] + events = [ + Event(timestamp=now, duration=0, data={"key": "just a test value"}), + Event(timestamp=now, duration=0, data={}), + ] + events = tag(events, classes) + + assert len(events[0].data["$tags"]) == 2 + assert len(events[1].data["$tags"]) == 0 + + +def test_union_no_overlap(): + from pprint import pprint + + now = datetime(2018, 1, 1, 0, 0) + td1h = timedelta(hours=1) + events1 = [ + Event(timestamp=now + 2 * i * td1h, duration=td1h, data={"test": 1}) + for i in range(3) + ] + events2 = [ + Event(timestamp=now + (2 * i + 0.5) * td1h, duration=td1h, data={"test": 2}) + for i in range(3) + ] + + events_union = union_no_overlap(events1, events2) + # pprint(events_union) + dur = sum((e.duration for e in events_union), timedelta(0)) + assert dur == timedelta(hours=4, minutes=30) + assert sorted(events_union, key=lambda e: e.timestamp) + + events_union = union_no_overlap(events2, events1) + # pprint(events_union) + dur = sum((e.duration for e in events_union), timedelta(0)) + assert dur == timedelta(hours=4, minutes=30) + assert sorted(events_union, key=lambda e: e.timestamp) + + events1 = [ + Event(timestamp=now + (2 * i) * td1h, duration=td1h, data={"test": 1}) + for i in range(3) + ] + events2 = [Event(timestamp=now, duration=5 * td1h, data={"test": 2})] + events_union = union_no_overlap(events1, events2) + pprint(events_union) + dur = sum((e.duration for e in events_union), timedelta(0)) + assert dur == timedelta(hours=5, minutes=0) + assert sorted(events_union, key=lambda e: e.timestamp) + + +def test_merge_subwatcher_fields_basic(): + """Fully overlapping subwatcher fields are injected without changing duration.""" + now = datetime(2024, 1, 1, 12, 0, tzinfo=timezone.utc) + td1h = timedelta(hours=1) + + base = [ + Event( + timestamp=now, + duration=td1h, + data={"app": "vim", "title": "file.py"}, + ) + ] + sub = [ + Event( + timestamp=now, + duration=td1h, + data={"project": "myproject", "file": "file.py", "language": "python"}, + ) + ] + result = merge_subwatcher_fields(base, sub, ["project", "file", "language"]) + + assert len(result) == 1 + # Original base fields preserved + assert result[0].data["app"] == "vim" + assert result[0].data["title"] == "file.py" + # Subwatcher fields injected + assert result[0].data["project"] == "myproject" + assert result[0].data["language"] == "python" + # Exact overlap means no extra segmentation + assert result[0].timestamp == now + assert result[0].duration == td1h + + +def test_merge_subwatcher_fields_partial_overlap_splits_base(): + """Partial overlap only enriches the covered slice of the base event.""" + now = datetime(2024, 1, 1, 12, 0, tzinfo=timezone.utc) + td15m = timedelta(minutes=15) + td30m = timedelta(minutes=30) + td1h = timedelta(hours=1) + + base = [Event(timestamp=now, duration=td1h, data={"app": "vim"})] + sub = [ + Event( + timestamp=now + td15m, + duration=td30m, + data={"project": "myproject"}, + ) + ] + + result = merge_subwatcher_fields(base, sub, ["project"]) + + assert len(result) == 3 + assert [event.duration for event in result] == [td15m, td30m, td15m] + assert [event.timestamp for event in result] == [ + now, + now + td15m, + now + td15m + td30m, + ] + assert "project" not in result[0].data + assert result[1].data["project"] == "myproject" + assert "project" not in result[2].data + assert sum_durations(result) == td1h + + +def test_merge_subwatcher_fields_overlapping_subwatchers_prefer_latest_start(): + """Newer overlapping subwatcher events take over from their own start time.""" + now = datetime(2024, 1, 1, 12, 0, tzinfo=timezone.utc) + td20m = timedelta(minutes=20) + td40m = timedelta(minutes=40) + td1h = timedelta(hours=1) + + base = [Event(timestamp=now, duration=td1h, data={"app": "vim"})] + sub = [ + Event(timestamp=now, duration=td40m, data={"project": "alpha"}), + Event(timestamp=now + td20m, duration=td40m, data={"project": "beta"}), + ] + + result = merge_subwatcher_fields(base, sub, ["project"]) + + assert len(result) == 2 + assert [event.timestamp for event in result] == [now, now + td20m] + assert [event.duration for event in result] == [td20m, td40m] + assert [event.data.get("project") for event in result] == ["alpha", "beta"] + + by_project = { + event.data.get("project"): event.duration + for event in merge_events_by_keys(result, ["project"]) + } + assert by_project["alpha"] == td20m + assert by_project["beta"] == td40m + assert sum_durations(result) == td1h + + +def test_merge_subwatcher_fields_no_overlap(): + """Base events with no overlapping subwatcher event are returned unchanged.""" + now = datetime(2024, 1, 1, 12, 0, tzinfo=timezone.utc) + td1h = timedelta(hours=1) + + base = [Event(timestamp=now, duration=td1h, data={"app": "vim"})] + # Subwatcher event is entirely after the base event + sub = [ + Event( + timestamp=now + 2 * td1h, + duration=td1h, + data={"project": "other"}, + ) + ] + result = merge_subwatcher_fields(base, sub, ["project"]) + + assert len(result) == 1 + assert "project" not in result[0].data + assert result[0].data["app"] == "vim" + + +def test_merge_subwatcher_fields_base_wins_conflict(): + """With conflict='base_wins' (default), existing base keys are not overwritten.""" + now = datetime(2024, 1, 1, 12, 0, tzinfo=timezone.utc) + td1h = timedelta(hours=1) + + base = [Event(timestamp=now, duration=td1h, data={"app": "vim", "file": "base.py"})] + sub = [Event(timestamp=now, duration=td1h, data={"file": "sub.py", "project": "p"})] + + result = merge_subwatcher_fields( + base, sub, ["file", "project"], conflict="base_wins" + ) + # base's "file" must not be overwritten + assert result[0].data["file"] == "base.py" + # "project" not in base → injected from sub + assert result[0].data["project"] == "p" + + +def test_merge_subwatcher_fields_sub_wins_conflict(): + """With conflict='sub_wins', subwatcher fields overwrite base fields.""" + now = datetime(2024, 1, 1, 12, 0, tzinfo=timezone.utc) + td1h = timedelta(hours=1) + + base = [Event(timestamp=now, duration=td1h, data={"app": "vim", "file": "base.py"})] + sub = [Event(timestamp=now, duration=td1h, data={"file": "sub.py"})] + + result = merge_subwatcher_fields(base, sub, ["file"], conflict="sub_wins") + assert result[0].data["file"] == "sub.py" + + +def test_merge_subwatcher_fields_multiple_subsegments_preserve_duration(): + """Repeated subwatcher values aggregate to their true covered duration.""" + now = datetime(2024, 1, 1, 12, 0, tzinfo=timezone.utc) + td15m = timedelta(minutes=15) + td1h = timedelta(hours=1) + + base = [Event(timestamp=now, duration=td1h, data={"app": "vim"})] + sub = [ + Event(timestamp=now, duration=td15m, data={"project": "alpha"}), + Event(timestamp=now + td15m, duration=td15m, data={"project": "beta"}), + Event( + timestamp=now + 2 * td15m, + duration=td15m, + data={"project": "alpha"}, + ), + ] + + result = merge_subwatcher_fields(base, sub, ["project"]) + + by_project = { + event.data.get("project"): event.duration + for event in merge_events_by_keys(result, ["project"]) + } + by_app = { + event.data.get("app"): event.duration + for event in merge_events_by_keys(result, ["app"]) + } + + assert by_project["alpha"] == 2 * td15m + assert by_project["beta"] == td15m + assert by_project[None] == td15m + assert by_app["vim"] == td1h + assert sum_durations(result) == td1h + + +def test_merge_subwatcher_fields_empty_inputs(): + """Empty sub or keys returns a defensive copy of base (not the same list).""" + now = datetime(2024, 1, 1, 12, 0, tzinfo=timezone.utc) + td1h = timedelta(hours=1) + base = [Event(timestamp=now, duration=td1h, data={"app": "vim"})] + + # Empty subwatcher list — returns a new list, data unchanged + result = merge_subwatcher_fields(base, [], ["project"]) + assert result[0].data == {"app": "vim"} + assert result is not base + + # Empty keys list + sub = [Event(timestamp=now, duration=td1h, data={"project": "p"})] + result = merge_subwatcher_fields(base, sub, []) + assert "project" not in result[0].data + assert result is not base + + # Both empty + result = merge_subwatcher_fields(base, [], []) + assert result[0].data == {"app": "vim"} + assert result is not base + + +def test_merge_subwatcher_fields_invalid_conflict(): + """Invalid conflict value raises ValueError immediately.""" + now = datetime(2024, 1, 1, 12, 0, tzinfo=timezone.utc) + td1h = timedelta(hours=1) + base = [Event(timestamp=now, duration=td1h, data={"app": "vim"})] + sub = [Event(timestamp=now, duration=td1h, data={"project": "p"})] + + with pytest.raises(ValueError, match="conflict must be"): + merge_subwatcher_fields(base, sub, ["project"], conflict="invalid") diff --git a/tests/utils.py b/tests/utils.py new file mode 100644 index 0000000..8a085f5 --- /dev/null +++ b/tests/utils.py @@ -0,0 +1,52 @@ +import logging +import random +from datetime import datetime, timezone + +from aw_datastore import Datastore, get_storage_methods + +logging.basicConfig(level=logging.DEBUG) + +# Useful when you just want some placeholder time in your events, saves typing +now = datetime.now(timezone.utc) + + +class TempTestBucket: + """Context manager for creating a test bucket""" + + def __init__(self, datastore): + self.ds = datastore + self.bucket_id = f"test-{random.randint(0, 10**4)}" + + def __enter__(self): + self.ds.create_bucket( + bucket_id=self.bucket_id, + type="testtype", + client="testclient", + hostname="testhost", + name="testname", + ) + return self.ds[self.bucket_id] + + def __exit__(self, *_): + self.ds.delete_bucket(bucket_id=self.bucket_id) + + def __repr__(self): + return f"" + + +_storage_methods = get_storage_methods() + + +def param_datastore_objects(): + return [ + Datastore(storage_strategy=strategy, testing=True) + for name, strategy in _storage_methods.items() + ] + + +def param_testing_buckets_cm(): + datastores = [ + Datastore(storage_strategy=strategy, testing=True) + for name, strategy in _storage_methods.items() + ] + return [TempTestBucket(ds) for ds in datastores]