snapshot of https://github.com/ActivityWatch/aw-core.git (master, 2026-09-16; blobs > 131072B omitted)

aw_datastore/__init__.py

@@ -0,0 +1,19 @@
+from typing import Callable, Dict
+
+from . import storages
+from .datastore import Datastore
+from .migration import check_for_migration
+
+
+def get_storage_methods() -> Dict[str, Callable[..., storages.AbstractStorage]]:
+    from .storages import MemoryStorage, PeeweeStorage, SqliteStorage
+
+    methods: Dict[str, Callable[..., storages.AbstractStorage]] = {
+        PeeweeStorage.sid: PeeweeStorage,
+        MemoryStorage.sid: MemoryStorage,
+        SqliteStorage.sid: SqliteStorage,
+    }
+    return methods
+
+
+__all__ = ["Datastore", "get_storage_methods", "check_for_migration"]

aw_datastore/benchmark.py

@@ -0,0 +1,97 @@
+#!/usr/bin/env python3
+import sys
+from contextlib import contextmanager
+from datetime import datetime, timedelta, timezone
+from typing import Callable
+
+from aw_core.models import Event
+
+from aw_datastore import get_storage_methods
+from aw_datastore.storages import AbstractStorage
+
+td1s = timedelta(seconds=1)
+
+
+def create_test_events(n):
+    now = datetime.now(timezone.utc) - timedelta(days=1000)
+
+    events = []
+    for i in range(n):
+        events.append(
+            Event(timestamp=now + i * td1s, duration=td1s, data={"label": "asd"})
+        )
+
+    return events
+
+
+@contextmanager
+def temporary_bucket(ds):
+    bucket_id = "test_bucket"
+    try:
+        ds.delete_bucket(bucket_id)
+    except Exception:
+        pass
+    bucket = ds.create_bucket(bucket_id, "testingtype", "test-client", "testing-box")
+    yield bucket
+    ds.delete_bucket(bucket_id)
+
+
+def benchmark(storage: Callable[..., AbstractStorage]):
+    raise NotImplementedError(
+        "No longer implemented as ttt/takethetime dependency is removed"
+    )
+
+    # if storage.__name__ == "PeeweeStorage":
+    #     ds = Datastore(storage, testing=True, filepath="test.db")
+    # else:
+    #     ds = Datastore(storage, testing=True)
+
+    # num_single_events = 50
+    # num_replace_events = 50
+    # num_bulk_events = 20_000
+    # num_events = num_single_events + num_replace_events + num_bulk_events + 1
+    # num_final_events = num_single_events + num_bulk_events + 1
+
+    # events = create_test_events(num_events)
+    # single_events = events[:num_single_events]
+    # replace_events = events[num_single_events : num_single_events + num_replace_events]
+    # bulk_events = events[num_single_events + num_replace_events : -1]
+
+    # print(storage.__name__)
+
+    # with temporary_bucket(ds) as bucket:
+    #     with ttt(" sum"):
+    #         with ttt(f" single insert {num_single_events} events"):
+    #             for event in single_events:
+    #                 bucket.insert(event)
+
+    #         with ttt(f" bulk insert {num_bulk_events} events"):
+    #             bucket.insert(bulk_events)
+
+    #         with ttt(f" replace last {num_replace_events}"):
+    #             for e in replace_events:
+    #                 bucket.replace_last(e)
+
+    #         with ttt(" insert 1 event"):
+    #             bucket.insert(events[-1])
+
+    #         with ttt(" get one"):
+    #             events_tmp = bucket.get(limit=1)
+
+    #         with ttt(" get all"):
+    #             events_tmp = bucket.get(limit=-1)
+    #             assert len(events_tmp) == num_final_events
+
+    #         with ttt(" get range"):
+    #             events_tmp = bucket.get(
+    #                 limit=-1,
+    #                 starttime=events[1].timestamp + 0.01 * td1s,
+    #                 endtime=events[-1].timestamp + events[-1].duration,
+    #             )
+    #             assert len(events_tmp) == num_final_events - 1
+
+
+if __name__ == "__main__":
+    for storage in get_storage_methods().values():
+        if len(sys.argv) <= 1 or storage.__name__ in sys.argv:
+            benchmark(storage)

aw_datastore/datastore.py

@@ -0,0 +1,189 @@
+import logging
+from datetime import datetime, timedelta, timezone
+from typing import (
+    Callable,
+    Dict,
+    List,
+    Optional,
+    Union,
+)
+
+from aw_core.models import Event
+
+from .storages import AbstractStorage
+
+logger = logging.getLogger(__name__)
+
+
+class Datastore:
+    def __init__(
+        self,
+        storage_strategy: Callable[..., AbstractStorage],
+        testing=False,
+        **kwargs,
+    ) -> None:
+        self.logger = logger.getChild("Datastore")
+        self.bucket_instances: Dict[str, Bucket] = dict()
+
+        self.storage_strategy = storage_strategy(testing=testing, **kwargs)
+
+    def __repr__(self):
+        return f"<Datastore object using {self.storage_strategy.__class__.__name__}>"
+
+    def __getitem__(self, bucket_id: str) -> "Bucket":
+        # If this bucket doesn't have a initialized object, create it
+        if bucket_id not in self.bucket_instances:
+            # If the bucket exists in the database, create an object representation of it
+            if bucket_id in self.buckets():
+                bucket = Bucket(self, bucket_id)
+                self.bucket_instances[bucket_id] = bucket
+            else:
+                self.logger.error(
+                    f"Cannot create a Bucket object for {bucket_id} because it doesn't exist in the database"
+                )
+                raise KeyError
+
+        return self.bucket_instances[bucket_id]
+
+    def create_bucket(
+        self,
+        bucket_id: str,
+        type: str,
+        client: str,
+        hostname: str,
+        created: Optional[datetime] = None,
+        name: Optional[str] = None,
+        data: Optional[dict] = None,
+    ) -> "Bucket":
+        created = created or datetime.now(timezone.utc)
+        self.logger.info(f"Creating bucket '{bucket_id}'")
+        self.storage_strategy.create_bucket(
+            bucket_id, type, client, hostname, created.isoformat(), name=name, data=data
+        )
+        return self[bucket_id]
+
+    def update_bucket(self, bucket_id: str, **kwargs):
+        self.logger.info(f"Updating bucket '{bucket_id}'")
+        return self.storage_strategy.update_bucket(bucket_id, **kwargs)
+
+    def delete_bucket(self, bucket_id: str):
+        self.logger.info(f"Deleting bucket '{bucket_id}'")
+        if bucket_id in self.bucket_instances:
+            del self.bucket_instances[bucket_id]
+        return self.storage_strategy.delete_bucket(bucket_id)
+
+    def buckets(self):
+        return self.storage_strategy.buckets()
+
+
+class Bucket:
+    def __init__(self, datastore: Datastore, bucket_id: str) -> None:
+        self.logger = logger.getChild("Bucket")
+        self.ds = datastore
+        self.bucket_id = bucket_id
+
+    def metadata(self) -> dict:
+        return self.ds.storage_strategy.get_metadata(self.bucket_id)
+
+    def get(
+        self,
+        limit: int = -1,
+        starttime: Optional[datetime] = None,
+        endtime: Optional[datetime] = None,
+    ) -> List[Event]:
+        """Returns events sorted in descending order by timestamp"""
+        # Resolution is rounded down since not all datastores like microsecond precision
+        if starttime:
+            starttime = starttime.replace(
+                microsecond=1000 * int(starttime.microsecond / 1000)
+            )
+        if endtime:
+            # Rounding up here in order to ensure events aren't missed
+            # second_offset and microseconds modulo required since replace() only takes microseconds up to 999999 (doesn't handle overflow)
+            milliseconds = 1 + int(endtime.microsecond / 1000)
+            second_offset = int(milliseconds / 1000)  # usually 0, rarely 1
+            microseconds = (
+                (1000 * milliseconds) % 1000000
+            )  # will likely just be 1000 * milliseconds, if it overflows it would become zero
+            endtime = endtime.replace(microsecond=microseconds) + timedelta(
+                seconds=second_offset
+            )
+
+        return self.ds.storage_strategy.get_events(
+            self.bucket_id, limit, starttime, endtime
+        )
+
+    def get_by_id(self, event_id) -> Optional[Event]:
+        """Will return the event with the provided ID, or None if not found."""
+        return self.ds.storage_strategy.get_event(self.bucket_id, event_id)
+
+    def get_eventcount(
+        self, starttime: Optional[datetime] = None, endtime: Optional[datetime] = None
+    ) -> int:
+        return self.ds.storage_strategy.get_eventcount(
+            self.bucket_id, starttime, endtime
+        )
+
+    def insert(self, events: Union[Event, List[Event]]) -> Optional[Event]:
+        """
+        Inserts one or several events.
+        If a single event is inserted, return the event with its id assigned.
+        If several events are inserted, returns None. (This is due to there being no efficient way of getting ids out when doing bulk inserts with some datastores such as peewee/SQLite)
+        """
+
+        # NOTE: Should we keep the timestamp checking?
+        warn_older_event = False
+
+        # Get last event for timestamp check after insert
+        if warn_older_event:
+            last_event_list = self.get(1)
+            last_event = None
+            if last_event_list:
+                last_event = last_event_list[0]
+
+        now = datetime.now(tz=timezone.utc)
+
+        inserted: Optional[Event] = None
+
+        # Call insert
+        if isinstance(events, Event):
+            oldest_event: Optional[Event] = events
+            if events.timestamp + events.duration > now:
+                self.logger.warning(
+                    f"Event inserted into bucket {self.bucket_id} reaches into the future. Current UTC time: {str(now)}. Event data: {str(events)}"
+                )
+            inserted = self.ds.storage_strategy.insert_one(self.bucket_id, events)
+            # assert inserted
+        elif isinstance(events, list):
+            if events:
+                oldest_event = sorted(events, key=lambda k: k["timestamp"])[0]
+            else:  # pragma: no cover
+                oldest_event = None
+            for event in events:
+                if event.timestamp + event.duration > now:
+                    self.logger.warning(
+                        f"Event inserted into bucket {self.bucket_id} reaches into the future. Current UTC time: {str(now)}. Event data: {str(event)}"
+                    )
+            self.ds.storage_strategy.insert_many(self.bucket_id, events)
+        else:
+            raise TypeError
+
+        # Warn if timestamp is older than last event
+        if warn_older_event and last_event and oldest_event:
+            if oldest_event.timestamp < last_event.timestamp:  # pragma: no cover
+                self.logger.warning(
+                    f"""Inserting event that has a older timestamp than previous event!
+Previous: {last_event}
+Inserted: {oldest_event}"""
+                )
+
+        return inserted
+
+    def delete(self, event_id):
+        return self.ds.storage_strategy.delete(self.bucket_id, event_id)
+
+    def replace_last(self, event):
+        return self.ds.storage_strategy.replace_last(self.bucket_id, event)
+
+    def replace(self, event_id, event):
+        return self.ds.storage_strategy.replace(self.bucket_id, event_id, event)

aw_datastore/migration.py

@@ -0,0 +1,62 @@
+import logging
+import os
+from typing import List, Optional
+
+from aw_core.dirs import get_data_dir, legacy_testing_suffix
+
+from .storages import AbstractStorage
+
+logger = logging.getLogger(__name__)
+
+
+def detect_db_files(
+    data_dir: str, datastore_name: Optional[str] = None, version=None
+) -> List[str]:
+    db_files = [filename for filename in os.listdir(data_dir)]
+    if datastore_name:
+        db_files = [
+            filename
+            for filename in db_files
+            if filename.split(".")[0] == datastore_name
+        ]
+    if version:
+        db_files = [
+            filename for filename in db_files if filename.split(".")[1] == f"v{version}"
+        ]
+    return db_files
+
+
+def check_for_migration(datastore: AbstractStorage):
+    data_dir = get_data_dir("aw-server")
+
+    if datastore.sid == "sqlite":
+        peewee_type = "peewee-sqlite"
+        peewee_name = peewee_type + legacy_testing_suffix(datastore.testing)
+        # Migrate from peewee v2
+        peewee_db_v2 = detect_db_files(data_dir, peewee_name, 2)
+        if len(peewee_db_v2) > 0:
+            peewee_v2_to_sqlite_v1(datastore)
+
+
+def peewee_v2_to_sqlite_v1(datastore):
+    logger.info("Migrating database from peewee v2 to sqlite v1")
+    from .storages import PeeweeStorage
+
+    pw_db = PeeweeStorage(datastore.testing)
+    # Fetch buckets and events
+    buckets = pw_db.buckets()
+    # Insert buckets and events to new db
+    for bucket_id in buckets:
+        logger.info(f"Migrating bucket {bucket_id}")
+        bucket = buckets[bucket_id]
+        datastore.create_bucket(
+            bucket["id"],
+            bucket["type"],
+            bucket["client"],
+            bucket["hostname"],
+            bucket["created"],
+            bucket["name"],
+        )
+        bucket_events = pw_db.get_events(bucket_id, -1)
+        datastore.insert_many(bucket_id, bucket_events)
+    logger.info("Migration of peewee v2 to sqlite v1 finished")

aw_datastore/storages/__init__.py

@@ -0,0 +1,15 @@
+import logging as _logging
+
+logger: _logging.Logger = _logging.getLogger(__name__)
+
+from .abstract import AbstractStorage
+from .memory import MemoryStorage
+from .peewee import PeeweeStorage
+from .sqlite import SqliteStorage
+
+__all__ = [
+    "AbstractStorage",
+    "MemoryStorage",
+    "PeeweeStorage",
+    "SqliteStorage",
+]

aw_datastore/storages/abstract.py

@@ -0,0 +1,101 @@
+from abc import ABCMeta, abstractmethod
+from datetime import datetime
+from typing import Dict, List, Optional
+
+from aw_core.models import Event
+
+
+class AbstractStorage(metaclass=ABCMeta):
+    """
+    Interface for storage methods.
+    """
+
+    sid = "Storage id not set, fix me"
+
+    @abstractmethod
+    def __init__(self, testing: bool) -> None:
+        self.testing = True
+        raise NotImplementedError
+
+    @abstractmethod
+    def buckets(self) -> Dict[str, dict]:
+        raise NotImplementedError
+
+    @abstractmethod
+    def create_bucket(
+        self,
+        bucket_id: str,
+        type_id: str,
+        client: str,
+        hostname: str,
+        created: str,
+        name: Optional[str] = None,
+        data: Optional[dict] = None,
+    ) -> None:
+        raise NotImplementedError
+
+    @abstractmethod
+    def update_bucket(
+        self,
+        bucket_id: str,
+        type_id: Optional[str] = None,
+        client: Optional[str] = None,
+        hostname: Optional[str] = None,
+        name: Optional[str] = None,
+        data: Optional[dict] = None,
+    ) -> None:
+        raise NotImplementedError
+
+    @abstractmethod
+    def delete_bucket(self, bucket_id: str) -> None:
+        raise NotImplementedError
+
+    @abstractmethod
+    def get_metadata(self, bucket_id: str) -> dict:
+        raise NotImplementedError
+
+    @abstractmethod
+    def get_event(
+        self,
+        bucket_id: str,
+        event_id: int,
+    ) -> Optional[Event]:
+        raise NotImplementedError
+
+    @abstractmethod
+    def get_events(
+        self,
+        bucket_id: str,
+        limit: int,
+        starttime: Optional[datetime] = None,
+        endtime: Optional[datetime] = None,
+    ) -> List[Event]:
+        raise NotImplementedError
+
+    def get_eventcount(
+        self,
+        bucket_id: str,
+        starttime: Optional[datetime] = None,
+        endtime: Optional[datetime] = None,
+    ) -> int:
+        raise NotImplementedError
+
+    @abstractmethod
+    def insert_one(self, bucket_id: str, event: Event) -> Event:
+        raise NotImplementedError
+
+    def insert_many(self, bucket_id: str, events: List[Event]) -> None:
+        for event in events:
+            self.insert_one(bucket_id, event)
+
+    @abstractmethod
+    def delete(self, bucket_id: str, event_id: int) -> bool:
+        raise NotImplementedError
+
+    @abstractmethod
+    def replace(self, bucket_id: str, event_id: int, event: Event) -> bool:
+        raise NotImplementedError
+
+    @abstractmethod
+    def replace_last(self, bucket_id: str, event: Event) -> None:
+        raise NotImplementedError

aw_datastore/storages/memory.py

@@ -0,0 +1,187 @@
+import copy
+import sys
+from datetime import datetime
+from typing import Dict, List, Optional
+
+from aw_core.models import Event
+
+from . import logger
+from .abstract import AbstractStorage
+
+
+class MemoryStorage(AbstractStorage):
+    """For storage of data in-memory, useful primarily in testing"""
+
+    sid = "memory"
+
+    def __init__(self, testing: bool) -> None:
+        self.logger = logger.getChild(self.sid)
+        # self.logger.warning("Using in-memory storage, any events stored will not be persistent and will be lost when server is shut down. Use the --storage parameter to set a different storage method.")
+        self.db: Dict[str, List[Event]] = {}
+        self._metadata: Dict[str, dict] = dict()
+
+    def create_bucket(
+        self,
+        bucket_id,
+        type_id,
+        client,
+        hostname,
+        created,
+        name=None,
+        data=None,
+    ) -> None:
+        if not name:
+            name = bucket_id
+        self._metadata[bucket_id] = {
+            "id": bucket_id,
+            "name": name,
+            "type": type_id,
+            "client": client,
+            "hostname": hostname,
+            "created": created,
+            "data": data or {},
+        }
+        self.db[bucket_id] = []
+
+    def update_bucket(
+        self,
+        bucket_id: str,
+        type_id: Optional[str] = None,
+        client: Optional[str] = None,
+        hostname: Optional[str] = None,
+        name: Optional[str] = None,
+        data: Optional[dict] = None,
+    ) -> None:
+        if bucket_id in self._metadata:
+            if type_id:
+                self._metadata[bucket_id]["type"] = type_id
+            if client:
+                self._metadata[bucket_id]["client"] = client
+            if hostname:
+                self._metadata[bucket_id]["hostname"] = hostname
+            if name:
+                self._metadata[bucket_id]["name"] = name
+            if data:
+                self._metadata[bucket_id]["data"] = data
+        else:
+            raise ValueError("Bucket did not exist, could not update")
+
+    def delete_bucket(self, bucket_id: str) -> None:
+        if bucket_id in self.db:
+            del self.db[bucket_id]
+        if bucket_id in self._metadata:
+            del self._metadata[bucket_id]
+        else:
+            raise ValueError("Bucket did not exist, could not delete")
+
+    def buckets(self):
+        buckets = dict()
+        for bucket_id in self.db:
+            buckets[bucket_id] = self.get_metadata(bucket_id)
+        return buckets
+
+    def get_event(
+        self,
+        bucket_id: str,
+        event_id: int,
+    ) -> Optional[Event]:
+        event = self._get_event(bucket_id, event_id)
+        return copy.deepcopy(event)
+
+    def get_events(
+        self,
+        bucket: str,
+        limit: int,
+        starttime: Optional[datetime] = None,
+        endtime: Optional[datetime] = None,
+    ) -> List[Event]:
+        events = self.db[bucket]
+
+        # Sort by timestamp
+        events = sorted(events, key=lambda k: k["timestamp"])[::-1]
+
+        # Filter by date
+        if starttime:
+            events = [e for e in events if starttime <= (e.timestamp + e.duration)]
+        if endtime:
+            events = [e for e in events if e.timestamp <= endtime]
+
+        # Limit
+        if limit == 0:
+            return []
+        elif limit < 0:
+            limit = sys.maxsize
+        events = events[:limit]
+        # Return
+        return copy.deepcopy(events)
+
+    def get_eventcount(
+        self,
+        bucket: str,
+        starttime: Optional[datetime] = None,
+        endtime: Optional[datetime] = None,
+    ) -> int:
+        return len(
+            [
+                e
+                for e in self.db[bucket]
+                if (not starttime or starttime <= e.timestamp)
+                and (not endtime or e.timestamp <= endtime)
+            ]
+        )
+
+    def get_metadata(self, bucket_id: str):
+        if bucket_id in self._metadata:
+            return self._metadata[bucket_id]
+        else:
+            raise ValueError("Bucket did not exist, could not get metadata")
+
+    def insert_one(self, bucket: str, event: Event) -> Event:
+        if event.id is not None:
+            self.replace(bucket, event.id, event)
+        else:
+            # We need to copy the event to avoid setting the ID on the passed event
+            event = copy.copy(event)
+            if self.db[bucket]:
+                event.id = max(int(e.id or 0) for e in self.db[bucket]) + 1
+            else:
+                event.id = 0
+            self.db[bucket].append(event)
+        return event
+
+    def delete(self, bucket_id, event_id):
+        for idx in (
+            idx
+            for idx, event in reversed(list(enumerate(self.db[bucket_id])))
+            if event.id == event_id
+        ):
+            self.db[bucket_id].pop(idx)
+            return True
+        return False
+
+    def _get_event(self, bucket_id, event_id) -> Optional[Event]:
+        events = [
+            event
+            for idx, event in reversed(list(enumerate(self.db[bucket_id])))
+            if event.id == event_id
+        ]
+        if len(events) < 1:
+            return None
+        else:
+            return events[0]
+
+    def replace(self, bucket_id, event_id, event):
+        for idx in (
+            idx
+            for idx, event in reversed(list(enumerate(self.db[bucket_id])))
+            if event.id == event_id
+        ):
+            # We need to copy the event to avoid setting the ID on the passed event
+            event = copy.copy(event)
+            event.id = event_id
+            self.db[bucket_id][idx] = event
+
+    def replace_last(self, bucket_id, event):
+        # NOTE: This does not actually get the most recent event, only the last inserted
+        last = sorted(self.db[bucket_id], key=lambda e: e.timestamp)[-1]
+        self.replace(bucket_id, last.id, event)

aw_datastore/storages/peewee.py

@@ -0,0 +1,451 @@
+import json
+import logging
+import os
+from datetime import datetime, timedelta, timezone
+from typing import (
+    Any,
+    Dict,
+    List,
+    Optional,
+)
+
+import iso8601
+from aw_core.dirs import get_data_dir, legacy_testing_suffix
+from aw_core.models import Event
+from playhouse.migrate import SqliteMigrator, migrate
+
+try:
+    from playhouse.sqlite_ext import SqliteExtDatabase
+except ImportError:
+    # peewee 4 is missing SqliteExtDatabase
+    from peewee import SqliteDatabase as SqliteExtDatabase
+
+import peewee
+from peewee import (
+    AutoField,
+    CharField,
+    DateTimeField,
+    DecimalField,
+    ForeignKeyField,
+    IntegerField,
+    Model,
+)
+
+from .abstract import AbstractStorage
+
+logger = logging.getLogger(__name__)
+
+# Prevent debug output from propagating
+peewee_logger = logging.getLogger("peewee")
+peewee_logger.setLevel(logging.INFO)
+
+# Init'd later in the PeeweeStorage constructor.
+#   See: http://docs.peewee-orm.com/en/latest/peewee/database.html#run-time-database-configuration
+# Another option would be to use peewee's Proxy.
+#   See: http://docs.peewee-orm.com/en/latest/peewee/database.html#dynamic-db
+_db = SqliteExtDatabase(None)
+
+
+LATEST_VERSION = 2
+
+
+def auto_migrate(path: str) -> None:
+    db = SqliteExtDatabase(path)
+    migrator = SqliteMigrator(db)
+
+    # check if bucketmodel has datastr field
+    info = db.execute_sql("PRAGMA table_info(bucketmodel)")
+    has_datastr = any(row[1] == "datastr" for row in info)
+
+    if not has_datastr:
+        datastr_field = CharField(default="{}")
+        with db.atomic():
+            migrate(migrator.add_column("bucketmodel", "datastr", datastr_field))
+
+    # Replace the single-column event indexes with a composite index.
+    # Every event query filters on bucket_id and orders by timestamp (or
+    # narrows to a timestamp range), so the composite index serves the seek,
+    # the range and the ORDER BY in one pass. With only single-column indexes
+    # SQLite reads all of a bucket's rows and sorts them in a temp B-tree,
+    # even for the LIMIT 1 last-event query issued on every heartbeat merge.
+    # Dropping the old indexes also makes inserts cheaper.
+    indexes = {row[1] for row in db.execute_sql("PRAGMA index_list(eventmodel)")}
+    if "eventmodel_bucket_id_timestamp" not in indexes:
+        logger.info(
+            "Migrating event indexes, this can take a few minutes on large databases..."
+        )
+        # Drops run before the create so the pages they free are reused to
+        # build the new index within the same transaction; creating first
+        # would permanently grow the database file by the new index's size.
+        with db.atomic():
+            db.execute_sql("DROP INDEX IF EXISTS eventmodel_bucket_id")
+            db.execute_sql("DROP INDEX IF EXISTS eventmodel_timestamp")
+            db.execute_sql(
+                "CREATE INDEX eventmodel_bucket_id_timestamp"
+                " ON eventmodel (bucket_id, timestamp)"
+            )
+
+    db.close()
+
+
+def chunks(ls, n):
+    """Yield successive n-sized chunks from ls.
+    From: https://stackoverflow.com/a/312464/965332"""
+    for i in range(0, len(ls), n):
+        yield ls[i : i + n]
+
+
+def dt_plus_duration(dt, duration):
+    # See peewee docs on datemath: https://docs.peewee-orm.com/en/latest/peewee/hacks.html#date-math
+    return peewee.fn.strftime(
+        "%Y-%m-%d %H:%M:%f+00:00",
+        (peewee.fn.julianday(dt) - 2440587.5) * 86400.0 + duration,
+        "unixepoch",
+    )
+
+
+class BaseModel(Model):
+    class Meta:
+        database = _db
+
+
+class BucketModel(BaseModel):
+    key = IntegerField(primary_key=True)
+    id = CharField(unique=True)
+    created = DateTimeField(default=lambda: datetime.now(timezone.utc))
+    name = CharField(null=True)
+    type = CharField()
+    client = CharField()
+    hostname = CharField()
+    datastr = CharField(null=True)  # JSON-encoded object
+
+    def json(self):
+        return {
+            "id": self.id,
+            "created": iso8601.parse_date(self.created)
+            .astimezone(timezone.utc)
+            .isoformat(),
+            "name": self.name,
+            "type": self.type,
+            "client": self.client,
+            "hostname": self.hostname,
+            "data": json.loads(self.datastr) if self.datastr else {},
+        }
+
+
+class EventModel(BaseModel):
+    # Events are covered by the composite (bucket_id, timestamp) index created
+    # in auto_migrate(); it is kept out of the model so create_table doesn't
+    # build it before auto_migrate has dropped the old single-column indexes.
+    id = AutoField()
+    bucket = ForeignKeyField(BucketModel, backref="events", index=False)
+    timestamp = DateTimeField(default=lambda: datetime.now(timezone.utc))
+    duration = DecimalField()
+    datastr = CharField()
+
+    @classmethod
+    def from_event(cls, bucket_key, event: Event):
+        return cls(
+            bucket=bucket_key,
+            id=event.id,
+            timestamp=event.timestamp,
+            duration=event.duration.total_seconds(),
+            datastr=json.dumps(event.data),
+        )
+
+    def json(self):
+        return {
+            "id": self.id,
+            "timestamp": self.timestamp,
+            "duration": float(self.duration),
+            "data": json.loads(self.datastr),
+        }
+
+
+class PeeweeStorage(AbstractStorage):
+    sid = "peewee"
+
+    def __init__(self, testing: bool = True, filepath: Optional[str] = None) -> None:
+        data_dir = get_data_dir("aw-server")
+
+        if not filepath:
+            filename = (
+                "peewee-sqlite"
+                + legacy_testing_suffix(testing)
+                + f".v{LATEST_VERSION}"
+                + ".db"
+            )
+            filepath = os.path.join(data_dir, filename)
+        self.db = _db
+        self.db.init(
+            filepath,
+            pragmas={
+                "journal_mode": "wal",
+            },
+        )
+        logger.info(f"Using database file: {filepath}")
+        self.db.connect()
+
+        self.bucket_keys: Dict[str, int] = {}
+        BucketModel.create_table(safe=True)
+        EventModel.create_table(safe=True)
+
+        # Migrate database if needed, requires closing the connection first
+        self.db.close()
+        auto_migrate(filepath)
+        self.db.connect()
+
+        # Update bucket keys
+        self.update_bucket_keys()
+
+    def update_bucket_keys(self) -> None:
+        buckets = BucketModel.select()
+        self.bucket_keys = {bucket.id: bucket.key for bucket in buckets}
+
+    def buckets(self) -> Dict[str, Dict[str, Any]]:
+        return {bucket.id: bucket.json() for bucket in BucketModel.select()}
+
+    def create_bucket(
+        self,
+        bucket_id: str,
+        type_id: str,
+        client: str,
+        hostname: str,
+        created: str,
+        name: Optional[str] = None,
+        data: Optional[Dict[str, Any]] = None,
+    ):
+        BucketModel.create(
+            id=bucket_id,
+            type=type_id,
+            client=client,
+            hostname=hostname,
+            created=created,
+            name=name,
+            datastr=json.dumps(data or {}),
+        )
+        self.update_bucket_keys()
+
+    def update_bucket(
+        self,
+        bucket_id: str,
+        type_id: Optional[str] = None,
+        client: Optional[str] = None,
+        hostname: Optional[str] = None,
+        name: Optional[str] = None,
+        data: Optional[dict] = None,
+    ) -> None:
+        if bucket_id in self.bucket_keys:
+            bucket = BucketModel.get(BucketModel.key == self.bucket_keys[bucket_id])
+
+            if type_id is not None:
+                bucket.type = type_id
+            if client is not None:
+                bucket.client = client
+            if hostname is not None:
+                bucket.hostname = hostname
+            if name is not None:
+                bucket.name = name
+            if data is not None:
+                bucket.datastr = json.dumps(data)  # Encoding data dictionary to JSON
+
+            bucket.save()
+        else:
+            raise ValueError("Bucket did not exist, could not update")
+
+    def delete_bucket(self, bucket_id: str) -> None:
+        if bucket_id in self.bucket_keys:
+            EventModel.delete().where(
+                EventModel.bucket == self.bucket_keys[bucket_id]
+            ).execute()
+            BucketModel.delete().where(
+                BucketModel.key == self.bucket_keys[bucket_id]
+            ).execute()
+            self.update_bucket_keys()
+        else:
+            raise ValueError("Bucket did not exist, could not delete")
+
+    def get_metadata(self, bucket_id: str):
+        if bucket_id in self.bucket_keys:
+            bucket = BucketModel.get(
+                BucketModel.key == self.bucket_keys[bucket_id]
+            ).json()
+            return bucket
+        else:
+            raise ValueError("Bucket did not exist, could not get metadata")
+
+    def insert_one(self, bucket_id: str, event: Event) -> Event:
+        e = EventModel.from_event(self.bucket_keys[bucket_id], event)
+        e.save()
+        event.id = e.id
+        return event
+
+    def insert_many(self, bucket_id, events: List[Event]) -> None:
+        # NOTE: Events need to be handled differently depending on
+        #       if they're upserts or inserts (have id's or not).
+
+        # These events are updates which need to be applied one by one
+        events_updates = [e for e in events if e.id is not None]
+        for e in events_updates:
+            self.insert_one(bucket_id, e)
+
+        # These events can be inserted with insert_many
+        events_dictlist = [
+            {
+                "bucket": self.bucket_keys[bucket_id],
+                "timestamp": event.timestamp,
+                "duration": event.duration.total_seconds(),
+                "datastr": json.dumps(event.data),
+            }
+            for event in events
+            if event.id is None
+        ]
+
+        # Chunking into lists of length 100 is needed here due to SQLITE_MAX_COMPOUND_SELECT
+        # and SQLITE_LIMIT_VARIABLE_NUMBER under Windows.
+        # See: https://github.com/coleifer/peewee/issues/948
+        for chunk in chunks(events_dictlist, 100):
+            EventModel.insert_many(chunk).execute()
+
+    def _get_event(self, bucket_id, event_id) -> Optional[EventModel]:
+        try:
+            return (
+                EventModel.select()
+                .where(EventModel.id == event_id)
+                .where(EventModel.bucket == self.bucket_keys[bucket_id])
+                .get()
+            )
+        except peewee.DoesNotExist:
+            return None
+
+    def _get_last(self, bucket_id) -> EventModel:
+        return (
+            EventModel.select()
+            .where(EventModel.bucket == self.bucket_keys[bucket_id])
+            .order_by(EventModel.timestamp.desc())
+            .get()
+        )
+
+    def replace_last(self, bucket_id, event):
+        e = self._get_last(bucket_id)
+        e.timestamp = event.timestamp
+        e.duration = event.duration.total_seconds()
+        e.datastr = json.dumps(event.data)
+        e.save()
+        event.id = e.id
+        return event
+
+    def delete(self, bucket_id, event_id):
+        return (
+            EventModel.delete()
+            .where(EventModel.id == event_id)
+            .where(EventModel.bucket == self.bucket_keys[bucket_id])
+            .execute()
+        )
+
+    def replace(self, bucket_id, event_id, event):
+        e = self._get_event(bucket_id, event_id)
+        e.timestamp = event.timestamp
+        e.duration = event.duration.total_seconds()
+        e.datastr = json.dumps(event.data)
+        e.save()
+        event.id = e.id
+        return event
+
+    def get_event(
+        self,
+        bucket_id: str,
+        event_id: int,
+    ) -> Optional[Event]:
+        """
+        Fetch a single event from a bucket.
+        """
+        res = self._get_event(bucket_id, event_id)
+        return Event(**EventModel.json(res)) if res else None
+
+    def get_events(
+        self,
+        bucket_id: str,
+        limit: int,
+        starttime: Optional[datetime] = None,
+        endtime: Optional[datetime] = None,
+    ):
+        """
+        Fetch events from a certain bucket, optionally from a given range of time.
+
+        Example raw query:
+
+            SELECT strftime(
+              "%Y-%m-%d %H:%M:%f+00:00",
+              ((julianday(timestamp) - 2440587.5) * 86400),
+              'unixepoch'
+            )
+            FROM eventmodel
+            WHERE eventmodel.timestamp > '2021-06-20'
+            LIMIT 10;
+
+        """
+        if limit == 0:
+            return []
+        q = (
+            EventModel.select()
+            .where(EventModel.bucket == self.bucket_keys[bucket_id])
+            .order_by(EventModel.timestamp.desc())
+            .limit(limit)
+        )
+
+        q = self._where_range(q, starttime, endtime)
+
+        res = q.execute()
+        events = [Event(**e) for e in list(map(EventModel.json, res))]
+
+        # Trim events that are out of range (as done in aw-server-rust)
+        # TODO: Do the same for the other storage methods
+        for e in events:
+            if starttime:
+                if e.timestamp < starttime:
+                    e_end = e.timestamp + e.duration
+                    e.timestamp = starttime
+                    e.duration = e_end - e.timestamp
+            if endtime:
+                if e.timestamp + e.duration > endtime:
+                    e.duration = endtime - e.timestamp
+
+        return events
+
+    def get_eventcount(
+        self,
+        bucket_id: str,
+        starttime: Optional[datetime] = None,
+        endtime: Optional[datetime] = None,
+    ) -> int:
+        q = EventModel.select().where(EventModel.bucket == self.bucket_keys[bucket_id])
+        q = self._where_range(q, starttime, endtime)
+        return q.count()
+
+    def _where_range(
+        self,
+        q,
+        starttime: Optional[datetime] = None,
+        endtime: Optional[datetime] = None,
+    ):
+        # Important to normalize datetimes to UTC, otherwise any UTC offset will be ignored
+        if starttime:
+            starttime = starttime.astimezone(timezone.utc)
+        if endtime:
+            endtime = endtime.astimezone(timezone.utc)
+
+        if starttime:
+            # Faster WHERE to speed up slow query below, leads to ~2-3x speedup
+            # We'll assume events aren't >24h
+            q = q.where(starttime - timedelta(hours=24) <= EventModel.timestamp)
+
+            # This can be slow on large databases...
+            # Tried creating various indexes and using SQLite's unlikely() function, but it had no effect
+            q = q.where(
+                starttime <= dt_plus_duration(EventModel.timestamp, EventModel.duration)
+            )
+        if endtime:
+            q = q.where(EventModel.timestamp <= endtime)
+
+        return q

aw_datastore/storages/sqlite.py

@@ -0,0 +1,378 @@
+import json
+import logging
+import os
+import sqlite3
+from datetime import datetime, timedelta, timezone
+from typing import Iterable, List, Optional
+
+from aw_core.dirs import get_data_dir, legacy_testing_suffix
+from aw_core.models import Event
+
+from .abstract import AbstractStorage
+
+logger = logging.getLogger(__name__)
+
+LATEST_VERSION = 1
+
+# The max integer value in SQLite is signed 8 Bytes / 64 bits
+MAX_TIMESTAMP = 2**63 - 1
+
+CREATE_BUCKETS_TABLE = """
+    CREATE TABLE IF NOT EXISTS buckets (
+        rowid INTEGER PRIMARY KEY AUTOINCREMENT,
+        id TEXT UNIQUE NOT NULL,
+        name TEXT,
+        type TEXT NOT NULL,
+        client TEXT NOT NULL,
+        hostname TEXT NOT NULL,
+        created TEXT NOT NULL,
+        datastr TEXT NOT NULL
+    )
+"""
+
+CREATE_EVENTS_TABLE = """
+    CREATE TABLE IF NOT EXISTS events (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        bucketrow INTEGER NOT NULL,
+        starttime INTEGER NOT NULL,
+        endtime INTEGER NOT NULL,
+        datastr TEXT NOT NULL,
+        FOREIGN KEY (bucketrow) REFERENCES buckets(rowid)
+    )
+"""
+
+INDEX_BUCKETS_TABLE_ID = """
+    CREATE INDEX IF NOT EXISTS event_index_id ON events(id);
+"""
+
+INDEX_EVENTS_TABLE_STARTTIME = """
+    CREATE INDEX IF NOT EXISTS event_index_starttime ON events(bucketrow, starttime);
+"""
+INDEX_EVENTS_TABLE_ENDTIME = """
+    CREATE INDEX IF NOT EXISTS event_index_endtime ON events(bucketrow, endtime);
+"""
+
+
+def _rows_to_events(rows: Iterable) -> List[Event]:
+    events = []
+    for row in rows:
+        eid = row[0]
+        starttime = datetime.fromtimestamp(row[1] / 1000000, timezone.utc)
+        endtime = datetime.fromtimestamp(row[2] / 1000000, timezone.utc)
+        duration = endtime - starttime
+        data = json.loads(row[3])
+        events.append(Event(id=eid, timestamp=starttime, duration=duration, data=data))
+    return events
+
+
+class SqliteStorage(AbstractStorage):
+    sid = "sqlite"
+
+    def __init__(
+        self, testing, filepath: Optional[str] = None, enable_lazy_commit=True
+    ) -> None:
+        self.testing = testing
+        self.enable_lazy_commit = enable_lazy_commit
+
+        # Ignore the migration check if custom filepath is set
+        ignore_migration_check = filepath is not None
+
+        ds_name = self.sid + legacy_testing_suffix(testing)
+        if not filepath:
+            data_dir = get_data_dir("aw-server")
+            filename = ds_name + f".v{LATEST_VERSION}" + ".db"
+            filepath = os.path.join(data_dir, filename)
+
+        new_db_file = not os.path.exists(filepath)
+        self.conn = sqlite3.connect(filepath)
+        logger.info(f"Using database file: {filepath}")
+
+        # Create tables
+        self.conn.execute(CREATE_BUCKETS_TABLE)
+        self.conn.execute(CREATE_EVENTS_TABLE)
+        self.conn.execute(INDEX_BUCKETS_TABLE_ID)
+        self.conn.execute(INDEX_EVENTS_TABLE_STARTTIME)
+        self.conn.execute(INDEX_EVENTS_TABLE_ENDTIME)
+        self.conn.execute("PRAGMA journal_mode=WAL;")
+        self.commit()
+
+        if new_db_file and not ignore_migration_check:
+            logger.info("Created new SQlite db file")
+
+            from aw_datastore import check_for_migration  # fmt: skip
+
+            check_for_migration(self)
+
+        self.last_commit = datetime.now()
+        self.num_uncommitted_statements = 0
+
+    def commit(self):
+        """
+        Useful for debugging and trying to lower the amount of
+        unnecessary commits
+        """
+        self.conn.commit()
+        self.last_commit = datetime.now()
+        self.num_uncommitted_statements = 0
+
+    def conditional_commit(self, num_statements):
+        """
+        Only commit transactions if:
+         - We have a lot of statements in our transaction
+         - Was a while ago since last commit
+        This is because sqlite is very slow with small inserts, this
+        is a way to batch them together and lower CPU+disk usage
+        """
+        if self.enable_lazy_commit:
+            self.num_uncommitted_statements += num_statements
+            if self.num_uncommitted_statements > 50:
+                self.commit()
+            if (self.last_commit - datetime.now()) > timedelta(seconds=10):
+                self.commit()
+        else:
+            self.commit()
+
+    def buckets(self):
+        buckets = {}
+        c = self.conn.cursor()
+        for row in c.execute(
+            "SELECT id, name, type, client, hostname, created, datastr FROM buckets"
+        ):
+            buckets[row[0]] = {
+                "id": row[0],
+                "name": row[1],
+                "type": row[2],
+                "client": row[3],
+                "hostname": row[4],
+                "created": row[5],
+                "data": json.loads(row[6] or "{}"),
+            }
+        return buckets
+
+    def create_bucket(
+        self,
+        bucket_id: str,
+        type_id: str,
+        client: str,
+        hostname: str,
+        created: str,
+        name: Optional[str] = None,
+        data: Optional[dict] = None,
+    ):
+        self.conn.execute(
+            "INSERT INTO buckets(id, name, type, client, hostname, created, datastr) "
+            + "VALUES (?, ?, ?, ?, ?, ?, ?)",
+            [
+                bucket_id,
+                name,
+                type_id,
+                client,
+                hostname,
+                created,
+                json.dumps(data or {}),
+            ],
+        )
+        self.commit()
+        return self.get_metadata(bucket_id)
+
+    def update_bucket(
+        self,
+        bucket_id: str,
+        type_id: Optional[str] = None,
+        client: Optional[str] = None,
+        hostname: Optional[str] = None,
+        name: Optional[str] = None,
+        data: Optional[dict] = None,
+    ):
+        update_values = [
+            ("type", type_id),
+            ("client", client),
+            ("hostname", hostname),
+            ("name", name),
+            ("datastr", json.dumps(data) if data is not None else None),
+        ]
+        updates, values = zip(*[(k, v) for k, v in update_values if v is not None])
+        if not updates:
+            raise ValueError("At least one field must be updated.")
+
+        sql = (
+            "UPDATE buckets SET "
+            + ", ".join(f"{u} = ?" for u in updates)
+            + " WHERE id = ?"
+        )
+        self.conn.execute(sql, (*values, bucket_id))
+        self.commit()
+        return self.get_metadata(bucket_id)
+
+    def delete_bucket(self, bucket_id: str):
+        self.conn.execute(
+            "DELETE FROM events WHERE bucketrow IN (SELECT rowid FROM buckets WHERE id = ?)",
+            [bucket_id],
+        )
+        cursor = self.conn.execute("DELETE FROM buckets WHERE id = ?", [bucket_id])
+        self.commit()
+        if cursor.rowcount != 1:
+            raise ValueError("Bucket did not exist, could not delete")
+
+    def get_metadata(self, bucket_id: str):
+        c = self.conn.cursor()
+        res = c.execute(
+            "SELECT id, name, type, client, hostname, created, datastr FROM buckets WHERE id = ?",
+            [bucket_id],
+        )
+        row = res.fetchone()
+        if row is not None:
+            return {
+                "id": row[0],
+                "name": row[1],
+                "type": row[2],
+                "client": row[3],
+                "hostname": row[4],
+                "created": row[5],
+                "data": json.loads(row[6] or "{}"),
+            }
+        else:
+            raise ValueError("Bucket did not exist, could not get metadata")
+
+    def insert_one(self, bucket_id: str, event: Event) -> Event:
+        c = self.conn.cursor()
+        starttime = event.timestamp.timestamp() * 1000000
+        endtime = starttime + (event.duration.total_seconds() * 1000000)
+        datastr = json.dumps(event.data)
+        c.execute(
+            "INSERT INTO events(bucketrow, starttime, endtime, datastr) "
+            + "VALUES ((SELECT rowid FROM buckets WHERE id = ?), ?, ?, ?)",
+            [bucket_id, starttime, endtime, datastr],
+        )
+        event.id = c.lastrowid
+        self.conditional_commit(1)
+        return event
+
+    def insert_many(self, bucket_id, events: List[Event]) -> None:
+        # FIXME: Is this true not only for peewee but sqlite aswell?
+        # Chunking into lists of length 100 is needed here due to SQLITE_MAX_COMPOUND_SELECT
+        # and SQLITE_LIMIT_VARIABLE_NUMBER under Windows.
+        # See: https://github.com/coleifer/peewee/issues/948
+
+        # First, upsert events with id's set
+        events_upsert = [e for e in events if e.id is not None]
+        for e in events_upsert:
+            self.replace(bucket_id, e.id, e)
+
+        # Then insert events without id's set
+        events_insert = [e for e in events if e.id is None]
+        event_rows = []
+        for event in events_insert:
+            starttime = event.timestamp.timestamp() * 1000000
+            endtime = starttime + (event.duration.total_seconds() * 1000000)
+            datastr = json.dumps(event.data)
+            event_rows.append((bucket_id, starttime, endtime, datastr))
+        query = (
+            "INSERT INTO events(bucketrow, starttime, endtime, datastr) "
+            + "VALUES ((SELECT rowid FROM buckets WHERE id = ?), ?, ?, ?)"
+        )
+        self.conn.executemany(query, event_rows)
+        self.conditional_commit(len(event_rows))
+
+    def replace_last(self, bucket_id, event):
+        starttime = event.timestamp.timestamp() * 1000000
+        endtime = starttime + (event.duration.total_seconds() * 1000000)
+        datastr = json.dumps(event.data)
+        query = """UPDATE events
+                   SET starttime = ?, endtime = ?, datastr = ?
+                   WHERE id = (
+                        SELECT id FROM events WHERE endtime =
+                            (SELECT max(endtime) FROM events WHERE bucketrow =
+                                (SELECT rowid FROM buckets WHERE id = ?) LIMIT 1))"""
+        self.conn.execute(query, [starttime, endtime, datastr, bucket_id])
+        self.conditional_commit(1)
+        return True
+
+    def delete(self, bucket_id, event_id):
+        query = (
+            "DELETE FROM events "
+            + "WHERE id = ? AND bucketrow = (SELECT b.rowid FROM buckets b WHERE b.id = ?)"
+        )
+        cursor = self.conn.execute(query, [event_id, bucket_id])
+        return cursor.rowcount == 1
+
+    def replace(self, bucket_id, event_id, event) -> bool:
+        starttime = event.timestamp.timestamp() * 1000000
+        endtime = starttime + (event.duration.total_seconds() * 1000000)
+        datastr = json.dumps(event.data)
+        query = """UPDATE events
+                     SET bucketrow = (SELECT rowid FROM buckets WHERE id = ?),
+                         starttime = ?,
+                         endtime = ?,
+                         datastr = ?
+                     WHERE id = ?"""
+        self.conn.execute(query, [bucket_id, starttime, endtime, datastr, event_id])
+        self.conditional_commit(1)
+        return True
+
+    def get_event(
+        self,
+        bucket_id: str,
+        event_id: int,
+    ) -> Optional[Event]:
+        self.commit()
+        c = self.conn.cursor()
+        query = """
+            SELECT id, starttime, endtime, datastr
+            FROM events
+            WHERE bucketrow = (SELECT rowid FROM buckets WHERE id = ?) AND id = ?
+            LIMIT 1
+        """
+        rows = c.execute(query, [bucket_id, event_id])
+        events = _rows_to_events(rows)
+        if events:
+            return events[0]
+        else:
+            return None
+
+    def get_events(
+        self,
+        bucket_id: str,
+        limit: int,
+        starttime: Optional[datetime] = None,
+        endtime: Optional[datetime] = None,
+    ):
+        if limit == 0:
+            return []
+        elif limit < 0:
+            limit = -1
+        self.commit()
+        c = self.conn.cursor()
+        starttime_i = starttime.timestamp() * 1000000 if starttime else 0
+        endtime_i = endtime.timestamp() * 1000000 if endtime else MAX_TIMESTAMP
+        query = """
+            SELECT id, starttime, endtime, datastr
+            FROM events
+            WHERE bucketrow = (SELECT rowid FROM buckets WHERE id = ?)
+            AND endtime >= ? AND starttime <= ?
+            ORDER BY endtime DESC LIMIT ?
+        """
+        rows = c.execute(query, [bucket_id, starttime_i, endtime_i, limit])
+        events = _rows_to_events(rows)
+        return events
+
+    def get_eventcount(
+        self,
+        bucket_id: str,
+        starttime: Optional[datetime] = None,
+        endtime: Optional[datetime] = None,
+    ):
+        self.commit()
+        c = self.conn.cursor()
+        starttime_i = starttime.timestamp() * 1000000 if starttime else 0
+        endtime_i = endtime.timestamp() * 1000000 if endtime else MAX_TIMESTAMP
+        query = (
+            "SELECT count(*) "
+            + "FROM events "
+            + "WHERE bucketrow = (SELECT rowid FROM buckets WHERE id = ?) "
+            + "AND endtime >= ? AND starttime <= ?"
+        )
+        rows = c.execute(query, [bucket_id, starttime_i, endtime_i])
+        row = rows.fetchone()
+        eventcount = row[0]
+        return eventcount