snapshot of https://github.com/zulip/zulip.git @ 48c69911c7dd4e03b20552eb3bd77ad8151c1357 (2026-09-15)

analytics/lib/fixtures.py

@@ -0,0 +1,77 @@
+from math import sqrt
+from random import Random
+
+from analytics.lib.counts import CountStat
+
+
+def generate_time_series_data(
+    days: int = 100,
+    business_hours_base: float = 10,
+    non_business_hours_base: float = 10,
+    growth: float = 1,
+    autocorrelation: float = 0,
+    spikiness: float = 1,
+    holiday_rate: float = 0,
+    frequency: str = CountStat.DAY,
+    partial_sum: bool = False,
+    random_seed: int = 26,
+) -> list[int]:
+    """
+    Generate semi-realistic looking time series data for testing analytics graphs.
+
+    days -- Number of days of data. Is the number of data points generated if
+        frequency is CountStat.DAY.
+    business_hours_base -- Average value during a business hour (or day) at beginning of
+        time series, if frequency is CountStat.HOUR (CountStat.DAY, respectively).
+    non_business_hours_base -- The above, for non-business hours/days.
+    growth -- Ratio between average values at end of time series and beginning of time series.
+    autocorrelation -- Makes neighboring data points look more like each other. At 0 each
+        point is unaffected by the previous point, and at 1 each point is a deterministic
+        function of the previous point.
+    spikiness -- 0 means no randomness (other than holiday_rate), higher values increase
+        the variance.
+    holiday_rate -- Fraction of days randomly set to 0, largely for testing how we handle 0s.
+    frequency -- Should be CountStat.HOUR or CountStat.DAY.
+    partial_sum -- If True, return partial sum of the series.
+    random_seed -- Seed for random number generator.
+    """
+    rng = Random(random_seed)
+
+    if frequency == CountStat.HOUR:
+        length = days * 24
+        seasonality = [non_business_hours_base] * 24 * 7
+        for day in range(5):
+            for hour in range(8):
+                seasonality[24 * day + hour] = business_hours_base
+        holidays = []
+        for i in range(days):
+            holidays.extend([rng.random() < holiday_rate] * 24)
+    elif frequency == CountStat.DAY:
+        length = days
+        seasonality = [8 * business_hours_base + 16 * non_business_hours_base] * 5 + [
+            24 * non_business_hours_base
+        ] * 2
+        holidays = [rng.random() < holiday_rate for i in range(days)]
+    else:
+        raise AssertionError(f"Unknown frequency: {frequency}")
+    if length < 2:
+        raise AssertionError(
+            f"Must be generating at least 2 data points. Currently generating {length}"
+        )
+    growth_base = growth ** (1.0 / (length - 1))
+    values_no_noise = [seasonality[i % len(seasonality)] * (growth_base**i) for i in range(length)]
+
+    noise_scalars = [rng.gauss(0, 1)]
+    for i in range(1, length):
+        noise_scalars.append(
+            noise_scalars[-1] * autocorrelation + rng.gauss(0, 1) * (1 - autocorrelation)
+        )
+
+    values = [
+        0 if holiday else int(v + sqrt(v) * noise_scalar * spikiness)
+        for v, noise_scalar, holiday in zip(values_no_noise, noise_scalars, holidays, strict=False)
+    ]
+    if partial_sum:
+        for i in range(1, length):
+            values[i] = values[i - 1] + values[i]
+    return [max(v, 0) for v in values]

analytics/lib/time_utils.py

@@ -0,0 +1,33 @@
+from datetime import datetime, timedelta
+
+from analytics.lib.counts import CountStat
+from zerver.lib.timestamp import floor_to_day, floor_to_hour, verify_UTC
+
+
+# If min_length is None, returns end_times from ceiling(start) to floor(end), inclusive.
+# If min_length is greater than 0, pads the list to the left.
+# So informally, time_range(Sep 20, Sep 22, day, None) returns [Sep 20, Sep 21, Sep 22],
+# and time_range(Sep 20, Sep 22, day, 5) returns [Sep 18, Sep 19, Sep 20, Sep 21, Sep 22]
+def time_range(
+    start: datetime, end: datetime, frequency: str, min_length: int | None
+) -> list[datetime]:
+    verify_UTC(start)
+    verify_UTC(end)
+    if frequency == CountStat.HOUR:
+        end = floor_to_hour(end)
+        step = timedelta(hours=1)
+    elif frequency == CountStat.DAY:
+        end = floor_to_day(end)
+        step = timedelta(days=1)
+    else:
+        raise AssertionError(f"Unknown frequency: {frequency}")
+
+    times = []
+    if min_length is not None:
+        start = min(start, end - (min_length - 1) * step)
+    current = end
+    while current >= start:
+        times.append(current)
+        current -= step
+    times.reverse()
+    return times

analytics/management/commands/check_analytics_state.py

@@ -0,0 +1,92 @@
+from dataclasses import dataclass
+from datetime import timedelta
+from typing import Any, Literal
+
+from django.utils.timezone import now as timezone_now
+from typing_extensions import override
+
+from analytics.lib.counts import ALL_COUNT_STATS, CountStat
+from analytics.models import installation_epoch
+from scripts.lib.zulip_tools import atomic_nagios_write
+from zerver.lib.management import ZulipBaseCommand
+from zerver.lib.timestamp import TimeZoneNotUTCError, floor_to_day, floor_to_hour, verify_UTC
+from zerver.models import Realm
+
+states = {
+    0: "OK",
+    1: "WARNING",
+    2: "CRITICAL",
+    3: "UNKNOWN",
+}
+
+
+@dataclass
+class NagiosResult:
+    status: Literal["ok", "warning", "critical", "unknown"]
+    message: str
+
+
+class Command(ZulipBaseCommand):
+    help = """Checks FillState table.
+
+    Run as a cron job that runs every hour."""
+
+    @override
+    def handle(self, *args: Any, **options: Any) -> None:
+        fill_state = self.get_fill_state()
+        atomic_nagios_write("check-analytics-state", fill_state.status, fill_state.message)
+
+    def get_fill_state(self) -> NagiosResult:
+        if not Realm.objects.exists():
+            return NagiosResult(status="ok", message="No realms exist, so not checking FillState.")
+
+        warning_unfilled_properties = []
+        critical_unfilled_properties = []
+        for property, stat in ALL_COUNT_STATS.items():
+            last_fill = stat.last_successful_fill()
+            if last_fill is None:
+                last_fill = installation_epoch()
+            try:
+                verify_UTC(last_fill)
+            except TimeZoneNotUTCError:
+                return NagiosResult(
+                    status="critical", message=f"FillState not in UTC for {property}"
+                )
+
+            if stat.frequency == CountStat.DAY:
+                floor_function = floor_to_day
+                warning_threshold = timedelta(hours=26)
+                critical_threshold = timedelta(hours=50)
+            else:  # CountStat.HOUR
+                floor_function = floor_to_hour
+                warning_threshold = timedelta(minutes=90)
+                critical_threshold = timedelta(minutes=150)
+
+            if floor_function(last_fill) != last_fill:
+                return NagiosResult(
+                    status="critical",
+                    message=f"FillState not on {stat.frequency} boundary for {property}",
+                )
+
+            time_to_last_fill = timezone_now() - last_fill
+            if time_to_last_fill > critical_threshold:
+                critical_unfilled_properties.append(property)
+            elif time_to_last_fill > warning_threshold:
+                warning_unfilled_properties.append(property)
+
+        if len(critical_unfilled_properties) == 0 and len(warning_unfilled_properties) == 0:
+            return NagiosResult(status="ok", message="FillState looks fine.")
+        if len(critical_unfilled_properties) == 0:
+            return NagiosResult(
+                status="warning",
+                message="Missed filling {} once.".format(
+                    ", ".join(warning_unfilled_properties),
+                ),
+            )
+        return NagiosResult(
+            status="critical",
+            message="Missed filling {} once. Missed filling {} at least twice.".format(
+                ", ".join(warning_unfilled_properties),
+                ", ".join(critical_unfilled_properties),
+            ),
+        )

analytics/management/commands/clear_analytics_tables.py

@@ -0,0 +1,25 @@
+from argparse import ArgumentParser
+from typing import Any
+
+from django.core.management.base import CommandError
+from typing_extensions import override
+
+from analytics.lib.counts import do_drop_all_analytics_tables
+from zerver.lib.management import ZulipBaseCommand
+
+
+class Command(ZulipBaseCommand):
+    help = """Clear analytics tables."""
+
+    @override
+    def add_arguments(self, parser: ArgumentParser) -> None:
+        parser.add_argument("--force", action="store_true", help="Clear analytics tables.")
+
+    @override
+    def handle(self, *args: Any, **options: Any) -> None:
+        if options["force"]:
+            do_drop_all_analytics_tables()
+        else:
+            raise CommandError(
+                "Would delete all data from analytics tables (!); use --force to do so."
+            )

analytics/management/commands/clear_single_stat.py

@@ -0,0 +1,27 @@
+from argparse import ArgumentParser
+from typing import Any
+
+from django.core.management.base import CommandError
+from typing_extensions import override
+
+from analytics.lib.counts import ALL_COUNT_STATS, do_drop_single_stat
+from zerver.lib.management import ZulipBaseCommand
+
+
+class Command(ZulipBaseCommand):
+    help = """Clear analytics tables."""
+
+    @override
+    def add_arguments(self, parser: ArgumentParser) -> None:
+        parser.add_argument("--force", action="store_true", help="Actually do it.")
+        parser.add_argument("--property", help="The property of the stat to be cleared.")
+
+    @override
+    def handle(self, *args: Any, **options: Any) -> None:
+        property = options["property"]
+        if property not in ALL_COUNT_STATS:
+            raise CommandError(f"Invalid property: {property}")
+        if not options["force"]:
+            raise CommandError("No action taken. Use --force.")
+
+        do_drop_single_stat(property)

analytics/management/commands/populate_analytics_db.py

@@ -0,0 +1,353 @@
+from collections.abc import Mapping
+from datetime import timedelta
+from typing import Any, TypeAlias
+
+from django.core.files.uploadedfile import UploadedFile
+from django.utils.timezone import now as timezone_now
+from typing_extensions import override
+
+from analytics.lib.counts import COUNT_STATS, CountStat, do_drop_all_analytics_tables
+from analytics.lib.fixtures import generate_time_series_data
+from analytics.lib.time_utils import time_range
+from analytics.models import (
+    BaseCount,
+    FillState,
+    InstallationCount,
+    RealmCount,
+    StreamCount,
+    UserCount,
+)
+from zerver.actions.create_realm import do_create_realm
+from zerver.lib.create_user import create_user
+from zerver.lib.management import ZulipBaseCommand
+from zerver.lib.storage import static_path
+from zerver.lib.stream_color import STREAM_ASSIGNMENT_COLORS
+from zerver.lib.stream_subscription import create_stream_subscription
+from zerver.lib.streams import get_default_values_for_stream_permission_group_settings
+from zerver.lib.timestamp import floor_to_day
+from zerver.lib.upload import upload_message_attachment_from_request
+from zerver.models import Attachment, Client, Realm, RealmAuditLog, Recipient, Stream, UserProfile
+from zerver.models.groups import NamedUserGroup, SystemGroups, UserGroupMembership
+from zerver.models.realm_audit_logs import AuditLogEventType
+
+
+class Command(ZulipBaseCommand):
+    help = """Populates analytics tables with randomly generated data."""
+
+    DAYS_OF_DATA = 100
+    random_seed = 26
+
+    def generate_fixture_data(
+        self,
+        stat: CountStat,
+        business_hours_base: float,
+        non_business_hours_base: float,
+        growth: float,
+        autocorrelation: float,
+        spikiness: float,
+        holiday_rate: float = 0,
+        partial_sum: bool = False,
+    ) -> list[int]:
+        self.random_seed += 1
+        return generate_time_series_data(
+            days=self.DAYS_OF_DATA,
+            business_hours_base=business_hours_base,
+            non_business_hours_base=non_business_hours_base,
+            growth=growth,
+            autocorrelation=autocorrelation,
+            spikiness=spikiness,
+            holiday_rate=holiday_rate,
+            frequency=stat.frequency,
+            partial_sum=partial_sum,
+            random_seed=self.random_seed,
+        )
+
+    @override
+    def handle(self, *args: Any, **options: Any) -> None:
+        # TODO: This should arguably only delete the objects
+        # associated with the "analytics" realm.
+        do_drop_all_analytics_tables()
+
+        # Delete attachment added for summary statistic before deleting
+        # the Realm object.
+        Attachment.objects.filter(realm__string_id="analytics").delete()
+
+        # This also deletes any objects with this realm as a foreign key
+        Realm.objects.filter(string_id="analytics").delete()
+
+        # Because we just deleted a bunch of objects in the database
+        # directly (rather than deleting individual objects in Django,
+        # in which case our post_save hooks would have flushed the
+        # individual objects from memcached for us), we need to flush
+        # memcached in order to ensure deleted objects aren't still
+        # present in the memcached cache.
+        from zerver.apps import flush_cache
+
+        flush_cache(None)
+
+        installation_time = timezone_now() - timedelta(days=self.DAYS_OF_DATA)
+        last_end_time = floor_to_day(timezone_now())
+        realm = do_create_realm(
+            string_id="analytics", name="Analytics", date_created=installation_time
+        )
+
+        owners_system_group = NamedUserGroup.objects.get(
+            name=SystemGroups.OWNERS, realm_for_sharding=realm, is_system_group=True
+        )
+        guests_system_group = NamedUserGroup.objects.get(
+            name=SystemGroups.EVERYONE, realm_for_sharding=realm, is_system_group=True
+        )
+
+        shylock = create_user(
+            "shylock@analytics.ds",
+            "Shylock",
+            realm,
+            full_name="Shylock",
+            role=UserProfile.ROLE_REALM_OWNER,
+            force_date_joined=installation_time,
+        )
+        UserGroupMembership.objects.create(user_profile=shylock, user_group=owners_system_group)
+
+        # Create guest user for set_guest_users_statistic.
+        bassanio = create_user(
+            "bassanio@analytics.ds",
+            "Bassanio",
+            realm,
+            full_name="Bassanio",
+            role=UserProfile.ROLE_GUEST,
+            force_date_joined=installation_time,
+        )
+        UserGroupMembership.objects.create(user_profile=bassanio, user_group=guests_system_group)
+
+        stream = Stream.objects.create(
+            name="all",
+            realm=realm,
+            date_created=installation_time,
+            **get_default_values_for_stream_permission_group_settings(realm),
+        )
+        recipient = Recipient.objects.create(type_id=stream.id, type=Recipient.STREAM)
+        stream.recipient = recipient
+        stream.save(update_fields=["recipient"])
+
+        # Subscribe shylock to the stream to avoid invariant failures.
+        create_stream_subscription(
+            user_profile=shylock,
+            recipient=recipient,
+            stream=stream,
+            color=STREAM_ASSIGNMENT_COLORS[0],
+        )
+        RealmAuditLog.objects.create(
+            realm=realm,
+            modified_user=shylock,
+            modified_stream=stream,
+            event_last_message_id=0,
+            event_type=AuditLogEventType.SUBSCRIPTION_CREATED,
+            event_time=installation_time,
+        )
+
+        # Create an attachment in the database for set_storage_space_used_statistic.
+        IMAGE_FILE_PATH = static_path("images/test-images/checkbox.png")
+        with open(IMAGE_FILE_PATH, "rb") as fp:
+            upload_message_attachment_from_request(UploadedFile(fp), shylock)
+
+        FixtureData: TypeAlias = Mapping[str | int | None, list[int]]
+
+        def insert_fixture_data(
+            stat: CountStat,
+            fixture_data: FixtureData,
+            table: type[BaseCount],
+        ) -> None:
+            end_times = time_range(
+                last_end_time, last_end_time, stat.frequency, len(next(iter(fixture_data.values())))
+            )
+            id_args: dict[str, Realm | UserProfile | Stream] = {}
+            if table == RealmCount:
+                id_args = {"realm": realm}
+            if table == UserCount:
+                id_args = {"realm": realm, "user": shylock}
+            if table == StreamCount:
+                id_args = {"realm": realm, "stream": stream}
+
+            for subgroup, values in fixture_data.items():
+                table._default_manager.bulk_create(
+                    table(
+                        property=stat.property,
+                        subgroup=subgroup,
+                        end_time=end_time,
+                        value=value,
+                        **id_args,
+                    )
+                    for end_time, value in zip(end_times, values, strict=False)
+                    if value != 0
+                )
+
+        stat = COUNT_STATS["1day_actives::day"]
+        realm_data: FixtureData = {
+            None: self.generate_fixture_data(stat, 0.08, 0.02, 3, 0.3, 6, partial_sum=True),
+        }
+        insert_fixture_data(stat, realm_data, RealmCount)
+        installation_data: FixtureData = {
+            None: self.generate_fixture_data(stat, 0.8, 0.2, 4, 0.3, 6, partial_sum=True),
+        }
+        insert_fixture_data(stat, installation_data, InstallationCount)
+        FillState.objects.create(
+            property=stat.property, end_time=last_end_time, state=FillState.DONE
+        )
+
+        stat = COUNT_STATS["7day_actives::day"]
+        realm_data = {
+            None: self.generate_fixture_data(stat, 0.2, 0.07, 3, 0.3, 6, partial_sum=True),
+        }
+        insert_fixture_data(stat, realm_data, RealmCount)
+        installation_data = {
+            None: self.generate_fixture_data(stat, 2, 0.7, 4, 0.3, 6, partial_sum=True),
+        }
+        insert_fixture_data(stat, installation_data, InstallationCount)
+        FillState.objects.create(
+            property=stat.property, end_time=last_end_time, state=FillState.DONE
+        )
+
+        stat = COUNT_STATS["realm_active_humans::day"]
+        realm_data = {
+            None: self.generate_fixture_data(stat, 0.8, 0.08, 3, 0.5, 3, partial_sum=True),
+        }
+        insert_fixture_data(stat, realm_data, RealmCount)
+        installation_data = {
+            None: self.generate_fixture_data(stat, 1, 0.3, 4, 0.5, 3, partial_sum=True),
+        }
+        insert_fixture_data(stat, installation_data, InstallationCount)
+        FillState.objects.create(
+            property=stat.property, end_time=last_end_time, state=FillState.DONE
+        )
+
+        stat = COUNT_STATS["active_users_audit:is_bot:day"]
+        realm_data = {
+            "false": self.generate_fixture_data(stat, 1, 0.2, 3.5, 0.8, 2, partial_sum=True),
+            "true": self.generate_fixture_data(stat, 0.3, 0.05, 3, 0.3, 2, partial_sum=True),
+        }
+        insert_fixture_data(stat, realm_data, RealmCount)
+        installation_data = {
+            "false": self.generate_fixture_data(stat, 3, 1, 4, 0.8, 2, partial_sum=True),
+            "true": self.generate_fixture_data(stat, 1, 0.4, 4, 0.8, 2, partial_sum=True),
+        }
+        insert_fixture_data(stat, installation_data, InstallationCount)
+        FillState.objects.create(
+            property=stat.property, end_time=last_end_time, state=FillState.DONE
+        )
+
+        stat = COUNT_STATS["messages_sent:is_bot:hour"]
+        user_data: FixtureData = {
+            "false": self.generate_fixture_data(stat, 2, 1, 1.5, 0.6, 8, holiday_rate=0.1),
+        }
+        insert_fixture_data(stat, user_data, UserCount)
+        realm_data = {
+            "false": self.generate_fixture_data(stat, 35, 15, 6, 0.6, 4),
+            "true": self.generate_fixture_data(stat, 15, 15, 3, 0.4, 2),
+        }
+        insert_fixture_data(stat, realm_data, RealmCount)
+        installation_data = {
+            "false": self.generate_fixture_data(stat, 350, 150, 6, 0.6, 4),
+            "true": self.generate_fixture_data(stat, 150, 150, 3, 0.4, 2),
+        }
+        insert_fixture_data(stat, installation_data, InstallationCount)
+        FillState.objects.create(
+            property=stat.property, end_time=last_end_time, state=FillState.DONE
+        )
+
+        stat = COUNT_STATS["messages_sent:message_type:day"]
+        user_data = {
+            "public_stream": self.generate_fixture_data(stat, 1.5, 1, 3, 0.6, 8),
+            "private_message": self.generate_fixture_data(stat, 0.5, 0.3, 1, 0.6, 8),
+            "huddle_message": self.generate_fixture_data(stat, 0.2, 0.2, 2, 0.6, 8),
+        }
+        insert_fixture_data(stat, user_data, UserCount)
+        realm_data = {
+            "public_stream": self.generate_fixture_data(stat, 30, 8, 5, 0.6, 4),
+            "private_stream": self.generate_fixture_data(stat, 7, 7, 5, 0.6, 4),
+            "private_message": self.generate_fixture_data(stat, 13, 5, 5, 0.6, 4),
+            "huddle_message": self.generate_fixture_data(stat, 6, 3, 3, 0.6, 4),
+        }
+        insert_fixture_data(stat, realm_data, RealmCount)
+        installation_data = {
+            "public_stream": self.generate_fixture_data(stat, 300, 80, 5, 0.6, 4),
+            "private_stream": self.generate_fixture_data(stat, 70, 70, 5, 0.6, 4),
+            "private_message": self.generate_fixture_data(stat, 130, 50, 5, 0.6, 4),
+            "huddle_message": self.generate_fixture_data(stat, 60, 30, 3, 0.6, 4),
+        }
+        insert_fixture_data(stat, installation_data, InstallationCount)
+        FillState.objects.create(
+            property=stat.property, end_time=last_end_time, state=FillState.DONE
+        )
+
+        website, _created = Client.objects.get_or_create(name="website")
+        old_desktop, _created = Client.objects.get_or_create(name="desktop app Linux 0.3.7")
+        android, _created = Client.objects.get_or_create(name="ZulipAndroid")
+        iOS, _created = Client.objects.get_or_create(name="ZulipiOS")
+        react_native, _created = Client.objects.get_or_create(name="ZulipMobile")
+        flutter, _created = Client.objects.get_or_create(name="ZulipFlutter")
+        API, _created = Client.objects.get_or_create(name="API: Python")
+        irc_mirror, _created = Client.objects.get_or_create(name="irc_mirror")
+        unused, _created = Client.objects.get_or_create(name="unused")
+        long_webhook, _created = Client.objects.get_or_create(name="ZulipLooooooooooongNameWebhook")
+
+        stat = COUNT_STATS["messages_sent:client:day"]
+        user_data = {
+            website.id: self.generate_fixture_data(stat, 2, 1, 1.5, 0.6, 8),
+            irc_mirror.id: self.generate_fixture_data(stat, 0, 0.3, 1.5, 0.6, 8),
+        }
+        insert_fixture_data(stat, user_data, UserCount)
+        realm_data = {
+            website.id: self.generate_fixture_data(stat, 30, 20, 5, 0.6, 3),
+            old_desktop.id: self.generate_fixture_data(stat, 5, 3, 8, 0.6, 3),
+            android.id: self.generate_fixture_data(stat, 5, 5, 2, 0.6, 3),
+            iOS.id: self.generate_fixture_data(stat, 5, 5, 2, 0.6, 3),
+            react_native.id: self.generate_fixture_data(stat, 5, 5, 10, 0.6, 3),
+            flutter.id: self.generate_fixture_data(stat, 5, 5, 10, 0.6, 3),
+            API.id: self.generate_fixture_data(stat, 5, 5, 5, 0.6, 3),
+            irc_mirror.id: self.generate_fixture_data(stat, 1, 1, 3, 0.6, 3),
+            unused.id: self.generate_fixture_data(stat, 0, 0, 0, 0, 0),
+            long_webhook.id: self.generate_fixture_data(stat, 5, 5, 2, 0.6, 3),
+        }
+        insert_fixture_data(stat, realm_data, RealmCount)
+        installation_data = {
+            website.id: self.generate_fixture_data(stat, 300, 200, 5, 0.6, 3),
+            old_desktop.id: self.generate_fixture_data(stat, 50, 30, 8, 0.6, 3),
+            android.id: self.generate_fixture_data(stat, 50, 50, 2, 0.6, 3),
+            iOS.id: self.generate_fixture_data(stat, 50, 50, 2, 0.6, 3),
+            flutter.id: self.generate_fixture_data(stat, 5, 5, 10, 0.6, 3),
+            react_native.id: self.generate_fixture_data(stat, 5, 5, 10, 0.6, 3),
+            API.id: self.generate_fixture_data(stat, 50, 50, 5, 0.6, 3),
+            irc_mirror.id: self.generate_fixture_data(stat, 10, 10, 3, 0.6, 3),
+            unused.id: self.generate_fixture_data(stat, 0, 0, 0, 0, 0),
+            long_webhook.id: self.generate_fixture_data(stat, 50, 50, 2, 0.6, 3),
+        }
+        insert_fixture_data(stat, installation_data, InstallationCount)
+        FillState.objects.create(
+            property=stat.property, end_time=last_end_time, state=FillState.DONE
+        )
+
+        stat = COUNT_STATS["messages_in_stream:is_bot:day"]
+        realm_data = {
+            "false": self.generate_fixture_data(stat, 30, 5, 6, 0.6, 4),
+            "true": self.generate_fixture_data(stat, 20, 2, 3, 0.2, 3),
+        }
+        insert_fixture_data(stat, realm_data, RealmCount)
+        stream_data: Mapping[int | str | None, list[int]] = {
+            "false": self.generate_fixture_data(stat, 10, 7, 5, 0.6, 4),
+            "true": self.generate_fixture_data(stat, 5, 3, 2, 0.4, 2),
+        }
+        insert_fixture_data(stat, stream_data, StreamCount)
+        FillState.objects.create(
+            property=stat.property, end_time=last_end_time, state=FillState.DONE
+        )
+
+        stat = COUNT_STATS["messages_read::hour"]
+        user_data = {
+            None: self.generate_fixture_data(stat, 7, 3, 2, 0.6, 8, holiday_rate=0.1),
+        }
+        insert_fixture_data(stat, user_data, UserCount)
+        realm_data = {None: self.generate_fixture_data(stat, 50, 35, 6, 0.6, 4)}
+        insert_fixture_data(stat, realm_data, RealmCount)
+        FillState.objects.create(
+            property=stat.property, end_time=last_end_time, state=FillState.DONE
+        )

analytics/management/commands/update_analytics_counts.py

@@ -0,0 +1,99 @@
+import hashlib
+import time
+from argparse import ArgumentParser
+from datetime import timezone
+from typing import Any
+
+from django.conf import settings
+from django.utils.dateparse import parse_datetime
+from django.utils.timezone import now as timezone_now
+from typing_extensions import override
+
+from analytics.lib.counts import ALL_COUNT_STATS, logger, process_count_stat
+from zerver.lib.management import ZulipBaseCommand, abort_cron_during_deploy, abort_unless_locked
+from zerver.lib.remote_server import send_server_data_to_push_bouncer, should_send_analytics_data
+from zerver.lib.timestamp import floor_to_hour
+from zerver.models import Realm
+
+
+class Command(ZulipBaseCommand):
+    help = """Fills Analytics tables.
+
+    Run as a cron job that runs every hour."""
+
+    @override
+    def add_arguments(self, parser: ArgumentParser) -> None:
+        parser.add_argument(
+            "--time",
+            "-t",
+            help="Update stat tables from current state to --time. Defaults to the current time.",
+            default=timezone_now().isoformat(),
+        )
+        parser.add_argument("--utc", action="store_true", help="Interpret --time in UTC.")
+        parser.add_argument(
+            "--stat", "-s", help="CountStat to process. If omitted, all stats are processed."
+        )
+        parser.add_argument(
+            "--verbose", action="store_true", help="Print timing information to stdout."
+        )
+
+    @override
+    @abort_cron_during_deploy
+    @abort_unless_locked
+    def handle(self, *args: Any, **options: Any) -> None:
+        self.run_update_analytics_counts(options)
+
+    def run_update_analytics_counts(self, options: dict[str, Any]) -> None:
+        # installation_epoch relies on there being at least one realm; we
+        # shouldn't run the analytics code if that condition isn't satisfied
+        if not Realm.objects.exists():
+            logger.info("No realms, stopping update_analytics_counts")
+            return
+
+        fill_to_time = parse_datetime(options["time"])
+        assert fill_to_time is not None
+        if options["utc"]:
+            fill_to_time = fill_to_time.replace(tzinfo=timezone.utc)
+        if fill_to_time.tzinfo is None:
+            raise ValueError(
+                "--time must be time-zone-aware. Maybe you meant to use the --utc option?"
+            )
+
+        fill_to_time = floor_to_hour(fill_to_time.astimezone(timezone.utc))
+
+        if options["stat"] is not None:
+            stats = [ALL_COUNT_STATS[options["stat"]]]
+        else:
+            stats = list(ALL_COUNT_STATS.values())
+
+        logger.info("Starting updating analytics counts through %s", fill_to_time)
+        if options["verbose"]:
+            start = time.time()
+            last = start
+
+        for stat in stats:
+            process_count_stat(stat, fill_to_time)
+            if options["verbose"]:
+                print(f"Updated {stat.property} in {time.time() - last:.3f}s")
+                last = time.time()
+
+        if options["verbose"]:
+            print(
+                f"Finished updating analytics counts through {fill_to_time} in {time.time() - start:.3f}s"
+            )
+        logger.info("Finished updating analytics counts through %s", fill_to_time)
+
+        if should_send_analytics_data():
+            # Based on the specific value of the setting, the exact details to send
+            # will be decided. However, we proceed just based on this not being falsey.
+
+            # Skew 0-10 minutes based on a hash of settings.ZULIP_ORG_ID, so
+            # that each server will report in at a somewhat consistent time.
+            assert settings.ZULIP_ORG_ID
+            delay = int.from_bytes(
+                hashlib.sha256(settings.ZULIP_ORG_ID.encode()).digest(), byteorder="big"
+            ) % (60 * 10)
+            logger.info("Sleeping %d seconds before reporting...", delay)
+            time.sleep(delay)
+
+            send_server_data_to_push_bouncer(consider_usage_statistics=True, raise_on_error=True)

analytics/migrations/0001_initial.py

@@ -0,0 +1,208 @@
+import django.db.models.deletion
+from django.conf import settings
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("zerver", "0030_realm_org_type"),
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="Anomaly",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        verbose_name="ID", serialize=False, auto_created=True, primary_key=True
+                    ),
+                ),
+                ("info", models.CharField(max_length=1000)),
+            ],
+            bases=(models.Model,),
+        ),
+        migrations.CreateModel(
+            name="HuddleCount",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        verbose_name="ID", serialize=False, auto_created=True, primary_key=True
+                    ),
+                ),
+                (
+                    "huddle",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE, to="zerver.Recipient"
+                    ),
+                ),
+                (
+                    "user",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL
+                    ),
+                ),
+                ("property", models.CharField(max_length=40)),
+                ("end_time", models.DateTimeField()),
+                ("interval", models.CharField(max_length=20)),
+                ("value", models.BigIntegerField()),
+                (
+                    "anomaly",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        to="analytics.Anomaly",
+                        null=True,
+                    ),
+                ),
+            ],
+            bases=(models.Model,),
+        ),
+        migrations.CreateModel(
+            name="InstallationCount",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        verbose_name="ID", serialize=False, auto_created=True, primary_key=True
+                    ),
+                ),
+                ("property", models.CharField(max_length=40)),
+                ("end_time", models.DateTimeField()),
+                ("interval", models.CharField(max_length=20)),
+                ("value", models.BigIntegerField()),
+                (
+                    "anomaly",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        to="analytics.Anomaly",
+                        null=True,
+                    ),
+                ),
+            ],
+            bases=(models.Model,),
+        ),
+        migrations.CreateModel(
+            name="RealmCount",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        verbose_name="ID", serialize=False, auto_created=True, primary_key=True
+                    ),
+                ),
+                (
+                    "realm",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE, to="zerver.Realm"
+                    ),
+                ),
+                ("property", models.CharField(max_length=40)),
+                ("end_time", models.DateTimeField()),
+                ("interval", models.CharField(max_length=20)),
+                ("value", models.BigIntegerField()),
+                (
+                    "anomaly",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        to="analytics.Anomaly",
+                        null=True,
+                    ),
+                ),
+            ],
+            bases=(models.Model,),
+        ),
+        migrations.CreateModel(
+            name="StreamCount",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        verbose_name="ID", serialize=False, auto_created=True, primary_key=True
+                    ),
+                ),
+                (
+                    "realm",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE, to="zerver.Realm"
+                    ),
+                ),
+                (
+                    "stream",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE, to="zerver.Stream"
+                    ),
+                ),
+                ("property", models.CharField(max_length=40)),
+                ("end_time", models.DateTimeField()),
+                ("interval", models.CharField(max_length=20)),
+                ("value", models.BigIntegerField()),
+                (
+                    "anomaly",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        to="analytics.Anomaly",
+                        null=True,
+                    ),
+                ),
+            ],
+            bases=(models.Model,),
+        ),
+        migrations.CreateModel(
+            name="UserCount",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        verbose_name="ID", serialize=False, auto_created=True, primary_key=True
+                    ),
+                ),
+                (
+                    "realm",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE, to="zerver.Realm"
+                    ),
+                ),
+                (
+                    "user",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL
+                    ),
+                ),
+                ("property", models.CharField(max_length=40)),
+                ("end_time", models.DateTimeField()),
+                ("interval", models.CharField(max_length=20)),
+                ("value", models.BigIntegerField()),
+                (
+                    "anomaly",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        to="analytics.Anomaly",
+                        null=True,
+                    ),
+                ),
+            ],
+            bases=(models.Model,),
+        ),
+        migrations.AlterUniqueTogether(
+            name="usercount",
+            unique_together={("user", "property", "end_time", "interval")},
+        ),
+        migrations.AlterUniqueTogether(
+            name="streamcount",
+            unique_together={("stream", "property", "end_time", "interval")},
+        ),
+        migrations.AlterUniqueTogether(
+            name="realmcount",
+            unique_together={("realm", "property", "end_time", "interval")},
+        ),
+        migrations.AlterUniqueTogether(
+            name="installationcount",
+            unique_together={("property", "end_time", "interval")},
+        ),
+        migrations.AlterUniqueTogether(
+            name="huddlecount",
+            unique_together={("huddle", "property", "end_time", "interval")},
+        ),
+    ]

analytics/migrations/0001_squashed_0021_alter_fillstate_id.py

@@ -0,0 +1,224 @@
+# Generated by Django 5.0.7 on 2024-08-13 20:16
+
+import django.db.models.deletion
+from django.conf import settings
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    replaces = [
+        ("analytics", "0001_initial"),
+        ("analytics", "0002_remove_huddlecount"),
+        ("analytics", "0003_fillstate"),
+        ("analytics", "0004_add_subgroup"),
+        ("analytics", "0005_alter_field_size"),
+        ("analytics", "0006_add_subgroup_to_unique_constraints"),
+        ("analytics", "0007_remove_interval"),
+        ("analytics", "0008_add_count_indexes"),
+        ("analytics", "0009_remove_messages_to_stream_stat"),
+        ("analytics", "0010_clear_messages_sent_values"),
+        ("analytics", "0011_clear_analytics_tables"),
+        ("analytics", "0012_add_on_delete"),
+        ("analytics", "0013_remove_anomaly"),
+        ("analytics", "0014_remove_fillstate_last_modified"),
+        ("analytics", "0015_clear_duplicate_counts"),
+        ("analytics", "0016_unique_constraint_when_subgroup_null"),
+        ("analytics", "0017_regenerate_partial_indexes"),
+        ("analytics", "0018_remove_usercount_active_users_audit"),
+        ("analytics", "0019_remove_unused_counts"),
+        ("analytics", "0020_alter_installationcount_id_alter_realmcount_id_and_more"),
+        ("analytics", "0021_alter_fillstate_id"),
+    ]
+
+    initial = True
+
+    dependencies = [
+        # Needed for foreign keys to core models like Realm.
+        ("zerver", "0001_initial"),
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="InstallationCount",
+            fields=[
+                (
+                    "id",
+                    models.BigAutoField(
+                        auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
+                    ),
+                ),
+                ("property", models.CharField(max_length=32)),
+                ("end_time", models.DateTimeField()),
+                ("value", models.BigIntegerField()),
+                ("subgroup", models.CharField(max_length=16, null=True)),
+            ],
+            options={
+                "unique_together": set(),
+                "constraints": [
+                    models.UniqueConstraint(
+                        condition=models.Q(("subgroup__isnull", False)),
+                        fields=("property", "subgroup", "end_time"),
+                        name="unique_installation_count",
+                    ),
+                    models.UniqueConstraint(
+                        condition=models.Q(("subgroup__isnull", True)),
+                        fields=("property", "end_time"),
+                        name="unique_installation_count_null_subgroup",
+                    ),
+                ],
+            },
+        ),
+        migrations.CreateModel(
+            name="RealmCount",
+            fields=[
+                (
+                    "id",
+                    models.BigAutoField(
+                        auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
+                    ),
+                ),
+                (
+                    "realm",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE, to="zerver.realm"
+                    ),
+                ),
+                ("property", models.CharField(max_length=32)),
+                ("end_time", models.DateTimeField()),
+                ("value", models.BigIntegerField()),
+                ("subgroup", models.CharField(max_length=16, null=True)),
+            ],
+            options={
+                "indexes": [
+                    models.Index(
+                        fields=["property", "end_time"],
+                        name="analytics_realmcount_property_end_time_3b60396b_idx",
+                    )
+                ],
+                "unique_together": set(),
+                "constraints": [
+                    models.UniqueConstraint(
+                        condition=models.Q(("subgroup__isnull", False)),
+                        fields=("realm", "property", "subgroup", "end_time"),
+                        name="unique_realm_count",
+                    ),
+                    models.UniqueConstraint(
+                        condition=models.Q(("subgroup__isnull", True)),
+                        fields=("realm", "property", "end_time"),
+                        name="unique_realm_count_null_subgroup",
+                    ),
+                ],
+            },
+        ),
+        migrations.CreateModel(
+            name="StreamCount",
+            fields=[
+                (
+                    "id",
+                    models.BigAutoField(
+                        auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
+                    ),
+                ),
+                (
+                    "realm",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE, to="zerver.realm"
+                    ),
+                ),
+                (
+                    "stream",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE, to="zerver.stream"
+                    ),
+                ),
+                ("property", models.CharField(max_length=32)),
+                ("end_time", models.DateTimeField()),
+                ("value", models.BigIntegerField()),
+                ("subgroup", models.CharField(max_length=16, null=True)),
+            ],
+            options={
+                "indexes": [
+                    models.Index(
+                        fields=["property", "realm", "end_time"],
+                        name="analytics_streamcount_property_realm_id_end_time_155ae930_idx",
+                    )
+                ],
+                "unique_together": set(),
+                "constraints": [
+                    models.UniqueConstraint(
+                        condition=models.Q(("subgroup__isnull", False)),
+                        fields=("stream", "property", "subgroup", "end_time"),
+                        name="unique_stream_count",
+                    ),
+                    models.UniqueConstraint(
+                        condition=models.Q(("subgroup__isnull", True)),
+                        fields=("stream", "property", "end_time"),
+                        name="unique_stream_count_null_subgroup",
+                    ),
+                ],
+            },
+        ),
+        migrations.CreateModel(
+            name="UserCount",
+            fields=[
+                (
+                    "id",
+                    models.BigAutoField(
+                        auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
+                    ),
+                ),
+                (
+                    "realm",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE, to="zerver.realm"
+                    ),
+                ),
+                (
+                    "user",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL
+                    ),
+                ),
+                ("property", models.CharField(max_length=32)),
+                ("end_time", models.DateTimeField()),
+                ("value", models.BigIntegerField()),
+                ("subgroup", models.CharField(max_length=16, null=True)),
+            ],
+            options={
+                "indexes": [
+                    models.Index(
+                        fields=["property", "realm", "end_time"],
+                        name="analytics_usercount_property_realm_id_end_time_591dbec1_idx",
+                    )
+                ],
+                "unique_together": set(),
+                "constraints": [
+                    models.UniqueConstraint(
+                        condition=models.Q(("subgroup__isnull", False)),
+                        fields=("user", "property", "subgroup", "end_time"),
+                        name="unique_user_count",
+                    ),
+                    models.UniqueConstraint(
+                        condition=models.Q(("subgroup__isnull", True)),
+                        fields=("user", "property", "end_time"),
+                        name="unique_user_count_null_subgroup",
+                    ),
+                ],
+            },
+        ),
+        migrations.CreateModel(
+            name="FillState",
+            fields=[
+                (
+                    "id",
+                    models.BigAutoField(
+                        auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
+                    ),
+                ),
+                ("property", models.CharField(max_length=40, unique=True)),
+                ("end_time", models.DateTimeField()),
+                ("state", models.PositiveSmallIntegerField()),
+            ],
+        ),
+    ]

analytics/migrations/0002_remove_huddlecount.py

@@ -0,0 +1,29 @@
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("analytics", "0001_initial"),
+    ]
+
+    operations = [
+        migrations.AlterUniqueTogether(
+            name="huddlecount",
+            unique_together=set(),
+        ),
+        migrations.RemoveField(
+            model_name="huddlecount",
+            name="anomaly",
+        ),
+        migrations.RemoveField(
+            model_name="huddlecount",
+            name="huddle",
+        ),
+        migrations.RemoveField(
+            model_name="huddlecount",
+            name="user",
+        ),
+        migrations.DeleteModel(
+            name="HuddleCount",
+        ),
+    ]

analytics/migrations/0003_fillstate.py

@@ -0,0 +1,26 @@
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("analytics", "0002_remove_huddlecount"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="FillState",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        verbose_name="ID", serialize=False, auto_created=True, primary_key=True
+                    ),
+                ),
+                ("property", models.CharField(unique=True, max_length=40)),
+                ("end_time", models.DateTimeField()),
+                ("state", models.PositiveSmallIntegerField()),
+                ("last_modified", models.DateTimeField(auto_now=True)),
+            ],
+            bases=(models.Model,),
+        ),
+    ]

analytics/migrations/0004_add_subgroup.py

@@ -0,0 +1,30 @@
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("analytics", "0003_fillstate"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="installationcount",
+            name="subgroup",
+            field=models.CharField(max_length=16, null=True),
+        ),
+        migrations.AddField(
+            model_name="realmcount",
+            name="subgroup",
+            field=models.CharField(max_length=16, null=True),
+        ),
+        migrations.AddField(
+            model_name="streamcount",
+            name="subgroup",
+            field=models.CharField(max_length=16, null=True),
+        ),
+        migrations.AddField(
+            model_name="usercount",
+            name="subgroup",
+            field=models.CharField(max_length=16, null=True),
+        ),
+    ]

analytics/migrations/0005_alter_field_size.py

@@ -0,0 +1,50 @@
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("analytics", "0004_add_subgroup"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="installationcount",
+            name="interval",
+            field=models.CharField(max_length=8),
+        ),
+        migrations.AlterField(
+            model_name="installationcount",
+            name="property",
+            field=models.CharField(max_length=32),
+        ),
+        migrations.AlterField(
+            model_name="realmcount",
+            name="interval",
+            field=models.CharField(max_length=8),
+        ),
+        migrations.AlterField(
+            model_name="realmcount",
+            name="property",
+            field=models.CharField(max_length=32),
+        ),
+        migrations.AlterField(
+            model_name="streamcount",
+            name="interval",
+            field=models.CharField(max_length=8),
+        ),
+        migrations.AlterField(
+            model_name="streamcount",
+            name="property",
+            field=models.CharField(max_length=32),
+        ),
+        migrations.AlterField(
+            model_name="usercount",
+            name="interval",
+            field=models.CharField(max_length=8),
+        ),
+        migrations.AlterField(
+            model_name="usercount",
+            name="property",
+            field=models.CharField(max_length=32),
+        ),
+    ]

analytics/migrations/0006_add_subgroup_to_unique_constraints.py

@@ -0,0 +1,26 @@
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("analytics", "0005_alter_field_size"),
+    ]
+
+    operations = [
+        migrations.AlterUniqueTogether(
+            name="installationcount",
+            unique_together={("property", "subgroup", "end_time", "interval")},
+        ),
+        migrations.AlterUniqueTogether(
+            name="realmcount",
+            unique_together={("realm", "property", "subgroup", "end_time", "interval")},
+        ),
+        migrations.AlterUniqueTogether(
+            name="streamcount",
+            unique_together={("stream", "property", "subgroup", "end_time", "interval")},
+        ),
+        migrations.AlterUniqueTogether(
+            name="usercount",
+            unique_together={("user", "property", "subgroup", "end_time", "interval")},
+        ),
+    ]

analytics/migrations/0007_remove_interval.py

@@ -0,0 +1,43 @@
+# Generated by Django 1.10.4 on 2017-01-16 20:50
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("analytics", "0006_add_subgroup_to_unique_constraints"),
+    ]
+
+    operations = [
+        migrations.AlterUniqueTogether(
+            name="installationcount",
+            unique_together={("property", "subgroup", "end_time")},
+        ),
+        migrations.RemoveField(
+            model_name="installationcount",
+            name="interval",
+        ),
+        migrations.AlterUniqueTogether(
+            name="realmcount",
+            unique_together={("realm", "property", "subgroup", "end_time")},
+        ),
+        migrations.RemoveField(
+            model_name="realmcount",
+            name="interval",
+        ),
+        migrations.AlterUniqueTogether(
+            name="streamcount",
+            unique_together={("stream", "property", "subgroup", "end_time")},
+        ),
+        migrations.RemoveField(
+            model_name="streamcount",
+            name="interval",
+        ),
+        migrations.AlterUniqueTogether(
+            name="usercount",
+            unique_together={("user", "property", "subgroup", "end_time")},
+        ),
+        migrations.RemoveField(
+            model_name="usercount",
+            name="interval",
+        ),
+    ]

analytics/migrations/0008_add_count_indexes.py

@@ -0,0 +1,33 @@
+# Generated by Django 1.10.5 on 2017-02-01 22:28
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("zerver", "0050_userprofile_avatar_version"),
+        ("analytics", "0007_remove_interval"),
+    ]
+
+    operations = [
+        migrations.AddIndex(
+            model_name="realmcount",
+            index=models.Index(
+                fields=["property", "end_time"],
+                name="analytics_realmcount_property_end_time_3b60396b_idx",
+            ),
+        ),
+        migrations.AddIndex(
+            model_name="streamcount",
+            index=models.Index(
+                fields=["property", "realm", "end_time"],
+                name="analytics_streamcount_property_realm_id_end_time_155ae930_idx",
+            ),
+        ),
+        migrations.AddIndex(
+            model_name="usercount",
+            index=models.Index(
+                fields=["property", "realm", "end_time"],
+                name="analytics_usercount_property_realm_id_end_time_591dbec1_idx",
+            ),
+        ),
+    ]

analytics/migrations/0009_remove_messages_to_stream_stat.py

@@ -0,0 +1,30 @@
+from django.db import migrations
+from django.db.backends.base.schema import BaseDatabaseSchemaEditor
+from django.db.migrations.state import StateApps
+
+
+def delete_messages_sent_to_stream_stat(
+    apps: StateApps, schema_editor: BaseDatabaseSchemaEditor
+) -> None:
+    UserCount = apps.get_model("analytics", "UserCount")
+    StreamCount = apps.get_model("analytics", "StreamCount")
+    RealmCount = apps.get_model("analytics", "RealmCount")
+    InstallationCount = apps.get_model("analytics", "InstallationCount")
+    FillState = apps.get_model("analytics", "FillState")
+
+    property = "messages_sent_to_stream:is_bot"
+    UserCount.objects.filter(property=property).delete()
+    StreamCount.objects.filter(property=property).delete()
+    RealmCount.objects.filter(property=property).delete()
+    InstallationCount.objects.filter(property=property).delete()
+    FillState.objects.filter(property=property).delete()
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("analytics", "0008_add_count_indexes"),
+    ]
+
+    operations = [
+        migrations.RunPython(delete_messages_sent_to_stream_stat, elidable=True),
+    ]

analytics/migrations/0010_clear_messages_sent_values.py

@@ -0,0 +1,28 @@
+from django.db import migrations
+from django.db.backends.base.schema import BaseDatabaseSchemaEditor
+from django.db.migrations.state import StateApps
+
+
+def clear_message_sent_by_message_type_values(
+    apps: StateApps, schema_editor: BaseDatabaseSchemaEditor
+) -> None:
+    UserCount = apps.get_model("analytics", "UserCount")
+    StreamCount = apps.get_model("analytics", "StreamCount")
+    RealmCount = apps.get_model("analytics", "RealmCount")
+    InstallationCount = apps.get_model("analytics", "InstallationCount")
+    FillState = apps.get_model("analytics", "FillState")
+
+    property = "messages_sent:message_type:day"
+    UserCount.objects.filter(property=property).delete()
+    StreamCount.objects.filter(property=property).delete()
+    RealmCount.objects.filter(property=property).delete()
+    InstallationCount.objects.filter(property=property).delete()
+    FillState.objects.filter(property=property).delete()
+
+
+class Migration(migrations.Migration):
+    dependencies = [("analytics", "0009_remove_messages_to_stream_stat")]
+
+    operations = [
+        migrations.RunPython(clear_message_sent_by_message_type_values, elidable=True),
+    ]

analytics/migrations/0011_clear_analytics_tables.py

@@ -0,0 +1,27 @@
+from django.db import migrations
+from django.db.backends.base.schema import BaseDatabaseSchemaEditor
+from django.db.migrations.state import StateApps
+
+
+def clear_analytics_tables(apps: StateApps, schema_editor: BaseDatabaseSchemaEditor) -> None:
+    UserCount = apps.get_model("analytics", "UserCount")
+    StreamCount = apps.get_model("analytics", "StreamCount")
+    RealmCount = apps.get_model("analytics", "RealmCount")
+    InstallationCount = apps.get_model("analytics", "InstallationCount")
+    FillState = apps.get_model("analytics", "FillState")
+
+    UserCount.objects.all().delete()
+    StreamCount.objects.all().delete()
+    RealmCount.objects.all().delete()
+    InstallationCount.objects.all().delete()
+    FillState.objects.all().delete()
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("analytics", "0010_clear_messages_sent_values"),
+    ]
+
+    operations = [
+        migrations.RunPython(clear_analytics_tables, elidable=True),
+    ]

analytics/migrations/0012_add_on_delete.py

@@ -0,0 +1,41 @@
+# Generated by Django 1.11.6 on 2018-01-29 08:14
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("analytics", "0011_clear_analytics_tables"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="installationcount",
+            name="anomaly",
+            field=models.ForeignKey(
+                null=True, on_delete=django.db.models.deletion.SET_NULL, to="analytics.Anomaly"
+            ),
+        ),
+        migrations.AlterField(
+            model_name="realmcount",
+            name="anomaly",
+            field=models.ForeignKey(
+                null=True, on_delete=django.db.models.deletion.SET_NULL, to="analytics.Anomaly"
+            ),
+        ),
+        migrations.AlterField(
+            model_name="streamcount",
+            name="anomaly",
+            field=models.ForeignKey(
+                null=True, on_delete=django.db.models.deletion.SET_NULL, to="analytics.Anomaly"
+            ),
+        ),
+        migrations.AlterField(
+            model_name="usercount",
+            name="anomaly",
+            field=models.ForeignKey(
+                null=True, on_delete=django.db.models.deletion.SET_NULL, to="analytics.Anomaly"
+            ),
+        ),
+    ]

analytics/migrations/0013_remove_anomaly.py

@@ -0,0 +1,31 @@
+# Generated by Django 1.11.18 on 2019-02-02 02:47
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("analytics", "0012_add_on_delete"),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name="installationcount",
+            name="anomaly",
+        ),
+        migrations.RemoveField(
+            model_name="realmcount",
+            name="anomaly",
+        ),
+        migrations.RemoveField(
+            model_name="streamcount",
+            name="anomaly",
+        ),
+        migrations.RemoveField(
+            model_name="usercount",
+            name="anomaly",
+        ),
+        migrations.DeleteModel(
+            name="Anomaly",
+        ),
+    ]

analytics/migrations/0014_remove_fillstate_last_modified.py

@@ -0,0 +1,16 @@
+# Generated by Django 1.11.26 on 2020-01-27 04:32
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("analytics", "0013_remove_anomaly"),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name="fillstate",
+            name="last_modified",
+        ),
+    ]

analytics/migrations/0015_clear_duplicate_counts.py

@@ -0,0 +1,66 @@
+from django.db import migrations
+from django.db.backends.base.schema import BaseDatabaseSchemaEditor
+from django.db.migrations.state import StateApps
+from django.db.models import Count, Sum
+
+
+def clear_duplicate_counts(apps: StateApps, schema_editor: BaseDatabaseSchemaEditor) -> None:
+    """This is a preparatory migration for our Analytics tables.
+
+    The backstory is that Django's unique_together indexes do not properly
+    handle the subgroup=None corner case (allowing duplicate rows that have a
+    subgroup of None), which meant that in race conditions, rather than updating
+    an existing row for the property/(realm, stream, user)/time with subgroup=None, Django would
+    create a duplicate row.
+
+    In the next migration, we'll add a proper constraint to fix this bug, but
+    we need to fix any existing problematic rows before we can add that constraint.
+
+    We fix this in an appropriate fashion for each type of CountStat object; mainly
+    this means deleting the extra rows, but for LoggingCountStat objects, we need to
+    additionally combine the sums.
+    """
+    count_tables = dict(
+        realm=apps.get_model("analytics", "RealmCount"),
+        user=apps.get_model("analytics", "UserCount"),
+        stream=apps.get_model("analytics", "StreamCount"),
+        installation=apps.get_model("analytics", "InstallationCount"),
+    )
+
+    for name, count_table in count_tables.items():
+        value = [name, "property", "end_time"]
+        if name == "installation":
+            value = ["property", "end_time"]
+        counts = (
+            count_table.objects.filter(subgroup=None)
+            .values(*value)
+            .annotate(Count("id"), Sum("value"))
+            .filter(id__count__gt=1)
+        )
+
+        for count in counts:
+            count.pop("id__count")
+            total_value = count.pop("value__sum")
+            duplicate_counts = list(count_table.objects.filter(**count))
+            first_count = duplicate_counts[0]
+            if count["property"] in ["invites_sent::day", "active_users_log:is_bot:day"]:
+                # For LoggingCountStat objects, the right fix is to combine the totals;
+                # for other CountStat objects, we expect the duplicates to have the same value.
+                # And so all we need to do is delete them.
+                first_count.value = total_value
+                first_count.save()
+            to_cleanup = duplicate_counts[1:]
+            for duplicate_count in to_cleanup:
+                duplicate_count.delete()
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("analytics", "0014_remove_fillstate_last_modified"),
+    ]
+
+    operations = [
+        migrations.RunPython(
+            clear_duplicate_counts, reverse_code=migrations.RunPython.noop, elidable=True
+        ),
+    ]

analytics/migrations/0016_unique_constraint_when_subgroup_null.py

@@ -0,0 +1,92 @@
+# Generated by Django 2.2.10 on 2020-02-29 19:40
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("analytics", "0015_clear_duplicate_counts"),
+    ]
+
+    operations = [
+        migrations.AlterUniqueTogether(
+            name="installationcount",
+            unique_together=set(),
+        ),
+        migrations.AlterUniqueTogether(
+            name="realmcount",
+            unique_together=set(),
+        ),
+        migrations.AlterUniqueTogether(
+            name="streamcount",
+            unique_together=set(),
+        ),
+        migrations.AlterUniqueTogether(
+            name="usercount",
+            unique_together=set(),
+        ),
+        migrations.AddConstraint(
+            model_name="installationcount",
+            constraint=models.UniqueConstraint(
+                condition=models.Q(subgroup__isnull=False),
+                fields=("property", "subgroup", "end_time"),
+                name="unique_installation_count",
+            ),
+        ),
+        migrations.AddConstraint(
+            model_name="installationcount",
+            constraint=models.UniqueConstraint(
+                condition=models.Q(subgroup__isnull=True),
+                fields=("property", "end_time"),
+                name="unique_installation_count_null_subgroup",
+            ),
+        ),
+        migrations.AddConstraint(
+            model_name="realmcount",
+            constraint=models.UniqueConstraint(
+                condition=models.Q(subgroup__isnull=False),
+                fields=("realm", "property", "subgroup", "end_time"),
+                name="unique_realm_count",
+            ),
+        ),
+        migrations.AddConstraint(
+            model_name="realmcount",
+            constraint=models.UniqueConstraint(
+                condition=models.Q(subgroup__isnull=True),
+                fields=("realm", "property", "end_time"),
+                name="unique_realm_count_null_subgroup",
+            ),
+        ),
+        migrations.AddConstraint(
+            model_name="streamcount",
+            constraint=models.UniqueConstraint(
+                condition=models.Q(subgroup__isnull=False),
+                fields=("stream", "property", "subgroup", "end_time"),
+                name="unique_stream_count",
+            ),
+        ),
+        migrations.AddConstraint(
+            model_name="streamcount",
+            constraint=models.UniqueConstraint(
+                condition=models.Q(subgroup__isnull=True),
+                fields=("stream", "property", "end_time"),
+                name="unique_stream_count_null_subgroup",
+            ),
+        ),
+        migrations.AddConstraint(
+            model_name="usercount",
+            constraint=models.UniqueConstraint(
+                condition=models.Q(subgroup__isnull=False),
+                fields=("user", "property", "subgroup", "end_time"),
+                name="unique_user_count",
+            ),
+        ),
+        migrations.AddConstraint(
+            model_name="usercount",
+            constraint=models.UniqueConstraint(
+                condition=models.Q(subgroup__isnull=True),
+                fields=("user", "property", "end_time"),
+                name="unique_user_count_null_subgroup",
+            ),
+        ),
+    ]

analytics/migrations/0017_regenerate_partial_indexes.py

@@ -0,0 +1,114 @@
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("analytics", "0016_unique_constraint_when_subgroup_null"),
+    ]
+
+    # If the server was installed between 7.0 and 7.4 (or main between
+    # 2c20028aa451 and 7807bff52635), it contains indexes which (when
+    # running 7.5 or 7807bff52635 or higher) are never used, because
+    # they contain an improper cast
+    # (https://code.djangoproject.com/ticket/34840).
+    #
+    # We regenerate the indexes here, by dropping and re-creating
+    # them, so that we know that they are properly formed.
+    operations = [
+        migrations.RemoveConstraint(
+            model_name="installationcount",
+            name="unique_installation_count",
+        ),
+        migrations.AddConstraint(
+            model_name="installationcount",
+            constraint=models.UniqueConstraint(
+                condition=models.Q(subgroup__isnull=False),
+                fields=("property", "subgroup", "end_time"),
+                name="unique_installation_count",
+            ),
+        ),
+        migrations.RemoveConstraint(
+            model_name="installationcount",
+            name="unique_installation_count_null_subgroup",
+        ),
+        migrations.AddConstraint(
+            model_name="installationcount",
+            constraint=models.UniqueConstraint(
+                condition=models.Q(subgroup__isnull=True),
+                fields=("property", "end_time"),
+                name="unique_installation_count_null_subgroup",
+            ),
+        ),
+        migrations.RemoveConstraint(
+            model_name="realmcount",
+            name="unique_realm_count",
+        ),
+        migrations.AddConstraint(
+            model_name="realmcount",
+            constraint=models.UniqueConstraint(
+                condition=models.Q(subgroup__isnull=False),
+                fields=("realm", "property", "subgroup", "end_time"),
+                name="unique_realm_count",
+            ),
+        ),
+        migrations.RemoveConstraint(
+            model_name="realmcount",
+            name="unique_realm_count_null_subgroup",
+        ),
+        migrations.AddConstraint(
+            model_name="realmcount",
+            constraint=models.UniqueConstraint(
+                condition=models.Q(subgroup__isnull=True),
+                fields=("realm", "property", "end_time"),
+                name="unique_realm_count_null_subgroup",
+            ),
+        ),
+        migrations.RemoveConstraint(
+            model_name="streamcount",
+            name="unique_stream_count",
+        ),
+        migrations.AddConstraint(
+            model_name="streamcount",
+            constraint=models.UniqueConstraint(
+                condition=models.Q(subgroup__isnull=False),
+                fields=("stream", "property", "subgroup", "end_time"),
+                name="unique_stream_count",
+            ),
+        ),
+        migrations.RemoveConstraint(
+            model_name="streamcount",
+            name="unique_stream_count_null_subgroup",
+        ),
+        migrations.AddConstraint(
+            model_name="streamcount",
+            constraint=models.UniqueConstraint(
+                condition=models.Q(subgroup__isnull=True),
+                fields=("stream", "property", "end_time"),
+                name="unique_stream_count_null_subgroup",
+            ),
+        ),
+        migrations.RemoveConstraint(
+            model_name="usercount",
+            name="unique_user_count",
+        ),
+        migrations.AddConstraint(
+            model_name="usercount",
+            constraint=models.UniqueConstraint(
+                condition=models.Q(subgroup__isnull=False),
+                fields=("user", "property", "subgroup", "end_time"),
+                name="unique_user_count",
+            ),
+        ),
+        migrations.RemoveConstraint(
+            model_name="usercount",
+            name="unique_user_count_null_subgroup",
+        ),
+        migrations.AddConstraint(
+            model_name="usercount",
+            constraint=models.UniqueConstraint(
+                condition=models.Q(subgroup__isnull=True),
+                fields=("user", "property", "end_time"),
+                name="unique_user_count_null_subgroup",
+            ),
+        ),
+    ]

analytics/migrations/0018_remove_usercount_active_users_audit.py

@@ -0,0 +1,16 @@
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+    elidable = True
+
+    dependencies = [
+        ("analytics", "0017_regenerate_partial_indexes"),
+    ]
+
+    operations = [
+        migrations.RunSQL(
+            "DELETE FROM analytics_usercount WHERE property = 'active_users_audit:is_bot:day'",
+            elidable=True,
+        )
+    ]

analytics/migrations/0019_remove_unused_counts.py

@@ -0,0 +1,27 @@
+from django.db import migrations
+
+REMOVED_COUNTS = (
+    "active_users_log:is_bot:day",
+    "active_users:is_bot:day",
+)
+
+
+class Migration(migrations.Migration):
+    elidable = True
+
+    dependencies = [
+        ("analytics", "0018_remove_usercount_active_users_audit"),
+    ]
+
+    operations = [
+        migrations.RunSQL(
+            [
+                ("DELETE FROM analytics_realmcount WHERE property IN %s", (REMOVED_COUNTS,)),
+                (
+                    "DELETE FROM analytics_installationcount WHERE property IN %s",
+                    (REMOVED_COUNTS,),
+                ),
+            ],
+            elidable=True,
+        )
+    ]

analytics/migrations/0020_alter_installationcount_id_alter_realmcount_id_and_more.py

@@ -0,0 +1,40 @@
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    atomic = False
+
+    dependencies = [
+        ("analytics", "0019_remove_unused_counts"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="installationcount",
+            name="id",
+            field=models.BigAutoField(
+                auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
+            ),
+        ),
+        migrations.AlterField(
+            model_name="realmcount",
+            name="id",
+            field=models.BigAutoField(
+                auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
+            ),
+        ),
+        migrations.AlterField(
+            model_name="streamcount",
+            name="id",
+            field=models.BigAutoField(
+                auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
+            ),
+        ),
+        migrations.AlterField(
+            model_name="usercount",
+            name="id",
+            field=models.BigAutoField(
+                auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
+            ),
+        ),
+    ]

analytics/migrations/0021_alter_fillstate_id.py

@@ -0,0 +1,17 @@
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("analytics", "0020_alter_installationcount_id_alter_realmcount_id_and_more"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="fillstate",
+            name="id",
+            field=models.BigAutoField(
+                auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
+            ),
+        ),
+    ]

analytics/models.py

@@ -0,0 +1,158 @@
+from datetime import datetime
+
+from django.db import models
+from django.db.models import Q, UniqueConstraint
+from typing_extensions import override
+
+from zerver.lib.timestamp import floor_to_day
+from zerver.models import Realm, Stream, UserProfile
+
+
+class FillState(models.Model):
+    property = models.CharField(max_length=40, unique=True)
+    end_time = models.DateTimeField()
+
+    # Valid states are {DONE, STARTED}
+    DONE = 1
+    STARTED = 2
+    state = models.PositiveSmallIntegerField()
+
+    @override
+    def __str__(self) -> str:
+        return f"{self.property} {self.end_time} {self.state}"
+
+
+# The earliest/starting end_time in FillState
+# We assume there is at least one realm
+def installation_epoch() -> datetime:
+    earliest_realm_creation = Realm.objects.aggregate(models.Min("date_created"))[
+        "date_created__min"
+    ]
+    return floor_to_day(earliest_realm_creation)
+
+
+class BaseCount(models.Model):
+    # Note: When inheriting from BaseCount, you may want to rearrange
+    # the order of the columns in the migration to make sure they
+    # match how you'd like the table to be arranged.
+    property = models.CharField(max_length=32)
+    subgroup = models.CharField(max_length=16, null=True)
+    end_time = models.DateTimeField()
+    value = models.BigIntegerField()
+
+    class Meta:
+        abstract = True
+
+
+class InstallationCount(BaseCount):
+    class Meta:
+        # Handles invalid duplicate InstallationCount data
+        constraints = [
+            UniqueConstraint(
+                fields=["property", "subgroup", "end_time"],
+                condition=Q(subgroup__isnull=False),
+                name="unique_installation_count",
+            ),
+            UniqueConstraint(
+                fields=["property", "end_time"],
+                condition=Q(subgroup__isnull=True),
+                name="unique_installation_count_null_subgroup",
+            ),
+        ]
+
+    @override
+    def __str__(self) -> str:
+        return f"{self.property} {self.subgroup} {self.value}"
+
+
+class RealmCount(BaseCount):
+    realm = models.ForeignKey(Realm, on_delete=models.CASCADE)
+
+    class Meta:
+        # Handles invalid duplicate RealmCount data
+        constraints = [
+            UniqueConstraint(
+                fields=["realm", "property", "subgroup", "end_time"],
+                condition=Q(subgroup__isnull=False),
+                name="unique_realm_count",
+            ),
+            UniqueConstraint(
+                fields=["realm", "property", "end_time"],
+                condition=Q(subgroup__isnull=True),
+                name="unique_realm_count_null_subgroup",
+            ),
+        ]
+        indexes = [
+            models.Index(
+                fields=["property", "end_time"],
+                name="analytics_realmcount_property_end_time_3b60396b_idx",
+            )
+        ]
+
+    @override
+    def __str__(self) -> str:
+        return f"{self.realm!r} {self.property} {self.subgroup} {self.value}"
+
+
+class UserCount(BaseCount):
+    user = models.ForeignKey(UserProfile, on_delete=models.CASCADE)
+    realm = models.ForeignKey(Realm, on_delete=models.CASCADE)
+
+    class Meta:
+        # Handles invalid duplicate UserCount data
+        constraints = [
+            UniqueConstraint(
+                fields=["user", "property", "subgroup", "end_time"],
+                condition=Q(subgroup__isnull=False),
+                name="unique_user_count",
+            ),
+            UniqueConstraint(
+                fields=["user", "property", "end_time"],
+                condition=Q(subgroup__isnull=True),
+                name="unique_user_count_null_subgroup",
+            ),
+        ]
+        # This index dramatically improves the performance of
+        # aggregating from users to realms
+        indexes = [
+            models.Index(
+                fields=["property", "realm", "end_time"],
+                name="analytics_usercount_property_realm_id_end_time_591dbec1_idx",
+            )
+        ]
+
+    @override
+    def __str__(self) -> str:
+        return f"{self.user!r} {self.property} {self.subgroup} {self.value}"
+
+
+class StreamCount(BaseCount):
+    stream = models.ForeignKey(Stream, on_delete=models.CASCADE)
+    realm = models.ForeignKey(Realm, on_delete=models.CASCADE)
+
+    class Meta:
+        # Handles invalid duplicate StreamCount data
+        constraints = [
+            UniqueConstraint(
+                fields=["stream", "property", "subgroup", "end_time"],
+                condition=Q(subgroup__isnull=False),
+                name="unique_stream_count",
+            ),
+            UniqueConstraint(
+                fields=["stream", "property", "end_time"],
+                condition=Q(subgroup__isnull=True),
+                name="unique_stream_count_null_subgroup",
+            ),
+        ]
+        # This index dramatically improves the performance of
+        # aggregating from streams to realms
+        indexes = [
+            models.Index(
+                fields=["property", "realm", "end_time"],
+                name="analytics_streamcount_property_realm_id_end_time_155ae930_idx",
+            )
+        ]
+
+    @override
+    def __str__(self) -> str:
+        return f"{self.stream!r} {self.property} {self.subgroup} {self.value} {self.id}"

analytics/tests/test_fixtures.py

@@ -0,0 +1,40 @@
+from analytics.lib.counts import CountStat
+from analytics.lib.fixtures import generate_time_series_data
+from zerver.lib.test_classes import ZulipTestCase
+
+
+# A very light test suite; the code being tested is not run in production.
+class TestFixtures(ZulipTestCase):
+    def test_deterministic_settings(self) -> None:
+        # test basic business_hour / non_business_hour calculation
+        # test we get an array of the right length with frequency=CountStat.DAY
+        data = generate_time_series_data(
+            days=7, business_hours_base=20, non_business_hours_base=15, spikiness=0
+        )
+        self.assertEqual(data, [400, 400, 400, 400, 400, 360, 360])
+
+        data = generate_time_series_data(
+            days=1,
+            business_hours_base=2000,
+            non_business_hours_base=1500,
+            growth=2,
+            spikiness=0,
+            frequency=CountStat.HOUR,
+        )
+        # test we get an array of the right length with frequency=CountStat.HOUR
+        self.assert_length(data, 24)
+        # test that growth doesn't affect the first data point
+        self.assertEqual(data[0], 2000)
+        # test that the last data point is growth times what it otherwise would be
+        self.assertEqual(data[-1], 1500 * 2)
+
+        # test autocorrelation == 1, since that's the easiest value to test
+        data = generate_time_series_data(
+            days=1,
+            business_hours_base=2000,
+            non_business_hours_base=2000,
+            autocorrelation=1,
+            frequency=CountStat.HOUR,
+        )
+        self.assertEqual(data[0], data[1])
+        self.assertEqual(data[0], data[-1])

analytics/tests/test_stats_views.py

@@ -0,0 +1,689 @@
+from datetime import datetime, timedelta, timezone
+
+from django.utils.timezone import now as timezone_now
+from typing_extensions import override
+
+from analytics.lib.counts import COUNT_STATS, CountStat
+from analytics.lib.time_utils import time_range
+from analytics.models import FillState, RealmCount, StreamCount, UserCount
+from analytics.views.stats import rewrite_client_arrays, sort_by_totals, sort_client_labels
+from zerver.lib.test_classes import ZulipTestCase
+from zerver.lib.timestamp import ceiling_to_day, ceiling_to_hour, datetime_to_timestamp
+from zerver.models import Client
+from zerver.models.realms import get_realm
+
+
+class TestStatsEndpoint(ZulipTestCase):
+    def test_stats(self) -> None:
+        self.user = self.example_user("hamlet")
+        self.login_user(self.user)
+        result = self.client_get("/stats")
+        self.assertEqual(result.status_code, 200)
+        # Check that we get something back
+        self.assert_in_response("Zulip analytics for", result)
+
+    def test_guest_user_cant_access_stats(self) -> None:
+        self.user = self.example_user("polonius")
+        self.login_user(self.user)
+        result = self.client_get("/stats")
+        self.assert_json_error(result, "Not allowed for guest users", 400)
+
+        result = self.client_get("/json/analytics/chart_data")
+        self.assert_json_error(result, "Not allowed for guest users", 400)
+
+    def test_stats_for_realm(self) -> None:
+        user = self.example_user("hamlet")
+        self.login_user(user)
+
+        result = self.client_get("/stats/realm/zulip/")
+        self.assertEqual(result.status_code, 302)
+
+        result = self.client_get("/stats/realm/not_existing_realm/")
+        self.assertEqual(result.status_code, 302)
+
+        user = self.example_user("hamlet")
+        user.is_staff = True
+        user.save(update_fields=["is_staff"])
+
+        result = self.client_get("/stats/realm/not_existing_realm/")
+        self.assertEqual(result.status_code, 404)
+
+        result = self.client_get("/stats/realm/zulip/")
+        self.assertEqual(result.status_code, 200)
+        self.assert_in_response("Zulip analytics for", result)
+
+    def test_stats_for_installation(self) -> None:
+        user = self.example_user("hamlet")
+        self.login_user(user)
+
+        result = self.client_get("/stats/installation")
+        self.assertEqual(result.status_code, 302)
+
+        user = self.example_user("hamlet")
+        user.is_staff = True
+        user.save(update_fields=["is_staff"])
+
+        result = self.client_get("/stats/installation")
+        self.assertEqual(result.status_code, 200)
+        self.assert_in_response("Zulip analytics for", result)
+
+
+class TestGetChartData(ZulipTestCase):
+    @override
+    def setUp(self) -> None:
+        super().setUp()
+        self.realm = get_realm("zulip")
+        self.user = self.example_user("hamlet")
+        self.stream_id = self.get_stream_id(self.get_streams(self.user)[0])
+        self.login_user(self.user)
+        self.end_times_hour = [
+            ceiling_to_hour(self.realm.date_created) + timedelta(hours=i) for i in range(4)
+        ]
+        self.end_times_day = [
+            ceiling_to_day(self.realm.date_created) + timedelta(days=i) for i in range(4)
+        ]
+
+    def data(self, i: int) -> list[int]:
+        return [0, 0, i, 0]
+
+    def insert_data(
+        self, stat: CountStat, realm_subgroups: list[str | None], user_subgroups: list[str]
+    ) -> None:
+        if stat.frequency == CountStat.HOUR:
+            insert_time = self.end_times_hour[2]
+            fill_time = self.end_times_hour[-1]
+        if stat.frequency == CountStat.DAY:
+            insert_time = self.end_times_day[2]
+            fill_time = self.end_times_day[-1]
+
+        RealmCount.objects.bulk_create(
+            RealmCount(
+                property=stat.property,
+                subgroup=subgroup,
+                end_time=insert_time,
+                value=100 + i,
+                realm=self.realm,
+            )
+            for i, subgroup in enumerate(realm_subgroups)
+        )
+        UserCount.objects.bulk_create(
+            UserCount(
+                property=stat.property,
+                subgroup=subgroup,
+                end_time=insert_time,
+                value=200 + i,
+                realm=self.realm,
+                user=self.user,
+            )
+            for i, subgroup in enumerate(user_subgroups)
+        )
+        StreamCount.objects.bulk_create(
+            StreamCount(
+                property=stat.property,
+                subgroup=subgroup,
+                end_time=insert_time,
+                value=100 + i,
+                stream_id=self.stream_id,
+                realm=self.realm,
+            )
+            for i, subgroup in enumerate(realm_subgroups)
+        )
+        FillState.objects.create(property=stat.property, end_time=fill_time, state=FillState.DONE)
+
+    def test_number_of_humans(self) -> None:
+        stat = COUNT_STATS["realm_active_humans::day"]
+        self.insert_data(stat, [None], [])
+        stat = COUNT_STATS["1day_actives::day"]
+        self.insert_data(stat, [None], [])
+        stat = COUNT_STATS["active_users_audit:is_bot:day"]
+        self.insert_data(stat, ["false"], [])
+        result = self.client_get("/json/analytics/chart_data", {"chart_name": "number_of_humans"})
+        data = self.assert_json_success(result)
+        self.assertEqual(
+            data,
+            {
+                "msg": "",
+                "end_times": [datetime_to_timestamp(dt) for dt in self.end_times_day],
+                "frequency": CountStat.DAY,
+                "everyone": {
+                    "_1day": self.data(100),
+                    "_15day": self.data(100),
+                    "all_time": self.data(100),
+                },
+                "display_order": None,
+                "result": "success",
+            },
+        )
+
+    def test_messages_sent_over_time(self) -> None:
+        stat = COUNT_STATS["messages_sent:is_bot:hour"]
+        self.insert_data(stat, ["true", "false"], ["false"])
+        result = self.client_get(
+            "/json/analytics/chart_data", {"chart_name": "messages_sent_over_time"}
+        )
+        data = self.assert_json_success(result)
+        self.assertEqual(
+            data,
+            {
+                "msg": "",
+                "end_times": [datetime_to_timestamp(dt) for dt in self.end_times_hour],
+                "frequency": CountStat.HOUR,
+                "everyone": {"bot": self.data(100), "human": self.data(101)},
+                "user": {"bot": self.data(0), "human": self.data(200)},
+                "display_order": None,
+                "result": "success",
+            },
+        )
+
+    def test_messages_sent_by_message_type(self) -> None:
+        stat = COUNT_STATS["messages_sent:message_type:day"]
+        self.insert_data(
+            stat, ["public_stream", "private_message"], ["public_stream", "private_stream"]
+        )
+        result = self.client_get(
+            "/json/analytics/chart_data", {"chart_name": "messages_sent_by_message_type"}
+        )
+        data = self.assert_json_success(result)
+        self.assertEqual(
+            data,
+            {
+                "msg": "",
+                "end_times": [datetime_to_timestamp(dt) for dt in self.end_times_day],
+                "frequency": CountStat.DAY,
+                "everyone": {
+                    "Public channels": self.data(100),
+                    "Private channels": self.data(0),
+                    "Direct messages": self.data(101),
+                    "Group direct messages": self.data(0),
+                },
+                "user": {
+                    "Public channels": self.data(200),
+                    "Private channels": self.data(201),
+                    "Direct messages": self.data(0),
+                    "Group direct messages": self.data(0),
+                },
+                "display_order": [
+                    "Direct messages",
+                    "Public channels",
+                    "Private channels",
+                    "Group direct messages",
+                ],
+                "result": "success",
+            },
+        )
+
+    def test_messages_sent_by_client(self) -> None:
+        stat = COUNT_STATS["messages_sent:client:day"]
+        client1 = Client.objects.create(name="client 1")
+        client2 = Client.objects.create(name="client 2")
+        client3 = Client.objects.create(name="client 3")
+        client4 = Client.objects.create(name="client 4")
+        self.insert_data(
+            stat,
+            [str(client4.id), str(client3.id), str(client2.id)],
+            [str(client3.id), str(client1.id)],
+        )
+        result = self.client_get(
+            "/json/analytics/chart_data", {"chart_name": "messages_sent_by_client"}
+        )
+        data = self.assert_json_success(result)
+        self.assertEqual(
+            data,
+            {
+                "msg": "",
+                "end_times": [datetime_to_timestamp(dt) for dt in self.end_times_day],
+                "frequency": CountStat.DAY,
+                "everyone": {
+                    "client 4": self.data(100),
+                    "client 3": self.data(101),
+                    "client 2": self.data(102),
+                },
+                "user": {"client 3": self.data(200), "client 1": self.data(201)},
+                "display_order": ["client 1", "client 2", "client 3", "client 4"],
+                "result": "success",
+            },
+        )
+
+    def test_messages_read_over_time(self) -> None:
+        stat = COUNT_STATS["messages_read::hour"]
+        self.insert_data(stat, [None], [])
+        result = self.client_get(
+            "/json/analytics/chart_data", {"chart_name": "messages_read_over_time"}
+        )
+        data = self.assert_json_success(result)
+        self.assertEqual(
+            data,
+            {
+                "msg": "",
+                "end_times": [datetime_to_timestamp(dt) for dt in self.end_times_hour],
+                "frequency": CountStat.HOUR,
+                "everyone": {"read": self.data(100)},
+                "user": {"read": self.data(0)},
+                "display_order": None,
+                "result": "success",
+            },
+        )
+
+    def test_messages_sent_by_stream(self) -> None:
+        stat = COUNT_STATS["messages_in_stream:is_bot:day"]
+        self.insert_data(stat, ["true", "false"], [])
+
+        result = self.client_get(
+            f"/json/analytics/chart_data/stream/{self.stream_id}",
+            {
+                "chart_name": "messages_sent_by_stream",
+            },
+        )
+        data = self.assert_json_success(result)
+        self.assertEqual(
+            data,
+            {
+                "msg": "",
+                "end_times": [datetime_to_timestamp(dt) for dt in self.end_times_day],
+                "frequency": CountStat.DAY,
+                "everyone": {"bot": self.data(100), "human": self.data(101)},
+                "display_order": None,
+                "result": "success",
+            },
+        )
+
+        result = self.api_get(
+            self.example_user("polonius"),
+            f"/api/v1/analytics/chart_data/stream/{self.stream_id}",
+            {
+                "chart_name": "messages_sent_by_stream",
+            },
+        )
+        self.assert_json_error(result, "Not allowed for guest users")
+
+        # Verify we correctly forbid access to stats of streams in other realms.
+        result = self.api_get(
+            self.mit_user("sipbtest"),
+            f"/api/v1/analytics/chart_data/stream/{self.stream_id}",
+            {
+                "chart_name": "messages_sent_by_stream",
+            },
+            subdomain="zephyr",
+        )
+        self.assert_json_error(result, "Invalid channel ID")
+
+    def test_include_empty_subgroups(self) -> None:
+        FillState.objects.create(
+            property="realm_active_humans::day",
+            end_time=self.end_times_day[0],
+            state=FillState.DONE,
+        )
+        result = self.client_get("/json/analytics/chart_data", {"chart_name": "number_of_humans"})
+        data = self.assert_json_success(result)
+        self.assertEqual(data["everyone"], {"_1day": [0], "_15day": [0], "all_time": [0]})
+        self.assertFalse("user" in data)
+
+        FillState.objects.create(
+            property="messages_sent:is_bot:hour",
+            end_time=self.end_times_hour[0],
+            state=FillState.DONE,
+        )
+        result = self.client_get(
+            "/json/analytics/chart_data", {"chart_name": "messages_sent_over_time"}
+        )
+        data = self.assert_json_success(result)
+        self.assertEqual(data["everyone"], {"human": [0], "bot": [0]})
+        self.assertEqual(data["user"], {"human": [0], "bot": [0]})
+
+        FillState.objects.create(
+            property="messages_sent:message_type:day",
+            end_time=self.end_times_day[0],
+            state=FillState.DONE,
+        )
+        result = self.client_get(
+            "/json/analytics/chart_data", {"chart_name": "messages_sent_by_message_type"}
+        )
+        data = self.assert_json_success(result)
+        self.assertEqual(
+            data["everyone"],
+            {
+                "Public channels": [0],
+                "Private channels": [0],
+                "Direct messages": [0],
+                "Group direct messages": [0],
+            },
+        )
+        self.assertEqual(
+            data["user"],
+            {
+                "Public channels": [0],
+                "Private channels": [0],
+                "Direct messages": [0],
+                "Group direct messages": [0],
+            },
+        )
+
+        FillState.objects.create(
+            property="messages_sent:client:day",
+            end_time=self.end_times_day[0],
+            state=FillState.DONE,
+        )
+        result = self.client_get(
+            "/json/analytics/chart_data", {"chart_name": "messages_sent_by_client"}
+        )
+        data = self.assert_json_success(result)
+        self.assertEqual(data["everyone"], {})
+        self.assertEqual(data["user"], {})
+
+    def test_start_and_end(self) -> None:
+        stat = COUNT_STATS["realm_active_humans::day"]
+        self.insert_data(stat, [None], [])
+        stat = COUNT_STATS["1day_actives::day"]
+        self.insert_data(stat, [None], [])
+        stat = COUNT_STATS["active_users_audit:is_bot:day"]
+        self.insert_data(stat, ["false"], [])
+        end_time_timestamps = [datetime_to_timestamp(dt) for dt in self.end_times_day]
+
+        # valid start and end
+        result = self.client_get(
+            "/json/analytics/chart_data",
+            {
+                "chart_name": "number_of_humans",
+                "start": end_time_timestamps[1],
+                "end": end_time_timestamps[2],
+            },
+        )
+        data = self.assert_json_success(result)
+        self.assertEqual(data["end_times"], end_time_timestamps[1:3])
+        self.assertEqual(
+            data["everyone"], {"_1day": [0, 100], "_15day": [0, 100], "all_time": [0, 100]}
+        )
+
+        # start later then end
+        result = self.client_get(
+            "/json/analytics/chart_data",
+            {
+                "chart_name": "number_of_humans",
+                "start": end_time_timestamps[2],
+                "end": end_time_timestamps[1],
+            },
+        )
+        self.assert_json_error_contains(result, "Start time is later than")
+
+    def test_min_length(self) -> None:
+        stat = COUNT_STATS["realm_active_humans::day"]
+        self.insert_data(stat, [None], [])
+        stat = COUNT_STATS["1day_actives::day"]
+        self.insert_data(stat, [None], [])
+        stat = COUNT_STATS["active_users_audit:is_bot:day"]
+        self.insert_data(stat, ["false"], [])
+        # test min_length is too short to change anything
+        result = self.client_get(
+            "/json/analytics/chart_data", {"chart_name": "number_of_humans", "min_length": 2}
+        )
+        data = self.assert_json_success(result)
+        self.assertEqual(
+            data["end_times"], [datetime_to_timestamp(dt) for dt in self.end_times_day]
+        )
+        self.assertEqual(
+            data["everyone"],
+            {"_1day": self.data(100), "_15day": self.data(100), "all_time": self.data(100)},
+        )
+        # test min_length larger than filled data
+        result = self.client_get(
+            "/json/analytics/chart_data", {"chart_name": "number_of_humans", "min_length": 5}
+        )
+        data = self.assert_json_success(result)
+        end_times = [
+            ceiling_to_day(self.realm.date_created) + timedelta(days=i) for i in range(-1, 4)
+        ]
+        self.assertEqual(data["end_times"], [datetime_to_timestamp(dt) for dt in end_times])
+        self.assertEqual(
+            data["everyone"],
+            {
+                "_1day": [0, *self.data(100)],
+                "_15day": [0, *self.data(100)],
+                "all_time": [0, *self.data(100)],
+            },
+        )
+
+    def test_non_existent_chart(self) -> None:
+        result = self.client_get("/json/analytics/chart_data", {"chart_name": "does_not_exist"})
+        self.assert_json_error_contains(result, "Unknown chart name")
+
+    def test_analytics_not_running(self) -> None:
+        realm = get_realm("zulip")
+
+        self.assertEqual(FillState.objects.count(), 0)
+
+        realm.date_created = timezone_now() - timedelta(days=3)
+        realm.save(update_fields=["date_created"])
+        with self.assertLogs(level="WARNING") as m:
+            result = self.client_get(
+                "/json/analytics/chart_data", {"chart_name": "messages_sent_over_time"}
+            )
+            self.assertEqual(
+                m.output,
+                [
+                    f"WARNING:root:User from realm zulip attempted to access /stats, but the computed start time: {realm.date_created} (creation of realm or installation) is later than the computed end time: 0001-01-01 00:00:00+00:00 (last successful analytics update). Is the analytics cron job running?"
+                ],
+            )
+
+        self.assert_json_error_contains(result, "No analytics data available")
+
+        realm.date_created = timezone_now() - timedelta(days=1, hours=2)
+        realm.save(update_fields=["date_created"])
+        with self.assertLogs(level="WARNING") as m:
+            result = self.client_get(
+                "/json/analytics/chart_data", {"chart_name": "messages_sent_over_time"}
+            )
+            self.assertEqual(
+                m.output,
+                [
+                    f"WARNING:root:User from realm zulip attempted to access /stats, but the computed start time: {realm.date_created} (creation of realm or installation) is later than the computed end time: 0001-01-01 00:00:00+00:00 (last successful analytics update). Is the analytics cron job running?"
+                ],
+            )
+
+        self.assert_json_error_contains(result, "No analytics data available")
+
+        realm.date_created = timezone_now() - timedelta(days=1, minutes=10)
+        realm.save(update_fields=["date_created"])
+        result = self.client_get(
+            "/json/analytics/chart_data", {"chart_name": "messages_sent_over_time"}
+        )
+        self.assert_json_success(result)
+
+        realm.date_created = timezone_now() - timedelta(hours=10)
+        realm.save(update_fields=["date_created"])
+        result = self.client_get(
+            "/json/analytics/chart_data", {"chart_name": "messages_sent_over_time"}
+        )
+        self.assert_json_success(result)
+
+        end_time = timezone_now() - timedelta(days=5)
+        fill_state = FillState.objects.create(
+            property="messages_sent:is_bot:hour", end_time=end_time, state=FillState.DONE
+        )
+
+        realm.date_created = timezone_now() - timedelta(days=3)
+        realm.save(update_fields=["date_created"])
+        with self.assertLogs(level="WARNING") as m:
+            result = self.client_get(
+                "/json/analytics/chart_data", {"chart_name": "messages_sent_over_time"}
+            )
+            self.assertEqual(
+                m.output,
+                [
+                    f"WARNING:root:User from realm zulip attempted to access /stats, but the computed start time: {realm.date_created} (creation of realm or installation) is later than the computed end time: {end_time} (last successful analytics update). Is the analytics cron job running?"
+                ],
+            )
+
+        self.assert_json_error_contains(result, "No analytics data available")
+
+        realm.date_created = timezone_now() - timedelta(days=1, minutes=10)
+        realm.save(update_fields=["date_created"])
+        result = self.client_get(
+            "/json/analytics/chart_data", {"chart_name": "messages_sent_over_time"}
+        )
+        self.assert_json_success(result)
+
+        end_time = timezone_now() - timedelta(days=2)
+        fill_state.end_time = end_time
+        fill_state.save(update_fields=["end_time"])
+
+        realm.date_created = timezone_now() - timedelta(days=3)
+        realm.save(update_fields=["date_created"])
+        result = self.client_get(
+            "/json/analytics/chart_data", {"chart_name": "messages_sent_over_time"}
+        )
+        self.assert_json_success(result)
+
+        realm.date_created = timezone_now() - timedelta(days=1, hours=2)
+        realm.save(update_fields=["date_created"])
+        with self.assertLogs(level="WARNING") as m:
+            result = self.client_get(
+                "/json/analytics/chart_data", {"chart_name": "messages_sent_over_time"}
+            )
+            self.assertEqual(
+                m.output,
+                [
+                    f"WARNING:root:User from realm zulip attempted to access /stats, but the computed start time: {realm.date_created} (creation of realm or installation) is later than the computed end time: {end_time} (last successful analytics update). Is the analytics cron job running?"
+                ],
+            )
+
+        self.assert_json_error_contains(result, "No analytics data available")
+
+        realm.date_created = timezone_now() - timedelta(days=1, minutes=10)
+        realm.save(update_fields=["date_created"])
+        result = self.client_get(
+            "/json/analytics/chart_data", {"chart_name": "messages_sent_over_time"}
+        )
+        self.assert_json_success(result)
+
+    def test_get_chart_data_for_realm(self) -> None:
+        user = self.example_user("hamlet")
+        self.login_user(user)
+
+        result = self.client_get(
+            "/json/analytics/chart_data/realm/zulip", {"chart_name": "number_of_humans"}
+        )
+        self.assert_json_error(result, "Must be a server administrator", 400)
+
+        user = self.example_user("hamlet")
+        user.is_staff = True
+        user.save(update_fields=["is_staff"])
+        stat = COUNT_STATS["realm_active_humans::day"]
+        self.insert_data(stat, [None], [])
+
+        result = self.client_get(
+            "/json/analytics/chart_data/realm/not_existing_realm",
+            {"chart_name": "number_of_humans"},
+        )
+        self.assert_json_error(result, "Invalid organization", 400)
+
+        result = self.client_get(
+            "/json/analytics/chart_data/realm/zulip", {"chart_name": "number_of_humans"}
+        )
+        self.assert_json_success(result)
+
+    def test_get_chart_data_for_installation(self) -> None:
+        user = self.example_user("hamlet")
+        self.login_user(user)
+
+        result = self.client_get(
+            "/json/analytics/chart_data/installation", {"chart_name": "number_of_humans"}
+        )
+        self.assert_json_error(result, "Must be a server administrator", 400)
+
+        user = self.example_user("hamlet")
+        user.is_staff = True
+        user.save(update_fields=["is_staff"])
+        stat = COUNT_STATS["realm_active_humans::day"]
+        self.insert_data(stat, [None], [])
+
+        result = self.client_get(
+            "/json/analytics/chart_data/installation", {"chart_name": "number_of_humans"}
+        )
+        self.assert_json_success(result)
+
+
+class TestGetChartDataHelpers(ZulipTestCase):
+    def test_sort_by_totals(self) -> None:
+        empty: list[int] = []
+        value_arrays = {"c": [0, 1], "a": [9], "b": [1, 1, 1], "d": empty}
+        self.assertEqual(sort_by_totals(value_arrays), ["a", "b", "c", "d"])
+
+    def test_sort_client_labels(self) -> None:
+        data = {
+            "everyone": {"a": [16], "c": [15], "b": [14], "e": [13], "d": [12], "h": [11]},
+            "user": {"a": [6], "b": [5], "d": [4], "e": [3], "f": [2], "g": [1]},
+        }
+        self.assertEqual(sort_client_labels(data), ["a", "b", "c", "d", "e", "f", "g", "h"])
+
+
+class TestTimeRange(ZulipTestCase):
+    def test_time_range(self) -> None:
+        HOUR = timedelta(hours=1)
+        DAY = timedelta(days=1)
+
+        a_time = datetime(2016, 3, 14, 22, 59, tzinfo=timezone.utc)
+        floor_hour = datetime(2016, 3, 14, 22, tzinfo=timezone.utc)
+        floor_day = datetime(2016, 3, 14, tzinfo=timezone.utc)
+
+        # test start == end
+        self.assertEqual(time_range(a_time, a_time, CountStat.HOUR, None), [])
+        self.assertEqual(time_range(a_time, a_time, CountStat.DAY, None), [])
+        # test start == end == boundary, and min_length == 0
+        self.assertEqual(time_range(floor_hour, floor_hour, CountStat.HOUR, 0), [floor_hour])
+        self.assertEqual(time_range(floor_day, floor_day, CountStat.DAY, 0), [floor_day])
+        # test start and end on different boundaries
+        self.assertEqual(
+            time_range(floor_hour, floor_hour + HOUR, CountStat.HOUR, None),
+            [floor_hour, floor_hour + HOUR],
+        )
+        self.assertEqual(
+            time_range(floor_day, floor_day + DAY, CountStat.DAY, None),
+            [floor_day, floor_day + DAY],
+        )
+        # test min_length
+        self.assertEqual(
+            time_range(floor_hour, floor_hour + HOUR, CountStat.HOUR, 4),
+            [floor_hour - 2 * HOUR, floor_hour - HOUR, floor_hour, floor_hour + HOUR],
+        )
+        self.assertEqual(
+            time_range(floor_day, floor_day + DAY, CountStat.DAY, 4),
+            [floor_day - 2 * DAY, floor_day - DAY, floor_day, floor_day + DAY],
+        )
+
+
+class TestMapArrays(ZulipTestCase):
+    def test_map_arrays(self) -> None:
+        a = {
+            "desktop app 1.0": [1, 2, 3],
+            "desktop app 2.0": [10, 12, 13],
+            "desktop app 3.0": [21, 22, 23],
+            "website": [1, 2, 3],
+            "ZulipiOS": [1, 2, 3],
+            "ZulipElectron": [2, 5, 7],
+            "ZulipMobile": [1, 2, 3],
+            "ZulipMobile/flutter": [1, 1, 1],
+            "ZulipFlutter": [1, 1, 1],
+            "ZulipPython": [1, 2, 3],
+            "API: Python": [1, 2, 3],
+            "SomethingRandom": [4, 5, 6],
+            "ZulipGitHubWebhook": [7, 7, 9],
+            "ZulipAndroid": [64, 63, 65],
+            "ZulipTerminal": [9, 10, 11],
+        }
+        result = rewrite_client_arrays(a)
+        self.assertEqual(
+            result,
+            {
+                "Old desktop app": [32, 36, 39],
+                "Ancient iOS app": [1, 2, 3],
+                "Desktop app": [2, 5, 7],
+                "Old mobile app (React Native)": [1, 2, 3],
+                "Mobile app (Flutter)": [2, 2, 2],
+                "Web app": [1, 2, 3],
+                "Python API": [2, 4, 6],
+                "SomethingRandom": [4, 5, 6],
+                "GitHub webhook": [7, 7, 9],
+                "Ancient Android app": [64, 63, 65],
+                "Terminal app": [9, 10, 11],
+            },
+        )

analytics/urls.py

@@ -0,0 +1,74 @@
+from django.conf import settings
+from django.conf.urls import include
+from django.urls import path
+from django.urls.resolvers import URLPattern, URLResolver
+
+from analytics.views.stats import (
+    get_chart_data,
+    get_chart_data_for_installation,
+    get_chart_data_for_realm,
+    get_chart_data_for_stream,
+    stats,
+    stats_for_installation,
+    stats_for_realm,
+)
+from zerver.lib.rest import rest_path
+
+i18n_urlpatterns: list[URLPattern | URLResolver] = [
+    # Server admin (user_profile.is_staff) visible stats pages
+    path("stats/realm/<realm_str>/", stats_for_realm),
+    path("stats/installation", stats_for_installation),
+    # User-visible stats page
+    path("stats", stats, name="stats"),
+]
+
+if settings.ZILENCER_ENABLED:
+    from analytics.views.stats import stats_for_remote_installation, stats_for_remote_realm
+
+    i18n_urlpatterns += [
+        path("stats/remote/<int:remote_server_id>/installation", stats_for_remote_installation),
+        path(
+            "stats/remote/<int:remote_server_id>/realm/<int:remote_realm_id>/",
+            stats_for_remote_realm,
+        ),
+    ]
+
+# These endpoints are a part of the API (V1), which uses:
+# * REST verbs
+# * Basic auth (username:password is email:apiKey)
+# * Takes and returns json-formatted data
+#
+# See rest_dispatch in zerver.lib.rest for an explanation of auth methods used
+#
+# All of these paths are accessed by either a /json or /api prefix
+v1_api_and_json_patterns = [
+    # get data for the graphs at /stats
+    rest_path("analytics/chart_data", GET=get_chart_data),
+    rest_path("analytics/chart_data/stream/<stream_id>", GET=get_chart_data_for_stream),
+    rest_path("analytics/chart_data/realm/<realm_str>", GET=get_chart_data_for_realm),
+    rest_path("analytics/chart_data/installation", GET=get_chart_data_for_installation),
+]
+
+if settings.ZILENCER_ENABLED:
+    from analytics.views.stats import (
+        get_chart_data_for_remote_installation,
+        get_chart_data_for_remote_realm,
+    )
+
+    v1_api_and_json_patterns += [
+        rest_path(
+            "analytics/chart_data/remote/<int:remote_server_id>/installation",
+            GET=get_chart_data_for_remote_installation,
+        ),
+        rest_path(
+            "analytics/chart_data/remote/<int:remote_server_id>/realm/<int:remote_realm_id>",
+            GET=get_chart_data_for_remote_realm,
+        ),
+    ]
+
+i18n_urlpatterns += [
+    path("api/v1/", include(v1_api_and_json_patterns)),
+    path("json/", include(v1_api_and_json_patterns)),
+]
+
+urlpatterns = i18n_urlpatterns

analytics/views/stats.py

@@ -0,0 +1,649 @@
+import logging
+from collections import defaultdict
+from datetime import datetime, timedelta, timezone
+from typing import Annotated, Any, Optional, TypeAlias, TypeVar
+
+from django.conf import settings
+from django.db.models import QuerySet
+from django.http import HttpRequest, HttpResponse, HttpResponseNotFound
+from django.shortcuts import render
+from django.utils import translation
+from django.utils.timezone import now as timezone_now
+from django.utils.translation import gettext as _
+from pydantic import BeforeValidator, Json, NonNegativeInt
+
+from analytics.lib.counts import COUNT_STATS, CountStat
+from analytics.lib.time_utils import time_range
+from analytics.models import (
+    BaseCount,
+    InstallationCount,
+    RealmCount,
+    StreamCount,
+    UserCount,
+    installation_epoch,
+)
+from zerver.decorator import (
+    require_non_guest_user,
+    require_server_admin,
+    require_server_admin_api,
+    to_utc_datetime,
+    zulip_login_required,
+)
+from zerver.lib.exceptions import JsonableError
+from zerver.lib.i18n import get_and_set_request_language, get_language_translation_data
+from zerver.lib.response import json_success
+from zerver.lib.streams import access_stream_by_id
+from zerver.lib.timestamp import convert_to_UTC
+from zerver.lib.typed_endpoint import PathOnly, typed_endpoint
+from zerver.models import Client, Realm, Stream, UserProfile
+from zerver.models.realms import get_realm
+
+if settings.ZILENCER_ENABLED:
+    from zilencer.models import RemoteInstallationCount, RemoteRealmCount, RemoteZulipServer
+
+MAX_TIME_FOR_FULL_ANALYTICS_GENERATION = timedelta(days=1, minutes=30)
+
+
+def is_analytics_ready(realm: Realm) -> bool:
+    return (timezone_now() - realm.date_created) > MAX_TIME_FOR_FULL_ANALYTICS_GENERATION
+
+
+def render_stats(
+    request: HttpRequest,
+    data_url_suffix: str,
+    realm: Realm | None,
+    *,
+    title: str | None = None,
+    analytics_ready: bool = True,
+) -> HttpResponse:
+    assert request.user.is_authenticated
+
+    if realm is not None:
+        # Same query to get guest user count as in get_seat_count in corporate/lib/stripe.py.
+        guest_users = UserProfile.objects.filter(
+            realm=realm, is_active=True, is_bot=False, role=UserProfile.ROLE_GUEST
+        ).count()
+        space_used = realm.currently_used_upload_space_bytes()
+        if title:
+            pass
+        else:
+            title = realm.name or realm.string_id
+    else:
+        assert title
+        guest_users = None
+        space_used = None
+
+    request_language = get_and_set_request_language(
+        request,
+        request.user.default_language,
+        translation.get_language_from_path(request.path_info),
+    )
+
+    # Sync this with stats_params_schema in base_page_params.ts.
+    page_params = dict(
+        page_type="stats",
+        data_url_suffix=data_url_suffix,
+        upload_space_used=space_used,
+        guest_users=guest_users,
+        translation_data=get_language_translation_data(request_language),
+    )
+
+    return render(
+        request,
+        "analytics/stats.html",
+        context=dict(
+            target_name=title,
+            page_params=page_params,
+            analytics_ready=analytics_ready,
+        ),
+    )
+
+
+@zulip_login_required
+def stats(request: HttpRequest) -> HttpResponse:
+    assert request.user.is_authenticated
+    realm = request.user.realm
+    if request.user.is_guest:
+        # TODO: Make @zulip_login_required pass the UserProfile so we
+        # can use @require_human_non_guest_user
+        raise JsonableError(_("Not allowed for guest users"))
+    return render_stats(request, "", realm, analytics_ready=is_analytics_ready(realm))
+
+
+@require_server_admin
+@typed_endpoint
+def stats_for_realm(request: HttpRequest, *, realm_str: PathOnly[str]) -> HttpResponse:
+    try:
+        realm = get_realm(realm_str)
+    except Realm.DoesNotExist:
+        return HttpResponseNotFound()
+
+    return render_stats(
+        request,
+        f"/realm/{realm_str}",
+        realm,
+        analytics_ready=is_analytics_ready(realm),
+    )
+
+
+@require_server_admin
+@typed_endpoint
+def stats_for_remote_realm(
+    request: HttpRequest, *, remote_server_id: PathOnly[int], remote_realm_id: PathOnly[int]
+) -> HttpResponse:
+    assert settings.ZILENCER_ENABLED
+    server = RemoteZulipServer.objects.get(id=remote_server_id)
+    return render_stats(
+        request,
+        f"/remote/{server.id}/realm/{remote_realm_id}",
+        None,
+        title=f"Realm {remote_realm_id} on server {server.hostname}",
+    )
+
+
+@require_server_admin_api
+@typed_endpoint
+def get_chart_data_for_realm(
+    request: HttpRequest,
+    user_profile: UserProfile,
+    /,
+    *,
+    realm_str: PathOnly[str],
+    chart_name: str,
+    min_length: Json[NonNegativeInt] | None = None,
+    start: Annotated[datetime | None, BeforeValidator(to_utc_datetime)] = None,
+    end: Annotated[datetime | None, BeforeValidator(to_utc_datetime)] = None,
+) -> HttpResponse:
+    try:
+        realm = get_realm(realm_str)
+    except Realm.DoesNotExist:
+        raise JsonableError(_("Invalid organization"))
+
+    return do_get_chart_data(
+        request,
+        user_profile,
+        realm=realm,
+        chart_name=chart_name,
+        min_length=min_length,
+        start=start,
+        end=end,
+    )
+
+
+@require_non_guest_user
+@typed_endpoint
+def get_chart_data_for_stream(
+    request: HttpRequest,
+    user_profile: UserProfile,
+    *,
+    stream_id: PathOnly[int],
+    chart_name: str,
+    min_length: Json[NonNegativeInt] | None = None,
+    start: Annotated[datetime | None, BeforeValidator(to_utc_datetime)] = None,
+    end: Annotated[datetime | None, BeforeValidator(to_utc_datetime)] = None,
+) -> HttpResponse:
+    stream, _sub = access_stream_by_id(
+        user_profile,
+        stream_id,
+        require_content_access=False,
+    )
+
+    return do_get_chart_data(
+        request,
+        user_profile,
+        stream=stream,
+        chart_name=chart_name,
+        min_length=min_length,
+        start=start,
+        end=end,
+    )
+
+
+@require_server_admin_api
+@typed_endpoint
+def get_chart_data_for_remote_realm(
+    request: HttpRequest,
+    user_profile: UserProfile,
+    /,
+    *,
+    remote_server_id: PathOnly[int],
+    remote_realm_id: PathOnly[int],
+    chart_name: str,
+    min_length: Json[NonNegativeInt] | None = None,
+    start: Annotated[datetime | None, BeforeValidator(to_utc_datetime)] = None,
+    end: Annotated[datetime | None, BeforeValidator(to_utc_datetime)] = None,
+) -> HttpResponse:
+    assert settings.ZILENCER_ENABLED
+    server = RemoteZulipServer.objects.get(id=remote_server_id)
+    return do_get_chart_data(
+        request,
+        user_profile,
+        server=server,
+        remote=True,
+        remote_realm_id=remote_realm_id,
+        chart_name=chart_name,
+        min_length=min_length,
+        start=start,
+        end=end,
+    )
+
+
+@require_server_admin
+def stats_for_installation(request: HttpRequest) -> HttpResponse:
+    assert request.user.is_authenticated
+    return render_stats(request, "/installation", None, title="installation")
+
+
+@require_server_admin
+def stats_for_remote_installation(request: HttpRequest, remote_server_id: int) -> HttpResponse:
+    assert settings.ZILENCER_ENABLED
+    server = RemoteZulipServer.objects.get(id=remote_server_id)
+    return render_stats(
+        request,
+        f"/remote/{server.id}/installation",
+        None,
+        title=f"remote installation {server.hostname}",
+    )
+
+
+@require_server_admin_api
+@typed_endpoint
+def get_chart_data_for_installation(
+    request: HttpRequest,
+    user_profile: UserProfile,
+    /,
+    *,
+    chart_name: str,
+    min_length: Json[NonNegativeInt] | None = None,
+    start: Annotated[datetime | None, BeforeValidator(to_utc_datetime)] = None,
+    end: Annotated[datetime | None, BeforeValidator(to_utc_datetime)] = None,
+) -> HttpResponse:
+    return do_get_chart_data(
+        request,
+        user_profile,
+        for_installation=True,
+        chart_name=chart_name,
+        min_length=min_length,
+        start=start,
+        end=end,
+    )
+
+
+@require_server_admin_api
+@typed_endpoint
+def get_chart_data_for_remote_installation(
+    request: HttpRequest,
+    user_profile: UserProfile,
+    /,
+    *,
+    remote_server_id: PathOnly[int],
+    chart_name: str,
+    min_length: Json[NonNegativeInt] | None = None,
+    start: Annotated[datetime | None, BeforeValidator(to_utc_datetime)] = None,
+    end: Annotated[datetime | None, BeforeValidator(to_utc_datetime)] = None,
+) -> HttpResponse:
+    assert settings.ZILENCER_ENABLED
+    server = RemoteZulipServer.objects.get(id=remote_server_id)
+    return do_get_chart_data(
+        request,
+        user_profile,
+        for_installation=True,
+        remote=True,
+        server=server,
+        chart_name=chart_name,
+        min_length=min_length,
+        start=start,
+        end=end,
+    )
+
+
+@require_non_guest_user
+@typed_endpoint
+def get_chart_data(
+    request: HttpRequest,
+    user_profile: UserProfile,
+    *,
+    chart_name: str,
+    min_length: Json[NonNegativeInt] | None = None,
+    start: Annotated[datetime | None, BeforeValidator(to_utc_datetime)] = None,
+    end: Annotated[datetime | None, BeforeValidator(to_utc_datetime)] = None,
+) -> HttpResponse:
+    return do_get_chart_data(
+        request,
+        user_profile,
+        chart_name=chart_name,
+        min_length=min_length,
+        start=start,
+        end=end,
+    )
+
+
+@require_non_guest_user
+def do_get_chart_data(
+    request: HttpRequest,
+    user_profile: UserProfile,
+    *,
+    # Common parameters supported by all stats endpoints.
+    chart_name: str,
+    min_length: NonNegativeInt | None = None,
+    start: datetime | None = None,
+    end: datetime | None = None,
+    # The following parameters are only used by wrapping functions for
+    # various contexts; the callers are responsible for validating them.
+    realm: Realm | None = None,
+    for_installation: bool = False,
+    remote: bool = False,
+    remote_realm_id: int | None = None,
+    server: Optional["RemoteZulipServer"] = None,
+    stream: Stream | None = None,
+) -> HttpResponse:
+    TableType: TypeAlias = (
+        type["RemoteInstallationCount"]
+        | type[InstallationCount]
+        | type["RemoteRealmCount"]
+        | type[RealmCount]
+    )
+    if for_installation:
+        if remote:
+            assert settings.ZILENCER_ENABLED
+            aggregate_table: TableType = RemoteInstallationCount
+            assert server is not None
+        else:
+            aggregate_table = InstallationCount
+    else:
+        if remote:
+            assert settings.ZILENCER_ENABLED
+            aggregate_table = RemoteRealmCount
+            assert server is not None
+            assert remote_realm_id is not None
+        else:
+            aggregate_table = RealmCount
+
+    tables: (
+        tuple[TableType] | tuple[TableType, type[UserCount]] | tuple[TableType, type[StreamCount]]
+    )
+
+    if chart_name == "number_of_humans":
+        stats = [
+            COUNT_STATS["1day_actives::day"],
+            COUNT_STATS["realm_active_humans::day"],
+            COUNT_STATS["active_users_audit:is_bot:day"],
+        ]
+        tables = (aggregate_table,)
+        subgroup_to_label: dict[CountStat, dict[str | None, str]] = {
+            stats[0]: {None: "_1day"},
+            stats[1]: {None: "_15day"},
+            stats[2]: {"false": "all_time"},
+        }
+        labels_sort_function = None
+        include_empty_subgroups = True
+    elif chart_name == "messages_sent_over_time":
+        stats = [COUNT_STATS["messages_sent:is_bot:hour"]]
+        tables = (aggregate_table, UserCount)
+        subgroup_to_label = {stats[0]: {"false": "human", "true": "bot"}}
+        labels_sort_function = None
+        include_empty_subgroups = True
+    elif chart_name == "messages_sent_by_message_type":
+        stats = [COUNT_STATS["messages_sent:message_type:day"]]
+        tables = (aggregate_table, UserCount)
+        subgroup_to_label = {
+            stats[0]: {
+                "public_stream": _("Public channels"),
+                "private_stream": _("Private channels"),
+                "private_message": _("Direct messages"),
+                "huddle_message": _("Group direct messages"),
+            }
+        }
+        labels_sort_function = lambda data: sort_by_totals(data["everyone"])
+        include_empty_subgroups = True
+    elif chart_name == "messages_sent_by_client":
+        stats = [COUNT_STATS["messages_sent:client:day"]]
+        tables = (aggregate_table, UserCount)
+        # Note that the labels are further re-written by client_label_map
+        subgroup_to_label = {
+            stats[0]: {str(id): name for id, name in Client.objects.values_list("id", "name")}
+        }
+        labels_sort_function = sort_client_labels
+        include_empty_subgroups = False
+    elif chart_name == "messages_read_over_time":
+        stats = [COUNT_STATS["messages_read::hour"]]
+        tables = (aggregate_table, UserCount)
+        subgroup_to_label = {stats[0]: {None: "read"}}
+        labels_sort_function = None
+        include_empty_subgroups = True
+    elif chart_name == "messages_sent_by_stream":
+        if stream is None:
+            raise JsonableError(
+                _("Missing channel for chart: {chart_name}").format(chart_name=chart_name)
+            )
+        stats = [COUNT_STATS["messages_in_stream:is_bot:day"]]
+        tables = (aggregate_table, StreamCount)
+        subgroup_to_label = {stats[0]: {"false": "human", "true": "bot"}}
+        labels_sort_function = None
+        include_empty_subgroups = True
+    else:
+        raise JsonableError(_("Unknown chart name: {chart_name}").format(chart_name=chart_name))
+
+    # Most likely someone using our API endpoint. The /stats page does not
+    # pass a start or end in its requests.
+    if start is not None:
+        start = convert_to_UTC(start)
+    if end is not None:
+        end = convert_to_UTC(end)
+    if start is not None and end is not None and start > end:
+        raise JsonableError(
+            _("Start time is later than end time. Start: {start}, End: {end}").format(
+                start=start,
+                end=end,
+            )
+        )
+
+    if realm is None:
+        # Note that this value is invalid for Remote tables; be
+        # careful not to access it in those code paths.
+        realm = user_profile.realm
+
+    if remote:
+        # For remote servers, we don't have fillstate data, and thus
+        # should simply use the first and last data points for the
+        # table.
+        assert server is not None
+        assert aggregate_table is RemoteInstallationCount or aggregate_table is RemoteRealmCount
+        if not aggregate_table.objects.filter(server=server).exists():
+            raise JsonableError(
+                _("No analytics data available. Please contact your server administrator.")
+            )
+        if start is None:
+            first = aggregate_table.objects.filter(server=server).order_by("remote_id").first()
+            assert first is not None
+            start = first.end_time
+        if end is None:
+            last = aggregate_table.objects.filter(server=server).order_by("remote_id").last()
+            assert last is not None
+            end = last.end_time
+    else:
+        # Otherwise, we can use tables on the current server to
+        # determine a nice range, and some additional validation.
+        if start is None:
+            if for_installation:
+                start = installation_epoch()
+            else:
+                start = realm.date_created
+        if end is None:
+            end = max(
+                stat.last_successful_fill() or datetime.min.replace(tzinfo=timezone.utc)
+                for stat in stats
+            )
+
+        if start > end and (timezone_now() - start > MAX_TIME_FOR_FULL_ANALYTICS_GENERATION):
+            logging.warning(
+                "User from realm %s attempted to access /stats, but the computed "
+                "start time: %s (creation of realm or installation) is later than the computed "
+                "end time: %s (last successful analytics update). Is the "
+                "analytics cron job running?",
+                realm.string_id,
+                start,
+                end,
+            )
+            raise JsonableError(
+                _("No analytics data available. Please contact your server administrator.")
+            )
+
+    assert len({stat.frequency for stat in stats}) == 1
+    end_times = time_range(start, end, stats[0].frequency, min_length)
+    data: dict[str, Any] = {
+        "end_times": [int(end_time.timestamp()) for end_time in end_times],
+        "frequency": stats[0].frequency,
+    }
+
+    aggregation_level = {
+        InstallationCount: "everyone",
+        RealmCount: "everyone",
+        UserCount: "user",
+        StreamCount: "everyone",
+    }
+    if settings.ZILENCER_ENABLED:
+        aggregation_level[RemoteInstallationCount] = "everyone"
+        aggregation_level[RemoteRealmCount] = "everyone"
+
+    # -1 is a placeholder value, since there is no relevant filtering on InstallationCount
+    id_value = {
+        InstallationCount: -1,
+        RealmCount: realm.id,
+        UserCount: user_profile.id,
+    }
+    if stream is not None:
+        id_value[StreamCount] = stream.id
+
+    if settings.ZILENCER_ENABLED:
+        if server is not None:
+            id_value[RemoteInstallationCount] = server.id
+        # TODO: RemoteRealmCount logic doesn't correctly handle
+        # filtering by server_id as well.
+        if remote_realm_id is not None:
+            id_value[RemoteRealmCount] = remote_realm_id
+
+    for table in tables:
+        data[aggregation_level[table]] = {}
+        for stat in stats:
+            data[aggregation_level[table]].update(
+                get_time_series_by_subgroup(
+                    stat,
+                    table,
+                    id_value[table],
+                    end_times,
+                    subgroup_to_label[stat],
+                    include_empty_subgroups,
+                )
+            )
+
+    if labels_sort_function is not None:
+        data["display_order"] = labels_sort_function(data)
+    else:
+        data["display_order"] = None
+    return json_success(request, data=data)
+
+
+def sort_by_totals(value_arrays: dict[str, list[int]]) -> list[str]:
+    totals = sorted(((sum(values), label) for label, values in value_arrays.items()), reverse=True)
+    return [label for total, label in totals]
+
+
+# For any given user, we want to show a fixed set of clients in the chart,
+# regardless of the time aggregation or whether we're looking at realm or
+# user data. This fixed set ideally includes the clients most important in
+# understanding the realm's traffic and the user's traffic. This function
+# tries to rank the clients so that taking the first N elements of the
+# sorted list has a reasonable chance of doing so.
+def sort_client_labels(data: dict[str, dict[str, list[int]]]) -> list[str]:
+    realm_order = sort_by_totals(data["everyone"])
+    user_order = sort_by_totals(data["user"])
+    label_sort_values: dict[str, float] = {label: i for i, label in enumerate(realm_order)}
+    for i, label in enumerate(user_order):
+        label_sort_values[label] = min(i - 0.1, label_sort_values.get(label, i))
+    return [label for label, sort_value in sorted(label_sort_values.items(), key=lambda x: x[1])]
+
+
+CountT = TypeVar("CountT", bound=BaseCount)
+
+
+def table_filtered_to_id(table: type[CountT], key_id: int) -> QuerySet[CountT]:
+    if table == RealmCount:
+        return table._default_manager.filter(realm_id=key_id)
+    elif table == UserCount:
+        return table._default_manager.filter(user_id=key_id)
+    elif table == StreamCount:
+        return table._default_manager.filter(stream_id=key_id)
+    elif table == InstallationCount:
+        return table._default_manager.all()
+    elif settings.ZILENCER_ENABLED and table == RemoteInstallationCount:
+        return table._default_manager.filter(server_id=key_id)
+    elif settings.ZILENCER_ENABLED and table == RemoteRealmCount:
+        return table._default_manager.filter(realm_id=key_id)
+    else:
+        raise AssertionError(f"Unknown table: {table}")
+
+
+def client_label_map(name: str) -> str:
+    if name == "website":
+        return "Web app"
+    if name.startswith("desktop app"):
+        return "Old desktop app"
+    if name == "ZulipElectron":
+        return "Desktop app"
+    if name == "ZulipTerminal":
+        return "Terminal app"
+    if name == "ZulipAndroid":
+        return "Ancient Android app"
+    if name == "ZulipiOS":
+        return "Ancient iOS app"
+    if name == "ZulipMobile":
+        return "Old mobile app (React Native)"
+    if name in ["ZulipFlutter", "ZulipMobile/flutter"]:
+        return "Mobile app (Flutter)"
+    if name in ["ZulipPython", "API: Python"]:
+        return "Python API"
+    if name.startswith("Zulip") and name.endswith("Webhook"):
+        return name.removeprefix("Zulip").removesuffix("Webhook") + " webhook"
+    return name
+
+
+def rewrite_client_arrays(value_arrays: dict[str, list[int]]) -> dict[str, list[int]]:
+    mapped_arrays: dict[str, list[int]] = {}
+    for label, array in value_arrays.items():
+        mapped_label = client_label_map(label)
+        if mapped_label in mapped_arrays:
+            for i in range(len(array)):
+                mapped_arrays[mapped_label][i] += array[i]
+        else:
+            mapped_arrays[mapped_label] = array.copy()
+    return mapped_arrays
+
+
+def get_time_series_by_subgroup(
+    stat: CountStat,
+    table: type[BaseCount],
+    key_id: int,
+    end_times: list[datetime],
+    subgroup_to_label: dict[str | None, str],
+    include_empty_subgroups: bool,
+) -> dict[str, list[int]]:
+    queryset = (
+        table_filtered_to_id(table, key_id)
+        .filter(property=stat.property)
+        .values_list("subgroup", "end_time", "value")
+    )
+    value_dicts: dict[str | None, dict[datetime, int]] = defaultdict(lambda: defaultdict(int))
+    for subgroup, end_time, value in queryset:
+        value_dicts[subgroup][end_time] = value
+    value_arrays = {}
+    for subgroup, label in subgroup_to_label.items():
+        if (subgroup in value_dicts) or include_empty_subgroups:
+            value_arrays[label] = [value_dicts[subgroup][end_time] for end_time in end_times]
+
+    if stat == COUNT_STATS["messages_sent:client:day"]:
+        # HACK: We rewrite these arrays to collapse the Client objects
+        # with similar names into a single sum, and generally give
+        # them better names
+        return rewrite_client_arrays(value_arrays)
+    return value_arrays