snapshot of https://github.com/zulip/zulip.git @ 48c69911c7dd4e03b20552eb3bd77ad8151c1357 (2026-09-15)
Some checks failed
Code scanning / CodeQL (push) Failing after 15s
Zulip production suite / Ubuntu 22.04 production build (push) Failing after 26s
Zulip production suite / Debian 12 production install with custom db name and user (push) Has been skipped
Zulip production suite / Ubuntu 22.04 production install and PostgreSQL upgrade with pgroonga (push) Has been skipped
Zulip production suite / Ubuntu 24.04 production install (push) Has been skipped
Zulip production suite / Ubuntu 26.04 production install (push) Has been skipped
Zulip production suite / Debian 13 production install (push) Has been skipped
Zulip production suite / 7.0 Version Upgrade (push) Has been skipped
Zulip production suite / 8.0 Version Upgrade (push) Has been skipped
Zulip production suite / 6.0 Version Upgrade (push) Has been skipped
Zulip production suite / 10.0 Version Upgrade (push) Has been skipped
Zulip production suite / 9.0 Version Upgrade (push) Has been skipped
Zulip production suite / 12.0 Version Upgrade (push) Has been skipped
Zulip production suite / 11.0 Version Upgrade (push) Has been skipped
GitHub Actions Security Analysis with zizmor / zizmor (push) Failing after 3s
Zulip CI / Debian 12 (Python 3.11, backend + documentation) (push) Failing after 26s
Zulip CI / Ubuntu 22.04 (Python 3.10, backend + frontend) (push) Failing after 3s
Zulip CI / Ubuntu 24.04 (Python 3.12, backend) (push) Failing after 25s
Zulip CI / Ubuntu 26.04 (Python 3.14, backend) (push) Failing after 25s
Zulip CI / Debian 13 (Python 3.13, backend) (push) Failing after 28s
Zulip production suite / Required jobs (push) Has been cancelled
Zulip CI / Required jobs (push) Has been cancelled
API Documentation Update Check / check-feature-level-updated (push) Has been cancelled
API Documentation Update Check / notify-if-api-docs-changed (push) Has been cancelled
Some checks failed
Code scanning / CodeQL (push) Failing after 15s
Zulip production suite / Ubuntu 22.04 production build (push) Failing after 26s
Zulip production suite / Debian 12 production install with custom db name and user (push) Has been skipped
Zulip production suite / Ubuntu 22.04 production install and PostgreSQL upgrade with pgroonga (push) Has been skipped
Zulip production suite / Ubuntu 24.04 production install (push) Has been skipped
Zulip production suite / Ubuntu 26.04 production install (push) Has been skipped
Zulip production suite / Debian 13 production install (push) Has been skipped
Zulip production suite / 7.0 Version Upgrade (push) Has been skipped
Zulip production suite / 8.0 Version Upgrade (push) Has been skipped
Zulip production suite / 6.0 Version Upgrade (push) Has been skipped
Zulip production suite / 10.0 Version Upgrade (push) Has been skipped
Zulip production suite / 9.0 Version Upgrade (push) Has been skipped
Zulip production suite / 12.0 Version Upgrade (push) Has been skipped
Zulip production suite / 11.0 Version Upgrade (push) Has been skipped
GitHub Actions Security Analysis with zizmor / zizmor (push) Failing after 3s
Zulip CI / Debian 12 (Python 3.11, backend + documentation) (push) Failing after 26s
Zulip CI / Ubuntu 22.04 (Python 3.10, backend + frontend) (push) Failing after 3s
Zulip CI / Ubuntu 24.04 (Python 3.12, backend) (push) Failing after 25s
Zulip CI / Ubuntu 26.04 (Python 3.14, backend) (push) Failing after 25s
Zulip CI / Debian 13 (Python 3.13, backend) (push) Failing after 28s
Zulip production suite / Required jobs (push) Has been cancelled
Zulip CI / Required jobs (push) Has been cancelled
API Documentation Update Check / check-feature-level-updated (push) Has been cancelled
API Documentation Update Check / notify-if-api-docs-changed (push) Has been cancelled
This commit is contained in:
0
analytics/management/commands/__init__.py
Normal file
0
analytics/management/commands/__init__.py
Normal file
92
analytics/management/commands/check_analytics_state.py
Normal file
92
analytics/management/commands/check_analytics_state.py
Normal file
@@ -0,0 +1,92 @@
|
||||
from dataclasses import dataclass
|
||||
from datetime import timedelta
|
||||
from typing import Any, Literal
|
||||
|
||||
from django.utils.timezone import now as timezone_now
|
||||
from typing_extensions import override
|
||||
|
||||
from analytics.lib.counts import ALL_COUNT_STATS, CountStat
|
||||
from analytics.models import installation_epoch
|
||||
from scripts.lib.zulip_tools import atomic_nagios_write
|
||||
from zerver.lib.management import ZulipBaseCommand
|
||||
from zerver.lib.timestamp import TimeZoneNotUTCError, floor_to_day, floor_to_hour, verify_UTC
|
||||
from zerver.models import Realm
|
||||
|
||||
states = {
|
||||
0: "OK",
|
||||
1: "WARNING",
|
||||
2: "CRITICAL",
|
||||
3: "UNKNOWN",
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class NagiosResult:
|
||||
status: Literal["ok", "warning", "critical", "unknown"]
|
||||
message: str
|
||||
|
||||
|
||||
class Command(ZulipBaseCommand):
|
||||
help = """Checks FillState table.
|
||||
|
||||
Run as a cron job that runs every hour."""
|
||||
|
||||
@override
|
||||
def handle(self, *args: Any, **options: Any) -> None:
|
||||
fill_state = self.get_fill_state()
|
||||
atomic_nagios_write("check-analytics-state", fill_state.status, fill_state.message)
|
||||
|
||||
def get_fill_state(self) -> NagiosResult:
|
||||
if not Realm.objects.exists():
|
||||
return NagiosResult(status="ok", message="No realms exist, so not checking FillState.")
|
||||
|
||||
warning_unfilled_properties = []
|
||||
critical_unfilled_properties = []
|
||||
for property, stat in ALL_COUNT_STATS.items():
|
||||
last_fill = stat.last_successful_fill()
|
||||
if last_fill is None:
|
||||
last_fill = installation_epoch()
|
||||
try:
|
||||
verify_UTC(last_fill)
|
||||
except TimeZoneNotUTCError:
|
||||
return NagiosResult(
|
||||
status="critical", message=f"FillState not in UTC for {property}"
|
||||
)
|
||||
|
||||
if stat.frequency == CountStat.DAY:
|
||||
floor_function = floor_to_day
|
||||
warning_threshold = timedelta(hours=26)
|
||||
critical_threshold = timedelta(hours=50)
|
||||
else: # CountStat.HOUR
|
||||
floor_function = floor_to_hour
|
||||
warning_threshold = timedelta(minutes=90)
|
||||
critical_threshold = timedelta(minutes=150)
|
||||
|
||||
if floor_function(last_fill) != last_fill:
|
||||
return NagiosResult(
|
||||
status="critical",
|
||||
message=f"FillState not on {stat.frequency} boundary for {property}",
|
||||
)
|
||||
|
||||
time_to_last_fill = timezone_now() - last_fill
|
||||
if time_to_last_fill > critical_threshold:
|
||||
critical_unfilled_properties.append(property)
|
||||
elif time_to_last_fill > warning_threshold:
|
||||
warning_unfilled_properties.append(property)
|
||||
|
||||
if len(critical_unfilled_properties) == 0 and len(warning_unfilled_properties) == 0:
|
||||
return NagiosResult(status="ok", message="FillState looks fine.")
|
||||
if len(critical_unfilled_properties) == 0:
|
||||
return NagiosResult(
|
||||
status="warning",
|
||||
message="Missed filling {} once.".format(
|
||||
", ".join(warning_unfilled_properties),
|
||||
),
|
||||
)
|
||||
return NagiosResult(
|
||||
status="critical",
|
||||
message="Missed filling {} once. Missed filling {} at least twice.".format(
|
||||
", ".join(warning_unfilled_properties),
|
||||
", ".join(critical_unfilled_properties),
|
||||
),
|
||||
)
|
||||
25
analytics/management/commands/clear_analytics_tables.py
Normal file
25
analytics/management/commands/clear_analytics_tables.py
Normal file
@@ -0,0 +1,25 @@
|
||||
from argparse import ArgumentParser
|
||||
from typing import Any
|
||||
|
||||
from django.core.management.base import CommandError
|
||||
from typing_extensions import override
|
||||
|
||||
from analytics.lib.counts import do_drop_all_analytics_tables
|
||||
from zerver.lib.management import ZulipBaseCommand
|
||||
|
||||
|
||||
class Command(ZulipBaseCommand):
|
||||
help = """Clear analytics tables."""
|
||||
|
||||
@override
|
||||
def add_arguments(self, parser: ArgumentParser) -> None:
|
||||
parser.add_argument("--force", action="store_true", help="Clear analytics tables.")
|
||||
|
||||
@override
|
||||
def handle(self, *args: Any, **options: Any) -> None:
|
||||
if options["force"]:
|
||||
do_drop_all_analytics_tables()
|
||||
else:
|
||||
raise CommandError(
|
||||
"Would delete all data from analytics tables (!); use --force to do so."
|
||||
)
|
||||
27
analytics/management/commands/clear_single_stat.py
Normal file
27
analytics/management/commands/clear_single_stat.py
Normal file
@@ -0,0 +1,27 @@
|
||||
from argparse import ArgumentParser
|
||||
from typing import Any
|
||||
|
||||
from django.core.management.base import CommandError
|
||||
from typing_extensions import override
|
||||
|
||||
from analytics.lib.counts import ALL_COUNT_STATS, do_drop_single_stat
|
||||
from zerver.lib.management import ZulipBaseCommand
|
||||
|
||||
|
||||
class Command(ZulipBaseCommand):
|
||||
help = """Clear analytics tables."""
|
||||
|
||||
@override
|
||||
def add_arguments(self, parser: ArgumentParser) -> None:
|
||||
parser.add_argument("--force", action="store_true", help="Actually do it.")
|
||||
parser.add_argument("--property", help="The property of the stat to be cleared.")
|
||||
|
||||
@override
|
||||
def handle(self, *args: Any, **options: Any) -> None:
|
||||
property = options["property"]
|
||||
if property not in ALL_COUNT_STATS:
|
||||
raise CommandError(f"Invalid property: {property}")
|
||||
if not options["force"]:
|
||||
raise CommandError("No action taken. Use --force.")
|
||||
|
||||
do_drop_single_stat(property)
|
||||
353
analytics/management/commands/populate_analytics_db.py
Normal file
353
analytics/management/commands/populate_analytics_db.py
Normal file
@@ -0,0 +1,353 @@
|
||||
from collections.abc import Mapping
|
||||
from datetime import timedelta
|
||||
from typing import Any, TypeAlias
|
||||
|
||||
from django.core.files.uploadedfile import UploadedFile
|
||||
from django.utils.timezone import now as timezone_now
|
||||
from typing_extensions import override
|
||||
|
||||
from analytics.lib.counts import COUNT_STATS, CountStat, do_drop_all_analytics_tables
|
||||
from analytics.lib.fixtures import generate_time_series_data
|
||||
from analytics.lib.time_utils import time_range
|
||||
from analytics.models import (
|
||||
BaseCount,
|
||||
FillState,
|
||||
InstallationCount,
|
||||
RealmCount,
|
||||
StreamCount,
|
||||
UserCount,
|
||||
)
|
||||
from zerver.actions.create_realm import do_create_realm
|
||||
from zerver.lib.create_user import create_user
|
||||
from zerver.lib.management import ZulipBaseCommand
|
||||
from zerver.lib.storage import static_path
|
||||
from zerver.lib.stream_color import STREAM_ASSIGNMENT_COLORS
|
||||
from zerver.lib.stream_subscription import create_stream_subscription
|
||||
from zerver.lib.streams import get_default_values_for_stream_permission_group_settings
|
||||
from zerver.lib.timestamp import floor_to_day
|
||||
from zerver.lib.upload import upload_message_attachment_from_request
|
||||
from zerver.models import Attachment, Client, Realm, RealmAuditLog, Recipient, Stream, UserProfile
|
||||
from zerver.models.groups import NamedUserGroup, SystemGroups, UserGroupMembership
|
||||
from zerver.models.realm_audit_logs import AuditLogEventType
|
||||
|
||||
|
||||
class Command(ZulipBaseCommand):
|
||||
help = """Populates analytics tables with randomly generated data."""
|
||||
|
||||
DAYS_OF_DATA = 100
|
||||
random_seed = 26
|
||||
|
||||
def generate_fixture_data(
|
||||
self,
|
||||
stat: CountStat,
|
||||
business_hours_base: float,
|
||||
non_business_hours_base: float,
|
||||
growth: float,
|
||||
autocorrelation: float,
|
||||
spikiness: float,
|
||||
holiday_rate: float = 0,
|
||||
partial_sum: bool = False,
|
||||
) -> list[int]:
|
||||
self.random_seed += 1
|
||||
return generate_time_series_data(
|
||||
days=self.DAYS_OF_DATA,
|
||||
business_hours_base=business_hours_base,
|
||||
non_business_hours_base=non_business_hours_base,
|
||||
growth=growth,
|
||||
autocorrelation=autocorrelation,
|
||||
spikiness=spikiness,
|
||||
holiday_rate=holiday_rate,
|
||||
frequency=stat.frequency,
|
||||
partial_sum=partial_sum,
|
||||
random_seed=self.random_seed,
|
||||
)
|
||||
|
||||
@override
|
||||
def handle(self, *args: Any, **options: Any) -> None:
|
||||
# TODO: This should arguably only delete the objects
|
||||
# associated with the "analytics" realm.
|
||||
do_drop_all_analytics_tables()
|
||||
|
||||
# Delete attachment added for summary statistic before deleting
|
||||
# the Realm object.
|
||||
Attachment.objects.filter(realm__string_id="analytics").delete()
|
||||
|
||||
# This also deletes any objects with this realm as a foreign key
|
||||
Realm.objects.filter(string_id="analytics").delete()
|
||||
|
||||
# Because we just deleted a bunch of objects in the database
|
||||
# directly (rather than deleting individual objects in Django,
|
||||
# in which case our post_save hooks would have flushed the
|
||||
# individual objects from memcached for us), we need to flush
|
||||
# memcached in order to ensure deleted objects aren't still
|
||||
# present in the memcached cache.
|
||||
from zerver.apps import flush_cache
|
||||
|
||||
flush_cache(None)
|
||||
|
||||
installation_time = timezone_now() - timedelta(days=self.DAYS_OF_DATA)
|
||||
last_end_time = floor_to_day(timezone_now())
|
||||
realm = do_create_realm(
|
||||
string_id="analytics", name="Analytics", date_created=installation_time
|
||||
)
|
||||
|
||||
owners_system_group = NamedUserGroup.objects.get(
|
||||
name=SystemGroups.OWNERS, realm_for_sharding=realm, is_system_group=True
|
||||
)
|
||||
guests_system_group = NamedUserGroup.objects.get(
|
||||
name=SystemGroups.EVERYONE, realm_for_sharding=realm, is_system_group=True
|
||||
)
|
||||
|
||||
shylock = create_user(
|
||||
"shylock@analytics.ds",
|
||||
"Shylock",
|
||||
realm,
|
||||
full_name="Shylock",
|
||||
role=UserProfile.ROLE_REALM_OWNER,
|
||||
force_date_joined=installation_time,
|
||||
)
|
||||
UserGroupMembership.objects.create(user_profile=shylock, user_group=owners_system_group)
|
||||
|
||||
# Create guest user for set_guest_users_statistic.
|
||||
bassanio = create_user(
|
||||
"bassanio@analytics.ds",
|
||||
"Bassanio",
|
||||
realm,
|
||||
full_name="Bassanio",
|
||||
role=UserProfile.ROLE_GUEST,
|
||||
force_date_joined=installation_time,
|
||||
)
|
||||
UserGroupMembership.objects.create(user_profile=bassanio, user_group=guests_system_group)
|
||||
|
||||
stream = Stream.objects.create(
|
||||
name="all",
|
||||
realm=realm,
|
||||
date_created=installation_time,
|
||||
**get_default_values_for_stream_permission_group_settings(realm),
|
||||
)
|
||||
recipient = Recipient.objects.create(type_id=stream.id, type=Recipient.STREAM)
|
||||
stream.recipient = recipient
|
||||
stream.save(update_fields=["recipient"])
|
||||
|
||||
# Subscribe shylock to the stream to avoid invariant failures.
|
||||
create_stream_subscription(
|
||||
user_profile=shylock,
|
||||
recipient=recipient,
|
||||
stream=stream,
|
||||
color=STREAM_ASSIGNMENT_COLORS[0],
|
||||
)
|
||||
RealmAuditLog.objects.create(
|
||||
realm=realm,
|
||||
modified_user=shylock,
|
||||
modified_stream=stream,
|
||||
event_last_message_id=0,
|
||||
event_type=AuditLogEventType.SUBSCRIPTION_CREATED,
|
||||
event_time=installation_time,
|
||||
)
|
||||
|
||||
# Create an attachment in the database for set_storage_space_used_statistic.
|
||||
IMAGE_FILE_PATH = static_path("images/test-images/checkbox.png")
|
||||
with open(IMAGE_FILE_PATH, "rb") as fp:
|
||||
upload_message_attachment_from_request(UploadedFile(fp), shylock)
|
||||
|
||||
FixtureData: TypeAlias = Mapping[str | int | None, list[int]]
|
||||
|
||||
def insert_fixture_data(
|
||||
stat: CountStat,
|
||||
fixture_data: FixtureData,
|
||||
table: type[BaseCount],
|
||||
) -> None:
|
||||
end_times = time_range(
|
||||
last_end_time, last_end_time, stat.frequency, len(next(iter(fixture_data.values())))
|
||||
)
|
||||
id_args: dict[str, Realm | UserProfile | Stream] = {}
|
||||
if table == RealmCount:
|
||||
id_args = {"realm": realm}
|
||||
if table == UserCount:
|
||||
id_args = {"realm": realm, "user": shylock}
|
||||
if table == StreamCount:
|
||||
id_args = {"realm": realm, "stream": stream}
|
||||
|
||||
for subgroup, values in fixture_data.items():
|
||||
table._default_manager.bulk_create(
|
||||
table(
|
||||
property=stat.property,
|
||||
subgroup=subgroup,
|
||||
end_time=end_time,
|
||||
value=value,
|
||||
**id_args,
|
||||
)
|
||||
for end_time, value in zip(end_times, values, strict=False)
|
||||
if value != 0
|
||||
)
|
||||
|
||||
stat = COUNT_STATS["1day_actives::day"]
|
||||
realm_data: FixtureData = {
|
||||
None: self.generate_fixture_data(stat, 0.08, 0.02, 3, 0.3, 6, partial_sum=True),
|
||||
}
|
||||
insert_fixture_data(stat, realm_data, RealmCount)
|
||||
installation_data: FixtureData = {
|
||||
None: self.generate_fixture_data(stat, 0.8, 0.2, 4, 0.3, 6, partial_sum=True),
|
||||
}
|
||||
insert_fixture_data(stat, installation_data, InstallationCount)
|
||||
FillState.objects.create(
|
||||
property=stat.property, end_time=last_end_time, state=FillState.DONE
|
||||
)
|
||||
|
||||
stat = COUNT_STATS["7day_actives::day"]
|
||||
realm_data = {
|
||||
None: self.generate_fixture_data(stat, 0.2, 0.07, 3, 0.3, 6, partial_sum=True),
|
||||
}
|
||||
insert_fixture_data(stat, realm_data, RealmCount)
|
||||
installation_data = {
|
||||
None: self.generate_fixture_data(stat, 2, 0.7, 4, 0.3, 6, partial_sum=True),
|
||||
}
|
||||
insert_fixture_data(stat, installation_data, InstallationCount)
|
||||
FillState.objects.create(
|
||||
property=stat.property, end_time=last_end_time, state=FillState.DONE
|
||||
)
|
||||
|
||||
stat = COUNT_STATS["realm_active_humans::day"]
|
||||
realm_data = {
|
||||
None: self.generate_fixture_data(stat, 0.8, 0.08, 3, 0.5, 3, partial_sum=True),
|
||||
}
|
||||
insert_fixture_data(stat, realm_data, RealmCount)
|
||||
installation_data = {
|
||||
None: self.generate_fixture_data(stat, 1, 0.3, 4, 0.5, 3, partial_sum=True),
|
||||
}
|
||||
insert_fixture_data(stat, installation_data, InstallationCount)
|
||||
FillState.objects.create(
|
||||
property=stat.property, end_time=last_end_time, state=FillState.DONE
|
||||
)
|
||||
|
||||
stat = COUNT_STATS["active_users_audit:is_bot:day"]
|
||||
realm_data = {
|
||||
"false": self.generate_fixture_data(stat, 1, 0.2, 3.5, 0.8, 2, partial_sum=True),
|
||||
"true": self.generate_fixture_data(stat, 0.3, 0.05, 3, 0.3, 2, partial_sum=True),
|
||||
}
|
||||
insert_fixture_data(stat, realm_data, RealmCount)
|
||||
installation_data = {
|
||||
"false": self.generate_fixture_data(stat, 3, 1, 4, 0.8, 2, partial_sum=True),
|
||||
"true": self.generate_fixture_data(stat, 1, 0.4, 4, 0.8, 2, partial_sum=True),
|
||||
}
|
||||
insert_fixture_data(stat, installation_data, InstallationCount)
|
||||
FillState.objects.create(
|
||||
property=stat.property, end_time=last_end_time, state=FillState.DONE
|
||||
)
|
||||
|
||||
stat = COUNT_STATS["messages_sent:is_bot:hour"]
|
||||
user_data: FixtureData = {
|
||||
"false": self.generate_fixture_data(stat, 2, 1, 1.5, 0.6, 8, holiday_rate=0.1),
|
||||
}
|
||||
insert_fixture_data(stat, user_data, UserCount)
|
||||
realm_data = {
|
||||
"false": self.generate_fixture_data(stat, 35, 15, 6, 0.6, 4),
|
||||
"true": self.generate_fixture_data(stat, 15, 15, 3, 0.4, 2),
|
||||
}
|
||||
insert_fixture_data(stat, realm_data, RealmCount)
|
||||
installation_data = {
|
||||
"false": self.generate_fixture_data(stat, 350, 150, 6, 0.6, 4),
|
||||
"true": self.generate_fixture_data(stat, 150, 150, 3, 0.4, 2),
|
||||
}
|
||||
insert_fixture_data(stat, installation_data, InstallationCount)
|
||||
FillState.objects.create(
|
||||
property=stat.property, end_time=last_end_time, state=FillState.DONE
|
||||
)
|
||||
|
||||
stat = COUNT_STATS["messages_sent:message_type:day"]
|
||||
user_data = {
|
||||
"public_stream": self.generate_fixture_data(stat, 1.5, 1, 3, 0.6, 8),
|
||||
"private_message": self.generate_fixture_data(stat, 0.5, 0.3, 1, 0.6, 8),
|
||||
"huddle_message": self.generate_fixture_data(stat, 0.2, 0.2, 2, 0.6, 8),
|
||||
}
|
||||
insert_fixture_data(stat, user_data, UserCount)
|
||||
realm_data = {
|
||||
"public_stream": self.generate_fixture_data(stat, 30, 8, 5, 0.6, 4),
|
||||
"private_stream": self.generate_fixture_data(stat, 7, 7, 5, 0.6, 4),
|
||||
"private_message": self.generate_fixture_data(stat, 13, 5, 5, 0.6, 4),
|
||||
"huddle_message": self.generate_fixture_data(stat, 6, 3, 3, 0.6, 4),
|
||||
}
|
||||
insert_fixture_data(stat, realm_data, RealmCount)
|
||||
installation_data = {
|
||||
"public_stream": self.generate_fixture_data(stat, 300, 80, 5, 0.6, 4),
|
||||
"private_stream": self.generate_fixture_data(stat, 70, 70, 5, 0.6, 4),
|
||||
"private_message": self.generate_fixture_data(stat, 130, 50, 5, 0.6, 4),
|
||||
"huddle_message": self.generate_fixture_data(stat, 60, 30, 3, 0.6, 4),
|
||||
}
|
||||
insert_fixture_data(stat, installation_data, InstallationCount)
|
||||
FillState.objects.create(
|
||||
property=stat.property, end_time=last_end_time, state=FillState.DONE
|
||||
)
|
||||
|
||||
website, _created = Client.objects.get_or_create(name="website")
|
||||
old_desktop, _created = Client.objects.get_or_create(name="desktop app Linux 0.3.7")
|
||||
android, _created = Client.objects.get_or_create(name="ZulipAndroid")
|
||||
iOS, _created = Client.objects.get_or_create(name="ZulipiOS")
|
||||
react_native, _created = Client.objects.get_or_create(name="ZulipMobile")
|
||||
flutter, _created = Client.objects.get_or_create(name="ZulipFlutter")
|
||||
API, _created = Client.objects.get_or_create(name="API: Python")
|
||||
irc_mirror, _created = Client.objects.get_or_create(name="irc_mirror")
|
||||
unused, _created = Client.objects.get_or_create(name="unused")
|
||||
long_webhook, _created = Client.objects.get_or_create(name="ZulipLooooooooooongNameWebhook")
|
||||
|
||||
stat = COUNT_STATS["messages_sent:client:day"]
|
||||
user_data = {
|
||||
website.id: self.generate_fixture_data(stat, 2, 1, 1.5, 0.6, 8),
|
||||
irc_mirror.id: self.generate_fixture_data(stat, 0, 0.3, 1.5, 0.6, 8),
|
||||
}
|
||||
insert_fixture_data(stat, user_data, UserCount)
|
||||
realm_data = {
|
||||
website.id: self.generate_fixture_data(stat, 30, 20, 5, 0.6, 3),
|
||||
old_desktop.id: self.generate_fixture_data(stat, 5, 3, 8, 0.6, 3),
|
||||
android.id: self.generate_fixture_data(stat, 5, 5, 2, 0.6, 3),
|
||||
iOS.id: self.generate_fixture_data(stat, 5, 5, 2, 0.6, 3),
|
||||
react_native.id: self.generate_fixture_data(stat, 5, 5, 10, 0.6, 3),
|
||||
flutter.id: self.generate_fixture_data(stat, 5, 5, 10, 0.6, 3),
|
||||
API.id: self.generate_fixture_data(stat, 5, 5, 5, 0.6, 3),
|
||||
irc_mirror.id: self.generate_fixture_data(stat, 1, 1, 3, 0.6, 3),
|
||||
unused.id: self.generate_fixture_data(stat, 0, 0, 0, 0, 0),
|
||||
long_webhook.id: self.generate_fixture_data(stat, 5, 5, 2, 0.6, 3),
|
||||
}
|
||||
insert_fixture_data(stat, realm_data, RealmCount)
|
||||
installation_data = {
|
||||
website.id: self.generate_fixture_data(stat, 300, 200, 5, 0.6, 3),
|
||||
old_desktop.id: self.generate_fixture_data(stat, 50, 30, 8, 0.6, 3),
|
||||
android.id: self.generate_fixture_data(stat, 50, 50, 2, 0.6, 3),
|
||||
iOS.id: self.generate_fixture_data(stat, 50, 50, 2, 0.6, 3),
|
||||
flutter.id: self.generate_fixture_data(stat, 5, 5, 10, 0.6, 3),
|
||||
react_native.id: self.generate_fixture_data(stat, 5, 5, 10, 0.6, 3),
|
||||
API.id: self.generate_fixture_data(stat, 50, 50, 5, 0.6, 3),
|
||||
irc_mirror.id: self.generate_fixture_data(stat, 10, 10, 3, 0.6, 3),
|
||||
unused.id: self.generate_fixture_data(stat, 0, 0, 0, 0, 0),
|
||||
long_webhook.id: self.generate_fixture_data(stat, 50, 50, 2, 0.6, 3),
|
||||
}
|
||||
insert_fixture_data(stat, installation_data, InstallationCount)
|
||||
FillState.objects.create(
|
||||
property=stat.property, end_time=last_end_time, state=FillState.DONE
|
||||
)
|
||||
|
||||
stat = COUNT_STATS["messages_in_stream:is_bot:day"]
|
||||
realm_data = {
|
||||
"false": self.generate_fixture_data(stat, 30, 5, 6, 0.6, 4),
|
||||
"true": self.generate_fixture_data(stat, 20, 2, 3, 0.2, 3),
|
||||
}
|
||||
insert_fixture_data(stat, realm_data, RealmCount)
|
||||
stream_data: Mapping[int | str | None, list[int]] = {
|
||||
"false": self.generate_fixture_data(stat, 10, 7, 5, 0.6, 4),
|
||||
"true": self.generate_fixture_data(stat, 5, 3, 2, 0.4, 2),
|
||||
}
|
||||
insert_fixture_data(stat, stream_data, StreamCount)
|
||||
FillState.objects.create(
|
||||
property=stat.property, end_time=last_end_time, state=FillState.DONE
|
||||
)
|
||||
|
||||
stat = COUNT_STATS["messages_read::hour"]
|
||||
user_data = {
|
||||
None: self.generate_fixture_data(stat, 7, 3, 2, 0.6, 8, holiday_rate=0.1),
|
||||
}
|
||||
insert_fixture_data(stat, user_data, UserCount)
|
||||
realm_data = {None: self.generate_fixture_data(stat, 50, 35, 6, 0.6, 4)}
|
||||
insert_fixture_data(stat, realm_data, RealmCount)
|
||||
FillState.objects.create(
|
||||
property=stat.property, end_time=last_end_time, state=FillState.DONE
|
||||
)
|
||||
99
analytics/management/commands/update_analytics_counts.py
Normal file
99
analytics/management/commands/update_analytics_counts.py
Normal file
@@ -0,0 +1,99 @@
|
||||
import hashlib
|
||||
import time
|
||||
from argparse import ArgumentParser
|
||||
from datetime import timezone
|
||||
from typing import Any
|
||||
|
||||
from django.conf import settings
|
||||
from django.utils.dateparse import parse_datetime
|
||||
from django.utils.timezone import now as timezone_now
|
||||
from typing_extensions import override
|
||||
|
||||
from analytics.lib.counts import ALL_COUNT_STATS, logger, process_count_stat
|
||||
from zerver.lib.management import ZulipBaseCommand, abort_cron_during_deploy, abort_unless_locked
|
||||
from zerver.lib.remote_server import send_server_data_to_push_bouncer, should_send_analytics_data
|
||||
from zerver.lib.timestamp import floor_to_hour
|
||||
from zerver.models import Realm
|
||||
|
||||
|
||||
class Command(ZulipBaseCommand):
|
||||
help = """Fills Analytics tables.
|
||||
|
||||
Run as a cron job that runs every hour."""
|
||||
|
||||
@override
|
||||
def add_arguments(self, parser: ArgumentParser) -> None:
|
||||
parser.add_argument(
|
||||
"--time",
|
||||
"-t",
|
||||
help="Update stat tables from current state to --time. Defaults to the current time.",
|
||||
default=timezone_now().isoformat(),
|
||||
)
|
||||
parser.add_argument("--utc", action="store_true", help="Interpret --time in UTC.")
|
||||
parser.add_argument(
|
||||
"--stat", "-s", help="CountStat to process. If omitted, all stats are processed."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--verbose", action="store_true", help="Print timing information to stdout."
|
||||
)
|
||||
|
||||
@override
|
||||
@abort_cron_during_deploy
|
||||
@abort_unless_locked
|
||||
def handle(self, *args: Any, **options: Any) -> None:
|
||||
self.run_update_analytics_counts(options)
|
||||
|
||||
def run_update_analytics_counts(self, options: dict[str, Any]) -> None:
|
||||
# installation_epoch relies on there being at least one realm; we
|
||||
# shouldn't run the analytics code if that condition isn't satisfied
|
||||
if not Realm.objects.exists():
|
||||
logger.info("No realms, stopping update_analytics_counts")
|
||||
return
|
||||
|
||||
fill_to_time = parse_datetime(options["time"])
|
||||
assert fill_to_time is not None
|
||||
if options["utc"]:
|
||||
fill_to_time = fill_to_time.replace(tzinfo=timezone.utc)
|
||||
if fill_to_time.tzinfo is None:
|
||||
raise ValueError(
|
||||
"--time must be time-zone-aware. Maybe you meant to use the --utc option?"
|
||||
)
|
||||
|
||||
fill_to_time = floor_to_hour(fill_to_time.astimezone(timezone.utc))
|
||||
|
||||
if options["stat"] is not None:
|
||||
stats = [ALL_COUNT_STATS[options["stat"]]]
|
||||
else:
|
||||
stats = list(ALL_COUNT_STATS.values())
|
||||
|
||||
logger.info("Starting updating analytics counts through %s", fill_to_time)
|
||||
if options["verbose"]:
|
||||
start = time.time()
|
||||
last = start
|
||||
|
||||
for stat in stats:
|
||||
process_count_stat(stat, fill_to_time)
|
||||
if options["verbose"]:
|
||||
print(f"Updated {stat.property} in {time.time() - last:.3f}s")
|
||||
last = time.time()
|
||||
|
||||
if options["verbose"]:
|
||||
print(
|
||||
f"Finished updating analytics counts through {fill_to_time} in {time.time() - start:.3f}s"
|
||||
)
|
||||
logger.info("Finished updating analytics counts through %s", fill_to_time)
|
||||
|
||||
if should_send_analytics_data():
|
||||
# Based on the specific value of the setting, the exact details to send
|
||||
# will be decided. However, we proceed just based on this not being falsey.
|
||||
|
||||
# Skew 0-10 minutes based on a hash of settings.ZULIP_ORG_ID, so
|
||||
# that each server will report in at a somewhat consistent time.
|
||||
assert settings.ZULIP_ORG_ID
|
||||
delay = int.from_bytes(
|
||||
hashlib.sha256(settings.ZULIP_ORG_ID.encode()).digest(), byteorder="big"
|
||||
) % (60 * 10)
|
||||
logger.info("Sleeping %d seconds before reporting...", delay)
|
||||
time.sleep(delay)
|
||||
|
||||
send_server_data_to_push_bouncer(consider_usage_statistics=True, raise_on_error=True)
|
||||
Reference in New Issue
Block a user