snapshot of https://github.com/zulip/zulip.git @ 48c69911c7dd4e03b20552eb3bd77ad8151c1357 (2026-09-15)

analytics/migrations/0001_initial.py

@@ -0,0 +1,208 @@
+import django.db.models.deletion
+from django.conf import settings
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("zerver", "0030_realm_org_type"),
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="Anomaly",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        verbose_name="ID", serialize=False, auto_created=True, primary_key=True
+                    ),
+                ),
+                ("info", models.CharField(max_length=1000)),
+            ],
+            bases=(models.Model,),
+        ),
+        migrations.CreateModel(
+            name="HuddleCount",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        verbose_name="ID", serialize=False, auto_created=True, primary_key=True
+                    ),
+                ),
+                (
+                    "huddle",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE, to="zerver.Recipient"
+                    ),
+                ),
+                (
+                    "user",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL
+                    ),
+                ),
+                ("property", models.CharField(max_length=40)),
+                ("end_time", models.DateTimeField()),
+                ("interval", models.CharField(max_length=20)),
+                ("value", models.BigIntegerField()),
+                (
+                    "anomaly",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        to="analytics.Anomaly",
+                        null=True,
+                    ),
+                ),
+            ],
+            bases=(models.Model,),
+        ),
+        migrations.CreateModel(
+            name="InstallationCount",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        verbose_name="ID", serialize=False, auto_created=True, primary_key=True
+                    ),
+                ),
+                ("property", models.CharField(max_length=40)),
+                ("end_time", models.DateTimeField()),
+                ("interval", models.CharField(max_length=20)),
+                ("value", models.BigIntegerField()),
+                (
+                    "anomaly",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        to="analytics.Anomaly",
+                        null=True,
+                    ),
+                ),
+            ],
+            bases=(models.Model,),
+        ),
+        migrations.CreateModel(
+            name="RealmCount",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        verbose_name="ID", serialize=False, auto_created=True, primary_key=True
+                    ),
+                ),
+                (
+                    "realm",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE, to="zerver.Realm"
+                    ),
+                ),
+                ("property", models.CharField(max_length=40)),
+                ("end_time", models.DateTimeField()),
+                ("interval", models.CharField(max_length=20)),
+                ("value", models.BigIntegerField()),
+                (
+                    "anomaly",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        to="analytics.Anomaly",
+                        null=True,
+                    ),
+                ),
+            ],
+            bases=(models.Model,),
+        ),
+        migrations.CreateModel(
+            name="StreamCount",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        verbose_name="ID", serialize=False, auto_created=True, primary_key=True
+                    ),
+                ),
+                (
+                    "realm",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE, to="zerver.Realm"
+                    ),
+                ),
+                (
+                    "stream",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE, to="zerver.Stream"
+                    ),
+                ),
+                ("property", models.CharField(max_length=40)),
+                ("end_time", models.DateTimeField()),
+                ("interval", models.CharField(max_length=20)),
+                ("value", models.BigIntegerField()),
+                (
+                    "anomaly",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        to="analytics.Anomaly",
+                        null=True,
+                    ),
+                ),
+            ],
+            bases=(models.Model,),
+        ),
+        migrations.CreateModel(
+            name="UserCount",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        verbose_name="ID", serialize=False, auto_created=True, primary_key=True
+                    ),
+                ),
+                (
+                    "realm",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE, to="zerver.Realm"
+                    ),
+                ),
+                (
+                    "user",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL
+                    ),
+                ),
+                ("property", models.CharField(max_length=40)),
+                ("end_time", models.DateTimeField()),
+                ("interval", models.CharField(max_length=20)),
+                ("value", models.BigIntegerField()),
+                (
+                    "anomaly",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        to="analytics.Anomaly",
+                        null=True,
+                    ),
+                ),
+            ],
+            bases=(models.Model,),
+        ),
+        migrations.AlterUniqueTogether(
+            name="usercount",
+            unique_together={("user", "property", "end_time", "interval")},
+        ),
+        migrations.AlterUniqueTogether(
+            name="streamcount",
+            unique_together={("stream", "property", "end_time", "interval")},
+        ),
+        migrations.AlterUniqueTogether(
+            name="realmcount",
+            unique_together={("realm", "property", "end_time", "interval")},
+        ),
+        migrations.AlterUniqueTogether(
+            name="installationcount",
+            unique_together={("property", "end_time", "interval")},
+        ),
+        migrations.AlterUniqueTogether(
+            name="huddlecount",
+            unique_together={("huddle", "property", "end_time", "interval")},
+        ),
+    ]

analytics/migrations/0001_squashed_0021_alter_fillstate_id.py

@@ -0,0 +1,224 @@
+# Generated by Django 5.0.7 on 2024-08-13 20:16
+
+import django.db.models.deletion
+from django.conf import settings
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    replaces = [
+        ("analytics", "0001_initial"),
+        ("analytics", "0002_remove_huddlecount"),
+        ("analytics", "0003_fillstate"),
+        ("analytics", "0004_add_subgroup"),
+        ("analytics", "0005_alter_field_size"),
+        ("analytics", "0006_add_subgroup_to_unique_constraints"),
+        ("analytics", "0007_remove_interval"),
+        ("analytics", "0008_add_count_indexes"),
+        ("analytics", "0009_remove_messages_to_stream_stat"),
+        ("analytics", "0010_clear_messages_sent_values"),
+        ("analytics", "0011_clear_analytics_tables"),
+        ("analytics", "0012_add_on_delete"),
+        ("analytics", "0013_remove_anomaly"),
+        ("analytics", "0014_remove_fillstate_last_modified"),
+        ("analytics", "0015_clear_duplicate_counts"),
+        ("analytics", "0016_unique_constraint_when_subgroup_null"),
+        ("analytics", "0017_regenerate_partial_indexes"),
+        ("analytics", "0018_remove_usercount_active_users_audit"),
+        ("analytics", "0019_remove_unused_counts"),
+        ("analytics", "0020_alter_installationcount_id_alter_realmcount_id_and_more"),
+        ("analytics", "0021_alter_fillstate_id"),
+    ]
+
+    initial = True
+
+    dependencies = [
+        # Needed for foreign keys to core models like Realm.
+        ("zerver", "0001_initial"),
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="InstallationCount",
+            fields=[
+                (
+                    "id",
+                    models.BigAutoField(
+                        auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
+                    ),
+                ),
+                ("property", models.CharField(max_length=32)),
+                ("end_time", models.DateTimeField()),
+                ("value", models.BigIntegerField()),
+                ("subgroup", models.CharField(max_length=16, null=True)),
+            ],
+            options={
+                "unique_together": set(),
+                "constraints": [
+                    models.UniqueConstraint(
+                        condition=models.Q(("subgroup__isnull", False)),
+                        fields=("property", "subgroup", "end_time"),
+                        name="unique_installation_count",
+                    ),
+                    models.UniqueConstraint(
+                        condition=models.Q(("subgroup__isnull", True)),
+                        fields=("property", "end_time"),
+                        name="unique_installation_count_null_subgroup",
+                    ),
+                ],
+            },
+        ),
+        migrations.CreateModel(
+            name="RealmCount",
+            fields=[
+                (
+                    "id",
+                    models.BigAutoField(
+                        auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
+                    ),
+                ),
+                (
+                    "realm",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE, to="zerver.realm"
+                    ),
+                ),
+                ("property", models.CharField(max_length=32)),
+                ("end_time", models.DateTimeField()),
+                ("value", models.BigIntegerField()),
+                ("subgroup", models.CharField(max_length=16, null=True)),
+            ],
+            options={
+                "indexes": [
+                    models.Index(
+                        fields=["property", "end_time"],
+                        name="analytics_realmcount_property_end_time_3b60396b_idx",
+                    )
+                ],
+                "unique_together": set(),
+                "constraints": [
+                    models.UniqueConstraint(
+                        condition=models.Q(("subgroup__isnull", False)),
+                        fields=("realm", "property", "subgroup", "end_time"),
+                        name="unique_realm_count",
+                    ),
+                    models.UniqueConstraint(
+                        condition=models.Q(("subgroup__isnull", True)),
+                        fields=("realm", "property", "end_time"),
+                        name="unique_realm_count_null_subgroup",
+                    ),
+                ],
+            },
+        ),
+        migrations.CreateModel(
+            name="StreamCount",
+            fields=[
+                (
+                    "id",
+                    models.BigAutoField(
+                        auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
+                    ),
+                ),
+                (
+                    "realm",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE, to="zerver.realm"
+                    ),
+                ),
+                (
+                    "stream",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE, to="zerver.stream"
+                    ),
+                ),
+                ("property", models.CharField(max_length=32)),
+                ("end_time", models.DateTimeField()),
+                ("value", models.BigIntegerField()),
+                ("subgroup", models.CharField(max_length=16, null=True)),
+            ],
+            options={
+                "indexes": [
+                    models.Index(
+                        fields=["property", "realm", "end_time"],
+                        name="analytics_streamcount_property_realm_id_end_time_155ae930_idx",
+                    )
+                ],
+                "unique_together": set(),
+                "constraints": [
+                    models.UniqueConstraint(
+                        condition=models.Q(("subgroup__isnull", False)),
+                        fields=("stream", "property", "subgroup", "end_time"),
+                        name="unique_stream_count",
+                    ),
+                    models.UniqueConstraint(
+                        condition=models.Q(("subgroup__isnull", True)),
+                        fields=("stream", "property", "end_time"),
+                        name="unique_stream_count_null_subgroup",
+                    ),
+                ],
+            },
+        ),
+        migrations.CreateModel(
+            name="UserCount",
+            fields=[
+                (
+                    "id",
+                    models.BigAutoField(
+                        auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
+                    ),
+                ),
+                (
+                    "realm",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE, to="zerver.realm"
+                    ),
+                ),
+                (
+                    "user",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL
+                    ),
+                ),
+                ("property", models.CharField(max_length=32)),
+                ("end_time", models.DateTimeField()),
+                ("value", models.BigIntegerField()),
+                ("subgroup", models.CharField(max_length=16, null=True)),
+            ],
+            options={
+                "indexes": [
+                    models.Index(
+                        fields=["property", "realm", "end_time"],
+                        name="analytics_usercount_property_realm_id_end_time_591dbec1_idx",
+                    )
+                ],
+                "unique_together": set(),
+                "constraints": [
+                    models.UniqueConstraint(
+                        condition=models.Q(("subgroup__isnull", False)),
+                        fields=("user", "property", "subgroup", "end_time"),
+                        name="unique_user_count",
+                    ),
+                    models.UniqueConstraint(
+                        condition=models.Q(("subgroup__isnull", True)),
+                        fields=("user", "property", "end_time"),
+                        name="unique_user_count_null_subgroup",
+                    ),
+                ],
+            },
+        ),
+        migrations.CreateModel(
+            name="FillState",
+            fields=[
+                (
+                    "id",
+                    models.BigAutoField(
+                        auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
+                    ),
+                ),
+                ("property", models.CharField(max_length=40, unique=True)),
+                ("end_time", models.DateTimeField()),
+                ("state", models.PositiveSmallIntegerField()),
+            ],
+        ),
+    ]

analytics/migrations/0002_remove_huddlecount.py

@@ -0,0 +1,29 @@
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("analytics", "0001_initial"),
+    ]
+
+    operations = [
+        migrations.AlterUniqueTogether(
+            name="huddlecount",
+            unique_together=set(),
+        ),
+        migrations.RemoveField(
+            model_name="huddlecount",
+            name="anomaly",
+        ),
+        migrations.RemoveField(
+            model_name="huddlecount",
+            name="huddle",
+        ),
+        migrations.RemoveField(
+            model_name="huddlecount",
+            name="user",
+        ),
+        migrations.DeleteModel(
+            name="HuddleCount",
+        ),
+    ]

analytics/migrations/0003_fillstate.py

@@ -0,0 +1,26 @@
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("analytics", "0002_remove_huddlecount"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="FillState",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        verbose_name="ID", serialize=False, auto_created=True, primary_key=True
+                    ),
+                ),
+                ("property", models.CharField(unique=True, max_length=40)),
+                ("end_time", models.DateTimeField()),
+                ("state", models.PositiveSmallIntegerField()),
+                ("last_modified", models.DateTimeField(auto_now=True)),
+            ],
+            bases=(models.Model,),
+        ),
+    ]

analytics/migrations/0004_add_subgroup.py

@@ -0,0 +1,30 @@
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("analytics", "0003_fillstate"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="installationcount",
+            name="subgroup",
+            field=models.CharField(max_length=16, null=True),
+        ),
+        migrations.AddField(
+            model_name="realmcount",
+            name="subgroup",
+            field=models.CharField(max_length=16, null=True),
+        ),
+        migrations.AddField(
+            model_name="streamcount",
+            name="subgroup",
+            field=models.CharField(max_length=16, null=True),
+        ),
+        migrations.AddField(
+            model_name="usercount",
+            name="subgroup",
+            field=models.CharField(max_length=16, null=True),
+        ),
+    ]

analytics/migrations/0005_alter_field_size.py

@@ -0,0 +1,50 @@
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("analytics", "0004_add_subgroup"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="installationcount",
+            name="interval",
+            field=models.CharField(max_length=8),
+        ),
+        migrations.AlterField(
+            model_name="installationcount",
+            name="property",
+            field=models.CharField(max_length=32),
+        ),
+        migrations.AlterField(
+            model_name="realmcount",
+            name="interval",
+            field=models.CharField(max_length=8),
+        ),
+        migrations.AlterField(
+            model_name="realmcount",
+            name="property",
+            field=models.CharField(max_length=32),
+        ),
+        migrations.AlterField(
+            model_name="streamcount",
+            name="interval",
+            field=models.CharField(max_length=8),
+        ),
+        migrations.AlterField(
+            model_name="streamcount",
+            name="property",
+            field=models.CharField(max_length=32),
+        ),
+        migrations.AlterField(
+            model_name="usercount",
+            name="interval",
+            field=models.CharField(max_length=8),
+        ),
+        migrations.AlterField(
+            model_name="usercount",
+            name="property",
+            field=models.CharField(max_length=32),
+        ),
+    ]

analytics/migrations/0006_add_subgroup_to_unique_constraints.py

@@ -0,0 +1,26 @@
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("analytics", "0005_alter_field_size"),
+    ]
+
+    operations = [
+        migrations.AlterUniqueTogether(
+            name="installationcount",
+            unique_together={("property", "subgroup", "end_time", "interval")},
+        ),
+        migrations.AlterUniqueTogether(
+            name="realmcount",
+            unique_together={("realm", "property", "subgroup", "end_time", "interval")},
+        ),
+        migrations.AlterUniqueTogether(
+            name="streamcount",
+            unique_together={("stream", "property", "subgroup", "end_time", "interval")},
+        ),
+        migrations.AlterUniqueTogether(
+            name="usercount",
+            unique_together={("user", "property", "subgroup", "end_time", "interval")},
+        ),
+    ]

analytics/migrations/0007_remove_interval.py

@@ -0,0 +1,43 @@
+# Generated by Django 1.10.4 on 2017-01-16 20:50
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("analytics", "0006_add_subgroup_to_unique_constraints"),
+    ]
+
+    operations = [
+        migrations.AlterUniqueTogether(
+            name="installationcount",
+            unique_together={("property", "subgroup", "end_time")},
+        ),
+        migrations.RemoveField(
+            model_name="installationcount",
+            name="interval",
+        ),
+        migrations.AlterUniqueTogether(
+            name="realmcount",
+            unique_together={("realm", "property", "subgroup", "end_time")},
+        ),
+        migrations.RemoveField(
+            model_name="realmcount",
+            name="interval",
+        ),
+        migrations.AlterUniqueTogether(
+            name="streamcount",
+            unique_together={("stream", "property", "subgroup", "end_time")},
+        ),
+        migrations.RemoveField(
+            model_name="streamcount",
+            name="interval",
+        ),
+        migrations.AlterUniqueTogether(
+            name="usercount",
+            unique_together={("user", "property", "subgroup", "end_time")},
+        ),
+        migrations.RemoveField(
+            model_name="usercount",
+            name="interval",
+        ),
+    ]

analytics/migrations/0008_add_count_indexes.py

@@ -0,0 +1,33 @@
+# Generated by Django 1.10.5 on 2017-02-01 22:28
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("zerver", "0050_userprofile_avatar_version"),
+        ("analytics", "0007_remove_interval"),
+    ]
+
+    operations = [
+        migrations.AddIndex(
+            model_name="realmcount",
+            index=models.Index(
+                fields=["property", "end_time"],
+                name="analytics_realmcount_property_end_time_3b60396b_idx",
+            ),
+        ),
+        migrations.AddIndex(
+            model_name="streamcount",
+            index=models.Index(
+                fields=["property", "realm", "end_time"],
+                name="analytics_streamcount_property_realm_id_end_time_155ae930_idx",
+            ),
+        ),
+        migrations.AddIndex(
+            model_name="usercount",
+            index=models.Index(
+                fields=["property", "realm", "end_time"],
+                name="analytics_usercount_property_realm_id_end_time_591dbec1_idx",
+            ),
+        ),
+    ]

analytics/migrations/0009_remove_messages_to_stream_stat.py

@@ -0,0 +1,30 @@
+from django.db import migrations
+from django.db.backends.base.schema import BaseDatabaseSchemaEditor
+from django.db.migrations.state import StateApps
+
+
+def delete_messages_sent_to_stream_stat(
+    apps: StateApps, schema_editor: BaseDatabaseSchemaEditor
+) -> None:
+    UserCount = apps.get_model("analytics", "UserCount")
+    StreamCount = apps.get_model("analytics", "StreamCount")
+    RealmCount = apps.get_model("analytics", "RealmCount")
+    InstallationCount = apps.get_model("analytics", "InstallationCount")
+    FillState = apps.get_model("analytics", "FillState")
+
+    property = "messages_sent_to_stream:is_bot"
+    UserCount.objects.filter(property=property).delete()
+    StreamCount.objects.filter(property=property).delete()
+    RealmCount.objects.filter(property=property).delete()
+    InstallationCount.objects.filter(property=property).delete()
+    FillState.objects.filter(property=property).delete()
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("analytics", "0008_add_count_indexes"),
+    ]
+
+    operations = [
+        migrations.RunPython(delete_messages_sent_to_stream_stat, elidable=True),
+    ]

analytics/migrations/0010_clear_messages_sent_values.py

@@ -0,0 +1,28 @@
+from django.db import migrations
+from django.db.backends.base.schema import BaseDatabaseSchemaEditor
+from django.db.migrations.state import StateApps
+
+
+def clear_message_sent_by_message_type_values(
+    apps: StateApps, schema_editor: BaseDatabaseSchemaEditor
+) -> None:
+    UserCount = apps.get_model("analytics", "UserCount")
+    StreamCount = apps.get_model("analytics", "StreamCount")
+    RealmCount = apps.get_model("analytics", "RealmCount")
+    InstallationCount = apps.get_model("analytics", "InstallationCount")
+    FillState = apps.get_model("analytics", "FillState")
+
+    property = "messages_sent:message_type:day"
+    UserCount.objects.filter(property=property).delete()
+    StreamCount.objects.filter(property=property).delete()
+    RealmCount.objects.filter(property=property).delete()
+    InstallationCount.objects.filter(property=property).delete()
+    FillState.objects.filter(property=property).delete()
+
+
+class Migration(migrations.Migration):
+    dependencies = [("analytics", "0009_remove_messages_to_stream_stat")]
+
+    operations = [
+        migrations.RunPython(clear_message_sent_by_message_type_values, elidable=True),
+    ]

analytics/migrations/0011_clear_analytics_tables.py

@@ -0,0 +1,27 @@
+from django.db import migrations
+from django.db.backends.base.schema import BaseDatabaseSchemaEditor
+from django.db.migrations.state import StateApps
+
+
+def clear_analytics_tables(apps: StateApps, schema_editor: BaseDatabaseSchemaEditor) -> None:
+    UserCount = apps.get_model("analytics", "UserCount")
+    StreamCount = apps.get_model("analytics", "StreamCount")
+    RealmCount = apps.get_model("analytics", "RealmCount")
+    InstallationCount = apps.get_model("analytics", "InstallationCount")
+    FillState = apps.get_model("analytics", "FillState")
+
+    UserCount.objects.all().delete()
+    StreamCount.objects.all().delete()
+    RealmCount.objects.all().delete()
+    InstallationCount.objects.all().delete()
+    FillState.objects.all().delete()
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("analytics", "0010_clear_messages_sent_values"),
+    ]
+
+    operations = [
+        migrations.RunPython(clear_analytics_tables, elidable=True),
+    ]

analytics/migrations/0012_add_on_delete.py

@@ -0,0 +1,41 @@
+# Generated by Django 1.11.6 on 2018-01-29 08:14
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("analytics", "0011_clear_analytics_tables"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="installationcount",
+            name="anomaly",
+            field=models.ForeignKey(
+                null=True, on_delete=django.db.models.deletion.SET_NULL, to="analytics.Anomaly"
+            ),
+        ),
+        migrations.AlterField(
+            model_name="realmcount",
+            name="anomaly",
+            field=models.ForeignKey(
+                null=True, on_delete=django.db.models.deletion.SET_NULL, to="analytics.Anomaly"
+            ),
+        ),
+        migrations.AlterField(
+            model_name="streamcount",
+            name="anomaly",
+            field=models.ForeignKey(
+                null=True, on_delete=django.db.models.deletion.SET_NULL, to="analytics.Anomaly"
+            ),
+        ),
+        migrations.AlterField(
+            model_name="usercount",
+            name="anomaly",
+            field=models.ForeignKey(
+                null=True, on_delete=django.db.models.deletion.SET_NULL, to="analytics.Anomaly"
+            ),
+        ),
+    ]

analytics/migrations/0013_remove_anomaly.py

@@ -0,0 +1,31 @@
+# Generated by Django 1.11.18 on 2019-02-02 02:47
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("analytics", "0012_add_on_delete"),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name="installationcount",
+            name="anomaly",
+        ),
+        migrations.RemoveField(
+            model_name="realmcount",
+            name="anomaly",
+        ),
+        migrations.RemoveField(
+            model_name="streamcount",
+            name="anomaly",
+        ),
+        migrations.RemoveField(
+            model_name="usercount",
+            name="anomaly",
+        ),
+        migrations.DeleteModel(
+            name="Anomaly",
+        ),
+    ]

analytics/migrations/0014_remove_fillstate_last_modified.py

@@ -0,0 +1,16 @@
+# Generated by Django 1.11.26 on 2020-01-27 04:32
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("analytics", "0013_remove_anomaly"),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name="fillstate",
+            name="last_modified",
+        ),
+    ]

analytics/migrations/0015_clear_duplicate_counts.py

@@ -0,0 +1,66 @@
+from django.db import migrations
+from django.db.backends.base.schema import BaseDatabaseSchemaEditor
+from django.db.migrations.state import StateApps
+from django.db.models import Count, Sum
+
+
+def clear_duplicate_counts(apps: StateApps, schema_editor: BaseDatabaseSchemaEditor) -> None:
+    """This is a preparatory migration for our Analytics tables.
+
+    The backstory is that Django's unique_together indexes do not properly
+    handle the subgroup=None corner case (allowing duplicate rows that have a
+    subgroup of None), which meant that in race conditions, rather than updating
+    an existing row for the property/(realm, stream, user)/time with subgroup=None, Django would
+    create a duplicate row.
+
+    In the next migration, we'll add a proper constraint to fix this bug, but
+    we need to fix any existing problematic rows before we can add that constraint.
+
+    We fix this in an appropriate fashion for each type of CountStat object; mainly
+    this means deleting the extra rows, but for LoggingCountStat objects, we need to
+    additionally combine the sums.
+    """
+    count_tables = dict(
+        realm=apps.get_model("analytics", "RealmCount"),
+        user=apps.get_model("analytics", "UserCount"),
+        stream=apps.get_model("analytics", "StreamCount"),
+        installation=apps.get_model("analytics", "InstallationCount"),
+    )
+
+    for name, count_table in count_tables.items():
+        value = [name, "property", "end_time"]
+        if name == "installation":
+            value = ["property", "end_time"]
+        counts = (
+            count_table.objects.filter(subgroup=None)
+            .values(*value)
+            .annotate(Count("id"), Sum("value"))
+            .filter(id__count__gt=1)
+        )
+
+        for count in counts:
+            count.pop("id__count")
+            total_value = count.pop("value__sum")
+            duplicate_counts = list(count_table.objects.filter(**count))
+            first_count = duplicate_counts[0]
+            if count["property"] in ["invites_sent::day", "active_users_log:is_bot:day"]:
+                # For LoggingCountStat objects, the right fix is to combine the totals;
+                # for other CountStat objects, we expect the duplicates to have the same value.
+                # And so all we need to do is delete them.
+                first_count.value = total_value
+                first_count.save()
+            to_cleanup = duplicate_counts[1:]
+            for duplicate_count in to_cleanup:
+                duplicate_count.delete()
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("analytics", "0014_remove_fillstate_last_modified"),
+    ]
+
+    operations = [
+        migrations.RunPython(
+            clear_duplicate_counts, reverse_code=migrations.RunPython.noop, elidable=True
+        ),
+    ]

analytics/migrations/0016_unique_constraint_when_subgroup_null.py

@@ -0,0 +1,92 @@
+# Generated by Django 2.2.10 on 2020-02-29 19:40
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("analytics", "0015_clear_duplicate_counts"),
+    ]
+
+    operations = [
+        migrations.AlterUniqueTogether(
+            name="installationcount",
+            unique_together=set(),
+        ),
+        migrations.AlterUniqueTogether(
+            name="realmcount",
+            unique_together=set(),
+        ),
+        migrations.AlterUniqueTogether(
+            name="streamcount",
+            unique_together=set(),
+        ),
+        migrations.AlterUniqueTogether(
+            name="usercount",
+            unique_together=set(),
+        ),
+        migrations.AddConstraint(
+            model_name="installationcount",
+            constraint=models.UniqueConstraint(
+                condition=models.Q(subgroup__isnull=False),
+                fields=("property", "subgroup", "end_time"),
+                name="unique_installation_count",
+            ),
+        ),
+        migrations.AddConstraint(
+            model_name="installationcount",
+            constraint=models.UniqueConstraint(
+                condition=models.Q(subgroup__isnull=True),
+                fields=("property", "end_time"),
+                name="unique_installation_count_null_subgroup",
+            ),
+        ),
+        migrations.AddConstraint(
+            model_name="realmcount",
+            constraint=models.UniqueConstraint(
+                condition=models.Q(subgroup__isnull=False),
+                fields=("realm", "property", "subgroup", "end_time"),
+                name="unique_realm_count",
+            ),
+        ),
+        migrations.AddConstraint(
+            model_name="realmcount",
+            constraint=models.UniqueConstraint(
+                condition=models.Q(subgroup__isnull=True),
+                fields=("realm", "property", "end_time"),
+                name="unique_realm_count_null_subgroup",
+            ),
+        ),
+        migrations.AddConstraint(
+            model_name="streamcount",
+            constraint=models.UniqueConstraint(
+                condition=models.Q(subgroup__isnull=False),
+                fields=("stream", "property", "subgroup", "end_time"),
+                name="unique_stream_count",
+            ),
+        ),
+        migrations.AddConstraint(
+            model_name="streamcount",
+            constraint=models.UniqueConstraint(
+                condition=models.Q(subgroup__isnull=True),
+                fields=("stream", "property", "end_time"),
+                name="unique_stream_count_null_subgroup",
+            ),
+        ),
+        migrations.AddConstraint(
+            model_name="usercount",
+            constraint=models.UniqueConstraint(
+                condition=models.Q(subgroup__isnull=False),
+                fields=("user", "property", "subgroup", "end_time"),
+                name="unique_user_count",
+            ),
+        ),
+        migrations.AddConstraint(
+            model_name="usercount",
+            constraint=models.UniqueConstraint(
+                condition=models.Q(subgroup__isnull=True),
+                fields=("user", "property", "end_time"),
+                name="unique_user_count_null_subgroup",
+            ),
+        ),
+    ]

analytics/migrations/0017_regenerate_partial_indexes.py

@@ -0,0 +1,114 @@
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("analytics", "0016_unique_constraint_when_subgroup_null"),
+    ]
+
+    # If the server was installed between 7.0 and 7.4 (or main between
+    # 2c20028aa451 and 7807bff52635), it contains indexes which (when
+    # running 7.5 or 7807bff52635 or higher) are never used, because
+    # they contain an improper cast
+    # (https://code.djangoproject.com/ticket/34840).
+    #
+    # We regenerate the indexes here, by dropping and re-creating
+    # them, so that we know that they are properly formed.
+    operations = [
+        migrations.RemoveConstraint(
+            model_name="installationcount",
+            name="unique_installation_count",
+        ),
+        migrations.AddConstraint(
+            model_name="installationcount",
+            constraint=models.UniqueConstraint(
+                condition=models.Q(subgroup__isnull=False),
+                fields=("property", "subgroup", "end_time"),
+                name="unique_installation_count",
+            ),
+        ),
+        migrations.RemoveConstraint(
+            model_name="installationcount",
+            name="unique_installation_count_null_subgroup",
+        ),
+        migrations.AddConstraint(
+            model_name="installationcount",
+            constraint=models.UniqueConstraint(
+                condition=models.Q(subgroup__isnull=True),
+                fields=("property", "end_time"),
+                name="unique_installation_count_null_subgroup",
+            ),
+        ),
+        migrations.RemoveConstraint(
+            model_name="realmcount",
+            name="unique_realm_count",
+        ),
+        migrations.AddConstraint(
+            model_name="realmcount",
+            constraint=models.UniqueConstraint(
+                condition=models.Q(subgroup__isnull=False),
+                fields=("realm", "property", "subgroup", "end_time"),
+                name="unique_realm_count",
+            ),
+        ),
+        migrations.RemoveConstraint(
+            model_name="realmcount",
+            name="unique_realm_count_null_subgroup",
+        ),
+        migrations.AddConstraint(
+            model_name="realmcount",
+            constraint=models.UniqueConstraint(
+                condition=models.Q(subgroup__isnull=True),
+                fields=("realm", "property", "end_time"),
+                name="unique_realm_count_null_subgroup",
+            ),
+        ),
+        migrations.RemoveConstraint(
+            model_name="streamcount",
+            name="unique_stream_count",
+        ),
+        migrations.AddConstraint(
+            model_name="streamcount",
+            constraint=models.UniqueConstraint(
+                condition=models.Q(subgroup__isnull=False),
+                fields=("stream", "property", "subgroup", "end_time"),
+                name="unique_stream_count",
+            ),
+        ),
+        migrations.RemoveConstraint(
+            model_name="streamcount",
+            name="unique_stream_count_null_subgroup",
+        ),
+        migrations.AddConstraint(
+            model_name="streamcount",
+            constraint=models.UniqueConstraint(
+                condition=models.Q(subgroup__isnull=True),
+                fields=("stream", "property", "end_time"),
+                name="unique_stream_count_null_subgroup",
+            ),
+        ),
+        migrations.RemoveConstraint(
+            model_name="usercount",
+            name="unique_user_count",
+        ),
+        migrations.AddConstraint(
+            model_name="usercount",
+            constraint=models.UniqueConstraint(
+                condition=models.Q(subgroup__isnull=False),
+                fields=("user", "property", "subgroup", "end_time"),
+                name="unique_user_count",
+            ),
+        ),
+        migrations.RemoveConstraint(
+            model_name="usercount",
+            name="unique_user_count_null_subgroup",
+        ),
+        migrations.AddConstraint(
+            model_name="usercount",
+            constraint=models.UniqueConstraint(
+                condition=models.Q(subgroup__isnull=True),
+                fields=("user", "property", "end_time"),
+                name="unique_user_count_null_subgroup",
+            ),
+        ),
+    ]

analytics/migrations/0018_remove_usercount_active_users_audit.py

@@ -0,0 +1,16 @@
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+    elidable = True
+
+    dependencies = [
+        ("analytics", "0017_regenerate_partial_indexes"),
+    ]
+
+    operations = [
+        migrations.RunSQL(
+            "DELETE FROM analytics_usercount WHERE property = 'active_users_audit:is_bot:day'",
+            elidable=True,
+        )
+    ]

analytics/migrations/0019_remove_unused_counts.py

@@ -0,0 +1,27 @@
+from django.db import migrations
+
+REMOVED_COUNTS = (
+    "active_users_log:is_bot:day",
+    "active_users:is_bot:day",
+)
+
+
+class Migration(migrations.Migration):
+    elidable = True
+
+    dependencies = [
+        ("analytics", "0018_remove_usercount_active_users_audit"),
+    ]
+
+    operations = [
+        migrations.RunSQL(
+            [
+                ("DELETE FROM analytics_realmcount WHERE property IN %s", (REMOVED_COUNTS,)),
+                (
+                    "DELETE FROM analytics_installationcount WHERE property IN %s",
+                    (REMOVED_COUNTS,),
+                ),
+            ],
+            elidable=True,
+        )
+    ]

analytics/migrations/0020_alter_installationcount_id_alter_realmcount_id_and_more.py

@@ -0,0 +1,40 @@
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    atomic = False
+
+    dependencies = [
+        ("analytics", "0019_remove_unused_counts"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="installationcount",
+            name="id",
+            field=models.BigAutoField(
+                auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
+            ),
+        ),
+        migrations.AlterField(
+            model_name="realmcount",
+            name="id",
+            field=models.BigAutoField(
+                auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
+            ),
+        ),
+        migrations.AlterField(
+            model_name="streamcount",
+            name="id",
+            field=models.BigAutoField(
+                auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
+            ),
+        ),
+        migrations.AlterField(
+            model_name="usercount",
+            name="id",
+            field=models.BigAutoField(
+                auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
+            ),
+        ),
+    ]

analytics/migrations/0021_alter_fillstate_id.py

@@ -0,0 +1,17 @@
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("analytics", "0020_alter_installationcount_id_alter_realmcount_id_and_more"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="fillstate",
+            name="id",
+            field=models.BigAutoField(
+                auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
+            ),
+        ),
+    ]