snapshot of https://github.com/zulip/zulip.git @ 48c69911c7dd4e03b20552eb3bd77ad8151c1357 (2026-09-15)

zerver/models/linkifiers.py

@@ -0,0 +1,237 @@
+import re2
+import uri_template
+from django.core.exceptions import ValidationError
+from django.db import models
+from django.db.models import CASCADE
+from django.db.models.signals import post_delete, post_save
+from django.utils.translation import gettext as _
+from typing_extensions import override
+
+from zerver.lib import cache
+from zerver.lib.cache import cache_delete, cache_with_key
+from zerver.lib.exceptions import MissingDependentParameterError
+from zerver.lib.per_request_cache import cache_for_current_request, flush_per_request_cache
+from zerver.lib.types import LinkifierDict
+from zerver.models.realms import Realm
+
+
+def normalize_optional_text(value: str | None) -> str | None:
+    if value is None:
+        return None
+    value = value.strip()
+    return value or None
+
+
+# A function with the same name implements this on the JS side.
+# Please replicate any changes here to that function as well.
+def expand_reverse_template(
+    reverse_template: str, match: "re2._Match[str]", group_set: set[str]
+) -> str:
+    output: list[str] = []
+    index = 0
+    while index < len(reverse_template):
+        if reverse_template.startswith("{{", index):
+            output.append("{")
+            index += 2
+        elif reverse_template.startswith("}}", index):
+            output.append("}")
+            index += 2
+        elif reverse_template[index] == "{":
+            end_index = reverse_template.find("}", index + 1)
+            if end_index == -1:
+                raise ValidationError(_("Invalid auto-conversion field: missing '}' character."))
+            name = reverse_template[index + 1 : end_index]
+            if not name:
+                raise ValidationError(_("Invalid auto-conversion field: empty field name."))
+            if name not in group_set:
+                raise ValidationError(
+                    _(
+                        "Group %(name)r in auto-conversion field is not present in linkifier pattern."
+                    ),
+                    params={"name": name},
+                )
+            # For optional groups, match.group might return None, which we cannot
+            # append in the output to be .join(ed) later, so we add an empty string
+            # in that case.
+            output.append(match.group(name) or "")
+            index = end_index + 1
+        else:
+            output.append(reverse_template[index])
+            index += 1
+    return "".join(output)
+
+
+def filter_pattern_validator(value: str) -> "re2._Regexp[str]":
+    try:
+        # Do not write errors to stderr (this still raises exceptions)
+        options = re2.Options()
+        options.log_errors = False
+
+        regex = re2.compile(value, options=options)
+    except re2.error as e:
+        if len(e.args) >= 1:
+            if isinstance(e.args[0], str):  # nocoverage
+                raise ValidationError(_("Bad regular expression: {regex}").format(regex=e.args[0]))
+            if isinstance(e.args[0], bytes):
+                raise ValidationError(
+                    _("Bad regular expression: {regex}").format(regex=e.args[0].decode())
+                )
+        raise ValidationError(_("Unknown regular expression error"))  # nocoverage
+
+    return regex
+
+
+def url_template_validator(value: str) -> None:
+    """Validate as a URL template"""
+    if not uri_template.validate(value):
+        raise ValidationError(_("Invalid URL template."))
+
+
+class RealmFilter(models.Model):
+    """Realm-specific regular expressions to automatically linkify certain
+    strings inside the Markdown processor.  See "Custom filters" in the settings UI.
+    """
+
+    realm = models.ForeignKey(Realm, on_delete=CASCADE)
+    pattern = models.TextField()
+    url_template = models.TextField(validators=[url_template_validator])
+    example_input = models.TextField(blank=True, null=True)
+    reverse_template = models.TextField(blank=True, null=True)
+    alternative_url_templates = models.JSONField(default=list, blank=True)
+    # Linkifiers are applied in a message/topic in order; the processing order
+    # is important when there are overlapping patterns.
+    order = models.IntegerField(default=0)
+
+    class Meta:
+        unique_together = ("realm", "pattern")
+
+    @override
+    def __str__(self) -> str:
+        return f"{self.realm.string_id}: {self.pattern} {self.url_template}"
+
+    @override
+    def clean(self) -> None:
+        """Validate whether the set of parameters in the URL template
+        match the set of parameters in the regular expression.
+
+        Django's `full_clean` calls `clean_fields` followed by `clean` method
+        and stores all ValidationErrors from all stages to return as JSON.
+        """
+
+        # Extract variables present in the pattern
+        pattern = filter_pattern_validator(self.pattern)
+        group_set = set(pattern.groupindex.keys())
+
+        example_input = normalize_optional_text(self.example_input)
+        reverse_template = normalize_optional_text(self.reverse_template)
+
+        if reverse_template is not None and example_input is None:
+            raise MissingDependentParameterError(
+                required_parameter="Example text",
+                set_parameter="auto-conversion field",
+            )
+
+        if example_input is not None:
+            match = pattern.search(example_input)
+            if match is None:
+                raise ValidationError(_("Example input does not match the linkifier pattern."))
+            elif reverse_template is not None:
+                reversed_input = expand_reverse_template(reverse_template, match, group_set)
+                if reversed_input != example_input:
+                    raise ValidationError(_("Example text does not match auto-conversion field."))
+
+        # Do not continue the check if the url template is invalid to begin with.
+        # The ValidationError for invalid template will only be raised by the validator
+        # set on the url_template field instead of here to avoid duplicates.
+        if not uri_template.validate(self.url_template):
+            return
+
+        # Extract variables used in the URL template.
+        template_variables_set = set(uri_template.URITemplate(self.url_template).variable_names)
+
+        # Report patterns missing in linkifier pattern.
+        missing_in_pattern_set = template_variables_set - group_set
+        if len(missing_in_pattern_set) > 0:
+            name = min(missing_in_pattern_set)
+            raise ValidationError(
+                _("Group %(name)r in URL template is not present in linkifier pattern."),
+                params={"name": name},
+            )
+
+        missing_in_url_set = group_set - template_variables_set
+        # Report patterns missing in URL template.
+        if len(missing_in_url_set) > 0:
+            # We just report the first missing pattern here. Users can
+            # incrementally resolve errors if there are multiple
+            # missing patterns.
+            name = min(missing_in_url_set)
+            raise ValidationError(
+                _("Group %(name)r in linkifier pattern is not present in URL template."),
+                params={"name": name},
+            )
+
+        if self.url_template in self.alternative_url_templates:
+            raise ValidationError(
+                _("An alternative URL template cannot be the same as the URL template.")
+            )
+
+        if len(self.alternative_url_templates) != len(set(self.alternative_url_templates)):
+            raise ValidationError(_("Alternative URL templates must be unique."))
+
+        for alternative_template in self.alternative_url_templates:
+            url_template_validator(alternative_template)
+
+            alternative_template_variables = set(
+                uri_template.URITemplate(alternative_template).variable_names
+            )
+
+            missing_in_pattern = alternative_template_variables - group_set
+            if len(missing_in_pattern) > 0:
+                name = min(missing_in_pattern)
+                raise ValidationError(
+                    _(
+                        "Group %(name)r in alternative URL template is not present in linkifier pattern."
+                    ),
+                    params={"name": name},
+                )
+
+            missing_in_alternative = group_set - alternative_template_variables
+            if len(missing_in_alternative) > 0:
+                name = min(missing_in_alternative)
+                raise ValidationError(
+                    _(
+                        "Group %(name)r in linkifier pattern is not present in alternative URL template."
+                    ),
+                    params={"name": name},
+                )
+
+
+def get_linkifiers_cache_key(realm_id: int) -> str:
+    return f"{cache.KEY_PREFIX}:all_linkifiers_for_realm:{realm_id}"
+
+
+@cache_for_current_request
+@cache_with_key(get_linkifiers_cache_key, timeout=3600 * 24 * 7)
+def linkifiers_for_realm(realm_id: int) -> list[LinkifierDict]:
+    return [
+        LinkifierDict(
+            pattern=linkifier.pattern,
+            url_template=linkifier.url_template,
+            id=linkifier.id,
+            example_input=linkifier.example_input,
+            reverse_template=linkifier.reverse_template,
+            alternative_url_templates=linkifier.alternative_url_templates,
+        )
+        for linkifier in RealmFilter.objects.filter(realm_id=realm_id).order_by("order")
+    ]
+
+
+def flush_linkifiers(*, instance: RealmFilter, **kwargs: object) -> None:
+    realm_id = instance.realm_id
+    cache_delete(get_linkifiers_cache_key(realm_id))
+    flush_per_request_cache("linkifiers_for_realm")
+    flush_per_request_cache("topic_links")
+
+
+post_save.connect(flush_linkifiers, sender=RealmFilter)
+post_delete.connect(flush_linkifiers, sender=RealmFilter)