snapshot of https://github.com/zulip/zulip.git @ 48c69911c7dd4e03b20552eb3bd77ad8151c1357 (2026-09-15)

zerver/lib/url_preview/parsers/__init__.py

@@ -0,0 +1,4 @@
+from zerver.lib.url_preview.parsers.generic import GenericParser
+from zerver.lib.url_preview.parsers.open_graph import OpenGraphParser
+
+__all__ = ["GenericParser", "OpenGraphParser"]

zerver/lib/url_preview/parsers/base.py

@@ -0,0 +1,19 @@
+from email.message import EmailMessage
+
+from zerver.lib.url_preview.types import UrlEmbedData
+
+
+class BaseParser:
+    def __init__(self, html_source: bytes, content_type: str | None) -> None:
+        # We import BeautifulSoup here, because it's not used by most
+        # processes in production, and bs4 is big enough that
+        # importing it adds 10s of milliseconds to manage.py startup.
+        from bs4 import BeautifulSoup
+
+        m = EmailMessage()
+        m["Content-Type"] = content_type
+        charset = m.get_content_charset()
+        self._soup = BeautifulSoup(html_source, "lxml", from_encoding=charset)
+
+    def extract_data(self) -> UrlEmbedData:
+        raise NotImplementedError

zerver/lib/url_preview/parsers/generic.py

@@ -0,0 +1,61 @@
+from urllib.parse import urlsplit
+
+from bs4.element import Tag
+from typing_extensions import override
+
+from zerver.lib.url_preview.parsers.base import BaseParser
+from zerver.lib.url_preview.types import UrlEmbedData
+
+
+class GenericParser(BaseParser):
+    @override
+    def extract_data(self) -> UrlEmbedData:
+        return UrlEmbedData(
+            title=self._get_title(),
+            description=self._get_description(),
+            image=self._get_image(),
+        )
+
+    def _get_title(self) -> str | None:
+        soup = self._soup
+        if soup.title and soup.title.text != "":
+            return soup.title.text
+        if soup.h1 and soup.h1.text != "":
+            return soup.h1.text
+        return None
+
+    def _get_description(self) -> str | None:
+        soup = self._soup
+        meta_description = soup.find("meta", attrs={"name": "description"})
+        if isinstance(meta_description, Tag) and meta_description.get("content", "") != "":
+            assert isinstance(meta_description["content"], str)
+            return meta_description["content"]
+        first_h1 = soup.find("h1")
+        if first_h1:
+            first_p = first_h1.find_next("p")
+            if first_p and first_p.text != "":
+                return first_p.text
+        first_p = soup.find("p")
+        if first_p and first_p.text != "":
+            return first_p.text
+        return None
+
+    def _get_image(self) -> str | None:
+        """
+        Finding a first image after the h1 header.
+        Presumably it will be the main image.
+        """
+        soup = self._soup
+        first_h1 = soup.find("h1")
+        if first_h1:
+            first_image = first_h1.find_next_sibling("img", src=True)
+            if isinstance(first_image, Tag) and first_image["src"] != "":
+                assert isinstance(first_image["src"], str)
+                try:
+                    # We use urlsplit and not URLValidator because we
+                    # need to support relative URLs.
+                    urlsplit(first_image["src"])
+                except ValueError:
+                    return None
+                return first_image["src"]
+        return None

zerver/lib/url_preview/parsers/open_graph.py

@@ -0,0 +1,36 @@
+from urllib.parse import urlsplit
+
+from typing_extensions import override
+
+from zerver.lib.url_preview.types import UrlEmbedData
+
+from .base import BaseParser
+
+
+class OpenGraphParser(BaseParser):
+    @override
+    def extract_data(self) -> UrlEmbedData:
+        meta = self._soup.find_all("meta")
+
+        data = UrlEmbedData()
+
+        for tag in meta:
+            if not tag.has_attr("property"):
+                continue
+            if not tag.has_attr("content"):
+                continue
+
+            if tag["property"] == "og:title":
+                data.title = tag["content"]
+            elif tag["property"] == "og:description":
+                data.description = tag["content"]
+            elif tag["property"] == "og:image":
+                try:
+                    # We use urlsplit and not URLValidator because we
+                    # need to support relative URLs.
+                    urlsplit(tag["content"])
+                except ValueError:
+                    continue
+                data.image = tag["content"]
+
+        return data