snapshot of https://github.com/zulip/zulip.git @ 48c69911c7dd4e03b20552eb3bd77ad8151c1357 (2026-09-15)

zerver/lib/url_preview/oembed.py

@@ -0,0 +1,52 @@
+import json
+
+import requests
+from pyoembed import PyOembedException, oEmbed
+
+from zerver.lib.url_preview.types import UrlEmbedData, UrlOEmbedData
+
+
+def get_oembed_data(url: str, maxwidth: int = 640, maxheight: int = 480) -> UrlEmbedData | None:
+    # pyoembed makes its requests directly, not through OutgoingSession,
+    # but they still go through Smokescreen: requests lib honors the
+    # HTTP_proxy/HTTPS_proxy variables we set in every process's environment.
+    try:
+        data = oEmbed(url, maxwidth=maxwidth, maxheight=maxheight)
+    except (PyOembedException, json.decoder.JSONDecodeError, requests.exceptions.ConnectionError):
+        return None
+
+    oembed_resource_type = data.get("type", "")
+    image = data.get("url", data.get("image"))
+    thumbnail = data.get("thumbnail_url")
+    html = data.get("html", "")
+    if oembed_resource_type == "photo" and image:
+        return UrlOEmbedData(
+            image=image,
+            type="photo",
+            title=data.get("title"),
+            description=data.get("description"),
+        )
+
+    if oembed_resource_type == "video" and html and thumbnail:
+        return UrlOEmbedData(
+            image=thumbnail,
+            type="video",
+            html=strip_cdata(html),
+            title=data.get("title"),
+            description=data.get("description"),
+        )
+
+    # Otherwise, use the title/description from pyembed as the basis
+    # for our other parsers
+    return UrlEmbedData(
+        title=data.get("title"),
+        description=data.get("description"),
+    )
+
+
+def strip_cdata(html: str) -> str:
+    # Work around a bug in SoundCloud's XML generation:
+    # <html>&lt;![CDATA[&lt;iframe ...&gt;&lt;/iframe&gt;]]&gt;</html>
+    if html.startswith("<![CDATA[") and html.endswith("]]>"):
+        html = html[9:-3]
+    return html

zerver/lib/url_preview/parsers/__init__.py

@@ -0,0 +1,4 @@
+from zerver.lib.url_preview.parsers.generic import GenericParser
+from zerver.lib.url_preview.parsers.open_graph import OpenGraphParser
+
+__all__ = ["GenericParser", "OpenGraphParser"]

zerver/lib/url_preview/parsers/base.py

@@ -0,0 +1,19 @@
+from email.message import EmailMessage
+
+from zerver.lib.url_preview.types import UrlEmbedData
+
+
+class BaseParser:
+    def __init__(self, html_source: bytes, content_type: str | None) -> None:
+        # We import BeautifulSoup here, because it's not used by most
+        # processes in production, and bs4 is big enough that
+        # importing it adds 10s of milliseconds to manage.py startup.
+        from bs4 import BeautifulSoup
+
+        m = EmailMessage()
+        m["Content-Type"] = content_type
+        charset = m.get_content_charset()
+        self._soup = BeautifulSoup(html_source, "lxml", from_encoding=charset)
+
+    def extract_data(self) -> UrlEmbedData:
+        raise NotImplementedError

zerver/lib/url_preview/parsers/generic.py

@@ -0,0 +1,61 @@
+from urllib.parse import urlsplit
+
+from bs4.element import Tag
+from typing_extensions import override
+
+from zerver.lib.url_preview.parsers.base import BaseParser
+from zerver.lib.url_preview.types import UrlEmbedData
+
+
+class GenericParser(BaseParser):
+    @override
+    def extract_data(self) -> UrlEmbedData:
+        return UrlEmbedData(
+            title=self._get_title(),
+            description=self._get_description(),
+            image=self._get_image(),
+        )
+
+    def _get_title(self) -> str | None:
+        soup = self._soup
+        if soup.title and soup.title.text != "":
+            return soup.title.text
+        if soup.h1 and soup.h1.text != "":
+            return soup.h1.text
+        return None
+
+    def _get_description(self) -> str | None:
+        soup = self._soup
+        meta_description = soup.find("meta", attrs={"name": "description"})
+        if isinstance(meta_description, Tag) and meta_description.get("content", "") != "":
+            assert isinstance(meta_description["content"], str)
+            return meta_description["content"]
+        first_h1 = soup.find("h1")
+        if first_h1:
+            first_p = first_h1.find_next("p")
+            if first_p and first_p.text != "":
+                return first_p.text
+        first_p = soup.find("p")
+        if first_p and first_p.text != "":
+            return first_p.text
+        return None
+
+    def _get_image(self) -> str | None:
+        """
+        Finding a first image after the h1 header.
+        Presumably it will be the main image.
+        """
+        soup = self._soup
+        first_h1 = soup.find("h1")
+        if first_h1:
+            first_image = first_h1.find_next_sibling("img", src=True)
+            if isinstance(first_image, Tag) and first_image["src"] != "":
+                assert isinstance(first_image["src"], str)
+                try:
+                    # We use urlsplit and not URLValidator because we
+                    # need to support relative URLs.
+                    urlsplit(first_image["src"])
+                except ValueError:
+                    return None
+                return first_image["src"]
+        return None

zerver/lib/url_preview/parsers/open_graph.py

@@ -0,0 +1,36 @@
+from urllib.parse import urlsplit
+
+from typing_extensions import override
+
+from zerver.lib.url_preview.types import UrlEmbedData
+
+from .base import BaseParser
+
+
+class OpenGraphParser(BaseParser):
+    @override
+    def extract_data(self) -> UrlEmbedData:
+        meta = self._soup.find_all("meta")
+
+        data = UrlEmbedData()
+
+        for tag in meta:
+            if not tag.has_attr("property"):
+                continue
+            if not tag.has_attr("content"):
+                continue
+
+            if tag["property"] == "og:title":
+                data.title = tag["content"]
+            elif tag["property"] == "og:description":
+                data.description = tag["content"]
+            elif tag["property"] == "og:image":
+                try:
+                    # We use urlsplit and not URLValidator because we
+                    # need to support relative URLs.
+                    urlsplit(tag["content"])
+                except ValueError:
+                    continue
+                data.image = tag["content"]
+
+        return data

zerver/lib/url_preview/preview.py

@@ -0,0 +1,115 @@
+import re
+from collections.abc import Callable
+from re import Match
+from typing import Any
+from urllib.parse import urljoin
+
+import magic
+import requests
+from django.conf import settings
+from django.utils.encoding import smart_str
+
+from version import ZULIP_VERSION
+from zerver.lib.cache import cache_with_key, preview_url_cache_key
+from zerver.lib.outgoing_http import OutgoingSession
+from zerver.lib.pysa import mark_sanitized
+from zerver.lib.url_preview.oembed import get_oembed_data
+from zerver.lib.url_preview.parsers import GenericParser, OpenGraphParser
+from zerver.lib.url_preview.types import UrlEmbedData, UrlOEmbedData
+
+# Based on django.core.validators.URLValidator, with ftp support removed.
+link_regex = re.compile(
+    r"^(?:http)s?://"  # http:// or https://
+    r"(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|"  # domain...
+    r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})"  # ...or ip
+    r"(?::\d+)?"  # optional port
+    r"(?:/?|[/?]\S+)$",
+    re.IGNORECASE,
+)
+
+# Use Chrome User-Agent, since some sites refuse to work on old browsers
+ZULIP_URL_PREVIEW_USER_AGENT = (
+    f"Mozilla/5.0 (compatible; ZulipURLPreview/{ZULIP_VERSION}; +{settings.ROOT_DOMAIN_URI})"
+)
+
+# FIXME: This header and timeout are not used by pyoembed, when trying to autodiscover!
+HEADERS = {"User-Agent": ZULIP_URL_PREVIEW_USER_AGENT}
+TIMEOUT = 15
+
+
+class PreviewSession(OutgoingSession):
+    def __init__(self) -> None:
+        super().__init__(role="preview", timeout=TIMEOUT, headers=HEADERS)
+
+
+def is_link(url: str) -> Match[str] | None:
+    return link_regex.match(smart_str(url))
+
+
+def guess_mimetype_from_content(response: requests.Response) -> str:
+    mime_magic = magic.Magic(mime=True)
+    try:
+        content = next(response.iter_content(1000))
+    except StopIteration:
+        content = ""
+    return mime_magic.from_buffer(content)
+
+
+def valid_content_type(url: str) -> bool:
+    try:
+        response = PreviewSession().get(url, stream=True)
+    except requests.RequestException:
+        return False
+
+    if not response.ok:
+        return False
+
+    content_type = response.headers.get("content-type")
+    # Be accommodating of bad servers: assume content may be html if no content-type header
+    if not content_type or content_type.startswith("text/html"):
+        # Verify that the content is actually HTML if the server claims it is
+        content_type = guess_mimetype_from_content(response)
+    return content_type.startswith("text/html")
+
+
+def catch_network_errors(func: Callable[..., Any]) -> Callable[..., Any]:
+    def wrapper(*args: Any, **kwargs: Any) -> Any:
+        try:
+            return func(*args, **kwargs)
+        except requests.exceptions.RequestException:
+            pass
+
+    return wrapper
+
+
+@catch_network_errors
+@cache_with_key(preview_url_cache_key)
+def get_link_embed_data(url: str, maxwidth: int = 640, maxheight: int = 480) -> UrlEmbedData | None:
+    if not is_link(url):
+        return None
+
+    if not valid_content_type(url):
+        return None
+
+    # The oembed data from pyoembed may be complete enough to return
+    # as-is; if so, we use it.  Otherwise, we use it as a _base_ for
+    # the other, less sophisticated techniques which we apply as
+    # successive fallbacks.
+    data = get_oembed_data(url, maxwidth=maxwidth, maxheight=maxheight)
+    if data is not None and isinstance(data, UrlOEmbedData):
+        return data
+
+    response = PreviewSession().get(mark_sanitized(url), stream=True)
+    if not response.ok:
+        return None
+
+    if data is None:
+        data = UrlEmbedData()
+
+    for parser_class in (OpenGraphParser, GenericParser):
+        parser = parser_class(response.content, response.headers.get("Content-Type"))
+        data.merge(parser.extract_data())
+
+    if data.image:
+        data.image = urljoin(response.url, data.image)
+    return data

zerver/lib/url_preview/types.py

@@ -0,0 +1,25 @@
+from dataclasses import dataclass
+from typing import Literal
+
+
+@dataclass
+class UrlEmbedData:
+    type: str | None = None
+    html: str | None = None
+    title: str | None = None
+    description: str | None = None
+    image: str | None = None
+
+    def merge(self, other: "UrlEmbedData") -> None:
+        if self.title is None and other.title is not None:
+            self.title = other.title
+        if self.description is None and other.description is not None:
+            self.description = other.description
+        if self.image is None and other.image is not None:
+            self.image = other.image
+
+
+@dataclass
+class UrlOEmbedData(UrlEmbedData):
+    type: Literal["photo", "video"]
+    html: str | None = None