upstream-zulip/tools/closed-by-commits

#!/usr/bin/env -S uv run --script --frozen --only-group release-tools  # -*-python-*-

import json
import re
import time
import urllib.parse
from collections import defaultdict
from dataclasses import dataclass, field
from functools import cache
from typing import Annotated, Any

import requests
import typer
from github import Auth, Github
from rich.console import Console
from rich.progress import Progress
from typing_extensions import override


def encode_hash_component(s: str) -> str:
    hash_replacements = {
        "%": ".",
        "(": ".28",
        ")": ".29",
        ".": ".2E",
    }
    encoded = urllib.parse.quote(s, safe="*")
    return "".join(hash_replacements.get(c, c) for c in encoded)


@cache
def search_czo_for_number(prefix: str, number: int) -> frozenset[str]:
    params = {
        "anchor": "newest",
        "num_before": "100",
        "num_after": "0",
        "narrow": json.dumps(
            [
                {"negated": False, "operator": "search", "operand": f'"#{prefix}{number}"'},
                {"negated": False, "operator": "channels", "operand": "web-public"},
            ]
        ),
    }

    try:
        response = requests.get("https://chat.zulip.org/json/messages", params=params, timeout=30)
        response.raise_for_status()

        data = response.json()
        messages = data.get("messages", [])

        # Extract unique topic URLs
        urls = set()
        for msg in messages:
            stream_id = msg.get("stream_id")
            display_recipient = msg.get("display_recipient")
            subject = msg.get("subject")

            assert stream_id
            assert display_recipient

            encoded_recipient = encode_hash_component(display_recipient.replace(" ", "-"))
            encoded_subject = encode_hash_component(subject)
            url = f"https://chat.zulip.org/#narrow/channel/{stream_id}-{encoded_recipient}/topic/{encoded_subject}"
            urls.add(url)

        return frozenset(urls)

    except requests.exceptions.RequestException as e:
        assert e.response
        if e.response.status_code != 429:
            raise
        retry_after = int(e.response.headers["Retry-After"]) + 1
        time.sleep(retry_after)
        return search_czo_for_number(prefix, number)


@dataclass
class Issue:
    number: int
    title: str
    czo_urls: set[str] = field(default_factory=set)
    closed_by_prs: list["PullRequest"] = field(default_factory=list)
    duplicate_issue_ids: set[int] = field(default_factory=set)

    @override
    def __hash__(self) -> int:
        return hash(self.number)

    @override
    def __eq__(self, other: object) -> bool:
        return isinstance(other, Issue) and self.number == other.number


@dataclass
class PullRequest:
    number: int
    title: str
    czo_urls: set[str] = field(default_factory=set)

    @override
    def __hash__(self) -> int:
        return hash(self.number)

    @override
    def __eq__(self, other: object) -> bool:
        return isinstance(other, PullRequest) and self.number == other.number


class CommitRangeAnalyzer:
    COMMIT_PRS_QUERY = """
    query($oid: GitObjectID!, $repo: String!) {
      repository(owner: "zulip", name: $repo) {
        object(oid: $oid) {
          ... on Commit {
            messageBody
            associatedPullRequests(first: 10) {
              nodes {
                number
                title
                url
                body
                comments(first: 100) {
                  nodes {
                    body
                  }
                }
                closingIssuesReferences(first: 50) {
                  nodes {
                    number
                    title
                    url
                    body
                    comments(first: 100) {
                      nodes {
                        body
                      }
                    }
                    timelineItems(first:100, itemTypes:MARKED_AS_DUPLICATE_EVENT) {
                      ... on IssueTimelineItemsConnection {
                        nodes {
                          ... on MarkedAsDuplicateEvent {
                            duplicate {
                              ... on Issue {
                                number
                              }
                            }
                          }
                        }
                      }
                    }
                  }
                }
              }
            }
          }
        }
      }
    }
    """

    ISSUE_QUERY = """
    query($number: Int!, $repo: String!) {
      repository(owner: "zulip", name: $repo) {
        issue(number: $number) {
          number
          title
          body
          comments(first: 100) {
            nodes {
              body
            }
          }
          timelineItems(first:100, itemTypes:MARKED_AS_DUPLICATE_EVENT) {
            ... on IssueTimelineItemsConnection {
              nodes {
                ... on MarkedAsDuplicateEvent {
                  duplicate {
                    ... on Issue {
                      number
                    }
                  }
                }
              }
            }
          }
        }
      }
    }
    """

    def __init__(self, token: str, reponame: str, czo_issue_prefix: str) -> None:
        self.github = Github(auth=Auth.Token(token))
        self.reponame = reponame
        self.czo_issue_prefix = czo_issue_prefix

    @staticmethod
    def _extract_czo_urls(text: str | None) -> set[str]:
        if not text:
            return set()

        matches = re.findall(r"https://chat\.zulip\.org/[^\s\)\]\>]+", text)

        urls = set()
        for url in matches:
            # We strip off and remove /with/... and /near/... to
            # reduce the number of unique links which are generated.
            parsed_url = re.match(r"(.*)/topic/([^/]+)(/(near|with)/.*)?$", url)
            if not parsed_url:
                continue
            urls.add(
                parsed_url[1]
                + "/topic/"
                # Normalize the topic by decoding and re-encoding
                + encode_hash_component(urllib.parse.unquote(parsed_url[2].replace(".", "%")))
            )

        return urls

    @staticmethod
    def _extract_issue_numbers(reponame: str, text: str | None) -> set[int]:
        if not text:
            return set()

        # https://docs.github.com/en/issues/tracking-your-work-with-issues/using-issues/linking-a-pull-request-to-an-issue#linking-a-pull-request-to-an-issue-using-a-keyword
        pattern = rf"\b(?:close[sd]?|fix(?:es|ed)?|resolve[sd]?):?\s+(?:zulip/{reponame})?#(\d+)\b"
        matches = re.findall(pattern, text, re.IGNORECASE)
        return {int(num) for num in matches}

    def _issue_from_node(self, issue_node: dict[str, Any], console: Console) -> Issue:
        issue = Issue(
            number=issue_node["number"],
            title=issue_node["title"].strip(),
        )
        issue_comments = issue_node.get("comments", {}).get("nodes", [])
        for node in [issue_node, *issue_comments]:
            issue.czo_urls.update(self._extract_czo_urls(node.get("body")))
        issue.czo_urls.update(search_czo_for_number(self.czo_issue_prefix, issue.number))

        issue.duplicate_issue_ids = {
            event["duplicate"]["number"] for event in issue_node["timelineItems"]["nodes"]
        }
        for issue_id in issue.duplicate_issue_ids:
            if duplicate_info := self._fetch_issue(issue_id, console):
                issue.czo_urls.update(duplicate_info.czo_urls)
                issue.duplicate_issue_ids.update(duplicate_info.duplicate_issue_ids)
        return issue

    def _fetch_issue(self, number: int, console: Console) -> Issue | None:
        """Fetch issue metadata from GitHub."""
        try:
            _, result = self.github.requester.graphql_query(
                self.ISSUE_QUERY, {"number": number, "repo": self.reponame}
            )

            if "errors" in result:
                console.log(
                    "Failed to fetch zulip/%s#%d: %s", self.reponame, number, result["errors"]
                )
                return None

            data = result.get("data", {})
            issue_node = data.get("repository", {}).get("issue")

            if not issue_node:
                return None

            return self._issue_from_node(issue_node, console)

        except Exception:
            return None

    def get_issues_for_commit(
        self, commit_sha: str, console: Console
    ) -> tuple[dict[Issue, set[PullRequest]], set[int]]:
        _, result = self.github.requester.graphql_query(
            self.COMMIT_PRS_QUERY, {"oid": commit_sha, "repo": self.reponame}
        )

        if "errors" in result:
            error_messages = [e.get("message", str(e)) for e in result["errors"]]
            raise RuntimeError(f"GraphQL errors: {', '.join(error_messages)}")

        data = result.get("data", {})

        if not data.get("repository", {}).get("object"):
            return dict(), set()

        commit_obj = data["repository"]["object"]
        commit_message = commit_obj.get("messageBody", "")
        pr_nodes = commit_obj.get("associatedPullRequests", {}).get("nodes", [])

        results: dict[Issue, set[PullRequest]] = defaultdict(set)
        all_seen_prs: set[int] = set()
        for pr_node in pr_nodes:
            pr = PullRequest(
                number=pr_node["number"],
                title=pr_node["title"].strip(),
            )
            all_seen_prs.add(pr.number)
            pr_comments = pr_node.get("comments", {}).get("nodes", [])
            for node in [pr_node, *pr_comments]:
                pr.czo_urls.update(self._extract_czo_urls(node.get("body")))
            pr.czo_urls.update(search_czo_for_number(self.czo_issue_prefix, pr.number))

            # Get issues from PR metadata
            issues_dict = {}
            issue_nodes = pr_node.get("closingIssuesReferences", {}).get("nodes", [])
            for issue_node in issue_nodes:
                if issue_node is None:
                    continue

                issue = self._issue_from_node(issue_node, console)
                issues_dict[issue.number] = issue

            # Extract additional issue numbers from commit message
            for issue_num in self._extract_issue_numbers(self.reponame, commit_message):
                if issue_num not in issues_dict:
                    maybe_issue = self._fetch_issue(issue_num, console)
                    if maybe_issue is None:
                        continue
                    issues_dict[issue_num] = maybe_issue

            for issue in issues_dict.values():
                results[issue].add(pr)

        if not pr_nodes:
            for issue_num in self._extract_issue_numbers(self.reponame, commit_message):
                maybe_issue = self._fetch_issue(issue_num, console)
                if maybe_issue is None:
                    continue
                results[maybe_issue].update()

        return results, all_seen_prs

    def analyze_range(self, base: str, head: str) -> list[Issue]:
        console = Console(stderr=True, log_path=False)

        repository = self.github.get_repo(f"zulip/{self.reponame}")
        comparison = repository.compare(base, head)
        commit_shas = [commit.sha for commit in comparison.commits]
        console.log(f"Found {len(commit_shas)} commits")

        all_seen_prs = set()
        issue_to_prs: dict[Issue, set[PullRequest]] = defaultdict(set)
        with Progress(console=console) as progress:
            task = progress.add_task("Processing commits...", total=len(commit_shas))
            for sha in commit_shas:
                progress.console.log(f"Processing {sha}")

                this_issue_to_prs, this_seen_prs = self.get_issues_for_commit(sha, console)
                for issue, prs in this_issue_to_prs.items():
                    issue_to_prs[issue].update(prs)
                all_seen_prs.update(this_seen_prs)
                progress.advance(task)

        for issue, prs in issue_to_prs.items():
            for pr in prs:
                issue.czo_urls.update(pr.czo_urls)

            issue.closed_by_prs = sorted(prs, key=lambda p: p.number)

        unique_prs = len({pr for prs in issue_to_prs.values() for pr in prs})
        console.log(
            f"Found {len(all_seen_prs)} unique PRs, of which {unique_prs} PRs closed {len(issue_to_prs)} issues",
        )

        return sorted(issue_to_prs.keys(), key=lambda x: x.number)


def validate_github_token(value: str) -> str:
    # https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/about-authentication-to-github#githubs-token-formats
    if value.startswith("github_"):
        return value
    if re.match(r"gh[pousr]_", value):
        return value
    raise typer.BadParameter("Github access tokens start with `github_`, or `gh`")


from enum import Enum


class ZulipRepo(str, Enum):
    zulip = ("zulip",)
    flutter = "zulip-flutter"


repo_issue_prefixes = {
    ZulipRepo.zulip: "",
    ZulipRepo.flutter: "F",
}


def main(
    base_commit: Annotated[str, typer.Argument(help="Commit-ish, resolved on the server")],
    head_commit: Annotated[str, typer.Argument(help="Commit-ish, resolved on the server")],
    token: Annotated[
        str,
        typer.Option(
            metavar="TOKEN",
            envvar="GITHUB_TOKEN",
            show_envvar=True,
            callback=validate_github_token,
            help=(
                "Github access token; can be a fine-grained personal access token "
                "with read-only access to 'Public repositories'.  "
                "See https://github.com/settings/personal-access-tokens/new"
            ),
        ),
    ],
    repo: ZulipRepo = ZulipRepo.zulip,
) -> int:
    """Find issues which are closed in a commit range."""

    lines = []
    prefix = repo_issue_prefixes[repo]
    analyzer = CommitRangeAnalyzer(token, repo.value, prefix)
    for issue in analyzer.analyze_range(base_commit, head_commit):
        lines.append(f"#### #{prefix}{issue.number}: {issue.title}")
        if issue.duplicate_issue_ids:
            lines.append(
                " - **Duplicates:** "
                + ", ".join(f"#{prefix}{number}" for number in issue.duplicate_issue_ids)
            )
        lines.extend(
            f" - **Closed by:** #{prefix}{pr.number}: {pr.title}" for pr in issue.closed_by_prs
        )
        lines.extend(f" - {czo_url}" for czo_url in sorted(issue.czo_urls))

        lines.append("")

    print("\n".join(lines))

    return 0


if __name__ == "__main__":
    typer.run(main)