From d30a4348a665f8827b4d17ebfbdbe51adf6e7e7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexis=20M=C3=A9taireau?= Date: Tue, 29 Oct 2024 22:37:14 +0100 Subject: [PATCH 1/2] Add a script to help generate release notes from merged pull requests --- dev_scripts/generate-release-notes.py | 67 +++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 dev_scripts/generate-release-notes.py diff --git a/dev_scripts/generate-release-notes.py b/dev_scripts/generate-release-notes.py new file mode 100644 index 0000000..463b123 --- /dev/null +++ b/dev_scripts/generate-release-notes.py @@ -0,0 +1,67 @@ +import sys + +import requests + +REPOSITORY = "https://github.com/freedomofpress/dangerzone/" +TEMPLATE = "- {title} ([#{number}]({url}))" + + +def get_prs_since_last_release(owner, repo): + session = requests.Session() + session.headers["Accept"] = "application/vnd.github.v3+json" + + # Try to get latest release + response = session.get( + f"https://api.github.com/repos/{owner}/{repo}/releases/latest" + ) + since = None + if response.ok: + since = response.json()["published_at"] + + # Get merged PRs + response = session.get( + f"https://api.github.com/repos/{owner}/{repo}/pulls", + params={ + "state": "closed", + "sort": "updated", + "direction": "desc", + "per_page": 100, + }, + ) + response.raise_for_status() + + prs = [] + for pr in response.json(): + if not pr["merged_at"]: + continue + if since and pr["merged_at"] <= since: + break + + prs.append( + TEMPLATE.format(title=pr["title"], number=pr["number"], url=pr["html_url"]) + ) + + return prs + + +def main(): + try: + url_path = REPOSITORY.rstrip("/").split("github.com/")[1] + owner, repo = url_path.split("/")[-2:] + except (ValueError, IndexError): + print("Error: Invalid GitHub URL", file=sys.stderr) + sys.exit(1) + + try: + notes = get_prs_since_last_release(owner, repo) + print("\n".join(notes)) + except requests.exceptions.HTTPError as e: + print(f"Error: {e.response.json().get('message', str(e))}", file=sys.stderr) + sys.exit(1) + except Exception as e: + print(f"Error: {e}", file=sys.stderr) + sys.exit(1) + + +if __name__ == "__main__": + main() From bcc1aecc78c8128c9a9446c6997f4af51b8cfff0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexis=20M=C3=A9taireau?= Date: Wed, 30 Oct 2024 14:59:18 +0100 Subject: [PATCH 2/2] fixup! Also list closed issues, use httpx to paralellise the requests --- dev_scripts/generate-release-notes.py | 233 +++++++++++++++++++++++--- pyproject.toml | 3 + 2 files changed, 210 insertions(+), 26 deletions(-) diff --git a/dev_scripts/generate-release-notes.py b/dev_scripts/generate-release-notes.py index 463b123..15ff528 100644 --- a/dev_scripts/generate-release-notes.py +++ b/dev_scripts/generate-release-notes.py @@ -1,50 +1,227 @@ +import argparse +import asyncio +import re import sys +from datetime import datetime +from typing import Dict, List, Optional, Tuple -import requests +import httpx REPOSITORY = "https://github.com/freedomofpress/dangerzone/" TEMPLATE = "- {title} ([#{number}]({url}))" -def get_prs_since_last_release(owner, repo): - session = requests.Session() - session.headers["Accept"] = "application/vnd.github.v3+json" +def parse_version(version: str) -> Tuple[int, int]: + """Extract major.minor from version string, ignoring patch""" + match = re.match(r"v?(\d+)\.(\d+)", version) + if not match: + raise ValueError(f"Invalid version format: {version}") + return (int(match.group(1)), int(match.group(2))) - # Try to get latest release - response = session.get( - f"https://api.github.com/repos/{owner}/{repo}/releases/latest" + +async def get_last_minor_release( + client: httpx.AsyncClient, owner: str, repo: str +) -> Optional[str]: + """Get the latest minor release date (ignoring patches)""" + response = await client.get(f"https://api.github.com/repos/{owner}/{repo}/releases") + response.raise_for_status() + releases = response.json() + + if not releases: + return None + + # Get the latest minor version by comparing major.minor numbers + current_version = parse_version(releases[0]["tag_name"]) + latest_date = None + + for release in releases: + try: + version = parse_version(release["tag_name"]) + if version < current_version: + latest_date = release["published_at"] + break + except ValueError: + continue + + return latest_date + + +async def get_issue_details( + client: httpx.AsyncClient, owner: str, repo: str, issue_number: int +) -> Optional[dict]: + """Get issue title and number if it exists""" + response = await client.get( + f"https://api.github.com/repos/{owner}/{repo}/issues/{issue_number}" ) - since = None - if response.ok: - since = response.json()["published_at"] + if response.is_success: + data = response.json() + return { + "title": data["title"], + "number": data["number"], + "url": data["html_url"], + } + return None - # Get merged PRs - response = session.get( - f"https://api.github.com/repos/{owner}/{repo}/pulls", + +def extract_issue_number(pr_body: Optional[str]) -> Optional[int]: + """Extract issue number from PR body looking for common formats like 'Fixes #123' or 'Closes #123'""" + if not pr_body: + return None + + patterns = [ + r"(?:closes|fixes|resolves)\s*#(\d+)", + r"(?:close|fix|resolve)\s*#(\d+)", + ] + + for pattern in patterns: + match = re.search(pattern, pr_body.lower()) + if match: + return int(match.group(1)) + + return None + + +async def verify_commit_in_master( + client: httpx.AsyncClient, owner: str, repo: str, commit_id: str +) -> bool: + """Verify if a commit exists in master""" + response = await client.get( + f"https://api.github.com/repos/{owner}/{repo}/commits/{commit_id}" + ) + return response.is_success and response.json().get("commit") is not None + + +async def process_issue_events( + client: httpx.AsyncClient, owner: str, repo: str, issue: Dict +) -> Optional[Dict]: + """Process events for a single issue""" + events_response = await client.get(f"{issue['url']}/events") + if not events_response.is_success: + return None + + for event in events_response.json(): + if event["event"] == "closed" and event.get("commit_id"): + if await verify_commit_in_master(client, owner, repo, event["commit_id"]): + return { + "title": issue["title"], + "number": issue["number"], + "url": issue["html_url"], + } + return None + + +async def get_closed_issues( + client: httpx.AsyncClient, owner: str, repo: str, since: str +) -> List[Dict]: + """Get issues closed by commits to master since the given date""" + response = await client.get( + f"https://api.github.com/repos/{owner}/{repo}/issues", params={ "state": "closed", "sort": "updated", "direction": "desc", + "since": since, "per_page": 100, }, ) response.raise_for_status() - prs = [] - for pr in response.json(): - if not pr["merged_at"]: - continue - if since and pr["merged_at"] <= since: - break + tasks = [] + since_date = datetime.strptime(since, "%Y-%m-%dT%H:%M:%SZ") - prs.append( - TEMPLATE.format(title=pr["title"], number=pr["number"], url=pr["html_url"]) + for issue in response.json(): + if "pull_request" in issue: + continue + + closed_at = datetime.strptime(issue["closed_at"], "%Y-%m-%dT%H:%M:%SZ") + if closed_at <= since_date: + continue + + tasks.append(process_issue_events(client, owner, repo, issue)) + + results = await asyncio.gather(*tasks) + return [r for r in results if r is not None] + + +async def process_pull_request( + client: httpx.AsyncClient, + owner: str, + repo: str, + pr: Dict, + closed_issues: List[Dict], +) -> Optional[str]: + """Process a single pull request""" + issue_number = extract_issue_number(pr.get("body")) + if issue_number: + issue = await get_issue_details(client, owner, repo, issue_number) + if issue: + if not any(i["number"] == issue["number"] for i in closed_issues): + return TEMPLATE.format(**issue) + return None + + return TEMPLATE.format(title=pr["title"], number=pr["number"], url=pr["html_url"]) + + +async def get_changes_since_last_release( + owner: str, repo: str, token: Optional[str] = None +) -> List[str]: + headers = { + "Accept": "application/vnd.github.v3+json", + } + if token: + headers["Authorization"] = f"token {token}" + else: + print( + "Warning: No token provided. API rate limiting may occur.", file=sys.stderr ) - return prs + async with httpx.AsyncClient(headers=headers, timeout=30.0) as client: + # Get the date of last minor release + since = await get_last_minor_release(client, owner, repo) + if not since: + return [] + + changes = [] + + # Get issues closed by commits to master + closed_issues = await get_closed_issues(client, owner, repo, since) + changes.extend([TEMPLATE.format(**issue) for issue in closed_issues]) + + # Get merged PRs + response = await client.get( + f"https://api.github.com/repos/{owner}/{repo}/pulls", + params={ + "state": "closed", + "sort": "updated", + "direction": "desc", + "per_page": 100, + }, + ) + response.raise_for_status() + + # Process PRs in parallel + pr_tasks = [] + for pr in response.json(): + if not pr["merged_at"]: + continue + if since and pr["merged_at"] <= since: + break + + pr_tasks.append( + process_pull_request(client, owner, repo, pr, closed_issues) + ) + + pr_results = await asyncio.gather(*pr_tasks) + changes.extend([r for r in pr_results if r is not None]) + + return changes -def main(): +async def main_async(): + parser = argparse.ArgumentParser(description="Generate release notes from GitHub") + parser.add_argument("--token", "-t", help="GitHub API token") + args = parser.parse_args() + try: url_path = REPOSITORY.rstrip("/").split("github.com/")[1] owner, repo = url_path.split("/")[-2:] @@ -53,15 +230,19 @@ def main(): sys.exit(1) try: - notes = get_prs_since_last_release(owner, repo) + notes = await get_changes_since_last_release(owner, repo, args.token) print("\n".join(notes)) - except requests.exceptions.HTTPError as e: - print(f"Error: {e.response.json().get('message', str(e))}", file=sys.stderr) + except httpx.HTTPError as e: + print(f"Error: {e}", file=sys.stderr) sys.exit(1) except Exception as e: print(f"Error: {e}", file=sys.stderr) sys.exit(1) +def main(): + asyncio.run(main_async()) + + if __name__ == "__main__": main() diff --git a/pyproject.toml b/pyproject.toml index 160cbaa..9377010 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,6 +56,9 @@ pytest-subprocess = "^1.5.2" [tool.poetry.group.container.dependencies] pymupdf = "^1.24.10" +[tool.poetry.group.dev.dependencies] +httpx = "^0.27.2" + [tool.isort] profile = "black" skip_gitignore = true