AI Agent Development #7: Capstone Project — An Issue Triage Agent

import requests

API = "https://api.github.com"
HEADERS = {"Authorization": f"Bearer {GITHUB_TOKEN}"}
MAX_BODY = 3000   # Part 4: tool result diet

def list_open_issues(repo: str) -> str:
    """Return up to 10 unlabeled open issues, newest first."""
    r = requests.get(f"{API}/repos/{repo}/issues",
                     params={"state": "open", "per_page": 30}, headers=HEADERS)
    r.raise_for_status()
    unlabeled = [i for i in r.json() if not i["labels"] and "pull_request" not in i]
    lines = [f"#{i['number']} {i['title']}" for i in unlabeled[:10]]
    return "\n".join(lines) or "There are no unlabeled open issues."

def get_issue(repo: str, number: int) -> str:
    """Fetch the title and body of an issue by number. Always read the body before classifying."""
    r = requests.get(f"{API}/repos/{repo}/issues/{number}", headers=HEADERS)
    r.raise_for_status()
    issue = r.json()
    body = (issue["body"] or "")[:MAX_BODY]
    return f"Title: {issue['title']}\nBody:\n{body}"

def add_labels(repo: str, number: int, labels: list) -> str:
    """Attach labels to an issue. Reversible, but visible on a public repository, so it requires approval."""
    r = requests.post(f"{API}/repos/{repo}/issues/{number}/labels",
                      json={"labels": labels}, headers=HEADERS)
    r.raise_for_status()
    return f"Attached labels {labels} to #{number}."

def post_comment(repo: str, number: int, body: str) -> str:
    """Post a comment on an issue. A publicly visible action, so it always requires approval."""
    r = requests.post(f"{API}/repos/{repo}/issues/{number}/comments",
                      json={"body": body}, headers=HEADERS)
    r.raise_for_status()
    return f"Posted a comment on #{number}."

NEEDS_APPROVAL = {"add_labels", "post_comment"}

def execute_tool(block) -> dict:
    if block.name in NEEDS_APPROVAL:
        print(f"\n[Approval request] {block.name} {block.input}")
        if input("Run it? (y/n) ").strip().lower() != "y":
            return {
                "type": "tool_result",
                "tool_use_id": block.id,
                "content": "The user did not approve. Summarize your proposals and report them instead.",
                "is_error": True,
            }
    try:
        return {"type": "tool_result", "tool_use_id": block.id,
                "content": run_tool(block.name, block.input)}
    except Exception as e:
        return {"type": "tool_result", "tool_use_id": block.id,
                "content": f"Tool execution failed: {e}", "is_error": True}

SYSTEM = """You are a GitHub issue triage agent.

Procedure:
1. Check unlabeled issues with list_open_issues and list your processing plan.
2. For each issue, read the body with get_issue before classifying. Never classify from the title alone.
3. Classification criteria: bug if it contains a reproducible error, enhancement if it
   proposes a new feature, question if it asks about usage, documentation if it suggests doc improvements.
4. Propose labels with add_labels. If you are not confident, do not attach a label and report why.
5. Only for question issues, draft a reply and propose it with post_comment.

Rules:
- Never retry an action once its approval has been denied.
- At the end, report the issues handled, labels attached, and issues held back in a table.
"""

GOLDEN = [
    {"number": 101, "expected": ["bug"]},
    {"number": 95,  "expected": ["question"]},
    {"number": 88,  "expected": ["enhancement"]},
    # about 20 past issues are enough to detect changes
]

def evaluate(repo: str) -> float:
    correct = 0
    for case in GOLDEN:
        proposed = triage_one(repo, case["number"])   # a mode that only returns proposals, no approval gate
        if set(proposed) == set(case["expected"]):
            correct += 1
        else:
            print(f"#{case['number']}: expected {case['expected']} / proposed {proposed}")
    return correct / len(GOLDEN)

print(f"Classification accuracy: {evaluate('owner/repo'):.0%}")