agent-orchestrator/tangled_pr.py

#!/usr/bin/env python3
"""File a Tangled pull request as the bot, by reusing a browser session cookie.

WHY: Tangled's appview only indexes a pull when it's created through its
OAuth-authenticated web endpoint (POST /{owner}/{repo}/pulls/), which fetches the
patch from the knot and inserts it into the appview DB directly. Writing the
sh.tangled.repo.pull record straight to the PDS (com.atproto.repo.createRecord)
does NOT get indexed (verified: even a byte-identical knot patch fails). There is
no client CLI. So the automation is: log into tangled.org ONCE in a browser as the
bot, copy the session cookie here (gitignored), and this tool reuses it.

The endpoint needs no CSRF token and no patch upload — just the session cookie and
the branch names; the appview generates the patch (from the knot) and indexes it.
Branch-based PR requires push access to the repo (the bot owns its fork, so OK).

COOKIE FILE (tools/.tangled-session, gitignored), a single line — paste the whole
Cookie header value from the logged-in browser (both appview-* cookies):
  TANGLED_COOKIE=appview-session-v2=<...>; appview-accounts-v2=<...>

RELIABILITY (rewritten 2026-08-21). This tool used to judge success ONLY by an
HX-Redirect header on the POST response. A create that SUCCEEDED but answered without
that header read as a failure, the caller retried, and Tangled grew duplicate pulls —
that is exactly how #397, #398 and #399 were created for one branch. A response header
describes what the server meant to say; it is not the artifact. So now:

  * BEFORE posting, look for an existing OPEN pull for this source branch. If one
    exists, refuse and name it — a duplicate cannot be created even if a caller retries.
  * AFTER posting, confirm against the PULLS LIST, not the response: a new pull number
    that did not exist before and whose page names this source branch IS the success,
    with or without a redirect header.
  * Only report failure when no such pull appeared. A false failure is worse than a
    loud error here, because the caller's remedy is to retry.

USAGE:
  tangled_pr.py --owner notplants-bot.bsky.social --repo lichen.page \
      --target main --source hardening-review \
      [--fork did:plc:<forkRepoDid>] [--title "..."] [--body "..."] [--dry-run]
"""
import argparse, os, sys, urllib.request, urllib.parse, urllib.error

BASE = "https://tangled.org"

def load_cookie(path=None):
    """The cookie lives in the encrypted store (tangled.cookie) — see engine/README.md (Secrets).
    `path` is a legacy escape hatch: a TANGLED_COOKIE=... file, used only if explicitly passed."""
    if path:
        for line in open(path):
            if line.strip().startswith("TANGLED_COOKIE="):
                return line.strip()[len("TANGLED_COOKIE="):]
        sys.exit(f"{path} has no TANGLED_COOKIE= line")
    sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
    import secrets as _store
    c = _store.get("tangled.cookie")
    if not c:
        sys.exit("no tangled.cookie in the secret store — add it with: sops /secrets/store.yaml")
    return c


# ── verification against the artifact, not the response ─────────────────────────
# These are pure so they can be unit-tested without a network: see tests/test_tools.py.

import re

def pull_numbers(html):
    """Every pull number linked from a pulls index page."""
    return {int(n) for n in re.findall(r'/pulls/(\d+)\b', html or "")}

def page_names_branch(html, branch):
    """True if a pull's page mentions this source branch.

    Substring rather than token: Tangled renders the branch inside markup we do not
    control, and a false NEGATIVE here would resurrect the duplicate bug. A false
    positive is caught by the caller, which only asks about pulls it just created.
    """
    return bool(branch) and branch in (html or "")

def classify(before, after, branch_pages):
    """Decide what a POST did, from the pulls list before and after it.

    before/after: sets of pull numbers. branch_pages: {number: names_our_branch}.
    Returns (verdict, pull_number) where verdict is:
      "created"    a new pull naming our branch appeared
      "unrelated"  new pulls appeared but none is ours (someone else was filing)
      "none"       nothing new appeared — a real failure
    """
    fresh = sorted(after - before)
    ours = [n for n in fresh if branch_pages.get(n)]
    if ours:
        return "created", max(ours)
    if fresh:
        return "unrelated", None
    return "none", None

def fetch(url, cookie, timeout=30):
    """GET a page as the bot. Returns "" on any error: callers treat an unreadable page as
    'cannot confirm', never as 'confirmed absent' — the difference is the duplicate bug."""
    try:
        r = urllib.request.Request(url, headers={"Cookie": cookie, "User-Agent": "tangled-pr-bot"})
        return urllib.request.urlopen(r, timeout=timeout).read().decode("utf-8", "replace")
    except Exception:
        return ""

def index_pulls(owner, repo, cookie, _fetch=fetch):
    return pull_numbers(_fetch(f"{BASE}/{owner}/{repo}/pulls", cookie))

def pull_names_branch(owner, repo, n, branch, cookie, _fetch=fetch):
    return page_names_branch(_fetch(f"{BASE}/{owner}/{repo}/pulls/{n}", cookie), branch)

def existing_pull_for(owner, repo, branch, cookie, numbers, scan=12, _fetch=fetch):
    """The newest pull whose page names this branch, scanning back `scan` pulls.

    Bounded on purpose: an unbounded scan would fetch hundreds of pages on every call.
    Scanning the newest N is enough for the duplicate this guards against, which is a
    retry seconds after the original. Returns None if none is found within the window —
    which is 'not seen', not 'does not exist', and the post-check is the real backstop.
    """
    for n in sorted(numbers, reverse=True)[:scan]:
        if pull_names_branch(owner, repo, n, branch, cookie, _fetch):
            return n
    return None

def main():
    ap = argparse.ArgumentParser(description="file a Tangled PR via a reused session cookie")
    ap.add_argument("--owner", required=True)
    ap.add_argument("--repo", required=True)
    ap.add_argument("--target", required=True, help="target branch (merge into), e.g. main")
    ap.add_argument("--source", required=True, help="source branch (the changes), e.g. hardening-review")
    ap.add_argument("--fork", default="", help="fork repoDid for a cross-fork PR (omit for same-repo)")
    ap.add_argument("--title", default="")
    ap.add_argument("--body", default="")
    ap.add_argument("--cookie-file", default=None, help="legacy: read the cookie from this file instead of the store")
    ap.add_argument("--dry-run", action="store_true", help="check for an existing pull and show the request; do not POST")
    ap.add_argument("--allow-duplicate", action="store_true", help="post even if a pull for this branch already exists")
    a = ap.parse_args()

    cookie = load_cookie(a.cookie_file)
    # Tangled's PR form is htmx: it POSTs to /pulls/new (NOT /pulls/, which is 405) and only processes
    # the create when it sees the HX-Request header — otherwise it just re-renders the page (a 200 that
    # creates nothing). Success is signalled by an HX-Redirect header pointing at the new pull.
    # PRE-CHECK: refuse to create a second pull for a branch that already has one. This is what
    # makes a retry harmless rather than duplicating — the failure mode that produced #397-#399.
    before = index_pulls(a.owner, a.repo, cookie)
    if not before:
        print("WARNING: could not read the pulls index — cannot check for an existing pull, and "
              "cannot confirm the result afterwards. Proceeding, but verify by hand.")
    dup = existing_pull_for(a.owner, a.repo, a.source, cookie, before) if before else None
    if dup and not a.allow_duplicate:
        print(f"REFUSING: #{dup} already exists for source branch {a.source!r}.")
        print(f"  {BASE}/{a.owner}/{a.repo}/pulls/{dup}")
        print("  To push new commits to an existing pull use tangled_pr_resubmit.py.")
        print("  To file anyway (rarely right): --allow-duplicate")
        sys.exit(3)

    new_url = f"{BASE}/{a.owner}/{a.repo}/pulls/new"
    form = {
        "source": "branch",           # branch-compare mode (each PR targets the branch below it)
        "targetBranch": a.target,
        "sourceBranch": a.source,
        "title": a.title, "titleDirty": "true",
        "body": a.body,   "bodyDirty":  "true",
    }
    if a.fork:  form["fork"] = a.fork
    data = urllib.parse.urlencode(form).encode()
    req = urllib.request.Request(new_url, data=data, method="POST", headers={
        "Cookie": cookie,
        "Content-Type": "application/x-www-form-urlencoded",
        "User-Agent": "tangled-pr-bot",
        "HX-Request": "true",
        "HX-Current-URL": new_url,
        "Referer": new_url,
    })

    if a.dry_run:
        print(f"DRY RUN: would POST {new_url}")
        print(f"  target={a.target!r} source={a.source!r} title={a.title[:60]!r}")
        print(f"  existing pull for this branch: {dup if dup else 'none found in the scan window'}")
        return

    class NoRedirect(urllib.request.HTTPRedirectHandler):
        def redirect_request(self, *args, **kw):  # capture the redirect instead of following it
            return None
    opener = urllib.request.build_opener(NoRedirect)
    try:
        r = opener.open(req, timeout=90)
        hdrs, code, body = r.headers, r.getcode(), r.read(3000).decode("utf-8", "replace")
    except urllib.error.HTTPError as e:
        hdrs, code, body = e.headers, e.code, e.read(3000).decode("utf-8", "replace")
    # htmx success is signalled by HX-Redirect (…/pulls/<n>), not a normal 3xx Location.
    target = hdrs.get("HX-Redirect", "") or hdrs.get("HX-Location", "") or hdrs.get("Location", "")
    print(f"HTTP {code}" + (f" -> {target}" if target else ""))

    if "/login" in target or "oauth" in target.lower():
        sys.exit("AUTH FAILED: session cookie expired/invalid — refresh the cookie in the secret "
                 "store (scripts/get-tangled-cookie.py)")

    # POST-CHECK: the artifact decides, not the header. A create that answered without a redirect
    # is still a create; reporting it as a failure is what made callers retry into duplicates.
    after = index_pulls(a.owner, a.repo, cookie)
    branch_pages = {n: pull_names_branch(a.owner, a.repo, n, a.source, cookie)
                    for n in sorted(after - before, reverse=True)[:12]}
    verdict, num = classify(before, after, branch_pages)

    if verdict == "created":
        print(f"OK: pull #{num} created -> {BASE}/{a.owner}/{a.repo}/pulls/{num}")
        if not target:
            print("  (no redirect header — confirmed against the pulls list instead)")
        return
    if verdict == "unrelated":
        print("  new pulls appeared but none names this source branch — not ours.")
    snippet = " ".join(body.split())[:600]
    print("FAILED: no pull for this branch appeared. Response snippet:")
    print("  " + snippet)
    sys.exit(2)

if __name__ == "__main__":
    main()