From 300e69d3b3371ce812460144e2e51c39743e3b90 Mon Sep 17 00:00:00 2001 From: notplants-bot Date: Sat, 1 Aug 2026 16:58:26 +0000 Subject: [PATCH] refactor(secrets): consumers read the store; drop materialized copies MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Materializing wrote a second plaintext file per secret, which is the problem the store was meant to solve: two files drift, and the copy is what ends up committed or grepped. - tangled_pr / tangled_pr_edit / tangled_repo now read tangled.cookie from the store. engine/.tangled-session is deleted; --cookie-file remains as a legacy escape hatch. - materialize() is replaced by run-time injection that leaves nothing at rest: exec-env -- cmd group as env vars (use this instead of a systemd EnvironmentFile — same effect, no plaintext on disk) with-file -- cmd {} 0600 file in a private tmpdir, removed when cmd exits, for consumers that insist on a path (ssh -i, a TLS key) Co-Authored-By: Claude --- README.md | 21 +++++++++--- secrets.py | 82 +++++++++++++++++++++++++++------------------- tangled_pr.py | 25 ++++++++------ tangled_pr_edit.py | 25 ++++++++------ tangled_repo.py | 24 +++++++++----- 5 files changed, 110 insertions(+), 67 deletions(-) diff --git a/README.md b/README.md index ee35ee7..10ad7b3 100644 --- a/README.md +++ b/README.md @@ -385,10 +385,23 @@ python3 engine/secrets.py materialize tangled-session # write a runtime file f sops /secrets/store.yaml # add/edit: decrypts to $EDITOR, re-encrypts on save ``` -**Materialized files.** Some consumers read a fixed path and can't be taught otherwise (a systemd -`EnvironmentFile`, an ssh `IdentityFile`, nix's `authKeyFile`). Those files still exist at 0600, -but **the store is the source of truth** — `materialize` rewrites them from it. Never hand-edit a -materialized file: edit the store and re-materialize, or the two silently drift. +**No second copies.** A secret must never be written to a second file "so something can read +it" — copies drift from the store, get committed, and widen what a stray `grep` or an attacker +finds. Our own code imports this module. Anything else gets the value at **run time**: + +```sh +# a group as environment variables — nothing touches the disk +python3 engine/secrets.py exec-env cc_ci_testenv -- some-command + +# a consumer that insists on a path: 0600 file in a private tmpdir, deleted when the command exits +python3 engine/secrets.py with-file ssh_keys.tangled-ed25519 -- ssh -i {} host +``` + +For **systemd**, wrap `ExecStart` in `exec-env` rather than using an `EnvironmentFile`: same +effect, no plaintext at rest. The genuine exceptions are OS-level paths that are read before any +of this exists — nix's `authKeyFile`, sshd host keys, and ssh client keys used by bare `git push`. +Those stay where the OS expects them; do not also copy them into the store, or you have two +sources of truth again. **Rules of thumb** diff --git a/secrets.py b/secrets.py index 805ec14..4601944 100755 --- a/secrets.py +++ b/secrets.py @@ -20,25 +20,26 @@ USAGE (CLI): python3 engine/secrets.py get tangled.cookie # value to stdout (careful in logs) python3 engine/secrets.py materialize # write a runtime file a consumer needs -MATERIALIZED FILES: some consumers read a fixed path and cannot be taught otherwise (a -systemd EnvironmentFile, an ssh IdentityFile, `nix`'s authKeyFile). Those files still exist -on disk at 0600, but the STORE IS THE SOURCE OF TRUTH — `materialize` rewrites them from it. -Never edit a materialized file by hand; edit the store (`sops /secrets/store.yaml`) and -re-materialize, or the two silently drift. +NO SECOND COPIES. A secret must not be written to a second file "so something can read it" — +copies drift from the store, get committed, and multiply what an attacker (or a careless +`grep`) can find. Consumers read the store: our own code imports this module; anything else +gets the value injected at RUN TIME and nothing is left at rest. + + secrets.py exec-env cc_ci_testenv -- some-command # group as env vars, no file + secrets.py with-file ssh_keys.tangled-ed25519 -- ssh -i {} host # 0600 file in a private + # tmpdir, deleted when the command exits + +For systemd, wrap ExecStart in `exec-env` instead of using an EnvironmentFile — same effect, +no plaintext on disk. The few OS-level paths that genuinely cannot be taught this (nix's +`authKeyFile`, sshd host keys) are the exception, and are noted in engine/README.md. ADDING A SECRET: sops /secrets/store.yaml (opens decrypted in $EDITOR, re-encrypts on save) """ -import json, os, subprocess, sys, pathlib +import json, os, subprocess, sys, pathlib, tempfile, shutil STORE = os.environ.get("AO_SECRETS_STORE", "/secrets/store.yaml") AGE_KEY = os.environ.get("SOPS_AGE_KEY_FILE", os.path.expanduser("~/.config/sops/age/keys.txt")) -# name -> (path, mode). Files a consumer reads from a fixed location; see MATERIALIZED FILES. -MATERIALIZE = { - "tangled-session": ("engine/.tangled-session", 0o600), # relative to the project dir - "cc-ci-testenv": ("/srv/cc-ci/.testenv", 0o600), -} - def _load(): """Decrypt the store. Fails loudly: a silent empty dict would look like 'no secrets'.""" @@ -67,23 +68,32 @@ def get_group(name): return _load().get(name, {}) -def materialize(name, project_dir="."): - """Write a runtime file from the store. Returns the path written.""" - if name == "tangled-session": - cookie = get("tangled.cookie") - if not cookie: - sys.exit("store has no tangled.cookie") - p = pathlib.Path(project_dir) / MATERIALIZE[name][0] - body = f"TANGLED_COOKIE={cookie}\n" - elif name == "cc-ci-testenv": - p = pathlib.Path(MATERIALIZE[name][0]) - body = "".join(f"{k}={v}\n" for k, v in get_group("cc_ci_testenv").items()) - else: - sys.exit(f"unknown materialize target {name!r} — known: {', '.join(MATERIALIZE)}") - p.parent.mkdir(parents=True, exist_ok=True) - p.write_text(body) - p.chmod(MATERIALIZE[name][1]) - return str(p) +def exec_env(group, argv): + """Run argv with `group`'s keys added to the environment. Nothing touches the disk.""" + vals = get_group(group) + if not vals: + sys.exit(f"no group {group!r} in the store (secrets.py list)") + env = {**os.environ, **{k: str(v) for k, v in vals.items()}} + return subprocess.call(argv, env=env) + + +def with_file(dotted, argv): + """Run argv with `{}` replaced by a 0600 temp file holding the value. + + The file lives in a private 0700 dir and is removed when the command exits — so a consumer + that insists on a path (ssh -i, a TLS key) never leaves a lasting second copy of the secret. + """ + val = get(dotted) + if val is None: + sys.exit(f"no such key: {dotted}") + d = tempfile.mkdtemp(prefix="ao-secret-") # mkdtemp is 0700 + try: + p = pathlib.Path(d) / dotted.split(".")[-1] + p.write_text(val if isinstance(val, str) else json.dumps(val)) + p.chmod(0o600) + return subprocess.call([a.replace("{}", str(p)) for a in argv]) + finally: + shutil.rmtree(d, ignore_errors=True) def main(): @@ -102,12 +112,16 @@ def main(): if v is None: sys.exit(f"no such key: {argv[1]}") print(v if isinstance(v, str) else json.dumps(v)) - elif cmd == "materialize": - if len(argv) < 2: - sys.exit(f"usage: secrets.py materialize <{'|'.join(MATERIALIZE)}>") - print("wrote", materialize(argv[1], argv[2] if len(argv) > 2 else ".")) + elif cmd in ("exec-env", "with-file"): + if "--" not in argv: + sys.exit(f"usage: secrets.py {cmd} -- ") + i = argv.index("--") + if i != 2: + sys.exit(f"usage: secrets.py {cmd} -- ") + run = exec_env if cmd == "exec-env" else with_file + sys.exit(run(argv[1], argv[i + 1:])) else: - sys.exit(f"unknown command {cmd!r} — list | get | materialize") + sys.exit(f"unknown command {cmd!r} — list | get | exec-env | with-file") if __name__ == "__main__": diff --git a/tangled_pr.py b/tangled_pr.py index abc5328..9fe75d6 100755 --- a/tangled_pr.py +++ b/tangled_pr.py @@ -26,15 +26,20 @@ import argparse, os, sys, urllib.request, urllib.parse, urllib.error BASE = "https://tangled.org" -def load_cookie(path): - if not os.path.exists(path): - sys.exit(f"no cookie file at {path} — do the one-time browser login " - f"(see machine-docs/tangled-pr-automation.md)") - for line in open(path): - line = line.strip() - if line.startswith("TANGLED_COOKIE="): - return line[len("TANGLED_COOKIE="):] - sys.exit("cookie file present but has no TANGLED_COOKIE= line") +def load_cookie(path=None): + """The cookie lives in the encrypted store (tangled.cookie) — see engine/README.md (Secrets). + `path` is a legacy escape hatch: a TANGLED_COOKIE=... file, used only if explicitly passed.""" + if path: + for line in open(path): + if line.strip().startswith("TANGLED_COOKIE="): + return line.strip()[len("TANGLED_COOKIE="):] + sys.exit(f"{path} has no TANGLED_COOKIE= line") + sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + import secrets as _store + c = _store.get("tangled.cookie") + if not c: + sys.exit("no tangled.cookie in the secret store — add it with: sops /secrets/store.yaml") + return c def main(): ap = argparse.ArgumentParser(description="file a Tangled PR via a reused session cookie") @@ -45,7 +50,7 @@ def main(): ap.add_argument("--fork", default="", help="fork repoDid for a cross-fork PR (omit for same-repo)") ap.add_argument("--title", default="") ap.add_argument("--body", default="") - ap.add_argument("--cookie-file", default=os.path.join(os.path.dirname(os.path.abspath(__file__)), ".tangled-session")) + ap.add_argument("--cookie-file", default=None, help="legacy: read the cookie from this file instead of the store") a = ap.parse_args() cookie = load_cookie(a.cookie_file) diff --git a/tangled_pr_edit.py b/tangled_pr_edit.py index 0ad7756..08d51f5 100644 --- a/tangled_pr_edit.py +++ b/tangled_pr_edit.py @@ -22,15 +22,20 @@ import argparse, html, os, re, sys, urllib.request, urllib.parse, urllib.error BASE = "https://tangled.org" -def load_cookie(path): - if not os.path.exists(path): - sys.exit(f"no cookie file at {path} — do the one-time browser login " - f"(see machine-docs/tangled-pr-automation.md)") - for line in open(path): - line = line.strip() - if line.startswith("TANGLED_COOKIE="): - return line[len("TANGLED_COOKIE="):] - sys.exit("cookie file present but has no TANGLED_COOKIE= line") +def load_cookie(path=None): + """The cookie lives in the encrypted store (tangled.cookie) — see engine/README.md (Secrets). + `path` is a legacy escape hatch: a TANGLED_COOKIE=... file, used only if explicitly passed.""" + if path: + for line in open(path): + if line.strip().startswith("TANGLED_COOKIE="): + return line.strip()[len("TANGLED_COOKIE="):] + sys.exit(f"{path} has no TANGLED_COOKIE= line") + sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + import secrets as _store + c = _store.get("tangled.cookie") + if not c: + sys.exit("no tangled.cookie in the secret store — add it with: sops /secrets/store.yaml") + return c def fetch_form(edit_url, cookie): """GET the htmx edit form; return (title, body) as the appview holds them.""" @@ -54,7 +59,7 @@ def main(): ap.add_argument("--body", default=None) ap.add_argument("--body-file", default=None, help="read the new body from a file (overrides --body)") ap.add_argument("--show", action="store_true", help="print current title+body and exit (no edit)") - ap.add_argument("--cookie-file", default=os.path.join(os.path.dirname(os.path.abspath(__file__)), ".tangled-session")) + ap.add_argument("--cookie-file", default=None, help="legacy: read the cookie from this file instead of the store") a = ap.parse_args() cookie = load_cookie(a.cookie_file) diff --git a/tangled_repo.py b/tangled_repo.py index 32d35ab..4378b53 100755 --- a/tangled_repo.py +++ b/tangled_repo.py @@ -13,14 +13,20 @@ import argparse, os, sys, urllib.parse, urllib.request BASE = "https://tangled.org" -def load_cookie(path): - if not os.path.exists(path): - sys.exit(f"no cookie file at {path} — refresh it with scripts/get-tangled-cookie.py") - for line in open(path): - line = line.strip() - if line.startswith("TANGLED_COOKIE="): - return line[len("TANGLED_COOKIE="):] - sys.exit("cookie file present but has no TANGLED_COOKIE= line") +def load_cookie(path=None): + """The cookie lives in the encrypted store (tangled.cookie) — see engine/README.md (Secrets). + `path` is a legacy escape hatch: a TANGLED_COOKIE=... file, used only if explicitly passed.""" + if path: + for line in open(path): + if line.strip().startswith("TANGLED_COOKIE="): + return line.strip()[len("TANGLED_COOKIE="):] + sys.exit(f"{path} has no TANGLED_COOKIE= line") + sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + import secrets as _store + c = _store.get("tangled.cookie") + if not c: + sys.exit("no tangled.cookie in the secret store — add it with: sops /secrets/store.yaml") + return c def main(): ap = argparse.ArgumentParser(description="create a Tangled repo via a reused session cookie") @@ -28,7 +34,7 @@ def main(): ap.add_argument("--description", default="") ap.add_argument("--branch", default="main", help="default branch (form default: main)") ap.add_argument("--domain", default="knot1.tangled.sh", help="knot to host on (radio value)") - ap.add_argument("--cookie-file", default=os.path.join(os.path.dirname(os.path.abspath(__file__)), ".tangled-session")) + ap.add_argument("--cookie-file", default=None, help="legacy: read the cookie from this file instead of the store") a = ap.parse_args() cookie = load_cookie(a.cookie_file)