feat(1e): HC3 additive generic + op/assertion split (orchestrator owns the op)

- orchestrator: per mutating tier, run optional pre-op seed hook (ops.py pre_<op>) → perform the op
  ONCE (harness-owned) → run generic assertion (unless opted out) AND overlay assertion, both against
  the shared post-op deployment. Op results passed op→assertion via run-scoped CCCI_OP_STATE_FILE.
- opt-out: CCCI_SKIP_GENERIC / CCCI_SKIP_GENERIC_<OP> / recipe_meta.SKIP_GENERIC (declarative).
- generic.py: split do_* into op primitives (perform_upgrade/backup/restore) + assertions
  (assert_upgraded/backup_artifact/restore_healthy) reading op_state(); deployed_identity now returns
  {version,image,chaos} (chaos label ready for HC1).
- generic test_<op>.py + all 6 recipe overlays migrated to assertion-only; pre-op seeding moved to
  per-recipe ops.py (pre_upgrade/pre_backup/pre_restore). install overlays unchanged (no op).
- deploy-count stays 1 (op primitives never call deploy_app). lint PASS; 8 unit tests PASS on cc-ci.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

tests/keycloak/ops.py

@@ -0,0 +1,33 @@
+"""keycloak — pre-op seed hooks (Phase 1e HC3). The orchestrator runs these BEFORE the op; the
+matching test_<op>.py asserts post-op (assertion-only). The data marker is a realm in mariadb,
+written via the keycloak admin API (kc_admin)."""
+
+import os
+import sys
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..", "runner"))
+import kc_admin  # noqa: E402
+from harness import generic  # noqa: E402
+
+
+def _token(domain):
+    return kc_admin.admin_token(domain, kc_admin.admin_password(domain))
+
+
+def pre_upgrade(domain, meta):
+    # create the marker realm (DB data) before the upgrade so the overlay can prove it survives
+    assert kc_admin.create_marker_realm(domain, _token(domain)) in (201, 409)
+
+
+def pre_backup(domain, meta):
+    # establish the marker realm before the backup op captures mariadb
+    assert kc_admin.create_marker_realm(domain, _token(domain)) in (201, 409)
+
+
+def pre_restore(domain, meta):
+    # backup-bot-two cycles the keycloak container during backup → wait for serving, re-auth, then
+    # delete the realm (diverge from the backup) so a successful restore is observable
+    generic.assert_serving(domain, meta)
+    tok = _token(domain)
+    assert kc_admin.delete_marker_realm(domain, tok) in (204, 200)
+    assert not kc_admin.marker_realm_exists(domain, tok), "delete did not take"

tests/keycloak/test_backup.py

@@ -1,7 +1,9 @@
-"""keycloak — BACKUP overlay (Phase 1d, DG4): seed a known state (the marker realm in mariadb),
-back it up (assert a snapshot artifact), then mutate (delete the realm) so the RESTORE overlay
-(test_restore.py) can prove the backed-up state returns. Runs on the shared deployment; the mutated
-state persists for the restore tier."""
+"""keycloak — BACKUP overlay (Phase 1e HC3): assertion-only + additive.
+
+ops.pre_backup created the marker realm before the backup op captured mariadb; the orchestrator
+performed the backup once (generic tier asserted a snapshot artifact). This overlay ADDS: the marker
+realm is present at backup time. backup-bot-two cycles the container during backup, so wait for
+serving + re-auth first. The backup→restore divergence (deleting the realm) is in ops.pre_restore."""
 
 import os
 import sys
@@ -11,22 +13,7 @@ import kc_admin  # noqa: E402
 from harness import generic  # noqa: E402
 
 
-def test_backup_captures_state(live_app, meta):
-    domain = live_app
-    pw = kc_admin.admin_password(domain)
-    tok = kc_admin.admin_token(domain, pw)
-
-    # 1) create the marker realm, then back up (reuse the generic op: backup + assert a snapshot)
-    assert kc_admin.create_marker_realm(domain, tok) in (201, 409)
-    assert kc_admin.marker_realm_exists(domain, tok)
-    snap = generic.do_backup(domain)
-    assert snap, "backup produced no snapshot artifact"
-
-    # backup-bot-two cycles the keycloak container during backup, so the admin API is briefly 502.
-    # Wait for it to be serving again, then re-auth, before mutating via the HTTP admin API.
-    generic.assert_serving(domain, meta)
-    tok = kc_admin.admin_token(domain, pw)
-
-    # 2) mutate: delete the realm (diverge from the backup)
-    assert kc_admin.delete_marker_realm(domain, tok) in (204, 200)
-    assert not kc_admin.marker_realm_exists(domain, tok), "delete did not take"
+def test_backup_captures_realm(live_app, meta):
+    generic.assert_serving(live_app, meta)  # container cycled during backup; wait for it to be back
+    tok = kc_admin.admin_token(live_app, kc_admin.admin_password(live_app))
+    assert kc_admin.marker_realm_exists(live_app, tok), "marker realm not present at backup time"

tests/keycloak/test_restore.py

@@ -1,22 +1,16 @@
-"""keycloak — RESTORE overlay (Phase 1d, DG4): data-integrity, extends the generic restore.
+"""keycloak — RESTORE overlay (Phase 1e HC3): data-integrity, assertion-only + additive.
 
-Runs after the backup overlay (test_backup.py) on the SAME shared deployment, which left the marker
-realm deleted after backing it up. This restores the snapshot via the shared op helper
-(`generic.do_restore`, which also asserts the app is healthy + serving afterwards), then asserts the
-marker realm returned (mariadb restored to the backed-up state) — the app-specific data integrity
-the generic restore cannot check. Assertion-only (no deploy/teardown)."""
+ops.pre_restore deleted the marker realm (diverge from the backup); the orchestrator restored once
+(generic tier asserted healthy/serving). This overlay ADDS: the marker realm returned (mariadb
+restored to the backed-up state). Re-auths post-restore."""
 
 import os
 import sys
 
 sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..", "runner"))
 import kc_admin  # noqa: E402
-from harness import generic  # noqa: E402
 
 
-def test_restore_returns_state(live_app, meta):
-    domain = live_app
-    generic.do_restore(domain, meta)  # restore + assert healthy/serving
-    pw = kc_admin.admin_password(domain)
-    tok = kc_admin.admin_token(domain, pw)
-    assert kc_admin.marker_realm_exists(domain, tok), "restore did not bring back the realm"
+def test_restore_returns_realm(live_app):
+    tok = kc_admin.admin_token(live_app, kc_admin.admin_password(live_app))
+    assert kc_admin.marker_realm_exists(live_app, tok), "restore did not bring back the realm"

tests/keycloak/test_upgrade.py

@@ -1,28 +1,16 @@
-"""keycloak — UPGRADE overlay (Phase 1d, DG4): data-continuity, extends the generic upgrade.
+"""keycloak — UPGRADE overlay (Phase 1e HC3): data-continuity, assertion-only + additive.
 
-The orchestrator deployed the previous published version ONCE; this overlay creates a marker realm
-(DB data in mariadb) on the live app, performs the in-place upgrade via the shared op helper
-(`generic.do_upgrade`, which also asserts reconverge + serving + that the deployment moved), then
-asserts the realm SURVIVED (mariadb data preserved). Assertion-only on the shared deployment."""
+ops.pre_upgrade created a marker realm (mariadb) before the upgrade; the orchestrator performed the
+upgrade once (generic tier asserted reconverge/serving/moved). This overlay ADDS: the realm survived
+(mariadb data preserved). Re-auths post-upgrade."""
 
 import os
 import sys
 
 sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..", "runner"))
 import kc_admin  # noqa: E402
-from harness import generic  # noqa: E402
 
 
-def test_upgrade_preserves_realm(live_app, meta):
-    domain = live_app
-    pw = kc_admin.admin_password(domain)
-    tok = kc_admin.admin_token(domain, pw)
-    assert kc_admin.create_marker_realm(domain, tok) in (201, 409)
-    assert kc_admin.marker_realm_exists(domain, tok), "marker realm not created"
-
-    # in-place upgrade previous -> target (reuses the generic op: upgrade + assert reconverge/serving)
-    generic.do_upgrade(domain, os.environ.get("VERSION") or None, meta)
-
-    # re-auth (token from the old instance is fine, but get a fresh one post-upgrade) and verify
-    tok2 = kc_admin.admin_token(domain, pw)
-    assert kc_admin.marker_realm_exists(domain, tok2), "realm did not survive the upgrade"
+def test_upgrade_preserves_realm(live_app):
+    tok = kc_admin.admin_token(live_app, kc_admin.admin_password(live_app))
+    assert kc_admin.marker_realm_exists(live_app, tok), "realm did not survive the upgrade"