From 751fa518e65364ca836649cbb51303e212d8c9ac Mon Sep 17 00:00:00 2001 From: kawaiipunk Date: Wed, 18 Nov 2026 16:45:00 +0000 Subject: [PATCH] Document behaviour, invariants and architecture - expand README: post cadence, architecture, config reference, ops, troubleshooting, limitations - add AGENTS.md with hard rules and domain gotchas - comment the non-obvious parts: window clamping, reminder fallback, series vs occurrence, outbox lifecycle, image failure handling --- AGENTS.md | 108 ++++++++++++++++ README.md | 243 ++++++++++++++++++++++++------------ src/calbot/expand.py | 21 +++- src/calbot/feed.py | 7 ++ src/calbot/format.py | 12 ++ src/calbot/images.py | 12 ++ src/calbot/scheduler.py | 61 ++++++++- src/calbot/signal_client.py | 2 + src/calbot/store.py | 24 +++- src/calbot/watchdog.py | 6 + 10 files changed, 413 insertions(+), 83 deletions(-) create mode 100644 AGENTS.md diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..113daf5 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,108 @@ +# AGENTS.md + +Guidance for AI agents (and humans) changing this repository. + +## What this is + +A single-purpose service: it polls an iCalendar feed (The Events Calendar / +MCR Commons) and posts events, with poster images, to one Signal group. Signal +has no bot API, so sending goes through `signal-cli-rest-api` in Docker. + +Read `README.md` for the user-facing behaviour and setup. + +## Layout + +``` +src/calbot/ + config.py TOML loading, defaults, duration/time parsers + feed.py conditional GET; validators committed only after parse + expand.py iCalendar -> Event; series-ID heuristics; recurrence expansion + format.py the group's post template (title/hashtags/time/venue/link) + images.py page scraping (venue, og:image) + poster download/resize/cache + store.py SQLite schema and all persistence + scheduler.py windows, reminder planning, dispatch/batching, Bot.tick + signal_client.py /v2/send HTTP client + watchdog.py throttled operator alerts + __main__.py CLI: run loop, --dump, --check, --once +tests/ pytest; fixtures are *.ics.txt (never *.ics) +``` + +## Commands + +```bash +python -m venv .venv && . .venv/bin/activate +pip install -e '.[dev]' + +pytest # must pass, no network +calbot --config config.toml --dump # render the feed without sending +calbot --config config.toml --check # config + Signal account check +calbot --config config.toml --once # one tick +``` + +`--dump` uses a throwaway database and never sends; keep it that way. + +## Hard rules + +1. **Never commit calendar data.** `*.ics` is gitignored; test fixtures must be + named `*.ics.txt`. `config.toml`, `signal-data/`, `data/`, `cache/` are + local-only. +2. **UTC inside, local only at the edges.** Store and compare UTC. Convert to + `[calendar].timezone` only for window maths and rendering. Never use naive + `datetime.now()` in code that persists a timestamp. +3. **`tick()` must never raise.** Feed, scrape, image, and send problems are + logged and reported to the watchdog; the loop has to survive all of them. + The one catch-all in `__main__` is a backstop, not a design. +4. **Images are optional; text is not.** Any image or scraping failure must + degrade to a text-only post. Never let enrichment raise through `_compose`. +5. **Plan once, persist it.** Reminder times and outbox rows are written when + an event is first seen. Do not recompute schedules on every tick; that + reorders posts when the feed changes or the process restarts. +6. **Delivery is at-least-once.** Write `deliveries` and delete the outbox row + only after a successful send. A crash may duplicate one post; that is + accepted. Do not add silent "mark sent before send" shortcuts. +7. **No network in tests.** Inject `now`, and use the fake sender/scraper/image + service patterns in `tests/test_scheduler.py` (`httpx.MockTransport` for the + feed). + +## Domain gotchas + +- **Series vs occurrence.** TEC's UID is + `{postID}-{startUnix}-{endUnix}@{host}`, and the feed pre-expands + recurrences. Announcements are deduped per `series_id`; reminders per + `Event.key`. Changing this will spam the group with one post per occurrence. +- **Feed is capped** (30 events by default) and contains no past events. Use + the coverage horizon (`max(start)` of the snapshot) before marking events + `gone`; otherwise a busy week "cancels" next month's events. +- **No `LOCATION` on many events.** Venue/postcode come from the event page's + JSON-LD (`Event.location.name` / `address.postalCode`), then the description. + This is cached per URL in `pages`. +- **Poster URLs are full-size originals**, sometimes WebP or a 20 MB phone + photo. Always cap the download, downscale, flatten transparency onto white, + and re-encode as JPEG. +- **Posting windows** (`08:00–10:00`, `18:00–21:00`) are a preference, not a + rule: `clamp_send_at` breaks them when the next window would miss the event + minus `urgent_guard_minutes`. +- **Reminder fallback.** If `start − lead` has already passed when the event is + discovered, the reminder moves to the morning of the event (or earlier, if + the event starts before that morning time). +- **Batching.** All outbox rows due in a tick become one message (text blocks + joined by blank lines, at most `max_images_per_post` attachments). + +## Changing behaviour + +- **New post kind** (e.g. digest): add a `kind` to `outbox`, plan rows in + `_sync`/a new planner, make `_dispatch` treat it like the others, add a + template in `format.py`, and test planning + dispatch with a fake clock. +- **New config option**: add a dataclass field in `config.py` with a sensible + default, parse it in `load_config`, document it in `config.example.toml` and + the README table. +- **Schema change**: append to `SCHEMA` in `store.py`. There is no migration + framework; deleting `data/calbot.sqlite3` re-seeds silently. + +## Before you finish + +- `pytest` green, `python -m pyflakes src tests` clean. +- New behaviour covered by a test (especially failure paths). +- `config.example.toml`, README and AGENTS.md still accurate. +- `git status` shows no `.ics`, `config.toml`, `data/`, `cache/` or + `signal-data/` files. diff --git a/README.md b/README.md index 2632bb4..26e0ee3 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,9 @@ # calbot — iCalendar → Signal -Watches an iCalendar feed (built for The Events Calendar / MCR Commons) and posts -events to a Signal group, with the event's poster image attached. - -Posts look like this: +Polls an iCalendar feed and posts events to a Signal group, with the event's +poster image attached. Built for [MCR Commons](https://commonscalendar.uk) (The +Events Calendar), but works with any feed that exposes `UID`, `SUMMARY`, +`DTSTART`, `DTEND` and optionally `ATTACH`, `LOCATION`, `URL`. ``` Folk Night @@ -13,46 +13,81 @@ Old Print Works B12 9AH https://commonscalendar.uk/event/folk-night/ ``` -## How it behaves +## Behaviour -- **Announcements** — a newly seen event is posted within one refresh interval, - deferred to the next posting window (`08:00–10:00` or `18:00–21:00` by default). -- **Reminders** — every event is posted again 3 days before it starts. If the - event was added less than 3 days ahead, the reminder goes out on the morning - of the event instead. An imminent event breaks the posting window rather than - arriving late. -- **Images** — taken from the feed's `ATTACH` property (The Events Calendar - emits the featured image), falling back to the event page's JSON-LD / `og:image`. - Every post carries an image. -- **Recurring events** — the feed pre-expands occurrences, but announcements are - deduplicated per series, so adding a weekly event produces one announcement. -- **First run** — seeds silently (`announce.backfill = false`); reminders still - work for everything already in the feed. -- **Watchdog** — messages you on Signal if the feed stops working, parses to - nothing, or sending keeps failing. +Each event produces **at most two posts**; every post carries the poster image +when one exists. -## Setup +| Post | When | +|---|---| +| Announcement | As soon as the event is first seen in the feed | +| Reminder | 3 days before it starts | +| Reminder (fallback) | 09:00 on the day, if the event was added under 3 days out | -### 1. Link a Signal account +Posts are deferred to the next preferred window — **08:00–10:00** or +**18:00–21:00** local time — unless waiting would mean posting after the event +starts (or within an hour of it), in which case they go out immediately. -Linking as a secondary device is the quickest route: the bot posts **as your own -number**. For a separate bot identity you must register a dedicated number -instead (see `signal-cli-rest-api` docs). +Everything due in the same window is **batched into one message**. That is also +what merges the "new event" and "today" posts for an event added on the day. + +Other rules: + +- **First run seeds silently** (`announce.backfill = false`), so enabling the + bot does not dump the whole calendar into the group. Reminders still work for + everything already in the feed. +- **Recurring events are announced once per series**, not once per occurrence. + The feed pre-expands recurrences; announcements are keyed on the series. +- **Events that disappear from the feed** (cancelled) have their pending posts + dropped. Events beyond the feed's coverage are left alone — The Events + Calendar only exports ~30 upcoming events. +- **The watchdog** messages your own number ("Note to Self") if the feed keeps + failing, parses to nothing, or sending fails repeatedly. + +## How it works + +``` + feed (ICS) ──► parse/expand ──► SQLite ──► dispatch ──► signal-cli-rest-api + │ │ + reminder_plans + outbox Signal group +``` + +One tick every 30 s: + +1. **Refresh** — conditional GET (ETag/Last-Modified, every `refresh_minutes`). + The feed is parsed and expanded, then reconciled with the database. New + event rows get reminder plans; a new series gets an announcement. +2. **Dispatch** — every outbox row whose time has come is grouped by event, + rendered, and sent as one message. +3. **Heartbeat** — optional once-a-day "alive" note. + +Scheduling decisions are persisted, so restarts and feed edits cannot reorder +or duplicate posts. Delivery is at-least-once: a crash between sending and +committing can duplicate exactly one post. + +## Quick start + +Requirements: Docker (and Docker Compose). A Signal account to link. + +### 1. Start signal-cli-rest-api and link a device ```bash mkdir -p signal-data data cache docker compose up -d signal-api ``` -Open and scan the QR -code from **Signal → Settings → Linked devices**. +Open and scan it from +**Signal → Settings → Linked devices**. + +> Linking as a secondary device means the bot posts **as your own number**. +> For a separate identity, register a dedicated number instead (see the +> `signal-cli-rest-api` docs on `/v1/register`). + +Fetch the linked number and the group ID: ```bash -# the linked number: curl -s http://localhost:8080/v1/accounts - -# groups for that number — copy the internal_id (starts with "group."): -curl -s http://localhost:8080/v1/groups/+447700900000 | jq +curl -s http://localhost:8080/v1/groups/+447700900000 | jq -r '.[].internal_id' ``` ### 2. Configure @@ -60,67 +95,119 @@ curl -s http://localhost:8080/v1/groups/+447700900000 | jq ```bash cp config.example.toml config.toml chmod 600 config.toml -$EDITOR config.toml ``` -Fill in `signal.number`, `signal.group_id`, and `calendar.url`. `webcal://` -URLs are accepted and fetched over https. +At minimum fill in `signal.number`, `signal.group_id` (including the `group.` +prefix) and `calendar.url`. `webcal://` URLs are accepted and fetched over +https. -### 3. Check, preview, run +### 3. Verify, preview, run ```bash -# is the account linked and the config valid? -docker compose run --rm bot python -m calbot --check - -# print the rendered posts for the feed without sending anything -docker compose run --rm bot python -m calbot --dump - -# start the bot +docker compose run --rm bot python -m calbot --check # config + account +docker compose run --rm bot python -m calbot --dump # rendered posts, no sending docker compose up -d bot docker compose logs -f bot ``` -`--once` runs a single tick and exits. +## CLI -## Install by hand +| Command | Purpose | +|---|---| +| `calbot --config config.toml` | Run the loop | +| `calbot --once` | Run a single tick and exit | +| `calbot --dump` | Print rendered posts for the feed; uses a temporary DB, sends nothing | +| `calbot --check` | Validate config and confirm the Signal account is linked | + +All commands accept `--log-level` (or `LOG_LEVEL`). + +## Configuration + +`config.example.toml` is commented and complete. The essentials: + +| Key | Default | Meaning | +|---|---|---| +| `signal.api_url` | — | `http://signal-api:8080` inside Compose | +| `signal.number` | — | The linked account's number, international format | +| `signal.group_id` | — | `internal_id` from `/v1/groups/`, starts with `group.` | +| `signal.text_mode` | `normal` | `styled` enables Signal markup | +| `calendar.url` | — | ICS feed (webcal accepted) | +| `calendar.timezone` | `Europe/London` | Used for windows, hashtags and display | +| `calendar.refresh_minutes` | `15` | Feed poll interval | +| `calendar.window_days` | `60` | How far ahead to keep events | +| `posting.windows` | `08:00-10:00`, `18:00-21:00` | Preferred send times | +| `posting.urgent_guard_minutes` | `60` | Break a window when the event is this close | +| `announce.enabled` | `true` | Post when a new series appears | +| `announce.max_days_ahead` | `30` | Ignore announcements further out | +| `announce.backfill` | `false` | Announce everything on first run | +| `reminders.enabled` | `true` | Post before events | +| `reminders.lead_times` | `["3d"]` | Any number of leads (`"3d"`, `"2h"`, `"90m"`) | +| `reminders.fallback_to_day_of` | `true` | Day-of reminder for late additions | +| `reminders.day_of_time` | `09:00` | When that day-of reminder goes out | +| `images.enabled` | `true` | Attach posters | +| `images.max_pixels` | `1600` | Longest edge after resizing | +| `images.max_bytes` | `400000` | JPEG size target | +| `images.max_images_per_post` | `3` | Cap for batched posts | +| `images.scrape_event_page` | `true` | Fetch pages for venue/`og:image` | +| `format.show_*` | all `true` | Toggle each line of the post | +| `format.shorten_links` | `false` | Shorten links via TinyURL | +| `watchdog.enabled` | `true` | Operator alerts | +| `watchdog.notify_number` | own number | Where alerts go | +| `watchdog.feed_failures` | `3` | Consecutive failures before alerting | +| `watchdog.daily_heartbeat` | `false` | Daily "alive" message | + +## Operations + +| Task | How | +|---|---| +| Follow logs | `docker compose logs -f bot` | +| Apply config changes | edit `config.toml`, then `docker compose restart bot` | +| Update code | `git pull && docker compose up -d --build` | +| Reset duplicate/seen state | stop bot, delete `data/calbot.sqlite3`, start (re-seeds silently) | +| Clear image cache | `rm -rf cache/images` | +| Back up | `signal-data/` (linked-device keys) and `config.toml` | + +**Back up `signal-data/`.** It contains the linked-device credentials. Losing it +means re-scanning the QR code; if the phone is re-registered, the link breaks +and must be re-paired. + +## Troubleshooting + +| Symptom | Check | +|---|---| +| `Signal account … not found` | The QR was not scanned, or `signal.number` is wrong (`curl /v1/accounts`). | +| Nothing posts | `--dump` works? Then check windows and whether the first run only seeded. Logs show planned `send_at` only at debug level — check `outbox` in SQLite if needed. | +| Posts appear as you | Expected with a linked device; use a dedicated number for a bot identity. | +| No venue line | The feed has no `LOCATION` and the event page has no JSON-LD venue. | +| No image | Event has no featured image and no `og:image`, or the download exceeded the 20 MB cap. Text still posts. | +| Duplicate post after a crash | At-least-once delivery; expected and bounded to one. | +| Watchdog alert about an empty feed | The site returned an empty calendar; stored events are left untouched. | +| `404` from the feed | The Events Calendar export URL changed; test it in a browser first. | + +## Limitations + +- Signal has no bot API; this relies on `signal-cli` and an unofficial REST + wrapper. Account linking can break when the phone is re-registered. +- Signal message text is not styled by default; hashtags are plain text (Signal + does not linkify them). +- The feed is a snapshot (TEC caps it at ~30 events), not a full history. There + is no reliable cancellation signal beyond "it vanished". +- Rescheduled events keep the same post ID, so they get a fresh reminder but no + new announcement. +- No lockfile yet: `Dockerfile` runs `pip install .`, so builds are not + byte-reproducible. + +## Development ```bash python -m venv .venv && . .venv/bin/activate pip install -e '.[dev]' -calbot --config config.toml --dump -calbot --config config.toml -``` - -## Files that matter - -| Path | Purpose | -|---|---| -| `config.toml` | numbers, group, feed, behaviour (never committed) | -| `signal-data/` | linked-device keys — **back this up**, losing it means re-linking | -| `data/calbot.sqlite3` | event/outbox/delivery state (safe to delete to re-seed) | -| `cache/images/` | downscaled posters | - -Deleting `data/calbot.sqlite3` resets dedupe state: on the next run everything -is seeded silently again, so you will not get a flood, but new-series detection -restarts from that moment. - -## Notes and limits - -- Signal has no bot API; this uses [`signal-cli`](https://github.com/AsamK/signal-cli) - via [`signal-cli-rest-api`](https://github.com/bbernhard/signal-cli-rest-api). -- Delivery is **at-least-once**. A crash between sending and recording can - duplicate one post. -- The feed is capped by The Events Calendar (30 events by default) and may omit - past events; the bot only knows what the feed contains. -- Venue/postcode come from the ICS `LOCATION` when present (this feed often has - none), then from the event page's structured data, then from the description. - -## Tests - -```bash -pip install -e '.[dev]' pytest ``` -Fixtures are stored as `tests/fixtures/*.ics.txt` so that `*.ics` can stay -gitignored. +Tests are offline. They inject a fake clock, sender, scraper and image service, +and use `httpx.MockTransport` for the feed. Calendar fixtures live in +`tests/fixtures/*.ics.txt` — `.ics` is gitignored so raw feeds can never be +committed. + +See `AGENTS.md` for the invariants and gotchas before making changes. diff --git a/src/calbot/expand.py b/src/calbot/expand.py index b082af4..8fcfa16 100644 --- a/src/calbot/expand.py +++ b/src/calbot/expand.py @@ -19,6 +19,12 @@ TEC_UID_RE = re.compile(r"^(\d+)-\d+-\d+@") @dataclass(frozen=True) class Event: + """One concrete occurrence, independent of the feed's file format. + + Times are timezone-aware UTC. ``key`` is unique per occurrence while + ``series_id`` groups recurrences (and TEC's pre-expanded copies) together. + """ + key: str uid: str series_id: str @@ -70,6 +76,13 @@ def _image_url(component) -> str: def _series_id(uid: str, summary: str, start: datetime, duration: timedelta) -> str: + """Identify the series an occurrence belongs to. + + The Events Calendar's UID is ``{postID}-{startUnix}-{endUnix}@{host}``, so + the post ID is a stable series key even after the feed pre-expands the + recurrences. Other feeds fall back to a fingerprint of title, start time + of day and duration. + """ match = TEC_UID_RE.match(uid) if match: return f"tec:{match.group(1)}" @@ -79,6 +92,7 @@ def _series_id(uid: str, summary: str, start: datetime, duration: timedelta) -> def _build_event(component, default_tz: ZoneInfo) -> Event | None: + """Convert one VEVENT (or expanded recurrence) into an Event.""" dtstart = component.get("DTSTART") if dtstart is None: return None @@ -139,7 +153,12 @@ def parse_events( window_end: datetime, default_timezone: str = "Europe/London", ) -> list[Event]: - """Parse an iCalendar document and return events overlapping the window.""" + """Parse an iCalendar document and return events overlapping the window. + + Uses ``recurring-ical-events`` so RRULE/EXDATE/RECURRENCE-ID feeds work, + and falls back to the raw VEVENTs if expansion fails (The Events Calendar + already pre-expands, so the fallback loses nothing there). + """ default_tz = _zone(default_timezone) calendar = icalendar.Calendar.from_ical(raw) diff --git a/src/calbot/feed.py b/src/calbot/feed.py index 7344216..54f4f0d 100644 --- a/src/calbot/feed.py +++ b/src/calbot/feed.py @@ -36,6 +36,13 @@ class FeedResponse: class Feed: + """The iCalendar feed, fetched with conditional requests. + + ``fetch`` deliberately does not persist ETag/Last-Modified; the caller + calls ``commit`` once the body has parsed, so a malformed response cannot + lock itself behind a 304. + """ + def __init__(self, url: str, store: Store, refresh_minutes: int, http: httpx.Client): self.url = normalize_feed_url(url) self.store = store diff --git a/src/calbot/format.py b/src/calbot/format.py index 5e3bb73..86dad6a 100644 --- a/src/calbot/format.py +++ b/src/calbot/format.py @@ -60,6 +60,7 @@ def _clock(moment: datetime) -> tuple[str, str]: def format_time_range(start: datetime, end: datetime) -> str: + """``4.30-6.30pm``; the meridiem is repeated only when it changes.""" start_text, start_meridiem = _clock(start) end_text, end_meridiem = _clock(end) if not end or end <= start: @@ -83,6 +84,17 @@ def render_post( postcode: str = "", link: str = "", ) -> str: + """Render the agreed group format, one field per line:: + + Folk Night + #November #November14 + 4.30-6.30pm + Old Print Works B12 9AH + https://commonscalendar.uk/event/folk-night/ + + Every line is optional (driven by ``[format]`` and by whether the data + exists), which is why the result is assembled rather than templated. + """ start = event.start.astimezone(local_tz) end = event.end.astimezone(local_tz) diff --git a/src/calbot/images.py b/src/calbot/images.py index bc6914d..d53a2f7 100644 --- a/src/calbot/images.py +++ b/src/calbot/images.py @@ -148,6 +148,12 @@ def _first_image(value) -> str: class ImageService: + """Downloads, downscales and caches posters, returning base64 data URIs. + + signal-cli-rest-api takes attachments inline (``base64_attachments``), so + there is no upload step. Resized JPEGs are cached on disk by URL hash. + """ + def __init__(self, config: ImagesConfig, cache_dir: Path, http: httpx.Client): self.config = config self.cache_dir = cache_dir / "images" @@ -155,6 +161,11 @@ class ImageService: self.http = http def data_uri(self, url: str) -> str | None: + """Return a Signal-ready ``data:`` URI, or None. + + Images are strictly optional: every failure path here returns None so + the text post is never blocked by a broken poster. + """ if not self.config.enabled or not url: return None try: @@ -200,6 +211,7 @@ class ImageService: return b"".join(chunks) def _resize(self, raw: bytes) -> bytes | None: + """Decode, orient, downscale and re-encode as JPEG under ``max_bytes``.""" try: with Image.open(io.BytesIO(raw)) as image: image = ImageOps.exif_transpose(image) diff --git a/src/calbot/scheduler.py b/src/calbot/scheduler.py index 031274a..d65fc06 100644 --- a/src/calbot/scheduler.py +++ b/src/calbot/scheduler.py @@ -1,4 +1,17 @@ -"""Decision logic and the main tick loop.""" +"""Decision logic and the main tick loop. + +Flow of one tick: + +1. ``_refresh_feed`` — re-fetch the iCalendar feed when it is due, expand it + into events and reconcile them with the database (``_sync``). +2. ``_dispatch`` — send everything whose outbox row is due, grouped into as + few Signal messages as possible. +3. ``_heartbeat`` — optional once-a-day "still alive" note. + +All scheduling decisions are persisted (``reminder_plans`` and ``outbox``) at +discovery time, so a restart, a feed edit or a sliding window cannot reshuffle +posts that were already planned. Sending is at-least-once. +""" from __future__ import annotations @@ -28,6 +41,7 @@ MAX_SEND_ATTEMPTS = 5 def in_window(moment: datetime, windows: tuple[tuple[time, time], ...]) -> bool: + """True when the local clock time falls inside one of the preferred windows.""" clock = moment.timetz().replace(tzinfo=None) return any(start <= clock < end for start, end in windows) @@ -35,6 +49,7 @@ def in_window(moment: datetime, windows: tuple[tuple[time, time], ...]) -> bool: def next_window_start( moment: datetime, windows: tuple[tuple[time, time], ...] ) -> datetime: + """First window opening strictly after ``moment`` (same timezone).""" for offset in range(0, 4): day = moment.date() + timedelta(days=offset) for start, _end in windows: @@ -51,7 +66,13 @@ def clamp_send_at( windows: tuple[tuple[time, time], ...], guard: timedelta, ) -> datetime: - """Defer a post to the next preferred window unless that misses the deadline.""" + """Defer a post to the next preferred window unless that misses the deadline. + + ``deadline`` is when the post stops being useful (usually the event start). + If the next window opens after ``deadline - guard``, the post goes out as + soon as it is due instead: a reminder that arrives after the event is + worse than one that arrives outside the preferred hours. + """ base = max(due, now) if in_window(base, windows): return base @@ -68,7 +89,16 @@ def reminder_time( config: Config, local_tz: ZoneInfo, ) -> datetime | None: - """When the reminder for this lead should go out (local time), or None.""" + """When the reminder for this lead should go out (local time), or None. + + Normally that is ``start - lead`` moved into the next posting window. + + If the event was discovered *after* that moment ("added 2 days out" with a + 3-day lead) the fallback is the morning of the event, so the group still + gets one heads-up rather than none. For a timed event that starts before + the configured morning time, the fallback moves earlier so it cannot land + after the event has begun. + """ start = event.start.astimezone(local_tz) seen = first_seen.astimezone(local_tz) lead_delta = timedelta(seconds=parse_duration(lead)) @@ -126,6 +156,7 @@ class Bot: # -- loop ------------------------------------------------------------ def tick(self, now: datetime | None = None) -> None: + """One full cycle. ``now`` is injectable so tests can travel in time.""" now = now or datetime.now(timezone.utc) self._refresh_feed(now) self._dispatch(now) @@ -134,6 +165,7 @@ class Bot: # -- feed + planning ------------------------------------------------- def _refresh_feed(self, now: datetime) -> None: + """Fetch (when due), parse and reconcile. Never raises.""" if not self.feed.due(now): return try: @@ -174,6 +206,14 @@ class Bot: self._sync(events, now) def _sync(self, events: list[Event], now: datetime) -> None: + """Reconcile the feed snapshot with the database. + + Only genuinely new event rows cause work: they get reminder plans, and + their series gets an announcement if this is the first occurrence we + have seen. Recurring series are pre-expanded by the feed, so keying on + the series (not the occurrence) is what stops a new weekly event from + producing thirty announcements. + """ first_run = self.store.get_meta("seeded") is None seeding = first_run and not self.config.announce.backfill @@ -200,6 +240,7 @@ class Bot: log.info("first run: seeded %d events", len(events)) def _plan_announcement(self, event: Event, now: datetime) -> None: + """Queue a "new event" post for the first occurrence of a series.""" if not self.config.announce.enabled: return if event.end <= now: @@ -235,6 +276,7 @@ class Bot: ) def _plan_reminders(self, event: Event, now: datetime) -> None: + """Queue the T-minus reminders for a newly discovered event.""" if not self.config.reminders.enabled or event.end <= now: return for lead in self.config.reminders.lead_times: @@ -256,6 +298,13 @@ class Bot: # -- sending --------------------------------------------------------- def _dispatch(self, now: datetime) -> None: + """Send every due outbox row, grouped into one message per batch. + + Rows are grouped by event so an announcement and a reminder that fall + in the same window become a single post rather than two. Items that + have become pointless (event over/started, removed from the feed) are + recorded as delivered and dropped so they are never retried. + """ items = self.store.due_outbox(now) if not items: return @@ -321,6 +370,7 @@ class Bot: def _record_failure( self, batch: list[tuple[Event, list[OutboxItem]]], reason: str, now: datetime ) -> None: + """Count a failed attempt; drop the post after MAX_SEND_ATTEMPTS.""" self.watchdog.send_failed(reason, now) for _event, group in batch: for item in group: @@ -335,6 +385,11 @@ class Bot: self.store.drop_outbox(item.id) def _compose(self, batch: list[tuple[Event, list[OutboxItem]]]) -> tuple[str, list[str]]: + """Render the batch into one message and up to N base64 attachments. + + A single event gets its own image; a multi-event batch is capped at + ``images.max_images_per_post`` so the payload stays reasonable. + """ blocks: list[str] = [] attachments: list[str] = [] limit = self.config.images.max_images_per_post if len(batch) > 1 else 1 diff --git a/src/calbot/signal_client.py b/src/calbot/signal_client.py index b4b2641..c7e0365 100644 --- a/src/calbot/signal_client.py +++ b/src/calbot/signal_client.py @@ -14,6 +14,8 @@ class SignalError(Exception): class SignalClient: + """Thin wrapper over ``signal-cli-rest-api``'s ``/v2/send``.""" + def __init__(self, base_url: str, number: str, timeout: float = 60.0): self.base_url = base_url.rstrip("/") self.number = number diff --git a/src/calbot/store.py b/src/calbot/store.py index 1e37207..c623038 100644 --- a/src/calbot/store.py +++ b/src/calbot/store.py @@ -1,4 +1,23 @@ -"""SQLite persistence: events, series, reminder plans, outbox, deliveries.""" +"""SQLite persistence: events, series, reminder plans, outbox, deliveries. + +Tables, and why each exists: + +* ``meta`` — feed validators, last fetch attempt, "seeded" marker. +* ``events`` — one row per occurrence; ``gone`` means it disappeared from + the feed (cancelled), not merely that our window moved past it. +* ``series`` — announcement dedupe. The feed pre-expands recurrences, so + without this a new weekly event would be announced once per occurrence. +* ``reminder_plans`` — the computed reminder time, fixed at discovery so a + restart or config edit cannot reshuffle it. +* ``outbox`` — pending sends. Rows are deleted only after a successful + send, which makes delivery at-least-once. +* ``deliveries`` — permanent record of what was already sent. +* ``pages`` — cached venue/image scraped from an event page. +* ``short_links`` — cached URL shortener results. + +All datetimes are stored as ISO-8601 UTC strings; ISO ordering is used for +"due" comparisons, which is safe because the format is fixed and offset-aware. +""" from __future__ import annotations @@ -89,6 +108,7 @@ CREATE TABLE IF NOT EXISTS short_links ( def iso(moment: datetime) -> str: + """Normalise to UTC so string comparison and ordering are meaningful.""" return moment.astimezone(timezone.utc).isoformat() @@ -284,6 +304,7 @@ class Store: send_at: datetime, now: datetime, ) -> None: + """Queue a send. ``(kind, ref, period)`` is unique, so replanning is a no-op.""" self.conn.execute( """ INSERT OR IGNORE INTO outbox(kind, ref, period, event_key, send_at, created_at) @@ -294,6 +315,7 @@ class Store: self.conn.commit() def due_outbox(self, now: datetime) -> list[OutboxItem]: + """Pending sends whose window has opened, oldest first.""" rows = self.conn.execute( "SELECT * FROM outbox WHERE send_at <= ? ORDER BY send_at, id", (iso(now),), diff --git a/src/calbot/watchdog.py b/src/calbot/watchdog.py index 03f8fdf..b068b0c 100644 --- a/src/calbot/watchdog.py +++ b/src/calbot/watchdog.py @@ -12,6 +12,12 @@ log = logging.getLogger(__name__) class Watchdog: + """Alerts the operator (own number by default) when the pipeline breaks. + + Alerts are throttled per issue and never raise: if Signal itself is down + there is nowhere to send, so the failure is only logged. + """ + def __init__( self, config: WatchdogConfig,