Fix watchdog gaps, image failures and stale reminders
- catch httpx errors on send/heartbeat so a dead signal-api alerts instead of failing silently - never let compose/images block the text post; count failed attempts - drop reminders for events removed from the feed, using the feed's coverage horizon so the 30-event cap does not mark future events gone - persist feed validators only after a successful parse - do not seed from an empty feed - flatten transparent images onto white; catch decompression bombs - --dump uses a throwaway database - avoid duplicated postcode in the venue line - remove dead code; add regression tests
This commit is contained in:
+50
-38
@@ -7,8 +7,10 @@ import logging
|
||||
import os
|
||||
import signal
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
import httpx
|
||||
@@ -44,46 +46,56 @@ def _http_client() -> httpx.Client:
|
||||
|
||||
|
||||
def dump(config: Config) -> int:
|
||||
"""Print rendered posts for the next events, without touching Signal."""
|
||||
http = _http_client()
|
||||
feed = Feed(config.calendar.url, Store(config.db_path), config.calendar.refresh_minutes, http)
|
||||
try:
|
||||
raw = feed.fetch(datetime.now(timezone.utc))
|
||||
except httpx.HTTPError as exc:
|
||||
print(f"feed fetch failed: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
if raw is None:
|
||||
print("feed unchanged (304); nothing to dump")
|
||||
return 0
|
||||
|
||||
"""Print rendered posts for the next events, without touching Signal or state."""
|
||||
now = datetime.now(timezone.utc)
|
||||
events = parse_events(
|
||||
raw,
|
||||
now,
|
||||
now + timedelta(days=config.calendar.window_days),
|
||||
config.calendar.timezone,
|
||||
)
|
||||
local_tz = ZoneInfo(config.calendar.timezone)
|
||||
for event in events:
|
||||
venue = venue_from_location(event.location) or venue_from_description(event.description)
|
||||
postcode = find_postcode(event.location, event.description)
|
||||
print("=" * 60)
|
||||
print(f"{event.start.astimezone(local_tz):%Y-%m-%d %H:%M} [{event.series_id}]")
|
||||
print("-" * 60)
|
||||
print(
|
||||
render_post(
|
||||
event,
|
||||
config.format,
|
||||
local_tz,
|
||||
venue=venue,
|
||||
postcode=postcode,
|
||||
link=event.url,
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
store = Store(Path(tmp) / "dump.sqlite3")
|
||||
http = _http_client()
|
||||
try:
|
||||
feed = Feed(
|
||||
config.calendar.url, store, config.calendar.refresh_minutes, http
|
||||
)
|
||||
)
|
||||
print(f"image: {event.image_url or '(none)'}")
|
||||
print("=" * 60)
|
||||
print(f"{len(events)} event(s)")
|
||||
return 0
|
||||
try:
|
||||
response = feed.fetch(now)
|
||||
except httpx.HTTPError as exc:
|
||||
print(f"feed fetch failed: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
if response.content is None:
|
||||
print("feed said not-modified (304); nothing to dump")
|
||||
return 0
|
||||
|
||||
events = parse_events(
|
||||
response.content,
|
||||
now,
|
||||
now + timedelta(days=config.calendar.window_days),
|
||||
config.calendar.timezone,
|
||||
)
|
||||
local_tz = ZoneInfo(config.calendar.timezone)
|
||||
for event in events:
|
||||
venue = venue_from_location(event.location) or venue_from_description(
|
||||
event.description
|
||||
)
|
||||
postcode = find_postcode(event.location, event.description)
|
||||
print("=" * 60)
|
||||
print(f"{event.start.astimezone(local_tz):%Y-%m-%d %H:%M} [{event.series_id}]")
|
||||
print("-" * 60)
|
||||
print(
|
||||
render_post(
|
||||
event,
|
||||
config.format,
|
||||
local_tz,
|
||||
venue=venue,
|
||||
postcode=postcode,
|
||||
link=event.url,
|
||||
)
|
||||
)
|
||||
print(f"image: {event.image_url or '(none)'}")
|
||||
print("=" * 60)
|
||||
print(f"{len(events)} event(s)")
|
||||
return 0
|
||||
finally:
|
||||
store.close()
|
||||
http.close()
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
|
||||
@@ -15,7 +15,6 @@ import recurring_ical_events
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
TEC_UID_RE = re.compile(r"^(\d+)-\d+-\d+@")
|
||||
DATE_LIKE_RE = re.compile(r"^\d{8}$")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -33,6 +32,7 @@ class Event:
|
||||
image_url: str = ""
|
||||
tz: str = "UTC"
|
||||
status: str = ""
|
||||
gone: bool = False
|
||||
|
||||
|
||||
def _zone(name: str) -> ZoneInfo:
|
||||
@@ -51,11 +51,6 @@ def _as_datetime(value: datetime | date, tz: ZoneInfo) -> datetime:
|
||||
return datetime.combine(value, time.min, tzinfo=tz)
|
||||
|
||||
|
||||
def _all_day_flag(component) -> bool:
|
||||
prop = component.get("DTSTART")
|
||||
return prop is not None and not isinstance(getattr(prop, "dt", None), datetime)
|
||||
|
||||
|
||||
def _image_url(component) -> str:
|
||||
for name in ("ATTACH", "IMAGE"):
|
||||
value = component.get(name)
|
||||
|
||||
+31
-10
@@ -3,6 +3,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
import httpx
|
||||
@@ -21,6 +22,19 @@ def normalize_feed_url(url: str) -> str:
|
||||
return url
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class FeedResponse:
|
||||
"""A fetch result whose validators are committed only after a good parse."""
|
||||
|
||||
content: bytes | None = None
|
||||
etag: str | None = None
|
||||
last_modified: str | None = None
|
||||
|
||||
@property
|
||||
def not_modified(self) -> bool:
|
||||
return self.content is None
|
||||
|
||||
|
||||
class Feed:
|
||||
def __init__(self, url: str, store: Store, refresh_minutes: int, http: httpx.Client):
|
||||
self.url = normalize_feed_url(url)
|
||||
@@ -34,8 +48,8 @@ class Feed:
|
||||
return True
|
||||
return now - from_iso(last) >= timedelta(minutes=self.refresh_minutes)
|
||||
|
||||
def fetch(self, now: datetime) -> bytes | None:
|
||||
"""Return new feed bytes, or None when unchanged / not modified."""
|
||||
def fetch(self, now: datetime) -> FeedResponse:
|
||||
"""Fetch the feed. Validators are *not* stored here; see commit()."""
|
||||
headers = {"User-Agent": USER_AGENT, "Accept": "text/calendar, */*"}
|
||||
etag = self.store.get_meta("feed_etag")
|
||||
modified = self.store.get_meta("feed_last_modified")
|
||||
@@ -49,13 +63,20 @@ class Feed:
|
||||
|
||||
if response.status_code == 304:
|
||||
log.debug("feed not modified")
|
||||
return None
|
||||
response.raise_for_status()
|
||||
return FeedResponse()
|
||||
|
||||
if response.headers.get("etag"):
|
||||
self.store.set_meta("feed_etag", response.headers["etag"])
|
||||
if response.headers.get("last-modified"):
|
||||
self.store.set_meta("feed_last_modified", response.headers["last-modified"])
|
||||
self.store.set_meta("feed_last_success", iso(datetime.now(timezone.utc)))
|
||||
response.raise_for_status()
|
||||
log.info("fetched feed: %d bytes", len(response.content))
|
||||
return response.content
|
||||
return FeedResponse(
|
||||
content=response.content,
|
||||
etag=response.headers.get("etag"),
|
||||
last_modified=response.headers.get("last-modified"),
|
||||
)
|
||||
|
||||
def commit(self, response: FeedResponse) -> None:
|
||||
"""Persist validators after the payload parsed successfully."""
|
||||
if response.etag:
|
||||
self.store.set_meta("feed_etag", response.etag)
|
||||
if response.last_modified:
|
||||
self.store.set_meta("feed_last_modified", response.last_modified)
|
||||
self.store.set_meta("feed_last_success", iso(datetime.now(timezone.utc)))
|
||||
|
||||
@@ -102,6 +102,9 @@ def render_post(
|
||||
if config.show_location:
|
||||
name = _clean(venue) or venue_from_location(event.location)
|
||||
code = postcode or find_postcode(event.location, event.description)
|
||||
if code and code.upper() in name.upper():
|
||||
# LOCATION sometimes already ends with the postcode.
|
||||
code = ""
|
||||
place = " ".join(part for part in (name, code) if part).strip()
|
||||
if place:
|
||||
lines.append(place)
|
||||
|
||||
+18
-7
@@ -8,12 +8,12 @@ import io
|
||||
import json
|
||||
import logging
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
import httpx
|
||||
from bs4 import BeautifulSoup
|
||||
from PIL import Image, ImageOps, UnidentifiedImageError
|
||||
from PIL import Image, ImageOps
|
||||
|
||||
from .config import ImagesConfig
|
||||
from .store import PageInfo, Store
|
||||
@@ -74,7 +74,7 @@ class PageScraper:
|
||||
venue=details.venue,
|
||||
postcode=details.postcode,
|
||||
)
|
||||
self.store.put_page(event_url, info, datetime.now())
|
||||
self.store.put_page(event_url, info, datetime.now(timezone.utc))
|
||||
return info
|
||||
|
||||
@staticmethod
|
||||
@@ -159,8 +159,8 @@ class ImageService:
|
||||
return None
|
||||
try:
|
||||
payload = self._cached_or_download(url)
|
||||
except (httpx.HTTPError, OSError) as exc:
|
||||
log.warning("image fetch failed for %s: %s", url, exc)
|
||||
except Exception as exc: # noqa: BLE001 - images are strictly optional
|
||||
log.warning("image failed for %s: %s", url, exc)
|
||||
return None
|
||||
if payload is None:
|
||||
return None
|
||||
@@ -203,9 +203,20 @@ class ImageService:
|
||||
try:
|
||||
with Image.open(io.BytesIO(raw)) as image:
|
||||
image = ImageOps.exif_transpose(image)
|
||||
image = image.convert("RGB")
|
||||
image.thumbnail((self.config.max_pixels, self.config.max_pixels))
|
||||
|
||||
# Composite transparency onto white; converting straight to RGB
|
||||
# turns transparent logos into black boxes.
|
||||
if image.mode in ("RGBA", "LA") or (
|
||||
image.mode == "P" and "transparency" in image.info
|
||||
):
|
||||
rgba = image.convert("RGBA")
|
||||
canvas = Image.new("RGB", rgba.size, (255, 255, 255))
|
||||
canvas.paste(rgba, mask=rgba.getchannel("A"))
|
||||
image = canvas
|
||||
else:
|
||||
image = image.convert("RGB")
|
||||
|
||||
quality = 82
|
||||
while True:
|
||||
buffer = io.BytesIO()
|
||||
@@ -219,6 +230,6 @@ class ImageService:
|
||||
(int(image.width * 0.8), int(image.height * 0.8))
|
||||
)
|
||||
quality = 70
|
||||
except (UnidentifiedImageError, OSError, ValueError) as exc:
|
||||
except Exception as exc: # noqa: BLE001 - including DecompressionBombError
|
||||
log.warning("could not decode image: %s", exc)
|
||||
return None
|
||||
|
||||
+93
-58
@@ -14,7 +14,7 @@ from .feed import Feed
|
||||
from .format import find_postcode, render_post, venue_from_description, venue_from_location
|
||||
from .images import ImageService, PageScraper
|
||||
from .signal_client import SignalClient, SignalError
|
||||
from .store import OutboxItem, Store, iso
|
||||
from .store import OutboxItem, PageInfo, Store, iso
|
||||
from .watchdog import Watchdog
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
@@ -137,12 +137,12 @@ class Bot:
|
||||
if not self.feed.due(now):
|
||||
return
|
||||
try:
|
||||
raw = self.feed.fetch(now)
|
||||
if raw is None:
|
||||
response = self.feed.fetch(now)
|
||||
if response.content is None:
|
||||
self.watchdog.feed_ok()
|
||||
return
|
||||
events = parse_events(
|
||||
raw,
|
||||
response.content,
|
||||
now,
|
||||
now + timedelta(days=self.config.calendar.window_days),
|
||||
self.config.calendar.timezone,
|
||||
@@ -152,15 +152,24 @@ class Bot:
|
||||
self.watchdog.feed_failed(str(exc), now)
|
||||
return
|
||||
|
||||
if not events and self.store.count_events() > 0:
|
||||
self.watchdog.alert(
|
||||
"empty-feed",
|
||||
"Feed parsed to zero events while the database still has events. "
|
||||
"Leaving stored events untouched.",
|
||||
now,
|
||||
)
|
||||
if not events:
|
||||
# Never seed from an empty feed: it is far more likely to be a
|
||||
# transient publishing problem than "the calendar has no events".
|
||||
if self.store.count_events() > 0:
|
||||
self.watchdog.alert(
|
||||
"empty-feed",
|
||||
"Feed parsed to zero events while the database still has events. "
|
||||
"Leaving stored events untouched.",
|
||||
now,
|
||||
)
|
||||
else:
|
||||
log.warning("feed returned no events; not seeding yet")
|
||||
self.watchdog.feed_failed("feed parsed to zero events", now)
|
||||
return
|
||||
|
||||
# Only persist validators once the payload has parsed cleanly, so a
|
||||
# broken response cannot pin itself behind a 304.
|
||||
self.feed.commit(response)
|
||||
self.watchdog.feed_ok()
|
||||
self._sync(events, now)
|
||||
|
||||
@@ -183,7 +192,9 @@ class Bot:
|
||||
if new_series:
|
||||
self._plan_announcement(event, now)
|
||||
|
||||
self.store.mark_gone([event.key for event in events], now)
|
||||
self.store.mark_gone(
|
||||
[event.key for event in events], now, max(event.start for event in events)
|
||||
)
|
||||
if first_run:
|
||||
self.store.set_meta("seeded", iso(now))
|
||||
log.info("first run: seeded %d events", len(events))
|
||||
@@ -263,8 +274,10 @@ class Bot:
|
||||
|
||||
valid: list[OutboxItem] = []
|
||||
for item in group:
|
||||
usable = event.end > now and (
|
||||
item.kind != "reminder" or event.all_day or event.start > now
|
||||
usable = (
|
||||
not event.gone
|
||||
and event.end > now
|
||||
and (item.kind != "reminder" or event.all_day or event.start > now)
|
||||
)
|
||||
if usable:
|
||||
valid.append(item)
|
||||
@@ -278,7 +291,12 @@ class Bot:
|
||||
if not batch:
|
||||
return
|
||||
|
||||
message, attachments = self._compose(batch)
|
||||
try:
|
||||
message, attachments = self._compose(batch)
|
||||
except Exception as exc: # noqa: BLE001 - compose must not wedge the outbox
|
||||
log.exception("could not compose post")
|
||||
self._record_failure(batch, str(exc), now)
|
||||
return
|
||||
try:
|
||||
self.sender.send(
|
||||
message,
|
||||
@@ -286,20 +304,9 @@ class Bot:
|
||||
attachments=attachments,
|
||||
text_mode=self.config.signal.text_mode,
|
||||
)
|
||||
except SignalError as exc:
|
||||
except (SignalError, httpx.HTTPError) as exc:
|
||||
log.error("send failed: %s", exc)
|
||||
self.watchdog.send_failed(str(exc), now)
|
||||
for _event, group in batch:
|
||||
for item in group:
|
||||
attempts = self.store.bump_attempts(item.id)
|
||||
if attempts >= MAX_SEND_ATTEMPTS:
|
||||
log.error("giving up on %s after %d attempts", item.ref, attempts)
|
||||
self.watchdog.alert(
|
||||
"outbox",
|
||||
f"Dropping a post after {attempts} failed attempts.",
|
||||
now,
|
||||
)
|
||||
self.store.drop_outbox(item.id)
|
||||
self._record_failure(batch, str(exc), now)
|
||||
return
|
||||
|
||||
self.watchdog.send_ok()
|
||||
@@ -311,44 +318,72 @@ class Bot:
|
||||
self.store.drop_outbox(item.id)
|
||||
log.info("posted %d item(s) covering %d event(s)", len(items), len(batch))
|
||||
|
||||
def _record_failure(
|
||||
self, batch: list[tuple[Event, list[OutboxItem]]], reason: str, now: datetime
|
||||
) -> None:
|
||||
self.watchdog.send_failed(reason, now)
|
||||
for _event, group in batch:
|
||||
for item in group:
|
||||
attempts = self.store.bump_attempts(item.id)
|
||||
if attempts >= MAX_SEND_ATTEMPTS:
|
||||
log.error("giving up on %s after %d attempts", item.ref, attempts)
|
||||
self.watchdog.alert(
|
||||
"outbox",
|
||||
f"Dropping a post after {attempts} failed attempts.",
|
||||
now,
|
||||
)
|
||||
self.store.drop_outbox(item.id)
|
||||
|
||||
def _compose(self, batch: list[tuple[Event, list[OutboxItem]]]) -> tuple[str, list[str]]:
|
||||
blocks: list[str] = []
|
||||
attachments: list[str] = []
|
||||
limit = self.config.images.max_images_per_post if len(batch) > 1 else 1
|
||||
|
||||
for event, _items in batch:
|
||||
info = self.scraper.details(event.url)
|
||||
venue = (
|
||||
venue_from_location(event.location)
|
||||
or info.venue
|
||||
or venue_from_description(event.description)
|
||||
)
|
||||
postcode = (
|
||||
find_postcode(event.location, event.description) or info.postcode
|
||||
)
|
||||
link = event.url
|
||||
if self.config.format.shorten_links and link:
|
||||
link = self._shorten(link)
|
||||
|
||||
blocks.append(
|
||||
render_post(
|
||||
event,
|
||||
self.config.format,
|
||||
self.local_tz,
|
||||
venue=venue,
|
||||
postcode=postcode,
|
||||
link=link,
|
||||
)
|
||||
)
|
||||
|
||||
if len(attachments) < limit:
|
||||
image_url = event.image_url or info.image_url
|
||||
data_uri = self.images.data_uri(image_url) if image_url else None
|
||||
if data_uri:
|
||||
attachments.append(data_uri)
|
||||
block, attachment = self._compose_one(event, limit - len(attachments))
|
||||
blocks.append(block)
|
||||
if attachment:
|
||||
attachments.append(attachment)
|
||||
|
||||
return "\n\n".join(blocks), attachments
|
||||
|
||||
def _compose_one(self, event: Event, images_left: int) -> tuple[str, str | None]:
|
||||
"""Render one event; enrichment and image failures degrade to text only."""
|
||||
info = PageInfo(image_url="", venue="", postcode="")
|
||||
try:
|
||||
info = self.scraper.details(event.url)
|
||||
except Exception: # noqa: BLE001 - never block a post on scraping
|
||||
log.exception("page scrape failed for %s", event.url)
|
||||
|
||||
venue = (
|
||||
venue_from_location(event.location)
|
||||
or info.venue
|
||||
or venue_from_description(event.description)
|
||||
)
|
||||
postcode = find_postcode(event.location, event.description) or info.postcode
|
||||
link = event.url
|
||||
if self.config.format.shorten_links and link:
|
||||
link = self._shorten(link)
|
||||
|
||||
text = render_post(
|
||||
event,
|
||||
self.config.format,
|
||||
self.local_tz,
|
||||
venue=venue,
|
||||
postcode=postcode,
|
||||
link=link,
|
||||
)
|
||||
|
||||
attachment = None
|
||||
if images_left > 0:
|
||||
image_url = event.image_url or info.image_url
|
||||
if image_url:
|
||||
try:
|
||||
attachment = self.images.data_uri(image_url)
|
||||
except Exception: # noqa: BLE001 - images are best effort
|
||||
log.exception("image failed for %s", image_url)
|
||||
return text, attachment
|
||||
|
||||
def _shorten(self, url: str) -> str:
|
||||
cached = self.store.get_short_link(url)
|
||||
if cached:
|
||||
@@ -383,7 +418,7 @@ class Bot:
|
||||
[self.watchdog.recipient],
|
||||
text_mode=self.config.signal.text_mode,
|
||||
)
|
||||
except SignalError as exc:
|
||||
except (SignalError, httpx.HTTPError) as exc:
|
||||
log.warning("heartbeat failed: %s", exc)
|
||||
return
|
||||
self.store.set_meta("last_heartbeat", today)
|
||||
|
||||
+21
-11
@@ -143,10 +143,6 @@ class Store:
|
||||
|
||||
# -- events ----------------------------------------------------------
|
||||
|
||||
def has_events(self) -> bool:
|
||||
row = self.conn.execute("SELECT 1 FROM events LIMIT 1").fetchone()
|
||||
return row is not None
|
||||
|
||||
def event_exists(self, event_key: str) -> bool:
|
||||
row = self.conn.execute(
|
||||
"SELECT 1 FROM events WHERE event_key = ?", (event_key,)
|
||||
@@ -194,15 +190,28 @@ class Store:
|
||||
self.conn.commit()
|
||||
return is_new
|
||||
|
||||
def mark_gone(self, seen_keys: Iterable[str], now: datetime) -> None:
|
||||
def mark_gone(
|
||||
self, seen_keys: Iterable[str], now: datetime, up_to: datetime | None = None
|
||||
) -> None:
|
||||
"""Flag events missing from the feed as gone.
|
||||
|
||||
``up_to`` is the end of the feed's coverage. The Events Calendar feed is
|
||||
capped at ~30 events, so anything starting after the last event in the
|
||||
feed is simply unknown, not cancelled.
|
||||
"""
|
||||
seen = set(seen_keys)
|
||||
rows = self.conn.execute("SELECT event_key FROM events WHERE gone = 0").fetchall()
|
||||
rows = self.conn.execute(
|
||||
"SELECT event_key, start_utc FROM events WHERE gone = 0"
|
||||
).fetchall()
|
||||
for row in rows:
|
||||
if row["event_key"] not in seen:
|
||||
self.conn.execute(
|
||||
"UPDATE events SET gone = 1, last_seen_at = ? WHERE event_key = ?",
|
||||
(iso(now), row["event_key"]),
|
||||
)
|
||||
if row["event_key"] in seen:
|
||||
continue
|
||||
if up_to is not None and from_iso(row["start_utc"]) > up_to:
|
||||
continue
|
||||
self.conn.execute(
|
||||
"UPDATE events SET gone = 1, last_seen_at = ? WHERE event_key = ?",
|
||||
(iso(now), row["event_key"]),
|
||||
)
|
||||
self.conn.commit()
|
||||
|
||||
def get_event(self, event_key: str) -> Event | None:
|
||||
@@ -224,6 +233,7 @@ class Store:
|
||||
image_url=row["image_url"],
|
||||
tz=row["event_tz"],
|
||||
status=row["status"],
|
||||
gone=bool(row["gone"]),
|
||||
)
|
||||
|
||||
def count_events(self) -> int:
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
import httpx
|
||||
|
||||
from calbot.feed import Feed, FeedResponse, normalize_feed_url
|
||||
from calbot.store import Store
|
||||
|
||||
NOW = datetime(2026, 11, 18, 12, 0, tzinfo=timezone.utc)
|
||||
EMPTY_FEED = b"BEGIN:VCALENDAR\r\nVERSION:2.0\r\nPRODID:-//test//EN\r\nEND:VCALENDAR\r\n"
|
||||
|
||||
|
||||
def test_webcal_is_fetched_over_https():
|
||||
assert normalize_feed_url("webcal://example.test/x.ics") == "https://example.test/x.ics"
|
||||
assert normalize_feed_url("https://example.test/x.ics") == "https://example.test/x.ics"
|
||||
|
||||
|
||||
def test_validators_are_only_stored_after_commit(tmp_path):
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
if request.headers.get("if-none-match") == '"abc"':
|
||||
return httpx.Response(304)
|
||||
return httpx.Response(200, content=EMPTY_FEED, headers={"ETag": '"abc"'})
|
||||
|
||||
store = Store(Path(tmp_path) / "db.sqlite3")
|
||||
client = httpx.Client(transport=httpx.MockTransport(handler))
|
||||
feed = Feed("webcal://example.test/feed.ics", store, 15, client)
|
||||
|
||||
response = feed.fetch(NOW)
|
||||
assert isinstance(response, FeedResponse)
|
||||
assert response.content == EMPTY_FEED
|
||||
# A parse failure would skip commit(), so nothing may be persisted yet.
|
||||
assert store.get_meta("feed_etag") is None
|
||||
|
||||
feed.commit(response)
|
||||
assert store.get_meta("feed_etag") == '"abc"'
|
||||
|
||||
second = feed.fetch(NOW)
|
||||
assert second.not_modified
|
||||
assert second.content is None
|
||||
|
||||
|
||||
def test_fetch_raises_on_http_error(tmp_path):
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(500, text="boom")
|
||||
|
||||
store = Store(Path(tmp_path) / "db.sqlite3")
|
||||
client = httpx.Client(transport=httpx.MockTransport(handler))
|
||||
feed = Feed("https://example.test/feed.ics", store, 15, client)
|
||||
try:
|
||||
feed.fetch(NOW)
|
||||
except httpx.HTTPStatusError:
|
||||
pass
|
||||
else: # pragma: no cover
|
||||
raise AssertionError("expected HTTPStatusError")
|
||||
@@ -96,3 +96,16 @@ def test_missing_fields_are_omitted():
|
||||
text = render_post(event, FormatConfig(), LONDON, venue="", postcode="", link="")
|
||||
lines = text.splitlines()
|
||||
assert lines == ["Folk Night", "#November #November14", "4.30-6.30pm"]
|
||||
|
||||
|
||||
def test_postcode_is_not_duplicated_when_part_of_venue():
|
||||
event = make_event(location="Old Print Works B12 9AH", url="")
|
||||
text = render_post(
|
||||
event,
|
||||
FormatConfig(),
|
||||
LONDON,
|
||||
venue="Old Print Works B12 9AH",
|
||||
postcode="B12 9AH",
|
||||
link="",
|
||||
)
|
||||
assert text.splitlines()[3] == "Old Print Works B12 9AH"
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
|
||||
import httpx
|
||||
from PIL import Image
|
||||
|
||||
from calbot.config import ImagesConfig
|
||||
from calbot.images import ImageService
|
||||
|
||||
|
||||
def make_service(tmp_path, **overrides) -> ImageService:
|
||||
return ImageService(ImagesConfig(**overrides), tmp_path, httpx.Client())
|
||||
|
||||
|
||||
def test_transparent_png_is_flattened_onto_white(tmp_path):
|
||||
buffer = io.BytesIO()
|
||||
Image.new("RGBA", (120, 120), (255, 0, 0, 0)).save(buffer, "PNG")
|
||||
|
||||
service = make_service(tmp_path)
|
||||
result = service._resize(buffer.getvalue())
|
||||
assert result
|
||||
|
||||
image = Image.open(io.BytesIO(result))
|
||||
assert image.mode == "RGB"
|
||||
red, green, blue = image.getpixel((60, 60))
|
||||
assert min(red, green, blue) > 240
|
||||
|
||||
|
||||
def test_oversized_image_is_downscaled(tmp_path):
|
||||
buffer = io.BytesIO()
|
||||
Image.new("RGB", (4000, 3000), (10, 80, 200)).save(buffer, "PNG")
|
||||
service = make_service(tmp_path, max_pixels=400, max_bytes=50_000)
|
||||
result = service._resize(buffer.getvalue())
|
||||
assert result
|
||||
assert len(result) <= 50_000
|
||||
image = Image.open(io.BytesIO(result))
|
||||
assert max(image.size) <= 400
|
||||
|
||||
|
||||
def test_decompression_bomb_does_not_propagate(tmp_path):
|
||||
service = make_service(tmp_path)
|
||||
|
||||
def boom(url):
|
||||
raise Image.DecompressionBombError("too many pixels")
|
||||
|
||||
service._cached_or_download = boom # type: ignore[method-assign]
|
||||
assert service.data_uri("https://example.test/huge.jpg") is None
|
||||
+149
-7
@@ -18,7 +18,7 @@ from calbot.config import (
|
||||
WatchdogConfig,
|
||||
)
|
||||
from calbot.expand import Event
|
||||
from calbot.feed import Feed, normalize_feed_url
|
||||
from calbot.feed import Feed, FeedResponse, normalize_feed_url
|
||||
from calbot.scheduler import (
|
||||
Bot,
|
||||
clamp_send_at,
|
||||
@@ -47,6 +47,7 @@ def make_event(
|
||||
start: datetime,
|
||||
end: datetime | None = None,
|
||||
all_day: bool = False,
|
||||
image_url: str = "",
|
||||
) -> Event:
|
||||
return Event(
|
||||
key=f"{uid}|{start.astimezone(UTC).isoformat()}",
|
||||
@@ -58,6 +59,7 @@ def make_event(
|
||||
all_day=all_day,
|
||||
location="Old Print Works\nB12 9AH",
|
||||
url="https://commonscalendar.uk/event/folk-night/",
|
||||
image_url=image_url,
|
||||
tz="Europe/London",
|
||||
)
|
||||
|
||||
@@ -170,18 +172,57 @@ class FakeScraper:
|
||||
return PageInfo(image_url="", venue="", postcode="")
|
||||
|
||||
|
||||
def build_bot(tmp_path, sender):
|
||||
config = make_config(tmp_path)
|
||||
class BoomScraper:
|
||||
def details(self, url):
|
||||
raise RuntimeError("page down")
|
||||
|
||||
|
||||
class BoomImages:
|
||||
def data_uri(self, url):
|
||||
raise RuntimeError("image down")
|
||||
|
||||
|
||||
class FailingSender:
|
||||
def __init__(self, exc):
|
||||
self.exc = exc
|
||||
self.attempts = 0
|
||||
|
||||
def send(self, message, recipients, attachments=None, text_mode="normal"):
|
||||
self.attempts += 1
|
||||
raise self.exc
|
||||
|
||||
|
||||
class FakeFeed:
|
||||
def __init__(self, content: bytes):
|
||||
self.content = content
|
||||
self.committed = False
|
||||
|
||||
def due(self, now):
|
||||
return True
|
||||
|
||||
def fetch(self, now):
|
||||
return FeedResponse(content=self.content)
|
||||
|
||||
def commit(self, response):
|
||||
self.committed = True
|
||||
|
||||
|
||||
def build_bot(tmp_path, sender, **overrides):
|
||||
config = overrides.pop("config", None) or make_config(tmp_path)
|
||||
store = Store(config.db_path)
|
||||
http = httpx.Client()
|
||||
bot = Bot(
|
||||
config=config,
|
||||
store=store,
|
||||
feed=Feed(config.calendar.url, store, 15, http),
|
||||
feed=overrides.pop(
|
||||
"feed", Feed(config.calendar.url, store, 15, http)
|
||||
),
|
||||
sender=sender,
|
||||
images=FakeImages(),
|
||||
scraper=FakeScraper(),
|
||||
watchdog=Watchdog(config.watchdog, sender, "+447700900000"),
|
||||
images=overrides.pop("images", FakeImages()),
|
||||
scraper=overrides.pop("scraper", FakeScraper()),
|
||||
watchdog=overrides.pop(
|
||||
"watchdog", Watchdog(config.watchdog, sender, "+447700900000")
|
||||
),
|
||||
http=http,
|
||||
)
|
||||
return bot, store
|
||||
@@ -244,3 +285,104 @@ def test_stale_reminder_is_dropped_not_sent(tmp_path):
|
||||
bot._dispatch(local(2026, 11, 18, 18, 0)) # event already started
|
||||
assert sender.sent == []
|
||||
assert store.due_outbox(local(2026, 11, 18, 18, 5)) == []
|
||||
|
||||
|
||||
# -- failure handling -------------------------------------------------------
|
||||
|
||||
|
||||
def test_signal_connection_error_does_not_wedge_the_outbox(tmp_path):
|
||||
sender = FailingSender(httpx.ConnectError("connection refused"))
|
||||
config = make_config(
|
||||
tmp_path, watchdog=WatchdogConfig(enabled=True, send_failures=1)
|
||||
)
|
||||
watchdog = Watchdog(config.watchdog, sender, "+447700900000")
|
||||
bot, store = build_bot(tmp_path, sender, config=config, watchdog=watchdog)
|
||||
|
||||
event = make_event(uid="20", series_id="tec:20", start=local(2026, 11, 25, 19, 0))
|
||||
bot._sync([event], local(2026, 11, 18, 12, 0))
|
||||
bot._sync(
|
||||
[event, make_event(uid="21", series_id="tec:21", start=local(2026, 11, 26, 19, 0))],
|
||||
local(2026, 11, 18, 12, 1),
|
||||
)
|
||||
|
||||
now = local(2026, 11, 18, 18, 0)
|
||||
bot._dispatch(now) # must not raise
|
||||
|
||||
pending = store.due_outbox(now)
|
||||
assert pending, "the failed post should stay queued"
|
||||
assert pending[0].attempts == 1
|
||||
# The announcement stays queued, so it is not marked as delivered.
|
||||
assert store.due_outbox(now)[0].kind == "announce"
|
||||
|
||||
|
||||
def test_scraper_and_image_failures_still_send_text(tmp_path):
|
||||
sender = FakeSender()
|
||||
bot, store = build_bot(
|
||||
tmp_path,
|
||||
sender,
|
||||
images=BoomImages(),
|
||||
scraper=BoomScraper(),
|
||||
)
|
||||
|
||||
event = make_event(
|
||||
uid="22",
|
||||
series_id="tec:22",
|
||||
start=local(2026, 11, 25, 19, 0),
|
||||
image_url="https://example.test/poster.png",
|
||||
)
|
||||
bot._sync([event], local(2026, 11, 18, 12, 0))
|
||||
bot._sync(
|
||||
[event, make_event(uid="23", series_id="tec:23", start=local(2026, 11, 26, 19, 0))],
|
||||
local(2026, 11, 18, 12, 1),
|
||||
)
|
||||
bot._dispatch(local(2026, 11, 18, 18, 0))
|
||||
|
||||
assert len(sender.sent) == 1
|
||||
assert sender.sent[0]["attachments"] == []
|
||||
assert sender.sent[0]["message"].startswith("Folk Night")
|
||||
|
||||
|
||||
# -- gone events ------------------------------------------------------------
|
||||
|
||||
|
||||
def test_reminder_for_gone_event_is_dropped(tmp_path):
|
||||
sender = FakeSender()
|
||||
bot, store = build_bot(tmp_path, sender)
|
||||
|
||||
event = make_event(uid="30", series_id="tec:30", start=local(2026, 11, 25, 14, 0))
|
||||
bot._sync([event], local(2026, 11, 18, 12, 0))
|
||||
store.mark_gone([], local(2026, 11, 18, 13, 0), up_to=event.end + timedelta(days=2))
|
||||
|
||||
bot._dispatch(local(2026, 11, 22, 18, 5))
|
||||
assert sender.sent == []
|
||||
assert store.due_outbox(local(2026, 11, 22, 18, 10)) == []
|
||||
|
||||
|
||||
def test_events_beyond_feed_coverage_are_not_marked_gone(tmp_path):
|
||||
sender = FakeSender()
|
||||
bot, store = build_bot(tmp_path, sender)
|
||||
|
||||
event = make_event(uid="31", series_id="tec:31", start=local(2026, 12, 20, 14, 0))
|
||||
bot._sync([event], local(2026, 11, 18, 12, 0))
|
||||
store.mark_gone([], local(2026, 11, 18, 13, 0), up_to=local(2026, 11, 30, 0, 0))
|
||||
|
||||
stored = store.get_event(event.key)
|
||||
assert stored is not None and stored.gone is False
|
||||
|
||||
|
||||
# -- feed seeding -----------------------------------------------------------
|
||||
|
||||
|
||||
def test_empty_first_feed_does_not_seed(tmp_path):
|
||||
sender = FakeSender()
|
||||
empty = (
|
||||
b"BEGIN:VCALENDAR\r\nVERSION:2.0\r\nPRODID:-//test//EN\r\n"
|
||||
b"END:VCALENDAR\r\n"
|
||||
)
|
||||
feed = FakeFeed(empty)
|
||||
bot, store = build_bot(tmp_path, sender, feed=feed)
|
||||
|
||||
bot._refresh_feed(local(2026, 11, 18, 12, 0))
|
||||
|
||||
assert store.get_meta("seeded") is None
|
||||
assert feed.committed is False
|
||||
|
||||
Reference in New Issue
Block a user