Fix watchdog gaps, image failures and stale reminders
- catch httpx errors on send/heartbeat so a dead signal-api alerts instead of failing silently - never let compose/images block the text post; count failed attempts - drop reminders for events removed from the feed, using the feed's coverage horizon so the 30-event cap does not mark future events gone - persist feed validators only after a successful parse - do not seed from an empty feed - flatten transparent images onto white; catch decompression bombs - --dump uses a throwaway database - avoid duplicated postcode in the venue line - remove dead code; add regression tests
This commit is contained in:
+50
-38
@@ -7,8 +7,10 @@ import logging
|
|||||||
import os
|
import os
|
||||||
import signal
|
import signal
|
||||||
import sys
|
import sys
|
||||||
|
import tempfile
|
||||||
import time
|
import time
|
||||||
from datetime import datetime, timedelta, timezone
|
from datetime import datetime, timedelta, timezone
|
||||||
|
from pathlib import Path
|
||||||
from zoneinfo import ZoneInfo
|
from zoneinfo import ZoneInfo
|
||||||
|
|
||||||
import httpx
|
import httpx
|
||||||
@@ -44,46 +46,56 @@ def _http_client() -> httpx.Client:
|
|||||||
|
|
||||||
|
|
||||||
def dump(config: Config) -> int:
|
def dump(config: Config) -> int:
|
||||||
"""Print rendered posts for the next events, without touching Signal."""
|
"""Print rendered posts for the next events, without touching Signal or state."""
|
||||||
http = _http_client()
|
|
||||||
feed = Feed(config.calendar.url, Store(config.db_path), config.calendar.refresh_minutes, http)
|
|
||||||
try:
|
|
||||||
raw = feed.fetch(datetime.now(timezone.utc))
|
|
||||||
except httpx.HTTPError as exc:
|
|
||||||
print(f"feed fetch failed: {exc}", file=sys.stderr)
|
|
||||||
return 1
|
|
||||||
if raw is None:
|
|
||||||
print("feed unchanged (304); nothing to dump")
|
|
||||||
return 0
|
|
||||||
|
|
||||||
now = datetime.now(timezone.utc)
|
now = datetime.now(timezone.utc)
|
||||||
events = parse_events(
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
raw,
|
store = Store(Path(tmp) / "dump.sqlite3")
|
||||||
now,
|
http = _http_client()
|
||||||
now + timedelta(days=config.calendar.window_days),
|
try:
|
||||||
config.calendar.timezone,
|
feed = Feed(
|
||||||
)
|
config.calendar.url, store, config.calendar.refresh_minutes, http
|
||||||
local_tz = ZoneInfo(config.calendar.timezone)
|
|
||||||
for event in events:
|
|
||||||
venue = venue_from_location(event.location) or venue_from_description(event.description)
|
|
||||||
postcode = find_postcode(event.location, event.description)
|
|
||||||
print("=" * 60)
|
|
||||||
print(f"{event.start.astimezone(local_tz):%Y-%m-%d %H:%M} [{event.series_id}]")
|
|
||||||
print("-" * 60)
|
|
||||||
print(
|
|
||||||
render_post(
|
|
||||||
event,
|
|
||||||
config.format,
|
|
||||||
local_tz,
|
|
||||||
venue=venue,
|
|
||||||
postcode=postcode,
|
|
||||||
link=event.url,
|
|
||||||
)
|
)
|
||||||
)
|
try:
|
||||||
print(f"image: {event.image_url or '(none)'}")
|
response = feed.fetch(now)
|
||||||
print("=" * 60)
|
except httpx.HTTPError as exc:
|
||||||
print(f"{len(events)} event(s)")
|
print(f"feed fetch failed: {exc}", file=sys.stderr)
|
||||||
return 0
|
return 1
|
||||||
|
if response.content is None:
|
||||||
|
print("feed said not-modified (304); nothing to dump")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
events = parse_events(
|
||||||
|
response.content,
|
||||||
|
now,
|
||||||
|
now + timedelta(days=config.calendar.window_days),
|
||||||
|
config.calendar.timezone,
|
||||||
|
)
|
||||||
|
local_tz = ZoneInfo(config.calendar.timezone)
|
||||||
|
for event in events:
|
||||||
|
venue = venue_from_location(event.location) or venue_from_description(
|
||||||
|
event.description
|
||||||
|
)
|
||||||
|
postcode = find_postcode(event.location, event.description)
|
||||||
|
print("=" * 60)
|
||||||
|
print(f"{event.start.astimezone(local_tz):%Y-%m-%d %H:%M} [{event.series_id}]")
|
||||||
|
print("-" * 60)
|
||||||
|
print(
|
||||||
|
render_post(
|
||||||
|
event,
|
||||||
|
config.format,
|
||||||
|
local_tz,
|
||||||
|
venue=venue,
|
||||||
|
postcode=postcode,
|
||||||
|
link=event.url,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
print(f"image: {event.image_url or '(none)'}")
|
||||||
|
print("=" * 60)
|
||||||
|
print(f"{len(events)} event(s)")
|
||||||
|
return 0
|
||||||
|
finally:
|
||||||
|
store.close()
|
||||||
|
http.close()
|
||||||
|
|
||||||
|
|
||||||
def main(argv: list[str] | None = None) -> int:
|
def main(argv: list[str] | None = None) -> int:
|
||||||
|
|||||||
@@ -15,7 +15,6 @@ import recurring_ical_events
|
|||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
TEC_UID_RE = re.compile(r"^(\d+)-\d+-\d+@")
|
TEC_UID_RE = re.compile(r"^(\d+)-\d+-\d+@")
|
||||||
DATE_LIKE_RE = re.compile(r"^\d{8}$")
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
@dataclass(frozen=True)
|
||||||
@@ -33,6 +32,7 @@ class Event:
|
|||||||
image_url: str = ""
|
image_url: str = ""
|
||||||
tz: str = "UTC"
|
tz: str = "UTC"
|
||||||
status: str = ""
|
status: str = ""
|
||||||
|
gone: bool = False
|
||||||
|
|
||||||
|
|
||||||
def _zone(name: str) -> ZoneInfo:
|
def _zone(name: str) -> ZoneInfo:
|
||||||
@@ -51,11 +51,6 @@ def _as_datetime(value: datetime | date, tz: ZoneInfo) -> datetime:
|
|||||||
return datetime.combine(value, time.min, tzinfo=tz)
|
return datetime.combine(value, time.min, tzinfo=tz)
|
||||||
|
|
||||||
|
|
||||||
def _all_day_flag(component) -> bool:
|
|
||||||
prop = component.get("DTSTART")
|
|
||||||
return prop is not None and not isinstance(getattr(prop, "dt", None), datetime)
|
|
||||||
|
|
||||||
|
|
||||||
def _image_url(component) -> str:
|
def _image_url(component) -> str:
|
||||||
for name in ("ATTACH", "IMAGE"):
|
for name in ("ATTACH", "IMAGE"):
|
||||||
value = component.get(name)
|
value = component.get(name)
|
||||||
|
|||||||
+31
-10
@@ -3,6 +3,7 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
|
from dataclasses import dataclass
|
||||||
from datetime import datetime, timedelta, timezone
|
from datetime import datetime, timedelta, timezone
|
||||||
|
|
||||||
import httpx
|
import httpx
|
||||||
@@ -21,6 +22,19 @@ def normalize_feed_url(url: str) -> str:
|
|||||||
return url
|
return url
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class FeedResponse:
|
||||||
|
"""A fetch result whose validators are committed only after a good parse."""
|
||||||
|
|
||||||
|
content: bytes | None = None
|
||||||
|
etag: str | None = None
|
||||||
|
last_modified: str | None = None
|
||||||
|
|
||||||
|
@property
|
||||||
|
def not_modified(self) -> bool:
|
||||||
|
return self.content is None
|
||||||
|
|
||||||
|
|
||||||
class Feed:
|
class Feed:
|
||||||
def __init__(self, url: str, store: Store, refresh_minutes: int, http: httpx.Client):
|
def __init__(self, url: str, store: Store, refresh_minutes: int, http: httpx.Client):
|
||||||
self.url = normalize_feed_url(url)
|
self.url = normalize_feed_url(url)
|
||||||
@@ -34,8 +48,8 @@ class Feed:
|
|||||||
return True
|
return True
|
||||||
return now - from_iso(last) >= timedelta(minutes=self.refresh_minutes)
|
return now - from_iso(last) >= timedelta(minutes=self.refresh_minutes)
|
||||||
|
|
||||||
def fetch(self, now: datetime) -> bytes | None:
|
def fetch(self, now: datetime) -> FeedResponse:
|
||||||
"""Return new feed bytes, or None when unchanged / not modified."""
|
"""Fetch the feed. Validators are *not* stored here; see commit()."""
|
||||||
headers = {"User-Agent": USER_AGENT, "Accept": "text/calendar, */*"}
|
headers = {"User-Agent": USER_AGENT, "Accept": "text/calendar, */*"}
|
||||||
etag = self.store.get_meta("feed_etag")
|
etag = self.store.get_meta("feed_etag")
|
||||||
modified = self.store.get_meta("feed_last_modified")
|
modified = self.store.get_meta("feed_last_modified")
|
||||||
@@ -49,13 +63,20 @@ class Feed:
|
|||||||
|
|
||||||
if response.status_code == 304:
|
if response.status_code == 304:
|
||||||
log.debug("feed not modified")
|
log.debug("feed not modified")
|
||||||
return None
|
return FeedResponse()
|
||||||
response.raise_for_status()
|
|
||||||
|
|
||||||
if response.headers.get("etag"):
|
response.raise_for_status()
|
||||||
self.store.set_meta("feed_etag", response.headers["etag"])
|
|
||||||
if response.headers.get("last-modified"):
|
|
||||||
self.store.set_meta("feed_last_modified", response.headers["last-modified"])
|
|
||||||
self.store.set_meta("feed_last_success", iso(datetime.now(timezone.utc)))
|
|
||||||
log.info("fetched feed: %d bytes", len(response.content))
|
log.info("fetched feed: %d bytes", len(response.content))
|
||||||
return response.content
|
return FeedResponse(
|
||||||
|
content=response.content,
|
||||||
|
etag=response.headers.get("etag"),
|
||||||
|
last_modified=response.headers.get("last-modified"),
|
||||||
|
)
|
||||||
|
|
||||||
|
def commit(self, response: FeedResponse) -> None:
|
||||||
|
"""Persist validators after the payload parsed successfully."""
|
||||||
|
if response.etag:
|
||||||
|
self.store.set_meta("feed_etag", response.etag)
|
||||||
|
if response.last_modified:
|
||||||
|
self.store.set_meta("feed_last_modified", response.last_modified)
|
||||||
|
self.store.set_meta("feed_last_success", iso(datetime.now(timezone.utc)))
|
||||||
|
|||||||
@@ -102,6 +102,9 @@ def render_post(
|
|||||||
if config.show_location:
|
if config.show_location:
|
||||||
name = _clean(venue) or venue_from_location(event.location)
|
name = _clean(venue) or venue_from_location(event.location)
|
||||||
code = postcode or find_postcode(event.location, event.description)
|
code = postcode or find_postcode(event.location, event.description)
|
||||||
|
if code and code.upper() in name.upper():
|
||||||
|
# LOCATION sometimes already ends with the postcode.
|
||||||
|
code = ""
|
||||||
place = " ".join(part for part in (name, code) if part).strip()
|
place = " ".join(part for part in (name, code) if part).strip()
|
||||||
if place:
|
if place:
|
||||||
lines.append(place)
|
lines.append(place)
|
||||||
|
|||||||
+18
-7
@@ -8,12 +8,12 @@ import io
|
|||||||
import json
|
import json
|
||||||
import logging
|
import logging
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from datetime import datetime
|
from datetime import datetime, timezone
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
import httpx
|
import httpx
|
||||||
from bs4 import BeautifulSoup
|
from bs4 import BeautifulSoup
|
||||||
from PIL import Image, ImageOps, UnidentifiedImageError
|
from PIL import Image, ImageOps
|
||||||
|
|
||||||
from .config import ImagesConfig
|
from .config import ImagesConfig
|
||||||
from .store import PageInfo, Store
|
from .store import PageInfo, Store
|
||||||
@@ -74,7 +74,7 @@ class PageScraper:
|
|||||||
venue=details.venue,
|
venue=details.venue,
|
||||||
postcode=details.postcode,
|
postcode=details.postcode,
|
||||||
)
|
)
|
||||||
self.store.put_page(event_url, info, datetime.now())
|
self.store.put_page(event_url, info, datetime.now(timezone.utc))
|
||||||
return info
|
return info
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
@@ -159,8 +159,8 @@ class ImageService:
|
|||||||
return None
|
return None
|
||||||
try:
|
try:
|
||||||
payload = self._cached_or_download(url)
|
payload = self._cached_or_download(url)
|
||||||
except (httpx.HTTPError, OSError) as exc:
|
except Exception as exc: # noqa: BLE001 - images are strictly optional
|
||||||
log.warning("image fetch failed for %s: %s", url, exc)
|
log.warning("image failed for %s: %s", url, exc)
|
||||||
return None
|
return None
|
||||||
if payload is None:
|
if payload is None:
|
||||||
return None
|
return None
|
||||||
@@ -203,9 +203,20 @@ class ImageService:
|
|||||||
try:
|
try:
|
||||||
with Image.open(io.BytesIO(raw)) as image:
|
with Image.open(io.BytesIO(raw)) as image:
|
||||||
image = ImageOps.exif_transpose(image)
|
image = ImageOps.exif_transpose(image)
|
||||||
image = image.convert("RGB")
|
|
||||||
image.thumbnail((self.config.max_pixels, self.config.max_pixels))
|
image.thumbnail((self.config.max_pixels, self.config.max_pixels))
|
||||||
|
|
||||||
|
# Composite transparency onto white; converting straight to RGB
|
||||||
|
# turns transparent logos into black boxes.
|
||||||
|
if image.mode in ("RGBA", "LA") or (
|
||||||
|
image.mode == "P" and "transparency" in image.info
|
||||||
|
):
|
||||||
|
rgba = image.convert("RGBA")
|
||||||
|
canvas = Image.new("RGB", rgba.size, (255, 255, 255))
|
||||||
|
canvas.paste(rgba, mask=rgba.getchannel("A"))
|
||||||
|
image = canvas
|
||||||
|
else:
|
||||||
|
image = image.convert("RGB")
|
||||||
|
|
||||||
quality = 82
|
quality = 82
|
||||||
while True:
|
while True:
|
||||||
buffer = io.BytesIO()
|
buffer = io.BytesIO()
|
||||||
@@ -219,6 +230,6 @@ class ImageService:
|
|||||||
(int(image.width * 0.8), int(image.height * 0.8))
|
(int(image.width * 0.8), int(image.height * 0.8))
|
||||||
)
|
)
|
||||||
quality = 70
|
quality = 70
|
||||||
except (UnidentifiedImageError, OSError, ValueError) as exc:
|
except Exception as exc: # noqa: BLE001 - including DecompressionBombError
|
||||||
log.warning("could not decode image: %s", exc)
|
log.warning("could not decode image: %s", exc)
|
||||||
return None
|
return None
|
||||||
|
|||||||
+93
-58
@@ -14,7 +14,7 @@ from .feed import Feed
|
|||||||
from .format import find_postcode, render_post, venue_from_description, venue_from_location
|
from .format import find_postcode, render_post, venue_from_description, venue_from_location
|
||||||
from .images import ImageService, PageScraper
|
from .images import ImageService, PageScraper
|
||||||
from .signal_client import SignalClient, SignalError
|
from .signal_client import SignalClient, SignalError
|
||||||
from .store import OutboxItem, Store, iso
|
from .store import OutboxItem, PageInfo, Store, iso
|
||||||
from .watchdog import Watchdog
|
from .watchdog import Watchdog
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
@@ -137,12 +137,12 @@ class Bot:
|
|||||||
if not self.feed.due(now):
|
if not self.feed.due(now):
|
||||||
return
|
return
|
||||||
try:
|
try:
|
||||||
raw = self.feed.fetch(now)
|
response = self.feed.fetch(now)
|
||||||
if raw is None:
|
if response.content is None:
|
||||||
self.watchdog.feed_ok()
|
self.watchdog.feed_ok()
|
||||||
return
|
return
|
||||||
events = parse_events(
|
events = parse_events(
|
||||||
raw,
|
response.content,
|
||||||
now,
|
now,
|
||||||
now + timedelta(days=self.config.calendar.window_days),
|
now + timedelta(days=self.config.calendar.window_days),
|
||||||
self.config.calendar.timezone,
|
self.config.calendar.timezone,
|
||||||
@@ -152,15 +152,24 @@ class Bot:
|
|||||||
self.watchdog.feed_failed(str(exc), now)
|
self.watchdog.feed_failed(str(exc), now)
|
||||||
return
|
return
|
||||||
|
|
||||||
if not events and self.store.count_events() > 0:
|
if not events:
|
||||||
self.watchdog.alert(
|
# Never seed from an empty feed: it is far more likely to be a
|
||||||
"empty-feed",
|
# transient publishing problem than "the calendar has no events".
|
||||||
"Feed parsed to zero events while the database still has events. "
|
if self.store.count_events() > 0:
|
||||||
"Leaving stored events untouched.",
|
self.watchdog.alert(
|
||||||
now,
|
"empty-feed",
|
||||||
)
|
"Feed parsed to zero events while the database still has events. "
|
||||||
|
"Leaving stored events untouched.",
|
||||||
|
now,
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
log.warning("feed returned no events; not seeding yet")
|
||||||
|
self.watchdog.feed_failed("feed parsed to zero events", now)
|
||||||
return
|
return
|
||||||
|
|
||||||
|
# Only persist validators once the payload has parsed cleanly, so a
|
||||||
|
# broken response cannot pin itself behind a 304.
|
||||||
|
self.feed.commit(response)
|
||||||
self.watchdog.feed_ok()
|
self.watchdog.feed_ok()
|
||||||
self._sync(events, now)
|
self._sync(events, now)
|
||||||
|
|
||||||
@@ -183,7 +192,9 @@ class Bot:
|
|||||||
if new_series:
|
if new_series:
|
||||||
self._plan_announcement(event, now)
|
self._plan_announcement(event, now)
|
||||||
|
|
||||||
self.store.mark_gone([event.key for event in events], now)
|
self.store.mark_gone(
|
||||||
|
[event.key for event in events], now, max(event.start for event in events)
|
||||||
|
)
|
||||||
if first_run:
|
if first_run:
|
||||||
self.store.set_meta("seeded", iso(now))
|
self.store.set_meta("seeded", iso(now))
|
||||||
log.info("first run: seeded %d events", len(events))
|
log.info("first run: seeded %d events", len(events))
|
||||||
@@ -263,8 +274,10 @@ class Bot:
|
|||||||
|
|
||||||
valid: list[OutboxItem] = []
|
valid: list[OutboxItem] = []
|
||||||
for item in group:
|
for item in group:
|
||||||
usable = event.end > now and (
|
usable = (
|
||||||
item.kind != "reminder" or event.all_day or event.start > now
|
not event.gone
|
||||||
|
and event.end > now
|
||||||
|
and (item.kind != "reminder" or event.all_day or event.start > now)
|
||||||
)
|
)
|
||||||
if usable:
|
if usable:
|
||||||
valid.append(item)
|
valid.append(item)
|
||||||
@@ -278,7 +291,12 @@ class Bot:
|
|||||||
if not batch:
|
if not batch:
|
||||||
return
|
return
|
||||||
|
|
||||||
message, attachments = self._compose(batch)
|
try:
|
||||||
|
message, attachments = self._compose(batch)
|
||||||
|
except Exception as exc: # noqa: BLE001 - compose must not wedge the outbox
|
||||||
|
log.exception("could not compose post")
|
||||||
|
self._record_failure(batch, str(exc), now)
|
||||||
|
return
|
||||||
try:
|
try:
|
||||||
self.sender.send(
|
self.sender.send(
|
||||||
message,
|
message,
|
||||||
@@ -286,20 +304,9 @@ class Bot:
|
|||||||
attachments=attachments,
|
attachments=attachments,
|
||||||
text_mode=self.config.signal.text_mode,
|
text_mode=self.config.signal.text_mode,
|
||||||
)
|
)
|
||||||
except SignalError as exc:
|
except (SignalError, httpx.HTTPError) as exc:
|
||||||
log.error("send failed: %s", exc)
|
log.error("send failed: %s", exc)
|
||||||
self.watchdog.send_failed(str(exc), now)
|
self._record_failure(batch, str(exc), now)
|
||||||
for _event, group in batch:
|
|
||||||
for item in group:
|
|
||||||
attempts = self.store.bump_attempts(item.id)
|
|
||||||
if attempts >= MAX_SEND_ATTEMPTS:
|
|
||||||
log.error("giving up on %s after %d attempts", item.ref, attempts)
|
|
||||||
self.watchdog.alert(
|
|
||||||
"outbox",
|
|
||||||
f"Dropping a post after {attempts} failed attempts.",
|
|
||||||
now,
|
|
||||||
)
|
|
||||||
self.store.drop_outbox(item.id)
|
|
||||||
return
|
return
|
||||||
|
|
||||||
self.watchdog.send_ok()
|
self.watchdog.send_ok()
|
||||||
@@ -311,44 +318,72 @@ class Bot:
|
|||||||
self.store.drop_outbox(item.id)
|
self.store.drop_outbox(item.id)
|
||||||
log.info("posted %d item(s) covering %d event(s)", len(items), len(batch))
|
log.info("posted %d item(s) covering %d event(s)", len(items), len(batch))
|
||||||
|
|
||||||
|
def _record_failure(
|
||||||
|
self, batch: list[tuple[Event, list[OutboxItem]]], reason: str, now: datetime
|
||||||
|
) -> None:
|
||||||
|
self.watchdog.send_failed(reason, now)
|
||||||
|
for _event, group in batch:
|
||||||
|
for item in group:
|
||||||
|
attempts = self.store.bump_attempts(item.id)
|
||||||
|
if attempts >= MAX_SEND_ATTEMPTS:
|
||||||
|
log.error("giving up on %s after %d attempts", item.ref, attempts)
|
||||||
|
self.watchdog.alert(
|
||||||
|
"outbox",
|
||||||
|
f"Dropping a post after {attempts} failed attempts.",
|
||||||
|
now,
|
||||||
|
)
|
||||||
|
self.store.drop_outbox(item.id)
|
||||||
|
|
||||||
def _compose(self, batch: list[tuple[Event, list[OutboxItem]]]) -> tuple[str, list[str]]:
|
def _compose(self, batch: list[tuple[Event, list[OutboxItem]]]) -> tuple[str, list[str]]:
|
||||||
blocks: list[str] = []
|
blocks: list[str] = []
|
||||||
attachments: list[str] = []
|
attachments: list[str] = []
|
||||||
limit = self.config.images.max_images_per_post if len(batch) > 1 else 1
|
limit = self.config.images.max_images_per_post if len(batch) > 1 else 1
|
||||||
|
|
||||||
for event, _items in batch:
|
for event, _items in batch:
|
||||||
info = self.scraper.details(event.url)
|
block, attachment = self._compose_one(event, limit - len(attachments))
|
||||||
venue = (
|
blocks.append(block)
|
||||||
venue_from_location(event.location)
|
if attachment:
|
||||||
or info.venue
|
attachments.append(attachment)
|
||||||
or venue_from_description(event.description)
|
|
||||||
)
|
|
||||||
postcode = (
|
|
||||||
find_postcode(event.location, event.description) or info.postcode
|
|
||||||
)
|
|
||||||
link = event.url
|
|
||||||
if self.config.format.shorten_links and link:
|
|
||||||
link = self._shorten(link)
|
|
||||||
|
|
||||||
blocks.append(
|
|
||||||
render_post(
|
|
||||||
event,
|
|
||||||
self.config.format,
|
|
||||||
self.local_tz,
|
|
||||||
venue=venue,
|
|
||||||
postcode=postcode,
|
|
||||||
link=link,
|
|
||||||
)
|
|
||||||
)
|
|
||||||
|
|
||||||
if len(attachments) < limit:
|
|
||||||
image_url = event.image_url or info.image_url
|
|
||||||
data_uri = self.images.data_uri(image_url) if image_url else None
|
|
||||||
if data_uri:
|
|
||||||
attachments.append(data_uri)
|
|
||||||
|
|
||||||
return "\n\n".join(blocks), attachments
|
return "\n\n".join(blocks), attachments
|
||||||
|
|
||||||
|
def _compose_one(self, event: Event, images_left: int) -> tuple[str, str | None]:
|
||||||
|
"""Render one event; enrichment and image failures degrade to text only."""
|
||||||
|
info = PageInfo(image_url="", venue="", postcode="")
|
||||||
|
try:
|
||||||
|
info = self.scraper.details(event.url)
|
||||||
|
except Exception: # noqa: BLE001 - never block a post on scraping
|
||||||
|
log.exception("page scrape failed for %s", event.url)
|
||||||
|
|
||||||
|
venue = (
|
||||||
|
venue_from_location(event.location)
|
||||||
|
or info.venue
|
||||||
|
or venue_from_description(event.description)
|
||||||
|
)
|
||||||
|
postcode = find_postcode(event.location, event.description) or info.postcode
|
||||||
|
link = event.url
|
||||||
|
if self.config.format.shorten_links and link:
|
||||||
|
link = self._shorten(link)
|
||||||
|
|
||||||
|
text = render_post(
|
||||||
|
event,
|
||||||
|
self.config.format,
|
||||||
|
self.local_tz,
|
||||||
|
venue=venue,
|
||||||
|
postcode=postcode,
|
||||||
|
link=link,
|
||||||
|
)
|
||||||
|
|
||||||
|
attachment = None
|
||||||
|
if images_left > 0:
|
||||||
|
image_url = event.image_url or info.image_url
|
||||||
|
if image_url:
|
||||||
|
try:
|
||||||
|
attachment = self.images.data_uri(image_url)
|
||||||
|
except Exception: # noqa: BLE001 - images are best effort
|
||||||
|
log.exception("image failed for %s", image_url)
|
||||||
|
return text, attachment
|
||||||
|
|
||||||
def _shorten(self, url: str) -> str:
|
def _shorten(self, url: str) -> str:
|
||||||
cached = self.store.get_short_link(url)
|
cached = self.store.get_short_link(url)
|
||||||
if cached:
|
if cached:
|
||||||
@@ -383,7 +418,7 @@ class Bot:
|
|||||||
[self.watchdog.recipient],
|
[self.watchdog.recipient],
|
||||||
text_mode=self.config.signal.text_mode,
|
text_mode=self.config.signal.text_mode,
|
||||||
)
|
)
|
||||||
except SignalError as exc:
|
except (SignalError, httpx.HTTPError) as exc:
|
||||||
log.warning("heartbeat failed: %s", exc)
|
log.warning("heartbeat failed: %s", exc)
|
||||||
return
|
return
|
||||||
self.store.set_meta("last_heartbeat", today)
|
self.store.set_meta("last_heartbeat", today)
|
||||||
|
|||||||
+21
-11
@@ -143,10 +143,6 @@ class Store:
|
|||||||
|
|
||||||
# -- events ----------------------------------------------------------
|
# -- events ----------------------------------------------------------
|
||||||
|
|
||||||
def has_events(self) -> bool:
|
|
||||||
row = self.conn.execute("SELECT 1 FROM events LIMIT 1").fetchone()
|
|
||||||
return row is not None
|
|
||||||
|
|
||||||
def event_exists(self, event_key: str) -> bool:
|
def event_exists(self, event_key: str) -> bool:
|
||||||
row = self.conn.execute(
|
row = self.conn.execute(
|
||||||
"SELECT 1 FROM events WHERE event_key = ?", (event_key,)
|
"SELECT 1 FROM events WHERE event_key = ?", (event_key,)
|
||||||
@@ -194,15 +190,28 @@ class Store:
|
|||||||
self.conn.commit()
|
self.conn.commit()
|
||||||
return is_new
|
return is_new
|
||||||
|
|
||||||
def mark_gone(self, seen_keys: Iterable[str], now: datetime) -> None:
|
def mark_gone(
|
||||||
|
self, seen_keys: Iterable[str], now: datetime, up_to: datetime | None = None
|
||||||
|
) -> None:
|
||||||
|
"""Flag events missing from the feed as gone.
|
||||||
|
|
||||||
|
``up_to`` is the end of the feed's coverage. The Events Calendar feed is
|
||||||
|
capped at ~30 events, so anything starting after the last event in the
|
||||||
|
feed is simply unknown, not cancelled.
|
||||||
|
"""
|
||||||
seen = set(seen_keys)
|
seen = set(seen_keys)
|
||||||
rows = self.conn.execute("SELECT event_key FROM events WHERE gone = 0").fetchall()
|
rows = self.conn.execute(
|
||||||
|
"SELECT event_key, start_utc FROM events WHERE gone = 0"
|
||||||
|
).fetchall()
|
||||||
for row in rows:
|
for row in rows:
|
||||||
if row["event_key"] not in seen:
|
if row["event_key"] in seen:
|
||||||
self.conn.execute(
|
continue
|
||||||
"UPDATE events SET gone = 1, last_seen_at = ? WHERE event_key = ?",
|
if up_to is not None and from_iso(row["start_utc"]) > up_to:
|
||||||
(iso(now), row["event_key"]),
|
continue
|
||||||
)
|
self.conn.execute(
|
||||||
|
"UPDATE events SET gone = 1, last_seen_at = ? WHERE event_key = ?",
|
||||||
|
(iso(now), row["event_key"]),
|
||||||
|
)
|
||||||
self.conn.commit()
|
self.conn.commit()
|
||||||
|
|
||||||
def get_event(self, event_key: str) -> Event | None:
|
def get_event(self, event_key: str) -> Event | None:
|
||||||
@@ -224,6 +233,7 @@ class Store:
|
|||||||
image_url=row["image_url"],
|
image_url=row["image_url"],
|
||||||
tz=row["event_tz"],
|
tz=row["event_tz"],
|
||||||
status=row["status"],
|
status=row["status"],
|
||||||
|
gone=bool(row["gone"]),
|
||||||
)
|
)
|
||||||
|
|
||||||
def count_events(self) -> int:
|
def count_events(self) -> int:
|
||||||
|
|||||||
@@ -0,0 +1,56 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import httpx
|
||||||
|
|
||||||
|
from calbot.feed import Feed, FeedResponse, normalize_feed_url
|
||||||
|
from calbot.store import Store
|
||||||
|
|
||||||
|
NOW = datetime(2026, 11, 18, 12, 0, tzinfo=timezone.utc)
|
||||||
|
EMPTY_FEED = b"BEGIN:VCALENDAR\r\nVERSION:2.0\r\nPRODID:-//test//EN\r\nEND:VCALENDAR\r\n"
|
||||||
|
|
||||||
|
|
||||||
|
def test_webcal_is_fetched_over_https():
|
||||||
|
assert normalize_feed_url("webcal://example.test/x.ics") == "https://example.test/x.ics"
|
||||||
|
assert normalize_feed_url("https://example.test/x.ics") == "https://example.test/x.ics"
|
||||||
|
|
||||||
|
|
||||||
|
def test_validators_are_only_stored_after_commit(tmp_path):
|
||||||
|
def handler(request: httpx.Request) -> httpx.Response:
|
||||||
|
if request.headers.get("if-none-match") == '"abc"':
|
||||||
|
return httpx.Response(304)
|
||||||
|
return httpx.Response(200, content=EMPTY_FEED, headers={"ETag": '"abc"'})
|
||||||
|
|
||||||
|
store = Store(Path(tmp_path) / "db.sqlite3")
|
||||||
|
client = httpx.Client(transport=httpx.MockTransport(handler))
|
||||||
|
feed = Feed("webcal://example.test/feed.ics", store, 15, client)
|
||||||
|
|
||||||
|
response = feed.fetch(NOW)
|
||||||
|
assert isinstance(response, FeedResponse)
|
||||||
|
assert response.content == EMPTY_FEED
|
||||||
|
# A parse failure would skip commit(), so nothing may be persisted yet.
|
||||||
|
assert store.get_meta("feed_etag") is None
|
||||||
|
|
||||||
|
feed.commit(response)
|
||||||
|
assert store.get_meta("feed_etag") == '"abc"'
|
||||||
|
|
||||||
|
second = feed.fetch(NOW)
|
||||||
|
assert second.not_modified
|
||||||
|
assert second.content is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_fetch_raises_on_http_error(tmp_path):
|
||||||
|
def handler(request: httpx.Request) -> httpx.Response:
|
||||||
|
return httpx.Response(500, text="boom")
|
||||||
|
|
||||||
|
store = Store(Path(tmp_path) / "db.sqlite3")
|
||||||
|
client = httpx.Client(transport=httpx.MockTransport(handler))
|
||||||
|
feed = Feed("https://example.test/feed.ics", store, 15, client)
|
||||||
|
try:
|
||||||
|
feed.fetch(NOW)
|
||||||
|
except httpx.HTTPStatusError:
|
||||||
|
pass
|
||||||
|
else: # pragma: no cover
|
||||||
|
raise AssertionError("expected HTTPStatusError")
|
||||||
@@ -96,3 +96,16 @@ def test_missing_fields_are_omitted():
|
|||||||
text = render_post(event, FormatConfig(), LONDON, venue="", postcode="", link="")
|
text = render_post(event, FormatConfig(), LONDON, venue="", postcode="", link="")
|
||||||
lines = text.splitlines()
|
lines = text.splitlines()
|
||||||
assert lines == ["Folk Night", "#November #November14", "4.30-6.30pm"]
|
assert lines == ["Folk Night", "#November #November14", "4.30-6.30pm"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_postcode_is_not_duplicated_when_part_of_venue():
|
||||||
|
event = make_event(location="Old Print Works B12 9AH", url="")
|
||||||
|
text = render_post(
|
||||||
|
event,
|
||||||
|
FormatConfig(),
|
||||||
|
LONDON,
|
||||||
|
venue="Old Print Works B12 9AH",
|
||||||
|
postcode="B12 9AH",
|
||||||
|
link="",
|
||||||
|
)
|
||||||
|
assert text.splitlines()[3] == "Old Print Works B12 9AH"
|
||||||
|
|||||||
@@ -0,0 +1,48 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import io
|
||||||
|
|
||||||
|
import httpx
|
||||||
|
from PIL import Image
|
||||||
|
|
||||||
|
from calbot.config import ImagesConfig
|
||||||
|
from calbot.images import ImageService
|
||||||
|
|
||||||
|
|
||||||
|
def make_service(tmp_path, **overrides) -> ImageService:
|
||||||
|
return ImageService(ImagesConfig(**overrides), tmp_path, httpx.Client())
|
||||||
|
|
||||||
|
|
||||||
|
def test_transparent_png_is_flattened_onto_white(tmp_path):
|
||||||
|
buffer = io.BytesIO()
|
||||||
|
Image.new("RGBA", (120, 120), (255, 0, 0, 0)).save(buffer, "PNG")
|
||||||
|
|
||||||
|
service = make_service(tmp_path)
|
||||||
|
result = service._resize(buffer.getvalue())
|
||||||
|
assert result
|
||||||
|
|
||||||
|
image = Image.open(io.BytesIO(result))
|
||||||
|
assert image.mode == "RGB"
|
||||||
|
red, green, blue = image.getpixel((60, 60))
|
||||||
|
assert min(red, green, blue) > 240
|
||||||
|
|
||||||
|
|
||||||
|
def test_oversized_image_is_downscaled(tmp_path):
|
||||||
|
buffer = io.BytesIO()
|
||||||
|
Image.new("RGB", (4000, 3000), (10, 80, 200)).save(buffer, "PNG")
|
||||||
|
service = make_service(tmp_path, max_pixels=400, max_bytes=50_000)
|
||||||
|
result = service._resize(buffer.getvalue())
|
||||||
|
assert result
|
||||||
|
assert len(result) <= 50_000
|
||||||
|
image = Image.open(io.BytesIO(result))
|
||||||
|
assert max(image.size) <= 400
|
||||||
|
|
||||||
|
|
||||||
|
def test_decompression_bomb_does_not_propagate(tmp_path):
|
||||||
|
service = make_service(tmp_path)
|
||||||
|
|
||||||
|
def boom(url):
|
||||||
|
raise Image.DecompressionBombError("too many pixels")
|
||||||
|
|
||||||
|
service._cached_or_download = boom # type: ignore[method-assign]
|
||||||
|
assert service.data_uri("https://example.test/huge.jpg") is None
|
||||||
+149
-7
@@ -18,7 +18,7 @@ from calbot.config import (
|
|||||||
WatchdogConfig,
|
WatchdogConfig,
|
||||||
)
|
)
|
||||||
from calbot.expand import Event
|
from calbot.expand import Event
|
||||||
from calbot.feed import Feed, normalize_feed_url
|
from calbot.feed import Feed, FeedResponse, normalize_feed_url
|
||||||
from calbot.scheduler import (
|
from calbot.scheduler import (
|
||||||
Bot,
|
Bot,
|
||||||
clamp_send_at,
|
clamp_send_at,
|
||||||
@@ -47,6 +47,7 @@ def make_event(
|
|||||||
start: datetime,
|
start: datetime,
|
||||||
end: datetime | None = None,
|
end: datetime | None = None,
|
||||||
all_day: bool = False,
|
all_day: bool = False,
|
||||||
|
image_url: str = "",
|
||||||
) -> Event:
|
) -> Event:
|
||||||
return Event(
|
return Event(
|
||||||
key=f"{uid}|{start.astimezone(UTC).isoformat()}",
|
key=f"{uid}|{start.astimezone(UTC).isoformat()}",
|
||||||
@@ -58,6 +59,7 @@ def make_event(
|
|||||||
all_day=all_day,
|
all_day=all_day,
|
||||||
location="Old Print Works\nB12 9AH",
|
location="Old Print Works\nB12 9AH",
|
||||||
url="https://commonscalendar.uk/event/folk-night/",
|
url="https://commonscalendar.uk/event/folk-night/",
|
||||||
|
image_url=image_url,
|
||||||
tz="Europe/London",
|
tz="Europe/London",
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -170,18 +172,57 @@ class FakeScraper:
|
|||||||
return PageInfo(image_url="", venue="", postcode="")
|
return PageInfo(image_url="", venue="", postcode="")
|
||||||
|
|
||||||
|
|
||||||
def build_bot(tmp_path, sender):
|
class BoomScraper:
|
||||||
config = make_config(tmp_path)
|
def details(self, url):
|
||||||
|
raise RuntimeError("page down")
|
||||||
|
|
||||||
|
|
||||||
|
class BoomImages:
|
||||||
|
def data_uri(self, url):
|
||||||
|
raise RuntimeError("image down")
|
||||||
|
|
||||||
|
|
||||||
|
class FailingSender:
|
||||||
|
def __init__(self, exc):
|
||||||
|
self.exc = exc
|
||||||
|
self.attempts = 0
|
||||||
|
|
||||||
|
def send(self, message, recipients, attachments=None, text_mode="normal"):
|
||||||
|
self.attempts += 1
|
||||||
|
raise self.exc
|
||||||
|
|
||||||
|
|
||||||
|
class FakeFeed:
|
||||||
|
def __init__(self, content: bytes):
|
||||||
|
self.content = content
|
||||||
|
self.committed = False
|
||||||
|
|
||||||
|
def due(self, now):
|
||||||
|
return True
|
||||||
|
|
||||||
|
def fetch(self, now):
|
||||||
|
return FeedResponse(content=self.content)
|
||||||
|
|
||||||
|
def commit(self, response):
|
||||||
|
self.committed = True
|
||||||
|
|
||||||
|
|
||||||
|
def build_bot(tmp_path, sender, **overrides):
|
||||||
|
config = overrides.pop("config", None) or make_config(tmp_path)
|
||||||
store = Store(config.db_path)
|
store = Store(config.db_path)
|
||||||
http = httpx.Client()
|
http = httpx.Client()
|
||||||
bot = Bot(
|
bot = Bot(
|
||||||
config=config,
|
config=config,
|
||||||
store=store,
|
store=store,
|
||||||
feed=Feed(config.calendar.url, store, 15, http),
|
feed=overrides.pop(
|
||||||
|
"feed", Feed(config.calendar.url, store, 15, http)
|
||||||
|
),
|
||||||
sender=sender,
|
sender=sender,
|
||||||
images=FakeImages(),
|
images=overrides.pop("images", FakeImages()),
|
||||||
scraper=FakeScraper(),
|
scraper=overrides.pop("scraper", FakeScraper()),
|
||||||
watchdog=Watchdog(config.watchdog, sender, "+447700900000"),
|
watchdog=overrides.pop(
|
||||||
|
"watchdog", Watchdog(config.watchdog, sender, "+447700900000")
|
||||||
|
),
|
||||||
http=http,
|
http=http,
|
||||||
)
|
)
|
||||||
return bot, store
|
return bot, store
|
||||||
@@ -244,3 +285,104 @@ def test_stale_reminder_is_dropped_not_sent(tmp_path):
|
|||||||
bot._dispatch(local(2026, 11, 18, 18, 0)) # event already started
|
bot._dispatch(local(2026, 11, 18, 18, 0)) # event already started
|
||||||
assert sender.sent == []
|
assert sender.sent == []
|
||||||
assert store.due_outbox(local(2026, 11, 18, 18, 5)) == []
|
assert store.due_outbox(local(2026, 11, 18, 18, 5)) == []
|
||||||
|
|
||||||
|
|
||||||
|
# -- failure handling -------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def test_signal_connection_error_does_not_wedge_the_outbox(tmp_path):
|
||||||
|
sender = FailingSender(httpx.ConnectError("connection refused"))
|
||||||
|
config = make_config(
|
||||||
|
tmp_path, watchdog=WatchdogConfig(enabled=True, send_failures=1)
|
||||||
|
)
|
||||||
|
watchdog = Watchdog(config.watchdog, sender, "+447700900000")
|
||||||
|
bot, store = build_bot(tmp_path, sender, config=config, watchdog=watchdog)
|
||||||
|
|
||||||
|
event = make_event(uid="20", series_id="tec:20", start=local(2026, 11, 25, 19, 0))
|
||||||
|
bot._sync([event], local(2026, 11, 18, 12, 0))
|
||||||
|
bot._sync(
|
||||||
|
[event, make_event(uid="21", series_id="tec:21", start=local(2026, 11, 26, 19, 0))],
|
||||||
|
local(2026, 11, 18, 12, 1),
|
||||||
|
)
|
||||||
|
|
||||||
|
now = local(2026, 11, 18, 18, 0)
|
||||||
|
bot._dispatch(now) # must not raise
|
||||||
|
|
||||||
|
pending = store.due_outbox(now)
|
||||||
|
assert pending, "the failed post should stay queued"
|
||||||
|
assert pending[0].attempts == 1
|
||||||
|
# The announcement stays queued, so it is not marked as delivered.
|
||||||
|
assert store.due_outbox(now)[0].kind == "announce"
|
||||||
|
|
||||||
|
|
||||||
|
def test_scraper_and_image_failures_still_send_text(tmp_path):
|
||||||
|
sender = FakeSender()
|
||||||
|
bot, store = build_bot(
|
||||||
|
tmp_path,
|
||||||
|
sender,
|
||||||
|
images=BoomImages(),
|
||||||
|
scraper=BoomScraper(),
|
||||||
|
)
|
||||||
|
|
||||||
|
event = make_event(
|
||||||
|
uid="22",
|
||||||
|
series_id="tec:22",
|
||||||
|
start=local(2026, 11, 25, 19, 0),
|
||||||
|
image_url="https://example.test/poster.png",
|
||||||
|
)
|
||||||
|
bot._sync([event], local(2026, 11, 18, 12, 0))
|
||||||
|
bot._sync(
|
||||||
|
[event, make_event(uid="23", series_id="tec:23", start=local(2026, 11, 26, 19, 0))],
|
||||||
|
local(2026, 11, 18, 12, 1),
|
||||||
|
)
|
||||||
|
bot._dispatch(local(2026, 11, 18, 18, 0))
|
||||||
|
|
||||||
|
assert len(sender.sent) == 1
|
||||||
|
assert sender.sent[0]["attachments"] == []
|
||||||
|
assert sender.sent[0]["message"].startswith("Folk Night")
|
||||||
|
|
||||||
|
|
||||||
|
# -- gone events ------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def test_reminder_for_gone_event_is_dropped(tmp_path):
|
||||||
|
sender = FakeSender()
|
||||||
|
bot, store = build_bot(tmp_path, sender)
|
||||||
|
|
||||||
|
event = make_event(uid="30", series_id="tec:30", start=local(2026, 11, 25, 14, 0))
|
||||||
|
bot._sync([event], local(2026, 11, 18, 12, 0))
|
||||||
|
store.mark_gone([], local(2026, 11, 18, 13, 0), up_to=event.end + timedelta(days=2))
|
||||||
|
|
||||||
|
bot._dispatch(local(2026, 11, 22, 18, 5))
|
||||||
|
assert sender.sent == []
|
||||||
|
assert store.due_outbox(local(2026, 11, 22, 18, 10)) == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_events_beyond_feed_coverage_are_not_marked_gone(tmp_path):
|
||||||
|
sender = FakeSender()
|
||||||
|
bot, store = build_bot(tmp_path, sender)
|
||||||
|
|
||||||
|
event = make_event(uid="31", series_id="tec:31", start=local(2026, 12, 20, 14, 0))
|
||||||
|
bot._sync([event], local(2026, 11, 18, 12, 0))
|
||||||
|
store.mark_gone([], local(2026, 11, 18, 13, 0), up_to=local(2026, 11, 30, 0, 0))
|
||||||
|
|
||||||
|
stored = store.get_event(event.key)
|
||||||
|
assert stored is not None and stored.gone is False
|
||||||
|
|
||||||
|
|
||||||
|
# -- feed seeding -----------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def test_empty_first_feed_does_not_seed(tmp_path):
|
||||||
|
sender = FakeSender()
|
||||||
|
empty = (
|
||||||
|
b"BEGIN:VCALENDAR\r\nVERSION:2.0\r\nPRODID:-//test//EN\r\n"
|
||||||
|
b"END:VCALENDAR\r\n"
|
||||||
|
)
|
||||||
|
feed = FakeFeed(empty)
|
||||||
|
bot, store = build_bot(tmp_path, sender, feed=feed)
|
||||||
|
|
||||||
|
bot._refresh_feed(local(2026, 11, 18, 12, 0))
|
||||||
|
|
||||||
|
assert store.get_meta("seeded") is None
|
||||||
|
assert feed.committed is False
|
||||||
|
|||||||
Reference in New Issue
Block a user