Fix watchdog gaps, image failures and stale reminders

- catch httpx errors on send/heartbeat so a dead signal-api alerts instead
  of failing silently
- never let compose/images block the text post; count failed attempts
- drop reminders for events removed from the feed, using the feed's
  coverage horizon so the 30-event cap does not mark future events gone
- persist feed validators only after a successful parse
- do not seed from an empty feed
- flatten transparent images onto white; catch decompression bombs
- --dump uses a throwaway database
- avoid duplicated postcode in the venue line
- remove dead code; add regression tests
This commit is contained in:
2026-11-18 16:42:47 +00:00
parent 234c990e84
commit bdd41929d4
11 changed files with 483 additions and 137 deletions
+56
View File
@@ -0,0 +1,56 @@
from __future__ import annotations
from datetime import datetime, timezone
from pathlib import Path
import httpx
from calbot.feed import Feed, FeedResponse, normalize_feed_url
from calbot.store import Store
NOW = datetime(2026, 11, 18, 12, 0, tzinfo=timezone.utc)
EMPTY_FEED = b"BEGIN:VCALENDAR\r\nVERSION:2.0\r\nPRODID:-//test//EN\r\nEND:VCALENDAR\r\n"
def test_webcal_is_fetched_over_https():
assert normalize_feed_url("webcal://example.test/x.ics") == "https://example.test/x.ics"
assert normalize_feed_url("https://example.test/x.ics") == "https://example.test/x.ics"
def test_validators_are_only_stored_after_commit(tmp_path):
def handler(request: httpx.Request) -> httpx.Response:
if request.headers.get("if-none-match") == '"abc"':
return httpx.Response(304)
return httpx.Response(200, content=EMPTY_FEED, headers={"ETag": '"abc"'})
store = Store(Path(tmp_path) / "db.sqlite3")
client = httpx.Client(transport=httpx.MockTransport(handler))
feed = Feed("webcal://example.test/feed.ics", store, 15, client)
response = feed.fetch(NOW)
assert isinstance(response, FeedResponse)
assert response.content == EMPTY_FEED
# A parse failure would skip commit(), so nothing may be persisted yet.
assert store.get_meta("feed_etag") is None
feed.commit(response)
assert store.get_meta("feed_etag") == '"abc"'
second = feed.fetch(NOW)
assert second.not_modified
assert second.content is None
def test_fetch_raises_on_http_error(tmp_path):
def handler(request: httpx.Request) -> httpx.Response:
return httpx.Response(500, text="boom")
store = Store(Path(tmp_path) / "db.sqlite3")
client = httpx.Client(transport=httpx.MockTransport(handler))
feed = Feed("https://example.test/feed.ics", store, 15, client)
try:
feed.fetch(NOW)
except httpx.HTTPStatusError:
pass
else: # pragma: no cover
raise AssertionError("expected HTTPStatusError")
+13
View File
@@ -96,3 +96,16 @@ def test_missing_fields_are_omitted():
text = render_post(event, FormatConfig(), LONDON, venue="", postcode="", link="")
lines = text.splitlines()
assert lines == ["Folk Night", "#November #November14", "4.30-6.30pm"]
def test_postcode_is_not_duplicated_when_part_of_venue():
event = make_event(location="Old Print Works B12 9AH", url="")
text = render_post(
event,
FormatConfig(),
LONDON,
venue="Old Print Works B12 9AH",
postcode="B12 9AH",
link="",
)
assert text.splitlines()[3] == "Old Print Works B12 9AH"
+48
View File
@@ -0,0 +1,48 @@
from __future__ import annotations
import io
import httpx
from PIL import Image
from calbot.config import ImagesConfig
from calbot.images import ImageService
def make_service(tmp_path, **overrides) -> ImageService:
return ImageService(ImagesConfig(**overrides), tmp_path, httpx.Client())
def test_transparent_png_is_flattened_onto_white(tmp_path):
buffer = io.BytesIO()
Image.new("RGBA", (120, 120), (255, 0, 0, 0)).save(buffer, "PNG")
service = make_service(tmp_path)
result = service._resize(buffer.getvalue())
assert result
image = Image.open(io.BytesIO(result))
assert image.mode == "RGB"
red, green, blue = image.getpixel((60, 60))
assert min(red, green, blue) > 240
def test_oversized_image_is_downscaled(tmp_path):
buffer = io.BytesIO()
Image.new("RGB", (4000, 3000), (10, 80, 200)).save(buffer, "PNG")
service = make_service(tmp_path, max_pixels=400, max_bytes=50_000)
result = service._resize(buffer.getvalue())
assert result
assert len(result) <= 50_000
image = Image.open(io.BytesIO(result))
assert max(image.size) <= 400
def test_decompression_bomb_does_not_propagate(tmp_path):
service = make_service(tmp_path)
def boom(url):
raise Image.DecompressionBombError("too many pixels")
service._cached_or_download = boom # type: ignore[method-assign]
assert service.data_uri("https://example.test/huge.jpg") is None
+149 -7
View File
@@ -18,7 +18,7 @@ from calbot.config import (
WatchdogConfig,
)
from calbot.expand import Event
from calbot.feed import Feed, normalize_feed_url
from calbot.feed import Feed, FeedResponse, normalize_feed_url
from calbot.scheduler import (
Bot,
clamp_send_at,
@@ -47,6 +47,7 @@ def make_event(
start: datetime,
end: datetime | None = None,
all_day: bool = False,
image_url: str = "",
) -> Event:
return Event(
key=f"{uid}|{start.astimezone(UTC).isoformat()}",
@@ -58,6 +59,7 @@ def make_event(
all_day=all_day,
location="Old Print Works\nB12 9AH",
url="https://commonscalendar.uk/event/folk-night/",
image_url=image_url,
tz="Europe/London",
)
@@ -170,18 +172,57 @@ class FakeScraper:
return PageInfo(image_url="", venue="", postcode="")
def build_bot(tmp_path, sender):
config = make_config(tmp_path)
class BoomScraper:
def details(self, url):
raise RuntimeError("page down")
class BoomImages:
def data_uri(self, url):
raise RuntimeError("image down")
class FailingSender:
def __init__(self, exc):
self.exc = exc
self.attempts = 0
def send(self, message, recipients, attachments=None, text_mode="normal"):
self.attempts += 1
raise self.exc
class FakeFeed:
def __init__(self, content: bytes):
self.content = content
self.committed = False
def due(self, now):
return True
def fetch(self, now):
return FeedResponse(content=self.content)
def commit(self, response):
self.committed = True
def build_bot(tmp_path, sender, **overrides):
config = overrides.pop("config", None) or make_config(tmp_path)
store = Store(config.db_path)
http = httpx.Client()
bot = Bot(
config=config,
store=store,
feed=Feed(config.calendar.url, store, 15, http),
feed=overrides.pop(
"feed", Feed(config.calendar.url, store, 15, http)
),
sender=sender,
images=FakeImages(),
scraper=FakeScraper(),
watchdog=Watchdog(config.watchdog, sender, "+447700900000"),
images=overrides.pop("images", FakeImages()),
scraper=overrides.pop("scraper", FakeScraper()),
watchdog=overrides.pop(
"watchdog", Watchdog(config.watchdog, sender, "+447700900000")
),
http=http,
)
return bot, store
@@ -244,3 +285,104 @@ def test_stale_reminder_is_dropped_not_sent(tmp_path):
bot._dispatch(local(2026, 11, 18, 18, 0)) # event already started
assert sender.sent == []
assert store.due_outbox(local(2026, 11, 18, 18, 5)) == []
# -- failure handling -------------------------------------------------------
def test_signal_connection_error_does_not_wedge_the_outbox(tmp_path):
sender = FailingSender(httpx.ConnectError("connection refused"))
config = make_config(
tmp_path, watchdog=WatchdogConfig(enabled=True, send_failures=1)
)
watchdog = Watchdog(config.watchdog, sender, "+447700900000")
bot, store = build_bot(tmp_path, sender, config=config, watchdog=watchdog)
event = make_event(uid="20", series_id="tec:20", start=local(2026, 11, 25, 19, 0))
bot._sync([event], local(2026, 11, 18, 12, 0))
bot._sync(
[event, make_event(uid="21", series_id="tec:21", start=local(2026, 11, 26, 19, 0))],
local(2026, 11, 18, 12, 1),
)
now = local(2026, 11, 18, 18, 0)
bot._dispatch(now) # must not raise
pending = store.due_outbox(now)
assert pending, "the failed post should stay queued"
assert pending[0].attempts == 1
# The announcement stays queued, so it is not marked as delivered.
assert store.due_outbox(now)[0].kind == "announce"
def test_scraper_and_image_failures_still_send_text(tmp_path):
sender = FakeSender()
bot, store = build_bot(
tmp_path,
sender,
images=BoomImages(),
scraper=BoomScraper(),
)
event = make_event(
uid="22",
series_id="tec:22",
start=local(2026, 11, 25, 19, 0),
image_url="https://example.test/poster.png",
)
bot._sync([event], local(2026, 11, 18, 12, 0))
bot._sync(
[event, make_event(uid="23", series_id="tec:23", start=local(2026, 11, 26, 19, 0))],
local(2026, 11, 18, 12, 1),
)
bot._dispatch(local(2026, 11, 18, 18, 0))
assert len(sender.sent) == 1
assert sender.sent[0]["attachments"] == []
assert sender.sent[0]["message"].startswith("Folk Night")
# -- gone events ------------------------------------------------------------
def test_reminder_for_gone_event_is_dropped(tmp_path):
sender = FakeSender()
bot, store = build_bot(tmp_path, sender)
event = make_event(uid="30", series_id="tec:30", start=local(2026, 11, 25, 14, 0))
bot._sync([event], local(2026, 11, 18, 12, 0))
store.mark_gone([], local(2026, 11, 18, 13, 0), up_to=event.end + timedelta(days=2))
bot._dispatch(local(2026, 11, 22, 18, 5))
assert sender.sent == []
assert store.due_outbox(local(2026, 11, 22, 18, 10)) == []
def test_events_beyond_feed_coverage_are_not_marked_gone(tmp_path):
sender = FakeSender()
bot, store = build_bot(tmp_path, sender)
event = make_event(uid="31", series_id="tec:31", start=local(2026, 12, 20, 14, 0))
bot._sync([event], local(2026, 11, 18, 12, 0))
store.mark_gone([], local(2026, 11, 18, 13, 0), up_to=local(2026, 11, 30, 0, 0))
stored = store.get_event(event.key)
assert stored is not None and stored.gone is False
# -- feed seeding -----------------------------------------------------------
def test_empty_first_feed_does_not_seed(tmp_path):
sender = FakeSender()
empty = (
b"BEGIN:VCALENDAR\r\nVERSION:2.0\r\nPRODID:-//test//EN\r\n"
b"END:VCALENDAR\r\n"
)
feed = FakeFeed(empty)
bot, store = build_bot(tmp_path, sender, feed=feed)
bot._refresh_feed(local(2026, 11, 18, 12, 0))
assert store.get_meta("seeded") is None
assert feed.committed is False