Fix watchdog gaps, image failures and stale reminders
- catch httpx errors on send/heartbeat so a dead signal-api alerts instead of failing silently - never let compose/images block the text post; count failed attempts - drop reminders for events removed from the feed, using the feed's coverage horizon so the 30-event cap does not mark future events gone - persist feed validators only after a successful parse - do not seed from an empty feed - flatten transparent images onto white; catch decompression bombs - --dump uses a throwaway database - avoid duplicated postcode in the venue line - remove dead code; add regression tests
This commit is contained in:
@@ -0,0 +1,56 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
import httpx
|
||||
|
||||
from calbot.feed import Feed, FeedResponse, normalize_feed_url
|
||||
from calbot.store import Store
|
||||
|
||||
NOW = datetime(2026, 11, 18, 12, 0, tzinfo=timezone.utc)
|
||||
EMPTY_FEED = b"BEGIN:VCALENDAR\r\nVERSION:2.0\r\nPRODID:-//test//EN\r\nEND:VCALENDAR\r\n"
|
||||
|
||||
|
||||
def test_webcal_is_fetched_over_https():
|
||||
assert normalize_feed_url("webcal://example.test/x.ics") == "https://example.test/x.ics"
|
||||
assert normalize_feed_url("https://example.test/x.ics") == "https://example.test/x.ics"
|
||||
|
||||
|
||||
def test_validators_are_only_stored_after_commit(tmp_path):
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
if request.headers.get("if-none-match") == '"abc"':
|
||||
return httpx.Response(304)
|
||||
return httpx.Response(200, content=EMPTY_FEED, headers={"ETag": '"abc"'})
|
||||
|
||||
store = Store(Path(tmp_path) / "db.sqlite3")
|
||||
client = httpx.Client(transport=httpx.MockTransport(handler))
|
||||
feed = Feed("webcal://example.test/feed.ics", store, 15, client)
|
||||
|
||||
response = feed.fetch(NOW)
|
||||
assert isinstance(response, FeedResponse)
|
||||
assert response.content == EMPTY_FEED
|
||||
# A parse failure would skip commit(), so nothing may be persisted yet.
|
||||
assert store.get_meta("feed_etag") is None
|
||||
|
||||
feed.commit(response)
|
||||
assert store.get_meta("feed_etag") == '"abc"'
|
||||
|
||||
second = feed.fetch(NOW)
|
||||
assert second.not_modified
|
||||
assert second.content is None
|
||||
|
||||
|
||||
def test_fetch_raises_on_http_error(tmp_path):
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(500, text="boom")
|
||||
|
||||
store = Store(Path(tmp_path) / "db.sqlite3")
|
||||
client = httpx.Client(transport=httpx.MockTransport(handler))
|
||||
feed = Feed("https://example.test/feed.ics", store, 15, client)
|
||||
try:
|
||||
feed.fetch(NOW)
|
||||
except httpx.HTTPStatusError:
|
||||
pass
|
||||
else: # pragma: no cover
|
||||
raise AssertionError("expected HTTPStatusError")
|
||||
@@ -96,3 +96,16 @@ def test_missing_fields_are_omitted():
|
||||
text = render_post(event, FormatConfig(), LONDON, venue="", postcode="", link="")
|
||||
lines = text.splitlines()
|
||||
assert lines == ["Folk Night", "#November #November14", "4.30-6.30pm"]
|
||||
|
||||
|
||||
def test_postcode_is_not_duplicated_when_part_of_venue():
|
||||
event = make_event(location="Old Print Works B12 9AH", url="")
|
||||
text = render_post(
|
||||
event,
|
||||
FormatConfig(),
|
||||
LONDON,
|
||||
venue="Old Print Works B12 9AH",
|
||||
postcode="B12 9AH",
|
||||
link="",
|
||||
)
|
||||
assert text.splitlines()[3] == "Old Print Works B12 9AH"
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
|
||||
import httpx
|
||||
from PIL import Image
|
||||
|
||||
from calbot.config import ImagesConfig
|
||||
from calbot.images import ImageService
|
||||
|
||||
|
||||
def make_service(tmp_path, **overrides) -> ImageService:
|
||||
return ImageService(ImagesConfig(**overrides), tmp_path, httpx.Client())
|
||||
|
||||
|
||||
def test_transparent_png_is_flattened_onto_white(tmp_path):
|
||||
buffer = io.BytesIO()
|
||||
Image.new("RGBA", (120, 120), (255, 0, 0, 0)).save(buffer, "PNG")
|
||||
|
||||
service = make_service(tmp_path)
|
||||
result = service._resize(buffer.getvalue())
|
||||
assert result
|
||||
|
||||
image = Image.open(io.BytesIO(result))
|
||||
assert image.mode == "RGB"
|
||||
red, green, blue = image.getpixel((60, 60))
|
||||
assert min(red, green, blue) > 240
|
||||
|
||||
|
||||
def test_oversized_image_is_downscaled(tmp_path):
|
||||
buffer = io.BytesIO()
|
||||
Image.new("RGB", (4000, 3000), (10, 80, 200)).save(buffer, "PNG")
|
||||
service = make_service(tmp_path, max_pixels=400, max_bytes=50_000)
|
||||
result = service._resize(buffer.getvalue())
|
||||
assert result
|
||||
assert len(result) <= 50_000
|
||||
image = Image.open(io.BytesIO(result))
|
||||
assert max(image.size) <= 400
|
||||
|
||||
|
||||
def test_decompression_bomb_does_not_propagate(tmp_path):
|
||||
service = make_service(tmp_path)
|
||||
|
||||
def boom(url):
|
||||
raise Image.DecompressionBombError("too many pixels")
|
||||
|
||||
service._cached_or_download = boom # type: ignore[method-assign]
|
||||
assert service.data_uri("https://example.test/huge.jpg") is None
|
||||
+149
-7
@@ -18,7 +18,7 @@ from calbot.config import (
|
||||
WatchdogConfig,
|
||||
)
|
||||
from calbot.expand import Event
|
||||
from calbot.feed import Feed, normalize_feed_url
|
||||
from calbot.feed import Feed, FeedResponse, normalize_feed_url
|
||||
from calbot.scheduler import (
|
||||
Bot,
|
||||
clamp_send_at,
|
||||
@@ -47,6 +47,7 @@ def make_event(
|
||||
start: datetime,
|
||||
end: datetime | None = None,
|
||||
all_day: bool = False,
|
||||
image_url: str = "",
|
||||
) -> Event:
|
||||
return Event(
|
||||
key=f"{uid}|{start.astimezone(UTC).isoformat()}",
|
||||
@@ -58,6 +59,7 @@ def make_event(
|
||||
all_day=all_day,
|
||||
location="Old Print Works\nB12 9AH",
|
||||
url="https://commonscalendar.uk/event/folk-night/",
|
||||
image_url=image_url,
|
||||
tz="Europe/London",
|
||||
)
|
||||
|
||||
@@ -170,18 +172,57 @@ class FakeScraper:
|
||||
return PageInfo(image_url="", venue="", postcode="")
|
||||
|
||||
|
||||
def build_bot(tmp_path, sender):
|
||||
config = make_config(tmp_path)
|
||||
class BoomScraper:
|
||||
def details(self, url):
|
||||
raise RuntimeError("page down")
|
||||
|
||||
|
||||
class BoomImages:
|
||||
def data_uri(self, url):
|
||||
raise RuntimeError("image down")
|
||||
|
||||
|
||||
class FailingSender:
|
||||
def __init__(self, exc):
|
||||
self.exc = exc
|
||||
self.attempts = 0
|
||||
|
||||
def send(self, message, recipients, attachments=None, text_mode="normal"):
|
||||
self.attempts += 1
|
||||
raise self.exc
|
||||
|
||||
|
||||
class FakeFeed:
|
||||
def __init__(self, content: bytes):
|
||||
self.content = content
|
||||
self.committed = False
|
||||
|
||||
def due(self, now):
|
||||
return True
|
||||
|
||||
def fetch(self, now):
|
||||
return FeedResponse(content=self.content)
|
||||
|
||||
def commit(self, response):
|
||||
self.committed = True
|
||||
|
||||
|
||||
def build_bot(tmp_path, sender, **overrides):
|
||||
config = overrides.pop("config", None) or make_config(tmp_path)
|
||||
store = Store(config.db_path)
|
||||
http = httpx.Client()
|
||||
bot = Bot(
|
||||
config=config,
|
||||
store=store,
|
||||
feed=Feed(config.calendar.url, store, 15, http),
|
||||
feed=overrides.pop(
|
||||
"feed", Feed(config.calendar.url, store, 15, http)
|
||||
),
|
||||
sender=sender,
|
||||
images=FakeImages(),
|
||||
scraper=FakeScraper(),
|
||||
watchdog=Watchdog(config.watchdog, sender, "+447700900000"),
|
||||
images=overrides.pop("images", FakeImages()),
|
||||
scraper=overrides.pop("scraper", FakeScraper()),
|
||||
watchdog=overrides.pop(
|
||||
"watchdog", Watchdog(config.watchdog, sender, "+447700900000")
|
||||
),
|
||||
http=http,
|
||||
)
|
||||
return bot, store
|
||||
@@ -244,3 +285,104 @@ def test_stale_reminder_is_dropped_not_sent(tmp_path):
|
||||
bot._dispatch(local(2026, 11, 18, 18, 0)) # event already started
|
||||
assert sender.sent == []
|
||||
assert store.due_outbox(local(2026, 11, 18, 18, 5)) == []
|
||||
|
||||
|
||||
# -- failure handling -------------------------------------------------------
|
||||
|
||||
|
||||
def test_signal_connection_error_does_not_wedge_the_outbox(tmp_path):
|
||||
sender = FailingSender(httpx.ConnectError("connection refused"))
|
||||
config = make_config(
|
||||
tmp_path, watchdog=WatchdogConfig(enabled=True, send_failures=1)
|
||||
)
|
||||
watchdog = Watchdog(config.watchdog, sender, "+447700900000")
|
||||
bot, store = build_bot(tmp_path, sender, config=config, watchdog=watchdog)
|
||||
|
||||
event = make_event(uid="20", series_id="tec:20", start=local(2026, 11, 25, 19, 0))
|
||||
bot._sync([event], local(2026, 11, 18, 12, 0))
|
||||
bot._sync(
|
||||
[event, make_event(uid="21", series_id="tec:21", start=local(2026, 11, 26, 19, 0))],
|
||||
local(2026, 11, 18, 12, 1),
|
||||
)
|
||||
|
||||
now = local(2026, 11, 18, 18, 0)
|
||||
bot._dispatch(now) # must not raise
|
||||
|
||||
pending = store.due_outbox(now)
|
||||
assert pending, "the failed post should stay queued"
|
||||
assert pending[0].attempts == 1
|
||||
# The announcement stays queued, so it is not marked as delivered.
|
||||
assert store.due_outbox(now)[0].kind == "announce"
|
||||
|
||||
|
||||
def test_scraper_and_image_failures_still_send_text(tmp_path):
|
||||
sender = FakeSender()
|
||||
bot, store = build_bot(
|
||||
tmp_path,
|
||||
sender,
|
||||
images=BoomImages(),
|
||||
scraper=BoomScraper(),
|
||||
)
|
||||
|
||||
event = make_event(
|
||||
uid="22",
|
||||
series_id="tec:22",
|
||||
start=local(2026, 11, 25, 19, 0),
|
||||
image_url="https://example.test/poster.png",
|
||||
)
|
||||
bot._sync([event], local(2026, 11, 18, 12, 0))
|
||||
bot._sync(
|
||||
[event, make_event(uid="23", series_id="tec:23", start=local(2026, 11, 26, 19, 0))],
|
||||
local(2026, 11, 18, 12, 1),
|
||||
)
|
||||
bot._dispatch(local(2026, 11, 18, 18, 0))
|
||||
|
||||
assert len(sender.sent) == 1
|
||||
assert sender.sent[0]["attachments"] == []
|
||||
assert sender.sent[0]["message"].startswith("Folk Night")
|
||||
|
||||
|
||||
# -- gone events ------------------------------------------------------------
|
||||
|
||||
|
||||
def test_reminder_for_gone_event_is_dropped(tmp_path):
|
||||
sender = FakeSender()
|
||||
bot, store = build_bot(tmp_path, sender)
|
||||
|
||||
event = make_event(uid="30", series_id="tec:30", start=local(2026, 11, 25, 14, 0))
|
||||
bot._sync([event], local(2026, 11, 18, 12, 0))
|
||||
store.mark_gone([], local(2026, 11, 18, 13, 0), up_to=event.end + timedelta(days=2))
|
||||
|
||||
bot._dispatch(local(2026, 11, 22, 18, 5))
|
||||
assert sender.sent == []
|
||||
assert store.due_outbox(local(2026, 11, 22, 18, 10)) == []
|
||||
|
||||
|
||||
def test_events_beyond_feed_coverage_are_not_marked_gone(tmp_path):
|
||||
sender = FakeSender()
|
||||
bot, store = build_bot(tmp_path, sender)
|
||||
|
||||
event = make_event(uid="31", series_id="tec:31", start=local(2026, 12, 20, 14, 0))
|
||||
bot._sync([event], local(2026, 11, 18, 12, 0))
|
||||
store.mark_gone([], local(2026, 11, 18, 13, 0), up_to=local(2026, 11, 30, 0, 0))
|
||||
|
||||
stored = store.get_event(event.key)
|
||||
assert stored is not None and stored.gone is False
|
||||
|
||||
|
||||
# -- feed seeding -----------------------------------------------------------
|
||||
|
||||
|
||||
def test_empty_first_feed_does_not_seed(tmp_path):
|
||||
sender = FakeSender()
|
||||
empty = (
|
||||
b"BEGIN:VCALENDAR\r\nVERSION:2.0\r\nPRODID:-//test//EN\r\n"
|
||||
b"END:VCALENDAR\r\n"
|
||||
)
|
||||
feed = FakeFeed(empty)
|
||||
bot, store = build_bot(tmp_path, sender, feed=feed)
|
||||
|
||||
bot._refresh_feed(local(2026, 11, 18, 12, 0))
|
||||
|
||||
assert store.get_meta("seeded") is None
|
||||
assert feed.committed is False
|
||||
|
||||
Reference in New Issue
Block a user