v0.9.0: play counts that can't conflict
The merge windows kept coming because finishing a track rewrote all 15 MB of library.json. Both machines did that, so Syncthing saw two edits to one big file between syncs and produced a conflict file roughly once per song — and the merge then took max() of the two counts, discarding whichever side had played less. The .resolved/ backups showed the last nine library.json merges were ~98% play counts plus exactly one real edit, with the same ~45 tracks disagreeing every time and the count only creeping down over a day. library.json now holds only a base count. Each machine owns plays/<machine-id>.json with its own per-track totals, and the effective count is base + the sum of every journal. Only the owner writes its journal, so play data can't conflict; the journal stores totals rather than an append log, so there's no compaction step to double-count in; and a machine still on older code keeps bumping its own base, which stays additive with our journal. Two places carry the whole hazard, and both are pinned by tests: save_tracks writes journal.base_fields(), never the Track's effective count, and PlayJournal.load must be given a base-valued library — which is why reload_from_disk folds the journals onto the disk copy before reconciling. On the real 21k library: three plays write 83 bytes and leave library.json untouched. Claude-Session: https://claude.ai/code/session_01M8Mzze7shr5pZoEgKQU5NW
This commit is contained in:
@@ -0,0 +1,191 @@
|
||||
"""Per-machine play journals — the reason play counts can no longer conflict.
|
||||
|
||||
``library.json`` used to carry the live play count, so finishing a track
|
||||
rewrote all 15 MB of it. Both machines did that, so Syncthing saw two edits to
|
||||
one big file between syncs and produced a conflict file roughly once per song.
|
||||
The merge then took ``max()`` of the two counts, silently discarding whichever
|
||||
side had played less.
|
||||
|
||||
Now ``library.json`` holds only a **base** count, and each machine additionally
|
||||
owns exactly one file ``<data_dir>/plays/<machine-id>.json`` recording *its own*
|
||||
per-track totals. Effective count = base + the sum of every journal.
|
||||
|
||||
Three properties fall out of that:
|
||||
|
||||
* **No conflict is possible.** A journal file is only ever written by the
|
||||
machine that owns it, so two machines never touch the same file.
|
||||
* **No compaction step.** The journal stores running totals, not an append log,
|
||||
so it is bounded by "tracks this machine has played" and never needs folding
|
||||
back into the base — which is what would risk double-counting.
|
||||
* **Version skew is safe.** A machine still on older code keeps bumping its own
|
||||
base in ``library.json``; we never write base, so the sum stays correct. When
|
||||
it updates, its already-counted plays stay frozen in base and it starts its
|
||||
own journal.
|
||||
|
||||
The machine id lives in the config dir, deliberately *outside* the synced data
|
||||
dir — a synced id would make two machines share one journal.
|
||||
"""
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
from lintunes.config import config_path
|
||||
from lintunes.storage.json_storage import read_json, write_json
|
||||
|
||||
|
||||
JOURNAL_DIR = "plays"
|
||||
|
||||
# The four Track fields a journal contributes to. Everything else about a track
|
||||
# still lives in library.json.
|
||||
PLAY_FIELDS = ("play_count", "skip_count", "play_date_utc", "skip_date")
|
||||
|
||||
|
||||
def machine_id() -> str:
|
||||
"""This machine's stable id, created on first use. Kept next to config.json
|
||||
(not in the data dir) so Syncthing never copies it to the other machine."""
|
||||
path = config_path().parent / "machine_id"
|
||||
try:
|
||||
text = path.read_text(encoding="utf-8").strip()
|
||||
if text:
|
||||
return text
|
||||
except OSError:
|
||||
pass
|
||||
new_id = uuid.uuid4().hex[:12]
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(new_id + "\n", encoding="utf-8")
|
||||
return new_id
|
||||
|
||||
|
||||
def _newer(a, b):
|
||||
"""The later of two ISO-8601 stamps, either of which may be None."""
|
||||
if a is None:
|
||||
return b
|
||||
if b is None:
|
||||
return a
|
||||
return a if a >= b else b
|
||||
|
||||
|
||||
class PlayJournal:
|
||||
"""The base counts from library.json plus every machine's journal.
|
||||
|
||||
``load()`` snapshots the base off the freshly loaded Tracks and then folds
|
||||
the journals in, so the rest of the app keeps reading one plain
|
||||
``track.play_count``. Only the save path needs to know the difference —
|
||||
see ``base_fields``.
|
||||
"""
|
||||
|
||||
def __init__(self, machine=None):
|
||||
self.machine = machine or machine_id()
|
||||
self._base = {} # track_id -> {field: value} as stored in library.json
|
||||
self._own = {} # track_id -> {"plays", "skips", "last_played", "last_skipped"}
|
||||
self._others = {} # machine id -> same shape as _own
|
||||
self.dirty = False
|
||||
|
||||
# ---- loading ----
|
||||
|
||||
def load(self, data_dir: Path, library):
|
||||
"""Read every journal in the data dir and fold them into ``library``.
|
||||
|
||||
``library`` must be one whose tracks still hold the **base** counts —
|
||||
the freshly loaded library at startup, or the disk copy during a reload.
|
||||
Passing an already-folded library would promote the effective count to
|
||||
base and double-count it.
|
||||
"""
|
||||
self._base = {
|
||||
tid: {f: getattr(track, f) for f in PLAY_FIELDS}
|
||||
for tid, track in library.tracks.items()
|
||||
}
|
||||
# Our own journal is only ever written by us, so unsaved bumps in memory
|
||||
# are always more current than the file — never re-read over them.
|
||||
keep_own = self._own if self.dirty else None
|
||||
self._own = {}
|
||||
self._others = {}
|
||||
journal_dir = Path(data_dir) / JOURNAL_DIR
|
||||
if journal_dir.is_dir():
|
||||
for path in sorted(journal_dir.glob("*.json")):
|
||||
if path.name.endswith(".tmp"):
|
||||
continue
|
||||
try:
|
||||
entries = read_json(path)
|
||||
except (ValueError, OSError):
|
||||
continue # a half-synced journal must never abort startup
|
||||
entries = {int(k): v for k, v in entries.items()
|
||||
if isinstance(v, dict)}
|
||||
if path.stem == self.machine:
|
||||
self._own = entries
|
||||
else:
|
||||
self._others[path.stem] = entries
|
||||
if keep_own is not None:
|
||||
self._own = keep_own
|
||||
else:
|
||||
self.dirty = False
|
||||
self.apply(library)
|
||||
|
||||
def apply(self, library):
|
||||
"""Set every track's play fields to base + the sum of all journals."""
|
||||
for tid, track in library.tracks.items():
|
||||
added = sum(e[tid].get("plays", 0)
|
||||
for e in self._all_journals() if tid in e)
|
||||
skipped_n = sum(e[tid].get("skips", 0)
|
||||
for e in self._all_journals() if tid in e)
|
||||
base = self._base.get(tid)
|
||||
if base is None:
|
||||
# A track added after the snapshot. Its current count already
|
||||
# includes anything recorded into the journal since, so back
|
||||
# that out rather than baking it into the base.
|
||||
base = {f: getattr(track, f) for f in PLAY_FIELDS}
|
||||
base["play_count"] = (base["play_count"] or 0) - added
|
||||
base["skip_count"] = (base["skip_count"] or 0) - skipped_n
|
||||
self._base[tid] = base
|
||||
plays = (base["play_count"] or 0) + added
|
||||
skips = (base["skip_count"] or 0) + skipped_n
|
||||
played = base["play_date_utc"]
|
||||
skipped = base["skip_date"]
|
||||
for entries in self._all_journals():
|
||||
e = entries.get(tid)
|
||||
if not e:
|
||||
continue
|
||||
played = _newer(played, e.get("last_played"))
|
||||
skipped = _newer(skipped, e.get("last_skipped"))
|
||||
track.play_count = plays
|
||||
track.skip_count = skips
|
||||
track.play_date_utc = played
|
||||
track.skip_date = skipped
|
||||
|
||||
def _all_journals(self):
|
||||
yield self._own
|
||||
yield from self._others.values()
|
||||
|
||||
# ---- recording ----
|
||||
|
||||
def record_play(self, track_id: int, when: str):
|
||||
entry = self._own.setdefault(track_id, {})
|
||||
entry["plays"] = entry.get("plays", 0) + 1
|
||||
entry["last_played"] = when
|
||||
self.dirty = True
|
||||
|
||||
def record_skip(self, track_id: int, when: str):
|
||||
entry = self._own.setdefault(track_id, {})
|
||||
entry["skips"] = entry.get("skips", 0) + 1
|
||||
entry["last_skipped"] = when
|
||||
self.dirty = True
|
||||
|
||||
# ---- saving ----
|
||||
|
||||
def base_fields(self, track_id: int) -> dict:
|
||||
"""What library.json should hold for this track. Writing the *effective*
|
||||
count here instead would double-count it on the next load — this is the
|
||||
one place in the change where that can go wrong."""
|
||||
base = self._base.get(track_id)
|
||||
if base is None: # never seen at load time — nothing to preserve
|
||||
return {"play_count": 0, "skip_count": 0,
|
||||
"play_date_utc": None, "skip_date": None}
|
||||
return dict(base)
|
||||
|
||||
def save(self, data_dir: Path):
|
||||
"""Write this machine's journal only. Nothing else ever writes this file,
|
||||
so it cannot conflict."""
|
||||
journal_dir = Path(data_dir) / JOURNAL_DIR
|
||||
journal_dir.mkdir(parents=True, exist_ok=True)
|
||||
write_json(journal_dir / f"{self.machine}.json",
|
||||
{str(k): v for k, v in sorted(self._own.items())})
|
||||
self.dirty = False
|
||||
Reference in New Issue
Block a user