trav
3214bb7d96
The merge windows kept coming because finishing a track rewrote all 15 MB of library.json. Both machines did that, so Syncthing saw two edits to one big file between syncs and produced a conflict file roughly once per song — and the merge then took max() of the two counts, discarding whichever side had played less. The .resolved/ backups showed the last nine library.json merges were ~98% play counts plus exactly one real edit, with the same ~45 tracks disagreeing every time and the count only creeping down over a day. library.json now holds only a base count. Each machine owns plays/<machine-id>.json with its own per-track totals, and the effective count is base + the sum of every journal. Only the owner writes its journal, so play data can't conflict; the journal stores totals rather than an append log, so there's no compaction step to double-count in; and a machine still on older code keeps bumping its own base, which stays additive with our journal. Two places carry the whole hazard, and both are pinned by tests: save_tracks writes journal.base_fields(), never the Track's effective count, and PlayJournal.load must be given a base-valued library — which is why reload_from_disk folds the journals onto the disk copy before reconciling. On the real 21k library: three plays write 83 bytes and leave library.json untouched. Claude-Session: https://claude.ai/code/session_01M8Mzze7shr5pZoEgKQU5NW
326 lines
14 KiB
Python
326 lines
14 KiB
Python
import re
|
|
import shutil
|
|
from dataclasses import dataclass, field
|
|
from datetime import datetime
|
|
from pathlib import Path
|
|
|
|
from lintunes.storage.json_storage import read_json, write_json
|
|
|
|
|
|
# Syncthing conflict file pattern: filename.sync-conflict-20240101-123456-ABCDEFG.json
|
|
CONFLICT_PATTERN = re.compile(r"^(.+)\.sync-conflict-(\d{8}-\d{6})-\w+(\.\w+)$")
|
|
|
|
# How many pre-merge snapshots to keep in <data_dir>/.resolved. Each library.json
|
|
# merge stores two 15 MB copies, and the folder rides the Syncthing share, so an
|
|
# unbounded history is replicated to every machine (it had reached 1.7 GB).
|
|
BACKUPS_TO_KEEP = 10
|
|
|
|
|
|
@dataclass
|
|
class ConflictSummary:
|
|
"""A human-readable record of one merged Syncthing conflict, surfaced to the
|
|
user and used to offer a restore from the backed-up pre-merge copies."""
|
|
file: str # display name, e.g. "library.json" or a playlist name
|
|
kind: str # "library" | "playlist" | "metadata" | "other"
|
|
lines: list[str] = field(default_factory=list) # what was reconciled
|
|
backup_dir: str = "" # <data_dir>/.resolved/<timestamp>
|
|
|
|
|
|
def resolve_conflicts(data_dir: Path) -> list[ConflictSummary]:
|
|
"""Merge any Syncthing ``*.sync-conflict-*`` files into their originals.
|
|
|
|
Before touching anything, both the current (original) file and the incoming
|
|
conflict file are copied into ``<data_dir>/.resolved/<timestamp>/`` so the
|
|
merge is always reversible. Returns one ConflictSummary per conflict (empty
|
|
list when there was nothing to do)."""
|
|
if not data_dir.exists():
|
|
return []
|
|
_prune_backups(data_dir)
|
|
|
|
conflict_files = _find_conflict_files(data_dir)
|
|
if not conflict_files:
|
|
return []
|
|
|
|
stamp = datetime.now().strftime("%Y%m%d-%H%M%S")
|
|
backup_dir = data_dir / ".resolved" / stamp
|
|
(backup_dir / "original").mkdir(parents=True, exist_ok=True)
|
|
(backup_dir / "incoming").mkdir(parents=True, exist_ok=True)
|
|
|
|
summaries = []
|
|
for conflict_path, original_path in conflict_files:
|
|
rel = original_path.relative_to(data_dir)
|
|
# Back up both sides first (belt and suspenders).
|
|
if original_path.exists():
|
|
_copy(original_path, backup_dir / "original" / rel)
|
|
_copy(conflict_path, backup_dir / "incoming" / rel)
|
|
|
|
try:
|
|
if not original_path.exists():
|
|
summary = ConflictSummary(
|
|
str(rel), "other",
|
|
["Deleted here but edited elsewhere — kept the deletion; "
|
|
"the other copy is in the backup."])
|
|
elif original_path.name == "library.json":
|
|
summary = _merge_library(original_path, conflict_path)
|
|
elif original_path.name == "library_metadata.json":
|
|
summary = _merge_metadata(original_path, conflict_path)
|
|
elif original_path.parent.name == "playlists":
|
|
summary = _merge_playlist(original_path, conflict_path)
|
|
elif original_path.parent.name == "plays":
|
|
summary = _merge_play_journal(original_path, conflict_path)
|
|
else:
|
|
summary = _keep_newer(original_path, conflict_path)
|
|
except Exception as e: # never let a bad file abort startup/live reload
|
|
summary = ConflictSummary(
|
|
str(rel), "other",
|
|
[f"Could not merge automatically ({e}); both versions are in the "
|
|
"backup, current copy left as-is."])
|
|
|
|
summary.backup_dir = str(backup_dir)
|
|
summaries.append(summary)
|
|
# Remove the conflict file from the data dir (its copy is in the backup)
|
|
# so it isn't reprocessed and Syncthing stops flagging it.
|
|
conflict_path.unlink(missing_ok=True)
|
|
|
|
return summaries
|
|
|
|
|
|
def restore_backup(backup_dir: Path, data_dir: Path) -> list[str]:
|
|
"""Undo a merge: copy the pre-merge ``original/`` snapshot back over the
|
|
current files. Returns the list of restored relative paths."""
|
|
original_root = Path(backup_dir) / "original"
|
|
restored = []
|
|
if not original_root.exists():
|
|
return restored
|
|
for src in original_root.rglob("*"):
|
|
if src.is_file():
|
|
rel = src.relative_to(original_root)
|
|
dest = Path(data_dir) / rel
|
|
dest.parent.mkdir(parents=True, exist_ok=True)
|
|
shutil.copy2(src, dest)
|
|
restored.append(str(rel))
|
|
return restored
|
|
|
|
|
|
def _prune_backups(data_dir: Path, keep: int = BACKUPS_TO_KEEP):
|
|
"""Drop all but the newest `keep` snapshots in <data_dir>/.resolved.
|
|
|
|
Snapshot directories are named with a sortable timestamp, so newest-last by
|
|
name is newest-last by time."""
|
|
root = data_dir / ".resolved"
|
|
if not root.is_dir():
|
|
return
|
|
snapshots = sorted((d for d in root.iterdir() if d.is_dir()),
|
|
key=lambda d: d.name)
|
|
for stale in snapshots[:max(0, len(snapshots) - keep)]:
|
|
shutil.rmtree(stale, ignore_errors=True)
|
|
|
|
|
|
def _find_conflict_files(data_dir: Path) -> list[tuple[Path, Path]]:
|
|
results = []
|
|
for path in data_dir.rglob("*.sync-conflict-*"):
|
|
if ".resolved" in path.parts: # don't reprocess our own backups
|
|
continue
|
|
match = CONFLICT_PATTERN.match(path.name)
|
|
if match:
|
|
original_name = match.group(1) + match.group(3)
|
|
original_path = path.parent / original_name
|
|
results.append((path, original_path))
|
|
return results
|
|
|
|
|
|
def _merge_library(original_path: Path, conflict_path: Path) -> ConflictSummary:
|
|
original = read_json(original_path)
|
|
conflict = read_json(conflict_path)
|
|
|
|
added = 0
|
|
changed = 0
|
|
examples: list[str] = []
|
|
for tid, conflict_track_data in conflict.items():
|
|
if tid not in original:
|
|
original[tid] = conflict_track_data
|
|
added += 1
|
|
continue
|
|
notes = _merge_track_fields(original[tid], conflict_track_data)
|
|
if notes:
|
|
changed += 1
|
|
if len(examples) < 8:
|
|
name = original[tid].get("name") or f"track {tid}"
|
|
examples.append(f"“{name}”: {notes[0]}")
|
|
|
|
write_json(original_path, original)
|
|
|
|
lines = []
|
|
if changed:
|
|
lines.append(f"{changed} track(s) reconciled "
|
|
"(play counts kept highest, newest edits win).")
|
|
if added:
|
|
lines.append(f"{added} track(s) present only on the other machine were kept.")
|
|
if not lines:
|
|
lines.append("No differences needed reconciling.")
|
|
lines.extend(examples)
|
|
return ConflictSummary("library.json", "library", lines)
|
|
|
|
|
|
# Every field _merge_track_fields reads or writes. If two copies of a track
|
|
# agree on all of these, merging them is a no-op — which lets callers skip the
|
|
# work entirely (see library_manager._reconcile_track).
|
|
MERGEABLE_FIELDS = (
|
|
"play_count", "skip_count", "play_date_utc", "skip_date", "date_added",
|
|
"loved", "rating", "name", "artist", "album_artist", "album", "genre",
|
|
"composer", "grouping", "comments", "sort_name", "sort_artist",
|
|
"sort_album_artist", "sort_album", "sort_composer", "location",
|
|
"date_modified",
|
|
)
|
|
|
|
|
|
def _merge_track_fields(orig: dict, conflict: dict) -> list[str]:
|
|
"""Merge conflict's track fields into orig in place; return change notes."""
|
|
notes = []
|
|
# play_count, skip_count: take max
|
|
for field_name, label in (("play_count", "play count"), ("skip_count", "skip count")):
|
|
if field_name in conflict:
|
|
new = max(orig.get(field_name, 0), conflict[field_name])
|
|
if new != orig.get(field_name, 0):
|
|
notes.append(f"{label} {orig.get(field_name, 0)} → {new}")
|
|
orig[field_name] = new
|
|
|
|
# play_date_utc, skip_date: take most recent
|
|
for field_name in ("play_date_utc", "skip_date"):
|
|
if field_name in conflict:
|
|
orig_val = orig.get(field_name)
|
|
conf_val = conflict[field_name]
|
|
if orig_val is None or (conf_val and conf_val > orig_val):
|
|
orig[field_name] = conf_val
|
|
|
|
# date_added: take earliest
|
|
if "date_added" in conflict:
|
|
orig_val = orig.get("date_added")
|
|
conf_val = conflict["date_added"]
|
|
if orig_val is None or (conf_val and conf_val < orig_val):
|
|
orig["date_added"] = conf_val
|
|
|
|
# loved: OR (true wins). Legacy field — unused by LinTunes but preserved so a
|
|
# merge never silently clears it.
|
|
if conflict.get("loved", False):
|
|
orig["loved"] = True
|
|
|
|
# rating + editable metadata: newest date_modified wins
|
|
orig_mod = orig.get("date_modified")
|
|
conf_mod = conflict.get("date_modified")
|
|
if conf_mod and (not orig_mod or conf_mod > orig_mod):
|
|
if conflict.get("rating") != orig.get("rating") and "rating" in conflict:
|
|
notes.append(f"rating {_stars(orig.get('rating'))} → "
|
|
f"{_stars(conflict.get('rating'))} (newer)")
|
|
# location rides along: renaming artist/album moves the file (and bumps
|
|
# date_modified), so the machine with the newest edit also knows where
|
|
# the file now lives. Both sides are relative here (raw JSON) and both
|
|
# absolute in the reload_from_disk reconcile — always like-with-like.
|
|
for field_name in ("rating", "name", "artist", "album_artist", "album",
|
|
"genre", "composer", "grouping", "comments",
|
|
"sort_name", "sort_artist", "sort_album_artist",
|
|
"sort_album", "sort_composer", "location",
|
|
"date_modified"):
|
|
if field_name in conflict:
|
|
orig[field_name] = conflict[field_name]
|
|
return notes
|
|
|
|
|
|
def _merge_playlist(original_path: Path, conflict_path: Path) -> ConflictSummary:
|
|
original = read_json(original_path)
|
|
conflict = read_json(conflict_path)
|
|
name = original.get("name") or conflict.get("name") or original_path.stem
|
|
|
|
# Smart playlists: membership is derived, so keep the newer criteria and let
|
|
# it recompute (unioning derived track_ids would resurrect non-matches).
|
|
if original.get("playlist_type") == "smart" or conflict.get("playlist_type") == "smart":
|
|
newer = _keep_newer(original_path, conflict_path)
|
|
newer.file, newer.kind = name, "playlist"
|
|
newer.lines = ["Smart-playlist rules taken from the most recently edited copy."]
|
|
return newer
|
|
|
|
orig_ids = list(original.get("track_ids", []))
|
|
conf_ids = list(conflict.get("track_ids", []))
|
|
orig_set, conf_set = set(orig_ids), set(conf_ids)
|
|
|
|
conflict_newer = conflict_path.stat().st_mtime > original_path.stat().st_mtime
|
|
if conflict_newer:
|
|
base_order = conf_ids
|
|
extra = [tid for tid in orig_ids if tid not in conf_set]
|
|
original["track_ids"] = base_order + extra
|
|
if "settings" in conflict:
|
|
original["settings"] = conflict["settings"]
|
|
order_src = "the other machine"
|
|
else:
|
|
extra = [tid for tid in conf_ids if tid not in orig_set]
|
|
original["track_ids"] = orig_ids + extra
|
|
order_src = "this machine"
|
|
|
|
write_json(original_path, original)
|
|
|
|
added = len(set(original["track_ids"]) - orig_set)
|
|
lines = [f"Order kept from {order_src} (most recently edited)."]
|
|
if added:
|
|
lines.append(f"{added} track(s) that were only in the other copy were kept "
|
|
"(nothing is removed on a merge).")
|
|
return ConflictSummary(name, "playlist", lines)
|
|
|
|
|
|
def _merge_play_journal(original_path: Path, conflict_path: Path) -> ConflictSummary:
|
|
"""Insurance only. A journal is owned by one machine, so it should never
|
|
conflict — but if a filesystem or clock oddity produces one, falling through
|
|
to _keep_newer would throw away a machine's entire play history. Take the
|
|
higher total per track instead."""
|
|
original = read_json(original_path)
|
|
incoming = read_json(conflict_path)
|
|
merged = dict(original)
|
|
changed = 0
|
|
for tid, entry in incoming.items():
|
|
mine = merged.get(tid)
|
|
if mine is None:
|
|
merged[tid] = entry
|
|
changed += 1
|
|
continue
|
|
combined = dict(mine)
|
|
for count in ("plays", "skips"):
|
|
combined[count] = max(mine.get(count, 0), entry.get(count, 0))
|
|
for stamp in ("last_played", "last_skipped"):
|
|
a, b = mine.get(stamp), entry.get(stamp)
|
|
combined[stamp] = a if b is None else (b if a is None else max(a, b))
|
|
if combined != mine:
|
|
changed += 1
|
|
merged[tid] = combined
|
|
write_json(original_path, merged)
|
|
return ConflictSummary(
|
|
f"plays/{original_path.name}", "plays",
|
|
[f"{changed} track(s) of play history reconciled (highest total kept)."]
|
|
if changed else ["No differences needed reconciling."])
|
|
|
|
|
|
def _merge_metadata(original_path: Path, conflict_path: Path) -> ConflictSummary:
|
|
summary = _keep_newer(original_path, conflict_path)
|
|
summary.file, summary.kind = "library_metadata.json", "metadata"
|
|
summary.lines = ["Library columns/settings taken from the most recently edited copy."]
|
|
return summary
|
|
|
|
|
|
def _keep_newer(original_path: Path, conflict_path: Path) -> ConflictSummary:
|
|
took = "this machine"
|
|
if conflict_path.stat().st_mtime > original_path.stat().st_mtime:
|
|
shutil.copy2(conflict_path, original_path)
|
|
took = "the other machine"
|
|
return ConflictSummary(original_path.name, "other",
|
|
[f"Kept the copy from {took} (most recently edited)."])
|
|
|
|
|
|
def _stars(rating) -> str:
|
|
try:
|
|
return f"{int(rating) // 20}★"
|
|
except (TypeError, ValueError):
|
|
return "0★"
|
|
|
|
|
|
def _copy(src: Path, dest: Path):
|
|
dest.parent.mkdir(parents=True, exist_ok=True)
|
|
shutil.copy2(src, dest)
|