diff --git a/CLAUDE.md b/CLAUDE.md index 373956f..f407e42 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -395,6 +395,13 @@ persistence) → GUI (Qt widgets that read the manager and connect to its signal model exists) and "gentle" (default: a quarter of the cores, nice 19). Splits are cached per model (`//`; Round 93's sit in `/` and count as htdemucs) and a deck takes the best one cached. + **Round 100**: `` is a hash of the *audio packets* (`ffmpeg -c copy + -f hash`), not path + size + mtime — loading a song with no BPM writes + its BPM tag, which orphaned every such split at once. `index.json` + memoizes path + size + mtime → key; old path-keyed folders are renamed + on sight. The cache is held to config.json's `dj_stem_cache_gb` + (Preferences › DJ, default 1): after each split the least recently used + songs go (`stems.prune`). **Prepare for DJ** (a playlist's or songs' right-click; `dj/prepare.py`) does the BPM + split ahead of time with the status-bar progress/cancel. Each deck also has a volume fader (outer edge, perceptual), a ½×/1×/2× diff --git a/lintunes/__init__.py b/lintunes/__init__.py index e8b48d9..2f63ee8 100644 --- a/lintunes/__init__.py +++ b/lintunes/__init__.py @@ -1,3 +1,3 @@ """LinTunes — iTunes-style music library manager and player for Linux.""" -__version__ = "0.67.3" +__version__ = "0.68.0" diff --git a/lintunes/dj/stems.py b/lintunes/dj/stems.py index 9da0285..6f0b7b7 100644 --- a/lintunes/dj/stems.py +++ b/lintunes/dj/stems.py @@ -2,11 +2,22 @@ Nothing separates a voice from a mix in real time on a laptop, so a song is split once, in the background, and kept: -``$XDG_CACHE_HOME/lintunes/stems///{vocals,no_vocals}.flac``, -keyed by path + size + mtime like the loudness cache (machine-local, never -the synced data dir). A cached song's slider works the moment it loads, and -a deck uses the best split it finds whatever model made it. (Round 93's -first splits sit directly in ``/``; they were htdemucs.) +``$XDG_CACHE_HOME/lintunes/stems///{vocals,no_vocals}.flac`` +(machine-local, never the synced data dir). A cached song's slider works the +moment it loads, and a deck uses the best split it finds whatever model made +it. (Round 93's first splits sit directly in ``/``; they were htdemucs.) + +The key is the song's **audio**, not its file (Round 100): a hash of the +audio packets (``ffmpeg -c copy -f hash``, ~0.2 s, no decoding), so a tag +edit or a move doesn't lose the split. It used to be path + size + mtime, +and loading a song with no BPM *writes* its BPM tag — so every such split +was orphaned the moment it was made. ``index.json`` remembers path + size + +mtime → key, so the hash runs once per version of a file; a folder still +under its old path key is renamed to the audio key when found. + +The cache is held to ``config.json``'s ``dj_stem_cache_gb`` (1 GB unless +set; this machine's disk, so not a synced pref): after each split, the songs +used longest ago go first. Using a split (``cached``) counts as using it. Demucs runs as a runtime tool, never a pip dependency of LinTunes: PyTorch is ~1 GB, so it lives in its own venv at ``$XDG_DATA_HOME/lintunes/demucs`` @@ -20,6 +31,7 @@ One split at a time: two would only halve each other's speed. """ import hashlib +import json import os import re import shutil @@ -30,6 +42,8 @@ from pathlib import Path from PyQt6.QtCore import QObject, pyqtSignal +from lintunes.config import load_config + # id → (label, what it costs). Best first: a deck uses the best split it has. # Measured on this laptop (Round 94, 40 s of audio, 2 threads): htdemucs # 35 s, at overlap 0.1 27 s; hdemucs_mmi 42 s and mdx 156 s were slower, @@ -95,13 +109,90 @@ def cache_root() -> Path: return Path(base) / "lintunes" / "stems" -def cache_key(path: str) -> str | None: +CACHE_LIMIT_KEY = "dj_stem_cache_gb" +DEFAULT_CACHE_GB = 1.0 +_INDEX = "index.json" +_index_lock = threading.Lock() + + +def cache_limit_gb(config: dict | None = None) -> float: + config = load_config() if config is None else config + try: + value = float(config.get(CACHE_LIMIT_KEY, DEFAULT_CACHE_GB)) + except (TypeError, ValueError): + return DEFAULT_CACHE_GB + return value if value >= 0 else DEFAULT_CACHE_GB + + +def _file_id(path: str) -> str | None: try: st = os.stat(path) except OSError: return None - raw = f"{os.path.abspath(path)}|{st.st_size}|{int(st.st_mtime)}" - return hashlib.sha1(raw.encode()).hexdigest()[:20] + return f"{os.path.abspath(path)}|{st.st_size}|{int(st.st_mtime)}" + + +def _path_key(path: str) -> str | None: + """The key before Round 100: path + size + mtime.""" + raw = _file_id(path) + return hashlib.sha1(raw.encode()).hexdigest()[:20] if raw else None + + +def audio_hash(path: str) -> str | None: + """A hash of the audio packets alone, so tags don't change it.""" + try: + done = subprocess.run( + ["ffmpeg", "-v", "error", "-nostdin", "-i", path, "-map", "0:a:0", + "-c", "copy", "-f", "hash", "-hash", "sha256", "-"], + capture_output=True, text=True, timeout=60) + except (OSError, subprocess.SubprocessError): + return None + out = done.stdout.strip() + if done.returncode != 0 or not out.startswith("SHA256="): + return None + return out[len("SHA256="):] + + +def _read_index() -> dict: + try: + with open(cache_root() / _INDEX, encoding="utf-8") as f: + data = json.load(f) + return data if isinstance(data, dict) else {} + except (OSError, ValueError): + return {} + + +def _write_index(index: dict): + root = cache_root() + try: + root.mkdir(parents=True, exist_ok=True) + tmp = root / (_INDEX + ".tmp") + tmp.write_text(json.dumps(index), encoding="utf-8") + tmp.replace(root / _INDEX) + except OSError: + pass + + +def cache_key(path: str) -> str | None: + """The song's audio key (the path key when ffmpeg can't read it).""" + file_id = _file_id(path) + if file_id is None: + return None + with _index_lock: + known = _read_index().get(file_id) + if known: + return known + digest = audio_hash(path) + if digest is None: + return _path_key(path) + key = "a" + digest[:24] + with _index_lock: + index = _read_index() + prefix = os.path.abspath(path) + "|" + index = {k: v for k, v in index.items() if not k.startswith(prefix)} + index[file_id] = key + _write_index(index) + return key def _files(folder: Path) -> dict[str, Path] | None: @@ -109,6 +200,13 @@ def _files(folder: Path) -> dict[str, Path] | None: return files if all(f.is_file() for f in files.values()) else None +def _touch(folder: Path): + try: + os.utime(folder) + except OSError: + pass + + def cached(path: str, model: str | None = None) -> dict[str, Path] | None: """The stems of ``path``: ``model``'s when given, else the best there are (``MODELS`` order). None when it hasn't been split.""" @@ -116,16 +214,63 @@ def cached(path: str, model: str | None = None) -> dict[str, Path] | None: if key is None: return None root = cache_root() / key + legacy = _path_key(path) + if legacy and legacy != key and not root.exists() and (cache_root() / legacy).is_dir(): + try: + (cache_root() / legacy).rename(root) + except OSError: + root = cache_root() / legacy order = [model] if model else list(MODELS) for name in order: hit = _files(root / name) if hit is None and name == LEGACY_MODEL: hit = _files(root) if hit is not None: + _touch(root) return hit return None +def _size(folder: Path) -> int: + total = 0 + for f in folder.rglob("*"): + try: + if f.is_file(): + total += f.stat().st_size + except OSError: + pass + return total + + +def prune(limit_gb: float | None = None, keep: str | None = None) -> int: + """Delete the songs used longest ago until the cache is under + ``limit_gb`` (config.json's when None). ``keep`` (a key) is never + deleted. Returns how many songs went.""" + limit = int((cache_limit_gb() if limit_gb is None else limit_gb) * 1024 ** 3) + root = cache_root() + try: + folders = [d for d in root.iterdir() if d.is_dir()] + except OSError: + return 0 + entries = [] + for d in folders: + try: + entries.append((d.stat().st_mtime, d, _size(d))) + except OSError: + pass + total = sum(size for _m, _d, size in entries) + removed = 0 + for _mtime, folder, size in sorted(entries, key=lambda e: e[0]): + if total <= limit: + break + if folder.name == keep: + continue + shutil.rmtree(folder, ignore_errors=True) + total -= size + removed += 1 + return removed + + def parse_progress(text: str) -> int | None: found = _PERCENT.findall(text) return int(found[-1]) if found else None @@ -178,6 +323,8 @@ def split(path: str, model: str, gentle: bool, progress=None, target = folder / f"{name}.flac" shutil.move(str(made[0].parent / f"{name}.flac"), target) out[name] = target + _touch(cache_root() / key) + prune(keep=key) return out finally: shutil.rmtree(work, ignore_errors=True) @@ -200,11 +347,19 @@ class StemSplitter(QObject): self._lock = threading.Lock() def request(self, path: str): - """Split ``path`` unless any split of it is cached (then report it).""" + """Split ``path`` unless any split of it is cached (then report it). + The lookup may hash the audio (~0.2 s), so it runs off this thread.""" + threading.Thread(target=self._look_up, args=(path,), daemon=True, + name="lintunes-stems-lookup").start() + + def _look_up(self, path: str): hit = cached(path) if hit is not None: self.finished.emit(path, hit) return + self._enqueue(path) + + def _enqueue(self, path: str): with self._lock: if path in self._queue: return diff --git a/lintunes/gui/preferences_dialog.py b/lintunes/gui/preferences_dialog.py index 669eb72..bf4f9fc 100644 --- a/lintunes/gui/preferences_dialog.py +++ b/lintunes/gui/preferences_dialog.py @@ -1,4 +1,5 @@ import math +import threading from PyQt6.QtWidgets import ( QDialog, QVBoxLayout, QHBoxLayout, QGroupBox, QLabel, QSlider, @@ -9,7 +10,7 @@ from PyQt6.QtWidgets import ( QMessageBox, QApplication, ) from PyQt6.QtCore import Qt, QTimer -from PyQt6.QtGui import QFont, QColor, QFontInfo +from PyQt6.QtGui import QDoubleValidator, QFont, QColor, QFontInfo from lintunes import loudness, theme from lintunes.config import load_config, save_config @@ -273,10 +274,24 @@ class PreferencesDialog(QDialog): gentle.setChecked(stems.gentle_choice(self._prefs)) gentle.toggled.connect(lambda on: self._prefs.set("dj_stem_gentle", on)) form.addRow("", gentle) + cache = QLineEdit(f"{stems.cache_limit_gb():g}") + cache.setValidator(QDoubleValidator(0.0, 10000.0, 2, cache)) + cache.setMaximumWidth(80) + cache.setToolTip("Separated songs are kept up to this much disk space " + "on this computer; past it, the songs used longest " + "ago are deleted (they're split again if needed).") + cache.editingFinished.connect(self._save_stem_cache_limit) + cache_row = QHBoxLayout() + cache_row.addWidget(cache) + cache_row.addWidget(QLabel("GB")) + cache_row.addStretch(1) + form.addRow("Keep up to:", cache_row) + self._dj_cache_edit = cache if stems.available(): note = QLabel("Demucs is set up. Songs are split when they load onto " "a deck, or ahead of time with a playlist's or " - "song's “Prepare for DJ”. Splits are kept.") + "song's “Prepare for DJ”. Splits are kept, up to " + "the size above.") else: note = QLabel("Demucs isn't set up on this computer (a one-time ~1 GB " "download). In a terminal:\n" + stems.setup_command()) @@ -287,6 +302,23 @@ class PreferencesDialog(QDialog): self._dj_gentle_check = gentle return group + def _save_stem_cache_limit(self): + """config.json, not preferences: how much disk to give the splits + is this machine's business. A lower limit applies at once.""" + from lintunes.dj import stems + try: + value = float(self._dj_cache_edit.text().replace(",", ".")) + except ValueError: + self._dj_cache_edit.setText(f"{stems.cache_limit_gb():g}") + return + config = load_config() + if config.get(stems.CACHE_LIMIT_KEY) == value: + return + config[stems.CACHE_LIMIT_KEY] = value + save_config(config) + threading.Thread(target=stems.prune, args=(value,), daemon=True, + name="lintunes-stems-prune").start() + # ---- Wrapped ---- def _build_wrapped_group(self) -> QGroupBox: diff --git a/tests/test_round100.py b/tests/test_round100.py index 84f1c6f..2a72211 100644 --- a/tests/test_round100.py +++ b/tests/test_round100.py @@ -33,3 +33,118 @@ def test_dragging_narrower_snaps_back_and_saves_the_minimum(qapp): def test_wider_widths_are_left_alone(qapp): view = _view({"rating": 200}) assert view.columnWidth(view.model_.fields.index("rating")) == 200 + + +# ---- DJ stems: a cache that survives a tag edit, held to a size ---- + +import os +import shutil +import subprocess +import time + +import pytest + +from lintunes.dj import stems + + +@pytest.fixture +def cache(tmp_path, monkeypatch): + monkeypatch.setenv("XDG_CACHE_HOME", str(tmp_path / "cache")) + monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path / "config")) + return tmp_path + + +def _mp3(path): + subprocess.run(["ffmpeg", "-v", "error", "-y", "-f", "lavfi", "-i", + "sine=f=440:d=2", str(path)], check=True) + return str(path) + + +def _make(folder, size=1): + folder.mkdir(parents=True, exist_ok=True) + for name in stems.STEMS: + (folder / f"{name}.flac").write_bytes(b"f" * size) + + +needs_ffmpeg = pytest.mark.skipif(shutil.which("ffmpeg") is None, + reason="needs the ffmpeg CLI") + + +@needs_ffmpeg +def test_writing_the_bpm_tag_keeps_the_split(cache): + from mutagen.id3 import ID3, TBPM + song = _mp3(cache / "a.mp3") + _make(stems.cache_root() / stems.cache_key(song) / "htdemucs") + tags = ID3() + tags.add(TBPM(encoding=3, text=["128"])) + tags.save(song) # what loading a song with no BPM does + os.utime(song, (time.time() + 60, time.time() + 60)) + assert stems.cached(song) is not None + + +@needs_ffmpeg +def test_a_split_under_the_old_path_key_is_moved_to_the_audio_key(cache): + song = _mp3(cache / "a.mp3") + _make(stems.cache_root() / stems._path_key(song) / "htdemucs") + assert stems.cached(song) is not None + assert (stems.cache_root() / stems.cache_key(song) / "htdemucs").is_dir() + assert not (stems.cache_root() / stems._path_key(song)).exists() + + +@needs_ffmpeg +def test_the_hash_runs_once_per_version_of_a_file(cache, monkeypatch): + song = _mp3(cache / "a.mp3") + calls = [] + real = stems.audio_hash + monkeypatch.setattr(stems, "audio_hash", + lambda p: calls.append(p) or real(p)) + first = stems.cache_key(song) + assert stems.cache_key(song) == first + assert len(calls) == 1 + + +def test_unreadable_audio_falls_back_to_the_path_key(cache): + song = cache / "a.mp3" + song.write_bytes(b"x") + assert stems.cache_key(str(song)) == stems._path_key(str(song)) + + +def test_prune_drops_the_songs_used_longest_ago(cache): + root = stems.cache_root() + now = time.time() + for i, name in enumerate(["old", "mid", "new"]): + _make(root / name / "htdemucs", size=400 * 1024) # 800 KiB a song + os.utime(root / name, (now - 100 + i, now - 100 + i)) + removed = stems.prune(limit_gb=2 * 800 / 1024 ** 2) # room for two + assert removed == 1 + assert sorted(p.name for p in root.iterdir()) == ["mid", "new"] + stems.prune(limit_gb=0, keep="mid") + assert [p.name for p in root.iterdir()] == ["mid"] + + +def test_the_limit_comes_from_config_json(cache): + from lintunes.config import save_config + assert stems.cache_limit_gb() == 1.0 + save_config({stems.CACHE_LIMIT_KEY: 2.5}) + assert stems.cache_limit_gb() == 2.5 + assert stems.cache_limit_gb({stems.CACHE_LIMIT_KEY: "junk"}) == 1.0 + + +def test_preferences_saves_the_limit_to_config_json(qapp, cache, monkeypatch): + from lintunes.config import load_config + from lintunes.gui.preferences_dialog import PreferencesDialog + from lintunes.preferences import Preferences + from tests.test_round40 import _FakeLastFm + pruned = [] + monkeypatch.setattr(stems, "prune", lambda limit: pruned.append(limit)) + dialog = PreferencesDialog(Preferences(cache), _FakeLastFm()) + assert dialog._dj_cache_edit.text() == "1" + dialog._dj_cache_edit.setText("3.5") + dialog._save_stem_cache_limit() + assert load_config()[stems.CACHE_LIMIT_KEY] == 3.5 + for _ in range(50): + if pruned: + break + time.sleep(0.01) + assert pruned == [3.5] + dialog.deleteLater() diff --git a/tests/test_round94.py b/tests/test_round94.py index 9de95b9..a14af80 100644 --- a/tests/test_round94.py +++ b/tests/test_round94.py @@ -136,6 +136,7 @@ class TestStems: song = self._song(tmp_path, monkeypatch) monkeypatch.setattr(stems, "available", lambda: True) monkeypatch.setattr(stems.os, "cpu_count", lambda: 8) + monkeypatch.setattr(stems, "audio_hash", lambda path: None) seen = {} class Proc: