v0.68.0: DJ vocal splits survive tag edits; a size limit for their cache

This commit is contained in:
2026-10-19 22:16:39 -07:00
parent 1d215fd045
commit d1174c6b35
6 changed files with 322 additions and 12 deletions
+7
View File
@@ -395,6 +395,13 @@ persistence) → GUI (Qt widgets that read the manager and connect to its signal
model exists) and "gentle" (default: a quarter of the cores, nice 19).
Splits are cached per model (`<key>/<model>/`; Round 93's sit in
`<key>/` and count as htdemucs) and a deck takes the best one cached.
**Round 100**: `<key>` is a hash of the *audio packets* (`ffmpeg -c copy
-f hash`), not path + size + mtime — loading a song with no BPM writes
its BPM tag, which orphaned every such split at once. `index.json`
memoizes path + size + mtime → key; old path-keyed folders are renamed
on sight. The cache is held to config.json's `dj_stem_cache_gb`
(Preferences › DJ, default 1): after each split the least recently used
songs go (`stems.prune`).
**Prepare for DJ** (a playlist's or songs' right-click; `dj/prepare.py`)
does the BPM + split ahead of time with the status-bar progress/cancel.
Each deck also has a volume fader (outer edge, perceptual), a ½×/1×/2×
+1 -1
View File
@@ -1,3 +1,3 @@
"""LinTunes — iTunes-style music library manager and player for Linux."""
__version__ = "0.67.3"
__version__ = "0.68.0"
+164 -9
View File
@@ -2,11 +2,22 @@
Nothing separates a voice from a mix in real time on a laptop, so a song is
split once, in the background, and kept:
``$XDG_CACHE_HOME/lintunes/stems/<key>/<model>/{vocals,no_vocals}.flac``,
keyed by path + size + mtime like the loudness cache (machine-local, never
the synced data dir). A cached song's slider works the moment it loads, and
a deck uses the best split it finds whatever model made it. (Round 93's
first splits sit directly in ``<key>/``; they were htdemucs.)
``$XDG_CACHE_HOME/lintunes/stems/<key>/<model>/{vocals,no_vocals}.flac``
(machine-local, never the synced data dir). A cached song's slider works the
moment it loads, and a deck uses the best split it finds whatever model made
it. (Round 93's first splits sit directly in ``<key>/``; they were htdemucs.)
The key is the song's **audio**, not its file (Round 100): a hash of the
audio packets (``ffmpeg -c copy -f hash``, ~0.2 s, no decoding), so a tag
edit or a move doesn't lose the split. It used to be path + size + mtime,
and loading a song with no BPM *writes* its BPM tag — so every such split
was orphaned the moment it was made. ``index.json`` remembers path + size +
mtime → key, so the hash runs once per version of a file; a folder still
under its old path key is renamed to the audio key when found.
The cache is held to ``config.json``'s ``dj_stem_cache_gb`` (1 GB unless
set; this machine's disk, so not a synced pref): after each split, the songs
used longest ago go first. Using a split (``cached``) counts as using it.
Demucs runs as a runtime tool, never a pip dependency of LinTunes: PyTorch is
~1 GB, so it lives in its own venv at ``$XDG_DATA_HOME/lintunes/demucs``
@@ -20,6 +31,7 @@ One split at a time: two would only halve each other's speed.
"""
import hashlib
import json
import os
import re
import shutil
@@ -30,6 +42,8 @@ from pathlib import Path
from PyQt6.QtCore import QObject, pyqtSignal
from lintunes.config import load_config
# id → (label, what it costs). Best first: a deck uses the best split it has.
# Measured on this laptop (Round 94, 40 s of audio, 2 threads): htdemucs
# 35 s, at overlap 0.1 27 s; hdemucs_mmi 42 s and mdx 156 s were slower,
@@ -95,13 +109,90 @@ def cache_root() -> Path:
return Path(base) / "lintunes" / "stems"
def cache_key(path: str) -> str | None:
CACHE_LIMIT_KEY = "dj_stem_cache_gb"
DEFAULT_CACHE_GB = 1.0
_INDEX = "index.json"
_index_lock = threading.Lock()
def cache_limit_gb(config: dict | None = None) -> float:
config = load_config() if config is None else config
try:
value = float(config.get(CACHE_LIMIT_KEY, DEFAULT_CACHE_GB))
except (TypeError, ValueError):
return DEFAULT_CACHE_GB
return value if value >= 0 else DEFAULT_CACHE_GB
def _file_id(path: str) -> str | None:
try:
st = os.stat(path)
except OSError:
return None
raw = f"{os.path.abspath(path)}|{st.st_size}|{int(st.st_mtime)}"
return hashlib.sha1(raw.encode()).hexdigest()[:20]
return f"{os.path.abspath(path)}|{st.st_size}|{int(st.st_mtime)}"
def _path_key(path: str) -> str | None:
"""The key before Round 100: path + size + mtime."""
raw = _file_id(path)
return hashlib.sha1(raw.encode()).hexdigest()[:20] if raw else None
def audio_hash(path: str) -> str | None:
"""A hash of the audio packets alone, so tags don't change it."""
try:
done = subprocess.run(
["ffmpeg", "-v", "error", "-nostdin", "-i", path, "-map", "0:a:0",
"-c", "copy", "-f", "hash", "-hash", "sha256", "-"],
capture_output=True, text=True, timeout=60)
except (OSError, subprocess.SubprocessError):
return None
out = done.stdout.strip()
if done.returncode != 0 or not out.startswith("SHA256="):
return None
return out[len("SHA256="):]
def _read_index() -> dict:
try:
with open(cache_root() / _INDEX, encoding="utf-8") as f:
data = json.load(f)
return data if isinstance(data, dict) else {}
except (OSError, ValueError):
return {}
def _write_index(index: dict):
root = cache_root()
try:
root.mkdir(parents=True, exist_ok=True)
tmp = root / (_INDEX + ".tmp")
tmp.write_text(json.dumps(index), encoding="utf-8")
tmp.replace(root / _INDEX)
except OSError:
pass
def cache_key(path: str) -> str | None:
"""The song's audio key (the path key when ffmpeg can't read it)."""
file_id = _file_id(path)
if file_id is None:
return None
with _index_lock:
known = _read_index().get(file_id)
if known:
return known
digest = audio_hash(path)
if digest is None:
return _path_key(path)
key = "a" + digest[:24]
with _index_lock:
index = _read_index()
prefix = os.path.abspath(path) + "|"
index = {k: v for k, v in index.items() if not k.startswith(prefix)}
index[file_id] = key
_write_index(index)
return key
def _files(folder: Path) -> dict[str, Path] | None:
@@ -109,6 +200,13 @@ def _files(folder: Path) -> dict[str, Path] | None:
return files if all(f.is_file() for f in files.values()) else None
def _touch(folder: Path):
try:
os.utime(folder)
except OSError:
pass
def cached(path: str, model: str | None = None) -> dict[str, Path] | None:
"""The stems of ``path``: ``model``'s when given, else the best there
are (``MODELS`` order). None when it hasn't been split."""
@@ -116,16 +214,63 @@ def cached(path: str, model: str | None = None) -> dict[str, Path] | None:
if key is None:
return None
root = cache_root() / key
legacy = _path_key(path)
if legacy and legacy != key and not root.exists() and (cache_root() / legacy).is_dir():
try:
(cache_root() / legacy).rename(root)
except OSError:
root = cache_root() / legacy
order = [model] if model else list(MODELS)
for name in order:
hit = _files(root / name)
if hit is None and name == LEGACY_MODEL:
hit = _files(root)
if hit is not None:
_touch(root)
return hit
return None
def _size(folder: Path) -> int:
total = 0
for f in folder.rglob("*"):
try:
if f.is_file():
total += f.stat().st_size
except OSError:
pass
return total
def prune(limit_gb: float | None = None, keep: str | None = None) -> int:
"""Delete the songs used longest ago until the cache is under
``limit_gb`` (config.json's when None). ``keep`` (a key) is never
deleted. Returns how many songs went."""
limit = int((cache_limit_gb() if limit_gb is None else limit_gb) * 1024 ** 3)
root = cache_root()
try:
folders = [d for d in root.iterdir() if d.is_dir()]
except OSError:
return 0
entries = []
for d in folders:
try:
entries.append((d.stat().st_mtime, d, _size(d)))
except OSError:
pass
total = sum(size for _m, _d, size in entries)
removed = 0
for _mtime, folder, size in sorted(entries, key=lambda e: e[0]):
if total <= limit:
break
if folder.name == keep:
continue
shutil.rmtree(folder, ignore_errors=True)
total -= size
removed += 1
return removed
def parse_progress(text: str) -> int | None:
found = _PERCENT.findall(text)
return int(found[-1]) if found else None
@@ -178,6 +323,8 @@ def split(path: str, model: str, gentle: bool, progress=None,
target = folder / f"{name}.flac"
shutil.move(str(made[0].parent / f"{name}.flac"), target)
out[name] = target
_touch(cache_root() / key)
prune(keep=key)
return out
finally:
shutil.rmtree(work, ignore_errors=True)
@@ -200,11 +347,19 @@ class StemSplitter(QObject):
self._lock = threading.Lock()
def request(self, path: str):
"""Split ``path`` unless any split of it is cached (then report it)."""
"""Split ``path`` unless any split of it is cached (then report it).
The lookup may hash the audio (~0.2 s), so it runs off this thread."""
threading.Thread(target=self._look_up, args=(path,), daemon=True,
name="lintunes-stems-lookup").start()
def _look_up(self, path: str):
hit = cached(path)
if hit is not None:
self.finished.emit(path, hit)
return
self._enqueue(path)
def _enqueue(self, path: str):
with self._lock:
if path in self._queue:
return
+34 -2
View File
@@ -1,4 +1,5 @@
import math
import threading
from PyQt6.QtWidgets import (
QDialog, QVBoxLayout, QHBoxLayout, QGroupBox, QLabel, QSlider,
@@ -9,7 +10,7 @@ from PyQt6.QtWidgets import (
QMessageBox, QApplication,
)
from PyQt6.QtCore import Qt, QTimer
from PyQt6.QtGui import QFont, QColor, QFontInfo
from PyQt6.QtGui import QDoubleValidator, QFont, QColor, QFontInfo
from lintunes import loudness, theme
from lintunes.config import load_config, save_config
@@ -273,10 +274,24 @@ class PreferencesDialog(QDialog):
gentle.setChecked(stems.gentle_choice(self._prefs))
gentle.toggled.connect(lambda on: self._prefs.set("dj_stem_gentle", on))
form.addRow("", gentle)
cache = QLineEdit(f"{stems.cache_limit_gb():g}")
cache.setValidator(QDoubleValidator(0.0, 10000.0, 2, cache))
cache.setMaximumWidth(80)
cache.setToolTip("Separated songs are kept up to this much disk space "
"on this computer; past it, the songs used longest "
"ago are deleted (they're split again if needed).")
cache.editingFinished.connect(self._save_stem_cache_limit)
cache_row = QHBoxLayout()
cache_row.addWidget(cache)
cache_row.addWidget(QLabel("GB"))
cache_row.addStretch(1)
form.addRow("Keep up to:", cache_row)
self._dj_cache_edit = cache
if stems.available():
note = QLabel("Demucs is set up. Songs are split when they load onto "
"a deck, or ahead of time with a playlist's or "
"song's “Prepare for DJ”. Splits are kept.")
"song's “Prepare for DJ”. Splits are kept, up to "
"the size above.")
else:
note = QLabel("Demucs isn't set up on this computer (a one-time ~1 GB "
"download). In a terminal:\n" + stems.setup_command())
@@ -287,6 +302,23 @@ class PreferencesDialog(QDialog):
self._dj_gentle_check = gentle
return group
def _save_stem_cache_limit(self):
"""config.json, not preferences: how much disk to give the splits
is this machine's business. A lower limit applies at once."""
from lintunes.dj import stems
try:
value = float(self._dj_cache_edit.text().replace(",", "."))
except ValueError:
self._dj_cache_edit.setText(f"{stems.cache_limit_gb():g}")
return
config = load_config()
if config.get(stems.CACHE_LIMIT_KEY) == value:
return
config[stems.CACHE_LIMIT_KEY] = value
save_config(config)
threading.Thread(target=stems.prune, args=(value,), daemon=True,
name="lintunes-stems-prune").start()
# ---- Wrapped ----
def _build_wrapped_group(self) -> QGroupBox:
+115
View File
@@ -33,3 +33,118 @@ def test_dragging_narrower_snaps_back_and_saves_the_minimum(qapp):
def test_wider_widths_are_left_alone(qapp):
view = _view({"rating": 200})
assert view.columnWidth(view.model_.fields.index("rating")) == 200
# ---- DJ stems: a cache that survives a tag edit, held to a size ----
import os
import shutil
import subprocess
import time
import pytest
from lintunes.dj import stems
@pytest.fixture
def cache(tmp_path, monkeypatch):
monkeypatch.setenv("XDG_CACHE_HOME", str(tmp_path / "cache"))
monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path / "config"))
return tmp_path
def _mp3(path):
subprocess.run(["ffmpeg", "-v", "error", "-y", "-f", "lavfi", "-i",
"sine=f=440:d=2", str(path)], check=True)
return str(path)
def _make(folder, size=1):
folder.mkdir(parents=True, exist_ok=True)
for name in stems.STEMS:
(folder / f"{name}.flac").write_bytes(b"f" * size)
needs_ffmpeg = pytest.mark.skipif(shutil.which("ffmpeg") is None,
reason="needs the ffmpeg CLI")
@needs_ffmpeg
def test_writing_the_bpm_tag_keeps_the_split(cache):
from mutagen.id3 import ID3, TBPM
song = _mp3(cache / "a.mp3")
_make(stems.cache_root() / stems.cache_key(song) / "htdemucs")
tags = ID3()
tags.add(TBPM(encoding=3, text=["128"]))
tags.save(song) # what loading a song with no BPM does
os.utime(song, (time.time() + 60, time.time() + 60))
assert stems.cached(song) is not None
@needs_ffmpeg
def test_a_split_under_the_old_path_key_is_moved_to_the_audio_key(cache):
song = _mp3(cache / "a.mp3")
_make(stems.cache_root() / stems._path_key(song) / "htdemucs")
assert stems.cached(song) is not None
assert (stems.cache_root() / stems.cache_key(song) / "htdemucs").is_dir()
assert not (stems.cache_root() / stems._path_key(song)).exists()
@needs_ffmpeg
def test_the_hash_runs_once_per_version_of_a_file(cache, monkeypatch):
song = _mp3(cache / "a.mp3")
calls = []
real = stems.audio_hash
monkeypatch.setattr(stems, "audio_hash",
lambda p: calls.append(p) or real(p))
first = stems.cache_key(song)
assert stems.cache_key(song) == first
assert len(calls) == 1
def test_unreadable_audio_falls_back_to_the_path_key(cache):
song = cache / "a.mp3"
song.write_bytes(b"x")
assert stems.cache_key(str(song)) == stems._path_key(str(song))
def test_prune_drops_the_songs_used_longest_ago(cache):
root = stems.cache_root()
now = time.time()
for i, name in enumerate(["old", "mid", "new"]):
_make(root / name / "htdemucs", size=400 * 1024) # 800 KiB a song
os.utime(root / name, (now - 100 + i, now - 100 + i))
removed = stems.prune(limit_gb=2 * 800 / 1024 ** 2) # room for two
assert removed == 1
assert sorted(p.name for p in root.iterdir()) == ["mid", "new"]
stems.prune(limit_gb=0, keep="mid")
assert [p.name for p in root.iterdir()] == ["mid"]
def test_the_limit_comes_from_config_json(cache):
from lintunes.config import save_config
assert stems.cache_limit_gb() == 1.0
save_config({stems.CACHE_LIMIT_KEY: 2.5})
assert stems.cache_limit_gb() == 2.5
assert stems.cache_limit_gb({stems.CACHE_LIMIT_KEY: "junk"}) == 1.0
def test_preferences_saves_the_limit_to_config_json(qapp, cache, monkeypatch):
from lintunes.config import load_config
from lintunes.gui.preferences_dialog import PreferencesDialog
from lintunes.preferences import Preferences
from tests.test_round40 import _FakeLastFm
pruned = []
monkeypatch.setattr(stems, "prune", lambda limit: pruned.append(limit))
dialog = PreferencesDialog(Preferences(cache), _FakeLastFm())
assert dialog._dj_cache_edit.text() == "1"
dialog._dj_cache_edit.setText("3.5")
dialog._save_stem_cache_limit()
assert load_config()[stems.CACHE_LIMIT_KEY] == 3.5
for _ in range(50):
if pruned:
break
time.sleep(0.01)
assert pruned == [3.5]
dialog.deleteLater()
+1
View File
@@ -136,6 +136,7 @@ class TestStems:
song = self._song(tmp_path, monkeypatch)
monkeypatch.setattr(stems, "available", lambda: True)
monkeypatch.setattr(stems.os, "cpu_count", lambda: 8)
monkeypatch.setattr(stems, "audio_hash", lambda path: None)
seen = {}
class Proc: