v0.68.0: DJ vocal splits survive tag edits; a size limit for their cache

This commit is contained in:
2026-10-19 22:16:39 -07:00
parent 1d215fd045
commit d1174c6b35
6 changed files with 322 additions and 12 deletions
+7
View File
@@ -395,6 +395,13 @@ persistence) → GUI (Qt widgets that read the manager and connect to its signal
model exists) and "gentle" (default: a quarter of the cores, nice 19). model exists) and "gentle" (default: a quarter of the cores, nice 19).
Splits are cached per model (`<key>/<model>/`; Round 93's sit in Splits are cached per model (`<key>/<model>/`; Round 93's sit in
`<key>/` and count as htdemucs) and a deck takes the best one cached. `<key>/` and count as htdemucs) and a deck takes the best one cached.
**Round 100**: `<key>` is a hash of the *audio packets* (`ffmpeg -c copy
-f hash`), not path + size + mtime — loading a song with no BPM writes
its BPM tag, which orphaned every such split at once. `index.json`
memoizes path + size + mtime → key; old path-keyed folders are renamed
on sight. The cache is held to config.json's `dj_stem_cache_gb`
(Preferences › DJ, default 1): after each split the least recently used
songs go (`stems.prune`).
**Prepare for DJ** (a playlist's or songs' right-click; `dj/prepare.py`) **Prepare for DJ** (a playlist's or songs' right-click; `dj/prepare.py`)
does the BPM + split ahead of time with the status-bar progress/cancel. does the BPM + split ahead of time with the status-bar progress/cancel.
Each deck also has a volume fader (outer edge, perceptual), a ½×/1×/2× Each deck also has a volume fader (outer edge, perceptual), a ½×/1×/2×
+1 -1
View File
@@ -1,3 +1,3 @@
"""LinTunes — iTunes-style music library manager and player for Linux.""" """LinTunes — iTunes-style music library manager and player for Linux."""
__version__ = "0.67.3" __version__ = "0.68.0"
+164 -9
View File
@@ -2,11 +2,22 @@
Nothing separates a voice from a mix in real time on a laptop, so a song is Nothing separates a voice from a mix in real time on a laptop, so a song is
split once, in the background, and kept: split once, in the background, and kept:
``$XDG_CACHE_HOME/lintunes/stems/<key>/<model>/{vocals,no_vocals}.flac``, ``$XDG_CACHE_HOME/lintunes/stems/<key>/<model>/{vocals,no_vocals}.flac``
keyed by path + size + mtime like the loudness cache (machine-local, never (machine-local, never the synced data dir). A cached song's slider works the
the synced data dir). A cached song's slider works the moment it loads, and moment it loads, and a deck uses the best split it finds whatever model made
a deck uses the best split it finds whatever model made it. (Round 93's it. (Round 93's first splits sit directly in ``<key>/``; they were htdemucs.)
first splits sit directly in ``<key>/``; they were htdemucs.)
The key is the song's **audio**, not its file (Round 100): a hash of the
audio packets (``ffmpeg -c copy -f hash``, ~0.2 s, no decoding), so a tag
edit or a move doesn't lose the split. It used to be path + size + mtime,
and loading a song with no BPM *writes* its BPM tag — so every such split
was orphaned the moment it was made. ``index.json`` remembers path + size +
mtime → key, so the hash runs once per version of a file; a folder still
under its old path key is renamed to the audio key when found.
The cache is held to ``config.json``'s ``dj_stem_cache_gb`` (1 GB unless
set; this machine's disk, so not a synced pref): after each split, the songs
used longest ago go first. Using a split (``cached``) counts as using it.
Demucs runs as a runtime tool, never a pip dependency of LinTunes: PyTorch is Demucs runs as a runtime tool, never a pip dependency of LinTunes: PyTorch is
~1 GB, so it lives in its own venv at ``$XDG_DATA_HOME/lintunes/demucs`` ~1 GB, so it lives in its own venv at ``$XDG_DATA_HOME/lintunes/demucs``
@@ -20,6 +31,7 @@ One split at a time: two would only halve each other's speed.
""" """
import hashlib import hashlib
import json
import os import os
import re import re
import shutil import shutil
@@ -30,6 +42,8 @@ from pathlib import Path
from PyQt6.QtCore import QObject, pyqtSignal from PyQt6.QtCore import QObject, pyqtSignal
from lintunes.config import load_config
# id → (label, what it costs). Best first: a deck uses the best split it has. # id → (label, what it costs). Best first: a deck uses the best split it has.
# Measured on this laptop (Round 94, 40 s of audio, 2 threads): htdemucs # Measured on this laptop (Round 94, 40 s of audio, 2 threads): htdemucs
# 35 s, at overlap 0.1 27 s; hdemucs_mmi 42 s and mdx 156 s were slower, # 35 s, at overlap 0.1 27 s; hdemucs_mmi 42 s and mdx 156 s were slower,
@@ -95,13 +109,90 @@ def cache_root() -> Path:
return Path(base) / "lintunes" / "stems" return Path(base) / "lintunes" / "stems"
def cache_key(path: str) -> str | None: CACHE_LIMIT_KEY = "dj_stem_cache_gb"
DEFAULT_CACHE_GB = 1.0
_INDEX = "index.json"
_index_lock = threading.Lock()
def cache_limit_gb(config: dict | None = None) -> float:
config = load_config() if config is None else config
try:
value = float(config.get(CACHE_LIMIT_KEY, DEFAULT_CACHE_GB))
except (TypeError, ValueError):
return DEFAULT_CACHE_GB
return value if value >= 0 else DEFAULT_CACHE_GB
def _file_id(path: str) -> str | None:
try: try:
st = os.stat(path) st = os.stat(path)
except OSError: except OSError:
return None return None
raw = f"{os.path.abspath(path)}|{st.st_size}|{int(st.st_mtime)}" return f"{os.path.abspath(path)}|{st.st_size}|{int(st.st_mtime)}"
return hashlib.sha1(raw.encode()).hexdigest()[:20]
def _path_key(path: str) -> str | None:
"""The key before Round 100: path + size + mtime."""
raw = _file_id(path)
return hashlib.sha1(raw.encode()).hexdigest()[:20] if raw else None
def audio_hash(path: str) -> str | None:
"""A hash of the audio packets alone, so tags don't change it."""
try:
done = subprocess.run(
["ffmpeg", "-v", "error", "-nostdin", "-i", path, "-map", "0:a:0",
"-c", "copy", "-f", "hash", "-hash", "sha256", "-"],
capture_output=True, text=True, timeout=60)
except (OSError, subprocess.SubprocessError):
return None
out = done.stdout.strip()
if done.returncode != 0 or not out.startswith("SHA256="):
return None
return out[len("SHA256="):]
def _read_index() -> dict:
try:
with open(cache_root() / _INDEX, encoding="utf-8") as f:
data = json.load(f)
return data if isinstance(data, dict) else {}
except (OSError, ValueError):
return {}
def _write_index(index: dict):
root = cache_root()
try:
root.mkdir(parents=True, exist_ok=True)
tmp = root / (_INDEX + ".tmp")
tmp.write_text(json.dumps(index), encoding="utf-8")
tmp.replace(root / _INDEX)
except OSError:
pass
def cache_key(path: str) -> str | None:
"""The song's audio key (the path key when ffmpeg can't read it)."""
file_id = _file_id(path)
if file_id is None:
return None
with _index_lock:
known = _read_index().get(file_id)
if known:
return known
digest = audio_hash(path)
if digest is None:
return _path_key(path)
key = "a" + digest[:24]
with _index_lock:
index = _read_index()
prefix = os.path.abspath(path) + "|"
index = {k: v for k, v in index.items() if not k.startswith(prefix)}
index[file_id] = key
_write_index(index)
return key
def _files(folder: Path) -> dict[str, Path] | None: def _files(folder: Path) -> dict[str, Path] | None:
@@ -109,6 +200,13 @@ def _files(folder: Path) -> dict[str, Path] | None:
return files if all(f.is_file() for f in files.values()) else None return files if all(f.is_file() for f in files.values()) else None
def _touch(folder: Path):
try:
os.utime(folder)
except OSError:
pass
def cached(path: str, model: str | None = None) -> dict[str, Path] | None: def cached(path: str, model: str | None = None) -> dict[str, Path] | None:
"""The stems of ``path``: ``model``'s when given, else the best there """The stems of ``path``: ``model``'s when given, else the best there
are (``MODELS`` order). None when it hasn't been split.""" are (``MODELS`` order). None when it hasn't been split."""
@@ -116,16 +214,63 @@ def cached(path: str, model: str | None = None) -> dict[str, Path] | None:
if key is None: if key is None:
return None return None
root = cache_root() / key root = cache_root() / key
legacy = _path_key(path)
if legacy and legacy != key and not root.exists() and (cache_root() / legacy).is_dir():
try:
(cache_root() / legacy).rename(root)
except OSError:
root = cache_root() / legacy
order = [model] if model else list(MODELS) order = [model] if model else list(MODELS)
for name in order: for name in order:
hit = _files(root / name) hit = _files(root / name)
if hit is None and name == LEGACY_MODEL: if hit is None and name == LEGACY_MODEL:
hit = _files(root) hit = _files(root)
if hit is not None: if hit is not None:
_touch(root)
return hit return hit
return None return None
def _size(folder: Path) -> int:
total = 0
for f in folder.rglob("*"):
try:
if f.is_file():
total += f.stat().st_size
except OSError:
pass
return total
def prune(limit_gb: float | None = None, keep: str | None = None) -> int:
"""Delete the songs used longest ago until the cache is under
``limit_gb`` (config.json's when None). ``keep`` (a key) is never
deleted. Returns how many songs went."""
limit = int((cache_limit_gb() if limit_gb is None else limit_gb) * 1024 ** 3)
root = cache_root()
try:
folders = [d for d in root.iterdir() if d.is_dir()]
except OSError:
return 0
entries = []
for d in folders:
try:
entries.append((d.stat().st_mtime, d, _size(d)))
except OSError:
pass
total = sum(size for _m, _d, size in entries)
removed = 0
for _mtime, folder, size in sorted(entries, key=lambda e: e[0]):
if total <= limit:
break
if folder.name == keep:
continue
shutil.rmtree(folder, ignore_errors=True)
total -= size
removed += 1
return removed
def parse_progress(text: str) -> int | None: def parse_progress(text: str) -> int | None:
found = _PERCENT.findall(text) found = _PERCENT.findall(text)
return int(found[-1]) if found else None return int(found[-1]) if found else None
@@ -178,6 +323,8 @@ def split(path: str, model: str, gentle: bool, progress=None,
target = folder / f"{name}.flac" target = folder / f"{name}.flac"
shutil.move(str(made[0].parent / f"{name}.flac"), target) shutil.move(str(made[0].parent / f"{name}.flac"), target)
out[name] = target out[name] = target
_touch(cache_root() / key)
prune(keep=key)
return out return out
finally: finally:
shutil.rmtree(work, ignore_errors=True) shutil.rmtree(work, ignore_errors=True)
@@ -200,11 +347,19 @@ class StemSplitter(QObject):
self._lock = threading.Lock() self._lock = threading.Lock()
def request(self, path: str): def request(self, path: str):
"""Split ``path`` unless any split of it is cached (then report it).""" """Split ``path`` unless any split of it is cached (then report it).
The lookup may hash the audio (~0.2 s), so it runs off this thread."""
threading.Thread(target=self._look_up, args=(path,), daemon=True,
name="lintunes-stems-lookup").start()
def _look_up(self, path: str):
hit = cached(path) hit = cached(path)
if hit is not None: if hit is not None:
self.finished.emit(path, hit) self.finished.emit(path, hit)
return return
self._enqueue(path)
def _enqueue(self, path: str):
with self._lock: with self._lock:
if path in self._queue: if path in self._queue:
return return
+34 -2
View File
@@ -1,4 +1,5 @@
import math import math
import threading
from PyQt6.QtWidgets import ( from PyQt6.QtWidgets import (
QDialog, QVBoxLayout, QHBoxLayout, QGroupBox, QLabel, QSlider, QDialog, QVBoxLayout, QHBoxLayout, QGroupBox, QLabel, QSlider,
@@ -9,7 +10,7 @@ from PyQt6.QtWidgets import (
QMessageBox, QApplication, QMessageBox, QApplication,
) )
from PyQt6.QtCore import Qt, QTimer from PyQt6.QtCore import Qt, QTimer
from PyQt6.QtGui import QFont, QColor, QFontInfo from PyQt6.QtGui import QDoubleValidator, QFont, QColor, QFontInfo
from lintunes import loudness, theme from lintunes import loudness, theme
from lintunes.config import load_config, save_config from lintunes.config import load_config, save_config
@@ -273,10 +274,24 @@ class PreferencesDialog(QDialog):
gentle.setChecked(stems.gentle_choice(self._prefs)) gentle.setChecked(stems.gentle_choice(self._prefs))
gentle.toggled.connect(lambda on: self._prefs.set("dj_stem_gentle", on)) gentle.toggled.connect(lambda on: self._prefs.set("dj_stem_gentle", on))
form.addRow("", gentle) form.addRow("", gentle)
cache = QLineEdit(f"{stems.cache_limit_gb():g}")
cache.setValidator(QDoubleValidator(0.0, 10000.0, 2, cache))
cache.setMaximumWidth(80)
cache.setToolTip("Separated songs are kept up to this much disk space "
"on this computer; past it, the songs used longest "
"ago are deleted (they're split again if needed).")
cache.editingFinished.connect(self._save_stem_cache_limit)
cache_row = QHBoxLayout()
cache_row.addWidget(cache)
cache_row.addWidget(QLabel("GB"))
cache_row.addStretch(1)
form.addRow("Keep up to:", cache_row)
self._dj_cache_edit = cache
if stems.available(): if stems.available():
note = QLabel("Demucs is set up. Songs are split when they load onto " note = QLabel("Demucs is set up. Songs are split when they load onto "
"a deck, or ahead of time with a playlist's or " "a deck, or ahead of time with a playlist's or "
"song's “Prepare for DJ”. Splits are kept.") "song's “Prepare for DJ”. Splits are kept, up to "
"the size above.")
else: else:
note = QLabel("Demucs isn't set up on this computer (a one-time ~1 GB " note = QLabel("Demucs isn't set up on this computer (a one-time ~1 GB "
"download). In a terminal:\n" + stems.setup_command()) "download). In a terminal:\n" + stems.setup_command())
@@ -287,6 +302,23 @@ class PreferencesDialog(QDialog):
self._dj_gentle_check = gentle self._dj_gentle_check = gentle
return group return group
def _save_stem_cache_limit(self):
"""config.json, not preferences: how much disk to give the splits
is this machine's business. A lower limit applies at once."""
from lintunes.dj import stems
try:
value = float(self._dj_cache_edit.text().replace(",", "."))
except ValueError:
self._dj_cache_edit.setText(f"{stems.cache_limit_gb():g}")
return
config = load_config()
if config.get(stems.CACHE_LIMIT_KEY) == value:
return
config[stems.CACHE_LIMIT_KEY] = value
save_config(config)
threading.Thread(target=stems.prune, args=(value,), daemon=True,
name="lintunes-stems-prune").start()
# ---- Wrapped ---- # ---- Wrapped ----
def _build_wrapped_group(self) -> QGroupBox: def _build_wrapped_group(self) -> QGroupBox:
+115
View File
@@ -33,3 +33,118 @@ def test_dragging_narrower_snaps_back_and_saves_the_minimum(qapp):
def test_wider_widths_are_left_alone(qapp): def test_wider_widths_are_left_alone(qapp):
view = _view({"rating": 200}) view = _view({"rating": 200})
assert view.columnWidth(view.model_.fields.index("rating")) == 200 assert view.columnWidth(view.model_.fields.index("rating")) == 200
# ---- DJ stems: a cache that survives a tag edit, held to a size ----
import os
import shutil
import subprocess
import time
import pytest
from lintunes.dj import stems
@pytest.fixture
def cache(tmp_path, monkeypatch):
monkeypatch.setenv("XDG_CACHE_HOME", str(tmp_path / "cache"))
monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path / "config"))
return tmp_path
def _mp3(path):
subprocess.run(["ffmpeg", "-v", "error", "-y", "-f", "lavfi", "-i",
"sine=f=440:d=2", str(path)], check=True)
return str(path)
def _make(folder, size=1):
folder.mkdir(parents=True, exist_ok=True)
for name in stems.STEMS:
(folder / f"{name}.flac").write_bytes(b"f" * size)
needs_ffmpeg = pytest.mark.skipif(shutil.which("ffmpeg") is None,
reason="needs the ffmpeg CLI")
@needs_ffmpeg
def test_writing_the_bpm_tag_keeps_the_split(cache):
from mutagen.id3 import ID3, TBPM
song = _mp3(cache / "a.mp3")
_make(stems.cache_root() / stems.cache_key(song) / "htdemucs")
tags = ID3()
tags.add(TBPM(encoding=3, text=["128"]))
tags.save(song) # what loading a song with no BPM does
os.utime(song, (time.time() + 60, time.time() + 60))
assert stems.cached(song) is not None
@needs_ffmpeg
def test_a_split_under_the_old_path_key_is_moved_to_the_audio_key(cache):
song = _mp3(cache / "a.mp3")
_make(stems.cache_root() / stems._path_key(song) / "htdemucs")
assert stems.cached(song) is not None
assert (stems.cache_root() / stems.cache_key(song) / "htdemucs").is_dir()
assert not (stems.cache_root() / stems._path_key(song)).exists()
@needs_ffmpeg
def test_the_hash_runs_once_per_version_of_a_file(cache, monkeypatch):
song = _mp3(cache / "a.mp3")
calls = []
real = stems.audio_hash
monkeypatch.setattr(stems, "audio_hash",
lambda p: calls.append(p) or real(p))
first = stems.cache_key(song)
assert stems.cache_key(song) == first
assert len(calls) == 1
def test_unreadable_audio_falls_back_to_the_path_key(cache):
song = cache / "a.mp3"
song.write_bytes(b"x")
assert stems.cache_key(str(song)) == stems._path_key(str(song))
def test_prune_drops_the_songs_used_longest_ago(cache):
root = stems.cache_root()
now = time.time()
for i, name in enumerate(["old", "mid", "new"]):
_make(root / name / "htdemucs", size=400 * 1024) # 800 KiB a song
os.utime(root / name, (now - 100 + i, now - 100 + i))
removed = stems.prune(limit_gb=2 * 800 / 1024 ** 2) # room for two
assert removed == 1
assert sorted(p.name for p in root.iterdir()) == ["mid", "new"]
stems.prune(limit_gb=0, keep="mid")
assert [p.name for p in root.iterdir()] == ["mid"]
def test_the_limit_comes_from_config_json(cache):
from lintunes.config import save_config
assert stems.cache_limit_gb() == 1.0
save_config({stems.CACHE_LIMIT_KEY: 2.5})
assert stems.cache_limit_gb() == 2.5
assert stems.cache_limit_gb({stems.CACHE_LIMIT_KEY: "junk"}) == 1.0
def test_preferences_saves_the_limit_to_config_json(qapp, cache, monkeypatch):
from lintunes.config import load_config
from lintunes.gui.preferences_dialog import PreferencesDialog
from lintunes.preferences import Preferences
from tests.test_round40 import _FakeLastFm
pruned = []
monkeypatch.setattr(stems, "prune", lambda limit: pruned.append(limit))
dialog = PreferencesDialog(Preferences(cache), _FakeLastFm())
assert dialog._dj_cache_edit.text() == "1"
dialog._dj_cache_edit.setText("3.5")
dialog._save_stem_cache_limit()
assert load_config()[stems.CACHE_LIMIT_KEY] == 3.5
for _ in range(50):
if pruned:
break
time.sleep(0.01)
assert pruned == [3.5]
dialog.deleteLater()
+1
View File
@@ -136,6 +136,7 @@ class TestStems:
song = self._song(tmp_path, monkeypatch) song = self._song(tmp_path, monkeypatch)
monkeypatch.setattr(stems, "available", lambda: True) monkeypatch.setattr(stems, "available", lambda: True)
monkeypatch.setattr(stems.os, "cpu_count", lambda: 8) monkeypatch.setattr(stems.os, "cpu_count", lambda: 8)
monkeypatch.setattr(stems, "audio_hash", lambda path: None)
seen = {} seen = {}
class Proc: class Proc: