feat(worker): separate download staging volume (STAGING_DIR)

Staging is no longer forced under the library. The worker reads STAGING_ROOT
(container path, default /music/.staging) and stages per-job downloads there;
docker-compose mounts ${STAGING_DIR:-${MUSIC_DIR}/.staging} at /staging with
STAGING_ROOT=/staging. Set STAGING_DIR to a fast local disk when the library
(MUSIC_DIR) is a network share so temp download I/O stays off the share.

Safe cross-volume: import_album already assembles into a temp dir on the
library volume and swaps atomically there, so partial downloads never touch
the library and the final swap stays atomic. clear_staging_root now clears the
root's contents (mount-safe) rather than removing the root. Documented in
.env.example; new tests cover the separate-root path.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Jonathan
2026-07-12 23:44:27 +02:00
parent 891793953f
commit e825e3d7ad
7 changed files with 84 additions and 16 deletions
+16 -7
View File
@@ -4,7 +4,9 @@ from lyra_worker.library import clear_staging_root, import_album, staging_dir
def test_staging_dir_path():
assert staging_dir("/music", "job-1") == "/music/.staging/job-1"
# staging_dir now takes the staging root directly (the default root is <library>/.staging)
assert staging_dir("/music/.staging", "job-1") == "/music/.staging/job-1"
assert staging_dir("/staging", "job-1") == "/staging/job-1" # separate-volume root
def test_import_album_moves_audio_and_cover_into_fresh_final(tmp_path):
@@ -53,9 +55,16 @@ def test_import_album_leaves_no_temp_siblings(tmp_path):
assert os.listdir(tmp_path / "Artist") == ["Album"] # no .importing / .old leftovers
def test_clear_staging_root_removes_staging(tmp_path):
stg = tmp_path / ".staging" / "j1"
stg.mkdir(parents=True)
(stg / "x").write_bytes(b"")
clear_staging_root(str(tmp_path))
assert not (tmp_path / ".staging").exists()
def test_clear_staging_root_removes_jobs_but_keeps_root(tmp_path):
# Clears per-job dirs (and stray files) under the root, but leaves the root itself so a
# mounted staging volume stays mounted.
root = tmp_path / "staging"
(root / "j1").mkdir(parents=True)
(root / "j1" / "x").write_bytes(b"")
(root / "stray.txt").write_text("orphan")
clear_staging_root(str(root))
assert root.exists() and list(os.listdir(root)) == []
def test_clear_staging_root_tolerates_missing(tmp_path):
clear_staging_root(str(tmp_path / "nope")) # no-op, must not raise
+29
View File
@@ -37,6 +37,35 @@ class FlakyAdapter:
return DownloadResult(ok=True, path=dest, track_count=3)
class CompleteAdapter(FlakyAdapter):
"""Always downloads the full 3-track set — for exercising a clean import path."""
def download(self, candidate, dest, on_progress):
os.makedirs(dest, exist_ok=True)
for i in range(3):
open(os.path.join(dest, f"{i} track.flac"), "wb").close()
return DownloadResult(ok=True, path=dest, track_count=3)
def test_staging_on_a_separate_root_still_imports(conn, tmp_path):
# The SMB-share use case: staging on a different root than the library. Download stages
# under STAGING_ROOT, the album imports into the library, and the default library/.staging
# is never used.
lib = tmp_path / "library"
stg = tmp_path / "staging"
job_id = insert_request(conn, artist="Radiohead", album="In Rainbows")
claim_next(conn)
run_pipeline(conn, job_id, [CompleteAdapter()], dest_root=str(lib), staging_root=str(stg))
with conn.cursor() as cur:
cur.execute('SELECT state FROM "Job" WHERE id = %s', (job_id,))
assert cur.fetchone()[0] == "imported"
assert sorted(os.listdir(lib / "Radiohead" / "In Rainbows")) == ["0 track.flac", "1 track.flac", "2 track.flac"]
assert stg.is_dir() # staging used the separate root...
assert not (lib / ".staging").exists() # ...not the default library/.staging
assert list(stg.iterdir()) == [] # per-job staging cleaned up afterwards
def test_partial_download_leaves_library_untouched(conn, tmp_path):
adapter = FlakyAdapter() # first call is partial
job_id = insert_request(conn, artist="Radiohead", album="In Rainbows")