feat: download to staging + atomically promote complete album into the library

Rewrite run_pipeline's download->import section to download into a per-job
staging dir (cleared before each candidate), verify completeness there,
promote only a complete album into the library via import_album, tag the
final dir, then DB-import. The staging dir is always removed in a
try/finally, so a partial/failed download never touches or corrupts the
library, and a retry can no longer leave orphan files behind.

Also point test_upgrade_e2e.py's run_pipeline call at dest_root="/tmp/lib"
(matching every other pipeline test) since import_album now unconditionally
creates the final dir, which requires a writable dest_root.

worker/lyra_worker/pipeline.py

@@ -1,3 +1,4 @@
+import shutil
 from dataclasses import replace
 from typing import Sequence
 
@@ -5,7 +6,7 @@ import psycopg
 
 from lyra_worker.adapters.base import SourceAdapter
 from lyra_worker.confidence import score_confidence
-from lyra_worker.library import album_dir
+from lyra_worker.library import album_dir, import_album, staging_dir
 from lyra_worker.quality import quality_class
 from lyra_worker.ranker import rank_candidates
 from lyra_worker.types import Candidate, MBTarget
@@ -181,40 +182,45 @@ def run_pipeline(
         _fail(conn, job_id, "no candidate above confidence threshold")
         return
 
-    # 4. download (fall-through)
+    # 4. download (fall-through) into an isolated per-job staging dir
     _set_state(conn, job_id, "downloading", "download")
     by_source = {a.name: a for a in adapters}
-    dest = album_dir(dest_root, target)
+    staging = staging_dir(dest_root, job_id)
     winner = None
     result = None
-    for candidate in ranked:
-        adapter = by_source.get(candidate.source)
-        if adapter is None:
-            continue
-        result = adapter.download(candidate, dest, lambda _pct: None)
-        if result.ok:
-            winner = candidate
-            _mark_chosen(conn, job_id, candidate.source_ref)
-            break
-    if winner is None or result is None or not result.ok:
-        _fail(conn, job_id, "all downloads failed")
-        return
+    try:
+        for candidate in ranked:
+            adapter = by_source.get(candidate.source)
+            if adapter is None:
+                continue
+            shutil.rmtree(staging, ignore_errors=True)  # clean slate per attempt
+            result = adapter.download(candidate, staging, lambda _pct: None)
+            if result.ok:
+                winner = candidate
+                _mark_chosen(conn, job_id, candidate.source_ref)
+                break
+        if winner is None or result is None or not result.ok:
+            _fail(conn, job_id, "all downloads failed")
+            return
 
-    # 5. tag (integrity check against the canonical track count when known)
-    _set_state(conn, job_id, "tagging", "tag")
-    expected = target.track_count if target.track_count is not None else winner.track_count
-    if result.track_count < expected:
-        _fail(conn, job_id, "incomplete download")
-        return
+        # 5. verify completeness on the staging copy, then promote + tag
+        _set_state(conn, job_id, "tagging", "tag")
+        expected = target.track_count if target.track_count is not None else winner.track_count
+        if result.track_count < expected:
+            _fail(conn, job_id, "incomplete download")
+            return
 
-    library_path = result.path or dest
+        final = album_dir(dest_root, target)
+        import_album(staging, final)  # clean audio(+cover) move into the library, replacing any prior copy
 
-    if tagger is not None:
-        try:
-            tagger.tag_album(library_path, target)
-        except Exception as e:  # a tagging failure must not discard a good download
-            print(f"pipeline: tagging failed for job {job_id}: {e}", flush=True)
+        if tagger is not None:
+            try:
+                tagger.tag_album(final, target)
+            except Exception as e:  # a tagging failure must not discard a good download
+                print(f"pipeline: tagging failed for job {job_id}: {e}", flush=True)
 
-    # 6. import
-    _set_state(conn, job_id, "imported", "import")
-    _import(conn, job_id, target, winner, library_path)
+        # 6. import (DB)
+        _set_state(conn, job_id, "imported", "import")
+        _import(conn, job_id, target, winner, final)
+    finally:
+        shutil.rmtree(staging, ignore_errors=True)  # a partial/failed download never persists