Jonathan
1fc61a0f5a
Two compounding causes of 'many downloads of one album from different sources' in slskd: (1) the slskd client raised on timeout/error WITHOUT cancelling the transfer it enqueued, so as the pipeline fell through to other peers each left a transfer running; (2) the incomplete-download fall-through was uncapped, grinding through every candidate of a popular album (~200 Soulseek peers), amplified now that Qobuz pacing pushes most jobs to Soulseek. - SlskdClient.download now cancels its enqueued transfers (DELETE) on any abandon. - The download loop caps at _MAX_DOWNLOAD_ATTEMPTS (6) candidates. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
126 lines
4.7 KiB
Python
126 lines
4.7 KiB
Python
"""When the top-ranked source downloads but is incomplete (e.g. Qobuz consistently fails one
|
|
track), the pipeline must fall through to the next-ranked source instead of failing the whole
|
|
job — only giving up (needs_attention) when every candidate is incomplete."""
|
|
import os
|
|
|
|
from lyra_worker.claim import claim_next
|
|
from lyra_worker.pipeline import run_pipeline
|
|
from lyra_worker.types import Candidate, DownloadResult, Quality
|
|
from tests.conftest import insert_request
|
|
|
|
_HIRES = Quality(fmt="FLAC", lossless=True, bit_depth=24, sample_rate=96000)
|
|
_CD = Quality(fmt="FLAC", lossless=True, bit_depth=16, sample_rate=44100)
|
|
|
|
|
|
def _stage(dest, n):
|
|
os.makedirs(dest, exist_ok=True)
|
|
for i in range(n):
|
|
with open(os.path.join(dest, f"{i + 1:02d}.flac"), "wb") as fh:
|
|
fh.write(b"audio")
|
|
|
|
|
|
class _Qobuz:
|
|
"""Higher-quality (ranks first) Qobuz that promises 3 tracks but delivers `deliver`."""
|
|
name = "qobuz"
|
|
tier = 0
|
|
|
|
def __init__(self, deliver):
|
|
self._deliver = deliver
|
|
|
|
def health(self):
|
|
return True
|
|
|
|
def search(self, target):
|
|
return [Candidate(source="qobuz", source_ref="q", matched_artist=target.artist,
|
|
matched_album=target.album, quality=_HIRES, track_count=3, source_tier=0)]
|
|
|
|
def download(self, candidate, dest, on_progress):
|
|
_stage(dest, self._deliver)
|
|
return DownloadResult(ok=True, path=dest, track_count=self._deliver)
|
|
|
|
|
|
class _Soulseek:
|
|
"""Lower-quality (ranks second) Soulseek that promises 3 tracks and delivers `deliver`."""
|
|
name = "soulseek"
|
|
tier = 1
|
|
|
|
def __init__(self, deliver):
|
|
self._deliver = deliver
|
|
|
|
def health(self):
|
|
return True
|
|
|
|
def search(self, target):
|
|
return [Candidate(source="soulseek", source_ref="s", matched_artist=target.artist,
|
|
matched_album=target.album, quality=_CD, track_count=3, source_tier=1)]
|
|
|
|
def download(self, candidate, dest, on_progress):
|
|
_stage(dest, self._deliver)
|
|
return DownloadResult(ok=True, path=dest, track_count=self._deliver)
|
|
|
|
|
|
def _state_error(conn, job_id):
|
|
with conn.cursor() as cur:
|
|
cur.execute('SELECT state, error FROM "Job" WHERE id = %s', (job_id,))
|
|
return cur.fetchone()
|
|
|
|
|
|
def _chosen_source(conn, job_id):
|
|
with conn.cursor() as cur:
|
|
cur.execute('SELECT source FROM "Candidate" WHERE "jobId" = %s AND chosen = true', (job_id,))
|
|
row = cur.fetchone()
|
|
return row[0] if row else None
|
|
|
|
|
|
def test_incomplete_top_source_falls_through_to_complete_next(conn, tmp_path):
|
|
# Qobuz (ranks first on quality) delivers only 2 of its promised 3 tracks -> incomplete.
|
|
# The pipeline must try the next source (Soulseek, complete) rather than failing the job.
|
|
job_id = insert_request(conn, artist="Radiohead", album="In Rainbows")
|
|
claim_next(conn)
|
|
run_pipeline(conn, job_id, [_Qobuz(deliver=2), _Soulseek(deliver=3)], dest_root=str(tmp_path))
|
|
assert _state_error(conn, job_id) == ("imported", None)
|
|
assert _chosen_source(conn, job_id) == "soulseek"
|
|
|
|
|
|
def test_all_sources_incomplete_still_needs_attention(conn, tmp_path):
|
|
# Every candidate is incomplete -> no fallback succeeds -> needs_attention (incomplete).
|
|
job_id = insert_request(conn, artist="Radiohead", album="In Rainbows")
|
|
claim_next(conn)
|
|
run_pipeline(conn, job_id, [_Qobuz(deliver=2), _Soulseek(deliver=1)], dest_root=str(tmp_path))
|
|
assert _state_error(conn, job_id) == ("needs_attention", "incomplete download")
|
|
assert not (tmp_path / "Radiohead" / "In Rainbows").exists() # nothing imported
|
|
|
|
|
|
class _CountingIncompleteQobuz:
|
|
"""Offers `n` candidates that all download but come back incomplete; counts download calls."""
|
|
name = "qobuz"
|
|
tier = 0
|
|
|
|
def __init__(self, n):
|
|
self.n = n
|
|
self.calls = 0
|
|
|
|
def health(self):
|
|
return True
|
|
|
|
def search(self, target):
|
|
q = _HIRES
|
|
return [Candidate(source="qobuz", source_ref=f"r{i}", matched_artist=target.artist,
|
|
matched_album=target.album, quality=q, track_count=3, source_tier=0)
|
|
for i in range(self.n)]
|
|
|
|
def download(self, candidate, dest, on_progress):
|
|
self.calls += 1
|
|
_stage(dest, 1) # 1 of 3 → always incomplete
|
|
return DownloadResult(ok=True, path=dest, track_count=1)
|
|
|
|
|
|
def test_fall_through_is_capped(conn):
|
|
from lyra_worker.pipeline import _MAX_DOWNLOAD_ATTEMPTS
|
|
a = _CountingIncompleteQobuz(n=20)
|
|
job_id = insert_request(conn, artist="Radiohead", album="In Rainbows")
|
|
claim_next(conn)
|
|
run_pipeline(conn, job_id, [a], dest_root="/tmp/lib-cap")
|
|
assert _state_error(conn, job_id) == ("needs_attention", "incomplete download")
|
|
assert a.calls == _MAX_DOWNLOAD_ATTEMPTS # capped, not all 20 peers tried
|