fix(worker): cancel abandoned slskd transfers + cap download fall-through
Two compounding causes of 'many downloads of one album from different sources' in slskd: (1) the slskd client raised on timeout/error WITHOUT cancelling the transfer it enqueued, so as the pipeline fell through to other peers each left a transfer running; (2) the incomplete-download fall-through was uncapped, grinding through every candidate of a popular album (~200 Soulseek peers), amplified now that Qobuz pacing pushes most jobs to Soulseek. - SlskdClient.download now cancels its enqueued transfers (DELETE) on any abandon. - The download loop caps at _MAX_DOWNLOAD_ATTEMPTS (6) candidates. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -89,3 +89,37 @@ def test_all_sources_incomplete_still_needs_attention(conn, tmp_path):
|
||||
run_pipeline(conn, job_id, [_Qobuz(deliver=2), _Soulseek(deliver=1)], dest_root=str(tmp_path))
|
||||
assert _state_error(conn, job_id) == ("needs_attention", "incomplete download")
|
||||
assert not (tmp_path / "Radiohead" / "In Rainbows").exists() # nothing imported
|
||||
|
||||
|
||||
class _CountingIncompleteQobuz:
|
||||
"""Offers `n` candidates that all download but come back incomplete; counts download calls."""
|
||||
name = "qobuz"
|
||||
tier = 0
|
||||
|
||||
def __init__(self, n):
|
||||
self.n = n
|
||||
self.calls = 0
|
||||
|
||||
def health(self):
|
||||
return True
|
||||
|
||||
def search(self, target):
|
||||
q = _HIRES
|
||||
return [Candidate(source="qobuz", source_ref=f"r{i}", matched_artist=target.artist,
|
||||
matched_album=target.album, quality=q, track_count=3, source_tier=0)
|
||||
for i in range(self.n)]
|
||||
|
||||
def download(self, candidate, dest, on_progress):
|
||||
self.calls += 1
|
||||
_stage(dest, 1) # 1 of 3 → always incomplete
|
||||
return DownloadResult(ok=True, path=dest, track_count=1)
|
||||
|
||||
|
||||
def test_fall_through_is_capped(conn):
|
||||
from lyra_worker.pipeline import _MAX_DOWNLOAD_ATTEMPTS
|
||||
a = _CountingIncompleteQobuz(n=20)
|
||||
job_id = insert_request(conn, artist="Radiohead", album="In Rainbows")
|
||||
claim_next(conn)
|
||||
run_pipeline(conn, job_id, [a], dest_root="/tmp/lib-cap")
|
||||
assert _state_error(conn, job_id) == ("needs_attention", "incomplete download")
|
||||
assert a.calls == _MAX_DOWNLOAD_ATTEMPTS # capped, not all 20 peers tried
|
||||
|
||||
@@ -0,0 +1,72 @@
|
||||
"""When an slskd download is abandoned (times out or errors), the enqueued transfers must be
|
||||
cancelled — otherwise, as the pipeline falls through to other peers for the same album, abandoned
|
||||
transfers pile up in slskd (many downloads of one album from different sources)."""
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
import lyra_worker.adapters._slskd as slskd_mod
|
||||
from lyra_worker.adapters._slskd import SlskdClient
|
||||
|
||||
|
||||
class _Resp:
|
||||
def __init__(self, payload=None):
|
||||
self._p = payload if payload is not None else {}
|
||||
|
||||
def raise_for_status(self):
|
||||
pass
|
||||
|
||||
def json(self):
|
||||
return self._p
|
||||
|
||||
|
||||
class _FakeRequests:
|
||||
"""Enqueue OK; polling always reports the file still in progress (never completes); records
|
||||
DELETEs so the test can assert the abandoned transfer was cancelled."""
|
||||
def __init__(self, state="InProgress"):
|
||||
self.deleted = []
|
||||
self._poll = {"directories": [{"files": [
|
||||
{"filename": "f1.flac", "size": 5, "state": state, "id": "t1"},
|
||||
]}]}
|
||||
|
||||
def post(self, url, **kw):
|
||||
return _Resp({})
|
||||
|
||||
def get(self, url, **kw):
|
||||
return _Resp(self._poll)
|
||||
|
||||
def delete(self, url, **kw):
|
||||
self.deleted.append(url)
|
||||
return _Resp({})
|
||||
|
||||
|
||||
def _client(tmp_path):
|
||||
return SlskdClient({"slskd.url": "http://x", "slskd.api_key": "k",
|
||||
"slskd.downloadsDir": str(tmp_path)})
|
||||
|
||||
|
||||
_REF = json.dumps({"username": "peerA", "files": [{"filename": "f1.flac", "size": 5}]})
|
||||
|
||||
|
||||
def test_cancels_transfer_on_timeout(tmp_path, monkeypatch):
|
||||
fake = _FakeRequests(state="InProgress")
|
||||
monkeypatch.setattr(slskd_mod, "requests", fake)
|
||||
monkeypatch.setattr(slskd_mod.time, "sleep", lambda _s: None)
|
||||
monkeypatch.setattr(slskd_mod, "_XFER_POLLS", 2)
|
||||
|
||||
with pytest.raises(TimeoutError):
|
||||
_client(tmp_path).download(_REF, str(tmp_path / "dest"), lambda _p: None)
|
||||
|
||||
assert any(u.endswith("/api/v0/transfers/downloads/peerA/t1") for u in fake.deleted), fake.deleted
|
||||
|
||||
|
||||
def test_cancels_transfer_on_error_state(tmp_path, monkeypatch):
|
||||
fake = _FakeRequests(state="Completed, Errored")
|
||||
monkeypatch.setattr(slskd_mod, "requests", fake)
|
||||
monkeypatch.setattr(slskd_mod.time, "sleep", lambda _s: None)
|
||||
monkeypatch.setattr(slskd_mod, "_XFER_POLLS", 5)
|
||||
|
||||
with pytest.raises(RuntimeError):
|
||||
_client(tmp_path).download(_REF, str(tmp_path / "dest"), lambda _p: None)
|
||||
|
||||
assert any(u.endswith("/peerA/t1") for u in fake.deleted), fake.deleted
|
||||
Reference in New Issue
Block a user