fix(worker): smarter Soulseek peer selection + stall/progress-aware transfers
Draining the queue via Soulseek was stalling: slow peers (~100KB/s) never finished a full FLAC album within the fixed 10-min timeout, got cancelled at ~89%, and the fall-through restarted from scratch on the next peer — while single-song folders named like the album were also being tried. - Stall detection: abandon a peer with no byte progress for ~90s (queued/dead) instead of waiting the full timeout; a peer that IS progressing keeps its slot up to a 60-min backstop, so a slow-but-working transfer can finish. - Rank search candidates by peer: free upload slot, then speed, then shortest queue. - Drop candidates with far fewer tracks than the release (single-song folders) up front, so no download attempt is wasted on them. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Jonathan
@@ -1,12 +1,12 @@
|
||||
"""When an slskd download is abandoned (times out or errors), the enqueued transfers must be
|
||||
cancelled — otherwise, as the pipeline falls through to other peers for the same album, abandoned
|
||||
transfers pile up in slskd (many downloads of one album from different sources)."""
|
||||
"""slskd download behaviour: abandoned transfers are cancelled (no pile-up), a stalled peer is
|
||||
dropped fast, a slow-but-progressing peer is kept, and search candidates are ranked fast/free
|
||||
peers first."""
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
import lyra_worker.adapters._slskd as slskd_mod
|
||||
from lyra_worker.adapters._slskd import SlskdClient
|
||||
from lyra_worker.adapters._slskd import SlskdClient, _parse_search_responses
|
||||
|
||||
|
||||
class _Resp:
|
||||
@@ -20,53 +20,91 @@ class _Resp:
|
||||
return self._p
|
||||
|
||||
|
||||
class _FakeRequests:
|
||||
"""Enqueue OK; polling always reports the file still in progress (never completes); records
|
||||
DELETEs so the test can assert the abandoned transfer was cancelled."""
|
||||
def __init__(self, state="InProgress"):
|
||||
class _SeqRequests:
|
||||
"""post OK; get() returns successive poll payloads (last one repeats); records DELETEs."""
|
||||
def __init__(self, polls):
|
||||
self._polls = list(polls)
|
||||
self._i = 0
|
||||
self.deleted = []
|
||||
self._poll = {"directories": [{"files": [
|
||||
{"filename": "f1.flac", "size": 5, "state": state, "id": "t1"},
|
||||
]}]}
|
||||
|
||||
def post(self, url, **kw):
|
||||
return _Resp({})
|
||||
|
||||
def get(self, url, **kw):
|
||||
return _Resp(self._poll)
|
||||
p = self._polls[min(self._i, len(self._polls) - 1)]
|
||||
self._i += 1
|
||||
return _Resp(p)
|
||||
|
||||
def delete(self, url, **kw):
|
||||
self.deleted.append(url)
|
||||
return _Resp({})
|
||||
|
||||
|
||||
def _poll(state, bytes_transferred, fid="t1"):
|
||||
return {"directories": [{"files": [
|
||||
{"filename": "f1.flac", "size": 100, "state": state,
|
||||
"bytesTransferred": bytes_transferred, "id": fid},
|
||||
]}]}
|
||||
|
||||
|
||||
def _client(tmp_path):
|
||||
return SlskdClient({"slskd.url": "http://x", "slskd.api_key": "k",
|
||||
"slskd.downloadsDir": str(tmp_path)})
|
||||
|
||||
|
||||
_REF = json.dumps({"username": "peerA", "files": [{"filename": "f1.flac", "size": 5}]})
|
||||
_REF = json.dumps({"username": "peerA", "files": [{"filename": "f1.flac", "size": 100}]})
|
||||
|
||||
|
||||
def test_cancels_transfer_on_timeout(tmp_path, monkeypatch):
|
||||
fake = _FakeRequests(state="InProgress")
|
||||
monkeypatch.setattr(slskd_mod, "requests", fake)
|
||||
def test_stalled_peer_is_abandoned_and_cancelled(tmp_path, monkeypatch):
|
||||
# No byte progress across polls → stall → TimeoutError → the transfer is cancelled.
|
||||
monkeypatch.setattr(slskd_mod, "requests", _SeqRequests([_poll("InProgress", 0)]))
|
||||
monkeypatch.setattr(slskd_mod.time, "sleep", lambda _s: None)
|
||||
monkeypatch.setattr(slskd_mod, "_XFER_POLLS", 2)
|
||||
monkeypatch.setattr(slskd_mod, "_STALL_POLLS", 3)
|
||||
fake = slskd_mod.requests
|
||||
|
||||
with pytest.raises(TimeoutError):
|
||||
_client(tmp_path).download(_REF, str(tmp_path / "dest"), lambda _p: None)
|
||||
|
||||
assert any(u.endswith("/api/v0/transfers/downloads/peerA/t1") for u in fake.deleted), fake.deleted
|
||||
assert any(u.endswith("/api/v0/transfers/downloads/peerA/t1") for u in fake.deleted)
|
||||
|
||||
|
||||
def test_cancels_transfer_on_error_state(tmp_path, monkeypatch):
|
||||
fake = _FakeRequests(state="Completed, Errored")
|
||||
monkeypatch.setattr(slskd_mod, "requests", fake)
|
||||
def test_error_state_is_cancelled(tmp_path, monkeypatch):
|
||||
monkeypatch.setattr(slskd_mod, "requests", _SeqRequests([_poll("Completed, Errored", 0)]))
|
||||
monkeypatch.setattr(slskd_mod.time, "sleep", lambda _s: None)
|
||||
monkeypatch.setattr(slskd_mod, "_XFER_POLLS", 5)
|
||||
|
||||
fake = slskd_mod.requests
|
||||
with pytest.raises(RuntimeError):
|
||||
_client(tmp_path).download(_REF, str(tmp_path / "dest"), lambda _p: None)
|
||||
assert any(u.endswith("/peerA/t1") for u in fake.deleted)
|
||||
|
||||
assert any(u.endswith("/peerA/t1") for u in fake.deleted), fake.deleted
|
||||
|
||||
def test_slow_but_progressing_peer_is_not_abandoned(tmp_path, monkeypatch):
|
||||
# Bytes climb for more polls than _STALL_POLLS, then complete → success, not a premature stall.
|
||||
(tmp_path / "01.flac").write_bytes(b"audio") # so _retrieve finds the finished file
|
||||
ref = json.dumps({"username": "peerA", "files": [{"filename": r"d\Album\01.flac", "size": 5}]})
|
||||
polls = [
|
||||
{"directories": [{"files": [{"filename": r"d\Album\01.flac", "size": 5,
|
||||
"state": "InProgress", "bytesTransferred": b, "id": "t1"}]}]}
|
||||
for b in (10, 20, 30, 40, 50)
|
||||
] + [
|
||||
{"directories": [{"files": [{"filename": r"d\Album\01.flac", "size": 5,
|
||||
"state": "Completed, Succeeded", "bytesTransferred": 5, "id": "t1"}]}]}
|
||||
]
|
||||
monkeypatch.setattr(slskd_mod, "requests", _SeqRequests(polls))
|
||||
monkeypatch.setattr(slskd_mod.time, "sleep", lambda _s: None)
|
||||
monkeypatch.setattr(slskd_mod, "_STALL_POLLS", 2) # would fire early if progress were ignored
|
||||
|
||||
out = _client(tmp_path).download(ref, str(tmp_path / "dest"), lambda _p: None)
|
||||
assert out["track_count"] == 1
|
||||
assert slskd_mod.requests.deleted == [] # completed cleanly → nothing cancelled
|
||||
|
||||
|
||||
def test_parse_ranks_free_and_fast_peers_first():
|
||||
responses = [
|
||||
{"username": "slow", "hasFreeUploadSlot": True, "uploadSpeed": 100, "queueLength": 0,
|
||||
"files": [{"filename": r"a\Artist - Album\01.flac", "size": 5}]},
|
||||
{"username": "fast", "hasFreeUploadSlot": True, "uploadSpeed": 9000, "queueLength": 0,
|
||||
"files": [{"filename": r"b\Artist - Album\01.flac", "size": 5}]},
|
||||
{"username": "queued", "hasFreeUploadSlot": False, "uploadSpeed": 99999, "queueLength": 40,
|
||||
"files": [{"filename": r"c\Artist - Album\01.flac", "size": 5}]},
|
||||
]
|
||||
peers = [json.loads(c["source_ref"])["username"] for c in _parse_search_responses(responses)]
|
||||
assert peers == ["fast", "slow", "queued"] # free+fast first; no-slot peer last despite speed
|
||||
|
||||
Reference in New Issue
Block a user