fix(worker): smarter Soulseek peer selection + stall/progress-aware transfers
Draining the queue via Soulseek was stalling: slow peers (~100KB/s) never finished a full FLAC album within the fixed 10-min timeout, got cancelled at ~89%, and the fall-through restarted from scratch on the next peer — while single-song folders named like the album were also being tried. - Stall detection: abandon a peer with no byte progress for ~90s (queued/dead) instead of waiting the full timeout; a peer that IS progressing keeps its slot up to a 60-min backstop, so a slow-but-working transfer can finish. - Rank search candidates by peer: free upload slot, then speed, then shortest queue. - Drop candidates with far fewer tracks than the release (single-song folders) up front, so no download attempt is wasted on them. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -10,8 +10,9 @@ _DEFAULT_DOWNLOADS_ROOT = "/slskd-downloads"
|
||||
|
||||
_AUDIO_EXT = {"flac", "mp3", "m4a", "ogg", "opus", "wav", "aac"}
|
||||
_LOSSLESS_EXT = {"flac", "wav"}
|
||||
_SEARCH_POLLS = 40 # * 3s ≈ 2 min
|
||||
_XFER_POLLS = 200 # * 3s ≈ 10 min
|
||||
_SEARCH_POLLS = 40 # * 3s ≈ 2 min
|
||||
_MAX_XFER_POLLS = 1200 # * 3s ≈ 60 min absolute backstop for a slow-but-progressing peer
|
||||
_STALL_POLLS = 30 # * 3s ≈ 90s of no byte progress → abandon a queued/dead peer and fall through
|
||||
_POLL_SECONDS = 3
|
||||
|
||||
|
||||
@@ -27,6 +28,52 @@ def _dirname(path: str) -> str:
|
||||
return ""
|
||||
|
||||
|
||||
def _parse_search_responses(responses: list) -> list[dict]:
|
||||
"""Turn slskd search responses into album candidates (one per peer+directory), ordered so the
|
||||
peers most likely to deliver quickly come first: a free upload slot, then higher upload speed,
|
||||
then a shorter queue. The pipeline tries candidates in this order, so ranking fast/free peers
|
||||
first (over slow or queued ones) is what makes the Soulseek fall-through actually converge.
|
||||
Pure — no I/O — so it is unit-tested offline."""
|
||||
scored: list[tuple] = []
|
||||
for resp in responses:
|
||||
username = resp.get("username", "")
|
||||
has_slot = 1 if resp.get("hasFreeUploadSlot") else 0
|
||||
speed = int(resp.get("uploadSpeed") or 0)
|
||||
queue = int(resp.get("queueLength") or 0)
|
||||
by_dir: dict[str, list] = {}
|
||||
for f in resp.get("files", []) or []:
|
||||
name = f.get("filename", "")
|
||||
ext = name.rsplit(".", 1)[-1].lower() if "." in name else ""
|
||||
if ext not in _AUDIO_EXT:
|
||||
continue
|
||||
by_dir.setdefault(_dirname(name), []).append(
|
||||
{"filename": name, "size": f.get("size", 0), "ext": ext}
|
||||
)
|
||||
for directory, dfiles in by_dir.items():
|
||||
if not dfiles:
|
||||
continue
|
||||
lossless = all(f["ext"] in _LOSSLESS_EXT for f in dfiles)
|
||||
dirname = _basename(directory)
|
||||
if " - " in dirname:
|
||||
guess_artist, guess_album = (p.strip() for p in dirname.split(" - ", 1))
|
||||
else:
|
||||
guess_artist, guess_album = "", dirname
|
||||
candidate = {
|
||||
"source_ref": json.dumps({
|
||||
"username": username,
|
||||
"files": [{"filename": f["filename"], "size": f["size"]} for f in dfiles],
|
||||
}),
|
||||
"title": guess_album,
|
||||
"artist": guess_artist,
|
||||
"track_count": len(dfiles),
|
||||
"format": "FLAC" if lossless else "MP3",
|
||||
"bitrate": None,
|
||||
}
|
||||
scored.append((has_slot, speed, -queue, candidate))
|
||||
scored.sort(key=lambda t: (t[0], t[1], t[2]), reverse=True) # free + fast + short-queue first
|
||||
return [c for *_rest, c in scored]
|
||||
|
||||
|
||||
class SlskdClient:
|
||||
"""Real Soulseek client talking to an slskd daemon's REST API.
|
||||
|
||||
@@ -75,43 +122,7 @@ class SlskdClient:
|
||||
break
|
||||
|
||||
responses = self._get(f"/api/v0/searches/{search_id}/responses")
|
||||
candidates: list[dict] = []
|
||||
for resp in responses:
|
||||
username = resp.get("username", "")
|
||||
by_dir: dict[str, list] = {}
|
||||
for f in resp.get("files", []) or []:
|
||||
name = f.get("filename", "")
|
||||
ext = name.rsplit(".", 1)[-1].lower() if "." in name else ""
|
||||
if ext not in _AUDIO_EXT:
|
||||
continue
|
||||
by_dir.setdefault(_dirname(name), []).append(
|
||||
{"filename": name, "size": f.get("size", 0), "ext": ext}
|
||||
)
|
||||
for directory, dfiles in by_dir.items():
|
||||
if not dfiles:
|
||||
continue
|
||||
lossless = all(f["ext"] in _LOSSLESS_EXT for f in dfiles)
|
||||
dirname = _basename(directory)
|
||||
if " - " in dirname:
|
||||
guess_artist, guess_album = (p.strip() for p in dirname.split(" - ", 1))
|
||||
else:
|
||||
guess_artist, guess_album = "", dirname
|
||||
candidates.append(
|
||||
{
|
||||
"source_ref": json.dumps(
|
||||
{
|
||||
"username": username,
|
||||
"files": [{"filename": f["filename"], "size": f["size"]} for f in dfiles],
|
||||
}
|
||||
),
|
||||
"title": guess_album,
|
||||
"artist": guess_artist,
|
||||
"track_count": len(dfiles),
|
||||
"format": "FLAC" if lossless else "MP3",
|
||||
"bitrate": None,
|
||||
}
|
||||
)
|
||||
return candidates
|
||||
return _parse_search_responses(responses if isinstance(responses, list) else [])
|
||||
|
||||
def download(self, source_ref: str, dest: str, on_progress: Callable[[float], None]) -> dict:
|
||||
ref = json.loads(source_ref)
|
||||
@@ -141,14 +152,18 @@ class SlskdClient:
|
||||
|
||||
total = len(files)
|
||||
completed = False
|
||||
for _ in range(_XFER_POLLS):
|
||||
last_bytes = -1
|
||||
stall = 0
|
||||
for _ in range(_MAX_XFER_POLLS):
|
||||
time.sleep(_POLL_SECONDS)
|
||||
data = self._get(f"/api/v0/transfers/downloads/{username}")
|
||||
states: list[str] = []
|
||||
bytes_now = 0
|
||||
for directory in data.get("directories", []) if isinstance(data, dict) else []:
|
||||
for f in directory.get("files", []):
|
||||
if f.get("filename") in wanted: # only the files we enqueued
|
||||
states.append(str(f.get("state", "")))
|
||||
bytes_now += int(f.get("bytesTransferred") or 0)
|
||||
if f.get("id"): # remember the transfer so we can cancel it if abandoned
|
||||
xfer_ids.add(str(f["id"]))
|
||||
if states:
|
||||
@@ -159,6 +174,16 @@ class SlskdClient:
|
||||
if done >= total:
|
||||
completed = True
|
||||
break
|
||||
# Stall detection: a peer making byte progress keeps its slot (up to the absolute
|
||||
# backstop above) so a slow-but-working transfer can finish; one that sends no bytes
|
||||
# for _STALL_POLLS (~90s) is queued/dead — abandon it fast and fall through.
|
||||
if bytes_now > last_bytes:
|
||||
last_bytes = bytes_now
|
||||
stall = 0
|
||||
else:
|
||||
stall += 1
|
||||
if stall >= _STALL_POLLS:
|
||||
raise TimeoutError(f"soulseek transfer stalled for {username}")
|
||||
|
||||
if not completed:
|
||||
raise TimeoutError(f"soulseek download did not complete for {username}")
|
||||
|
||||
Reference in New Issue
Block a user