fix(worker): cancel abandoned slskd transfers + cap download fall-through
Two compounding causes of 'many downloads of one album from different sources' in slskd: (1) the slskd client raised on timeout/error WITHOUT cancelling the transfer it enqueued, so as the pipeline fell through to other peers each left a transfer running; (2) the incomplete-download fall-through was uncapped, grinding through every candidate of a popular album (~200 Soulseek peers), amplified now that Qobuz pacing pushes most jobs to Soulseek. - SlskdClient.download now cancels its enqueued transfers (DELETE) on any abandon. - The download loop caps at _MAX_DOWNLOAD_ATTEMPTS (6) candidates. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -119,7 +119,17 @@ class SlskdClient:
|
||||
files = ref["files"]
|
||||
wanted = {f["filename"] for f in files}
|
||||
os.makedirs(dest, exist_ok=True)
|
||||
xfer_ids: set[str] = set() # slskd transfer ids we started, for cancel-on-abandon
|
||||
try:
|
||||
return self._download(username, files, wanted, dest, on_progress, xfer_ids)
|
||||
except Exception:
|
||||
# Abandoning this peer (timeout / error / the pipeline moving on): cancel the transfers
|
||||
# we enqueued so they don't keep running and pile up in slskd as the job falls through
|
||||
# to other peers/sources for the same album.
|
||||
self._cancel_transfers(username, xfer_ids)
|
||||
raise
|
||||
|
||||
def _download(self, username, files, wanted, dest, on_progress, xfer_ids: set) -> dict:
|
||||
r = requests.post(
|
||||
f"{self._url}/api/v0/transfers/downloads/{username}",
|
||||
headers=self._headers(),
|
||||
@@ -139,6 +149,8 @@ class SlskdClient:
|
||||
for f in directory.get("files", []):
|
||||
if f.get("filename") in wanted: # only the files we enqueued
|
||||
states.append(str(f.get("state", "")))
|
||||
if f.get("id"): # remember the transfer so we can cancel it if abandoned
|
||||
xfer_ids.add(str(f["id"]))
|
||||
if states:
|
||||
if any(bad in s for s in states for bad in ("Errored", "Rejected", "Cancelled")):
|
||||
raise RuntimeError("soulseek transfer failed")
|
||||
@@ -162,6 +174,19 @@ class SlskdClient:
|
||||
)
|
||||
return {"track_count": moved, "path": dest}
|
||||
|
||||
def _cancel_transfers(self, username: str, xfer_ids: set) -> None:
|
||||
"""Cancel the given in-flight slskd transfers for a peer. Best-effort: a cleanup failure
|
||||
must never mask the original download failure."""
|
||||
for tid in xfer_ids:
|
||||
try:
|
||||
requests.delete(
|
||||
f"{self._url}/api/v0/transfers/downloads/{username}/{tid}",
|
||||
headers=self._headers(),
|
||||
timeout=15,
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def _retrieve(self, files: list[dict], dest: str) -> int:
|
||||
"""Move the just-finished slskd files out of its downloads dir into `dest`.
|
||||
|
||||
|
||||
@@ -24,6 +24,11 @@ _AUDIO_EXT = {".flac", ".mp3", ".m4a", ".opus", ".ogg", ".aac", ".wav"}
|
||||
# track COUNT right. Generous, so edition/encoding differences don't false-positive; bonus
|
||||
# tracks (over-long) are always fine.
|
||||
_DURATION_MIN_RATIO = 0.85
|
||||
# Cap how many ranked candidates a single job will actually download+verify before giving up.
|
||||
# Without this, the incomplete-download fall-through can grind through dozens of sources (a popular
|
||||
# album returns ~200 Soulseek candidates), each a slow attempt that also leaves an abandoned slskd
|
||||
# transfer. The best candidates rank first, so 6 attempts is plenty.
|
||||
_MAX_DOWNLOAD_ATTEMPTS = 6
|
||||
|
||||
|
||||
def _measure_staged_duration_s(staging: str) -> float | None:
|
||||
@@ -397,10 +402,14 @@ def run_pipeline(
|
||||
_poller.start()
|
||||
try:
|
||||
try:
|
||||
attempts = 0
|
||||
for candidate in ranked:
|
||||
adapter = by_source.get(candidate.source)
|
||||
if adapter is None:
|
||||
continue
|
||||
if attempts >= _MAX_DOWNLOAD_ATTEMPTS:
|
||||
break # don't grind through every peer/source of a popular album
|
||||
attempts += 1
|
||||
_mark_chosen(conn, job_id, candidate.source_ref) # reflect the source now in flight
|
||||
shutil.rmtree(staging, ignore_errors=True) # clean slate per attempt
|
||||
_reports.clear() # this attempt hasn't reported yet → poller estimates until it does
|
||||
|
||||
Reference in New Issue
Block a user