Jonathan
f74fd5ab88
SlskdClient enqueued the transfer and polled to Completed but never moved any
files into `dest` — slskd saves them to its OWN downloads dir, so staging stayed
empty and every Soulseek job failed "incomplete download". Qobuz/YouTube write
straight into staging and were unaffected.
- SlskdClient._retrieve() moves the finished files out of slskd's downloads dir
into staging after completion, matching tolerantly by basename and preferring
a parent-folder + size match (slskd's on-disk layout varies by version).
- download() returns the moved count as track_count so the pipeline's
completeness check reflects what actually landed; raises loudly if nothing was
found (misconfigured/unmounted downloads dir).
- Downloads dir resolves slskd.downloadsDir Config > SLSKD_DOWNLOADS_ROOT env >
/slskd-downloads default; docker-compose mounts ${SLSKD_DOWNLOADS_DIR} there.
- .env.example documents SLSKD_DOWNLOADS_DIR; retrieval unit-tested offline.
slskd runs as an external daemon, so the shared dir must be a host path both
slskd and the worker mount — set once Lyra runs on the slskd host. Not yet
verified live end-to-end (no slskd access from this VM).
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
215 lines
8.4 KiB
Python
215 lines
8.4 KiB
Python
import json
|
|
import os
|
|
import shutil
|
|
import time
|
|
from typing import Callable
|
|
|
|
import requests
|
|
|
|
_DEFAULT_DOWNLOADS_ROOT = "/slskd-downloads"
|
|
|
|
_AUDIO_EXT = {"flac", "mp3", "m4a", "ogg", "opus", "wav", "aac"}
|
|
_LOSSLESS_EXT = {"flac", "wav"}
|
|
_SEARCH_POLLS = 40 # * 3s ≈ 2 min
|
|
_XFER_POLLS = 200 # * 3s ≈ 10 min
|
|
_POLL_SECONDS = 3
|
|
|
|
|
|
def _basename(path: str) -> str:
|
|
return path.rsplit("\\", 1)[-1].rsplit("/", 1)[-1]
|
|
|
|
|
|
def _dirname(path: str) -> str:
|
|
if "\\" in path:
|
|
return path.rsplit("\\", 1)[0]
|
|
if "/" in path:
|
|
return path.rsplit("/", 1)[0]
|
|
return ""
|
|
|
|
|
|
class SlskdClient:
|
|
"""Real Soulseek client talking to an slskd daemon's REST API.
|
|
|
|
NOT unit-tested offline; see test_soulseek_live.py. slskd response/transfer
|
|
JSON shapes may vary by version — validate with the live test.
|
|
"""
|
|
|
|
def __init__(self, config: dict):
|
|
self._url = (config.get("slskd.url", "") or "").rstrip("/")
|
|
self._key = config.get("slskd.api_key", "")
|
|
# Where slskd (an external daemon) writes finished downloads. slskd saves into ITS
|
|
# OWN directory, so this path must be a share/mount both slskd and this worker see.
|
|
# Precedence: the slskd.downloadsDir Config row (Settings-overridable) > the
|
|
# SLSKD_DOWNLOADS_ROOT env (the container mount point) > a sensible default.
|
|
self._downloads_dir = (
|
|
config.get("slskd.downloadsDir")
|
|
or os.environ.get("SLSKD_DOWNLOADS_ROOT")
|
|
or _DEFAULT_DOWNLOADS_ROOT
|
|
)
|
|
|
|
def is_configured(self) -> bool:
|
|
return bool(self._url and self._key)
|
|
|
|
def _headers(self) -> dict:
|
|
return {"X-API-Key": self._key, "Content-Type": "application/json"}
|
|
|
|
def _get(self, path: str) -> dict | list:
|
|
r = requests.get(f"{self._url}{path}", headers=self._headers(), timeout=30)
|
|
r.raise_for_status()
|
|
return r.json()
|
|
|
|
def search_album(self, artist: str, album: str) -> list[dict]:
|
|
r = requests.post(
|
|
f"{self._url}/api/v0/searches",
|
|
headers=self._headers(),
|
|
json={"searchText": f"{artist} {album}"},
|
|
timeout=30,
|
|
)
|
|
r.raise_for_status()
|
|
search_id = r.json()["id"]
|
|
|
|
for _ in range(_SEARCH_POLLS):
|
|
time.sleep(_POLL_SECONDS)
|
|
state = self._get(f"/api/v0/searches/{search_id}")
|
|
if state.get("isComplete") or "Completed" in str(state.get("state", "")):
|
|
break
|
|
|
|
responses = self._get(f"/api/v0/searches/{search_id}/responses")
|
|
candidates: list[dict] = []
|
|
for resp in responses:
|
|
username = resp.get("username", "")
|
|
by_dir: dict[str, list] = {}
|
|
for f in resp.get("files", []) or []:
|
|
name = f.get("filename", "")
|
|
ext = name.rsplit(".", 1)[-1].lower() if "." in name else ""
|
|
if ext not in _AUDIO_EXT:
|
|
continue
|
|
by_dir.setdefault(_dirname(name), []).append(
|
|
{"filename": name, "size": f.get("size", 0), "ext": ext}
|
|
)
|
|
for directory, dfiles in by_dir.items():
|
|
if not dfiles:
|
|
continue
|
|
lossless = all(f["ext"] in _LOSSLESS_EXT for f in dfiles)
|
|
dirname = _basename(directory)
|
|
if " - " in dirname:
|
|
guess_artist, guess_album = (p.strip() for p in dirname.split(" - ", 1))
|
|
else:
|
|
guess_artist, guess_album = "", dirname
|
|
candidates.append(
|
|
{
|
|
"source_ref": json.dumps(
|
|
{
|
|
"username": username,
|
|
"files": [{"filename": f["filename"], "size": f["size"]} for f in dfiles],
|
|
}
|
|
),
|
|
"title": guess_album,
|
|
"artist": guess_artist,
|
|
"track_count": len(dfiles),
|
|
"format": "FLAC" if lossless else "MP3",
|
|
"bitrate": None,
|
|
}
|
|
)
|
|
return candidates
|
|
|
|
def download(self, source_ref: str, dest: str, on_progress: Callable[[float], None]) -> dict:
|
|
ref = json.loads(source_ref)
|
|
username = ref["username"]
|
|
files = ref["files"]
|
|
wanted = {f["filename"] for f in files}
|
|
os.makedirs(dest, exist_ok=True)
|
|
|
|
r = requests.post(
|
|
f"{self._url}/api/v0/transfers/downloads/{username}",
|
|
headers=self._headers(),
|
|
json=[{"filename": f["filename"], "size": f["size"]} for f in files],
|
|
timeout=30,
|
|
)
|
|
r.raise_for_status()
|
|
on_progress(0.0)
|
|
|
|
total = len(files)
|
|
completed = False
|
|
for _ in range(_XFER_POLLS):
|
|
time.sleep(_POLL_SECONDS)
|
|
data = self._get(f"/api/v0/transfers/downloads/{username}")
|
|
states: list[str] = []
|
|
for directory in data.get("directories", []) if isinstance(data, dict) else []:
|
|
for f in directory.get("files", []):
|
|
if f.get("filename") in wanted: # only the files we enqueued
|
|
states.append(str(f.get("state", "")))
|
|
if states:
|
|
if any(bad in s for s in states for bad in ("Errored", "Rejected", "Cancelled")):
|
|
raise RuntimeError("soulseek transfer failed")
|
|
done = sum(1 for s in states if "Completed" in s)
|
|
on_progress(min(1.0, done / max(total, 1)))
|
|
if done >= total:
|
|
completed = True
|
|
break
|
|
|
|
if not completed:
|
|
raise TimeoutError(f"soulseek download did not complete for {username}")
|
|
|
|
# slskd saved the files into its OWN downloads dir — retrieve them into `dest` so the
|
|
# pipeline (which verifies + imports against `dest`/staging) actually sees the audio.
|
|
moved = self._retrieve(files, dest)
|
|
if moved == 0:
|
|
raise RuntimeError(
|
|
f"soulseek transfer completed but no files were found under "
|
|
f"{self._downloads_dir!r} to move into staging — check that the slskd "
|
|
f"downloads dir is shared/mounted (SLSKD_DOWNLOADS_ROOT / slskd.downloadsDir)"
|
|
)
|
|
return {"track_count": moved, "path": dest}
|
|
|
|
def _retrieve(self, files: list[dict], dest: str) -> int:
|
|
"""Move the just-finished slskd files out of its downloads dir into `dest`.
|
|
|
|
slskd's on-disk layout can vary by version/config, so match tolerantly: for each
|
|
wanted file, find on-disk candidates by basename anywhere under the downloads root,
|
|
then prefer the one whose parent-folder name matches the remote directory and whose
|
|
size matches. Returns the number of files actually moved. Offline-testable.
|
|
"""
|
|
root = self._downloads_dir
|
|
if not root or not os.path.isdir(root):
|
|
raise RuntimeError(
|
|
f"slskd downloads dir not found: {root!r} — mount slskd's downloads "
|
|
f"directory into the worker (SLSKD_DOWNLOADS_ROOT / slskd.downloadsDir)"
|
|
)
|
|
os.makedirs(dest, exist_ok=True)
|
|
|
|
wanted: dict[str, dict] = {} # basename(lower) -> {parent, size}
|
|
for f in files:
|
|
name = f.get("filename", "")
|
|
wanted[_basename(name).lower()] = {
|
|
"parent": _basename(_dirname(name)).lower(),
|
|
"size": f.get("size"),
|
|
}
|
|
|
|
found: dict[str, list[str]] = {b: [] for b in wanted}
|
|
for cur_root, _dirs, fnames in os.walk(root):
|
|
for fn in fnames:
|
|
key = fn.lower()
|
|
if key in found:
|
|
found[key].append(os.path.join(cur_root, fn))
|
|
|
|
moved = 0
|
|
for basename, paths in found.items():
|
|
if not paths:
|
|
continue
|
|
want = wanted[basename]
|
|
|
|
def _score(p: str) -> tuple[int, int]:
|
|
parent_ok = int(os.path.basename(os.path.dirname(p)).lower() == want["parent"])
|
|
size_ok = 0
|
|
try:
|
|
size_ok = int(bool(want["size"]) and os.path.getsize(p) == want["size"])
|
|
except OSError:
|
|
pass
|
|
return (parent_ok, size_ok)
|
|
|
|
best = max(paths, key=_score)
|
|
shutil.move(best, os.path.join(dest, os.path.basename(best)))
|
|
moved += 1
|
|
return moved
|