diff --git a/.env.example b/.env.example index 84f83cd..cc15017 100644 --- a/.env.example +++ b/.env.example @@ -14,3 +14,10 @@ MUSIC_DIR=./music # share (SMB/NFS) so temporary downloads don't hammer the share; the final import copies the # finished album onto the library volume and swaps it in atomically. # STAGING_DIR=/srv/lyra-staging + +# Host directory where slskd (the external Soulseek daemon) writes FINISHED downloads, +# bind-mounted into the worker at /slskd-downloads. The worker moves completed Soulseek +# transfers out of here into staging, so this MUST be the same directory slskd downloads to. +# Set this only once Lyra runs on the same host as slskd; until then Soulseek can search but +# not deliver. Defaults to ./slskd-downloads (an empty local dir) so the stack still starts. +# SLSKD_DOWNLOADS_DIR=/var/lib/slskd/downloads diff --git a/.gitignore b/.gitignore index de411da..2127310 100644 --- a/.gitignore +++ b/.gitignore @@ -40,3 +40,4 @@ Thumbs.db .idea/ .vscode/ *.swp +slskd-downloads/ diff --git a/docker-compose.yml b/docker-compose.yml index fe9b2fb..f8544ba 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -36,6 +36,7 @@ services: DATABASE_URL: ${DATABASE_URL} LYRA_SECRET_KEY: ${LYRA_SECRET_KEY} STAGING_ROOT: /staging + SLSKD_DOWNLOADS_ROOT: /slskd-downloads # Qobuz's CDN also resolves to IPv6, but the container has no IPv6 route # ("Network is unreachable"); disable IPv6 so downloads use IPv4 only. sysctls: @@ -45,6 +46,13 @@ services: # Per-job download staging on its own volume. Defaults to MUSIC_DIR/.staging (prior # behavior); set STAGING_DIR to a fast local path when MUSIC_DIR is a network share. - ${STAGING_DIR:-${MUSIC_DIR:-./music}/.staging}:/staging + # slskd (an EXTERNAL daemon, not in this compose stack) writes finished Soulseek + # downloads into its own directory; the worker moves them out of here into staging. + # Point SLSKD_DOWNLOADS_DIR at slskd's downloads dir on the host (must be the SAME + # path slskd writes to — set it once Lyra runs on the slskd host). Defaults to an + # empty local dir so the mount is always valid; Soulseek delivery only works once + # this points at the real slskd downloads directory. + - ${SLSKD_DOWNLOADS_DIR:-./slskd-downloads}:/slskd-downloads depends_on: db: condition: service_healthy diff --git a/worker/lyra_worker/adapters/_slskd.py b/worker/lyra_worker/adapters/_slskd.py index 8541755..c668424 100644 --- a/worker/lyra_worker/adapters/_slskd.py +++ b/worker/lyra_worker/adapters/_slskd.py @@ -1,10 +1,13 @@ import json import os +import shutil import time from typing import Callable import requests +_DEFAULT_DOWNLOADS_ROOT = "/slskd-downloads" + _AUDIO_EXT = {"flac", "mp3", "m4a", "ogg", "opus", "wav", "aac"} _LOSSLESS_EXT = {"flac", "wav"} _SEARCH_POLLS = 40 # * 3s ≈ 2 min @@ -34,6 +37,15 @@ class SlskdClient: def __init__(self, config: dict): self._url = (config.get("slskd.url", "") or "").rstrip("/") self._key = config.get("slskd.api_key", "") + # Where slskd (an external daemon) writes finished downloads. slskd saves into ITS + # OWN directory, so this path must be a share/mount both slskd and this worker see. + # Precedence: the slskd.downloadsDir Config row (Settings-overridable) > the + # SLSKD_DOWNLOADS_ROOT env (the container mount point) > a sensible default. + self._downloads_dir = ( + config.get("slskd.downloadsDir") + or os.environ.get("SLSKD_DOWNLOADS_ROOT") + or _DEFAULT_DOWNLOADS_ROOT + ) def is_configured(self) -> bool: return bool(self._url and self._key) @@ -138,4 +150,65 @@ class SlskdClient: if not completed: raise TimeoutError(f"soulseek download did not complete for {username}") - return {"track_count": total, "path": dest} + + # slskd saved the files into its OWN downloads dir — retrieve them into `dest` so the + # pipeline (which verifies + imports against `dest`/staging) actually sees the audio. + moved = self._retrieve(files, dest) + if moved == 0: + raise RuntimeError( + f"soulseek transfer completed but no files were found under " + f"{self._downloads_dir!r} to move into staging — check that the slskd " + f"downloads dir is shared/mounted (SLSKD_DOWNLOADS_ROOT / slskd.downloadsDir)" + ) + return {"track_count": moved, "path": dest} + + def _retrieve(self, files: list[dict], dest: str) -> int: + """Move the just-finished slskd files out of its downloads dir into `dest`. + + slskd's on-disk layout can vary by version/config, so match tolerantly: for each + wanted file, find on-disk candidates by basename anywhere under the downloads root, + then prefer the one whose parent-folder name matches the remote directory and whose + size matches. Returns the number of files actually moved. Offline-testable. + """ + root = self._downloads_dir + if not root or not os.path.isdir(root): + raise RuntimeError( + f"slskd downloads dir not found: {root!r} — mount slskd's downloads " + f"directory into the worker (SLSKD_DOWNLOADS_ROOT / slskd.downloadsDir)" + ) + os.makedirs(dest, exist_ok=True) + + wanted: dict[str, dict] = {} # basename(lower) -> {parent, size} + for f in files: + name = f.get("filename", "") + wanted[_basename(name).lower()] = { + "parent": _basename(_dirname(name)).lower(), + "size": f.get("size"), + } + + found: dict[str, list[str]] = {b: [] for b in wanted} + for cur_root, _dirs, fnames in os.walk(root): + for fn in fnames: + key = fn.lower() + if key in found: + found[key].append(os.path.join(cur_root, fn)) + + moved = 0 + for basename, paths in found.items(): + if not paths: + continue + want = wanted[basename] + + def _score(p: str) -> tuple[int, int]: + parent_ok = int(os.path.basename(os.path.dirname(p)).lower() == want["parent"]) + size_ok = 0 + try: + size_ok = int(bool(want["size"]) and os.path.getsize(p) == want["size"]) + except OSError: + pass + return (parent_ok, size_ok) + + best = max(paths, key=_score) + shutil.move(best, os.path.join(dest, os.path.basename(best))) + moved += 1 + return moved diff --git a/worker/tests/test_slskd_retrieve.py b/worker/tests/test_slskd_retrieve.py new file mode 100644 index 0000000..0f4a890 --- /dev/null +++ b/worker/tests/test_slskd_retrieve.py @@ -0,0 +1,73 @@ +"""Offline tests for SlskdClient._retrieve — the step that moves slskd's finished +downloads out of its own dir into the pipeline's staging dir. The search/download REST +flow stays live-only (test_soulseek_live.py); retrieval is pure filesystem, so it is +unit-testable here.""" +import os + +from lyra_worker.adapters._slskd import SlskdClient + + +def _client(downloads_dir: str) -> SlskdClient: + return SlskdClient({"slskd.url": "http://x", "slskd.api_key": "k", + "slskd.downloadsDir": downloads_dir}) + + +def _write(path: str, content: bytes = b"audio") -> None: + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "wb") as fh: + fh.write(content) + + +def test_retrieve_moves_files_from_slskd_dir(tmp_path): + downloads = tmp_path / "slskd" + dest = tmp_path / "staging" + # slskd's typical layout: // + _write(str(downloads / "Artist - Album" / "01 One.flac")) + _write(str(downloads / "Artist - Album" / "02 Two.flac")) + files = [ + {"filename": r"@@abc\music\Artist - Album\01 One.flac", "size": 5}, + {"filename": r"@@abc\music\Artist - Album\02 Two.flac", "size": 5}, + ] + + moved = _client(str(downloads))._retrieve(files, str(dest)) + + assert moved == 2 + assert sorted(os.listdir(dest)) == ["01 One.flac", "02 Two.flac"] + # the source files were moved, not copied + assert os.listdir(downloads / "Artist - Album") == [] + + +def test_retrieve_disambiguates_by_parent_and_size(tmp_path): + downloads = tmp_path / "slskd" + dest = tmp_path / "staging" + # a decoy with the same basename in a different folder + wrong size + _write(str(downloads / "Other Album" / "01 One.flac"), b"wrongsize!!") + _write(str(downloads / "Artist - Album" / "01 One.flac"), b"right") + files = [{"filename": r"x\Artist - Album\01 One.flac", "size": len(b"right")}] + + moved = _client(str(downloads))._retrieve(files, str(dest)) + + assert moved == 1 + with open(dest / "01 One.flac", "rb") as fh: + assert fh.read() == b"right" # picked the parent+size match, not the decoy + + +def test_retrieve_returns_zero_when_nothing_matches(tmp_path): + downloads = tmp_path / "slskd" + downloads.mkdir() + dest = tmp_path / "staging" + files = [{"filename": r"x\Album\missing.flac", "size": 1}] + + assert _client(str(downloads))._retrieve(files, str(dest)) == 0 + + +def test_retrieve_raises_when_downloads_dir_absent(tmp_path): + dest = tmp_path / "staging" + client = _client(str(tmp_path / "does-not-exist")) + files = [{"filename": r"x\Album\a.flac", "size": 1}] + try: + client._retrieve(files, str(dest)) + except RuntimeError as e: + assert "downloads dir not found" in str(e) + else: + raise AssertionError("expected RuntimeError for a missing downloads dir")