diff --git a/worker/lyra_worker/pipeline.py b/worker/lyra_worker/pipeline.py index f54bd0e..5180246 100644 --- a/worker/lyra_worker/pipeline.py +++ b/worker/lyra_worker/pipeline.py @@ -2,6 +2,7 @@ import os import shutil import threading import time +from concurrent.futures import ThreadPoolExecutor from dataclasses import replace from typing import Callable, Sequence @@ -94,6 +95,16 @@ def _poll_download_progress(job_id: str, staging: str, expected: int, stop: "thr conn.close() +def _search_adapter(adapter: SourceAdapter, target: MBTarget) -> list[Candidate]: + """Search one adapter, swallowing failures — a down source contributes no candidates rather + than crashing the job. Runs on a worker thread (searches are parallelized across sources).""" + try: + return list(adapter.search(target)) + except Exception as e: + print(f"pipeline: adapter {adapter.name} search failed: {e}", flush=True) + return [] + + def _set_state(conn: psycopg.Connection, job_id: str, state: str, stage: str) -> None: with conn.cursor() as cur: cur.execute( @@ -306,15 +317,16 @@ def run_pipeline( conn.commit() return - # 2. match — search all adapters; score + persist EVERY candidate found + # 2. match — search all adapters CONCURRENTLY (Soulseek's search polls up to ~2 min; run it + # alongside Qobuz's instant search so total match time = the slowest source, not their sum). + # Only Qobuz touches streamrip's shared asyncio loop and there's a single Qobuz adapter, so no + # two threads drive that loop at once. Results are collected in adapter order (map preserves + # input order) so candidate ordering stays deterministic. _set_state(conn, job_id, "matching", "match") found: list[Candidate] = [] - for adapter in adapters: - try: - results = adapter.search(target) - except Exception as e: # a down source contributes no candidates, never crashes the job - print(f"pipeline: adapter {adapter.name} search failed: {e}", flush=True) - continue + with ThreadPoolExecutor(max_workers=max(1, len(adapters))) as pool: + per_adapter = list(pool.map(lambda a: (a, _search_adapter(a, target)), adapters)) + for adapter, results in per_adapter: for c in results: found.append(replace(c, source_tier=adapter.tier, confidence=score_confidence(target, c))) _persist_candidates(conn, job_id, found) diff --git a/worker/tests/test_pipeline.py b/worker/tests/test_pipeline.py index 107d3a9..f83be3d 100644 --- a/worker/tests/test_pipeline.py +++ b/worker/tests/test_pipeline.py @@ -195,6 +195,33 @@ def test_below_threshold_candidates_are_persisted_for_diagnosis(conn): assert cur.fetchone()[0] >= 1 # found-but-rejected candidate retained for diagnosis +def test_searches_run_concurrently_across_adapters(conn): + # Two sources that each take ~0.5s to search must run in parallel: total match time is ~the + # slowest single search (~0.5s), not the sum (~1.0s). Also verifies concurrency drops no + # candidates — both sources' contributions are persisted. + import time as _t + + class _SlowQobuz(FakeQobuz): + def search(self, target): + _t.sleep(0.5) + return super().search(target) + + class _SlowSoulseek(FakeSoulseek): + def search(self, target): + _t.sleep(0.5) + return super().search(target) + + job_id = insert_request(conn, artist="Radiohead", album="In Rainbows") + claim_next(conn) + start = _t.monotonic() + run_pipeline(conn, job_id, [_SlowQobuz(), _SlowSoulseek()], dest_root="/tmp/lib") + elapsed = _t.monotonic() - start + assert elapsed < 0.9 # concurrent (~0.5s), not sequential (~1.0s) + with conn.cursor() as cur: + cur.execute('SELECT count(DISTINCT source) FROM "Candidate" WHERE "jobId" = %s', (job_id,)) + assert cur.fetchone()[0] == 2 # both adapters contributed candidates + + def test_duplicate_adapter_names_raise(conn): import pytest from lyra_worker.adapters.fakes import FakeQobuz