fix(worker): parallelize per-source searches in the match phase

Soulseek's slskd search polls up to ~2 min; searching adapters sequentially
added that on top of Qobuz's instant search. Run the per-adapter searches
concurrently via a ThreadPoolExecutor so total match time is the slowest
single source, not their sum. Only Qobuz drives streamrip's shared asyncio
loop and there's one Qobuz adapter, so no two threads use that loop at once.
Results collected in adapter order (map preserves order) for deterministic
candidate ordering; per-adapter failures still swallowed (down source = no
candidates).

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Jonathan
2026-07-15 02:37:08 +02:00
parent 0325467aba
commit 0c5e73eb17
2 changed files with 46 additions and 7 deletions
+19 -7
View File
@@ -2,6 +2,7 @@ import os
import shutil
import threading
import time
from concurrent.futures import ThreadPoolExecutor
from dataclasses import replace
from typing import Callable, Sequence
@@ -94,6 +95,16 @@ def _poll_download_progress(job_id: str, staging: str, expected: int, stop: "thr
conn.close()
def _search_adapter(adapter: SourceAdapter, target: MBTarget) -> list[Candidate]:
"""Search one adapter, swallowing failures — a down source contributes no candidates rather
than crashing the job. Runs on a worker thread (searches are parallelized across sources)."""
try:
return list(adapter.search(target))
except Exception as e:
print(f"pipeline: adapter {adapter.name} search failed: {e}", flush=True)
return []
def _set_state(conn: psycopg.Connection, job_id: str, state: str, stage: str) -> None:
with conn.cursor() as cur:
cur.execute(
@@ -306,15 +317,16 @@ def run_pipeline(
conn.commit()
return
# 2. match — search all adapters; score + persist EVERY candidate found
# 2. match — search all adapters CONCURRENTLY (Soulseek's search polls up to ~2 min; run it
# alongside Qobuz's instant search so total match time = the slowest source, not their sum).
# Only Qobuz touches streamrip's shared asyncio loop and there's a single Qobuz adapter, so no
# two threads drive that loop at once. Results are collected in adapter order (map preserves
# input order) so candidate ordering stays deterministic.
_set_state(conn, job_id, "matching", "match")
found: list[Candidate] = []
for adapter in adapters:
try:
results = adapter.search(target)
except Exception as e: # a down source contributes no candidates, never crashes the job
print(f"pipeline: adapter {adapter.name} search failed: {e}", flush=True)
continue
with ThreadPoolExecutor(max_workers=max(1, len(adapters))) as pool:
per_adapter = list(pool.map(lambda a: (a, _search_adapter(a, target)), adapters))
for adapter, results in per_adapter:
for c in results:
found.append(replace(c, source_tier=adapter.tier, confidence=score_confidence(target, c)))
_persist_candidates(conn, job_id, found)
+27
View File
@@ -195,6 +195,33 @@ def test_below_threshold_candidates_are_persisted_for_diagnosis(conn):
assert cur.fetchone()[0] >= 1 # found-but-rejected candidate retained for diagnosis
def test_searches_run_concurrently_across_adapters(conn):
# Two sources that each take ~0.5s to search must run in parallel: total match time is ~the
# slowest single search (~0.5s), not the sum (~1.0s). Also verifies concurrency drops no
# candidates — both sources' contributions are persisted.
import time as _t
class _SlowQobuz(FakeQobuz):
def search(self, target):
_t.sleep(0.5)
return super().search(target)
class _SlowSoulseek(FakeSoulseek):
def search(self, target):
_t.sleep(0.5)
return super().search(target)
job_id = insert_request(conn, artist="Radiohead", album="In Rainbows")
claim_next(conn)
start = _t.monotonic()
run_pipeline(conn, job_id, [_SlowQobuz(), _SlowSoulseek()], dest_root="/tmp/lib")
elapsed = _t.monotonic() - start
assert elapsed < 0.9 # concurrent (~0.5s), not sequential (~1.0s)
with conn.cursor() as cur:
cur.execute('SELECT count(DISTINCT source) FROM "Candidate" WHERE "jobId" = %s', (job_id,))
assert cur.fetchone()[0] == 2 # both adapters contributed candidates
def test_duplicate_adapter_names_raise(conn):
import pytest
from lyra_worker.adapters.fakes import FakeQobuz