feat(worker): recover blocklisted Soulseek searches via album/artist-only fallback
The Soulseek server silently drops searches containing certain blocklisted terms
(major-label artists / album titles — Adele, Rihanna, Beyoncé, "Lemonade", …),
returning 0 responses ("TimedOut"). The blocked term poisons the whole query even
though the OTHER field usually isn't blocked and the content is on the network
(e.g. searching "Beyoncé …" → 0, but "I Am… Sasha Fierce" alone → 250 with 136
real Beyoncé folders).
When the primary "{artist} {album}" search returns nothing, retry with each field
alone (normalized to fold accents/punctuation) and keep only results whose file
path still matches the dropped field (_keep_matching), so a same-titled album by a
different artist is never grabbed. Recovers blocklisted albums with a distinctive
title; safely yields nothing when the title is too generic (e.g. Adele "21") rather
than mis-grabbing. Fallback only fires on a 0-response primary, so the happy path is
unchanged.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -2,6 +2,7 @@ import json
|
|||||||
import os
|
import os
|
||||||
import shutil
|
import shutil
|
||||||
import time
|
import time
|
||||||
|
import unicodedata
|
||||||
from typing import Callable
|
from typing import Callable
|
||||||
|
|
||||||
import requests
|
import requests
|
||||||
@@ -28,6 +29,39 @@ def _dirname(path: str) -> str:
|
|||||||
return ""
|
return ""
|
||||||
|
|
||||||
|
|
||||||
|
def _norm(s: str) -> str:
|
||||||
|
"""Lowercase, strip diacritics, and reduce to alphanumeric tokens, so 'Beyoncé' matches a
|
||||||
|
peer's 'Beyonce' folder and punctuation ('I Am… Sasha Fierce') doesn't defeat the match."""
|
||||||
|
s = unicodedata.normalize("NFKD", s or "")
|
||||||
|
s = "".join(ch for ch in s if not unicodedata.combining(ch))
|
||||||
|
s = "".join(ch if ch.isalnum() else " " for ch in s.lower())
|
||||||
|
return " ".join(s.split())
|
||||||
|
|
||||||
|
|
||||||
|
def _path_matches(needle: str, path: str) -> bool:
|
||||||
|
"""True if `needle` (an artist or album name) appears in a file path, tolerant of accents and
|
||||||
|
punctuation. Every significant token (≥3 chars) must be present; for a very short needle
|
||||||
|
(e.g. 'U2') the whole normalized string must appear."""
|
||||||
|
npath = _norm(path)
|
||||||
|
tokens = [t for t in _norm(needle).split() if len(t) >= 3]
|
||||||
|
if not tokens:
|
||||||
|
whole = _norm(needle)
|
||||||
|
return bool(whole) and whole in npath
|
||||||
|
return all(t in npath for t in tokens)
|
||||||
|
|
||||||
|
|
||||||
|
def _keep_matching(responses: list, needle: str) -> list:
|
||||||
|
"""Drop files — then now-empty responses — whose path doesn't match `needle`. Makes an album-
|
||||||
|
or artist-only fallback search precise so we never grab a same-titled release by a different
|
||||||
|
artist. Pure — unit-tested offline."""
|
||||||
|
out = []
|
||||||
|
for resp in responses:
|
||||||
|
files = [f for f in (resp.get("files") or []) if _path_matches(needle, f.get("filename", ""))]
|
||||||
|
if files:
|
||||||
|
out.append({**resp, "files": files})
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
def _parse_search_responses(responses: list) -> list[dict]:
|
def _parse_search_responses(responses: list) -> list[dict]:
|
||||||
"""Turn slskd search responses into album candidates (one per peer+directory), ordered so the
|
"""Turn slskd search responses into album candidates (one per peer+directory), ordered so the
|
||||||
peers most likely to deliver quickly come first: a free upload slot, then higher upload speed,
|
peers most likely to deliver quickly come first: a free upload slot, then higher upload speed,
|
||||||
@@ -105,11 +139,12 @@ class SlskdClient:
|
|||||||
r.raise_for_status()
|
r.raise_for_status()
|
||||||
return r.json()
|
return r.json()
|
||||||
|
|
||||||
def search_album(self, artist: str, album: str) -> list[dict]:
|
def _run_search(self, text: str) -> list:
|
||||||
|
"""POST a search, wait for it to complete, and return the raw responses (a list)."""
|
||||||
r = requests.post(
|
r = requests.post(
|
||||||
f"{self._url}/api/v0/searches",
|
f"{self._url}/api/v0/searches",
|
||||||
headers=self._headers(),
|
headers=self._headers(),
|
||||||
json={"searchText": f"{artist} {album}"},
|
json={"searchText": text},
|
||||||
timeout=30,
|
timeout=30,
|
||||||
)
|
)
|
||||||
r.raise_for_status()
|
r.raise_for_status()
|
||||||
@@ -122,7 +157,27 @@ class SlskdClient:
|
|||||||
break
|
break
|
||||||
|
|
||||||
responses = self._get(f"/api/v0/searches/{search_id}/responses")
|
responses = self._get(f"/api/v0/searches/{search_id}/responses")
|
||||||
return _parse_search_responses(responses if isinstance(responses, list) else [])
|
return responses if isinstance(responses, list) else []
|
||||||
|
|
||||||
|
def search_album(self, artist: str, album: str) -> list[dict]:
|
||||||
|
responses = self._run_search(f"{artist} {album}")
|
||||||
|
if not responses:
|
||||||
|
# The Soulseek server silently drops searches containing certain blocklisted terms
|
||||||
|
# (major-label artists / album titles like Adele, Rihanna, Beyoncé, "Lemonade") →
|
||||||
|
# 0 responses. The blocked term poisons the whole query, but the OTHER field usually
|
||||||
|
# isn't blocked, so retry with each field alone (normalized) and keep only results
|
||||||
|
# whose path still matches the field we dropped. That recovers e.g. Beyoncé's
|
||||||
|
# "I Am… Sasha Fierce" via the album title, without ever grabbing a same-titled album
|
||||||
|
# by someone else.
|
||||||
|
for query, needle in ((album, artist), (artist, album)):
|
||||||
|
q = _norm(query)
|
||||||
|
if not q:
|
||||||
|
continue
|
||||||
|
alt = _keep_matching(self._run_search(q), needle)
|
||||||
|
if alt:
|
||||||
|
responses = alt
|
||||||
|
break
|
||||||
|
return _parse_search_responses(responses)
|
||||||
|
|
||||||
def download(self, source_ref: str, dest: str, on_progress: Callable[[float], None]) -> dict:
|
def download(self, source_ref: str, dest: str, on_progress: Callable[[float], None]) -> dict:
|
||||||
ref = json.loads(source_ref)
|
ref = json.loads(source_ref)
|
||||||
|
|||||||
@@ -0,0 +1,150 @@
|
|||||||
|
"""Soulseek search fallback: the Soulseek server drops searches containing blocklisted terms
|
||||||
|
(major-label artists / album titles) → 0 responses. search_album retries with album-only then
|
||||||
|
artist-only, keeping only path-matching results so it never grabs a same-titled foreign album."""
|
||||||
|
import json
|
||||||
|
|
||||||
|
import lyra_worker.adapters._slskd as slskd_mod
|
||||||
|
from lyra_worker.adapters._slskd import (
|
||||||
|
SlskdClient,
|
||||||
|
_keep_matching,
|
||||||
|
_norm,
|
||||||
|
_path_matches,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# --- pure helpers -----------------------------------------------------------------------------
|
||||||
|
|
||||||
|
def test_norm_strips_diacritics_and_punctuation():
|
||||||
|
assert _norm("Beyoncé") == "beyonce"
|
||||||
|
assert _norm("I Am… Sasha Fierce") == "i am sasha fierce"
|
||||||
|
assert _norm("AC/DC") == "ac dc"
|
||||||
|
|
||||||
|
|
||||||
|
def test_path_matches_is_accent_and_punctuation_tolerant():
|
||||||
|
p = r"@@bckfj\All\Beyonce\I Am... Sasha Fierce CD1\03 Halo.flac"
|
||||||
|
assert _path_matches("Beyoncé", p) # accent folded
|
||||||
|
assert _path_matches("I Am… Sasha Fierce", p) # ellipsis / punctuation ignored
|
||||||
|
assert not _path_matches("Rihanna", p)
|
||||||
|
|
||||||
|
|
||||||
|
def test_path_matches_short_needle_requires_whole_string():
|
||||||
|
assert _path_matches("U2", r"x\U2 - Achtung Baby\01.flac")
|
||||||
|
assert not _path_matches("U2", r"x\Blur - 13\01.flac")
|
||||||
|
|
||||||
|
|
||||||
|
def test_keep_matching_drops_foreign_and_empty_responses():
|
||||||
|
responses = [
|
||||||
|
{"username": "a", "files": [{"filename": r"x\Rihanna - Unapologetic\01.flac"}]},
|
||||||
|
{"username": "b", "files": [{"filename": r"x\Someone Else - Unapologetic\01.flac"}]},
|
||||||
|
]
|
||||||
|
kept = _keep_matching(responses, "Rihanna")
|
||||||
|
assert [r["username"] for r in kept] == ["a"]
|
||||||
|
|
||||||
|
|
||||||
|
# --- fallback orchestration (HTTP faked) ------------------------------------------------------
|
||||||
|
|
||||||
|
class _Resp:
|
||||||
|
def __init__(self, payload):
|
||||||
|
self._p = payload
|
||||||
|
|
||||||
|
def raise_for_status(self):
|
||||||
|
pass
|
||||||
|
|
||||||
|
def json(self):
|
||||||
|
return self._p
|
||||||
|
|
||||||
|
|
||||||
|
class _FakeSearch:
|
||||||
|
"""Routes slskd search HTTP by URL: POST returns an id, GET on the search returns a completed
|
||||||
|
state, GET on /responses returns the canned responses for that query. Records queries sent."""
|
||||||
|
def __init__(self, by_query):
|
||||||
|
self.by_query = by_query # {searchText: [responses]}
|
||||||
|
self.queries = [] # searchText values POSTed, in order
|
||||||
|
self._id_query = {}
|
||||||
|
self._n = 0
|
||||||
|
|
||||||
|
def post(self, url, **kw):
|
||||||
|
text = kw["json"]["searchText"]
|
||||||
|
self.queries.append(text)
|
||||||
|
self._n += 1
|
||||||
|
sid = f"s{self._n}"
|
||||||
|
self._id_query[sid] = text
|
||||||
|
return _Resp({"id": sid})
|
||||||
|
|
||||||
|
def get(self, url, **kw):
|
||||||
|
if url.endswith("/responses"):
|
||||||
|
sid = url.rsplit("/", 2)[-2]
|
||||||
|
return _Resp(self.by_query.get(self._id_query[sid], []))
|
||||||
|
return _Resp({"state": "Completed, TimedOut"})
|
||||||
|
|
||||||
|
|
||||||
|
def _client():
|
||||||
|
return SlskdClient({"slskd.url": "http://x", "slskd.api_key": "k", "slskd.downloadsDir": "/x"})
|
||||||
|
|
||||||
|
|
||||||
|
def _install(monkeypatch, fake):
|
||||||
|
monkeypatch.setattr(slskd_mod, "requests", fake)
|
||||||
|
monkeypatch.setattr(slskd_mod.time, "sleep", lambda _s: None)
|
||||||
|
|
||||||
|
|
||||||
|
def test_primary_hit_does_not_trigger_fallback(monkeypatch):
|
||||||
|
fake = _FakeSearch({
|
||||||
|
"Rihanna Unapologetic": [
|
||||||
|
{"username": "a", "hasFreeUploadSlot": True, "uploadSpeed": 1, "queueLength": 0,
|
||||||
|
"files": [{"filename": r"x\Rihanna - Unapologetic\01.flac", "size": 5}]},
|
||||||
|
],
|
||||||
|
})
|
||||||
|
_install(monkeypatch, fake)
|
||||||
|
out = _client().search_album("Rihanna", "Unapologetic")
|
||||||
|
assert len(out) == 1
|
||||||
|
assert fake.queries == ["Rihanna Unapologetic"] # no fallback searches
|
||||||
|
|
||||||
|
|
||||||
|
def test_album_only_fallback_recovers_blocklisted_artist(monkeypatch):
|
||||||
|
# Primary (artist blocked) → 0; album-only returns the artist's folder → recovered.
|
||||||
|
fake = _FakeSearch({
|
||||||
|
"Beyoncé I Am… Sasha Fierce": [], # blocked
|
||||||
|
"i am sasha fierce": [
|
||||||
|
{"username": "a", "hasFreeUploadSlot": True, "uploadSpeed": 1, "queueLength": 0,
|
||||||
|
"files": [{"filename": r"x\Beyonce - I Am... Sasha Fierce\01.flac", "size": 5}]},
|
||||||
|
{"username": "b", "hasFreeUploadSlot": True, "uploadSpeed": 1, "queueLength": 0,
|
||||||
|
"files": [{"filename": r"x\Other Artist - I Am Sasha Fierce\01.flac", "size": 5}]},
|
||||||
|
],
|
||||||
|
})
|
||||||
|
_install(monkeypatch, fake)
|
||||||
|
out = _client().search_album("Beyoncé", "I Am… Sasha Fierce")
|
||||||
|
assert fake.queries == ["Beyoncé I Am… Sasha Fierce", "i am sasha fierce"]
|
||||||
|
peers = [json.loads(c["source_ref"])["username"] for c in out]
|
||||||
|
assert peers == ["a"] # foreign same-titled album dropped
|
||||||
|
|
||||||
|
|
||||||
|
def test_artist_only_fallback_when_album_title_blocked(monkeypatch):
|
||||||
|
# Album-only also blocked/empty → fall through to artist-only, filtered by album in path.
|
||||||
|
fake = _FakeSearch({
|
||||||
|
"Foo Lemonade": [],
|
||||||
|
"lemonade": [], # album title itself blocked
|
||||||
|
"foo": [
|
||||||
|
{"username": "a", "hasFreeUploadSlot": True, "uploadSpeed": 1, "queueLength": 0,
|
||||||
|
"files": [{"filename": r"x\Foo - Lemonade\01.flac", "size": 5}]},
|
||||||
|
],
|
||||||
|
})
|
||||||
|
_install(monkeypatch, fake)
|
||||||
|
out = _client().search_album("Foo", "Lemonade")
|
||||||
|
assert fake.queries == ["Foo Lemonade", "lemonade", "foo"]
|
||||||
|
assert len(out) == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_fallback_yields_nothing_when_title_too_generic(monkeypatch):
|
||||||
|
# 'Adele 21': artist blocked, album title '21' returns lots of non-Adele content → filtered
|
||||||
|
# to nothing rather than grabbing a wrong album.
|
||||||
|
fake = _FakeSearch({
|
||||||
|
"Adele 21": [],
|
||||||
|
"21": [
|
||||||
|
{"username": "z", "hasFreeUploadSlot": True, "uploadSpeed": 1, "queueLength": 0,
|
||||||
|
"files": [{"filename": r"x\21 Savage - Savage Mode\01.flac", "size": 5}]},
|
||||||
|
],
|
||||||
|
"adele": [], # artist blocked
|
||||||
|
})
|
||||||
|
_install(monkeypatch, fake)
|
||||||
|
out = _client().search_album("Adele", "21")
|
||||||
|
assert out == []
|
||||||
Reference in New Issue
Block a user