feat(worker): share the MusicBrainz cache from scans/discovery

The worker's MB browser (discography + artist search, used by monitor sweeps,
library scans, and Last.fm discovery) re-fetched every time. Route it through
the same ApiCache table + logical keys the web app uses, so a discography one
process fetched warms the other, and repeat lookups stop hitting MB.

- apicache.py cached_api(): read-through, serve-stale-on-outage, never caches
  null, best-effort (a cache-DB error degrades to a live fetch, never breaks
  scan/sweep/discovery). Its own DEDICATED autocommit connection so it can't
  commit the worker's in-flight transaction, and autocommit avoids freezing
  Postgres now() (which would break TTL math).
- CachingMbBrowser decorates MusicBrainzBrowser, serializing to the SAME
  camelCase JSON shape the web writes (round-trip tested). registry.build_browser
  wraps it when a DSN is present; the Last.fm source's browser rides it too.
- Hourly prune drops ApiCache rows > 90d so the table stays bounded.
- worker 213 tests / 7-skip.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Jonathan
2026-07-14 11:17:51 +02:00
parent bdbf9d237e
commit 034ae40084
7 changed files with 291 additions and 9 deletions
+2
View File
@@ -30,6 +30,7 @@ def conn():
cur.execute('DELETE FROM "DiscoverySuggestion"')
cur.execute('DELETE FROM "DiscoverySeedContribution"')
cur.execute('DELETE FROM "ScanWorkItem"')
cur.execute('DELETE FROM "ApiCache"')
cur.execute('DELETE FROM "Config"')
connection.commit()
yield connection
@@ -47,6 +48,7 @@ def conn():
cur.execute('DELETE FROM "DiscoverySuggestion"')
cur.execute('DELETE FROM "DiscoverySeedContribution"')
cur.execute('DELETE FROM "ScanWorkItem"')
cur.execute('DELETE FROM "ApiCache"')
cur.execute('DELETE FROM "Config"')
connection.commit()
connection.close()
+104
View File
@@ -0,0 +1,104 @@
"""Read-through + shared-shape tests for the worker cache. cached_api uses the real
lyra_test DB (via the conn fixture); MB is never hit (fetch_fresh is a fake)."""
from lyra_worker.apicache import cached_api, norm_key
from lyra_worker._mbbrowser import (
_artist_hit_from_json, _artist_hit_to_json, _rg_from_json, _rg_to_json,
)
from lyra_worker.browser import ArtistHit, ReleaseGroupInfo
def _backdate(conn, key, seconds):
with conn.cursor() as cur:
cur.execute(
'UPDATE "ApiCache" SET "fetchedAt" = now() - make_interval(secs => %s) WHERE key = %s',
(seconds, key),
)
conn.commit()
def test_read_through_fetches_once(conn):
calls = []
def fresh():
calls.append(1)
return {"v": 1}
assert cached_api(conn, "k1", 3600, fresh) == {"v": 1}
assert cached_api(conn, "k1", 3600, fresh) == {"v": 1}
assert len(calls) == 1 # second served from cache
def test_refetches_after_ttl(conn):
calls = []
def fresh():
calls.append(1)
return {"v": len(calls)}
cached_api(conn, "k2", 3600, fresh)
_backdate(conn, "k2", 7200) # older than the ttl
cached_api(conn, "k2", 3600, fresh)
assert len(calls) == 2
def test_serves_stale_on_fetch_error(conn):
cached_api(conn, "k3", 3600, lambda: {"v": "old"})
_backdate(conn, "k3", 7200)
def boom():
raise RuntimeError("mb down")
assert cached_api(conn, "k3", 3600, boom) == {"v": "old"}
def test_reraises_when_no_cached_row(conn):
def boom():
raise RuntimeError("mb down")
try:
cached_api(conn, "k4-missing", 3600, boom)
except RuntimeError as e:
assert "mb down" in str(e)
else:
raise AssertionError("expected the fetch error to propagate")
def test_does_not_cache_none(conn):
calls = []
def fresh():
calls.append(1)
return None
cached_api(conn, "k5", 3600, fresh)
cached_api(conn, "k5", 3600, fresh)
assert len(calls) == 2 # None never cached
with conn.cursor() as cur:
cur.execute('SELECT 1 FROM "ApiCache" WHERE key = %s', ("k5",))
assert cur.fetchone() is None
def test_norm_key():
assert norm_key(" Radiohead ") == "radiohead"
def test_release_group_json_roundtrip():
r = ReleaseGroupInfo(rg_mbid="rg1", title="OK Computer", primary_type="Album",
secondary_types=("Live",), first_release_date="1997-05-21")
assert _rg_from_json(_rg_to_json(r)) == r
# a bare group (no type/date) round-trips to the worker's empty-string defaults
bare = ReleaseGroupInfo(rg_mbid="rg2", title="X")
assert _rg_from_json(_rg_to_json(bare)) == bare
def test_artist_hit_json_roundtrip():
a = ArtistHit(mbid="a1", name="Radiohead", disambiguation="UK band")
assert _artist_hit_from_json(_artist_hit_to_json(a)) == a
def test_worker_shape_matches_web_camelcase():
# the stored dict MUST use the same camelCase keys the web app writes/reads
d = _rg_to_json(ReleaseGroupInfo(rg_mbid="rg", title="T", primary_type="Album",
secondary_types=(), first_release_date="2000"))
assert set(d) == {"rgMbid", "title", "primaryType", "secondaryTypes", "firstReleaseDate"}
assert set(_artist_hit_to_json(ArtistHit(mbid="a", name="N"))) == {"mbid", "name", "disambiguation"}