feat(worker): share the MusicBrainz cache from scans/discovery
The worker's MB browser (discography + artist search, used by monitor sweeps, library scans, and Last.fm discovery) re-fetched every time. Route it through the same ApiCache table + logical keys the web app uses, so a discography one process fetched warms the other, and repeat lookups stop hitting MB. - apicache.py cached_api(): read-through, serve-stale-on-outage, never caches null, best-effort (a cache-DB error degrades to a live fetch, never breaks scan/sweep/discovery). Its own DEDICATED autocommit connection so it can't commit the worker's in-flight transaction, and autocommit avoids freezing Postgres now() (which would break TTL math). - CachingMbBrowser decorates MusicBrainzBrowser, serializing to the SAME camelCase JSON shape the web writes (round-trip tested). registry.build_browser wraps it when a DSN is present; the Last.fm source's browser rides it too. - Hourly prune drops ApiCache rows > 90d so the table stays bounded. - worker 213 tests / 7-skip. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,104 @@
|
||||
"""Read-through + shared-shape tests for the worker cache. cached_api uses the real
|
||||
lyra_test DB (via the conn fixture); MB is never hit (fetch_fresh is a fake)."""
|
||||
from lyra_worker.apicache import cached_api, norm_key
|
||||
from lyra_worker._mbbrowser import (
|
||||
_artist_hit_from_json, _artist_hit_to_json, _rg_from_json, _rg_to_json,
|
||||
)
|
||||
from lyra_worker.browser import ArtistHit, ReleaseGroupInfo
|
||||
|
||||
|
||||
def _backdate(conn, key, seconds):
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
'UPDATE "ApiCache" SET "fetchedAt" = now() - make_interval(secs => %s) WHERE key = %s',
|
||||
(seconds, key),
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
|
||||
def test_read_through_fetches_once(conn):
|
||||
calls = []
|
||||
|
||||
def fresh():
|
||||
calls.append(1)
|
||||
return {"v": 1}
|
||||
|
||||
assert cached_api(conn, "k1", 3600, fresh) == {"v": 1}
|
||||
assert cached_api(conn, "k1", 3600, fresh) == {"v": 1}
|
||||
assert len(calls) == 1 # second served from cache
|
||||
|
||||
|
||||
def test_refetches_after_ttl(conn):
|
||||
calls = []
|
||||
|
||||
def fresh():
|
||||
calls.append(1)
|
||||
return {"v": len(calls)}
|
||||
|
||||
cached_api(conn, "k2", 3600, fresh)
|
||||
_backdate(conn, "k2", 7200) # older than the ttl
|
||||
cached_api(conn, "k2", 3600, fresh)
|
||||
assert len(calls) == 2
|
||||
|
||||
|
||||
def test_serves_stale_on_fetch_error(conn):
|
||||
cached_api(conn, "k3", 3600, lambda: {"v": "old"})
|
||||
_backdate(conn, "k3", 7200)
|
||||
|
||||
def boom():
|
||||
raise RuntimeError("mb down")
|
||||
|
||||
assert cached_api(conn, "k3", 3600, boom) == {"v": "old"}
|
||||
|
||||
|
||||
def test_reraises_when_no_cached_row(conn):
|
||||
def boom():
|
||||
raise RuntimeError("mb down")
|
||||
|
||||
try:
|
||||
cached_api(conn, "k4-missing", 3600, boom)
|
||||
except RuntimeError as e:
|
||||
assert "mb down" in str(e)
|
||||
else:
|
||||
raise AssertionError("expected the fetch error to propagate")
|
||||
|
||||
|
||||
def test_does_not_cache_none(conn):
|
||||
calls = []
|
||||
|
||||
def fresh():
|
||||
calls.append(1)
|
||||
return None
|
||||
|
||||
cached_api(conn, "k5", 3600, fresh)
|
||||
cached_api(conn, "k5", 3600, fresh)
|
||||
assert len(calls) == 2 # None never cached
|
||||
with conn.cursor() as cur:
|
||||
cur.execute('SELECT 1 FROM "ApiCache" WHERE key = %s', ("k5",))
|
||||
assert cur.fetchone() is None
|
||||
|
||||
|
||||
def test_norm_key():
|
||||
assert norm_key(" Radiohead ") == "radiohead"
|
||||
|
||||
|
||||
def test_release_group_json_roundtrip():
|
||||
r = ReleaseGroupInfo(rg_mbid="rg1", title="OK Computer", primary_type="Album",
|
||||
secondary_types=("Live",), first_release_date="1997-05-21")
|
||||
assert _rg_from_json(_rg_to_json(r)) == r
|
||||
# a bare group (no type/date) round-trips to the worker's empty-string defaults
|
||||
bare = ReleaseGroupInfo(rg_mbid="rg2", title="X")
|
||||
assert _rg_from_json(_rg_to_json(bare)) == bare
|
||||
|
||||
|
||||
def test_artist_hit_json_roundtrip():
|
||||
a = ArtistHit(mbid="a1", name="Radiohead", disambiguation="UK band")
|
||||
assert _artist_hit_from_json(_artist_hit_to_json(a)) == a
|
||||
|
||||
|
||||
def test_worker_shape_matches_web_camelcase():
|
||||
# the stored dict MUST use the same camelCase keys the web app writes/reads
|
||||
d = _rg_to_json(ReleaseGroupInfo(rg_mbid="rg", title="T", primary_type="Album",
|
||||
secondary_types=(), first_release_date="2000"))
|
||||
assert set(d) == {"rgMbid", "title", "primaryType", "secondaryTypes", "firstReleaseDate"}
|
||||
assert set(_artist_hit_to_json(ArtistHit(mbid="a", name="N"))) == {"mbid", "name", "disambiguation"}
|
||||
Reference in New Issue
Block a user