fix(worker): retry transient MusicBrainz failures with backoff
The matching stage makes up to three musicbrainzngs calls per job, none of which retried. A dropped TLS connection (surfacing as NetworkError, "<urlopen error [SSL: UNEXPECTED_EOF_WHILE_READING]>") or a 503 rate-limit (ResponseError) failed the entire pipeline for the job, which then re-queued and re-ran from scratch — ~36% of jobs churned this way over a 6h window. Wrap the three MB calls in _with_retry (4 attempts, 1s/2s/4s exponential backoff), catching only the two transient musicbrainzngs error types so real bugs still propagate. musicbrainzngs stays lazily imported to keep module import offline. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -1,4 +1,7 @@
|
||||
from lyra_worker._musicbrainz import _best_release_group
|
||||
import musicbrainzngs
|
||||
import pytest
|
||||
|
||||
from lyra_worker._musicbrainz import _best_release_group, _with_retry
|
||||
|
||||
# Offline unit tests for the pure release-group selection logic. The live resolve()
|
||||
# path (network) is covered by test_musicbrainz_live.py.
|
||||
@@ -60,3 +63,73 @@ def test_prefers_credited_artist_over_same_titled_other_artist_album():
|
||||
]
|
||||
best = _best_release_group("Lake Street Dive", "Fun Machine", groups)
|
||||
assert best["artist-credit"][0]["artist"]["name"] == "Lake Street Dive"
|
||||
|
||||
|
||||
# --- _with_retry: transient MusicBrainz failures (SSL EOF, 503 rate-limit) ---
|
||||
|
||||
|
||||
def test_retry_returns_value_without_backoff_when_first_call_succeeds():
|
||||
sleeps: list[float] = []
|
||||
calls = {"n": 0}
|
||||
|
||||
def fn():
|
||||
calls["n"] += 1
|
||||
return "ok"
|
||||
|
||||
assert _with_retry(fn, sleep=sleeps.append) == "ok"
|
||||
assert calls["n"] == 1
|
||||
assert sleeps == [] # no backoff when nothing failed
|
||||
|
||||
|
||||
def test_retry_recovers_from_transient_network_error():
|
||||
# The real symptom: <urlopen error [SSL: UNEXPECTED_EOF_WHILE_READING]> arrives
|
||||
# as musicbrainzngs.NetworkError. A couple of retries clears it.
|
||||
sleeps: list[float] = []
|
||||
calls = {"n": 0}
|
||||
|
||||
def fn():
|
||||
calls["n"] += 1
|
||||
if calls["n"] < 3:
|
||||
raise musicbrainzngs.NetworkError("SSL: UNEXPECTED_EOF_WHILE_READING")
|
||||
return "recovered"
|
||||
|
||||
assert _with_retry(fn, sleep=sleeps.append) == "recovered"
|
||||
assert calls["n"] == 3
|
||||
assert sleeps == [1.0, 2.0] # exponential backoff before each retry
|
||||
|
||||
|
||||
def test_retry_recovers_from_transient_response_error():
|
||||
# MusicBrainz's own 503 rate-limit surfaces as ResponseError.
|
||||
calls = {"n": 0}
|
||||
|
||||
def fn():
|
||||
calls["n"] += 1
|
||||
if calls["n"] < 2:
|
||||
raise musicbrainzngs.ResponseError("503 Service Unavailable")
|
||||
return "ok"
|
||||
|
||||
assert _with_retry(fn, sleep=lambda _: None) == "ok"
|
||||
assert calls["n"] == 2
|
||||
|
||||
|
||||
def test_retry_reraises_after_exhausting_attempts():
|
||||
sleeps: list[float] = []
|
||||
|
||||
def fn():
|
||||
raise musicbrainzngs.NetworkError("still down")
|
||||
|
||||
with pytest.raises(musicbrainzngs.NetworkError):
|
||||
_with_retry(fn, attempts=4, sleep=sleeps.append)
|
||||
assert sleeps == [1.0, 2.0, 4.0] # 3 backoffs across 4 attempts
|
||||
|
||||
|
||||
def test_retry_does_not_swallow_non_transient_errors():
|
||||
calls = {"n": 0}
|
||||
|
||||
def fn():
|
||||
calls["n"] += 1
|
||||
raise ValueError("bug, not a network blip")
|
||||
|
||||
with pytest.raises(ValueError):
|
||||
_with_retry(fn, sleep=lambda _: None)
|
||||
assert calls["n"] == 1 # non-transient errors propagate immediately, no retry
|
||||
|
||||
Reference in New Issue
Block a user