Jonathan/Lyra
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(worker): fall through to the next source on an incomplete download

Browse Source 
Completeness/duration verification was applied only to the first candidate that
downloaded successfully; if that copy was incomplete or too short the whole job
failed, even when a later-ranked source had a complete copy (e.g. John Mayer 'The
Search for Everything' — Qobuz reliably fails one hi-res track, so it never fell
back to the deluxe edition or Soulseek).

Extract the checks into a pure _download_problem() and run it INSIDE the download
loop: a candidate that downloads but fails verification now falls through to the
next-ranked source, and the job only fails (needs_attention) when every candidate
is incomplete — reporting the last shortfall reason.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Jonathan
2026-07-15 13:41:04 +02:00
parent b0eec41823
commit d47ed12e8e
2 changed files with 139 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files
worker/lyra_worker/pipeline.py
+48 -40
View File
@@ -70,6 +70,40 @@ def _count_staged_audio(staging: str) -> int:
return n
def _download_problem(result, staging: str, target, winner, measure_duration) -> str | None:
"""Why a completed download is unusable — ``"incomplete download"`` / ``"download too short
(...)"`` — or None if it's good. Completeness is judged against the CHOSEN SOURCE's own track
count (``winner.track_count``), NOT MusicBrainz's canonical release: MB picks an arbitrary
releases[0] that is often a deluxe/expanded edition with more tracks (and runtime) than the
standard album a source legitimately delivers, so gating on ``target.track_count`` falsely
rejects complete standard-edition downloads (The Script "No Sound Without Silence" 11 vs MB 12,
Skillet "Unleashed" 12 vs MB's 20-track deluxe). Pure — no DB I/O — so the download loop can
call it per candidate and fall through to the next source on a problem."""
expected = winner.track_count
# (a) Truncated: fewer files than the source promised (a track failed mid-download). Count the
# NON-EMPTY audio on disk too: a silently-skipped track leaves a 0-byte placeholder the adapter
# still counts. (`staged and ...` keeps fake adapters that stage nothing on the count path.)
staged = _count_staged_audio(staging)
truncated = result.track_count < expected or bool(staged and staged < expected)
# (b) Implausibly small vs MB's album — a lone-track "full album" video / wrong match, as
# distinct from a legitimately smaller edition. A real edition keeps more than half MB's tracks.
too_small = target.track_count is not None and expected * 2 <= target.track_count
if truncated or too_small:
return "incomplete download"
# Duration: a truncated file or short preview substituted for a real track shows up as playtime
# well under the expected total. Scale MB's total to the DELIVERED edition's size so a smaller
# edition isn't judged against a larger one's runtime. Only when the total is known (measured is
# None for the offline fakes).
if target.total_duration_s:
expected_total = target.total_duration_s
if target.track_count:
expected_total *= winner.track_count / target.track_count
measured = measure_duration(staging)
if measured is not None and measured < expected_total * _DURATION_MIN_RATIO:
return f"download too short ({measured / 60:.0f} of ~{expected_total / 60:.0f} min)"
return None
def _poll_download_progress(job_id: str, staging: str, expected: int, stop: "threading.Event",
reports: "threading.Event") -> None:
"""Until `stop`, periodically write Job.downloadProgress = files-in-staging / expected
@@ -343,7 +377,7 @@ def run_pipeline(
by_source = {a.name: a for a in adapters}
staging = staging_dir(staging_root or f"{dest_root}/.staging", job_id)
winner = None
result = None
last_problem = None # why the most recent downloaded-but-rejected candidate was unusable
_set_download_progress(conn, job_id, 0.0) # reset for this run
expected = target.track_count or (ranked[0].track_count if ranked else 0)
_stop = threading.Event()
@@ -362,54 +396,28 @@ def run_pipeline(
shutil.rmtree(staging, ignore_errors=True) # clean slate per attempt
_reports.clear() # this attempt hasn't reported yet → poller estimates until it does
result = adapter.download(candidate, staging, _make_on_progress(conn, job_id, _reports))
if result.ok:
if not result.ok:
continue
# Verify completeness on the staging copy. A download that lands incomplete or too
# short falls through to the NEXT-ranked source (e.g. Qobuz repeatedly fails one
# track → try the deluxe edition or Soulseek) rather than failing the whole job on
# the first source's shortfall.
last_problem = _download_problem(result, staging, target, candidate, measure_duration)
if last_problem is None:
winner = candidate
break
finally:
_stop.set()
_poller.join(timeout=3)
if winner is None or result is None or not result.ok:
_fail(conn, job_id, "all downloads failed")
if winner is None:
# No source produced a complete album: surface why the last one was rejected (incomplete
# / too short), or "all downloads failed" if none even downloaded.
_fail(conn, job_id, last_problem or "all downloads failed")
return
_set_download_progress(conn, job_id, 1.0) # download done — UI shows 100% into Finishing
# 5. verify completeness on the staging copy, then promote + tag
# 5. promote + tag the verified winner
_set_state(conn, job_id, "tagging", "tag")
# Judge completeness against what the CHOSEN SOURCE offered, NOT MusicBrainz's canonical
# release: MB picks an arbitrary releases[0] that is often a deluxe/expanded edition with
# more tracks than the standard album a source legitimately delivers, so gating purely on
# target.track_count falsely rejects complete standard-edition downloads (e.g. The Script
# "No Sound Without Silence" 11 vs MB 12, Skillet "Unleashed" 12 vs MB's 20-track deluxe).
expected = winner.track_count
# (a) Truncated: fewer files than the source promised (a track failed mid-download).
# Count the NON-EMPTY audio actually on disk too: a silently-skipped track leaves a 0-byte
# placeholder the adapter still counts, so trust the files when any are staged. (`staged
# and ...` keeps fake adapters that stage nothing on the count path.)
staged = _count_staged_audio(staging)
truncated = result.track_count < expected or bool(staged and staged < expected)
# (b) Implausibly small vs MB's album — a lone-track "full album" video or a wrong match,
# as distinct from a legitimately smaller edition (standard vs deluxe). A real edition
# keeps more than half of MB's tracks; a 1-of-12 match does not.
too_small = target.track_count is not None and expected * 2 <= target.track_count
if truncated or too_small:
_fail(conn, job_id, "incomplete download")
return
# Duration check: even with the right track count, a truncated file or a short preview
# substituted for a real track shows up as a total playtime well under the expected total.
# Scale MB's total to the DELIVERED edition's size (source vs MB track count) so a smaller
# standard edition isn't measured against a larger deluxe edition's runtime. Only applies
# when the total is known (measured is None for the offline fakes).
if target.total_duration_s:
expected_total = target.total_duration_s
if target.track_count:
expected_total *= winner.track_count / target.track_count
measured = measure_duration(staging)
if measured is not None and measured < expected_total * _DURATION_MIN_RATIO:
_fail(conn, job_id,
f"download too short ({measured / 60:.0f} of ~{expected_total / 60:.0f} min)")
return
final = album_dir(dest_root, target)
existing_q = _library_quality(conn, target)
new_q = quality_class(winner.quality)