From bc42e546f0c5a987018cecb1084e20c5c4f6dac8 Mon Sep 17 00:00:00 2001 From: Jonathan Date: Tue, 14 Jul 2026 13:17:20 +0200 Subject: [PATCH] fix(worker): raise http.client header limit so Qobuz tracks stop failing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bruno Mars "24K Magic" track 1 repeatedly downloaded as a 0-byte file. The worker log showed the real cause: streamrip's track download hit HTTPException("got more than 100 headers") — Qobuz's CDN returns >100 response headers for some tracks and Python's http.client rejects them at the default _MAXHEADERS=100, so streamrip skips the track and leaves an empty placeholder. - Raise http.client._MAXHEADERS to 1000 on import of the Qobuz adapter, so those responses parse and the track downloads. - _count_staged_audio now skips 0-byte files (a failed-track placeholder is not a real track) — improves the download-progress estimate and stops counting dead tracks. (A verify-level "reject 0-byte tracks" safety net is a follow-up under #16 per-track duration matching — it needs the pipeline fakes to write real files, deferred to avoid disproportionate churn here.) worker 223 tests / 7-skip. Co-Authored-By: Claude Opus 4.8 (1M context) --- worker/lyra_worker/adapters/_streamrip.py | 7 +++++++ worker/lyra_worker/pipeline.py | 13 ++++++++++--- worker/tests/test_download_progress.py | 15 +++++++++++---- 3 files changed, 28 insertions(+), 7 deletions(-) diff --git a/worker/lyra_worker/adapters/_streamrip.py b/worker/lyra_worker/adapters/_streamrip.py index 60da230..30e517a 100644 --- a/worker/lyra_worker/adapters/_streamrip.py +++ b/worker/lyra_worker/adapters/_streamrip.py @@ -1,8 +1,15 @@ import asyncio import hashlib +import http.client import os from typing import Callable +# Qobuz's CDN returns responses with more than Python's default 100-header limit for some +# tracks, so http.client raises HTTPException("got more than 100 headers"), streamrip skips +# that track, and it lands as a 0-byte file (seen on Bruno Mars "24K Magic" — track 1). Raise +# the limit process-wide so those downloads succeed. (Set on import of the Qobuz adapter.) +http.client._MAXHEADERS = 1000 + def _hashed_password(password: str) -> str: """streamrip's Qobuz email/password login expects the password as an MD5 hex digest.""" diff --git a/worker/lyra_worker/pipeline.py b/worker/lyra_worker/pipeline.py index 0527eae..5e0a0dd 100644 --- a/worker/lyra_worker/pipeline.py +++ b/worker/lyra_worker/pipeline.py @@ -18,12 +18,19 @@ _AUDIO_EXT = {".flac", ".mp3", ".m4a", ".opus", ".ogg", ".aac", ".wav"} def _count_staged_audio(staging: str) -> int: - """Count audio files anywhere under the staging dir (streamrip nests them in a subfolder).""" + """Count NON-EMPTY audio files anywhere under the staging dir (streamrip nests them in a + subfolder). 0-byte files are skipped: streamrip leaves an empty placeholder when a track + download fails (e.g. Qobuz's >100-header responses), and a dead track must not count toward + completeness — otherwise a broken album imports as 'complete'.""" n = 0 - for _root, _dirs, files in os.walk(staging): + for root, _dirs, files in os.walk(staging): for f in files: if os.path.splitext(f)[1].lower() in _AUDIO_EXT: - n += 1 + try: + if os.path.getsize(os.path.join(root, f)) > 0: + n += 1 + except OSError: + pass return n diff --git a/worker/tests/test_download_progress.py b/worker/tests/test_download_progress.py index c578f4e..d2c1034 100644 --- a/worker/tests/test_download_progress.py +++ b/worker/tests/test_download_progress.py @@ -41,14 +41,21 @@ def test_on_progress_clamps_out_of_range(conn): def test_count_staged_audio_counts_nested_audio_files_only(tmp_path): nested = tmp_path / "Artist" / "Album" nested.mkdir(parents=True) - (nested / "01.flac").write_bytes(b"") - (nested / "02.flac").write_bytes(b"") - (nested / "03.flac").write_bytes(b"") - (nested / "cover.jpg").write_bytes(b"") + (nested / "01.flac").write_bytes(b"audio") + (nested / "02.flac").write_bytes(b"audio") + (nested / "03.flac").write_bytes(b"audio") + (nested / "cover.jpg").write_bytes(b"img") assert _count_staged_audio(str(tmp_path)) == 3 +def test_count_staged_audio_skips_zero_byte_files(tmp_path): + # a 0-byte file is a failed-track placeholder (e.g. streamrip skipping a track) → not a track + (tmp_path / "01 Good.flac").write_bytes(b"audio") + (tmp_path / "02 Dead.flac").write_bytes(b"") + assert _count_staged_audio(str(tmp_path)) == 1 + + def test_count_staged_audio_empty_dir_is_zero(tmp_path): assert _count_staged_audio(str(tmp_path)) == 0