fix(web): stream large album uploads to disk (busboy) instead of buffering
Manual import used request.formData(), which buffers the whole body and caps out around ~15MB — a real FLAC album is hundreds of MB (Hollywood's Bleeding is 584MB), so it failed with "expected multipart form data". Now the multipart body is STREAMED with busboy: each file pipes straight to a hidden .import-<uuid> staging dir on the library filesystem, then the dir is atomically renamed into place. No full-body buffering, no size cap. Cleaned up on any failure; guards (400 missing fields/audio, 409 exists) unchanged. web 161 tests. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Jonathan
Generated
+31
-2
@@ -7,8 +7,11 @@
|
||||
"": {
|
||||
"name": "lyra-web",
|
||||
"version": "0.1.0",
|
||||
"hasInstallScript": true,
|
||||
"dependencies": {
|
||||
"@prisma/client": "6.1.0",
|
||||
"@types/busboy": "^1.5.4",
|
||||
"busboy": "^1.6.0",
|
||||
"next": "15.5.20",
|
||||
"react": "19.0.0",
|
||||
"react-dom": "19.0.0"
|
||||
@@ -1457,6 +1460,15 @@
|
||||
"tslib": "^2.8.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/busboy": {
|
||||
"version": "1.5.4",
|
||||
"resolved": "https://registry.npmjs.org/@types/busboy/-/busboy-1.5.4.tgz",
|
||||
"integrity": "sha512-kG7WrUuAKK0NoyxfQHsVE6j1m01s6kMma64E+OZenQABMQyTJop1DumUWcLwAQ2JzpefU7PDYoRDKl8uZosFjw==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@types/node": "*"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/estree": {
|
||||
"version": "1.0.9",
|
||||
"resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.9.tgz",
|
||||
@@ -1468,7 +1480,6 @@
|
||||
"version": "22.10.2",
|
||||
"resolved": "https://registry.npmjs.org/@types/node/-/node-22.10.2.tgz",
|
||||
"integrity": "sha512-Xxr6BBRCAOQixvonOye19wnzyDiUtTeqldOOmj3CkeblonbccA12PFwlufvRdrpjXxqnmUaeiU5EOA+7s5diUQ==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"undici-types": "~6.20.0"
|
||||
@@ -1643,6 +1654,17 @@
|
||||
"node": ">=12"
|
||||
}
|
||||
},
|
||||
"node_modules/busboy": {
|
||||
"version": "1.6.0",
|
||||
"resolved": "https://registry.npmjs.org/busboy/-/busboy-1.6.0.tgz",
|
||||
"integrity": "sha512-8SFQbg/0hQ9xy3UNTB0YEnsNBbWfhf7RtnzpL7TkBiTBRfrQ9Fxcnz7VJsleJpyp6rVLvXiuORqjlHi5q+PYuA==",
|
||||
"dependencies": {
|
||||
"streamsearch": "^1.1.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=10.16.0"
|
||||
}
|
||||
},
|
||||
"node_modules/cac": {
|
||||
"version": "6.7.14",
|
||||
"resolved": "https://registry.npmjs.org/cac/-/cac-6.7.14.tgz",
|
||||
@@ -2157,6 +2179,14 @@
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/streamsearch": {
|
||||
"version": "1.1.0",
|
||||
"resolved": "https://registry.npmjs.org/streamsearch/-/streamsearch-1.1.0.tgz",
|
||||
"integrity": "sha512-Mcc5wHehp9aXz1ax6bZUyY5afg9u2rv5cqQI3mRrYkGC8rW2hM02jWuwjtL++LS5qinSyhj2QfLyNsuc+VsExg==",
|
||||
"engines": {
|
||||
"node": ">=10.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/styled-jsx": {
|
||||
"version": "5.1.6",
|
||||
"resolved": "https://registry.npmjs.org/styled-jsx/-/styled-jsx-5.1.6.tgz",
|
||||
@@ -2248,7 +2278,6 @@
|
||||
"version": "6.20.0",
|
||||
"resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.20.0.tgz",
|
||||
"integrity": "sha512-Ny6QZ2Nju20vw1SRHe3d9jVu6gJ+4e3+MMpqu7pqE5HT6WsTSlce++GQmK5UXS8mzV8DSYHrQH+Xrf2jVcuKNg==",
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/vite": {
|
||||
|
||||
+5
-3
@@ -12,17 +12,19 @@
|
||||
"db:dev": "prisma migrate dev"
|
||||
},
|
||||
"dependencies": {
|
||||
"@prisma/client": "6.1.0",
|
||||
"@types/busboy": "^1.5.4",
|
||||
"busboy": "^1.6.0",
|
||||
"next": "15.5.20",
|
||||
"react": "19.0.0",
|
||||
"react-dom": "19.0.0",
|
||||
"@prisma/client": "6.1.0"
|
||||
"react-dom": "19.0.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"typescript": "5.7.2",
|
||||
"@types/node": "22.10.2",
|
||||
"@types/react": "19.0.1",
|
||||
"@types/react-dom": "19.0.1",
|
||||
"prisma": "6.1.0",
|
||||
"typescript": "5.7.2",
|
||||
"vitest": "2.1.8"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,17 +1,22 @@
|
||||
import { mkdir, rename, writeFile, rm } from "node:fs/promises";
|
||||
import { existsSync } from "node:fs";
|
||||
import { resolve, join, extname } from "node:path";
|
||||
import { createWriteStream, existsSync } from "node:fs";
|
||||
import { mkdir, rename, rm } from "node:fs/promises";
|
||||
import { resolve, join, dirname, extname } from "node:path";
|
||||
import { Readable } from "node:stream";
|
||||
import { pipeline } from "node:stream/promises";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import Busboy from "busboy";
|
||||
import { prisma } from "@/lib/db";
|
||||
|
||||
// Manually import an album (a ripped CD / files already on the PC): upload the audio (+ an
|
||||
// optional cover) with artist/album/year, and Lyra writes it into the library. The web
|
||||
// container bind-mounts /music, so it can write directly. A later library Scan will
|
||||
// MB-resolve it (cover art, canonical types) and re-probe quality.
|
||||
// container bind-mounts /music, so it can write directly. A later library Scan MB-resolves it
|
||||
// (cover art, canonical types) and re-probes quality.
|
||||
//
|
||||
// The body is STREAMED to disk with busboy rather than buffered via request.formData(), which
|
||||
// caps out around ~15MB — real FLAC albums are hundreds of MB.
|
||||
|
||||
const AUDIO_EXT = new Set([".flac", ".mp3", ".m4a", ".opus", ".ogg", ".aac", ".wav"]);
|
||||
const LOSSLESS_EXT = new Set([".flac", ".wav"]);
|
||||
// filesystem-illegal path chars → "_" (matches the worker's safe_name; control chars can't
|
||||
// reach here through a form field, so the range is omitted).
|
||||
const ILLEGAL = /[<>:"/\\|?*]/g;
|
||||
|
||||
function safeName(s: string): string {
|
||||
@@ -20,54 +25,79 @@ function safeName(s: string): string {
|
||||
}
|
||||
|
||||
export async function POST(request: Request) {
|
||||
let form: FormData;
|
||||
try {
|
||||
form = await request.formData();
|
||||
} catch {
|
||||
const ct = request.headers.get("content-type") ?? "";
|
||||
if (!ct.includes("multipart/form-data") || !request.body) {
|
||||
return Response.json({ error: "expected multipart form data" }, { status: 400 });
|
||||
}
|
||||
|
||||
const artist = String(form.get("artist") ?? "").trim();
|
||||
const album = String(form.get("album") ?? "").trim();
|
||||
const year = String(form.get("year") ?? "").trim();
|
||||
if (!artist || !album) return Response.json({ error: "artist and album are required" }, { status: 400 });
|
||||
if (year && !/^\d{4}$/.test(year)) return Response.json({ error: "year must be 4 digits" }, { status: 400 });
|
||||
|
||||
const files = form.getAll("files").filter((f): f is File => f instanceof File);
|
||||
const audio = files
|
||||
.filter((f) => AUDIO_EXT.has(extname(f.name).toLowerCase()))
|
||||
.sort((a, b) => a.name.localeCompare(b.name));
|
||||
if (audio.length === 0) return Response.json({ error: "no audio files provided" }, { status: 400 });
|
||||
const cover = files.find((f) => /\.(jpe?g|png)$/i.test(f.name));
|
||||
|
||||
const root = resolve(process.env.LYRA_LIBRARY_ROOT ?? "/music");
|
||||
// stage under a hidden dir on the library filesystem so the final move is an atomic rename
|
||||
// (the scan skips dot-dirs). Cleaned up on any failure.
|
||||
const staging = join(root, `.import-${randomUUID()}`);
|
||||
await mkdir(staging, { recursive: true });
|
||||
|
||||
const fields: Record<string, string> = {};
|
||||
const audioNames: string[] = [];
|
||||
let hasCover = false;
|
||||
|
||||
try {
|
||||
await new Promise<void>((res, rej) => {
|
||||
const bb = Busboy({ headers: { "content-type": ct } });
|
||||
const writes: Promise<void>[] = [];
|
||||
bb.on("field", (name, val) => {
|
||||
fields[name] = val;
|
||||
});
|
||||
bb.on("file", (_name, stream, info) => {
|
||||
const ext = extname(info.filename).toLowerCase();
|
||||
if (AUDIO_EXT.has(ext)) {
|
||||
const safe = safeName(info.filename);
|
||||
audioNames.push(safe);
|
||||
writes.push(pipeline(stream, createWriteStream(join(staging, safe))));
|
||||
} else if (/\.(jpe?g|png)$/i.test(info.filename) && !hasCover) {
|
||||
hasCover = true;
|
||||
writes.push(pipeline(stream, createWriteStream(join(staging, "cover.jpg"))));
|
||||
} else {
|
||||
stream.resume(); // discard anything that isn't audio or a cover
|
||||
}
|
||||
});
|
||||
bb.on("error", rej);
|
||||
bb.on("close", () => Promise.all(writes).then(() => res(), rej));
|
||||
Readable.fromWeb(request.body as import("node:stream/web").ReadableStream).pipe(bb);
|
||||
});
|
||||
} catch {
|
||||
await rm(staging, { recursive: true, force: true });
|
||||
return Response.json({ error: "failed to write the upload to disk" }, { status: 500 });
|
||||
}
|
||||
|
||||
const artist = (fields.artist ?? "").trim();
|
||||
const album = (fields.album ?? "").trim();
|
||||
const year = (fields.year ?? "").trim();
|
||||
const audio = audioNames.sort((a, b) => a.localeCompare(b));
|
||||
|
||||
const bad =
|
||||
!artist || !album
|
||||
? "artist and album are required"
|
||||
: audio.length === 0
|
||||
? "no audio files provided"
|
||||
: year && !/^\d{4}$/.test(year)
|
||||
? "year must be 4 digits"
|
||||
: null;
|
||||
if (bad) {
|
||||
await rm(staging, { recursive: true, force: true });
|
||||
return Response.json({ error: bad }, { status: 400 });
|
||||
}
|
||||
|
||||
const albumFolder = year ? `${safeName(album)} (${year})` : safeName(album);
|
||||
const finalDir = join(root, safeName(artist), albumFolder);
|
||||
|
||||
if (existsSync(finalDir)) {
|
||||
return Response.json({ error: "a folder for that album already exists on disk" }, { status: 409 });
|
||||
}
|
||||
if (await prisma.libraryItem.findFirst({ where: { artist, album } })) {
|
||||
if (existsSync(finalDir) || (await prisma.libraryItem.findFirst({ where: { artist, album } }))) {
|
||||
await rm(staging, { recursive: true, force: true });
|
||||
return Response.json({ error: "this album is already in the library" }, { status: 409 });
|
||||
}
|
||||
|
||||
// Assemble in a sibling temp dir then rename into place, so a failed/partial upload never
|
||||
// leaves a half-written album in the library.
|
||||
const tmpDir = finalDir + ".importing";
|
||||
await rm(tmpDir, { recursive: true, force: true });
|
||||
await mkdir(tmpDir, { recursive: true });
|
||||
try {
|
||||
for (const f of audio) {
|
||||
await writeFile(join(tmpDir, safeName(f.name)), Buffer.from(await f.arrayBuffer()));
|
||||
}
|
||||
if (cover) await writeFile(join(tmpDir, "cover.jpg"), Buffer.from(await cover.arrayBuffer()));
|
||||
await rename(tmpDir, finalDir);
|
||||
} catch {
|
||||
await rm(tmpDir, { recursive: true, force: true });
|
||||
return Response.json({ error: "failed to write the album to disk" }, { status: 500 });
|
||||
}
|
||||
await mkdir(dirname(finalDir), { recursive: true });
|
||||
await rename(staging, finalDir);
|
||||
|
||||
const firstExt = extname(audio[0].name).toLowerCase();
|
||||
const firstExt = extname(audio[0]).toLowerCase();
|
||||
await prisma.libraryItem.create({
|
||||
data: {
|
||||
artist,
|
||||
@@ -78,7 +108,7 @@ export async function POST(request: Request) {
|
||||
// rough class from the extension (2 = CD-lossless, 1 = lossy); a later scan re-probes and
|
||||
// can upgrade a hi-res file to class 3.
|
||||
qualityClass: LOSSLESS_EXT.has(firstExt) ? 2 : 1,
|
||||
trackNames: audio.map((f) => safeName(f.name)),
|
||||
trackNames: audio,
|
||||
},
|
||||
});
|
||||
|
||||
|
||||
Reference in New Issue
Block a user