diff --git a/documentation/backend/services/auth.md b/documentation/backend/services/auth.md index 062d001c6..da32ecfd6 100644 --- a/documentation/backend/services/auth.md +++ b/documentation/backend/services/auth.md @@ -316,6 +316,7 @@ oidc.admin_claim_value = 'readmeabook-admin' - **Sessions follow approval status:** `requireAuth` (access tokens and API tokens) and `/api/auth/refresh` now read `registrationStatus` in their existing user lookup and refuse `pending_approval`/`rejected` accounts like a deleted one (401); null counts as approved. Token login refuses them like the regular sign-in, and login and API tokens can no longer be created for them. One rule for every check: `getRegistrationRefusal` (`src/lib/services/auth/local-account-status.ts`). - **Download links check the account:** `GET /api/requests/:id/download?token=` now checks the link's account like a sign-in does (one `getSessionRefusal` rule shared with `requireAuth`): missing, deleted, pending or rejected accounts, and links made before the account's sessions were revoked, get 401 (`Invalid or expired download token`); an account without download access now (`resolveDownloadAccess`, same rule as `POST /api/requests/:id/download-token`) gets 403. Live accounts with access: unchanged, no session cookie needed. - **Multi-file download zip entry names:** a request with several files streams one zip (`archiver` 8, `ZipArchive`). Each entry is the file's base name with any `/`, `\` or `:` replaced by `_`, so every file sits at the zip root under its full name. Before, a `\` in a Linux file name became a folder separator in the entry name, and archiver dropped a leading `word:` (`Prologue: The Start.mp3` downloaded as ` The Start.mp3`). +- **#490 Multi-file download cancel, read error, slow reader:** the zip is pull-based (`streamZip`, `src/app/api/requests/[id]/download/route.ts`): files are added one at a time, so only the file being zipped is open, and the next bytes are made only when the client reads. A cancel closes that file. A read error (or an archiver warning) is logged once with the request id and ends the response with an error. Before, a cancel left the file open and threw `ERR_INVALID_STATE: Controller is already closed`, a read error was uncaught and the download never ended, and a paused client made the server queue most of the zip in memory. Single-file downloads, entry names and compression level unchanged. - **#215 Soft-deleted users counted as local users/admins:** `hasLocalUsers`/`hasLocalAdmins` (providers + admin settings) had no `deletedAt: null`, so a deleted local admin kept the login form shown and the Auth tab lockout warnings quiet. Fix: count non-deleted users only. First-user checks unchanged (deleted users still count). - **Admin job routes:** `/api/admin/jobs`, `/api/admin/jobs/:id`, `/api/admin/jobs/:id/trigger` and `/api/admin/job-status/:id` use `requireAuth` + `requireAdmin` like the other admin routes: the account must still exist, its sessions must not be revoked, and the session must be an admin session. API tokens are refused (not on the allowlist). - **Author and series catalog routes:** `/api/authors/:asin`, `/api/authors/:asin/books`, `/api/authors/search`, `/api/series/:asin` and `/api/series/search` use `requireAuth` like the other signed-in routes: the account must still exist, be approved, and its sessions must not be revoked. API tokens get 403 (not on the allowlist; previously 401). Popular, new releases, category and audiobook search are unchanged. diff --git a/src/app/api/requests/[id]/download/route.ts b/src/app/api/requests/[id]/download/route.ts index ad97afa13..4d1436efa 100644 --- a/src/app/api/requests/[id]/download/route.ts +++ b/src/app/api/requests/[id]/download/route.ts @@ -26,6 +26,79 @@ function sanitizeFilename(name: string): string { .slice(0, 200); } +/** + * Zips the files as the client reads the response. A cancel or a failure closes every file the + * zip opened; a failure is logged once and ends the response with an error. + */ +function streamZip(requestId: string, filePaths: string[]): ReadableStream { + // Flat entry names: '/' or '\' becomes '_', so each file stays at the zip root; so does ':', + // since archiver drops a leading 'word:' from entry names. Stat first, so a file that is gone + // fails the request before the zip starts. + const entries = filePaths.map(filePath => ({ + filePath, + name: path.basename(filePath).replace(/[\\/:]/g, '_'), + stats: fs.lstatSync(filePath), + })); + + const output = new PassThrough(); + const archive = new ZipArchive({ zlib: { level: 6 } }); + let current: fs.ReadStream | null = null; + let failed = false; + const fail = (err: Error) => { + if (failed) return; + failed = true; + logger.error('Zip download failed', { requestId, error: err.message }); + output.destroy(err); + }; + archive.on('error', fail); + // A file archiver can't add: fail rather than send a zip without it + archive.on('warning', fail); + // Done, cancelled or failed: stop the archive and close the file being read, if any + output.once('close', () => { + archive.abort(); + current?.destroy(); + }); + archive.pipe(output); + + // One entry at a time: archiver starts reading a stream as soon as it is appended, so the next + // file is opened only when the previous entry is written, and only one file is open at a time + let next = 0; + const appendNext = () => { + if (failed || output.destroyed) return; + if (next === entries.length) { + // Failures arrive on 'error'; the promise only repeats them + archive.finalize().catch(() => {}); + return; + } + const { filePath, name, stats } = entries[next++]; + if (stats.isFile()) { + current = fs.createReadStream(filePath); + current.on('error', fail); + archive.append(current, { name, stats }); + } else { + // A directory or symlink: archiver writes the entry without opening a file + current = null; + archive.file(filePath, { name, stats }); + } + }; + archive.on('entry', appendNext); + appendNext(); + + // Pull-based: the next chunk is read only when the client has taken the last one, so a slow + // reader holds the archive back instead of the zip queueing in memory + const chunks = output[Symbol.asyncIterator](); + return new ReadableStream({ + async pull(controller) { + const { value, done } = await chunks.next(); + if (done) controller.close(); + else controller.enqueue(new Uint8Array(value)); + }, + cancel() { + output.destroy(); + }, + }); +} + /** * GET /api/requests/[id]/download?token= * Token-authenticated file download — no session cookie required. @@ -141,32 +214,8 @@ export async function GET( }); } - // Multiple files — stream zip via archiver (avoids loading all files into memory) - const passThrough = new PassThrough(); - const archive = new ZipArchive({ zlib: { level: 6 } }); - archive.pipe(passThrough); - for (const filePath of matchingFiles) { - // Flat entry names: '/' or '\' becomes '_', so each file stays at the zip root; so does ':', - // since archiver drops a leading 'word:' from entry names - archive.file(filePath, { name: path.basename(filePath).replace(/[\\/:]/g, '_') }); - } - archive.finalize(); - - const zipReadable = new ReadableStream({ - start(controller) { - passThrough.on('data', chunk => controller.enqueue(new Uint8Array(chunk))); - passThrough.on('end', () => controller.close()); - passThrough.on('error', err => { - logger.error('Zip stream error', { error: err.message }); - controller.error(err); - }); - }, - cancel() { - archive.abort(); - }, - }); - - return new NextResponse(zipReadable, { + // Multiple files: a zip built as the client reads it + return new NextResponse(streamZip(id, matchingFiles), { headers: { 'Content-Type': 'application/zip', 'Content-Disposition': `attachment; filename="${sanitizedTitle}.zip"`, diff --git a/tests/api/requests-download.zip-stream.test.ts b/tests/api/requests-download.zip-stream.test.ts new file mode 100644 index 000000000..7221e6506 --- /dev/null +++ b/tests/api/requests-download.zip-stream.test.ts @@ -0,0 +1,227 @@ +/** + * Component: Request File Download Route Tests (zip cancel, read error, slow reader) + * Documentation: documentation/backend/services/auth.md + * + * Runs the real route and the real archiver over real temp files. File handles and bytes read + * are counted by wrapping fs.open, fs.read and fs.close, which every fs.ReadStream calls. + */ + +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { NextRequest } from 'next/server'; +import crypto from 'crypto'; +import fs from 'fs'; +import os from 'os'; +import path from 'path'; +import { createPrismaMock } from '../helpers/prisma'; + +const prismaMock = createPrismaMock(); +const { logged } = vi.hoisted(() => ({ + logged: [] as Array<{ level: string; message: string; metadata?: Record }>, +})); + +vi.mock('@/lib/db', () => ({ + prisma: prismaMock, +})); + +vi.mock('@/lib/utils/logger', () => { + const make = () => ({ + debug: () => {}, + info: () => {}, + warn: (message: string, metadata?: Record) => logged.push({ level: 'warn', message, metadata }), + error: (message: string, metadata?: Record) => logged.push({ level: 'error', message, metadata }), + }); + return { RMABLogger: { create: make, forJob: make } }; +}); + +type Callback = (...args: unknown[]) => void; +type FsFn = (...args: unknown[]) => unknown; +const mutableFs = fs as unknown as Record<'open' | 'read' | 'close', FsFn>; +const realFs = { open: mutableFs.open, read: mutableFs.read, close: mutableFs.close }; + +const MB = 1024 * 1024; +const openFds = new Map(); +let opened = 0; +let bytesRead = 0; +let failReadsOf: string | null = null; + +/** Tracks every file the route opens under `dir`; reads of `failReadsOf` fail with EIO */ +function instrumentFs(dir: string) { + mutableFs.open = function (this: unknown, file: unknown, ...rest: unknown[]) { + const cb = rest[rest.length - 1]; + // archiver passes paths with '/' separators, so compare resolved paths + if (typeof file === 'string' && path.resolve(file).startsWith(dir) && typeof cb === 'function') { + rest[rest.length - 1] = (err: unknown, fd: number) => { + if (!err) { + opened++; + openFds.set(fd, file); + } + (cb as Callback)(err, fd); + }; + } + return realFs.open.call(fs, file, ...rest); + }; + mutableFs.read = function (this: unknown, fd: unknown, ...rest: unknown[]) { + const cb = rest[rest.length - 1]; + const file = typeof fd === 'number' ? openFds.get(fd) : undefined; + if (file && typeof cb === 'function') { + if (failReadsOf && path.basename(file) === failReadsOf) { + const err = Object.assign(new Error('EIO: i/o error, read'), { code: 'EIO', errno: -5, syscall: 'read' }); + process.nextTick(cb as Callback, err); + return undefined; + } + rest[rest.length - 1] = (err: unknown, n: number, buf: unknown) => { + if (!err) bytesRead += n; + (cb as Callback)(err, n, buf); + }; + } + return realFs.read.call(fs, fd, ...rest); + }; + mutableFs.close = function (this: unknown, fd: unknown, ...rest: unknown[]) { + if (typeof fd === 'number') openFds.delete(fd); + return realFs.close.call(fs, fd, ...rest); + }; +} + +async function startZip(): Promise> { + const { generateDownloadToken } = await import('@/lib/utils/jwt'); + const { GET } = await import('@/app/api/requests/[id]/download/route'); + const token = generateDownloadToken('user-1', 'req-1'); + const response = await GET( + new NextRequest(`http://localhost/api/requests/req-1/download?token=${token}`), + { params: Promise.resolve({ id: 'req-1' }) } + ); + expect(response.status).toBe(200); + expect(response.headers.get('content-type')).toBe('application/zip'); + return response.body!.getReader() as ReadableStreamDefaultReader; +} + +const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +async function waitFor(condition: () => boolean, ms: number): Promise { + const end = Date.now() + ms; + while (!condition() && Date.now() < end) await sleep(20); + return condition(); +} + +/** Reads to the end; 'timeout' when the response neither ends nor fails within `ms` */ +async function drain(reader: ReadableStreamDefaultReader, ms: number) { + const read = (async () => { + try { + for (;;) { + if ((await reader.read()).done) return 'done' as const; + } + } catch { + return 'errored' as const; + } + })(); + return Promise.race([read, sleep(ms).then(() => 'timeout' as const)]); +} + +describe('GET /api/requests/[id]/download (zip stream)', () => { + let bookDir: string; + let reader: ReadableStreamDefaultReader | null; + const uncaught: unknown[] = []; + const onUncaught = (err: unknown) => uncaught.push(err); + + function writeParts(count: number, bytes: number) { + const data = crypto.randomBytes(bytes); // random bytes don't compress: the zip is as big as the files + for (let i = 1; i <= count; i++) fs.writeFileSync(path.join(bookDir, `Part ${i}.mp3`), data); + } + + beforeEach(() => { + vi.clearAllMocks(); + logged.length = 0; + uncaught.length = 0; + openFds.clear(); + opened = 0; + bytesRead = 0; + failReadsOf = null; + reader = null; + bookDir = path.resolve(fs.mkdtempSync(path.join(os.tmpdir(), 'rmab-download-zip-stream-'))); + prismaMock.request.findFirst.mockResolvedValue({ + id: 'req-1', + userId: 'user-1', + status: 'downloaded', + type: 'audiobook', + audiobook: { title: 'My Book', filePath: bookDir }, + }); + prismaMock.configuration.findUnique.mockResolvedValue(null); + prismaMock.user.findUnique.mockResolvedValue({ + role: 'user', + downloadAccess: null, + deletedAt: null, + registrationStatus: 'approved', + sessionsInvalidatedAt: null, + }); + process.on('uncaughtException', onUncaught); + instrumentFs(bookDir); + }); + + afterEach(async () => { + await reader?.cancel().catch(() => {}); + await sleep(50); + Object.assign(mutableFs, realFs); + process.off('uncaughtException', onUncaught); + try { + fs.rmSync(bookDir, { recursive: true, force: true, maxRetries: 3 }); + } catch { + // a handle left open by a failing run keeps the folder on Windows + } + }); + + it('closes every file it opened when the client cancels partway', async () => { + writeParts(3, 2 * MB); + reader = await startZip(); + + // past the first entry's header, so a file is being read when the client goes away + let received = 0; + while (received < 512 * 1024) received += (await reader.read()).value?.byteLength ?? 0; + expect(opened).toBeGreaterThan(0); + await reader.cancel(); + reader = null; + + await waitFor(() => openFds.size === 0, 3000); + await sleep(200); + expect({ stillOpen: openFds.size, uncaught: uncaught.map(String) }).toEqual({ stillOpen: 0, uncaught: [] }); + }); + + it('ends the response with an error when a file cannot be read, and logs it once', async () => { + writeParts(3, MB / 4); + failReadsOf = 'Part 2.mp3'; + reader = await startZip(); + + const outcome = await drain(reader, 5000); + + await waitFor(() => openFds.size === 0, 3000); + expect({ outcome, stillOpen: openFds.size, uncaught: uncaught.map(String) }).toEqual({ + outcome: 'errored', + stillOpen: 0, + uncaught: [], + }); + const errors = logged.filter((entry) => entry.level === 'error'); + expect(errors).toHaveLength(1); + expect(errors[0].metadata).toMatchObject({ requestId: 'req-1', error: expect.stringContaining('EIO') }); + }); + + it('reads the files only as fast as the client reads the zip', async () => { + writeParts(5, 20 * MB); // a 100 MB zip + reader = await startZip(); + + await reader.read(); + await sleep(3000); // the client stops reading + const readWhilePaused = bytesRead; + console.log(`[zip-stream] read from disk while the client paused: ${(readWhilePaused / MB).toFixed(1)} MB of 100 MB`); + + expect(readWhilePaused).toBeLessThan(16 * MB); + + // The client reads on: the zip carries on from where it stopped + let received = 0; + while (received < 4 * MB) received += (await reader.read()).value?.byteLength ?? 0; + expect(bytesRead).toBeGreaterThan(readWhilePaused); + await reader.cancel(); + reader = null; + + await waitFor(() => openFds.size === 0, 3000); + expect({ stillOpen: openFds.size, uncaught: uncaught.map(String) }).toEqual({ stillOpen: 0, uncaught: [] }); + }, 40000); +});