diff --git a/documentation/phase3/ranking-algorithm.md b/documentation/phase3/ranking-algorithm.md index 477dcbed7..cab49cf20 100644 --- a/documentation/phase3/ranking-algorithm.md +++ b/documentation/phase3/ranking-algorithm.md @@ -1,7 +1,7 @@ # Intelligent Ranking Algorithm **Status:** ✅ Implemented | Comprehensive edge case test coverage -**Tests:** tests/utils/ranking-algorithm.test.ts, ranking-non-latin.test.ts, ranking-match-gates.test.ts, ranking-oracle-223.test.ts, match-gates.test.ts, match-normalize.test.ts, tests/processors/search-indexers-non-latin.test.ts +**Tests:** tests/utils/ranking-algorithm.test.ts, ranking-non-latin.test.ts, ranking-match-gates.test.ts, ranking-oracle-223.test.ts, match-gates.test.ts, match-normalize.test.ts, ranking-size-floor.test.ts, tests/processors/search-indexers-non-latin.test.ts, search-indexers-size-floor.test.ts **Automatic-search gates (#223):** [match-gates.md](match-gates.md): one-word titles, required numbers, author whole words, volume conflict (title-only subset of #330) Evaluates and scores torrents to automatically select best audiobook download. @@ -141,6 +141,13 @@ Evaluates and scores torrents to automatically select best audiobook download. - 1 seeder: 0pts, 10 seeders: 6pts, 100 seeders: 12pts, 1000+: 15pts - Note: Usenet/NZB results without seeders get full 15 pts (centralized availability) +**Pre-filters (dropped before scoring, `rankTorrents`)** +- **< 20 MB:** always dropped (ebooks, samples) +- **Size floor (`minMBPerMinute`, opt-in):** with a known runtime, drops results under `minMBPerMinute` MB per minute. No runtime: no change +- **Who sets it:** only automatic search (`search-indexers.processor.ts`), with `AUTO_SEARCH_MIN_MB_PER_MINUTE = 0.15` (about 21 kbps). Every automatic path (request create, approve, watched lists, RSS, retries, admin re-search) runs through that processor. Interactive search (`requests/[id]/interactive-search`, `audiobooks/search-torrents`) doesn't set it, so it still lists undersized results +- **Why 0.15:** one file of a multi-file Usenet post (57 MB for a 1182-min book, 0.05 MB/min) scored over 50 and was grabbed as the whole book; a complete 0.21 MB/min release must still pass +- **Log:** `Filtered out N results < 20 MB` gains `or < 0.15 MB/min` only when a runtime is known + ## Bonus Points System **Extensible multiplicative bonus system** for external quality factors: @@ -222,6 +229,7 @@ interface RankTorrentsOptions { indexerPriorities?: Map; // indexerId -> priority (1-25) flagConfigs?: IndexerFlagConfig[]; // Flag bonus configurations requireAuthor?: boolean; // Enforce author check (default: true) + minMBPerMinute?: number; // Size floor when runtime known (automatic search: 0.15) } interface BonusModifier { @@ -368,6 +376,7 @@ const bestEbook = ranked[0]; // Safe to auto-download ``` ## Fixed Issues +- **#311 (single-file grabs):** automatic search grabbed one 57 MB chapter file of a 97-file Usenet post for a 1182-min book (score 79.7) and imported it as the book. Fixed by the size floor above; pinned in `tests/utils/ranking-size-floor.test.ts` and `tests/processors/search-indexers-size-floor.test.ts` - **#466 (apostrophe rows):** `L.Intruse…` for "L'intruse", `Changer.L.Eau.Des.Fleurs…`, `Enders.Game…` for "Ender's Game" scored 0 at word coverage (the apostrophe was kept in the word). Fixed by the apostrophe fold; pinned in `tests/utils/ranking-match-gates.test.ts`. Subtitles after `. ` / ` - ` (same issue) still open - **Known limit:** a release that splits an English ending (`Ender.s.Game`) misses `enders` (on main only a request with a curly `’` matched it) - **Known limit:** a request written without the apostrophe ("Enders Game") does not match `Ender’s Game` (as on main): a release possessive needs the apostrophe in the request, so "The Titans" never matches `The Titan's Curse` diff --git a/src/lib/processors/search-indexers.processor.ts b/src/lib/processors/search-indexers.processor.ts index 3561c711a..663283016 100644 --- a/src/lib/processors/search-indexers.processor.ts +++ b/src/lib/processors/search-indexers.processor.ts @@ -6,7 +6,7 @@ import { SearchIndexersPayload, getJobQueueService } from '../services/job-queue.service'; import { prisma } from '../db'; import { getProwlarrService } from '../integrations/prowlarr.service'; -import { getRankingAlgorithm } from '../utils/ranking-algorithm'; +import { getRankingAlgorithm, AUTO_SEARCH_MIN_MB_PER_MINUTE } from '../utils/ranking-algorithm'; import { groupIndexersByCategories, getGroupDescription } from '../utils/indexer-grouping'; import { RMABLogger } from '../utils/logger'; import { getLanguageForRegion } from '../constants/language-config'; @@ -173,7 +173,8 @@ export async function processSearchIndexers(payload: SearchIndexersPayload): Pro const langConfig = getLanguageForRegion(region); // Rank results with indexer priorities and flag configs - // Note: rankTorrents now filters out results < 20 MB internally + // Note: rankTorrents filters out results < 20 MB internally, and here also results under + // the MB-per-minute floor when the runtime is known (one file of a multi-file post, #311) // Use effectiveSearchTitle so custom search terms are respected for ranking // requireAuthor: true (default) - strict filtering for automatic selection const rankedResults = ranker.rankTorrents(searchResults, { @@ -186,12 +187,14 @@ export async function processSearchIndexers(payload: SearchIndexersPayload): Pro requireAuthor: true, // Automatic mode - prevent wrong authors stopWords: langConfig.stopWords, characterReplacements: langConfig.characterReplacements, + minMBPerMinute: AUTO_SEARCH_MIN_MB_PER_MINUTE, // Automatic only; interactive lists everything }); - // Log filter results + // Log filter results (the floor applies only when the runtime is known) const postFilterCount = rankedResults.length; if (postFilterCount < preFilterCount) { - logger.info(`Filtered out ${preFilterCount - postFilterCount} results < ${sizeMBThreshold} MB`); + const floorNote = durationMinutes ? ` or < ${AUTO_SEARCH_MIN_MB_PER_MINUTE} MB/min` : ''; + logger.info(`Filtered out ${preFilterCount - postFilterCount} results < ${sizeMBThreshold} MB${floorNote}`); } // Dual threshold filtering: diff --git a/src/lib/utils/ranking-algorithm.ts b/src/lib/utils/ranking-algorithm.ts index d81c8bc8e..d963ef356 100644 --- a/src/lib/utils/ranking-algorithm.ts +++ b/src/lib/utils/ranking-algorithm.ts @@ -51,8 +51,12 @@ export interface RankTorrentsOptions { requireAuthor?: boolean; // Enforce author presence check (default: true) stopWords?: string[]; // Language-specific stop words for matching characterReplacements?: Record; // Language-specific char replacements (e.g. ß→ss) + minMBPerMinute?: number; // Size floor when runtime is known (opt-in; automatic search only) } +/** Size floor for automatic search, about 21 kbps: drops a single file of a multi-file post (#311) */ +export const AUTO_SEARCH_MIN_MB_PER_MINUTE = 0.15; + export interface EbookTorrentRequest { title: string; author: string; @@ -129,11 +133,17 @@ export class RankingAlgorithm { requireAuthor = true, // Safe default: require author in automatic mode stopWords, characterReplacements, + minMBPerMinute, } = options; - // Filter out files < 20 MB (likely ebooks/samples) + // Filter out files < 20 MB (likely ebooks/samples), and, when a floor is set and the + // runtime is known, results too small for the runtime (one file of a multi-file post) + const runtime = audiobook.durationMinutes; + const floorMB = minMBPerMinute !== undefined && runtime !== undefined && runtime > 0 + ? minMBPerMinute * runtime + : 0; const filteredTorrents = torrents.filter((torrent) => { const sizeMB = torrent.size / (1024 * 1024); - return sizeMB >= 20; + return sizeMB >= 20 && sizeMB >= floorMB; }); const ranked = filteredTorrents.map((torrent) => { diff --git a/tests/api/audiobooks-search.routes.test.ts b/tests/api/audiobooks-search.routes.test.ts index 79aa3fb9b..022ca6774 100644 --- a/tests/api/audiobooks-search.routes.test.ts +++ b/tests/api/audiobooks-search.routes.test.ts @@ -113,6 +113,8 @@ describe('Audiobooks search torrents route', () => { expect(payload.success).toBe(true); expect(payload.results[0].rank).toBe(1); expect(rankTorrentsMock).toHaveBeenCalled(); + // Interactive search shows undersized releases too: only automatic search sets the size floor + expect(rankTorrentsMock.mock.calls[0][2]).not.toHaveProperty('minMBPerMinute'); }); it('ranks all candidates before returning the top 100 with truncation metadata', async () => { diff --git a/tests/api/requests-actions.routes.test.ts b/tests/api/requests-actions.routes.test.ts index 3e4697006..bcde33832 100644 --- a/tests/api/requests-actions.routes.test.ts +++ b/tests/api/requests-actions.routes.test.ts @@ -135,6 +135,8 @@ describe('Request action routes', () => { expect.objectContaining({ title: 'Title', author: 'Author', durationMinutes: 600 }), expect.any(Object) ); + // Interactive search shows undersized releases too: only automatic search sets the size floor + expect(rankTorrentsMock.mock.calls[0][2]).not.toHaveProperty('minMBPerMinute'); }); it('performs interactive search without runtime when no ASIN', async () => { diff --git a/tests/processors/search-indexers-size-floor.test.ts b/tests/processors/search-indexers-size-floor.test.ts new file mode 100644 index 000000000..d0427a96d --- /dev/null +++ b/tests/processors/search-indexers-size-floor.test.ts @@ -0,0 +1,111 @@ +/** + * Component: Search Indexers Processor Size Floor Tests + * Documentation: documentation/phase3/ranking-algorithm.md + */ + +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { createPrismaMock } from '../helpers/prisma'; +import { createJobQueueMock } from '../helpers/job-queue'; + +const MB = 1024 * 1024; + +const prismaMock = createPrismaMock(); +const configMock = vi.hoisted(() => ({ get: vi.fn(), getAudibleRegion: vi.fn() })); +const jobQueueMock = createJobQueueMock(); +const prowlarrMock = vi.hoisted(() => ({ search: vi.fn(), searchWithVariations: vi.fn() })); +// Plain holder, not a factory mockImplementation, so clearAllMocks cannot wipe it +const runtime = vi.hoisted(() => ({ minutes: null as number | null })); + +vi.mock('@/lib/db', () => ({ prisma: prismaMock })); +vi.mock('@/lib/services/config.service', () => ({ getConfigService: () => configMock })); +vi.mock('@/lib/services/job-queue.service', () => ({ getJobQueueService: () => jobQueueMock })); +vi.mock('@/lib/integrations/prowlarr.service', () => ({ getProwlarrService: () => prowlarrMock })); +vi.mock('@/lib/integrations/audible.service', () => ({ + getAudibleService: () => ({ getRuntime: async () => runtime.minutes }), +})); + +const release = (guid: string, title: string, sizeMB: number) => ({ + indexer: 'Usenet', + indexerId: 1, + title, + size: sizeMB * MB, + seeders: undefined, + publishDate: new Date('2024-01-01T00:00:00Z'), + downloadUrl: `https://indexer/${guid}`, + guid, + protocol: 'usenet', +}); + +// One chapter file of a multi-file Usenet post: 57 MB for a 1182-minute book +const singleFile = release('part-44', 'Alastair Reynolds - The Prefect - 44 of 97.mp3', 57); +const fullRelease = release('full', 'Alastair Reynolds - The Prefect (Unabridged) MP3', 1100); + +const audiobook = { id: 'ab-1', title: 'The Prefect', author: 'Alastair Reynolds', asin: 'B0PREFECT1' }; + +const filterLogLines = (spy: { mock: { calls: unknown[][] } }) => + spy.mock.calls.map((c) => String(c[0])).filter((line) => line.includes('Filtered out')); + +async function runSearch() { + const { processSearchIndexers } = await import('@/lib/processors/search-indexers.processor'); + return processSearchIndexers({ requestId: 'req-1', audiobook, jobId: 'job-1' }); +} + +describe('processSearchIndexers size floor', () => { + beforeEach(() => { + vi.clearAllMocks(); + runtime.minutes = 1182; + prismaMock.request.updateMany.mockResolvedValue({ count: 1 }); + prismaMock.blockedRelease.findMany.mockResolvedValue([]); + configMock.getAudibleRegion.mockResolvedValue('us'); + configMock.get.mockImplementation(async (key: string) => { + if (key === 'prowlarr_indexers') { + return JSON.stringify([{ id: 1, name: 'Usenet', protocol: 'usenet', priority: 10, categories: [3030] }]); + } + return null; + }); + }); + + it('does not grab a single file of a multi-file post when the runtime is known', async () => { + prowlarrMock.searchWithVariations.mockResolvedValue([singleFile]); + const log = vi.spyOn(console, 'log').mockImplementation(() => {}); + + const result = await runSearch(); + const lines = filterLogLines(log); + log.mockRestore(); + + expect(result.success).toBe(false); + expect(jobQueueMock.addDownloadJob).not.toHaveBeenCalled(); + expect(prismaMock.request.updateMany).toHaveBeenLastCalledWith( + expect.objectContaining({ data: expect.objectContaining({ status: 'awaiting_search' }) }) + ); + expect(lines).toEqual([expect.stringContaining('Filtered out 1 results < 20 MB or < 0.15 MB/min')]); + }); + + it('grabs the full release and leaves the single file out of the fallbacks', async () => { + prowlarrMock.searchWithVariations.mockResolvedValue([singleFile, fullRelease]); + + await runSearch(); + + expect(jobQueueMock.addDownloadJob).toHaveBeenCalledTimes(1); + const [, , best, alternatives] = jobQueueMock.addDownloadJob.mock.calls[0]; + expect(best.guid).toBe('full'); + expect(alternatives.map((r: { guid: string }) => r.guid)).not.toContain('part-44'); + }); + + it('leaves results alone when no runtime is known', async () => { + runtime.minutes = null; + const tiny = release('tiny', 'Alastair Reynolds - The Prefect sample.mp3', 10); + prowlarrMock.searchWithVariations.mockResolvedValue([singleFile, tiny]); + const log = vi.spyOn(console, 'log').mockImplementation(() => {}); + + await runSearch(); + const lines = filterLogLines(log); + log.mockRestore(); + + // The log names the floor only when a runtime is known + expect(lines).toEqual([expect.stringContaining('Filtered out 1 results < 20 MB')]); + expect(lines[0]).not.toContain('MB/min'); + expect(jobQueueMock.addDownloadJob).toHaveBeenCalledTimes(1); + expect(jobQueueMock.addDownloadJob.mock.calls[0][2].guid).toBe('part-44'); + }); +}); diff --git a/tests/utils/ranking-size-floor.test.ts b/tests/utils/ranking-size-floor.test.ts new file mode 100644 index 000000000..b4d5029b3 --- /dev/null +++ b/tests/utils/ranking-size-floor.test.ts @@ -0,0 +1,64 @@ +/** + * Component: Ranking Size Floor (MB per minute) Tests + * Documentation: documentation/phase3/ranking-algorithm.md + */ + +import { describe, expect, it } from 'vitest'; +import { AUTO_SEARCH_MIN_MB_PER_MINUTE, rankTorrents } from '@/lib/utils/ranking-algorithm'; + +const MB = 1024 * 1024; +const RUNTIME = 1182; // minutes + +const book = { title: 'The Prefect', author: 'Alastair Reynolds', durationMinutes: RUNTIME }; + +const release = (guid: string, title: string, sizeMB: number) => ({ + indexer: 'Usenet', + title, + size: sizeMB * MB, + seeders: undefined, + publishDate: new Date('2024-01-01T00:00:00Z'), + downloadUrl: `https://indexer/${guid}`, + guid, + protocol: 'usenet', +}); + +// One chapter file out of a 97-file Usenet post: 57 MB for a 1182-minute book (0.05 MB/min) +const singleFile = release('part-44', 'Alastair Reynolds - The Prefect - 44 of 97.mp3', 57); +// A complete low-bitrate release at 0.21 MB/min (about 29 kbps) +const lowBitrate = release('low', 'Alastair Reynolds - The Prefect (Unabridged) MP3', Math.ceil(0.21 * RUNTIME)); + +describe('automatic search size floor (minMBPerMinute)', () => { + it('sits at 0.15 MB per minute (about 21 kbps)', () => { + expect(AUTO_SEARCH_MIN_MB_PER_MINUTE).toBe(0.15); + }); + + it('drops a 57 MB single file for a 1182-minute book', () => { + const ranked = rankTorrents([singleFile], book, { minMBPerMinute: AUTO_SEARCH_MIN_MB_PER_MINUTE }); + expect(ranked).toEqual([]); + }); + + it('keeps a complete release at 0.21 MB per minute', () => { + const ranked = rankTorrents([singleFile, lowBitrate], book, { minMBPerMinute: AUTO_SEARCH_MIN_MB_PER_MINUTE }); + expect(ranked.map((r) => r.guid)).toEqual(['low']); + }); + + it('keeps everything when the runtime is unknown', () => { + const ranked = rankTorrents( + [singleFile, lowBitrate], + { title: book.title, author: book.author }, + { minMBPerMinute: AUTO_SEARCH_MIN_MB_PER_MINUTE } + ); + expect(ranked.map((r) => r.guid).sort()).toEqual(['low', 'part-44']); + }); + + it('keeps everything when the option is not set (interactive search)', () => { + const ranked = rankTorrents([singleFile, lowBitrate], book, { requireAuthor: false }); + expect(ranked.map((r) => r.guid).sort()).toEqual(['low', 'part-44']); + }); + + it('still applies the 20 MB filter alongside the floor', () => { + const tiny = release('tiny', 'Alastair Reynolds - The Prefect (Unabridged) MP3', 10); + const ranked = rankTorrents([tiny], { title: book.title, author: book.author }, { minMBPerMinute: AUTO_SEARCH_MIN_MB_PER_MINUTE }); + expect(ranked).toEqual([]); + }); +});