Skip to content

Commit b8b918d

Browse files
author
TubeAgentKit
committed
v0.3.0: batch API, webhook signature helper, transcript provenance fields
- createBatch / getBatch / waitForBatch for POST/GET /batch. - verifyWebhookSignature() for X-GYT-Signature, Web Crypto based (edge friendly) with a node:crypto fallback for Node 18; tested against a vector produced by the API's own signer. - TranscriptData gains requested_language, caption_type, cached, fetched_at; new BatchData / BatchItem / CaptionType types. - README sections, API coverage table, live batch test.
1 parent d5596c2 commit b8b918d

8 files changed

Lines changed: 438 additions & 2 deletions

File tree

‎README.md‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,49 @@ console.log(segments?.[0]); // { start: 3.96, duration: 4.56, text: "So, Reed, e
4848

4949
Each segment is `{ start, duration, text }` with `start` and `duration` in seconds (exported as the `Segment` type).
5050

51+
### Where a transcript came from
52+
53+
| Field | Meaning |
54+
| --- | --- |
55+
| `language_code` | The caption track actually returned |
56+
| `requested_language` | What you asked for. If it differs from `language_code`, YouTube didn't have that language |
57+
| `caption_type` | `"manual"` (uploaded by the creator), `"auto"` (YouTube speech recognition), or `null` if unknown |
58+
| `cached` | `true` when served from the stored copy rather than fetched from YouTube just now |
59+
| `fetched_at` | ISO 8601 time it was fetched from YouTube |
60+
61+
### Batch: many videos at once
62+
63+
Queue up to 100 videos in one call; transcripts are fetched in the background. Submitting is free, each video that returns a transcript costs 1 credit, and failed videos are never charged (10 videos where 2 have no captions = 8 credits).
64+
65+
```ts
66+
const { batch_id } = await client.createBatch({ videos: ["jNQXAC9IVRw", "https://youtu.be/dQw4w9WgXcQ"] });
67+
const result = await client.waitForBatch(batch_id); // polls, then collects every page
68+
69+
for (const item of result.items ?? []) {
70+
if (item.status === "succeeded") console.log(item.video_id, item.caption_type, item.transcript?.slice(0, 80));
71+
else console.log(item.video_id, "failed:", item.error_code); // e.g. TRANSCRIPT_DISABLED
72+
}
73+
console.log("credits used:", result.credits_charged);
74+
```
75+
76+
`getBatch({ id, offset, limit })` returns the status and one page if you'd rather poll yourself. Pass `idempotencyKey` to `createBatch` so a retried call returns the same batch instead of queuing a second one. Batches are kept for 7 days, and an account can have 5 unfinished batches at a time.
77+
78+
To be notified instead of polling, pass `webhookUrl` (public https). The response includes a `webhook_secret` (shown once); each delivery is signed, so check it against the raw body before trusting it:
79+
80+
```ts
81+
import { verifyWebhookSignature } from "@tubeagentkit/getyoutubetranscript";
82+
83+
// e.g. a Next.js route handler
84+
export async function POST(request: Request) {
85+
const body = await request.text(); // raw body, not re-serialized JSON
86+
if (!(await verifyWebhookSignature(body, request.headers.get("x-gyt-signature"), process.env.GYT_WEBHOOK_SECRET!))) {
87+
return new Response("invalid signature", { status: 401 });
88+
}
89+
const event = JSON.parse(body); // { event: "batch.completed", batch_id, results_url, ... }
90+
return new Response("ok");
91+
}
92+
```
93+
5194
## Getting an API key
5295

5396
Every request needs an API key. New accounts get **100 free credits, no card required** - grab one at [getyoutubetranscript.com](https://getyoutubetranscript.com).
@@ -120,6 +163,8 @@ try {
120163
| Method | Endpoint | Credits |
121164
| --- | --- | --- |
122165
| `getTranscript(params)` | `GET /transcript` (optional `timestamps: true`) | 1 |
166+
| `createBatch(params)` | `POST /batch` (up to 100 videos) | 1 per successful video |
167+
| `getBatch(params)` / `waitForBatch(id)` | `GET /batch` | free |
123168
| `search(params)` | `GET /search` | 1 |
124169
| `resolveChannel(params)` | `GET /resolve` | free |
125170
| `getPlaylist(params)` | `GET /playlist` | 1 |

‎package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"name": "@tubeagentkit/getyoutubetranscript",
3-
"version": "0.2.2",
3+
"version": "0.3.0",
44
"description": "YouTube transcript API for Node.js and TypeScript: get YouTube video transcripts, captions and subtitles with timestamps, search YouTube, and list channel and playlist videos. No proxies or headless browser.",
55
"type": "module",
66
"main": "./dist/index.cjs",

‎src/client.ts‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,12 @@ import { GetYouTubeTranscriptError } from "./errors.js";
22
import type {
33
ApiErrorBody,
44
ApiSuccessBody,
5+
BatchData,
6+
BatchItem,
57
ChannelLatestData,
8+
CreateBatchParams,
9+
GetBatchParams,
10+
WaitForBatchOptions,
611
ChannelVideosData,
712
CreditsData,
813
GetChannelLatestParams,
@@ -124,6 +129,19 @@ export class GetYouTubeTranscript {
124129
});
125130
}
126131

132+
private post<T>(path: string, body: Record<string, unknown>, extraHeaders: Record<string, string> = {}): Promise<T> {
133+
return request<T>(this.fetchImpl, `${this.baseUrl}${path}`, {
134+
method: "POST",
135+
headers: {
136+
Authorization: `Bearer ${this.apiKey}`,
137+
"Content-Type": "application/json",
138+
...extraHeaders,
139+
},
140+
// JSON.stringify drops undefined values, so unset options are omitted.
141+
body: JSON.stringify(body),
142+
});
143+
}
144+
127145
/**
128146
* Full transcript for one video, plus title/author/thumbnail metadata. 1 credit.
129147
* Pass `timestamps: true` to also get per-line `segments` (same credit).
@@ -136,6 +154,57 @@ export class GetYouTubeTranscript {
136154
});
137155
}
138156

157+
/**
158+
* Queue transcripts for up to 100 videos in one call. Free to submit:
159+
* 1 credit per video that returns a transcript, failed videos are never
160+
* charged. Follow with `waitForBatch` (or `getBatch`), or pass
161+
* `webhookUrl` to be notified.
162+
*/
163+
createBatch(params: CreateBatchParams): Promise<BatchData> {
164+
if (!params?.videos?.length) {
165+
return Promise.reject(new Error("GetYouTubeTranscript.createBatch: `videos` must contain at least one video"));
166+
}
167+
return this.post<BatchData>(
168+
"/batch",
169+
{
170+
videos: params.videos,
171+
language: params.language,
172+
timestamps: params.timestamps ? true : undefined,
173+
webhook_url: params.webhookUrl,
174+
},
175+
params.idempotencyKey ? { "Idempotency-Key": params.idempotencyKey } : {},
176+
);
177+
}
178+
179+
/** A batch's status and one page of results, in submission order. Free. */
180+
getBatch(params: GetBatchParams): Promise<BatchData> {
181+
return this.get<BatchData>("/batch", { id: params.id, offset: params.offset, limit: params.limit });
182+
}
183+
184+
/** Poll until a batch completes, then return it with every item. Free. */
185+
async waitForBatch(id: string, options: WaitForBatchOptions = {}): Promise<BatchData> {
186+
const { pollIntervalMs = 3000, timeoutMs = 900_000, pageSize = 50 } = options;
187+
const deadline = Date.now() + timeoutMs;
188+
189+
let batch = await this.getBatch({ id, limit: 1 });
190+
while (batch.status !== "completed") {
191+
if (Date.now() >= deadline) {
192+
throw new Error(`GetYouTubeTranscript.waitForBatch: batch ${id} not completed after ${timeoutMs} ms (${batch.pending} pending)`);
193+
}
194+
await new Promise((resolve) => setTimeout(resolve, pollIntervalMs));
195+
batch = await this.getBatch({ id, limit: 1 });
196+
}
197+
198+
const items: BatchItem[] = [];
199+
let offset: number | null | undefined = 0;
200+
while (offset !== null && offset !== undefined) {
201+
batch = await this.getBatch({ id, offset, limit: pageSize });
202+
items.push(...(batch.items ?? []));
203+
offset = batch.next_offset;
204+
}
205+
return { ...batch, items, next_offset: null };
206+
}
207+
139208
/**
140209
* Search YouTube videos or channels. Provide `q` for a first page, or
141210
* `pageToken` (from a previous response's `continuation_token`) to

‎src/index.ts‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,20 @@
11
export { GetYouTubeTranscript, signup, verifySignup } from "./client.js";
22
export { GetYouTubeTranscriptError } from "./errors.js";
3+
export { verifyWebhookSignature } from "./webhooks.js";
4+
export type { VerifyWebhookOptions } from "./webhooks.js";
35
export type {
46
ApiErrorBody,
57
ApiSuccessBody,
8+
BatchData,
9+
BatchItem,
10+
BatchStatus,
11+
CaptionType,
612
ChannelLatestData,
713
ChannelResult,
814
ChannelVideo,
915
ChannelVideosData,
16+
CreateBatchParams,
17+
GetBatchParams,
1018
GetChannelLatestParams,
1119
GetPlaylistParams,
1220
GetTranscriptParams,
@@ -23,6 +31,8 @@ export type {
2331
SearchPagination,
2432
SignupResponse,
2533
TranscriptData,
34+
TranscriptFields,
2635
VerifySignupResponse,
2736
VideoResult,
37+
WaitForBatchOptions,
2838
} from "./types.js";

‎src/types.ts‎

Lines changed: 90 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,19 +46,108 @@ export interface Segment {
4646
text: string;
4747
}
4848

49-
export interface TranscriptData {
49+
/** `manual` = uploaded by the creator, `auto` = YouTube speech recognition, `null` = unknown (older transcripts). */
50+
export type CaptionType = "manual" | "auto" | null;
51+
52+
/** Fields shared by a transcript and a succeeded batch item. */
53+
export interface TranscriptFields {
5054
video_id: string;
55+
/** The caption track actually returned. */
5156
language_code: string;
57+
/** What was asked for. Differs from `language_code` when YouTube didn't have that language. */
58+
requested_language: string;
59+
caption_type: CaptionType;
5260
title: string;
5361
author_name: string;
5462
author_url: string;
5563
thumbnail_url: string;
5664
transcript: string;
5765
word_count: number;
66+
/** ISO 8601 time the transcript was fetched from YouTube. */
67+
fetched_at: string | null;
5868
/** Per-line timing. Present only when the request set `timestamps: true`. */
5969
segments?: Segment[];
6070
}
6171

72+
export interface TranscriptData extends TranscriptFields {
73+
/** `true` when served from the stored copy rather than fetched from YouTube just now. */
74+
cached: boolean;
75+
}
76+
77+
// ---------------------------------------------------------------------------
78+
// POST /batch, GET /batch
79+
// ---------------------------------------------------------------------------
80+
81+
export interface CreateBatchParams {
82+
/** 1-100 video URLs or 11-character IDs. Duplicates are fetched once. */
83+
videos: string[];
84+
/** Caption language code for every video. Defaults to 'en'. */
85+
language?: string;
86+
/** Include per-line `segments` in the results. */
87+
timestamps?: boolean;
88+
/** Public https URL (port 443) that receives a signed `batch.completed` POST. See `verifyWebhookSignature`. */
89+
webhookUrl?: string;
90+
/** Sent as the `Idempotency-Key` header: a retry with the same key returns the original batch. */
91+
idempotencyKey?: string;
92+
}
93+
94+
export interface GetBatchParams {
95+
/** The `batch_id` from `createBatch`. */
96+
id: string;
97+
/** Items to skip. Defaults to 0. */
98+
offset?: number;
99+
/** Items per page, 1-50. Defaults to 20. */
100+
limit?: number;
101+
}
102+
103+
export interface WaitForBatchOptions {
104+
/** Milliseconds between status checks. Defaults to 3000. */
105+
pollIntervalMs?: number;
106+
/** Give up after this many milliseconds. Defaults to 900000 (15 minutes). */
107+
timeoutMs?: number;
108+
/** Items fetched per request once complete, 1-50. Defaults to 50. */
109+
pageSize?: number;
110+
}
111+
112+
export type BatchStatus = "queued" | "processing" | "completed";
113+
114+
/** One video in a batch: succeeded items carry the transcript fields, failed ones an `error_code`. */
115+
export interface BatchItem extends Partial<TranscriptFields> {
116+
/** Index in the submitted `videos` list (after de-duplication). */
117+
position: number;
118+
video_id: string;
119+
status: "pending" | "succeeded" | "failed";
120+
/** `true` when this video used a credit. */
121+
charged: boolean;
122+
/** e.g. `TRANSCRIPT_DISABLED`, `VIDEO_UNAVAILABLE`, `PAYMENT_REQUIRED`. */
123+
error_code?: string;
124+
}
125+
126+
export interface BatchData {
127+
batch_id: string;
128+
status: BatchStatus;
129+
language: string;
130+
timestamps: boolean;
131+
total: number;
132+
succeeded: number;
133+
failed: number;
134+
pending: number;
135+
webhook_status: "none" | "pending" | "delivered" | "failed";
136+
created_at: string;
137+
completed_at: string | null;
138+
results_url: string;
139+
/** `createBatch` only, and only when `webhookUrl` was given. Shown once. */
140+
webhook_secret?: string;
141+
/** `createBatch`: `true` when the Idempotency-Key matched an earlier batch. */
142+
idempotent_replay?: boolean;
143+
/** `getBatch` / `waitForBatch`. */
144+
credits_charged?: number;
145+
/** `getBatch`: one page. `waitForBatch`: every item. */
146+
items?: BatchItem[];
147+
/** `getBatch`: offset of the next page, `null` on the last one. */
148+
next_offset?: number | null;
149+
}
150+
62151
// ---------------------------------------------------------------------------
63152
// GET /search
64153
// ---------------------------------------------------------------------------

‎src/webhooks.ts‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
/**
2+
* Verify batch-completion webhooks sent by the GetYouTubeTranscript API.
3+
*
4+
* Each delivery carries an `X-GYT-Signature: t=<unix seconds>,v1=<hex>`
5+
* header, where `v1` is the HMAC-SHA256 of `"<t>.<raw request body>"` keyed
6+
* with the `webhook_secret` that `createBatch` returned.
7+
*
8+
* Uses Web Crypto, so it also runs on edge runtimes; on Node 18, where the
9+
* global isn't enabled by default, it falls back to `node:crypto`'s copy.
10+
*/
11+
12+
export interface VerifyWebhookOptions {
13+
/** Reject signatures older (or newer) than this many seconds, to limit replays. 0 skips the check. Defaults to 300. */
14+
toleranceSeconds?: number;
15+
/** Current Unix time in seconds, for tests. Defaults to now. */
16+
now?: number;
17+
}
18+
19+
async function subtleCrypto(): Promise<SubtleCrypto> {
20+
if (globalThis.crypto?.subtle) return globalThis.crypto.subtle;
21+
// Node 18 without --experimental-global-webcrypto. The specifier is kept out of the
22+
// type graph so the package doesn't need @types/node to build.
23+
const nodeCryptoSpecifier = "node:crypto";
24+
const nodeCrypto = (await import(/* @vite-ignore */ nodeCryptoSpecifier)) as { webcrypto: { subtle: SubtleCrypto } };
25+
return nodeCrypto.webcrypto.subtle;
26+
}
27+
28+
function toHex(buffer: ArrayBuffer): string {
29+
return Array.from(new Uint8Array(buffer), (byte) => byte.toString(16).padStart(2, "0")).join("");
30+
}
31+
32+
/** Compares in time independent of where the strings first differ. */
33+
function constantTimeEqual(a: string, b: string): boolean {
34+
if (a.length !== b.length) return false;
35+
let diff = 0;
36+
for (let i = 0; i < a.length; i++) diff |= a.charCodeAt(i) ^ b.charCodeAt(i);
37+
return diff === 0;
38+
}
39+
40+
/**
41+
* Resolves to `true` if a webhook delivery is genuine and recent.
42+
*
43+
* @param body The raw request body exactly as received (e.g. `await request.text()`).
44+
* Don't re-serialize parsed JSON: any whitespace change breaks the signature.
45+
* @param signatureHeader The `X-GYT-Signature` header value.
46+
* @param secret The batch's `webhook_secret` (`whsec_...`).
47+
*/
48+
export async function verifyWebhookSignature(
49+
body: string | Uint8Array,
50+
signatureHeader: string | null | undefined,
51+
secret: string,
52+
options: VerifyWebhookOptions = {},
53+
): Promise<boolean> {
54+
if (!signatureHeader || !secret) return false;
55+
const parts = new Map(
56+
signatureHeader.split(",").map((part) => {
57+
const index = part.indexOf("=");
58+
return [part.slice(0, index), part.slice(index + 1)] as const;
59+
}),
60+
);
61+
const timestamp = parts.get("t") ?? "";
62+
const signature = parts.get("v1") ?? "";
63+
if (!/^\d+$/.test(timestamp) || !signature) return false;
64+
65+
const { toleranceSeconds = 300, now = Date.now() / 1000 } = options;
66+
if (toleranceSeconds && Math.abs(now - Number(timestamp)) > toleranceSeconds) return false;
67+
68+
const encoder = new TextEncoder();
69+
const rawBody = typeof body === "string" ? encoder.encode(body) : body;
70+
const signed = new Uint8Array(timestamp.length + 1 + rawBody.length);
71+
signed.set(encoder.encode(`${timestamp}.`));
72+
signed.set(rawBody, timestamp.length + 1);
73+
74+
const subtle = await subtleCrypto();
75+
const key = await subtle.importKey("raw", encoder.encode(secret), { name: "HMAC", hash: "SHA-256" }, false, ["sign"]);
76+
const expected = toHex(await subtle.sign("HMAC", key, signed));
77+
return constantTimeEqual(expected, signature);
78+
}

0 commit comments

Comments
 (0)