diff --git a/CHANGELOG.md b/CHANGELOG.md index efab3fd..98fbb49 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,25 @@ # Changelog +## [4.36.3] + +- Target the canonical `/v1` sync API routes (`/v1/transcribe`, `/v1/warm`); the unprefixed paths remain served for older SDK versions + +## [4.36.2] + +- Add opt-in `timestamps` sync config option — when `true`, sync words carry accurate `start`/`end` timings at a small latency cost. By default no timings are computed or returned; `SyncWord.start`/`end` are now optional and absent unless requested + +## [4.36.1] + +- Rename the sync transcription surface introduced in 4.36.0 (never published under the old names): `client.direct` → `client.sync`, `Direct*` types → `Sync*`, `directBaseUrl` → `syncBaseUrl`; `word_boost` → `keyterms_prompt` +- The default sync speech model is now `universal-3-5-pro` (sent as the `X-AAI-Model` routing header) + +## [4.36.0] + +- Add `client.sync` — synchronous transcription: audio in, transcript out, one request against the sync API host, with no job id or polling. Accepts a local file path, raw bytes, a Blob, or a readable stream (not URLs), plus config for `prompt`, `keyterms_prompt`, `conversation_context`, `language_codes`, and raw PCM (`sample_rate` + `channels`) +- Add `SyncTranscriber.warm()` — pre-opens the connection to the sync API so the next `transcribe()` skips the DNS + TCP + TLS handshake +- Add `SyncTranscriptError` with `status`, machine-readable `errorCode`, and `retryAfter` (seconds) on 429/503 responses +- Add `syncBaseUrl` client option (defaults to `https://sync.assemblyai.com`) + ## [4.35.4] - `sampleRate` is now optional when `encoding` is `opus` or `ogg_opus` (the Opus stream is self-describing and the server ignores the value). It remains required for PCM encodings and for dual-channel mode; omitting it there now throws at construction time diff --git a/CLAUDE.md b/CLAUDE.md index b503922..4405b61 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -42,6 +42,7 @@ const client = new AssemblyAI({ - `client.transcripts.get(id)` — Retrieve a transcript by ID - `client.transcripts.list()` — List transcripts with pagination - `client.transcripts.delete(id)` — Delete a transcript +- `client.sync.transcribe(audio, config?, options?)` — Synchronous transcription: audio in, transcript out, one request (no polling) - `client.streaming.transcriber(params)` — Create a real-time streaming session ## Common patterns @@ -174,6 +175,83 @@ const srt = await client.transcripts.subtitles(id, "srt"); const vtt = await client.transcripts.subtitles(id, "vtt"); ``` +## Sync transcription (pre-recorded, single request) + +`client.sync` posts a whole audio file and returns the finished transcript in one +round trip — no job id, no polling, no status enum. It targets the sync API host +(`sync.assemblyai.com`, override with the `syncBaseUrl` client option), distinct +from `client.transcripts`' async job API. Use it for short clips where you want the +answer inline; use `client.transcripts` for long-form audio, URLs, or the rich +audio-intelligence features (speaker labels, chapters, sentiment, …) the sync API +doesn't expose. This is the Node counterpart of the Python SDK's `SyncTranscriber`. + +```typescript +const result = await client.sync.transcribe("./call.wav"); +console.log(result.text, result.session_id); +for (const w of result.words) { + console.log(w.text, w.confidence); // w.start/w.end need timestamps: true (see below) +} +``` + +**Input**: a local file path (Node/Bun/Deno), raw bytes (`Uint8Array`/`ArrayBuffer`), +a Blob/File, or a readable stream. **Not** a URL — pass a path/bytes or use +`client.transcripts` for URL ingestion. + +**Config** (all optional, second argument): + +```typescript +const result = await client.sync.transcribe("./call.wav", { + prompt: "Transcribe verbatim. Preserve disfluencies.", // max 4096 chars, rejected over + keyterms_prompt: ["AssemblyAI", "Lemur", "U3-Pro"], // max 2048 chars total, rejected over + language_codes: ["es"], // or e.g. ["en", "es"] for multilingual; defaults to English; ignored when prompt is set + conversation_context: [ + // prior turns oldest-first; capped at 100 turns / 4096 chars — trimmed, not rejected + "I'd like to book a flight to Denver.", + "Sure, what date were you thinking?", + ], +}); +``` + +**Word timestamps** are opt-in. By default words carry `text` and `confidence` only — +`start`/`end` are not calculated. `timestamps: true` computes accurate per-word +timings at a small latency cost: + +```typescript +const result = await client.sync.transcribe("./call.wav", { timestamps: true }); +for (const w of result.words) { + console.log(w.text, w.start, w.end); // milliseconds +} +``` + +**Raw PCM** (S16LE) needs `sample_rate` + `channels`; WAV reads them from its header. +Setting either field routes the audio as `audio/pcm`, and both must be present: + +```typescript +const result = await client.sync.transcribe(rawPcmBytes, { + sample_rate: 16000, + channels: 1, +}); +``` + +**Model**: `config.model` defaults to `"universal-3-5-pro"` and is sent as the +`X-AAI-Model` routing header — never in the request body. + +**Errors**: failures throw `SyncTranscriptError` with `.status`, a +machine-readable `.errorCode` (snake_cased problem-details title: `bad_audio`, +`audio_too_short`, `audio_too_large`, `capacity_exceeded`, `inference_timeout`, …), +and `.retryAfter` (seconds) on 429/503. Audio limits: 80 ms–120 s, ≤40 MB, 16-bit, +mono/stereo, sample rate ∈ {8000, 16000, 22050, 24000, 32000, 44100, 48000}. + +**Pre-warming**: the sync API is one request/response, so a `transcribe()` that +connects on demand pays the full DNS + TCP + TLS handshake on the critical path. +Call `await client.sync.warm()` as soon as you know audio is coming (e.g. while +it is still being recorded); it returns `true` once the socket is open, `false` on +a transport failure. Call it shortly before `transcribe()` — the pooled connection +idles out after a few seconds. + +**Client-side timeout**: third argument — `client.sync.transcribe(audio, {}, { timeout: 30_000 })` +(default 60 s, kept above the server's 30 s deadline). + ## Important gotchas - **.transcribe() polls until complete** — use .submit() for fire-and-forget diff --git a/README.md b/README.md index b02be9c..5870051 100644 --- a/README.md +++ b/README.md @@ -311,6 +311,104 @@ const res = await client.transcripts.delete(transcript.id); +### Transcribe audio synchronously + +`client.sync` posts a whole audio file and returns the finished transcript in one +round trip — no job id, no polling. Use it for short clips where you want the answer +inline; use `client.transcripts` for long-form audio, URLs, or the rich +audio-intelligence features the sync API doesn't expose. + +```typescript +const result = await client.sync.transcribe("./call.wav"); +console.log(result.text, result.session_id); +``` + +The input can be a local file path, raw audio bytes, a Blob, or a readable +stream — but not a URL. + +
+Configure the transcription + +```typescript +const result = await client.sync.transcribe("./call.wav", { + prompt: "Transcribe verbatim. Preserve disfluencies.", // max 4096 chars + keyterms_prompt: ["AssemblyAI", "Lemur"], // max 2048 chars total + language_codes: ["es"], // or e.g. ["en", "es"] for multilingual; defaults to English + conversation_context: [ + // prior turns, oldest first + "I'd like to book a flight to Denver.", + "Sure, what date were you thinking?", + ], +}); +``` + +Raw S16LE PCM audio needs `sample_rate` and `channels`; WAV reads them from its +header. + +```typescript +const result = await client.sync.transcribe(rawPcmBytes, { + sample_rate: 16_000, + channels: 1, +}); +``` + +
+ +
+Get word timestamps + +Word timestamps are opt-in. By default each word in `result.words` carries +`text` and `confidence` only — `start`/`end` are absent. Set `timestamps: true` +to get accurate per-word timings at a small latency cost. + +```typescript +const result = await client.sync.transcribe("./call.wav", { + timestamps: true, +}); +for (const word of result.words) { + console.log(word.text, word.start, word.end); // milliseconds +} +``` + +
+ +
+Pre-warm the connection + +The sync API is a single request/response, so a `transcribe()` that connects on +demand pays the full DNS + TCP + TLS handshake on the critical path. Call `warm()` +as soon as you know audio is coming — for example while it is still being +recorded — so the next `transcribe()` reuses the open connection. + +```typescript +await client.sync.warm(); // fire as recording starts +const audio = await recordUntilDone(); +const result = await client.sync.transcribe(audio); // reuses the hot connection +``` + +
+ +
+Handle errors + +Failures throw a `SyncTranscriptError` with the HTTP `status`, a machine-readable +`errorCode` (`bad_audio`, `audio_too_large`, `capacity_exceeded`, …), and +`retryAfter` (seconds) on 429/503 responses. + +```typescript +import { SyncTranscriptError } from "assemblyai"; + +try { + const result = await client.sync.transcribe("./call.wav"); +} catch (error) { + if (error instanceof SyncTranscriptError) { + console.error(error.status, error.errorCode, error.retryAfter); + } +} +``` + +
+ ### Transcribe streaming audio Refer to [AssemblyAI's streaming documentation](https://www.assemblyai.com/docs/streaming/getting-started/transcribe-streaming-audio) for full code examples. diff --git a/package.json b/package.json index 084397e..20a6c43 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "assemblyai", - "version": "4.35.4", + "version": "4.36.3", "description": "The AssemblyAI JavaScript SDK provides an easy-to-use interface for interacting with the AssemblyAI API, which supports async and real-time transcription, as well as the latest LeMUR models.", "engines": { "node": ">=18" diff --git a/src/services/base.ts b/src/services/base.ts index 7e7875f..312645f 100644 --- a/src/services/base.ts +++ b/src/services/base.ts @@ -19,26 +19,29 @@ export abstract class BaseService { this.userAgent = buildUserAgent(params.userAgent || {}); } } - protected async fetch( + protected async fetchResponse( input: string, init?: RequestInit | undefined, ): Promise { init = { ...DEFAULT_FETCH_INIT, ...init }; - let headers = { + let headers: Record = { Authorization: this.params.apiKey, - "Content-Type": "application/json", }; + // FormData bodies must let fetch set the multipart boundary itself. + if (!(init.body instanceof FormData)) { + headers["Content-Type"] = "application/json"; + } if (DEFAULT_FETCH_INIT?.headers) headers = { ...headers, ...DEFAULT_FETCH_INIT.headers }; - if (init?.headers) headers = { ...headers, ...init.headers }; + if (init?.headers) + headers = { ...headers, ...(init.headers as Record) }; if (this.userAgent) { - (headers as Record)["User-Agent"] = this.userAgent; + headers["User-Agent"] = this.userAgent; if (conditions.browser || conditions.default) { // chromium browsers have a bug where the user agent can't be modified if (typeof window !== "undefined" && "chrome" in window) { - (headers as Record)["AssemblyAI-Agent"] = - this.userAgent; + headers["AssemblyAI-Agent"] = this.userAgent; } } } @@ -46,7 +49,14 @@ export abstract class BaseService { if (!input.startsWith("http")) input = this.params.baseUrl + input; - const response = await fetch(input, init); + return await fetch(input, init); + } + + protected async fetch( + input: string, + init?: RequestInit | undefined, + ): Promise { + const response = await this.fetchResponse(input, init); if (response.status >= 400) { let json: JsonError | undefined; diff --git a/src/services/index.ts b/src/services/index.ts index 0c12913..d92bde1 100644 --- a/src/services/index.ts +++ b/src/services/index.ts @@ -1,4 +1,6 @@ import { BaseServiceParams } from ".."; +import { SyncTranscriber } from "./sync"; +import { SyncTranscriptError } from "../utils/errors"; import { LemurService } from "./lemur"; import { RealtimeTranscriber, @@ -23,6 +25,7 @@ import { const defaultBaseUrl = "https://api.assemblyai.com"; const defaultStreamingUrl = "https://streaming.assemblyai.com"; +const defaultSyncUrl = "https://sync.assemblyai.com"; class AssemblyAI { /** @@ -50,6 +53,11 @@ class AssemblyAI { */ public streaming: StreamingTranscriberFactory; + /** + * The synchronous transcription service. + */ + public sync: SyncTranscriber; + /** * Create a new AssemblyAI client. * @param params - The parameters for the service, including the API key and base URL, if any. @@ -69,11 +77,22 @@ class AssemblyAI { ...params, baseUrl: params.streamingBaseUrl || defaultStreamingUrl, }); + + let syncBaseUrl = params.syncBaseUrl || defaultSyncUrl; + if (syncBaseUrl.endsWith("/")) { + syncBaseUrl = syncBaseUrl.slice(0, -1); + } + this.sync = new SyncTranscriber({ + ...params, + baseUrl: syncBaseUrl, + }); } } export { AssemblyAI, + SyncTranscriber, + SyncTranscriptError, LemurService, RealtimeTranscriberFactory, RealtimeTranscriber, diff --git a/src/services/sync/index.ts b/src/services/sync/index.ts new file mode 100644 index 0000000..6261f89 --- /dev/null +++ b/src/services/sync/index.ts @@ -0,0 +1 @@ +export * from "./service"; diff --git a/src/services/sync/service.ts b/src/services/sync/service.ts new file mode 100644 index 0000000..d65e3b2 --- /dev/null +++ b/src/services/sync/service.ts @@ -0,0 +1,369 @@ +import { readFile } from "#fs"; +import { BaseService } from "../base"; +import { + BaseServiceParams, + SyncAudioInput, + SyncTranscribeOptions, + SyncTranscriptResponse, + SyncTranscriptionConfig, +} from "../.."; +import { defaultSyncSpeechModel } from "../../types/sync"; +import { SyncTranscriptError } from "../../utils/errors/sync"; +import { getPath } from "../../utils/path"; + +// Canonical paths since the sync API gained a /v1 prefix (#18103); the +// unprefixed routes remain served for SDK versions that predate it. +const transcribeEndpoint = "/v1/transcribe"; +const warmEndpoint = "/v1/warm"; +const modelHeader = "X-AAI-Model"; +// Kept above the server's 30 s deadline so the client doesn't race it. +const defaultTimeoutMs = 60_000; +const warmTimeoutMs = 10_000; +const maxPromptLength = 4096; +const maxKeytermsPromptLength = 2048; +const maxContextTurns = 100; +const maxContextLength = 4096; +// Extensions that signal raw S16LE PCM rather than a WAV container. +const pcmSuffixes = [".pcm", ".raw"]; + +/** + * The synchronous transcription service: audio in, transcript out, + * one request. + * + * Unlike `client.transcripts` (which submits a job to the async API and + * polls for completion), `SyncTranscriber` posts the audio to the sync + * API and returns the finished transcript in the HTTP response. There is no + * job id or status to poll. Accepts a local file path, raw audio bytes, a + * Blob, or a readable stream — but not a URL. + */ +export class SyncTranscriber extends BaseService { + /** + * Create a new synchronous transcription service. + * @param params - The parameters to use for the service. + */ + constructor(params: BaseServiceParams) { + super(params); + } + + /** + * Transcribe audio and return the finished transcript in one request. + * @param audio - A local file path, raw audio bytes, a Blob, or a readable + * stream. Raw PCM also requires `sample_rate` and `channels` on the config. + * @param config - Options for this transcription request. + * @param options - Client-side options, such as the request timeout. + * @returns A promise that resolves to the finished transcript. + * @throws SyncTranscriptError when the request fails. + */ + async transcribe( + audio: SyncAudioInput, + config: SyncTranscriptionConfig = {}, + options: SyncTranscribeOptions = {}, + ): Promise { + const { bytes, filename, contentType } = await resolveAudio(audio, config); + + const body = new FormData(); + body.append( + "audio", + new Blob([bytes as BlobPart], { type: contentType }), + filename, + ); + const configJson = buildConfigJson(config); + if (configJson) { + body.append( + "config", + new Blob([JSON.stringify(configJson)], { type: "application/json" }), + ); + } + + const response = await this.fetchResponse(transcribeEndpoint, { + method: "POST", + body, + headers: { [modelHeader]: config.model ?? defaultSyncSpeechModel }, + signal: AbortSignal.timeout(options.timeout ?? defaultTimeoutMs), + }); + if (response.status !== 200) throw await errorFromResponse(response); + return (await response.json()) as SyncTranscriptResponse; + } + + /** + * Open the connection to the sync API ahead of time. + * + * The sync API is a single request/response, so a `transcribe()` that + * opens its connection on demand pays the full DNS + TCP + TLS handshake + * on the critical path. Call `warm()` as soon as you know audio is coming — + * typically while the clip is still being recorded — so the next + * `transcribe()` reuses the already-open connection. `warm()` is idempotent + * and cheap; call it shortly before `transcribe()` so the pooled connection + * hasn't idled out. + * @param params - Optionally the model to route the probe to, so the warmed + * connection lands on the same backend as the eventual transcription. + * @returns A promise that resolves to `true` once the connection is open + * (any HTTP response — even a non-200 — means the socket is + * established), or `false` if the connection could not be opened. + */ + async warm(params?: { model?: string }): Promise { + try { + await this.fetchResponse(warmEndpoint, { + method: "GET", + headers: { [modelHeader]: params?.model ?? defaultSyncSpeechModel }, + signal: AbortSignal.timeout(warmTimeoutMs), + }); + return true; + } catch { + return false; + } + } +} + +type ResolvedAudio = { + bytes: Uint8Array; + filename: string; + contentType: string; +}; + +/** + * Read the audio input into bytes and decide its multipart content type. + * + * PCM is selected when the source has a `.pcm`/`.raw` extension or when + * `sample_rate`/`channels` are set on the config (the fields the sync API + * requires only for raw PCM) — and both must then be present. Everything + * else is treated as a WAV container. URLs are rejected — the sync API has + * no URL ingestion. + */ +async function resolveAudio( + input: SyncAudioInput, + config: SyncTranscriptionConfig, +): Promise { + let bytes: Uint8Array; + let filename: string | undefined; + let suffix = ""; + + if (typeof input === "string") { + if (/^https?:\/\//i.test(input)) { + throw new Error( + "SyncTranscriber does not accept URLs. Pass a local file path or " + + "audio bytes, or use client.transcripts for URL/async transcription.", + ); + } + if (input.startsWith("data:")) { + bytes = dataUrlToBytes(input); + } else { + const path = getPath(input) ?? input; + bytes = await readStream(await readFile(path)); + filename = basename(path); + suffix = extname(filename); + } + } else if (input instanceof Uint8Array) { + bytes = input; + } else if (input instanceof ArrayBuffer) { + bytes = new Uint8Array(input); + } else if (input instanceof Blob) { + bytes = new Uint8Array(await input.arrayBuffer()); + // File instances carry a name; the File global itself needs Node >= 20. + const name = (input as { name?: string }).name; + if (name) { + filename = basename(name); + suffix = extname(filename); + } + } else if (isWebReadableStream(input)) { + bytes = await readStream(input); + } else if (isAsyncIterable(input)) { + bytes = await readAsyncIterable(input); + // fs.ReadStream carries the path it was opened from. + const path = (input as { path?: string | Buffer }).path; + if (typeof path === "string") { + filename = basename(path); + suffix = extname(filename); + } + } else { + throw new TypeError("unsupported audio input type"); + } + + const wantsPcm = + config.sample_rate !== undefined || config.channels !== undefined; + const isPcm = pcmSuffixes.includes(suffix) || wantsPcm; + if ( + isPcm && + (config.sample_rate === undefined || config.channels === undefined) + ) { + throw new Error( + "raw PCM audio requires both sample_rate and channels in the config", + ); + } + + const contentType = isPcm ? "audio/pcm" : "audio/wav"; + if (!filename) filename = isPcm ? "audio.pcm" : "audio.wav"; + + return { bytes, filename, contentType }; +} + +/** + * Serialize the config to the JSON `config` part, validating and normalizing + * field values to match the server's caps. The routing `model` is never + * included — it travels in the `X-AAI-Model` header. Returns `undefined` + * when there is nothing to send, so the part can be omitted entirely. + */ +function buildConfigJson( + config: SyncTranscriptionConfig, +): Record | undefined { + if (config.prompt !== undefined && config.prompt.length > maxPromptLength) { + throw new Error( + `prompt exceeds ${maxPromptLength} characters (got ${config.prompt.length})`, + ); + } + + const json: Record = {}; + if (config.prompt !== undefined) json["prompt"] = config.prompt; + const keytermsPrompt = normalizeKeytermsPrompt(config.keyterms_prompt); + if (keytermsPrompt) json["keyterms_prompt"] = keytermsPrompt; + const context = normalizeConversationContext(config.conversation_context); + if (context) json["conversation_context"] = context; + if (config.language_codes !== undefined) + json["language_codes"] = config.language_codes; + if (config.sample_rate !== undefined) + json["sample_rate"] = config.sample_rate; + if (config.channels !== undefined) json["channels"] = config.channels; + if (config.timestamps !== undefined) json["timestamps"] = config.timestamps; + + return Object.keys(json).length > 0 ? json : undefined; +} + +function normalizeKeytermsPrompt( + keytermsPrompt?: string[], +): string[] | undefined { + if (!keytermsPrompt) return undefined; + const terms = keytermsPrompt + .map((term) => term.trim()) + .filter((term) => term.length > 0); + const total = terms.reduce((sum, term) => sum + term.length, 0); + if (total > maxKeytermsPromptLength) { + throw new Error( + `keyterms_prompt exceeds ${maxKeytermsPromptLength} characters (got ${total})`, + ); + } + return terms.length > 0 ? terms : undefined; +} + +function normalizeConversationContext( + context?: string | string[], +): string[] | undefined { + if (context === undefined) return undefined; + let turns = (typeof context === "string" ? [context] : context) + .map((turn) => turn.trim()) + .filter((turn) => turn.length > 0); + let total = turns.reduce((sum, turn) => sum + turn.length, 0); + // Over-cap context is trimmed oldest-first, never rejected. + while ( + turns.length > 0 && + (turns.length > maxContextTurns || total > maxContextLength) + ) { + total -= turns[0].length; + turns = turns.slice(1); + } + return turns.length > 0 ? turns : undefined; +} + +/** + * Build a SyncTranscriptError from a non-200 response. The primary format + * is an RFC 9457 problem-details body (`status`/`title`/`detail`); legacy + * `{error_code, message}` and `{detail}`-only bodies are also accepted. + */ +async function errorFromResponse( + response: Response, +): Promise { + let errorCode: string | undefined; + let message: string | undefined; + + const text = await response.text(); + try { + const body = JSON.parse(text); + if (body && typeof body === "object" && !Array.isArray(body)) { + if (typeof body.error_code === "string") errorCode = body.error_code; + if (errorCode === undefined && typeof body.title === "string") { + errorCode = body.title.toLowerCase().replace(/ /g, "_"); + } + if (typeof body.detail === "string") message = body.detail; + else if (typeof body.message === "string") message = body.message; + } + } catch { + if (text) message = text; + } + if (!message) { + message = `sync transcription failed with status ${response.status}`; + } + + const retryHeader = response.headers.get("retry-after"); + const retryAfter = + retryHeader && /^\d+$/.test(retryHeader) + ? parseInt(retryHeader, 10) + : undefined; + + return new SyncTranscriptError( + message, + response.status, + errorCode, + retryAfter, + ); +} + +function basename(path: string): string { + return path.split(/[\\/]/).pop() ?? path; +} + +function extname(filename: string): string { + const dotIndex = filename.lastIndexOf("."); + return dotIndex > 0 ? filename.slice(dotIndex).toLowerCase() : ""; +} + +function dataUrlToBytes(dataUrl: string): Uint8Array { + const base64 = dataUrl.split(",")[1]; + const binary = atob(base64); + const bytes = new Uint8Array(binary.length); + for (let i = 0; i < binary.length; i++) bytes[i] = binary.charCodeAt(i); + return bytes; +} + +function isWebReadableStream( + input: unknown, +): input is ReadableStream { + return typeof (input as ReadableStream)?.getReader === "function"; +} + +function isAsyncIterable(input: unknown): input is AsyncIterable { + return ( + typeof (input as AsyncIterable)?.[Symbol.asyncIterator] === + "function" + ); +} + +async function readStream( + stream: ReadableStream, +): Promise { + const chunks: Uint8Array[] = []; + const reader = stream.getReader(); + for (;;) { + const { done, value } = await reader.read(); + if (done) break; + chunks.push(value); + } + return concatChunks(chunks); +} + +async function readAsyncIterable( + iterable: AsyncIterable, +): Promise { + const chunks: Uint8Array[] = []; + for await (const chunk of iterable) chunks.push(chunk); + return concatChunks(chunks); +} + +function concatChunks(chunks: Uint8Array[]): Uint8Array { + const total = chunks.reduce((sum, chunk) => sum + chunk.length, 0); + const bytes = new Uint8Array(total); + let offset = 0; + for (const chunk of chunks) { + bytes.set(chunk, offset); + offset += chunk.length; + } + return bytes; +} diff --git a/src/types/index.ts b/src/types/index.ts index 512ebad..70f9c1d 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -1,3 +1,4 @@ +export * from "./sync"; export * from "./files"; export * from "./transcripts"; export * from "./realtime"; diff --git a/src/types/services/index.ts b/src/types/services/index.ts index 17ac1bc..e881863 100644 --- a/src/types/services/index.ts +++ b/src/types/services/index.ts @@ -4,6 +4,7 @@ type BaseServiceParams = { apiKey: string; baseUrl?: string; streamingBaseUrl?: string; + syncBaseUrl?: string; /** * The AssemblyAI user agent to use for requests. * The provided components will be merged into the default AssemblyAI user agent. diff --git a/src/types/sync/index.ts b/src/types/sync/index.ts new file mode 100644 index 0000000..68750c9 --- /dev/null +++ b/src/types/sync/index.ts @@ -0,0 +1,145 @@ +import { LiteralUnion } from "../helpers"; + +/** + * The speech models available on the synchronous transcription API. + */ +export type SyncSpeechModel = LiteralUnion<"universal-3-5-pro", string>; + +/** + * The default speech model for synchronous transcription. + */ +export const defaultSyncSpeechModel: SyncSpeechModel = "universal-3-5-pro"; + +/** + * Audio input for synchronous transcription: a local file path or + * data URL (file system access requires Node.js, Bun, or Deno), raw audio + * bytes, a Blob/File, or a readable stream. + * + * URLs are not accepted — the sync API has no URL ingestion; use + * `client.transcripts` for URL or asynchronous transcription. + */ +export type SyncAudioInput = + | string + | Uint8Array + | ArrayBuffer + | Blob + | ReadableStream + | NodeJS.ReadableStream; + +/** + * Options for a synchronous transcription request. + * + * `sample_rate` and `channels` are required only for raw PCM audio — WAV + * carries them in its header. `model` is sent as the `X-AAI-Model` routing + * header and is never included in the request body. + */ +export type SyncTranscriptionConfig = { + /** + * The sync speech model to route to, sent as the `X-AAI-Model` header. + * Defaults to `"universal-3-5-pro"`. + */ + model?: SyncSpeechModel; + /** + * Custom transcription instruction. Maximum 4096 characters — longer + * prompts are rejected. + */ + prompt?: string; + /** + * Terms to bias the decoder towards. Whitespace is stripped and empty + * terms are dropped. Maximum 2048 characters in total — longer lists are + * rejected. + */ + keyterms_prompt?: string[]; + /** + * Prior turns from the same conversation, oldest first, most recent last. + * A single string is treated as one turn. Capped at 100 turns and 4096 + * characters in total — over-cap context is trimmed (oldest turns dropped + * first), not rejected. + */ + conversation_context?: string | string[]; + /** + * ISO 639-1 codes for the language(s) of the audio — a single-element + * array (e.g. `["es"]`) for monolingual audio, or several codes (e.g. + * `["en", "es"]`) for multilingual audio. Ignored when `prompt` is set. + * Defaults to English. + */ + language_codes?: string[]; + /** + * The source sample rate in Hz. Required for raw PCM audio; ignored for + * WAV. + */ + sample_rate?: number; + /** + * The channel count (1 for mono, 2 for stereo). Required for raw PCM + * audio; ignored for WAV. + */ + channels?: number; + /** + * Whether to compute per-word `start`/`end` timestamps. When `true`, + * words carry accurate timestamps at a small latency cost. Defaults to + * `false`: no timestamps are returned. + */ + timestamps?: boolean; +}; + +/** + * Client-side options for a synchronous transcription request. + * These are not sent to the server. + */ +export type SyncTranscribeOptions = { + /** + * The request timeout in milliseconds. Defaults to 60 000, which is kept + * above the server's 30 s deadline so the client doesn't race it. + */ + timeout?: number; +}; + +/** + * A single word in a sync transcript. + * + * `start`/`end` are in milliseconds and present only when the request set + * `timestamps: true`; otherwise they are omitted. + */ +export type SyncWord = { + /** The text of the word. */ + text: string; + /** + * The start time of the word in milliseconds. Absent unless `timestamps` + * was requested. + */ + start?: number; + /** + * The end time of the word in milliseconds. Absent unless `timestamps` + * was requested. + */ + end?: number; + /** The confidence score of the word, in the range 0-1. */ + confidence: number; +}; + +/** + * The result of a synchronous transcription request. + */ +export type SyncTranscriptResponse = { + /** The full transcript text. */ + text: string; + /** + * Per-word confidence, plus `start`/`end` timings when the request set + * `timestamps: true`. + */ + words: SyncWord[]; + /** The overall transcript confidence, in the range 0-1. */ + confidence: number; + /** The total audio duration in milliseconds. */ + audio_duration_ms: number; + /** + * The server-generated UUID for this request. Record it to correlate a + * request with support. + */ + session_id: string; + /** + * The end-to-end server-side request time in milliseconds. `undefined` + * when the server predates the field. + */ + request_time_ms?: number; +}; diff --git a/src/utils/errors/index.ts b/src/utils/errors/index.ts index 9813c5f..ef13934 100644 --- a/src/utils/errors/index.ts +++ b/src/utils/errors/index.ts @@ -1,3 +1,5 @@ +export { SyncTranscriptError } from "./sync"; + export { RealtimeError, RealtimeErrorType, diff --git a/src/utils/errors/sync.ts b/src/utils/errors/sync.ts new file mode 100644 index 0000000..52a3180 --- /dev/null +++ b/src/utils/errors/sync.ts @@ -0,0 +1,25 @@ +/** + * Error thrown when a synchronous transcription request fails. + */ +export class SyncTranscriptError extends Error { + override name = "SyncTranscriptError"; + + /** + * Create a new SyncTranscriptError. + * @param message - The human-readable error message. + * @param status - The HTTP status code of the failed request. + * @param errorCode - Machine-readable code — the snake_cased + * problem-details `title` from the server (e.g. `bad_audio`, + * `audio_too_large`, `capacity_exceeded`, `inference_timeout`). + * @param retryAfter - Seconds to wait before retrying, from the + * `Retry-After` header on 429/503 responses. + */ + constructor( + message: string, + public readonly status?: number, + public readonly errorCode?: string, + public readonly retryAfter?: number, + ) { + super(message); + } +} diff --git a/tests/integration/sync.test.ts b/tests/integration/sync.test.ts new file mode 100644 index 0000000..4128a85 --- /dev/null +++ b/tests/integration/sync.test.ts @@ -0,0 +1,33 @@ +import path from "path"; +import "dotenv/config"; +import { AssemblyAI } from "../../src"; + +const testDir = process.env["TESTDATA_DIR"] ?? "tests/static"; + +const client = new AssemblyAI({ + apiKey: process.env.ASSEMBLYAI_API_KEY!, +}); + +describe("sync", () => { + it("should transcribe a local file in one request", async () => { + const result = await client.sync.transcribe( + path.join(testDir, "gore-short.wav"), + ); + + expect(result.text).toBeTruthy(); + expect(result.session_id).toBeTruthy(); + expect(result.audio_duration_ms).toBeGreaterThan(0); + expect(result.words.length).toBeGreaterThan(0); + expect(result.words[0].end).toBeGreaterThan(result.words[0].start); + }); + + it("should warm the connection and then transcribe", async () => { + const warmed = await client.sync.warm(); + expect(warmed).toBe(true); + + const result = await client.sync.transcribe( + path.join(testDir, "gore-short.wav"), + ); + expect(result.text).toBeTruthy(); + }); +}); diff --git a/tests/unit/sync.test.ts b/tests/unit/sync.test.ts new file mode 100644 index 0000000..534665d --- /dev/null +++ b/tests/unit/sync.test.ts @@ -0,0 +1,389 @@ +import { createReadStream } from "fs"; +import fetchMock from "jest-fetch-mock"; +import path from "path"; +import { SyncTranscriptError } from "../../src"; +import { createClient, requestMatches } from "./utils"; + +fetchMock.enableMocks(); + +const testDir = process.env["TESTDATA_DIR"] ?? "tests/static"; + +const assembly = createClient(); + +const fakeWavBytes = new TextEncoder().encode("RIFFfake-wav-bytes"); + +const okResponse = { + text: "hello world", + words: [ + { text: "hello", start: 0, end: 200, confidence: 0.9 }, + { text: "world", start: 220, end: 400, confidence: 0.95 }, + ], + confidence: 0.92, + audio_duration_ms: 400, + session_id: "eb92c4ff-4bbb-429f-9b99-7279d7fe738f", + request_time_ms: 243.7, +}; + +function mockOk() { + fetchMock.doMockOnceIf( + requestMatches({ url: "/v1/transcribe", method: "POST" }), + JSON.stringify(okResponse), + ); +} + +type NamedBlob = Blob & { name?: string }; + +function requestBody(): FormData { + return fetchMock.mock.calls[0][1]!.body as FormData; +} + +function requestHeaders(): Record { + return fetchMock.mock.calls[0][1]!.headers as Record; +} + +async function configPart(): Promise | null> { + const part = requestBody().get("config"); + if (part === null) return null; + return JSON.parse(await (part as Blob).text()); +} + +beforeEach(() => { + fetchMock.resetMocks(); + fetchMock.doMock(); +}); + +describe("sync", () => { + it("should transcribe bytes and parse the response", async () => { + mockOk(); + const result = await assembly.sync.transcribe(fakeWavBytes); + expect(result.text).toBe("hello world"); + expect(result.session_id).toBe(okResponse.session_id); + expect(result.words[0].start).toBe(0); + expect(result.words[0].end).toBe(200); + expect(result.words[1].text).toBe("world"); + expect(result.request_time_ms).toBe(243.7); + }); + + it("should parse a response without request_time_ms", async () => { + const response: Partial = { ...okResponse }; + delete response.request_time_ms; + fetchMock.doMockOnceIf( + requestMatches({ url: "/v1/transcribe", method: "POST" }), + JSON.stringify(response), + ); + const result = await assembly.sync.transcribe(fakeWavBytes); + expect(result.request_time_ms).toBeUndefined(); + }); + + it("should send the model header and a WAV part", async () => { + mockOk(); + await assembly.sync.transcribe(fakeWavBytes); + expect(requestHeaders()["X-AAI-Model"]).toBe("universal-3-5-pro"); + const audio = requestBody().get("audio") as Blob; + expect(audio.type).toBe("audio/wav"); + expect(requestBody().get("config")).toBeNull(); + }); + + it("should send the prompt and normalized keyterms_prompt", async () => { + mockOk(); + await assembly.sync.transcribe(fakeWavBytes, { + prompt: "Transcribe verbatim.", + keyterms_prompt: ["AssemblyAI", " Lemur ", ""], + }); + const config = await configPart(); + expect(config).toEqual({ + prompt: "Transcribe verbatim.", + keyterms_prompt: ["AssemblyAI", "Lemur"], + }); + }); + + it("should never send the model in the config part", async () => { + mockOk(); + await assembly.sync.transcribe(fakeWavBytes, { + model: "some-other-model", + prompt: "Transcribe verbatim.", + }); + expect(requestHeaders()["X-AAI-Model"]).toBe("some-other-model"); + const config = await configPart(); + expect(config).not.toHaveProperty("model"); + }); + + it("should omit the config part when only the model is set", async () => { + mockOk(); + await assembly.sync.transcribe(fakeWavBytes, { + model: "some-other-model", + }); + expect(requestBody().get("config")).toBeNull(); + }); + + it("should send conversation_context turns, stripped with empties dropped", async () => { + mockOk(); + await assembly.sync.transcribe(fakeWavBytes, { + conversation_context: [ + "I'd like to book a flight to Denver.", + " Sure, what date were you thinking? ", + "", + ], + }); + const config = await configPart(); + expect(config?.conversation_context).toEqual([ + "I'd like to book a flight to Denver.", + "Sure, what date were you thinking?", + ]); + }); + + it("should coerce a conversation_context string to a one-turn list", async () => { + mockOk(); + await assembly.sync.transcribe(fakeWavBytes, { + conversation_context: "Sure, what date were you thinking?", + }); + const config = await configPart(); + expect(config?.conversation_context).toEqual([ + "Sure, what date were you thinking?", + ]); + }); + + it("should trim the oldest conversation turns over the char cap", async () => { + mockOk(); + await assembly.sync.transcribe(fakeWavBytes, { + conversation_context: ["a".repeat(3000), "b".repeat(3000)], + }); + const config = await configPart(); + expect(config?.conversation_context).toEqual(["b".repeat(3000)]); + }); + + it("should trim the oldest conversation turns over the turn cap", async () => { + mockOk(); + const turns = Array.from({ length: 120 }, (_, i) => `turn ${i}`); + await assembly.sync.transcribe(fakeWavBytes, { + conversation_context: turns, + }); + const config = await configPart(); + expect(config?.conversation_context).toEqual(turns.slice(20)); + }); + + it("should trim to nothing when a single turn is over the char cap", async () => { + mockOk(); + await assembly.sync.transcribe(fakeWavBytes, { + conversation_context: ["a".repeat(5000)], + }); + expect(requestBody().get("config")).toBeNull(); + }); + + it("should send a single-element language_codes list", async () => { + mockOk(); + await assembly.sync.transcribe(fakeWavBytes, { + language_codes: ["es"], + }); + const config = await configPart(); + expect(config?.language_codes).toEqual(["es"]); + }); + + it("should send a language_codes list", async () => { + mockOk(); + await assembly.sync.transcribe(fakeWavBytes, { + language_codes: ["en", "es"], + }); + const config = await configPart(); + expect(config?.language_codes).toEqual(["en", "es"]); + }); + + it("should omit the config part for a default config", async () => { + mockOk(); + await assembly.sync.transcribe(fakeWavBytes); + expect(requestBody().get("config")).toBeNull(); + }); + + it("should send the timestamps flag when opted in", async () => { + mockOk(); + await assembly.sync.transcribe(fakeWavBytes, { timestamps: true }); + const config = await configPart(); + expect(config).toEqual({ timestamps: true }); + }); + + it("should parse words without start/end timings", async () => { + // Without timestamps in the config, the server omits the fields + // rather than sending null. + const response = { + ...okResponse, + words: [ + { text: "hello", confidence: 0.9 }, + { text: "world", confidence: 0.95 }, + ], + }; + fetchMock.doMockOnceIf( + requestMatches({ url: "/v1/transcribe", method: "POST" }), + JSON.stringify(response), + ); + const result = await assembly.sync.transcribe(fakeWavBytes); + expect(result.words[0].text).toBe("hello"); + expect(result.words[0].start).toBeUndefined(); + expect(result.words[0].end).toBeUndefined(); + expect(result.words[1].confidence).toBe(0.95); + }); + + it("should send a PCM part with rate and channels", async () => { + mockOk(); + await assembly.sync.transcribe(new Uint8Array(200), { + sample_rate: 16000, + channels: 1, + }); + const audio = requestBody().get("audio") as Blob; + expect(audio.type).toBe("audio/pcm"); + const config = await configPart(); + expect(config).toEqual({ sample_rate: 16000, channels: 1 }); + }); + + it("should reject PCM without channels before any request", async () => { + await expect( + assembly.sync.transcribe(new Uint8Array(200), { sample_rate: 16000 }), + ).rejects.toThrow("sample_rate and channels"); + expect(fetchMock).not.toHaveBeenCalled(); + }); + + it("should reject URLs", async () => { + await expect( + assembly.sync.transcribe("https://example.com/audio.wav"), + ).rejects.toThrow("does not accept URLs"); + expect(fetchMock).not.toHaveBeenCalled(); + }); + + it("should ship a file path input under its own name", async () => { + mockOk(); + const result = await assembly.sync.transcribe( + path.join(testDir, "gore-short.wav"), + ); + expect(result.text).toBe("hello world"); + const audio = requestBody().get("audio") as NamedBlob; + expect(audio.name).toBe("gore-short.wav"); + expect(audio.type).toBe("audio/wav"); + }); + + it("should transcribe a Node stream and use its file name", async () => { + mockOk(); + const stream = createReadStream(path.join(testDir, "gore-short.wav")); + const result = await assembly.sync.transcribe(stream); + expect(result.text).toBe("hello world"); + const audio = requestBody().get("audio") as NamedBlob; + expect(audio.name).toBe("gore-short.wav"); + }); + + it("should transcribe a web ReadableStream", async () => { + mockOk(); + const stream = new ReadableStream({ + start(controller) { + controller.enqueue(fakeWavBytes); + controller.close(); + }, + }); + const result = await assembly.sync.transcribe(stream); + expect(result.text).toBe("hello world"); + }); + + it("should transcribe a Blob and use its file name", async () => { + mockOk(); + // Named like a File without the File global, which needs Node >= 20. + const file: NamedBlob = new Blob([fakeWavBytes as BlobPart]); + file.name = "call.wav"; + const result = await assembly.sync.transcribe(file); + expect(result.text).toBe("hello world"); + const audio = requestBody().get("audio") as NamedBlob; + expect(audio.name).toBe("call.wav"); + }); + + it("should reject an oversized keyterms_prompt", async () => { + await expect( + assembly.sync.transcribe(fakeWavBytes, { + keyterms_prompt: ["x".repeat(3000)], + }), + ).rejects.toThrow("keyterms_prompt exceeds"); + expect(fetchMock).not.toHaveBeenCalled(); + }); + + it("should reject an oversized prompt", async () => { + await expect( + assembly.sync.transcribe(fakeWavBytes, { prompt: "x".repeat(5000) }), + ).rejects.toThrow("prompt exceeds"); + expect(fetchMock).not.toHaveBeenCalled(); + }); + + it("should map a problem-details envelope to a SyncTranscriptError", async () => { + fetchMock.mockResponseOnce( + JSON.stringify({ + status: 413, + title: "Audio Too Large", + detail: "too long", + }), + { status: 413 }, + ); + const promise = assembly.sync.transcribe(fakeWavBytes); + await expect(promise).rejects.toThrow(SyncTranscriptError); + await expect(promise).rejects.toMatchObject({ + message: "too long", + status: 413, + errorCode: "audio_too_large", + }); + }); + + it("should map a legacy error envelope to a SyncTranscriptError", async () => { + fetchMock.mockResponseOnce( + JSON.stringify({ error_code: "audio_too_large", message: "too long" }), + { status: 413 }, + ); + await expect(assembly.sync.transcribe(fakeWavBytes)).rejects.toMatchObject({ + message: "too long", + status: 413, + errorCode: "audio_too_large", + }); + }); + + it("should surface retryAfter on rate limits", async () => { + fetchMock.mockResponseOnce( + JSON.stringify({ + status: 429, + title: "Too Many Requests", + detail: "Too many requests", + }), + { status: 429, headers: { "Retry-After": "5" } }, + ); + await expect(assembly.sync.transcribe(fakeWavBytes)).rejects.toMatchObject({ + status: 429, + errorCode: "too_many_requests", + retryAfter: 5, + }); + }); + + it("should map a detail-only envelope without an error code", async () => { + fetchMock.mockResponseOnce(JSON.stringify({ detail: "Invalid API key" }), { + status: 401, + }); + await expect(assembly.sync.transcribe(fakeWavBytes)).rejects.toMatchObject({ + message: "Invalid API key", + status: 401, + errorCode: undefined, + }); + }); + + it("should warm the connection with the model header", async () => { + fetchMock.doMockOnceIf(requestMatches({ url: "/v1/warm", method: "GET" })); + const warmed = await assembly.sync.warm(); + expect(warmed).toBe(true); + expect(requestHeaders()["X-AAI-Model"]).toBe("universal-3-5-pro"); + }); + + it("should warm with the provided model", async () => { + fetchMock.doMockOnceIf(requestMatches({ url: "/v1/warm", method: "GET" })); + await assembly.sync.warm({ model: "some-other-model" }); + expect(requestHeaders()["X-AAI-Model"]).toBe("some-other-model"); + }); + + it("should return true from warm on a non-200 response", async () => { + fetchMock.mockResponseOnce("", { status: 404 }); + expect(await assembly.sync.warm()).toBe(true); + }); + + it("should return false from warm on a transport error", async () => { + fetchMock.mockRejectOnce(new TypeError("connection refused")); + expect(await assembly.sync.warm()).toBe(false); + }); +}); diff --git a/tests/unit/utils.ts b/tests/unit/utils.ts index 7847dff..adc9aa8 100644 --- a/tests/unit/utils.ts +++ b/tests/unit/utils.ts @@ -16,4 +16,8 @@ export function requestMatches(params: { } export const createClient = () => - new AssemblyAI({ baseUrl: defaultBaseUrl, apiKey: defaultApiKey }); + new AssemblyAI({ + baseUrl: defaultBaseUrl, + syncBaseUrl: defaultBaseUrl, + apiKey: defaultApiKey, + });