fix: NoSuchKey, The specified key does not exist.

This commit is contained in:
Rotimi Best
2026-06-07 23:47:14 +01:00
parent d7225a77b2
commit 5f07184b29
4 changed files with 111 additions and 10 deletions
@@ -339,9 +339,9 @@ export async function encodeAndUploadHls(opts: EncodeAndUploadOptions): Promise<
emit({ stage: 'uploading', percent: 73 });
await uploader.flush();
// HLS audio segments are enough for server-side Whisper prep — no second
// full pass through the source file for a separate audio.m4a upload.
const audioKey = audioTrack ? `${assetId}/audio/playlist.m3u8` : undefined;
// Prefer the dedicated HLS audio playlist when present; otherwise fall
// back to the highest uploaded video rendition (muxed audio is fine for Whisper).
const audioKey = audioTrack ? resolveTranscriptionAudioKey(assetId, rungs, uploader) : undefined;
// --- Thumbnails ----------------------------------------------------
emit({ stage: 'thumbnail', percent: 86 });
@@ -744,6 +744,7 @@ class HlsUploader {
private uploadedCount = 0;
private inflight: Promise<void>[] = [];
private uploadFailures: unknown[] = [];
private readonly uploadedPaths = new Set<string>();
private readonly uploadGate = new ConcurrencyGate(HLS_UPLOAD_CONCURRENCY);
constructor(
@@ -758,6 +759,10 @@ class HlsUploader {
}
) {}
hasUploaded(relativePath: string): boolean {
return this.uploadedPaths.has(relativePath);
}
targetFor(path: string, mimeType: string): BufferTarget {
return new BufferTarget({
onFinalize: (buffer) => {
@@ -801,6 +806,7 @@ class HlsUploader {
});
this.options.presignedUrls.set(path, presigned);
this.uploadedPaths.add(path);
this.totalBytes += u8.byteLength;
this.uploadedCount += 1;
this.options.onUploaded({ totalBytes: this.totalBytes, count: this.uploadedCount });
@@ -960,6 +966,28 @@ function pad(n: number, width: number): string {
return n.toString().padStart(width, '0');
}
function resolveTranscriptionAudioKey(
assetId: string,
rungs: HlsRungConfig[],
uploader: HlsUploader
): string | undefined {
if (uploader.hasUploaded('audio/playlist.m3u8')) {
return `${assetId}/audio/playlist.m3u8`;
}
for (let index = rungs.length - 1; index >= 0; index -= 1) {
const rung = rungs[index];
if (!rung) continue;
const playlistPath = `${rung.name}/playlist.m3u8`;
if (uploader.hasUploaded(playlistPath)) {
return `${assetId}/${playlistPath}`;
}
}
return undefined;
}
/** Rough byte budget for progress UI — actual HLS output varies with content complexity. */
function estimateHlsUploadBytes(rungs: HlsRungConfig[], durationSeconds: number, hasAudio: boolean): number {
const segmentCount = Math.max(1, Math.ceil(durationSeconds / HLS_TARGET_SEGMENT_SECONDS)) + 5;
@@ -2,12 +2,13 @@ import { mkdir, readFile, rename, rm, stat, writeFile } from 'node:fs/promises';
import path from 'node:path';
import { tmpdir } from 'node:os';
import { getJobStep, updateMediaJob, upsertJobStep } from '@cio/db/queries';
import { getAssetById, getJobStep, updateMediaJob, upsertJobStep } from '@cio/db/queries';
import { ZExtractAudioPayload, type TExtractAudioPayload } from '@cio/jobs/payloads/media';
import {
downloadObjectToTempFile,
mediaBucket,
objectExists,
safeUnlink,
uploadFileToBucket,
videosBucket
@@ -83,6 +84,60 @@ async function downloadHlsPlaylistToTempFile(
return { inputPath, tempDir };
}
type HlsAssetMetadata = {
hlsRenditions?: string[];
};
function readHlsRenditions(metadata: unknown): string[] {
if (typeof metadata !== 'object' || metadata === null || Array.isArray(metadata)) {
return [];
}
return (metadata as HlsAssetMetadata).hlsRenditions ?? [];
}
/**
* Pick the first videos-bucket object that actually exists. HLS assets may
* store a dedicated audio playlist, a legacy audio.m4a, or only muxed audio
* inside a video rendition playlist.
*/
async function resolveTranscriptionSourceKey(assetId: string, payloadStorageKey: string): Promise<string> {
const asset = await getAssetById(assetId);
if (!asset) {
throw new Error(`Asset ${assetId} not found`);
}
const candidates: string[] = [];
const push = (key: string | null | undefined) => {
if (key && !candidates.includes(key)) {
candidates.push(key);
}
};
push(asset.hlsAudioKey);
push(payloadStorageKey);
push(asset.storageKey);
if (asset.hlsManifestKey) {
const prefix = `${assetId}/`;
push(`${prefix}audio/playlist.m3u8`);
push(`${prefix}audio.m4a`);
for (const rung of [...readHlsRenditions(asset.metadata)].reverse()) {
push(`${prefix}${rung}/playlist.m3u8`);
}
}
const bucket = videosBucket();
for (const key of candidates) {
if (await objectExists(bucket, key)) {
log.info('extract-audio-source-resolved', { assetId, key });
return key;
}
}
throw new Error(`No transcription source object found for asset ${assetId} (tried ${candidates.join(', ')})`);
}
async function resolveAudioSourcePath(
bucket: string,
storageKey: string
@@ -125,7 +180,8 @@ export async function processExtractAudio(rawData: unknown): Promise<ExtractAudi
let inputTempDir: string | undefined;
let outputPath: string | undefined;
try {
const source = await resolveAudioSourcePath(videosBucket(), storageKey);
const resolvedStorageKey = await resolveTranscriptionSourceKey(assetId, storageKey);
const source = await resolveAudioSourcePath(videosBucket(), resolvedStorageKey);
inputPath = source.inputPath;
inputTempDir = source.tempDir;
+19 -1
View File
@@ -1,4 +1,4 @@
import { GetObjectCommand, PutObjectCommand } from '@aws-sdk/client-s3';
import { GetObjectCommand, HeadObjectCommand, NotFound, PutObjectCommand } from '@aws-sdk/client-s3';
import { Readable } from 'node:stream';
import { createWriteStream } from 'node:fs';
import { mkdir, readFile, unlink } from 'node:fs/promises';
@@ -13,6 +13,24 @@ import { getS3Client, getStorageConfig } from '../config/storage';
* Workers prefer disk over buffers so ffmpeg can seek without holding the
* whole video in RAM.
*/
export async function objectExists(bucket: string, key: string): Promise<boolean> {
try {
await getS3Client().send(new HeadObjectCommand({ Bucket: bucket, Key: key }));
return true;
} catch (error) {
if (error instanceof NotFound || (error as { name?: string }).name === 'NotFound') {
return false;
}
const status = (error as { $metadata?: { httpStatusCode?: number } }).$metadata?.httpStatusCode;
if (status === 404) {
return false;
}
throw error;
}
}
export async function downloadObjectToTempFile(bucket: string, key: string, fileNameHint?: string): Promise<string> {
const dir = path.join(tmpdir(), 'cio-jobs');
await mkdir(dir, { recursive: true });
+3 -4
View File
@@ -1209,10 +1209,9 @@ export async function finalizeHls1080Service(orgId: string, assetId: string, dat
}
/**
* HLS assets ship audio as a separate `audio.m4a` already extracted by
* Mediabunny in the browser. The existing transcription pipeline accepts
* any media key in the videos bucket — its extract-audio leg is a no-op
* on input that's already audio.
* HLS assets ship audio as HLS segments (dedicated `audio/` playlist or muxed
* into a video rendition). The transcription pipeline demuxes those segments
* server-side before calling Whisper.
*/
async function enqueueAudioTranscriptionForAsset(input: {
organizationId: string;