feat: make video uploads faster

- Implemented a new endpoint to delete failed manual p1080 renditions, restoring the master manifest and updating metadata. - Enhanced the HLS upload process with better error handling and cleanup for aborted uploads. - Introduced tests for HLS encoder functionality to ensure correct selection of rungs based on video source properties. - Updated video upload component to manage reserved asset IDs and perform cleanup on abo
2026-06-07 23:20:42 +01:00
parent 27a612e92f
commit 4eee866e85
6 changed files with 357 additions and 52 deletions
@@ -19,6 +19,7 @@ import {
 import { ZUpdateTranscript } from '@cio/utils/validation/media';
 import {
  abortHlsAssetService,
+  abortHls1080RenditionService,
  attachAssetService,
  batchPresignHlsService,
  batchPresignHls1080Service,
@@ -228,6 +229,22 @@ export const assetsRouter = new Hono()
      return handleError(c, error, 'Failed to abort HLS asset');
    }
  })
+  /**
+   * DELETE /organization/assets/:assetId/hls/1080
+   * Drop a failed manual p1080 rendition: delete partial segments, restore
+   * the master manifest, and reset rendition metadata.
+   */
+  .delete('/:assetId/hls/1080', authMiddleware, orgMemberMiddleware, zValidator('param', ZAssetGetParam), async (c) => {
+    try {
+      const orgId = c.req.header('cio-org-id')!;
+      const { assetId } = c.req.valid('param');
+      const updated = await abortHls1080RenditionService(orgId, assetId);
+
+      return c.json({ success: true, data: updated }, 200);
+    } catch (error) {
+      return handleError(c, error, 'Failed to abort 1080p HLS rendition');
+    }
+  })
  /**
   * POST /organization/assets/:assetId/hls/cookie
   * After entitlement check, set a 15-minute HMAC-signed cookie scoped to
@@ -0,0 +1,31 @@
+import { describe, expect, it, vi } from 'vitest';
+
+vi.mock('$lib/utils/services/api', () => ({
+  classroomio: {}
+}));
+
+import { ALL_HLS_RUNGS, selectRungsForSource } from './hls-encoder';
+
+function mockTrack(height: number) {
+  return { displayHeight: height, displayWidth: Math.round(height * (16 / 9)) };
+}
+
+describe('selectRungsForSource', () => {
+  it('returns 360p and 720p for HD sources under 100MB', () => {
+    const rungs = selectRungsForSource(mockTrack(1080) as never, 720, 50 * 1024 * 1024);
+
+    expect(rungs.map((rung) => rung.name)).toEqual(['p360', 'p720']);
+  });
+
+  it('returns only the highest rung for large HD sources', () => {
+    const rungs = selectRungsForSource(mockTrack(1080) as never, 720, 195 * 1024 * 1024);
+
+    expect(rungs).toEqual([ALL_HLS_RUNGS.find((rung) => rung.name === 'p720')]);
+  });
+
+  it('returns a single 360p rung for low-resolution sources', () => {
+    const rungs = selectRungsForSource(mockTrack(480) as never, 720, 200 * 1024 * 1024);
+
+    expect(rungs.map((rung) => rung.name)).toEqual(['p360']);
+  });
+});
@@ -16,7 +16,6 @@ import {
  HlsOutputFormat,
  Input,
  MpegTsOutputFormat,
-  Mp4OutputFormat,
  Output,
  PathedTarget,
  ALL_FORMATS,
@@ -70,18 +69,34 @@ export const ALL_HLS_RUNGS: HlsRungConfig[] = [
 export const P1080_RUNG = ALL_HLS_RUNGS.find((rung) => rung.name === 'p1080')!;

 const DEFAULT_MAX_RUNG_HEIGHT = 720;
+/** Above this size, encode a single ladder rung to avoid doubling encode time on large sources. */
+const LARGE_FILE_SINGLE_RUNG_BYTES = 100 * 1024 * 1024;

 /** Source-capped rungs up to `maxHeight` — never upscales beyond the source. */
-export function selectRungsForSource(track: InputVideoTrack, maxHeight = DEFAULT_MAX_RUNG_HEIGHT): HlsRungConfig[] {
+export function selectRungsForSource(
+  track: InputVideoTrack,
+  maxHeight = DEFAULT_MAX_RUNG_HEIGHT,
+  fileSizeBytes?: number
+): HlsRungConfig[] {
  const sourceHeight = track.displayHeight ?? 0;
+  const eligible = ALL_HLS_RUNGS.filter((rung) => rung.height <= Math.min(maxHeight, sourceHeight));

-  return ALL_HLS_RUNGS.filter((rung) => rung.height <= Math.min(maxHeight, sourceHeight));
+  if (!eligible.length) return [];
+
+  // Large uploads spend most of the wait in WebCodecs. A single rung (typically
+  // 720p) finishes roughly twice as fast as a 360p+720p ladder while still
+  // delivering HD playback; smaller files keep the full ABR ladder.
+  if (fileSizeBytes != null && fileSizeBytes >= LARGE_FILE_SINGLE_RUNG_BYTES && eligible.length > 1) {
+    return [eligible[eligible.length - 1]!];
+  }
+
+  return eligible;
 }

 const AUDIO_BITRATE = 128_000;
 const HLS_TARGET_SEGMENT_SECONDS = 4;
 /** Cap parallel segment PUTs — unbounded concurrency can overwhelm R2 and fail mid-upload. */
-const HLS_UPLOAD_CONCURRENCY = 8;
+const HLS_UPLOAD_CONCURRENCY = 16;
 const HLS_PUT_MAX_ATTEMPTS = 4;
 /** Server validation allows up to 2000 paths per presign request. */
 const PRESIGN_BATCH_SIZE = 1500;
@@ -103,6 +118,7 @@ export interface EncodeAndUploadOptions {
  title?: string;
  rungs?: HlsRungConfig[];
  onProgress?: (progress: HlsEncodeProgress) => void;
+  onAssetReserved?: (assetId: string) => void;
  signal?: AbortSignal;
 }

@@ -113,6 +129,54 @@ export interface EncodeAndUpload1080Options {
  signal?: AbortSignal;
 }

+const CLEANUP_MAX_ATTEMPTS = 3;
+
+/** Best-effort cleanup for a reserved `processing` HLS asset after failure or cancel. */
+export async function cleanupAbortedHlsUpload(assetId: string, signal?: AbortSignal): Promise<void> {
+  for (let attempt = 1; attempt <= CLEANUP_MAX_ATTEMPTS; attempt += 1) {
+    try {
+      const response = await classroomio.organization.assets[':assetId'].hls.$delete({
+        param: { assetId },
+        signal
+      });
+
+      if (response.ok) return;
+
+      if (response.status === 404 || response.status === 409) return;
+    } catch (error) {
+      if (attempt === CLEANUP_MAX_ATTEMPTS) {
+        console.warn('Failed to abort HLS asset after encode error', error);
+        return;
+      }
+    }
+
+    await sleep(Math.min(500 * 2 ** (attempt - 1), 4000));
+  }
+}
+
+/** Best-effort cleanup for a failed manual p1080 rendition on an active asset. */
+export async function cleanupAbortedHls1080Upload(assetId: string, signal?: AbortSignal): Promise<void> {
+  for (let attempt = 1; attempt <= CLEANUP_MAX_ATTEMPTS; attempt += 1) {
+    try {
+      const response = await classroomio.organization.assets[':assetId'].hls['1080'].$delete({
+        param: { assetId },
+        signal
+      });
+
+      if (response.ok) return;
+
+      if (response.status === 404 || response.status === 409) return;
+    } catch (error) {
+      if (attempt === CLEANUP_MAX_ATTEMPTS) {
+        console.warn('Failed to abort partial 1080p HLS rendition', error);
+        return;
+      }
+    }
+
+    await sleep(Math.min(500 * 2 ** (attempt - 1), 4000));
+  }
+}
+
 /**
 * End-to-end: encode the file to HLS, push to R2, finalize the asset.
 * Resolves with the new `assetId` and HLS manifest key.
@@ -148,7 +212,7 @@ export async function encodeAndUploadHls(opts: EncodeAndUploadOptions): Promise<
    const aspectRatio = computeAspectRatioLabel(videoTrack);
    const sourceWidth = videoTrack.displayWidth ?? 0;
    const sourceHeight = videoTrack.displayHeight ?? 0;
-    const rungs = opts.rungs ?? selectRungsForSource(videoTrack, DEFAULT_MAX_RUNG_HEIGHT);
+    const rungs = opts.rungs ?? selectRungsForSource(videoTrack, DEFAULT_MAX_RUNG_HEIGHT, opts.file.size);
    if (rungs.length === 0) {
      throw new HlsEncoderError('encode_failed', 'Source resolution is too low for HLS encoding');
    }
@@ -168,6 +232,7 @@ export async function encodeAndUploadHls(opts: EncodeAndUploadOptions): Promise<
    }
    const { assetId } = initBody.data;
    reservedAssetId = assetId;
+    opts.onAssetReserved?.(assetId);

    // --- Pre-presign every file Mediabunny will write ------------------
    // Segments are deterministic from rung name + segment index, so we
@@ -177,12 +242,20 @@ export async function encodeAndUploadHls(opts: EncodeAndUploadOptions): Promise<
    emit({ stage: 'encoding', percent: 1 });
    const expectedPaths = predictHlsPaths(rungs, duration, Boolean(audioTrack));
    const presignedUrls = await batchPresignPaths(assetId, expectedPaths);
+    const expectedUploadBytes = estimateHlsUploadBytes(rungs, duration, Boolean(audioTrack));

    const uploader = new HlsUploader({
      assetId,
      presignedUrls,
      onUploaded: ({ totalBytes, count }) => {
-        emit({ uploadedBytes: totalBytes, uploadedCount: count });
+        const uploadFraction = Math.min(1, totalBytes / expectedUploadBytes);
+        const uploadPercent = 73 + Math.round(uploadFraction * 22);
+        emit({
+          uploadedBytes: totalBytes,
+          uploadedCount: count,
+          stage: uploadFraction > 0 ? 'uploading' : progress.stage,
+          percent: Math.max(progress.percent, uploadPercent)
+        });
      },
      signal: opts.signal
    });
@@ -247,8 +320,12 @@ export async function encodeAndUploadHls(opts: EncodeAndUploadOptions): Promise<
    });

    conversion.onProgress = (fraction) => {
-      // Reserve 70% of the bar for the encode itself; uploads finish behind it.
-      emit({ percent: Math.min(72, 2 + Math.round(fraction * 70)) });
+      // Reserve ~70% of the bar for encode; segment uploads continue in parallel.
+      const encodePercent = Math.min(72, 2 + Math.round(fraction * 70));
+      emit({
+        stage: progress.stage === 'uploading' ? 'uploading' : 'encoding',
+        percent: Math.max(progress.percent, encodePercent)
+      });
    };

    if (!conversion.isValid) {
@@ -259,30 +336,12 @@ export async function encodeAndUploadHls(opts: EncodeAndUploadOptions): Promise<
    }

    await conversion.execute();
+    emit({ stage: 'uploading', percent: 73 });
    await uploader.flush();

-    // --- Audio-only output for transcription --------------------------
-    let audioKey: string | undefined;
-    if (audioTrack) {
-      const audioOutput = new Output({
-        format: new Mp4OutputFormat(),
-        target: new BufferTarget({
-          onFinalize: async (buffer) => {
-            audioKey = await uploader.uploadSingle('audio.m4a', 'audio/mp4', buffer);
-          }
-        })
-      });
-      const audioConversion = await Conversion.init({
-        input,
-        output: audioOutput,
-        video: { discard: true },
-        audio: { codec: 'aac', bitrate: AUDIO_BITRATE, numberOfChannels: 1, sampleRate: 16_000 },
-        showWarnings: false
-      });
-      if (audioConversion.isValid) {
-        await audioConversion.execute();
-      }
-    }
+    // HLS audio segments are enough for server-side Whisper prep — no second
+    // full pass through the source file for a separate audio.m4a upload.
+    const audioKey = audioTrack ? `${assetId}/audio/playlist.m3u8` : undefined;

    // --- Thumbnails ----------------------------------------------------
    emit({ stage: 'thumbnail', percent: 86 });
@@ -339,11 +398,7 @@ export async function encodeAndUploadHls(opts: EncodeAndUploadOptions): Promise<
    // already sees the original failure, and a stuck `processing` row is
    // recoverable via an admin sweep anyway.
    if (reservedAssetId) {
-      try {
-        await classroomio.organization.assets[':assetId'].hls.$delete({ param: { assetId: reservedAssetId } });
-      } catch (cleanupError) {
-        console.warn('Failed to abort HLS asset after encode error', cleanupError);
-      }
+      await cleanupAbortedHlsUpload(reservedAssetId, opts.signal);
    }

    throw error;
@@ -484,6 +539,8 @@ export async function encodeAndUploadHls1080(opts: EncodeAndUpload1080Options):
    const code = error instanceof HlsEncoderError ? error.code : 'encode_failed';
    const message = error instanceof Error ? error.message : 'Unknown encoder error';
    emit({ stage: 'failed', error: `${code}: ${message}` });
+
+    await cleanupAbortedHls1080Upload(opts.assetId, opts.signal);
    throw error;
  }
 }
@@ -512,9 +569,8 @@ interface PresignedUrl {
 *
 * When the source has audio, Mediabunny's HLS output emits a separate
 * `audio/` playlist + its own segments alongside the video rungs (the
- * master playlist groups them via `#EXT-X-MEDIA`). We presign those too.
- * The trailing `audio.m4a` is the separate transcription-only output we
- * run after the HLS conversion.
+ * master playlist groups them via `#EXT-X-MEDIA`). We presign those too;
+ * the jobs worker demuxes that playlist for Whisper transcription.
 */
 function predictHlsPaths(rungs: HlsRungConfig[], duration: number, hasAudio: boolean): string[] {
  const paths: string[] = ['master.m3u8'];
@@ -531,7 +587,6 @@ function predictHlsPaths(rungs: HlsRungConfig[], duration: number, hasAudio: boo
    for (let i = 1; i <= segmentCount; i++) {
      paths.push(`audio/seg-${pad(i, 5)}.ts`);
    }
-    paths.push('audio.m4a');
  }

  return paths;
@@ -705,8 +760,8 @@ class HlsUploader {

  targetFor(path: string, mimeType: string): BufferTarget {
    return new BufferTarget({
-      onFinalize: async (buffer) => {
-        await this.uploadSingle(path, mimeType, buffer);
+      onFinalize: (buffer) => {
+        void this.uploadSingle(path, mimeType, buffer);
      }
    });
  }
@@ -731,9 +786,7 @@ class HlsUploader {
      }

      const contentType = uploadContentTypeForPath(path, mimeType);
-      const owned = new Uint8Array(u8.byteLength);
-      owned.set(u8);
-      const body = new Blob([owned], { type: contentType });
+      const body = new Blob([u8], { type: contentType });
      presigned = await putPresignedObject({
        path,
        presigned,
@@ -748,7 +801,7 @@ class HlsUploader {
      });
      this.options.presignedUrls.set(path, presigned);

-      this.totalBytes += owned.byteLength;
+      this.totalBytes += u8.byteLength;
      this.uploadedCount += 1;
      this.options.onUploaded({ totalBytes: this.totalBytes, count: this.uploadedCount });
      return presigned.key;
@@ -907,6 +960,17 @@ function pad(n: number, width: number): string {
  return n.toString().padStart(width, '0');
 }

+/** Rough byte budget for progress UI — actual HLS output varies with content complexity. */
+function estimateHlsUploadBytes(rungs: HlsRungConfig[], durationSeconds: number, hasAudio: boolean): number {
+  const segmentCount = Math.max(1, Math.ceil(durationSeconds / HLS_TARGET_SEGMENT_SECONDS)) + 5;
+  const playlistOverhead = 32 * 1024 * (rungs.length + (hasAudio ? 2 : 1));
+  const videoBytes =
+    rungs.reduce((sum, rung) => sum + rung.videoBitrate * durationSeconds, 0) / 8 + segmentCount * rungs.length * 4096;
+  const audioBytes = hasAudio ? (AUDIO_BITRATE * durationSeconds) / 8 + segmentCount * 4096 : 0;
+
+  return Math.max(1, Math.round(playlistOverhead + videoBytes + audioBytes));
+}
+
 async function fetchExistingMasterManifest(assetId: string, signal?: AbortSignal): Promise<string> {
  await classroomio.organization.assets[':assetId'].hls.cookie.$post({ param: { assetId } });
  const response = await fetch(`/hls/${assetId}/master.m3u8`, { credentials: 'include', signal });
@@ -7,7 +7,7 @@
  import { t } from '$lib/utils/functions/translations';
  import { VideoUploader } from '$lib/utils/services/courses/presign';
  import { MediaUploader } from './media-uploader';
-  import { encodeAndUploadHls, type HlsEncodeProgress } from './hls-encoder';
+  import { encodeAndUploadHls, cleanupAbortedHlsUpload, type HlsEncodeProgress } from './hls-encoder';
  import { env as publicEnv } from '$env/dynamic/public';
  import { Button } from '@cio/ui/base/button';
  import * as FileDropZone from '@cio/ui/custom/file-drop-zone';
@@ -27,6 +27,7 @@
  let fileSize: number | undefined = $state();
  let hlsProgress: HlsEncodeProgress | null = $state(null);
  let hlsAbortController: AbortController | null = null;
+  let hlsReservedAssetId: string | null = $state(null);

  /**
   * HLS is the default upload path on Cloud. Self-hosted installs fall
@@ -122,16 +123,21 @@

    if (hlsEnabled) {
      hlsAbortController = new AbortController();
+      hlsReservedAssetId = null;
      try {
        const result = await encodeAndUploadHls({
          file: videoFile,
          title: videoFile.name,
          signal: hlsAbortController.signal,
+          onAssetReserved: (assetId) => {
+            hlsReservedAssetId = assetId;
+          },
          onProgress: (p) => {
            hlsProgress = p;
            $lessonVideoUpload.uploadProgress = p.percent;
          }
        });
+        hlsReservedAssetId = null;

        const lessonVideoPosition = Array.isArray(lessonApi.lesson?.videos) ? lessonApi.lesson?.videos.length : 0;
        if (_lessonId) {
@@ -184,6 +190,7 @@
        return;
      } catch (err) {
        console.error('HLS encode failed', err);
+        hlsReservedAssetId = null;
        formRes = {
          type: 'INTERNAL_ERROR',
          status: 500,
@@ -344,6 +351,12 @@
    videoUploader.abortController?.abort();
    hlsAbortController?.abort();

+    const reservedAssetId = hlsReservedAssetId;
+    hlsReservedAssetId = null;
+    if (reservedAssetId) {
+      void cleanupAbortedHlsUpload(reservedAssetId);
+    }
+
    cancelVideoUpload();

    formRes = null;
@@ -1,4 +1,4 @@
-import { mkdir, stat } from 'node:fs/promises';
+import { mkdir, readFile, rename, rm, stat, writeFile } from 'node:fs/promises';
 import path from 'node:path';
 import { tmpdir } from 'node:os';

@@ -45,6 +45,57 @@ function transcodeToMp3(inputPath: string, outputPath: string): Promise<void> {
  ]);
 }

+/**
+ * Download an HLS media playlist and its segment objects from the videos
+ * bucket into a temp directory so ffmpeg can demux the audio track.
+ */
+async function downloadHlsPlaylistToTempFile(
+  bucket: string,
+  playlistKey: string
+): Promise<{ inputPath: string; tempDir: string }> {
+  const downloadedPlaylist = await downloadObjectToTempFile(bucket, playlistKey, 'playlist.m3u8');
+  const rawPlaylist = await readFile(downloadedPlaylist, 'utf-8');
+  await safeUnlink(downloadedPlaylist);
+
+  const tempDir = path.join(tmpdir(), 'cio-jobs', 'hls-audio', `${Date.now()}_${process.pid}`);
+  await mkdir(tempDir, { recursive: true });
+
+  const playlistPrefix = path.posix.dirname(playlistKey);
+  const localLines: string[] = [];
+
+  for (const line of rawPlaylist.split('\n')) {
+    const trimmed = line.trim();
+    if (!trimmed || trimmed.startsWith('#')) {
+      localLines.push(line);
+      continue;
+    }
+
+    const segmentKey = trimmed.includes('/') ? trimmed : `${playlistPrefix}/${trimmed}`;
+    const localSegmentPath = path.join(tempDir, path.basename(trimmed));
+    const downloadedSegment = await downloadObjectToTempFile(bucket, segmentKey, path.basename(trimmed));
+    await rename(downloadedSegment, localSegmentPath);
+    localLines.push(path.basename(trimmed));
+  }
+
+  const inputPath = path.join(tempDir, 'playlist.m3u8');
+  await writeFile(inputPath, localLines.join('\n'));
+
+  return { inputPath, tempDir };
+}
+
+async function resolveAudioSourcePath(
+  bucket: string,
+  storageKey: string
+): Promise<{ inputPath: string; tempDir?: string }> {
+  if (storageKey.endsWith('.m3u8')) {
+    return downloadHlsPlaylistToTempFile(bucket, storageKey);
+  }
+
+  return {
+    inputPath: await downloadObjectToTempFile(bucket, storageKey, 'audio-source.bin')
+  };
+}
+
 /**
 * Extract a 16 kHz mono MP3 (32 kbps) from the source video for Whisper (25 MB
 * API limit). The transcribe step reads the object from the media bucket.
@@ -71,9 +122,12 @@ export async function processExtractAudio(rawData: unknown): Promise<ExtractAudi
  });

  let inputPath: string | undefined;
+  let inputTempDir: string | undefined;
  let outputPath: string | undefined;
  try {
-    inputPath = await downloadObjectToTempFile(videosBucket(), storageKey, 'audio-source.bin');
+    const source = await resolveAudioSourcePath(videosBucket(), storageKey);
+    inputPath = source.inputPath;
+    inputTempDir = source.tempDir;

    const dir = path.join(tmpdir(), 'cio-jobs', 'audio');
    await mkdir(dir, { recursive: true });
@@ -112,7 +166,11 @@ export async function processExtractAudio(rawData: unknown): Promise<ExtractAudi
    });
    throw error;
  } finally {
-    await safeUnlink(inputPath);
+    if (inputTempDir) {
+      await rm(inputTempDir, { recursive: true, force: true }).catch(() => undefined);
+    } else {
+      await safeUnlink(inputPath);
+    }
    await safeUnlink(outputPath);
  }
 }