feat: let AI chat read lesson video transcripts on demand

Add get_lesson_transcript (teacher) and read_lesson_transcript (student)
tools that fetch a lesson's uploaded-video transcript from media_transcript
by the video assetIds. The agents call them only when a question is about
the video's spoken content, so text-only chats keep a cacheable prefix.
This commit is contained in:
Rotimi Best
2026-06-07 22:30:12 +01:00
parent 4221f93584
commit 27a612e92f
6 changed files with 97 additions and 1 deletions
@@ -7,6 +7,7 @@ import * as schema from '@cio/db/schema';
import type { AiTutorSettings } from '@cio/ai-assistant';
import { getLesson } from '@cio/core/services/lesson/lesson';
import { getLessonVideoTranscript } from '@cio/core/services/agent/lesson-transcript';
import { getExercise } from '@cio/core/services/exercise/exercise';
import { listCourseSections } from '@cio/core/services/course/section';
import { AppError } from '@api/utils/errors';
@@ -191,6 +192,19 @@ export function buildStudentAgentTools(orgId: string, userId: string, courseId:
}
}),
read_lesson_transcript: tool({
description:
"Read the transcript of a lesson's uploaded video(s). A video's spoken content is NOT part of the lesson body, so use this whenever the learner asks about what the video says, explains, or demonstrates. Only uploaded videos are transcribed — embedded links (YouTube, etc.) return no transcript.",
inputSchema: readLessonParam,
execute: async (args) => {
return executeStudentTool('read_lesson_transcript', { orgId, userId, courseId, args }, async () => {
await verifyLessonBelongsToCourse(args.lessonId, courseId);
return getLessonVideoTranscript(args.lessonId, orgId);
});
}
}),
read_exercise: tool({
description:
'Read an exercise prompt — the question text and options visible to a student. Answer keys, correct flags, and marking schemes are stripped before returning.',
@@ -181,6 +181,7 @@ You have read-only, course-scoped tools:
- \`list_course_outline\` — see all sections, lessons, and exercises in this course.
- \`read_lesson\` — fetch the body of a specific lesson by ID.
- \`read_lesson_transcript\` — fetch the transcript of a lesson's uploaded video. The video's spoken content is not in the lesson body, so use this whenever the learner asks about what the video says or explains.
- \`read_exercise\` — fetch an exercise prompt (no answer keys).
- \`search_course\` — keyword search across this course\u2019s lessons and exercise prompts.
+1 -1
View File
@@ -37,7 +37,7 @@ export function buildTeacherSystemPrompt(context: AgentContext, options?: BuildT
## Your Capabilities
You have access to specific tools listed below. Use them to read course content, create sections and lessons, update existing course-outline section metadata via update_section, update existing lesson metadata via update_lesson, write lesson content, create exercises with questions, create exercise question blocks via create_exercise_section, edit exercise-level metadata (title, description, linked lesson, course section placement, order, due date, lock state, allow-multiple-attempts) via update_exercise, edit each existing in-exercise question block heading and intro via update_exercise_section (ids from get_exercise_details sections array — not course section ids), add or update questions inside an existing exercise (use add_questions with exerciseSectionId from that sections array when the exercise groups questions into blocks), update course landing-page copy/settings, check go-live readiness, and publish the course when it is ready. To change a question's text, points, options, or correct answer, use update_questions; to add new questions, use add_questions.
You have access to specific tools listed below. Use them to read course content, read a lesson's uploaded video transcript via get_lesson_transcript (the video's spoken content is not in the lesson body — use this whenever a question is about what the video says or explains), create sections and lessons, update existing course-outline section metadata via update_section, update existing lesson metadata via update_lesson, write lesson content, create exercises with questions, create exercise question blocks via create_exercise_section, edit exercise-level metadata (title, description, linked lesson, course section placement, order, due date, lock state, allow-multiple-attempts) via update_exercise, edit each existing in-exercise question block heading and intro via update_exercise_section (ids from get_exercise_details sections array — not course section ids), add or update questions inside an existing exercise (use add_questions with exerciseSectionId from that sections array when the exercise groups questions into blocks), update course landing-page copy/settings, check go-live readiness, and publish the course when it is ready. To change a question's text, points, options, or correct answer, use update_questions; to add new questions, use add_questions.
## Question Types
@@ -14,6 +14,7 @@ import { ZCourseLandingPageUpdate, ZCourseLandingPageMetadataUpdate } from '@cio
export const emptyParam = z.object({});
export const lessonReadParam = z.object({ lessonId: z.string(), locale: z.string().default('en') });
export const lessonTranscriptParam = z.object({ lessonId: z.string() });
export const exerciseReadParam = z.object({ exerciseId: z.string() });
export const createSectionParam = z.object({ title: z.string().min(1), order: z.number().int().min(0) });
export const updateSectionParam = z
@@ -27,6 +27,7 @@ import {
} from '../exercise/exercise';
import { reorderCourseContent } from '../course/content';
import { normalizeAgentLessonContent } from './lesson-content';
import { getLessonVideoTranscript } from './lesson-transcript';
import { buildUpdatedQuestions } from './question-update';
import { updateCourseLandingPageService } from '../course/landing-page';
import { getCourseGoLiveReadiness, publishCourseWhenReady } from '../course/go-live-readiness';
@@ -50,6 +51,7 @@ import {
fetchDocumentationUrlParam,
goLiveParam,
lessonReadParam,
lessonTranscriptParam,
reorderContentParam,
updateContentParam,
updateCourseLandingPageParam,
@@ -487,6 +489,19 @@ export function buildAgentTools(
}
}),
get_lesson_transcript: tool({
description:
"Get the transcript of a lesson's uploaded video(s). The spoken content of a video is NOT part of the lesson's HTML content, so call this whenever a question is about what the video says, explains, or demonstrates. Only uploaded videos are transcribed — embedded links (YouTube, etc.) return no transcript.",
inputSchema: lessonTranscriptParam,
execute: async (args) => {
return executeAgentTool('get_lesson_transcript', { orgId, userId, courseId, args }, async () => {
await verifyLessonBelongsToCourse(args.lessonId, courseId);
return getLessonVideoTranscript(args.lessonId, orgId);
});
}
}),
get_exercise_details: tool({
description:
'Get an exercise with sections, questions, and answer options from this course. Each sections entry has an id — use it with update_exercise_section to change that question block title or description.',
@@ -0,0 +1,65 @@
import { listMediaTranscriptsByAssetIds } from '@cio/db/queries/media-transcript';
import { getLesson } from '../lesson/lesson';
export interface LessonVideoTranscriptResult {
lessonId: string;
title: string;
hasTranscript: boolean;
transcript: string | null;
message?: string;
}
/**
* Assembles the transcript text for a lesson's uploaded video(s).
*
* Lesson `videos[]` carry an `assetId` for uploaded videos; the transcription
* pipeline stores Whisper output in `media_transcript` keyed by that assetId.
* YouTube/embedded links have no assetId and are never transcribed.
*/
export async function getLessonVideoTranscript(lessonId: string, orgId: string): Promise<LessonVideoTranscriptResult> {
const lesson = await getLesson(lessonId);
const lessonWithVideos = lesson as {
id: string;
title: string;
videos?: Array<{ assetId?: string }> | null;
};
const assetIds = (lessonWithVideos.videos ?? [])
.map((video) => video.assetId)
.filter((assetId): assetId is string => typeof assetId === 'string' && assetId.length > 0);
if (assetIds.length === 0) {
return {
lessonId: lesson.id,
title: lesson.title,
hasTranscript: false,
transcript: null,
message:
'This lesson has no uploaded video with a transcript. Only uploaded videos are transcribed — embedded links (YouTube, Google Drive, etc.) are not.'
};
}
const transcripts = await listMediaTranscriptsByAssetIds(assetIds, orgId);
const transcript = transcripts
.map((row) => row.text?.trim())
.filter((text): text is string => Boolean(text))
.join('\n\n');
if (!transcript) {
return {
lessonId: lesson.id,
title: lesson.title,
hasTranscript: false,
transcript: null,
message:
'This lesson has an uploaded video, but no transcript is available yet — transcription may still be processing.'
};
}
return {
lessonId: lesson.id,
title: lesson.title,
hasTranscript: true,
transcript
};
}