From 27a612e92fb1e57a58d325f8f88681043359821c Mon Sep 17 00:00:00 2001 From: Rotimi Best Date: Sun, 7 Jun 2026 22:30:12 +0100 Subject: [PATCH] feat: let AI chat read lesson video transcripts on demand Add get_lesson_transcript (teacher) and read_lesson_transcript (student) tools that fetch a lesson's uploaded-video transcript from media_transcript by the video assetIds. The agents call them only when a question is about the video's spoken content, so text-only chats keep a cacheable prefix. --- apps/api/src/services/agent/student-tools.ts | 14 ++++ packages/ai-assistant/src/prompt/student.ts | 1 + packages/ai-assistant/src/prompt/teacher.ts | 2 +- .../src/services/agent/agent-tool-schemas.ts | 1 + .../core/src/services/agent/chat-tools.ts | 15 +++++ .../src/services/agent/lesson-transcript.ts | 65 +++++++++++++++++++ 6 files changed, 97 insertions(+), 1 deletion(-) create mode 100644 packages/core/src/services/agent/lesson-transcript.ts diff --git a/apps/api/src/services/agent/student-tools.ts b/apps/api/src/services/agent/student-tools.ts index 9ee15771..cf3a2e12 100644 --- a/apps/api/src/services/agent/student-tools.ts +++ b/apps/api/src/services/agent/student-tools.ts @@ -7,6 +7,7 @@ import * as schema from '@cio/db/schema'; import type { AiTutorSettings } from '@cio/ai-assistant'; import { getLesson } from '@cio/core/services/lesson/lesson'; +import { getLessonVideoTranscript } from '@cio/core/services/agent/lesson-transcript'; import { getExercise } from '@cio/core/services/exercise/exercise'; import { listCourseSections } from '@cio/core/services/course/section'; import { AppError } from '@api/utils/errors'; @@ -191,6 +192,19 @@ export function buildStudentAgentTools(orgId: string, userId: string, courseId: } }), + read_lesson_transcript: tool({ + description: + "Read the transcript of a lesson's uploaded video(s). A video's spoken content is NOT part of the lesson body, so use this whenever the learner asks about what the video says, explains, or demonstrates. Only uploaded videos are transcribed — embedded links (YouTube, etc.) return no transcript.", + inputSchema: readLessonParam, + execute: async (args) => { + return executeStudentTool('read_lesson_transcript', { orgId, userId, courseId, args }, async () => { + await verifyLessonBelongsToCourse(args.lessonId, courseId); + + return getLessonVideoTranscript(args.lessonId, orgId); + }); + } + }), + read_exercise: tool({ description: 'Read an exercise prompt — the question text and options visible to a student. Answer keys, correct flags, and marking schemes are stripped before returning.', diff --git a/packages/ai-assistant/src/prompt/student.ts b/packages/ai-assistant/src/prompt/student.ts index 1cd936f8..2918622c 100644 --- a/packages/ai-assistant/src/prompt/student.ts +++ b/packages/ai-assistant/src/prompt/student.ts @@ -181,6 +181,7 @@ You have read-only, course-scoped tools: - \`list_course_outline\` — see all sections, lessons, and exercises in this course. - \`read_lesson\` — fetch the body of a specific lesson by ID. +- \`read_lesson_transcript\` — fetch the transcript of a lesson's uploaded video. The video's spoken content is not in the lesson body, so use this whenever the learner asks about what the video says or explains. - \`read_exercise\` — fetch an exercise prompt (no answer keys). - \`search_course\` — keyword search across this course\u2019s lessons and exercise prompts. diff --git a/packages/ai-assistant/src/prompt/teacher.ts b/packages/ai-assistant/src/prompt/teacher.ts index 8cdcd913..735cda46 100644 --- a/packages/ai-assistant/src/prompt/teacher.ts +++ b/packages/ai-assistant/src/prompt/teacher.ts @@ -37,7 +37,7 @@ export function buildTeacherSystemPrompt(context: AgentContext, options?: BuildT ## Your Capabilities -You have access to specific tools listed below. Use them to read course content, create sections and lessons, update existing course-outline section metadata via update_section, update existing lesson metadata via update_lesson, write lesson content, create exercises with questions, create exercise question blocks via create_exercise_section, edit exercise-level metadata (title, description, linked lesson, course section placement, order, due date, lock state, allow-multiple-attempts) via update_exercise, edit each existing in-exercise question block heading and intro via update_exercise_section (ids from get_exercise_details sections array — not course section ids), add or update questions inside an existing exercise (use add_questions with exerciseSectionId from that sections array when the exercise groups questions into blocks), update course landing-page copy/settings, check go-live readiness, and publish the course when it is ready. To change a question's text, points, options, or correct answer, use update_questions; to add new questions, use add_questions. +You have access to specific tools listed below. Use them to read course content, read a lesson's uploaded video transcript via get_lesson_transcript (the video's spoken content is not in the lesson body — use this whenever a question is about what the video says or explains), create sections and lessons, update existing course-outline section metadata via update_section, update existing lesson metadata via update_lesson, write lesson content, create exercises with questions, create exercise question blocks via create_exercise_section, edit exercise-level metadata (title, description, linked lesson, course section placement, order, due date, lock state, allow-multiple-attempts) via update_exercise, edit each existing in-exercise question block heading and intro via update_exercise_section (ids from get_exercise_details sections array — not course section ids), add or update questions inside an existing exercise (use add_questions with exerciseSectionId from that sections array when the exercise groups questions into blocks), update course landing-page copy/settings, check go-live readiness, and publish the course when it is ready. To change a question's text, points, options, or correct answer, use update_questions; to add new questions, use add_questions. ## Question Types diff --git a/packages/core/src/services/agent/agent-tool-schemas.ts b/packages/core/src/services/agent/agent-tool-schemas.ts index dbf11562..e6ad4e0f 100644 --- a/packages/core/src/services/agent/agent-tool-schemas.ts +++ b/packages/core/src/services/agent/agent-tool-schemas.ts @@ -14,6 +14,7 @@ import { ZCourseLandingPageUpdate, ZCourseLandingPageMetadataUpdate } from '@cio export const emptyParam = z.object({}); export const lessonReadParam = z.object({ lessonId: z.string(), locale: z.string().default('en') }); +export const lessonTranscriptParam = z.object({ lessonId: z.string() }); export const exerciseReadParam = z.object({ exerciseId: z.string() }); export const createSectionParam = z.object({ title: z.string().min(1), order: z.number().int().min(0) }); export const updateSectionParam = z diff --git a/packages/core/src/services/agent/chat-tools.ts b/packages/core/src/services/agent/chat-tools.ts index 0897a5be..e6ba4932 100644 --- a/packages/core/src/services/agent/chat-tools.ts +++ b/packages/core/src/services/agent/chat-tools.ts @@ -27,6 +27,7 @@ import { } from '../exercise/exercise'; import { reorderCourseContent } from '../course/content'; import { normalizeAgentLessonContent } from './lesson-content'; +import { getLessonVideoTranscript } from './lesson-transcript'; import { buildUpdatedQuestions } from './question-update'; import { updateCourseLandingPageService } from '../course/landing-page'; import { getCourseGoLiveReadiness, publishCourseWhenReady } from '../course/go-live-readiness'; @@ -50,6 +51,7 @@ import { fetchDocumentationUrlParam, goLiveParam, lessonReadParam, + lessonTranscriptParam, reorderContentParam, updateContentParam, updateCourseLandingPageParam, @@ -487,6 +489,19 @@ export function buildAgentTools( } }), + get_lesson_transcript: tool({ + description: + "Get the transcript of a lesson's uploaded video(s). The spoken content of a video is NOT part of the lesson's HTML content, so call this whenever a question is about what the video says, explains, or demonstrates. Only uploaded videos are transcribed — embedded links (YouTube, etc.) return no transcript.", + inputSchema: lessonTranscriptParam, + execute: async (args) => { + return executeAgentTool('get_lesson_transcript', { orgId, userId, courseId, args }, async () => { + await verifyLessonBelongsToCourse(args.lessonId, courseId); + + return getLessonVideoTranscript(args.lessonId, orgId); + }); + } + }), + get_exercise_details: tool({ description: 'Get an exercise with sections, questions, and answer options from this course. Each sections entry has an id — use it with update_exercise_section to change that question block title or description.', diff --git a/packages/core/src/services/agent/lesson-transcript.ts b/packages/core/src/services/agent/lesson-transcript.ts new file mode 100644 index 00000000..fc38d1a8 --- /dev/null +++ b/packages/core/src/services/agent/lesson-transcript.ts @@ -0,0 +1,65 @@ +import { listMediaTranscriptsByAssetIds } from '@cio/db/queries/media-transcript'; +import { getLesson } from '../lesson/lesson'; + +export interface LessonVideoTranscriptResult { + lessonId: string; + title: string; + hasTranscript: boolean; + transcript: string | null; + message?: string; +} + +/** + * Assembles the transcript text for a lesson's uploaded video(s). + * + * Lesson `videos[]` carry an `assetId` for uploaded videos; the transcription + * pipeline stores Whisper output in `media_transcript` keyed by that assetId. + * YouTube/embedded links have no assetId and are never transcribed. + */ +export async function getLessonVideoTranscript(lessonId: string, orgId: string): Promise { + const lesson = await getLesson(lessonId); + const lessonWithVideos = lesson as { + id: string; + title: string; + videos?: Array<{ assetId?: string }> | null; + }; + + const assetIds = (lessonWithVideos.videos ?? []) + .map((video) => video.assetId) + .filter((assetId): assetId is string => typeof assetId === 'string' && assetId.length > 0); + + if (assetIds.length === 0) { + return { + lessonId: lesson.id, + title: lesson.title, + hasTranscript: false, + transcript: null, + message: + 'This lesson has no uploaded video with a transcript. Only uploaded videos are transcribed — embedded links (YouTube, Google Drive, etc.) are not.' + }; + } + + const transcripts = await listMediaTranscriptsByAssetIds(assetIds, orgId); + const transcript = transcripts + .map((row) => row.text?.trim()) + .filter((text): text is string => Boolean(text)) + .join('\n\n'); + + if (!transcript) { + return { + lessonId: lesson.id, + title: lesson.title, + hasTranscript: false, + transcript: null, + message: + 'This lesson has an uploaded video, but no transcript is available yet — transcription may still be processing.' + }; + } + + return { + lessonId: lesson.id, + title: lesson.title, + hasTranscript: true, + transcript + }; +}