feat: let AI chat read lesson video transcripts on demand
Add get_lesson_transcript (teacher) and read_lesson_transcript (student) tools that fetch a lesson's uploaded-video transcript from media_transcript by the video assetIds. The agents call them only when a question is about the video's spoken content, so text-only chats keep a cacheable prefix.
This commit is contained in:
@@ -7,6 +7,7 @@ import * as schema from '@cio/db/schema';
|
|||||||
import type { AiTutorSettings } from '@cio/ai-assistant';
|
import type { AiTutorSettings } from '@cio/ai-assistant';
|
||||||
|
|
||||||
import { getLesson } from '@cio/core/services/lesson/lesson';
|
import { getLesson } from '@cio/core/services/lesson/lesson';
|
||||||
|
import { getLessonVideoTranscript } from '@cio/core/services/agent/lesson-transcript';
|
||||||
import { getExercise } from '@cio/core/services/exercise/exercise';
|
import { getExercise } from '@cio/core/services/exercise/exercise';
|
||||||
import { listCourseSections } from '@cio/core/services/course/section';
|
import { listCourseSections } from '@cio/core/services/course/section';
|
||||||
import { AppError } from '@api/utils/errors';
|
import { AppError } from '@api/utils/errors';
|
||||||
@@ -191,6 +192,19 @@ export function buildStudentAgentTools(orgId: string, userId: string, courseId:
|
|||||||
}
|
}
|
||||||
}),
|
}),
|
||||||
|
|
||||||
|
read_lesson_transcript: tool({
|
||||||
|
description:
|
||||||
|
"Read the transcript of a lesson's uploaded video(s). A video's spoken content is NOT part of the lesson body, so use this whenever the learner asks about what the video says, explains, or demonstrates. Only uploaded videos are transcribed — embedded links (YouTube, etc.) return no transcript.",
|
||||||
|
inputSchema: readLessonParam,
|
||||||
|
execute: async (args) => {
|
||||||
|
return executeStudentTool('read_lesson_transcript', { orgId, userId, courseId, args }, async () => {
|
||||||
|
await verifyLessonBelongsToCourse(args.lessonId, courseId);
|
||||||
|
|
||||||
|
return getLessonVideoTranscript(args.lessonId, orgId);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}),
|
||||||
|
|
||||||
read_exercise: tool({
|
read_exercise: tool({
|
||||||
description:
|
description:
|
||||||
'Read an exercise prompt — the question text and options visible to a student. Answer keys, correct flags, and marking schemes are stripped before returning.',
|
'Read an exercise prompt — the question text and options visible to a student. Answer keys, correct flags, and marking schemes are stripped before returning.',
|
||||||
|
|||||||
@@ -181,6 +181,7 @@ You have read-only, course-scoped tools:
|
|||||||
|
|
||||||
- \`list_course_outline\` — see all sections, lessons, and exercises in this course.
|
- \`list_course_outline\` — see all sections, lessons, and exercises in this course.
|
||||||
- \`read_lesson\` — fetch the body of a specific lesson by ID.
|
- \`read_lesson\` — fetch the body of a specific lesson by ID.
|
||||||
|
- \`read_lesson_transcript\` — fetch the transcript of a lesson's uploaded video. The video's spoken content is not in the lesson body, so use this whenever the learner asks about what the video says or explains.
|
||||||
- \`read_exercise\` — fetch an exercise prompt (no answer keys).
|
- \`read_exercise\` — fetch an exercise prompt (no answer keys).
|
||||||
- \`search_course\` — keyword search across this course\u2019s lessons and exercise prompts.
|
- \`search_course\` — keyword search across this course\u2019s lessons and exercise prompts.
|
||||||
|
|
||||||
|
|||||||
@@ -37,7 +37,7 @@ export function buildTeacherSystemPrompt(context: AgentContext, options?: BuildT
|
|||||||
|
|
||||||
## Your Capabilities
|
## Your Capabilities
|
||||||
|
|
||||||
You have access to specific tools listed below. Use them to read course content, create sections and lessons, update existing course-outline section metadata via update_section, update existing lesson metadata via update_lesson, write lesson content, create exercises with questions, create exercise question blocks via create_exercise_section, edit exercise-level metadata (title, description, linked lesson, course section placement, order, due date, lock state, allow-multiple-attempts) via update_exercise, edit each existing in-exercise question block heading and intro via update_exercise_section (ids from get_exercise_details sections array — not course section ids), add or update questions inside an existing exercise (use add_questions with exerciseSectionId from that sections array when the exercise groups questions into blocks), update course landing-page copy/settings, check go-live readiness, and publish the course when it is ready. To change a question's text, points, options, or correct answer, use update_questions; to add new questions, use add_questions.
|
You have access to specific tools listed below. Use them to read course content, read a lesson's uploaded video transcript via get_lesson_transcript (the video's spoken content is not in the lesson body — use this whenever a question is about what the video says or explains), create sections and lessons, update existing course-outline section metadata via update_section, update existing lesson metadata via update_lesson, write lesson content, create exercises with questions, create exercise question blocks via create_exercise_section, edit exercise-level metadata (title, description, linked lesson, course section placement, order, due date, lock state, allow-multiple-attempts) via update_exercise, edit each existing in-exercise question block heading and intro via update_exercise_section (ids from get_exercise_details sections array — not course section ids), add or update questions inside an existing exercise (use add_questions with exerciseSectionId from that sections array when the exercise groups questions into blocks), update course landing-page copy/settings, check go-live readiness, and publish the course when it is ready. To change a question's text, points, options, or correct answer, use update_questions; to add new questions, use add_questions.
|
||||||
|
|
||||||
## Question Types
|
## Question Types
|
||||||
|
|
||||||
|
|||||||
@@ -14,6 +14,7 @@ import { ZCourseLandingPageUpdate, ZCourseLandingPageMetadataUpdate } from '@cio
|
|||||||
|
|
||||||
export const emptyParam = z.object({});
|
export const emptyParam = z.object({});
|
||||||
export const lessonReadParam = z.object({ lessonId: z.string(), locale: z.string().default('en') });
|
export const lessonReadParam = z.object({ lessonId: z.string(), locale: z.string().default('en') });
|
||||||
|
export const lessonTranscriptParam = z.object({ lessonId: z.string() });
|
||||||
export const exerciseReadParam = z.object({ exerciseId: z.string() });
|
export const exerciseReadParam = z.object({ exerciseId: z.string() });
|
||||||
export const createSectionParam = z.object({ title: z.string().min(1), order: z.number().int().min(0) });
|
export const createSectionParam = z.object({ title: z.string().min(1), order: z.number().int().min(0) });
|
||||||
export const updateSectionParam = z
|
export const updateSectionParam = z
|
||||||
|
|||||||
@@ -27,6 +27,7 @@ import {
|
|||||||
} from '../exercise/exercise';
|
} from '../exercise/exercise';
|
||||||
import { reorderCourseContent } from '../course/content';
|
import { reorderCourseContent } from '../course/content';
|
||||||
import { normalizeAgentLessonContent } from './lesson-content';
|
import { normalizeAgentLessonContent } from './lesson-content';
|
||||||
|
import { getLessonVideoTranscript } from './lesson-transcript';
|
||||||
import { buildUpdatedQuestions } from './question-update';
|
import { buildUpdatedQuestions } from './question-update';
|
||||||
import { updateCourseLandingPageService } from '../course/landing-page';
|
import { updateCourseLandingPageService } from '../course/landing-page';
|
||||||
import { getCourseGoLiveReadiness, publishCourseWhenReady } from '../course/go-live-readiness';
|
import { getCourseGoLiveReadiness, publishCourseWhenReady } from '../course/go-live-readiness';
|
||||||
@@ -50,6 +51,7 @@ import {
|
|||||||
fetchDocumentationUrlParam,
|
fetchDocumentationUrlParam,
|
||||||
goLiveParam,
|
goLiveParam,
|
||||||
lessonReadParam,
|
lessonReadParam,
|
||||||
|
lessonTranscriptParam,
|
||||||
reorderContentParam,
|
reorderContentParam,
|
||||||
updateContentParam,
|
updateContentParam,
|
||||||
updateCourseLandingPageParam,
|
updateCourseLandingPageParam,
|
||||||
@@ -487,6 +489,19 @@ export function buildAgentTools(
|
|||||||
}
|
}
|
||||||
}),
|
}),
|
||||||
|
|
||||||
|
get_lesson_transcript: tool({
|
||||||
|
description:
|
||||||
|
"Get the transcript of a lesson's uploaded video(s). The spoken content of a video is NOT part of the lesson's HTML content, so call this whenever a question is about what the video says, explains, or demonstrates. Only uploaded videos are transcribed — embedded links (YouTube, etc.) return no transcript.",
|
||||||
|
inputSchema: lessonTranscriptParam,
|
||||||
|
execute: async (args) => {
|
||||||
|
return executeAgentTool('get_lesson_transcript', { orgId, userId, courseId, args }, async () => {
|
||||||
|
await verifyLessonBelongsToCourse(args.lessonId, courseId);
|
||||||
|
|
||||||
|
return getLessonVideoTranscript(args.lessonId, orgId);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}),
|
||||||
|
|
||||||
get_exercise_details: tool({
|
get_exercise_details: tool({
|
||||||
description:
|
description:
|
||||||
'Get an exercise with sections, questions, and answer options from this course. Each sections entry has an id — use it with update_exercise_section to change that question block title or description.',
|
'Get an exercise with sections, questions, and answer options from this course. Each sections entry has an id — use it with update_exercise_section to change that question block title or description.',
|
||||||
|
|||||||
@@ -0,0 +1,65 @@
|
|||||||
|
import { listMediaTranscriptsByAssetIds } from '@cio/db/queries/media-transcript';
|
||||||
|
import { getLesson } from '../lesson/lesson';
|
||||||
|
|
||||||
|
export interface LessonVideoTranscriptResult {
|
||||||
|
lessonId: string;
|
||||||
|
title: string;
|
||||||
|
hasTranscript: boolean;
|
||||||
|
transcript: string | null;
|
||||||
|
message?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Assembles the transcript text for a lesson's uploaded video(s).
|
||||||
|
*
|
||||||
|
* Lesson `videos[]` carry an `assetId` for uploaded videos; the transcription
|
||||||
|
* pipeline stores Whisper output in `media_transcript` keyed by that assetId.
|
||||||
|
* YouTube/embedded links have no assetId and are never transcribed.
|
||||||
|
*/
|
||||||
|
export async function getLessonVideoTranscript(lessonId: string, orgId: string): Promise<LessonVideoTranscriptResult> {
|
||||||
|
const lesson = await getLesson(lessonId);
|
||||||
|
const lessonWithVideos = lesson as {
|
||||||
|
id: string;
|
||||||
|
title: string;
|
||||||
|
videos?: Array<{ assetId?: string }> | null;
|
||||||
|
};
|
||||||
|
|
||||||
|
const assetIds = (lessonWithVideos.videos ?? [])
|
||||||
|
.map((video) => video.assetId)
|
||||||
|
.filter((assetId): assetId is string => typeof assetId === 'string' && assetId.length > 0);
|
||||||
|
|
||||||
|
if (assetIds.length === 0) {
|
||||||
|
return {
|
||||||
|
lessonId: lesson.id,
|
||||||
|
title: lesson.title,
|
||||||
|
hasTranscript: false,
|
||||||
|
transcript: null,
|
||||||
|
message:
|
||||||
|
'This lesson has no uploaded video with a transcript. Only uploaded videos are transcribed — embedded links (YouTube, Google Drive, etc.) are not.'
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const transcripts = await listMediaTranscriptsByAssetIds(assetIds, orgId);
|
||||||
|
const transcript = transcripts
|
||||||
|
.map((row) => row.text?.trim())
|
||||||
|
.filter((text): text is string => Boolean(text))
|
||||||
|
.join('\n\n');
|
||||||
|
|
||||||
|
if (!transcript) {
|
||||||
|
return {
|
||||||
|
lessonId: lesson.id,
|
||||||
|
title: lesson.title,
|
||||||
|
hasTranscript: false,
|
||||||
|
transcript: null,
|
||||||
|
message:
|
||||||
|
'This lesson has an uploaded video, but no transcript is available yet — transcription may still be processing.'
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
lessonId: lesson.id,
|
||||||
|
title: lesson.title,
|
||||||
|
hasTranscript: true,
|
||||||
|
transcript
|
||||||
|
};
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user