import { describe, expect, it } from "vitest"; import { answerDiagnosticQuestion, getNextDiagnosticQuestion } from "../src/server/diagnostics.js"; import { createSession, getProgressSummary, postMessage } from "../src/server/services.js"; import { diagnosticHardCap } from "../src/server/diagnostic-strategy.js"; import { getCatalogDiagnosticsConcepts } from "../src/server/course-catalog.js"; import { selectExercise } from "../src/tools/exercise-tools.js"; import { getConceptMastery, getRecentLearningContext, getStudentProfile, recordLearningEvent, tagMistake, updateMastery } from "../src/tools/progress-tools.js"; import { createId, nowIso } from "../src/security/ids.js"; import { createTestRuntime } from "./utils/runtime.js"; import { diagnosticTutor, upsertMasteryFixture } from "./utils/content-fixtures.js"; describe("diagnostics and progress", () => { it("initializes a local profile through the first-use diagnostic flow", async () => { const runtime = await createTestRuntime({ tutor: diagnosticTutor() }); const session = createSession(runtime, { resume: false }); const next = await getNextDiagnosticQuestion(runtime, session.session_id); expect(next.completed).toBe(false); expect(next.question?.id).toBeTruthy(); const answer = await answerDiagnosticQuestion(runtime, session.session_id, next.diagnostic_id, { question_id: next.question!.id, answer: { choice_id: "a" }, }); expect(answer.accepted).toBe(true); const progress = getProgressSummary(runtime); expect(progress.profile_summary).toContain("Python"); expect(progress.mastery).toEqual([]); }); it("starts without seeded diagnostic questions and asks the tutor to design one on demand", async () => { const runtime = await createTestRuntime({ tutor: { generate: async () => JSON.stringify({ prompt_md: "下面代码执行后会输出什么?\n```python\nx = 2\nprint(x + 3)\n```", choices: [ { id: "a", text: "5" }, { id: "b", text: "23" }, { id: "c", text: "x + 3" }, ], answer_choice_id: "a", difficulty: 1, }), }, }); const session = createSession(runtime, { resume: false }); expect(runtime.db.query<{ count: number }>("SELECT COUNT(*) AS count FROM diagnostic_questions WHERE status = 'published'").get()?.count).toBe(0); const next = await getNextDiagnosticQuestion(runtime, session.session_id); expect(next.completed).toBe(false); expect(next.question?.prompt_md).toContain("x = 2"); const generated = runtime.db.query<{ version: string; prompt_md: string }>("SELECT version, prompt_md FROM diagnostic_questions WHERE status = 'published' LIMIT 1").get(); expect(generated).toMatchObject({ version: "agent-designed" }); expect(generated?.prompt_md).not.toContain("下面哪段代码会输出 0 到 4"); }); it("keeps course progress at the diagnostic gate before a starting level is known", async () => { const runtime = await createTestRuntime({ tutor: diagnosticTutor() }); const session = createSession(runtime, { resume: false }); const initial = getProgressSummary(runtime); expect(initial.current_level).toBe("未诊断"); expect(initial.course_progress_percent).toBe(0); expect(initial.current_chapter_title).toBe("初始测评"); expect(initial.diagnostic).toMatchObject({ answered: 0, total: diagnosticHardCap(getCatalogDiagnosticsConcepts(runtime).length), completed: false }); expect(initial.curriculum.map((chapter) => chapter.title)).toEqual([ "入门与基础", "数据处理", "程序组织", "类与对象", "对象模型", "生成器", "进阶主题", "测试与调试", "包与工程化", ]); expect(initial.curriculum.map((chapter) => chapter.id)).toEqual([ "introduction", "working-with-data", "program-organization", "classes-and-objects", "object-model", "generators", "advanced-topics", "testing-debugging", "packages", ]); expect(initial.curriculum.map((chapter) => chapter.id)).not.toContain("chapter_1"); expect(initial.curriculum.map((chapter) => chapter.title)).not.toContain("KB 概念"); expect(initial.curriculum.length).toBeGreaterThan(2); const next = await getNextDiagnosticQuestion(runtime, session.session_id); await answerDiagnosticQuestion(runtime, session.session_id, next.diagnostic_id, { question_id: next.question!.id, answer: { choice_id: "a" }, }); const afterOneAnswer = getProgressSummary(runtime); expect(afterOneAnswer.current_level).toBe("未诊断"); expect(afterOneAnswer.course_progress_percent).toBe(0); expect(afterOneAnswer.current_chapter_title).toBe("初始测评"); expect(afterOneAnswer.diagnostic).toMatchObject({ answered: 1, total: diagnosticHardCap(getCatalogDiagnosticsConcepts(runtime).length), completed: false }); }); it("does not let profile current level unlock diagnostic completion or exercises", async () => { const runtime = await createTestRuntime({ tutor: diagnosticTutor() }); const session = createSession(runtime, { resume: false }); runtime.db.query("UPDATE local_profile SET profile_json = ?, updated_at = ? WHERE id = 'local'").run([ JSON.stringify({ profile_summary: "Legacy profile says the learner has a level.", current_level: "初级", current_goal: "This must remain display-only.", }), nowIso(), ]); const profile = await getStudentProfile(runtime); const progress = getProgressSummary(runtime); const selected = await selectExercise(runtime, { concept_ids: ["loop"], difficulty: 2, mode: "practice" }, { sessionId: session.session_id }); expect(profile.data.diagnostic_completed).toBe(false); expect(progress.current_level).toBe("未诊断"); expect(progress.course_progress_percent).toBe(0); expect(progress.current_chapter_id).toBe("diagnostic"); expect(selected).toMatchObject({ ok: false, code: "DIAGNOSTIC_REQUIRED" }); }); it("does not award course progress merely for completed placement", async () => { const runtime = await createTestRuntime({ tutor: diagnosticTutor() }); const session = createSession(runtime, { resume: false }); runtime.db.query( "INSERT INTO diagnostic_sessions(id, session_id, status, target_concepts_json, stop_reason, started_at, ended_at) VALUES (?, ?, 'completed', ?, 'test_complete', ?, ?)", ).run([createId("diag"), session.session_id, JSON.stringify(["loop"]), nowIso(), nowIso()]); const progress = getProgressSummary(runtime); expect(progress.diagnostic.completed).toBe(true); expect(progress.course_progress_percent).toBe(0); }); it("closes diagnostics conservatively at the hard cap and keeps ordinary practice guidance-first", async () => { const runtime = await createTestRuntime({ tutor: diagnosticTutor() }); const session = createSession(runtime, { resume: false }); const first = await getNextDiagnosticQuestion(runtime, session.session_id); const diagnosticSession = runtime.db.query<{ id: string }>( "SELECT id FROM diagnostic_sessions WHERE session_id = ?", ).get([session.session_id]); expect(diagnosticSession?.id).toBeTruthy(); const hardCap = diagnosticHardCap(getCatalogDiagnosticsConcepts(runtime).length); seedDiagnosticAttemptsAtCap(runtime, diagnosticSession!.id, session.session_id, hardCap); const next = await getNextDiagnosticQuestion(runtime, session.session_id); const row = runtime.db.query<{ status: string; stop_reason: string | null }>( "SELECT status, stop_reason FROM diagnostic_sessions WHERE id = ?", ).get([diagnosticSession!.id]); const selected = await selectExercise(runtime, { concept_ids: [first.question!.concept_ids[0]], difficulty: 1, mode: "practice" }, { sessionId: session.session_id }); const progress = getProgressSummary(runtime, { sessionId: session.session_id }); expect(next.completed).toBe(true); expect(next.question).toBeUndefined(); expect(next.progress.estimated_remaining_min).toBe(0); expect(row).toMatchObject({ status: "completed", stop_reason: "hard_cap_reached_low_confidence" }); expect(progress.progress_decision).toMatchObject({ diagnostic_state: "completed", practice_state: "guidance_first", }); expect(selected).toMatchObject({ ok: false, code: "DIAGNOSTIC_REQUIRED" }); }); it("maintains adaptive diagnostic concept state and continues after the first answer", async () => { const runtime = await createTestRuntime({ tutor: diagnosticTutor() }); const session = createSession(runtime, { resume: false }); const first = await getNextDiagnosticQuestion(runtime, session.session_id); expect(first.completed).toBe(false); expect(first.progress.total).toBe(diagnosticHardCap(getCatalogDiagnosticsConcepts(runtime).length)); const diagnosticSession = runtime.db.query<{ id: string; status: string; target_concepts_json: string }>( "SELECT id, status, target_concepts_json FROM diagnostic_sessions WHERE session_id = ?", ).get([session.session_id]); expect(diagnosticSession).toMatchObject({ status: "active" }); expect(JSON.parse(diagnosticSession?.target_concepts_json ?? "[]")).toHaveLength(getCatalogDiagnosticsConcepts(runtime).length); const states = runtime.db.query<{ concept_id: string; evidence_count: number; band: string }>( "SELECT concept_id, evidence_count, band FROM diagnostic_concept_state WHERE diagnostic_session_id = ?", ).all([diagnosticSession!.id]); expect(states).toHaveLength(getCatalogDiagnosticsConcepts(runtime).length); expect(states.every((state) => state.evidence_count === 0 && state.band === "unknown")).toBe(true); const generatedItem = runtime.db.query<{ concept_ids_json: string; prompt_md: string; answer_key_private_json: string; validation_status: string; schema_version: string }>( "SELECT concept_ids_json, prompt_md, answer_key_private_json, validation_status, schema_version FROM generated_items WHERE id = ?", ).get([first.question!.id]); expect(generatedItem).toMatchObject({ validation_status: "validated", schema_version: "generated_diagnostic_item.v1" }); expect(generatedItem?.answer_key_private_json).toContain("choice"); expect(generatedItem?.prompt_md).not.toContain(generatedItem?.answer_key_private_json ?? "answer"); const answered = await answerDiagnosticQuestion(runtime, session.session_id, first.diagnostic_id, { question_id: first.question!.id, answer: { choice_id: "a" }, }); expect(answered.completed).toBe(false); const next = await getNextDiagnosticQuestion(runtime, session.session_id); expect(next.completed).toBe(false); expect(next.question?.id).not.toBe(first.question?.id); expect(next.question?.concept_ids[0]).not.toBe(first.question?.concept_ids[0]); const updatedState = runtime.db.query<{ evidence_count: number; confidence: number; band: string }>( "SELECT evidence_count, confidence, band FROM diagnostic_concept_state WHERE diagnostic_session_id = ? AND concept_id = ?", ).get([diagnosticSession!.id, first.question!.concept_ids[0]]); expect(updatedState?.evidence_count).toBe(1); expect(updatedState?.confidence).toBeGreaterThan(0); expect(updatedState?.band).not.toBe("unknown"); }); it("validates concept ids and computes mastery server-side", async () => { const runtime = await createTestRuntime(); const session = createSession(runtime, { resume: false }); const turn = await postMessage(runtime, session.session_id, { message: "帮我看循环", attachments: [] }); const before = await getConceptMastery(runtime, { concept_ids: ["loop"] }); expect(before.ok).toBe(true); expect(before.data.concepts[0]?.confidence).toBeNull(); const event = await recordLearningEvent(runtime, { event_type: "debug_hint_given", concept_ids: ["loop"], evidence: { session_turn_id: turn.turn_id, summary: "定位 for 语句缺少冒号" }, }); expect(event.ok).toBe(true); const updated = await updateMastery(runtime, { turn_id: turn.turn_id, concept_ids: ["loop"], outcome: "completed_with_hint", difficulty: 2, hint_count: 1, evidence: { summary: "学生修复循环语法" }, }); expect(updated.ok).toBe(true); const after = await getConceptMastery(runtime, { concept_ids: ["loop"] }); expect(after.data.concepts[0]?.mastery_level ?? 0).toBeGreaterThan(before.data.concepts[0]!.mastery_level ?? 0); }); it("does not show inactive catalog concepts in weak learning context", async () => { const runtime = await createTestRuntime(); upsertMasteryFixture(runtime, "loop", { mastery: 5, confidence: 0.9, readiness: 0, evidenceCount: 5, reviewPriority: 10 }); runtime.db.query("UPDATE concepts SET catalog_status = 'inactive' WHERE id = 'loop'").run(); const context = await getRecentLearningContext(runtime, { event_limit: 3 }); expect(context.ok).toBe(true); expect(context.data.weak_concepts.map((concept) => concept.concept_id)).not.toContain("loop"); expect(context.data.concept_mastery.map((concept) => concept.concept_id)).not.toContain("loop"); }); it("deduplicates learning events per turn without merging identical events from different turns", async () => { const runtime = await createTestRuntime(); const session = createSession(runtime, { resume: false }); const firstTurn = await postMessage(runtime, session.session_id, { message: "第一次调试循环", attachments: [] }); const secondTurn = await postMessage(runtime, session.session_id, { message: "第二次调试循环", attachments: [] }); const first = await recordLearningEvent(runtime, { event_type: "debug_hint_given", concept_ids: ["loop"], evidence: { session_turn_id: firstTurn.turn_id, summary: "定位 for 语句缺少冒号" }, }); const duplicateFirst = await recordLearningEvent(runtime, { event_type: "debug_hint_given", concept_ids: ["loop"], evidence: { session_turn_id: firstTurn.turn_id, summary: "定位 for 语句缺少冒号" }, }); const second = await recordLearningEvent(runtime, { event_type: "debug_hint_given", concept_ids: ["loop"], evidence: { session_turn_id: secondTurn.turn_id, summary: "定位 for 语句缺少冒号" }, }); expect(first.ok).toBe(true); expect(duplicateFirst.data.event_id).toBe(first.data.event_id); expect(second.ok).toBe(true); expect(second.data.event_id).not.toBe(first.data.event_id); }); it("rejects unknown concept ids before reading mastery", async () => { const runtime = await createTestRuntime(); await expect(getConceptMastery(runtime, { concept_ids: ["../../secrets"] })).resolves.toMatchObject({ ok: false, code: "VALIDATION_ERROR", }); await expect(getConceptMastery(runtime, { concept_ids: [] })).resolves.toMatchObject({ ok: false, code: "VALIDATION_ERROR", }); await expect(getRecentLearningContext(runtime, { event_limit: 99 })).resolves.toMatchObject({ ok: false, code: "VALIDATION_ERROR", }); }); it("rejects learning writes for turns that do not belong to a local session", async () => { const runtime = await createTestRuntime(); await expect(recordLearningEvent(runtime, { event_type: "debug_hint_given", concept_ids: ["loop"], evidence: { session_turn_id: "turn_missing", summary: "不能写入不存在的 turn" }, })).resolves.toMatchObject({ ok: false, code: "SESSION_NOT_FOUND", }); await expect(updateMastery(runtime, { turn_id: "turn_missing", concept_ids: ["loop"], outcome: "completed_with_hint", evidence: { summary: "不能用不存在的 turn 更新掌握度" }, })).resolves.toMatchObject({ ok: false, code: "SESSION_NOT_FOUND", }); }); it("validates mistake tag writes with schema and local allowlists", async () => { const runtime = await createTestRuntime(); const session = createSession(runtime, { resume: false }); const turn = await postMessage(runtime, session.session_id, { message: "我总是漏冒号", attachments: [] }); const tag = runtime.db.query<{ id: string; metadata_json: string }>( "SELECT id, metadata_json FROM mistake_tags WHERE catalog_status = 'active' ORDER BY id ASC LIMIT 1", ).get(); expect(tag?.id).toMatch(/^kb-/); expect(JSON.parse(tag?.metadata_json ?? "{}")).toMatchObject({ source_type: "mistake_tag" }); await expect(tagMistake(runtime, { turn_id: turn.turn_id, concept_ids: ["loop"], mistake_tag_ids: [tag!.id], evidence: { summary: "for 语句末尾缺少冒号" }, mastery_level: 100, })).resolves.toMatchObject({ ok: true, }); await expect(tagMistake(runtime, { turn_id: turn.turn_id, concept_ids: ["loop"], mistake_tag_ids: [], evidence: { summary: "空错因列表不能写入" }, })).resolves.toMatchObject({ ok: false, code: "VALIDATION_ERROR", }); }); }); function seedDiagnosticAttemptsAtCap( runtime: Awaited>, diagnosticSessionId: string, sessionId: string, count: number, ): void { const now = nowIso(); runtime.db.query( "UPDATE diagnostic_concept_state SET mastery = 0, confidence = 0, evidence_count = 0, uncertainty = 0.9, band = 'unknown', conflicting_evidence_count = 0 WHERE diagnostic_session_id = ?", ).run([diagnosticSessionId]); const existingQuestions = runtime.db.query<{ id: string; concept_ids_json: string }>( "SELECT id, concept_ids_json FROM generated_items WHERE diagnostic_session_id = ?", ).all([diagnosticSessionId]); for (const question of existingQuestions) { runtime.db.query( "INSERT OR IGNORE INTO diagnostic_attempts(id, question_id, session_id, answer_json, result_summary_json, created_at) VALUES (?, ?, ?, ?, ?, ?)", ).run([ createId("diag"), question.id, sessionId, JSON.stringify({ choice_id: "b" }), JSON.stringify({ outcome: "incorrect", concept_ids: JSON.parse(question.concept_ids_json) }), now, ]); } for (let index = existingQuestions.length; index < count; index++) { const questionId = createId("diag"); const conceptId = index === 0 ? "loop" : "variable"; runtime.db.query( "INSERT INTO diagnostic_questions(id, concept_ids_json, question_type, prompt_md, choices_json, answer_key_ref, difficulty, status, version, created_at, updated_at) VALUES (?, ?, 'multiple_choice', ?, ?, 'answer:choice:a', 1, 'published', 'test', ?, ?)", ).run([ questionId, JSON.stringify([conceptId]), `Question ${index}`, JSON.stringify([{ id: "a", text: "A" }]), now, now, ]); runtime.db.query( "INSERT INTO generated_items(id, diagnostic_session_id, concept_ids_json, item_type, prompt_md, choices_json, answer_key_private_json, rubric_private, difficulty, expected_evidence, validation_status, generator_model_version, generator_prompt_version, schema_version, created_at) VALUES (?, ?, ?, 'multiple_choice', ?, ?, ?, '', 1, 'recognition', 'validated', 'test', 'test', 'generated_diagnostic_item.v1', ?)", ).run([ questionId, diagnosticSessionId, JSON.stringify([conceptId]), `Question ${index}`, JSON.stringify([{ id: "a", text: "A" }]), JSON.stringify({ choice: "a" }), now, ]); runtime.db.query( "INSERT INTO diagnostic_attempts(id, question_id, session_id, answer_json, result_summary_json, created_at) VALUES (?, ?, ?, ?, ?, ?)", ).run([ createId("diag"), questionId, sessionId, JSON.stringify({ choice_id: "b" }), JSON.stringify({ outcome: "incorrect" }), now, ]); } }