use std::{
collections::HashSet,
path::{Path, PathBuf},
sync::mpsc::{Receiver, TryRecvError, channel},
};
use anyhow::{Context, Result, bail};
use serde::Deserialize;
use crate::{
ask::{self, parse_json},
config::{AskConfig, ExamConfig, Strictness},
deck::{self, Deck, SourceLayers},
store::Store,
workspace,
};
const MAX_SOURCE_BYTES: usize = 100_000;
#[derive(Clone, Debug, PartialEq, Eq, Deserialize)]
pub struct ExamQuestion {
pub prompt: String,
pub points: Vec<String>,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum Verdict {
Pass,
Partial,
Fail,
}
impl Verdict {
pub fn label(self) -> &'static str {
match self {
Verdict::Pass => "PASS",
Verdict::Partial => "PARTIAL",
Verdict::Fail => "FAIL",
}
}
}
#[derive(Clone, Debug, PartialEq, Eq, Deserialize)]
pub struct AnswerGrade {
pub verdict: Verdict,
pub feedback: String,
#[serde(default)]
pub missed: Vec<String>,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct ExamResult {
pub passed: bool,
pub grades: Vec<AnswerGrade>,
}
impl ExamResult {
pub fn gaps(&self) -> Vec<String> {
let mut gaps = Vec::new();
for grade in &self.grades {
if grade.verdict == Verdict::Pass {
continue;
}
for point in &grade.missed {
let point = point.trim();
if !point.is_empty() && !gaps.iter().any(|g: &String| g == point) {
gaps.push(point.to_string());
}
}
}
gaps
}
}
pub fn generate_questions(
deck: &Deck,
cfg: &ExamConfig,
ask_cfg: &AskConfig,
) -> Result<Vec<ExamQuestion>> {
let sources = examination_sources(deck);
if sources.is_empty() {
bail!("the deck and its workspace declare no `source:` to examine against");
}
if let Some(error) = source_boundary_error(deck) {
bail!("{error}");
}
let root = workspace::content_root(&deck.path);
generate_questions_from(&sources, Some(&root), cfg, ask_cfg)
}
fn examination_sources(deck: &Deck) -> SourceLayers {
deck.source_layers()
}
fn source_boundary_error(deck: &Deck) -> Option<String> {
(workspace::root_for_deck(&deck.path).is_some() && deck.deck_token.is_some())
.then(|| crate::assets::validate_member(deck).err())
.flatten()
.map(|error| format!("{error:#}"))
}
pub fn ensure_backend_can_examine(deck: &Deck, ask_cfg: &AskConfig) -> Result<()> {
let sources = examination_sources(deck);
for source in sources.own.iter().chain(&sources.workspace) {
crate::backend::ensure_source_reachable(ask_cfg, deck::is_url(source))?;
}
Ok(())
}
fn generate_questions_from(
sources: &SourceLayers,
base: Option<&Path>,
cfg: &ExamConfig,
ask_cfg: &AskConfig,
) -> Result<Vec<ExamQuestion>> {
for source in sources.own.iter().chain(&sources.workspace) {
crate::backend::ensure_source_reachable(ask_cfg, deck::is_url(source))?;
}
let prompt = questions_prompt(sources, base, cfg)?;
let raw = ask::run(&run_config(cfg, ask_cfg), &prompt, &[])?;
let parsed: QuestionsDto = parse_json(&raw).context("parsing the generated questions")?;
if parsed.questions.is_empty() {
bail!("the model returned no questions");
}
for q in &parsed.questions {
if q.prompt.trim().is_empty() || q.points.is_empty() {
bail!("the model returned a malformed question (empty prompt or no points)");
}
}
Ok(parsed.questions)
}
pub fn grade_answers(
questions: &[ExamQuestion],
answers: &[String],
strictness: Strictness,
cfg: &ExamConfig,
ask_cfg: &AskConfig,
) -> Result<ExamResult> {
if questions.len() != answers.len() {
bail!(
"expected {} answers, got {}",
questions.len(),
answers.len()
);
}
let prompt = grade_prompt(questions, answers, strictness);
let raw = ask::run(&run_config(cfg, ask_cfg), &prompt, &[])?;
let parsed: GradesDto = parse_json(&raw).context("parsing the grades")?;
if parsed.grades.len() != questions.len() {
bail!(
"the model graded {} of {} answers",
parsed.grades.len(),
questions.len()
);
}
Ok(ExamResult {
passed: passed(&parsed.grades, cfg.pass_threshold),
grades: parsed.grades,
})
}
pub fn grade_compression(
description: &str,
points: &[String],
compression: &str,
strictness: Strictness,
cfg: &ExamConfig,
ask_cfg: &AskConfig,
) -> Result<ExamResult> {
let prompt = grade_compression_prompt(description, points, compression, strictness);
let mut run = run_config(cfg, ask_cfg);
run.allowed_tools.clear(); let raw = ask::run(&run, &prompt, &[])?;
let grade: AnswerGrade = parse_json(&raw).context("parsing the compression grade")?;
Ok(ExamResult {
passed: grade.verdict == Verdict::Pass,
grades: vec![grade],
})
}
pub fn remediation_cards(gaps: &[String], cfg: &ExamConfig, ask_cfg: &AskConfig) -> Result<String> {
if gaps.is_empty() {
bail!("no gaps to remediate");
}
let prompt = remediation_prompt(gaps);
let mut cfg_run = run_config(cfg, ask_cfg);
cfg_run.allowed_tools.clear(); let raw = ask::run(&cfg_run, &prompt, &[])?;
let cards = clean_deck_output(&raw);
if !has_card_fronts(&cards) {
bail!("the model replied without any cards — try remediating again");
}
Ok(cards)
}
pub fn spawn_questions(
sources: SourceLayers,
base: Option<PathBuf>,
cfg: ExamConfig,
ask_cfg: AskConfig,
) -> Receiver<Result<Vec<ExamQuestion>, String>> {
let (tx, rx) = channel();
std::thread::spawn(move || {
let r = generate_questions_from(&sources, base.as_deref(), &cfg, &ask_cfg)
.map_err(|e| format!("{e:#}"));
let _ = tx.send(r);
});
rx
}
pub fn spawn_grade(
questions: Vec<ExamQuestion>,
answers: Vec<String>,
strictness: Strictness,
cfg: ExamConfig,
ask_cfg: AskConfig,
) -> Receiver<Result<ExamResult, String>> {
let (tx, rx) = channel();
std::thread::spawn(move || {
let r = grade_answers(&questions, &answers, strictness, &cfg, &ask_cfg)
.map_err(|e| format!("{e:#}"));
let _ = tx.send(r);
});
rx
}
pub fn spawn_grade_compression(
description: String,
points: Vec<String>,
compression: String,
strictness: Strictness,
cfg: ExamConfig,
ask_cfg: AskConfig,
) -> Receiver<Result<ExamResult, String>> {
let (tx, rx) = channel();
std::thread::spawn(move || {
let r = grade_compression(
&description,
&points,
&compression,
strictness,
&cfg,
&ask_cfg,
)
.map_err(|e| format!("{e:#}"));
let _ = tx.send(r);
});
rx
}
pub fn spawn_remediation(
gaps: Vec<String>,
cfg: ExamConfig,
ask_cfg: AskConfig,
) -> Receiver<Result<String, String>> {
let (tx, rx) = channel();
std::thread::spawn(move || {
let r = remediation_cards(&gaps, &cfg, &ask_cfg).map_err(|e| format!("{e:#}"));
let _ = tx.send(r);
});
rx
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum Phase {
Generating,
Answering,
Grading,
Results,
Remediating,
Remediated,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum Effect {
Passed,
TraceFailed,
RemediationCards(String),
}
enum Pending {
Questions(Receiver<Result<Vec<ExamQuestion>, String>>),
Grade(Receiver<Result<ExamResult, String>>),
Remediation(Receiver<Result<String, String>>),
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum SittingKind {
Source,
Trace,
}
pub struct Sitting {
kind: SittingKind,
subject: String,
deck_token: String,
deck_path: Option<std::path::PathBuf>,
deck_fingerprints: HashSet<u64>,
strictness: Strictness,
cfg: ExamConfig,
ask_cfg: AskConfig,
phase: Phase,
questions: Vec<ExamQuestion>,
answers: Vec<String>,
current: usize,
result: Option<ExamResult>,
pending: Option<Pending>,
error: Option<String>,
pending_since: Option<u64>,
remediated_count: Option<usize>,
}
impl Sitting {
pub fn start(deck: &Deck, strictness: Strictness, cfg: ExamConfig, ask_cfg: AskConfig) -> Self {
let boundary_error = source_boundary_error(deck);
let source_ready = boundary_error.is_none();
let pending = source_ready.then(|| {
Pending::Questions(spawn_questions(
examination_sources(deck),
Some(workspace::content_root(&deck.path)),
cfg.clone(),
ask_cfg.clone(),
))
});
Self {
kind: SittingKind::Source,
subject: deck.subject.clone(),
deck_token: deck.deck_token.clone().unwrap_or_default(),
deck_path: Some(deck.path.clone()),
deck_fingerprints: deck.cards.iter().map(|c| c.block_fingerprint).collect(),
strictness,
cfg,
ask_cfg,
phase: if source_ready {
Phase::Generating
} else {
Phase::Results
},
questions: Vec::new(),
answers: Vec::new(),
current: 0,
result: None,
pending,
error: boundary_error,
pending_since: source_ready.then(crate::time::now_ms),
remediated_count: None,
}
}
pub fn start_trace(
description: String,
rubric: Vec<String>,
subject: String,
deck_token: String,
strictness: Strictness,
cfg: ExamConfig,
ask_cfg: AskConfig,
) -> Self {
let question = ExamQuestion {
prompt: description,
points: rubric,
};
Self {
kind: SittingKind::Trace,
subject,
deck_token,
deck_path: None,
deck_fingerprints: HashSet::new(),
strictness,
cfg,
ask_cfg,
phase: Phase::Answering,
questions: vec![question],
answers: vec![String::new()],
current: 0,
result: None,
pending: None,
error: None,
pending_since: None,
remediated_count: None,
}
}
pub fn kind(&self) -> SittingKind {
self.kind
}
pub fn phase(&self) -> &Phase {
&self.phase
}
pub fn subject(&self) -> &str {
&self.subject
}
pub fn strictness(&self) -> Strictness {
self.strictness
}
pub fn error(&self) -> Option<&str> {
self.error.as_deref()
}
pub fn thinking(&self) -> bool {
self.pending.is_some()
}
pub fn elapsed_secs(&self) -> Option<u64> {
self.elapsed_secs_at(crate::time::now_ms())
}
fn elapsed_secs_at(&self, now_ms: u64) -> Option<u64> {
self.pending_since
.map(|since| now_ms.saturating_sub(since) / 1000)
}
pub fn total(&self) -> usize {
self.questions.len()
}
pub fn current_index(&self) -> usize {
self.current
}
pub fn questions(&self) -> &[ExamQuestion] {
&self.questions
}
pub fn answers(&self) -> &[String] {
&self.answers
}
pub fn result(&self) -> Option<&ExamResult> {
self.result.as_ref()
}
pub fn question(&self) -> Option<&ExamQuestion> {
self.questions.get(self.current)
}
pub fn answer(&self) -> &str {
self.answers
.get(self.current)
.map(String::as_str)
.unwrap_or("")
}
pub fn on_last(&self) -> bool {
!self.questions.is_empty() && self.current + 1 == self.questions.len()
}
pub fn set_answer(&mut self, text: String) {
if self.phase == Phase::Answering
&& let Some(slot) = self.answers.get_mut(self.current)
{
*slot = text;
}
}
pub fn set_answers(&mut self, answers: Vec<String>) -> bool {
if self.phase == Phase::Answering && answers.len() == self.questions.len() {
self.answers = answers;
true
} else {
false
}
}
pub fn next(&mut self) {
if self.current + 1 < self.questions.len() {
self.current += 1;
}
}
pub fn prev(&mut self) {
self.current = self.current.saturating_sub(1);
}
pub fn goto(&mut self, i: usize) {
if i < self.questions.len() {
self.current = i;
}
}
fn accept_questions(&mut self, questions: Vec<ExamQuestion>) {
self.answers = vec![String::new(); questions.len()];
self.questions = questions;
self.current = 0;
self.phase = Phase::Answering;
}
pub fn submit(&mut self) {
if self.phase != Phase::Answering {
return;
}
self.error = None;
let rx = match self.kind {
SittingKind::Source => spawn_grade(
self.questions.clone(),
self.answers.clone(),
self.strictness,
self.cfg.clone(),
self.ask_cfg.clone(),
),
SittingKind::Trace => {
let q = &self.questions[0];
spawn_grade_compression(
q.prompt.clone(),
q.points.clone(),
self.answers[0].clone(),
self.strictness,
self.cfg.clone(),
self.ask_cfg.clone(),
)
}
};
self.pending = Some(Pending::Grade(rx));
self.pending_since = Some(crate::time::now_ms());
self.phase = Phase::Grading;
}
pub fn gaps(&self) -> Vec<String> {
self.result
.as_ref()
.map(ExamResult::gaps)
.unwrap_or_default()
}
pub fn can_remediate(&self) -> bool {
self.kind == SittingKind::Source
&& self.phase == Phase::Results
&& self.result.as_ref().is_some_and(|r| !r.passed)
&& !self.gaps().is_empty()
}
pub fn remediated_count(&self) -> Option<usize> {
self.remediated_count
}
pub fn remediate(&mut self) {
if !self.can_remediate() {
return;
}
self.error = None;
self.pending = Some(Pending::Remediation(spawn_remediation(
self.gaps(),
self.cfg.clone(),
self.ask_cfg.clone(),
)));
self.pending_since = Some(crate::time::now_ms());
self.phase = Phase::Remediating;
}
pub fn advance(&mut self, _now_ms: u64) -> Option<Effect> {
let reply = match &self.pending {
None => return None,
Some(Pending::Questions(rx)) => match rx.try_recv() {
Ok(r) => Reply::Questions(r),
Err(TryRecvError::Empty) => return None,
Err(TryRecvError::Disconnected) => Reply::Questions(Err(thread_gone())),
},
Some(Pending::Grade(rx)) => match rx.try_recv() {
Ok(r) => Reply::Grade(r),
Err(TryRecvError::Empty) => return None,
Err(TryRecvError::Disconnected) => Reply::Grade(Err(thread_gone())),
},
Some(Pending::Remediation(rx)) => match rx.try_recv() {
Ok(r) => Reply::Remediation(r),
Err(TryRecvError::Empty) => return None,
Err(TryRecvError::Disconnected) => Reply::Remediation(Err(thread_gone())),
},
};
self.pending = None;
self.pending_since = None;
match reply {
Reply::Questions(Ok(qs)) => {
self.accept_questions(qs);
None
}
Reply::Questions(Err(e)) => {
self.error = Some(e);
None
}
Reply::Grade(Ok(result)) => {
let effect = if result.passed {
Some(Effect::Passed)
} else if self.kind == SittingKind::Trace {
Some(Effect::TraceFailed)
} else {
None
};
self.result = Some(result);
self.phase = Phase::Results;
effect
}
Reply::Grade(Err(e)) => {
self.error = Some(e);
self.phase = Phase::Answering;
None
}
Reply::Remediation(Ok(cards)) => {
self.phase = Phase::Remediated;
Some(Effect::RemediationCards(cards))
}
Reply::Remediation(Err(e)) => {
self.error = Some(e);
self.phase = Phase::Results;
None
}
}
}
pub fn poll(
&mut self,
store: &mut Store,
now_ms: u64,
retire_after_days: Option<u32>,
) -> ExamPollOutcome {
let was_pending = self.pending.is_some();
let effect = self.advance(now_ms);
let advanced = was_pending && self.pending.is_none();
let mut store_mutated = false;
match effect {
Some(Effect::Passed) => {
store.set_deck_mastered(&self.deck_token, now_ms);
store_mutated = true;
}
Some(Effect::TraceFailed) => {
store.set_exam_failed(&self.deck_token, now_ms);
store_mutated = true;
}
Some(Effect::RemediationCards(cards)) => {
store_mutated = true;
match crate::store::store_remediation_cards(
store,
self.deck_path.as_deref(),
&self.deck_token,
&self.deck_fingerprints,
&cards,
now_ms,
retire_after_days,
) {
Ok(n) => self.remediated_count = Some(n),
Err(e) => {
self.error = Some(format!("{e}"));
self.phase = Phase::Results;
}
}
}
None => {}
}
ExamPollOutcome {
advanced,
store_mutated,
}
}
}
pub struct ExamPollOutcome {
pub advanced: bool,
pub store_mutated: bool,
}
enum Reply {
Questions(Result<Vec<ExamQuestion>, String>),
Grade(Result<ExamResult, String>),
Remediation(Result<String, String>),
}
fn thread_gone() -> String {
"the exam helper exited unexpectedly".to_string()
}
pub use crate::store::cooldown_remaining_ms;
fn passed(grades: &[AnswerGrade], threshold: f64) -> bool {
if grades.is_empty() {
return false;
}
let passes = grades.iter().filter(|g| g.verdict == Verdict::Pass).count();
(passes as f64) / (grades.len() as f64) >= threshold
}
fn run_config(cfg: &ExamConfig, ask_cfg: &AskConfig) -> AskConfig {
AskConfig {
model: cfg.model.clone().or_else(|| ask_cfg.model.clone()),
timeout_secs: cfg.timeout_secs,
cwd: None,
source_access: false,
..ask_cfg.clone()
}
}
#[derive(Default)]
struct LayerSection {
urls: Vec<String>,
files: Vec<(String, String)>,
}
fn split_layer(sources: &[String], base: Option<&Path>) -> Result<LayerSection> {
let mut urls = Vec::new();
let mut files = Vec::new();
for src in sources {
if deck::is_url(src) {
urls.push(src.clone());
continue;
}
let Some(path) = crate::source::source_path(src, base) else {
continue;
};
let text = std::fs::read_to_string(&path)
.with_context(|| format!("cannot read `source:` {}", path.display()))?;
let declared = src.trim();
let label = if Path::new(declared).has_root() {
path.file_name()
.map(|n| n.to_string_lossy().into_owned())
.unwrap_or_else(|| declared.to_string())
} else {
declared.to_string()
};
let (truncated_text, was_truncated) = truncate(&text);
if was_truncated {
eprintln!(
"note: `{}` is larger than {} KB and was truncated for the exam prompt",
label,
MAX_SOURCE_BYTES / 1_000
);
}
files.push((label, truncated_text));
}
Ok(LayerSection { urls, files })
}
fn source_section(sources: &SourceLayers, base: Option<&Path>) -> Result<String> {
let own = split_layer(&sources.own, base)?;
let workspace = split_layer(&sources.workspace, base)?;
if own.urls.is_empty()
&& own.files.is_empty()
&& workspace.urls.is_empty()
&& workspace.files.is_empty()
{
bail!("none of the declared `source:` paths could be read to examine against");
}
let mut out = String::new();
if !own.urls.is_empty() {
out.push_str(
"Read these source pages with the WebFetch tool (fetch each once); \
they are the deck's own sources, the primary grounding for this exam:\n",
);
for url in &own.urls {
out.push_str(" - ");
out.push_str(url);
out.push('\n');
}
}
if !workspace.urls.is_empty() {
out.push_str(
"Also read these workspace source pages (supporting context, \
not the primary grounding):\n",
);
for url in &workspace.urls {
out.push_str(" - ");
out.push_str(url);
out.push('\n');
}
}
for (label, text) in &own.files {
out.push_str(&format!(
"\nSource file `{label}` (a deck source, primary grounding for this exam):\n<<<SOURCE\n{text}\nSOURCE\n"
));
}
for (label, text) in &workspace.files {
out.push_str(&format!(
"\nWorkspace source file `{label}` (supporting context, not the primary grounding):\n<<<SOURCE\n{text}\nSOURCE\n"
));
}
Ok(out)
}
fn truncate(text: &str) -> (String, bool) {
if text.len() <= MAX_SOURCE_BYTES {
return (text.to_string(), false);
}
let end = (0..=MAX_SOURCE_BYTES)
.rev()
.find(|&end| text.is_char_boundary(end))
.unwrap_or_default();
(
format!("{}\n[... source truncated ...]", &text[..end]),
true,
)
}
fn questions_prompt(
sources: &SourceLayers,
base: Option<&Path>,
cfg: &ExamConfig,
) -> Result<String> {
let sources = source_section(sources, base)?;
let mut prompt = format!(
"You are a strict examiner writing an oral exam that verifies a student \
truly UNDERSTANDS a topic — not whether they memorized isolated facts.\n\n\
{sources}\n\
Write exactly {n} open-ended questions grounded in the source above. \
Favor APPLICATION (\"given X, what happens / what would you do?\") and \
CONNECTIONS (how ideas relate, contrast or build on each other) over \
plain recall. Each question must be answerable from the source and \
demand reasoning in the student's own words.\n\n\
For each question, list the key points a correct answer MUST contain — \
the specific, source-grounded facts or reasoning steps you will grade \
against. Be concrete; these are the rubric.\n\n\
Output ONLY JSON in exactly this shape, no prose, no code fences:\n\
{{\"questions\": [{{\"prompt\": \"...\", \"points\": [\"...\", \"...\"]}}]}}",
n = cfg.num_questions,
);
if let Some(extra) = cfg
.extra
.as_deref()
.map(str::trim)
.filter(|s| !s.is_empty())
{
prompt.push_str("\n\nAdditional instructions:\n");
prompt.push_str(extra);
}
Ok(prompt)
}
fn strictness_criteria(strictness: Strictness) -> &'static str {
match strictness {
Strictness::Strict => {
"\
Grade strictly for COMPLETENESS — treat the rubric as a checklist. The answer \
passes only if it covers EVERY key point (wording may differ, but each point \
must be present). List in `missed` every key point the answer omits or gets \
wrong; here an omitted point IS a gap, because this material requires recalling \
all of it.\n\
- \"pass\": every key point is present.\n\
- \"partial\": some present, others missing or wrong.\n\
- \"fail\": most key points missing or wrong."
}
Strictness::Balanced => {
"\
Grade for UNDERSTANDING, not completeness of phrasing. A key point counts as \
covered if the answer shows the student grasps it — briefly, in their own \
words, or by clear implication. Put a point in `missed` ONLY if the answer is \
wrong about it or shows the student genuinely does not grasp it — NEVER merely \
because they didn't spell it out or skipped a secondary detail.\n\
- \"pass\": the answer demonstrates the question's core idea.\n\
- \"partial\": part understood, but something important is wrong or absent.\n\
- \"fail\": little or no understanding."
}
Strictness::Lenient => {
"\
Grade generously: the goal is only to catch real misunderstandings. Give the \
student the benefit of the doubt on anything plausibly correct or partially \
stated. An answer covering only some of the key points still passes when what \
it says is correct; incompleteness alone is never \"partial\" here. Reserve \
\"partial\" for an actual error. Put a point in `missed` ONLY if the answer is \
clearly wrong about it or the question was essentially not answered.\n\
- \"pass\": broadly correct, even if thin or incomplete.\n\
- \"partial\": partly right but with a clear error.\n\
- \"fail\": wrong or essentially unanswered."
}
}
}
fn grade_prompt(questions: &[ExamQuestion], answers: &[String], strictness: Strictness) -> String {
let mut prompt = String::from(
"You are an examiner grading an understanding exam. For each question you \
are given the key points a correct answer covers (the rubric) and the \
student's answer.\n",
);
prompt.push_str(strictness_criteria(strictness));
prompt.push_str(
"\nKeep `feedback` to one or two sentences, and cite the specific missed \
point(s) in `missed` per the rule above.\n\n",
);
for (i, (q, a)) in questions.iter().zip(answers).enumerate() {
prompt.push_str(&format!("Question {}: {}\n", i + 1, q.prompt));
prompt.push_str("Key points (rubric):\n");
for point in &q.points {
prompt.push_str(" - ");
prompt.push_str(point);
prompt.push('\n');
}
let answer = a.trim();
let answer = if answer.is_empty() {
"(no answer given)"
} else {
answer
};
prompt.push_str("Student's answer: ");
prompt.push_str(answer);
prompt.push_str("\n\n");
}
prompt.push_str(
"Output ONLY JSON in exactly this shape, one grade per question in order, \
no prose, no code fences:\n\
{\"grades\": [{\"verdict\": \"pass|partial|fail\", \"feedback\": \"...\", \
\"missed\": [\"...\"]}]}",
);
prompt
}
fn compression_strictness_criteria(strictness: Strictness) -> &'static str {
match strictness {
Strictness::Strict => {
"\
Grade strictly: the retrace must name every load-bearing step and the causal \
link from each to the next. A missing step, a wrong order, or a hand-waved \"and \
then it works\" is a gap.\n\
- \"pass\": the whole chain is re-derived — each step and link present.\n\
- \"partial\": the gist is right but a step or a link is missing or wrong.\n\
- \"fail\": the chain is mostly absent, out of order, or wrong."
}
Strictness::Balanced => {
"\
Grade for whether the learner can RE-DERIVE the path: the main steps and how \
each leads to the next, in their own words. Forgive omitted minor detail and \
loose wording — a two-sentence gist that captures the causal spine passes. Mark \
a point missed only when a load-bearing step or link is absent or wrong.\n\
- \"pass\": the causal spine is re-derived end to end.\n\
- \"partial\": part of the chain is there, but a key step or link is missing or \
wrong.\n\
- \"fail\": little of the path's chain is reconstructed."
}
Strictness::Lenient => {
"\
Grade generously: pass if the compression broadly captures the path's shape — \
roughly where it starts, the main move, and where it ends — even if thin. Fail \
only a retrace that is clearly wrong or essentially absent.\n\
- \"pass\": broadly traces the path, even thinly.\n\
- \"partial\": gestures at the path but with a clear error or gap.\n\
- \"fail\": wrong or essentially no retrace."
}
}
}
fn grade_compression_prompt(
description: &str,
points: &[String],
compression: &str,
strictness: Strictness,
) -> String {
let mut prompt = String::from(
"You are grading the final exam of a guided predict-and-verify walk \
through a source (a \"trace\"). The learner walked the path hop by hop; \
now, from memory, they RETRACE THE WHOLE PATH in a sentence or two. This \
compression IS the exam: it verifies they can RE-DERIVE the path (the \
steps and how they connect), not merely recognize each step.\n\n\
The path answers this question:\n",
);
prompt.push_str(description.trim());
prompt.push_str(
"\n\nA faithful retrace re-derives these load-bearing points, in order \
(drawn from the path's checkpoints — the ground truth):\n",
);
for point in points {
prompt.push_str(" - ");
prompt.push_str(point);
prompt.push('\n');
}
prompt.push_str("\nThe learner's compression:\n");
let answer = compression.trim();
prompt.push_str(if answer.is_empty() {
"(no answer given)"
} else {
answer
});
prompt.push_str("\n\n");
prompt.push_str(compression_strictness_criteria(strictness));
prompt.push_str(
"\nJudge the CAUSAL CHAIN, not verbatim coverage — a short retrace that \
re-derives the spine passes even if it omits detail, while a confident \
retrace that gets the mechanism wrong does not. Put the specific missing \
or wrong steps in `missed`. Keep `feedback` to one or two sentences.\n\
Output ONLY JSON in exactly this shape, no prose, no code fences:\n\
{\"verdict\": \"pass|partial|fail\", \"feedback\": \"...\", \"missed\": [\"...\"]}",
);
prompt
}
fn remediation_prompt(gaps: &[String]) -> String {
let mut prompt = String::from(
"A student failed an understanding exam on the concepts below. Turn them \
into spaced-repetition cards that close each gap.\n\n\
FIRST consolidate the list: some of these concepts overlap or restate one \
another. MERGE overlapping concepts and write ONE card per DISTINCT idea — \
never two cards that test the same thing. Fewer, non-overlapping cards are \
better than covering every line verbatim; only merge true duplicates, \
never drop a distinct idea.\n\n\
CHOOSE THE CARD TYPE per gap, by what the gap actually is:\n\
- A missed FACT or TERM (a definition, a value, what lives where) -> a \
cheap recall card. Prefer a cloze: a `## ` front with a short \
instruction, an answer line that states the fact as plain prose, then \
one `<!-- blank: span hidden=\"...\" -->` comment line per hidden \
span, where `hidden` repeats the exact answer text to hide. If there \
is no natural word to blank out, use a plain `## ` card with the \
answer on the line below instead.\n\
- A missed CONCEPT, MECHANISM or CONNECTION (a \"why\", \"how\" or \"what \
happens if\") -> an understanding card: a `## ` open prompt with the \
key points a good answer covers, one per line below it. This forces the \
student to re-derive the idea, which is what the exam re-tests. When \
unsure, prefer the understanding card.\n\n\
CONCEPTS THE STUDENT MISSED:\n",
);
for gap in gaps {
prompt.push_str(" - ");
prompt.push_str(gap);
prompt.push('\n');
}
prompt.push_str(
"\nFORMAT — a Markdown deck, cards one after another. A card is a `## ` \
front at column 0 (never indented); its answer is the plain \
(unindented) line(s) below it; a `> [!NOTE]` line after them opens an \
optional note (a caveat, example or why it matters) whose body lines \
each start with `> `. A blockquote with no badge first line is a \
quotation and belongs to the answer. No frontmatter, no \
headings other than the `## ` fronts. One example of each card type:\n\
## Recall how a String is laid out in memory.\n\
A String stores a pointer, length and capacity on the stack, and its \
bytes live on the heap.\n\
<!-- blank: span hidden=\"pointer\" -->\n\
<!-- blank: span hidden=\"length\" -->\n\
<!-- blank: span hidden=\"capacity\" -->\n\
<!-- blank: span hidden=\"heap\" -->\n\
## What does `drop` do for a String, and when?\n\
It returns the String's heap buffer to the allocator, at the end of the \
owning scope.\n\
## Why does moving a String invalidate the original binding?\n\
Both bindings would otherwise point at the same heap buffer.\n\
Dropping both would free it twice (a double free).\n\
So Rust invalidates the source instead of allowing two owners.\n\
> [!NOTE]\n\
> A move, not a shallow copy you can keep using.\n\n\
Before finishing, re-read your cards as a set and merge any two that test \
the same idea, so every card is distinct.\n\
Output ONLY the deck text — no markdown code fences, no preamble, no \
closing remarks.",
);
prompt
}
fn has_card_fronts(text: &str) -> bool {
text.lines().any(|l| {
crate::parser::heading_depth(l).is_some_and(|(d, _)| crate::parser::is_card_depth(d))
})
}
fn clean_deck_output(raw: &str) -> String {
let lines: Vec<&str> = raw.lines().collect();
let Some(start) = lines.iter().position(|l| {
crate::parser::heading_depth(l).is_some_and(|(d, _)| crate::parser::is_card_depth(d))
}) else {
return raw.trim().to_string();
};
let end = lines[start + 1..]
.iter()
.rposition(|line| {
let trimmed = line.trim();
!trimmed.is_empty() && !trimmed.starts_with("```")
})
.map(|offset| start + offset + 2)
.unwrap_or(start + 1);
lines[start..end].join("\n")
}
#[derive(Debug, Deserialize)]
struct QuestionsDto {
questions: Vec<ExamQuestion>,
}
#[derive(Debug, Deserialize)]
struct GradesDto {
grades: Vec<AnswerGrade>,
}
#[cfg(all(test, unix))]
mod tests {
use super::*;
fn deck_with_sources(srcs: &[&str]) -> Deck {
Deck {
path: std::path::PathBuf::from("/alix-nonexistent-test-root/d.md"),
subject: "d.md".to_string(),
deck_token: None,
cards: Vec::new(),
definitions: Vec::new(),
links: Vec::new(),
requires: Vec::new(),
sources: srcs.iter().map(|s| s.to_string()).collect(),
settings: Default::default(),
title: None,
description: None,
trace: None,
load_warnings: Vec::new(),
}
}
fn url_layers(own: &[&str]) -> SourceLayers {
SourceLayers {
own: own.iter().map(|s| s.to_string()).collect(),
workspace: Vec::new(),
}
}
#[test]
fn generated_card_text_is_recognized_at_every_depth() {
assert!(has_card_fronts("### only a sub-card\nanswer\n"));
assert!(has_card_fronts("#### deep\nanswer\n"));
assert!(!has_card_fronts("# just a section\nprose\n"));
}
#[test]
fn a_workspace_source_joins_deck_sources_as_its_own_labeled_layer() {
let dir = tempfile::tempdir().unwrap();
let members = dir.path().join(crate::workspace::DECKS);
std::fs::create_dir_all(&members).unwrap();
std::fs::write(
dir.path().join(crate::workspace::MANIFEST),
"source = \"https://example.org/current\"\n",
)
.unwrap();
let path = members.join("d.md");
std::fs::write(
&path,
"---\nformat-version: 1\nid: \"deck-deck1\"\nsource: https://example.org/own\n---\n## q\na\n",
)
.unwrap();
let deck = Deck::load(&path).unwrap();
assert_eq!(
SourceLayers {
own: vec!["https://example.org/own".to_string()],
workspace: vec!["https://example.org/current".to_string()],
},
examination_sources(&deck)
);
assert!(deck.has_exam());
}
#[test]
fn a_workspace_source_duplicated_by_the_deck_is_not_layered_twice() {
let dir = tempfile::tempdir().unwrap();
let members = dir.path().join(crate::workspace::DECKS);
std::fs::create_dir_all(&members).unwrap();
std::fs::write(
dir.path().join(crate::workspace::MANIFEST),
"source = \"https://example.org/own\"\n",
)
.unwrap();
let path = members.join("d.md");
std::fs::write(
&path,
"---\nformat-version: 1\nid: \"deck-deck1\"\nsource: https://example.org/own\n---\n## q\na\n",
)
.unwrap();
let deck = Deck::load(&path).unwrap();
assert_eq!(
url_layers(&["https://example.org/own"]),
examination_sources(&deck)
);
}
#[test]
fn exam_prompt_signatures_ban_section_like_names_and_pin_trace_grounding() {
let source = include_str!("exam.rs");
let builders: [(&str, &[&str]); 4] = [
("fn questions_prompt(", &[]),
("fn grade_prompt(", &[]),
(
"fn grade_compression_prompt(",
&["description: &str", "points: &[String]"],
),
("fn remediation_prompt(", &[]),
];
for (builder, trace_grounding) in builders {
let at = source
.find(builder)
.unwrap_or_else(|| panic!("{builder} moved or was renamed"));
let rest = &source[at..];
let end = rest
.find(" -> ")
.unwrap_or_else(|| panic!("{builder} signature"));
let params = &rest[..end];
for banned in ["context", "section", "cards", "deck_body"] {
assert!(
!params.contains(banned),
"{builder} takes `{banned}`: section/card context must not enter \
exam prompt signatures.\n{params}"
);
}
for required in trace_grounding {
assert!(
params.contains(required),
"{builder} must expose `{required}` as explicit trace grounding.\n{params}"
);
}
}
}
#[test]
fn questions_prompt_labels_deck_and_workspace_layers() {
let sources = SourceLayers {
own: vec!["https://example.org/own".to_string()],
workspace: vec!["https://example.org/context".to_string()],
};
let p = questions_prompt(&sources, None, &ExamConfig::default()).unwrap();
let own_at = p.find("https://example.org/own").unwrap();
let workspace_at = p.find("https://example.org/context").unwrap();
assert!(p.contains("primary grounding"));
assert!(p.contains("supporting context"));
assert!(own_at < workspace_at, "{p}");
}
#[test]
fn questions_prompt_distinguishes_sources_that_share_a_basename() {
let dir = tempfile::tempdir().unwrap();
for (parent, text) in [("a", "alpha source"), ("b", "beta source")] {
let source_dir = dir.path().join(parent);
std::fs::create_dir(&source_dir).unwrap();
std::fs::write(source_dir.join("notes.md"), text).unwrap();
}
let sources = SourceLayers {
own: vec!["a/notes.md".to_string(), "b/notes.md".to_string()],
workspace: Vec::new(),
};
let prompt = questions_prompt(&sources, Some(dir.path()), &ExamConfig::default()).unwrap();
assert!(
prompt.contains("Source file `a/notes.md`"),
"the first source is not labeled by what the deck declared: {prompt}"
);
assert!(
prompt.contains("Source file `b/notes.md`"),
"the second source is not labeled by what the deck declared: {prompt}"
);
}
#[test]
fn an_absolute_source_is_labeled_by_its_basename_and_a_relative_one_by_its_path() {
let dir = tempfile::tempdir().unwrap();
let nested = dir.path().join("deep");
std::fs::create_dir(&nested).unwrap();
std::fs::write(nested.join("relative.md"), "relative body").unwrap();
std::fs::write(dir.path().join("absolute.md"), "absolute body").unwrap();
let absolute = dir.path().join("absolute.md");
let sources = SourceLayers {
own: vec![
"deep/relative.md".to_string(),
absolute.display().to_string(),
],
workspace: Vec::new(),
};
let prompt = questions_prompt(&sources, Some(dir.path()), &ExamConfig::default()).unwrap();
assert!(
prompt.contains("Source file `deep/relative.md`"),
"a relative declaration should reach the prompt as declared: {prompt}"
);
assert!(
prompt.contains("Source file `absolute.md`"),
"an absolute declaration should reach the prompt as its basename: {prompt}"
);
assert!(
!prompt.contains(&absolute.display().to_string()),
"the operator's directory layout reached the prompt: {prompt}"
);
}
#[test]
fn same_named_absolute_sources_in_different_layers_have_distinct_headings() {
let dir = tempfile::tempdir().unwrap();
let own = dir.path().join("own").join("notes.md");
let workspace = dir.path().join("workspace").join("notes.md");
std::fs::create_dir(own.parent().unwrap()).unwrap();
std::fs::create_dir(workspace.parent().unwrap()).unwrap();
std::fs::write(&own, "primary body").unwrap();
std::fs::write(&workspace, "supporting body").unwrap();
let sources = SourceLayers {
own: vec![own.display().to_string()],
workspace: vec![workspace.display().to_string()],
};
let prompt = questions_prompt(&sources, None, &ExamConfig::default()).unwrap();
assert_eq!(
1,
prompt
.matches("Source file `notes.md` (a deck source, primary grounding")
.count(),
"deck heading: {prompt}"
);
assert_eq!(
1,
prompt
.matches("Workspace source file `notes.md` (supporting context")
.count(),
"workspace heading: {prompt}"
);
}
#[test]
fn questions_prompt_carries_source_strictness_and_json_shape() {
let sources = url_layers(&["https://example.org/ownership"]);
let p = questions_prompt(&sources, None, &ExamConfig::default()).unwrap();
assert!(p.contains("https://example.org/ownership"));
assert!(p.contains("WebFetch"));
assert!(p.contains("strict examiner"));
assert!(p.contains("APPLICATION"));
assert!(p.contains("\"questions\""));
assert!(p.contains("exactly 5 open-ended questions"));
}
#[test]
fn questions_prompt_honors_num_questions_and_extra() {
let sources = url_layers(&["https://x"]);
let cfg = ExamConfig {
num_questions: 3,
extra: Some("Focus on lifetimes.".to_string()),
..ExamConfig::default()
};
let p = questions_prompt(&sources, None, &cfg).unwrap();
assert!(p.contains("exactly 3 open-ended questions"));
assert!(p.contains("Additional instructions:"));
assert!(p.contains("Focus on lifetimes."));
}
#[test]
fn grade_prompt_includes_rubric_and_answers() {
let qs = vec![ExamQuestion {
prompt: "Why move?".to_string(),
points: vec!["avoids double free".to_string()],
}];
let p = grade_prompt(
&qs,
&["because ownership".to_string()],
Strictness::Balanced,
);
assert!(p.contains("Question 1: Why move?"));
assert!(p.contains("avoids double free"));
assert!(p.contains("because ownership"));
assert!(p.contains("examiner"));
assert!(p.contains("\"grades\""));
}
#[test]
fn grade_prompt_marks_empty_answers() {
let qs = vec![ExamQuestion {
prompt: "Q".to_string(),
points: vec!["p".to_string()],
}];
let p = grade_prompt(&qs, &[" ".to_string()], Strictness::Balanced);
assert!(p.contains("(no answer given)"));
}
#[test]
fn grade_prompt_criteria_vary_by_strictness() {
let qs = vec![ExamQuestion {
prompt: "Q".to_string(),
points: vec!["p".to_string()],
}];
let a = vec!["a".to_string()];
let strict = grade_prompt(&qs, &a, Strictness::Strict);
let balanced = grade_prompt(&qs, &a, Strictness::Balanced);
let lenient = grade_prompt(&qs, &a, Strictness::Lenient);
assert!(strict.contains("COMPLETENESS"));
assert!(strict.contains("EVERY key point"));
assert!(balanced.contains("UNDERSTANDING, not completeness"));
assert!(!balanced.contains("COMPLETENESS — treat the rubric"));
assert!(lenient.contains("benefit of the doubt"));
assert!(lenient.contains("incompleteness alone is never"));
assert!(!strict.contains("incompleteness alone is never"));
assert!(!balanced.contains("incompleteness alone is never"));
}
#[test]
fn remediation_prompt_asks_for_understanding_cards() {
let p = remediation_prompt(&["the aliasing rule".to_string()]);
assert!(p.contains("the aliasing rule"));
assert!(p.contains("understanding card"));
assert!(p.contains("Output ONLY the deck text"));
}
#[test]
fn remediation_prompt_example_writes_a_badged_note() {
let prompt = remediation_prompt(&["the aliasing rule".to_string()]);
assert!(
prompt.contains("> [!NOTE]\n> A move, not a shallow copy you can keep using."),
"the example must model the note grammar the prose requires: {prompt}"
);
}
#[test]
fn remediation_prompt_picks_card_type_per_gap() {
let p = remediation_prompt(&["x".to_string()]);
assert!(p.contains("missed FACT or TERM"));
assert!(p.contains("missed CONCEPT"));
assert!(p.contains("<!-- blank: span hidden="));
assert!(!p.contains("\\blank{"));
assert!(p.contains("understanding card"));
assert!(!p.contains("indented answer"));
assert!(p.contains("## "));
}
#[test]
fn remediation_prompt_recall_example_parses_as_four_blank_cards() {
let prompt = remediation_prompt(&["x".to_string()]);
let start = prompt
.find("## Recall how a String is laid out in memory.")
.expect("the recall example starts");
let tail = &prompt[start..];
let end = tail
.find("\n## What does `drop` do for a String, and when?")
.expect("the next example bounds the recall card");
let cards = crate::parser::parse_str("model-output.md", &tail[..end]).unwrap();
assert_eq!(
4,
cards.len(),
"the prompt's four hidden facts must become four review cards: {cards:?}"
);
assert!(
cards.iter().all(|card| card.region.is_some()),
"every generated recall card must be a span-derived card: {cards:?}"
);
}
#[test]
fn remediation_prompt_asks_to_dedup_overlapping_gaps() {
let p = remediation_prompt(&[
"String stores a pointer, length and capacity on the stack".to_string(),
"A String keeps its bytes on the heap and a pointer on the stack".to_string(),
]);
assert!(p.contains("String stores a pointer"));
assert!(p.contains("keeps its bytes on the heap"));
assert!(p.contains("MERGE overlapping concepts"));
assert!(p.contains("ONE card per DISTINCT idea"));
assert!(p.contains("merge any two that test the same idea"));
}
#[test]
fn parse_questions_from_fenced_json() {
let raw = "Here you go:\n```json\n{\"questions\": [{\"prompt\": \"Q1\", \
\"points\": [\"a\", \"b\"]}]}\n```";
let dto: QuestionsDto = parse_json(raw).unwrap();
assert_eq!(1, dto.questions.len());
assert_eq!("Q1", dto.questions[0].prompt);
assert_eq!(vec!["a", "b"], dto.questions[0].points);
}
#[test]
fn parse_grades_with_verdicts() {
let raw = "{\"grades\": [\
{\"verdict\": \"pass\", \"feedback\": \"good\", \"missed\": []},\
{\"verdict\": \"fail\", \"feedback\": \"no\", \"missed\": [\"x\"]}]}";
let dto: GradesDto = parse_json(raw).unwrap();
assert_eq!(Verdict::Pass, dto.grades[0].verdict);
assert_eq!(Verdict::Fail, dto.grades[1].verdict);
assert_eq!(vec!["x"], dto.grades[1].missed);
}
#[test]
fn missed_defaults_when_absent() {
let raw = "{\"grades\": [{\"verdict\": \"pass\", \"feedback\": \"ok\"}]}";
let dto: GradesDto = parse_json(raw).unwrap();
assert!(dto.grades[0].missed.is_empty());
}
#[test]
fn invalid_json_is_a_clear_error() {
let err = parse_json::<GradesDto>("not json at all").unwrap_err();
assert!(format!("{err:#}").contains("valid JSON"));
}
#[test]
fn threshold_all_pass_passes() {
let grades = vec![
AnswerGrade {
verdict: Verdict::Pass,
feedback: String::new(),
missed: Vec::new(),
},
AnswerGrade {
verdict: Verdict::Pass,
feedback: String::new(),
missed: Vec::new(),
},
];
assert!(passed(&grades, 1.0));
}
#[test]
fn threshold_one_partial_fails_strict() {
let grades = vec![
AnswerGrade {
verdict: Verdict::Pass,
feedback: String::new(),
missed: Vec::new(),
},
AnswerGrade {
verdict: Verdict::Partial,
feedback: String::new(),
missed: vec!["m".to_string()],
},
];
assert!(!passed(&grades, 1.0));
assert!(passed(&grades, 0.5));
}
#[test]
fn gaps_collects_missed_from_non_passes_deduped() {
let result = ExamResult {
passed: false,
grades: vec![
AnswerGrade {
verdict: Verdict::Pass,
feedback: String::new(),
missed: vec!["ignored (passed)".to_string()],
},
AnswerGrade {
verdict: Verdict::Fail,
feedback: String::new(),
missed: vec!["dup".to_string(), "unique".to_string()],
},
AnswerGrade {
verdict: Verdict::Partial,
feedback: String::new(),
missed: vec!["dup".to_string()],
},
],
};
assert_eq!(vec!["dup", "unique"], result.gaps());
}
#[test]
fn clean_deck_output_strips_fences() {
let raw = "```text\n## Q\nA\n```";
assert_eq!("## Q\nA", clean_deck_output(raw));
}
#[test]
fn deck_cleanup_handles_empty_and_trailing_boundaries() {
assert_eq!("## Q", clean_deck_output("preamble\n## Q\n\n```\n"));
assert_eq!(
"## Q\n```not trailing\nA",
clean_deck_output("preamble\n## Q\n```not trailing\nA\n```\n\n")
);
}
#[test]
fn source_truncation_is_exact_and_utf8_safe() {
let exact = "a".repeat(MAX_SOURCE_BYTES);
assert_eq!((exact.clone(), false), truncate(&exact));
let mut oversized = "a".repeat(MAX_SOURCE_BYTES - 1);
oversized.push('é');
let (truncated, did_truncate) = truncate(&oversized);
assert!(did_truncate);
assert_eq!(
format!(
"{}\n[... source truncated ...]",
"a".repeat(MAX_SOURCE_BYTES - 1)
),
truncated
);
}
#[test]
fn questions_prompt_tells_the_examiner_when_a_source_was_truncated() {
let dir = tempfile::tempdir().unwrap();
std::fs::write(
dir.path().join("large.md"),
"a".repeat(MAX_SOURCE_BYTES + 1),
)
.unwrap();
let sources = SourceLayers {
own: vec!["large.md".to_string()],
workspace: Vec::new(),
};
let prompt = questions_prompt(&sources, Some(dir.path()), &ExamConfig::default()).unwrap();
assert!(
prompt.contains("[... source truncated ...]"),
"the pasted source carries no truncation marker, so the examiner reads a partial source as whole"
);
assert!(
prompt.contains("Each question must be answerable from the source"),
"the answerability instruction is gone, so the marker guards nothing"
);
}
use crate::{
parser,
session::is_retired_id,
testutil::{ask_config, exec_lock, fake_cli, fake_reply},
};
#[test]
fn verdict_labels() {
assert_eq!("PASS", Verdict::Pass.label());
assert_eq!("PARTIAL", Verdict::Partial.label());
assert_eq!("FAIL", Verdict::Fail.label());
}
#[test]
fn generate_questions_rejects_a_deck_without_a_source() {
let deck = deck_with_sources(&[]);
let err = generate_questions(
&deck,
&ExamConfig::default(),
&ask_config(std::path::Path::new("unused")),
)
.unwrap_err();
assert!(format!("{err:#}").contains("no `source:`"));
}
#[test]
fn workspace_asset_boundary_failures_surface_through_the_sitting() {
let dir = tempfile::tempdir().unwrap();
let members = dir.path().join(crate::workspace::DECKS);
std::fs::create_dir(&members).unwrap();
std::fs::write(dir.path().join(crate::workspace::MANIFEST), "").unwrap();
let path = members.join("d.md");
std::fs::write(
&path,
"---\nformat-version: 1\nid: deck-9w2c7x4k1m8q3z5t0v6b2n4d8f\nsource: https://example.org/source\n---\n## q\na\n<!-- at: source.md:1 fingerprint: xxh64-0000000000000007 asset: missing.txt -->\n<!-- id: card-4jkya9q3m8z0tw5v9y2b4n6d8f -->\n",
)
.unwrap();
let deck = Deck::load(&path).unwrap();
let boundary = source_boundary_error(&deck).unwrap();
assert!(boundary.contains("missing.txt"), "{boundary}");
let sitting = Sitting::start(
&deck,
Strictness::Lenient,
ExamConfig::default(),
AskConfig::default(),
);
assert_eq!(Strictness::Lenient, sitting.strictness());
assert_eq!(Some(boundary.as_str()), sitting.error());
assert_eq!(None, sitting.elapsed_secs());
}
#[test]
fn sitting_navigation_accepts_only_question_indices() {
let mut sitting = Sitting::start_trace(
"first".to_string(),
vec!["point".to_string()],
"trace".to_string(),
"deck-trace".to_string(),
Strictness::Strict,
ExamConfig::default(),
AskConfig::default(),
);
sitting.accept_questions(vec![
ExamQuestion {
prompt: "first".to_string(),
points: vec!["one".to_string()],
},
ExamQuestion {
prompt: "second".to_string(),
points: vec!["two".to_string()],
},
]);
assert_eq!(Strictness::Strict, sitting.strictness());
assert_eq!(0, sitting.current_index());
sitting.next();
assert_eq!(1, sitting.current_index());
sitting.next();
assert_eq!(1, sitting.current_index());
sitting.goto(0);
assert_eq!(0, sitting.current_index());
sitting.goto(2);
assert_eq!(0, sitting.current_index());
}
#[test]
fn elapsed_seconds_use_a_saturating_millisecond_boundary() {
let mut sitting = Sitting::start_trace(
"trace".to_string(),
vec!["point".to_string()],
"trace".to_string(),
"deck-trace".to_string(),
Strictness::Balanced,
ExamConfig::default(),
AskConfig::default(),
);
assert_eq!(None, sitting.elapsed_secs_at(3_500));
sitting.pending_since = Some(1_000);
assert_eq!(Some(2), sitting.elapsed_secs_at(3_500));
assert_eq!(Some(0), sitting.elapsed_secs_at(500));
sitting.pending_since = Some(crate::time::now_ms().saturating_sub(2_500));
assert!(sitting.elapsed_secs().is_some_and(|elapsed| elapsed >= 2));
}
#[test]
fn exam_run_configuration_overrides_only_its_owned_fields() {
let ask = AskConfig {
model: Some("fallback".to_string()),
timeout_secs: 12,
cwd: Some(PathBuf::from("/source")),
source_access: true,
effort: Some("high".to_string()),
..AskConfig::default()
};
let exam = ExamConfig {
model: Some("exam-model".to_string()),
timeout_secs: 34,
..ExamConfig::default()
};
let actual = run_config(&exam, &ask);
assert_eq!(Some("exam-model".to_string()), actual.model);
assert_eq!(34, actual.timeout_secs);
assert_eq!(None, actual.cwd);
assert!(!actual.source_access);
assert_eq!(Some("high".to_string()), actual.effort);
}
#[test]
fn disconnected_exam_workers_have_a_specific_error() {
assert_eq!("the exam helper exited unexpectedly", thread_gone());
}
#[test]
fn generate_questions_from_rejects_url_source_on_fetch_incapable_backend() {
use crate::config::BackendKind;
let ask_cfg = AskConfig {
backend: BackendKind::Codex,
command: "/dev/null".to_string(),
timeout_secs: 5,
..AskConfig::default()
};
let rx = spawn_questions(
url_layers(&["https://example.org/doc"]),
None,
ExamConfig::default(),
ask_cfg,
);
let err = rx.recv().unwrap().unwrap_err();
assert!(
err.contains("codex") && err.contains("can't fetch a url"),
"expected capability-refusal message, got: {err}"
);
}
#[test]
fn ensure_backend_can_examine_rejects_url_source_on_fetch_incapable_backend() {
use crate::config::BackendKind;
let deck = deck_with_sources(&["https://example.org/doc"]);
let ask_cfg = AskConfig {
backend: BackendKind::Codex,
command: "/dev/null".to_string(),
timeout_secs: 5,
..AskConfig::default()
};
let err = ensure_backend_can_examine(&deck, &ask_cfg).unwrap_err();
let msg = format!("{err:#}");
assert!(
msg.contains("codex") && msg.contains("can't fetch a url"),
"expected capability-refusal message, got: {msg}"
);
}
#[test]
fn generate_questions_rejects_an_empty_reply() {
let _lock = exec_lock();
let dir = tempfile::tempdir().unwrap();
let cli = fake_reply(dir.path(), "{\"questions\":[]}");
let deck = deck_with_sources(&["https://x"]);
let err = generate_questions(&deck, &ExamConfig::default(), &ask_config(&cli)).unwrap_err();
assert!(format!("{err:#}").contains("no questions"));
}
#[test]
fn generate_questions_rejects_a_malformed_question() {
let _lock = exec_lock();
for (label, reply) in [
(
"empty prompt",
"{\"questions\":[{\"prompt\":\"\",\"points\":[\"point\"]}]}",
),
(
"empty points",
"{\"questions\":[{\"prompt\":\"question\",\"points\":[]}]}",
),
] {
let dir = tempfile::tempdir().unwrap();
let cli = fake_reply(dir.path(), reply);
let deck = deck_with_sources(&["https://x"]);
let err =
generate_questions(&deck, &ExamConfig::default(), &ask_config(&cli)).unwrap_err();
assert!(
format!("{err:#}").contains("malformed"),
"{label} must be rejected"
);
}
}
#[test]
fn grade_answers_rejects_a_count_mismatch() {
let qs = vec![
ExamQuestion {
prompt: "Q1".to_string(),
points: vec!["p".to_string()],
},
ExamQuestion {
prompt: "Q2".to_string(),
points: vec!["p".to_string()],
},
];
let err = grade_answers(
&qs,
&["only one".to_string()],
Strictness::Balanced,
&ExamConfig::default(),
&ask_config(std::path::Path::new("unused")),
)
.unwrap_err();
assert!(format!("{err:#}").contains("expected 2 answers, got 1"));
}
#[test]
fn grade_answers_rejects_a_wrong_grade_count() {
let _lock = exec_lock();
let dir = tempfile::tempdir().unwrap();
let cli = fake_reply(
dir.path(),
"{\"grades\":[{\"verdict\":\"pass\",\"feedback\":\"ok\",\"missed\":[]}]}",
);
let qs = vec![
ExamQuestion {
prompt: "Q1".to_string(),
points: vec!["p".to_string()],
},
ExamQuestion {
prompt: "Q2".to_string(),
points: vec!["p".to_string()],
},
];
let err = grade_answers(
&qs,
&["a".to_string(), "b".to_string()],
Strictness::Balanced,
&ExamConfig::default(),
&ask_config(&cli),
)
.unwrap_err();
assert!(format!("{err:#}").contains("graded 1 of 2"));
}
#[test]
fn remediation_cards_rejects_no_gaps() {
let err = remediation_cards(
&[],
&ExamConfig::default(),
&ask_config(std::path::Path::new("unused")),
)
.unwrap_err();
assert!(format!("{err:#}").contains("no gaps"));
}
#[test]
fn generate_questions_end_to_end() {
let _lock = exec_lock();
let dir = tempfile::tempdir().unwrap();
let json = "{\"questions\":[{\"prompt\":\"Why move?\",\
\"points\":[\"avoids double free\"]}]}";
let cli = fake_reply(dir.path(), json);
let deck = deck_with_sources(&["https://x"]);
let cfg = ExamConfig {
num_questions: 1,
..ExamConfig::default()
};
let qs = generate_questions(&deck, &cfg, &ask_config(&cli)).unwrap();
assert_eq!(1, qs.len());
assert_eq!("Why move?", qs[0].prompt);
assert_eq!(vec!["avoids double free"], qs[0].points);
}
#[test]
fn grade_answers_end_to_end_pass() {
let _lock = exec_lock();
let dir = tempfile::tempdir().unwrap();
let json = "{\"grades\":[{\"verdict\":\"pass\",\"feedback\":\"good\",\"missed\":[]}]}";
let cli = fake_reply(dir.path(), json);
let qs = vec![ExamQuestion {
prompt: "Q".to_string(),
points: vec!["p".to_string()],
}];
let result = grade_answers(
&qs,
&["my answer".to_string()],
Strictness::Balanced,
&ExamConfig::default(),
&ask_config(&cli),
)
.unwrap();
assert!(result.passed);
assert_eq!(Verdict::Pass, result.grades[0].verdict);
}
#[test]
fn remediation_cards_end_to_end() {
let _lock = exec_lock();
let dir = tempfile::tempdir().unwrap();
let cli = fake_reply(dir.path(), "```text\n## Why?\npoint\n```\n");
let cards = remediation_cards(
&["the gap".to_string()],
&ExamConfig::default(),
&ask_config(&cli),
)
.unwrap();
assert_eq!("## Why?\npoint", cards);
}
#[test]
fn remediation_cards_rejects_a_reply_without_cards() {
let _lock = exec_lock();
let dir = tempfile::tempdir().unwrap();
let cli = fake_reply(dir.path(), "Sure, here is some advice on those concepts.");
let err = remediation_cards(
&["the gap".to_string()],
&ExamConfig::default(),
&ask_config(&cli),
)
.unwrap_err();
assert!(format!("{err:#}").contains("without any cards"));
}
#[test]
fn local_file_source_is_embedded() {
let dir = std::env::temp_dir().join(format!("alix-exam-src-{}", std::process::id()));
std::fs::create_dir_all(&dir).unwrap();
let src = dir.join("notes.md");
std::fs::write(&src, "the ground truth text").unwrap();
let deck = Deck {
path: dir.join("d.md"),
subject: "d.md".to_string(),
deck_token: None,
cards: Vec::new(),
definitions: Vec::new(),
links: Vec::new(),
requires: Vec::new(),
sources: vec!["notes.md".to_string()],
settings: Default::default(),
title: None,
description: None,
trace: None,
load_warnings: Vec::new(),
};
let section = source_section(&deck.source_layers(), deck.path.parent()).unwrap();
assert!(section.contains("the ground truth text"));
assert!(section.contains("notes.md"));
assert!(section.contains("primary grounding"));
std::fs::remove_dir_all(&dir).ok();
}
fn drain(s: &mut Sitting, store: &mut Store) {
for _ in 0..500 {
if s.poll(store, 0, Some(crate::session::DEFAULT_RETIRE_AFTER_DAYS))
.advanced
{
return;
}
std::thread::sleep(std::time::Duration::from_millis(10));
}
panic!("exam background call did not complete");
}
fn advance_until_idle(s: &mut Sitting) -> Option<Effect> {
for _ in 0..500 {
let effect = s.advance(0);
if !s.thinking() {
return effect;
}
std::thread::sleep(std::time::Duration::from_millis(10));
}
panic!("exam background call did not complete");
}
fn branching_cli(dir: &std::path::Path, grades: &str) -> std::path::PathBuf {
let body = format!(
"input=$(cat)\n\
case \"$input\" in\n\
*'\"grades\"'*) printf '%s' '{grades}' ;;\n\
*'\"questions\"'*) printf '%s' '{{\"questions\":[{{\"prompt\":\"Q1\",\"points\":[\"p1\"]}},{{\"prompt\":\"Q2\",\"points\":[\"p2\"]}}]}}' ;;\n\
*) printf '## Why does X?\\npoint one\\n' ;;\n\
esac"
);
fake_cli(dir, &body)
}
fn sidecar_cards(deck_path: &std::path::Path, deck_id: &str) -> Vec<crate::card::Card> {
let text =
std::fs::read_to_string(crate::personal::sidecar_path(deck_path)).unwrap_or_default();
parser::parse_str(deck_id, &text).unwrap_or_default()
}
fn sourced_deck(dir: &std::path::Path) -> Deck {
let path = dir.join("d.md");
std::fs::write(
&path,
"---\nformat-version: 1\nid: \"deck-d1\"\nsource: https://x\n---\n## c\na\n<!-- id: card-qc -->\n",
)
.unwrap();
Deck::load(&path).unwrap()
}
#[test]
fn sitting_drives_generate_answer_grade_remediate_on_fail() {
let _lock = exec_lock();
let dir = tempfile::tempdir().unwrap();
let cli = branching_cli(
dir.path(),
"{\"grades\":[{\"verdict\":\"fail\",\"feedback\":\"no\",\"missed\":[\"the move rule\"]},\
{\"verdict\":\"pass\",\"feedback\":\"ok\",\"missed\":[]}]}",
);
let deck = sourced_deck(dir.path());
let mut store = Store::open(dir.path().join("p.json")).unwrap();
let mut s = Sitting::start(
&deck,
Strictness::Balanced,
ExamConfig::default(),
ask_config(&cli),
);
assert_eq!(&Phase::Generating, s.phase());
drain(&mut s, &mut store);
assert_eq!(&Phase::Answering, s.phase());
assert_eq!(2, s.total());
assert!(!s.on_last());
s.set_answer("a1".to_string());
s.next();
assert!(s.on_last());
s.set_answer("a2".to_string());
s.prev();
assert_eq!("a1", s.answer());
s.next();
s.submit();
assert_eq!(&Phase::Grading, s.phase());
drain(&mut s, &mut store);
assert_eq!(&Phase::Results, s.phase());
assert!(!s.result().unwrap().passed);
assert_eq!(vec!["the move rule"], s.gaps());
assert!(!store.deck_mastered("deck-d1"));
assert!(s.can_remediate());
let snapshot = std::fs::read(dir.path().join("d.md")).unwrap();
s.remediate();
assert_eq!(&Phase::Remediating, s.phase());
drain(&mut s, &mut store);
assert_eq!(&Phase::Remediated, s.phase());
assert_eq!(snapshot, std::fs::read(dir.path().join("d.md")).unwrap());
let personal = sidecar_cards(&dir.path().join("d.md"), "deck-d1");
assert_eq!(1, personal.len());
assert_eq!("Why does X?", personal[0].front);
assert_eq!(vec!["point one".to_string()], personal[0].back);
}
#[test]
fn exam_fail_reports_the_remediation_count() {
let _lock = exec_lock();
let dir = tempfile::tempdir().unwrap();
let cli = branching_cli(
dir.path(),
"{\"grades\":[{\"verdict\":\"fail\",\"feedback\":\"no\",\"missed\":[\"the move rule\"]},\
{\"verdict\":\"pass\",\"feedback\":\"ok\",\"missed\":[]}]}",
);
let deck = sourced_deck(dir.path());
let mut store = Store::open(dir.path().join("p.json")).unwrap();
let mut s = Sitting::start(
&deck,
Strictness::Balanced,
ExamConfig::default(),
ask_config(&cli),
);
drain(&mut s, &mut store);
s.set_answer("a1".to_string());
s.next();
s.set_answer("a2".to_string());
s.submit();
drain(&mut s, &mut store);
assert!(s.can_remediate());
assert_eq!(None, s.remediated_count());
s.remediate();
drain(&mut s, &mut store);
assert_eq!(&Phase::Remediated, s.phase());
let personal = sidecar_cards(&dir.path().join("d.md"), "deck-d1");
assert_eq!(1, personal.len());
assert_eq!(Some(personal.len()), s.remediated_count());
}
#[test]
fn sitting_pass_marks_mastered() {
let _lock = exec_lock();
let dir = tempfile::tempdir().unwrap();
let cli = branching_cli(
dir.path(),
"{\"grades\":[{\"verdict\":\"pass\",\"feedback\":\"ok\",\"missed\":[]},\
{\"verdict\":\"pass\",\"feedback\":\"ok\",\"missed\":[]}]}",
);
let deck = sourced_deck(dir.path());
let mut store = Store::open(dir.path().join("p.json")).unwrap();
let mut s = Sitting::start(
&deck,
Strictness::Strict,
ExamConfig::default(),
ask_config(&cli),
);
drain(&mut s, &mut store);
s.set_answer("a".to_string());
s.next();
s.set_answer("b".to_string());
s.submit();
drain(&mut s, &mut store);
assert_eq!(&Phase::Results, s.phase());
assert!(s.result().unwrap().passed);
assert!(store.deck_mastered("deck-d1"));
assert!(!s.can_remediate());
}
#[test]
fn a_failed_exam_writes_remediation_to_the_sidecar_and_leaves_the_deck_unchanged() {
let _lock = exec_lock();
let dir = tempfile::tempdir().unwrap();
let cli = branching_cli(
dir.path(),
"{\"grades\":[{\"verdict\":\"fail\",\"feedback\":\"no\",\"missed\":[\"the move rule\"]},\
{\"verdict\":\"pass\",\"feedback\":\"ok\",\"missed\":[]}]}",
);
let deck = sourced_deck(dir.path());
let deck_path = dir.path().join("d.md");
let before = std::fs::read(&deck_path).unwrap();
let mut store = Store::open(dir.path().join("p.json")).unwrap();
let mut s = Sitting::start(
&deck,
Strictness::Balanced,
ExamConfig::default(),
ask_config(&cli),
);
drain(&mut s, &mut store);
s.set_answer("a1".to_string());
s.next();
s.set_answer("a2".to_string());
s.submit();
drain(&mut s, &mut store);
assert!(s.can_remediate());
s.remediate();
drain(&mut s, &mut store);
assert_eq!(&Phase::Remediated, s.phase());
let after = std::fs::read(&deck_path).unwrap();
assert_eq!(before, after, "remediation must never touch the deck file");
let sidecar = std::fs::read_to_string(crate::personal::sidecar_path(&deck_path))
.expect("remediation wrote a sidecar beside the deck");
assert!(
sidecar.contains("for: deck-d1"),
"the sidecar names its deck: {sidecar}"
);
let cards = crate::parser::parse_str("deck-d1", &sidecar).unwrap();
assert_eq!(1, cards.len(), "one remediation card: {sidecar}");
assert!(
cards[0].id().is_some_and(|id| store.get(&id).is_some()),
"the store carries the card's schedule, the file carries its content"
);
}
#[test]
fn a_re_pass_does_not_retire_the_remediation_batch() {
let _lock = exec_lock();
let dir = tempfile::tempdir().unwrap();
let cli_dir1 = tempfile::tempdir().unwrap();
let cli1 = branching_cli(
cli_dir1.path(),
"{\"grades\":[{\"verdict\":\"fail\",\"feedback\":\"no\",\"missed\":[\"the move rule\"]},\
{\"verdict\":\"pass\",\"feedback\":\"ok\",\"missed\":[]}]}",
);
let deck = sourced_deck(dir.path());
let mut store = Store::open(dir.path().join("p.json")).unwrap();
let mut s = Sitting::start(
&deck,
Strictness::Balanced,
ExamConfig::default(),
ask_config(&cli1),
);
drain(&mut s, &mut store);
s.set_answer("a1".to_string());
s.next();
s.set_answer("a2".to_string());
s.submit();
drain(&mut s, &mut store);
assert!(!s.result().unwrap().passed);
s.remediate();
drain(&mut s, &mut store);
assert_eq!(&Phase::Remediated, s.phase());
assert!(!sidecar_cards(&dir.path().join("d.md"), "deck-d1").is_empty());
let cli_dir2 = tempfile::tempdir().unwrap();
let cli2 = branching_cli(
cli_dir2.path(),
"{\"grades\":[{\"verdict\":\"pass\",\"feedback\":\"ok\",\"missed\":[]},\
{\"verdict\":\"pass\",\"feedback\":\"ok\",\"missed\":[]}]}",
);
let mut s2 = Sitting::start(
&deck,
Strictness::Balanced,
ExamConfig::default(),
ask_config(&cli2),
);
drain(&mut s2, &mut store);
s2.set_answer("a".to_string());
s2.next();
s2.set_answer("b".to_string());
s2.submit();
drain(&mut s2, &mut store);
assert!(s2.result().unwrap().passed);
assert!(store.deck_mastered("deck-d1"));
for card in sidecar_cards(&dir.path().join("d.md"), "deck-d1") {
let id = card.id().expect("a personal card carries its id");
assert!(
!is_retired_id(&id, &store, Some(crate::session::DEFAULT_RETIRE_AFTER_DAYS)),
"a passing re-sit must not retire remediation cards"
);
}
}
#[test]
fn advance_surfaces_remediation_text_without_a_store() {
let _lock = exec_lock();
let dir = tempfile::tempdir().unwrap();
let cli = branching_cli(
dir.path(),
"{\"grades\":[{\"verdict\":\"fail\",\"feedback\":\"no\",\"missed\":[\"the move rule\"]},\
{\"verdict\":\"pass\",\"feedback\":\"ok\",\"missed\":[]}]}",
);
let deck = sourced_deck(dir.path());
let mut store = Store::open(dir.path().join("p.json")).unwrap();
let mut s = Sitting::start(
&deck,
Strictness::Balanced,
ExamConfig::default(),
ask_config(&cli),
);
drain(&mut s, &mut store);
s.set_answer("a1".to_string());
s.next();
s.set_answer("a2".to_string());
s.submit();
drain(&mut s, &mut store);
assert!(s.can_remediate());
s.remediate();
assert_eq!(&Phase::Remediating, s.phase());
let effect = advance_until_idle(&mut s);
let Some(Effect::RemediationCards(text)) = effect else {
panic!("expected Some(Effect::RemediationCards(_)), got {effect:?}");
};
let synth = parser::parse_str("d.md", &text).unwrap();
assert_eq!("Why does X?", synth[0].front);
assert_eq!(vec!["point one".to_string()], synth[0].back);
assert_eq!(&Phase::Remediated, s.phase());
assert_eq!(None, s.remediated_count());
}
#[test]
fn advance_keeps_grade_fail_storeless() {
let _lock = exec_lock();
let dir = tempfile::tempdir().unwrap();
let cli = branching_cli(
dir.path(),
"{\"grades\":[{\"verdict\":\"fail\",\"feedback\":\"no\",\"missed\":[\"the move rule\"]},\
{\"verdict\":\"pass\",\"feedback\":\"ok\",\"missed\":[]}]}",
);
let deck = sourced_deck(dir.path());
let mut s = Sitting::start(
&deck,
Strictness::Balanced,
ExamConfig::default(),
ask_config(&cli),
);
assert_eq!(None, advance_until_idle(&mut s));
assert_eq!(&Phase::Answering, s.phase());
s.set_answer("a1".to_string());
s.next();
s.set_answer("a2".to_string());
s.submit();
let effect = advance_until_idle(&mut s);
assert_eq!(None, effect);
assert_eq!(&Phase::Results, s.phase());
assert!(!s.result().unwrap().passed);
}
#[test]
fn set_answers_rejects_wrong_arity_and_wrong_phase() {
let _lock = exec_lock();
let dir = tempfile::tempdir().unwrap();
let cli = branching_cli(
dir.path(),
"{\"grades\":[{\"verdict\":\"pass\",\"feedback\":\"ok\",\"missed\":[]},\
{\"verdict\":\"pass\",\"feedback\":\"ok\",\"missed\":[]}]}",
);
let deck = sourced_deck(dir.path());
let mut s = Sitting::start(
&deck,
Strictness::Balanced,
ExamConfig::default(),
ask_config(&cli),
);
assert_eq!(&Phase::Generating, s.phase());
assert!(!s.set_answers(Vec::new()));
assert!(s.answers().is_empty());
advance_until_idle(&mut s);
assert_eq!(&Phase::Answering, s.phase());
assert_eq!(2, s.total());
assert!(!s.set_answers(vec!["only one".to_string()]));
assert_eq!(vec!["".to_string(), "".to_string()], s.answers());
assert!(s.set_answers(vec!["a1".to_string(), "a2".to_string()]));
assert_eq!(vec!["a1".to_string(), "a2".to_string()], s.answers());
}
#[test]
fn grade_compression_prompt_carries_path_points_and_answer() {
let p = grade_compression_prompt(
"how a keypress becomes a saved grade",
&[
"the keypress posts only the grade".to_string(),
"the server reschedules the card".to_string(),
],
"you press a key and it saves",
Strictness::Balanced,
);
assert!(p.contains("how a keypress becomes a saved grade"));
assert!(p.contains("posts only the grade"));
assert!(p.contains("you press a key and it saves"));
assert!(p.contains("RE-DERIVE"));
assert!(p.contains("\"verdict\""));
}
#[test]
fn grade_compression_prompt_marks_an_empty_answer() {
let p = grade_compression_prompt("d", &["p".to_string()], " ", Strictness::Strict);
assert!(p.contains("(no answer given)"));
}
#[test]
fn compression_criteria_vary_by_strictness() {
assert!(compression_strictness_criteria(Strictness::Strict).contains("every load-bearing"));
assert!(compression_strictness_criteria(Strictness::Balanced).contains("RE-DERIVE"));
assert!(compression_strictness_criteria(Strictness::Lenient).contains("generously"));
}
#[test]
fn grade_compression_end_to_end_pass() {
let _lock = exec_lock();
let dir = tempfile::tempdir().unwrap();
let cli = fake_reply(
dir.path(),
"{\"verdict\":\"pass\",\"feedback\":\"passed the chain\",\"missed\":[]}",
);
let result = grade_compression(
"how X becomes Y",
&["step one".to_string()],
"my retrace",
Strictness::Balanced,
&ExamConfig::default(),
&ask_config(&cli),
)
.unwrap();
assert!(result.passed);
assert_eq!(Verdict::Pass, result.grades[0].verdict);
}
fn trace_sitting(cli: &std::path::Path) -> Sitting {
Sitting::start_trace(
"how a moves".to_string(),
vec!["it advances".to_string()],
"t.txt".to_string(),
"t1".to_string(),
Strictness::Balanced,
ExamConfig::default(),
ask_config(cli),
)
}
#[test]
fn trace_sitting_passes_and_masters() {
let _lock = exec_lock();
let dir = tempfile::tempdir().unwrap();
let cli = fake_reply(
dir.path(),
"{\"verdict\":\"pass\",\"feedback\":\"good\",\"missed\":[]}",
);
let mut store = Store::open(dir.path().join("p.json")).unwrap();
let mut s = trace_sitting(&cli);
assert_eq!(SittingKind::Trace, s.kind());
assert_eq!(&Phase::Answering, s.phase());
assert_eq!(1, s.total());
s.set_answer("my retrace of the path".to_string());
s.submit();
assert_eq!(&Phase::Grading, s.phase());
drain(&mut s, &mut store);
assert_eq!(&Phase::Results, s.phase());
assert!(s.result().unwrap().passed);
assert!(store.deck_mastered("t1"));
assert!(!s.can_remediate());
}
#[test]
fn trace_sitting_fail_starts_cooldown_without_mastering() {
let _lock = exec_lock();
let dir = tempfile::tempdir().unwrap();
let cli = fake_reply(
dir.path(),
"{\"verdict\":\"fail\",\"feedback\":\"missed the return path\",\"missed\":[\"the return\"]}",
);
let mut store = Store::open(dir.path().join("p.json")).unwrap();
let mut s = trace_sitting(&cli);
s.set_answer("wrong".to_string());
s.submit();
drain(&mut s, &mut store);
assert_eq!(&Phase::Results, s.phase());
assert!(!s.result().unwrap().passed);
assert!(!store.deck_mastered("t1"));
assert!(store.exam_failed_at("t1").is_some());
assert!(!s.can_remediate());
}
}
#[cfg(all(test, windows))]
mod windows_tests {
use super::*;
#[test]
fn a_windows_root_relative_source_does_not_send_its_directory_layout() {
let dir = tempfile::tempdir().unwrap();
let source = dir.path().join("private").join("notes.md");
std::fs::create_dir(source.parent().unwrap()).unwrap();
std::fs::write(&source, "private body").unwrap();
let prefix_len = source.components().next().unwrap().as_os_str().len();
let declared = source.to_string_lossy()[prefix_len..].to_string();
assert!(Path::new(&declared).has_root(), "{declared}");
assert!(!Path::new(&declared).is_absolute(), "{declared}");
let sources = SourceLayers {
own: vec![declared.clone()],
workspace: Vec::new(),
};
let prompt = questions_prompt(&sources, Some(dir.path()), &ExamConfig::default()).unwrap();
assert!(
prompt.contains("Source file `notes.md`"),
"a drive-rooted declaration should reach the prompt as its basename: {prompt}"
);
assert!(
!prompt.contains(&declared),
"the operator's root-relative directory layout reached the prompt: {prompt}"
);
}
}