1use crate::ast::{
6 Block, Blockquote, Doc, Example, Fence, Heading, ListItem, Paragraph, Row, SegmentOffset,
7 Table, TableOrFence, ThematicBreak,
8};
9use crate::diagnostics::{Diagnostic, DiagnosticCode, Severity};
10use crate::error::{StepError, StepFailure};
11use crate::execute::{ExecutePorts, StepObservation, StepOutcome, collect_examples};
12use crate::plan::{ExecutionPlan, PlannedExample, PlannedStep, plan};
13use crate::reference::OathWorkspace;
14use crate::registry::Registry;
15use crate::span::Span;
16use crate::value::Value;
17use std::any::Any;
18use std::cell::RefCell;
19use std::collections::{BTreeMap, HashMap};
20use std::rc::Rc;
21
22pub use crate::expression::parameter_type_names;
23
24pub struct BundleArtifacts {
26 pub doc: Value,
27 pub registry: Value,
28 pub plan: Value,
29 pub trace: Value,
30}
31
32fn obj(pairs: Vec<(&str, Value)>) -> Value {
33 let mut m = BTreeMap::new();
34 for (k, v) in pairs {
35 m.insert(k.to_string(), v);
36 }
37 Value::Map(m)
38}
39
40fn vint(n: usize) -> Value {
41 Value::Int(n as i64)
42}
43
44pub fn to_doc_artifact(doc: &Doc) -> Value {
50 obj(vec![
51 ("path", Value::from(doc.path.as_str())),
52 ("examples", Value::List(doc.examples.iter().map(example).collect())),
53 (
54 "orphanAttachments",
55 Value::List(doc.orphan_attachments.iter().map(table_or_fence).collect()),
56 ),
57 ("headings", Value::List(doc.headings.iter().map(heading).collect())),
58 ])
59}
60
61fn span(s: Span) -> Value {
62 obj(vec![
63 ("startOffset", vint(s.start_offset)),
64 ("endOffset", vint(s.end_offset)),
65 ("startLine", vint(s.start_line)),
66 ("startCol", vint(s.start_col)),
67 ("endLine", vint(s.end_line)),
68 ("endCol", vint(s.end_col)),
69 ])
70}
71
72fn segment_offset(o: &SegmentOffset) -> Value {
73 obj(vec![
74 ("textOffset", vint(o.text_offset)),
75 ("sourceOffset", vint(o.source_offset)),
76 ])
77}
78
79fn segment_map(map: &[SegmentOffset]) -> Value {
80 Value::List(map.iter().map(segment_offset).collect())
81}
82
83fn row(r: &Row) -> Value {
84 obj(vec![
85 ("cells", Value::List(r.cells.iter().map(|c| Value::from(c.as_str())).collect())),
86 ("cellSpans", Value::List(r.cell_spans.iter().map(|s| span(*s)).collect())),
87 ("span", span(r.span)),
88 ])
89}
90
91fn table(t: &Table) -> Value {
92 obj(vec![
93 ("kind", Value::from("table")),
94 ("span", span(t.span)),
95 ("header", row(&t.header)),
96 ("rows", Value::List(t.rows.iter().map(row).collect())),
97 ])
98}
99
100fn fence(f: &Fence) -> Value {
101 obj(vec![
102 ("kind", Value::from("fence")),
103 ("span", span(f.span)),
104 ("info", Value::from(f.info.as_str())),
105 ("body", Value::from(f.body.as_str())),
106 ("bodySpan", span(f.body_span)),
107 ])
108}
109
110fn heading(h: &Heading) -> Value {
111 obj(vec![
112 ("kind", Value::from("heading")),
113 ("level", vint(h.level)),
114 ("text", Value::from(h.text.as_str())),
115 ("span", span(h.span)),
116 ])
117}
118
119fn paragraph(p: &Paragraph) -> Value {
120 obj(vec![
121 ("kind", Value::from("paragraph")),
122 ("text", Value::from(p.text.as_str())),
123 ("span", span(p.span)),
124 ("segmentMap", segment_map(&p.segment_map)),
125 ])
126}
127
128fn list_item(l: &ListItem) -> Value {
129 obj(vec![
130 ("kind", Value::from("list_item")),
131 ("text", Value::from(l.text.as_str())),
132 ("span", span(l.span)),
133 ("segmentMap", segment_map(&l.segment_map)),
134 ("ordered", Value::Bool(l.ordered)),
135 ("markerSpan", span(l.marker_span)),
136 ])
137}
138
139fn blockquote(b: &Blockquote) -> Value {
140 obj(vec![
141 ("kind", Value::from("blockquote")),
142 ("text", Value::from(b.text.as_str())),
143 ("span", span(b.span)),
144 ("segmentMap", segment_map(&b.segment_map)),
145 ])
146}
147
148fn thematic_break(t: &ThematicBreak) -> Value {
149 obj(vec![
150 ("kind", Value::from("thematic_break")),
151 ("span", span(t.span)),
152 ])
153}
154
155fn block(b: &Block) -> Value {
156 match b {
157 Block::Heading(h) => heading(h),
158 Block::Paragraph(p) => paragraph(p),
159 Block::ListItem(l) => list_item(l),
160 Block::Blockquote(b) => blockquote(b),
161 Block::Table(t) => table(t),
162 Block::Fence(f) => fence(f),
163 Block::ThematicBreak(t) => thematic_break(t),
164 }
165}
166
167fn table_or_fence(tf: &TableOrFence) -> Value {
168 match tf {
169 TableOrFence::Table(t) => table(t),
170 TableOrFence::Fence(f) => fence(f),
171 }
172}
173
174fn example(e: &Example) -> Value {
175 obj(vec![
176 (
177 "scopeStack",
178 Value::List(
179 e.scope_stack
180 .iter()
181 .map(|s| Value::from(s.as_str()))
182 .collect(),
183 ),
184 ),
185 ("span", span(e.span)),
186 ("body", Value::List(e.body.iter().map(block).collect())),
187 ("precededByDelimiter", Value::Bool(e.preceded_by_delimiter)),
188 ])
189}
190
191pub fn to_registry_artifact(registry: &Registry) -> Value {
197 let steps: Vec<Value> = registry
198 .steps
199 .iter()
200 .map(|s| {
201 obj(vec![
202 ("expression", Value::from(s.expression.as_str())),
203 (
204 "parameterTypeNames",
205 Value::List(
206 parameter_type_names(&s.expression)
207 .into_iter()
208 .map(Value::from)
209 .collect(),
210 ),
211 ),
212 ])
213 })
214 .collect();
215 let parameter_types: Vec<Value> = registry
216 .custom_parameter_types
217 .iter()
218 .map(|p| {
219 obj(vec![
220 ("name", Value::from(p.name.as_str())),
221 ("regexp", Value::from(p.regexp.as_str())),
222 ])
223 })
224 .collect();
225 obj(vec![
226 ("steps", Value::List(steps)),
227 ("parameterTypes", Value::List(parameter_types)),
228 ])
229}
230
231pub fn to_plan_artifact(plan: &ExecutionPlan) -> Value {
237 let source = &plan.doc.source;
238 obj(vec![
239 (
240 "examples",
241 Value::List(
242 plan.examples
243 .iter()
244 .map(|ex| planned_example(source, ex))
245 .collect(),
246 ),
247 ),
248 ("diagnostics", Value::List(plan.diagnostics.iter().map(diagnostic).collect())),
249 ])
250}
251
252fn planned_example(source: &str, ex: &PlannedExample) -> Value {
253 let mut pairs = vec![
254 ("name", Value::from(ex.name.as_str())),
255 (
256 "scopeStack",
257 Value::List(
258 ex.scope_stack
259 .iter()
260 .map(|s| Value::from(s.as_str()))
261 .collect(),
262 ),
263 ),
264 ("span", span(ex.span)),
265 ("expectedOutcome", Value::from(ex.expected_outcome.as_deref().unwrap_or("pass"))),
266 ];
267 if let Some(msg) = &ex.expected_error_message {
268 pairs.push(("expectedErrorMessage", Value::from(msg.as_str())));
269 }
270 pairs.push(("steps", Value::List(ex.steps.iter().map(|s| planned_step(source, s)).collect())));
271 obj(pairs)
272}
273
274fn planned_step(_source: &str, step: &PlannedStep) -> Value {
275 let param_names = parameter_type_names(&step.step_def.expression);
276 let args: Vec<Value> = step
277 .param_texts
278 .iter()
279 .enumerate()
280 .map(|(i, text)| {
281 obj(vec![
282 ("value", Value::from(text.as_str())),
283 (
284 "parameterType",
285 param_names
286 .get(i)
287 .map_or(Value::Null, |n| Value::from(n.as_str())),
288 ),
289 ])
290 })
291 .collect();
292
293 let mut pairs = vec![
294 ("text", Value::from(step.text.as_str())),
295 ("matchSpan", span(step.match_span)),
296 ("paramSpans", Value::List(step.param_spans.iter().map(|s| span(*s)).collect())),
297 ("matchedExpression", Value::from(step.step_def.expression.as_str())),
298 ("args", Value::List(args)),
299 ];
300 if let Some(p) = &step.doc_path {
303 pairs.push(("docPath", Value::from(p.as_str())));
304 }
305 if let Some(t) = &step.data_table {
306 pairs.push(("dataTable", table(t)));
307 }
308 if let Some(f) = &step.doc_string {
309 pairs.push(("docString", doc_string(f)));
310 }
311 obj(pairs)
312}
313
314fn doc_string(f: &Fence) -> Value {
315 obj(vec![
316 ("content", Value::from(f.body.as_str())),
317 ("contentType", Value::from(f.info.as_str())),
318 ("span", span(f.body_span)),
319 ])
320}
321
322fn diagnostic(d: &Diagnostic) -> Value {
323 obj(vec![
324 ("code", Value::from(diagnostic_code(d.code))),
325 ("severity", Value::from(severity(d.severity))),
326 ("span", span(d.span)),
327 ])
328}
329
330fn diagnostic_code(code: DiagnosticCode) -> &'static str {
331 match code {
332 DiagnosticCode::AmbiguousMatch => "ambiguous-match",
333 DiagnosticCode::ErrorFenceWithoutStep => "error-fence-without-step",
334 DiagnosticCode::Drift => "drift",
335 DiagnosticCode::ReferenceNotFound => "reference-not-found",
336 DiagnosticCode::ReferenceEmpty => "reference-empty",
337 DiagnosticCode::ReferenceCycle => "reference-cycle",
338 DiagnosticCode::AmbiguousAnchor => "ambiguous-anchor",
339 }
340}
341
342fn severity(s: Severity) -> &'static str {
343 match s {
344 Severity::Error => "error",
345 Severity::Warning => "warning",
346 Severity::Info => "info",
347 }
348}
349
350pub fn to_failure_artifact(failure: Option<&StepFailure>, match_span: Span) -> Value {
357 let line = match_span.start_line;
358 let anchor_span = match failure {
359 Some(f) => crate::failure_anchor::anchor(&f.error, match_span),
360 None => match_span,
361 };
362 let anchor = span(anchor_span);
363
364 let err = failure.map(|f| &f.error);
365 match err {
366 Some(StepError::CellMismatch(cells)) => {
367 let failing: Vec<Value> = cells.iter().filter(|c| !c.ok).map(failure_cell).collect();
368 obj(vec![
369 ("kind", Value::from("cell-mismatch")),
370 ("line", vint(line)),
371 ("anchor", anchor),
372 ("message", Value::from(err.unwrap().message().as_str())),
373 ("cells", Value::List(failing)),
374 ])
375 }
376 Some(e @ StepError::ReturnShape(_)) => {
377 kind_line_anchor("return-shape", line, anchor, Some(e.message()))
378 }
379 Some(e @ StepError::UnexpectedPass) => {
380 kind_line_anchor("unexpected-pass", line, anchor, Some(e.message()))
381 }
382 _ => kind_line_anchor("thrown", line, anchor, None),
383 }
384}
385
386fn failure_cell(c: &crate::cell_diff::CellDiff) -> Value {
387 obj(vec![
388 ("column", Value::from(c.column.as_str())),
389 ("expected", Value::from(c.expected.as_str())),
390 ("actual", Value::from(c.actual.as_str())),
391 ("span", span(c.span)),
392 ])
393}
394
395fn kind_line_anchor(kind: &str, line: usize, anchor: Value, message: Option<String>) -> Value {
396 let mut fields = vec![
397 ("kind", Value::from(kind)),
398 ("line", vint(line)),
399 ("anchor", anchor),
400 ];
401 if let Some(m) = message {
402 fields.push(("message", Value::from(m.as_str())));
403 }
404 obj(fields)
405}
406
407fn file_stem(path: &str) -> String {
410 let base = path.rsplit(['/', '\\']).next().unwrap_or(path);
411 match base.rfind('.') {
412 Some(dot) if dot > 0 => base[..dot].to_string(),
413 _ => base.to_string(),
414 }
415}
416
417pub fn run_conformance(
420 doc: &Doc,
421 registry: &Registry,
422 context_factory: &dyn Fn() -> Rc<dyn Any>,
423 workspace: &OathWorkspace,
426) -> BundleArtifacts {
427 let execution = plan(doc, registry, workspace);
428
429 let observed: Rc<RefCell<HashMap<usize, Vec<StepObservation>>>> =
430 Rc::new(RefCell::new(HashMap::new()));
431 let observed_writer = observed.clone();
432 let ports = ExecutePorts {
433 reporter: Box::new(|_| {}),
434 create_context: Some(Box::new(|_| context_factory())),
435 observer: Some(Box::new(move |o: StepObservation| {
436 observed_writer
437 .borrow_mut()
438 .entry(o.example_index)
439 .or_default()
440 .push(o);
441 })),
442 };
443
444 let queue = collect_examples(&execution, &ports);
445 let mut trace_examples = Vec::with_capacity(queue.len());
446 for (k, queued) in queue.iter().enumerate() {
447 let outcome = if queued.run().is_err() {
448 "fail"
449 } else {
450 "pass"
451 };
452
453 let planned = &execution.examples[k];
454 let empty = Vec::new();
455 let obs_map = observed.borrow();
456 let obs = obs_map.get(&k).unwrap_or(&empty);
457
458 let mut steps = Vec::with_capacity(planned.steps.len());
459 for (i, step) in planned.steps.iter().enumerate() {
460 let ordinal = i + 1;
461 let mut chosen: Option<&StepObservation> = None;
463 for o in obs {
464 if o.ordinal != ordinal {
465 continue;
466 }
467 chosen = Some(o);
468 if o.outcome == StepOutcome::Fail {
469 break;
470 }
471 }
472 let step_outcome = chosen.map_or("skipped", |o| o.outcome.as_str());
473
474 let context_key = obj(vec![
475 ("exampleName", Value::from(queued.name.as_str())),
476 (
477 "stepFile",
478 Value::from(file_stem(&step.step_def.expression_source_file).as_str()),
479 ),
480 ]);
481 let mut step_pairs = vec![
482 ("exampleName", Value::from(queued.name.as_str())),
483 ("ordinal", vint(ordinal)),
484 ("stepText", Value::from(step.text.as_str())),
485 ("matchedExpression", Value::from(step.step_def.expression.as_str())),
486 ("contextKey", context_key),
487 ("outcome", Value::from(step_outcome)),
488 ];
489 if step_outcome == "fail" {
490 let failure = chosen.and_then(|o| o.error.as_ref());
491 step_pairs.push(("failure", to_failure_artifact(failure, step.match_span)));
492 }
493 steps.push(obj(step_pairs));
494 }
495
496 trace_examples.push(obj(vec![
497 ("name", Value::from(queued.name.as_str())),
498 ("outcome", Value::from(outcome)),
499 ("steps", Value::List(steps)),
500 ]));
501 }
502
503 let trace = obj(vec![("examples", Value::List(trace_examples))]);
504
505 BundleArtifacts {
506 doc: to_doc_artifact(doc),
507 registry: to_registry_artifact(registry),
508 plan: to_plan_artifact(&execution),
509 trace,
510 }
511}