1use serde::Serialize;
4use vyre_foundation::serial::wire::encode::{
5 ScanDatabaseHeader, ScanDatabaseSectionKind, UnsupportedScanFeature,
6};
7use vyre_lower::{
8 DescriptorIntentKind, DescriptorIntentSet, DescriptorIntentStrategy, IntentAnnotatedDescriptor,
9};
10
11pub const SCAN_EXPLAIN_REPORT_SCHEMA_VERSION: u32 = 1;
12
13#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize)]
14pub enum ScanFactorRole {
15 Literal,
16 Prefix,
17 Suffix,
18 Infix,
19 Outfix,
20}
21
22#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize)]
23pub struct ScanExplainFactor {
24 pub role: ScanFactorRole,
25 pub pattern_index: u32,
26 pub bytes: Vec<u8>,
27 pub digest: u64,
28}
29
30#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize)]
31pub struct ScanExplainEngine {
32 pub strategy: DescriptorIntentStrategy,
33 pub reason: String,
34}
35
36#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize)]
37pub enum ScanExplainExactnessClass {
38 Exact,
39 PrefilterVerified,
40 VerifierRequired,
41 Unsupported,
42}
43
44#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize)]
45pub struct ScanExplainRejectedEngine {
46 pub engine_id: String,
47 pub pattern_index: u32,
48 pub reason: String,
49 pub exactness_class: ScanExplainExactnessClass,
50}
51
52#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize)]
53pub struct ScanExplainVerifierFragment {
54 pub source: String,
55 pub bytes: u64,
56 pub digest: u64,
57}
58
59#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize)]
60pub struct ScanExplainRouteEvidence {
61 pub exactness_class: ScanExplainExactnessClass,
62 pub verifier_cost_estimate_bytes: u64,
63 pub literal_selectivity_basis: String,
64}
65
66#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
67#[non_exhaustive]
68pub struct ScanExplainReport {
69 pub schema_version: u32,
70 pub pattern_set_id: String,
71 pub extracted_factors: Vec<ScanExplainFactor>,
72 pub selected_engines: Vec<ScanExplainEngine>,
73 pub rejected_engines: Vec<ScanExplainRejectedEngine>,
74 pub verifier_fragments: Vec<ScanExplainVerifierFragment>,
75 pub route_evidence: ScanExplainRouteEvidence,
76 pub streaming_state_bytes: u64,
77 pub table_bytes: u64,
78 pub baseline_id: String,
79 pub unsupported_features: Vec<UnsupportedScanFeature>,
80 pub artifact_links: Vec<String>,
81}
82
83impl ScanExplainReport {
84 #[must_use]
85 pub fn is_complete(&self) -> bool {
86 self.schema_version == SCAN_EXPLAIN_REPORT_SCHEMA_VERSION
87 && !self.pattern_set_id.trim().is_empty()
88 && !self.extracted_factors.is_empty()
89 && !self.selected_engines.is_empty()
90 && !self.route_evidence.literal_selectivity_basis.trim().is_empty()
91 && !self.baseline_id.trim().is_empty()
92 && !self.artifact_links.is_empty()
93 && self.table_bytes > 0
94 }
95}
96
97#[derive(Debug, Clone, PartialEq, Eq)]
98pub enum ScanExplainError {
99 MissingPatternSetId,
100 MissingFactors,
101 MissingBaselineId,
102 MissingArtifactLinks,
103 MissingDescriptorIntents,
104 MissingScanDatabase,
105}
106
107impl std::fmt::Display for ScanExplainError {
108 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
109 match self {
110 Self::MissingPatternSetId => {
111 f.write_str("scan explain report is missing pattern_set_id")
112 }
113 Self::MissingFactors => f.write_str("scan explain report has no extracted factors"),
114 Self::MissingBaselineId => f.write_str("scan explain report is missing baseline_id"),
115 Self::MissingArtifactLinks => {
116 f.write_str("scan explain report is missing artifact links")
117 }
118 Self::MissingDescriptorIntents => {
119 f.write_str("scan explain report is missing descriptor intents")
120 }
121 Self::MissingScanDatabase => {
122 f.write_str("scan explain report is missing scan database header")
123 }
124 }
125 }
126}
127
128impl std::error::Error for ScanExplainError {}
129
130pub fn scan_explain_report(
131 pattern_set_id: impl Into<String>,
132 extracted_factors: Vec<ScanExplainFactor>,
133 descriptor: &IntentAnnotatedDescriptor,
134 scan_database: &ScanDatabaseHeader,
135 baseline_id: impl Into<String>,
136 artifact_links: Vec<String>,
137) -> Result<ScanExplainReport, ScanExplainError> {
138 let pattern_set_id = pattern_set_id.into();
139 if pattern_set_id.trim().is_empty() {
140 return Err(ScanExplainError::MissingPatternSetId);
141 }
142 if extracted_factors.is_empty() {
143 return Err(ScanExplainError::MissingFactors);
144 }
145 let baseline_id = baseline_id.into();
146 if baseline_id.trim().is_empty() {
147 return Err(ScanExplainError::MissingBaselineId);
148 }
149 if artifact_links.is_empty() {
150 return Err(ScanExplainError::MissingArtifactLinks);
151 }
152
153 let selected_engines = selected_engines_from_intents(&descriptor.intents)?;
154 let verifier_fragments =
155 verifier_fragments_from_sources(&descriptor.intents, scan_database)?;
156 let rejected_engines = rejected_engines_from_unsupported_features(scan_database);
157 let streaming_state_bytes =
158 streaming_state_bytes_from_sources(&descriptor.intents, scan_database);
159 let table_bytes = scan_database
160 .table_sections
161 .iter()
162 .map(|section| section.byte_len)
163 .sum();
164 let route_evidence = route_evidence_from_sources(
165 &extracted_factors,
166 &selected_engines,
167 &rejected_engines,
168 &verifier_fragments,
169 scan_database,
170 );
171
172 Ok(ScanExplainReport {
173 schema_version: SCAN_EXPLAIN_REPORT_SCHEMA_VERSION,
174 pattern_set_id,
175 extracted_factors,
176 selected_engines,
177 rejected_engines,
178 verifier_fragments,
179 route_evidence,
180 streaming_state_bytes,
181 table_bytes,
182 baseline_id,
183 unsupported_features: scan_database.unsupported_features.clone(),
184 artifact_links,
185 })
186}
187
188fn selected_engines_from_intents(
189 intents: &DescriptorIntentSet,
190) -> Result<Vec<ScanExplainEngine>, ScanExplainError> {
191 if intents.intents.is_empty() {
192 return Err(ScanExplainError::MissingDescriptorIntents);
193 }
194 let mut engines = Vec::new();
195 for intent in &intents.intents {
196 let strategy = intent.kind.strategy();
197 if engines
198 .iter()
199 .any(|engine: &ScanExplainEngine| engine.strategy == strategy)
200 {
201 continue;
202 }
203 engines.push(ScanExplainEngine {
204 strategy,
205 reason: intent_reason(intent.kind),
206 });
207 }
208 Ok(engines)
209}
210
211fn rejected_engines_from_unsupported_features(
212 scan_database: &ScanDatabaseHeader,
213) -> Vec<ScanExplainRejectedEngine> {
214 let mut rejected = Vec::new();
215 for feature in &scan_database.unsupported_features {
216 for engine_id in rejected_candidate_engines(&feature.feature) {
217 rejected.push(ScanExplainRejectedEngine {
218 engine_id: engine_id.to_string(),
219 pattern_index: feature.pattern_index,
220 reason: sanitized_rejection_reason(&feature.feature),
221 exactness_class: ScanExplainExactnessClass::VerifierRequired,
222 });
223 }
224 }
225 rejected
226}
227
228fn rejected_candidate_engines(feature: &str) -> &'static [&'static str] {
229 let lowered = feature.to_ascii_lowercase();
230 if lowered.contains("look") || lowered.contains("capture") || lowered.contains("verifier") {
231 &["cuda", "wgpu", "metal", "dpu", "fpga"]
232 } else {
233 &["cuda", "wgpu", "metal"]
234 }
235}
236
237fn sanitized_rejection_reason(feature: &str) -> String {
238 let mut sanitized = feature.replace("/credentials/", "/credentials/<redacted>/");
239 sanitized = sanitized.replace("C:\\credentials\\", "C:\\credentials\\<redacted>\\");
240 for marker in ["token=", "secret=", "password=", "api_key="] {
241 if let Some(index) = sanitized.to_ascii_lowercase().find(marker) {
242 let keep = index + marker.len();
243 sanitized.truncate(keep);
244 sanitized.push_str("<redacted>");
245 break;
246 }
247 }
248 if sanitized.trim().is_empty() {
249 "candidate route rejected by scan database feature policy".to_string()
250 } else {
251 sanitized
252 }
253}
254
255fn route_evidence_from_sources(
256 extracted_factors: &[ScanExplainFactor],
257 selected_engines: &[ScanExplainEngine],
258 rejected_engines: &[ScanExplainRejectedEngine],
259 verifier_fragments: &[ScanExplainVerifierFragment],
260 scan_database: &ScanDatabaseHeader,
261) -> ScanExplainRouteEvidence {
262 let verifier_cost_estimate_bytes = verifier_fragments
263 .iter()
264 .map(|fragment| fragment.bytes)
265 .sum::<u64>()
266 + scan_database.unsupported_features.len() as u64 * 64;
267 let has_prefilter = selected_engines
268 .iter()
269 .any(|engine| engine.strategy == DescriptorIntentStrategy::Prefilter);
270 let exactness_class = if rejected_engines
271 .iter()
272 .any(|engine| engine.exactness_class == ScanExplainExactnessClass::Unsupported)
273 {
274 ScanExplainExactnessClass::Unsupported
275 } else if !verifier_fragments.is_empty() && !rejected_engines.is_empty() {
276 ScanExplainExactnessClass::VerifierRequired
277 } else if has_prefilter && !verifier_fragments.is_empty() {
278 ScanExplainExactnessClass::PrefilterVerified
279 } else {
280 ScanExplainExactnessClass::Exact
281 };
282
283 ScanExplainRouteEvidence {
284 exactness_class,
285 verifier_cost_estimate_bytes,
286 literal_selectivity_basis: literal_selectivity_basis(extracted_factors),
287 }
288}
289
290fn literal_selectivity_basis(extracted_factors: &[ScanExplainFactor]) -> String {
291 let literal_factors = extracted_factors
292 .iter()
293 .filter(|factor| {
294 matches!(
295 factor.role,
296 ScanFactorRole::Literal
297 | ScanFactorRole::Prefix
298 | ScanFactorRole::Suffix
299 | ScanFactorRole::Infix
300 )
301 })
302 .collect::<Vec<_>>();
303 let total_bytes = literal_factors
304 .iter()
305 .map(|factor| factor.bytes.len())
306 .sum::<usize>();
307 let shortest = literal_factors
308 .iter()
309 .map(|factor| factor.bytes.len())
310 .min()
311 .unwrap_or(0);
312 format!(
313 "literal_factors={};total_literal_bytes={total_bytes};shortest_literal_bytes={shortest}",
314 literal_factors.len()
315 )
316}
317
318fn verifier_fragments_from_sources(
319 intents: &DescriptorIntentSet,
320 scan_database: &ScanDatabaseHeader,
321) -> Result<Vec<ScanExplainVerifierFragment>, ScanExplainError> {
322 if scan_database.table_sections.is_empty() {
323 return Err(ScanExplainError::MissingScanDatabase);
324 }
325 let mut fragments = Vec::new();
326 for intent in &intents.intents {
327 if intent.kind == DescriptorIntentKind::Verifier {
328 fragments.push(ScanExplainVerifierFragment {
329 source: "descriptor-intent".to_string(),
330 bytes: 0,
331 digest: intent.section_digest,
332 });
333 }
334 }
335 for section in &scan_database.table_sections {
336 if section.kind == ScanDatabaseSectionKind::VerifierFragments {
337 fragments.push(ScanExplainVerifierFragment {
338 source: "scan-database-section".to_string(),
339 bytes: section.byte_len,
340 digest: section.section_digest,
341 });
342 }
343 }
344 Ok(fragments)
345}
346
347fn streaming_state_bytes_from_sources(
348 intents: &DescriptorIntentSet,
349 scan_database: &ScanDatabaseHeader,
350) -> u64 {
351 let intent_bytes = intents
352 .intents
353 .iter()
354 .filter(|intent| intent.kind == DescriptorIntentKind::StreamingState)
355 .map(|intent| u64::from(intent.stream_state_bytes))
356 .sum::<u64>();
357 let section_bytes = scan_database
358 .table_sections
359 .iter()
360 .filter(|section| section.kind == ScanDatabaseSectionKind::StreamingState)
361 .map(|section| section.byte_len)
362 .sum::<u64>();
363 intent_bytes.max(section_bytes)
364}
365
366fn intent_reason(kind: DescriptorIntentKind) -> String {
367 match kind {
368 DescriptorIntentKind::LiteralPrefilter => {
369 "literal factors route to a prefilter before automata or verifier work".to_string()
370 }
371 DescriptorIntentKind::AutomataTransition => {
372 "automata transition tables route to the scan transition engine".to_string()
373 }
374 DescriptorIntentKind::Verifier => {
375 "verifier fragments confirm candidates after prefilter or automata matches".to_string()
376 }
377 DescriptorIntentKind::OutputCompaction => {
378 "output compaction owns sparse match materialization".to_string()
379 }
380 DescriptorIntentKind::RelationSeed => {
381 "relation seeds hand scan output into graph or flow facts".to_string()
382 }
383 DescriptorIntentKind::StreamingState => {
384 "streaming state carries cross-chunk scan state".to_string()
385 }
386 }
387}
388
389#[cfg(test)]
390mod tests {
391 use super::*;
392 use vyre_foundation::ir::DataType;
393 use vyre_foundation::serial::wire::encode::{
394 ScanDatabaseCompatibilityRecord, ScanDatabaseMode,
395 ScanDatabaseReaderCompatibility, ScanDatabaseSectionHeader,
396 };
397 use vyre_lower::{
398 BindingLayout, BindingSlot, BindingVisibility, DescriptorIntent,
399 DescriptorIntentKind, DescriptorIntentSet, Dispatch, KernelBody,
400 KernelDescriptor, KernelOp, KernelOpKind, LiteralValue, MemoryClass,
401 };
402
403 fn descriptor(intents: DescriptorIntentSet) -> IntentAnnotatedDescriptor {
404 let descriptor = KernelDescriptor {
405 id: "scan-explain-test".to_string(),
406 bindings: BindingLayout {
407 slots: vec![BindingSlot {
408 slot: 0,
409 element_type: DataType::U32,
410 element_count: None,
411 memory_class: MemoryClass::Global,
412 visibility: BindingVisibility::ReadWrite,
413 name: "scan_table".to_string(),
414 }],
415 },
416 dispatch: Dispatch::new(64, 1, 1),
417 body: KernelBody {
418 ops: vec![KernelOp {
419 kind: KernelOpKind::Literal,
420 operands: vec![0],
421 result: Some(0),
422 }],
423 child_bodies: vec![],
424 literals: vec![LiteralValue::U32(1)],
425 },
426 };
427 IntentAnnotatedDescriptor::try_new(descriptor, intents).unwrap()
428 }
429
430 fn literal_intents() -> DescriptorIntentSet {
431 DescriptorIntentSet::new(vec![
432 DescriptorIntent::new(DescriptorIntentKind::LiteralPrefilter, 10)
433 .with_binding_slot(0)
434 .with_op_result(0),
435 DescriptorIntent::new(DescriptorIntentKind::Verifier, 11)
436 .with_binding_slot(0)
437 .with_op_result(0),
438 DescriptorIntent::new(DescriptorIntentKind::OutputCompaction, 12)
439 .with_binding_slot(0)
440 .with_op_result(0),
441 ])
442 }
443
444 fn regex_intents() -> DescriptorIntentSet {
445 DescriptorIntentSet::new(vec![
446 DescriptorIntent::new(DescriptorIntentKind::LiteralPrefilter, 20)
447 .with_binding_slot(0)
448 .with_op_result(0),
449 DescriptorIntent::new(DescriptorIntentKind::AutomataTransition, 21)
450 .with_binding_slot(0)
451 .with_op_result(0),
452 DescriptorIntent::new(DescriptorIntentKind::Verifier, 22)
453 .with_binding_slot(0)
454 .with_op_result(0),
455 DescriptorIntent::new(DescriptorIntentKind::StreamingState, 23)
456 .with_binding_slot(0)
457 .with_stream_state_bytes(128),
458 ])
459 }
460
461 fn database() -> ScanDatabaseHeader {
462 ScanDatabaseHeader {
463 pattern_set_digest: [4u8; 32],
464 compiler_version: "vyre-debug-scan-explain-test".to_string(),
465 mode: ScanDatabaseMode::Streaming,
466 table_sections: vec![
467 ScanDatabaseSectionHeader {
468 kind: ScanDatabaseSectionKind::LiteralTable,
469 offset: 0,
470 byte_len: 64,
471 section_digest: 100,
472 },
473 ScanDatabaseSectionHeader {
474 kind: ScanDatabaseSectionKind::VerifierFragments,
475 offset: 64,
476 byte_len: 32,
477 section_digest: 101,
478 },
479 ScanDatabaseSectionHeader {
480 kind: ScanDatabaseSectionKind::StreamingState,
481 offset: 96,
482 byte_len: 96,
483 section_digest: 102,
484 },
485 ],
486 unsupported_features: vec![UnsupportedScanFeature {
487 pattern_index: 9,
488 feature: "Fix: look-around stays verifier-only".to_string(),
489 }],
490 compatibility: ScanDatabaseCompatibilityRecord {
491 construct_tier_digest: 0x51ca_51ca,
492 dialect_digest: 0xd1a1_ec7,
493 reader_compatibility: ScanDatabaseReaderCompatibility::RequiresVerifier,
494 },
495 }
496 }
497
498 #[test]
499 fn scan_explain_report_covers_literal_case() {
500 let report = scan_explain_report(
501 "literal-pattern-set",
502 vec![ScanExplainFactor {
503 role: ScanFactorRole::Literal,
504 pattern_index: 0,
505 bytes: b"needle".to_vec(),
506 digest: 44,
507 }],
508 &descriptor(literal_intents()),
509 &database(),
510 "hyperscan-nsdi-literal",
511 vec!["release/evidence/benchmarks/frontier-leaderboard.json".to_string()],
512 )
513 .unwrap();
514
515 assert!(report.is_complete());
516 assert!(report
517 .selected_engines
518 .iter()
519 .any(|engine| engine.strategy == DescriptorIntentStrategy::Prefilter));
520 assert_eq!(report.verifier_fragments.len(), 2);
521 assert_eq!(report.table_bytes, 192);
522 assert_eq!(
523 report.route_evidence.literal_selectivity_basis,
524 "literal_factors=1;total_literal_bytes=6;shortest_literal_bytes=6"
525 );
526 assert!(report
527 .rejected_engines
528 .iter()
529 .any(|engine| engine.engine_id == "cuda"));
530 }
531
532 #[test]
533 fn scan_explain_report_covers_regex_streaming_case() {
534 let report = scan_explain_report(
535 "regex-pattern-set",
536 vec![
537 ScanExplainFactor {
538 role: ScanFactorRole::Prefix,
539 pattern_index: 1,
540 bytes: b"abc".to_vec(),
541 digest: 45,
542 },
543 ScanExplainFactor {
544 role: ScanFactorRole::Suffix,
545 pattern_index: 1,
546 bytes: b"xyz".to_vec(),
547 digest: 46,
548 },
549 ],
550 &descriptor(regex_intents()),
551 &database(),
552 "hyperscan-nsdi-regex",
553 vec!["release/evidence/benchmarks/frontier-leaderboard.json".to_string()],
554 )
555 .unwrap();
556
557 assert!(report
558 .selected_engines
559 .iter()
560 .any(|engine| engine.strategy == DescriptorIntentStrategy::Automata));
561 assert_eq!(report.streaming_state_bytes, 128);
562 assert_eq!(report.unsupported_features.len(), 1);
563 assert_eq!(
564 report.route_evidence.exactness_class,
565 ScanExplainExactnessClass::VerifierRequired
566 );
567 assert!(report.route_evidence.verifier_cost_estimate_bytes >= 96);
568 assert!(report.rejected_engines.iter().any(|engine| {
569 engine.engine_id == "dpu"
570 && engine.reason.contains("verifier-only")
571 && !engine.reason.contains("credentials")
572 }));
573 }
574
575 #[test]
576 fn scan_explain_report_rejects_missing_artifact_links() {
577 let error = scan_explain_report(
578 "regex-pattern-set",
579 vec![ScanExplainFactor {
580 role: ScanFactorRole::Infix,
581 pattern_index: 2,
582 bytes: b"mid".to_vec(),
583 digest: 47,
584 }],
585 &descriptor(regex_intents()),
586 &database(),
587 "hyperscan-nsdi-regex",
588 Vec::new(),
589 )
590 .unwrap_err();
591
592 assert_eq!(error, ScanExplainError::MissingArtifactLinks);
593 }
594}