use crate::snippets::cache::ValidationCache;
use crate::snippets::error::Result;
use crate::snippets::session::{SessionSpec, prepare_sessions_isolated};
use crate::snippets::types::{
DowngradeReason, RunSummary, SideEffectClass, Snippet, SnippetAnnotationKind, SnippetStatus, ValidationLevel,
ValidationResult,
};
use crate::snippets::validators::ValidatorRegistry;
use rayon::prelude::*;
use std::collections::{BTreeMap, HashMap};
use std::sync::Mutex;
use std::time::Instant;
mod batch;
use batch::validate_batches;
pub struct RunnerConfig {
pub level: ValidationLevel,
pub parallelism: usize,
pub timeout_secs: u64,
pub fail_fast: bool,
pub deny_unclassified: bool,
pub allowed_side_effects: Vec<SideEffectClass>,
pub cache_dir: Option<std::path::PathBuf>,
pub changed_only: bool,
pub sessions: HashMap<String, SessionSpec>,
}
impl Default for RunnerConfig {
fn default() -> Self {
Self {
level: ValidationLevel::Syntax,
parallelism: available_parallelism(),
timeout_secs: 120,
fail_fast: false,
deny_unclassified: false,
allowed_side_effects: Vec::new(),
cache_dir: Some(std::path::PathBuf::from(".alef/snippets")),
changed_only: false,
sessions: HashMap::new(),
}
}
}
fn available_parallelism() -> usize {
std::thread::available_parallelism().map_or(4, std::num::NonZeroUsize::get)
}
pub fn run_validation(snippets: &[Snippet], registry: &ValidatorRegistry, config: &RunnerConfig) -> Result<RunSummary> {
let preparation = prepare_sessions_isolated(&config.sessions, config.timeout_secs);
let sessions = preparation.sessions;
let session_errors = preparation.errors;
let session_locks = sessions
.keys()
.map(|target| (target.clone(), Mutex::new(())))
.collect::<HashMap<_, _>>();
let pool = rayon::ThreadPoolBuilder::new()
.num_threads(config.parallelism)
.build()
.map_err(|err| crate::snippets::error::Error::Other(format!("failed to build thread pool: {err}")))?;
let fail_fast = config.fail_fast;
let calling_span = tracing::Span::current();
let results: Vec<ValidationResult> = pool.install(|| {
let _entered = calling_span.enter();
if fail_fast {
fail_fast_results(snippets, registry, config, &sessions, &session_errors, &session_locks)
} else {
parallel_results(snippets, registry, config, &sessions, &session_errors, &session_locks)
}
});
Ok(RunSummary::from_results(results))
}
fn fail_fast_results(
snippets: &[Snippet],
registry: &ValidatorRegistry,
config: &RunnerConfig,
sessions: &HashMap<String, crate::snippets::session::ValidationSession>,
session_errors: &HashMap<String, String>,
session_locks: &HashMap<String, Mutex<()>>,
) -> Vec<ValidationResult> {
tracing::info!(
snippet_count = snippets.len(),
timeout_secs = config.timeout_secs,
"Starting fail-fast snippet validation"
);
let started = Instant::now();
let reporter = FailureReporter::new(snippets);
let mut results = Vec::with_capacity(snippets.len());
for snippet in snippets {
let preparation_error = session_preparation_error(snippet, sessions, session_errors);
let session = session_for(snippet, sessions);
let lock = session_key(snippet, sessions).and_then(|key| session_locks.get(key));
let result = validate_one(snippet, registry, config, session, lock, preparation_error);
reporter.record(&result);
let should_stop =
preparation_error.is_none() && matches!(result.status, SnippetStatus::Fail | SnippetStatus::Error);
results.push(result);
if should_stop {
break;
}
}
let duration_ms = u64::try_from(started.elapsed().as_millis()).unwrap_or(u64::MAX);
tracing::info!(
snippet_count = results.len(),
duration_ms,
"Finished fail-fast snippet validation"
);
results
}
fn parallel_results(
snippets: &[Snippet],
registry: &ValidatorRegistry,
config: &RunnerConfig,
sessions: &HashMap<String, crate::snippets::session::ValidationSession>,
session_errors: &HashMap<String, String>,
session_locks: &HashMap<String, Mutex<()>>,
) -> Vec<ValidationResult> {
let reporter = FailureReporter::new(snippets);
let batched = validate_batches(
snippets,
registry,
config,
sessions,
session_errors,
session_locks,
&reporter,
);
let fallback_counts = fallback_counts_by_language(snippets, &batched);
for (language, count) in &fallback_counts {
tracing::info!(
language = %language,
snippet_count = count,
timeout_secs = config.timeout_secs,
"Starting per-snippet validation"
);
}
let started = Instant::now();
let results = snippets
.par_iter()
.enumerate()
.map(|(index, snippet)| {
if let Some(result) = batched[index].clone() {
return result;
}
let session = session_for(snippet, sessions);
let lock = session_key(snippet, sessions).and_then(|key| session_locks.get(key));
let result = validate_one(snippet, registry, config, session, lock, None);
reporter.record(&result);
result
})
.collect();
let duration_ms = u64::try_from(started.elapsed().as_millis()).unwrap_or(u64::MAX);
for (language, count) in &fallback_counts {
tracing::info!(
language = %language,
snippet_count = count,
duration_ms,
"Finished per-snippet validation"
);
}
results
}
fn fallback_counts_by_language(
snippets: &[Snippet],
batched: &[Option<ValidationResult>],
) -> BTreeMap<crate::snippets::types::Language, usize> {
let mut counts = BTreeMap::new();
for (snippet, entry) in snippets.iter().zip(batched) {
if entry.is_none() {
*counts.entry(snippet.language).or_insert(0_usize) += 1;
}
}
counts
}
const FAILURE_PROGRESS_STRIDE: usize = 25;
const FAILURE_MESSAGE_PREVIEW_CHARS: usize = 400;
#[derive(Clone, Copy, Default)]
struct LanguageTally {
completed: usize,
failed: usize,
unavailable: usize,
}
struct FailureReporter {
totals: BTreeMap<crate::snippets::types::Language, usize>,
tallies: Mutex<BTreeMap<crate::snippets::types::Language, LanguageTally>>,
span: tracing::Span,
}
impl FailureReporter {
fn new(snippets: &[Snippet]) -> Self {
let mut totals = BTreeMap::new();
for snippet in snippets {
*totals.entry(snippet.language).or_insert(0_usize) += 1;
}
Self {
totals,
tallies: Mutex::new(BTreeMap::new()),
span: tracing::Span::current(),
}
}
fn record(&self, result: &ValidationResult) {
let language = result.snippet.language;
let failed = matches!(result.status, SnippetStatus::Fail | SnippetStatus::Error);
let unavailable = matches!(result.status, SnippetStatus::Unavailable);
let Ok(mut tallies) = self.tallies.lock() else {
return;
};
let tally = tallies.entry(language).or_default();
tally.completed += 1;
if failed {
tally.failed += 1;
}
if unavailable {
tally.unavailable += 1;
}
let tally = *tally;
drop(tallies);
let snippet_count = self.totals.get(&language).copied().unwrap_or(tally.completed);
self.span.in_scope(|| {
if failed && tally.failed == 1 {
tracing::warn!(
language = %language,
path = %result.snippet.source_origin.path.display(),
line = result.snippet.source_origin.line,
snippet_count = snippet_count,
error = %failure_preview(result.message.as_deref()),
"First snippet validation failure for this language"
);
} else if unavailable && tally.unavailable == 1 {
tracing::warn!(
language = %language,
path = %result.snippet.source_origin.path.display(),
line = result.snippet.source_origin.line,
snippet_count = snippet_count,
unresolved_dependency = result.unresolved_dependency,
error = %failure_preview(result.message.as_deref()),
"First snippet validation unavailability for this language"
);
} else if failed && tally.failed % FAILURE_PROGRESS_STRIDE == 0 {
tracing::warn!(
language = %language,
failed = tally.failed,
completed = tally.completed,
snippet_count = snippet_count,
"Snippet validation failures accumulating"
);
}
if tally.completed < snippet_count {
return;
}
if tally.failed > 0 {
tracing::warn!(
language = %language,
failed = tally.failed,
unavailable = tally.unavailable,
snippet_count = snippet_count,
"Finished snippet validation for this language with failures"
);
} else if tally.unavailable > 0 {
tracing::warn!(
language = %language,
unavailable = tally.unavailable,
snippet_count = snippet_count,
"Finished snippet validation for this language with every result unvalidated"
);
} else {
tracing::debug!(
language = %language,
snippet_count = snippet_count,
"Finished snippet validation for this language"
);
}
});
}
}
fn failure_preview(message: Option<&str>) -> String {
let joined = message
.unwrap_or_default()
.lines()
.map(str::trim)
.filter(|line| !line.is_empty())
.collect::<Vec<_>>()
.join(" | ");
if joined.is_empty() {
return "<no validator output>".to_string();
}
match joined.char_indices().nth(FAILURE_MESSAGE_PREVIEW_CHARS) {
Some((index, _)) => format!("{}...", &joined[..index]),
None => joined,
}
}
type BatchKey = (crate::snippets::types::Language, Option<String>, ValidationLevel);
struct ValidationOutcome {
status: SnippetStatus,
message: Option<String>,
duration_ms: u64,
}
fn batch_level(
snippet: &Snippet,
registry: &ValidatorRegistry,
config: &RunnerConfig,
session: Option<&crate::snippets::session::ValidationSession>,
) -> Option<ValidationLevel> {
if cached_result(snippet, config, session).is_some() || side_effect_rejection(snippet, config).is_some() {
return None;
}
if let Some(annotation) = &snippet.annotation
&& annotation.kind == SnippetAnnotationKind::Skip
{
return None;
}
let validator = registry.get(snippet.language)?;
if !validator.supports_batching() {
return None;
}
let level = capped_level(snippet, config, validator);
validator.is_available_at(level).then_some(level)
}
fn annotation_level_limit(snippet: &Snippet) -> Option<ValidationLevel> {
snippet
.annotation
.as_ref()
.and_then(|annotation| match annotation.kind {
SnippetAnnotationKind::SyntaxOnly => Some(ValidationLevel::Syntax),
SnippetAnnotationKind::CompileOnly => Some(ValidationLevel::Compile),
SnippetAnnotationKind::TypeCheckOnly => Some(ValidationLevel::TypeCheck),
SnippetAnnotationKind::Skip => None,
})
}
fn effective_validation_level(snippet: &Snippet, requested: ValidationLevel) -> ValidationLevel {
[annotation_level_limit(snippet), snippet.metadata.level]
.into_iter()
.flatten()
.fold(requested, ValidationLevel::min)
}
fn capped_level(
snippet: &Snippet,
config: &RunnerConfig,
validator: &dyn crate::snippets::validators::SnippetValidator,
) -> ValidationLevel {
effective_validation_level(snippet, config.level)
.min(validator.max_level())
.min(validator.achievable_level(config.level))
}
fn structurally_unreachable(
validator: &dyn crate::snippets::validators::SnippetValidator,
requested: ValidationLevel,
) -> bool {
validator.max_level() < requested
|| (validator.achievable_level(requested) < requested && validator.achievable_level_is_structural(requested))
}
fn session_for<'a>(
snippet: &Snippet,
sessions: &'a HashMap<String, crate::snippets::session::ValidationSession>,
) -> Option<&'a crate::snippets::session::ValidationSession> {
snippet
.metadata
.target
.as_ref()
.and_then(|target| sessions.get(&crate::snippets::types::Language::normalize_session_target(target)))
.or_else(|| sessions.get(&snippet.language.to_string()))
}
fn session_key<'a>(
snippet: &Snippet,
sessions: &'a HashMap<String, crate::snippets::session::ValidationSession>,
) -> Option<&'a str> {
let target = snippet
.metadata
.target
.as_ref()
.map(|target| crate::snippets::types::Language::normalize_session_target(target));
if let Some(target) = target.as_deref()
&& sessions.contains_key(target)
{
return sessions.get_key_value(target).map(|(key, _)| key.as_str());
}
sessions
.get_key_value(&snippet.language.to_string())
.map(|(key, _)| key.as_str())
}
fn session_preparation_error<'a>(
snippet: &Snippet,
sessions: &HashMap<String, crate::snippets::session::ValidationSession>,
errors: &'a HashMap<String, String>,
) -> Option<&'a str> {
let target = snippet
.metadata
.target
.as_ref()
.map(|target| crate::snippets::types::Language::normalize_session_target(target));
if let Some(target) = target.as_deref() {
if let Some(error) = errors.get(target) {
return Some(error);
}
if sessions.contains_key(target) {
return None;
}
}
errors.get(&snippet.language.to_string()).map(String::as_str)
}
fn validate_one(
snippet: &Snippet,
registry: &ValidatorRegistry,
config: &RunnerConfig,
session: Option<&crate::snippets::session::ValidationSession>,
session_lock: Option<&Mutex<()>>,
session_preparation_error: Option<&str>,
) -> ValidationResult {
if let Some(message) = session_preparation_error {
return result(
snippet,
SnippetStatus::Error,
config.level,
config.level,
Some(message.to_owned()),
0,
);
}
if let Some(result) = cached_result(snippet, config, session) {
return result;
}
if let Some(message) = side_effect_rejection(snippet, config) {
return result(
snippet,
SnippetStatus::Skip,
config.level,
config.level,
Some(message),
0,
);
}
if let Some(annotation) = &snippet.annotation
&& annotation.kind == SnippetAnnotationKind::Skip
{
return result(
snippet,
SnippetStatus::Skip,
config.level,
config.level,
Some(skip_message("skipped via annotation", annotation.reason.as_deref())),
0,
);
}
let Some(validator) = registry.get(snippet.language) else {
return result(
snippet,
SnippetStatus::Unavailable,
config.level,
config.level,
Some(format!("no validator for {}", snippet.language)),
0,
);
};
let effective_level = capped_level(snippet, config, validator);
if !validator.is_available_at(effective_level) {
return result(
snippet,
SnippetStatus::Unavailable,
config.level,
config.level,
Some(format!("{} toolchain not found", snippet.language)),
0,
);
}
let start = Instant::now();
let validation = || validator.validate_in_session(snippet, effective_level, config.timeout_secs, session);
let validation_result = match session_lock {
Some(lock) => match lock.lock() {
Ok(_guard) => validation(),
Err(error) => Err(crate::snippets::error::Error::Other(format!(
"locking {} snippet validation session: {error}",
snippet.language
))),
},
None => validation(),
};
let (status, message) = match validation_result {
Ok((status, message)) => (status, message),
Err(err) => (SnippetStatus::Error, Some(err.to_string())),
};
let duration_ms = u64::try_from(start.elapsed().as_millis()).unwrap_or(u64::MAX);
finalize_result(
snippet,
validator,
config,
session,
effective_level,
ValidationOutcome {
status,
message,
duration_ms,
},
)
}
struct ResultClassification {
status: SnippetStatus,
capability_capped: bool,
downgrade_reason: Option<DowngradeReason>,
}
fn classify_result(
snippet: &Snippet,
validator: &dyn crate::snippets::validators::SnippetValidator,
config: &RunnerConfig,
effective_level: ValidationLevel,
status: SnippetStatus,
) -> ResultClassification {
if status != SnippetStatus::Pass || effective_level >= config.level {
return ResultClassification {
status,
capability_capped: false,
downgrade_reason: None,
};
}
let annotated_level = effective_validation_level(snippet, config.level);
let structural = structurally_unreachable(validator, config.level);
if annotated_level >= config.level && structural {
return ResultClassification {
status,
capability_capped: true,
downgrade_reason: Some(DowngradeReason::ValidatorCapability),
};
}
let declared_binds = snippet
.metadata
.level
.is_some_and(|level| config.level.min(level) == annotated_level);
if effective_level == annotated_level && annotated_level < config.level && declared_binds {
return ResultClassification {
status,
capability_capped: false,
downgrade_reason: Some(DowngradeReason::Declared),
};
}
let reason = if effective_level < annotated_level && structural {
DowngradeReason::ValidatorCapability
} else if effective_level < annotated_level {
DowngradeReason::Environment
} else {
DowngradeReason::Annotation
};
ResultClassification {
status: SnippetStatus::Downgraded,
capability_capped: false,
downgrade_reason: Some(reason),
}
}
fn finalize_result(
snippet: &Snippet,
validator: &dyn crate::snippets::validators::SnippetValidator,
config: &RunnerConfig,
session: Option<&crate::snippets::session::ValidationSession>,
effective_level: ValidationLevel,
outcome: ValidationOutcome,
) -> ValidationResult {
let ValidationOutcome {
mut status,
message,
duration_ms,
} = outcome;
let mut unresolved_dependency = false;
if status == SnippetStatus::Fail
&& let Some(error_output) = &message
&& validator.is_dependency_error(error_output)
{
if effective_level == ValidationLevel::Syntax {
status = SnippetStatus::Pass;
} else {
status = SnippetStatus::Unavailable;
unresolved_dependency = true;
}
}
let classification = classify_result(snippet, validator, config, effective_level, status);
let status = classification.status;
let message = if classification.downgrade_reason == Some(DowngradeReason::Declared) {
Some(format!(
"requested {}, validated at declared level {effective_level}",
config.level
))
} else if status == SnippetStatus::Downgraded {
Some(format!("requested {}, validated at {}", config.level, effective_level))
} else if classification.capability_capped {
Some(format!(
"requested {}, validated at {} ({} validator caps at {})",
config.level, effective_level, snippet.language, effective_level
))
} else if unresolved_dependency {
Some(format!(
"could not validate at {effective_level}: {} toolchain ran but reported a missing dependency or build \
artifact -- run `alef build` first if this crate validates snippets against built artifacts: {}",
snippet.language,
message.as_deref().unwrap_or("<no validator output>")
))
} else {
message
};
debug_assert!(
classification.downgrade_reason.is_some()
|| !(classification.status == SnippetStatus::Downgraded || classification.capability_capped),
"a Downgraded or capability_capped result must always carry a downgrade_reason"
);
let mut result = result(snippet, status, config.level, effective_level, message, duration_ms);
result.capability_capped = classification.capability_capped;
result.downgrade_reason = classification.downgrade_reason;
result.unresolved_dependency = unresolved_dependency;
if let Some(cache) = config.cache_dir.clone().map(ValidationCache::new)
&& let Err(error) = cache.store(
snippet,
config.level,
session.map(|value| value.fingerprint.as_str()),
&result,
)
{
tracing::warn!("writing snippet validation cache: {error}");
}
result
}
fn cached_result(
snippet: &Snippet,
config: &RunnerConfig,
session: Option<&crate::snippets::session::ValidationSession>,
) -> Option<ValidationResult> {
if !config.changed_only {
return None;
}
let cache = config.cache_dir.clone().map(ValidationCache::new)?;
let mut result = cache.load(snippet, config.level, session.map(|value| value.fingerprint.as_str()))?;
result.snippet = snippet.clone();
result.duration_ms = 0;
result.message = result.message.or_else(|| Some("cached".to_string()));
Some(result)
}
fn side_effect_rejection(snippet: &Snippet, config: &RunnerConfig) -> Option<String> {
if config.level != ValidationLevel::Run {
return None;
}
let Some(class) = snippet.metadata.side_effect else {
return config
.deny_unclassified
.then(|| "unclassified side effects are denied".to_string());
};
if class == SideEffectClass::Safe || config.allowed_side_effects.contains(&class) {
None
} else {
Some(format!("side effect class {class:?} is not allowed").to_lowercase())
}
}
fn result(
snippet: &Snippet,
status: SnippetStatus,
requested_level: ValidationLevel,
effective_level: ValidationLevel,
message: Option<String>,
duration_ms: u64,
) -> ValidationResult {
ValidationResult {
snippet: snippet.clone(),
status,
level: effective_level,
requested_level,
effective_level,
message,
duration_ms,
capability_capped: false,
downgrade_reason: None,
unresolved_dependency: false,
}
}
fn skip_message(message: &str, reason: Option<&str>) -> String {
match reason {
Some(reason) if !reason.is_empty() => format!("{message}: {reason}"),
_ => message.to_string(),
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::snippets::types::{SnippetMetadata, SourceOrigin};
use crate::snippets::validators::SnippetValidator;
use std::sync::Arc;
use tracing_test::traced_test;
struct RecordingValidator {
language: crate::snippets::types::Language,
batches: Arc<Mutex<Vec<(crate::snippets::types::Language, usize, bool)>>>,
singles: Arc<Mutex<usize>>,
}
#[cfg(unix)]
struct ExhaustingValidator {
timeouts: Arc<Mutex<Vec<u64>>>,
}
#[cfg(unix)]
impl SnippetValidator for ExhaustingValidator {
fn language(&self) -> crate::snippets::types::Language {
crate::snippets::types::Language::Bash
}
fn is_available(&self) -> bool {
true
}
fn validate(
&self,
_snippet: &Snippet,
_level: ValidationLevel,
timeout_secs: u64,
) -> Result<(SnippetStatus, Option<String>)> {
let call = {
let mut timeouts = self.timeouts.lock().expect("timeouts");
timeouts.push(timeout_secs);
timeouts.len()
};
if call == 1 {
let mut command = std::process::Command::new("sh");
command.args(["-c", "sleep 30 & wait"]);
crate::snippets::validators::run_command(&mut command, timeout_secs)?;
}
Ok((SnippetStatus::Pass, None))
}
fn max_level(&self) -> ValidationLevel {
ValidationLevel::Run
}
}
struct BudgetRecordingValidator {
singles: Arc<Mutex<Vec<u64>>>,
batched: Arc<Mutex<Vec<(usize, u64)>>>,
supports_batching: bool,
}
impl SnippetValidator for BudgetRecordingValidator {
fn language(&self) -> crate::snippets::types::Language {
crate::snippets::types::Language::Rust
}
fn is_available(&self) -> bool {
true
}
fn validate(
&self,
_snippet: &Snippet,
_level: ValidationLevel,
timeout_secs: u64,
) -> Result<(SnippetStatus, Option<String>)> {
self.singles.lock().expect("single timeouts").push(timeout_secs);
Ok((SnippetStatus::Pass, None))
}
fn validate_batch_in_session(
&self,
snippets: &[&Snippet],
_level: ValidationLevel,
timeout_secs: u64,
_session: Option<&crate::snippets::session::ValidationSession>,
) -> Option<Result<Vec<(SnippetStatus, Option<String>)>>> {
if !self.supports_batching {
return None;
}
self.batched
.lock()
.expect("batch timeouts")
.push((snippets.len(), timeout_secs));
Some(Ok(vec![(SnippetStatus::Pass, None); snippets.len()]))
}
fn max_level(&self) -> ValidationLevel {
ValidationLevel::Run
}
fn supports_batching(&self) -> bool {
self.supports_batching
}
}
impl SnippetValidator for RecordingValidator {
fn language(&self) -> crate::snippets::types::Language {
self.language
}
fn is_available(&self) -> bool {
true
}
fn validate(
&self,
_snippet: &Snippet,
_level: ValidationLevel,
_timeout_secs: u64,
) -> Result<(SnippetStatus, Option<String>)> {
*self.singles.lock().expect("single count") += 1;
Ok((SnippetStatus::Pass, None))
}
fn validate_batch_in_session(
&self,
snippets: &[&Snippet],
_level: ValidationLevel,
_timeout_secs: u64,
session: Option<&crate::snippets::session::ValidationSession>,
) -> Option<Result<Vec<(SnippetStatus, Option<String>)>>> {
self.batches
.lock()
.expect("batch records")
.push((self.language, snippets.len(), session.is_some()));
Some(Ok(vec![(SnippetStatus::Pass, None); snippets.len()]))
}
fn max_level(&self) -> ValidationLevel {
ValidationLevel::Run
}
fn supports_batching(&self) -> bool {
true
}
}
struct CappedValidator {
language: crate::snippets::types::Language,
ceiling: ValidationLevel,
}
impl SnippetValidator for CappedValidator {
fn language(&self) -> crate::snippets::types::Language {
self.language
}
fn is_available(&self) -> bool {
true
}
fn validate(
&self,
_snippet: &Snippet,
_level: ValidationLevel,
_timeout_secs: u64,
) -> Result<(SnippetStatus, Option<String>)> {
Ok((SnippetStatus::Pass, None))
}
fn max_level(&self) -> ValidationLevel {
self.ceiling
}
}
fn network_snippet() -> Snippet {
Snippet {
id: None,
path: "example.md".into(),
language: crate::snippets::types::Language::Rust,
title: None,
code: "fn main() {}".into(),
start_line: 1,
block_index: 0,
annotation: None,
metadata: SnippetMetadata {
side_effect: Some(SideEffectClass::Network),
..SnippetMetadata::default()
},
source_origin: SourceOrigin {
path: "example.md".into(),
line: 1,
block_index: 0,
},
}
}
#[test]
fn side_effect_policy_only_blocks_execution() {
let snippet = network_snippet();
let compile = RunnerConfig {
level: ValidationLevel::Compile,
..RunnerConfig::default()
};
let run = RunnerConfig {
level: ValidationLevel::Run,
..RunnerConfig::default()
};
assert_eq!(side_effect_rejection(&snippet, &compile), None);
assert_eq!(
side_effect_rejection(&snippet, &run).as_deref(),
Some("side effect class network is not allowed")
);
}
#[test]
fn annotations_cap_validation_instead_of_skipping_it() {
let mut snippet = network_snippet();
snippet.annotation = Some(crate::snippets::types::SnippetAnnotation {
kind: SnippetAnnotationKind::SyntaxOnly,
reason: None,
});
assert_eq!(
effective_validation_level(&snippet, ValidationLevel::TypeCheck),
ValidationLevel::Syntax
);
snippet.annotation = None;
assert_eq!(
effective_validation_level(&snippet, ValidationLevel::TypeCheck),
ValidationLevel::TypeCheck
);
}
#[test]
fn target_session_precedes_canonical_language_fallback() {
let mut snippet = network_snippet();
snippet.language = crate::snippets::types::Language::TypeScript;
snippet.metadata.target = Some("wasm".into());
let sessions = HashMap::from([
(
"typescript".into(),
crate::snippets::session::ValidationSession {
language: crate::snippets::types::Language::TypeScript,
working_directory: "bindings/node".into(),
manifest: None,
fingerprint: "node".into(),
env: Default::default(),
include_paths: Vec::new(),
rust_features: Vec::new(),
rust_dependencies: Default::default(),
},
),
(
"wasm".into(),
crate::snippets::session::ValidationSession {
language: crate::snippets::types::Language::TypeScript,
working_directory: "bindings/wasm".into(),
manifest: None,
fingerprint: "wasm".into(),
env: Default::default(),
include_paths: Vec::new(),
rust_features: Vec::new(),
rust_dependencies: Default::default(),
},
),
]);
assert_eq!(
session_for(&snippet, &sessions).map(|session| session.fingerprint.as_str()),
Some("wasm")
);
snippet.metadata.target = None;
assert_eq!(
session_for(&snippet, &sessions).map(|session| session.fingerprint.as_str()),
Some("node")
);
}
#[test]
fn groups_batches_by_language_session_and_preserves_order() {
let batches = Arc::new(Mutex::new(Vec::new()));
let singles = Arc::new(Mutex::new(0));
let mut registry = ValidatorRegistry::new();
for language in [
crate::snippets::types::Language::Rust,
crate::snippets::types::Language::Python,
] {
registry.register(Box::new(RecordingValidator {
language,
batches: Arc::clone(&batches),
singles: Arc::clone(&singles),
}));
}
let first_directory = tempfile::tempdir().expect("first session");
let second_directory = tempfile::tempdir().expect("second session");
let session = |directory: &std::path::Path| SessionSpec {
language: crate::snippets::types::Language::Rust,
working_directory: directory.into(),
manifest: None,
before: Vec::new(),
env: Default::default(),
include_paths: Vec::new(),
rust_features: Vec::new(),
rust_dependencies: Default::default(),
};
let config = RunnerConfig {
level: ValidationLevel::Compile,
cache_dir: None,
sessions: HashMap::from([
("alpha".into(), session(first_directory.path())),
("beta".into(), session(second_directory.path())),
]),
..RunnerConfig::default()
};
let mut snippets = vec![network_snippet(), network_snippet(), network_snippet()];
snippets[0].id = Some("first".into());
snippets[0].metadata.target = Some("alpha".into());
snippets[1].id = Some("second".into());
snippets[1].metadata.target = Some("beta".into());
snippets[2].id = Some("third".into());
snippets[2].language = crate::snippets::types::Language::Python;
let summary = run_validation(&snippets, ®istry, &config).expect("validation succeeds");
assert_eq!(
summary
.results
.iter()
.map(|value| value.snippet.id.as_deref())
.collect::<Vec<_>>(),
[Some("first"), Some("second"), Some("third")]
);
assert_eq!(*singles.lock().expect("single count"), 0);
let batches = batches.lock().expect("batch records");
assert_eq!(batches.len(), 3);
assert!(batches.iter().all(|(_, size, _)| *size == 1));
}
#[test]
fn session_preparation_errors_do_not_abort_healthy_targets() {
let batches = Arc::new(Mutex::new(Vec::new()));
let singles = Arc::new(Mutex::new(0));
let mut registry = ValidatorRegistry::new();
registry.register(Box::new(RecordingValidator {
language: crate::snippets::types::Language::Rust,
batches: Arc::clone(&batches),
singles,
}));
let directory = tempfile::tempdir().expect("session directory");
let session = |manifest| SessionSpec {
language: crate::snippets::types::Language::Rust,
working_directory: directory.path().into(),
manifest,
before: Vec::new(),
env: Default::default(),
include_paths: Vec::new(),
rust_features: Vec::new(),
rust_dependencies: Default::default(),
};
let config = RunnerConfig {
level: ValidationLevel::Compile,
cache_dir: None,
sessions: HashMap::from([
("broken".into(), session(Some(directory.path().join("missing.toml")))),
("healthy".into(), session(None)),
]),
..RunnerConfig::default()
};
let mut snippets = vec![network_snippet(), network_snippet()];
snippets[0].metadata.target = Some("broken".into());
snippets[1].metadata.target = Some("healthy".into());
let summary = run_validation(&snippets, ®istry, &config).expect("validation completes");
assert_eq!(summary.total, 2);
assert_eq!(summary.errors, 1);
assert_eq!(summary.passed, 1);
assert!(summary.has_failures());
assert_eq!(summary.results[0].status, SnippetStatus::Error);
assert!(
summary.results[0].message.as_deref().is_some_and(
|message| message.contains("target `broken`") && message.contains("manifest does not exist")
)
);
assert_eq!(summary.results[1].status, SnippetStatus::Pass);
assert_eq!(
batches.lock().expect("batch records").as_slice(),
&[(crate::snippets::types::Language::Rust, 1, true)]
);
}
#[test]
fn cached_cells_are_excluded_from_batches() {
let batches = Arc::new(Mutex::new(Vec::new()));
let singles = Arc::new(Mutex::new(0));
let mut registry = ValidatorRegistry::new();
registry.register(Box::new(RecordingValidator {
language: crate::snippets::types::Language::Rust,
batches: Arc::clone(&batches),
singles,
}));
let cache_directory = tempfile::tempdir().expect("cache directory");
let mut snippets = vec![network_snippet(), network_snippet()];
snippets[1].code = "fn main() { let _value = 2; }".into();
let cached = result(
&snippets[0],
SnippetStatus::Pass,
ValidationLevel::Compile,
ValidationLevel::Compile,
None,
1,
);
ValidationCache::new(cache_directory.path().into())
.store(&snippets[0], ValidationLevel::Compile, None, &cached)
.expect("cache entry");
let config = RunnerConfig {
level: ValidationLevel::Compile,
changed_only: true,
cache_dir: Some(cache_directory.path().into()),
..RunnerConfig::default()
};
let summary = run_validation(&snippets, ®istry, &config).expect("validation succeeds");
assert_eq!(summary.results.len(), 2);
assert_eq!(summary.results[0].duration_ms, 0);
assert_eq!(
batches.lock().expect("batch records").as_slice(),
&[(crate::snippets::types::Language::Rust, 1, false)]
);
}
#[cfg(unix)]
#[test]
fn a_snippet_that_times_out_does_not_consume_the_next_snippets_budget() {
let timeouts = Arc::new(Mutex::new(Vec::new()));
let mut registry = ValidatorRegistry::new();
registry.register(Box::new(ExhaustingValidator {
timeouts: Arc::clone(&timeouts),
}));
let mut snippets = vec![network_snippet(), network_snippet()];
for snippet in &mut snippets {
snippet.language = crate::snippets::types::Language::Bash;
}
let config = RunnerConfig {
level: ValidationLevel::Run,
parallelism: 1,
timeout_secs: 1,
cache_dir: None,
allowed_side_effects: vec![SideEffectClass::Network],
..RunnerConfig::default()
};
let summary = run_validation(&snippets, ®istry, &config).expect("validation completes");
assert_eq!(*timeouts.lock().expect("timeouts"), vec![1, 1]);
assert_eq!(summary.errors, 1);
assert_eq!(summary.passed, 1);
for value in &summary.results {
let message = value.message.as_deref().unwrap_or_default();
assert!(
!message.contains("batch"),
"reported against a batch command: {message}"
);
}
}
#[test]
fn non_batching_validators_receive_the_configured_timeout_per_snippet() {
let singles = Arc::new(Mutex::new(Vec::new()));
let mut registry = ValidatorRegistry::new();
registry.register(Box::new(BudgetRecordingValidator {
singles: Arc::clone(&singles),
batched: Arc::new(Mutex::new(Vec::new())),
supports_batching: false,
}));
let config = RunnerConfig {
level: ValidationLevel::Compile,
parallelism: 1,
timeout_secs: 7,
cache_dir: None,
..RunnerConfig::default()
};
let snippets = vec![network_snippet(), network_snippet(), network_snippet()];
let summary = run_validation(&snippets, ®istry, &config).expect("validation completes");
assert_eq!(*singles.lock().expect("single timeouts"), vec![7, 7, 7]);
assert_eq!(summary.passed, 3);
assert_eq!(summary.errors, 0);
}
#[test]
fn batching_validators_still_receive_the_group_budget_once() {
let singles = Arc::new(Mutex::new(Vec::new()));
let batched = Arc::new(Mutex::new(Vec::new()));
let mut registry = ValidatorRegistry::new();
registry.register(Box::new(BudgetRecordingValidator {
singles: Arc::clone(&singles),
batched: Arc::clone(&batched),
supports_batching: true,
}));
let config = RunnerConfig {
level: ValidationLevel::Compile,
parallelism: 1,
timeout_secs: 7,
cache_dir: None,
..RunnerConfig::default()
};
let snippets = vec![network_snippet(), network_snippet(), network_snippet()];
let summary = run_validation(&snippets, ®istry, &config).expect("validation completes");
assert_eq!(*batched.lock().expect("batch timeouts"), vec![(3, 7)]);
assert!(singles.lock().expect("single timeouts").is_empty());
assert_eq!(summary.passed, 3);
}
#[test]
fn validator_ceiling_passes_instead_of_downgrading() {
let mut registry = ValidatorRegistry::new();
registry.register(Box::new(CappedValidator {
language: crate::snippets::types::Language::Rust,
ceiling: ValidationLevel::Syntax,
}));
let config = RunnerConfig {
level: ValidationLevel::TypeCheck,
parallelism: 1,
cache_dir: None,
allowed_side_effects: vec![SideEffectClass::Network],
..RunnerConfig::default()
};
let summary = run_validation(&[network_snippet()], ®istry, &config).expect("validation completes");
assert_eq!(summary.results[0].status, SnippetStatus::Pass);
assert!(summary.results[0].capability_capped);
assert_eq!(summary.downgraded, 0);
assert_eq!(summary.capability_capped, 1);
assert_eq!(summary.results[0].effective_level, ValidationLevel::Syntax);
assert_eq!(
summary.results[0].downgrade_reason,
Some(DowngradeReason::ValidatorCapability)
);
}
#[test]
fn annotation_downgrade_is_not_treated_as_a_capability_ceiling() {
let mut registry = ValidatorRegistry::new();
registry.register(Box::new(CappedValidator {
language: crate::snippets::types::Language::Rust,
ceiling: ValidationLevel::Run,
}));
let mut snippet = network_snippet();
snippet.annotation = Some(crate::snippets::types::SnippetAnnotation {
kind: SnippetAnnotationKind::SyntaxOnly,
reason: None,
});
let config = RunnerConfig {
level: ValidationLevel::TypeCheck,
parallelism: 1,
cache_dir: None,
allowed_side_effects: vec![SideEffectClass::Network],
..RunnerConfig::default()
};
let summary = run_validation(&[snippet], ®istry, &config).expect("validation completes");
assert_eq!(summary.results[0].status, SnippetStatus::Downgraded);
assert!(!summary.results[0].capability_capped);
assert_eq!(summary.downgraded, 1);
assert_eq!(summary.capability_capped, 0);
assert_eq!(summary.results[0].downgrade_reason, Some(DowngradeReason::Annotation));
}
struct EnvironmentLimitedValidator {
language: crate::snippets::types::Language,
}
impl SnippetValidator for EnvironmentLimitedValidator {
fn language(&self) -> crate::snippets::types::Language {
self.language
}
fn is_available(&self) -> bool {
true
}
fn validate(
&self,
_snippet: &Snippet,
_level: ValidationLevel,
_timeout_secs: u64,
) -> Result<(SnippetStatus, Option<String>)> {
Ok((SnippetStatus::Pass, None))
}
fn max_level(&self) -> ValidationLevel {
ValidationLevel::Run
}
fn achievable_level(&self, requested: ValidationLevel) -> ValidationLevel {
if requested == ValidationLevel::TypeCheck {
ValidationLevel::Syntax
} else {
ValidationLevel::Run
}
}
}
#[test]
fn environment_limited_validator_downgrades_instead_of_capability_capping() {
let mut registry = ValidatorRegistry::new();
registry.register(Box::new(EnvironmentLimitedValidator {
language: crate::snippets::types::Language::Rust,
}));
let config = RunnerConfig {
level: ValidationLevel::TypeCheck,
parallelism: 1,
cache_dir: None,
allowed_side_effects: vec![SideEffectClass::Network],
..RunnerConfig::default()
};
let summary = run_validation(&[network_snippet()], ®istry, &config).expect("validation completes");
assert_eq!(summary.results[0].status, SnippetStatus::Downgraded);
assert!(!summary.results[0].capability_capped);
assert_eq!(summary.results[0].effective_level, ValidationLevel::Syntax);
assert_eq!(summary.downgraded, 1);
assert_eq!(summary.capability_capped, 0);
assert_eq!(summary.results[0].downgrade_reason, Some(DowngradeReason::Environment));
}
struct StructurallyCappedAchievableValidator {
language: crate::snippets::types::Language,
}
impl SnippetValidator for StructurallyCappedAchievableValidator {
fn language(&self) -> crate::snippets::types::Language {
self.language
}
fn is_available(&self) -> bool {
true
}
fn validate(
&self,
_snippet: &Snippet,
_level: ValidationLevel,
_timeout_secs: u64,
) -> Result<(SnippetStatus, Option<String>)> {
Ok((SnippetStatus::Pass, None))
}
fn max_level(&self) -> ValidationLevel {
ValidationLevel::Run
}
fn achievable_level(&self, requested: ValidationLevel) -> ValidationLevel {
if requested == ValidationLevel::TypeCheck {
ValidationLevel::Syntax
} else {
ValidationLevel::Run
}
}
fn achievable_level_is_structural(&self, requested: ValidationLevel) -> bool {
requested == ValidationLevel::TypeCheck
}
}
#[test]
fn structural_achievable_level_gap_is_capability_capped_not_downgraded() {
let mut registry = ValidatorRegistry::new();
registry.register(Box::new(StructurallyCappedAchievableValidator {
language: crate::snippets::types::Language::Rust,
}));
let config = RunnerConfig {
level: ValidationLevel::TypeCheck,
parallelism: 1,
cache_dir: None,
allowed_side_effects: vec![SideEffectClass::Network],
..RunnerConfig::default()
};
let summary = run_validation(&[network_snippet()], ®istry, &config).expect("validation completes");
assert_eq!(summary.results[0].status, SnippetStatus::Pass);
assert!(summary.results[0].capability_capped);
assert_eq!(summary.results[0].effective_level, ValidationLevel::Syntax);
assert_eq!(summary.downgraded, 0);
assert_eq!(summary.capability_capped, 1);
assert_eq!(
summary.results[0].downgrade_reason,
Some(DowngradeReason::ValidatorCapability)
);
}
#[test]
fn declared_level_contract_passes_instead_of_downgrading() {
let mut registry = ValidatorRegistry::new();
registry.register(Box::new(CappedValidator {
language: crate::snippets::types::Language::Rust,
ceiling: ValidationLevel::Run,
}));
let mut snippet = network_snippet();
snippet.metadata.level = Some(ValidationLevel::Syntax);
let config = RunnerConfig {
level: ValidationLevel::TypeCheck,
parallelism: 1,
cache_dir: None,
allowed_side_effects: vec![SideEffectClass::Network],
..RunnerConfig::default()
};
let summary = run_validation(&[snippet], ®istry, &config).expect("validation completes");
assert_eq!(summary.results[0].status, SnippetStatus::Pass);
assert!(!summary.results[0].capability_capped);
assert_eq!(summary.results[0].effective_level, ValidationLevel::Syntax);
assert_eq!(summary.downgraded, 0);
assert_eq!(summary.capability_capped, 0);
assert_eq!(summary.results[0].downgrade_reason, Some(DowngradeReason::Declared));
assert_eq!(
summary.results[0].message.as_deref(),
Some("requested typecheck, validated at declared level syntax")
);
}
#[test]
fn declared_typecheck_ceiling_names_the_clamped_run_request() {
let mut registry = ValidatorRegistry::new();
registry.register(Box::new(CappedValidator {
language: crate::snippets::types::Language::Rust,
ceiling: ValidationLevel::Run,
}));
let mut snippet = network_snippet();
snippet.metadata.level = Some(ValidationLevel::TypeCheck);
let config = RunnerConfig {
level: ValidationLevel::Run,
parallelism: 1,
cache_dir: None,
allowed_side_effects: vec![SideEffectClass::Network],
..RunnerConfig::default()
};
let summary = run_validation(&[snippet], ®istry, &config).expect("validation completes");
assert_eq!(summary.results[0].status, SnippetStatus::Pass);
assert_eq!(summary.results[0].effective_level, ValidationLevel::TypeCheck);
assert_eq!(summary.results[0].downgrade_reason, Some(DowngradeReason::Declared));
assert_eq!(
summary.results[0].message.as_deref(),
Some("requested run, validated at declared level typecheck"),
"a `run` request clamped by a snippet's declared level must name both the request and \
the level it was clamped to, not just the clamped level"
);
}
#[test]
fn legitimately_configured_lower_level_has_no_downgrade_reason() {
let mut registry = ValidatorRegistry::new();
registry.register(Box::new(CappedValidator {
language: crate::snippets::types::Language::Rust,
ceiling: ValidationLevel::Run,
}));
let config = RunnerConfig {
level: ValidationLevel::TypeCheck,
parallelism: 1,
cache_dir: None,
allowed_side_effects: vec![SideEffectClass::Network],
..RunnerConfig::default()
};
let summary = run_validation(&[network_snippet()], ®istry, &config).expect("validation completes");
assert_eq!(summary.results[0].status, SnippetStatus::Pass);
assert_eq!(summary.results[0].effective_level, ValidationLevel::TypeCheck);
assert_eq!(summary.results[0].downgrade_reason, None);
assert_eq!(summary.results[0].message, None);
}
#[test]
fn declared_level_the_validator_cannot_reach_still_downgrades() {
let mut registry = ValidatorRegistry::new();
registry.register(Box::new(EnvironmentLimitedValidator {
language: crate::snippets::types::Language::Rust,
}));
let mut snippet = network_snippet();
snippet.metadata.level = Some(ValidationLevel::Compile);
let config = RunnerConfig {
level: ValidationLevel::TypeCheck,
parallelism: 1,
cache_dir: None,
allowed_side_effects: vec![SideEffectClass::Network],
..RunnerConfig::default()
};
let summary = run_validation(&[snippet], ®istry, &config).expect("validation completes");
assert_eq!(summary.results[0].status, SnippetStatus::Downgraded);
assert!(!summary.results[0].capability_capped);
assert_eq!(summary.results[0].effective_level, ValidationLevel::Syntax);
assert_eq!(summary.results[0].downgrade_reason, Some(DowngradeReason::Environment));
}
#[test]
fn validator_at_or_above_requested_level_is_not_capped() {
let mut registry = ValidatorRegistry::new();
registry.register(Box::new(CappedValidator {
language: crate::snippets::types::Language::Rust,
ceiling: ValidationLevel::Run,
}));
let config = RunnerConfig {
level: ValidationLevel::Syntax,
parallelism: 1,
cache_dir: None,
allowed_side_effects: vec![SideEffectClass::Network],
..RunnerConfig::default()
};
let summary = run_validation(&[network_snippet()], ®istry, &config).expect("validation completes");
assert_eq!(summary.results[0].status, SnippetStatus::Pass);
assert!(!summary.results[0].capability_capped);
assert_eq!(summary.capability_capped, 0);
}
#[traced_test]
#[test]
fn non_batching_validator_skips_the_batch_log_and_uses_the_fallback_log() {
let mut registry = ValidatorRegistry::new();
registry.register(Box::new(CappedValidator {
language: crate::snippets::types::Language::Rust,
ceiling: ValidationLevel::Run,
}));
let config = RunnerConfig {
level: ValidationLevel::Syntax,
parallelism: 1,
cache_dir: None,
allowed_side_effects: vec![SideEffectClass::Network],
..RunnerConfig::default()
};
let summary =
run_validation(&[network_snippet(), network_snippet()], ®istry, &config).expect("validation completes");
assert_eq!(summary.passed, 2);
assert!(!logs_contain("Starting batched snippet validation"));
assert!(logs_contain("Starting per-snippet validation"));
assert!(logs_contain("Finished per-snippet validation"));
}
struct DecliningBatchValidator;
impl SnippetValidator for DecliningBatchValidator {
fn language(&self) -> crate::snippets::types::Language {
crate::snippets::types::Language::Rust
}
fn is_available(&self) -> bool {
true
}
fn validate(
&self,
_snippet: &Snippet,
_level: ValidationLevel,
_timeout_secs: u64,
) -> Result<(SnippetStatus, Option<String>)> {
Ok((SnippetStatus::Pass, None))
}
fn validate_batch_in_session(
&self,
_snippets: &[&Snippet],
_level: ValidationLevel,
_timeout_secs: u64,
_session: Option<&crate::snippets::session::ValidationSession>,
) -> Option<Result<Vec<(SnippetStatus, Option<String>)>>> {
None
}
fn max_level(&self) -> ValidationLevel {
ValidationLevel::Run
}
fn supports_batching(&self) -> bool {
true
}
}
#[traced_test]
#[test]
fn batching_validator_that_declines_a_group_logs_the_fallback_explicitly() {
let mut registry = ValidatorRegistry::new();
registry.register(Box::new(DecliningBatchValidator));
let config = RunnerConfig {
level: ValidationLevel::Syntax,
parallelism: 1,
cache_dir: None,
allowed_side_effects: vec![SideEffectClass::Network],
..RunnerConfig::default()
};
let summary = run_validation(&[network_snippet()], ®istry, &config).expect("validation completes");
assert_eq!(summary.results[0].status, SnippetStatus::Pass);
assert!(logs_contain("Starting batched snippet validation"));
assert!(logs_contain(
"Batch validation declined for this group; falling back to per-snippet validation"
));
assert!(logs_contain("Starting per-snippet validation"));
}
struct FailingValidator;
impl SnippetValidator for FailingValidator {
fn language(&self) -> crate::snippets::types::Language {
crate::snippets::types::Language::Java
}
fn is_available(&self) -> bool {
true
}
fn validate(
&self,
_snippet: &Snippet,
_level: ValidationLevel,
_timeout_secs: u64,
) -> Result<(SnippetStatus, Option<String>)> {
Ok((
SnippetStatus::Fail,
Some("Example.java:1: error: duplicate class: Example\n 1 error".into()),
))
}
fn max_level(&self) -> ValidationLevel {
ValidationLevel::Run
}
}
fn failing_snippet(language: crate::snippets::types::Language) -> Snippet {
let mut snippet = network_snippet();
snippet.language = language;
snippet
}
fn failure_result(snippet: &Snippet, message: &str) -> ValidationResult {
result(
snippet,
SnippetStatus::Fail,
ValidationLevel::Compile,
ValidationLevel::Compile,
Some(message.to_string()),
1,
)
}
#[traced_test]
#[test]
fn a_languages_first_failure_is_reported_before_its_stage_ends() {
let snippets = vec![
failing_snippet(crate::snippets::types::Language::Java),
failing_snippet(crate::snippets::types::Language::Java),
failing_snippet(crate::snippets::types::Language::Java),
];
let reporter = FailureReporter::new(&snippets);
reporter.record(&failure_result(
&snippets[0],
"error: cannot find symbol\n symbol: class Missing",
));
assert!(logs_contain("First snippet validation failure for this language"));
assert!(logs_contain("cannot find symbol | symbol: class Missing"));
assert!(!logs_contain("Finished snippet validation for this language"));
}
fn unavailable_result(snippet: &Snippet, message: &str) -> ValidationResult {
let mut value = result(
snippet,
SnippetStatus::Unavailable,
ValidationLevel::Compile,
ValidationLevel::Compile,
Some(message.to_string()),
1,
);
value.unresolved_dependency = true;
value
}
#[traced_test]
#[test]
fn a_languages_first_unavailable_result_is_reported_with_the_validator_message() {
let snippets = vec![
failing_snippet(crate::snippets::types::Language::Csharp),
failing_snippet(crate::snippets::types::Language::Csharp),
];
let reporter = FailureReporter::new(&snippets);
reporter.record(&unavailable_result(
&snippets[0],
"error NU1101: Unable to find package Contoso.Sample",
));
assert!(logs_contain(
"First snippet validation unavailability for this language"
));
assert!(logs_contain("Unable to find package Contoso.Sample"));
}
#[traced_test]
#[test]
fn a_language_with_no_validated_result_at_all_is_not_reported_as_finished_clean() {
let snippets = vec![
failing_snippet(crate::snippets::types::Language::Csharp),
failing_snippet(crate::snippets::types::Language::Csharp),
];
let reporter = FailureReporter::new(&snippets);
for snippet in &snippets {
reporter.record(&unavailable_result(snippet, "error NU1101: Unable to find package"));
}
assert!(logs_contain(
"Finished snippet validation for this language with every result unvalidated"
));
}
#[traced_test]
#[test]
fn failures_after_the_first_are_counted_rather_than_logged_one_line_each() {
let snippets = vec![failing_snippet(crate::snippets::types::Language::Java); FAILURE_PROGRESS_STRIDE + 1];
let reporter = FailureReporter::new(&snippets);
for snippet in snippets.iter().take(2) {
reporter.record(&failure_result(snippet, "compilation failed"));
}
assert!(logs_contain("First snippet validation failure for this language"));
assert!(!logs_contain("Snippet validation failures accumulating"));
for snippet in snippets.iter().take(FAILURE_PROGRESS_STRIDE).skip(2) {
reporter.record(&failure_result(snippet, "compilation failed"));
}
assert!(logs_contain("Snippet validation failures accumulating"));
assert!(!logs_contain("Finished snippet validation for this language"));
}
#[traced_test]
#[test]
fn a_failing_language_is_reported_through_the_real_validation_run() {
let mut registry = ValidatorRegistry::new();
registry.register(Box::new(FailingValidator));
let config = RunnerConfig {
level: ValidationLevel::Compile,
parallelism: 1,
cache_dir: None,
..RunnerConfig::default()
};
let snippets = vec![
failing_snippet(crate::snippets::types::Language::Java),
failing_snippet(crate::snippets::types::Language::Java),
];
let summary = run_validation(&snippets, ®istry, &config).expect("validation completes");
assert_eq!(summary.failed, 2);
assert!(logs_contain("First snippet validation failure for this language"));
assert!(logs_contain("duplicate class: Example"));
assert!(logs_contain(
"Finished snippet validation for this language with failures"
));
}
#[test]
fn a_failure_preview_is_a_single_bounded_line() {
let long = "x".repeat(FAILURE_MESSAGE_PREVIEW_CHARS + 50);
let preview = failure_preview(Some(long.as_str()));
assert_eq!(preview.len(), FAILURE_MESSAGE_PREVIEW_CHARS + 3);
assert!(preview.ends_with("..."));
assert_eq!(failure_preview(Some(" \n\n ")), "<no validator output>");
assert_eq!(failure_preview(None), "<no validator output>");
assert_eq!(failure_preview(Some("first\n\nsecond")), "first | second");
}
struct DependencyFailingValidator;
impl SnippetValidator for DependencyFailingValidator {
fn language(&self) -> crate::snippets::types::Language {
crate::snippets::types::Language::TypeScript
}
fn is_available(&self) -> bool {
true
}
fn validate(
&self,
_snippet: &Snippet,
_level: ValidationLevel,
_timeout_secs: u64,
) -> Result<(SnippetStatus, Option<String>)> {
unreachable!("this test drives finalize_result directly, not through validate")
}
fn max_level(&self) -> ValidationLevel {
ValidationLevel::Run
}
fn is_dependency_error(&self, error_output: &str) -> bool {
error_output.contains("Cannot find module")
}
}
#[test]
fn finalize_result_sets_unresolved_dependency_on_the_returned_result() {
let snippet = failing_snippet(crate::snippets::types::Language::TypeScript);
let validator = DependencyFailingValidator;
let config = RunnerConfig {
level: ValidationLevel::Compile,
cache_dir: None,
..RunnerConfig::default()
};
let outcome = ValidationOutcome {
status: SnippetStatus::Fail,
message: Some("error TS2307: Cannot find module 'widgets'".to_string()),
duration_ms: 5,
};
let result = finalize_result(&snippet, &validator, &config, None, ValidationLevel::Compile, outcome);
assert_eq!(result.status, SnippetStatus::Unavailable, "got: {result:?}");
assert!(
result.unresolved_dependency,
"finalize_result must set unresolved_dependency on the ValidationResult it returns, not just use it \
locally to reclassify status and message"
);
}
}