//! Index command-line prototype.
#[cfg(test)]
use std::cell::RefCell;
use std::collections::BTreeMap;
use std::env;
use std::fs;
use std::io::{self, Read};
use std::mem;
use std::path::{Path, PathBuf};
use std::process::ExitCode;
use std::sync::{Arc, Mutex};
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
mod compat_pack_runtime;
use compat_pack_runtime::{
PackSource, inspect_runtime_for_url, install_pack_file, lint_pack_file,
list_runtime_pack_files, rollback_pack_file, select_runtime_manifest_for_url, sign_pack_bytes,
verify_pack_signature,
};
use image::imageops::{BiLevel, FilterType, dither, resize};
use index_ai::{AiAction, OfflineProvider, PrivacyMode, run_ai_action};
use index_capture::{
ArtifactContext, ArtifactFreshness, ArtifactStore, CaptureArtifact, CaptureRequest,
IndexArtifact, capture_redacted, catalog_entry_for_fixture, preview_redacted,
validate_capture_bundle,
};
use index_core::{CookieJar, MemorySecureStorage, Redactor, SecureStorage};
use index_core::{
DiagnosticAction, DiagnosticConfidence, DiagnosticRecord, DiagnosticSeverity, DiagnosticSource,
FailureDiagnostic, FormSubmission, IndexDocument, IndexNode, IndexUrl, KnowledgeShelf, Link,
ReaderProfile, ResponseLogEntry, ShelfRecord, ShelfSearchResult,
};
use index_dom::{
IndexManifest, IndexManifestError, discover_index_manifest_link_from_html,
discover_index_manifest_link_from_http_link_header, parse_index_manifest,
well_known_index_manifest_url,
};
use index_extract::{
ExtractFormat, ExtractionLimits, export_section_markdown, extract_citations_tsv,
extract_document, try_extract_document,
};
use index_http::{Fetcher, FormSubmitter, Request, RetryingFetcher, SecureFetcher, UreqFetcher};
use index_renderer::{
RenderOptions, TuiDocumentResult, render_document,
run_tui_with_navigation_profile_forms_and_state_with_progress,
};
use index_security::{ContentLimits, check_content_size};
use index_transformer::{
TransformedDocumentCache, Transformer, apply_index_manifest_hints, state::Empty,
transform_html_cached,
};
const VERSION: &str = env!("CARGO_PKG_VERSION");
const IMAGE_PREVIEW_MAX_IMAGES_PER_DOCUMENT: usize = 8;
const IMAGE_PREVIEW_MAX_FETCH_BYTES: usize = 1024 * 1024;
const IMAGE_PREVIEW_MAX_WIDTH: u32 = 101;
const IMAGE_PREVIEW_MAX_HEIGHT: u32 = 67;
const IMAGE_PREVIEW_TIMEOUT: Duration = Duration::from_secs(4);
const ARTIFACT_MAX_AGE_SECS: u64 = 900;
const STAGE_FETCH_TIMEOUT: Duration = Duration::from_secs(120);
const STAGE_SNAPSHOT_TIMEOUT: Duration = Duration::from_secs(30);
const STAGE_PARSE_TIMEOUT: Duration = Duration::from_secs(15);
const STAGE_TRANSFORM_TIMEOUT: Duration = Duration::from_secs(120);
const STAGE_SCORE_TIMEOUT: Duration = Duration::from_secs(10);
const STAGE_STORE_TIMEOUT: Duration = Duration::from_secs(10);
const COMPATIBILITY_SLO_TOP100_PATH: &str = "docs/top100-corpus/matrix.tsv";
const COMPATIBILITY_SLO_FORUM_PATH: &str = "docs/forum-corpus/matrix.tsv";
const LIVE_VARIANCE_TARGETS_PATH: &str = "docs/live-variance/targets.tsv";
const LIVE_VARIANCE_RUNS_PATH: &str = "docs/live-variance/runs.tsv";
const COMPATIBILITY_SLO_READABILITY_FLOOR: f64 = 0.85;
const COMPATIBILITY_SLO_ACTIONABILITY_FLOOR: f64 = 0.40;
const COMPATIBILITY_SLO_FAILURE_QUALITY_FLOOR: f64 = 0.98;
const COMPATIBILITY_SLO_V2_DEFAULT_FAMILY_READABILITY_FLOOR: f64 = 0.35;
const COMPATIBILITY_SLO_V2_DEFAULT_FAMILY_ACTIONABILITY_FLOOR: f64 = 0.15;
const COMPATIBILITY_SLO_V2_DEFAULT_FAMILY_FAILURE_FLOOR: f64 = 0.98;
const COMPATIBILITY_BACKLOG_TOP_N_DEFAULT: usize = 20;
const LIVE_VARIANCE_WINDOW_DEFAULT: usize = 10;
const LIVE_VARIANCE_MAX_WINDOW: usize = 120;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum RuntimeStage {
Queued,
Fetching,
Snapshotting,
Parsing,
Transforming,
Scoring,
Storing,
Done,
Failed,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum RecoveryProfile {
Default,
AppShell,
}
impl RecoveryProfile {
const fn as_str(self) -> &'static str {
match self {
Self::Default => "default",
Self::AppShell => "app-shell",
}
}
}
impl RuntimeStage {
const fn as_str(self) -> &'static str {
match self {
Self::Queued => "queued",
Self::Fetching => "fetching",
Self::Snapshotting => "snapshotting",
Self::Parsing => "parsing",
Self::Transforming => "transforming",
Self::Scoring => "scoring",
Self::Storing => "storing",
Self::Done => "done",
Self::Failed => "failed",
}
}
const fn default_timeout(self) -> Option<Duration> {
match self {
Self::Fetching => Some(STAGE_FETCH_TIMEOUT),
Self::Snapshotting => Some(STAGE_SNAPSHOT_TIMEOUT),
Self::Parsing => Some(STAGE_PARSE_TIMEOUT),
Self::Transforming => Some(STAGE_TRANSFORM_TIMEOUT),
Self::Scoring => Some(STAGE_SCORE_TIMEOUT),
Self::Storing => Some(STAGE_STORE_TIMEOUT),
Self::Queued | Self::Done | Self::Failed => None,
}
}
}
fn recovery_profile_for_target(target: &str) -> RecoveryProfile {
let normalized = normalize_url_input(target);
let parsed = url::Url::parse(&normalized);
let host = parsed
.ok()
.and_then(|url| url.host_str().map(str::to_ascii_lowercase))
.unwrap_or_default();
if matches!(
host.as_str(),
"chatgpt.com"
| "claude.ai"
| "gemini.google.com"
| "deepseek.com"
| "chat.deepseek.com"
| "discord.com"
| "facebook.com"
| "instagram.com"
| "linkedin.com"
| "netflix.com"
| "roblox.com"
| "tiktok.com"
| "douyin.com"
| "x.com"
| "youtube.com"
| "music.youtube.com"
) {
return RecoveryProfile::AppShell;
}
RecoveryProfile::Default
}
fn stage_timeout_for_target(stage: RuntimeStage, target: &str) -> Option<Duration> {
let profile = recovery_profile_for_target(target);
match (profile, stage) {
(RecoveryProfile::AppShell, RuntimeStage::Fetching) => Some(Duration::from_secs(180)),
(RecoveryProfile::AppShell, RuntimeStage::Snapshotting) => Some(Duration::from_secs(45)),
(RecoveryProfile::AppShell, RuntimeStage::Transforming) => Some(Duration::from_secs(180)),
(RecoveryProfile::AppShell, RuntimeStage::Scoring) => Some(Duration::from_secs(15)),
(RecoveryProfile::AppShell, RuntimeStage::Storing) => Some(Duration::from_secs(15)),
_ => stage.default_timeout(),
}
}
fn recovery_fallback_order(profile: RecoveryProfile) -> &'static [&'static str] {
match profile {
RecoveryProfile::Default => &["static-dom", "accessibility-snapshot", "rendered-dom"],
RecoveryProfile::AppShell => &["accessibility-snapshot", "rendered-dom", "static-dom"],
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
struct StageFlow {
last: Option<RuntimeStage>,
}
impl StageFlow {
fn advance(&mut self, next: RuntimeStage) -> Result<(), String> {
if !is_valid_stage_transition(self.last, next) {
return Err(format!(
"invalid stage transition: {:?} -> {:?}",
self.last, next
));
}
self.last = Some(next);
Ok(())
}
}
fn is_valid_stage_transition(previous: Option<RuntimeStage>, next: RuntimeStage) -> bool {
matches!(
(previous, next),
(None, RuntimeStage::Queued)
| (Some(RuntimeStage::Queued), RuntimeStage::Fetching)
| (Some(RuntimeStage::Queued), RuntimeStage::Snapshotting)
| (Some(RuntimeStage::Queued), RuntimeStage::Parsing)
| (Some(RuntimeStage::Queued), RuntimeStage::Transforming)
| (Some(RuntimeStage::Queued), RuntimeStage::Failed)
| (Some(RuntimeStage::Fetching), RuntimeStage::Snapshotting)
| (Some(RuntimeStage::Fetching), RuntimeStage::Parsing)
| (Some(RuntimeStage::Fetching), RuntimeStage::Failed)
| (Some(RuntimeStage::Snapshotting), RuntimeStage::Parsing)
| (Some(RuntimeStage::Snapshotting), RuntimeStage::Failed)
| (Some(RuntimeStage::Parsing), RuntimeStage::Transforming)
| (Some(RuntimeStage::Parsing), RuntimeStage::Failed)
| (Some(RuntimeStage::Transforming), RuntimeStage::Scoring)
| (Some(RuntimeStage::Transforming), RuntimeStage::Failed)
| (Some(RuntimeStage::Scoring), RuntimeStage::Storing)
| (Some(RuntimeStage::Scoring), RuntimeStage::Done)
| (Some(RuntimeStage::Scoring), RuntimeStage::Failed)
| (Some(RuntimeStage::Storing), RuntimeStage::Done)
| (Some(RuntimeStage::Storing), RuntimeStage::Failed)
)
}
fn report_stage(
flow: &mut StageFlow,
stage: RuntimeStage,
target: &str,
report_progress: &mut dyn FnMut(String),
) -> Result<(), String> {
flow.advance(stage)?;
report_progress(format!("{} {target}", stage.as_str()));
Ok(())
}
fn enforce_stage_budget(
stage: RuntimeStage,
elapsed: Duration,
target: &str,
) -> Result<(), String> {
let Some(timeout) = stage_timeout_for_target(stage, target) else {
return Ok(());
};
if elapsed > timeout {
return Err(format!(
"stage {} timed out after {:?} for {target}",
stage.as_str(),
elapsed
));
}
Ok(())
}
fn candidate_manifest_urls(
page_url: &str,
html: &str,
http_link_header: Option<&str>,
) -> Vec<String> {
let mut candidates = Vec::new();
if let Some(url) = well_known_index_manifest_url(page_url) {
candidates.push(url);
}
if let Some(link_header) = http_link_header {
if let Some(url) = discover_index_manifest_link_from_http_link_header(link_header, page_url)
{
if !candidates.contains(&url) {
candidates.push(url);
}
}
}
if let Some(url) = discover_index_manifest_link_from_html(html, page_url) {
if !candidates.contains(&url) {
candidates.push(url);
}
}
candidates
}
fn load_index_manifest_for_response<F: Fetcher>(
fetcher: &F,
response: &index_http::Response,
) -> Option<IndexManifest> {
let page_url = response.final_url.as_str();
let candidates = candidate_manifest_urls(page_url, &response.body, None);
for candidate in candidates {
let url = match IndexUrl::parse(&candidate) {
Ok(url) => url,
Err(_) => continue,
};
let manifest_response = match fetcher.fetch(&Request { url }) {
Ok(response) => response,
Err(_) => continue,
};
let parsed = parse_index_manifest(
&manifest_response.body,
manifest_response.final_url.as_str(),
page_url,
);
if let Ok(manifest) = parsed {
return Some(manifest);
}
}
None
}
fn apply_runtime_pack_manifest_hints(document: &mut IndexDocument, final_url: &str) {
let paths = runtime_paths();
if let Ok(Some(selected)) = select_runtime_manifest_for_url(&paths.config_dir, final_url) {
apply_index_manifest_hints(document, &selected.manifest);
if let Some(quality) = document.metadata.quality.as_mut() {
quality.reasons.push(format!(
"compatibility pack {}:{} matched {}{}",
selected.source.as_str(),
selected.pack_id,
selected.rule_host,
selected.rule_path_prefix
));
}
}
}
fn main() -> ExitCode {
match run() {
Ok(CliAction::Print(output)) => {
println!("{output}");
ExitCode::SUCCESS
}
Ok(CliAction::Tui {
document,
profile,
url_history,
}) => {
let fetcher = RetryingFetcher::new(SecureFetcher::new(UreqFetcher::new()));
let transform_cache = Arc::new(Mutex::new(TransformedDocumentCache::new()));
let response_log_sequence = Arc::new(Mutex::new(0_u64));
let artifact_store = Arc::new(ArtifactStore::new(artifact_store_root()));
match run_tui_with_navigation_profile_forms_and_state_with_progress(
*document,
profile,
url_history,
Vec::new(),
{
let fetcher = fetcher.clone();
let transform_cache = Arc::clone(&transform_cache);
let response_log_sequence = Arc::clone(&response_log_sequence);
let artifact_store = Arc::clone(&artifact_store);
move |target, report_progress| {
let mut cache = transform_cache
.lock()
.map_err(|_| "transform cache lock poisoned".to_owned())?;
let mut sequence = response_log_sequence
.lock()
.map_err(|_| "response log lock poisoned".to_owned())?;
fetch_document_with_artifact_runtime(
&fetcher,
&mut cache,
&mut sequence,
&artifact_store,
target,
report_progress,
)
}
},
{
let fetcher = fetcher.clone();
let transform_cache = Arc::clone(&transform_cache);
let response_log_sequence = Arc::clone(&response_log_sequence);
let artifact_store = Arc::clone(&artifact_store);
move |submission, report_progress| {
let mut cache = transform_cache
.lock()
.map_err(|_| "transform cache lock poisoned".to_owned())?;
let mut sequence = response_log_sequence
.lock()
.map_err(|_| "response log lock poisoned".to_owned())?;
submit_form_with_artifact_runtime(
&fetcher,
&mut cache,
&mut sequence,
&artifact_store,
submission,
report_progress,
)
}
},
) {
Ok(()) => ExitCode::SUCCESS,
Err(error) => {
eprintln!("index: failed to run terminal UI: {error}");
ExitCode::from(1)
}
}
}
Ok(CliAction::Help(output)) => {
println!("{output}");
ExitCode::SUCCESS
}
Ok(CliAction::Version(output)) => {
println!("{output}");
ExitCode::SUCCESS
}
Err(error) => {
eprintln!("index: {error}");
ExitCode::from(1)
}
}
}
fn run() -> Result<CliAction, String> {
let args = env::args().skip(1).collect::<Vec<_>>();
let mut stdin = io::stdin();
run_with_args(args, &mut stdin)
}
#[derive(Debug, Clone, PartialEq, Eq)]
enum CliAction {
Help(String),
Print(String),
Tui {
document: Box<IndexDocument>,
profile: ReaderProfile,
url_history: Vec<String>,
},
Version(String),
}
fn run_with_args<I, S>(args: I, stdin: &mut dyn Read) -> Result<CliAction, String>
where
I: IntoIterator<Item = S>,
S: AsRef<str>,
{
let fetcher = RetryingFetcher::new(SecureFetcher::new(UreqFetcher::new()));
run_with_args_and_fetcher(args, stdin, &fetcher)
}
fn run_with_args_and_fetcher<I, S, F>(
args: I,
stdin: &mut dyn Read,
fetcher: &F,
) -> Result<CliAction, String>
where
I: IntoIterator<Item = S>,
S: AsRef<str>,
F: Fetcher,
{
let mut args = args.into_iter();
let mut output_mode = OutputMode::Tui;
let mut profile = ReaderProfile::Reader;
let Some(input) = args.next() else {
return Ok(CliAction::Tui {
document: Box::new(default_start_page()),
profile,
url_history: Vec::new(),
});
};
let mut input = input.as_ref().to_owned();
if input == "--profile" {
let Some(next_profile) = args.next() else {
return Ok(CliAction::Help(help()));
};
profile = ReaderProfile::parse(next_profile.as_ref())
.ok_or_else(|| format!("unsupported reader profile: {}", next_profile.as_ref()))?;
let Some(next_input) = args.next() else {
return Ok(CliAction::Tui {
document: Box::new(default_start_page()),
profile,
url_history: Vec::new(),
});
};
input = next_input.as_ref().to_owned();
}
if input == "--help" || input == "-h" {
return Ok(CliAction::Help(help()));
}
if input == "--version" || input == "-V" {
return Ok(CliAction::Version(version()));
}
if input == "--paths" {
return Ok(CliAction::Print(runtime_paths().display()));
}
if input == "doctor" {
if args.next().is_some() {
return Ok(CliAction::Help(help()));
}
return Ok(CliAction::Print(doctor_report(&runtime_paths())));
}
if input == "quickstart" {
if args.next().is_some() {
return Ok(CliAction::Help(help()));
}
return Ok(CliAction::Print(quickstart()));
}
if input == "compatibility-slo" {
return run_compatibility_slo_command(args);
}
if input == "compatibility-slo-v2" {
return run_compatibility_slo_v2_command(args);
}
if input == "compatibility-backlog" {
return run_compatibility_backlog_command(args);
}
if input == "compatibility-live-variance" {
return run_compatibility_live_variance_command(args);
}
if input == "compatibility-recovery-plan" {
return run_compatibility_recovery_plan_command(args);
}
if input == "compatibility-recovery-gate" {
return run_compatibility_recovery_gate_command(args);
}
if input == "compatibility-pack" {
return run_compatibility_pack_command(args);
}
if input == "auth-assist" {
return run_auth_assist_command(args);
}
if input == "challenge-diagnose" {
return run_challenge_diagnose_command(args);
}
if input == "idx" {
return run_idx_command(args);
}
if input == "--benchmark" {
let Some(target) = args.next() else {
return Ok(CliAction::Help(help()));
};
if args.next().is_some() {
return Err("too many arguments".to_owned());
}
return run_benchmark_command(target.as_ref(), stdin, fetcher);
}
if input == "--save" {
let Some(format) = args.next() else {
return Ok(CliAction::Help(help()));
};
let Some(format) = ExtractFormat::parse(format.as_ref()) else {
return Err(format!("unsupported save format: {}", format.as_ref()));
};
let Some(target) = args.next() else {
return Ok(CliAction::Help(help()));
};
let Some(output_path) = args.next() else {
return Ok(CliAction::Help(help()));
};
if args.next().is_some() {
return Err("too many arguments".to_owned());
}
return run_save_command(
format,
target.as_ref(),
output_path.as_ref(),
stdin,
fetcher,
);
}
if input == "--citations" {
let Some(target) = args.next() else {
return Ok(CliAction::Help(help()));
};
if args.next().is_some() {
return Err("too many arguments".to_owned());
}
let document = load_document(target.as_ref(), stdin, fetcher)?;
return Ok(CliAction::Print(extract_citations_tsv(&document)));
}
if input == "--section" {
let Some(selector) = args.next() else {
return Ok(CliAction::Help(help()));
};
let Some(target) = args.next() else {
return Ok(CliAction::Help(help()));
};
if args.next().is_some() {
return Err("too many arguments".to_owned());
}
let document = load_document(target.as_ref(), stdin, fetcher)?;
let section = export_section_markdown(&document, selector.as_ref())
.ok_or_else(|| format!("section not found: {}", selector.as_ref()))?;
return Ok(CliAction::Print(section));
}
if input == "--batch-extract" {
let Some(format) = args.next() else {
return Ok(CliAction::Help(help()));
};
let Some(format) = ExtractFormat::parse(format.as_ref()) else {
return Err(format!(
"unsupported batch extraction format: {}",
format.as_ref()
));
};
let targets = args.map(|arg| arg.as_ref().to_owned()).collect::<Vec<_>>();
if targets.is_empty() {
return Ok(CliAction::Help(help()));
}
return run_batch_extract_command(format, &targets);
}
if input == "adapter" {
return run_adapter_command(args);
}
if input == "artifact" {
return run_artifact_command(args);
}
if input == "capture" {
return run_capture_command(args, stdin);
}
if input == "shelf" {
return run_shelf_command(args, stdin, fetcher);
}
if input == "--plain" {
output_mode = OutputMode::Plain;
let Some(next_input) = args.next() else {
return Ok(CliAction::Help(help()));
};
input = next_input.as_ref().to_owned();
} else if input == "--extract" {
let Some(format) = args.next() else {
return Ok(CliAction::Help(help()));
};
let Some(format) = ExtractFormat::parse(format.as_ref()) else {
return Err(format!(
"unsupported extraction format: {}",
format.as_ref()
));
};
output_mode = OutputMode::Extract(format);
let Some(next_input) = args.next() else {
return Ok(CliAction::Help(help()));
};
input = next_input.as_ref().to_owned();
} else if input == "--ai-offline" {
let Some(action) = args.next() else {
return Ok(CliAction::Help(help()));
};
let Some(action) = AiAction::parse(action.as_ref()) else {
return Err(format!("unsupported AI action: {}", action.as_ref()));
};
output_mode = OutputMode::AiOffline(action);
let Some(next_input) = args.next() else {
return Ok(CliAction::Help(help()));
};
input = next_input.as_ref().to_owned();
}
if args.next().is_some() {
return Err("too many arguments".to_owned());
}
let document = load_document(&input, stdin, fetcher)?;
let url_history = should_fetch_input(&input).then(|| normalize_url_input(&input));
match output_mode {
OutputMode::Tui => Ok(CliAction::Tui {
document: Box::new(document),
profile,
url_history: url_history.into_iter().collect(),
}),
OutputMode::Plain => Ok(CliAction::Print(render_document(
&document,
RenderOptions::default(),
))),
OutputMode::Extract(format) => Ok(CliAction::Print(
try_extract_document(&document, format, ExtractionLimits::default())
.map_err(|error| error.to_string())?,
)),
OutputMode::AiOffline(action) => {
let response = run_ai_action(
&OfflineProvider,
&document,
action,
PrivacyMode::Redacted,
&Redactor::new(),
)
.map_err(|error| error.to_string())?;
Ok(CliAction::Print(response.text))
}
}
}
fn run_capture_command<I, S>(mut args: I, stdin: &mut dyn Read) -> Result<CliAction, String>
where
I: Iterator<Item = S>,
S: AsRef<str>,
{
let Some(flag) = args.next() else {
return Ok(CliAction::Help(help()));
};
if flag.as_ref() == "--validate" {
let Some(input) = args.next() else {
return Ok(CliAction::Help(help()));
};
if args.next().is_some() {
return Err("too many arguments".to_owned());
}
let artifact_text = read_input(input.as_ref(), stdin)?;
let artifact =
validate_capture_bundle(&artifact_text).map_err(|error| error.to_string())?;
return Ok(CliAction::Print(format!(
"index-capture-validation-v1\nsource_url: {}\nstatus: ok\n{}",
artifact.source_url,
artifact.submission_checklist()
)));
}
if flag.as_ref() == "--catalog-entry" {
let Some(path) = args.next() else {
return Ok(CliAction::Help(help()));
};
if args.next().is_some() {
return Err("too many arguments".to_owned());
}
return Ok(CliAction::Print(
catalog_entry_for_fixture(path.as_ref(), "unknown", 0)
.map_err(|error| error.to_string())?,
));
}
let preview = if flag.as_ref() == "--preview" {
let Some(redact_flag) = args.next() else {
return Ok(CliAction::Help(help()));
};
if redact_flag.as_ref() != "--redact" {
return Err(format!(
"unsupported capture option: {}",
redact_flag.as_ref()
));
}
true
} else if flag.as_ref() == "--redact" {
false
} else {
return Err(format!("unsupported capture option: {}", flag.as_ref()));
};
let Some(source_url) = args.next() else {
return Ok(CliAction::Help(help()));
};
let Some(input) = args.next() else {
return Ok(CliAction::Help(help()));
};
if args.next().is_some() {
return Err("too many arguments".to_owned());
}
let html = read_input(input.as_ref(), stdin)?;
let source_url = normalize_url_input(source_url.as_ref());
let request = CaptureRequest::new(&source_url, html).map_err(|error| error.to_string())?;
if preview {
Ok(CliAction::Print(preview_redacted(&request).to_text()))
} else {
Ok(CliAction::Print(capture_redacted(&request).to_text()))
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum OutputMode {
Tui,
Plain,
Extract(ExtractFormat),
AiOffline(AiAction),
}
fn read_input(input: &str, stdin: &mut dyn Read) -> Result<String, String> {
read_input_with_limits(input, stdin, ContentLimits::default())
}
fn default_start_page() -> IndexDocument {
let mut document = IndexDocument::titled("Index");
document.push(IndexNode::Heading {
level: 2,
text: "Start".to_owned(),
});
document.push(IndexNode::Paragraph(
"Open a page with :open <url>, or start Index with index example.org. Missing URL schemes default to HTTPS."
.to_owned(),
));
document.push(IndexNode::Paragraph(
"Use f to show link hints, / to search, j/k to scroll, and :quit to leave.".to_owned(),
));
document.push(IndexNode::Heading {
level: 2,
text: "Useful Commands".to_owned(),
});
document.push(IndexNode::List {
ordered: false,
items: vec![
":open <id-or-url> fetch and render a page".to_owned(),
"e edit form fields; tab moves fields; enter submits".to_owned(),
":extract markdown|links|json request scriptable output".to_owned(),
":submit <form> field=value resolve a semantic form action".to_owned(),
":pipe <cmd> request a confirmed local pipe action".to_owned(),
":ai explain|summarize|extract request an explicit local AI action".to_owned(),
],
});
document.push(IndexNode::Heading {
level: 2,
text: "Local Inputs".to_owned(),
});
document.push(IndexNode::Paragraph(
"Index also accepts local HTML files and stdin: index examples/sample.html or index - < file.html."
.to_owned(),
));
document.push(IndexNode::Paragraph(
"Need the command cheat sheet? Run index quickstart.".to_owned(),
));
document.push(IndexNode::Link(Link::new(
"Example Domain",
"https://example.org",
)));
document
}
fn load_document<F: Fetcher>(
input: &str,
stdin: &mut dyn Read,
fetcher: &F,
) -> Result<IndexDocument, String> {
if should_fetch_input(input) {
let normalized = normalize_url_input(input);
return match fetch_document(fetcher, &normalized) {
Ok(document) => Ok(document),
Err(error) => Ok(network_failure_document(&normalized, &error)),
};
}
let html = read_input(input, stdin)?;
Ok(transform_html(html))
}
fn network_failure_document(target: &str, error: &str) -> IndexDocument {
FailureDiagnostic::new(
"Network fetch failed",
DiagnosticSource::Network,
DiagnosticConfidence::Failed,
format!("could not fetch {target}: {error}"),
)
.with_fallback("no document was transformed")
.with_tried("URL normalization")
.with_tried("secure fetcher")
.with_tried("bounded retry policy")
.with_actions([
DiagnosticAction::Retry,
DiagnosticAction::Extract,
DiagnosticAction::Capture,
])
.with_command(format!(":open {target}"))
.with_command(":capture save network-failure.capture")
.with_record(
DiagnosticRecord::new(
DiagnosticSeverity::Error,
"INDEX-NETWORK-FAILED",
error.to_owned(),
)
.with_field("target", target),
)
.into_document()
}
fn fetch_document<F: Fetcher>(fetcher: &F, target: &str) -> Result<IndexDocument, String> {
let normalized = normalize_url_input(target);
let url = IndexUrl::parse(&normalized).map_err(|error| error.to_string())?;
if artifact_runtime_enabled() {
let artifact_store = ArtifactStore::new(artifact_store_root());
if let Some(artifact) = artifact_store
.load(&url, ArtifactContext::LiveGet)
.map_err(|error| error.to_string())?
{
if artifact.is_fresh(unix_timestamp_secs()) {
return document_from_index_artifact_uncached(&artifact);
}
}
let response = fetcher
.fetch(&Request { url: url.clone() })
.map_err(|error| error.to_string())?;
let mut document = transform_html(html_with_base(&response.final_url, &response.body));
document = hydrate_document_images(document);
apply_runtime_pack_manifest_hints(&mut document, response.final_url.as_str());
if let Ok(artifact) = IndexArtifact::from_document(
&document,
&url,
&response.final_url,
ArtifactContext::LiveGet,
unix_timestamp_secs(),
ARTIFACT_MAX_AGE_SECS,
) {
let _ = artifact_store.store(&artifact);
}
return Ok(document);
}
let response = fetcher
.fetch(&Request { url: url.clone() })
.map_err(|error| error.to_string())?;
let mut document = transform_html(html_with_base(&response.final_url, &response.body));
document = hydrate_document_images(document);
apply_runtime_pack_manifest_hints(&mut document, response.final_url.as_str());
Ok(document)
}
#[cfg(test)]
fn fetch_document_cached_with_log<F: Fetcher>(
fetcher: &F,
cache: &mut TransformedDocumentCache,
sequence: &RefCell<u64>,
target: &str,
) -> Result<TuiDocumentResult, String> {
let mut next_sequence = sequence.borrow_mut();
fetch_document_cached_with_log_with_counter(fetcher, cache, &mut next_sequence, target)
}
#[cfg(test)]
fn fetch_document_cached_with_log_with_counter<F: Fetcher>(
fetcher: &F,
cache: &mut TransformedDocumentCache,
sequence: &mut u64,
target: &str,
) -> Result<TuiDocumentResult, String> {
let mut ignore_progress = |_message: String| {};
fetch_document_cached_with_log_with_counter_with_progress(
fetcher,
cache,
sequence,
target,
&mut ignore_progress,
)
}
fn fetch_document_with_artifact_runtime<F: Fetcher + Clone + Send + 'static>(
fetcher: &F,
cache: &mut TransformedDocumentCache,
sequence: &mut u64,
artifact_store: &ArtifactStore,
target: &str,
report_progress: &mut dyn FnMut(String),
) -> Result<TuiDocumentResult, String> {
let normalized = normalize_url_input(target);
let canonical = IndexUrl::parse(&normalized).map_err(|error| error.to_string())?;
let mut flow = StageFlow::default();
report_stage(
&mut flow,
RuntimeStage::Queued,
&normalized,
report_progress,
)?;
if let Some(artifact) = artifact_store
.load(&canonical, ArtifactContext::LiveGet)
.map_err(|error| error.to_string())?
{
report_stage(
&mut flow,
RuntimeStage::Transforming,
&artifact.final_url,
report_progress,
)?;
let transform_started = Instant::now();
let document = document_from_index_artifact(cache, &artifact).inspect_err(|_error| {
let _ = report_stage(
&mut flow,
RuntimeStage::Failed,
&normalized,
report_progress,
);
})?;
enforce_stage_budget(
RuntimeStage::Transforming,
transform_started.elapsed(),
&artifact.final_url,
)?;
report_stage(
&mut flow,
RuntimeStage::Scoring,
&artifact.final_url,
report_progress,
)?;
let log = response_log_entry(
sequence,
"ARTIFACT",
&artifact.canonical_url,
&artifact.final_url,
Some("index/artifact+v1"),
&artifact.capture.redacted_html,
);
report_stage(
&mut flow,
RuntimeStage::Done,
&artifact.final_url,
report_progress,
)?;
let freshness = artifact.freshness(unix_timestamp_secs());
if matches!(freshness, ArtifactFreshness::Stale) {
let refresh_fetcher = fetcher.clone();
let refresh_store = artifact_store.clone();
let refresh_canonical = canonical.clone();
std::thread::spawn(move || {
let _ =
refresh_live_get_artifact(&refresh_fetcher, &refresh_store, &refresh_canonical);
});
}
return Ok(TuiDocumentResult::new(document)
.with_visited_url(artifact.final_url)
.with_response_log(log));
}
report_stage(
&mut flow,
RuntimeStage::Fetching,
&normalized,
report_progress,
)?;
let fetch_started = Instant::now();
let response = fetcher
.fetch(&Request {
url: canonical.clone(),
})
.map_err(|error| {
let _ = report_stage(
&mut flow,
RuntimeStage::Failed,
&normalized,
report_progress,
);
error.to_string()
})?;
enforce_stage_budget(
RuntimeStage::Fetching,
fetch_started.elapsed(),
response.final_url.as_str(),
)?;
report_stage(
&mut flow,
RuntimeStage::Snapshotting,
response.final_url.as_str(),
report_progress,
)?;
let snapshot_started = Instant::now();
let html = html_with_base(&response.final_url, &response.body);
let manifest = load_index_manifest_for_response(fetcher, &response);
enforce_stage_budget(
RuntimeStage::Snapshotting,
snapshot_started.elapsed(),
response.final_url.as_str(),
)?;
report_stage(
&mut flow,
RuntimeStage::Parsing,
response.final_url.as_str(),
report_progress,
)?;
let parse_started = Instant::now();
enforce_stage_budget(
RuntimeStage::Parsing,
parse_started.elapsed(),
response.final_url.as_str(),
)?;
report_stage(
&mut flow,
RuntimeStage::Transforming,
response.final_url.as_str(),
report_progress,
)?;
let transform_started = Instant::now();
let transformed = transform_html_cached(cache, Some(response.final_url.as_str()), html);
let mut document = hydrate_document_images(transformed);
if let Some(manifest) = manifest.as_ref() {
apply_index_manifest_hints(&mut document, manifest);
}
apply_runtime_pack_manifest_hints(&mut document, response.final_url.as_str());
enforce_stage_budget(
RuntimeStage::Transforming,
transform_started.elapsed(),
response.final_url.as_str(),
)?;
report_stage(
&mut flow,
RuntimeStage::Scoring,
response.final_url.as_str(),
report_progress,
)?;
let log = response_log_entry(
sequence,
"GET",
&normalized,
response.final_url.as_str(),
response.mime_type.as_deref(),
&response.body,
);
report_stage(
&mut flow,
RuntimeStage::Storing,
response.final_url.as_str(),
report_progress,
)?;
let store_started = Instant::now();
if let Ok(artifact) = IndexArtifact::from_document(
&document,
&canonical,
&response.final_url,
ArtifactContext::LiveGet,
unix_timestamp_secs(),
ARTIFACT_MAX_AGE_SECS,
) {
let _ = artifact_store.store(&artifact);
}
enforce_stage_budget(
RuntimeStage::Storing,
store_started.elapsed(),
response.final_url.as_str(),
)?;
report_stage(
&mut flow,
RuntimeStage::Done,
response.final_url.as_str(),
report_progress,
)?;
Ok(TuiDocumentResult::new(document)
.with_visited_url(response.final_url.as_str())
.with_response_log(log))
}
#[cfg(test)]
fn fetch_document_cached_with_log_with_counter_with_progress<F: Fetcher>(
fetcher: &F,
cache: &mut TransformedDocumentCache,
sequence: &mut u64,
target: &str,
report_progress: &mut dyn FnMut(String),
) -> Result<TuiDocumentResult, String> {
let normalized = normalize_url_input(target);
report_progress(format!("fetching {normalized}"));
let url = IndexUrl::parse(&normalized).map_err(|error| error.to_string())?;
let response = fetcher
.fetch(&Request { url })
.map_err(|error| error.to_string())?;
report_progress(format!("parsing {}", response.final_url.as_str()));
let html = html_with_base(&response.final_url, &response.body);
report_progress(format!("transforming {}", response.final_url.as_str()));
let transformed = transform_html_cached(cache, Some(response.final_url.as_str()), html);
report_progress(format!("rendering {}", response.final_url.as_str()));
let mut document = hydrate_document_images(transformed);
apply_runtime_pack_manifest_hints(&mut document, response.final_url.as_str());
let log = response_log_entry(
sequence,
"GET",
&normalized,
response.final_url.as_str(),
response.mime_type.as_deref(),
&response.body,
);
Ok(TuiDocumentResult::new(document)
.with_visited_url(response.final_url.as_str())
.with_response_log(log))
}
#[cfg(test)]
fn submit_form_cached<F: FormSubmitter>(
submitter: &F,
cache: &mut TransformedDocumentCache,
submission: &FormSubmission,
) -> Result<IndexDocument, String> {
let response = submitter
.submit_form(submission)
.map_err(|error| error.to_string())?;
let html = html_with_base(&response.final_url, &response.body);
let mut document = transform_html_cached(cache, Some(response.final_url.as_str()), html);
document = hydrate_document_images(document);
apply_runtime_pack_manifest_hints(&mut document, response.final_url.as_str());
Ok(document)
}
#[cfg(test)]
fn submit_form_cached_with_log<F: FormSubmitter>(
submitter: &F,
cache: &mut TransformedDocumentCache,
sequence: &RefCell<u64>,
submission: &FormSubmission,
) -> Result<TuiDocumentResult, String> {
let mut next_sequence = sequence.borrow_mut();
submit_form_cached_with_log_with_counter(submitter, cache, &mut next_sequence, submission)
}
#[cfg(test)]
fn submit_form_cached_with_log_with_counter<F: FormSubmitter>(
submitter: &F,
cache: &mut TransformedDocumentCache,
sequence: &mut u64,
submission: &FormSubmission,
) -> Result<TuiDocumentResult, String> {
let mut ignore_progress = |_message: String| {};
submit_form_cached_with_log_with_counter_with_progress(
submitter,
cache,
sequence,
submission,
&mut ignore_progress,
)
}
fn submit_form_with_artifact_runtime<F: FormSubmitter + Fetcher>(
submitter: &F,
cache: &mut TransformedDocumentCache,
sequence: &mut u64,
artifact_store: &ArtifactStore,
submission: &FormSubmission,
report_progress: &mut dyn FnMut(String),
) -> Result<TuiDocumentResult, String> {
let target = submission.action.as_str().to_owned();
let mut flow = StageFlow::default();
report_stage(&mut flow, RuntimeStage::Queued, &target, report_progress)?;
report_stage(&mut flow, RuntimeStage::Fetching, &target, report_progress)?;
let submit_started = Instant::now();
let response = submitter.submit_form(submission).map_err(|error| {
let _ = report_stage(&mut flow, RuntimeStage::Failed, &target, report_progress);
error.to_string()
})?;
enforce_stage_budget(
RuntimeStage::Fetching,
submit_started.elapsed(),
response.final_url.as_str(),
)?;
report_stage(
&mut flow,
RuntimeStage::Snapshotting,
response.final_url.as_str(),
report_progress,
)?;
let snapshot_started = Instant::now();
let html = html_with_base(&response.final_url, &response.body);
let manifest = load_index_manifest_for_response(submitter, &response);
enforce_stage_budget(
RuntimeStage::Snapshotting,
snapshot_started.elapsed(),
response.final_url.as_str(),
)?;
report_stage(
&mut flow,
RuntimeStage::Parsing,
response.final_url.as_str(),
report_progress,
)?;
let parse_started = Instant::now();
enforce_stage_budget(
RuntimeStage::Parsing,
parse_started.elapsed(),
response.final_url.as_str(),
)?;
report_stage(
&mut flow,
RuntimeStage::Transforming,
response.final_url.as_str(),
report_progress,
)?;
let transform_started = Instant::now();
let transformed = transform_html_cached(cache, Some(response.final_url.as_str()), html);
let mut document = hydrate_document_images(transformed);
if let Some(manifest) = manifest.as_ref() {
apply_index_manifest_hints(&mut document, manifest);
}
apply_runtime_pack_manifest_hints(&mut document, response.final_url.as_str());
enforce_stage_budget(
RuntimeStage::Transforming,
transform_started.elapsed(),
response.final_url.as_str(),
)?;
report_stage(
&mut flow,
RuntimeStage::Scoring,
response.final_url.as_str(),
report_progress,
)?;
let log = response_log_entry(
sequence,
submission.method.as_str(),
submission.action.as_str(),
response.final_url.as_str(),
response.mime_type.as_deref(),
&response.body,
);
report_stage(
&mut flow,
RuntimeStage::Storing,
response.final_url.as_str(),
report_progress,
)?;
let store_started = Instant::now();
let canonical = submission.action.clone();
if let Ok(artifact) = IndexArtifact::from_document(
&document,
&canonical,
&response.final_url,
ArtifactContext::LiveSubmit,
unix_timestamp_secs(),
ARTIFACT_MAX_AGE_SECS,
) {
let _ = artifact_store.store(&artifact);
}
enforce_stage_budget(
RuntimeStage::Storing,
store_started.elapsed(),
response.final_url.as_str(),
)?;
report_stage(
&mut flow,
RuntimeStage::Done,
response.final_url.as_str(),
report_progress,
)?;
Ok(TuiDocumentResult::new(document)
.with_visited_url(response.final_url.as_str())
.with_response_log(log))
}
#[cfg(test)]
fn submit_form_cached_with_log_with_counter_with_progress<F: FormSubmitter>(
submitter: &F,
cache: &mut TransformedDocumentCache,
sequence: &mut u64,
submission: &FormSubmission,
report_progress: &mut dyn FnMut(String),
) -> Result<TuiDocumentResult, String> {
report_progress(format!(
"submitting {} {}",
submission.method.as_str(),
submission.action
));
let response = submitter
.submit_form(submission)
.map_err(|error| error.to_string())?;
report_progress(format!("parsing {}", response.final_url.as_str()));
let html = html_with_base(&response.final_url, &response.body);
report_progress(format!("transforming {}", response.final_url.as_str()));
let transformed = transform_html_cached(cache, Some(response.final_url.as_str()), html);
report_progress(format!("rendering {}", response.final_url.as_str()));
let mut document = hydrate_document_images(transformed);
apply_runtime_pack_manifest_hints(&mut document, response.final_url.as_str());
let log = response_log_entry(
sequence,
submission.method.as_str(),
submission.action.as_str(),
response.final_url.as_str(),
response.mime_type.as_deref(),
&response.body,
);
Ok(TuiDocumentResult::new(document)
.with_visited_url(response.final_url.as_str())
.with_response_log(log))
}
fn document_from_index_artifact(
cache: &mut TransformedDocumentCache,
artifact: &IndexArtifact,
) -> Result<IndexDocument, String> {
let final_url = IndexUrl::parse(artifact.final_url.replace("[REDACTED]", "redacted"))
.map_err(|error| error.to_string())?;
let transformed = transform_html_cached(
cache,
Some(final_url.as_str()),
artifact.capture.redacted_html.clone(),
);
let mut document = hydrate_document_images(transformed);
apply_runtime_pack_manifest_hints(&mut document, final_url.as_str());
Ok(document)
}
fn document_from_index_artifact_uncached(
artifact: &IndexArtifact,
) -> Result<IndexDocument, String> {
let final_url = IndexUrl::parse(artifact.final_url.replace("[REDACTED]", "redacted"))
.map_err(|error| error.to_string())?;
let mut document = transform_html(html_with_base(&final_url, &artifact.capture.redacted_html));
document = hydrate_document_images(document);
apply_runtime_pack_manifest_hints(&mut document, final_url.as_str());
Ok(document)
}
fn refresh_live_get_artifact<F: Fetcher>(
fetcher: &F,
artifact_store: &ArtifactStore,
canonical: &IndexUrl,
) -> Result<(), String> {
let response = fetcher
.fetch(&Request {
url: canonical.clone(),
})
.map_err(|error| error.to_string())?;
let mut document = hydrate_document_images(transform_html(html_with_base(
&response.final_url,
&response.body,
)));
apply_runtime_pack_manifest_hints(&mut document, response.final_url.as_str());
let artifact = IndexArtifact::from_document(
&document,
canonical,
&response.final_url,
ArtifactContext::LiveGet,
unix_timestamp_secs(),
ARTIFACT_MAX_AGE_SECS,
)
.map_err(|error| error.to_string())?;
artifact_store
.store(&artifact)
.map_err(|error| error.to_string())?;
Ok(())
}
fn response_log_entry(
sequence: &mut u64,
method: &str,
requested_url: &str,
final_url: &str,
mime_type: Option<&str>,
body: &str,
) -> ResponseLogEntry {
*sequence = sequence.saturating_add(1);
ResponseLogEntry::new(
*sequence,
method,
requested_url,
final_url,
mime_type,
body,
512,
)
}
fn transform_html(html: String) -> IndexDocument {
Transformer::<Empty>::new()
.fetched(html)
.parse()
.extract()
.transform()
.into_document()
}
fn hydrate_document_images(document: IndexDocument) -> IndexDocument {
let agent: ureq::Agent = ureq::Agent::config_builder()
.https_only(false)
.timeout_global(Some(IMAGE_PREVIEW_TIMEOUT))
.user_agent("Index/0.1.0")
.build()
.into();
let mut loader = |src: &str| fetch_image_bytes(&agent, src);
hydrate_document_images_with_loader(document, &mut loader)
}
fn hydrate_document_images_with_loader<L>(
mut document: IndexDocument,
loader: &mut L,
) -> IndexDocument
where
L: FnMut(&str) -> Result<Vec<u8>, String>,
{
let mut budget = IMAGE_PREVIEW_MAX_IMAGES_PER_DOCUMENT;
document.nodes = hydrate_image_nodes(mem::take(&mut document.nodes), loader, &mut budget);
document
}
fn hydrate_image_nodes<L>(
nodes: Vec<IndexNode>,
loader: &mut L,
budget: &mut usize,
) -> Vec<IndexNode>
where
L: FnMut(&str) -> Result<Vec<u8>, String>,
{
let mut output = Vec::with_capacity(nodes.len());
for node in nodes {
match node {
IndexNode::Section {
role,
title,
collapsed,
nodes,
} => output.push(IndexNode::Section {
role,
title,
collapsed,
nodes: hydrate_image_nodes(nodes, loader, budget),
}),
IndexNode::Image { alt, src } => {
if *budget > 0 {
if let Some(src_url) = src.as_deref() {
*budget = budget.saturating_sub(1);
if let Some(preview_nodes) =
hydrate_image_node_preview(&alt, src_url, loader)
{
output.extend(preview_nodes);
continue;
}
}
}
output.push(IndexNode::Image { alt, src });
}
other => output.push(other),
}
}
output
}
fn hydrate_image_node_preview<L>(alt: &str, src: &str, loader: &mut L) -> Option<Vec<IndexNode>>
where
L: FnMut(&str) -> Result<Vec<u8>, String>,
{
let url = IndexUrl::parse(src).ok()?;
let bytes = loader(url.as_str()).ok()?;
let preview = dither_image_to_terminal_blocks(&bytes).ok()?;
if preview.trim().is_empty() {
return None;
}
let label = if alt.trim().is_empty() {
"image preview".to_owned()
} else {
format!("image preview: {alt}")
};
Some(vec![
IndexNode::Paragraph(format!("󰥶 {label}")),
IndexNode::CodeBlock {
language: Some("bw-dither".to_owned()),
code: preview,
},
IndexNode::Link(Link::new("image source", url.as_str())),
])
}
fn fetch_image_bytes(agent: &ureq::Agent, src: &str) -> Result<Vec<u8>, String> {
let mut response = agent.get(src).call().map_err(|error| error.to_string())?;
let bytes = response
.body_mut()
.with_config()
.limit(IMAGE_PREVIEW_MAX_FETCH_BYTES as u64)
.read_to_vec()
.map_err(|error| error.to_string())?;
if bytes.is_empty() {
return Err("empty image response".to_owned());
}
Ok(bytes)
}
fn dither_image_to_terminal_blocks(bytes: &[u8]) -> Result<String, String> {
let image = image::load_from_memory(bytes).map_err(|error| error.to_string())?;
let grayscale = image.to_luma8();
let (width, height) = grayscale.dimensions();
if width == 0 || height == 0 {
return Err("image has zero dimensions".to_owned());
}
let (resized_width, resized_height) = image_preview_dimensions(width, height);
let mut resized = resize(
&grayscale,
resized_width.max(1),
resized_height.max(1),
FilterType::Triangle,
);
dither(&mut resized, &BiLevel);
Ok(half_block_render(&resized))
}
fn image_preview_dimensions(width: u32, height: u32) -> (u32, u32) {
let width_scale = width as f32 / IMAGE_PREVIEW_MAX_WIDTH as f32;
let height_scale = height as f32 / IMAGE_PREVIEW_MAX_HEIGHT as f32;
let scale = width_scale.max(height_scale).max(1.0);
let scaled_width = ((width as f32 / scale).round() as u32).clamp(1, IMAGE_PREVIEW_MAX_WIDTH);
let scaled_height = ((height as f32 / scale).round() as u32).clamp(1, IMAGE_PREVIEW_MAX_HEIGHT);
(scaled_width, scaled_height)
}
fn half_block_render(image: &image::GrayImage) -> String {
let width = image.width() as usize;
let height = image.height() as usize;
let mut lines = Vec::new();
let mut row = 0usize;
while row < height {
let mut line = String::with_capacity(width);
for column in 0..width {
let top = image.get_pixel(column as u32, row as u32)[0] == 0;
let bottom = if row + 1 < height {
image.get_pixel(column as u32, (row + 1) as u32)[0] == 0
} else {
false
};
let symbol = match (top, bottom) {
(true, true) => 'â–ˆ',
(true, false) => 'â–€',
(false, true) => 'â–„',
(false, false) => ' ',
};
line.push(symbol);
}
lines.push(line.trim_end_matches(' ').to_owned());
row += 2;
}
while matches!(lines.last(), Some(last) if last.is_empty()) {
let _ = lines.pop();
}
lines.join("\n")
}
fn run_save_command<F: Fetcher>(
format: ExtractFormat,
input: &str,
output_path: &str,
stdin: &mut dyn Read,
fetcher: &F,
) -> Result<CliAction, String> {
let document = load_document(input, stdin, fetcher)?;
let output = try_extract_document(&document, format, ExtractionLimits::default())
.map_err(|error| error.to_string())?;
fs::write(output_path, output).map_err(|error| error.to_string())?;
Ok(CliAction::Print(format!(
"saved\t{}\t{}",
format.as_str(),
output_path
)))
}
fn run_batch_extract_command(
format: ExtractFormat,
inputs: &[String],
) -> Result<CliAction, String> {
let mut output = String::new();
for input in inputs {
let document = load_offline_document(input)?;
output.push_str("==> ");
output.push_str(input);
output.push_str(" <==\n");
output.push_str(
&try_extract_document(&document, format, ExtractionLimits::default())
.map_err(|error| error.to_string())?,
);
if !output.ends_with('\n') {
output.push('\n');
}
}
Ok(CliAction::Print(output))
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum CompatibilityCorpus {
Top100,
Forum,
}
impl CompatibilityCorpus {
const fn as_str(self) -> &'static str {
match self {
Self::Top100 => "top100",
Self::Forum => "forum",
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum CoverageExpectedPath {
Generic,
Adapter,
Failure,
}
impl CoverageExpectedPath {
fn parse(value: &str, context: &str) -> Result<Self, String> {
match value {
"generic" => Ok(Self::Generic),
"adapter" => Ok(Self::Adapter),
"failure" => Ok(Self::Failure),
_ => Err(format!("invalid expected path for {context}: {value}")),
}
}
const fn as_str(self) -> &'static str {
match self {
Self::Generic => "generic",
Self::Adapter => "adapter",
Self::Failure => "failure",
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum CoverageStatus {
Covered,
Partial,
Blocked,
Planned,
}
impl CoverageStatus {
fn parse(value: &str, context: &str) -> Result<Self, String> {
match value {
"covered" => Ok(Self::Covered),
"partial" => Ok(Self::Partial),
"blocked" => Ok(Self::Blocked),
"planned" => Ok(Self::Planned),
_ => Err(format!("invalid coverage status for {context}: {value}")),
}
}
const fn is_failure_guardrail_status(self) -> bool {
matches!(self, Self::Blocked | Self::Planned)
}
const fn as_str(self) -> &'static str {
match self {
Self::Covered => "covered",
Self::Partial => "partial",
Self::Blocked => "blocked",
Self::Planned => "planned",
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
struct CompatibilitySloRow {
corpus: CompatibilityCorpus,
domain: String,
family: String,
intent: Option<String>,
min_tier: u8,
current_tier: u8,
expected_path: CoverageExpectedPath,
status: CoverageStatus,
known_limit: String,
}
#[derive(Debug, Clone, Copy, PartialEq)]
struct CompatibilitySloThresholds {
readability_floor: f64,
actionability_floor: f64,
failure_quality_floor: f64,
}
impl Default for CompatibilitySloThresholds {
fn default() -> Self {
Self {
readability_floor: COMPATIBILITY_SLO_READABILITY_FLOOR,
actionability_floor: COMPATIBILITY_SLO_ACTIONABILITY_FLOOR,
failure_quality_floor: COMPATIBILITY_SLO_FAILURE_QUALITY_FLOOR,
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
struct CorpusCompatibilityScore {
corpus: CompatibilityCorpus,
total_rows: usize,
eligible_rows: usize,
readable_rows: usize,
actionable_rows: usize,
failure_rows: usize,
guarded_failure_rows: usize,
guardrail_failures: Vec<String>,
}
#[derive(Debug, Clone, PartialEq)]
struct CompatibilitySloReport {
top100_path: String,
forum_path: String,
thresholds: CompatibilitySloThresholds,
top100: CorpusCompatibilityScore,
forum: CorpusCompatibilityScore,
readability_score: f64,
actionability_score: f64,
failure_quality_score: f64,
violations: Vec<String>,
}
impl CompatibilitySloReport {
fn passed(&self) -> bool {
self.violations.is_empty()
}
fn to_text(&self) -> String {
let status = if self.passed() { "pass" } else { "fail" };
let mut output = vec![
"index-compatibility-slo-v1".to_owned(),
format!("top100_matrix: {}", self.top100_path),
format!("forum_matrix: {}", self.forum_path),
format!(
"thresholds: readability>={:.4} actionability>={:.4} failure_quality>={:.4}",
self.thresholds.readability_floor,
self.thresholds.actionability_floor,
self.thresholds.failure_quality_floor
),
format!(
"scores: readability={:.4} ({}/{}) actionability={:.4} ({}/{}) failure_quality={:.4} ({}/{})",
self.readability_score,
self.top100.readable_rows + self.forum.readable_rows,
self.top100.eligible_rows + self.forum.eligible_rows,
self.actionability_score,
self.top100.actionable_rows + self.forum.actionable_rows,
self.top100.eligible_rows + self.forum.eligible_rows,
self.failure_quality_score,
self.top100.guarded_failure_rows + self.forum.guarded_failure_rows,
self.top100.failure_rows + self.forum.failure_rows,
),
format!(
"corpus.top100: rows={} eligible={} readable={} actionable={} failures={} guarded_failures={}",
self.top100.total_rows,
self.top100.eligible_rows,
self.top100.readable_rows,
self.top100.actionable_rows,
self.top100.failure_rows,
self.top100.guarded_failure_rows,
),
format!(
"corpus.forum: rows={} eligible={} readable={} actionable={} failures={} guarded_failures={}",
self.forum.total_rows,
self.forum.eligible_rows,
self.forum.readable_rows,
self.forum.actionable_rows,
self.forum.failure_rows,
self.forum.guarded_failure_rows,
),
];
if self.violations.is_empty() {
output.push("violations: none".to_owned());
} else {
output.push("violations:".to_owned());
for violation in &self.violations {
output.push(format!("- {violation}"));
}
}
output.push(format!("result: {status}"));
output.join("\n")
}
}
#[derive(Debug, Clone, PartialEq)]
struct FamilyCompatibilityScore {
family: String,
rows_total: usize,
eligible_rows: usize,
readable_rows: usize,
actionable_rows: usize,
failure_rows: usize,
guarded_failure_rows: usize,
readability_score: f64,
actionability_score: f64,
failure_quality_score: f64,
thresholds: CompatibilitySloThresholds,
violations: Vec<String>,
requires_non_failure_evidence: bool,
}
#[derive(Debug, Clone, PartialEq)]
struct CompatibilitySloFamilyDelta {
family: String,
readability_delta: f64,
actionability_delta: f64,
failure_quality_delta: f64,
}
#[derive(Debug, Clone, PartialEq)]
struct CompatibilitySloDeltaReport {
readability_delta: f64,
actionability_delta: f64,
failure_quality_delta: f64,
families: Vec<CompatibilitySloFamilyDelta>,
}
#[derive(Debug, Clone, PartialEq)]
struct CompatibilitySloV2Report {
top100_path: String,
forum_path: String,
baseline_top100_path: Option<String>,
baseline_forum_path: Option<String>,
global: CompatibilitySloReport,
families: Vec<FamilyCompatibilityScore>,
delta: Option<CompatibilitySloDeltaReport>,
violations: Vec<String>,
evidence_warnings: Vec<String>,
planning_inputs: Vec<String>,
}
impl CompatibilitySloV2Report {
fn passed(&self) -> bool {
self.violations.is_empty()
}
fn to_text(&self) -> String {
let status = if self.passed() { "pass" } else { "fail" };
let mut output = vec![
"index-compatibility-slo-v2".to_owned(),
format!("top100_matrix: {}", self.top100_path),
format!("forum_matrix: {}", self.forum_path),
];
if let (Some(top100), Some(forum)) = (
self.baseline_top100_path.as_deref(),
self.baseline_forum_path.as_deref(),
) {
output.push(format!("baseline_top100_matrix: {top100}"));
output.push(format!("baseline_forum_matrix: {forum}"));
}
output.push(format!(
"global_scores: readability={:.4} actionability={:.4} failure_quality={:.4}",
self.global.readability_score,
self.global.actionability_score,
self.global.failure_quality_score
));
output.push(format!("family_rows: {}", self.families.len()));
output.push("family_columns: family\treadability\treadability_floor\tactionability\tactionability_floor\tfailure_quality\tfailure_quality_floor\teligible_rows\tfailure_rows\tresult".to_owned());
for family in &self.families {
output.push(format!(
"{}\t{:.4}\t{:.4}\t{:.4}\t{:.4}\t{:.4}\t{:.4}\t{}\t{}\t{}",
family.family,
family.readability_score,
family.thresholds.readability_floor,
family.actionability_score,
family.thresholds.actionability_floor,
family.failure_quality_score,
family.thresholds.failure_quality_floor,
family.eligible_rows,
family.failure_rows,
if family.violations.is_empty() {
"pass"
} else {
"fail"
},
));
}
if let Some(delta) = &self.delta {
output.push(format!(
"delta_global: readability={:+.4} actionability={:+.4} failure_quality={:+.4}",
delta.readability_delta, delta.actionability_delta, delta.failure_quality_delta
));
output.push("delta_family_columns: family\treadability_delta\tactionability_delta\tfailure_quality_delta".to_owned());
for family in &delta.families {
output.push(format!(
"{}\t{:+.4}\t{:+.4}\t{:+.4}",
family.family,
family.readability_delta,
family.actionability_delta,
family.failure_quality_delta
));
}
}
if self.violations.is_empty() {
output.push("violations: none".to_owned());
} else {
output.push("violations:".to_owned());
for violation in &self.violations {
output.push(format!("- {violation}"));
}
}
if self.evidence_warnings.is_empty() {
output.push("evidence_warnings: none".to_owned());
} else {
output.push("evidence_warnings:".to_owned());
for warning in &self.evidence_warnings {
output.push(format!("- {warning}"));
}
}
if self.planning_inputs.is_empty() {
output.push("planning_inputs: none".to_owned());
} else {
output.push("planning_inputs:".to_owned());
for input in &self.planning_inputs {
output.push(format!("- {input}"));
}
}
output.push(format!("result: {status}"));
output.join("\n")
}
}
fn run_compatibility_slo_command<I, S>(mut args: I) -> Result<CliAction, String>
where
I: Iterator<Item = S>,
S: AsRef<str>,
{
let (mut top100_path, mut forum_path) = default_compatibility_slo_paths();
let mut thresholds = CompatibilitySloThresholds::default();
while let Some(argument) = args.next() {
match argument.as_ref() {
"--top100" => {
let Some(path) = args.next() else {
return Ok(CliAction::Help(help()));
};
top100_path = path.as_ref().to_owned();
}
"--forum" => {
let Some(path) = args.next() else {
return Ok(CliAction::Help(help()));
};
forum_path = path.as_ref().to_owned();
}
"--min-readability" => {
let Some(value) = args.next() else {
return Ok(CliAction::Help(help()));
};
thresholds.readability_floor = parse_slo_floor("readability", value.as_ref())?;
}
"--min-actionability" => {
let Some(value) = args.next() else {
return Ok(CliAction::Help(help()));
};
thresholds.actionability_floor = parse_slo_floor("actionability", value.as_ref())?;
}
"--min-failure-quality" => {
let Some(value) = args.next() else {
return Ok(CliAction::Help(help()));
};
thresholds.failure_quality_floor =
parse_slo_floor("failure-quality", value.as_ref())?;
}
other => return Err(format!("unsupported compatibility-slo option: {other}")),
}
}
let report = compatibility_slo_report_from_paths(&top100_path, &forum_path, thresholds)?;
if report.passed() {
Ok(CliAction::Print(report.to_text()))
} else {
Err(report.to_text())
}
}
fn run_compatibility_slo_v2_command<I, S>(mut args: I) -> Result<CliAction, String>
where
I: Iterator<Item = S>,
S: AsRef<str>,
{
let (mut top100_path, mut forum_path) = default_compatibility_slo_paths();
let mut thresholds = CompatibilitySloThresholds::default();
let mut baseline_top100_path: Option<String> = None;
let mut baseline_forum_path: Option<String> = None;
while let Some(argument) = args.next() {
match argument.as_ref() {
"--top100" => {
let Some(path) = args.next() else {
return Ok(CliAction::Help(help()));
};
top100_path = path.as_ref().to_owned();
}
"--forum" => {
let Some(path) = args.next() else {
return Ok(CliAction::Help(help()));
};
forum_path = path.as_ref().to_owned();
}
"--baseline-top100" => {
let Some(path) = args.next() else {
return Ok(CliAction::Help(help()));
};
baseline_top100_path = Some(path.as_ref().to_owned());
}
"--baseline-forum" => {
let Some(path) = args.next() else {
return Ok(CliAction::Help(help()));
};
baseline_forum_path = Some(path.as_ref().to_owned());
}
"--min-readability" => {
let Some(value) = args.next() else {
return Ok(CliAction::Help(help()));
};
thresholds.readability_floor = parse_slo_floor("readability", value.as_ref())?;
}
"--min-actionability" => {
let Some(value) = args.next() else {
return Ok(CliAction::Help(help()));
};
thresholds.actionability_floor = parse_slo_floor("actionability", value.as_ref())?;
}
"--min-failure-quality" => {
let Some(value) = args.next() else {
return Ok(CliAction::Help(help()));
};
thresholds.failure_quality_floor =
parse_slo_floor("failure-quality", value.as_ref())?;
}
other => return Err(format!("unsupported compatibility-slo-v2 option: {other}")),
}
}
if baseline_top100_path.is_some() != baseline_forum_path.is_some() {
return Err(
"compatibility-slo-v2 requires both --baseline-top100 and --baseline-forum".to_owned(),
);
}
let report = compatibility_slo_v2_report_from_paths(
&top100_path,
&forum_path,
thresholds,
baseline_top100_path.as_deref(),
baseline_forum_path.as_deref(),
)?;
if report.passed() {
Ok(CliAction::Print(report.to_text()))
} else {
Err(report.to_text())
}
}
fn default_compatibility_slo_paths() -> (String, String) {
let root = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../..");
let top100 = root.join(COMPATIBILITY_SLO_TOP100_PATH);
let forum = root.join(COMPATIBILITY_SLO_FORUM_PATH);
(top100.display().to_string(), forum.display().to_string())
}
fn parse_slo_floor(name: &str, value: &str) -> Result<f64, String> {
let parsed = value
.parse::<f64>()
.map_err(|error| format!("invalid {name} floor value {value}: {error}"))?;
if !(0.0..=1.0).contains(&parsed) {
return Err(format!(
"invalid {name} floor value {value}: must be between 0.0 and 1.0"
));
}
Ok(parsed)
}
fn compatibility_slo_report_from_paths(
top100_path: &str,
forum_path: &str,
thresholds: CompatibilitySloThresholds,
) -> Result<CompatibilitySloReport, String> {
let top100_text = fs::read_to_string(top100_path)
.map_err(|error| format!("failed to read {top100_path}: {error}"))?;
let forum_text = fs::read_to_string(forum_path)
.map_err(|error| format!("failed to read {forum_path}: {error}"))?;
let top100_rows = parse_top100_slo_rows(&top100_text)?;
let forum_rows = parse_forum_slo_rows(&forum_text)?;
Ok(evaluate_compatibility_slo(
top100_path,
forum_path,
&top100_rows,
&forum_rows,
thresholds,
))
}
fn compatibility_slo_v2_report_from_paths(
top100_path: &str,
forum_path: &str,
thresholds: CompatibilitySloThresholds,
baseline_top100_path: Option<&str>,
baseline_forum_path: Option<&str>,
) -> Result<CompatibilitySloV2Report, String> {
let top100_text = fs::read_to_string(top100_path)
.map_err(|error| format!("failed to read {top100_path}: {error}"))?;
let forum_text = fs::read_to_string(forum_path)
.map_err(|error| format!("failed to read {forum_path}: {error}"))?;
let top100_rows = parse_top100_slo_rows(&top100_text)?;
let forum_rows = parse_forum_slo_rows(&forum_text)?;
let baseline_rows = if let (Some(baseline_top100_path), Some(baseline_forum_path)) =
(baseline_top100_path, baseline_forum_path)
{
let top100_baseline_text = fs::read_to_string(baseline_top100_path)
.map_err(|error| format!("failed to read {baseline_top100_path}: {error}"))?;
let forum_baseline_text = fs::read_to_string(baseline_forum_path)
.map_err(|error| format!("failed to read {baseline_forum_path}: {error}"))?;
Some((
parse_top100_slo_rows(&top100_baseline_text)?,
parse_forum_slo_rows(&forum_baseline_text)?,
))
} else {
None
};
let baseline = match (
baseline_top100_path,
baseline_forum_path,
baseline_rows.as_ref(),
) {
(Some(top100_path), Some(forum_path), Some((top100_rows, forum_rows))) => {
Some(CompatibilitySloV2Baseline {
top100_path,
forum_path,
top100_rows,
forum_rows,
})
}
_ => None,
};
Ok(evaluate_compatibility_slo_v2(
top100_path,
forum_path,
&top100_rows,
&forum_rows,
thresholds,
baseline,
))
}
#[derive(Debug, Clone, Copy)]
struct CompatibilitySloV2Baseline<'a> {
top100_path: &'a str,
forum_path: &'a str,
top100_rows: &'a [CompatibilitySloRow],
forum_rows: &'a [CompatibilitySloRow],
}
fn evaluate_compatibility_slo_v2(
top100_path: &str,
forum_path: &str,
top100_rows: &[CompatibilitySloRow],
forum_rows: &[CompatibilitySloRow],
thresholds: CompatibilitySloThresholds,
baseline: Option<CompatibilitySloV2Baseline<'_>>,
) -> CompatibilitySloV2Report {
let global =
evaluate_compatibility_slo(top100_path, forum_path, top100_rows, forum_rows, thresholds);
let families = score_compatibility_families(top100_rows, forum_rows);
let mut violations = global.violations.clone();
for family in &families {
for violation in &family.violations {
violations.push(format!("family {}: {violation}", family.family));
}
}
let delta = baseline.map(|baseline| {
compatibility_slo_delta(
top100_path,
forum_path,
top100_rows,
forum_rows,
baseline.top100_rows,
baseline.forum_rows,
thresholds,
)
});
if let Some(delta) = &delta {
if delta.readability_delta < 0.0 {
violations.push(format!(
"global readability regressed by {:.4}",
delta.readability_delta
));
}
if delta.actionability_delta < 0.0 {
violations.push(format!(
"global actionability regressed by {:.4}",
delta.actionability_delta
));
}
if delta.failure_quality_delta < 0.0 {
violations.push(format!(
"global failure-quality regressed by {:.4}",
delta.failure_quality_delta
));
}
}
let evidence_warnings = compatibility_slo_evidence_warnings(&families);
let planning_inputs =
compatibility_slo_planning_inputs(&families, delta.as_ref(), &evidence_warnings);
CompatibilitySloV2Report {
top100_path: top100_path.to_owned(),
forum_path: forum_path.to_owned(),
baseline_top100_path: baseline.map(|value| value.top100_path.to_owned()),
baseline_forum_path: baseline.map(|value| value.forum_path.to_owned()),
global,
families,
delta,
violations,
evidence_warnings,
planning_inputs,
}
}
fn compatibility_slo_evidence_warnings(families: &[FamilyCompatibilityScore]) -> Vec<String> {
families
.iter()
.filter(|family| {
family.rows_total > 0
&& family.eligible_rows == 0
&& family.requires_non_failure_evidence
})
.map(|family| {
format!(
"family {} has no known_limit=none rows (rows_total={}, failure_rows={}); add at least one non-limited fixture row",
family.family, family.rows_total, family.failure_rows
)
})
.collect()
}
fn canonical_family_name(family: &str) -> String {
match family.trim().to_ascii_lowercase().as_str() {
"reddit" | "generic-forum" => "social-community".to_owned(),
_ => family.trim().to_owned(),
}
}
fn score_compatibility_families(
top100_rows: &[CompatibilitySloRow],
forum_rows: &[CompatibilitySloRow],
) -> Vec<FamilyCompatibilityScore> {
use std::collections::BTreeSet;
let mut family_names = BTreeSet::new();
for row in top100_rows {
family_names.insert(canonical_family_name(&row.family));
}
for row in forum_rows {
family_names.insert(canonical_family_name(&row.family));
}
family_names
.into_iter()
.map(|family| {
let rows = top100_rows
.iter()
.chain(forum_rows.iter())
.filter(|row| canonical_family_name(&row.family) == family)
.collect::<Vec<_>>();
score_compatibility_family(&family, &rows)
})
.collect()
}
fn score_compatibility_family(
family: &str,
rows: &[&CompatibilitySloRow],
) -> FamilyCompatibilityScore {
let mut eligible_rows = 0usize;
let mut readable_rows = 0usize;
let mut actionable_rows = 0usize;
let mut failure_rows = 0usize;
let mut guarded_failure_rows = 0usize;
let mut guardrail_failures = Vec::new();
let mut requires_non_failure_evidence = false;
for row in rows {
if row.min_tier > 0 {
requires_non_failure_evidence = true;
}
if row.known_limit == "none" {
eligible_rows += 1;
if row.current_tier >= 1 {
readable_rows += 1;
}
if row.current_tier >= 2 {
actionable_rows += 1;
}
}
if row.expected_path == CoverageExpectedPath::Failure {
failure_rows += 1;
let guarded = row.known_limit != "none"
&& row.status.is_failure_guardrail_status()
&& row.current_tier <= 1
&& row.min_tier <= 1;
if guarded {
guarded_failure_rows += 1;
} else {
guardrail_failures.push(format!("{}:{}", row.corpus.as_str(), row.domain));
}
}
}
let thresholds = family_thresholds(family);
let readability_score = ratio(readable_rows, eligible_rows);
let actionability_score = ratio(actionable_rows, eligible_rows);
let failure_quality_score = ratio(guarded_failure_rows, failure_rows);
let mut violations = Vec::new();
if eligible_rows > 0 {
if readability_score < thresholds.readability_floor {
violations.push(format!(
"readability {:.4} below floor {:.4}",
readability_score, thresholds.readability_floor
));
}
if actionability_score < thresholds.actionability_floor {
violations.push(format!(
"actionability {:.4} below floor {:.4}",
actionability_score, thresholds.actionability_floor
));
}
}
if failure_rows > 0 && failure_quality_score < thresholds.failure_quality_floor {
violations.push(format!(
"failure-quality {:.4} below floor {:.4}",
failure_quality_score, thresholds.failure_quality_floor
));
}
if !guardrail_failures.is_empty() {
violations.push(format!(
"failure guardrail violations: {}",
guardrail_failures.join(", ")
));
}
FamilyCompatibilityScore {
family: family.to_owned(),
rows_total: rows.len(),
eligible_rows,
readable_rows,
actionable_rows,
failure_rows,
guarded_failure_rows,
readability_score,
actionability_score,
failure_quality_score,
thresholds,
violations,
requires_non_failure_evidence,
}
}
fn family_thresholds(family: &str) -> CompatibilitySloThresholds {
let normalized = family.trim().to_ascii_lowercase();
if normalized.contains("search")
|| normalized.contains("stackexchange")
|| normalized.contains("hacker-news")
|| normalized.contains("slashdot")
{
CompatibilitySloThresholds {
readability_floor: 0.35,
actionability_floor: 0.20,
failure_quality_floor: COMPATIBILITY_SLO_V2_DEFAULT_FAMILY_FAILURE_FLOOR,
}
} else if normalized.contains("commerce") || normalized.contains("marketplace") {
CompatibilitySloThresholds {
readability_floor: COMPATIBILITY_SLO_V2_DEFAULT_FAMILY_READABILITY_FLOOR,
actionability_floor: 0.05,
failure_quality_floor: COMPATIBILITY_SLO_V2_DEFAULT_FAMILY_FAILURE_FLOOR,
}
} else if normalized.contains("media") || normalized.contains("streaming") {
CompatibilitySloThresholds {
readability_floor: COMPATIBILITY_SLO_V2_DEFAULT_FAMILY_READABILITY_FLOOR,
actionability_floor: 0.14,
failure_quality_floor: COMPATIBILITY_SLO_V2_DEFAULT_FAMILY_FAILURE_FLOOR,
}
} else if normalized.contains("services") || normalized.contains("utility") {
CompatibilitySloThresholds {
readability_floor: COMPATIBILITY_SLO_V2_DEFAULT_FAMILY_READABILITY_FLOOR,
actionability_floor: 0.0,
failure_quality_floor: COMPATIBILITY_SLO_V2_DEFAULT_FAMILY_FAILURE_FLOOR,
}
} else if normalized.contains("knowledge") || normalized.contains("docs") {
CompatibilitySloThresholds {
readability_floor: 0.30,
actionability_floor: 0.14,
failure_quality_floor: COMPATIBILITY_SLO_V2_DEFAULT_FAMILY_FAILURE_FLOOR,
}
} else {
CompatibilitySloThresholds {
readability_floor: COMPATIBILITY_SLO_V2_DEFAULT_FAMILY_READABILITY_FLOOR,
actionability_floor: COMPATIBILITY_SLO_V2_DEFAULT_FAMILY_ACTIONABILITY_FLOOR,
failure_quality_floor: COMPATIBILITY_SLO_V2_DEFAULT_FAMILY_FAILURE_FLOOR,
}
}
}
fn compatibility_slo_delta(
top100_path: &str,
forum_path: &str,
top100_rows: &[CompatibilitySloRow],
forum_rows: &[CompatibilitySloRow],
baseline_top100_rows: &[CompatibilitySloRow],
baseline_forum_rows: &[CompatibilitySloRow],
thresholds: CompatibilitySloThresholds,
) -> CompatibilitySloDeltaReport {
use std::collections::BTreeSet;
let current =
evaluate_compatibility_slo(top100_path, forum_path, top100_rows, forum_rows, thresholds);
let baseline = evaluate_compatibility_slo(
top100_path,
forum_path,
baseline_top100_rows,
baseline_forum_rows,
thresholds,
);
let current_family_scores = score_compatibility_families(top100_rows, forum_rows);
let baseline_family_scores =
score_compatibility_families(baseline_top100_rows, baseline_forum_rows);
let mut families = BTreeSet::new();
for score in ¤t_family_scores {
families.insert(score.family.clone());
}
for score in &baseline_family_scores {
families.insert(score.family.clone());
}
let family_deltas = families
.into_iter()
.map(|family| {
let current_score = current_family_scores
.iter()
.find(|score| score.family == family);
let baseline_score = baseline_family_scores
.iter()
.find(|score| score.family == family);
CompatibilitySloFamilyDelta {
family,
readability_delta: current_score.map_or(0.0, |score| score.readability_score)
- baseline_score.map_or(0.0, |score| score.readability_score),
actionability_delta: current_score.map_or(0.0, |score| score.actionability_score)
- baseline_score.map_or(0.0, |score| score.actionability_score),
failure_quality_delta: current_score
.map_or(0.0, |score| score.failure_quality_score)
- baseline_score.map_or(0.0, |score| score.failure_quality_score),
}
})
.collect::<Vec<_>>();
CompatibilitySloDeltaReport {
readability_delta: current.readability_score - baseline.readability_score,
actionability_delta: current.actionability_score - baseline.actionability_score,
failure_quality_delta: current.failure_quality_score - baseline.failure_quality_score,
families: family_deltas,
}
}
fn compatibility_slo_planning_inputs(
families: &[FamilyCompatibilityScore],
delta: Option<&CompatibilitySloDeltaReport>,
evidence_warnings: &[String],
) -> Vec<String> {
let mut inputs = Vec::new();
for family in families {
if family
.violations
.iter()
.any(|violation| violation.starts_with("readability "))
{
inputs.push(format!("M74: readability gap for family {}", family.family));
}
if family
.violations
.iter()
.any(|violation| violation.starts_with("actionability "))
{
if family.family.contains("app-shell")
|| family.family.contains("commerce")
|| family.family.contains("media")
{
inputs.push(format!(
"M78: family-pack actionability gap for family {}",
family.family
));
} else {
inputs.push(format!(
"M75: actionability gap for family {}",
family.family
));
}
}
if family
.violations
.iter()
.any(|violation| violation.starts_with("failure-quality "))
|| family
.violations
.iter()
.any(|violation| violation.starts_with("failure guardrail violations"))
{
inputs.push(format!(
"M76: failure-quality gap for family {}",
family.family
));
}
}
if let Some(delta) = delta {
for family in &delta.families {
if family.readability_delta < 0.0 {
inputs.push(format!(
"M74: readability regression {:.4} for family {}",
family.readability_delta, family.family
));
}
if family.actionability_delta < 0.0 {
inputs.push(format!(
"M75: actionability regression {:.4} for family {}",
family.actionability_delta, family.family
));
}
if family.failure_quality_delta < 0.0 {
inputs.push(format!(
"M76: failure-quality regression {:.4} for family {}",
family.failure_quality_delta, family.family
));
}
}
}
for warning in evidence_warnings {
if let Some(family) = warning
.strip_prefix("family ")
.and_then(|rest| rest.split_once(' '))
.map(|(name, _)| name)
{
inputs.push(format!("M73: evidence-gap backlog for family {family}"));
} else {
inputs.push("M73: evidence-gap backlog".to_owned());
}
}
inputs.sort();
inputs.dedup();
inputs
}
fn evaluate_compatibility_slo(
top100_path: &str,
forum_path: &str,
top100_rows: &[CompatibilitySloRow],
forum_rows: &[CompatibilitySloRow],
thresholds: CompatibilitySloThresholds,
) -> CompatibilitySloReport {
let top100 = score_compatibility_corpus(CompatibilityCorpus::Top100, top100_rows);
let forum = score_compatibility_corpus(CompatibilityCorpus::Forum, forum_rows);
let readable_rows = top100.readable_rows + forum.readable_rows;
let actionable_rows = top100.actionable_rows + forum.actionable_rows;
let eligible_rows = top100.eligible_rows + forum.eligible_rows;
let guarded_failure_rows = top100.guarded_failure_rows + forum.guarded_failure_rows;
let failure_rows = top100.failure_rows + forum.failure_rows;
let readability_score = ratio(readable_rows, eligible_rows);
let actionability_score = ratio(actionable_rows, eligible_rows);
let failure_quality_score = ratio(guarded_failure_rows, failure_rows);
let mut violations = Vec::new();
if eligible_rows == 0 {
violations.push(
"eligible corpus rows are zero; readability/actionability are undefined".to_owned(),
);
}
if failure_rows == 0 {
violations.push(
"failure corpus rows are zero; failure-quality guardrail is undefined".to_owned(),
);
}
if readability_score < thresholds.readability_floor {
violations.push(format!(
"readability score {:.4} below floor {:.4}",
readability_score, thresholds.readability_floor
));
}
if actionability_score < thresholds.actionability_floor {
violations.push(format!(
"actionability score {:.4} below floor {:.4}",
actionability_score, thresholds.actionability_floor
));
}
if failure_quality_score < thresholds.failure_quality_floor {
violations.push(format!(
"failure-quality score {:.4} below floor {:.4}",
failure_quality_score, thresholds.failure_quality_floor
));
}
let mut guardrail_failures = top100.guardrail_failures.clone();
guardrail_failures.extend(forum.guardrail_failures.clone());
guardrail_failures.sort();
if !guardrail_failures.is_empty() {
violations.push(format!(
"failure guardrail violations in {} rows: {}",
guardrail_failures.len(),
guardrail_failures.join(", ")
));
}
CompatibilitySloReport {
top100_path: top100_path.to_owned(),
forum_path: forum_path.to_owned(),
thresholds,
top100,
forum,
readability_score,
actionability_score,
failure_quality_score,
violations,
}
}
fn score_compatibility_corpus(
corpus: CompatibilityCorpus,
rows: &[CompatibilitySloRow],
) -> CorpusCompatibilityScore {
let mut score = CorpusCompatibilityScore {
corpus,
total_rows: rows.len(),
eligible_rows: 0,
readable_rows: 0,
actionable_rows: 0,
failure_rows: 0,
guarded_failure_rows: 0,
guardrail_failures: Vec::new(),
};
for row in rows {
if row.known_limit == "none" {
score.eligible_rows += 1;
if row.current_tier >= 1 {
score.readable_rows += 1;
}
if row.current_tier >= 2 {
score.actionable_rows += 1;
}
}
if row.expected_path == CoverageExpectedPath::Failure {
score.failure_rows += 1;
let guarded = row.known_limit != "none"
&& row.status.is_failure_guardrail_status()
&& row.current_tier <= 1
&& row.min_tier <= 1;
if guarded {
score.guarded_failure_rows += 1;
} else {
score
.guardrail_failures
.push(format!("{}:{}", row.corpus.as_str(), row.domain));
}
}
}
score
}
fn ratio(numerator: usize, denominator: usize) -> f64 {
if denominator == 0 {
return 0.0;
}
numerator as f64 / denominator as f64
}
fn parse_top100_slo_rows(text: &str) -> Result<Vec<CompatibilitySloRow>, String> {
let mut rows = Vec::new();
for (line_number, line) in text.lines().enumerate() {
let trimmed = line.trim();
if trimmed.is_empty() || trimmed.starts_with('#') {
continue;
}
let fields = trimmed.split('\t').collect::<Vec<_>>();
if fields.len() < 9 {
return Err(format!(
"invalid top100 matrix row at line {}: expected 9 fields, got {}",
line_number + 1,
fields.len()
));
}
let domain = fields[0].to_owned();
let family = fields[1].trim().to_owned();
let intent = fields[2].trim().to_owned();
let context = format!("top100 row {} ({domain})", line_number + 1);
let min_tier = parse_slo_tier(fields[3], &context)?;
let current_tier = parse_slo_tier(fields[4], &context)?;
let expected_path = CoverageExpectedPath::parse(fields[6], &context)?;
let status = CoverageStatus::parse(fields[7], &context)?;
let known_limit = fields[8].trim().to_owned();
if family.is_empty() || intent.is_empty() || known_limit.is_empty() {
return Err(format!("missing required top100 fields for {context}"));
}
rows.push(CompatibilitySloRow {
corpus: CompatibilityCorpus::Top100,
domain,
family,
intent: Some(intent),
min_tier,
current_tier,
expected_path,
status,
known_limit,
});
}
Ok(rows)
}
fn parse_forum_slo_rows(text: &str) -> Result<Vec<CompatibilitySloRow>, String> {
let mut rows = Vec::new();
for (line_number, line) in text.lines().enumerate() {
let trimmed = line.trim();
if trimmed.is_empty() || trimmed.starts_with('#') {
continue;
}
let fields = trimmed.split('\t').collect::<Vec<_>>();
if fields.len() < 8 {
return Err(format!(
"invalid forum matrix row at line {}: expected 8 fields, got {}",
line_number + 1,
fields.len()
));
}
let domain = fields[0].to_owned();
let family = fields[1].trim().to_owned();
let context = format!("forum row {} ({domain})", line_number + 1);
let min_tier = parse_slo_tier(fields[2], &context)?;
let current_tier = parse_slo_tier(fields[3], &context)?;
let expected_path = CoverageExpectedPath::parse(fields[5], &context)?;
let status = CoverageStatus::parse(fields[6], &context)?;
let known_limit = fields[7].trim().to_owned();
if family.is_empty() || known_limit.is_empty() {
return Err(format!("missing required forum fields for {context}"));
}
rows.push(CompatibilitySloRow {
corpus: CompatibilityCorpus::Forum,
domain,
family,
intent: None,
min_tier,
current_tier,
expected_path,
status,
known_limit,
});
}
Ok(rows)
}
fn parse_slo_tier(value: &str, context: &str) -> Result<u8, String> {
let tier = value
.parse::<u8>()
.map_err(|error| format!("invalid tier for {context}: {value} ({error})"))?;
if tier > 5 {
return Err(format!("invalid tier for {context}: {value}"));
}
Ok(tier)
}
#[derive(Debug, Clone, PartialEq, Eq)]
struct CompatibilityBacklogRow {
corpus: CompatibilityCorpus,
domain: String,
family: String,
intent: Option<String>,
min_tier: u8,
current_tier: u8,
expected_path: CoverageExpectedPath,
status: CoverageStatus,
known_limit: String,
}
impl CompatibilityBacklogRow {
fn priority_band(&self) -> u8 {
if self.known_limit == "none" && self.current_tier < 2 {
return 0;
}
if self.known_limit == "none" {
return 1;
}
if self.expected_path == CoverageExpectedPath::Failure {
return 2;
}
3
}
fn priority_key(&self) -> (u8, u8, u8, &str, &str, &str) {
let intent = self.intent.as_deref().unwrap_or("");
(
self.priority_band(),
self.current_tier,
self.min_tier,
self.family.as_str(),
intent,
self.domain.as_str(),
)
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum CompatibilityMilestone {
M74,
M75,
M76,
M78,
}
impl CompatibilityMilestone {
const fn as_str(self) -> &'static str {
match self {
Self::M74 => "M74",
Self::M75 => "M75",
Self::M76 => "M76",
Self::M78 => "M78",
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
struct CompatibilityBacklogEntry {
row: CompatibilityBacklogRow,
milestone: CompatibilityMilestone,
rationale: String,
}
#[derive(Debug, Clone, PartialEq, Eq)]
struct CompatibilityBacklogReport {
top100_path: String,
forum_path: String,
top_n: usize,
rows_total: usize,
ranked: Vec<CompatibilityBacklogEntry>,
}
impl CompatibilityBacklogReport {
fn to_text(&self) -> String {
let mut output = vec![
"index-compatibility-backlog-v1".to_owned(),
format!("top100_matrix: {}", self.top100_path),
format!("forum_matrix: {}", self.forum_path),
format!("rows_total: {}", self.rows_total),
format!("top_n: {}", self.top_n),
"policy: known_limit=none first; lower current_tier first; deterministic tie-breakers".to_owned(),
"columns: rank\tcorpus\tdomain\tfamily\tintent\tknown_limit\texpected_path\tstatus\tmin_tier\tcurrent_tier\tmilestone\trationale".to_owned(),
];
for (index, entry) in self.ranked.iter().enumerate() {
output.push(format!(
"{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}",
index + 1,
entry.row.corpus.as_str(),
entry.row.domain,
entry.row.family,
entry.row.intent.as_deref().unwrap_or("-"),
entry.row.known_limit,
entry.row.expected_path.as_str(),
entry.row.status.as_str(),
entry.row.min_tier,
entry.row.current_tier,
entry.milestone.as_str(),
entry.rationale,
));
}
output.join("\n")
}
}
fn run_compatibility_backlog_command<I, S>(mut args: I) -> Result<CliAction, String>
where
I: Iterator<Item = S>,
S: AsRef<str>,
{
let (mut top100_path, mut forum_path) = default_compatibility_slo_paths();
let mut top_n = COMPATIBILITY_BACKLOG_TOP_N_DEFAULT;
while let Some(argument) = args.next() {
match argument.as_ref() {
"--top100" => {
let Some(path) = args.next() else {
return Ok(CliAction::Help(help()));
};
top100_path = path.as_ref().to_owned();
}
"--forum" => {
let Some(path) = args.next() else {
return Ok(CliAction::Help(help()));
};
forum_path = path.as_ref().to_owned();
}
"--top" => {
let Some(value) = args.next() else {
return Ok(CliAction::Help(help()));
};
top_n = value
.as_ref()
.parse::<usize>()
.map_err(|error| format!("invalid --top value {}: {error}", value.as_ref()))?;
if top_n == 0 {
return Err("invalid --top value 0: must be > 0".to_owned());
}
}
other => return Err(format!("unsupported compatibility-backlog option: {other}")),
}
}
let report = compatibility_backlog_report_from_paths(&top100_path, &forum_path, top_n)?;
Ok(CliAction::Print(report.to_text()))
}
fn compatibility_backlog_report_from_paths(
top100_path: &str,
forum_path: &str,
top_n: usize,
) -> Result<CompatibilityBacklogReport, String> {
let top100_text = fs::read_to_string(top100_path)
.map_err(|error| format!("failed to read {top100_path}: {error}"))?;
let forum_text = fs::read_to_string(forum_path)
.map_err(|error| format!("failed to read {forum_path}: {error}"))?;
let top100_rows = parse_top100_backlog_rows(&top100_text)?;
let forum_rows = parse_forum_backlog_rows(&forum_text)?;
Ok(evaluate_compatibility_backlog(
top100_path,
forum_path,
top_n,
&top100_rows,
&forum_rows,
))
}
fn evaluate_compatibility_backlog(
top100_path: &str,
forum_path: &str,
top_n: usize,
top100_rows: &[CompatibilityBacklogRow],
forum_rows: &[CompatibilityBacklogRow],
) -> CompatibilityBacklogReport {
let mut rows = Vec::new();
rows.extend_from_slice(top100_rows);
rows.extend_from_slice(forum_rows);
rows.retain(|row| {
!matches!(
row.status,
CoverageStatus::Covered | CoverageStatus::Blocked
)
});
rows.sort_by(|a, b| a.priority_key().cmp(&b.priority_key()));
let rows_total = rows.len();
let ranked = rows
.into_iter()
.take(top_n)
.map(|row| {
let (milestone, rationale) = map_backlog_row_to_milestone(&row);
CompatibilityBacklogEntry {
row,
milestone,
rationale,
}
})
.collect::<Vec<_>>();
CompatibilityBacklogReport {
top100_path: top100_path.to_owned(),
forum_path: forum_path.to_owned(),
top_n,
rows_total,
ranked,
}
}
fn map_backlog_row_to_milestone(row: &CompatibilityBacklogRow) -> (CompatibilityMilestone, String) {
let family = canonical_family_name(&row.family);
if row.expected_path == CoverageExpectedPath::Failure || row.known_limit != "none" {
return (
CompatibilityMilestone::M76,
"blocked-flow or failure-quality improvement".to_owned(),
);
}
if let Some(intent) = row.intent.as_deref() {
match intent {
"article-or-reference" | "portal-landing" => {
return (
CompatibilityMilestone::M74,
"generic readability and content selection".to_owned(),
);
}
"search-results" | "feed-or-thread" | "marketplace-listing" | "video-hub" => {
return (
CompatibilityMilestone::M75,
"generic actionability for lists, links, and flows".to_owned(),
);
}
"app-shell" => {
return (
CompatibilityMilestone::M78,
"family-pack expansion for app-shell structures".to_owned(),
);
}
_ => {}
}
}
if family.contains("forum")
|| family.contains("stackexchange")
|| family.contains("social-community")
|| family.contains("xenforo")
{
return (
CompatibilityMilestone::M75,
"thread/list actionability for community pages".to_owned(),
);
}
(
CompatibilityMilestone::M74,
"generic readability baseline lift".to_owned(),
)
}
fn parse_top100_backlog_rows(text: &str) -> Result<Vec<CompatibilityBacklogRow>, String> {
let mut rows = Vec::new();
for (line_number, line) in text.lines().enumerate() {
let trimmed = line.trim();
if trimmed.is_empty() || trimmed.starts_with('#') {
continue;
}
let fields = trimmed.split('\t').collect::<Vec<_>>();
if fields.len() < 9 {
return Err(format!(
"invalid top100 backlog row at line {}: expected 9 fields, got {}",
line_number + 1,
fields.len()
));
}
let domain = fields[0].trim().to_owned();
let family = canonical_family_name(fields[1]);
let intent = fields[2].trim().to_owned();
let context = format!("top100 row {} ({domain})", line_number + 1);
let min_tier = parse_slo_tier(fields[3], &context)?;
let current_tier = parse_slo_tier(fields[4], &context)?;
let expected_path = CoverageExpectedPath::parse(fields[6], &context)?;
let status = CoverageStatus::parse(fields[7], &context)?;
let known_limit = fields[8].trim().to_owned();
if family.is_empty() || intent.is_empty() || known_limit.is_empty() {
return Err(format!("missing required backlog fields for {context}"));
}
rows.push(CompatibilityBacklogRow {
corpus: CompatibilityCorpus::Top100,
domain,
family,
intent: Some(intent),
min_tier,
current_tier,
expected_path,
status,
known_limit,
});
}
Ok(rows)
}
fn parse_forum_backlog_rows(text: &str) -> Result<Vec<CompatibilityBacklogRow>, String> {
let mut rows = Vec::new();
for (line_number, line) in text.lines().enumerate() {
let trimmed = line.trim();
if trimmed.is_empty() || trimmed.starts_with('#') {
continue;
}
let fields = trimmed.split('\t').collect::<Vec<_>>();
if fields.len() < 8 {
return Err(format!(
"invalid forum backlog row at line {}: expected 8 fields, got {}",
line_number + 1,
fields.len()
));
}
let domain = fields[0].trim().to_owned();
let family = canonical_family_name(fields[1]);
let context = format!("forum row {} ({domain})", line_number + 1);
let min_tier = parse_slo_tier(fields[2], &context)?;
let current_tier = parse_slo_tier(fields[3], &context)?;
let expected_path = CoverageExpectedPath::parse(fields[5], &context)?;
let status = CoverageStatus::parse(fields[6], &context)?;
let known_limit = fields[7].trim().to_owned();
if family.is_empty() || known_limit.is_empty() {
return Err(format!("missing required backlog fields for {context}"));
}
rows.push(CompatibilityBacklogRow {
corpus: CompatibilityCorpus::Forum,
domain,
family,
intent: None,
min_tier,
current_tier,
expected_path,
status,
known_limit,
});
}
Ok(rows)
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum LiveOutcome {
Ok,
Timeout,
TransientError,
Blocked,
}
impl LiveOutcome {
fn parse(input: &str) -> Option<Self> {
match input.trim().to_ascii_lowercase().as_str() {
"ok" => Some(Self::Ok),
"timeout" => Some(Self::Timeout),
"transient-error" | "transient_error" | "transient" => Some(Self::TransientError),
"blocked" => Some(Self::Blocked),
_ => None,
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
struct LiveVarianceTarget {
origin: String,
family: String,
enabled: bool,
}
#[derive(Debug, Clone, PartialEq, Eq)]
struct LiveVarianceRow {
timestamp_utc: String,
origin: String,
outcome: LiveOutcome,
latency_ms: Option<u64>,
blocked_class: Option<String>,
}
#[derive(Debug, Clone, PartialEq)]
struct LiveVarianceOriginStats {
origin: String,
family: String,
samples: usize,
ok: usize,
timeout: usize,
transient_error: usize,
blocked: usize,
flake_rate: f64,
timeout_rate: f64,
recent_flake_rate: f64,
previous_flake_rate: f64,
trend_delta: f64,
}
#[derive(Debug, Clone, PartialEq)]
struct LiveVarianceReport {
targets_path: String,
runs_path: String,
window: usize,
rows_total: usize,
enabled_targets: usize,
total_samples: usize,
global_flake_rate: f64,
global_timeout_rate: f64,
origins: Vec<LiveVarianceOriginStats>,
blocked_class_counts: Vec<(String, usize)>,
}
impl LiveVarianceReport {
fn to_text(&self) -> String {
let mut output = vec![
"index-compatibility-live-variance-v1".to_owned(),
format!("targets: {}", self.targets_path),
format!("runs: {}", self.runs_path),
format!("window: {}", self.window),
format!("rows_total: {}", self.rows_total),
format!("enabled_targets: {}", self.enabled_targets),
format!("samples_total: {}", self.total_samples),
format!(
"global_rates: flake={:.4} timeout={:.4}",
self.global_flake_rate, self.global_timeout_rate
),
"origin_columns: origin\tfamily\tsamples\tok\ttimeout\ttransient_error\tblocked\tflake_rate\ttimeout_rate\trecent_flake\tprevious_flake\ttrend_delta".to_owned(),
];
for origin in &self.origins {
output.push(format!(
"{}\t{}\t{}\t{}\t{}\t{}\t{}\t{:.4}\t{:.4}\t{:.4}\t{:.4}\t{:+.4}",
origin.origin,
origin.family,
origin.samples,
origin.ok,
origin.timeout,
origin.transient_error,
origin.blocked,
origin.flake_rate,
origin.timeout_rate,
origin.recent_flake_rate,
origin.previous_flake_rate,
origin.trend_delta,
));
}
if self.blocked_class_counts.is_empty() {
output.push("blocked_classes: none".to_owned());
} else {
output.push("blocked_class_columns: class\tcount".to_owned());
for (class_name, count) in &self.blocked_class_counts {
output.push(format!("{class_name}\t{count}"));
}
}
output.push("result: pass".to_owned());
output.join("\n")
}
}
fn default_live_variance_paths() -> (String, String) {
let root = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../..");
(
root.join(LIVE_VARIANCE_TARGETS_PATH).display().to_string(),
root.join(LIVE_VARIANCE_RUNS_PATH).display().to_string(),
)
}
fn run_compatibility_live_variance_command<I, S>(mut args: I) -> Result<CliAction, String>
where
I: Iterator<Item = S>,
S: AsRef<str>,
{
let (mut targets_path, mut runs_path) = default_live_variance_paths();
let mut window = LIVE_VARIANCE_WINDOW_DEFAULT;
while let Some(argument) = args.next() {
match argument.as_ref() {
"--targets" => {
let Some(path) = args.next() else {
return Ok(CliAction::Help(help()));
};
targets_path = path.as_ref().to_owned();
}
"--runs" => {
let Some(path) = args.next() else {
return Ok(CliAction::Help(help()));
};
runs_path = path.as_ref().to_owned();
}
"--window" => {
let Some(value) = args.next() else {
return Ok(CliAction::Help(help()));
};
window = value.as_ref().parse::<usize>().map_err(|error| {
format!("invalid --window value {}: {error}", value.as_ref())
})?;
if window == 0 || window > LIVE_VARIANCE_MAX_WINDOW {
return Err(format!(
"invalid --window value {window}: expected 1..={LIVE_VARIANCE_MAX_WINDOW}"
));
}
}
other => {
return Err(format!(
"unsupported compatibility-live-variance option: {other}"
));
}
}
}
let report = compatibility_live_variance_report_from_paths(&targets_path, &runs_path, window)?;
Ok(CliAction::Print(report.to_text()))
}
fn compatibility_live_variance_report_from_paths(
targets_path: &str,
runs_path: &str,
window: usize,
) -> Result<LiveVarianceReport, String> {
let targets_text = fs::read_to_string(targets_path)
.map_err(|error| format!("failed to read {targets_path}: {error}"))?;
let runs_text = fs::read_to_string(runs_path)
.map_err(|error| format!("failed to read {runs_path}: {error}"))?;
let targets = parse_live_variance_targets(&targets_text)?;
let rows = parse_live_variance_rows(&runs_text)?;
Ok(evaluate_live_variance_report(
targets_path,
runs_path,
&targets,
&rows,
window,
))
}
fn evaluate_live_variance_report(
targets_path: &str,
runs_path: &str,
targets: &[LiveVarianceTarget],
rows: &[LiveVarianceRow],
window: usize,
) -> LiveVarianceReport {
let mut families = BTreeMap::new();
let mut enabled = BTreeMap::new();
for target in targets {
families.insert(target.origin.clone(), target.family.clone());
enabled.insert(target.origin.clone(), target.enabled);
}
let filtered = rows
.iter()
.filter(|row| enabled.get(&row.origin).copied().unwrap_or(true))
.cloned()
.collect::<Vec<_>>();
let mut rows_by_origin = BTreeMap::<String, Vec<LiveVarianceRow>>::new();
let mut blocked_counts = BTreeMap::<String, usize>::new();
let mut totals = (0usize, 0usize, 0usize);
for row in filtered {
totals.0 = totals.0.saturating_add(1);
if !matches!(row.outcome, LiveOutcome::Ok) {
totals.1 = totals.1.saturating_add(1);
}
if matches!(row.outcome, LiveOutcome::Timeout) {
totals.2 = totals.2.saturating_add(1);
}
if let Some(class_name) = row.blocked_class.as_ref() {
let counter = blocked_counts.entry(class_name.clone()).or_default();
*counter = counter.saturating_add(1);
}
rows_by_origin
.entry(row.origin.clone())
.or_default()
.push(row);
}
let origins = rows_by_origin
.into_iter()
.map(|(origin, entries)| {
let samples = entries.len();
let ok = entries
.iter()
.filter(|row| matches!(row.outcome, LiveOutcome::Ok))
.count();
let timeout = entries
.iter()
.filter(|row| matches!(row.outcome, LiveOutcome::Timeout))
.count();
let transient_error = entries
.iter()
.filter(|row| matches!(row.outcome, LiveOutcome::TransientError))
.count();
let blocked = entries
.iter()
.filter(|row| matches!(row.outcome, LiveOutcome::Blocked))
.count();
let flake = samples.saturating_sub(ok);
let flake_rate = live_ratio(flake, samples);
let timeout_rate = live_ratio(timeout, samples);
let recent = entries.iter().rev().take(window).collect::<Vec<_>>();
let previous = entries
.iter()
.rev()
.skip(window)
.take(window)
.collect::<Vec<_>>();
let recent_flake = recent
.iter()
.filter(|row| !matches!(row.outcome, LiveOutcome::Ok))
.count();
let previous_flake = previous
.iter()
.filter(|row| !matches!(row.outcome, LiveOutcome::Ok))
.count();
let recent_flake_rate = live_ratio(recent_flake, recent.len());
let previous_flake_rate = live_ratio(previous_flake, previous.len());
let trend_delta = recent_flake_rate - previous_flake_rate;
LiveVarianceOriginStats {
origin: origin.clone(),
family: families
.get(&origin)
.cloned()
.unwrap_or_else(|| "unknown".to_owned()),
samples,
ok,
timeout,
transient_error,
blocked,
flake_rate,
timeout_rate,
recent_flake_rate,
previous_flake_rate,
trend_delta,
}
})
.collect::<Vec<_>>();
LiveVarianceReport {
targets_path: targets_path.to_owned(),
runs_path: runs_path.to_owned(),
window,
rows_total: rows.len(),
enabled_targets: targets.iter().filter(|target| target.enabled).count(),
total_samples: totals.0,
global_flake_rate: live_ratio(totals.1, totals.0),
global_timeout_rate: live_ratio(totals.2, totals.0),
origins,
blocked_class_counts: blocked_counts.into_iter().collect(),
}
}
fn parse_live_variance_targets(input: &str) -> Result<Vec<LiveVarianceTarget>, String> {
let mut rows = Vec::new();
for (line_number, line) in input.lines().enumerate() {
let trimmed = line.trim();
if trimmed.is_empty() || trimmed.starts_with('#') {
continue;
}
let fields = trimmed.split('\t').collect::<Vec<_>>();
if fields.len() < 2 {
return Err(format!(
"invalid live-variance target row at line {}: expected at least 2 fields",
line_number + 1
));
}
let origin = fields[0].trim().to_ascii_lowercase();
if origin.is_empty() {
return Err(format!(
"invalid live-variance target row at line {}: origin is empty",
line_number + 1
));
}
let family = fields[1].trim().to_owned();
let enabled = fields
.get(2)
.map(|value| match value.trim().to_ascii_lowercase().as_str() {
"0" | "false" | "no" => Ok(false),
"1" | "true" | "yes" | "" => Ok(true),
other => Err(format!(
"invalid enabled flag at line {}: {}",
line_number + 1,
other
)),
})
.unwrap_or(Ok(true))?;
rows.push(LiveVarianceTarget {
origin,
family,
enabled,
});
}
rows.sort_by(|left, right| left.origin.cmp(&right.origin));
Ok(rows)
}
fn parse_live_variance_rows(input: &str) -> Result<Vec<LiveVarianceRow>, String> {
let mut rows = Vec::new();
for (line_number, line) in input.lines().enumerate() {
let trimmed = line.trim();
if trimmed.is_empty() || trimmed.starts_with('#') {
continue;
}
let fields = trimmed.split('\t').collect::<Vec<_>>();
if fields.len() < 3 {
return Err(format!(
"invalid live-variance row at line {}: expected at least 3 fields",
line_number + 1
));
}
let timestamp_utc = fields[0].trim().to_owned();
let origin = fields[1].trim().to_ascii_lowercase();
let Some(outcome) = LiveOutcome::parse(fields[2]) else {
return Err(format!(
"invalid live-variance outcome at line {}: {}",
line_number + 1,
fields[2].trim()
));
};
let latency_ms = fields
.get(3)
.map(|value| value.trim())
.filter(|value| !value.is_empty() && *value != "-")
.map(|value| {
value.parse::<u64>().map_err(|error| {
format!(
"invalid live-variance latency at line {}: {}",
line_number + 1,
error
)
})
})
.transpose()?;
let blocked_class = fields
.get(4)
.map(|value| value.trim())
.filter(|value| !value.is_empty() && *value != "-")
.map(ToOwned::to_owned);
if origin.is_empty() || timestamp_utc.is_empty() {
return Err(format!(
"invalid live-variance row at line {}: missing timestamp or origin",
line_number + 1
));
}
rows.push(LiveVarianceRow {
timestamp_utc,
origin,
outcome,
latency_ms,
blocked_class,
});
}
rows.sort_by(|left, right| {
(left.origin.as_str(), left.timestamp_utc.as_str())
.cmp(&(right.origin.as_str(), right.timestamp_utc.as_str()))
});
Ok(rows)
}
fn live_ratio(numerator: usize, denominator: usize) -> f64 {
if denominator == 0 {
0.0
} else {
numerator as f64 / denominator as f64
}
}
fn run_compatibility_recovery_plan_command<I, S>(mut args: I) -> Result<CliAction, String>
where
I: Iterator<Item = S>,
S: AsRef<str>,
{
let Some(target) = args.next() else {
return Ok(CliAction::Help(help()));
};
if args.next().is_some() {
return Err("too many arguments".to_owned());
}
let normalized = normalize_url_input(target.as_ref());
let profile = recovery_profile_for_target(&normalized);
let mut output = vec![
"index-compatibility-recovery-plan-v2".to_owned(),
format!("target: {normalized}"),
format!("profile: {}", profile.as_str()),
format!(
"fallback_order: {}",
recovery_fallback_order(profile).join(" -> ")
),
"stage_timeout_columns: stage\tseconds".to_owned(),
];
for stage in [
RuntimeStage::Fetching,
RuntimeStage::Snapshotting,
RuntimeStage::Parsing,
RuntimeStage::Transforming,
RuntimeStage::Scoring,
RuntimeStage::Storing,
] {
let seconds = stage_timeout_for_target(stage, &normalized)
.map(|value| value.as_secs().to_string())
.unwrap_or_else(|| "-".to_owned());
output.push(format!("{}\t{}", stage.as_str(), seconds));
}
output.push("result: pass".to_owned());
Ok(CliAction::Print(output.join("\n")))
}
fn auth_cookie_store_path() -> PathBuf {
Path::new(&runtime_paths().state_dir)
.join("auth")
.join("cookies.store")
}
fn run_auth_assist_command<I, S>(mut args: I) -> Result<CliAction, String>
where
I: Iterator<Item = S>,
S: AsRef<str>,
{
let Some(command) = args.next() else {
return Ok(CliAction::Help(help()));
};
match command.as_ref() {
"inspect" => {
let Some(target) = args.next() else {
return Ok(CliAction::Help(help()));
};
if args.next().is_some() {
return Err("too many arguments".to_owned());
}
Ok(CliAction::Print(auth_assist_inspect(target.as_ref())?))
}
"import" => {
let Some(path) = args.next() else {
return Ok(CliAction::Help(help()));
};
if args.next().is_some() {
return Err("too many arguments".to_owned());
}
Ok(CliAction::Print(auth_assist_import(path.as_ref())?))
}
"export" => {
let Some(path) = args.next() else {
return Ok(CliAction::Help(help()));
};
if args.next().is_some() {
return Err("too many arguments".to_owned());
}
Ok(CliAction::Print(auth_assist_export(path.as_ref())?))
}
"diagnose-submit" => {
let Some(url) = args.next() else {
return Ok(CliAction::Help(help()));
};
let Some(status) = args.next() else {
return Ok(CliAction::Help(help()));
};
let status_code = status
.as_ref()
.parse::<u16>()
.map_err(|error| format!("invalid status code {}: {error}", status.as_ref()))?;
let message = args
.map(|part| part.as_ref().to_owned())
.collect::<Vec<_>>();
if message.is_empty() {
return Ok(CliAction::Help(help()));
}
Ok(CliAction::Print(auth_assist_diagnose_submit(
url.as_ref(),
status_code,
&message.join(" "),
)))
}
other => Err(format!("unsupported auth-assist command: {other}")),
}
}
fn auth_assist_import(path: &str) -> Result<String, String> {
let bytes = fs::read(path).map_err(|error| format!("failed to read {path}: {error}"))?;
let mut storage = MemorySecureStorage::new();
storage
.store("cookies", &bytes)
.map_err(|error| format!("failed to stage cookies: {error}"))?;
let jar = CookieJar::load(&storage, "cookies")
.map_err(|error| format!("invalid cookie bundle: {error}"))?;
let normalized = jar
.header_for(&IndexUrl::parse("https://example.org").map_err(|error| error.to_string())?)
.unwrap_or_default();
let target = auth_cookie_store_path();
if let Some(parent) = target.parent() {
fs::create_dir_all(parent)
.map_err(|error| format!("failed to create {}: {error}", parent.display()))?;
}
fs::write(&target, &bytes)
.map_err(|error| format!("failed to write {}: {error}", target.display()))?;
Ok(format!(
"index-auth-assist-import-v1\nsource: {path}\nstored_to: {}\nprobe_cookie_header_len: {}\nresult: pass",
target.display(),
normalized.len()
))
}
fn auth_assist_export(path: &str) -> Result<String, String> {
let source = auth_cookie_store_path();
let bytes = fs::read(&source)
.map_err(|error| format!("failed to read {}: {error}", source.display()))?;
fs::write(path, bytes).map_err(|error| format!("failed to write {path}: {error}"))?;
Ok(format!(
"index-auth-assist-export-v1\nsource: {}\noutput: {path}\nresult: pass",
source.display()
))
}
fn auth_assist_inspect(target: &str) -> Result<String, String> {
let normalized = normalize_url_input(target);
let url = IndexUrl::parse(&normalized).map_err(|error| error.to_string())?;
let source = auth_cookie_store_path();
let bytes = match fs::read(&source) {
Ok(bytes) => bytes,
Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
return Ok(format!(
"index-auth-assist-inspect-v1\ntarget: {normalized}\nstore: {}\nsession: missing\nresult: pass",
source.display()
));
}
Err(error) => {
return Err(format!(
"failed to read auth cookie store {}: {error}",
source.display()
));
}
};
let mut storage = MemorySecureStorage::new();
storage
.store("cookies", &bytes)
.map_err(|error| format!("failed to stage cookies: {error}"))?;
let jar = CookieJar::load(&storage, "cookies")
.map_err(|error| format!("invalid cookie bundle: {error}"))?;
let header = jar.header_for(&url);
let names = header
.as_deref()
.map(cookie_header_names)
.unwrap_or_default()
.join(", ");
Ok(format!(
"index-auth-assist-inspect-v1\ntarget: {normalized}\nstore: {}\norigin_cookie_names: {}\nresult: pass",
source.display(),
if names.is_empty() {
"-".to_owned()
} else {
names
}
))
}
fn cookie_header_names(header: &str) -> Vec<String> {
let mut names = header
.split(';')
.filter_map(|part| part.split_once('='))
.map(|(name, _)| name.trim().to_owned())
.filter(|name| !name.is_empty())
.collect::<Vec<_>>();
names.sort();
names.dedup();
names
}
fn auth_assist_diagnose_submit(url: &str, status: u16, message: &str) -> String {
let normalized = normalize_url_input(url);
let lower = message.to_ascii_lowercase();
let (class, remediation) = if lower.contains("csrf")
|| lower.contains("xsrf")
|| (lower.contains("token") && matches!(status, 400 | 401 | 403 | 419))
{
(
"csrf-token-mismatch",
"refresh session cookies and resubmit with updated form token",
)
} else if matches!(status, 419 | 440) || lower.contains("expired session") {
(
"session-expired",
"re-authenticate and retry the form submission",
)
} else if status == 401 {
(
"auth-required",
"authenticate in browser context and import local session cookies",
)
} else if status == 403 {
(
"origin-policy-denied",
"check origin/session scope and token freshness",
)
} else {
(
"generic-submit-failure",
"capture the response and inspect form field mappings",
)
};
format!(
"index-auth-assist-diagnose-v1\ntarget: {normalized}\nstatus: {status}\nclass: {class}\nmessage: {message}\nremediation: {remediation}\nresult: pass"
)
}
fn run_challenge_diagnose_command<I, S>(mut args: I) -> Result<CliAction, String>
where
I: Iterator<Item = S>,
S: AsRef<str>,
{
let Some(target) = args.next() else {
return Ok(CliAction::Help(help()));
};
let remaining = args
.map(|part| part.as_ref().to_owned())
.collect::<Vec<_>>();
if remaining.is_empty() {
return Ok(CliAction::Help(help()));
}
let first = &remaining[0];
let text = match fs::read_to_string(first) {
Ok(contents) => contents,
Err(_) => remaining.join(" "),
};
let report = challenge_diagnose_report(target.as_ref(), &text);
Ok(CliAction::Print(report))
}
fn challenge_diagnose_report(target: &str, body: &str) -> String {
let normalized = normalize_url_input(target);
let lower = body.to_ascii_lowercase();
let (class_name, reason) = if lower.contains("captcha") {
("captcha", "captcha challenge detected")
} else if lower.contains("verify you are human")
|| lower.contains("cloudflare")
|| lower.contains("attention required")
|| lower.contains("bot detection")
{
("bot-gate", "anti-bot challenge detected")
} else if lower.contains("not available in your region")
|| lower.contains("geo-blocked")
|| lower.contains("country not supported")
{
("geo-gate", "geo restriction detected")
} else if lower.contains("age verification")
|| lower.contains("confirm your age")
|| lower.contains("you must be 18")
{
("age-gate", "age-gated workflow detected")
} else if lower.contains("sign in")
|| lower.contains("log in")
|| lower.contains("login required")
{
("auth-wall", "authentication gate detected")
} else if lower.contains("access denied") || lower.contains("blocked by policy") {
("policy-blocked", "policy or access control gate detected")
} else {
("none", "no strong challenge marker detected")
};
format!(
"index-challenge-diagnose-v1\ntarget: {normalized}\nclass: {class_name}\nreason: {reason}\nresult: pass"
)
}
#[derive(Debug, Clone, PartialEq)]
struct CompatibilityRecoveryGateReport {
top100_path: String,
forum_path: String,
live_targets_path: String,
live_runs_path: String,
live_window: usize,
max_live_flake_rate: f64,
max_live_timeout_rate: f64,
slo_v2_passed: bool,
live_flake_rate: f64,
live_timeout_rate: f64,
violations: Vec<String>,
}
impl CompatibilityRecoveryGateReport {
fn passed(&self) -> bool {
self.violations.is_empty()
}
fn to_text(&self) -> String {
let mut output = vec![
"index-compatibility-recovery-gate-v1".to_owned(),
format!("top100_matrix: {}", self.top100_path),
format!("forum_matrix: {}", self.forum_path),
format!("live_targets: {}", self.live_targets_path),
format!("live_runs: {}", self.live_runs_path),
format!("live_window: {}", self.live_window),
format!(
"thresholds: live_flake<={:.4} live_timeout<={:.4}",
self.max_live_flake_rate, self.max_live_timeout_rate
),
format!("slo_v2_passed: {}", self.slo_v2_passed),
format!(
"live_rates: flake={:.4} timeout={:.4}",
self.live_flake_rate, self.live_timeout_rate
),
];
if self.violations.is_empty() {
output.push("violations: none".to_owned());
} else {
output.push("violations:".to_owned());
for violation in &self.violations {
output.push(format!("- {violation}"));
}
}
output.push(format!(
"result: {}",
if self.passed() { "pass" } else { "fail" }
));
output.join("\n")
}
}
fn run_compatibility_recovery_gate_command<I, S>(mut args: I) -> Result<CliAction, String>
where
I: Iterator<Item = S>,
S: AsRef<str>,
{
let (mut top100_path, mut forum_path) = default_compatibility_slo_paths();
let (mut live_targets_path, mut live_runs_path) = default_live_variance_paths();
let mut thresholds = CompatibilitySloThresholds::default();
let mut live_window = LIVE_VARIANCE_WINDOW_DEFAULT;
let mut max_live_flake_rate = 0.35_f64;
let mut max_live_timeout_rate = 0.20_f64;
while let Some(argument) = args.next() {
match argument.as_ref() {
"--top100" => {
let Some(path) = args.next() else {
return Ok(CliAction::Help(help()));
};
top100_path = path.as_ref().to_owned();
}
"--forum" => {
let Some(path) = args.next() else {
return Ok(CliAction::Help(help()));
};
forum_path = path.as_ref().to_owned();
}
"--live-targets" => {
let Some(path) = args.next() else {
return Ok(CliAction::Help(help()));
};
live_targets_path = path.as_ref().to_owned();
}
"--live-runs" => {
let Some(path) = args.next() else {
return Ok(CliAction::Help(help()));
};
live_runs_path = path.as_ref().to_owned();
}
"--live-window" => {
let Some(value) = args.next() else {
return Ok(CliAction::Help(help()));
};
live_window = value.as_ref().parse::<usize>().map_err(|error| {
format!("invalid --live-window value {}: {error}", value.as_ref())
})?;
if live_window == 0 || live_window > LIVE_VARIANCE_MAX_WINDOW {
return Err(format!(
"invalid --live-window value {live_window}: expected 1..={LIVE_VARIANCE_MAX_WINDOW}"
));
}
}
"--max-live-flake-rate" => {
let Some(value) = args.next() else {
return Ok(CliAction::Help(help()));
};
max_live_flake_rate = parse_slo_floor("max-live-flake-rate", value.as_ref())?;
}
"--max-live-timeout-rate" => {
let Some(value) = args.next() else {
return Ok(CliAction::Help(help()));
};
max_live_timeout_rate = parse_slo_floor("max-live-timeout-rate", value.as_ref())?;
}
"--min-readability" => {
let Some(value) = args.next() else {
return Ok(CliAction::Help(help()));
};
thresholds.readability_floor = parse_slo_floor("readability", value.as_ref())?;
}
"--min-actionability" => {
let Some(value) = args.next() else {
return Ok(CliAction::Help(help()));
};
thresholds.actionability_floor = parse_slo_floor("actionability", value.as_ref())?;
}
"--min-failure-quality" => {
let Some(value) = args.next() else {
return Ok(CliAction::Help(help()));
};
thresholds.failure_quality_floor =
parse_slo_floor("failure-quality", value.as_ref())?;
}
other => {
return Err(format!(
"unsupported compatibility-recovery-gate option: {other}"
));
}
}
}
let slo = compatibility_slo_v2_report_from_paths(
&top100_path,
&forum_path,
thresholds,
Some(&top100_path),
Some(&forum_path),
)?;
let live = compatibility_live_variance_report_from_paths(
&live_targets_path,
&live_runs_path,
live_window,
)?;
let mut violations = Vec::new();
if !slo.passed() {
violations.push("compatibility-slo-v2 gate failed".to_owned());
}
if live.global_flake_rate > max_live_flake_rate {
violations.push(format!(
"live flake rate {:.4} exceeds threshold {:.4}",
live.global_flake_rate, max_live_flake_rate
));
}
if live.global_timeout_rate > max_live_timeout_rate {
violations.push(format!(
"live timeout rate {:.4} exceeds threshold {:.4}",
live.global_timeout_rate, max_live_timeout_rate
));
}
let report = CompatibilityRecoveryGateReport {
top100_path,
forum_path,
live_targets_path,
live_runs_path,
live_window,
max_live_flake_rate,
max_live_timeout_rate,
slo_v2_passed: slo.passed(),
live_flake_rate: live.global_flake_rate,
live_timeout_rate: live.global_timeout_rate,
violations,
};
if report.passed() {
Ok(CliAction::Print(report.to_text()))
} else {
Err(report.to_text())
}
}
fn run_compatibility_pack_command<I, S>(mut args: I) -> Result<CliAction, String>
where
I: Iterator<Item = S>,
S: AsRef<str>,
{
let Some(command) = args.next() else {
return Ok(CliAction::Help(help()));
};
match command.as_ref() {
"lint" => {
let Some(pack_path) = args.next() else {
return Ok(CliAction::Help(help()));
};
let Some(page_url) = args.next() else {
return Ok(CliAction::Help(help()));
};
if args.next().is_some() {
return Err("too many arguments".to_owned());
}
let report = lint_pack_file(pack_path.as_ref(), page_url.as_ref())?;
Ok(CliAction::Print(report))
}
"inspect" => {
let Some(page_url) = args.next() else {
return Ok(CliAction::Help(help()));
};
if args.next().is_some() {
return Err("too many arguments".to_owned());
}
let paths = runtime_paths();
let report = inspect_runtime_for_url(&paths.config_dir, page_url.as_ref())?;
Ok(CliAction::Print(report))
}
"install" => {
let Some(pack_path) = args.next() else {
return Ok(CliAction::Help(help()));
};
let mut source = PackSource::User;
if let Some(option) = args.next() {
source = match option.as_ref() {
"--trusted" => PackSource::Trusted,
"--user" => PackSource::User,
other => {
return Err(format!(
"unsupported compatibility-pack install option: {other}"
));
}
};
if args.next().is_some() {
return Err("too many arguments".to_owned());
}
}
let paths = runtime_paths();
let target = install_pack_file(&paths.config_dir, pack_path.as_ref(), source)?;
Ok(CliAction::Print(format!(
"index-compat-pack-install-v1\nsource: {}\npack: {}\ninstalled_to: {}\nresult: pass",
source.as_str(),
pack_path.as_ref(),
target.display()
)))
}
"update" => {
let Some(pack_id) = args.next() else {
return Ok(CliAction::Help(help()));
};
let Some(pack_path) = args.next() else {
return Ok(CliAction::Help(help()));
};
let mut source = PackSource::User;
if let Some(option) = args.next() {
source = match option.as_ref() {
"--trusted" => PackSource::Trusted,
"--user" => PackSource::User,
other => {
return Err(format!(
"unsupported compatibility-pack update option: {other}"
));
}
};
if args.next().is_some() {
return Err("too many arguments".to_owned());
}
}
let paths = runtime_paths();
let target = install_pack_file(&paths.config_dir, pack_path.as_ref(), source)?;
let expected_name = format!("{}.pack.json", pack_id.as_ref());
let actual_name = target
.file_name()
.and_then(|value| value.to_str())
.unwrap_or_default()
.to_owned();
if actual_name != expected_name {
return Err(format!(
"pack id mismatch: expected {}, installed {}",
pack_id.as_ref(),
actual_name
));
}
Ok(CliAction::Print(format!(
"index-compat-pack-update-v1\nsource: {}\npack_id: {}\npack: {}\nupdated_to: {}\nresult: pass",
source.as_str(),
pack_id.as_ref(),
pack_path.as_ref(),
target.display()
)))
}
"list" => {
if args.next().is_some() {
return Err("too many arguments".to_owned());
}
let paths = runtime_paths();
let files = list_runtime_pack_files(&paths.config_dir);
let mut output = vec![
"index-compat-pack-list-v1".to_owned(),
format!("count: {}", files.len()),
"columns: source\tpath".to_owned(),
];
output.extend(files);
output.push("result: pass".to_owned());
Ok(CliAction::Print(output.join("\n")))
}
"remove" => {
let Some(pack_id) = args.next() else {
return Ok(CliAction::Help(help()));
};
if args.next().is_some() {
return Err("too many arguments".to_owned());
}
let paths = runtime_paths();
let removed = remove_compatibility_pack(&paths.config_dir, pack_id.as_ref())?;
Ok(CliAction::Print(format!(
"index-compat-pack-remove-v1\npack_id: {}\nremoved_files: {}\nresult: pass",
pack_id.as_ref(),
removed
)))
}
"rollback" => {
let Some(pack_id) = args.next() else {
return Ok(CliAction::Help(help()));
};
if args.next().is_some() {
return Err("too many arguments".to_owned());
}
let paths = runtime_paths();
let restored = rollback_pack_file(&paths.config_dir, pack_id.as_ref())?;
Ok(CliAction::Print(format!(
"index-compat-pack-rollback-v1\npack_id: {}\nrestored_to: {}\nresult: pass",
pack_id.as_ref(),
restored.display()
)))
}
"sign" => {
let Some(pack_path) = args.next() else {
return Ok(CliAction::Help(help()));
};
let Some(key_id) = args.next() else {
return Ok(CliAction::Help(help()));
};
let Some(secret) = args.next() else {
return Ok(CliAction::Help(help()));
};
if args.next().is_some() {
return Err("too many arguments".to_owned());
}
let bytes = fs::read(pack_path.as_ref())
.map_err(|error| format!("failed to read {}: {error}", pack_path.as_ref()))?;
let signature = sign_pack_bytes(&bytes, key_id.as_ref(), secret.as_ref());
Ok(CliAction::Print(format!(
"index-compat-pack-sign-v1\npack: {}\nkey_id: {}\nsignature: {}\nresult: pass",
pack_path.as_ref(),
key_id.as_ref(),
signature
)))
}
"verify" => {
let Some(pack_path) = args.next() else {
return Ok(CliAction::Help(help()));
};
let Some(key_id) = args.next() else {
return Ok(CliAction::Help(help()));
};
let Some(secret) = args.next() else {
return Ok(CliAction::Help(help()));
};
let Some(signature) = args.next() else {
return Ok(CliAction::Help(help()));
};
if args.next().is_some() {
return Err("too many arguments".to_owned());
}
let bytes = fs::read(pack_path.as_ref())
.map_err(|error| format!("failed to read {}: {error}", pack_path.as_ref()))?;
if verify_pack_signature(&bytes, key_id.as_ref(), secret.as_ref(), signature.as_ref()) {
Ok(CliAction::Print(format!(
"index-compat-pack-verify-v1\npack: {}\nkey_id: {}\nresult: pass",
pack_path.as_ref(),
key_id.as_ref()
)))
} else {
Err(format!(
"index-compat-pack-verify-v1\npack: {}\nkey_id: {}\nresult: fail\nerror: signature mismatch",
pack_path.as_ref(),
key_id.as_ref()
))
}
}
other => Err(format!("unsupported compatibility-pack command: {other}")),
}
}
fn remove_compatibility_pack(config_dir: &str, pack_id: &str) -> Result<usize, String> {
let user_path = Path::new(config_dir)
.join("compat-packs/user")
.join(format!("{pack_id}.pack.json"));
let trusted_path = Path::new(config_dir)
.join("compat-packs/trusted")
.join(format!("{pack_id}.pack.json"));
let mut removed = 0usize;
for path in [user_path, trusted_path] {
match fs::remove_file(&path) {
Ok(()) => {
removed += 1;
}
Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
Err(error) => {
return Err(format!(
"failed to remove compatibility pack {}: {error}",
path.display()
));
}
}
}
Ok(removed)
}
fn run_adapter_command<I, S>(mut args: I) -> Result<CliAction, String>
where
I: Iterator<Item = S>,
S: AsRef<str>,
{
let Some(command) = args.next() else {
return Ok(CliAction::Help(help()));
};
match command.as_ref() {
"check" => {
let Some(fixture) = args.next() else {
return Ok(CliAction::Help(help()));
};
if args.next().is_some() {
return Err("too many arguments".to_owned());
}
Ok(CliAction::Print(adapter_check_report(fixture.as_ref())?))
}
"scaffold" => {
let Some(fixture) = args.next() else {
return Ok(CliAction::Help(help()));
};
if args.next().is_some() {
return Err("too many arguments".to_owned());
}
Ok(CliAction::Print(adapter_manifest_scaffold(
fixture.as_ref(),
)?))
}
"diff" => {
let Some(fixture) = args.next() else {
return Ok(CliAction::Help(help()));
};
let Some(manifest) = args.next() else {
return Ok(CliAction::Help(help()));
};
if args.next().is_some() {
return Err("too many arguments".to_owned());
}
Ok(CliAction::Print(adapter_manifest_diff(
fixture.as_ref(),
manifest.as_ref(),
)?))
}
other => Err(format!("unsupported adapter command: {other}")),
}
}
fn run_idx_command<I, S>(mut args: I) -> Result<CliAction, String>
where
I: Iterator<Item = S>,
S: AsRef<str>,
{
let Some(command) = args.next() else {
return Ok(CliAction::Help(help()));
};
match command.as_ref() {
"lint" => run_idx_lint_command(args),
other => Err(format!("unsupported idx command: {other}")),
}
}
fn run_idx_lint_command<I, S>(mut args: I) -> Result<CliAction, String>
where
I: Iterator<Item = S>,
S: AsRef<str>,
{
let Some(manifest_path) = args.next() else {
return Ok(CliAction::Help(help()));
};
let manifest_path = manifest_path.as_ref();
let Some(page_url) = args.next() else {
return Ok(CliAction::Help(help()));
};
let page_url = page_url.as_ref();
let mut source_url = well_known_index_manifest_url(page_url)
.ok_or_else(|| format!("invalid page URL for idx lint: {page_url}"))?;
while let Some(argument) = args.next() {
match argument.as_ref() {
"--source-url" => {
let Some(value) = args.next() else {
return Ok(CliAction::Help(help()));
};
source_url = value.as_ref().to_owned();
}
other => return Err(format!("unsupported idx lint option: {other}")),
}
}
let input = fs::read_to_string(manifest_path)
.map_err(|error| format!("failed to read {manifest_path}: {error}"))?;
let parsed = parse_index_manifest(&input, &source_url, page_url);
match parsed {
Ok(manifest) => Ok(CliAction::Print(idx_lint_pass_report(
manifest_path,
page_url,
&manifest,
))),
Err(error) => Err(idx_lint_failure_report(
manifest_path,
page_url,
&source_url,
&error,
)),
}
}
fn idx_lint_pass_report(manifest_path: &str, page_url: &str, manifest: &IndexManifest) -> String {
format!(
"index-idx-lint-v1\nmanifest: {manifest_path}\npage_url: {page_url}\nsource_url: {}\nstatus: pass\nscope: {}\ncontent.main_selector: {}\nregions: {}\nfields: {}\nforms: {}\ndates: {}\nchecks: same-origin=pass scope=pass safety=pass\nresult: pass",
manifest.source_url,
manifest.scope,
manifest.content.main_selector.as_deref().unwrap_or("-"),
manifest.regions.len(),
manifest.fields.len(),
manifest.forms.len(),
manifest.dates.len(),
)
}
fn idx_lint_failure_report(
manifest_path: &str,
page_url: &str,
source_url: &str,
error: &IndexManifestError,
) -> String {
format!(
"index-idx-lint-v1\nmanifest: {manifest_path}\npage_url: {page_url}\nsource_url: {source_url}\nstatus: fail\nerror: {error}\naction: {}\nresult: fail",
idx_lint_fix(error),
)
}
fn idx_lint_fix(error: &IndexManifestError) -> &'static str {
match error {
IndexManifestError::TooLarge { .. } => {
"reduce manifest size below 32KiB and remove non-essential hints"
}
IndexManifestError::InvalidJson(_) => "fix JSON syntax and validate with a local formatter",
IndexManifestError::UnsupportedVersion(_) => "set version to index.idx/v1",
IndexManifestError::InvalidSourceUrl(_) => {
"pass a valid same-origin source URL with --source-url"
}
IndexManifestError::InvalidPageUrl(_) => "provide a valid page URL for lint context",
IndexManifestError::CrossOrigin { .. } => {
"serve index.idx from the same origin as the target page"
}
IndexManifestError::InvalidScope(_) => "use an absolute scope path (for example /docs)",
IndexManifestError::OutOfScope { .. } => "adjust scope so it includes the tested page path",
IndexManifestError::TooManyHints { .. } => {
"reduce hint count in this category to <= 64 entries"
}
IndexManifestError::InvalidHint { .. } => {
"simplify selectors/text values and keep hints within protocol limits"
}
}
}
fn run_artifact_command<I, S>(mut args: I) -> Result<CliAction, String>
where
I: Iterator<Item = S>,
S: AsRef<str>,
{
let Some(command) = args.next() else {
return Ok(CliAction::Help(help()));
};
match command.as_ref() {
"inspect" => {
let Some(target) = args.next() else {
return Ok(CliAction::Help(help()));
};
if args.next().is_some() {
return Err("too many arguments".to_owned());
}
let normalized = normalize_url_input(target.as_ref());
let canonical = IndexUrl::parse(&normalized).map_err(|error| error.to_string())?;
let store = ArtifactStore::new(artifact_store_root());
let mut lines = vec![
"index-artifact-inspect-v1".to_owned(),
format!("target: {normalized}"),
format!("store_root: {}", store.root().display()),
];
for context in [
ArtifactContext::LiveGet,
ArtifactContext::LiveSubmit,
ArtifactContext::Offline,
] {
match store
.load(&canonical, context)
.map_err(|error| error.to_string())?
{
Some(artifact) => {
let freshness = match artifact.freshness(unix_timestamp_secs()) {
ArtifactFreshness::Fresh => "fresh",
ArtifactFreshness::Stale => "stale",
};
lines.push(format!(
"context: {}\n status: present\n freshness: {freshness}\n canonical_url: {}\n final_url: {}\n stored_at_unix_secs: {}\n max_age_secs: {}",
context.as_str(),
artifact.canonical_url,
artifact.final_url,
artifact.stored_at_unix_secs,
artifact.max_age_secs
));
}
None => {
lines.push(format!("context: {}\n status: missing", context.as_str()));
}
}
}
Ok(CliAction::Print(lines.join("\n")))
}
other => Err(format!("unsupported artifact command: {other}")),
}
}
fn adapter_check_report(fixture: &str) -> Result<String, String> {
let document = load_offline_document(fixture)?;
let quality = document.metadata.quality.as_ref();
let adapter = document
.metadata
.adapter_id
.as_ref()
.map_or("none", |id| id.as_str());
let category = quality.map_or("unknown", |quality| quality.category.as_str());
let score = quality.map_or(0, |quality| quality.score);
let fallback = if adapter == "none" {
quality.and_then(|quality| quality.reasons.first()).map_or(
"no adapter matched; generic transformer used",
String::as_str,
)
} else {
"adapter emitted task view"
};
let counts = adapter_counts(&document);
let markdown = extract_document(&document, ExtractFormat::Markdown);
Ok(format!(
"index-adapter-check-v1\nfixture: {fixture}\nadapter: {adapter}\nsupport_tier: {}\nquality: {category}\nquality_score: {score}\nfallback_reason: {fallback}\nnodes: {}\nlinks: {}\nforms: {}\ntables: {}\nregions: {}\nchecklist: docs/FIXTURE_INTAKE.md\n--- markdown ---\n{markdown}",
adapter_support_tier(category),
counts.nodes,
counts.links,
counts.forms,
counts.tables,
counts.regions,
))
}
fn adapter_support_tier(category: &str) -> u8 {
match category {
"adapter" => 3,
"strong-generic" => 2,
"partial-generic" | "fallback" => 1,
"failed" => 0,
_ => 0,
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
struct AdapterManifest {
adapter: String,
support_tier: u8,
nodes: usize,
links: usize,
forms: usize,
tables: usize,
regions: usize,
markdown_contains: Vec<String>,
}
fn adapter_manifest_scaffold(fixture: &str) -> Result<String, String> {
let document = load_offline_document(fixture)?;
let quality = document.metadata.quality.as_ref();
let adapter = document
.metadata
.adapter_id
.as_ref()
.map_or("none", |id| id.as_str());
let category = quality.map_or("unknown", |quality| quality.category.as_str());
let counts = adapter_counts(&document);
let markdown = extract_document(&document, ExtractFormat::Markdown);
let title_line = markdown.lines().next().unwrap_or_default();
Ok(format!(
"index-adapter-manifest-v1\nfixture: {fixture}\nadapter: {adapter}\nsupport_tier: {}\nnodes: {}\nlinks: {}\nforms: {}\ntables: {}\nregions: {}\nmarkdown_contains: {}\n",
adapter_support_tier(category),
counts.nodes,
counts.links,
counts.forms,
counts.tables,
counts.regions,
title_line
))
}
fn adapter_manifest_diff(fixture: &str, manifest_path: &str) -> Result<String, String> {
let expected = parse_adapter_manifest(
&fs::read_to_string(manifest_path)
.map_err(|error| format!("failed to read {manifest_path}: {error}"))?,
)?;
let document = load_offline_document(fixture)?;
let quality = document.metadata.quality.as_ref();
let adapter = document
.metadata
.adapter_id
.as_ref()
.map_or("none", |id| id.as_str());
let category = quality.map_or("unknown", |quality| quality.category.as_str());
let counts = adapter_counts(&document);
let markdown = extract_document(&document, ExtractFormat::Markdown);
let mut mismatches = Vec::new();
push_manifest_mismatch(&mut mismatches, "adapter", &expected.adapter, adapter);
push_manifest_mismatch(
&mut mismatches,
"support_tier",
&expected.support_tier.to_string(),
&adapter_support_tier(category).to_string(),
);
push_manifest_mismatch(
&mut mismatches,
"nodes",
&expected.nodes.to_string(),
&counts.nodes.to_string(),
);
push_manifest_mismatch(
&mut mismatches,
"links",
&expected.links.to_string(),
&counts.links.to_string(),
);
push_manifest_mismatch(
&mut mismatches,
"forms",
&expected.forms.to_string(),
&counts.forms.to_string(),
);
push_manifest_mismatch(
&mut mismatches,
"tables",
&expected.tables.to_string(),
&counts.tables.to_string(),
);
push_manifest_mismatch(
&mut mismatches,
"regions",
&expected.regions.to_string(),
&counts.regions.to_string(),
);
for needle in &expected.markdown_contains {
if !markdown.contains(needle) {
mismatches.push(format!("markdown_contains missing: {needle}"));
}
}
let status = if mismatches.is_empty() {
"ok"
} else {
"changed"
};
let mut output = format!(
"index-adapter-diff-v1\nfixture: {fixture}\nmanifest: {manifest_path}\nstatus: {status}\n"
);
for mismatch in mismatches {
output.push_str("mismatch: ");
output.push_str(&mismatch);
output.push('\n');
}
Ok(output)
}
fn parse_adapter_manifest(input: &str) -> Result<AdapterManifest, String> {
let mut lines = input.lines();
if lines.next() != Some("index-adapter-manifest-v1") {
return Err("adapter manifest missing header".to_owned());
}
let mut adapter = None;
let mut support_tier = None;
let mut counts = AdapterCounts::default();
let mut markdown_contains = Vec::new();
for line in lines {
let Some((key, value)) = line.split_once(": ") else {
continue;
};
match key {
"adapter" => adapter = Some(value.to_owned()),
"support_tier" => support_tier = Some(parse_manifest_usize(key, value)? as u8),
"nodes" => counts.nodes = parse_manifest_usize(key, value)?,
"links" => counts.links = parse_manifest_usize(key, value)?,
"forms" => counts.forms = parse_manifest_usize(key, value)?,
"tables" => counts.tables = parse_manifest_usize(key, value)?,
"regions" => counts.regions = parse_manifest_usize(key, value)?,
"markdown_contains" => markdown_contains.push(value.to_owned()),
_ => {}
}
}
Ok(AdapterManifest {
adapter: adapter.ok_or_else(|| "adapter manifest missing adapter".to_owned())?,
support_tier: support_tier
.ok_or_else(|| "adapter manifest missing support_tier".to_owned())?,
nodes: counts.nodes,
links: counts.links,
forms: counts.forms,
tables: counts.tables,
regions: counts.regions,
markdown_contains,
})
}
fn parse_manifest_usize(key: &str, value: &str) -> Result<usize, String> {
value
.parse::<usize>()
.map_err(|error| format!("invalid {key} in adapter manifest: {error}"))
}
fn push_manifest_mismatch(mismatches: &mut Vec<String>, key: &str, expected: &str, actual: &str) {
if expected != actual {
mismatches.push(format!("{key} expected {expected} actual {actual}"));
}
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
struct AdapterCounts {
nodes: usize,
links: usize,
forms: usize,
tables: usize,
regions: usize,
}
fn adapter_counts(document: &IndexDocument) -> AdapterCounts {
let mut counts = AdapterCounts::default();
for node in &document.nodes {
count_adapter_node(node, &mut counts);
}
counts
}
fn count_adapter_node(node: &IndexNode, counts: &mut AdapterCounts) {
counts.nodes += 1;
match node {
IndexNode::Link(_) => counts.links += 1,
IndexNode::Form(_) => counts.forms += 1,
IndexNode::Table { .. } => counts.tables += 1,
IndexNode::Section { nodes, .. } => {
counts.regions += 1;
for child in nodes {
count_adapter_node(child, counts);
}
}
_ => {}
}
}
fn run_shelf_command<I, S, F>(
args: I,
stdin: &mut dyn Read,
fetcher: &F,
) -> Result<CliAction, String>
where
I: Iterator<Item = S>,
S: AsRef<str>,
F: Fetcher,
{
let paths = shelf_paths()?;
run_shelf_command_with_paths(args, stdin, fetcher, &paths)
}
fn run_shelf_command_with_paths<I, S, F>(
mut args: I,
stdin: &mut dyn Read,
fetcher: &F,
paths: &ShelfPaths,
) -> Result<CliAction, String>
where
I: Iterator<Item = S>,
S: AsRef<str>,
F: Fetcher,
{
let Some(command) = args.next() else {
return Ok(CliAction::Help(help()));
};
match command.as_ref() {
"save" => {
let Some(input) = args.next() else {
return Ok(CliAction::Help(help()));
};
if args.next().is_some() {
return Err("too many arguments".to_owned());
}
run_shelf_save(input.as_ref(), stdin, fetcher, paths)
}
"list" => {
if args.next().is_some() {
return Err("too many arguments".to_owned());
}
let shelf = load_shelf(paths)?;
Ok(CliAction::Print(format_shelf_list(&shelf)))
}
"show" => {
let Some(id) = args.next() else {
return Ok(CliAction::Help(help()));
};
if args.next().is_some() {
return Err("too many arguments".to_owned());
}
let shelf = load_shelf(paths)?;
let record = shelf
.get(id.as_ref())
.ok_or_else(|| format!("shelf record not found: {}", id.as_ref()))?;
Ok(CliAction::Print(format_shelf_record(record)))
}
"search" => {
let (format, query) = parse_shelf_search_args(args)?;
if query.trim().is_empty() {
return Ok(CliAction::Help(help()));
}
let shelf = load_shelf(paths)?;
let results = shelf.search(&query, |record| {
fs::read_to_string(&record.markdown_path).ok()
});
Ok(CliAction::Print(format_shelf_search(
&query, format, &results,
)))
}
"tag" => {
let Some(id) = args.next() else {
return Ok(CliAction::Help(help()));
};
let Some(tag) = args.next() else {
return Ok(CliAction::Help(help()));
};
if args.next().is_some() {
return Err("too many arguments".to_owned());
}
update_shelf_record(id.as_ref(), |record| record.add_tag(tag.as_ref()), paths)?;
Ok(CliAction::Print(format!(
"shelf-tagged\t{}\t{}",
id.as_ref(),
tag.as_ref()
)))
}
"note" => {
let Some(id) = args.next() else {
return Ok(CliAction::Help(help()));
};
let note = args
.map(|part| part.as_ref().to_owned())
.collect::<Vec<_>>()
.join(" ");
if note.trim().is_empty() {
return Ok(CliAction::Help(help()));
}
update_shelf_record(id.as_ref(), |record| record.set_note(note.clone()), paths)?;
Ok(CliAction::Print(format!("shelf-noted\t{}", id.as_ref())))
}
other => Err(format!("unsupported shelf command: {other}")),
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum ShelfSearchFormat {
Links,
Markdown,
Json,
}
impl ShelfSearchFormat {
fn parse(input: &str) -> Option<Self> {
match input {
"links" => Some(Self::Links),
"markdown" => Some(Self::Markdown),
"json" => Some(Self::Json),
_ => None,
}
}
}
fn parse_shelf_search_args<I, S>(args: I) -> Result<(ShelfSearchFormat, String), String>
where
I: Iterator<Item = S>,
S: AsRef<str>,
{
let mut format = ShelfSearchFormat::Links;
let mut query = Vec::new();
let mut args = args.peekable();
while let Some(arg) = args.next() {
if arg.as_ref() == "--format" {
let Some(next_format) = args.next() else {
return Err("missing shelf search format".to_owned());
};
format = ShelfSearchFormat::parse(next_format.as_ref()).ok_or_else(|| {
format!("unsupported shelf search format: {}", next_format.as_ref())
})?;
} else {
query.push(arg.as_ref().to_owned());
query.extend(args.map(|part| part.as_ref().to_owned()));
break;
}
}
Ok((format, query.join(" ")))
}
fn run_shelf_save<F: Fetcher>(
input: &str,
stdin: &mut dyn Read,
fetcher: &F,
paths: &ShelfPaths,
) -> Result<CliAction, String> {
let document = load_document(input, stdin, fetcher)?;
fs::create_dir_all(&paths.exports_dir).map_err(|error| error.to_string())?;
let source_url = document.metadata.canonical_url.clone();
let quality = document
.metadata
.quality
.as_ref()
.map(|quality| quality.category.as_str().to_owned());
let citations = citation_urls(&document);
let mut record = ShelfRecord::new(
document.title.clone(),
source_url,
quality,
unix_timestamp(),
citations,
String::new(),
String::new(),
);
record.markdown_path = paths
.exports_dir
.join(format!("{}.md", record.id))
.display()
.to_string();
record.json_path = paths
.exports_dir
.join(format!("{}.json", record.id))
.display()
.to_string();
fs::write(
&record.markdown_path,
extract_document(&document, ExtractFormat::Markdown),
)
.map_err(|error| error.to_string())?;
fs::write(
&record.json_path,
extract_document(&document, ExtractFormat::Json),
)
.map_err(|error| error.to_string())?;
let mut shelf = load_shelf(paths)?;
shelf.upsert(record.clone());
save_shelf(&shelf, paths)?;
Ok(CliAction::Print(format!(
"shelf-saved\t{}\t{}",
record.id, record.title
)))
}
fn update_shelf_record(
id: &str,
update: impl FnOnce(&mut ShelfRecord),
paths: &ShelfPaths,
) -> Result<(), String> {
let mut shelf = load_shelf(paths)?;
let record = shelf
.get_mut(id)
.ok_or_else(|| format!("shelf record not found: {id}"))?;
update(record);
save_shelf(&shelf, paths)
}
fn load_shelf(paths: &ShelfPaths) -> Result<KnowledgeShelf, String> {
KnowledgeShelf::load_from_path(&paths.index_path).map_err(|error| error.to_string())
}
fn save_shelf(shelf: &KnowledgeShelf, paths: &ShelfPaths) -> Result<(), String> {
fs::create_dir_all(&paths.shelf_dir).map_err(|error| error.to_string())?;
shelf
.save_to_path(&paths.index_path)
.map_err(|error| error.to_string())
}
#[derive(Debug, Clone)]
struct ShelfPaths {
shelf_dir: std::path::PathBuf,
exports_dir: std::path::PathBuf,
index_path: std::path::PathBuf,
}
fn shelf_paths() -> Result<ShelfPaths, String> {
let shelf_dir = std::path::PathBuf::from(runtime_paths().state_dir).join("shelf");
let exports_dir = shelf_dir.join("exports");
let index_path = shelf_dir.join("index-shelf.txt");
fs::create_dir_all(&shelf_dir).map_err(|error| error.to_string())?;
Ok(ShelfPaths {
shelf_dir,
exports_dir,
index_path,
})
}
fn format_shelf_list(shelf: &KnowledgeShelf) -> String {
let mut lines = vec!["index-shelf-list-v1".to_owned()];
for record in shelf.iter() {
lines.push(format!(
"{}\t{}\t{}\t{}",
record.id,
record.title,
record.quality.as_deref().unwrap_or("unknown"),
record.source_url.as_deref().unwrap_or("")
));
}
lines.join("\n")
}
fn format_shelf_record(record: &ShelfRecord) -> String {
format!(
"index-shelf-record-v1\nid: {}\ntitle: {}\nsource_url: {}\nquality: {}\nsaved_at: {}\ntags: {}\nnote: {}\ncitations: {}\nmarkdown: {}\njson: {}",
record.id,
record.title,
record.source_url.as_deref().unwrap_or(""),
record.quality.as_deref().unwrap_or("unknown"),
record.saved_at,
record.tags.join(","),
record.note.as_deref().unwrap_or(""),
record.citations.join(","),
record.markdown_path,
record.json_path
)
}
fn format_shelf_search(
query: &str,
format: ShelfSearchFormat,
results: &[ShelfSearchResult],
) -> String {
match format {
ShelfSearchFormat::Links => format_shelf_search_links(query, results),
ShelfSearchFormat::Markdown => format_shelf_search_markdown(query, results),
ShelfSearchFormat::Json => format_shelf_search_json(query, results),
}
}
fn format_shelf_search_links(query: &str, results: &[ShelfSearchResult]) -> String {
let mut lines = vec![
"index-shelf-search-v1".to_owned(),
format!("query: {query}"),
"format: links".to_owned(),
];
for (index, result) in results.iter().enumerate() {
lines.push(format!(
"{}\t{}\t{}\t{}\t{}\t{}",
index + 1,
result.score,
result.id,
result.title,
result.source_url.as_deref().unwrap_or(""),
result.matched_fields.join(",")
));
}
lines.join("\n")
}
fn format_shelf_search_markdown(query: &str, results: &[ShelfSearchResult]) -> String {
let mut output = format!("# Shelf search: {query}\n\n");
for result in results {
output.push_str("- ");
if let Some(source_url) = &result.source_url {
output.push_str(&format!("[{}]({source_url})", result.title));
} else {
output.push_str(&result.title);
}
output.push_str(&format!(
" `{}` score {} fields {}\n",
result.id,
result.score,
result.matched_fields.join(",")
));
}
output
}
fn format_shelf_search_json(query: &str, results: &[ShelfSearchResult]) -> String {
let mut output = String::from("{\"query\":");
push_json_string(&mut output, query);
output.push_str(",\"results\":[");
for (index, result) in results.iter().enumerate() {
if index > 0 {
output.push(',');
}
output.push_str("{\"id\":");
push_json_string(&mut output, &result.id);
output.push_str(",\"title\":");
push_json_string(&mut output, &result.title);
output.push_str(",\"source_url\":");
if let Some(source_url) = &result.source_url {
push_json_string(&mut output, source_url);
} else {
output.push_str("null");
}
output.push_str(",\"score\":");
output.push_str(&result.score.to_string());
output.push_str(",\"matched_fields\":[");
for (field_index, field) in result.matched_fields.iter().enumerate() {
if field_index > 0 {
output.push(',');
}
push_json_string(&mut output, field);
}
output.push_str("]}");
}
output.push_str("]}");
output
}
fn push_json_string(output: &mut String, value: &str) {
output.push('"');
for ch in value.chars() {
match ch {
'"' => output.push_str("\\\""),
'\\' => output.push_str("\\\\"),
'\n' => output.push_str("\\n"),
'\r' => output.push_str("\\r"),
'\t' => output.push_str("\\t"),
ch if ch.is_control() => {
output.push_str("\\u");
output.push_str(&format!("{:04x}", ch as u32));
}
ch => output.push(ch),
}
}
output.push('"');
}
fn citation_urls(document: &IndexDocument) -> Vec<String> {
extract_citations_tsv(document)
.lines()
.skip(1)
.filter_map(|line| line.split('\t').nth(2).map(ToOwned::to_owned))
.collect()
}
fn unix_timestamp() -> String {
unix_timestamp_secs().to_string()
}
fn unix_timestamp_secs() -> u64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.map_or(0, |duration| duration.as_secs())
}
fn load_offline_document(input: &str) -> Result<IndexDocument, String> {
let contents = fs::read_to_string(input).map_err(|error| error.to_string())?;
if let Ok(artifact) = CaptureArtifact::from_text(&contents) {
let source_url =
IndexUrl::parse(&artifact.source_url).map_err(|error| error.to_string())?;
return Ok(transform_html(html_with_base(
&source_url,
&artifact.redacted_html,
)));
}
Ok(transform_html(contents))
}
fn run_benchmark_command<F: Fetcher>(
input: &str,
stdin: &mut dyn Read,
fetcher: &F,
) -> Result<CliAction, String> {
let mut cache = TransformedDocumentCache::new();
let source = benchmark_source(input, stdin, fetcher)?;
let first_start = Instant::now();
let first = transform_html_cached(
&mut cache,
source.source_url.as_deref(),
source.html.clone(),
);
let first_elapsed = first_start.elapsed().as_micros();
let second_start = Instant::now();
let second = transform_html_cached(
&mut cache,
source.source_url.as_deref(),
source.html.clone(),
);
let second_elapsed = second_start.elapsed().as_micros();
let report = BenchmarkReport {
source_label: source.label,
bytes: source.html.len(),
title: first.title,
nodes: document_node_count(&second),
links: document_link_count(&second),
cache_entries: cache.len(),
first_transform_micros: first_elapsed,
cached_transform_micros: second_elapsed,
};
Ok(CliAction::Print(report.display()))
}
#[derive(Debug, Clone, PartialEq, Eq)]
struct BenchmarkSource {
label: String,
source_url: Option<String>,
html: String,
}
fn benchmark_source<F: Fetcher>(
input: &str,
stdin: &mut dyn Read,
fetcher: &F,
) -> Result<BenchmarkSource, String> {
if should_fetch_input(input) {
let normalized = normalize_url_input(input);
let url = IndexUrl::parse(&normalized).map_err(|error| error.to_string())?;
let response = fetcher
.fetch(&Request { url })
.map_err(|error| error.to_string())?;
return Ok(BenchmarkSource {
label: normalized,
source_url: Some(response.final_url.to_string()),
html: html_with_base(&response.final_url, &response.body),
});
}
Ok(BenchmarkSource {
label: input.to_owned(),
source_url: None,
html: read_input(input, stdin)?,
})
}
#[derive(Debug, Clone, PartialEq, Eq)]
struct BenchmarkReport {
source_label: String,
bytes: usize,
title: String,
nodes: usize,
links: usize,
cache_entries: usize,
first_transform_micros: u128,
cached_transform_micros: u128,
}
impl BenchmarkReport {
fn display(&self) -> String {
format!(
"index-benchmark-v1\nsource: {}\nbytes: {}\ntitle: {}\nnodes: {}\nlinks: {}\ncache_entries: {}\nfirst_transform_micros: {}\ncached_transform_micros: {}",
self.source_label,
self.bytes,
self.title,
self.nodes,
self.links,
self.cache_entries,
self.first_transform_micros,
self.cached_transform_micros
)
}
}
fn document_node_count(document: &IndexDocument) -> usize {
document.nodes.iter().map(node_count).sum()
}
fn node_count(node: &IndexNode) -> usize {
match node {
IndexNode::Section { nodes, .. } => 1 + nodes.iter().map(node_count).sum::<usize>(),
_ => 1,
}
}
fn document_link_count(document: &IndexDocument) -> usize {
document.nodes.iter().map(link_count).sum()
}
fn link_count(node: &IndexNode) -> usize {
match node {
IndexNode::Link(_) => 1,
IndexNode::Section { nodes, .. } => nodes.iter().map(link_count).sum(),
_ => 0,
}
}
fn html_with_base(base_url: &IndexUrl, html: &str) -> String {
if html.to_lowercase().contains("<base ") {
return html.to_owned();
}
let base = format!(
r#"<base href="{}">"#,
base_url.as_str().replace('"', """)
);
let lower = html.to_lowercase();
if let Some(head_start) = lower.find("<head") {
if let Some(head_end) = html[head_start..].find('>') {
let insert_at = head_start + head_end + 1;
let mut output = String::with_capacity(html.len() + base.len());
output.push_str(&html[..insert_at]);
output.push_str(&base);
output.push_str(&html[insert_at..]);
return output;
}
}
format!("<head>{base}</head>\n{html}")
}
fn is_network_url(input: &str) -> bool {
input.starts_with("http://") || input.starts_with("https://")
}
fn should_fetch_input(input: &str) -> bool {
is_network_url(input) || is_schemeless_url_input(input)
}
fn normalize_url_input(input: &str) -> String {
let trimmed = input.trim();
if has_url_scheme(trimmed) {
trimmed.to_owned()
} else {
format!("https://{trimmed}")
}
}
fn is_schemeless_url_input(input: &str) -> bool {
let trimmed = input.trim();
if trimmed.is_empty()
|| trimmed == "-"
|| has_url_scheme(trimmed)
|| trimmed.starts_with('/')
|| trimmed.starts_with("./")
|| trimmed.starts_with("../")
|| fs::metadata(trimmed).is_ok()
{
return false;
}
let authority = trimmed
.split(['/', '?', '#'])
.next()
.unwrap_or_default()
.trim_end_matches('.');
authority.eq_ignore_ascii_case("localhost") || authority.contains('.')
}
fn has_url_scheme(input: &str) -> bool {
let Some(index) = input.find(':') else {
return false;
};
let scheme = &input[..index];
!scheme.is_empty()
&& scheme
.chars()
.all(|ch| ch.is_ascii_alphanumeric() || matches!(ch, '+' | '-' | '.'))
&& scheme
.chars()
.next()
.is_some_and(|ch| ch.is_ascii_alphabetic())
}
fn read_input_with_limits(
input: &str,
stdin: &mut dyn Read,
limits: ContentLimits,
) -> Result<String, String> {
if input == "-" {
let mut buffer = String::new();
stdin
.read_to_string(&mut buffer)
.map_err(|error| format!("failed to read stdin: {error}"))?;
check_content_size(&buffer, limits).map_err(|error| error.to_string())?;
return Ok(buffer);
}
if is_network_url(input) {
return Err(
"network URLs must be opened through the browser entry point, not as local files"
.to_owned(),
);
}
let html =
fs::read_to_string(input).map_err(|error| format!("failed to read {input}: {error}"))?;
check_content_size(&html, limits).map_err(|error| error.to_string())?;
Ok(html)
}
fn help() -> String {
"Index terminal browser prototype\n\nUsage:\n index\n index <url-or-local-html-file>\n index - < file.html\n index quickstart\n index --profile reader|docs|links|research|compact|verbose [url-or-local-html-file]\n index --plain <url-or-local-html-file>\n index --plain - < file.html\n index --extract markdown|links|json <url-or-local-html-file>\n index --extract markdown|links|json - < file.html\n index --save markdown|json <url-or-local-html-file> <output-file>\n index --citations <url-or-local-html-file>\n index --section <heading> <url-or-local-html-file>\n index --batch-extract markdown|json <local-html-or-capture-artifact>...\n index --ai-offline explain|summarize|extract <url-or-local-html-file>\n index --ai-offline explain|summarize|extract - < file.html\n index --benchmark <url-or-local-html-file>\n index --benchmark - < file.html\n index doctor\n index compatibility-slo [--top100 <matrix.tsv>] [--forum <matrix.tsv>] [--min-readability <0..1>] [--min-actionability <0..1>] [--min-failure-quality <0..1>]\n index compatibility-slo-v2 [--top100 <matrix.tsv>] [--forum <matrix.tsv>] [--baseline-top100 <matrix.tsv>] [--baseline-forum <matrix.tsv>] [--min-readability <0..1>] [--min-actionability <0..1>] [--min-failure-quality <0..1>]\n index compatibility-backlog [--top100 <matrix.tsv>] [--forum <matrix.tsv>] [--top <n>]\n index compatibility-live-variance [--targets <targets.tsv>] [--runs <runs.tsv>] [--window <n>]\n index compatibility-recovery-plan <url>\n index compatibility-recovery-gate [--top100 <matrix.tsv>] [--forum <matrix.tsv>] [--live-targets <targets.tsv>] [--live-runs <runs.tsv>] [--live-window <n>] [--max-live-flake-rate <0..1>] [--max-live-timeout-rate <0..1>] [--min-readability <0..1>] [--min-actionability <0..1>] [--min-failure-quality <0..1>]\n index compatibility-pack lint <pack-file> <page-url>\n index compatibility-pack inspect <page-url>\n index compatibility-pack install <pack-file> [--user|--trusted]\n index compatibility-pack update <pack-id> <pack-file> [--user|--trusted]\n index compatibility-pack list\n index compatibility-pack remove <pack-id>\n index compatibility-pack rollback <pack-id>\n index compatibility-pack sign <pack-file> <key-id> <secret>\n index compatibility-pack verify <pack-file> <key-id> <secret> <signature>\n index auth-assist inspect <url>\n index auth-assist import <cookie-bundle-file>\n index auth-assist export <output-file>\n index auth-assist diagnose-submit <url> <status-code> <message>\n index challenge-diagnose <url> <message-or-html-file>\n index idx lint <manifest-file> <page-url> [--source-url <manifest-url>]\n index adapter check <local-html-or-capture-artifact>\n index adapter scaffold <local-html-or-capture-artifact>\n index adapter diff <local-html-or-capture-artifact> <manifest>\n index artifact inspect <url>\n index shelf save <url-or-local-html-file>\n index shelf list\n index shelf show <id>\n index shelf search [--format links|markdown|json] <query>\n index shelf tag <id> <tag>\n index shelf note <id> <text>\n index capture --redact <url> <local-html-file>\n index capture --redact <url> - < file.html\n index capture --preview --redact <url> <local-html-file>\n index capture --validate <artifact-file>\n index capture --catalog-entry <fixture-path>\n index --paths\n index --version\n\nURL targets without a scheme default to HTTPS.\n\nKeys:\n j/k scroll, gg/G top/bottom, / search, f link hints, e edit form, tab next form field or open-history completion, enter submit form, :open <id-or-url>, :logs, :submit <form> field=value, :extract markdown|links|json, :pipe <cmd>, :ai explain|summarize|extract, :profile <name|auto>, :quit".to_owned()
}
fn version() -> String {
format!("index {VERSION}")
}
fn quickstart() -> String {
"index-quickstart-v1\nstep_1_open_url: index example.org\nstep_2_open_in_tui: :open <id-or-url>\nstep_3_find_links: press f\nstep_4_use_forms: press e, tab through fields, enter to submit\nstep_5_extract: :extract markdown|links|json\nstep_6_debug_network: index doctor and :logs\ntip: URL targets without a scheme default to HTTPS.".to_owned()
}
#[derive(Debug, Clone, PartialEq, Eq)]
struct RuntimePaths {
config_dir: String,
cache_dir: String,
state_dir: String,
}
impl RuntimePaths {
fn display(&self) -> String {
format!(
"config_dir={}\ncache_dir={}\nstate_dir={}",
self.config_dir, self.cache_dir, self.state_dir
)
}
}
fn doctor_report(paths: &RuntimePaths) -> String {
let config = redact_home_path(&paths.config_dir);
let cache = redact_home_path(&paths.cache_dir);
let state = redact_home_path(&paths.state_dir);
let config_check = path_health("config", &paths.config_dir);
let cache_check = path_health("cache", &paths.cache_dir);
let state_check = path_health("state", &paths.state_dir);
format!(
"index-doctor-v1\nversion={}\ntelemetry=local-only\nnetwork_probe=not-run\nnetwork_probe_policy=opt-in only; run normal URL commands when you choose to test live network access\nconfig_dir={config}\ncache_dir={cache}\nstate_dir={state}\nchecks:\n- {config_check}\n- {cache_check}\n- {state_check}\n- package: run `index --version` and compare with release notes\nsupport_report=review before sharing; no page content, cookies, or network probes are included",
version()
)
}
fn path_health(label: &str, path: &str) -> String {
let path = std::path::Path::new(path);
if path.is_dir() {
format!("{label}: ok")
} else {
format!(
"{label}: missing; remediation: create with `mkdir -p {}`",
redact_home_path(&path.display().to_string())
)
}
}
fn redact_home_path(path: &str) -> String {
let Ok(home) = env::var("HOME") else {
return path.to_owned();
};
if home.trim().is_empty() {
return path.to_owned();
}
if path == home {
return "$HOME".to_owned();
}
if let Some(rest) = path.strip_prefix(&format!("{home}/")) {
return format!("$HOME/{rest}");
}
path.to_owned()
}
fn runtime_paths() -> RuntimePaths {
RuntimePaths {
config_dir: format!("{}/index", env_dir("XDG_CONFIG_HOME", ".config")),
cache_dir: format!("{}/index", env_dir("XDG_CACHE_HOME", ".cache")),
state_dir: format!("{}/index", env_dir("XDG_STATE_HOME", ".local/state")),
}
}
fn artifact_store_root() -> PathBuf {
PathBuf::from(runtime_paths().cache_dir).join("artifacts")
}
fn artifact_runtime_enabled() -> bool {
!cfg!(test)
}
fn env_dir(variable: &str, fallback_suffix: &str) -> String {
if let Ok(value) = env::var(variable) {
if !value.trim().is_empty() {
return value;
}
}
if let Ok(home) = env::var("HOME") {
if !home.trim().is_empty() {
return format!("{home}/{fallback_suffix}");
}
}
format!("./{fallback_suffix}")
}
#[cfg(test)]
mod tests {
use std::cell::RefCell;
use std::io::Cursor;
use std::time::{Duration, SystemTime, UNIX_EPOCH};
use super::{
AdapterManifest, CliAction, CompatibilityBacklogRow, CompatibilityCorpus,
CompatibilityMilestone, CompatibilitySloFamilyDelta, CompatibilitySloRow,
CompatibilitySloThresholds, CoverageExpectedPath, CoverageStatus, FamilyCompatibilityScore,
IMAGE_PREVIEW_MAX_IMAGES_PER_DOCUMENT, LiveOutcome, LiveVarianceOriginStats,
LiveVarianceReport, RecoveryProfile, RuntimePaths, RuntimeStage, ShelfPaths,
ShelfSearchFormat, StageFlow, adapter_support_tier, auth_assist_diagnose_submit,
candidate_manifest_urls, compatibility_slo_planning_inputs,
compatibility_slo_v2_report_from_paths, default_start_page, doctor_report,
document_from_index_artifact_uncached, document_link_count, document_node_count,
enforce_stage_budget, evaluate_compatibility_backlog, evaluate_compatibility_slo,
evaluate_compatibility_slo_v2, fetch_document_cached_with_log,
fetch_document_cached_with_log_with_counter_with_progress,
fetch_document_with_artifact_runtime, format_shelf_list, format_shelf_record,
format_shelf_search, fs, html_with_base, hydrate_document_images_with_loader,
is_valid_stage_transition, load_offline_document, map_backlog_row_to_milestone,
normalize_url_input, parse_adapter_manifest, parse_forum_backlog_rows,
parse_live_variance_rows, parse_live_variance_targets, parse_shelf_search_args,
parse_slo_floor, parse_top100_backlog_rows, path_health, read_input,
read_input_with_limits, recovery_fallback_order, redact_home_path,
refresh_live_get_artifact, render_document, run_batch_extract_command, run_save_command,
run_shelf_command_with_paths, run_with_args, run_with_args_and_fetcher, submit_form_cached,
submit_form_cached_with_log, submit_form_cached_with_log_with_counter_with_progress,
submit_form_with_artifact_runtime, version,
};
use index_core::{
Form, IndexDocument, IndexNode, IndexUrl, Input, KnowledgeShelf, Link, ReaderProfile,
SectionRole, ShelfRecord, ShelfSearchResult,
};
use index_http::{FetchError, MemoryFetcher, Response, SecureFetcher};
use index_renderer::RenderOptions;
use index_security::ContentLimits;
use index_transformer::TransformedDocumentCache;
fn unique_temp_file(name: &str) -> std::path::PathBuf {
let nanos = match SystemTime::now().duration_since(UNIX_EPOCH) {
Ok(duration) => duration.as_nanos(),
Err(_) => 0,
};
std::env::temp_dir().join(format!("index-cli-{name}-{nanos}.html"))
}
fn workspace_path(path: &str) -> String {
std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
.join("../..")
.join(path)
.display()
.to_string()
}
#[test]
fn stage_flow_accepts_valid_runtime_sequence() {
let mut flow = StageFlow::default();
assert!(flow.advance(RuntimeStage::Queued).is_ok());
assert!(flow.advance(RuntimeStage::Fetching).is_ok());
assert!(flow.advance(RuntimeStage::Snapshotting).is_ok());
assert!(flow.advance(RuntimeStage::Parsing).is_ok());
assert!(flow.advance(RuntimeStage::Transforming).is_ok());
assert!(flow.advance(RuntimeStage::Scoring).is_ok());
assert!(flow.advance(RuntimeStage::Storing).is_ok());
assert!(flow.advance(RuntimeStage::Done).is_ok());
}
#[test]
fn stage_flow_rejects_invalid_runtime_sequence() {
assert!(!is_valid_stage_transition(
Some(RuntimeStage::Queued),
RuntimeStage::Done
));
let mut flow = StageFlow::default();
assert!(flow.advance(RuntimeStage::Queued).is_ok());
assert!(flow.advance(RuntimeStage::Done).is_err());
}
#[test]
fn stage_budget_enforcement_flags_timeout_excess() {
let target = "https://example.org/slow";
let ok = enforce_stage_budget(RuntimeStage::Fetching, Duration::from_secs(1), target);
assert!(ok.is_ok());
let too_slow =
enforce_stage_budget(RuntimeStage::Fetching, Duration::from_secs(121), target);
assert!(matches!(too_slow, Err(message) if message.contains("timed out")));
}
fn slo_row(
corpus: CompatibilityCorpus,
domain: &str,
min_tier: u8,
current_tier: u8,
expected_path: CoverageExpectedPath,
status: CoverageStatus,
known_limit: &str,
) -> CompatibilitySloRow {
CompatibilitySloRow {
corpus,
domain: domain.to_owned(),
family: if corpus == CompatibilityCorpus::Top100 {
"search-portal".to_owned()
} else {
"legacy-forum".to_owned()
},
intent: (corpus == CompatibilityCorpus::Top100).then(|| "search-results".to_owned()),
min_tier,
current_tier,
expected_path,
status,
known_limit: known_limit.to_owned(),
}
}
#[allow(clippy::too_many_arguments)]
fn slo_row_with_family(
corpus: CompatibilityCorpus,
domain: &str,
family: &str,
intent: Option<&str>,
min_tier: u8,
current_tier: u8,
expected_path: CoverageExpectedPath,
status: CoverageStatus,
known_limit: &str,
) -> CompatibilitySloRow {
CompatibilitySloRow {
corpus,
domain: domain.to_owned(),
family: family.to_owned(),
intent: intent.map(ToOwned::to_owned),
min_tier,
current_tier,
expected_path,
status,
known_limit: known_limit.to_owned(),
}
}
#[allow(clippy::too_many_arguments)]
fn backlog_row(
corpus: CompatibilityCorpus,
domain: &str,
family: &str,
intent: Option<&str>,
min_tier: u8,
current_tier: u8,
expected_path: CoverageExpectedPath,
status: CoverageStatus,
known_limit: &str,
) -> CompatibilityBacklogRow {
CompatibilityBacklogRow {
corpus,
domain: domain.to_owned(),
family: family.to_owned(),
intent: intent.map(ToOwned::to_owned),
min_tier,
current_tier,
expected_path,
status,
known_limit: known_limit.to_owned(),
}
}
#[test]
fn compatibility_slo_threshold_validation_enforces_floors() {
let top100_rows = vec![
slo_row(
CompatibilityCorpus::Top100,
"alpha.example",
1,
1,
CoverageExpectedPath::Generic,
CoverageStatus::Covered,
"none",
),
slo_row(
CompatibilityCorpus::Top100,
"beta.example",
1,
0,
CoverageExpectedPath::Generic,
CoverageStatus::Planned,
"none",
),
slo_row(
CompatibilityCorpus::Top100,
"blocked.example",
0,
0,
CoverageExpectedPath::Failure,
CoverageStatus::Blocked,
"auth-wall",
),
];
let forum_rows = vec![
slo_row(
CompatibilityCorpus::Forum,
"forum.example",
1,
2,
CoverageExpectedPath::Adapter,
CoverageStatus::Covered,
"none",
),
slo_row(
CompatibilityCorpus::Forum,
"forum-blocked.example",
0,
0,
CoverageExpectedPath::Failure,
CoverageStatus::Planned,
"js-required",
),
];
let strict = CompatibilitySloThresholds {
readability_floor: 0.9,
actionability_floor: 0.7,
failure_quality_floor: 1.0,
};
let report = evaluate_compatibility_slo(
"top100.tsv",
"forum.tsv",
&top100_rows,
&forum_rows,
strict,
);
assert!(!report.passed());
assert!(
report
.violations
.iter()
.any(|line| line.contains("readability score"))
);
assert!(
report
.violations
.iter()
.any(|line| line.contains("actionability score"))
);
}
#[test]
fn compatibility_slo_scores_are_deterministic_for_row_order() {
let rows_a = vec![
slo_row(
CompatibilityCorpus::Top100,
"one.example",
1,
2,
CoverageExpectedPath::Adapter,
CoverageStatus::Covered,
"none",
),
slo_row(
CompatibilityCorpus::Top100,
"two.example",
0,
0,
CoverageExpectedPath::Failure,
CoverageStatus::Blocked,
"auth-wall",
),
];
let rows_b = vec![rows_a[1].clone(), rows_a[0].clone()];
let forum_rows = vec![slo_row(
CompatibilityCorpus::Forum,
"forum.example",
1,
2,
CoverageExpectedPath::Adapter,
CoverageStatus::Covered,
"none",
)];
let thresholds = CompatibilitySloThresholds::default();
let report_a = evaluate_compatibility_slo(
"a-top100.tsv",
"a-forum.tsv",
&rows_a,
&forum_rows,
thresholds,
);
let report_b = evaluate_compatibility_slo(
"b-top100.tsv",
"b-forum.tsv",
&rows_b,
&forum_rows,
thresholds,
);
assert_eq!(report_a.readability_score, report_b.readability_score);
assert_eq!(report_a.actionability_score, report_b.actionability_score);
assert_eq!(
report_a.failure_quality_score,
report_b.failure_quality_score
);
assert_eq!(report_a.violations, report_b.violations);
}
#[test]
fn compatibility_slo_failure_guardrail_rejects_bad_failure_rows() {
let top100_rows = vec![slo_row(
CompatibilityCorpus::Top100,
"bad-failure.example",
0,
0,
CoverageExpectedPath::Failure,
CoverageStatus::Covered,
"none",
)];
let forum_rows = vec![slo_row(
CompatibilityCorpus::Forum,
"forum-ok.example",
1,
2,
CoverageExpectedPath::Adapter,
CoverageStatus::Covered,
"none",
)];
let report = evaluate_compatibility_slo(
"top100.tsv",
"forum.tsv",
&top100_rows,
&forum_rows,
CompatibilitySloThresholds {
readability_floor: 0.0,
actionability_floor: 0.0,
failure_quality_floor: 1.0,
},
);
assert!(!report.passed());
assert!(
report
.violations
.iter()
.any(|line| line.contains("failure guardrail violations"))
);
assert!(
report
.violations
.iter()
.any(|line| line.contains("top100:bad-failure.example"))
);
}
#[test]
fn compatibility_slo_v2_family_thresholds_enforce_per_family_floors() {
let top100_rows = vec![
slo_row_with_family(
CompatibilityCorpus::Top100,
"search-low.example",
"search-portal",
Some("search-results"),
1,
0,
CoverageExpectedPath::Adapter,
CoverageStatus::Planned,
"none",
),
slo_row_with_family(
CompatibilityCorpus::Top100,
"search-failure.example",
"search-portal",
Some("search-results"),
0,
0,
CoverageExpectedPath::Failure,
CoverageStatus::Blocked,
"auth-wall",
),
];
let forum_rows = Vec::new();
let report = evaluate_compatibility_slo_v2(
"top100.tsv",
"forum.tsv",
&top100_rows,
&forum_rows,
CompatibilitySloThresholds {
readability_floor: 0.0,
actionability_floor: 0.0,
failure_quality_floor: 0.0,
},
None,
);
assert!(!report.passed());
assert!(
report
.violations
.iter()
.any(|line| line.contains("family search-portal"))
);
assert!(
report
.planning_inputs
.iter()
.any(|line| line.contains("M74: readability gap"))
);
}
#[test]
fn compatibility_slo_v2_reports_evidence_gaps_for_known_limit_only_families() {
let top100_rows = vec![
slo_row_with_family(
CompatibilityCorpus::Top100,
"blocked-search.example",
"search-portal",
Some("search-results"),
1,
0,
CoverageExpectedPath::Failure,
CoverageStatus::Blocked,
"auth-wall",
),
slo_row_with_family(
CompatibilityCorpus::Top100,
"blocked-reddit.example",
"reddit",
Some("thread"),
0,
0,
CoverageExpectedPath::Failure,
CoverageStatus::Blocked,
"script-gate",
),
];
let forum_rows = Vec::new();
let report = evaluate_compatibility_slo_v2(
"top100.tsv",
"forum.tsv",
&top100_rows,
&forum_rows,
CompatibilitySloThresholds {
readability_floor: 0.0,
actionability_floor: 0.0,
failure_quality_floor: 0.0,
},
None,
);
assert_eq!(report.evidence_warnings.len(), 1);
assert!(
report
.evidence_warnings
.iter()
.any(|line| line.contains("family search-portal"))
);
assert!(
report
.planning_inputs
.iter()
.any(|line| line == "M73: evidence-gap backlog for family search-portal")
);
assert!(report.to_text().contains("evidence_warnings:"));
}
#[test]
fn compatibility_slo_v2_canonicalizes_reddit_family_aliases() {
let top100_rows = vec![slo_row_with_family(
CompatibilityCorpus::Top100,
"reddit-covered.example",
"social-community",
Some("feed-or-thread"),
1,
2,
CoverageExpectedPath::Adapter,
CoverageStatus::Covered,
"none",
)];
let forum_rows = vec![slo_row_with_family(
CompatibilityCorpus::Forum,
"reddit-blocked.example",
"reddit",
Some("thread"),
1,
1,
CoverageExpectedPath::Failure,
CoverageStatus::Blocked,
"js-required",
)];
let report = evaluate_compatibility_slo_v2(
"top100.tsv",
"forum.tsv",
&top100_rows,
&forum_rows,
CompatibilitySloThresholds {
readability_floor: 0.0,
actionability_floor: 0.0,
failure_quality_floor: 0.0,
},
None,
);
assert!(report.families.iter().any(|family| {
family.family == "social-community"
&& family.rows_total == 2
&& family.eligible_rows == 1
}));
assert!(
!report
.families
.iter()
.any(|family| family.family == "reddit")
);
}
#[test]
fn compatibility_slo_v2_delta_is_deterministic_for_row_order() {
let current_top100_a = vec![
slo_row_with_family(
CompatibilityCorpus::Top100,
"one.example",
"search-portal",
Some("search-results"),
1,
2,
CoverageExpectedPath::Adapter,
CoverageStatus::Covered,
"none",
),
slo_row_with_family(
CompatibilityCorpus::Top100,
"two.example",
"services-utility",
Some("app-shell"),
0,
0,
CoverageExpectedPath::Failure,
CoverageStatus::Blocked,
"auth-wall",
),
];
let current_top100_b = vec![current_top100_a[1].clone(), current_top100_a[0].clone()];
let current_forum_rows = vec![slo_row_with_family(
CompatibilityCorpus::Forum,
"forum.example",
"legacy-forum",
None,
1,
2,
CoverageExpectedPath::Adapter,
CoverageStatus::Covered,
"none",
)];
let baseline_top100_rows = vec![slo_row_with_family(
CompatibilityCorpus::Top100,
"one.example",
"search-portal",
Some("search-results"),
1,
1,
CoverageExpectedPath::Adapter,
CoverageStatus::Covered,
"none",
)];
let baseline_forum_rows = vec![slo_row_with_family(
CompatibilityCorpus::Forum,
"forum.example",
"legacy-forum",
None,
1,
1,
CoverageExpectedPath::Adapter,
CoverageStatus::Partial,
"none",
)];
let report_a = evaluate_compatibility_slo_v2(
"top100.tsv",
"forum.tsv",
¤t_top100_a,
¤t_forum_rows,
CompatibilitySloThresholds::default(),
Some(crate::CompatibilitySloV2Baseline {
top100_path: "baseline-top100.tsv",
forum_path: "baseline-forum.tsv",
top100_rows: &baseline_top100_rows,
forum_rows: &baseline_forum_rows,
}),
);
let report_b = evaluate_compatibility_slo_v2(
"top100.tsv",
"forum.tsv",
¤t_top100_b,
¤t_forum_rows,
CompatibilitySloThresholds::default(),
Some(crate::CompatibilitySloV2Baseline {
top100_path: "baseline-top100.tsv",
forum_path: "baseline-forum.tsv",
top100_rows: &baseline_top100_rows,
forum_rows: &baseline_forum_rows,
}),
);
assert_eq!(report_a.delta, report_b.delta);
assert_eq!(report_a.planning_inputs, report_b.planning_inputs);
}
#[test]
fn run_with_args_reports_compatibility_slo_from_custom_matrices()
-> Result<(), Box<dyn std::error::Error>> {
let top100_path = unique_temp_file("compatibility-slo-top100");
let forum_path = unique_temp_file("compatibility-slo-forum");
fs::write(
&top100_path,
"# domain\tfamily\tprimary_intent\tmin_tier\tcurrent_tier\tfixture\texpected_path\tstatus\tknown_limit\nexample.org\tsearch-portal\tsearch-results\t1\t2\tdocs/fixture.html\tadapter\tcovered\tnone\nblocked.example\tsearch-portal\tapp-shell\t0\t0\tdocs/fixture.html\tfailure\tblocked\tauth-wall\n",
)?;
fs::write(
&forum_path,
"# domain\tfamily\tmin_tier\tcurrent_tier\tfixture\texpected_path\tstatus\tknown_limit\nforum.example\tlegacy-forum\t1\t2\tdocs/fixture.html\tadapter\tcovered\tnone\n",
)?;
let mut stdin = Cursor::new(Vec::<u8>::new());
let output = run_with_args(
vec![
"compatibility-slo".to_owned(),
"--top100".to_owned(),
top100_path.display().to_string(),
"--forum".to_owned(),
forum_path.display().to_string(),
"--min-readability".to_owned(),
"0.5".to_owned(),
"--min-actionability".to_owned(),
"0.5".to_owned(),
"--min-failure-quality".to_owned(),
"1.0".to_owned(),
],
&mut stdin,
)?;
match output {
CliAction::Print(report) => {
assert!(report.contains("index-compatibility-slo-v1"));
assert!(report.contains("result: pass"));
}
other => {
return Err(std::io::Error::other(format!("unexpected output: {other:?}")).into());
}
}
let _ = fs::remove_file(top100_path);
let _ = fs::remove_file(forum_path);
Ok(())
}
#[test]
fn run_with_args_enforces_compatibility_slo_v2_gate() -> Result<(), Box<dyn std::error::Error>>
{
let top100_path = unique_temp_file("compatibility-slo-v2-top100");
let forum_path = unique_temp_file("compatibility-slo-v2-forum");
fs::write(
&top100_path,
"# domain\tfamily\tprimary_intent\tmin_tier\tcurrent_tier\tfixture\texpected_path\tstatus\tknown_limit\nexample.org\tsearch-portal\tsearch-results\t1\t0\tdocs/fixture.html\tadapter\tplanned\tnone\n",
)?;
fs::write(
&forum_path,
"# domain\tfamily\tmin_tier\tcurrent_tier\tfixture\texpected_path\tstatus\tknown_limit\nforum.example\tlegacy-forum\t1\t2\tdocs/fixture.html\tadapter\tcovered\tnone\n",
)?;
let mut stdin = Cursor::new(Vec::<u8>::new());
let output = run_with_args(
vec![
"compatibility-slo-v2".to_owned(),
"--top100".to_owned(),
top100_path.display().to_string(),
"--forum".to_owned(),
forum_path.display().to_string(),
"--min-readability".to_owned(),
"0.8".to_owned(),
"--min-actionability".to_owned(),
"0.5".to_owned(),
"--min-failure-quality".to_owned(),
"0.99".to_owned(),
],
&mut stdin,
);
match output {
Err(report) => {
assert!(report.contains("index-compatibility-slo-v2"));
assert!(report.contains("result: fail"));
assert!(report.contains("planning_inputs:"));
}
other => {
return Err(std::io::Error::other(format!(
"expected v2 gate failure, got {other:?}"
))
.into());
}
}
let _ = fs::remove_file(top100_path);
let _ = fs::remove_file(forum_path);
Ok(())
}
#[test]
fn compatibility_backlog_ranking_is_deterministic() {
let top100_rows = vec![
backlog_row(
CompatibilityCorpus::Top100,
"alpha.example",
"search-portal",
Some("search-results"),
1,
1,
CoverageExpectedPath::Generic,
CoverageStatus::Partial,
"none",
),
backlog_row(
CompatibilityCorpus::Top100,
"blocked.example",
"services-utility",
Some("app-shell"),
0,
0,
CoverageExpectedPath::Failure,
CoverageStatus::Blocked,
"auth-wall",
),
];
let forum_rows = vec![backlog_row(
CompatibilityCorpus::Forum,
"forum.example",
"legacy-forum",
None,
1,
2,
CoverageExpectedPath::Adapter,
CoverageStatus::Covered,
"none",
)];
let report_a =
evaluate_compatibility_backlog("a.tsv", "b.tsv", 10, &top100_rows, &forum_rows);
let mut top100_reordered = top100_rows.clone();
top100_reordered.reverse();
let report_b =
evaluate_compatibility_backlog("a.tsv", "b.tsv", 10, &top100_reordered, &forum_rows);
assert_eq!(report_a.ranked, report_b.ranked);
}
#[test]
fn compatibility_backlog_priority_prefers_known_limit_none_and_low_tier() {
let rows = vec![
backlog_row(
CompatibilityCorpus::Top100,
"tier2.example",
"search-portal",
Some("search-results"),
1,
2,
CoverageExpectedPath::Adapter,
CoverageStatus::Covered,
"none",
),
backlog_row(
CompatibilityCorpus::Top100,
"tier0.example",
"knowledge-reference",
Some("article-or-reference"),
1,
0,
CoverageExpectedPath::Generic,
CoverageStatus::Planned,
"none",
),
backlog_row(
CompatibilityCorpus::Top100,
"blocked.example",
"services-utility",
Some("app-shell"),
0,
0,
CoverageExpectedPath::Failure,
CoverageStatus::Blocked,
"auth-wall",
),
];
let report = evaluate_compatibility_backlog("top.tsv", "forum.tsv", 3, &rows, &[]);
assert_eq!(report.rows_total, 1);
assert_eq!(report.ranked.len(), 1);
assert_eq!(report.ranked[0].row.domain, "tier0.example");
}
#[test]
fn compatibility_backlog_row_to_milestone_mapping_is_valid() {
let readability = backlog_row(
CompatibilityCorpus::Top100,
"news.example",
"knowledge-reference",
Some("article-or-reference"),
1,
1,
CoverageExpectedPath::Generic,
CoverageStatus::Partial,
"none",
);
let actionability = backlog_row(
CompatibilityCorpus::Top100,
"search.example",
"search-portal",
Some("search-results"),
1,
1,
CoverageExpectedPath::Generic,
CoverageStatus::Partial,
"none",
);
let failure = backlog_row(
CompatibilityCorpus::Top100,
"blocked.example",
"services-utility",
Some("app-shell"),
0,
0,
CoverageExpectedPath::Failure,
CoverageStatus::Blocked,
"auth-wall",
);
let app_shell = backlog_row(
CompatibilityCorpus::Top100,
"shell.example",
"services-utility",
Some("app-shell"),
1,
1,
CoverageExpectedPath::Generic,
CoverageStatus::Partial,
"none",
);
assert_eq!(
map_backlog_row_to_milestone(&readability).0,
CompatibilityMilestone::M74
);
assert_eq!(
map_backlog_row_to_milestone(&actionability).0,
CompatibilityMilestone::M75
);
assert_eq!(
map_backlog_row_to_milestone(&failure).0,
CompatibilityMilestone::M76
);
assert_eq!(
map_backlog_row_to_milestone(&app_shell).0,
CompatibilityMilestone::M78
);
}
#[test]
fn compatibility_backlog_row_to_milestone_supports_social_community_alias_family() {
let community = backlog_row(
CompatibilityCorpus::Forum,
"community.example",
"social-community",
Some("feed-or-thread"),
1,
1,
CoverageExpectedPath::Generic,
CoverageStatus::Partial,
"none",
);
assert_eq!(
map_backlog_row_to_milestone(&community).0,
CompatibilityMilestone::M75
);
}
#[test]
fn parse_backlog_rows_canonicalize_reddit_and_generic_forum_families()
-> Result<(), Box<dyn std::error::Error>> {
let top100 = "# domain\tfamily\tprimary_intent\tmin_tier\tcurrent_tier\tfixture\texpected_path\tstatus\tknown_limit\nreddit.example\treddit\tfeed-or-thread\t1\t1\tdocs/fixture.html\tgeneric\tpartial\tnone\n";
let forum = "# domain\tfamily\tmin_tier\tcurrent_tier\tfixture\texpected_path\tstatus\tknown_limit\nforum.example\tgeneric-forum\t1\t1\tdocs/fixture.html\tgeneric\tpartial\tnone\n";
let parsed_top100 = parse_top100_backlog_rows(top100)?;
let parsed_forum = parse_forum_backlog_rows(forum)?;
assert_eq!(parsed_top100.len(), 1);
assert_eq!(parsed_forum.len(), 1);
assert_eq!(parsed_top100[0].family, "social-community");
assert_eq!(parsed_forum[0].family, "social-community");
Ok(())
}
#[test]
fn run_with_args_reports_compatibility_backlog_from_custom_matrices()
-> Result<(), Box<dyn std::error::Error>> {
let top100_path = unique_temp_file("compatibility-backlog-top100");
let forum_path = unique_temp_file("compatibility-backlog-forum");
fs::write(
&top100_path,
"# domain\tfamily\tprimary_intent\tmin_tier\tcurrent_tier\tfixture\texpected_path\tstatus\tknown_limit\nalpha.example\tsearch-portal\tsearch-results\t1\t1\tdocs/fixture.html\tgeneric\tpartial\tnone\nblocked.example\tservices-utility\tapp-shell\t0\t0\tdocs/fixture.html\tfailure\tblocked\tauth-wall\n",
)?;
fs::write(
&forum_path,
"# domain\tfamily\tmin_tier\tcurrent_tier\tfixture\texpected_path\tstatus\tknown_limit\nforum.example\tlegacy-forum\t1\t2\tdocs/fixture.html\tadapter\tcovered\tnone\n",
)?;
let mut stdin = Cursor::new(Vec::<u8>::new());
let output = run_with_args(
vec![
"compatibility-backlog".to_owned(),
"--top100".to_owned(),
top100_path.display().to_string(),
"--forum".to_owned(),
forum_path.display().to_string(),
"--top".to_owned(),
"2".to_owned(),
],
&mut stdin,
)?;
match output {
CliAction::Print(report) => {
assert!(report.contains("index-compatibility-backlog-v1"));
assert!(report.contains("alpha.example"));
assert!(!report.contains("blocked.example"));
assert!(report.contains("M75"));
}
other => {
return Err(std::io::Error::other(format!("unexpected output: {other:?}")).into());
}
}
let _ = fs::remove_file(top100_path);
let _ = fs::remove_file(forum_path);
Ok(())
}
#[test]
fn run_with_args_compatibility_pack_lint_reports_pass() -> Result<(), Box<dyn std::error::Error>>
{
let pack_path = unique_temp_file("compat-pack-lint");
fs::write(
&pack_path,
r#"{
"version": "index.pack/v1",
"id": "lint-pack",
"rules": [
{
"host": "example.org",
"path_prefix": "/docs",
"manifest": {
"version": "index.idx/v1",
"scope": "/docs",
"content": { "main_selector": "main article" }
}
}
]
}"#,
)?;
let mut stdin = Cursor::new(Vec::<u8>::new());
let output = run_with_args(
vec![
"compatibility-pack".to_owned(),
"lint".to_owned(),
pack_path.display().to_string(),
"https://example.org/docs/page".to_owned(),
],
&mut stdin,
)?;
match output {
CliAction::Print(report) => {
assert!(report.contains("index-compat-pack-lint-v1"));
assert!(report.contains("lint-pack"));
assert!(report.contains("result: pass"));
}
other => {
return Err(std::io::Error::other(format!("unexpected output: {other:?}")).into());
}
}
let _ = fs::remove_file(pack_path);
Ok(())
}
#[test]
fn run_with_args_compatibility_pack_install_rejects_unknown_option() {
let mut stdin = Cursor::new(Vec::<u8>::new());
let output = run_with_args(
vec![
"compatibility-pack".to_owned(),
"install".to_owned(),
"docs/index-idx/examples/article.index.idx.json".to_owned(),
"--remote".to_owned(),
],
&mut stdin,
);
assert!(
matches!(output, Err(error) if error.contains("unsupported compatibility-pack install option"))
);
}
#[test]
fn run_with_args_compatibility_pack_sign_and_verify_roundtrip()
-> Result<(), Box<dyn std::error::Error>> {
let pack_path = unique_temp_file("compat-pack-sign");
fs::write(
&pack_path,
r#"{"version":"index.pack/v1","id":"sign-pack","rules":[]}"#,
)?;
let mut stdin = Cursor::new(Vec::<u8>::new());
let signed = run_with_args(
vec![
"compatibility-pack".to_owned(),
"sign".to_owned(),
pack_path.display().to_string(),
"key-1".to_owned(),
"secret-1".to_owned(),
],
&mut stdin,
)?;
let signature = match signed {
CliAction::Print(report) => report
.lines()
.find_map(|line| line.strip_prefix("signature: "))
.ok_or_else(|| std::io::Error::other("missing signature line"))?
.to_owned(),
other => {
return Err(std::io::Error::other(format!("unexpected output: {other:?}")).into());
}
};
let mut stdin = Cursor::new(Vec::<u8>::new());
let verified = run_with_args(
vec![
"compatibility-pack".to_owned(),
"verify".to_owned(),
pack_path.display().to_string(),
"key-1".to_owned(),
"secret-1".to_owned(),
signature,
],
&mut stdin,
)?;
match verified {
CliAction::Print(report) => {
assert!(report.contains("index-compat-pack-verify-v1"));
assert!(report.contains("result: pass"));
}
other => {
return Err(std::io::Error::other(format!("unexpected output: {other:?}")).into());
}
}
let _ = fs::remove_file(pack_path);
Ok(())
}
#[test]
fn run_with_args_reports_compatibility_live_variance_from_custom_files()
-> Result<(), Box<dyn std::error::Error>> {
let targets_path = unique_temp_file("compat-live-targets");
let runs_path = unique_temp_file("compat-live-runs");
fs::write(
&targets_path,
"# origin\tfamily\tenabled\nnews.ycombinator.com\tsocial-community\t1\nreddit.com\tsocial-community\t1\nexample.org\tknowledge-reference\t0\n",
)?;
fs::write(
&runs_path,
"# timestamp\torigin\toutcome\tlatency_ms\tblocked_class\n2026-05-13T00:00:00Z\tnews.ycombinator.com\tok\t120\t-\n2026-05-13T00:01:00Z\tnews.ycombinator.com\ttimeout\t-\t-\n2026-05-13T00:00:00Z\treddit.com\tblocked\t80\tbot-gate\n2026-05-13T00:02:00Z\texample.org\ttransient-error\t200\t-\n",
)?;
let mut stdin = Cursor::new(Vec::<u8>::new());
let output = run_with_args(
vec![
"compatibility-live-variance".to_owned(),
"--targets".to_owned(),
targets_path.display().to_string(),
"--runs".to_owned(),
runs_path.display().to_string(),
"--window".to_owned(),
"1".to_owned(),
],
&mut stdin,
)?;
match output {
CliAction::Print(report) => {
assert!(report.contains("index-compatibility-live-variance-v1"));
assert!(report.contains("news.ycombinator.com"));
assert!(report.contains("reddit.com"));
assert!(report.contains("blocked_class_columns"));
assert!(report.contains("bot-gate"));
assert!(report.contains("global_rates: flake=0.6667 timeout=0.3333"));
}
other => {
return Err(std::io::Error::other(format!("unexpected output: {other:?}")).into());
}
}
let _ = fs::remove_file(targets_path);
let _ = fs::remove_file(runs_path);
Ok(())
}
#[test]
fn parse_live_variance_targets_rejects_invalid_rows() {
let missing_fields = parse_live_variance_targets("only-origin\n");
assert!(
matches!(missing_fields, Err(error) if error.contains("expected at least 2 fields"))
);
let bad_enabled = parse_live_variance_targets("example.org\tsearch-portal\tmaybe\n");
assert!(matches!(bad_enabled, Err(error) if error.contains("invalid enabled flag")));
}
#[test]
fn parse_live_variance_rows_rejects_invalid_rows() {
let missing_fields = parse_live_variance_rows("2026-01-01T00:00:00Z\texample.org\n");
assert!(
matches!(missing_fields, Err(error) if error.contains("expected at least 3 fields"))
);
let bad_outcome = parse_live_variance_rows("2026-01-01T00:00:00Z\texample.org\tunknown\n");
assert!(
matches!(bad_outcome, Err(error) if error.contains("invalid live-variance outcome"))
);
let bad_latency = parse_live_variance_rows("2026-01-01T00:00:00Z\texample.org\tok\tNaN\n");
assert!(
matches!(bad_latency, Err(error) if error.contains("invalid live-variance latency"))
);
let missing_origin = parse_live_variance_rows("2026-01-01T00:00:00Z\t\tok\t10\n");
assert!(
matches!(missing_origin, Err(error) if error.contains("missing timestamp or origin"))
);
}
#[test]
fn backlog_parsers_reject_invalid_rows() {
let invalid_top100 = parse_top100_backlog_rows("domain\tfamily\tintent\n");
assert!(matches!(invalid_top100, Err(error) if error.contains("expected 9 fields")));
let missing_required_top100 = parse_top100_backlog_rows(
"example.org\t\tsearch-results\t1\t1\tfixture\tgeneric\tpartial\tnone\n",
);
assert!(
matches!(missing_required_top100, Err(error) if error.contains("missing required backlog fields"))
);
let invalid_forum = parse_forum_backlog_rows("domain\tfamily\n");
assert!(matches!(invalid_forum, Err(error) if error.contains("expected 8 fields")));
let missing_required_forum =
parse_forum_backlog_rows("forum.example\t\t1\t1\tfixture\tgeneric\tpartial\tnone\n");
assert!(
matches!(missing_required_forum, Err(error) if error.contains("missing required backlog fields"))
);
}
#[test]
fn live_variance_report_text_handles_empty_blocked_classes() {
let report = LiveVarianceReport {
targets_path: "targets.tsv".to_owned(),
runs_path: "runs.tsv".to_owned(),
window: 3,
rows_total: 1,
enabled_targets: 1,
total_samples: 1,
global_flake_rate: 0.0,
global_timeout_rate: 0.0,
origins: vec![LiveVarianceOriginStats {
origin: "example.org".to_owned(),
family: "search-portal".to_owned(),
samples: 1,
ok: 1,
timeout: 0,
transient_error: 0,
blocked: 0,
flake_rate: 0.0,
timeout_rate: 0.0,
recent_flake_rate: 0.0,
previous_flake_rate: 0.0,
trend_delta: 0.0,
}],
blocked_class_counts: Vec::new(),
};
let text = report.to_text();
assert!(text.contains("blocked_classes: none"));
assert!(text.contains("origin_columns:"));
}
#[test]
fn live_outcome_parse_supports_expected_aliases() {
assert!(matches!(LiveOutcome::parse("ok"), Some(LiveOutcome::Ok)));
assert!(matches!(
LiveOutcome::parse("transient_error"),
Some(LiveOutcome::TransientError)
));
assert!(LiveOutcome::parse("mystery").is_none());
}
#[test]
fn run_with_args_live_variance_reports_option_errors() {
let mut stdin = Cursor::new(Vec::<u8>::new());
let missing_targets_value = run_with_args(
vec![
"compatibility-live-variance".to_owned(),
"--targets".to_owned(),
],
&mut stdin,
);
assert!(matches!(missing_targets_value, Ok(CliAction::Help(_))));
let mut stdin = Cursor::new(Vec::<u8>::new());
let invalid_window = run_with_args(
vec![
"compatibility-live-variance".to_owned(),
"--window".to_owned(),
"nan".to_owned(),
],
&mut stdin,
);
assert!(matches!(invalid_window, Err(error) if error.contains("invalid --window value")));
let mut stdin = Cursor::new(Vec::<u8>::new());
let out_of_range_window = run_with_args(
vec![
"compatibility-live-variance".to_owned(),
"--window".to_owned(),
"0".to_owned(),
],
&mut stdin,
);
assert!(matches!(out_of_range_window, Err(error) if error.contains("expected 1..=")));
let mut stdin = Cursor::new(Vec::<u8>::new());
let unsupported_option = run_with_args(
vec![
"compatibility-live-variance".to_owned(),
"--nope".to_owned(),
],
&mut stdin,
);
assert!(
matches!(unsupported_option, Err(error) if error.contains("unsupported compatibility-live-variance option"))
);
}
#[test]
fn compatibility_slo_planning_inputs_cover_family_and_evidence_paths() {
let thresholds = CompatibilitySloThresholds::default();
let families = vec![
FamilyCompatibilityScore {
family: "app-shell".to_owned(),
rows_total: 1,
eligible_rows: 1,
readable_rows: 1,
actionable_rows: 0,
failure_rows: 0,
guarded_failure_rows: 0,
readability_score: 1.0,
actionability_score: 0.0,
failure_quality_score: 1.0,
thresholds,
violations: vec!["actionability below floor".to_owned()],
requires_non_failure_evidence: false,
},
FamilyCompatibilityScore {
family: "search-portal".to_owned(),
rows_total: 1,
eligible_rows: 1,
readable_rows: 0,
actionable_rows: 0,
failure_rows: 1,
guarded_failure_rows: 0,
readability_score: 0.0,
actionability_score: 0.0,
failure_quality_score: 0.0,
thresholds,
violations: vec![
"readability below floor".to_owned(),
"actionability below floor".to_owned(),
"failure-quality below floor".to_owned(),
],
requires_non_failure_evidence: true,
},
];
let delta = Some(super::CompatibilitySloDeltaReport {
readability_delta: -0.10,
actionability_delta: -0.20,
failure_quality_delta: -0.30,
families: vec![CompatibilitySloFamilyDelta {
family: "search-portal".to_owned(),
readability_delta: -0.10,
actionability_delta: -0.20,
failure_quality_delta: -0.30,
}],
});
let warnings = vec![
"family search-portal missing non-failure evidence".to_owned(),
"missing evidence".to_owned(),
];
let inputs = compatibility_slo_planning_inputs(&families, delta.as_ref(), &warnings);
assert!(
inputs
.iter()
.any(|value| value.contains("M78: family-pack actionability gap"))
);
assert!(
inputs
.iter()
.any(|value| value.contains("M74: readability gap"))
);
assert!(
inputs
.iter()
.any(|value| value.contains("M75: actionability gap"))
);
assert!(
inputs
.iter()
.any(|value| value.contains("M76: failure-quality gap"))
);
assert!(
inputs
.iter()
.any(|value| value.contains("M73: evidence-gap backlog for family search-portal"))
);
assert!(
inputs
.iter()
.any(|value| value == "M73: evidence-gap backlog")
);
assert!(
inputs
.iter()
.any(|value| value.contains("M74: readability regression"))
);
assert!(
inputs
.iter()
.any(|value| value.contains("M75: actionability regression"))
);
assert!(
inputs
.iter()
.any(|value| value.contains("M76: failure-quality regression"))
);
}
#[test]
fn run_with_args_reports_recovery_plan_for_app_shell_target()
-> Result<(), Box<dyn std::error::Error>> {
let mut stdin = Cursor::new(Vec::<u8>::new());
let output = run_with_args(
vec![
"compatibility-recovery-plan".to_owned(),
"chatgpt.com".to_owned(),
],
&mut stdin,
)?;
match output {
CliAction::Print(report) => {
assert!(report.contains("index-compatibility-recovery-plan-v2"));
assert!(report.contains("target: https://chatgpt.com"));
assert!(report.contains("profile: app-shell"));
assert!(report.contains(
"fallback_order: accessibility-snapshot -> rendered-dom -> static-dom"
));
assert!(report.contains("fetching\t180"));
assert!(report.contains("snapshotting\t45"));
}
other => {
return Err(std::io::Error::other(format!("unexpected output: {other:?}")).into());
}
}
Ok(())
}
#[test]
fn run_with_args_auth_assist_diagnose_submit_classifies_token_failures()
-> Result<(), Box<dyn std::error::Error>> {
let mut stdin = Cursor::new(Vec::<u8>::new());
let output = run_with_args(
vec![
"auth-assist".to_owned(),
"diagnose-submit".to_owned(),
"news.ycombinator.com/login".to_owned(),
"403".to_owned(),
"csrf".to_owned(),
"token".to_owned(),
"expired".to_owned(),
],
&mut stdin,
)?;
match output {
CliAction::Print(report) => {
assert!(report.contains("index-auth-assist-diagnose-v1"));
assert!(report.contains("target: https://news.ycombinator.com/login"));
assert!(report.contains("class: csrf-token-mismatch"));
assert!(report.contains("remediation: refresh session cookies"));
}
other => {
return Err(std::io::Error::other(format!("unexpected output: {other:?}")).into());
}
}
Ok(())
}
#[test]
fn challenge_diagnose_report_classifies_blocked_flows() {
let report = super::challenge_diagnose_report(
"https://example.org",
"<html><body>Attention required by Cloudflare</body></html>",
);
assert!(report.contains("index-challenge-diagnose-v1"));
assert!(report.contains("class: bot-gate"));
assert!(report.contains("reason: anti-bot challenge detected"));
}
#[test]
fn auth_assist_diagnose_submit_defaults_to_https_and_reports_generic_failure() {
let report = auth_assist_diagnose_submit("example.org/login", 500, "unexpected error");
assert!(report.contains("target: https://example.org/login"));
assert!(report.contains("class: generic-submit-failure"));
}
#[test]
fn run_with_args_enforces_compatibility_recovery_gate_thresholds()
-> Result<(), Box<dyn std::error::Error>> {
let top100_path = unique_temp_file("compat-recovery-gate-top100");
let forum_path = unique_temp_file("compat-recovery-gate-forum");
let targets_path = unique_temp_file("compat-recovery-gate-targets");
let runs_path = unique_temp_file("compat-recovery-gate-runs");
fs::write(
&top100_path,
"# domain\tfamily\tprimary_intent\tmin_tier\tcurrent_tier\tfixture\texpected_path\tstatus\tknown_limit\nportal.example\tsearch-portal\tsearch-results\t1\t5\ta.html\tadapter\tcovered\tnone\nblocked.example\tservices-utility\tapp-shell\t0\t0\tb.html\tfailure\tblocked\tauth-wall\n",
)?;
fs::write(
&forum_path,
"# domain\tfamily\tmin_tier\tcurrent_tier\tfixture\texpected_path\tstatus\tknown_limit\nforum.example\tsocial-community\t1\t5\ta.html\tadapter\tcovered\tnone\n",
)?;
fs::write(
&targets_path,
"portal.example\tsearch-portal\t1\nforum.example\tsocial-community\t1\n",
)?;
fs::write(
&runs_path,
"2026-05-13T00:00:00Z\tportal.example\tok\t120\t-\n2026-05-13T00:01:00Z\tforum.example\ttimeout\t-\t-\n",
)?;
let mut stdin = Cursor::new(Vec::<u8>::new());
let output = run_with_args(
vec![
"compatibility-recovery-gate".to_owned(),
"--top100".to_owned(),
top100_path.display().to_string(),
"--forum".to_owned(),
forum_path.display().to_string(),
"--live-targets".to_owned(),
targets_path.display().to_string(),
"--live-runs".to_owned(),
runs_path.display().to_string(),
"--min-readability".to_owned(),
"0.0".to_owned(),
"--min-actionability".to_owned(),
"0.0".to_owned(),
"--min-failure-quality".to_owned(),
"0.0".to_owned(),
"--max-live-flake-rate".to_owned(),
"0.80".to_owned(),
"--max-live-timeout-rate".to_owned(),
"0.60".to_owned(),
],
&mut stdin,
)?;
match output {
CliAction::Print(report) => {
assert!(report.contains("index-compatibility-recovery-gate-v1"));
assert!(report.contains("result: pass"));
}
other => {
return Err(std::io::Error::other(format!("unexpected output: {other:?}")).into());
}
}
let mut stdin = Cursor::new(Vec::<u8>::new());
let failed = run_with_args(
vec![
"compatibility-recovery-gate".to_owned(),
"--top100".to_owned(),
top100_path.display().to_string(),
"--forum".to_owned(),
forum_path.display().to_string(),
"--live-targets".to_owned(),
targets_path.display().to_string(),
"--live-runs".to_owned(),
runs_path.display().to_string(),
"--min-readability".to_owned(),
"0.0".to_owned(),
"--min-actionability".to_owned(),
"0.0".to_owned(),
"--min-failure-quality".to_owned(),
"0.0".to_owned(),
"--max-live-flake-rate".to_owned(),
"0.20".to_owned(),
"--max-live-timeout-rate".to_owned(),
"0.20".to_owned(),
],
&mut stdin,
);
assert!(failed.is_err());
let error = failed.err().unwrap_or_default();
assert!(error.contains("index-compatibility-recovery-gate-v1"));
assert!(error.contains("result: fail"));
let _ = fs::remove_file(top100_path);
let _ = fs::remove_file(forum_path);
let _ = fs::remove_file(targets_path);
let _ = fs::remove_file(runs_path);
Ok(())
}
#[test]
fn manifest_candidate_discovery_prefers_well_known_then_header_then_html() {
let page_url = "https://example.org/docs/article";
let candidates = candidate_manifest_urls(
page_url,
r#"<html><head><link rel="index-manifest" href="/linked/index.idx"></head></html>"#,
Some(r#"</header/index.idx>; rel="index-manifest""#),
);
assert_eq!(
candidates,
vec![
"https://example.org/.well-known/index.idx".to_owned(),
"https://example.org/header/index.idx".to_owned(),
"https://example.org/linked/index.idx".to_owned(),
]
);
}
#[test]
fn artifact_runtime_emits_ordered_stage_progress_events()
-> Result<(), Box<dyn std::error::Error>> {
let mut fetcher = MemoryFetcher::new();
let target = IndexUrl::parse("https://example.org/docs")?;
fetcher.insert(
target,
"<html><head><title>Docs</title></head><body><main><p>Hello.</p></main></body></html>",
);
let mut cache = TransformedDocumentCache::default();
let mut sequence = 0_u64;
let store_root = unique_temp_file("artifact-stage-order");
let store = index_capture::ArtifactStore::new(&store_root);
let mut progress = Vec::new();
let result = fetch_document_with_artifact_runtime(
&fetcher,
&mut cache,
&mut sequence,
&store,
"https://example.org/docs",
&mut |message| progress.push(message),
)?;
assert_eq!(
result.visited_url.as_deref(),
Some("https://example.org/docs")
);
assert_eq!(
progress,
vec![
"queued https://example.org/docs".to_owned(),
"fetching https://example.org/docs".to_owned(),
"snapshotting https://example.org/docs".to_owned(),
"parsing https://example.org/docs".to_owned(),
"transforming https://example.org/docs".to_owned(),
"scoring https://example.org/docs".to_owned(),
"storing https://example.org/docs".to_owned(),
"done https://example.org/docs".to_owned(),
]
);
let _ = fs::remove_dir_all(&store_root);
Ok(())
}
#[test]
fn artifact_runtime_applies_index_manifest_date_hints() -> Result<(), Box<dyn std::error::Error>>
{
let mut fetcher = MemoryFetcher::new();
let page_url = IndexUrl::parse("https://example.org/docs/article")?;
fetcher.insert(
page_url.clone(),
r#"<html><head><title>Docs</title></head><body><main><p>Updated: 2026-05-11T12:34:00Z</p></main></body></html>"#,
);
let manifest_url = IndexUrl::parse("https://example.org/.well-known/index.idx")?;
fetcher.insert(
manifest_url,
r#"{
"version":"index.idx/v1",
"scope":"/docs",
"fields":[{"name":"updated","label":"Updated"}],
"dates":[{"field":"updated","style":"date"}]
}"#,
);
let mut cache = TransformedDocumentCache::default();
let mut sequence = 0_u64;
let store_root = unique_temp_file("artifact-manifest-hints");
let store = index_capture::ArtifactStore::new(&store_root);
let mut progress = Vec::new();
let result = fetch_document_with_artifact_runtime(
&fetcher,
&mut cache,
&mut sequence,
&store,
page_url.as_str(),
&mut |message| progress.push(message),
)?;
let rendered = render_document(&result.document, RenderOptions::default());
assert!(rendered.contains("Updated: 2026-05-11"));
assert!(!rendered.contains("2026-05-11T12:34:00Z"));
let _ = fs::remove_dir_all(&store_root);
Ok(())
}
#[test]
fn artifact_runtime_ignores_invalid_manifest_deterministically()
-> Result<(), Box<dyn std::error::Error>> {
let page_url = IndexUrl::parse("https://example.org/docs/article")?;
let html = r#"<html><head><title>Docs</title></head><body><main><p>Updated: 2026-05-11T12:34:00Z</p></main></body></html>"#;
let mut fetcher_without_manifest = MemoryFetcher::new();
fetcher_without_manifest.insert(page_url.clone(), html);
let mut fetcher_with_invalid_manifest = MemoryFetcher::new();
fetcher_with_invalid_manifest.insert(page_url.clone(), html);
let manifest_url = IndexUrl::parse("https://example.org/.well-known/index.idx")?;
fetcher_with_invalid_manifest.insert(
manifest_url,
r#"{"version":"index.idx/v1","scope":"docs","fields":[{"name":"updated","label":""}]}"#,
);
let mut cache_without_manifest = TransformedDocumentCache::default();
let mut cache_with_invalid_manifest = TransformedDocumentCache::default();
let mut sequence_a = 0_u64;
let mut sequence_b = 0_u64;
let store_root_a = unique_temp_file("artifact-manifest-invalid-a");
let store_root_b = unique_temp_file("artifact-manifest-invalid-b");
let store_a = index_capture::ArtifactStore::new(&store_root_a);
let store_b = index_capture::ArtifactStore::new(&store_root_b);
let mut progress = Vec::new();
let without_manifest = fetch_document_with_artifact_runtime(
&fetcher_without_manifest,
&mut cache_without_manifest,
&mut sequence_a,
&store_a,
page_url.as_str(),
&mut |message| progress.push(message),
)?;
let with_invalid_manifest = fetch_document_with_artifact_runtime(
&fetcher_with_invalid_manifest,
&mut cache_with_invalid_manifest,
&mut sequence_b,
&store_b,
page_url.as_str(),
&mut |message| progress.push(message),
)?;
let rendered_without_manifest =
render_document(&without_manifest.document, RenderOptions::default());
let rendered_with_invalid_manifest =
render_document(&with_invalid_manifest.document, RenderOptions::default());
assert_eq!(rendered_without_manifest, rendered_with_invalid_manifest);
let _ = fs::remove_dir_all(&store_root_a);
let _ = fs::remove_dir_all(&store_root_b);
Ok(())
}
#[test]
fn artifact_runtime_reuses_cached_artifact_without_network_fetch()
-> Result<(), Box<dyn std::error::Error>> {
let mut fetcher = MemoryFetcher::new();
let page_url = IndexUrl::parse("https://example.org/cache-hit")?;
fetcher.insert(
page_url.clone(),
"<html><head><title>Cached</title></head><body><main><p>Hello from cache.</p></main></body></html>",
);
let mut cache = TransformedDocumentCache::default();
let mut sequence = 0_u64;
let store_root = unique_temp_file("artifact-cache-hit");
let store = index_capture::ArtifactStore::new(&store_root);
let mut first_progress = Vec::new();
let first = fetch_document_with_artifact_runtime(
&fetcher,
&mut cache,
&mut sequence,
&store,
page_url.as_str(),
&mut |message| first_progress.push(message),
)?;
assert_eq!(first.visited_url.as_deref(), Some(page_url.as_str()));
assert!(
first_progress
.iter()
.any(|step| step.starts_with("fetching "))
);
let fetcher_without_network = MemoryFetcher::new();
let mut second_progress = Vec::new();
let second = fetch_document_with_artifact_runtime(
&fetcher_without_network,
&mut cache,
&mut sequence,
&store,
page_url.as_str(),
&mut |message| second_progress.push(message),
)?;
assert_eq!(second.visited_url.as_deref(), Some(page_url.as_str()));
assert_eq!(
second_progress,
vec![
format!("queued {}", page_url.as_str()),
format!("transforming {}", page_url.as_str()),
format!("scoring {}", page_url.as_str()),
format!("done {}", page_url.as_str()),
]
);
let _ = fs::remove_dir_all(&store_root);
Ok(())
}
#[test]
fn artifact_runtime_reports_failed_stage_for_invalid_cached_artifact_url()
-> Result<(), Box<dyn std::error::Error>> {
let canonical = IndexUrl::parse("https://example.org/invalid-artifact")?;
let final_url = IndexUrl::parse("https://example.org/rendered")?;
let mut document = IndexDocument::titled("Invalid Artifact");
document.push(IndexNode::Paragraph("body".to_owned()));
let mut artifact = index_capture::IndexArtifact::from_document(
&document,
&canonical,
&final_url,
index_capture::ArtifactContext::LiveGet,
0,
60,
)?;
artifact.final_url = "://bad-final-url".to_owned();
let store_root = unique_temp_file("artifact-invalid-final-url");
let store = index_capture::ArtifactStore::new(&store_root);
store.store(&artifact)?;
let fetcher = MemoryFetcher::new();
let mut cache = TransformedDocumentCache::default();
let mut sequence = 0_u64;
let mut progress = Vec::new();
let result = fetch_document_with_artifact_runtime(
&fetcher,
&mut cache,
&mut sequence,
&store,
canonical.as_str(),
&mut |message| progress.push(message),
);
assert!(result.is_err());
let error = result.err().unwrap_or_default();
assert!(!error.is_empty());
assert!(!progress.is_empty());
let _ = fs::remove_dir_all(&store_root);
Ok(())
}
#[test]
fn submit_form_artifact_runtime_tracks_stages_and_stores_artifact()
-> Result<(), Box<dyn std::error::Error>> {
let form = Form {
name: "login".to_owned(),
method: "POST".to_owned(),
action: "https://example.org/login".to_owned(),
inputs: vec![Input {
name: "user".to_owned(),
kind: "text".to_owned(),
value: Some("index".to_owned()),
required: true,
}],
buttons: Vec::new(),
};
let submission = form.submit(None, &[])?;
let mut fetcher = MemoryFetcher::new();
fetcher.insert_form_response(
&submission,
Response {
final_url: IndexUrl::parse("https://example.org/account")?,
redirects: Vec::new(),
mime_type: Some("text/html".to_owned()),
body: "<html><head><title>Account</title></head><body><main><p>Submitted form.</p></main></body></html>".to_owned(),
},
);
let store_root = unique_temp_file("submit-artifact-runtime");
let store = index_capture::ArtifactStore::new(&store_root);
let mut cache = TransformedDocumentCache::default();
let mut sequence = 0_u64;
let mut progress = Vec::new();
let result = submit_form_with_artifact_runtime(
&fetcher,
&mut cache,
&mut sequence,
&store,
&submission,
&mut |message| progress.push(message),
)?;
assert_eq!(
result.visited_url.as_deref(),
Some("https://example.org/account")
);
assert_eq!(
result.response_log.as_ref().map(|log| log.method.as_str()),
Some("POST")
);
assert_eq!(
progress,
vec![
"queued https://example.org/login".to_owned(),
"fetching https://example.org/login".to_owned(),
"snapshotting https://example.org/account".to_owned(),
"parsing https://example.org/account".to_owned(),
"transforming https://example.org/account".to_owned(),
"scoring https://example.org/account".to_owned(),
"storing https://example.org/account".to_owned(),
"done https://example.org/account".to_owned(),
]
);
assert!(
store
.load(
&submission.action,
index_capture::ArtifactContext::LiveSubmit,
)?
.is_some()
);
let mut failing_progress = Vec::new();
let empty_submitter = MemoryFetcher::new();
let result = submit_form_with_artifact_runtime(
&empty_submitter,
&mut cache,
&mut sequence,
&store,
&submission,
&mut |message| failing_progress.push(message),
);
assert!(result.is_err());
let error = result.err().unwrap_or_default();
assert!(!error.is_empty());
assert_eq!(
failing_progress,
vec![
"queued https://example.org/login".to_owned(),
"fetching https://example.org/login".to_owned(),
"failed https://example.org/login".to_owned(),
]
);
let _ = fs::remove_dir_all(&store_root);
Ok(())
}
#[test]
fn parse_slo_floor_rejects_out_of_range_values() {
let result = parse_slo_floor("readability", "1.2");
assert!(result.is_err());
let error = result.err().unwrap_or_default();
assert!(error.contains("must be between 0.0 and 1.0"));
}
#[test]
fn compatibility_slo_v2_report_reads_baselines_from_disk()
-> Result<(), Box<dyn std::error::Error>> {
let top100_path = unique_temp_file("compat-slo-v2-report-top100");
let forum_path = unique_temp_file("compat-slo-v2-report-forum");
let baseline_top100_path = unique_temp_file("compat-slo-v2-report-baseline-top100");
let baseline_forum_path = unique_temp_file("compat-slo-v2-report-baseline-forum");
fs::write(
&top100_path,
"# domain\tfamily\tprimary_intent\tmin_tier\tcurrent_tier\tfixture\texpected_path\tstatus\tknown_limit\nportal.example\tsearch-portal\tsearch-results\t2\t2\tdocs/fixture.html\tgeneric\tcovered\tnone\n",
)?;
fs::write(
&forum_path,
"# domain\tfamily\tmin_tier\tcurrent_tier\tfixture\texpected_path\tstatus\tknown_limit\nforum.example\tlegacy-forum\t2\t2\tdocs/fixture.html\tadapter\tcovered\tnone\n",
)?;
fs::write(
&baseline_top100_path,
"# domain\tfamily\tprimary_intent\tmin_tier\tcurrent_tier\tfixture\texpected_path\tstatus\tknown_limit\nportal.example\tsearch-portal\tsearch-results\t2\t2\tdocs/fixture.html\tgeneric\tcovered\tnone\n",
)?;
fs::write(
&baseline_forum_path,
"# domain\tfamily\tmin_tier\tcurrent_tier\tfixture\texpected_path\tstatus\tknown_limit\nforum.example\tlegacy-forum\t2\t2\tdocs/fixture.html\tadapter\tcovered\tnone\n",
)?;
let report = compatibility_slo_v2_report_from_paths(
top100_path.to_str().ok_or("top100 path utf8")?,
forum_path.to_str().ok_or("forum path utf8")?,
CompatibilitySloThresholds::default(),
Some(
baseline_top100_path
.to_str()
.ok_or("baseline top100 path utf8")?,
),
Some(
baseline_forum_path
.to_str()
.ok_or("baseline forum path utf8")?,
),
)?;
assert!(report.baseline_top100_path.is_some());
assert!(report.baseline_forum_path.is_some());
let _ = fs::remove_file(top100_path);
let _ = fs::remove_file(forum_path);
let _ = fs::remove_file(baseline_top100_path);
let _ = fs::remove_file(baseline_forum_path);
Ok(())
}
#[test]
fn run_with_args_idx_lint_reports_pass_for_valid_manifest()
-> Result<(), Box<dyn std::error::Error>> {
let manifest_path = unique_temp_file("idx-lint-valid");
fs::write(
&manifest_path,
r#"{
"version":"index.idx/v1",
"scope":"/docs",
"content":{"main_selector":"main article"},
"regions":[{"role":"related","selector":"aside.related","collapsed":true}],
"fields":[{"name":"updated","label":"Updated"}],
"forms":[{"name":"search","selector":"form.search","note":"Public search"}],
"dates":[{"field":"updated","style":"date"}]
}"#,
)?;
let mut stdin = Cursor::new(Vec::<u8>::new());
let output = run_with_args(
vec![
"idx".to_owned(),
"lint".to_owned(),
manifest_path.display().to_string(),
"https://example.org/docs/page".to_owned(),
],
&mut stdin,
)?;
match output {
CliAction::Print(report) => {
assert!(report.contains("index-idx-lint-v1"));
assert!(report.contains("status: pass"));
assert!(report.contains("checks: same-origin=pass scope=pass safety=pass"));
assert!(report.contains("result: pass"));
}
other => {
return Err(std::io::Error::other(format!("unexpected output: {other:?}")).into());
}
}
let _ = fs::remove_file(&manifest_path);
Ok(())
}
#[test]
fn run_with_args_idx_lint_fails_closed_for_cross_origin_source()
-> Result<(), Box<dyn std::error::Error>> {
let manifest_path = unique_temp_file("idx-lint-cross-origin");
fs::write(
&manifest_path,
r#"{"version":"index.idx/v1","scope":"/docs"}"#,
)?;
let mut stdin = Cursor::new(Vec::<u8>::new());
let error = match run_with_args(
vec![
"idx".to_owned(),
"lint".to_owned(),
manifest_path.display().to_string(),
"https://example.org/docs/page".to_owned(),
"--source-url".to_owned(),
"https://evil.example/index.idx".to_owned(),
],
&mut stdin,
) {
Err(error) => error,
Ok(other) => {
return Err(std::io::Error::other(format!(
"cross-origin lint should fail, got: {other:?}"
))
.into());
}
};
assert!(error.contains("index-idx-lint-v1"));
assert!(error.contains("status: fail"));
assert!(error.contains("same origin"));
assert!(error.contains("result: fail"));
let _ = fs::remove_file(&manifest_path);
Ok(())
}
#[test]
fn run_with_args_without_input_returns_default_start_page() {
let mut stdin = Cursor::new(Vec::<u8>::new());
let output = run_with_args(Vec::<String>::new(), &mut stdin);
assert!(matches!(output, Ok(CliAction::Tui { .. })));
if let Ok(CliAction::Tui {
document,
profile,
url_history,
}) = output
{
assert_eq!(profile, ReaderProfile::Reader);
assert!(url_history.is_empty());
assert_eq!(document.title, "Index");
assert!(
document
.nodes
.iter()
.any(|node| matches!(node, index_core::IndexNode::Paragraph(text) if text.contains(":open <url>")))
);
}
}
#[test]
fn default_start_page_contains_actionable_help() {
let document = default_start_page();
let rendered =
index_renderer::render_document(&document, index_renderer::RenderOptions::default());
assert!(rendered.contains("# Index"));
assert!(rendered.contains(":open <id-or-url>"));
assert!(rendered.contains("Example Domain -> https://example.org"));
}
fn image_node_count(nodes: &[IndexNode]) -> usize {
nodes
.iter()
.map(|node| match node {
IndexNode::Image { .. } => 1,
IndexNode::Section { nodes, .. } => image_node_count(nodes),
_ => 0,
})
.sum()
}
#[test]
fn hydrate_document_images_renders_dithered_preview() {
let bytes = include_bytes!("../../../assets/black-icon.png").to_vec();
let mut document = index_core::IndexDocument::titled("Image Preview");
document.push(IndexNode::Image {
alt: "Index icon".to_owned(),
src: Some("https://example.org/assets/icon.png".to_owned()),
});
let mut seen = Vec::new();
let mut loader = |src: &str| -> Result<Vec<u8>, String> {
seen.push(src.to_owned());
Ok(bytes.clone())
};
let hydrated = hydrate_document_images_with_loader(document, &mut loader);
let rendered =
index_renderer::render_document(&hydrated, index_renderer::RenderOptions::default());
assert_eq!(seen, vec!["https://example.org/assets/icon.png".to_owned()]);
assert_eq!(image_node_count(&hydrated.nodes), 0);
assert!(rendered.contains("bw-dither"));
assert!(rendered.contains("image source -> https://example.org/assets/icon.png"));
assert!(rendered.contains('â–ˆ') || rendered.contains('â–€') || rendered.contains('â–„'));
}
#[test]
fn hydrate_document_images_keeps_image_when_loader_fails() {
let mut document = index_core::IndexDocument::titled("Fallback");
document.push(IndexNode::Image {
alt: "Missing".to_owned(),
src: Some("https://example.org/missing.png".to_owned()),
});
let mut loader = |_src: &str| -> Result<Vec<u8>, String> { Err("fetch failed".to_owned()) };
let hydrated = hydrate_document_images_with_loader(document, &mut loader);
assert_eq!(image_node_count(&hydrated.nodes), 1);
}
#[test]
fn hydrate_document_images_respects_preview_budget() {
let bytes = include_bytes!("../../../assets/white-icon.png").to_vec();
let mut document = index_core::IndexDocument::titled("Budget");
for index in 0..(IMAGE_PREVIEW_MAX_IMAGES_PER_DOCUMENT + 2) {
document.push(IndexNode::Image {
alt: format!("image-{index}"),
src: Some(format!("https://example.org/{index}.png")),
});
}
let mut loader_calls = 0usize;
let mut loader = |_src: &str| -> Result<Vec<u8>, String> {
loader_calls += 1;
Ok(bytes.clone())
};
let hydrated = hydrate_document_images_with_loader(document, &mut loader);
assert_eq!(loader_calls, IMAGE_PREVIEW_MAX_IMAGES_PER_DOCUMENT);
assert_eq!(image_node_count(&hydrated.nodes), 2);
}
#[test]
fn hydrate_document_images_preserves_nested_sections_and_missing_sources() {
let mut document = index_core::IndexDocument::titled("Nested");
document.push(IndexNode::Section {
role: SectionRole::Main,
title: Some("Media".to_owned()),
collapsed: false,
nodes: vec![IndexNode::Image {
alt: String::new(),
src: None,
}],
});
let mut loader_calls = 0usize;
let mut loader = |_src: &str| -> Result<Vec<u8>, String> {
loader_calls += 1;
Ok(Vec::new())
};
let hydrated = hydrate_document_images_with_loader(document, &mut loader);
assert_eq!(loader_calls, 0);
assert!(matches!(
hydrated.nodes.first(),
Some(IndexNode::Section { nodes, .. })
if matches!(nodes.first(), Some(IndexNode::Image { src: None, .. }))
));
}
#[test]
fn runtime_stage_defaults_cover_default_profile_paths() {
assert_eq!(RecoveryProfile::Default.as_str(), "default");
assert_eq!(
recovery_fallback_order(RecoveryProfile::Default)[0],
"static-dom"
);
assert!(RuntimeStage::Scoring.default_timeout().is_some());
assert!(RuntimeStage::Queued.default_timeout().is_none());
}
#[test]
fn document_from_index_artifact_uncached_transforms_capture()
-> Result<(), Box<dyn std::error::Error>> {
let canonical = IndexUrl::parse("https://example.org/uncached-artifact")?;
let mut document = IndexDocument::titled("Uncached");
document.push(IndexNode::Paragraph("Body from artifact".to_owned()));
let artifact = index_capture::IndexArtifact::from_document(
&document,
&canonical,
&canonical,
index_capture::ArtifactContext::LiveGet,
0,
60,
)?;
let rendered = render_document(
&document_from_index_artifact_uncached(&artifact)?,
RenderOptions::default(),
);
assert!(rendered.contains("Body from artifact"));
Ok(())
}
#[test]
fn refresh_live_get_artifact_stores_refreshed_snapshot()
-> Result<(), Box<dyn std::error::Error>> {
let canonical = IndexUrl::parse("https://example.org/refresh-live-get")?;
let mut fetcher = MemoryFetcher::new();
fetcher.insert(
canonical.clone(),
"<html><head><title>Refreshed</title></head><body><main><p>Fresh.</p></main></body></html>",
);
let store_root = unique_temp_file("refresh-live-get");
let store = index_capture::ArtifactStore::new(&store_root);
refresh_live_get_artifact(&fetcher, &store, &canonical)?;
let stored = store
.load(&canonical, index_capture::ArtifactContext::LiveGet)?
.ok_or("missing stored artifact")?;
let rendered = render_document(
&document_from_index_artifact_uncached(&stored)?,
RenderOptions::default(),
);
assert!(rendered.contains("Fresh."));
let _ = fs::remove_dir_all(&store_root);
Ok(())
}
#[test]
fn run_save_and_batch_extract_helpers_cover_local_paths()
-> Result<(), Box<dyn std::error::Error>> {
let first = unique_temp_file("helper-save-input");
let second = unique_temp_file("helper-batch-input");
let output = unique_temp_file("helper-save-output");
fs::write(
&first,
"<html><body><main><p>first</p></main></body></html>",
)?;
fs::write(
&second,
"<html><body><main><p>second</p></main></body></html>",
)?;
let mut stdin = Cursor::new(Vec::<u8>::new());
let secure_fetcher = SecureFetcher::new(MemoryFetcher::new());
let save = run_save_command(
super::ExtractFormat::Markdown,
&first.display().to_string(),
&output.display().to_string(),
&mut stdin,
&secure_fetcher,
)?;
assert!(matches!(save, CliAction::Print(text) if text.contains("saved\tmarkdown")));
assert!(fs::read_to_string(&output)?.contains("first"));
let batch = run_batch_extract_command(
super::ExtractFormat::Markdown,
&[first.display().to_string(), second.display().to_string()],
)?;
assert!(matches!(
batch,
CliAction::Print(text) if text.contains("==> ") && text.contains("first") && text.contains("second")
));
fs::remove_file(&first)?;
fs::remove_file(&second)?;
fs::remove_file(&output)?;
Ok(())
}
#[test]
fn run_with_args_returns_version() {
let mut stdin = Cursor::new(Vec::<u8>::new());
let output = run_with_args(vec!["--version".to_owned()], &mut stdin);
assert_eq!(output, Ok(CliAction::Version(version())));
}
#[test]
fn run_with_args_returns_runtime_paths() {
let mut stdin = Cursor::new(Vec::<u8>::new());
let output = run_with_args(vec!["--paths".to_owned()], &mut stdin);
assert!(
matches!(output, Ok(CliAction::Print(paths)) if paths.contains("config_dir=") && paths.contains("cache_dir=") && paths.contains("state_dir="))
);
}
#[test]
fn run_with_args_returns_telemetry_free_doctor_report() {
let mut stdin = Cursor::new(Vec::<u8>::new());
let output = run_with_args(vec!["doctor".to_owned()], &mut stdin);
assert!(matches!(
output,
Ok(CliAction::Print(report))
if report.contains("index-doctor-v1")
&& report.contains("telemetry=local-only")
&& report.contains("network_probe=not-run")
&& report.contains("support_report=review before sharing")
&& !report.contains("https://")
));
}
#[test]
fn doctor_report_redacts_home_paths_and_has_local_remediation() {
let paths = RuntimePaths {
config_dir: "/tmp/index-doctor-test/config".to_owned(),
cache_dir: "/tmp/index-doctor-test/cache".to_owned(),
state_dir: "/tmp/index-doctor-test/state".to_owned(),
};
let report = doctor_report(&paths);
assert!(report.contains("config: missing; remediation: create with `mkdir -p"));
assert!(report.contains("cache: missing; remediation: create with `mkdir -p"));
assert!(report.contains("state: missing; remediation: create with `mkdir -p"));
assert_eq!(
redact_home_path("/definitely/not/home/index"),
"/definitely/not/home/index"
);
}
#[test]
fn run_with_args_returns_benchmark_report_from_stdin() {
let mut stdin = Cursor::new(
b"<html><title>Bench</title><main><p>Fast enough.</p><a href=\"/ref\">Ref</a></main></html>"
.to_vec(),
);
let output = run_with_args(vec!["--benchmark".to_owned(), "-".to_owned()], &mut stdin);
assert!(matches!(output, Ok(CliAction::Print(_))));
if let Ok(CliAction::Print(report)) = output {
assert!(report.contains("index-benchmark-v1"));
assert!(report.contains("source: -"));
assert!(report.contains("title: Bench"));
assert!(report.contains("cache_entries: 1"));
assert!(report.contains("cached_transform_micros:"));
}
}
#[test]
fn run_with_args_returns_benchmark_report_for_url() -> Result<(), Box<dyn std::error::Error>> {
let mut fetcher = MemoryFetcher::new();
let url = IndexUrl::parse("https://example.org/bench")?;
fetcher.insert(
url,
"<html><title>Remote Bench</title><main><p>Measured page.</p></main></html>",
);
let fetcher = SecureFetcher::new(fetcher);
let mut stdin = Cursor::new(Vec::<u8>::new());
let output = run_with_args_and_fetcher(
vec!["--benchmark".to_owned(), "example.org/bench".to_owned()],
&mut stdin,
&fetcher,
);
assert!(matches!(output, Ok(CliAction::Print(_))));
if let Ok(CliAction::Print(report)) = output {
assert!(report.contains("source: https://example.org/bench"));
assert!(report.contains("title: Remote Bench"));
}
Ok(())
}
#[test]
fn runtime_paths_display_is_stable() {
let paths = RuntimePaths {
config_dir: "/config/index".to_owned(),
cache_dir: "/cache/index".to_owned(),
state_dir: "/state/index".to_owned(),
};
assert_eq!(
paths.display(),
"config_dir=/config/index\ncache_dir=/cache/index\nstate_dir=/state/index"
);
}
#[test]
fn packaging_assets_describe_installed_command() {
let readme = include_str!("../../../README.md");
let man_page = include_str!("../../../docs/man/index.1");
let bash_completion = include_str!("../../../completions/index.bash");
let distro_notes = include_str!("../../../docs/packaging/DISTROS.md");
let clean_install = include_str!("../../../docs/packaging/CLEAN_INSTALL.md");
let package_manifest = include_str!("../../../packaging/package-manifest.tsv");
let package_smoke = include_str!("../../../scripts/package-smoke.sh");
let desktop_entry = include_str!("../../../packaging/index.desktop");
let branding = include_str!("../../../docs/BRANDING.md");
let white_icon = include_bytes!("../../../assets/white-icon.png");
let black_icon = include_bytes!("../../../assets/black-icon.png");
let packaged_banner = include_bytes!("../assets/white-banner.png");
assert!(readme.contains("assets/white-banner.png"));
assert!(man_page.contains(".B index"));
assert!(man_page.contains("--paths"));
assert!(bash_completion.contains("--version"));
assert!(distro_notes.contains("/usr/bin/index"));
assert!(distro_notes.contains("/usr/share/icons/hicolor/1024x1024/apps/index.png"));
assert!(clean_install.contains("Clean Install Validation"));
assert!(clean_install.contains("index --paths"));
assert!(package_manifest.contains("share/man/man1/index.1"));
assert!(package_manifest.contains("share/icons/hicolor/1024x1024/apps/index.png"));
assert!(package_smoke.contains("scripts/check-package-manifest.sh"));
assert!(package_smoke.contains("--plain"));
assert!(desktop_entry.contains("Icon=index"));
assert!(desktop_entry.contains("Terminal=true"));
assert!(branding.contains("JetBrainsMono Nerd Font Mono"));
assert!(white_icon.len() > 1024);
assert!(black_icon.len() > 1024);
assert!(packaged_banner.len() > 1024);
}
#[test]
fn artifact_runtime_assets_define_schema_policy_and_commands() {
let readme = include_str!("../../../README.md");
let roadmap = include_str!("../../../ROADMAP.md");
let changelog = include_str!("../../../CHANGELOG.md");
let architecture = include_str!("../../../docs/ARCHITECTURE.md");
let artifact_runtime = include_str!("../../../docs/ARTIFACT_RUNTIME.md");
let adr = include_str!("../../../docs/adr/0068-canonical-artifact-runtime.md");
let rfc = include_str!("../../../docs/rfc/0006-canonical-artifact-runtime.md");
assert!(readme.contains("index artifact inspect https://example.org/docs"));
assert!(readme.contains("docs/ARTIFACT_RUNTIME.md"));
assert!(roadmap.contains("Milestone 68 — Canonical Artifact Runtime"));
assert!(roadmap.contains("**Status:** complete."));
assert!(changelog.contains("Canonical artifact runtime milestone (M68)"));
assert!(architecture.contains("index-artifact-v1"));
assert!(artifact_runtime.contains("index-artifact-v1"));
assert!(artifact_runtime.contains("stale-while-revalidate"));
assert!(adr.contains("ArtifactStore"));
assert!(rfc.contains("canonical local artifact"));
}
#[test]
fn async_stage_assets_define_runtime_progress_contract() {
let readme = include_str!("../../../README.md");
let roadmap = include_str!("../../../ROADMAP.md");
let changelog = include_str!("../../../CHANGELOG.md");
let architecture = include_str!("../../../docs/ARCHITECTURE.md");
let async_stages = include_str!("../../../docs/ASYNC_STAGES.md");
let adr = include_str!("../../../docs/adr/0069-async-stage-execution-model.md");
let rfc = include_str!("../../../docs/rfc/0007-async-stage-execution-model.md");
assert!(readme.contains("docs/ASYNC_STAGES.md"));
assert!(roadmap.contains("Milestone 69 — Async Stage Execution Model"));
assert!(roadmap.contains("**Status:** complete."));
assert!(changelog.contains("Async stage execution milestone (M69)"));
assert!(architecture.contains("queued"));
assert!(async_stages.contains("snapshotting"));
assert!(async_stages.contains("Stage budgets"));
assert!(adr.contains("StageFlow"));
assert!(rfc.contains("queued"));
}
#[test]
fn index_idx_protocol_assets_define_safe_manifest_contract() {
let readme = include_str!("../../../README.md");
let roadmap = include_str!("../../../ROADMAP.md");
let changelog = include_str!("../../../CHANGELOG.md");
let architecture = include_str!("../../../docs/ARCHITECTURE.md");
let protocol = include_str!("../../../docs/INDEX_IDX_PROTOCOL.md");
let adr = include_str!("../../../docs/adr/0070-index-idx-protocol-v1.md");
let rfc = include_str!("../../../docs/rfc/0008-index-idx-protocol-v1.md");
assert!(readme.contains("docs/INDEX_IDX_PROTOCOL.md"));
assert!(roadmap.contains("Milestone 70 — `index.idx` Protocol v1"));
assert!(roadmap.contains("**Status:** complete."));
assert!(changelog.contains("`index.idx` protocol milestone (M70)"));
assert!(architecture.contains("optional same-origin manifest"));
assert!(protocol.contains("index.idx/v1"));
assert!(protocol.contains("fail closed"));
assert!(adr.contains("selector syntax/complexity checks"));
assert!(rfc.contains("Discovery order"));
}
#[test]
fn site_family_pack_assets_define_escalation_and_fixture_contract() {
let readme = include_str!("../../../README.md");
let roadmap = include_str!("../../../ROADMAP.md");
let changelog = include_str!("../../../CHANGELOG.md");
let catalog = include_str!("../../../docs/COVERAGE_CATALOG.md");
let discipline = include_str!("../../../docs/ADAPTER_DISCIPLINE.md");
let packs = include_str!("../../../docs/SITE_FAMILY_PACKS.md");
let adr = include_str!("../../../docs/adr/0071-site-family-compatibility-packs.md");
let rfc = include_str!("../../../docs/rfc/0009-site-family-compatibility-packs.md");
assert!(readme.contains("docs/SITE_FAMILY_PACKS.md"));
assert!(roadmap.contains("Milestone 71 — Site-Family Compatibility Packs"));
assert!(roadmap.contains("**Status:** complete."));
assert!(changelog.contains("Site-family compatibility packs milestone (M71)"));
assert!(catalog.contains("fixtures/adapters/family-forum.html"));
assert!(catalog.contains("fixtures/adapters/family-portal.html"));
assert!(discipline.contains("Family-pack coverage was evaluated"));
assert!(packs.contains("family-pack.docs"));
assert!(adr.contains("family-pack.portal"));
assert!(rfc.contains("shared heuristics"));
}
#[test]
fn alpha_hardening_assets_cover_release_paths() {
let alpha = include_str!("../../../docs/ALPHA.md");
let known_limits = include_str!("../../../docs/KNOWN_LIMITS.md");
let dogfooding = include_str!("../../../docs/DOGFOODING.md");
let smoke = include_str!("../../../scripts/alpha-smoke.sh");
assert!(alpha.contains("Alpha Release Checklist"));
assert!(alpha.contains("alpha-blocker/network"));
assert!(alpha.contains("fixture, capture artifact, diagnostic document"));
assert!(known_limits.contains("Index does not claim full web compatibility"));
assert!(known_limits.contains("No workflow should upload browsing state automatically"));
assert!(dogfooding.contains("Daily Notes Template"));
assert!(dogfooding.contains("crates/index-transformer/tests/fixtures/wiki-reference.html"));
assert!(smoke.contains("--plain"));
assert!(smoke.contains("--extract markdown"));
assert!(smoke.contains("capture --preview --redact"));
assert!(smoke.contains("adapter check"));
assert!(smoke.contains("shelf save"));
assert!(smoke.contains("INDEX_ALPHA_SMOKE_LIVE_URL"));
assert!(smoke.contains("INDEX_ALPHA_SMOKE_TUI"));
}
#[test]
fn dogfooding_corpus_assets_are_offline_and_tiered() {
let corpus_doc = include_str!("../../../docs/dogfooding/CORPUS.md");
let corpus = include_str!("../../../docs/dogfooding/corpus.tsv");
let live = include_str!("../../../docs/dogfooding/live-smoke.tsv");
let notes = include_str!("../../../docs/dogfooding/daily-notes-template.md");
let checker = include_str!("../../../scripts/check-dogfooding-corpus.sh");
assert!(corpus_doc.contains("without"));
assert!(corpus_doc.contains("fetching any URL"));
assert!(corpus.contains("mdn-adapter"));
assert!(corpus.contains("crates/index-transformer/tests/fixtures/wiki-reference.html"));
assert!(live.contains("https://developer.mozilla.org/"));
assert!(live.contains("expected_path"));
assert!(notes.contains("reproduction command"));
assert!(checker.contains("validate_committed"));
assert!(checker.contains("validate_live"));
assert!(checker.contains("http://*|https://*"));
}
#[test]
fn security_review_assets_cover_abuse_boundaries() {
let review = include_str!("../../../docs/SECURITY_REVIEW.md");
let abuse = include_str!("../../../docs/ABUSE_CASES.md");
let catalog = include_str!("../../../docs/security-abuse-cases.tsv");
let checker = include_str!("../../../scripts/check-security-review.sh");
let adr = include_str!("../../../docs/adr/0048-security-review-abuse-cases.md");
assert!(review.contains("Review Checklist"));
assert!(review.contains("Terminal Escape Audit"));
assert!(review.contains("Capture Privacy Audit"));
assert!(review.contains("Network Request Expansion Audit"));
assert!(review.contains("cargo deny check"));
assert!(abuse.contains("deferred-blocker"));
assert!(catalog.contains("terminal-control\tterminal"));
assert!(catalog.contains("capture-secret-url\tcapture"));
assert!(catalog.contains("unsafe-url-scheme\turl"));
assert!(catalog.contains("unsupported-content-type\tnetwork"));
assert!(catalog.contains("remote-local-read\tlocal-file"));
assert!(catalog.contains("dependency-advisory\tdependency"));
assert!(catalog.contains("ai-private-content\tai"));
assert!(checker.contains("make audit"));
assert!(checker.contains("cargo deny check"));
assert!(adr.contains("make security-review"));
}
#[test]
fn compatibility_assets_cover_accessibility_fallbacks() {
let compatibility = include_str!("../../../docs/COMPATIBILITY.md");
let validation = include_str!("../../../docs/COMPATIBILITY_VALIDATION.md");
let accessibility = include_str!("../../../docs/ACCESSIBILITY.md");
let checker = include_str!("../../../scripts/check-compatibility.sh");
let adr = include_str!("../../../docs/adr/0049-compatibility-accessibility-validation.md");
assert!(compatibility.contains("Capability Levels"));
assert!(validation.contains("True-color"));
assert!(validation.contains("Monochrome"));
assert!(validation.contains("Plain monospace font"));
assert!(validation.contains("No-animation mode"));
assert!(validation.contains("Narrow-Terminal Policy"));
assert!(accessibility.contains("prompt is always visible"));
assert!(accessibility.contains("screen-reader"));
assert!(checker.contains("JetBrainsMono Nerd Font Mono"));
assert!(checker.contains("plain-glyph"));
assert!(adr.contains("plain ASCII labels"));
}
#[test]
fn release_candidate_assets_cover_checksums_and_rollback() {
let release = include_str!("../../../docs/RELEASE.md");
let notes = include_str!("../../../docs/RELEASE_NOTES_TEMPLATE.md");
let script = include_str!("../../../scripts/release-candidate-dry-run.sh");
let adr = include_str!("../../../docs/adr/0050-release-candidate-process.md");
assert!(release.contains("Release Candidate Checklist"));
assert!(release.contains("Versioning Policy"));
assert!(release.contains("Artifact Checksums"));
assert!(release.contains("Rollback Notes"));
assert!(release.contains("Post-Release Smoke Checklist"));
assert!(notes.contains("Support level: alpha / beta / stable"));
assert!(notes.contains("Known limits"));
assert!(script.contains("scripts/package-smoke.sh"));
assert!(script.contains("dist/SHA256SUMS"));
assert!(script.contains("sha256sum"));
assert!(adr.contains("does not tag, sign, upload, or publish"));
}
#[test]
fn doctor_assets_document_local_only_support_reports() {
let diagnostics = include_str!("../../../docs/DIAGNOSTICS.md");
let doctor = include_str!("../../../docs/DOCTOR.md");
let adr = include_str!("../../../docs/adr/0051-telemetry-free-operational-diagnostics.md");
assert!(diagnostics.contains("index doctor"));
assert!(doctor.contains("index-doctor-v1"));
assert!(doctor.contains("telemetry=local-only"));
assert!(doctor.contains("network_probe=not-run"));
assert!(doctor.contains("Support Report Checklist"));
assert!(adr.contains("does not perform network IO"));
}
#[test]
fn beta_readiness_assets_define_narrow_support_gate() {
let beta = include_str!("../../../docs/BETA.md");
let report = include_str!("../../../docs/BETA_READINESS_REPORT.md");
let robustness = include_str!("../../../docs/ROBUSTNESS.md");
let robustness_report = include_str!("../../../docs/ROBUSTNESS_READINESS_REPORT.md");
let checker = include_str!("../../../scripts/check-beta-readiness.sh");
let adr = include_str!("../../../docs/adr/0052-beta-readiness-gate.md");
assert!(beta.contains("Beta Readiness Checklist"));
assert!(beta.contains("No known critical security or data-loss issue remains open"));
assert!(beta.contains("unsupported pages fail usefully"));
assert!(robustness.contains("Robustness Gate Checklist"));
assert!(robustness.contains("No-Hang Runtime Policy"));
assert!(robustness.contains("Deterministic Failure Policy"));
assert!(robustness_report.contains("Robustness gate is blocked"));
assert!(report.contains("Alpha Blocker Burn-Down"));
assert!(report.contains("Performance baseline"));
assert!(checker.contains("scripts/check-dogfooding-corpus.sh"));
assert!(checker.contains("scripts/check-top100-corpus.sh"));
assert!(checker.contains("scripts/check-security-review.sh"));
assert!(checker.contains("scripts/check-compatibility-slo.sh"));
assert!(checker.contains("scripts/check-compatibility-slo-v2.sh"));
assert!(checker.contains("scripts/check-package-manifest.sh"));
assert!(checker.contains("scripts/check-robustness-gate.sh"));
assert!(adr.contains("Beta support promises stay narrow"));
}
#[test]
fn stable_readiness_assets_require_beta_cycle_evidence() {
let stable = include_str!("../../../docs/STABLE.md");
let report = include_str!("../../../docs/STABLE_READINESS_REPORT.md");
let checker = include_str!("../../../scripts/check-stable-readiness.sh");
let adr = include_str!("../../../docs/adr/0053-stable-readiness-policy.md");
assert!(stable.contains("Stable Support Policy"));
assert!(stable.contains("Maintenance and Patch Policy"));
assert!(stable.contains("Compatibility Promise"));
assert!(stable.contains("Deprecation Policy"));
assert!(stable.contains("Security Response Process"));
assert!(stable.contains("Long-Term Fixture Stewardship"));
assert!(report.contains("Stable release is blocked"));
assert!(report.contains("Fixture Stewardship Notes"));
assert!(report.contains("Compatibility SLO"));
assert!(checker.contains("scripts/check-beta-readiness.sh"));
assert!(checker.contains("scripts/check-robustness-gate.sh"));
assert!(checker.contains("scripts/check-coverage-catalog.sh"));
assert!(checker.contains("scripts/check-dogfooding-corpus.sh"));
assert!(checker.contains("scripts/check-top100-corpus.sh"));
assert!(adr.contains("repeated release evidence"));
}
#[test]
fn robustness_gate_assets_define_local_reliability_command() {
let policy = include_str!("../../../docs/ROBUSTNESS.md");
let report = include_str!("../../../docs/ROBUSTNESS_READINESS_REPORT.md");
let checker = include_str!("../../../scripts/check-robustness-gate.sh");
let makefile = include_str!("../../../Makefile");
let release = include_str!("../../../docs/RELEASE.md");
let adr = include_str!("../../../docs/adr/0067-robustness-gate-v1.md");
assert!(policy.contains("Robustness Gate Checklist"));
assert!(policy.contains("No-Hang Runtime Policy"));
assert!(policy.contains("Deterministic Failure Policy"));
assert!(report.contains("Robustness gate is blocked"));
assert!(checker.contains("scripts/check-forum-corpus.sh"));
assert!(checker.contains("scripts/check-top100-corpus.sh"));
assert!(checker.contains("scripts/check-security-review.sh"));
assert!(checker.contains("scripts/check-compatibility-slo.sh"));
assert!(checker.contains("scripts/alpha-smoke.sh"));
assert!(makefile.contains("robustness-gate"));
assert!(release.contains("Run `make robustness-gate`."));
assert!(adr.contains("make robustness-gate"));
}
#[test]
fn compatibility_slo_assets_define_release_blocking_gate() {
let policy = include_str!("../../../docs/COMPATIBILITY_SLO.md");
let report = include_str!("../../../docs/COMPATIBILITY_SLO_REPORT.md");
let checker = include_str!("../../../scripts/check-compatibility-slo.sh");
let makefile = include_str!("../../../Makefile");
let beta_checker = include_str!("../../../scripts/check-beta-readiness.sh");
let adr = include_str!("../../../docs/adr/0072-compatibility-slo-gate-v1.md");
let rfc = include_str!("../../../docs/rfc/0010-compatibility-slo-gate-v1.md");
assert!(policy.contains("Compatibility SLO v1"));
assert!(policy.contains("Readability"));
assert!(policy.contains("Actionability"));
assert!(policy.contains("Failure Quality"));
assert!(report.contains("Compatibility SLO gate is blocked"));
assert!(checker.contains("index-cli -- compatibility-slo"));
assert!(checker.contains("docs/top100-corpus/matrix.tsv"));
assert!(checker.contains("docs/forum-corpus/matrix.tsv"));
assert!(makefile.contains("compatibility-slo"));
assert!(beta_checker.contains("scripts/check-compatibility-slo.sh"));
assert!(adr.contains("readability floor"));
assert!(rfc.contains("SLO Metrics"));
}
#[test]
fn compatibility_backlog_assets_define_ranked_burndown_queue() {
let burndown = include_str!("../../../docs/COMPATIBILITY_BURNDOWN.md");
let checker = include_str!("../../../scripts/check-compatibility-backlog.sh");
let makefile = include_str!("../../../Makefile");
let adr = include_str!("../../../docs/adr/0073-compatibility-backlog-queue-v1.md");
let rfc = include_str!("../../../docs/rfc/0011-compatibility-backlog-queue-v1.md");
let roadmap = include_str!("../../../ROADMAP.md");
assert!(burndown.contains("Compatibility Burn-Down Scoreboard"));
assert!(burndown.contains("index compatibility-backlog"));
assert!(burndown.contains("Top N Queue"));
assert!(checker.contains("compatibility-backlog"));
assert!(checker.contains("cmp -s"));
assert!(makefile.contains("compatibility-backlog"));
assert!(adr.contains("deterministic ranking queue"));
assert!(rfc.contains("Priority Policy"));
assert!(roadmap.contains("Milestone 73 — SLO Burn-Down Queue v1"));
}
#[test]
fn readability_v2_assets_define_fixture_and_verification_contract() {
let policy = include_str!("../../../docs/COMPATIBILITY_READABILITY_V2.md");
let checker = include_str!("../../../scripts/check-readability-lift-v2.sh");
let makefile = include_str!("../../../Makefile");
let adr = include_str!("../../../docs/adr/0074-generic-readability-lift-v2.md");
let rfc = include_str!("../../../docs/rfc/0012-generic-readability-lift-v2.md");
let roadmap = include_str!("../../../ROADMAP.md");
let changelog = include_str!("../../../CHANGELOG.md");
let catalog = include_str!("../../../docs/COVERAGE_CATALOG.md");
assert!(policy.contains("Generic Readability Lift v2"));
assert!(policy.contains("make readability-lift-v2"));
assert!(checker.contains("cargo test -p index-dom"));
assert!(checker.contains("static_reader readability_v2"));
assert!(makefile.contains("readability-lift-v2"));
assert!(adr.contains("deterministic readability-v2 policy"));
assert!(rfc.contains("root-selection scoring"));
assert!(roadmap.contains("Milestone 74 — Generic Readability Lift v2"));
assert!(changelog.contains("Generic readability lift milestone (M74)"));
assert!(catalog.contains("readability-v2-article.html"));
}
#[test]
fn actionability_v2_assets_define_link_form_and_forum_lift_contract() {
let policy = include_str!("../../../docs/COMPATIBILITY_ACTIONABILITY_V2.md");
let checker = include_str!("../../../scripts/check-actionability-lift-v2.sh");
let makefile = include_str!("../../../Makefile");
let adr = include_str!("../../../docs/adr/0075-generic-actionability-lift-v2.md");
let rfc = include_str!("../../../docs/rfc/0013-generic-actionability-lift-v2.md");
let roadmap = include_str!("../../../ROADMAP.md");
let changelog = include_str!("../../../CHANGELOG.md");
assert!(policy.contains("Generic Actionability Lift v2"));
assert!(policy.contains("forum \"next steps\""));
assert!(checker.contains("emit_links_ranks_navigation_and_result_links"));
assert!(checker.contains("form_submission_uses_default_field_values_and_allows_overrides"));
assert!(makefile.contains("actionability-lift-v2"));
assert!(adr.contains("actionability-v2 policy"));
assert!(rfc.contains("deterministic link ordering/deduplication"));
assert!(roadmap.contains("Milestone 75 — Generic Actionability Lift v2"));
assert!(changelog.contains("Generic actionability lift milestone (M75)"));
}
#[test]
fn failure_quality_v3_assets_define_blocked_flow_guardrail_contract() {
let policy = include_str!("../../../docs/FAILURE_QUALITY_V3.md");
let checker = include_str!("../../../scripts/check-failure-quality-v3.sh");
let makefile = include_str!("../../../Makefile");
let adr = include_str!("../../../docs/adr/0076-failure-quality-hardening-v3.md");
let rfc = include_str!("../../../docs/rfc/0014-failure-quality-hardening-v3.md");
let roadmap = include_str!("../../../ROADMAP.md");
let changelog = include_str!("../../../CHANGELOG.md");
assert!(policy.contains("Failure Quality Hardening v3"));
assert!(policy.contains("policy-blocked"));
assert!(checker.contains("blocked_flow_guardrails_cover_required_classes"));
assert!(checker.contains("unsupported_page_shape_never_looks_successful"));
assert!(checker.contains("blocked_top100_document_emits_remediation_and_capture_guidance"));
assert!(makefile.contains("failure-quality-v3"));
assert!(adr.contains("blocked-flow taxonomy coverage"));
assert!(rfc.contains("silent-success"));
assert!(roadmap.contains("Milestone 76 — Failure Quality Hardening v3"));
assert!(changelog.contains("Failure-quality hardening milestone (M76)"));
}
#[test]
fn index_idx_adoption_v1_assets_define_lint_and_template_contract() {
let adoption = include_str!("../../../docs/INDEX_IDX_ADOPTION.md");
let publisher = include_str!("../../../docs/INDEX_IDX_PUBLISHER_GUIDE.md");
let checker = include_str!("../../../scripts/check-index-idx-adoption-v1.sh");
let makefile = include_str!("../../../Makefile");
let adr = include_str!("../../../docs/adr/0077-index-idx-adoption-toolkit-v1.md");
let rfc = include_str!("../../../docs/rfc/0015-index-idx-adoption-toolkit-v1.md");
let roadmap = include_str!("../../../ROADMAP.md");
let changelog = include_str!("../../../CHANGELOG.md");
let template = include_str!("../../../docs/index-idx/TEMPLATE.index.idx.json");
let article_example =
include_str!("../../../docs/index-idx/examples/article.index.idx.json");
assert!(adoption.contains("`index.idx` Adoption Toolkit v1"));
assert!(adoption.contains("index idx lint"));
assert!(publisher.contains("Safe Selector Guidance"));
assert!(checker.contains("idx lint"));
assert!(checker.contains("cross-origin lint failure"));
assert!(makefile.contains("index-idx-adoption-v1"));
assert!(adr.contains("index idx lint"));
assert!(rfc.contains("local validation command"));
assert!(roadmap.contains("Milestone 77 — `index.idx` Adoption Toolkit v1"));
assert!(changelog.contains("`index.idx` adoption toolkit milestone (M77)"));
assert!(template.contains("\"version\": \"index.idx/v1\""));
assert!(article_example.contains("\"main_selector\": \"main article\""));
}
#[test]
fn family_pack_expansion_v2_assets_define_confidence_and_fixture_contract() {
let policy = include_str!("../../../docs/FAMILY_PACK_EXPANSION_V2.md");
let pack_policy = include_str!("../../../docs/SITE_FAMILY_PACKS.md");
let checker = include_str!("../../../scripts/check-family-pack-expansion-v2.sh");
let makefile = include_str!("../../../Makefile");
let adr = include_str!("../../../docs/adr/0078-family-pack-expansion-v2.md");
let rfc = include_str!("../../../docs/rfc/0016-family-pack-expansion-v2.md");
let roadmap = include_str!("../../../ROADMAP.md");
let changelog = include_str!("../../../CHANGELOG.md");
let catalog = include_str!("../../../docs/COVERAGE_CATALOG.md");
assert!(policy.contains("Family-Pack Expansion v2"));
assert!(policy.contains("family-pack.app-shell"));
assert!(policy.contains("family-pack.commerce-cards"));
assert!(policy.contains("family-pack.mixed-media"));
assert!(pack_policy.contains("Coverage targets"));
assert!(checker.contains("compatibility_pack_weak_signals_fall_back_to_generic"));
assert!(checker.contains("compatibility_pack_fixtures_cover_major_families"));
assert!(makefile.contains("family-pack-expansion-v2"));
assert!(adr.contains("minimum-confidence fallback"));
assert!(rfc.contains("confidence scoring"));
assert!(roadmap.contains("Milestone 78 — Family-Pack Expansion v2"));
assert!(changelog.contains("Family-pack expansion milestone (M78)"));
assert!(catalog.contains("family-app-shell.html"));
assert!(catalog.contains("family-commerce-cards.html"));
assert!(catalog.contains("family-mixed-media.html"));
}
#[test]
fn compatibility_slo_v2_assets_define_family_threshold_and_delta_gate() {
let policy = include_str!("../../../docs/COMPATIBILITY_SLO_V2.md");
let report = include_str!("../../../docs/COMPATIBILITY_SLO_V2_REPORT.md");
let checker = include_str!("../../../scripts/check-compatibility-slo-v2.sh");
let makefile = include_str!("../../../Makefile");
let release = include_str!("../../../docs/RELEASE.md");
let adr = include_str!("../../../docs/adr/0079-compatibility-slo-gate-v2.md");
let rfc = include_str!("../../../docs/rfc/0017-compatibility-slo-gate-v2.md");
let roadmap = include_str!("../../../ROADMAP.md");
let changelog = include_str!("../../../CHANGELOG.md");
let beta_checker = include_str!("../../../scripts/check-beta-readiness.sh");
assert!(policy.contains("Compatibility SLO v2"));
assert!(policy.contains("per-family"));
assert!(report.contains("Compatibility SLO v2 gate is blocked"));
assert!(checker.contains("compatibility-slo-v2"));
assert!(checker.contains("baseline-top100"));
assert!(checker.contains("cmp -s"));
assert!(makefile.contains("compatibility-slo-v2"));
assert!(release.contains("make compatibility-slo-v2"));
assert!(adr.contains("per-family threshold enforcement"));
assert!(rfc.contains("deterministic baseline delta"));
assert!(roadmap.contains("Milestone 79 — Compatibility SLO Gate v2"));
assert!(changelog.contains("Compatibility SLO gate milestone (M79)"));
assert!(beta_checker.contains("check-compatibility-slo-v2.sh"));
}
#[test]
fn compatibility_pack_runtime_v1_assets_define_safe_runtime_contract() {
let policy = include_str!("../../../docs/COMPATIBILITY_PACK_RUNTIME_V1.md");
let checker = include_str!("../../../scripts/check-compatibility-pack-runtime-v1.sh");
let makefile = include_str!("../../../Makefile");
let adr = include_str!("../../../docs/adr/0080-compatibility-pack-runtime-v1.md");
let rfc = include_str!("../../../docs/rfc/0018-compatibility-pack-runtime-v1.md");
let roadmap = include_str!("../../../ROADMAP.md");
let changelog = include_str!("../../../CHANGELOG.md");
let readme = include_str!("../../../README.md");
let builtin_pack = include_str!("../assets/compat/default.pack.json");
assert!(policy.contains("Compatibility Pack Runtime v1"));
assert!(policy.contains("user` > `trusted` > `built-in` > generic"));
assert!(checker.contains("compatibility-pack lint"));
assert!(checker.contains("compat_pack_runtime::tests"));
assert!(makefile.contains("compatibility-pack-runtime-v1"));
assert!(adr.contains("index.pack/v1"));
assert!(rfc.contains("compatibility-pack"));
assert!(roadmap.contains("Milestone 80 — Compatibility Pack Runtime v1"));
assert!(
roadmap.contains(
"## Milestone 80 — Compatibility Pack Runtime v1\n\n**Status:** complete."
)
);
assert!(changelog.contains("Compatibility pack runtime milestone (M80)"));
assert!(readme.contains("index compatibility-pack lint"));
assert!(builtin_pack.contains("\"version\": \"index.pack/v1\""));
}
#[test]
fn compat_lab_bootstrap_v1_assets_define_standalone_artifact_workflow() {
let policy = include_str!("../../../docs/COMPAT_LAB_BOOTSTRAP_V1.md");
let checker = include_str!("../../../scripts/check-compat-lab-bootstrap-v1.sh");
let makefile = include_str!("../../../Makefile");
let adr = include_str!("../../../docs/adr/0081-index-compat-lab-bootstrap-v1.md");
let rfc = include_str!("../../../docs/rfc/0019-index-compat-lab-bootstrap-v1.md");
let roadmap = include_str!("../../../ROADMAP.md");
let changelog = include_str!("../../../CHANGELOG.md");
let readme = include_str!("../../../README.md");
assert!(policy.contains("Compatibility Lab Bootstrap v1"));
assert!(policy.contains("index-compat-lab"));
assert!(checker.contains("index-compat-lab -- ingest"));
assert!(checker.contains("index-compat-lab -- scaffold"));
assert!(makefile.contains("compat-lab-bootstrap-v1"));
assert!(adr.contains("index-compat-lab"));
assert!(rfc.contains("standalone crate"));
assert!(roadmap.contains("Milestone 81 — `index-compat-lab` Crate Bootstrap"));
assert!(roadmap.contains(
"## Milestone 81 — `index-compat-lab` Crate Bootstrap\n\n**Status:** complete."
));
assert!(changelog.contains("bootstrap milestone (M81)"));
assert!(readme.contains("index-compat-lab"));
}
#[test]
fn compat_rule_synthesis_v1_assets_define_safety_lint_contract() {
let policy = include_str!("../../../docs/COMPAT_RULE_SYNTHESIS_V1.md");
let checker = include_str!("../../../scripts/check-compat-rule-synthesis-v1.sh");
let makefile = include_str!("../../../Makefile");
let adr = include_str!("../../../docs/adr/0082-compat-rule-synthesis-and-lint-v1.md");
let rfc = include_str!("../../../docs/rfc/0020-compat-rule-synthesis-and-lint-v1.md");
let roadmap = include_str!("../../../ROADMAP.md");
let changelog = include_str!("../../../CHANGELOG.md");
assert!(policy.contains("Rule Synthesis and Lint v1"));
assert!(policy.contains("merge-overrides"));
assert!(checker.contains("index-compat-lab -- synthesize"));
assert!(checker.contains("index-compat-lab -- lint"));
assert!(checker.contains("merge-overrides"));
assert!(makefile.contains("compat-rule-synthesis-v1"));
assert!(adr.contains("unsafe selectors"));
assert!(rfc.contains("deterministic override merge"));
assert!(roadmap.contains("Milestone 82 — Rule Synthesis and Safety Linting"));
assert!(roadmap.contains(
"## Milestone 82 — Rule Synthesis and Safety Linting\n\n**Status:** complete."
));
assert!(changelog.contains("Rule synthesis and safety linting milestone (M82)"));
}
#[test]
fn compat_pack_trust_v1_assets_define_signing_and_lifecycle_contract() {
let policy = include_str!("../../../docs/COMPAT_PACK_TRUST_V1.md");
let checker = include_str!("../../../scripts/check-compat-pack-trust-v1.sh");
let makefile = include_str!("../../../Makefile");
let adr = include_str!("../../../docs/adr/0083-compat-pack-trust-signing-v1.md");
let rfc = include_str!("../../../docs/rfc/0021-compat-pack-trust-signing-v1.md");
let roadmap = include_str!("../../../ROADMAP.md");
let changelog = include_str!("../../../CHANGELOG.md");
assert!(policy.contains("Trust and Registry v1"));
assert!(policy.contains("sign"));
assert!(policy.contains("verify"));
assert!(checker.contains("compatibility-pack sign"));
assert!(checker.contains("compatibility-pack verify"));
assert!(makefile.contains("compat-pack-trust-v1"));
assert!(adr.contains("provenance"));
assert!(rfc.contains("signing/verification"));
assert!(roadmap.contains("Milestone 83 — Trust, Signing, and Pack Registry v1"));
assert!(roadmap.contains(
"## Milestone 83 — Trust, Signing, and Pack Registry v1\n\n**Status:** complete."
));
assert!(changelog.contains("Trust/signing/registry milestone (M83)"));
}
#[test]
fn compat_pack_hotswap_v1_assets_define_rollback_contract() {
let policy = include_str!("../../../docs/COMPAT_PACK_HOTSWAP_V1.md");
let checker = include_str!("../../../scripts/check-compat-pack-hotswap-v1.sh");
let makefile = include_str!("../../../Makefile");
let adr = include_str!("../../../docs/adr/0084-compat-pack-hotswap-rollback-v1.md");
let rfc = include_str!("../../../docs/rfc/0022-compat-pack-hotswap-rollback-v1.md");
let roadmap = include_str!("../../../ROADMAP.md");
let changelog = include_str!("../../../CHANGELOG.md");
assert!(policy.contains("Hot-Swap and Rollback v1"));
assert!(policy.contains("rollback"));
assert!(checker.contains("rollback_restores_previous_pack_snapshot"));
assert!(checker.contains("compatibility-pack inspect"));
assert!(makefile.contains("compat-pack-hotswap-v1"));
assert!(adr.contains("rollback snapshots"));
assert!(rfc.contains("rollback"));
assert!(roadmap.contains("Milestone 84 — Runtime Hot-Swap and Rollback"));
assert!(
roadmap.contains(
"## Milestone 84 — Runtime Hot-Swap and Rollback\n\n**Status:** complete."
)
);
assert!(changelog.contains("Hot-swap and rollback milestone (M84)"));
}
#[test]
fn compat_pack_ci_v1_assets_define_composed_canary_gate() {
let policy = include_str!("../../../docs/COMPAT_PACK_CI_V1.md");
let checker = include_str!("../../../scripts/check-compat-pack-ci-v1.sh");
let makefile = include_str!("../../../Makefile");
let adr = include_str!("../../../docs/adr/0085-compat-pack-ci-canary-v1.md");
let rfc = include_str!("../../../docs/rfc/0023-compat-pack-ci-canary-v1.md");
let roadmap = include_str!("../../../ROADMAP.md");
let changelog = include_str!("../../../CHANGELOG.md");
assert!(policy.contains("CI and Canary Gate v1"));
assert!(checker.contains("check-compatibility-pack-runtime-v1.sh"));
assert!(checker.contains("check-compatibility-slo-v2.sh"));
assert!(checker.contains("check-compatibility-backlog.sh"));
assert!(makefile.contains("compat-pack-ci-v1"));
assert!(adr.contains("canary"));
assert!(rfc.contains("pack quality"));
assert!(roadmap.contains("Milestone 85 — Compatibility Pack CI and Canary Gate"));
assert!(roadmap.contains(
"## Milestone 85 — Compatibility Pack CI and Canary Gate\n\n**Status:** complete."
));
assert!(changelog.contains("Compatibility pack CI/canary milestone (M85)"));
}
#[test]
fn compat_no_binary_release_v1_assets_define_data_plane_release_contract() {
let policy = include_str!("../../../docs/COMPAT_DATA_RELEASE_V1.md");
let checker = include_str!("../../../scripts/check-compat-no-binary-release-v1.sh");
let makefile = include_str!("../../../Makefile");
let adr = include_str!("../../../docs/adr/0086-compatibility-data-release-workflow-v1.md");
let rfc = include_str!("../../../docs/rfc/0024-compatibility-data-release-workflow-v1.md");
let roadmap = include_str!("../../../ROADMAP.md");
let changelog = include_str!("../../../CHANGELOG.md");
assert!(policy.contains("Data-Only Release Workflow v1"));
assert!(policy.contains("Engine vs Data Boundary"));
assert!(checker.contains("check-compat-pack-ci-v1.sh"));
assert!(checker.contains("index-compat-lab -- scaffold"));
assert!(checker.contains("compatibility-pack verify"));
assert!(makefile.contains("compat-no-binary-release-v1"));
assert!(adr.contains("no-binary-release workflow"));
assert!(rfc.contains("compatibility-only releases"));
assert!(roadmap.contains("Milestone 86 — No-Binary-Release Compatibility Workflow v1"));
assert!(roadmap.contains(
"## Milestone 86 — No-Binary-Release Compatibility Workflow v1\n\n**Status:** complete."
));
assert!(changelog.contains("No-binary-release compatibility workflow milestone (M86)"));
}
#[test]
fn live_variance_v1_assets_define_opt_in_instability_contract() {
let policy = include_str!("../../../docs/LIVE_VARIANCE_V1.md");
let checker = include_str!("../../../scripts/check-live-variance-v1.sh");
let makefile = include_str!("../../../Makefile");
let adr = include_str!("../../../docs/adr/0087-live-variance-harness-v1.md");
let rfc = include_str!("../../../docs/rfc/0025-live-variance-harness-v1.md");
let roadmap = include_str!("../../../ROADMAP.md");
let changelog = include_str!("../../../CHANGELOG.md");
let targets = include_str!("../../../docs/compat-live/targets.tsv");
let runs = include_str!("../../../docs/compat-live/runs.tsv");
assert!(policy.contains("Live Variance Harness v1"));
assert!(policy.contains("local CI must remain deterministic offline"));
assert!(checker.contains("compatibility-live-variance"));
assert!(checker.contains("cmp -s"));
assert!(makefile.contains("live-variance-v1"));
assert!(adr.contains("live variance harness"));
assert!(rfc.contains("live variance harness"));
assert!(roadmap.contains("Milestone 87 — Live Variance Harness v1"));
assert!(
roadmap.contains("## Milestone 87 — Live Variance Harness v1\n\n**Status:** complete.")
);
assert!(changelog.contains("Live variance harness milestone (M87)"));
assert!(targets.contains("news.ycombinator.com"));
assert!(runs.contains("timestamp_utc"));
}
#[test]
fn app_shell_recovery_v2_assets_define_bounded_profiled_fallback_contract() {
let policy = include_str!("../../../docs/APP_SHELL_RECOVERY_V2.md");
let checker = include_str!("../../../scripts/check-app-shell-recovery-v2.sh");
let makefile = include_str!("../../../Makefile");
let adr = include_str!("../../../docs/adr/0088-js-app-shell-recovery-v2.md");
let rfc = include_str!("../../../docs/rfc/0026-js-app-shell-recovery-v2.md");
let roadmap = include_str!("../../../ROADMAP.md");
let changelog = include_str!("../../../CHANGELOG.md");
let async_stages = include_str!("../../../docs/ASYNC_STAGES.md");
assert!(policy.contains("JS/App-Shell Recovery v2"));
assert!(policy.contains("static-dom"));
assert!(checker.contains("compatibility-recovery-plan"));
assert!(checker.contains("profile: app-shell"));
assert!(makefile.contains("app-shell-recovery-v2"));
assert!(adr.contains("recovery profiles"));
assert!(rfc.contains("recovery profiles"));
assert!(roadmap.contains("Milestone 88 — JS/App-Shell Recovery v2"));
assert!(
roadmap.contains("## Milestone 88 — JS/App-Shell Recovery v2\n\n**Status:** complete.")
);
assert!(changelog.contains("JS/app-shell recovery milestone (M88)"));
assert!(async_stages.contains("App-shell profile"));
}
#[test]
fn auth_assist_v1_assets_define_local_session_diagnostics_contract() {
let policy = include_str!("../../../docs/AUTH_ASSIST_V1.md");
let checker = include_str!("../../../scripts/check-auth-assist-v1.sh");
let makefile = include_str!("../../../Makefile");
let adr = include_str!("../../../docs/adr/0089-session-aware-auth-assist-v1.md");
let rfc = include_str!("../../../docs/rfc/0027-session-aware-auth-assist-v1.md");
let roadmap = include_str!("../../../ROADMAP.md");
let changelog = include_str!("../../../CHANGELOG.md");
assert!(policy.contains("Session-Aware Auth Assist v1"));
assert!(policy.contains("local-only"));
assert!(checker.contains("auth-assist import"));
assert!(checker.contains("diagnose-submit"));
assert!(makefile.contains("auth-assist-v1"));
assert!(adr.contains("auth-assist"));
assert!(rfc.contains("auth-assist"));
assert!(roadmap.contains("Milestone 89 — Session-Aware Auth Assist v1"));
assert!(
roadmap.contains(
"## Milestone 89 — Session-Aware Auth Assist v1\n\n**Status:** complete."
)
);
assert!(changelog.contains("Session-aware auth assist milestone (M89)"));
}
#[test]
fn challenge_failure_ux_v1_assets_define_blocked_flow_reason_contract() {
let policy = include_str!("../../../docs/CHALLENGE_FAILURE_UX_V1.md");
let checker = include_str!("../../../scripts/check-challenge-failure-ux-v1.sh");
let makefile = include_str!("../../../Makefile");
let adr = include_str!("../../../docs/adr/0090-challenge-aware-failure-ux-v1.md");
let rfc = include_str!("../../../docs/rfc/0028-challenge-aware-failure-ux-v1.md");
let roadmap = include_str!("../../../ROADMAP.md");
let changelog = include_str!("../../../CHANGELOG.md");
assert!(policy.contains("Challenge-Aware Failure UX v1"));
assert!(policy.contains("captcha"));
assert!(checker.contains("challenge-diagnose"));
assert!(checker.contains("class: bot-gate"));
assert!(makefile.contains("challenge-failure-ux-v1"));
assert!(adr.contains("blocked-flow"));
assert!(rfc.contains("challenge"));
assert!(roadmap.contains("Milestone 90 — Challenge-Aware Failure UX v1"));
assert!(
roadmap.contains(
"## Milestone 90 — Challenge-Aware Failure UX v1\n\n**Status:** complete."
)
);
assert!(changelog.contains("Challenge-aware failure UX milestone (M90)"));
}
#[test]
fn layout_fidelity_v3_assets_define_spacing_and_code_fidelity_contract() {
let policy = include_str!("../../../docs/LAYOUT_FIDELITY_V3.md");
let checker = include_str!("../../../scripts/check-layout-fidelity-v3.sh");
let makefile = include_str!("../../../Makefile");
let adr = include_str!("../../../docs/adr/0091-layout-fidelity-lift-v3.md");
let rfc = include_str!("../../../docs/rfc/0029-layout-fidelity-lift-v3.md");
let roadmap = include_str!("../../../ROADMAP.md");
let changelog = include_str!("../../../CHANGELOG.md");
assert!(policy.contains("Layout Fidelity Lift v3"));
assert!(policy.contains("pre`/`code"));
assert!(checker.contains("preserves_pre_and_code_text_whitespace"));
assert!(checker.contains("renders_layout_spacers_as_extra_blank_lines"));
assert!(makefile.contains("layout-fidelity-v3"));
assert!(adr.contains("spacing rhythm"));
assert!(rfc.contains("card/grid"));
assert!(roadmap.contains("Milestone 91 — Layout Fidelity Lift v3"));
assert!(
roadmap.contains("## Milestone 91 — Layout Fidelity Lift v3\n\n**Status:** complete.")
);
assert!(changelog.contains("Layout fidelity lift milestone (M91)"));
}
#[test]
fn international_text_v2_assets_define_multilingual_guardrail_contract() {
let policy = include_str!("../../../docs/INTERNATIONAL_TEXT_V2.md");
let checker = include_str!("../../../scripts/check-international-text-v2.sh");
let makefile = include_str!("../../../Makefile");
let adr = include_str!("../../../docs/adr/0092-international-text-robustness-v2.md");
let rfc = include_str!("../../../docs/rfc/0030-international-text-robustness-v2.md");
let roadmap = include_str!("../../../ROADMAP.md");
let changelog = include_str!("../../../CHANGELOG.md");
assert!(policy.contains("International Text Robustness v2"));
assert!(policy.contains("mixed-direction"));
assert!(checker.contains("cjk_text_wraps_by_display_width_without_overflowing_columns"));
assert!(
checker.contains("shelf_search_matches_non_english_titles_notes_tags_and_headings")
);
assert!(makefile.contains("international-text-v2"));
assert!(adr.contains("multilingual"));
assert!(rfc.contains("multilingual"));
assert!(roadmap.contains("Milestone 92 — International Text Robustness v2"));
assert!(roadmap.contains(
"## Milestone 92 — International Text Robustness v2\n\n**Status:** complete."
));
assert!(changelog.contains("International text robustness milestone (M92)"));
}
#[test]
fn structured_data_recovery_v1_assets_define_metadata_recovery_contract() {
let policy = include_str!("../../../docs/STRUCTURED_DATA_RECOVERY_V1.md");
let checker = include_str!("../../../scripts/check-structured-data-recovery-v1.sh");
let makefile = include_str!("../../../Makefile");
let adr = include_str!("../../../docs/adr/0093-structured-data-recovery-v1.md");
let rfc = include_str!("../../../docs/rfc/0031-structured-data-recovery-v1.md");
let roadmap = include_str!("../../../ROADMAP.md");
let changelog = include_str!("../../../CHANGELOG.md");
assert!(policy.contains("Structured Data Recovery v1"));
assert!(policy.contains("metadata"));
assert!(checker.contains("extracts_metadata_and_resolves_relative_urls_against_base"));
assert!(checker.contains("apply_metadata_sets_document_metadata"));
assert!(makefile.contains("structured-data-recovery-v1"));
assert!(adr.contains("metadata"));
assert!(rfc.contains("metadata"));
assert!(roadmap.contains("Milestone 93 — Structured Data Recovery v1"));
assert!(
roadmap
.contains("## Milestone 93 — Structured Data Recovery v1\n\n**Status:** complete.")
);
assert!(changelog.contains("Structured data recovery milestone (M93)"));
}
#[test]
fn compat_data_plane_v2_assets_define_synthesis_quality_and_strict_lint_contract() {
let policy = include_str!("../../../docs/COMPAT_DATA_PLANE_V2.md");
let checker = include_str!("../../../scripts/check-compat-data-plane-v2.sh");
let makefile = include_str!("../../../Makefile");
let adr = include_str!("../../../docs/adr/0094-compatibility-data-plane-expansion-v2.md");
let rfc = include_str!("../../../docs/rfc/0032-compatibility-data-plane-expansion-v2.md");
let roadmap = include_str!("../../../ROADMAP.md");
let changelog = include_str!("../../../CHANGELOG.md");
assert!(policy.contains("Compatibility Data-Plane Expansion v2"));
assert!(checker.contains("quality: score="));
assert!(checker.contains("unsupported manifest version"));
assert!(makefile.contains("compat-data-plane-v2"));
assert!(adr.contains("quality scoring"));
assert!(rfc.contains("synthesis quality"));
assert!(roadmap.contains("Milestone 94 — Compatibility Data-Plane Expansion v2"));
assert!(roadmap.contains(
"## Milestone 94 — Compatibility Data-Plane Expansion v2\n\n**Status:** complete."
));
assert!(changelog.contains("Compatibility data-plane expansion milestone (M94)"));
}
#[test]
fn compatibility_recovery_gate_v1_assets_define_composed_release_evidence_contract() {
let policy = include_str!("../../../docs/COMPATIBILITY_RECOVERY_GATE_V1.md");
let report = include_str!("../../../docs/COMPATIBILITY_RECOVERY_GATE_V1_REPORT.md");
let checker = include_str!("../../../scripts/check-compatibility-recovery-gate-v1.sh");
let makefile = include_str!("../../../Makefile");
let release = include_str!("../../../docs/RELEASE.md");
let beta = include_str!("../../../docs/BETA.md");
let beta_report = include_str!("../../../docs/BETA_READINESS_REPORT.md");
let stable = include_str!("../../../docs/STABLE.md");
let stable_report = include_str!("../../../docs/STABLE_READINESS_REPORT.md");
let adr = include_str!("../../../docs/adr/0095-compatibility-recovery-gate-v1.md");
let rfc = include_str!("../../../docs/rfc/0033-compatibility-recovery-gate-v1.md");
let roadmap = include_str!("../../../ROADMAP.md");
let changelog = include_str!("../../../CHANGELOG.md");
assert!(policy.contains("Compatibility Recovery Gate v1"));
assert!(report.contains("gate result"));
assert!(checker.contains("check-live-variance-v1.sh"));
assert!(checker.contains("check-compat-data-plane-v2.sh"));
assert!(makefile.contains("compatibility-recovery-gate"));
assert!(release.contains("make compatibility-recovery-gate"));
assert!(beta.contains("make compatibility-recovery-gate"));
assert!(beta_report.contains("Compatibility recovery gate"));
assert!(stable.contains("make compatibility-recovery-gate"));
assert!(stable_report.contains("Compatibility recovery gate"));
assert!(adr.contains("SLO-v2 and live-variance"));
assert!(rfc.contains("compatibility-recovery-gate"));
assert!(roadmap.contains("Milestone 95 — Compatibility Recovery Gate v1"));
assert!(
roadmap.contains(
"## Milestone 95 — Compatibility Recovery Gate v1\n\n**Status:** complete."
)
);
assert!(changelog.contains("Compatibility recovery gate milestone (M95)"));
}
#[test]
fn greatness_gate_assets_define_strict_quality_contract() {
let makefile = include_str!("../../../Makefile");
let readme = include_str!("../../../README.md");
let gates = include_str!("../../../docs/GREATNESS_GATES.md");
let performance = include_str!("../../../scripts/check-performance-great.sh");
let security = include_str!("../../../scripts/check-security-best.sh");
let ux = include_str!("../../../scripts/check-ux-great.sh");
let readiness = include_str!("../../../scripts/check-readiness-great.sh");
let beta = include_str!("../../../docs/BETA_READINESS_REPORT.md");
let stable = include_str!("../../../docs/STABLE_READINESS_REPORT.md");
let robustness = include_str!("../../../docs/ROBUSTNESS_READINESS_REPORT.md");
let changelog = include_str!("../../../CHANGELOG.md");
assert!(makefile.contains("performance-great"));
assert!(makefile.contains("security-best"));
assert!(makefile.contains("ux-great"));
assert!(makefile.contains("readiness-great"));
assert!(readme.contains("index quickstart"));
assert!(readme.contains("make readiness-great"));
assert!(gates.contains("Greatness Gates"));
assert!(performance.contains("--benchmark examples/sample.html"));
assert!(performance.contains("PERFORMANCE_GREAT_MAX_FIRST_TRANSFORM_MICROS"));
assert!(security.contains("cargo deny check"));
assert!(security.contains("capture_redacts_credentials_cookies_and_private_fields"));
assert!(ux.contains("index quickstart"));
assert!(ux.contains("default_start_page_contains_actionable_help"));
assert!(readiness.contains("--min-readability 1.0"));
assert!(readiness.contains("check-stable-readiness.sh"));
assert!(beta.contains("Greatness gate"));
assert!(stable.contains("Greatness gate"));
assert!(robustness.contains("Greatness gate"));
assert!(changelog.contains("quality gates"));
}
#[test]
fn one_zero_gates_assets_define_release_closure_contract() {
let makefile = include_str!("../../../Makefile");
let readme = include_str!("../../../README.md");
let release = include_str!("../../../docs/RELEASE.md");
let stable_report = include_str!("../../../docs/STABLE_READINESS_REPORT.md");
let closure = include_str!("../../../docs/SECURITY_CLOSURE_V1.md");
let perf = include_str!("../../../docs/PERFORMANCE_CAPACITY_V1.md");
let ux = include_str!("../../../docs/UX_INTERACTION_V1.md");
let operability = include_str!("../../../docs/OPERABILITY_RELEASE_EVIDENCE_V1.md");
let contract = include_str!("../../../docs/CONTRACT_FREEZE_V1.md");
let rc = include_str!("../../../docs/ONE_ZERO_RC_V1.md");
let bundle_doc = include_str!("../../../docs/RELEASE_EVIDENCE_BUNDLE_V1.md");
let script_security = include_str!("../../../scripts/check-security-closure-gate-v1.sh");
let script_perf = include_str!("../../../scripts/check-performance-capacity-gate-v1.sh");
let script_ux = include_str!("../../../scripts/check-ux-interaction-gate-v1.sh");
let script_operability =
include_str!("../../../scripts/check-operability-release-evidence-gate-v1.sh");
let script_contract = include_str!("../../../scripts/check-contract-freeze-gate-v1.sh");
let script_rc = include_str!("../../../scripts/check-release-1-0-gate-v1.sh");
let script_bundle = include_str!("../../../scripts/generate-release-evidence-bundle-v1.sh");
let roadmap = include_str!("../../../ROADMAP.md");
let changelog = include_str!("../../../CHANGELOG.md");
assert!(makefile.contains("security-closure-v1"));
assert!(makefile.contains("performance-capacity-v1"));
assert!(makefile.contains("ux-interaction-v1"));
assert!(makefile.contains("operability-evidence-v1"));
assert!(makefile.contains("contract-freeze-v1"));
assert!(makefile.contains("release-1-0-gate-v1"));
assert!(readme.contains("make release-1-0-gate-v1"));
assert!(release.contains("make release-1-0-gate-v1"));
assert!(stable_report.contains("Security closure gate v1"));
assert!(stable_report.contains("1.0 release gate v1"));
assert!(closure.contains("Milestone: M96"));
assert!(perf.contains("Milestone: M97"));
assert!(ux.contains("Milestone: M98"));
assert!(operability.contains("Milestone: M99"));
assert!(contract.contains("Milestone: M100"));
assert!(rc.contains("Milestone: M101"));
assert!(bundle_doc.contains("BUNDLE_MANIFEST.tsv"));
assert!(script_security.contains("security-open-risks.tsv"));
assert!(script_perf.contains("check-performance-great.sh"));
assert!(script_ux.contains("check-ux-great.sh"));
assert!(script_operability.contains("generate-release-evidence-bundle-v1.sh"));
assert!(script_contract.contains("index.idx/v1"));
assert!(script_rc.contains("make fmt-check"));
assert!(script_rc.contains("make clippy"));
assert!(script_rc.contains("make check"));
assert!(script_rc.contains("make test"));
assert!(script_rc.contains("make doc"));
assert!(script_bundle.contains("BUNDLE_MANIFEST.tsv"));
assert!(
roadmap.contains("## Milestone 96 — Security Closure Gate v1\n\n**Status:** complete.")
);
assert!(roadmap.contains(
"## Milestone 101 — 1.0.0 Release Candidate and Final Cut\n\n**Status:** complete."
));
assert!(changelog.contains("1.0.0 closure milestones (M96-M101)"));
}
#[test]
fn run_with_args_rejects_too_many_arguments() {
let mut stdin = Cursor::new(Vec::<u8>::new());
let output = run_with_args(
vec!["examples/sample.html".to_owned(), "extra".to_owned()],
&mut stdin,
);
assert_eq!(output, Err("too many arguments".to_owned()));
}
#[test]
fn run_with_args_rejects_unknown_extract_format() {
let mut stdin = Cursor::new(Vec::<u8>::new());
let output = run_with_args(
vec!["--extract".to_owned(), "xml".to_owned(), "-".to_owned()],
&mut stdin,
);
assert_eq!(output, Err("unsupported extraction format: xml".to_owned()));
}
#[test]
fn run_with_args_rejects_unknown_ai_action() {
let mut stdin = Cursor::new(Vec::<u8>::new());
let output = run_with_args(
vec!["--ai-offline".to_owned(), "chat".to_owned(), "-".to_owned()],
&mut stdin,
);
assert_eq!(output, Err("unsupported AI action: chat".to_owned()));
}
#[test]
fn run_with_args_rejects_unknown_capture_option() {
let mut stdin = Cursor::new(Vec::<u8>::new());
let output = run_with_args(
vec![
"capture".to_owned(),
"--raw".to_owned(),
"https://example.org".to_owned(),
"-".to_owned(),
],
&mut stdin,
);
assert_eq!(output, Err("unsupported capture option: --raw".to_owned()));
}
#[test]
fn read_input_rejects_network_url() {
let mut stdin = Cursor::new(Vec::<u8>::new());
let result = read_input("https://example.com", &mut stdin);
assert!(result.is_err());
if let Err(error) = result {
assert!(error.contains("network URLs must be opened"));
}
}
#[test]
fn normalize_url_input_defaults_missing_scheme_to_https() {
assert_eq!(
normalize_url_input("example.org/docs"),
"https://example.org/docs"
);
assert_eq!(
normalize_url_input("https://example.org/docs"),
"https://example.org/docs"
);
assert_eq!(
normalize_url_input("http://example.org/docs"),
"http://example.org/docs"
);
}
#[test]
fn run_with_args_fetches_network_url_for_plain_output() -> Result<(), Box<dyn std::error::Error>>
{
let mut fetcher = MemoryFetcher::new();
let url = IndexUrl::parse("https://example.org/docs")?;
fetcher.insert(
url,
"<html><head><title>Remote</title></head><body><main><p>Fetched page.</p><a href=\"/next\">Next</a></main></body></html>",
);
let fetcher = SecureFetcher::new(fetcher);
let mut stdin = Cursor::new(Vec::<u8>::new());
let result = run_with_args_and_fetcher(
vec!["--plain".to_owned(), "https://example.org/docs".to_owned()],
&mut stdin,
&fetcher,
);
assert!(matches!(result, Ok(CliAction::Print(_))));
if let Ok(CliAction::Print(rendered)) = result {
assert!(rendered.contains("# Remote"));
assert!(rendered.contains("Fetched page."));
assert!(rendered.contains("Next -> https://example.org/next"));
}
Ok(())
}
#[test]
fn run_with_args_fetches_schemeless_url_as_https() -> Result<(), Box<dyn std::error::Error>> {
let mut fetcher = MemoryFetcher::new();
let url = IndexUrl::parse("https://example.org/docs")?;
fetcher.insert(
url,
"<html><head><title>Schemeless</title></head><body><main><p>Fetched over HTTPS.</p></main></body></html>",
);
let fetcher = SecureFetcher::new(fetcher);
let mut stdin = Cursor::new(Vec::<u8>::new());
let result = run_with_args_and_fetcher(
vec!["--plain".to_owned(), "example.org/docs".to_owned()],
&mut stdin,
&fetcher,
);
assert!(matches!(result, Ok(CliAction::Print(_))));
if let Ok(CliAction::Print(rendered)) = result {
assert!(rendered.contains("# Schemeless"));
assert!(rendered.contains("Fetched over HTTPS."));
}
Ok(())
}
#[test]
fn run_with_args_seeds_tui_url_history_for_initial_url()
-> Result<(), Box<dyn std::error::Error>> {
let mut fetcher = MemoryFetcher::new();
let url = IndexUrl::parse("https://example.org/docs")?;
fetcher.insert(
url,
"<html><head><title>Remote</title></head><body><main><p>Fetched page.</p></main></body></html>",
);
let fetcher = SecureFetcher::new(fetcher);
let mut stdin = Cursor::new(Vec::<u8>::new());
let result =
run_with_args_and_fetcher(vec!["example.org/docs".to_owned()], &mut stdin, &fetcher);
assert!(matches!(result, Ok(CliAction::Tui { .. })));
if let Ok(CliAction::Tui { url_history, .. }) = result {
assert_eq!(url_history, vec!["https://example.org/docs".to_owned()]);
}
Ok(())
}
#[test]
fn fetch_document_cached_with_log_returns_history_and_response_preview()
-> Result<(), Box<dyn std::error::Error>> {
let mut fetcher = MemoryFetcher::new();
let url = IndexUrl::parse("https://example.org/docs")?;
fetcher.insert(
url,
"<html><head><title>Remote</title></head><body><main><p>Fetched token=secret page.</p></main></body></html>",
);
let fetcher = SecureFetcher::new(fetcher);
let mut cache = TransformedDocumentCache::new();
let sequence = RefCell::new(0);
let result =
fetch_document_cached_with_log(&fetcher, &mut cache, &sequence, "example.org/docs")?;
assert_eq!(result.document.title, "Remote");
assert_eq!(
result.visited_url.as_deref(),
Some("https://example.org/docs")
);
let log = result.response_log.ok_or("missing response log")?;
assert_eq!(log.method, "GET");
assert!(!log.body_preview.contains("secret"));
assert!(log.body_preview.contains("[REDACTED]"));
Ok(())
}
#[test]
fn fetch_document_cached_progress_reports_all_runtime_steps()
-> Result<(), Box<dyn std::error::Error>> {
let mut fetcher = MemoryFetcher::new();
let url = IndexUrl::parse("https://example.org/progress")?;
fetcher.insert(
url,
"<html><head><title>Progress</title></head><body><main><p>Body.</p></main></body></html>",
);
let mut cache = TransformedDocumentCache::new();
let mut sequence = 0_u64;
let mut progress = Vec::new();
let result = fetch_document_cached_with_log_with_counter_with_progress(
&fetcher,
&mut cache,
&mut sequence,
"https://example.org/progress",
&mut |message| progress.push(message),
)?;
assert_eq!(
result.visited_url.as_deref(),
Some("https://example.org/progress")
);
assert_eq!(
progress,
vec![
"fetching https://example.org/progress".to_owned(),
"parsing https://example.org/progress".to_owned(),
"transforming https://example.org/progress".to_owned(),
"rendering https://example.org/progress".to_owned(),
]
);
Ok(())
}
#[test]
fn submit_form_cached_transforms_post_response() -> Result<(), Box<dyn std::error::Error>> {
let form = Form {
name: "login".to_owned(),
method: "POST".to_owned(),
action: "https://example.org/login".to_owned(),
inputs: vec![Input {
name: "user".to_owned(),
kind: "text".to_owned(),
value: Some("index".to_owned()),
required: true,
}],
buttons: Vec::new(),
};
let submission = form.submit(None, &[])?;
let mut fetcher = MemoryFetcher::new();
fetcher.insert_form_response(
&submission,
Response {
final_url: IndexUrl::parse("https://example.org/account")?,
redirects: Vec::new(),
mime_type: Some("text/html".to_owned()),
body: "<html><head><title>Account</title></head><body><main><p>Submitted form.</p></main></body></html>".to_owned(),
},
);
let fetcher = SecureFetcher::new(fetcher);
let mut cache = TransformedDocumentCache::new();
let document = submit_form_cached(&fetcher, &mut cache, &submission)?;
assert_eq!(document.title, "Account");
assert!(document.nodes.iter().any(
|node| matches!(node, IndexNode::Paragraph(text) if text.contains("Submitted form"))
));
let sequence = RefCell::new(0);
let result = submit_form_cached_with_log(&fetcher, &mut cache, &sequence, &submission)?;
assert_eq!(result.document.title, "Account");
assert_eq!(
result.visited_url.as_deref(),
Some("https://example.org/account")
);
let log = result.response_log.ok_or("missing response log")?;
assert_eq!(log.sequence, 1);
assert_eq!(log.method, "POST");
assert!(log.body_preview.contains("Submitted form"));
Ok(())
}
#[test]
fn submit_form_cached_progress_reports_all_runtime_steps()
-> Result<(), Box<dyn std::error::Error>> {
let form = Form {
name: "search".to_owned(),
method: "GET".to_owned(),
action: "https://example.org/search".to_owned(),
inputs: vec![Input {
name: "q".to_owned(),
kind: "text".to_owned(),
value: Some("index".to_owned()),
required: true,
}],
buttons: Vec::new(),
};
let submission = form.submit(None, &[])?;
let mut fetcher = MemoryFetcher::new();
fetcher.insert_response(
submission.action.clone(),
Response {
final_url: IndexUrl::parse("https://example.org/search?q=index")?,
redirects: Vec::new(),
mime_type: Some("text/html".to_owned()),
body: "<html><head><title>Search</title></head><body><main><p>Result.</p></main></body></html>".to_owned(),
},
);
let mut cache = TransformedDocumentCache::new();
let mut sequence = 0_u64;
let mut progress = Vec::new();
let result = submit_form_cached_with_log_with_counter_with_progress(
&fetcher,
&mut cache,
&mut sequence,
&submission,
&mut |message| progress.push(message),
)?;
assert_eq!(
result.visited_url.as_deref(),
Some("https://example.org/search?q=index")
);
assert_eq!(
progress,
vec![
"submitting GET https://example.org/search?q=index".to_owned(),
"parsing https://example.org/search?q=index".to_owned(),
"transforming https://example.org/search?q=index".to_owned(),
"rendering https://example.org/search?q=index".to_owned(),
]
);
Ok(())
}
#[test]
fn forum_navigation_fetch_logs_are_redacted_and_track_visited_url()
-> Result<(), Box<dyn std::error::Error>> {
let mut fetcher = MemoryFetcher::new();
let url = IndexUrl::parse("https://news.ycombinator.com/item?id=42")?;
fetcher.insert(
url,
"<html><head><title>HN Item</title><link rel=\"canonical\" href=\"https://news.ycombinator.com/item?id=42\"></head><body><main><h1>Story</h1><p>token=verysecret</p><a href=\"https://example.org/story\">Story</a></main></body></html>",
);
let fetcher = SecureFetcher::new(fetcher);
let mut cache = TransformedDocumentCache::new();
let sequence = RefCell::new(0);
let result = fetch_document_cached_with_log(
&fetcher,
&mut cache,
&sequence,
"news.ycombinator.com/item?id=42",
)?;
assert!(result.document.title.starts_with("Hacker News:"));
assert_eq!(
result.visited_url.as_deref(),
Some("https://news.ycombinator.com/item?id=42")
);
let log = result.response_log.ok_or("missing response log")?;
assert_eq!(log.method, "GET");
assert!(!log.body_preview.contains("verysecret"));
assert!(log.body_preview.contains("[REDACTED]"));
Ok(())
}
#[test]
fn forum_form_submission_maps_and_logs_redacted_preview()
-> Result<(), Box<dyn std::error::Error>> {
let fixture = fs::read_to_string(workspace_path(
"crates/index-transformer/tests/fixtures/adapters/legacy-forum-thread.html",
))?;
let document = super::transform_html(fixture);
let form = document
.nodes
.iter()
.find_map(|node| match node {
IndexNode::Form(form) if form.name == "legacy-search" => Some(form.clone()),
_ => None,
})
.ok_or("missing legacy-search form")?;
let submission = form.submit(
Some(&IndexUrl::parse("https://forums.tomshardware.com/")?),
&[("q", "password=super-secret")],
)?;
let mut fetcher = MemoryFetcher::new();
let requested = submission.action.clone();
fetcher.insert_response(
requested.clone(),
Response {
final_url: requested,
redirects: Vec::new(),
mime_type: Some("text/html".to_owned()),
body: "<html><head><title>Forum Search</title></head><body><main><p>results auth=secret-token</p></main></body></html>".to_owned(),
},
);
let fetcher = SecureFetcher::new(fetcher);
let mut cache = TransformedDocumentCache::new();
let sequence = RefCell::new(0);
let result = submit_form_cached_with_log(&fetcher, &mut cache, &sequence, &submission)?;
assert!(result.document.title.contains("Forum Search"));
let log = result.response_log.ok_or("missing response log")?;
assert_eq!(log.method, "GET");
assert!(!log.body_preview.contains("secret-token"));
assert!(log.body_preview.contains("[REDACTED]"));
Ok(())
}
#[test]
fn run_with_args_returns_diagnostic_document_for_network_failure()
-> Result<(), Box<dyn std::error::Error>> {
let fetcher = SecureFetcher::new(MemoryFetcher::new());
let mut stdin = Cursor::new(Vec::<u8>::new());
let result = run_with_args_and_fetcher(
vec![
"--plain".to_owned(),
"https://missing.example/page".to_owned(),
],
&mut stdin,
&fetcher,
);
assert!(matches!(result, Ok(CliAction::Print(_))));
if let Ok(CliAction::Print(rendered)) = result {
assert!(rendered.contains("# Network fetch failed"));
assert!(rendered.contains("could not fetch https://missing.example/page"));
assert!(rendered.contains("retry the request"));
assert!(rendered.contains("capture a local redacted fixture"));
}
Ok(())
}
#[test]
fn run_with_args_reports_network_failure_matrix() -> Result<(), Box<dyn std::error::Error>> {
let cases = [
(
"https://net.example/dns",
FetchError::Network("dns lookup failed".to_owned()),
"dns lookup failed",
),
(
"https://net.example/tls",
FetchError::Network("tls handshake failed".to_owned()),
"tls handshake failed",
),
(
"https://net.example/timeout",
FetchError::Timeout { timeout_ms: 1000 },
"timed out after 1000ms",
),
(
"https://net.example/status",
FetchError::HttpStatus {
status: 503,
url: "https://net.example/status".to_owned(),
},
"HTTP status 503",
),
(
"https://net.example/image",
FetchError::UnsupportedContentType("image/png".to_owned()),
"unsupported response content type: image/png",
),
];
for (url, error, expected) in cases {
let parsed = IndexUrl::parse(url)?;
let mut fetcher = MemoryFetcher::new();
fetcher.insert_error(parsed, error);
let fetcher = SecureFetcher::new(fetcher);
let mut stdin = Cursor::new(Vec::<u8>::new());
let result = run_with_args_and_fetcher(
vec!["--plain".to_owned(), url.to_owned()],
&mut stdin,
&fetcher,
);
assert!(matches!(result, Ok(CliAction::Print(_))));
if let Ok(CliAction::Print(rendered)) = result {
assert!(rendered.contains("# Network fetch failed"));
assert!(rendered.contains(expected));
assert!(rendered.contains("bounded retry policy"));
}
}
Ok(())
}
#[test]
fn run_with_args_prefers_existing_local_file_over_schemeless_url()
-> Result<(), Box<dyn std::error::Error>> {
let path = unique_temp_file("local-domain-like-name");
fs::write(
&path,
"<html><title>Local Dot Path</title><p>From file.</p></html>",
)?;
let fetcher = SecureFetcher::new(MemoryFetcher::new());
let mut stdin = Cursor::new(Vec::<u8>::new());
let result = run_with_args_and_fetcher(
vec![
"--plain".to_owned(),
path.to_str().unwrap_or_default().to_owned(),
],
&mut stdin,
&fetcher,
);
fs::remove_file(&path)?;
assert!(matches!(result, Ok(CliAction::Print(_))));
if let Ok(CliAction::Print(rendered)) = result {
assert!(rendered.contains("# Local Dot Path"));
assert!(rendered.contains("From file."));
}
Ok(())
}
#[test]
fn html_with_base_preserves_existing_base() -> Result<(), Box<dyn std::error::Error>> {
let url = IndexUrl::parse("https://example.org/docs")?;
let html = "<html><head><base href=\"https://docs.example/\"></head></html>";
assert_eq!(html_with_base(&url, html), html);
Ok(())
}
#[test]
fn read_input_reads_from_stdin() {
let mut stdin = Cursor::new(b"<title>stdin</title>".to_vec());
let result = read_input("-", &mut stdin);
assert_eq!(result, Ok("<title>stdin</title>".to_owned()));
}
#[test]
fn read_input_reads_local_file() -> Result<(), Box<dyn std::error::Error>> {
let path = unique_temp_file("read-input");
fs::write(&path, "<title>file</title>")?;
let mut stdin = Cursor::new(Vec::<u8>::new());
let result = read_input(path.to_str().unwrap_or_default(), &mut stdin);
fs::remove_file(&path)?;
assert_eq!(result, Ok("<title>file</title>".to_owned()));
Ok(())
}
#[test]
fn read_input_rejects_large_stdin() {
let mut stdin = Cursor::new(b"12345".to_vec());
let result = read_input_with_limits("-", &mut stdin, ContentLimits::new(4, 100, 20, 10));
assert!(matches!(result, Err(error) if error.contains("content size limit exceeded")));
}
#[test]
fn run_with_args_transforms_stdin_input() {
let mut stdin = Cursor::new(
b"<html><title>CLI</title><p>From stdin.</p><a href=\"https://example.com\">Go</a></html>"
.to_vec(),
);
let result = run_with_args(vec!["--plain".to_owned(), "-".to_owned()], &mut stdin);
assert!(result.is_ok());
assert!(matches!(result, Ok(CliAction::Print(_))));
if let Ok(CliAction::Print(rendered)) = result {
assert!(rendered.contains("# CLI"));
assert!(rendered.contains("From stdin."));
assert!(rendered.contains("Go -> https://example.com"));
}
}
#[test]
fn run_with_args_transforms_file_input() -> Result<(), Box<dyn std::error::Error>> {
let path = unique_temp_file("run-with-args");
fs::write(
&path,
"<html><title>CLI File</title><p>From file.</p><a href=\"https://example.com/docs\">Docs</a></html>",
)?;
let mut stdin = Cursor::new(Vec::<u8>::new());
let result = run_with_args(
vec![
"--plain".to_owned(),
path.to_str().unwrap_or_default().to_owned(),
],
&mut stdin,
);
fs::remove_file(&path)?;
assert!(result.is_ok());
assert!(matches!(result, Ok(CliAction::Print(_))));
if let Ok(CliAction::Print(rendered)) = result {
assert!(rendered.contains("# CLI File"));
assert!(rendered.contains("From file."));
assert!(rendered.contains("Docs -> https://example.com/docs"));
}
Ok(())
}
#[test]
fn run_with_args_extracts_markdown_from_stdin() {
let mut stdin = Cursor::new(
b"<html><title>CLI</title><h2>Section</h2><p>From stdin.</p></html>".to_vec(),
);
let result = run_with_args(
vec![
"--extract".to_owned(),
"markdown".to_owned(),
"-".to_owned(),
],
&mut stdin,
);
assert!(matches!(result, Ok(CliAction::Print(_))));
if let Ok(CliAction::Print(markdown)) = result {
assert!(markdown.contains("# CLI"));
assert!(markdown.contains("## Section"));
assert!(markdown.contains("From stdin."));
}
}
#[test]
fn run_with_args_extracts_links_from_stdin() {
let mut stdin =
Cursor::new(b"<html><title>CLI</title><a href=\"/docs\">Docs</a></html>".to_vec());
let result = run_with_args(
vec!["--extract".to_owned(), "links".to_owned(), "-".to_owned()],
&mut stdin,
);
assert_eq!(result, Ok(CliAction::Print("1\tDocs\t/docs\n".to_owned())));
}
#[test]
fn run_with_args_extracts_json_from_stdin() {
let mut stdin = Cursor::new(b"<html><title>CLI</title><p>From stdin.</p></html>".to_vec());
let result = run_with_args(
vec!["--extract".to_owned(), "json".to_owned(), "-".to_owned()],
&mut stdin,
);
assert!(matches!(result, Ok(CliAction::Print(_))));
if let Ok(CliAction::Print(json)) = result {
assert!(json.contains("\"title\": \"CLI\""));
assert!(json.contains("\"type\": \"paragraph\""));
}
}
#[test]
fn run_with_args_runs_offline_ai_from_stdin() {
let mut stdin = Cursor::new(
b"<html><title>CLI AI</title><p>token=secret Authorization: Bearer abc123</p></html>"
.to_vec(),
);
let result = run_with_args(
vec![
"--ai-offline".to_owned(),
"summarize".to_owned(),
"-".to_owned(),
],
&mut stdin,
);
assert!(matches!(result, Ok(CliAction::Print(_))));
if let Ok(CliAction::Print(output)) = result {
assert!(output.contains("Offline summary"));
assert!(output.contains("[REDACTED]"));
assert!(!output.contains("abc123"));
}
}
#[test]
fn run_with_args_captures_redacted_artifact_from_stdin() {
let mut stdin = Cursor::new(
b"<html><form><input name=\"password\" value=\"secret\"></form><p>public</p></html>"
.to_vec(),
);
let result = run_with_args(
vec![
"capture".to_owned(),
"--redact".to_owned(),
"https://example.org/login?token=url-secret".to_owned(),
"-".to_owned(),
],
&mut stdin,
);
assert!(matches!(result, Ok(CliAction::Print(_))));
if let Ok(CliAction::Print(output)) = result {
assert!(output.contains("index-capture-v1"));
assert!(output.contains("public"));
assert!(output.contains("[REDACTED]"));
assert!(!output.contains("secret"));
assert!(!output.contains("url-secret"));
}
}
#[test]
fn run_with_args_previews_capture_from_stdin() {
let mut stdin = Cursor::new(
b"<html><input name=\"password\" value=\"secret\"><p>public</p></html>".to_vec(),
);
let result = run_with_args(
vec![
"capture".to_owned(),
"--preview".to_owned(),
"--redact".to_owned(),
"https://example.org/login?token=url-secret".to_owned(),
"-".to_owned(),
],
&mut stdin,
);
assert!(matches!(result, Ok(CliAction::Print(_))));
if let Ok(CliAction::Print(output)) = result {
assert!(output.contains("index-capture-preview-v1"));
assert!(output.contains("redaction-summary-v1"));
assert!(output.contains("fixture-submission-checklist-v1"));
assert!(!output.contains("url-secret"));
assert!(!output.contains("secret"));
}
}
#[test]
fn run_with_args_validates_capture_artifact_from_stdin()
-> Result<(), Box<dyn std::error::Error>> {
let artifact = index_capture::capture_redacted(&index_capture::CaptureRequest::new(
"https://example.org/page?token=secret",
"<main>Public</main>",
)?)
.to_text();
let mut stdin = Cursor::new(artifact.into_bytes());
let result = run_with_args(
vec![
"capture".to_owned(),
"--validate".to_owned(),
"-".to_owned(),
],
&mut stdin,
);
assert!(matches!(result, Ok(CliAction::Print(_))));
if let Ok(CliAction::Print(output)) = result {
assert!(output.contains("index-capture-validation-v1"));
assert!(output.contains("status: ok"));
}
Ok(())
}
#[test]
fn run_with_args_emits_capture_catalog_entry() {
let mut stdin = Cursor::new(Vec::<u8>::new());
let output = run_with_args(
vec![
"capture".to_owned(),
"--catalog-entry".to_owned(),
workspace_path("examples/sample.html"),
],
&mut stdin,
);
assert!(
matches!(output, Ok(CliAction::Print(entry)) if entry.contains("examples/sample.html") && entry.contains("Tier 0"))
);
}
#[test]
fn run_with_args_captures_schemeless_source_url_as_https() {
let mut stdin = Cursor::new(b"<html><p>public</p></html>".to_vec());
let result = run_with_args(
vec![
"capture".to_owned(),
"--redact".to_owned(),
"example.org/login".to_owned(),
"-".to_owned(),
],
&mut stdin,
);
assert!(matches!(result, Ok(CliAction::Print(_))));
if let Ok(CliAction::Print(output)) = result {
assert!(output.contains("source_url: https://example.org/login"));
assert!(output.contains("index capture --redact https://example.org/login"));
}
}
#[test]
fn run_with_args_rejects_unsafe_capture_source_url() {
let mut stdin = Cursor::new(b"<main>Bad</main>".to_vec());
let result = run_with_args(
vec![
"capture".to_owned(),
"--redact".to_owned(),
"javascript:alert(1)".to_owned(),
"-".to_owned(),
],
&mut stdin,
);
assert!(matches!(result, Err(error) if error.contains("scheme is not allowed")));
}
#[test]
fn run_with_args_defaults_to_tui_action() {
let mut stdin = Cursor::new(b"<title>TUI</title><p>Interactive.</p>".to_vec());
let result = run_with_args(vec!["-".to_owned()], &mut stdin);
assert!(matches!(result, Ok(CliAction::Tui { .. })));
}
#[test]
fn run_with_args_accepts_initial_reader_profile() {
let mut stdin = Cursor::new(b"<title>TUI</title><p>Interactive.</p>".to_vec());
let result = run_with_args(
vec!["--profile".to_owned(), "docs".to_owned(), "-".to_owned()],
&mut stdin,
);
assert!(matches!(
result,
Ok(CliAction::Tui {
profile: ReaderProfile::Docs,
..
})
));
}
#[test]
fn run_with_args_rejects_unknown_reader_profile() {
let mut stdin = Cursor::new(Vec::<u8>::new());
let result = run_with_args(vec!["--profile".to_owned(), "loud".to_owned()], &mut stdin);
assert!(matches!(result, Err(error) if error.contains("unsupported reader profile")));
}
#[test]
fn run_with_args_help_paths_cover_missing_arguments() {
let help_cases: Vec<Vec<&str>> = vec![
vec!["--profile"],
vec!["--help"],
vec!["-h"],
vec!["doctor", "extra"],
vec!["quickstart", "extra"],
vec!["--benchmark"],
vec!["--save"],
vec!["--save", "markdown"],
vec!["--save", "markdown", "-"],
vec!["--citations"],
vec!["--section"],
vec!["--section", "Intro"],
vec!["--batch-extract"],
vec!["--batch-extract", "markdown"],
vec!["--plain"],
vec!["--extract"],
vec!["--extract", "markdown"],
vec!["--ai-offline"],
vec!["--ai-offline", "summarize"],
];
for case in help_cases {
let mut stdin = Cursor::new(Vec::<u8>::new());
let args = case
.iter()
.map(|value| (*value).to_owned())
.collect::<Vec<_>>();
let result = run_with_args(args, &mut stdin);
assert!(
matches!(result, Ok(CliAction::Help(_))),
"expected help for case {case:?}, got {result:?}"
);
}
let mut stdin = Cursor::new(Vec::<u8>::new());
let profile_only =
run_with_args(vec!["--profile".to_owned(), "docs".to_owned()], &mut stdin);
assert!(matches!(
profile_only,
Ok(CliAction::Tui {
profile: ReaderProfile::Docs,
..
})
));
let mut stdin = Cursor::new(Vec::<u8>::new());
let too_many = run_with_args(
vec!["--benchmark".to_owned(), "-".to_owned(), "extra".to_owned()],
&mut stdin,
);
assert_eq!(too_many, Err("too many arguments".to_owned()));
}
#[test]
fn run_with_args_returns_quickstart_report() {
let mut stdin = Cursor::new(Vec::<u8>::new());
let output = run_with_args(vec!["quickstart".to_owned()], &mut stdin);
assert!(
matches!(output, Ok(CliAction::Print(_))),
"expected quickstart report, got {output:?}"
);
if let Ok(CliAction::Print(report)) = output {
assert!(report.contains("index-quickstart-v1"));
assert!(report.contains("step_1_open_url"));
assert!(report.contains("step_4_use_forms"));
}
}
#[test]
fn run_with_args_capture_and_adapter_error_paths_are_actionable() {
let help_cases: Vec<Vec<&str>> = vec![
vec!["capture"],
vec!["capture", "--validate"],
vec!["capture", "--catalog-entry"],
vec!["capture", "--preview"],
vec!["capture", "--redact"],
vec!["adapter"],
vec!["adapter", "check"],
vec!["adapter", "scaffold"],
vec!["adapter", "diff"],
vec!["adapter", "diff", "fixture-only"],
vec!["artifact"],
vec!["artifact", "inspect"],
];
for case in help_cases {
let mut stdin = Cursor::new(Vec::<u8>::new());
let args = case
.iter()
.map(|value| (*value).to_owned())
.collect::<Vec<_>>();
let result = run_with_args(args, &mut stdin);
assert!(
matches!(result, Ok(CliAction::Help(_))),
"expected help for case {case:?}, got {result:?}"
);
}
let err_cases: Vec<(Vec<&str>, &str)> = vec![
(
vec!["capture", "--validate", "-", "extra"],
"too many arguments",
),
(
vec![
"capture",
"--catalog-entry",
"examples/sample.html",
"extra",
],
"too many arguments",
),
(
vec!["capture", "--preview", "--oops", "example.org", "-"],
"unsupported capture option: --oops",
),
(
vec!["capture", "--redact", "example.org", "-", "extra"],
"too many arguments",
),
(
vec!["adapter", "check", "fixture.html", "extra"],
"too many arguments",
),
(
vec!["adapter", "scaffold", "fixture.html", "extra"],
"too many arguments",
),
(
vec!["adapter", "diff", "fixture.html", "manifest.txt", "extra"],
"too many arguments",
),
(
vec!["artifact", "inspect", "example.org/docs", "extra"],
"too many arguments",
),
(
vec!["artifact", "unknown"],
"unsupported artifact command: unknown",
),
];
for (case, expected) in err_cases {
let mut stdin = Cursor::new(Vec::<u8>::new());
let args = case
.iter()
.map(|value| (*value).to_owned())
.collect::<Vec<_>>();
let result = run_with_args(args, &mut stdin);
assert_eq!(result, Err(expected.to_owned()));
}
}
#[test]
fn run_with_args_artifact_inspect_reports_context_matrix() {
let mut stdin = Cursor::new(Vec::<u8>::new());
let result = run_with_args(
vec![
"artifact".to_owned(),
"inspect".to_owned(),
"example.org/docs".to_owned(),
],
&mut stdin,
);
assert!(matches!(result, Ok(CliAction::Print(text))
if text.contains("index-artifact-inspect-v1")
&& text.contains("target: https://example.org/docs")
&& text.contains("context: live-get")
&& text.contains("context: live-submit")
&& text.contains("context: offline")));
}
#[test]
fn parse_adapter_manifest_and_tier_mapping_cover_error_paths() {
let missing_header = parse_adapter_manifest("adapter: docs");
assert_eq!(
missing_header,
Err("adapter manifest missing header".to_owned())
);
let invalid_number = parse_adapter_manifest(
"index-adapter-manifest-v1\nadapter: docs\nsupport_tier: nope\nnodes: 1\nlinks: 1\nforms: 0\ntables: 0\nregions: 0\n",
);
assert!(matches!(invalid_number, Err(error) if error.contains("invalid support_tier")));
let missing_adapter = parse_adapter_manifest(
"index-adapter-manifest-v1\nsupport_tier: 2\nnodes: 1\nlinks: 1\nforms: 0\ntables: 0\nregions: 0\n",
);
assert_eq!(
missing_adapter,
Err("adapter manifest missing adapter".to_owned())
);
let manifest = parse_adapter_manifest(
"index-adapter-manifest-v1\nadapter: top100.baseline\nsupport_tier: 2\nnodes: 7\nlinks: 3\nforms: 1\ntables: 1\nregions: 2\nmarkdown_contains: # Heading\nignored: value\n",
);
assert_eq!(
manifest,
Ok(AdapterManifest {
adapter: "top100.baseline".to_owned(),
support_tier: 2,
nodes: 7,
links: 3,
forms: 1,
tables: 1,
regions: 2,
markdown_contains: vec!["# Heading".to_owned()],
})
);
assert_eq!(adapter_support_tier("adapter"), 3);
assert_eq!(adapter_support_tier("strong-generic"), 2);
assert_eq!(adapter_support_tier("partial-generic"), 1);
assert_eq!(adapter_support_tier("fallback"), 1);
assert_eq!(adapter_support_tier("failed"), 0);
assert_eq!(adapter_support_tier("unknown"), 0);
}
#[test]
fn shelf_search_arg_parsing_and_formatting_cover_json_and_markdown_paths() {
let missing_format = parse_shelf_search_args(vec!["--format"].into_iter());
assert_eq!(
missing_format,
Err("missing shelf search format".to_owned())
);
let unsupported_format =
parse_shelf_search_args(vec!["--format", "xml", "query"].into_iter());
assert_eq!(
unsupported_format,
Err("unsupported shelf search format: xml".to_owned())
);
let parsed = parse_shelf_search_args(
vec!["--format", "markdown", "typed", "query", "terms"].into_iter(),
);
assert_eq!(
parsed,
Ok((ShelfSearchFormat::Markdown, "typed query terms".to_owned()))
);
let results = vec![ShelfSearchResult {
id: "rec-1".to_owned(),
title: "Line \"One\"\nTabbed\tValue".to_owned(),
source_url: None,
score: 88,
matched_fields: vec!["title".to_owned(), "markdown".to_owned()],
}];
let markdown = format_shelf_search("query", ShelfSearchFormat::Markdown, &results);
assert!(markdown.contains("- Line \"One\""));
assert!(markdown.contains("`rec-1` score 88 fields title,markdown"));
let json = format_shelf_search("query", ShelfSearchFormat::Json, &results);
assert!(json.contains("\"source_url\":null"));
assert!(json.contains("\\\"One\\\""));
assert!(json.contains("\\n"));
assert!(json.contains("\\t"));
assert!(json.contains("\"matched_fields\":[\"title\",\"markdown\"]"));
}
#[test]
fn load_offline_document_supports_capture_artifacts() -> Result<(), Box<dyn std::error::Error>>
{
let capture = index_capture::capture_redacted(&index_capture::CaptureRequest::new(
"https://example.org/articles/1",
"<html><head><title>Captured</title></head><body><main><p>Body</p><a href=\"/next\">Next</a></main></body></html>",
)?)
.to_text();
let path = unique_temp_file("capture-artifact");
fs::write(&path, capture)?;
let document = load_offline_document(path.to_str().unwrap_or_default())?;
fs::remove_file(&path)?;
assert_eq!(document.title, "Captured");
let rendered =
index_renderer::render_document(&document, index_renderer::RenderOptions::default());
assert!(rendered.contains("Body"));
assert!(rendered.contains("Next -> https://example.org/next"));
Ok(())
}
#[test]
fn document_counts_and_path_health_cover_section_recursion_and_existing_dirs() {
let mut nested = IndexDocument::titled("Nested");
nested.push(IndexNode::Section {
role: index_core::SectionRole::Main,
title: Some("Main".to_owned()),
collapsed: false,
nodes: vec![
IndexNode::Paragraph("Body".to_owned()),
IndexNode::Link(Link::new("Docs", "https://example.org/docs")),
],
});
nested.push(IndexNode::Link(Link::new("Root", "https://example.org")));
assert_eq!(document_node_count(&nested), 4);
assert_eq!(document_link_count(&nested), 2);
let existing_dir = std::env::temp_dir();
let ok = path_health("temp", existing_dir.to_str().unwrap_or_default());
assert_eq!(ok, "temp: ok");
}
#[test]
fn run_with_args_saves_markdown_export() -> Result<(), Box<dyn std::error::Error>> {
let input = unique_temp_file("save-input");
let output = unique_temp_file("save-output");
fs::write(&input, "<title>Save</title><main><p>Body</p></main>")?;
let mut stdin = Cursor::new(Vec::<u8>::new());
let result = run_with_args(
vec![
"--save".to_owned(),
"markdown".to_owned(),
input.display().to_string(),
output.display().to_string(),
],
&mut stdin,
);
assert!(matches!(result, Ok(CliAction::Print(_))));
assert!(fs::read_to_string(&output)?.contains("Body"));
fs::remove_file(&input)?;
fs::remove_file(&output)?;
Ok(())
}
#[test]
fn shelf_list_and_show_outputs_are_stable() {
let mut record = ShelfRecord::new(
"Shelf Page",
Some("https://example.org/shelf".to_owned()),
Some("strong-generic".to_owned()),
"12345",
vec!["https://example.org/ref".to_owned()],
"/tmp/shelf.md",
"/tmp/shelf.json",
);
record.add_tag("docs");
record.set_note("Read again");
let mut shelf = KnowledgeShelf::new();
shelf.upsert(record.clone());
let list = format_shelf_list(&shelf);
assert!(list.contains("index-shelf-list-v1"));
assert!(list.contains("Shelf Page"));
assert!(list.contains("strong-generic"));
let shown = format_shelf_record(&record);
assert!(shown.contains("index-shelf-record-v1"));
assert!(shown.contains("tags: docs"));
assert!(shown.contains("note: Read again"));
}
#[test]
fn shelf_commands_save_list_show_tag_and_note() -> Result<(), Box<dyn std::error::Error>> {
let root = unique_temp_file("shelf-root");
let shelf_dir = root.with_extension("dir");
let paths = ShelfPaths {
shelf_dir: shelf_dir.clone(),
exports_dir: shelf_dir.join("exports"),
index_path: shelf_dir.join("index-shelf.txt"),
};
let fetcher = SecureFetcher::new(MemoryFetcher::new());
let mut stdin = Cursor::new(
"<title>Shelf</title><main><p>Body</p><a href=\"https://example.org/ref\">Ref</a></main>",
);
let saved = run_shelf_command_with_paths(
vec!["save".to_owned(), "-".to_owned()].into_iter(),
&mut stdin,
&fetcher,
&paths,
)?;
let CliAction::Print(saved) = saved else {
return Err("expected print action".into());
};
assert!(saved.starts_with("shelf-saved\t"));
let id = saved
.split('\t')
.nth(1)
.ok_or("missing shelf id")?
.to_owned();
let mut stdin = Cursor::new(Vec::<u8>::new());
let listed = run_shelf_command_with_paths(
vec!["list".to_owned()].into_iter(),
&mut stdin,
&fetcher,
&paths,
)?;
assert!(matches!(listed, CliAction::Print(output) if output.contains(&id)));
let mut stdin = Cursor::new(Vec::<u8>::new());
run_shelf_command_with_paths(
vec!["tag".to_owned(), id.clone(), "docs".to_owned()].into_iter(),
&mut stdin,
&fetcher,
&paths,
)?;
let mut stdin = Cursor::new(Vec::<u8>::new());
run_shelf_command_with_paths(
vec![
"note".to_owned(),
id.clone(),
"read".to_owned(),
"again".to_owned(),
]
.into_iter(),
&mut stdin,
&fetcher,
&paths,
)?;
let mut stdin = Cursor::new(Vec::<u8>::new());
let shown = run_shelf_command_with_paths(
vec!["show".to_owned(), id].into_iter(),
&mut stdin,
&fetcher,
&paths,
)?;
let CliAction::Print(shown) = shown else {
return Err("expected show output".into());
};
assert!(shown.contains("tags: docs"));
assert!(shown.contains("note: read again"));
assert!(shown.contains("citations:"));
if shelf_dir.exists() {
std::fs::remove_dir_all(&shelf_dir)?;
}
Ok(())
}
#[test]
fn shelf_commands_report_help_and_missing_records() -> Result<(), Box<dyn std::error::Error>> {
let root = unique_temp_file("shelf-errors");
let shelf_dir = root.with_extension("dir");
let paths = ShelfPaths {
shelf_dir: shelf_dir.clone(),
exports_dir: shelf_dir.join("exports"),
index_path: shelf_dir.join("index-shelf.txt"),
};
let fetcher = SecureFetcher::new(MemoryFetcher::new());
let mut stdin = Cursor::new(Vec::<u8>::new());
let help = run_shelf_command_with_paths(
Vec::<String>::new().into_iter(),
&mut stdin,
&fetcher,
&paths,
)?;
assert!(matches!(help, CliAction::Help(_)));
let mut stdin = Cursor::new(Vec::<u8>::new());
let list = run_shelf_command_with_paths(
vec!["list".to_owned()].into_iter(),
&mut stdin,
&fetcher,
&paths,
)?;
assert!(matches!(list, CliAction::Print(output) if output == "index-shelf-list-v1"));
let mut stdin = Cursor::new(Vec::<u8>::new());
let show = run_shelf_command_with_paths(
vec!["show".to_owned(), "missing".to_owned()].into_iter(),
&mut stdin,
&fetcher,
&paths,
);
assert!(matches!(show, Err(error) if error.contains("not found")));
let mut stdin = Cursor::new(Vec::<u8>::new());
let tag_help = run_shelf_command_with_paths(
vec!["tag".to_owned(), "missing".to_owned()].into_iter(),
&mut stdin,
&fetcher,
&paths,
)?;
assert!(matches!(tag_help, CliAction::Help(_)));
let mut stdin = Cursor::new(Vec::<u8>::new());
let note_help = run_shelf_command_with_paths(
vec!["note".to_owned(), "missing".to_owned()].into_iter(),
&mut stdin,
&fetcher,
&paths,
)?;
assert!(matches!(note_help, CliAction::Help(_)));
let mut stdin = Cursor::new(Vec::<u8>::new());
let unsupported = run_shelf_command_with_paths(
vec!["remove".to_owned()].into_iter(),
&mut stdin,
&fetcher,
&paths,
);
assert!(matches!(unsupported, Err(error) if error.contains("unsupported shelf command")));
if shelf_dir.exists() {
std::fs::remove_dir_all(&shelf_dir)?;
}
Ok(())
}
#[test]
fn shelf_commands_reject_invalid_arities() -> Result<(), Box<dyn std::error::Error>> {
let root = unique_temp_file("shelf-arities");
let shelf_dir = root.with_extension("dir");
let paths = ShelfPaths {
shelf_dir: shelf_dir.clone(),
exports_dir: shelf_dir.join("exports"),
index_path: shelf_dir.join("index-shelf.txt"),
};
let fetcher = SecureFetcher::new(MemoryFetcher::new());
let mut stdin = Cursor::new(Vec::<u8>::new());
let save_help = run_shelf_command_with_paths(
vec!["save".to_owned()].into_iter(),
&mut stdin,
&fetcher,
&paths,
)?;
assert!(matches!(save_help, CliAction::Help(_)));
let mut stdin = Cursor::new(Vec::<u8>::new());
let save_too_many = run_shelf_command_with_paths(
vec!["save".to_owned(), "-".to_owned(), "extra".to_owned()].into_iter(),
&mut stdin,
&fetcher,
&paths,
);
assert!(matches!(save_too_many, Err(error) if error == "too many arguments"));
let mut stdin = Cursor::new(Vec::<u8>::new());
let list_too_many = run_shelf_command_with_paths(
vec!["list".to_owned(), "extra".to_owned()].into_iter(),
&mut stdin,
&fetcher,
&paths,
);
assert!(matches!(list_too_many, Err(error) if error == "too many arguments"));
let mut stdin = Cursor::new(Vec::<u8>::new());
let show_help = run_shelf_command_with_paths(
vec!["show".to_owned()].into_iter(),
&mut stdin,
&fetcher,
&paths,
)?;
assert!(matches!(show_help, CliAction::Help(_)));
let mut stdin = Cursor::new(Vec::<u8>::new());
let show_too_many = run_shelf_command_with_paths(
vec!["show".to_owned(), "one".to_owned(), "two".to_owned()].into_iter(),
&mut stdin,
&fetcher,
&paths,
);
assert!(matches!(show_too_many, Err(error) if error == "too many arguments"));
let mut stdin = Cursor::new(Vec::<u8>::new());
let tag_help = run_shelf_command_with_paths(
vec!["tag".to_owned()].into_iter(),
&mut stdin,
&fetcher,
&paths,
)?;
assert!(matches!(tag_help, CliAction::Help(_)));
let mut stdin = Cursor::new(Vec::<u8>::new());
let tag_too_many = run_shelf_command_with_paths(
vec![
"tag".to_owned(),
"one".to_owned(),
"two".to_owned(),
"three".to_owned(),
]
.into_iter(),
&mut stdin,
&fetcher,
&paths,
);
assert!(matches!(tag_too_many, Err(error) if error == "too many arguments"));
if shelf_dir.exists() {
std::fs::remove_dir_all(&shelf_dir)?;
}
Ok(())
}
#[test]
fn shelf_search_finds_local_exports_without_network() -> Result<(), Box<dyn std::error::Error>>
{
let root = unique_temp_file("shelf-search");
let shelf_dir = root.with_extension("dir");
let paths = ShelfPaths {
shelf_dir: shelf_dir.clone(),
exports_dir: shelf_dir.join("exports"),
index_path: shelf_dir.join("index-shelf.txt"),
};
fs::create_dir_all(&paths.exports_dir)?;
let mut record = ShelfRecord::new(
"Offline Rust Notes",
Some("https://example.org/rust".to_owned()),
Some("strong-generic".to_owned()),
"123",
vec!["https://example.org/citation".to_owned()],
paths.exports_dir.join("rust.md").display().to_string(),
paths.exports_dir.join("rust.json").display().to_string(),
);
record.add_tag("docs");
record.set_note("ownership");
fs::write(&record.markdown_path, "# Borrowing\nLocal export body")?;
let mut shelf = KnowledgeShelf::new();
shelf.upsert(record.clone());
shelf.save_to_path(&paths.index_path)?;
let fetcher = SecureFetcher::new(MemoryFetcher::new());
let mut stdin = Cursor::new(Vec::<u8>::new());
let links = run_shelf_command_with_paths(
vec!["search".to_owned(), "borrowing".to_owned()].into_iter(),
&mut stdin,
&fetcher,
&paths,
)?;
assert!(matches!(links, CliAction::Print(_)));
if let CliAction::Print(output) = links {
assert!(output.contains("index-shelf-search-v1"));
assert!(output.contains(&record.id));
assert!(output.contains("markdown_headings"));
}
let mut stdin = Cursor::new(Vec::<u8>::new());
let markdown = run_shelf_command_with_paths(
vec![
"search".to_owned(),
"--format".to_owned(),
"markdown".to_owned(),
"ownership".to_owned(),
]
.into_iter(),
&mut stdin,
&fetcher,
&paths,
)?;
assert!(
matches!(markdown, CliAction::Print(output) if output.contains("# Shelf search: ownership") && output.contains("Offline Rust Notes"))
);
let mut stdin = Cursor::new(Vec::<u8>::new());
let json = run_shelf_command_with_paths(
vec![
"search".to_owned(),
"--format".to_owned(),
"json".to_owned(),
"docs".to_owned(),
]
.into_iter(),
&mut stdin,
&fetcher,
&paths,
)?;
assert!(
matches!(json, CliAction::Print(output) if output.contains("\"query\":\"docs\"") && output.contains("\"matched_fields\":[\"tags\""))
);
if shelf_dir.exists() {
std::fs::remove_dir_all(&shelf_dir)?;
}
Ok(())
}
#[test]
fn shelf_search_reports_help_and_format_errors() -> Result<(), Box<dyn std::error::Error>> {
let root = unique_temp_file("shelf-search-errors");
let shelf_dir = root.with_extension("dir");
let paths = ShelfPaths {
shelf_dir: shelf_dir.clone(),
exports_dir: shelf_dir.join("exports"),
index_path: shelf_dir.join("index-shelf.txt"),
};
let fetcher = SecureFetcher::new(MemoryFetcher::new());
let mut stdin = Cursor::new(Vec::<u8>::new());
let help = run_shelf_command_with_paths(
vec!["search".to_owned()].into_iter(),
&mut stdin,
&fetcher,
&paths,
)?;
assert!(matches!(help, CliAction::Help(_)));
let mut stdin = Cursor::new(Vec::<u8>::new());
let missing_format = run_shelf_command_with_paths(
vec!["search".to_owned(), "--format".to_owned()].into_iter(),
&mut stdin,
&fetcher,
&paths,
);
assert!(matches!(missing_format, Err(error) if error == "missing shelf search format"));
let mut stdin = Cursor::new(Vec::<u8>::new());
let bad_format = run_shelf_command_with_paths(
vec![
"search".to_owned(),
"--format".to_owned(),
"xml".to_owned(),
"query".to_owned(),
]
.into_iter(),
&mut stdin,
&fetcher,
&paths,
);
assert!(
matches!(bad_format, Err(error) if error.contains("unsupported shelf search format"))
);
if shelf_dir.exists() {
std::fs::remove_dir_all(&shelf_dir)?;
}
Ok(())
}
#[test]
fn run_with_args_extracts_citations() {
let mut stdin = Cursor::new(
b"<title>Citations</title><main><a href=\"https://example.org/ref\">Ref</a><a href=\"/local\">Local</a></main>"
.to_vec(),
);
let result = run_with_args(vec!["--citations".to_owned(), "-".to_owned()], &mut stdin);
assert_eq!(
result,
Ok(CliAction::Print(
"1\tRef\thttps://example.org/ref\n".to_owned()
))
);
}
#[test]
fn run_with_args_exports_selected_section() {
let mut stdin = Cursor::new(
b"<title>Sections</title><main><h2>Keep</h2><p>Yes</p><h2>Skip</h2><p>No</p></main>"
.to_vec(),
);
let result = run_with_args(
vec!["--section".to_owned(), "Keep".to_owned(), "-".to_owned()],
&mut stdin,
);
assert_eq!(result, Ok(CliAction::Print("## Keep\n\nYes\n".to_owned())));
}
#[test]
fn run_with_args_batch_extracts_local_files_and_capture_artifacts()
-> Result<(), Box<dyn std::error::Error>> {
let html_path = unique_temp_file("batch-html");
let capture_path = unique_temp_file("batch-capture");
fs::write(&html_path, "<title>One</title><main><p>First</p></main>")?;
fs::write(
&capture_path,
"index-capture-v1\nsource_url: https://example.org/captured\nreproduce: index capture --redact https://example.org/captured fixture.html\n--- redacted_html ---\n<title>Two</title><main><p>Second</p></main>\n--- diagnostics ---\n",
)?;
let mut stdin = Cursor::new(Vec::<u8>::new());
let result = run_with_args(
vec![
"--batch-extract".to_owned(),
"markdown".to_owned(),
html_path.display().to_string(),
capture_path.display().to_string(),
],
&mut stdin,
);
assert!(matches!(result, Ok(CliAction::Print(_))));
if let Ok(CliAction::Print(output)) = result {
assert!(output.contains("==> "));
assert!(output.contains("First"));
assert!(output.contains("Second"));
}
fs::remove_file(&html_path)?;
fs::remove_file(&capture_path)?;
Ok(())
}
#[test]
fn adapter_check_reports_adapter_fixture() {
let mut stdin = Cursor::new(Vec::<u8>::new());
let result = run_with_args(
vec![
"adapter".to_owned(),
"check".to_owned(),
workspace_path(
"crates/index-transformer/tests/fixtures/adapters/gitlab-project.html",
),
],
&mut stdin,
);
assert!(matches!(result, Ok(CliAction::Print(_))));
if let Ok(CliAction::Print(report)) = result {
assert!(report.contains("index-adapter-check-v1"));
assert!(report.contains("adapter: gitlab"));
assert!(report.contains("support_tier: 3"));
assert!(report.contains("fallback_reason: adapter emitted task view"));
assert!(report.contains("checklist: docs/FIXTURE_INTAKE.md"));
assert!(report.contains("--- markdown ---"));
}
}
#[test]
fn adapter_check_report_matches_golden_snapshot() {
let mut stdin = Cursor::new(Vec::<u8>::new());
let result = run_with_args(
vec![
"adapter".to_owned(),
"check".to_owned(),
workspace_path(
"crates/index-transformer/tests/fixtures/adapters/mdn-reference.html",
),
],
&mut stdin,
);
assert!(matches!(result, Ok(CliAction::Print(_))));
if let Ok(CliAction::Print(report)) = result {
let report = report.replace(
&workspace_path(
"crates/index-transformer/tests/fixtures/adapters/mdn-reference.html",
),
"crates/index-transformer/tests/fixtures/adapters/mdn-reference.html",
);
assert_eq!(
report.trim_end(),
include_str!("../tests/golden/adapter-mdn-report.txt").trim_end()
);
}
}
#[test]
fn shelf_search_links_match_golden_snapshot() {
let mut record = ShelfRecord::new(
"Offline Rust Notes",
Some("https://example.org/rust".to_owned()),
Some("strong-generic".to_owned()),
"123",
vec!["https://example.org/citation".to_owned()],
"/tmp/index-shelf/rust.md",
"/tmp/index-shelf/rust.json",
);
record.add_tag("docs");
record.set_note("ownership");
let mut shelf = KnowledgeShelf::new();
shelf.upsert(record.clone());
let results = shelf.search("borrowing", |candidate| {
(candidate.id == record.id).then(|| "# Borrowing\nLocal export body".to_owned())
});
let output = format_shelf_search("borrowing", ShelfSearchFormat::Links, &results)
.replace(&record.id, "shelf-<id>");
assert_eq!(
output.trim_end(),
include_str!("../tests/golden/shelf-search.links").trim_end()
);
}
#[test]
fn adapter_check_reports_generic_fallback() {
let mut stdin = Cursor::new(Vec::<u8>::new());
let result = run_with_args(
vec![
"adapter".to_owned(),
"check".to_owned(),
workspace_path("crates/index-transformer/tests/fixtures/article.html"),
],
&mut stdin,
);
assert!(matches!(result, Ok(CliAction::Print(_))));
if let Ok(CliAction::Print(report)) = result {
assert!(report.contains("adapter: none"));
assert!(report.contains("support_tier: 2"));
assert!(report.contains("fallback_reason:"));
assert!(report.contains("links:"));
assert!(report.contains("forms: 0"));
}
}
#[test]
fn adapter_scaffold_and_diff_manifest_reports_stable_contract()
-> Result<(), Box<dyn std::error::Error>> {
let fixture =
workspace_path("crates/index-transformer/tests/fixtures/adapters/mdn-reference.html");
let manifest_path = unique_temp_file("adapter-manifest");
let mut stdin = Cursor::new(Vec::<u8>::new());
let scaffold = run_with_args(
vec!["adapter".to_owned(), "scaffold".to_owned(), fixture.clone()],
&mut stdin,
)?;
let CliAction::Print(manifest) = scaffold else {
return Err("adapter scaffold did not print manifest".into());
};
assert!(manifest.starts_with("index-adapter-manifest-v1"));
assert!(manifest.contains("adapter: mdn"));
assert!(manifest.contains("support_tier: 3"));
fs::write(&manifest_path, manifest)?;
let mut stdin = Cursor::new(Vec::<u8>::new());
let diff = run_with_args(
vec![
"adapter".to_owned(),
"diff".to_owned(),
fixture,
manifest_path.display().to_string(),
],
&mut stdin,
)?;
assert!(
matches!(diff, CliAction::Print(output) if output.contains("index-adapter-diff-v1") && output.contains("status: ok"))
);
fs::remove_file(&manifest_path)?;
Ok(())
}
#[test]
fn adapter_diff_reports_manifest_mismatches() -> Result<(), Box<dyn std::error::Error>> {
let fixture =
workspace_path("crates/index-transformer/tests/fixtures/adapters/mdn-reference.html");
let manifest_path = unique_temp_file("adapter-manifest-bad");
fs::write(
&manifest_path,
"index-adapter-manifest-v1\nadapter: none\nsupport_tier: 0\nnodes: 0\nlinks: 0\nforms: 0\ntables: 0\nregions: 0\nmarkdown_contains: missing text\n",
)?;
let mut stdin = Cursor::new(Vec::<u8>::new());
let diff = run_with_args(
vec![
"adapter".to_owned(),
"diff".to_owned(),
fixture,
manifest_path.display().to_string(),
],
&mut stdin,
)?;
assert!(
matches!(diff, CliAction::Print(output) if output.contains("status: changed") && output.contains("mismatch: adapter expected none actual mdn") && output.contains("markdown_contains missing"))
);
fs::remove_file(&manifest_path)?;
Ok(())
}
#[test]
fn adapter_command_reports_help_and_errors() {
let mut stdin = Cursor::new(Vec::<u8>::new());
let help = run_with_args(vec!["adapter".to_owned()], &mut stdin);
assert!(matches!(help, Ok(CliAction::Help(_))));
let mut stdin = Cursor::new(Vec::<u8>::new());
let check_help = run_with_args(vec!["adapter".to_owned(), "check".to_owned()], &mut stdin);
assert!(matches!(check_help, Ok(CliAction::Help(_))));
let mut stdin = Cursor::new(Vec::<u8>::new());
let too_many = run_with_args(
vec![
"adapter".to_owned(),
"check".to_owned(),
"one".to_owned(),
"two".to_owned(),
],
&mut stdin,
);
assert!(matches!(too_many, Err(error) if error == "too many arguments"));
let mut stdin = Cursor::new(Vec::<u8>::new());
let diff_help = run_with_args(vec!["adapter".to_owned(), "diff".to_owned()], &mut stdin);
assert!(matches!(diff_help, Ok(CliAction::Help(_))));
let mut stdin = Cursor::new(Vec::<u8>::new());
let unsupported =
run_with_args(vec!["adapter".to_owned(), "unknown".to_owned()], &mut stdin);
assert!(matches!(unsupported, Err(error) if error.contains("unsupported adapter command")));
}
}