use anyhow::Result;
use clap::{Parser, Subcommand};
use colored::*;
use serde_json::json;
use std::collections::BTreeSet;
mod audit;
mod actions;
mod brief;
mod crawl;
mod engine;
mod fetch;
mod geo_keyless;
mod gsc;
mod llm;
mod llms;
mod manifest;
mod narrative;
mod mcp;
mod paths;
mod policy;
mod rank;
mod robots;
mod rules;
mod schema;
mod serp;
mod sitemap;
mod vitals;
#[cfg(test)]
mod tests;
#[derive(Parser)]
#[command(name = "jev-seo")]
#[command(author = "Akash Priyadarshi")]
#[command(version)]
#[command(about = "FOSS zero-cost, agent-first SEO & GEO search radar powered by TypeSafe Jev", long_about = None)]
struct Cli {
#[command(subcommand)]
command: Commands,
#[arg(long, global = true, value_name = "NAME: value")]
header: Vec<String>,
#[arg(long, global = true)]
user: Option<String>,
#[arg(long, global = true)]
password: Option<String>,
}
#[derive(Subcommand)]
#[allow(clippy::large_enum_variant)]
enum Commands {
Keywords {
query: String,
#[arg(long)]
json: bool,
},
Query {
query: String,
#[arg(short, long, default_value_t = 10)]
limit: usize,
#[arg(long, default_value = "auto")]
provider: String,
#[arg(long, default_value = "advanced")]
depth: String,
#[arg(long, default_value = "general")]
topic: String,
#[arg(long)]
json: bool,
},
Link {
path: String,
#[arg(short, long, default_value_t = 10)]
limit: usize,
#[arg(long)]
json: bool,
#[arg(long)]
no_jev: bool,
#[arg(long, default_value_t = 0.25, value_name = "USD")]
jev_budget: f64,
},
Audit {
path: String,
#[arg(long)]
target_query: Option<String>,
#[arg(long)]
json: bool,
#[arg(long)]
min_pass: Option<f64>,
#[arg(long, default_value = "blocking")]
fail_on: String,
#[arg(long)]
forbid: Option<String>,
#[arg(long)]
max_critical: Option<usize>,
#[arg(long, value_name = "PATH")]
html: Option<String>,
#[arg(long, value_name = "PATH")]
pdf: Option<String>,
#[arg(long, value_name = "PATH")]
md: Option<String>,
#[arg(long, value_name = "PATH")]
csv: Option<String>,
#[arg(long, value_name = "PATH")]
digest: Option<String>,
#[arg(long, value_name = "PATH")]
actions_csv: Option<String>,
#[arg(long, value_name = "PATH")]
pairs_csv: Option<String>,
#[arg(long, value_name = "PATH")]
rescore: Option<String>,
#[arg(long, value_name = "DIR")]
manifest: Option<String>,
#[arg(long)]
no_jev: bool,
#[arg(long, default_value_t = 0.25, value_name = "USD")]
jev_budget: f64,
},
Geo {
target: String,
#[arg(long)]
query: String,
#[arg(long)]
json: bool,
#[arg(long, default_value_t = 0.25, value_name = "USD")]
jev_budget: f64,
},
Schema {
target: String,
#[arg(long)]
json: bool,
},
Robots {
domain: String,
#[arg(long)]
json: bool,
},
Brief {
topic: String,
#[arg(short, long, default_value_t = 5)]
limit: usize,
#[arg(long, default_value = "auto")]
provider: String,
#[arg(long)]
markdown: bool,
#[arg(long)]
json: bool,
},
Rank {
#[arg(long)]
domain: String,
#[arg(long)]
query: String,
},
Sitemap {
target: String,
#[arg(long)]
json: bool,
},
Crawl {
#[arg(value_name = "URL")]
url: Option<String>,
#[arg(long, default_value_t = crate::crawl::DEFAULT_MAX_PAGES)]
max_pages: usize,
#[arg(long, default_value = "auto")]
fetch: String,
#[arg(long, default_value_t = 0)]
max_credits: u32,
#[arg(long)]
json: bool,
#[arg(long)]
diff: bool,
#[arg(long, value_name = "PATH")]
csv: Option<String>,
#[arg(long, value_name = "PATH")]
rescore: Option<String>,
#[arg(long, value_name = "DIR")]
manifest: Option<String>,
#[arg(long)]
no_jev: bool,
#[arg(long, default_value_t = 0.25, value_name = "USD")]
jev_budget: f64,
#[arg(long)]
vitals: bool,
},
Llms {
domain: String,
#[arg(long)]
json: bool,
},
Doctor {
#[arg(long)]
json: bool,
},
Explain {
id: String,
#[arg(long)]
json: bool,
},
Report {
path: String,
#[arg(long, value_name = "PATH")]
baseline: String,
#[arg(long, value_name = "PATH")]
actions_csv: Option<String>,
#[arg(long)]
json: bool,
},
Bundle {
path: String,
#[arg(long, value_name = "PATH")]
out: String,
},
Drift {
op: String,
#[arg(long, value_name = "PATH")]
report: Option<String>,
#[arg(long)]
label: Option<String>,
#[arg(long)]
json: bool,
},
Gsc {
op: String,
#[arg(long)]
site: Option<String>,
#[arg(long)]
code: Option<String>,
#[arg(long, default_value_t = 10)]
limit: usize,
#[arg(long)]
json: bool,
},
Mcp,
}
fn apply_jev_budget_from(cli: &Cli) {
let usd = match &cli.command {
Commands::Audit { jev_budget, .. }
| Commands::Geo { jev_budget, .. }
| Commands::Crawl { jev_budget, .. } => *jev_budget,
_ => manifest::DEFAULT_JEV_BUDGET_USD,
};
manifest::set_jev_budget_usd(usd);
}
fn write_actions_csv(
path: &str,
findings: &[rules::Finding],
actions: &[actions::Action],
) -> Result<()> {
let body = actions::to_csv(actions);
manifest::gate_report(&body, findings, actions)?;
std::fs::write(path, body)?;
println!("Action tracker CSV written to {}", path.dimmed());
Ok(())
}
fn diff_audit_reports(current: &audit::DirectoryAuditReport, base: &audit::DirectoryAuditReport) -> serde_json::Value {
use std::collections::BTreeSet;
let score = |r: &audit::DirectoryAuditReport| r.pass_rate.round().clamp(0.0, 100.0) as u32;
let rules = |r: &audit::DirectoryAuditReport| -> BTreeSet<String> {
r.findings.iter().map(|f| f.rule_id.clone()).collect()
};
let cur = rules(current);
let old = rules(base);
let fixed: Vec<String> = old.difference(&cur).cloned().collect();
let new: Vec<String> = cur.difference(&old).cloned().collect();
let sc_now = score(current);
let sc_old = score(base);
json!({
"baseline_score": sc_old,
"current_score": sc_now,
"delta": sc_now as i32 - sc_old as i32,
"grade": crate::actions::grade(sc_now),
"rules_fixed": fixed,
"rules_new": new,
"findings_baseline": base.findings.len(),
"findings_current": current.findings.len(),
"files_baseline": base.total_files,
"files_current": current.total_files,
})
}
pub(crate) fn main_report_diff(
current: &audit::DirectoryAuditReport,
base: &audit::DirectoryAuditReport,
) -> serde_json::Value {
diff_audit_reports(current, base)
}
#[cfg(test)]
pub(crate) fn main_shim_diff(
current: &audit::DirectoryAuditReport,
base: &audit::DirectoryAuditReport,
) -> serde_json::Value {
diff_audit_reports(current, base)
}
fn main() -> Result<()> {
let cli = Cli::parse();
fetch::set_extra_headers(cli.user.as_deref(), cli.password.as_deref(), &cli.header);
apply_jev_budget_from(&cli);
match cli.command {
Commands::Keywords { query, json } => {
let suggestions = serp::get_autocomplete(&query)?;
if json {
println!("{}", serde_json::to_string_pretty(&suggestions)?);
return Ok(());
}
println!("{}", format!("Keyword Autocomplete for \"{}\":", query).cyan().bold());
for (idx, item) in suggestions.iter().enumerate() {
println!(" {}. {}", idx + 1, item);
}
let state = json!({
"root_query": query,
"suggestions": suggestions
});
let extras = policy::keyword_value_extras(&suggestions, 10);
if let Some((eval, v)) = gated_eval_with("keywords", state, extras) {
println!("\n{}", "Intent & Semantic Classification:".cyan().bold());
println!(" Primary Intent: {} (confidence: {:.2}){}{}", eval.intent.green(), eval.intent_confidence, policy::marker(v), runner_up_suffix(&eval.extra, v).dimmed());
println!(" Content Gap: {}", eval.content_gap.yellow());
print_keyword_values(&eval.extra, &suggestions);
println!(" Suggested Next: jev-seo {}", policy::route_for_intent(&eval.intent));
print_jev_spend_line();
}
}
Commands::Query { query, limit, provider, depth, topic, json } => {
let backend = match provider.as_str() {
"ddg" => serp::Provider::Ddg,
"tavily" => serp::Provider::Tavily,
"dfs" => serp::Provider::Dfs,
"auto" => serp::Provider::Auto,
other => anyhow::bail!("unknown --provider '{other}' (auto, ddg, tavily, dfs)"),
};
let opts = serp::SearchOpts { depth, topic, answer: false };
if backend == serp::Provider::Dfs && !serp::dfs_enabled() {
eprintln!("{}", "Note: DATAFORSEO keys missing; explicit dfs falls back to free scrape.".yellow());
}
if backend == serp::Provider::Tavily && !serp::tavily_enabled() {
eprintln!("{}", "Note: TAVILY_API_KEY missing; explicit tavily falls back to free scrape.".yellow());
}
eprintln!("{}", format!("Scraping live SERP for \"{}\" (limit: {})...", query, limit).dimmed());
let (items, served) = serp::scrape_serp_opts(&query, limit, backend, &opts)?;
if json {
println!("{}", serde_json::to_string_pretty(&items)?);
return Ok(());
}
let source_label = match served {
serp::Provider::Tavily => "Tavily",
serp::Provider::Dfs => "DataForSEO",
_ => "DuckDuckGo",
};
println!("\n{}", format!("Top Competitors on {}:", source_label).cyan().bold());
for item in &items {
println!(" #{:<2} {} - {}", item.position.to_string().green().bold(), item.title, item.url.dimmed());
if !item.snippet.is_empty() {
println!(" {}", item.snippet);
}
}
let mut rel_questions = serde_json::Map::new();
for (i, _) in items.iter().enumerate() {
rel_questions.insert(
format!("rel_{}", i),
json!({
"type": "score",
"instructions": format!("How relevant is result {} to the query in `target_query`?", i),
"criteria": ["Irrelevant or spam", "Tangential mention", "Relevant to the query", "Highly relevant, directly answers", "Exact best match"]
}),
);
}
let state = json!({
"target_query": query,
"competitor_serp": items
});
match engine::JevClient::new().map(|c| c.fanout_eval_with(state, serde_json::Value::Object(rel_questions))) {
Some(Ok(eval)) => {
let rel_conf: f64 = (0..items.len())
.map(|i| {
eval.extra
.get(&format!("rel_{}", i))
.and_then(|a| a.get("confidence"))
.and_then(|c| c.as_f64())
.unwrap_or(0.0)
})
.sum::<f64>()
/ items.len().max(1) as f64;
if rel_conf >= policy::thresholds("query").flag {
let mut ranked: Vec<(usize, f64)> = items
.iter()
.enumerate()
.map(|(i, _)| {
let s = eval
.extra
.get(&format!("rel_{}", i))
.and_then(|a| a.get("score"))
.and_then(|s| s.as_f64())
.unwrap_or(0.0);
(i, s)
})
.collect();
ranked.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal));
println!("\n{}", "Relevance Ranking (TypeSafe Jev):".cyan().bold());
for (rank, (i, score)) in ranked.iter().enumerate() {
let item = &items[*i];
println!(" #{:<2} (rel {:.1}) {} - {}", rank + 1, score, item.title, item.url.dimmed());
}
}
match policy::gate("query", eval.confidence()) {
policy::Verdict::Drop => eprintln!("{}", format!("Note: Jev unsure on gap analysis (confidence {:.2}), local results above stand.", eval.confidence()).yellow()),
v => {
println!("\n{}", "Competitive Gap Analysis (TypeSafe Jev):".cyan().bold());
println!(" Intent Class: {}{}", eval.intent.green(), runner_up_suffix(&eval.extra, v).dimmed());
println!(" AI Citations: {}/10 GEO Score{}", eval.geo_score, policy::marker(v));
println!(" Primary Gap: {}", eval.content_gap.yellow());
}
}
print_jev_spend_line();
}
Some(Err(e)) => eprintln!("{}", format!("Warning: Jev scoring failed ({e:#}), showing local-only output.").yellow()),
None => eprintln!("{}", "Note: TYPESAFE_API_KEY not set, showing local-only output.".yellow()),
}
}
Commands::Link { path, limit, json, no_jev, jev_budget } => {
let dir_report = audit::audit_path(&path)?;
manifest::set_jev_budget_usd(jev_budget);
let client = match (no_jev, engine::JevClient::new()) {
(true, _) | (_, None) => {
eprintln!("{}", "Note: link suggestions need Jev (unset --no-jev with TYPESAFE_API_KEY set).".yellow());
return Ok(());
}
(false, Some(c)) => c,
};
let pages: Vec<&audit::AuditReport> = dir_report
.reports
.iter()
.filter(|r| r.title.as_ref().is_some_and(|t| !t.trim().is_empty()))
.collect();
let stem_of = |p: &str| {
std::path::Path::new(p)
.file_stem()
.and_then(|s| s.to_str())
.unwrap_or(p)
.to_string()
};
let mut rows: Vec<serde_json::Value> = Vec::new();
let mut suggested = 0usize;
if !json {
println!("{}", "Internal-Link Suggestions (TypeSafe Jev):".cyan().bold());
}
for src in pages.iter().take(limit.max(1)) {
let src_stem = stem_of(&src.file_path);
let mut cands: Vec<(String, String)> = Vec::new();
for dst in &pages {
if dst.file_path == src.file_path {
continue;
}
let id = stem_of(&dst.file_path);
if src.internal_link_targets.iter().any(|t| t.contains(&id)) {
continue;
}
let summary = format!(
"{} — {}",
dst.title.clone().unwrap_or_default(),
dst.description.clone().unwrap_or_default()
);
cands.push((id, summary.chars().take(160).collect()));
if cands.len() >= 15 {
break;
}
}
if cands.is_empty() {
continue;
}
let excerpt: String = excerpt_local(&src.file_path).chars().take(1200).collect();
let state = json!({
"source": { "title": src.title, "excerpt": excerpt },
"candidates": cands.iter().map(|(id, s)| json!({"id": id, "summary": s})).collect::<Vec<_>>()
});
match client.fanout_eval_with(state, policy::link_question(&cands)) {
Ok(eval) => {
let ans = eval.extra.get("link_target");
let target = ans.and_then(|a| a.get("choice")).and_then(|c| c.as_str()).unwrap_or("no_link");
let conf = ans.and_then(|a| a.get("confidence")).and_then(|c| c.as_f64()).unwrap_or(0.5);
let v = policy::gate("link", conf);
if v == policy::Verdict::Drop {
continue;
}
if json {
rows.push(json!({"source": src.file_path, "target": target, "confidence": conf}));
} else {
println!(
" {} → {}{}",
src_stem.green(),
target.yellow(),
policy::marker(v)
);
}
suggested += 1;
}
Err(e) => eprintln!("{}", format!("Warning: link judge failed for {} ({e:#})", src.file_path).yellow()),
}
}
if json {
println!("{}", serde_json::to_string_pretty(&rows)?);
} else if suggested == 0 {
println!(" {}", "no suggestions: every page answered no_link or dropped below confidence.".dimmed());
}
print_jev_spend_line();
}
Commands::Audit { path, target_query, json, min_pass, fail_on, forbid, max_critical, html, pdf, md, csv, digest, actions_csv, pairs_csv, rescore, manifest, no_jev, jev_budget: _ } => {
let t0 = std::time::Instant::now();
let is_rescore = rescore.is_some();
let dir_report = match rescore {
Some(path) => {
let raw = std::fs::read_to_string(&path)?;
let saved: audit::DirectoryAuditReport = serde_json::from_str(&raw)?;
audit::with_findings(saved)
}
None => audit::with_findings(audit::audit_path(&path)?),
};
eprintln!(
"[jev-seo {:>02}:{:>02}] audited {} files in {:.1}s",
t0.elapsed().as_secs() / 60,
t0.elapsed().as_secs() % 60,
dir_report.total_files,
t0.elapsed().as_secs_f64()
);
let actions = crate::rules::actions_for(&dir_report.findings);
let jev_pages = if !is_rescore && !no_jev {
judge_audit_sample(&dir_report, target_query.as_deref(), 12)
} else {
Vec::new()
};
let jev_used = !jev_pages.is_empty() || (!no_jev && target_query.is_some() && manifest::jev_key_present());
let mut completeness = manifest::completeness_audit(dir_report.total_files, false, !jev_pages.is_empty() || (!no_jev && target_query.is_some()));
if !jev_pages.is_empty() {
completeness.notes.push(format!("Jev judged {} pages (sample)", jev_pages.len()));
completeness.full = false;
}
let mut export_dirs = manifest::auto_dirs(&[
html.as_deref(),
pdf.as_deref(),
md.as_deref(),
csv.as_deref(),
digest.as_deref(),
]);
if let Some(dir) = &manifest {
export_dirs = vec![std::path::PathBuf::from(dir)];
} else if !export_dirs.iter().any(|d| d.join("narrative.json").is_file()) {
let cwd = std::path::PathBuf::from(".");
if cwd.join("narrative.json").is_file() {
export_dirs.push(cwd);
}
}
let mut n_story: Option<narrative::Narrative> = None;
for d in export_dirs.iter() {
match narrative::load(d, &dir_report.findings, &actions, dir_report.total_files) {
Ok(Some(n)) => {
n_story = Some(n);
break;
}
Ok(None) => {}
Err(e) => anyhow::bail!("{e:#}"),
}
}
if n_story.is_none() {
n_story = Some(narrative::auto(
&dir_report.findings,
&actions,
dir_report.total_files,
dir_report.pass_rate,
));
}
let n = n_story.as_ref().expect("narrative always set");
if !n.unverified_numbers.is_empty() {
eprintln!(
"{} narrative numbers not in audit: {}",
"warn".yellow(),
n.unverified_numbers.join(", ")
);
}
if !n.effort_mismatches.is_empty() {
eprintln!(
"{} narrative effort mismatches: {}",
"warn".yellow(),
n.effort_mismatches.join("; ")
);
}
let n_md = narrative::to_markdown(n);
let n_html = narrative::to_html(n);
let body_html = html.as_ref().map(|_| audit::to_html(&dir_report) + &n_html);
let body_pdf = pdf.as_ref().map(|_| audit::to_pdf_with_narrative(&dir_report, n));
let body_md = md.as_ref().map(|_| audit::to_markdown(&dir_report) + &n_md);
let body_csv = csv.as_ref().map(|_| crate::rules::to_csv(&dir_report.findings));
let body_digest = digest.as_ref().map(|_| audit::digest(&dir_report, &actions));
if let Some(body) = &body_html {
manifest::gate_report(body, &dir_report.findings, &actions)?;
}
if let Some(body) = &body_pdf {
manifest::gate_report(&String::from_utf8_lossy(body), &dir_report.findings, &actions)?;
}
if let Some(body) = &body_md {
manifest::gate_report(body, &dir_report.findings, &actions)?;
}
if let Some(body) = &body_csv {
manifest::gate_report(body, &dir_report.findings, &actions)?;
}
if let Some(body) = &body_digest {
manifest::gate_report(body, &dir_report.findings, &actions)?;
}
if let Some(out) = &html {
std::fs::write(out, body_html.as_deref().unwrap_or_default())?;
println!("HTML report written to {}", out.dimmed());
}
if let Some(out) = &pdf {
std::fs::write(out, body_pdf.as_deref().unwrap_or_default())?;
println!("PDF report written to {}", out.dimmed());
}
if let Some(out) = &md {
std::fs::write(out, body_md.as_deref().unwrap_or_default())?;
println!("Markdown report written to {}", out.dimmed());
}
if let Some(out) = &csv {
std::fs::write(out, body_csv.as_deref().unwrap_or_default())?;
println!("CSV findings written to {}", out.dimmed());
}
if let Some(out) = &digest {
std::fs::write(out, body_digest.as_deref().unwrap_or_default())?;
println!("Digest written to {}", out.dimmed());
}
if let Some(out) = &actions_csv {
write_actions_csv(out, &dir_report.findings, &actions)?;
}
let texts_owned: Vec<String> = body_html
.iter()
.chain(body_md.iter())
.chain(body_csv.iter())
.chain(body_digest.iter())
.cloned()
.collect();
let texts: Vec<&str> = texts_owned.iter().map(|s| s.as_str()).collect();
let mut ledger = manifest::Ledger::new("audit", &path);
if !jev_pages.is_empty() {
ledger.note(format!("Jev page suite: {} pages judged", jev_pages.len()));
} else if target_query.is_some() {
ledger.note(if jev_used {
"Jev target-query scoring available"
} else {
"Jev target-query requested but TYPESAFE_API_KEY missing"
});
}
let score = manifest::ScoreCard {
score: dir_report.pass_rate.round().clamp(0.0, 100.0) as u32,
grade: crate::actions::grade(dir_report.pass_rate.round().clamp(0.0, 100.0) as u32).to_string(),
kind: "pass-rate".into(),
};
let run_manifest = manifest::ManifestInput {
command: "audit",
target: &path,
findings: &dir_report.findings,
actions: &actions,
texts: &texts,
score: Some(score),
completeness: completeness.clone(),
ledger,
}
.build();
if !run_manifest.validation.ok {
anyhow::bail!(
"run manifest citation gate failed: {:?}",
run_manifest.validation
);
}
let force = manifest.is_some();
let mut wrote_manifest = false;
for d in &export_dirs {
let (run_p, led_p) = manifest::write_pair(d, &run_manifest)?;
println!("Run manifest written to {}", run_p.display().to_string().dimmed());
println!("Spend ledger written to {}", led_p.display().to_string().dimmed());
wrote_manifest = true;
}
if !wrote_manifest && (force || !json) {
let (run_p, led_p) = manifest::write_pair(std::path::Path::new("."), &run_manifest)?;
println!("Run manifest written to {}", run_p.display().to_string().dimmed());
println!("Spend ledger written to {}", led_p.display().to_string().dimmed());
}
if json {
println!("{}", serde_json::to_string_pretty(&dir_report)?);
}
if let Some(floor) = min_pass {
if dir_report.pass_rate < floor {
anyhow::bail!(
"pass rate {:.1}% below gate {:.1}%",
dir_report.pass_rate,
floor
);
}
}
match fail_on.as_str() {
"blocking" => {
let blocked = rules::blocking_findings(&dir_report.findings);
if !blocked.is_empty() {
let ids: Vec<String> = blocked
.iter()
.map(|f| format!("{} ({})", f.rule_id, f.scope))
.collect();
anyhow::bail!("blocking findings fail the gate: {}", ids.join(", "));
}
}
"all" => {
if !dir_report.findings.is_empty() {
anyhow::bail!("{} findings fail the gate (--fail-on all)", dir_report.findings.len());
}
}
other => anyhow::bail!("unknown --fail-on '{other}' (blocking, all)"),
}
if let Some(list) = forbid.as_deref() {
let wanted: Vec<String> = list
.split(',')
.map(|s| s.trim().trim_start_matches("RULE-").to_ascii_uppercase())
.filter(|s| !s.is_empty())
.collect();
let hit: Vec<String> = dir_report
.findings
.iter()
.filter(|f| wanted.iter().any(|w| f.rule_id.to_ascii_uppercase() == *w))
.map(|f| format!("{} ({})", f.rule_id, f.scope))
.collect();
if !hit.is_empty() {
anyhow::bail!("forbidden findings fired: {}", hit.join(", "));
}
}
if let Some(max) = max_critical {
let n = dir_report
.findings
.iter()
.filter(|f| f.severity == rules::Severity::Critical)
.count();
if n > max {
anyhow::bail!("{n} critical findings exceed --max-critical {max}");
}
}
if json {
return Ok(());
}
if dir_report.total_files > 1 {
println!("\n{} {}", "Batch Directory SEO Audit:".cyan().bold(), dir_report.dir_path);
println!(" Total Files Audited: {}", dir_report.total_files);
println!(" Total Word Count: {}", dir_report.total_words);
println!(" Avg Words per File: {}", dir_report.avg_words_per_file);
println!(" Overall Check Pass: {:.1}%", dir_report.pass_rate);
manifest::print_banner(&completeness);
print_jev_page_suite(&jev_pages);
if !dir_report.duplicate_titles.is_empty() {
println!("\n{}", "Duplicate Titles Detected:".red().bold());
for (title, files) in &dir_report.duplicate_titles {
println!(" - \"{}\" in {} files:", title, files.len());
for f in files {
println!(" {}", f.dimmed());
}
}
} else {
println!(" Title Collisions: {}", "0 (Clean)".green());
}
if !dir_report.keyword_cannibalization.is_empty() {
println!("\n{}", "Keyword Cannibalization Detected:".yellow().bold());
for item in &dir_report.keyword_cannibalization {
println!(" - Target Stem: \"{}\" in {} pages:", item.keyword_stem.cyan(), item.colliding_files.len());
for f in &item.colliding_files {
println!(" {}", f.dimmed());
}
}
let pairs = audit::cannibalization_pairs(&dir_report);
if !pairs.is_empty() {
let verdicts = judge_pairs(&pairs, is_rescore || no_jev);
println!("\n{}", format!("Conflict pairs ({}):", pairs.len()).yellow().bold());
for (i, p) in pairs.iter().take(10).enumerate() {
let mark = match verdicts.get(i).copied().flatten() {
Some((true, c)) => format!(" [jev distinct {:.2}]", c).green().to_string(),
Some((false, c)) => format!(" [jev merge? {:.2}]", c).yellow().to_string(),
None => String::new(),
};
println!(
" \"{}\" {} ({}w) × {} ({}w) → keep {}{}",
p.keyword_stem.cyan(),
p.a,
p.a_words,
p.b,
p.b_words,
p.winner.green(),
mark
);
}
if pairs.len() > 10 {
println!(" ... and {} more pairs", pairs.len() - 10);
}
if let Some(out) = &pairs_csv {
let mut s = String::from("stem,a,b,a_words,b_words,winner\n");
let cell = |v: &str| format!("\"{}\"", v.replace('"', "\"\""));
for p in &pairs {
s.push_str(&format!(
"{},{},{},{},{},{}\n",
cell(&p.keyword_stem),
cell(&p.a),
cell(&p.b),
p.a_words,
p.b_words,
cell(&p.winner)
));
}
manifest::gate_report(&s, &dir_report.findings, &actions)?;
std::fs::write(out, s)?;
println!("Cannibalization pairs CSV written to {}", out.dimmed());
}
}
} else {
println!(" Cannibalization: {}", "0 (Unique Intent)".green());
}
if !dir_report.orphan_pages.is_empty() {
println!("\n{}", format!("Orphan Pages (0 incoming internal links, {} files):", dir_report.orphan_pages.len()).yellow().bold());
for f in dir_report.orphan_pages.iter().take(5) {
println!(" - {}", f.yellow());
}
if dir_report.orphan_pages.len() > 5 {
println!(" ... and {} more", dir_report.orphan_pages.len() - 5);
}
} else {
println!(" Orphan Pages: {}", "0 (Fully Interlinked)".green());
}
if !dir_report.thin_pages.is_empty() {
println!("\n{}", format!("Thin Pages (<300 words, {} files):", dir_report.thin_pages.len()).yellow().bold());
for (f, wc) in dir_report.thin_pages.iter().take(5) {
println!(" - {} ({} words)", f, wc);
}
if dir_report.thin_pages.len() > 5 {
println!(" ... and {} more", dir_report.thin_pages.len() - 5);
}
}
if !dir_report.missing_canonicals.is_empty() {
println!(" Missing Canonicals: {} files", dir_report.missing_canonicals.len().to_string().yellow());
}
println!("\n{}", "Summary Status: Audit complete across directory.".green());
let actions = crate::rules::actions_for(&dir_report.findings);
println!(" Rule findings: {}", dir_report.findings.len());
if actions.is_empty() {
println!(" Actions: {}", "none, directory is clean".green());
} else {
print_top_actions(&actions);
}
} else if let Some(report) = dir_report.reports.first() {
println!("\n{} {}", "On-Page SEO Audit:".cyan().bold(), report.file_path);
manifest::print_banner(&completeness);
println!(" Title: {}", report.title.as_deref().unwrap_or("N/A"));
println!(" Description: {}", report.description.as_deref().unwrap_or("N/A"));
println!(" Headings: H1: {}, H2: {}, H3: {}", report.h1_count, report.h2_count, report.h3_count);
if !report.heading_skipped_levels.is_empty() {
println!(" Skipped H*: {}", report.heading_skipped_levels.join(", ").yellow());
}
println!(" Word Count: {}", report.word_count);
if report.em_dash_count > 0 || !report.ai_slop_words_found.is_empty() {
println!(" AI Tells: {} em-dashes, words: [{}]", report.em_dash_count, report.ai_slop_words_found.join(", "));
}
println!(" Links: Internal: {}, External: {}", report.internal_links, report.external_links);
println!("\n{}", "Check Results:".bold());
for check in &report.checks {
let badge = if check.passed { "PASS".green().bold() } else { "WARN".yellow().bold() };
println!(" [{}] {:<20} - {}", badge, check.name, check.message);
}
let file_findings: Vec<crate::rules::Finding> = dir_report
.findings
.iter()
.filter(|f| f.scope == report.file_path)
.cloned()
.collect();
let failed = crate::rules::actions_for(&file_findings);
if !failed.is_empty() {
print_top_actions(&failed);
}
if let Some(query) = target_query {
if no_jev {
} else if let Some((eval, v)) = gated_eval_with("audit", json!({
"target_query": query,
"page_title": report.title,
"description": report.description,
"checks": report.checks,
"page": {
"title": report.title,
"description": report.description,
"text": excerpt_local(&report.file_path),
"word_count": report.word_count
}
}), merge_extras(policy::geo_questions(), policy::page_audit_extras())) {
println!("\n{}", "Semantic Gap Evaluation (TypeSafe Jev):".cyan().bold());
println!(" GEO Score: {}/10{}", eval.geo_score, policy::marker(v));
if let Some((comp, cconf)) = policy::composite_geo(&eval.extra) {
println!(" Composite GEO: {}/10 (confidence {:.2})", comp, cconf);
}
println!(" Direct Answer: {} (p={:.2})", if eval.direct_answer { "YES".green() } else { "NO".red() }, eval.direct_answer_p);
println!(" Content Gap: {}", eval.content_gap.yellow());
print_page_extras(&eval.extra);
}
} else if !jev_pages.is_empty() {
}
}
}
Commands::Geo { target, query, json, jev_budget: _ } => {
let raw = match crate::paths::read_user_file(&target, &["md", "mdx", "markdown", "html", "htm", "txt"]) {
Ok(c) => c,
Err(e) => {
eprintln!("{}", format!("Error: {e:#}").red());
return Ok(());
}
};
let lower = target.to_ascii_lowercase();
let is_html = lower.ends_with(".html") || lower.ends_with(".htm");
let text = if is_html {
crate::fetch::readable_text(&raw, 6000)
} else {
raw.chars().take(6000).collect::<String>()
};
let opening = is_html.then(|| crate::fetch::opening_after_h1(&raw, 500));
if let Some(client) = engine::JevClient::new() {
let wc = text.split_whitespace().count();
let state = engine::page_state(&query, Some(target.clone()), None, text, wc, opening);
match client.judge_page(state) {
Ok(eval) => {
if policy::injection_blocked(&eval.extra) {
eprintln!("{}", "Jev blocked: injection risk in content (preflight); no score.".yellow());
print_jev_spend_line();
return Ok(());
}
let v = policy::gate("geo", eval.geo_confidence);
if v == policy::Verdict::Drop {
eprintln!("{}", format!("Jev unsure (confidence {:.2}), no score.", eval.geo_confidence).yellow());
print_jev_spend_line();
return Ok(());
}
let m = policy::marker(v);
if json {
println!("{}", serde_json::to_string_pretty(&eval)?);
} else {
println!("\n{}", "Generative Engine Optimization (GEO) Report:".cyan().bold());
println!(" Target Query: {}", query);
println!(" GEO Score: {}/10{}", eval.geo_score, m);
if let Some((composite, cconf)) = policy::composite_geo(&eval.extra) {
println!(" Composite: {}/10 (confidence: {:.2})", composite, cconf);
}
let review = policy::needs_review(&eval.extra, "geo");
if !review.is_empty() {
println!(" Needs review: {} [{}]", review.len().to_string().yellow(), review.join(", ").dimmed());
}
println!(" Direct Answer: {} (p={:.2})", if eval.direct_answer { "YES".green() } else { "NO".red() }, eval.direct_answer_p);
println!(" Primary Gap: {}", eval.content_gap.yellow());
print_page_extras(&eval.extra);
if let Ok(db) = rank::DbStore::open() {
match db.record_geo(&target, &query, eval.geo_score) {
Ok(Some(prev)) if prev != eval.geo_score => {
let arrow = if eval.geo_score > prev { "▲".green() } else { "▼".red() };
println!(" Since Last: {} {} → {}", arrow, prev, eval.geo_score);
}
Ok(Some(prev)) => println!(" Since Last: {} (no change)", prev),
_ => println!(" Since Last: first recorded check"),
}
}
}
print_jev_spend_line();
}
Err(e) => eprintln!("{}", format!("Error: Jev scoring failed ({e:#}).").red()),
}
} else {
let k = geo_keyless::score(&text, &query);
if json {
println!("{}", serde_json::to_string_pretty(&k)?);
} else {
println!("\n{}", "GEO Report (keyless deterministic):".cyan().bold());
println!(" Target Query: {}", query);
println!(" GEO Score: {}/10 ({} / 100)", k.score_10, k.score_100);
for s in &k.signals {
println!(" - {}", s.dimmed());
}
println!(" {}", "Set TYPESAFE_API_KEY for Jev semantic judgment.".dimmed());
}
if let Ok(db) = rank::DbStore::open() {
let _ = db.record_geo(&target, &query, k.score_10);
}
}
}
Commands::Schema { target, json } => {
let report = schema::validate_target(&target)?;
if json {
println!("{}", serde_json::to_string_pretty(&report)?);
return Ok(());
}
println!("\n{} {}", "Schema.org Structured Data Audit:".cyan().bold(), report.target);
println!(" Schemas Found: {}", report.schemas_found);
println!(" Detected Types: {}", if report.types.is_empty() { "None".to_string() } else { report.types.join(", ") });
println!(" Completeness Score: {}/100", report.completeness_score);
println!(" Validation Status: {}", if report.is_valid { "VALID".green().bold() } else { "INVALID".red().bold() });
if !report.details.is_empty() {
println!("\n{}", "Schema Details:".bold());
for d in &report.details {
let status = if d.is_valid { "PASS".green() } else { "FAIL".red() };
println!(" [{}] @type: {}", status, d.schema_type.bold());
if !d.missing_required.is_empty() {
println!(" Missing required: {}", d.missing_required.join(", ").red());
}
if !d.missing_recommended.is_empty() {
println!(" Missing recommended: {}", d.missing_recommended.join(", ").yellow());
}
if let Some(dep) = &d.deprecation_notice {
println!(" DEPRECATION: {}", dep.yellow());
}
}
}
if !report.errors.is_empty() {
println!("\n{}", "Errors:".red().bold());
for e in &report.errors {
println!(" ✖ {}", e);
}
}
if !report.warnings.is_empty() {
println!("\n{}", "Warnings:".yellow().bold());
for w in &report.warnings {
println!(" ⚠ {}", w);
}
}
}
Commands::Robots { domain, json } => {
let report = robots::inspect_robots(&domain)?;
if json {
println!("{}", serde_json::to_string_pretty(&report)?);
return Ok(());
}
println!("\n{} {}", "robots.txt & AI Crawler Audit:".cyan().bold(), report.domain);
println!(" Robots URL: {}", report.robots_url.dimmed());
println!(" HTTP Status: {}", report.status_code);
println!(" Has robots.txt: {}", if report.has_robots { "YES".green() } else { "NO".red() });
println!(" Citation bots: {}/{} search crawlers unblocked", report.citation_bots_allowed, robots::CITATION_BOTS.len());
if !report.sitemaps.is_empty() {
println!("\n{}", "Sitemaps Discovered:".cyan().bold());
for s in &report.sitemaps {
println!(" - {}", s);
}
}
println!("\n{}", "AI Crawler Permissions:".bold());
for rule in &report.ai_bot_rules {
let badge = match rule.status {
robots::BotStatus::Allowed => "ALLOW".green().bold(),
robots::BotStatus::Disallowed => "BLOCK".red().bold(),
robots::BotStatus::DefaultStar => "DEFAULT(*)".yellow(),
};
println!(" [{:<10}] {:<16} ({})", badge, rule.bot_name.bold(), rule.purpose.dimmed());
println!(" {}", rule.rule_snippet.dimmed());
}
}
Commands::Brief { topic, limit, provider, markdown, json } => {
let backend = match provider.as_str() {
"ddg" => serp::Provider::Ddg,
"tavily" => serp::Provider::Tavily,
"dfs" => serp::Provider::Dfs,
"auto" => serp::Provider::Auto,
other => anyhow::bail!("unknown --provider '{other}' (auto, ddg, tavily, dfs)"),
};
let brief = brief::generate_brief_with(&topic, limit, backend)?;
if json {
println!("{}", serde_json::to_string_pretty(&brief)?);
return Ok(());
}
if markdown {
println!("{}", brief.to_markdown());
return Ok(());
}
println!("\n{} \"{}\"", "Content Brief Blueprint:".cyan().bold(), brief.topic);
println!(" Suggested Title: {}", brief.suggested_title.green().bold());
println!(" Target Length: {}", brief.target_word_count);
println!(" Search Intent: {}", brief.search_intent);
println!(" Audience: {}", brief.target_audience);
println!(" Differentiator: {}", brief.winning_angle.yellow());
println!("\n{}", "GEO 150-Word Opening Prescription:".cyan().bold());
println!(" {}", brief.geo_opening_prescription);
println!("\n{}", "Recommended Heading Outline (H2):".bold());
for h2 in &brief.recommended_h2_outline {
println!(" {}", h2);
}
println!("\n{}", "Top Competitors Analyzed:".bold());
for comp in &brief.competitor_benchmarks {
println!(" #{} {} ({})", comp.rank, comp.title, comp.url.dimmed());
}
}
Commands::Rank { domain, query } => {
println!("{}", format!("Searching rank for domain: \"{}\" on query: \"{}\"...", domain, query).dimmed());
let (items, served) = serp::scrape_serp(&query, 30)?;
let depth = serp::effective_limit(served, 30);
let position = items.iter().position(|i| paths::url_matches_domain(&i.url, &domain)).map(|p| p + 1);
let target_url = position.and_then(|p| items.get(p - 1)).map(|i| i.url.as_str());
let mut db = rank::DbStore::open()?;
let (prov_name, engine_name) = match served {
serp::Provider::Tavily => ("tavily", "tavily-search"),
serp::Provider::Dfs => ("dfs", "dataforseo-serp"),
_ => ("ddg", "duckduckgo-html"),
};
let delta = db.track_keyword(&domain, &query, position, target_url, prov_name, engine_name)?;
println!("\n{}", "Rank Tracking Result:".cyan().bold());
println!(" Domain: {}", delta.domain);
println!(" Keyword: {}", delta.term);
let rank_str = match delta.curr_rank {
Some(r) => format!("#{}", r).green().bold().to_string(),
None => format!("Not in top {depth}").red().to_string(),
};
println!(" Current: {}", rank_str);
if let Some(prev) = delta.prev_rank {
println!(" Previous: #{}", prev);
if let Some(curr) = delta.curr_rank {
if curr < prev {
println!(" Change: {}", format!("▲{}", prev - curr).green().bold());
} else if curr > prev {
println!(" Change: {}", format!("▼{}", curr - prev).red().bold());
} else {
println!(" Change: {}", "No change".dimmed());
}
}
} else {
println!(" Previous: First recorded check");
}
if let Ok(trail) = db.observation_trail(&domain, &query) {
for (pos, prov, eng) in trail.iter().take(5) {
let at = match pos {
Some(r) => format!("#{}", r),
None => "miss".to_string(),
};
println!(" Trail: {} via {} ({})", at.dimmed(), prov.dimmed(), eng.dimmed());
}
}
}
Commands::Sitemap { target, json } => {
let report = sitemap::audit_sitemap(&target)?;
if json {
println!("{}", serde_json::to_string_pretty(&report)?);
return Ok(());
}
println!("\n{} {}", "XML Sitemap & Hreflang Audit:".cyan().bold(), report.target);
println!(" Valid Sitemap: {}", if report.is_valid { "YES".green().bold() } else { "NO".red().bold() });
println!(" Total URLs: {}", report.total_urls);
println!(" HTTPS URLs: {}/{} ({:.1}%)",
report.https_urls,
report.total_urls,
if report.total_urls > 0 { (report.https_urls as f64 / report.total_urls as f64) * 100.0 } else { 0.0 }
);
if report.insecure_http_urls > 0 {
println!(" Insecure HTTP: {}", format!("{} URLs", report.insecure_http_urls).red().bold());
}
if report.urls_with_params > 0 {
println!(" Query Params: {}", format!("{} URLs with '?'", report.urls_with_params).yellow());
}
println!(" With <lastmod>: {}/{}", report.urls_with_lastmod, report.total_urls);
println!(" Hreflang Tags: {}", report.hreflang_count);
if !report.sample_urls.is_empty() {
println!("\n{}", "Sample URLs:".bold());
for u in &report.sample_urls {
println!(" - {}", u.dimmed());
}
}
if !report.errors.is_empty() {
println!("\n{}", "Errors:".red().bold());
for e in &report.errors {
println!(" ✖ {}", e);
}
}
if !report.warnings.is_empty() {
println!("\n{}", "Warnings:".yellow().bold());
for w in &report.warnings {
println!(" ⚠ {}", w);
}
}
}
Commands::Crawl { url, max_pages, fetch, max_credits, json, diff, csv, rescore, manifest, no_jev, jev_budget: _, vitals } => {
if let Some(path) = rescore {
if csv.is_some() || diff || manifest.is_some() || vitals {
eprintln!("{}", "Note: --rescore rebuilds from saved pages; --csv/--diff/--manifest/--vitals are ignored.".yellow());
}
let raw = std::fs::read_to_string(&path)?;
let saved: crawl::CrawlReport = serde_json::from_str(&raw)?;
let report = crawl::finish_report(crawl::ReportParts {
start_url: saved.start_url,
pages: saved.pages,
redirects: saved.redirects,
inbound: saved.inbound,
errors: saved.errors,
seeded_from_sitemap: saved.seeded_from_sitemap,
capped: saved.capped,
robots_honored: saved.robots_honored,
vitals: saved.vitals,
probes: vec![],
});
let completeness = manifest::completeness_crawl(&report, max_pages);
if json {
println!("{}", serde_json::to_string_pretty(&report)?);
} else {
println!(" Rescored: {}/100 ({})", report.score, crate::actions::grade(report.score));
manifest::print_banner(&completeness);
}
return Ok(());
}
let url = url.unwrap_or_else(|| {
eprintln!("{}", "Error: provide a start URL or --rescore PATH.".red());
std::process::exit(2);
});
eprintln!("{}", format!("Crawling {} (max {} pages)...", url, max_pages).dimmed());
let mode = match fetch.as_str() {
"direct" => crate::fetch::FetchMode::Direct,
"jina" => crate::fetch::FetchMode::Jina,
"firecrawl" => crate::fetch::FetchMode::Firecrawl,
_ => crate::fetch::FetchMode::Auto,
};
let mut budget = crate::fetch::Budget { max_credits, spent: 0 };
let mut report = crawl::crawl_site(&url, max_pages, mode, &mut budget)?;
if vitals {
eprintln!("{}", "Fetching PageSpeed vitals for start URL (free, keyless)...".dimmed());
let v = crate::vitals::fetch_home_vitals(&report.start_url);
if v.is_none() {
eprintln!("{}", "Note: PageSpeed unreachable or rate-limited; vitals skipped.".yellow());
}
crate::crawl::apply_vitals(&mut report, v);
}
let site_jev = if no_jev { None } else { judge_crawl_site(&report.start_url) };
let mut completeness_pre = manifest::completeness_crawl(&report, max_pages);
if site_jev.is_some() {
completeness_pre
.notes
.push("Jev site+GEO judged on homepage".into());
}
if json {
let mut ledger = manifest::Ledger::new("crawl", &report.start_url);
let backends = paid_backend_sources(&report.pages);
ledger.with_fetch(budget.spent, budget.max_credits, backends);
if site_jev.is_some() {
ledger.note("Jev site+GEO fan-out on homepage");
}
let run_manifest = manifest::ManifestInput {
command: "crawl",
target: &report.start_url,
findings: &report.findings,
actions: &report.actions,
texts: &[],
score: Some(manifest::ScoreCard {
score: report.score,
grade: report.grade.clone(),
kind: "health".into(),
}),
completeness: completeness_pre,
ledger,
}
.build();
if !run_manifest.validation.ok {
anyhow::bail!("run manifest citation gate failed: {:?}", run_manifest.validation);
}
let dirs = match &manifest {
Some(dir) => vec![std::path::PathBuf::from(dir)],
None => manifest::auto_dirs(&[csv.as_deref()]),
};
let dirs = if dirs.is_empty() {
vec![std::path::PathBuf::from(".")]
} else {
dirs
};
for d in &dirs {
let (run_p, led_p) = manifest::write_pair(d, &run_manifest)?;
eprintln!("Run manifest written to {}", run_p.display());
eprintln!("Spend ledger written to {}", led_p.display());
}
println!("{}", serde_json::to_string_pretty(&report)?);
return Ok(());
}
println!("\n{} {}", "Live Site Crawl:".cyan().bold(), report.start_url);
println!(" Pages crawled: {}", report.pages_crawled);
println!(" Health: {}/100 ({})", report.score, crate::actions::grade(report.score).green().bold());
let upgraded = report.pages.iter().filter(|p| p.upgraded).count();
if upgraded > 0 {
println!(" Upgraded: {} pages via alt backends", upgraded.to_string().yellow());
}
if budget.spent > 0 {
println!(" Paid spend: {} fetch credits", budget.spent.to_string().yellow());
}
println!(
" Areas: {}",
report
.areas
.iter()
.map(|a| format!("{} {}", crate::rules::label(&a.area), a.score))
.collect::<Vec<_>>()
.join(", ")
.dimmed()
);
match &report.vitals {
Some(v) => {
let lcp = v.lcp_ms.map(|x| format!("{}ms", x)).unwrap_or_else(|| "-".into());
let cls = v.cls_milli.map(crate::vitals::cls_display).unwrap_or_else(|| "-".into());
let inp = match v.inp_ms {
Some(x) if x > crate::vitals::INP_MS => format!("{}ms over", x),
Some(x) => format!("{}ms", x),
None => "-".into(),
};
let lab = if v.field { "field+lab" } else { "lab only" };
println!(" Vitals: LCP {} CLS {} INP {} ({})", lcp, cls, inp, lab.dimmed());
}
None => {
if vitals {
println!(" Vitals: {}", "unavailable (PageSpeed skipped)".yellow());
}
}
}
if report.broken.is_empty() {
println!(" Broken links: {}", "0 (Clean)".green());
} else {
println!(" Broken links: {}", report.broken.len().to_string().red().bold());
for p in report.broken.iter().take(5) {
println!(" ✖ [{}] {}", p.status, p.url.dimmed());
}
if report.broken.len() > 5 {
println!(" ... and {} more", report.broken.len() - 5);
}
}
let slow: Vec<&crawl::PageRecord> = report
.pages
.iter()
.filter(|p| p.status == 200 && p.elapsed_ms > crawl::SLOW_PAGE_MS)
.collect();
if slow.is_empty() {
println!(" Slow pages: {}", "0".green());
} else {
println!(" Slow pages: {}", format!("{} over {}ms", slow.len(), crawl::SLOW_PAGE_MS).yellow());
for p in slow.iter().take(5) {
println!(" {}ms {}", p.elapsed_ms, p.url.dimmed());
}
}
if report.redirects.is_empty() {
println!(" Redirects: {}", "0".green());
} else {
println!(" Redirects: {}", report.redirects.len().to_string().yellow());
for (from, to) in report.redirects.iter().take(3) {
println!(" {} -> {}", from.dimmed(), to.dimmed());
}
}
if report.orphans.is_empty() {
println!(" Orphans: {}", "0".green());
} else {
println!(" Orphans: {}", report.orphans.len().to_string().yellow());
for f in report.orphans.iter().take(5) {
println!(" {}", f.yellow());
}
}
if !report.errors.is_empty() {
println!(" Fetch errors: {}", report.errors.len().to_string().yellow());
for e in report.errors.iter().take(3) {
println!(" {}", e.dimmed());
}
}
if report.actions.is_empty() {
println!(" Actions: {}", "none, site is clean".green());
} else {
print_top_actions(&report.actions);
}
if diff {
match rank::DbStore::open()?.record_crawl_snapshot(
&report.start_url,
report.pages_crawled as i64,
report.broken.len() as i64,
)? {
Some((prev_pages, prev_broken)) => {
println!(
" Since last: {} pages (was {}), {} broken (was {})",
report.pages_crawled, prev_pages, report.broken.len(), prev_broken
);
}
None => println!(" Since last: first recorded snapshot"),
}
}
if let Some(out) = &csv {
let body = crate::rules::to_csv(&report.findings);
manifest::gate_report(&body, &report.findings, &report.actions)?;
std::fs::write(out, body)?;
println!("CSV findings written to {}", out.dimmed());
}
let mut completeness = manifest::completeness_crawl(&report, max_pages);
if site_jev.is_some() {
completeness
.notes
.push("Jev site+GEO judged on homepage".into());
}
manifest::print_banner(&completeness);
if let Some(eval) = &site_jev {
let v = policy::gate("crawl", eval.confidence());
println!("\n{}", "Jev site view (TypeSafe):".cyan().bold());
println!(
" GEO: {}/10{}",
eval.geo_score,
policy::marker(v)
);
if let Some((c, cf)) = policy::composite_geo(&eval.extra) {
println!(" Composite GEO: {}/10 (confidence {:.2})", c, cf);
}
println!(
" Intent: {} (confidence {:.2}){}",
eval.intent.green(),
eval.intent_confidence,
runner_up_suffix(&eval.extra, v).dimmed()
);
println!(" Value prop: {}", eval.content_gap.yellow());
print_page_extras(&eval.extra);
print_jev_spend_line();
}
let mut ledger = manifest::Ledger::new("crawl", &report.start_url);
if site_jev.is_some() {
ledger.note("Jev site+GEO fan-out on homepage");
}
let backends = paid_backend_sources(&report.pages);
ledger.with_fetch(budget.spent, budget.max_credits, backends);
if budget.spent > 0 {
ledger.note(format!("{} paid fetch credits spent (cap {})", budget.spent, budget.max_credits));
}
let run_manifest = manifest::ManifestInput {
command: "crawl",
target: &report.start_url,
findings: &report.findings,
actions: &report.actions,
texts: &[],
score: Some(manifest::ScoreCard {
score: report.score,
grade: report.grade.clone(),
kind: "health".into(),
}),
completeness: completeness.clone(),
ledger,
}
.build();
if !run_manifest.validation.ok {
anyhow::bail!("run manifest citation gate failed: {:?}", run_manifest.validation);
}
let mut export_dirs = manifest::auto_dirs(&[csv.as_deref()]);
if let Some(dir) = &manifest {
export_dirs = vec![std::path::PathBuf::from(dir)];
}
let force = manifest.is_some();
let mut wrote_manifest = false;
for d in &export_dirs {
let (run_p, led_p) = manifest::write_pair(d, &run_manifest)?;
println!("Run manifest written to {}", run_p.display().to_string().dimmed());
println!("Spend ledger written to {}", led_p.display().to_string().dimmed());
wrote_manifest = true;
}
if !wrote_manifest && (force || !json) {
let (run_p, led_p) = manifest::write_pair(std::path::Path::new("."), &run_manifest)?;
println!("Run manifest written to {}", run_p.display().to_string().dimmed());
println!("Spend ledger written to {}", led_p.display().to_string().dimmed());
}
}
Commands::Llms { domain, json } => {
let t0 = std::time::Instant::now();
let report = llms::check_llms(&domain)?;
eprintln!(
"[jev-seo {:>02}:{:>02}] readiness checked in {:.1}s",
t0.elapsed().as_secs() / 60,
t0.elapsed().as_secs() % 60,
t0.elapsed().as_secs_f64()
);
if json {
println!("{}", serde_json::to_string_pretty(&report)?);
return Ok(());
}
println!("\n{} {}", "Agent Readiness:".cyan().bold(), report.domain);
println!(" llms.txt: {}", if report.info.present { "YES".green() } else { "NO".red() });
if report.info.present {
println!(" Sections: {}", report.info.sections.len());
}
println!(" AI explicit: {}", if report.ai_allowed.is_empty() { "none".yellow().to_string() } else { report.ai_allowed.join(", ").green().to_string() });
println!(" AI default: {} bots inherit allow (*)", report.ai_default.len());
if !report.ai_blocked.is_empty() {
println!(" AI blocked: {}", report.ai_blocked.join(", ").red());
}
println!(" Score: {}/100", report.score);
if report.actions.is_empty() {
println!(" Actions: {}", "none, ready".green());
} else {
print_top_actions(&report.actions);
}
let completeness = manifest::completeness_llms(&report);
manifest::print_banner(&completeness);
println!(" {}", "Scores plumbing for ChatGPT, Perplexity, and Claude. Google Search ignores llms.txt.".dimmed());
println!("\n{}", "Checks:".bold());
for c in &report.checks {
let badge = if c.passed { "PASS".green().bold() } else { "WARN".yellow().bold() };
println!(" [{}] {:<22} - {}", badge, c.name, c.message);
}
}
Commands::Doctor { json } => {
let key_set = std::env::var("TYPESAFE_API_KEY").map(|k| !k.trim().is_empty()).unwrap_or(false);
let db_ok = rank::DbStore::open().is_ok();
let report = serde_json::json!({
"version": env!("CARGO_PKG_VERSION"),
"os": std::env::consts::OS,
"typesafe_key": if key_set { "set" } else { "missing" },
"database": if db_ok { "writable" } else { "error" },
});
if json {
println!("{}", serde_json::to_string_pretty(&report)?);
return Ok(());
}
println!("\n{}", "Doctor:".cyan().bold());
println!(" Version: {}", env!("CARGO_PKG_VERSION"));
println!(" Platform: {}", std::env::consts::OS);
println!(
" Jev API key: {}",
if key_set { "set (value hidden)".green().to_string() } else { "missing, Jev scores will skip".yellow().to_string() }
);
println!(" Database: {}", if db_ok { "writable".green().to_string() } else { "ERROR".red().to_string() });
}
Commands::Explain { id, json } => {
let text = rules::explain(&id)
.ok_or_else(|| anyhow::anyhow!("unknown rule id: {id} (try R01..R57 or RULE-R01..RULE-R57)"))?;
if json {
let bare = id.strip_prefix("RULE-").unwrap_or(&id).to_string();
let r = rules::rule(&bare).expect("explain found it");
println!(
"{}",
serde_json::to_string_pretty(&json!({
"id": r.id,
"area": rules::label(&r.area),
"severity": format!("{:?}", r.severity),
"gate": match rules::gate(r.id) {
rules::Gate::Blocking => "blocking",
rules::Gate::Advisory => "advisory",
},
"effort": r.effort,
"title": r.title,
"fix": r.fix,
}))?
);
return Ok(());
}
println!("{}", text.cyan());
}
Commands::Report { path, baseline, actions_csv, json } => {
let cur_src = crate::paths::read_user_file(&path, &["json"])?;
let base_src = crate::paths::read_user_file(&baseline, &["json"])?;
let cur: audit::DirectoryAuditReport = serde_json::from_str(&cur_src)?;
let base: audit::DirectoryAuditReport = serde_json::from_str(&base_src)?;
let diff = diff_audit_reports(&cur, &base);
let actions = crate::rules::actions_for(&cur.findings);
if let Some(out) = &actions_csv {
write_actions_csv(out, &cur.findings, &actions)?;
}
if json {
println!("{}", serde_json::to_string_pretty(&json!({
"diff": diff,
"actions": actions,
}))?);
return Ok(());
}
let delta = diff["delta"].as_i64().unwrap_or(0);
let sign = if delta > 0 { "+" } else { "" };
println!("\n{}", "Report diff (baseline → current):".cyan().bold());
println!(
" Score: {} → {} ({sign}{delta}) grade {}",
diff["baseline_score"],
diff["current_score"],
diff["grade"]
);
println!(
" Findings: {} → {}",
diff["findings_baseline"], diff["findings_current"]
);
let fixed: Vec<String> = serde_json::from_value(diff["rules_fixed"].clone())?;
let added: Vec<String> = serde_json::from_value(diff["rules_new"].clone())?;
if fixed.is_empty() && added.is_empty() {
println!(" Rules: no new or cleared rule ids");
} else {
if !fixed.is_empty() {
println!(" Cleared: {}", fixed.join(", ").green());
}
if !added.is_empty() {
println!(" New: {}", added.join(", ").yellow());
}
}
if !actions.is_empty() {
println!("\n{}", "Top actions:".cyan().bold());
for a in actions::top(&actions, 5) {
println!(
" [P{}] {} {} impact {}{}",
a.priority,
a.id.bold(),
a.title,
a.impact,
if a.quick_win { " quick-win".green().to_string() } else { String::new() }
);
}
}
}
Commands::Bundle { path, out } => {
let src = crate::paths::read_user_file(&path, &["json"])?;
let saved: audit::DirectoryAuditReport = serde_json::from_str(&src)?;
let rep = audit::with_findings(saved);
let actions = crate::rules::actions_for(&rep.findings);
let drift = rank::DbStore::open()
.ok()
.and_then(|db| db.drift_alerts(200).ok())
.unwrap_or_default();
let body = audit::bundle_markdown(&rep, &actions, &drift);
manifest::gate_report(&body, &rep.findings, &actions)?;
std::fs::write(&out, body)?;
println!("Bundle written to {}", out.dimmed());
}
Commands::Drift { op, report, label, json } => {
let label = label.unwrap_or_else(|| "latest".into());
let db = rank::DbStore::open()?;
match op.as_str() {
"baseline" => {
let path = report.unwrap_or_else(|| {
eprintln!("{}", "Error: baseline needs --report <audit JSON>.".red());
std::process::exit(2);
});
let src = crate::paths::read_user_file(&path, &["json"])?;
let parsed: audit::DirectoryAuditReport = serde_json::from_str(&src)?;
db.save_baseline(&label, &src)?;
if json {
println!("{}", serde_json::to_string_pretty(&serde_json::json!({
"saved": label, "files": parsed.total_files, "score": parsed.pass_rate,
}))?);
} else {
println!("Baseline '{}' saved: {} files, pass {:.1}%", label.dimmed(), parsed.total_files, parsed.pass_rate);
}
}
"compare" => {
let path = report.unwrap_or_else(|| {
eprintln!("{}", "Error: compare needs --report <audit JSON>.".red());
std::process::exit(2);
});
let base_src = db.load_baseline(&label)?.unwrap_or_else(|| {
eprintln!("{}", format!("Error: no baseline '{}' stored.", label).red());
std::process::exit(2);
});
let cur_src = crate::paths::read_user_file(&path, &["json"])?;
let cur: audit::DirectoryAuditReport = serde_json::from_str(&cur_src)?;
let base: audit::DirectoryAuditReport = serde_json::from_str(&base_src)?;
let diff = diff_audit_reports(&cur, &base);
let actions = crate::rules::actions_for(&cur.findings);
if json {
println!("{}", serde_json::to_string_pretty(&serde_json::json!({
"diff": diff, "actions": actions,
"pairs": crate::audit::cannibalization_pairs(&cur),
}))?);
return Ok(());
}
let delta = diff["delta"].as_i64().unwrap_or(0);
let sign = if delta > 0 { "+" } else { "" };
println!("\n{} '{}':", "Drift vs baseline".cyan().bold(), label);
println!(
" Score: {} → {} ({sign}{delta}) grade {}",
diff["baseline_score"], diff["current_score"], diff["grade"]
);
if !actions.is_empty() {
println!("\n{}", "Top actions:".cyan().bold());
for a in actions::top(&actions, 5) {
println!(" [P{}] {} {}", a.priority, a.id.bold(), a.title);
}
}
}
"history" => {
let labels = db.list_baselines()?;
if json {
println!("{}", serde_json::to_string_pretty(&labels)?);
} else if labels.is_empty() {
println!("No baselines stored. Save one: jev-seo drift baseline --report <audit JSON>");
} else {
println!("\n{}", "Stored baselines:".cyan().bold());
for (name, at) in &labels {
println!(" {:<20} {}", name.green(), at.dimmed());
}
}
}
other => {
eprintln!("{}", format!("Error: unknown drift action '{}', use baseline, compare, or history.", other).red());
std::process::exit(2);
}
}
}
Commands::Gsc { op, site, code, limit, json } => {
match op.as_str() {
"auth" => {
if let Some(device) = code {
gsc::auth_poll(&device)?;
} else {
gsc::auth_start()?;
}
}
"sites" => {
let sites = gsc::sites()?;
if json {
println!("{}", serde_json::to_string_pretty(&sites)?);
} else {
println!("\n{}", "Verified sites:".cyan().bold());
for s in &sites {
println!(" - {}", s);
}
}
}
"query" => {
let site = site.unwrap_or_else(|| {
eprintln!("{}", "Error: query needs --site <verified URL>.".red());
std::process::exit(2);
});
let rows = gsc::top_queries(&site, limit)?;
if json {
println!("{}", serde_json::to_string_pretty(&rows)?);
} else {
println!("\n{} {}", "Top queries:".cyan().bold(), site.dimmed());
for r in &rows {
println!(
" {:<40} clicks {:>6.0} pos {:>5.1}",
r.query.chars().take(40).collect::<String>(),
r.clicks,
r.position
);
}
}
}
"gap" => {
let site = site.unwrap_or_else(|| {
eprintln!("{}", "Error: gap needs --site <verified URL>.".red());
std::process::exit(2);
});
let rows = gsc::gap(&site, limit)?;
if json {
println!("{}", serde_json::to_string_pretty(&rows)?);
} else {
println!("\n{} {}", "Gap queue (impressions x weak position):".cyan().bold(), site.dimmed());
for r in &rows {
let cited = match r.cited_before {
Some(true) => "cited".green().to_string(),
Some(false) => "missed".yellow().to_string(),
None => "unchecked".dimmed().to_string(),
};
println!(
" {:<40} imp {:>7.0} pos {:>5.1} {}",
r.query.chars().take(40).collect::<String>(),
r.impressions,
r.position,
cited
);
}
}
}
other => {
eprintln!("{}", format!("Error: unknown gsc action '{}', use auth, sites, gap, or query.", other).red());
std::process::exit(2);
}
}
}
Commands::Mcp => {
mcp::run_stdio_server()?;
}
}
Ok(())
}
fn gated_eval_with(
command: &str,
state: serde_json::Value,
extra: serde_json::Value,
) -> Option<(engine::AnalysisResult, policy::Verdict)> {
let client = engine::JevClient::new()?;
match client.fanout_eval_with(state, extra) {
Ok(eval) => {
let v = policy::gate(command, eval.confidence());
if v == policy::Verdict::Drop {
eprintln!("{}", format!("Note: Jev unsure (confidence {:.2}), showing local-only output.", eval.confidence()).yellow());
print_jev_spend_line();
return None;
}
Some((eval, v))
}
Err(e) => {
eprintln!("{}", format!("Warning: Jev scoring failed ({e:#}), showing local-only output.").yellow());
None
}
}
}
fn merge_extras(a: serde_json::Value, b: serde_json::Value) -> serde_json::Value {
let mut out = a;
if let (Some(dst), Some(src)) = (out.as_object_mut(), b.as_object()) {
for (k, v) in src {
dst.insert(k.clone(), v.clone());
}
}
out
}
fn judge_pairs(pairs: &[audit::CannibalizationPair], skip: bool) -> Vec<Option<(bool, f64)>> {
let n = pairs.len().min(10);
if skip || n == 0 {
return vec![None; n];
}
let client = match engine::JevClient::new() {
Some(c) => c,
None => {
eprintln!("{}", "Note: pair judging needs TYPESAFE_API_KEY; word-count winners stand.".yellow());
return vec![None; n];
}
};
let state = serde_json::json!({
"pairs": pairs.iter().take(n).map(|p| serde_json::json!({
"stem": p.keyword_stem,
"a": excerpt_local(&p.a).chars().take(800).collect::<String>(),
"b": excerpt_local(&p.b).chars().take(800).collect::<String>(),
})).collect::<Vec<_>>(),
});
let eval = match client.fanout_eval_with(state, policy::pair_questions(n)) {
Ok(e) => e,
Err(e) => {
eprintln!("{}", format!("Warning: pair judge failed ({e:#}).").yellow());
return vec![None; n];
}
};
(0..n)
.map(|i| {
let key = format!("pair_{}", i);
let p = policy::noul_prob(&eval.extra.get(&key).cloned().unwrap_or_default());
if p >= 0.80 {
Some((true, p))
} else if p <= 0.20 {
Some((false, 1.0 - p))
} else {
None
}
})
.collect()
}
fn excerpt_local(path: &str) -> String {
let raw = std::fs::read_to_string(path).unwrap_or_default();
let cleaned: String = raw
.chars()
.filter(|c| !c.is_control() || *c == '\n' || *c == '\t')
.collect();
let lower = path.to_ascii_lowercase();
if lower.ends_with(".html") || lower.ends_with(".htm") {
crate::fetch::readable_text(&cleaned, 6000)
} else if lower.ends_with(".md") || lower.ends_with(".mdx") || lower.ends_with(".markdown") {
let body = gray_matter_strip(&cleaned);
let mut start = 0;
for (i, l) in body.lines().enumerate() {
let t = l.trim();
if t.starts_with("# ") {
start = i + 1;
break;
}
}
let after: String = body.lines().skip(start).collect::<Vec<_>>().join("\n");
crate::audit::md_plain_text(&after).chars().take(6000).collect()
} else {
cleaned.chars().take(6000).collect()
}
}
fn gray_matter_strip(cleaned: &str) -> String {
gray_matter::Matter::<gray_matter::engine::YAML>::new()
.parse(cleaned)
.content
}
fn judge_audit_sample(
dir_report: &audit::DirectoryAuditReport,
target_query: Option<&str>,
limit: usize,
) -> Vec<(String, engine::AnalysisResult, policy::Verdict)> {
let client = match engine::JevClient::new() {
Some(c) => c,
None => return Vec::new(),
};
let mut pages: Vec<&audit::AuditReport> = dir_report.reports.iter().collect();
pages.sort_by_key(|b| {
let stem = std::path::Path::new(&b.file_path)
.file_stem()
.and_then(|s| s.to_str())
.unwrap_or("")
.to_ascii_lowercase();
let home = ["index", "readme", "home", "main"].contains(&stem.as_str());
(!home, std::cmp::Reverse(b.word_count))
});
let mut out = Vec::new();
for report in pages.into_iter().take(limit) {
let content = excerpt_local(&report.file_path);
if content.trim().is_empty() {
continue;
}
let mut state = engine::page_state(
target_query.unwrap_or(""),
report.title.clone(),
report.description.clone(),
content,
report.word_count,
None,
);
if let Some(obj) = state.as_object_mut() {
obj.insert("checks".into(), serde_json::to_value(&report.checks).unwrap_or_default());
if let Some(q) = target_query {
obj.insert("query".into(), serde_json::Value::String(q.to_string()));
}
}
match client.judge_page(state) {
Ok(eval) => {
if policy::injection_blocked(&eval.extra) {
eprintln!(
"{}",
format!(
"Blocked: injection risk in {} (dedicated Noul pre-screen); quality suite not run.",
report.file_path
)
.yellow()
);
continue;
}
let v = policy::gate("audit", eval.confidence());
if v != policy::Verdict::Drop {
out.push((report.file_path.clone(), eval, v));
} else {
eprintln!(
"{}",
format!("Note: Jev unsure on {} (confidence {:.2}), local checks stand.", report.file_path, eval.confidence()).yellow()
);
}
}
Err(e) => {
eprintln!("{}", format!("Jev page judge failed for {}: {e:#}", report.file_path).yellow());
}
}
}
out
}
fn print_jev_page_suite(pages: &[(String, engine::AnalysisResult, policy::Verdict)]) {
if pages.is_empty() {
return;
}
println!("\n{}", format!("Jev page suite ({} pages, full fan-out):", pages.len()).cyan().bold());
for (path, eval, v) in pages {
let name = std::path::Path::new(path)
.file_name()
.and_then(|n| n.to_str())
.unwrap_or(path);
let review = policy::needs_review(&eval.extra, "audit");
println!(
" {} GEO {}/10 intent {} gap {}{}{}",
name,
eval.geo_score,
eval.intent,
eval.content_gap,
policy::marker(*v),
runner_up_suffix(&eval.extra, *v).dimmed()
);
print_page_extras(&eval.extra);
if !review.is_empty() {
println!(" needs review: {}", review.join(", ").dimmed());
}
}
print_jev_spend_line();
}
fn paid_backend_sources(pages: &[crawl::PageRecord]) -> Vec<String> {
let jina_paid = std::env::var("JINA_API_KEY")
.map(|k| !k.trim().is_empty())
.unwrap_or(false);
pages
.iter()
.filter(|p| p.source != "direct" && (p.source != "jina" || jina_paid))
.map(|p| p.source.clone())
.collect::<BTreeSet<_>>()
.into_iter()
.collect()
}
fn print_top_actions(actions: &[crate::actions::Action]) {
println!("\n{}", "Top Actions:".cyan().bold());
for a in crate::actions::top(actions, 5) {
println!(" [P{}|e{}|i{:>3}] {} {} {} {} - {}", a.priority, a.effort, a.impact, if a.quick_win { "QUICK".green().bold().to_string() } else { String::new() }, action_tags(a), a.id.bold(), a.title, a.evidence.dimmed());
}
}
fn action_tags(action: &crate::actions::Action) -> String {
let bare = action.id.strip_prefix("RULE-").unwrap_or(&action.id);
let r = match rules::rule(bare) {
Some(r) => r,
None => return String::new(),
};
let scope = action.evidence.split(',').next().unwrap_or("").trim();
let gate = match rules::effective_gate(bare, scope) {
rules::Gate::Blocking => "BLOCK".red().bold().to_string(),
rules::Gate::Advisory => "warn".yellow().to_string(),
};
format!("{} {}", gate, rules::truth_kind(r.id))
}
fn runner_up_suffix(extra: &serde_json::Map<String, serde_json::Value>, v: policy::Verdict) -> String {
if v == policy::Verdict::Flag {
if let Some(r) = policy::intent_runner_up(extra) {
return format!(" ← runner-up {r}");
}
}
String::new()
}
fn print_page_extras(extra: &serde_json::Map<String, serde_json::Value>) { let keys = [
"page_helpfulness",
"page_trust",
"page_specificity",
"title_fit",
"meta_fit",
"meta_verdict",
"importance",
"value_prop",
"geo_statistics",
"geo_directness",
"answer_first",
"clear_next_step",
"query_fit",
"entity_clarity",
"injection_risk",
];
let mut parts = Vec::new();
for k in keys {
if let Some(a) = extra.get(k) {
if let Some(s) = a.get("score").and_then(|x| x.as_f64()) {
let c = a.get("confidence").and_then(|x| x.as_f64()).unwrap_or(0.0);
let norm = normalize_score(a).unwrap_or(s);
parts.push(format!("{k}={norm:.2}({c:.2})"));
} else if let Some(n) = a.get("noul").or_else(|| a.get("probability")).and_then(|x| x.as_f64()) {
parts.push(format!("{k}=P{n:.2}"));
} else if let Some(ch) = a.get("choice").and_then(|x| x.as_str()) {
parts.push(format!("{k}={ch}"));
}
}
}
if !parts.is_empty() {
println!(" {}", parts.join(" · ").dimmed());
}
}
fn normalize_score(a: &serde_json::Value) -> Option<f64> {
let s = a.get("score")?.as_f64()?;
let legend = a.get("legend")?;
let top = legend
.as_object()?
.keys()
.filter_map(|k| k.parse::<f64>().ok())
.fold(0.0f64, f64::max);
if top <= 0.0 {
return Some(s.clamp(0.0, 1.0));
}
Some((s / top).clamp(0.0, 1.0))
}
fn print_keyword_values(extra: &serde_json::Map<String, serde_json::Value>, suggestions: &[String]) {
let mut rows = Vec::new();
for (i, s) in suggestions.iter().take(10).enumerate() {
if let Some(a) = extra.get(format!("kw_value_{i}").as_str()) {
if let Some(sc) = a.get("score").and_then(|x| x.as_f64()) {
rows.push(format!("{sc:.1} {}", s));
}
}
}
if !rows.is_empty() {
println!(" Query value: {}", rows.join(" · "));
}
}
fn print_jev_spend_line() {
let reqs = manifest::JEV_REQUESTS.load(std::sync::atomic::Ordering::Relaxed);
let toks = manifest::JEV_INPUT_TOKENS.load(std::sync::atomic::Ordering::Relaxed);
let cost = manifest::jev_cost_usd(toks);
if reqs > 0 {
eprintln!(
" Jev spend: {} request(s), {} input tokens, ${:.6} [{}|{}]",
reqs,
toks,
cost,
engine::jev_model(),
policy::QUESTION_VERSION
);
}
}
fn judge_crawl_site(start_url: &str) -> Option<engine::AnalysisResult> {
let client = engine::JevClient::new()?;
let target = crate::paths::reject_private_url(start_url).ok()?;
let resp = crate::fetch::with_extra_headers(
ureq::get(target.as_str())
.timeout(std::time::Duration::from_secs(15))
.set("User-Agent", crawl::CRAWL_UA),
)
.call()
.ok()?;
let landed = resp.get_url().to_string();
crate::paths::reject_redirect_target(&landed).ok()?;
let body = crate::fetch::capped_string(resp, crate::fetch::MAX_AUX_BYTES).ok()?;
if body.trim().is_empty() {
return None;
}
let tmp = std::env::temp_dir().join(format!("jev-seo-site-{}.html", std::process::id()));
std::fs::write(&tmp, &body).ok()?;
let report = audit::audit_file(tmp.to_str()?).ok();
let _ = std::fs::remove_file(&tmp);
let text: String = crate::fetch::readable_text(&body, 6000);
let wc = text.split_whitespace().count();
let state = engine::page_state(
start_url,
report.as_ref().and_then(|r| r.title.clone()),
report.as_ref().and_then(|r| r.description.clone()),
text,
report.as_ref().map(|r| r.word_count).unwrap_or(wc),
Some(crate::fetch::opening_after_h1(&body, 500)),
);
match client.judge_site(state) {
Ok(eval) => Some(eval),
Err(e) => {
eprintln!("{}", format!("Jev site judge failed: {e:#}").yellow());
None
}
}
}