aiseo 0.6.1

Agent-first CLI for SEO, GEO (generative engine optimisation), and AEO (answer engine optimisation) audits.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115

//! Keyword frequency and question extraction.
//!
//! Deliberately heuristic: token-counting + stop-word filter + question-form
//! sentence detection. An agent uses `primary` to decide whether the page's
//! actual top keyword matches the keyword the user is trying to rank for.

use once_cell::sync::Lazy;
use regex::Regex;
use serde::Serialize;
use std::collections::{BTreeMap, HashSet};

#[derive(Serialize)]
pub struct Keywords {
    /// Top single-word terms by frequency, after stop-word filtering.
    pub primary: Vec<KeywordHit>,
    /// Sentences that look like questions (end in `?`). Drives FAQ schema
    /// generation decisions.
    pub questions: Vec<String>,
    /// Top-N density: term → percentage of total non-stop tokens.
    pub density: BTreeMap<String, f64>,
}

#[derive(Serialize)]
pub struct KeywordHit {
    pub term: String,
    pub count: usize,
}

static STOP_WORDS: &[&str] = &[
    "a", "about", "above", "after", "again", "against", "all", "am", "an", "and",
    "any", "are", "as", "at", "be", "because", "been", "before", "being", "below",
    "between", "both", "but", "by", "can", "did", "do", "does", "doing", "down",
    "during", "each", "few", "for", "from", "further", "had", "has", "have",
    "having", "he", "her", "here", "hers", "herself", "him", "himself", "his",
    "how", "i", "if", "in", "into", "is", "it", "its", "itself", "just", "me",
    "might", "more", "most", "my", "myself", "no", "nor", "not", "now", "of",
    "off", "on", "once", "only", "or", "other", "our", "ours", "ourselves", "out",
    "over", "own", "same", "she", "should", "so", "some", "such", "than", "that",
    "the", "their", "theirs", "them", "themselves", "then", "there", "these",
    "they", "this", "those", "through", "to", "too", "under", "until", "up",
    "very", "was", "we", "were", "what", "when", "where", "which", "while", "who",
    "whom", "why", "will", "with", "would", "you", "your", "yours", "yourself",
    "yourselves",
];

static SENTENCE_END: Lazy<Regex> = Lazy::new(|| Regex::new(r"[.!?]+").unwrap());
static WORD: Lazy<Regex> = Lazy::new(|| Regex::new(r"\b[a-zA-Z][a-zA-Z\-']{2,}\b").unwrap());

pub fn extract(body_text: &str) -> Keywords {
    let stop: HashSet<&str> = STOP_WORDS.iter().copied().collect();

    let mut counts: BTreeMap<String, usize> = BTreeMap::new();
    let mut total: usize = 0;

    for m in WORD.find_iter(body_text) {
        let w = m.as_str().to_ascii_lowercase();
        if stop.contains(w.as_str()) {
            continue;
        }
        total += 1;
        *counts.entry(w).or_insert(0) += 1;
    }

    let mut sorted: Vec<(String, usize)> = counts.into_iter().collect();
    sorted.sort_by(|a, b| b.1.cmp(&a.1).then(a.0.cmp(&b.0)));

    let primary: Vec<KeywordHit> = sorted
        .iter()
        .take(5)
        .map(|(t, c)| KeywordHit {
            term: t.clone(),
            count: *c,
        })
        .collect();

    let mut density: BTreeMap<String, f64> = BTreeMap::new();
    if total > 0 {
        for (t, c) in sorted.iter().take(5) {
            density.insert(t.clone(), ((*c as f64 / total as f64) * 1000.0).round() / 10.0);
        }
    }

    let questions: Vec<String> = SENTENCE_END
        .split(body_text)
        .map(str::trim)
        .filter(|s| !s.is_empty())
        .filter_map(|s| {
            // SENTENCE_END strips punctuation, so we look at the original
            // text up to where the next `?` lands to keep `?` sentences.
            // Simpler: just keep sentences whose first word is a question
            // word, OR that the original text marks with `?`.
            None.or_else(|| {
                let lower = s.to_ascii_lowercase();
                let starts_with_question = [
                    "what ", "why ", "how ", "when ", "where ", "who ", "which ", "is ", "are ",
                    "do ", "does ", "can ", "should ", "will ",
                ]
                .iter()
                .any(|w| lower.starts_with(w));
                if starts_with_question {
                    Some(format!("{s}?"))
                } else {
                    None
                }
            })
        })
        .take(10)
        .collect();

    Keywords {
        primary,
        questions,
        density,
    }
}