#[derive(Debug, Default, Clone)]
pub struct QueryAnalysis {
pub identifiers: Vec<String>,
pub quoted: Vec<String>,
pub paths: Vec<String>,
pub routes: Vec<String>,
pub keywords: Vec<String>,
pub keys: Vec<String>,
}
const STOPWORDS: &[&str] = &[
"a",
"an",
"and",
"are",
"as",
"at",
"be",
"by",
"for",
"from",
"how",
"in",
"into",
"is",
"it",
"of",
"on",
"or",
"that",
"the",
"this",
"to",
"we",
"what",
"when",
"where",
"which",
"with",
"add",
"adds",
"adding",
"make",
"fix",
"fixes",
"fixing",
"implement",
"update",
"change",
"changes",
"should",
"can",
"need",
"needs",
"do",
"does",
"please",
"why",
];
fn is_identifier_like(word: &str) -> bool {
let has_upper = word.chars().any(|c| c.is_ascii_uppercase());
let has_lower = word.chars().any(|c| c.is_ascii_lowercase());
let mixed_case =
has_upper && has_lower && word.chars().next().is_some_and(|c| c.is_alphabetic());
let snake = word.contains('_');
let all_caps = word.len() > 1
&& word
.chars()
.all(|c| c.is_ascii_uppercase() || c.is_ascii_digit() || c == '_');
mixed_case || snake || all_caps
}
fn looks_like_path(word: &str) -> bool {
if word.starts_with("http://") || word.starts_with("https://") {
return false;
}
let known_ext = [
".rs", ".ts", ".tsx", ".js", ".jsx", ".py", ".go", ".lua", ".json", ".toml", ".yaml",
".yml", ".md", ".sql", ".sh", ".mly",
];
(word.contains('/') && !word.starts_with('/')) || known_ext.iter().any(|e| word.ends_with(e))
}
fn looks_like_key(word: &str) -> bool {
word.contains('.')
&& !word.contains('/')
&& word
.chars()
.all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '-' || c == '.')
&& crate::records::validate_key(word).is_ok()
}
pub fn analyze(task: &str) -> QueryAnalysis {
let mut a = QueryAnalysis::default();
let mut rest = String::with_capacity(task.len());
let mut chars = task.chars().peekable();
while let Some(c) = chars.next() {
if c == '"' || c == '\'' || c == '`' {
let mut phrase = String::new();
for q in chars.by_ref() {
if q == c {
break;
}
phrase.push(q);
}
if !phrase.is_empty() {
a.quoted.push(phrase.clone());
rest.push(' ');
rest.push_str(&phrase);
rest.push(' ');
}
} else {
rest.push(c);
}
}
for raw in rest.split(|c: char| c.is_whitespace() || ",;()[]{}<>!?".contains(c)) {
let word = raw.trim_matches(|c: char| ",.:;()[]{}<>\"'`!?".contains(c) && c != '.');
let word = word.trim_end_matches(['.', ':']);
if word.len() < 2 {
continue;
}
if word.starts_with('/') && word.len() > 2 && word[1..].contains(['/', '-']) {
a.routes.push(word.to_string());
continue;
}
if looks_like_path(word) {
a.paths.push(word.trim_start_matches("./").to_string());
continue;
}
if looks_like_key(word) {
a.keys.push(word.to_string());
continue;
}
if is_identifier_like(word) {
a.identifiers.push(word.to_string());
for part in split_ident(word) {
if part.len() >= 3 && !STOPWORDS.contains(&part.as_str()) {
a.keywords.push(part);
}
}
continue;
}
let lower = word.to_ascii_lowercase();
if lower.chars().all(|c| c.is_ascii_alphanumeric() || c == '-')
&& !STOPWORDS.contains(&lower.as_str())
&& lower.len() >= 3
{
a.keywords.push(lower);
}
}
let mut variants: Vec<String> = Vec::new();
for k in &a.keywords {
if let Some(stripped) = k.strip_suffix('s') {
if stripped.len() >= 3 {
variants.push(stripped.to_string());
}
} else if k.len() >= 3 {
variants.push(format!("{k}s"));
}
}
a.keywords.extend(variants);
for list in [
&mut a.identifiers,
&mut a.quoted,
&mut a.paths,
&mut a.routes,
&mut a.keywords,
&mut a.keys,
] {
list.dedup();
let mut seen = std::collections::HashSet::new();
list.retain(|w| seen.insert(w.clone()));
}
a
}
pub fn split_ident(word: &str) -> Vec<String> {
let mut parts: Vec<String> = Vec::new();
let mut current = String::new();
let mut prev_lower = false;
for c in word.chars() {
if c == '_' || c == '-' || c == '.' {
if !current.is_empty() {
parts.push(std::mem::take(&mut current));
}
prev_lower = false;
} else if c.is_ascii_uppercase() && prev_lower {
parts.push(std::mem::take(&mut current));
current.push(c.to_ascii_lowercase());
prev_lower = false;
} else {
prev_lower = c.is_ascii_lowercase() || c.is_ascii_digit();
current.push(c.to_ascii_lowercase());
}
}
if !current.is_empty() {
parts.push(current);
}
parts
}
pub fn estimate_tokens(text: &str) -> u32 {
let chars = text.chars().count() as u32;
(chars * 2).div_ceil(7).max(if chars > 0 { 1 } else { 0 })
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn extracts_forms() {
let a = analyze(
"Add retry to PaymentWebhookController.handle in apps/api/webhook.ts for \"transient failures\" covering payments.webhook.retry-policy and MAX_RETRIES via /api/payments/webhook",
);
assert!(
a.identifiers
.iter()
.any(|i| i == "PaymentWebhookController.handle"),
"{a:?}"
);
assert!(a.identifiers.iter().any(|i| i == "MAX_RETRIES"), "{a:?}");
assert!(a.paths.iter().any(|p| p == "apps/api/webhook.ts"), "{a:?}");
assert!(
a.keys.iter().any(|k| k == "payments.webhook.retry-policy"),
"{a:?}"
);
assert!(a.quoted.iter().any(|q| q == "transient failures"), "{a:?}");
assert!(
a.routes.iter().any(|r| r == "/api/payments/webhook"),
"{a:?}"
);
assert!(a.keywords.iter().any(|k| k == "retry"), "{a:?}");
}
#[test]
fn split_identifiers() {
assert_eq!(split_ident("PaymentWebhook"), vec!["payment", "webhook"]);
assert_eq!(split_ident("retry_policy"), vec!["retry", "policy"]);
assert_eq!(split_ident("MAX_RETRIES"), vec!["max", "retries"]);
}
#[test]
fn token_estimate_deterministic_and_conservative() {
assert_eq!(estimate_tokens(""), 0);
assert!(estimate_tokens("hello world") >= 3);
let long = "a".repeat(700);
assert_eq!(estimate_tokens(&long), 200);
}
}