use std::collections::BTreeSet;
use std::sync::LazyLock;
use regex::Regex;
const FILE_EXTENSIONS: &str = "py|rs|ts|tsx|js|jsx|json|jsonl|yaml|yml|toml|md|sql|sh|service|timer|\
conf|cfg|ini|txt|html|css|db|lock|gguf|env";
static BACKTICK_RE: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"`([^`\n]+)`").unwrap());
static TICKET_RE: LazyLock<Regex> =
LazyLock::new(|| Regex::new(r"\b[A-Z][A-Z0-9]+-\d+\b").unwrap());
static PATH_RE: LazyLock<Regex> =
LazyLock::new(|| Regex::new(r"(?:[\w.\-~]*/)+[\w.\-~]+").unwrap());
static FILENAME_RE: LazyLock<Regex> =
LazyLock::new(|| Regex::new(&format!(r"(?i)\b[\w\-]+\.(?:{FILE_EXTENSIONS})\b")).unwrap());
static NUMBER_RE: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"\d+(?:\.\d+)?").unwrap());
static NUMBER_BLOCKER_RE: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"^[\w.\-/]$").unwrap());
static DIGIT_RE: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"^\d$").unwrap());
fn char_matches(re: &Regex, c: char) -> bool {
re.is_match(c.encode_utf8(&mut [0u8; 4]))
}
fn is_path(token: &str) -> bool {
if ["/", "~", "./", "../"].iter().any(|p| token.starts_with(p)) {
return true;
}
let last = token.rsplit('/').next().unwrap_or("");
last.contains('.') || token.matches('/').count() >= 2
}
fn is_number(token: &str) -> bool {
let mut digits = token.chars().filter(|c| *c != '.').peekable();
digits.peek().is_some() && digits.all(|c| char_matches(&DIGIT_RE, c))
}
pub(crate) fn concrete_tokens(text: &str) -> BTreeSet<String> {
let mut tokens = BTreeSet::new();
for caps in BACKTICK_RE.captures_iter(text) {
tokens.insert(caps[1].trim().to_lowercase());
}
for m in TICKET_RE.find_iter(text) {
tokens.insert(m.as_str().to_lowercase());
}
for m in PATH_RE.find_iter(text) {
let path = m.as_str().trim_end_matches(['.', ',', ';', ':']);
if is_path(path) {
tokens.insert(path.to_lowercase());
}
}
for m in FILENAME_RE.find_iter(text) {
tokens.insert(m.as_str().to_lowercase());
}
for m in NUMBER_RE.find_iter(text) {
let blocked = text[..m.start()]
.chars()
.next_back()
.is_some_and(|c| char_matches(&NUMBER_BLOCKER_RE, c));
if !blocked && m.as_str().chars().count() >= 2 {
tokens.insert(m.as_str().to_string());
}
}
tokens.remove("");
tokens
.iter()
.filter(|t| is_number(t) || !tokens.iter().any(|o| o != *t && o.contains(t.as_str())))
.cloned()
.collect()
}
fn find(token: &str, text_norm: &str) -> Option<usize> {
if !is_number(token) {
return text_norm.find(token);
}
text_norm.match_indices(token).map(|(i, _)| i).find(|&i| {
let before = text_norm[..i].chars().next_back();
let after = text_norm[i + token.len()..].chars().next();
!before.is_some_and(|c| c == '.' || char_matches(&DIGIT_RE, c))
&& !after.is_some_and(|c| char_matches(&DIGIT_RE, c))
})
}
pub(crate) fn missing_tokens(members: &[&str], merged: &str) -> Vec<String> {
let merged_norm = merged.to_lowercase().replace('`', "");
let wanted: BTreeSet<String> = members.iter().flat_map(|m| concrete_tokens(m)).collect();
wanted
.into_iter()
.filter(|t| find(t, &merged_norm).is_none())
.collect()
}
pub(crate) fn ordered_identifiers<'a>(
text: &str,
tokens: impl IntoIterator<Item = &'a String>,
) -> Vec<String> {
let text_norm = text.to_lowercase();
let mut found: Vec<(usize, String)> = Vec::new();
for token in tokens {
let Some(byte_index) = find(token, &text_norm) else {
continue;
};
let index = text_norm[..byte_index].chars().count();
let original: String = text
.chars()
.skip(index)
.take(token.chars().count())
.collect();
let shown = if original.to_lowercase() == *token {
original
} else {
token.clone()
};
found.push((index, shown));
}
found.sort();
found.into_iter().map(|(_, shown)| shown).collect()
}
#[cfg(test)]
mod tests {
use super::*;
fn set(items: &[&str]) -> BTreeSet<String> {
items.iter().map(|s| s.to_string()).collect()
}
#[test]
fn concrete_tokens_cover_backticks_tickets_numbers_and_paths() {
let tokens = concrete_tokens(
"AUS-7903 inserts rows via `flask migrate` in src/app/config.py, \
cap 3600s with 60 padding at step 1",
);
assert!(
set(&[
"aus-7903",
"flask migrate",
"src/app/config.py",
"3600",
"60"
])
.is_subset(&tokens)
);
assert!(
!tokens.contains("1"),
"a single digit is usually a list marker, not a fact"
);
}
#[test]
fn missing_tokens_ignore_backticks_and_case() {
let members = [
"Set `MemoryMax=2400M` for AUS-12",
"the unit file lives in /etc/x.service",
];
let merged = "memorymax=2400M is set for aus-12; see /etc/x.service";
assert!(missing_tokens(&members, merged).is_empty());
assert_eq!(
missing_tokens(&members, "MemoryMax=2400M only"),
vec!["/etc/x.service", "aus-12"]
);
}
#[test]
fn ordered_identifiers_follow_first_appearance_with_original_casing() {
let text = "Ran `Cargo Test` on src/Main.rs for AUS-12, timeout 3600";
assert_eq!(
ordered_identifiers(text, &concrete_tokens(text)),
vec!["Cargo Test", "src/Main.rs", "AUS-12", "3600"]
);
}
#[test]
fn ordered_identifiers_place_a_number_at_its_own_occurrence() {
let text = "moved 8080 to 80";
assert_eq!(
ordered_identifiers(text, &set(&["80", "8080"])),
vec!["8080", "80"]
);
}
#[test]
fn ordered_identifiers_keep_only_the_given_tokens() {
let text = "AUS-1 then AUS-2 then AUS-3";
assert_eq!(
ordered_identifiers(text, &set(&["aus-3", "aus-1"])),
vec!["AUS-1", "AUS-3"]
);
}
#[test]
fn a_number_inside_a_word_is_not_a_token() {
let tokens = concrete_tokens("v12 at x.34 and a-56 but 90");
assert_eq!(tokens, set(&["90"]));
}
#[test]
fn a_token_inside_a_longer_one_is_dropped_unless_it_is_a_number() {
let tokens = concrete_tokens("`src/app` lives at src/app/config.py, port 8080 and 80");
assert_eq!(tokens, set(&["src/app/config.py", "8080", "80"]));
}
#[test]
fn prose_slashes_are_not_paths() {
assert!(concrete_tokens("read and/or write the input/output").is_empty());
}
#[test]
#[ignore]
fn tokens_parity() {
let path = std::env::var("SB_TOKENS_PARITY").expect("SB_TOKENS_PARITY names the file");
let cases: Vec<serde_json::Value> =
serde_json::from_str(&std::fs::read_to_string(path).unwrap()).unwrap();
let strings = |v: &serde_json::Value| -> Vec<String> {
v.as_array()
.unwrap()
.iter()
.map(|s| s.as_str().unwrap().to_string())
.collect()
};
let mut mismatches = 0;
for (i, case) in cases.iter().enumerate() {
let text = case["text"].as_str().unwrap();
let concrete = concrete_tokens(text);
let expected: BTreeSet<String> = strings(&case["concrete"]).into_iter().collect();
if concrete != expected {
mismatches += 1;
eprintln!(
"case {i} concrete: rust-only {:?} python-only {:?}",
concrete.difference(&expected).collect::<Vec<_>>(),
expected.difference(&concrete).collect::<Vec<_>>()
);
}
if ordered_identifiers(text, &expected) != strings(&case["ordered"]) {
mismatches += 1;
eprintln!("case {i} ordered differs");
}
for m in case["missing"].as_array().unwrap() {
if missing_tokens(&[text], m["merged"].as_str().unwrap()) != strings(&m["expected"])
{
mismatches += 1;
eprintln!("case {i} missing differs");
}
}
}
assert_eq!(
mismatches,
0,
"{mismatches} mismatches over {} cases",
cases.len()
);
}
}