sim_lib_forge/
tokens.rs

1//! Prompt-token estimation shared by FORGE authoring paths.
2
3/// Splits prose into the semantic token stream used by FORGE prompt budgets.
4pub fn semantic_tokens(prose: &str) -> Vec<String> {
5    prose
6        .split(|ch: char| !ch.is_ascii_alphanumeric() && ch != '-' && ch != '_' && ch != '/')
7        .filter_map(|raw| {
8            let token = raw.trim().to_ascii_lowercase();
9            (!token.is_empty() && !is_stop_word(&token)).then_some(token)
10        })
11        .collect()
12}
13
14/// Estimates prompt tokens with the FORGE baseline semantic-token counter.
15pub fn estimate_prompt_tokens(prose: &str) -> usize {
16    semantic_tokens(prose).len()
17}
18
19fn is_stop_word(token: &str) -> bool {
20    matches!(
21        token,
22        "a" | "an" | "and" | "for" | "in" | "of" | "on" | "please" | "the" | "to" | "with"
23    )
24}
sim_lib_forge/tokens.rs

sim_lib_forge/
tokens.rs
