1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
//! Small text-handling utilities shared across crates.
/// Returns the longest prefix of `s` whose byte length does not exceed
/// `max_bytes`, snapping back to the nearest preceding UTF-8 char boundary
/// when the budget lands inside a multi-byte character.
///
/// This is the safe replacement for `&s[..max_bytes]` when `s` may contain
/// non-ASCII text and the caller has a byte budget rather than a char budget.
pub fn utf8_prefix_at_or_before(s: &str, max_bytes: usize) -> &str {
if s.len() <= max_bytes {
return s;
}
let mut end = max_bytes;
while !s.is_char_boundary(end) && end > 0 {
end -= 1;
}
&s[..end]
}
/// Split a compound name into individual words.
///
/// Handles camelCase, `PascalCase`, and `snake_case`:
/// - `getUserName` → `["get", "User", "Name"]`
/// - `process_request` → `["process", "request"]`
/// - `MAX_RETRIES` → `["MAX", "RETRIES"]`
pub fn split_compound(name: &str) -> Vec<&str> {
if name.contains('_') {
return name.split('_').filter(|s| !s.is_empty()).collect();
}
// camelCase / PascalCase splitting
let bytes = name.as_bytes();
let mut parts = Vec::new();
let mut start = 0;
for i in 1..bytes.len() {
let cur = bytes[i] as char;
let prev = bytes[i - 1] as char;
// Split at lowercase→uppercase boundary (e.g. getUser → get|User)
let boundary = prev.is_ascii_lowercase() && cur.is_ascii_uppercase();
// Split at uppercase→uppercase+lowercase (e.g. XMLParser → XML|Parser)
let acronym_end = i + 1 < bytes.len()
&& prev.is_ascii_uppercase()
&& cur.is_ascii_uppercase()
&& (bytes[i + 1] as char).is_ascii_lowercase();
if boundary || acronym_end {
if i > start {
parts.push(&name[start..i]);
}
start = i;
}
}
if start < name.len() {
parts.push(&name[start..]);
}
parts
}
/// Returns `true` if `word` looks like CamelCase.
///
/// The word must contain at least one uppercase letter after the first character
/// and consist only of ASCII alphanumeric characters.
pub fn is_camel_case(word: &str) -> bool {
if word.len() < 2 {
return false;
}
// Must be all alphanumeric
if !word.chars().all(|c| c.is_ascii_alphanumeric()) {
return false;
}
// Must have at least one uppercase letter after the first char
word[1..].chars().any(|c| c.is_ascii_uppercase())
}
/// Space-joined camelCase word segments for the FTS `search_terms` column.
///
/// The unicode61 tokenizer already splits on every non-alphanumeric byte
/// (`_`, `::`, `/`, `.`), so `snake_case` names and path segments are indexed
/// as separate tokens on their own. What it cannot derive are the words
/// inside a camelCase identifier — `updateCloudClient` stays one token.
/// Emit only those inner words, deduplicated case-insensitively, so the
/// column adds signal without repeating what the tokenizer already produces.
#[must_use]
pub fn search_terms(name: &str, qualified_name: &str) -> String {
let mut seen = std::collections::HashSet::new();
let mut out: Vec<&str> = Vec::new();
for word in name
.split(|c: char| !c.is_ascii_alphanumeric())
.chain(qualified_name.split(|c: char| !c.is_ascii_alphanumeric()))
{
if !is_camel_case(word) {
continue;
}
for part in split_compound(word) {
if part.len() >= 2 && seen.insert(part.to_ascii_lowercase()) {
out.push(part);
}
}
}
out.join(" ")
}
#[cfg(test)]
mod tests {
use super::{is_camel_case, search_terms, split_compound, utf8_prefix_at_or_before};
#[test]
fn split_compound_handles_camel_snake_screaming() {
assert_eq!(split_compound("getUserName"), vec!["get", "User", "Name"]);
assert_eq!(
split_compound("process_request"),
vec!["process", "request"]
);
assert_eq!(split_compound("MAX_RETRIES"), vec!["MAX", "RETRIES"]);
assert_eq!(split_compound("XMLParser"), vec!["XML", "Parser"]);
}
#[test]
fn is_camel_case_classifies() {
assert!(is_camel_case("UserService"));
assert!(is_camel_case("processRequest"));
assert!(!is_camel_case("user"));
assert!(!is_camel_case("U"));
assert!(!is_camel_case("process_request"));
}
#[test]
fn search_terms_emits_camel_parts_only() {
// Snake_case name: unicode61 already splits it — nothing to add.
assert_eq!(
search_terms(
"rerank_candidates",
"src/context/ranking.rs::rerank_candidates"
),
""
);
// CamelCase name and path segment produce inner words.
let terms = search_terms(
"updateCloudClient",
"src/api/CloudSync.ts::updateCloudClient",
);
assert_eq!(terms, "update Cloud Client Sync");
}
#[test]
fn search_terms_dedupes_case_insensitively() {
let terms = search_terms("ParseJson", "parser/ParseJson.kt::ParseJson");
assert_eq!(terms, "Parse Json");
}
#[test]
fn returns_whole_string_when_under_budget() {
assert_eq!(utf8_prefix_at_or_before("hello", 10), "hello");
}
#[test]
fn returns_whole_string_when_at_budget() {
assert_eq!(utf8_prefix_at_or_before("hello", 5), "hello");
}
#[test]
fn truncates_ascii_at_budget() {
assert_eq!(utf8_prefix_at_or_before("abcdef", 3), "abc");
}
#[test]
fn walks_back_when_cut_lands_inside_multibyte_char() {
// "é" is 2 bytes (0xC3 0xA9). With 20 'a's the total is 22 bytes;
// a budget of 21 lands inside "é" and must walk back to 20.
let s = format!("{}é", "a".repeat(20));
assert_eq!(utf8_prefix_at_or_before(&s, 21), "a".repeat(20));
}
#[test]
fn returns_empty_when_budget_lands_inside_leading_multibyte() {
// 4-byte emoji at position 0; any budget < 4 (but > 0) walks back to 0.
let s = "🦀tail";
assert_eq!(utf8_prefix_at_or_before(s, 2), "");
}
#[test]
fn handles_empty_string() {
assert_eq!(utf8_prefix_at_or_before("", 10), "");
assert_eq!(utf8_prefix_at_or_before("", 0), "");
}
#[test]
fn handles_zero_budget() {
assert_eq!(utf8_prefix_at_or_before("abc", 0), "");
}
}