fxrank_lang_python/
source.rs

1/// Parse-position utilities for the Python frontend.
2///
3/// # BOM invariant
4///
5/// The frontend strips any leading UTF-8 BOM (`\u{feff}`) exactly once via
6/// `strip_bom` at the entry point, then uses that single stripped `&str` for
7/// all three consumers: `parse_module`, `tokenize`/`lambda_anchors`, and the
8/// pointer-arithmetic in `anchor_of_subslice`. Passing the same buffer to all
9/// three keeps every byte offset consistent — libcst's `byte_idx()` values,
10/// the token-stream positions, and `SpanIndex` line lookups all agree.
11use libcst_native::tokenize;
12
13/// Precomputed line-start byte offsets. The line is found in O(log n) (binary
14/// search over the line starts); the 1-based **char** column is then O(line length)
15/// (`chars().count()` over the line prefix, so multi-byte chars count as one).
16pub struct SpanIndex<'a> {
17    src: &'a str,
18    line_starts: Vec<usize>, // byte offset of the start of each line (line 1 = index 0)
19}
20
21impl<'a> SpanIndex<'a> {
22    pub fn new(src: &'a str) -> Self {
23        let mut line_starts = vec![0usize];
24        for (i, b) in src.bytes().enumerate() {
25            if b == b'\n' {
26                line_starts.push(i + 1);
27            }
28        }
29        SpanIndex { src, line_starts }
30    }
31
32    /// The source buffer this index was built from (for `anchor_of_subslice`).
33    pub fn src(&self) -> &'a str {
34        self.src
35    }
36
37    /// 1-based line, 1-based **char** column for a byte offset (`usize`, matching core).
38    pub fn line_col(&self, byte_off: usize) -> (usize, usize) {
39        let line_idx = match self.line_starts.binary_search(&byte_off) {
40            Ok(i) => i,
41            Err(i) => i - 1,
42        };
43        let line_start = self.line_starts[line_idx];
44        let col_chars = self.src[line_start..byte_off].chars().count();
45        (line_idx + 1, col_chars + 1)
46    }
47}
48
49/// Byte offset of a `&str` that is a subslice of `src` (pointer arithmetic).
50///
51/// **Precondition:** `sub` MUST point into `src`'s buffer (e.g. a libcst node's
52/// borrowed `&str` taken from the same parsed source). Passing an unrelated `&str`
53/// yields a meaningless offset. `pub(crate)` so this can't be misused from outside.
54pub(crate) fn anchor_of_subslice(src: &str, sub: &str) -> usize {
55    sub.as_ptr() as usize - src.as_ptr() as usize
56}
57
58/// (line, 1-based char col) of each `lambda` keyword token, in source order.
59///
60/// Returns `Some(anchors)` on success and `None` if tokenization fails.
61///
62/// # Precondition
63/// `src` MUST be the **same** (BOM-stripped) buffer that `parse_module` already
64/// accepted. Tokenizing is a strict subset of parsing, so any `src` that parsed
65/// also tokenizes; in practice this function therefore always returns `Some(…)`.
66/// The `None` branch exists to close the silent-drop hole that the old
67/// `unwrap_or_default()` created: if tokenization ever fails we now propagate the
68/// failure to the caller (`PythonFrontend::analyze`) so it can emit a `Diagnostic`
69/// and skip the file rather than silently emitting zero anchors and misattributing
70/// (or omitting) every lambda.
71///
72/// # Double-tokenize elimination
73/// The caller passes the returned `&[(usize, usize)]` slice directly into
74/// `functions::collect`, so tokenization happens **exactly once per file** — the
75/// old design called `lambda_anchors` a second time in `PythonFrontend::analyze`
76/// for the mismatch guard, producing two tokenizer passes per file.
77pub fn lambda_anchors(src: &str) -> Option<Vec<(usize, usize)>> {
78    tokenize(src)
79        .map(|toks| {
80            toks.iter()
81                .filter(|t| t.string == "lambda")
82                .map(|t| {
83                    (
84                        t.start_pos.line_number(),
85                        t.start_pos.char_column_number() + 1,
86                    )
87                })
88                .collect()
89        })
90        .ok()
91}
92
93/// Strip a leading UTF-8 BOM (`\u{feff}`) from `src`, if present.
94///
95/// Pass the result consistently to `parse_module`, `anchor_of_subslice`, and
96/// `SpanIndex::new` so that all byte offsets are relative to the same buffer.
97pub fn strip_bom(src: &str) -> &str {
98    src.strip_prefix('\u{feff}').unwrap_or(src)
99}
100
101#[cfg(test)]
102mod tests {
103    use super::*;
104
105    #[test]
106    fn line_col_counts_chars_not_bytes() {
107        let src = "x = 'é'\ndef f():\n    pass\n"; // 'é' is 2 bytes, 1 char
108        let idx = SpanIndex::new(src);
109        let byte_off = src.find("def").unwrap();
110        assert_eq!(idx.line_col(byte_off), (2, 1)); // line 2, char col 1
111    }
112
113    #[test]
114    fn anchor_of_subslice_is_exact() {
115        let src = "def greet():\n    pass\n";
116        let name = &src[4..9]; // "greet"
117        assert_eq!(anchor_of_subslice(src, name), 4);
118    }
119
120    #[test]
121    fn lambda_anchors_in_source_order() {
122        let src = "a = lambda: 1\nb = lambda y: y\n";
123        let anchors = lambda_anchors(src).expect("tokenize must succeed on valid Python");
124        assert_eq!(anchors, vec![(1, 5), (2, 5)]); // both `lambda` at char col 5
125    }
126
127    #[test]
128    fn strip_bom_removes_bom() {
129        let with_bom = "\u{feff}def f(): pass\n";
130        assert_eq!(strip_bom(with_bom), "def f(): pass\n");
131    }
132
133    #[test]
134    fn strip_bom_noop_without_bom() {
135        let src = "def f(): pass\n";
136        assert_eq!(strip_bom(src), src);
137    }
138}
fxrank_lang_python/source.rs

fxrank_lang_python/
source.rs
