Skip to main content

pesti_structural_tokenizer/
lib.rs

1//! Structural tokenizer for Rust source code using `syn`.
2//! Parses into AST via syn, then manually walks emitting semantic tokens.
3//! Supports budget-aware collapsing for LLM context size management.
4
5use std::collections::{HashMap, HashSet};
6use syn::{Item, parse_file};
7
8mod folder;
9pub use folder::{Span, SpanItem, SpanKind, fold};
10
11/// Structural token with position information
12#[derive(Debug, Clone)]
13pub struct StructuralToken {
14    pub kind: TokenKind,
15    pub text: String,
16    pub span: (usize, usize),
17}
18
19/// Kinds of structural tokens we emit
20#[derive(Debug, Clone, PartialEq)]
21pub enum TokenKind {
22    // Declarations
23    FnDecl,
24    LetStmt,
25    StructDecl,
26    EnumDecl,
27    TraitDecl,
28    ImplBlock,
29    UseStmt,
30    ModDecl,
31
32    // Control flow
33    IfElse,
34    WhileLoop,
35    ForLoop,
36    MatchExpr,
37    ReturnExpr,
38    BreakExpr,
39    ContinueExpr,
40
41    // Expressions & operations
42    CallExpr,
43    FieldAccess,
44    MethodCall,
45    BinaryOp,
46    UnaryOp,
47    ParenExpr,
48    ExprStmt,
49
50    // Literals
51    IntLit,
52    FloatLit,
53    StrLit,
54    CharLit,
55    BoolLit,
56    ByteLit,
57    ArrayLit,
58    TupleLit,
59    SliceExpr,
60
61    // Types & casts
62    TypeAnnotation,
63    CastExpr,
64    Dereference,
65    Reference,
66
67    // Blocks and delimiters
68    BlockStart,
69    BlockEnd,
70    ParenOpen,
71    ParenClose,
72    BraceOpen,
73    BraceClose,
74    BracketOpen,
75    BracketClose,
76    Semicolon,
77    Comma,
78    Colon,
79    Dot,
80    Arrow,
81    FatArrow,
82
83    // Identifiers and names
84    Ident(String),
85    Keyword(String),
86
87    // Rust-specific features
88    MacroInvocation,
89    Attribute,
90    Lifetime,
91    GenericParams,
92    WhereClause,
93    AsyncBlock,
94    MoveClosure,
95    Closure,
96    Punctuator,
97
98    // Visibility and modifiers
99    Pub,
100    Private,
101    Static,
102}
103
104impl std::fmt::Display for TokenKind {
105    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
106        match self {
107            TokenKind::FnDecl => write!(f, "FN_DECL"),
108            TokenKind::LetStmt => write!(f, "LET_STMT"),
109            TokenKind::StructDecl => write!(f, "STRUCT_DECL"),
110            TokenKind::EnumDecl => write!(f, "ENUM_DECL"),
111            TokenKind::TraitDecl => write!(f, "TRAIT_DECL"),
112            TokenKind::ImplBlock => write!(f, "IMPL_BLOCK"),
113            TokenKind::UseStmt => write!(f, "USE_STMT"),
114            TokenKind::ModDecl => write!(f, "MOD_DECL"),
115            TokenKind::IfElse => write!(f, "IF_ELSE"),
116            TokenKind::WhileLoop => write!(f, "WHILE_LOOP"),
117            TokenKind::ForLoop => write!(f, "FOR_LOOP"),
118            TokenKind::MatchExpr => write!(f, "MATCH_EXPR"),
119            TokenKind::ReturnExpr => write!(f, "RETURN_EXPR"),
120            TokenKind::BreakExpr => write!(f, "BREAK_EXPR"),
121            TokenKind::ContinueExpr => write!(f, "CONTINUE_EXPR"),
122            TokenKind::CallExpr => write!(f, "CALL_EXPR"),
123            TokenKind::FieldAccess => write!(f, "FIELD_ACCESS"),
124            TokenKind::MethodCall => write!(f, "METHOD_CALL"),
125            TokenKind::BinaryOp => write!(f, "BINARY_OP"),
126            TokenKind::UnaryOp => write!(f, "UNARY_OP"),
127            TokenKind::ParenExpr => write!(f, "PAREN_EXPR"),
128            TokenKind::ExprStmt => write!(f, "EXPR_STMT"),
129            TokenKind::IntLit => write!(f, "INT_LIT"),
130            TokenKind::FloatLit => write!(f, "FLOAT_LIT"),
131            TokenKind::StrLit => write!(f, "STR_LIT"),
132            TokenKind::CharLit => write!(f, "CHAR_LIT"),
133            TokenKind::BoolLit => write!(f, "BOOL_LIT"),
134            TokenKind::ByteLit => write!(f, "BYTE_LIT"),
135            TokenKind::ArrayLit => write!(f, "ARRAY_LIT"),
136            TokenKind::TupleLit => write!(f, "TUPLE_LIT"),
137            TokenKind::SliceExpr => write!(f, "SLICE_EXPR"),
138            TokenKind::TypeAnnotation => write!(f, "TYPE_ANNOT"),
139            TokenKind::CastExpr => write!(f, "CAST_EXPR"),
140            TokenKind::Dereference => write!(f, "DEREF"),
141            TokenKind::Reference => write!(f, "REF"),
142            TokenKind::BlockStart => write!(f, "BLOCK_START"),
143            TokenKind::BlockEnd => write!(f, "BLOCK_END"),
144            TokenKind::ParenOpen => write!(f, "("),
145            TokenKind::ParenClose => write!(f, ")"),
146            TokenKind::BraceOpen => write!(f, "{{"),
147            TokenKind::BraceClose => write!(f, "}}"),
148            TokenKind::BracketOpen => write!(f, "["),
149            TokenKind::BracketClose => write!(f, "]"),
150            TokenKind::Semicolon => write!(f, ";"),
151            TokenKind::Comma => write!(f, ","),
152            TokenKind::Colon => write!(f, ":"),
153            TokenKind::Dot => write!(f, "."),
154            TokenKind::Arrow => write!(f, "->"),
155            TokenKind::FatArrow => write!(f, "=>"),
156            TokenKind::Ident(name) => write!(f, "IDENT({})", name),
157            TokenKind::Keyword(kw) => write!(f, "KW({})", kw),
158            TokenKind::MacroInvocation => write!(f, "MACRO"),
159            TokenKind::Attribute => write!(f, "ATTR"),
160            TokenKind::Lifetime => write!(f, "LIFETIME"),
161            TokenKind::GenericParams => write!(f, "GENERIC_PARAMS"),
162            TokenKind::WhereClause => write!(f, "WHERE"),
163            TokenKind::AsyncBlock => write!(f, "ASYNC"),
164            TokenKind::MoveClosure => write!(f, "MOVE_CLOSURE"),
165            TokenKind::Closure => write!(f, "CLOSURE"),
166            TokenKind::Punctuator => write!(f, "PUNCT"),
167            TokenKind::Pub => write!(f, "pub"),
168            TokenKind::Private => write!(f, "private"),
169            TokenKind::Static => write!(f, "static"),
170        }
171    }
172}
173
174/// Errors from the structural tokenizer
175#[derive(Debug)]
176pub enum TokenizeError {
177    Parse { pos: usize, msg: String },
178}
179
180impl std::fmt::Display for TokenizeError {
181    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
182        match self {
183            TokenizeError::Parse { pos, msg } => {
184                write!(f, "Parse error at position {}: {}", pos, msg)
185            }
186        }
187    }
188}
189
190impl std::error::Error for TokenizeError {}
191
192/// Budget configuration for budget-aware tokenization.
193#[derive(Debug, Clone)]
194pub struct Budget {
195    /// Maximum number of structural tokens to emit.
196    pub max_tokens: usize,
197    /// Minimum statement count in a body worth eliding — smaller bodies are always emitted in full.
198    pub min_body_stmts: usize,
199}
200
201impl Default for Budget {
202    fn default() -> Self {
203        Budget {
204            max_tokens: 1000,
205            min_body_stmts: 5,
206        }
207    }
208}
209
210/// Tag for node kind used in elision summaries.
211#[derive(Debug, Clone, PartialEq, Eq, Hash)]
212pub enum NodeKindTag {
213    FnDecl,
214    MatchExpr,
215    Closure,
216    IfElse,
217    ForLoop,
218    WhileLoop,
219    Block,
220    Other,
221}
222
223impl std::fmt::Display for NodeKindTag {
224    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
225        match self {
226            NodeKindTag::FnDecl => write!(f, "fn"),
227            NodeKindTag::MatchExpr => write!(f, "match"),
228            NodeKindTag::Closure => write!(f, "closure"),
229            NodeKindTag::IfElse => write!(f, "if"),
230            NodeKindTag::ForLoop => write!(f, "for"),
231            NodeKindTag::WhileLoop => write!(f, "while"),
232            NodeKindTag::Block => write!(f, "block"),
233            NodeKindTag::Other => write!(f, "other"),
234        }
235    }
236}
237
238/// Marker for a subtree that was elided due to budget constraints.
239#[derive(Debug, Clone)]
240pub struct ElidedSpan {
241    /// Byte range of the elided source.
242    pub range: (usize, usize),
243    /// Number of nodes collapsed into this marker.
244    pub node_count: usize,
245    /// Breakdown by node kind for consumer to assess what was skipped.
246    pub kind_summary: Vec<(NodeKindTag, usize)>,
247}
248
249/// Emission result — either a token or an elision marker.
250#[derive(Debug, Clone)]
251pub enum Emission {
252    Node(StructuralToken),
253    Elided(ElidedSpan),
254}
255
256impl std::fmt::Display for Emission {
257    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
258        match self {
259            Emission::Node(tok) => write!(f, "{}", tok.kind),
260            Emission::Elided(span) => {
261                let summary_parts: Vec<String> = span
262                    .kind_summary
263                    .iter()
264                    .map(|(k, n)| format!("{}x{}", k, n))
265                    .collect();
266                write!(
267                    f,
268                    "ELIDED({} nodes: {})",
269                    span.node_count,
270                    summary_parts.join(", ")
271                )
272            }
273        }
274    }
275}
276
277/// The structural tokenizer — uses syn to parse, then manually walks AST.
278pub struct StructuralTokenizer {
279    #[allow(dead_code)]
280    vocab: HashMap<String, u32>,
281}
282
283impl StructuralTokenizer {
284    pub fn new() -> Self {
285        StructuralTokenizer {
286            vocab: HashMap::new(),
287        }
288    }
289
290    /// Tokenize Rust source into structural tokens (no budget).
291    pub fn tokenize(&self, src: &str) -> Result<Vec<StructuralToken>, TokenizeError> {
292        let ast = parse_file(src).map_err(|e| TokenizeError::Parse {
293            pos: 0,
294            msg: format!("syn parse error: {}", e),
295        })?;
296
297        let mut tokens = Vec::new();
298        self.walk_items(&ast.items, &mut tokens);
299        Ok(tokens)
300    }
301
302    /// Tokenize with budget-aware collapsing. Returns emissions including Elided markers.
303    pub fn tokenize_with_budget(
304        &self,
305        src: &str,
306        budget: Budget,
307    ) -> Result<Vec<Emission>, TokenizeError> {
308        let ast = parse_file(src).map_err(|e| TokenizeError::Parse {
309            pos: 0,
310            msg: format!("syn parse error: {}", e),
311        })?;
312
313        let mut emissions = Vec::new();
314        let mut budget_state = BudgetState {
315            max_tokens: budget.max_tokens,
316            emitted: 0,
317            min_body_stmts: budget.min_body_stmts,
318        };
319
320        self.walk_items_budgeted(&ast.items, &mut emissions, &mut budget_state);
321        Ok(emissions)
322    }
323
324    fn walk_items(&self, items: &[Item], tokens: &mut Vec<StructuralToken>) {
325        for item in items {
326            match item {
327                Item::Fn(f) => {
328                    let sig = format!("fn {}", f.sig.ident);
329                    self.emit(tokens, TokenKind::FnDecl, sig);
330
331                    // Walk function body statements
332                    for stmt in &f.block.stmts {
333                        self.walk_stmt(stmt, tokens);
334                    }
335                }
336                Item::Struct(s) => {
337                    let name = s.ident.to_string();
338                    self.emit(tokens, TokenKind::StructDecl, name);
339                    if let syn::Fields::Named(fields) = &s.fields {
340                        for field in &fields.named {
341                            if let Some(ident) = &field.ident {
342                                self.emit(
343                                    tokens,
344                                    TokenKind::Ident(ident.to_string()),
345                                    ident.to_string(),
346                                );
347                            }
348                        }
349                    }
350                }
351                Item::Enum(e) => {
352                    let name = e.ident.to_string();
353                    self.emit(tokens, TokenKind::EnumDecl, name);
354                    for variant in &e.variants {
355                        self.emit(
356                            tokens,
357                            TokenKind::Ident(variant.ident.to_string()),
358                            variant.ident.to_string(),
359                        );
360                    }
361                }
362                Item::Trait(t) => {
363                    let name = t.ident.to_string();
364                    self.emit(tokens, TokenKind::TraitDecl, name);
365                    for item in &t.items {
366                        if let syn::TraitItem::Fn(m) = item {
367                            self.emit(tokens, TokenKind::FnDecl, format!("fn {}", m.sig.ident));
368                        }
369                    }
370                }
371                Item::Impl(i) => {
372                    self.emit(tokens, TokenKind::ImplBlock, "impl".to_string());
373                    for item in &i.items {
374                        if let syn::ImplItem::Fn(m) = item {
375                            self.emit(tokens, TokenKind::FnDecl, format!("fn {}", m.sig.ident));
376                        }
377                    }
378                }
379                Item::Use(_u) => {
380                    self.emit(tokens, TokenKind::UseStmt, "use".to_string());
381                }
382                Item::Mod(m) => {
383                    let name = m.ident.to_string();
384                    self.emit(tokens, TokenKind::ModDecl, name);
385                    if let Some((_brace, items)) = &m.content {
386                        self.walk_items(items, tokens);
387                    }
388                }
389                _ => {}
390            }
391        }
392    }
393
394    fn walk_stmt(&self, stmt: &syn::Stmt, tokens: &mut Vec<StructuralToken>) {
395        match stmt {
396            syn::Stmt::Local(_let_stmt) => {
397                self.emit_kind(tokens, TokenKind::LetStmt);
398            }
399            syn::Stmt::Expr(expr, semi) => {
400                self.walk_expr(expr, tokens);
401                if semi.is_some() {
402                    self.emit_kind(tokens, TokenKind::Semicolon);
403                }
404            }
405            syn::Stmt::Item(item) => match item {
406                Item::Fn(inner) => {
407                    let sig = format!("fn {}", inner.sig.ident);
408                    self.emit(tokens, TokenKind::FnDecl, sig);
409                }
410                _ => {}
411            },
412            _ => {}
413        }
414    }
415
416    fn walk_expr(&self, expr: &syn::Expr, tokens: &mut Vec<StructuralToken>) {
417        match expr {
418            syn::Expr::If(eif) => {
419                self.emit_kind(tokens, TokenKind::IfElse);
420                // Walk into the then branch to find break/continue/etc
421                for stmt in &eif.then_branch.stmts {
422                    self.walk_stmt(stmt, tokens);
423                }
424            }
425            syn::Expr::While(_ewhile) => {
426                self.emit_kind(tokens, TokenKind::WhileLoop);
427            }
428            syn::Expr::ForLoop(efor) => {
429                self.emit_kind(tokens, TokenKind::ForLoop);
430                // Walk into loop body to find break/continue
431                for stmt in &efor.body.stmts {
432                    self.walk_stmt(stmt, tokens);
433                }
434            }
435            syn::Expr::Match(_ematch) => {
436                self.emit_kind(tokens, TokenKind::MatchExpr);
437            }
438            syn::Expr::Return(_eret) => {
439                self.emit_kind(tokens, TokenKind::ReturnExpr);
440            }
441            syn::Expr::Break(_) => {
442                self.emit_kind(tokens, TokenKind::BreakExpr);
443            }
444            syn::Expr::Continue(_) => {
445                self.emit_kind(tokens, TokenKind::ContinueExpr);
446            }
447            syn::Expr::Call(_ecall) => {
448                self.emit_kind(tokens, TokenKind::CallExpr);
449            }
450            syn::Expr::MethodCall(emethod) => {
451                let method_name = emethod.method.to_string();
452                self.emit(tokens, TokenKind::MethodCall, method_name);
453            }
454            syn::Expr::Field(efield) => {
455                let field_name = match &efield.member {
456                    syn::Member::Named(ident) => ident.to_string(),
457                    syn::Member::Unnamed(index) => index.index.to_string(),
458                };
459                self.emit(tokens, TokenKind::FieldAccess, field_name);
460            }
461            syn::Expr::Binary(ebin) => {
462                // BinOp doesn't impl Debug/Display — map to string manually
463                let op_str = match &ebin.op {
464                    syn::BinOp::Add(_) => "+",
465                    syn::BinOp::Sub(_) => "-",
466                    syn::BinOp::Mul(_) => "*",
467                    syn::BinOp::Div(_) => "/",
468                    syn::BinOp::Rem(_) => "%",
469                    syn::BinOp::And(_) => "&&",
470                    syn::BinOp::Or(_) => "||",
471                    syn::BinOp::Eq(_) => "==",
472                    syn::BinOp::Ne(_) => "!=",
473                    syn::BinOp::Lt(_) => "<",
474                    syn::BinOp::Le(_) => "<=",
475                    syn::BinOp::Gt(_) => ">",
476                    syn::BinOp::Ge(_) => ">=",
477                    syn::BinOp::Shl(_) => "<<",
478                    syn::BinOp::Shr(_) => ">>",
479                    syn::BinOp::BitAnd(_) => "&",
480                    syn::BinOp::BitOr(_) => "|",
481                    syn::BinOp::BitXor(_) => "^",
482                    _ => "?",
483                };
484                self.emit(tokens, TokenKind::BinaryOp, op_str.to_string());
485            }
486            syn::Expr::Unary(eunary) => {
487                let op_str = match &eunary.op {
488                    syn::UnOp::Deref(_) => "*",
489                    syn::UnOp::Not(_) => "!",
490                    syn::UnOp::Neg(_) => "-",
491                    _ => "?",
492                };
493                self.emit(tokens, TokenKind::UnaryOp, op_str.to_string());
494            }
495            syn::Expr::Paren(_eparen) => {
496                self.emit_kind(tokens, TokenKind::ParenExpr);
497            }
498            syn::Expr::Lit(elit) => match &elit.lit {
499                syn::Lit::Int(l) => {
500                    self.emit(tokens, TokenKind::IntLit, l.to_string());
501                }
502                syn::Lit::Float(f) => {
503                    self.emit(tokens, TokenKind::FloatLit, f.to_string());
504                }
505                syn::Lit::Str(s) => {
506                    let val = s.value();
507                    let truncated = if val.len() > 50 {
508                        format!("{}...", &val[..50])
509                    } else {
510                        val
511                    };
512                    self.emit(tokens, TokenKind::StrLit, truncated);
513                }
514                syn::Lit::Char(c) => {
515                    let ch = c.value();
516                    self.emit(tokens, TokenKind::CharLit, format!("{}", ch));
517                }
518                syn::Lit::Bool(b) => {
519                    self.emit(tokens, TokenKind::BoolLit, b.value().to_string());
520                }
521                _ => {}
522            },
523            syn::Expr::Array(_earray) => {
524                self.emit_kind(tokens, TokenKind::ArrayLit);
525            }
526            syn::Expr::Tuple(_etuple) => {
527                self.emit_kind(tokens, TokenKind::TupleLit);
528            }
529            syn::Expr::Index(_eindex) => {
530                self.emit_kind(tokens, TokenKind::SliceExpr);
531            }
532            syn::Expr::Cast(_ecast) => {
533                self.emit_kind(tokens, TokenKind::CastExpr);
534            }
535            // Deref is a Unary op in syn 2.x, not its own variant
536            syn::Expr::Reference(eref) => {
537                self.emit_kind(tokens, TokenKind::Reference);
538                if eref.mutability.is_some() {
539                    self.emit(
540                        tokens,
541                        TokenKind::Ident("mut".to_string()),
542                        "mut".to_string(),
543                    );
544                }
545            }
546            syn::Expr::Async(_easync) => {
547                self.emit_kind(tokens, TokenKind::AsyncBlock);
548            }
549            syn::Expr::Closure(_eclosure) => {
550                // Detect move by looking for "move" in source (syn doesn't expose this directly)
551                self.emit_kind(tokens, TokenKind::Closure);
552            }
553            syn::Expr::Macro(emacro) => {
554                let name = emacro
555                    .mac
556                    .path
557                    .segments
558                    .last()
559                    .map(|s| s.ident.to_string())
560                    .unwrap_or_default();
561                self.emit(tokens, TokenKind::MacroInvocation, name);
562            }
563            syn::Expr::Block(eblock) => {
564                // ExprBlock has a `block` field of type Block
565                for stmt in &eblock.block.stmts {
566                    self.walk_stmt(stmt, tokens);
567                }
568            }
569            syn::Expr::Path(epath) => {
570                let segments = epath
571                    .path
572                    .segments
573                    .iter()
574                    .map(|s| s.ident.to_string())
575                    .collect::<Vec<_>>();
576                if !segments.is_empty() {
577                    self.emit(
578                        tokens,
579                        TokenKind::Ident(segments.join("::")),
580                        segments.join("::"),
581                    );
582                }
583            }
584            _ => {}
585        }
586    }
587
588    fn emit(&self, tokens: &mut Vec<StructuralToken>, kind: TokenKind, text: String) {
589        tokens.push(StructuralToken {
590            kind,
591            text,
592            span: (0, 0), // No source spans in syn-based approach
593        });
594    }
595
596    fn emit_kind(&self, tokens: &mut Vec<StructuralToken>, kind: TokenKind) {
597        self.emit(tokens, kind, String::new());
598    }
599
600    // Budget-aware walkers
601
602    fn walk_items_budgeted(
603        &self,
604        items: &[Item],
605        emissions: &mut Vec<Emission>,
606        budget: &mut BudgetState,
607    ) {
608        for item in items {
609            if budget.emitted >= budget.max_tokens {
610                break;
611            }
612
613            match item {
614                Item::Fn(f) => {
615                    let sig = format!("fn {}", f.sig.ident);
616                    self.emit_emission(emissions, TokenKind::FnDecl, sig);
617                    budget.emitted += 1;
618
619                    // Check if body is worth eliding (many statements)
620                    let body_size = self.get_body_size(&f.block);
621                    if body_size >= budget.min_body_stmts {
622                        // Count what's in the body without emitting
623                        let (node_count, kind_summary) = self.count_body_nodes(&f.block);
624                        if node_count > 0 && budget.emitted < budget.max_tokens {
625                            // Emit elision marker instead of walking
626                            emissions.push(Emission::Elided(ElidedSpan {
627                                range: (0, 0), // span tracking not implemented yet
628                                node_count,
629                                kind_summary,
630                            }));
631                            budget.emitted += 1;
632                        }
633                    } else {
634                        // Small body — walk normally
635                        for stmt in &f.block.stmts {
636                            if budget.emitted >= budget.max_tokens {
637                                break;
638                            }
639                            self.walk_stmt_budgeted(stmt, emissions, budget);
640                        }
641                    }
642                }
643                Item::Struct(s) => {
644                    let name = s.ident.to_string();
645                    self.emit_emission(emissions, TokenKind::StructDecl, name);
646                    budget.emitted += 1;
647                }
648                Item::Enum(e) => {
649                    let name = e.ident.to_string();
650                    self.emit_emission(emissions, TokenKind::EnumDecl, name);
651                    budget.emitted += 1;
652                }
653                Item::Trait(t) => {
654                    let name = t.ident.to_string();
655                    self.emit_emission(emissions, TokenKind::TraitDecl, name);
656                    budget.emitted += 1;
657                }
658                Item::Impl(i) => {
659                    self.emit_emission(emissions, TokenKind::ImplBlock, "impl".to_string());
660                    budget.emitted += 1;
661                }
662                Item::Use(_u) => {
663                    self.emit_emission(emissions, TokenKind::UseStmt, "use".to_string());
664                    budget.emitted += 1;
665                }
666                Item::Mod(m) => {
667                    let name = m.ident.to_string();
668                    self.emit_emission(emissions, TokenKind::ModDecl, name);
669                    budget.emitted += 1;
670                    if let Some((_brace, items)) = &m.content {
671                        self.walk_items_budgeted(items, emissions, budget);
672                    }
673                }
674                _ => {}
675            }
676        }
677    }
678
679    fn walk_stmt_budgeted(
680        &self,
681        stmt: &syn::Stmt,
682        emissions: &mut Vec<Emission>,
683        budget: &mut BudgetState,
684    ) {
685        if budget.emitted >= budget.max_tokens {
686            return;
687        }
688
689        match stmt {
690            syn::Stmt::Local(_let_stmt) => {
691                self.emit_emission(emissions, TokenKind::LetStmt, String::new());
692                budget.emitted += 1;
693            }
694            syn::Stmt::Expr(expr, semi) => {
695                self.walk_expr_budgeted(expr, emissions, budget);
696                if semi.is_some() {
697                    self.emit_emission(emissions, TokenKind::Semicolon, String::new());
698                    budget.emitted += 1;
699                }
700            }
701            _ => {}
702        }
703    }
704
705    fn walk_expr_budgeted(
706        &self,
707        expr: &syn::Expr,
708        emissions: &mut Vec<Emission>,
709        budget: &mut BudgetState,
710    ) {
711        if budget.emitted >= budget.max_tokens {
712            return;
713        }
714
715        match expr {
716            syn::Expr::If(eif) => {
717                self.emit_emission(emissions, TokenKind::IfElse, String::new());
718                budget.emitted += 1;
719                for stmt in &eif.then_branch.stmts {
720                    self.walk_stmt_budgeted(stmt, emissions, budget);
721                }
722            }
723            syn::Expr::While(_ewhile) => {
724                self.emit_emission(emissions, TokenKind::WhileLoop, String::new());
725                budget.emitted += 1;
726            }
727            syn::Expr::ForLoop(efor) => {
728                self.emit_emission(emissions, TokenKind::ForLoop, String::new());
729                budget.emitted += 1;
730                for stmt in &efor.body.stmts {
731                    self.walk_stmt_budgeted(stmt, emissions, budget);
732                }
733            }
734            syn::Expr::Match(_ematch) => {
735                self.emit_emission(emissions, TokenKind::MatchExpr, String::new());
736                budget.emitted += 1;
737            }
738            syn::Expr::Return(_eret) => {
739                self.emit_emission(emissions, TokenKind::ReturnExpr, String::new());
740                budget.emitted += 1;
741            }
742            syn::Expr::Break(_) => {
743                self.emit_emission(emissions, TokenKind::BreakExpr, String::new());
744                budget.emitted += 1;
745            }
746            syn::Expr::Continue(_) => {
747                self.emit_emission(emissions, TokenKind::ContinueExpr, String::new());
748                budget.emitted += 1;
749            }
750            syn::Expr::Call(_ecall) => {
751                self.emit_emission(emissions, TokenKind::CallExpr, String::new());
752                budget.emitted += 1;
753            }
754            syn::Expr::MethodCall(emethod) => {
755                let method_name = emethod.method.to_string();
756                self.emit_emission(emissions, TokenKind::MethodCall, method_name);
757                budget.emitted += 1;
758            }
759            _ => {}
760        }
761    }
762
763    fn emit_emission(&self, emissions: &mut Vec<Emission>, kind: TokenKind, text: String) {
764        let token = StructuralToken {
765            kind,
766            text,
767            span: (0, 0),
768        };
769        emissions.push(Emission::Node(token));
770    }
771
772    fn get_body_size(&self, block: &syn::Block) -> usize {
773        // Count statements as proxy for body complexity/size
774        block.stmts.len()
775    }
776
777    fn count_body_nodes(&self, block: &syn::Block) -> (usize, Vec<(NodeKindTag, usize)>) {
778        let mut counts: HashMap<NodeKindTag, usize> = HashMap::new();
779        let mut total = 0;
780
781        for stmt in &block.stmts {
782            match stmt {
783                syn::Stmt::Local(_) => {
784                    *counts.entry(NodeKindTag::Other).or_insert(0) += 1;
785                    total += 1;
786                }
787                syn::Stmt::Expr(expr, _) => {
788                    self.count_expr_nodes(expr, &mut counts, &mut total);
789                }
790                _ => {}
791            }
792        }
793
794        let mut summary: Vec<(NodeKindTag, usize)> = counts.into_iter().collect();
795        summary.sort_by(|a, b| b.1.cmp(&a.1));
796        (total, summary)
797    }
798
799    fn count_expr_nodes(
800        &self,
801        expr: &syn::Expr,
802        counts: &mut HashMap<NodeKindTag, usize>,
803        total: &mut usize,
804    ) {
805        match expr {
806            syn::Expr::If(eif) => {
807                *counts.entry(NodeKindTag::IfElse).or_insert(0) += 1;
808                *total += 1;
809                for stmt in &eif.then_branch.stmts {
810                    self.count_stmt_nodes(stmt, counts, total);
811                }
812            }
813            syn::Expr::Match(_ematch) => {
814                *counts.entry(NodeKindTag::MatchExpr).or_insert(0) += 1;
815                *total += 1;
816            }
817            syn::Expr::ForLoop(efor) => {
818                *counts.entry(NodeKindTag::ForLoop).or_insert(0) += 1;
819                *total += 1;
820                for stmt in &efor.body.stmts {
821                    self.count_stmt_nodes(stmt, counts, total);
822                }
823            }
824            syn::Expr::While(_ewhile) => {
825                *counts.entry(NodeKindTag::WhileLoop).or_insert(0) += 1;
826                *total += 1;
827            }
828            syn::Expr::Closure(_eclosure) => {
829                *counts.entry(NodeKindTag::Closure).or_insert(0) += 1;
830                *total += 1;
831            }
832            _ => {}
833        }
834    }
835
836    fn count_stmt_nodes(
837        &self,
838        stmt: &syn::Stmt,
839        counts: &mut HashMap<NodeKindTag, usize>,
840        total: &mut usize,
841    ) {
842        match stmt {
843            syn::Stmt::Local(_) => {
844                *counts.entry(NodeKindTag::Other).or_insert(0) += 1;
845                *total += 1;
846            }
847            syn::Stmt::Expr(expr, _) => {
848                self.count_expr_nodes(expr, counts, total);
849            }
850            _ => {}
851        }
852    }
853}
854
855impl Default for StructuralTokenizer {
856    fn default() -> Self {
857        Self::new()
858    }
859}
860
861/// Internal budget state tracker.
862struct BudgetState {
863    max_tokens: usize,
864    emitted: usize,
865    min_body_stmts: usize,
866}
867
868#[cfg(test)]
869mod tests {
870    use super::*;
871
872    #[test]
873    fn test_fn_declaration() {
874        let tokenizer = StructuralTokenizer::new();
875        let src = "fn add(a: i32, b: i32) -> i32 { return a + b; }";
876        let tokens = tokenizer.tokenize(src).unwrap();
877
878        assert_eq!(tokens[0].kind, TokenKind::FnDecl);
879        assert_eq!(tokens[1].kind, TokenKind::ReturnExpr);
880    }
881
882    #[test]
883    fn test_let_statement() {
884        let tokenizer = StructuralTokenizer::new();
885        let src = "fn main() { let x: i32 = 42; }";
886        let tokens = tokenizer.tokenize(src).unwrap();
887
888        assert_eq!(tokens[0].kind, TokenKind::FnDecl);
889        assert_eq!(tokens[1].kind, TokenKind::LetStmt);
890    }
891
892    #[test]
893    fn test_return_expression() {
894        let tokenizer = StructuralTokenizer::new();
895        let src = "fn main() { return 42; }";
896        let tokens = tokenizer.tokenize(src).unwrap();
897
898        assert_eq!(tokens[0].kind, TokenKind::FnDecl);
899        assert_eq!(tokens[1].kind, TokenKind::ReturnExpr);
900    }
901
902    #[test]
903    fn test_block_structure() {
904        let tokenizer = StructuralTokenizer::new();
905        let src = "fn main() { if true { let x = 1; } else { let y = 2; } }";
906        let tokens = tokenizer.tokenize(src).unwrap();
907
908        assert!(tokens.iter().any(|t| t.kind == TokenKind::IfElse));
909    }
910
911    #[test]
912    fn test_struct_declaration() {
913        let tokenizer = StructuralTokenizer::new();
914        let src = "struct Point { x: i32, y: i32 }";
915        let tokens = tokenizer.tokenize(src).unwrap();
916        assert_eq!(tokens[0].kind, TokenKind::StructDecl);
917    }
918
919    #[test]
920    fn test_enum_declaration() {
921        let tokenizer = StructuralTokenizer::new();
922        let src = "enum Color { Red, Green, Blue }";
923        let tokens = tokenizer.tokenize(src).unwrap();
924        assert_eq!(tokens[0].kind, TokenKind::EnumDecl);
925    }
926
927    #[test]
928    fn test_trait_declaration() {
929        let tokenizer = StructuralTokenizer::new();
930        let src = "trait Display { fn fmt(&self) -> String; }";
931        let tokens = tokenizer.tokenize(src).unwrap();
932        assert_eq!(tokens[0].kind, TokenKind::TraitDecl);
933    }
934
935    #[test]
936    fn test_impl_block() {
937        let tokenizer = StructuralTokenizer::new();
938        let src = "impl Display for Point { fn fmt(&self) -> String { \"\".to_string() } }";
939        let tokens = tokenizer.tokenize(src).unwrap();
940        assert_eq!(tokens[0].kind, TokenKind::ImplBlock);
941    }
942
943    #[test]
944    fn test_break_continue() {
945        let tokenizer = StructuralTokenizer::new();
946        let src = "fn main() { for i in 0..10 { if i == 5 { break; } continue; } }";
947        let tokens = tokenizer.tokenize(src).unwrap();
948
949        assert!(tokens.iter().any(|t| t.kind == TokenKind::ForLoop));
950        assert!(tokens.iter().any(|t| t.kind == TokenKind::BreakExpr));
951        assert!(tokens.iter().any(|t| t.kind == TokenKind::ContinueExpr));
952    }
953
954    #[test]
955    fn test_complex_program() {
956        let tokenizer = StructuralTokenizer::new();
957        let src = r#"
958struct Counter {
959    count: i32,
960}
961
962impl Counter {
963    fn new() -> Counter {
964        Counter { count: 0 }
965    }
966
967    fn increment(&mut self) {
968        self.count = self.count + 1;
969    }
970
971    fn get(&self) -> i32 {
972        return self.count;
973    }
974}
975
976fn main() {
977    let c = Counter::new();
978    for i in 0..5 {
979        if i == 3 {
980            break;
981        }
982        c.increment();
983    }
984}
985"#;
986        let tokens = tokenizer.tokenize(src).unwrap();
987
988        assert!(tokens.iter().any(|t| t.kind == TokenKind::StructDecl));
989        assert!(tokens.iter().any(|t| t.kind == TokenKind::ImplBlock));
990        assert!(tokens.iter().any(|t| t.kind == TokenKind::FnDecl));
991    }
992
993    #[test]
994    fn test_budget_basic() {
995        let tokenizer = StructuralTokenizer::new();
996        let src = "fn foo() { let x = 1; let y = 2; return x + y; }";
997        let budget = Budget {
998            max_tokens: 5,
999            min_body_stmts: 1,
1000        };
1001        let emissions = tokenizer.tokenize_with_budget(src, budget).unwrap();
1002
1003        // Should emit fewer than unlimited mode
1004        assert!(emissions.len() <= 5);
1005    }
1006
1007    #[test]
1008    fn test_budget_elides_large_bodies() {
1009        // Build a function with a large body (many statements)
1010        let mut src = "fn big() {".to_string();
1011        for i in 0..20 {
1012            src.push_str(&format!("let x{} = {};\n", i, i));
1013        }
1014        src.push_str("}");
1015
1016        let tokenizer = StructuralTokenizer::new();
1017        let budget = Budget {
1018            max_tokens: 10,
1019            min_body_stmts: 1,
1020        };
1021        let emissions = tokenizer.tokenize_with_budget(&src, budget).unwrap();
1022
1023        // Should have FnDecl + Elided marker, not all the LetStmts
1024        assert!(
1025            emissions
1026                .iter()
1027                .any(|e| matches!(e, Emission::Node(t) if t.kind == TokenKind::FnDecl))
1028        );
1029        assert!(emissions.iter().any(|e| matches!(e, Emission::Elided(_))));
1030    }
1031
1032    #[test]
1033    fn test_elided_span_has_summary() {
1034        let src =
1035            "fn big() { for i in 0..10 { if i == 5 { break; } } match x { 1 => {}, _ => {} } }";
1036        let tokenizer = StructuralTokenizer::new();
1037        let budget = Budget {
1038            max_tokens: 5,
1039            min_body_stmts: 1,
1040        };
1041        let emissions = tokenizer.tokenize_with_budget(src, budget).unwrap();
1042
1043        // Find elided span and verify it has kind summary
1044        for e in &emissions {
1045            if let Emission::Elided(span) = e {
1046                assert!(!span.kind_summary.is_empty());
1047                assert!(span.node_count > 0);
1048            }
1049        }
1050    }
1051}