1use std::collections::{HashMap, HashSet};
6use syn::{Item, parse_file};
7
8mod folder;
9pub use folder::{Span, SpanItem, SpanKind, fold};
10
11#[derive(Debug, Clone)]
13pub struct StructuralToken {
14 pub kind: TokenKind,
15 pub text: String,
16 pub span: (usize, usize),
17}
18
19#[derive(Debug, Clone, PartialEq)]
21pub enum TokenKind {
22 FnDecl,
24 LetStmt,
25 StructDecl,
26 EnumDecl,
27 TraitDecl,
28 ImplBlock,
29 UseStmt,
30 ModDecl,
31
32 IfElse,
34 WhileLoop,
35 ForLoop,
36 MatchExpr,
37 ReturnExpr,
38 BreakExpr,
39 ContinueExpr,
40
41 CallExpr,
43 FieldAccess,
44 MethodCall,
45 BinaryOp,
46 UnaryOp,
47 ParenExpr,
48 ExprStmt,
49
50 IntLit,
52 FloatLit,
53 StrLit,
54 CharLit,
55 BoolLit,
56 ByteLit,
57 ArrayLit,
58 TupleLit,
59 SliceExpr,
60
61 TypeAnnotation,
63 CastExpr,
64 Dereference,
65 Reference,
66
67 BlockStart,
69 BlockEnd,
70 ParenOpen,
71 ParenClose,
72 BraceOpen,
73 BraceClose,
74 BracketOpen,
75 BracketClose,
76 Semicolon,
77 Comma,
78 Colon,
79 Dot,
80 Arrow,
81 FatArrow,
82
83 Ident(String),
85 Keyword(String),
86
87 MacroInvocation,
89 Attribute,
90 Lifetime,
91 GenericParams,
92 WhereClause,
93 AsyncBlock,
94 MoveClosure,
95 Closure,
96 Punctuator,
97
98 Pub,
100 Private,
101 Static,
102}
103
104impl std::fmt::Display for TokenKind {
105 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
106 match self {
107 TokenKind::FnDecl => write!(f, "FN_DECL"),
108 TokenKind::LetStmt => write!(f, "LET_STMT"),
109 TokenKind::StructDecl => write!(f, "STRUCT_DECL"),
110 TokenKind::EnumDecl => write!(f, "ENUM_DECL"),
111 TokenKind::TraitDecl => write!(f, "TRAIT_DECL"),
112 TokenKind::ImplBlock => write!(f, "IMPL_BLOCK"),
113 TokenKind::UseStmt => write!(f, "USE_STMT"),
114 TokenKind::ModDecl => write!(f, "MOD_DECL"),
115 TokenKind::IfElse => write!(f, "IF_ELSE"),
116 TokenKind::WhileLoop => write!(f, "WHILE_LOOP"),
117 TokenKind::ForLoop => write!(f, "FOR_LOOP"),
118 TokenKind::MatchExpr => write!(f, "MATCH_EXPR"),
119 TokenKind::ReturnExpr => write!(f, "RETURN_EXPR"),
120 TokenKind::BreakExpr => write!(f, "BREAK_EXPR"),
121 TokenKind::ContinueExpr => write!(f, "CONTINUE_EXPR"),
122 TokenKind::CallExpr => write!(f, "CALL_EXPR"),
123 TokenKind::FieldAccess => write!(f, "FIELD_ACCESS"),
124 TokenKind::MethodCall => write!(f, "METHOD_CALL"),
125 TokenKind::BinaryOp => write!(f, "BINARY_OP"),
126 TokenKind::UnaryOp => write!(f, "UNARY_OP"),
127 TokenKind::ParenExpr => write!(f, "PAREN_EXPR"),
128 TokenKind::ExprStmt => write!(f, "EXPR_STMT"),
129 TokenKind::IntLit => write!(f, "INT_LIT"),
130 TokenKind::FloatLit => write!(f, "FLOAT_LIT"),
131 TokenKind::StrLit => write!(f, "STR_LIT"),
132 TokenKind::CharLit => write!(f, "CHAR_LIT"),
133 TokenKind::BoolLit => write!(f, "BOOL_LIT"),
134 TokenKind::ByteLit => write!(f, "BYTE_LIT"),
135 TokenKind::ArrayLit => write!(f, "ARRAY_LIT"),
136 TokenKind::TupleLit => write!(f, "TUPLE_LIT"),
137 TokenKind::SliceExpr => write!(f, "SLICE_EXPR"),
138 TokenKind::TypeAnnotation => write!(f, "TYPE_ANNOT"),
139 TokenKind::CastExpr => write!(f, "CAST_EXPR"),
140 TokenKind::Dereference => write!(f, "DEREF"),
141 TokenKind::Reference => write!(f, "REF"),
142 TokenKind::BlockStart => write!(f, "BLOCK_START"),
143 TokenKind::BlockEnd => write!(f, "BLOCK_END"),
144 TokenKind::ParenOpen => write!(f, "("),
145 TokenKind::ParenClose => write!(f, ")"),
146 TokenKind::BraceOpen => write!(f, "{{"),
147 TokenKind::BraceClose => write!(f, "}}"),
148 TokenKind::BracketOpen => write!(f, "["),
149 TokenKind::BracketClose => write!(f, "]"),
150 TokenKind::Semicolon => write!(f, ";"),
151 TokenKind::Comma => write!(f, ","),
152 TokenKind::Colon => write!(f, ":"),
153 TokenKind::Dot => write!(f, "."),
154 TokenKind::Arrow => write!(f, "->"),
155 TokenKind::FatArrow => write!(f, "=>"),
156 TokenKind::Ident(name) => write!(f, "IDENT({})", name),
157 TokenKind::Keyword(kw) => write!(f, "KW({})", kw),
158 TokenKind::MacroInvocation => write!(f, "MACRO"),
159 TokenKind::Attribute => write!(f, "ATTR"),
160 TokenKind::Lifetime => write!(f, "LIFETIME"),
161 TokenKind::GenericParams => write!(f, "GENERIC_PARAMS"),
162 TokenKind::WhereClause => write!(f, "WHERE"),
163 TokenKind::AsyncBlock => write!(f, "ASYNC"),
164 TokenKind::MoveClosure => write!(f, "MOVE_CLOSURE"),
165 TokenKind::Closure => write!(f, "CLOSURE"),
166 TokenKind::Punctuator => write!(f, "PUNCT"),
167 TokenKind::Pub => write!(f, "pub"),
168 TokenKind::Private => write!(f, "private"),
169 TokenKind::Static => write!(f, "static"),
170 }
171 }
172}
173
174#[derive(Debug)]
176pub enum TokenizeError {
177 Parse { pos: usize, msg: String },
178}
179
180impl std::fmt::Display for TokenizeError {
181 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
182 match self {
183 TokenizeError::Parse { pos, msg } => {
184 write!(f, "Parse error at position {}: {}", pos, msg)
185 }
186 }
187 }
188}
189
190impl std::error::Error for TokenizeError {}
191
192#[derive(Debug, Clone)]
194pub struct Budget {
195 pub max_tokens: usize,
197 pub min_body_stmts: usize,
199}
200
201impl Default for Budget {
202 fn default() -> Self {
203 Budget {
204 max_tokens: 1000,
205 min_body_stmts: 5,
206 }
207 }
208}
209
210#[derive(Debug, Clone, PartialEq, Eq, Hash)]
212pub enum NodeKindTag {
213 FnDecl,
214 MatchExpr,
215 Closure,
216 IfElse,
217 ForLoop,
218 WhileLoop,
219 Block,
220 Other,
221}
222
223impl std::fmt::Display for NodeKindTag {
224 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
225 match self {
226 NodeKindTag::FnDecl => write!(f, "fn"),
227 NodeKindTag::MatchExpr => write!(f, "match"),
228 NodeKindTag::Closure => write!(f, "closure"),
229 NodeKindTag::IfElse => write!(f, "if"),
230 NodeKindTag::ForLoop => write!(f, "for"),
231 NodeKindTag::WhileLoop => write!(f, "while"),
232 NodeKindTag::Block => write!(f, "block"),
233 NodeKindTag::Other => write!(f, "other"),
234 }
235 }
236}
237
238#[derive(Debug, Clone)]
240pub struct ElidedSpan {
241 pub range: (usize, usize),
243 pub node_count: usize,
245 pub kind_summary: Vec<(NodeKindTag, usize)>,
247}
248
249#[derive(Debug, Clone)]
251pub enum Emission {
252 Node(StructuralToken),
253 Elided(ElidedSpan),
254}
255
256impl std::fmt::Display for Emission {
257 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
258 match self {
259 Emission::Node(tok) => write!(f, "{}", tok.kind),
260 Emission::Elided(span) => {
261 let summary_parts: Vec<String> = span
262 .kind_summary
263 .iter()
264 .map(|(k, n)| format!("{}x{}", k, n))
265 .collect();
266 write!(
267 f,
268 "ELIDED({} nodes: {})",
269 span.node_count,
270 summary_parts.join(", ")
271 )
272 }
273 }
274 }
275}
276
277pub struct StructuralTokenizer {
279 #[allow(dead_code)]
280 vocab: HashMap<String, u32>,
281}
282
283impl StructuralTokenizer {
284 pub fn new() -> Self {
285 StructuralTokenizer {
286 vocab: HashMap::new(),
287 }
288 }
289
290 pub fn tokenize(&self, src: &str) -> Result<Vec<StructuralToken>, TokenizeError> {
292 let ast = parse_file(src).map_err(|e| TokenizeError::Parse {
293 pos: 0,
294 msg: format!("syn parse error: {}", e),
295 })?;
296
297 let mut tokens = Vec::new();
298 self.walk_items(&ast.items, &mut tokens);
299 Ok(tokens)
300 }
301
302 pub fn tokenize_with_budget(
304 &self,
305 src: &str,
306 budget: Budget,
307 ) -> Result<Vec<Emission>, TokenizeError> {
308 let ast = parse_file(src).map_err(|e| TokenizeError::Parse {
309 pos: 0,
310 msg: format!("syn parse error: {}", e),
311 })?;
312
313 let mut emissions = Vec::new();
314 let mut budget_state = BudgetState {
315 max_tokens: budget.max_tokens,
316 emitted: 0,
317 min_body_stmts: budget.min_body_stmts,
318 };
319
320 self.walk_items_budgeted(&ast.items, &mut emissions, &mut budget_state);
321 Ok(emissions)
322 }
323
324 fn walk_items(&self, items: &[Item], tokens: &mut Vec<StructuralToken>) {
325 for item in items {
326 match item {
327 Item::Fn(f) => {
328 let sig = format!("fn {}", f.sig.ident);
329 self.emit(tokens, TokenKind::FnDecl, sig);
330
331 for stmt in &f.block.stmts {
333 self.walk_stmt(stmt, tokens);
334 }
335 }
336 Item::Struct(s) => {
337 let name = s.ident.to_string();
338 self.emit(tokens, TokenKind::StructDecl, name);
339 if let syn::Fields::Named(fields) = &s.fields {
340 for field in &fields.named {
341 if let Some(ident) = &field.ident {
342 self.emit(
343 tokens,
344 TokenKind::Ident(ident.to_string()),
345 ident.to_string(),
346 );
347 }
348 }
349 }
350 }
351 Item::Enum(e) => {
352 let name = e.ident.to_string();
353 self.emit(tokens, TokenKind::EnumDecl, name);
354 for variant in &e.variants {
355 self.emit(
356 tokens,
357 TokenKind::Ident(variant.ident.to_string()),
358 variant.ident.to_string(),
359 );
360 }
361 }
362 Item::Trait(t) => {
363 let name = t.ident.to_string();
364 self.emit(tokens, TokenKind::TraitDecl, name);
365 for item in &t.items {
366 if let syn::TraitItem::Fn(m) = item {
367 self.emit(tokens, TokenKind::FnDecl, format!("fn {}", m.sig.ident));
368 }
369 }
370 }
371 Item::Impl(i) => {
372 self.emit(tokens, TokenKind::ImplBlock, "impl".to_string());
373 for item in &i.items {
374 if let syn::ImplItem::Fn(m) = item {
375 self.emit(tokens, TokenKind::FnDecl, format!("fn {}", m.sig.ident));
376 }
377 }
378 }
379 Item::Use(_u) => {
380 self.emit(tokens, TokenKind::UseStmt, "use".to_string());
381 }
382 Item::Mod(m) => {
383 let name = m.ident.to_string();
384 self.emit(tokens, TokenKind::ModDecl, name);
385 if let Some((_brace, items)) = &m.content {
386 self.walk_items(items, tokens);
387 }
388 }
389 _ => {}
390 }
391 }
392 }
393
394 fn walk_stmt(&self, stmt: &syn::Stmt, tokens: &mut Vec<StructuralToken>) {
395 match stmt {
396 syn::Stmt::Local(_let_stmt) => {
397 self.emit_kind(tokens, TokenKind::LetStmt);
398 }
399 syn::Stmt::Expr(expr, semi) => {
400 self.walk_expr(expr, tokens);
401 if semi.is_some() {
402 self.emit_kind(tokens, TokenKind::Semicolon);
403 }
404 }
405 syn::Stmt::Item(item) => match item {
406 Item::Fn(inner) => {
407 let sig = format!("fn {}", inner.sig.ident);
408 self.emit(tokens, TokenKind::FnDecl, sig);
409 }
410 _ => {}
411 },
412 _ => {}
413 }
414 }
415
416 fn walk_expr(&self, expr: &syn::Expr, tokens: &mut Vec<StructuralToken>) {
417 match expr {
418 syn::Expr::If(eif) => {
419 self.emit_kind(tokens, TokenKind::IfElse);
420 for stmt in &eif.then_branch.stmts {
422 self.walk_stmt(stmt, tokens);
423 }
424 }
425 syn::Expr::While(_ewhile) => {
426 self.emit_kind(tokens, TokenKind::WhileLoop);
427 }
428 syn::Expr::ForLoop(efor) => {
429 self.emit_kind(tokens, TokenKind::ForLoop);
430 for stmt in &efor.body.stmts {
432 self.walk_stmt(stmt, tokens);
433 }
434 }
435 syn::Expr::Match(_ematch) => {
436 self.emit_kind(tokens, TokenKind::MatchExpr);
437 }
438 syn::Expr::Return(_eret) => {
439 self.emit_kind(tokens, TokenKind::ReturnExpr);
440 }
441 syn::Expr::Break(_) => {
442 self.emit_kind(tokens, TokenKind::BreakExpr);
443 }
444 syn::Expr::Continue(_) => {
445 self.emit_kind(tokens, TokenKind::ContinueExpr);
446 }
447 syn::Expr::Call(_ecall) => {
448 self.emit_kind(tokens, TokenKind::CallExpr);
449 }
450 syn::Expr::MethodCall(emethod) => {
451 let method_name = emethod.method.to_string();
452 self.emit(tokens, TokenKind::MethodCall, method_name);
453 }
454 syn::Expr::Field(efield) => {
455 let field_name = match &efield.member {
456 syn::Member::Named(ident) => ident.to_string(),
457 syn::Member::Unnamed(index) => index.index.to_string(),
458 };
459 self.emit(tokens, TokenKind::FieldAccess, field_name);
460 }
461 syn::Expr::Binary(ebin) => {
462 let op_str = match &ebin.op {
464 syn::BinOp::Add(_) => "+",
465 syn::BinOp::Sub(_) => "-",
466 syn::BinOp::Mul(_) => "*",
467 syn::BinOp::Div(_) => "/",
468 syn::BinOp::Rem(_) => "%",
469 syn::BinOp::And(_) => "&&",
470 syn::BinOp::Or(_) => "||",
471 syn::BinOp::Eq(_) => "==",
472 syn::BinOp::Ne(_) => "!=",
473 syn::BinOp::Lt(_) => "<",
474 syn::BinOp::Le(_) => "<=",
475 syn::BinOp::Gt(_) => ">",
476 syn::BinOp::Ge(_) => ">=",
477 syn::BinOp::Shl(_) => "<<",
478 syn::BinOp::Shr(_) => ">>",
479 syn::BinOp::BitAnd(_) => "&",
480 syn::BinOp::BitOr(_) => "|",
481 syn::BinOp::BitXor(_) => "^",
482 _ => "?",
483 };
484 self.emit(tokens, TokenKind::BinaryOp, op_str.to_string());
485 }
486 syn::Expr::Unary(eunary) => {
487 let op_str = match &eunary.op {
488 syn::UnOp::Deref(_) => "*",
489 syn::UnOp::Not(_) => "!",
490 syn::UnOp::Neg(_) => "-",
491 _ => "?",
492 };
493 self.emit(tokens, TokenKind::UnaryOp, op_str.to_string());
494 }
495 syn::Expr::Paren(_eparen) => {
496 self.emit_kind(tokens, TokenKind::ParenExpr);
497 }
498 syn::Expr::Lit(elit) => match &elit.lit {
499 syn::Lit::Int(l) => {
500 self.emit(tokens, TokenKind::IntLit, l.to_string());
501 }
502 syn::Lit::Float(f) => {
503 self.emit(tokens, TokenKind::FloatLit, f.to_string());
504 }
505 syn::Lit::Str(s) => {
506 let val = s.value();
507 let truncated = if val.len() > 50 {
508 format!("{}...", &val[..50])
509 } else {
510 val
511 };
512 self.emit(tokens, TokenKind::StrLit, truncated);
513 }
514 syn::Lit::Char(c) => {
515 let ch = c.value();
516 self.emit(tokens, TokenKind::CharLit, format!("{}", ch));
517 }
518 syn::Lit::Bool(b) => {
519 self.emit(tokens, TokenKind::BoolLit, b.value().to_string());
520 }
521 _ => {}
522 },
523 syn::Expr::Array(_earray) => {
524 self.emit_kind(tokens, TokenKind::ArrayLit);
525 }
526 syn::Expr::Tuple(_etuple) => {
527 self.emit_kind(tokens, TokenKind::TupleLit);
528 }
529 syn::Expr::Index(_eindex) => {
530 self.emit_kind(tokens, TokenKind::SliceExpr);
531 }
532 syn::Expr::Cast(_ecast) => {
533 self.emit_kind(tokens, TokenKind::CastExpr);
534 }
535 syn::Expr::Reference(eref) => {
537 self.emit_kind(tokens, TokenKind::Reference);
538 if eref.mutability.is_some() {
539 self.emit(
540 tokens,
541 TokenKind::Ident("mut".to_string()),
542 "mut".to_string(),
543 );
544 }
545 }
546 syn::Expr::Async(_easync) => {
547 self.emit_kind(tokens, TokenKind::AsyncBlock);
548 }
549 syn::Expr::Closure(_eclosure) => {
550 self.emit_kind(tokens, TokenKind::Closure);
552 }
553 syn::Expr::Macro(emacro) => {
554 let name = emacro
555 .mac
556 .path
557 .segments
558 .last()
559 .map(|s| s.ident.to_string())
560 .unwrap_or_default();
561 self.emit(tokens, TokenKind::MacroInvocation, name);
562 }
563 syn::Expr::Block(eblock) => {
564 for stmt in &eblock.block.stmts {
566 self.walk_stmt(stmt, tokens);
567 }
568 }
569 syn::Expr::Path(epath) => {
570 let segments = epath
571 .path
572 .segments
573 .iter()
574 .map(|s| s.ident.to_string())
575 .collect::<Vec<_>>();
576 if !segments.is_empty() {
577 self.emit(
578 tokens,
579 TokenKind::Ident(segments.join("::")),
580 segments.join("::"),
581 );
582 }
583 }
584 _ => {}
585 }
586 }
587
588 fn emit(&self, tokens: &mut Vec<StructuralToken>, kind: TokenKind, text: String) {
589 tokens.push(StructuralToken {
590 kind,
591 text,
592 span: (0, 0), });
594 }
595
596 fn emit_kind(&self, tokens: &mut Vec<StructuralToken>, kind: TokenKind) {
597 self.emit(tokens, kind, String::new());
598 }
599
600 fn walk_items_budgeted(
603 &self,
604 items: &[Item],
605 emissions: &mut Vec<Emission>,
606 budget: &mut BudgetState,
607 ) {
608 for item in items {
609 if budget.emitted >= budget.max_tokens {
610 break;
611 }
612
613 match item {
614 Item::Fn(f) => {
615 let sig = format!("fn {}", f.sig.ident);
616 self.emit_emission(emissions, TokenKind::FnDecl, sig);
617 budget.emitted += 1;
618
619 let body_size = self.get_body_size(&f.block);
621 if body_size >= budget.min_body_stmts {
622 let (node_count, kind_summary) = self.count_body_nodes(&f.block);
624 if node_count > 0 && budget.emitted < budget.max_tokens {
625 emissions.push(Emission::Elided(ElidedSpan {
627 range: (0, 0), node_count,
629 kind_summary,
630 }));
631 budget.emitted += 1;
632 }
633 } else {
634 for stmt in &f.block.stmts {
636 if budget.emitted >= budget.max_tokens {
637 break;
638 }
639 self.walk_stmt_budgeted(stmt, emissions, budget);
640 }
641 }
642 }
643 Item::Struct(s) => {
644 let name = s.ident.to_string();
645 self.emit_emission(emissions, TokenKind::StructDecl, name);
646 budget.emitted += 1;
647 }
648 Item::Enum(e) => {
649 let name = e.ident.to_string();
650 self.emit_emission(emissions, TokenKind::EnumDecl, name);
651 budget.emitted += 1;
652 }
653 Item::Trait(t) => {
654 let name = t.ident.to_string();
655 self.emit_emission(emissions, TokenKind::TraitDecl, name);
656 budget.emitted += 1;
657 }
658 Item::Impl(i) => {
659 self.emit_emission(emissions, TokenKind::ImplBlock, "impl".to_string());
660 budget.emitted += 1;
661 }
662 Item::Use(_u) => {
663 self.emit_emission(emissions, TokenKind::UseStmt, "use".to_string());
664 budget.emitted += 1;
665 }
666 Item::Mod(m) => {
667 let name = m.ident.to_string();
668 self.emit_emission(emissions, TokenKind::ModDecl, name);
669 budget.emitted += 1;
670 if let Some((_brace, items)) = &m.content {
671 self.walk_items_budgeted(items, emissions, budget);
672 }
673 }
674 _ => {}
675 }
676 }
677 }
678
679 fn walk_stmt_budgeted(
680 &self,
681 stmt: &syn::Stmt,
682 emissions: &mut Vec<Emission>,
683 budget: &mut BudgetState,
684 ) {
685 if budget.emitted >= budget.max_tokens {
686 return;
687 }
688
689 match stmt {
690 syn::Stmt::Local(_let_stmt) => {
691 self.emit_emission(emissions, TokenKind::LetStmt, String::new());
692 budget.emitted += 1;
693 }
694 syn::Stmt::Expr(expr, semi) => {
695 self.walk_expr_budgeted(expr, emissions, budget);
696 if semi.is_some() {
697 self.emit_emission(emissions, TokenKind::Semicolon, String::new());
698 budget.emitted += 1;
699 }
700 }
701 _ => {}
702 }
703 }
704
705 fn walk_expr_budgeted(
706 &self,
707 expr: &syn::Expr,
708 emissions: &mut Vec<Emission>,
709 budget: &mut BudgetState,
710 ) {
711 if budget.emitted >= budget.max_tokens {
712 return;
713 }
714
715 match expr {
716 syn::Expr::If(eif) => {
717 self.emit_emission(emissions, TokenKind::IfElse, String::new());
718 budget.emitted += 1;
719 for stmt in &eif.then_branch.stmts {
720 self.walk_stmt_budgeted(stmt, emissions, budget);
721 }
722 }
723 syn::Expr::While(_ewhile) => {
724 self.emit_emission(emissions, TokenKind::WhileLoop, String::new());
725 budget.emitted += 1;
726 }
727 syn::Expr::ForLoop(efor) => {
728 self.emit_emission(emissions, TokenKind::ForLoop, String::new());
729 budget.emitted += 1;
730 for stmt in &efor.body.stmts {
731 self.walk_stmt_budgeted(stmt, emissions, budget);
732 }
733 }
734 syn::Expr::Match(_ematch) => {
735 self.emit_emission(emissions, TokenKind::MatchExpr, String::new());
736 budget.emitted += 1;
737 }
738 syn::Expr::Return(_eret) => {
739 self.emit_emission(emissions, TokenKind::ReturnExpr, String::new());
740 budget.emitted += 1;
741 }
742 syn::Expr::Break(_) => {
743 self.emit_emission(emissions, TokenKind::BreakExpr, String::new());
744 budget.emitted += 1;
745 }
746 syn::Expr::Continue(_) => {
747 self.emit_emission(emissions, TokenKind::ContinueExpr, String::new());
748 budget.emitted += 1;
749 }
750 syn::Expr::Call(_ecall) => {
751 self.emit_emission(emissions, TokenKind::CallExpr, String::new());
752 budget.emitted += 1;
753 }
754 syn::Expr::MethodCall(emethod) => {
755 let method_name = emethod.method.to_string();
756 self.emit_emission(emissions, TokenKind::MethodCall, method_name);
757 budget.emitted += 1;
758 }
759 _ => {}
760 }
761 }
762
763 fn emit_emission(&self, emissions: &mut Vec<Emission>, kind: TokenKind, text: String) {
764 let token = StructuralToken {
765 kind,
766 text,
767 span: (0, 0),
768 };
769 emissions.push(Emission::Node(token));
770 }
771
772 fn get_body_size(&self, block: &syn::Block) -> usize {
773 block.stmts.len()
775 }
776
777 fn count_body_nodes(&self, block: &syn::Block) -> (usize, Vec<(NodeKindTag, usize)>) {
778 let mut counts: HashMap<NodeKindTag, usize> = HashMap::new();
779 let mut total = 0;
780
781 for stmt in &block.stmts {
782 match stmt {
783 syn::Stmt::Local(_) => {
784 *counts.entry(NodeKindTag::Other).or_insert(0) += 1;
785 total += 1;
786 }
787 syn::Stmt::Expr(expr, _) => {
788 self.count_expr_nodes(expr, &mut counts, &mut total);
789 }
790 _ => {}
791 }
792 }
793
794 let mut summary: Vec<(NodeKindTag, usize)> = counts.into_iter().collect();
795 summary.sort_by(|a, b| b.1.cmp(&a.1));
796 (total, summary)
797 }
798
799 fn count_expr_nodes(
800 &self,
801 expr: &syn::Expr,
802 counts: &mut HashMap<NodeKindTag, usize>,
803 total: &mut usize,
804 ) {
805 match expr {
806 syn::Expr::If(eif) => {
807 *counts.entry(NodeKindTag::IfElse).or_insert(0) += 1;
808 *total += 1;
809 for stmt in &eif.then_branch.stmts {
810 self.count_stmt_nodes(stmt, counts, total);
811 }
812 }
813 syn::Expr::Match(_ematch) => {
814 *counts.entry(NodeKindTag::MatchExpr).or_insert(0) += 1;
815 *total += 1;
816 }
817 syn::Expr::ForLoop(efor) => {
818 *counts.entry(NodeKindTag::ForLoop).or_insert(0) += 1;
819 *total += 1;
820 for stmt in &efor.body.stmts {
821 self.count_stmt_nodes(stmt, counts, total);
822 }
823 }
824 syn::Expr::While(_ewhile) => {
825 *counts.entry(NodeKindTag::WhileLoop).or_insert(0) += 1;
826 *total += 1;
827 }
828 syn::Expr::Closure(_eclosure) => {
829 *counts.entry(NodeKindTag::Closure).or_insert(0) += 1;
830 *total += 1;
831 }
832 _ => {}
833 }
834 }
835
836 fn count_stmt_nodes(
837 &self,
838 stmt: &syn::Stmt,
839 counts: &mut HashMap<NodeKindTag, usize>,
840 total: &mut usize,
841 ) {
842 match stmt {
843 syn::Stmt::Local(_) => {
844 *counts.entry(NodeKindTag::Other).or_insert(0) += 1;
845 *total += 1;
846 }
847 syn::Stmt::Expr(expr, _) => {
848 self.count_expr_nodes(expr, counts, total);
849 }
850 _ => {}
851 }
852 }
853}
854
855impl Default for StructuralTokenizer {
856 fn default() -> Self {
857 Self::new()
858 }
859}
860
861struct BudgetState {
863 max_tokens: usize,
864 emitted: usize,
865 min_body_stmts: usize,
866}
867
868#[cfg(test)]
869mod tests {
870 use super::*;
871
872 #[test]
873 fn test_fn_declaration() {
874 let tokenizer = StructuralTokenizer::new();
875 let src = "fn add(a: i32, b: i32) -> i32 { return a + b; }";
876 let tokens = tokenizer.tokenize(src).unwrap();
877
878 assert_eq!(tokens[0].kind, TokenKind::FnDecl);
879 assert_eq!(tokens[1].kind, TokenKind::ReturnExpr);
880 }
881
882 #[test]
883 fn test_let_statement() {
884 let tokenizer = StructuralTokenizer::new();
885 let src = "fn main() { let x: i32 = 42; }";
886 let tokens = tokenizer.tokenize(src).unwrap();
887
888 assert_eq!(tokens[0].kind, TokenKind::FnDecl);
889 assert_eq!(tokens[1].kind, TokenKind::LetStmt);
890 }
891
892 #[test]
893 fn test_return_expression() {
894 let tokenizer = StructuralTokenizer::new();
895 let src = "fn main() { return 42; }";
896 let tokens = tokenizer.tokenize(src).unwrap();
897
898 assert_eq!(tokens[0].kind, TokenKind::FnDecl);
899 assert_eq!(tokens[1].kind, TokenKind::ReturnExpr);
900 }
901
902 #[test]
903 fn test_block_structure() {
904 let tokenizer = StructuralTokenizer::new();
905 let src = "fn main() { if true { let x = 1; } else { let y = 2; } }";
906 let tokens = tokenizer.tokenize(src).unwrap();
907
908 assert!(tokens.iter().any(|t| t.kind == TokenKind::IfElse));
909 }
910
911 #[test]
912 fn test_struct_declaration() {
913 let tokenizer = StructuralTokenizer::new();
914 let src = "struct Point { x: i32, y: i32 }";
915 let tokens = tokenizer.tokenize(src).unwrap();
916 assert_eq!(tokens[0].kind, TokenKind::StructDecl);
917 }
918
919 #[test]
920 fn test_enum_declaration() {
921 let tokenizer = StructuralTokenizer::new();
922 let src = "enum Color { Red, Green, Blue }";
923 let tokens = tokenizer.tokenize(src).unwrap();
924 assert_eq!(tokens[0].kind, TokenKind::EnumDecl);
925 }
926
927 #[test]
928 fn test_trait_declaration() {
929 let tokenizer = StructuralTokenizer::new();
930 let src = "trait Display { fn fmt(&self) -> String; }";
931 let tokens = tokenizer.tokenize(src).unwrap();
932 assert_eq!(tokens[0].kind, TokenKind::TraitDecl);
933 }
934
935 #[test]
936 fn test_impl_block() {
937 let tokenizer = StructuralTokenizer::new();
938 let src = "impl Display for Point { fn fmt(&self) -> String { \"\".to_string() } }";
939 let tokens = tokenizer.tokenize(src).unwrap();
940 assert_eq!(tokens[0].kind, TokenKind::ImplBlock);
941 }
942
943 #[test]
944 fn test_break_continue() {
945 let tokenizer = StructuralTokenizer::new();
946 let src = "fn main() { for i in 0..10 { if i == 5 { break; } continue; } }";
947 let tokens = tokenizer.tokenize(src).unwrap();
948
949 assert!(tokens.iter().any(|t| t.kind == TokenKind::ForLoop));
950 assert!(tokens.iter().any(|t| t.kind == TokenKind::BreakExpr));
951 assert!(tokens.iter().any(|t| t.kind == TokenKind::ContinueExpr));
952 }
953
954 #[test]
955 fn test_complex_program() {
956 let tokenizer = StructuralTokenizer::new();
957 let src = r#"
958struct Counter {
959 count: i32,
960}
961
962impl Counter {
963 fn new() -> Counter {
964 Counter { count: 0 }
965 }
966
967 fn increment(&mut self) {
968 self.count = self.count + 1;
969 }
970
971 fn get(&self) -> i32 {
972 return self.count;
973 }
974}
975
976fn main() {
977 let c = Counter::new();
978 for i in 0..5 {
979 if i == 3 {
980 break;
981 }
982 c.increment();
983 }
984}
985"#;
986 let tokens = tokenizer.tokenize(src).unwrap();
987
988 assert!(tokens.iter().any(|t| t.kind == TokenKind::StructDecl));
989 assert!(tokens.iter().any(|t| t.kind == TokenKind::ImplBlock));
990 assert!(tokens.iter().any(|t| t.kind == TokenKind::FnDecl));
991 }
992
993 #[test]
994 fn test_budget_basic() {
995 let tokenizer = StructuralTokenizer::new();
996 let src = "fn foo() { let x = 1; let y = 2; return x + y; }";
997 let budget = Budget {
998 max_tokens: 5,
999 min_body_stmts: 1,
1000 };
1001 let emissions = tokenizer.tokenize_with_budget(src, budget).unwrap();
1002
1003 assert!(emissions.len() <= 5);
1005 }
1006
1007 #[test]
1008 fn test_budget_elides_large_bodies() {
1009 let mut src = "fn big() {".to_string();
1011 for i in 0..20 {
1012 src.push_str(&format!("let x{} = {};\n", i, i));
1013 }
1014 src.push_str("}");
1015
1016 let tokenizer = StructuralTokenizer::new();
1017 let budget = Budget {
1018 max_tokens: 10,
1019 min_body_stmts: 1,
1020 };
1021 let emissions = tokenizer.tokenize_with_budget(&src, budget).unwrap();
1022
1023 assert!(
1025 emissions
1026 .iter()
1027 .any(|e| matches!(e, Emission::Node(t) if t.kind == TokenKind::FnDecl))
1028 );
1029 assert!(emissions.iter().any(|e| matches!(e, Emission::Elided(_))));
1030 }
1031
1032 #[test]
1033 fn test_elided_span_has_summary() {
1034 let src =
1035 "fn big() { for i in 0..10 { if i == 5 { break; } } match x { 1 => {}, _ => {} } }";
1036 let tokenizer = StructuralTokenizer::new();
1037 let budget = Budget {
1038 max_tokens: 5,
1039 min_body_stmts: 1,
1040 };
1041 let emissions = tokenizer.tokenize_with_budget(src, budget).unwrap();
1042
1043 for e in &emissions {
1045 if let Emission::Elided(span) = e {
1046 assert!(!span.kind_summary.is_empty());
1047 assert!(span.node_count > 0);
1048 }
1049 }
1050 }
1051}