vyre_libs/parsing/python/parse/
structure.rs

1use super::{
2    find_matching_delimiter, find_matching_delimiter_into, load_u32, search_next_token,
3    search_next_token_into, search_prev_token, store_words, write_words,
4};
5use crate::parsing::composition::child_phase;
6use crate::parsing::python::lex::{
7    TOK_ASYNC, TOK_CLASS, TOK_COLON, TOK_COMMA, TOK_DEF, TOK_DOT, TOK_FROM, TOK_IDENTIFIER,
8    TOK_IMPORT, TOK_LBRACKET, TOK_LPAREN, TOK_RBRACKET, TOK_WITH,
9};
10use crate::parsing::python::{
11    DEF_RECORD_WORDS, IMPORT_RECORD_WORDS, INVALID_POS, WITH_RECORD_WORDS,
12};
13use crate::region::wrap_anonymous;
14use vyre::ir::{BufferAccess, BufferDecl, DataType, Expr, Node, Program};
15
16/// Extract `def`, `async def`, and `class` declarations.
17#[must_use]
18pub fn python312_extract_structure(
19    tok_types: &str,
20    tok_starts: &str,
21    tok_lens: &str,
22    out_records: &str,
23    out_counts: &str,
24    haystack_len: u32,
25) -> Program {
26    let t = Expr::InvocationId { axis: 0 };
27    let mut body = vec![
28        Node::let_bind("tok", load_u32(tok_types, t.clone())),
29        Node::let_bind("emit_kind", Expr::u32(0)),
30        Node::let_bind("keyword_pos", Expr::u32(INVALID_POS)),
31        Node::if_then(
32            Expr::eq(Expr::var("tok"), Expr::u32(TOK_DEF)),
33            vec![
34                Node::assign("emit_kind", Expr::u32(1)),
35                Node::assign("keyword_pos", t.clone()),
36            ],
37        ),
38        Node::if_then(
39            Expr::eq(Expr::var("tok"), Expr::u32(TOK_CLASS)),
40            vec![
41                Node::assign("emit_kind", Expr::u32(3)),
42                Node::assign("keyword_pos", t.clone()),
43            ],
44        ),
45    ];
46    body.extend(search_next_token(
47        "async_next",
48        Expr::add(t.clone(), Expr::u32(1)),
49        tok_types,
50        haystack_len,
51    ));
52    body.push(Node::if_then(
53        Expr::and(
54            Expr::eq(Expr::var("tok"), Expr::u32(TOK_ASYNC)),
55            Expr::eq(
56                load_u32(tok_types, Expr::var("async_next")),
57                Expr::u32(TOK_DEF),
58            ),
59        ),
60        vec![
61            Node::assign("emit_kind", Expr::u32(2)),
62            Node::assign("keyword_pos", Expr::var("async_next")),
63        ],
64    ));
65    body.extend(search_next_token(
66        "name_pos",
67        Expr::add(Expr::var("keyword_pos"), Expr::u32(1)),
68        tok_types,
69        haystack_len,
70    ));
71    body.extend(search_next_token(
72        "post_name",
73        Expr::add(Expr::var("name_pos"), Expr::u32(1)),
74        tok_types,
75        haystack_len,
76    ));
77    body.extend(find_matching_delimiter(
78        "type_params_end",
79        Expr::var("post_name"),
80        tok_types,
81        haystack_len,
82        TOK_LBRACKET,
83        TOK_RBRACKET,
84    ));
85    body.push(Node::if_then(
86        Expr::and(
87            Expr::ne(Expr::var("emit_kind"), Expr::u32(0)),
88            Expr::eq(
89                load_u32(tok_types, Expr::var("name_pos")),
90                Expr::u32(TOK_IDENTIFIER),
91            ),
92        ),
93        vec![
94            Node::let_bind("params_start", Expr::u32(INVALID_POS)),
95            Node::let_bind("params_end", Expr::u32(INVALID_POS)),
96            Node::let_bind("colon_pos", Expr::u32(INVALID_POS)),
97            // Hoist `after_type_params` and `after_params` to the
98            // outer scope so the if-block bodies (which assign them)
99            // and the later if-blocks (which read them) share one
100            // binding. Pre-T-V2 the per-branch `Node::let_bind` lived
101            // inside each block, the validator scoped the binding to
102            // the block, and the read sites failed with "reference to
103            // undeclared variable `after_type_params`" / `after_params`.
104            Node::let_bind("after_type_params", Expr::u32(INVALID_POS)),
105            Node::let_bind("after_params", Expr::u32(INVALID_POS)),
106            Node::if_then_else(
107                Expr::eq(
108                    load_u32(tok_types, Expr::var("post_name")),
109                    Expr::u32(TOK_LBRACKET),
110                ),
111                search_next_token_into(
112                    "after_type_params",
113                    Expr::add(Expr::var("type_params_end"), Expr::u32(1)),
114                    tok_types,
115                    haystack_len,
116                ),
117                vec![Node::assign("after_type_params", Expr::var("post_name"))],
118            ),
119            Node::if_then(
120                Expr::eq(
121                    load_u32(tok_types, Expr::var("after_type_params")),
122                    Expr::u32(TOK_LPAREN),
123                ),
124                vec![
125                    Node::assign("params_start", Expr::var("after_type_params")),
126                    Node::assign("params_end", Expr::u32(INVALID_POS)),
127                ]
128                .into_iter()
129                .chain(find_matching_delimiter_into(
130                    "params_end",
131                    Expr::var("after_type_params"),
132                    tok_types,
133                    haystack_len,
134                    TOK_LPAREN,
135                    crate::parsing::python::lex::TOK_RPAREN,
136                ))
137                .collect(),
138            ),
139            Node::if_then_else(
140                Expr::ne(Expr::var("params_end"), Expr::u32(INVALID_POS)),
141                search_next_token_into(
142                    "after_params",
143                    Expr::add(Expr::var("params_end"), Expr::u32(1)),
144                    tok_types,
145                    haystack_len,
146                ),
147                vec![Node::assign("after_params", Expr::var("after_type_params"))],
148            ),
149            Node::if_then(
150                Expr::eq(
151                    load_u32(tok_types, Expr::var("after_params")),
152                    Expr::u32(TOK_COLON),
153                ),
154                vec![Node::assign("colon_pos", Expr::var("after_params"))],
155            ),
156            Node::let_bind(
157                "slot",
158                Expr::atomic_add(out_counts, Expr::u32(0), Expr::u32(DEF_RECORD_WORDS)),
159            ),
160        ]
161        .into_iter()
162        .chain(store_words(
163            out_records,
164            "slot",
165            &[
166                Expr::var("emit_kind"),
167                load_u32(tok_starts, Expr::var("name_pos")),
168                load_u32(tok_lens, Expr::var("name_pos")),
169                Expr::var("params_start"),
170                Expr::var("params_end"),
171                Expr::var("colon_pos"),
172            ],
173        ))
174        .collect(),
175    ));
176
177    Program::wrapped(
178        vec![
179            BufferDecl::storage(tok_types, 0, BufferAccess::ReadOnly, DataType::U32)
180                .with_count(haystack_len),
181            BufferDecl::storage(tok_starts, 1, BufferAccess::ReadOnly, DataType::U32)
182                .with_count(haystack_len),
183            BufferDecl::storage(tok_lens, 2, BufferAccess::ReadOnly, DataType::U32)
184                .with_count(haystack_len),
185            BufferDecl::storage(out_records, 3, BufferAccess::ReadWrite, DataType::U32)
186                .with_count(haystack_len.saturating_mul(DEF_RECORD_WORDS)),
187            BufferDecl::storage(out_counts, 4, BufferAccess::ReadWrite, DataType::U32)
188                .with_count(1),
189        ],
190        [256, 1, 1],
191        vec![wrap_anonymous(
192            "vyre-libs::parsing::python312_extract_structure",
193            vec![child_phase(
194                "vyre-libs::parsing::python312_extract_structure",
195                vyre_primitives::text::line_index::OP_ID,
196                vec![Node::if_then(
197                    Expr::lt(t.clone(), Expr::u32(haystack_len)),
198                    body,
199                )],
200            )],
201        )],
202    )
203    .with_entry_op_id("vyre-libs::parsing::python312_extract_structure")
204    .with_non_composable_with_self(true)
205}
206
207/// Extract `import` and `from ... import ...` statements.
208#[must_use]
209pub fn python312_extract_imports(
210    tok_types: &str,
211    tok_starts: &str,
212    tok_lens: &str,
213    out_records: &str,
214    out_counts: &str,
215    haystack_len: u32,
216) -> Program {
217    let t = Expr::InvocationId { axis: 0 };
218    let mut body = vec![
219        Node::let_bind("tok", load_u32(tok_types, t.clone())),
220        Node::let_bind("record_kind", Expr::u32(0)),
221    ];
222    body.extend(search_prev_token("prev_tok", t.clone(), tok_types));
223    body.extend(search_next_token(
224        "next_tok",
225        Expr::add(t.clone(), Expr::u32(1)),
226        tok_types,
227        haystack_len,
228    ));
229    body.push(Node::if_then(
230        Expr::and(
231            Expr::eq(Expr::var("tok"), Expr::u32(TOK_IDENTIFIER)),
232            Expr::or(
233                Expr::eq(
234                    load_u32(tok_types, Expr::var("prev_tok")),
235                    Expr::u32(TOK_IMPORT),
236                ),
237                Expr::eq(
238                    load_u32(tok_types, Expr::var("prev_tok")),
239                    Expr::u32(TOK_FROM),
240                ),
241            ),
242        ),
243        vec![Node::assign(
244            "record_kind",
245            Expr::select(
246                Expr::eq(
247                    load_u32(tok_types, Expr::var("prev_tok")),
248                    Expr::u32(TOK_IMPORT),
249                ),
250                Expr::u32(1),
251                Expr::u32(2),
252            ),
253        )],
254    ));
255    body.push(Node::if_then(
256        Expr::and(
257            Expr::eq(Expr::var("tok"), Expr::u32(TOK_IDENTIFIER)),
258            Expr::eq(
259                load_u32(tok_types, Expr::var("prev_tok")),
260                Expr::u32(TOK_COMMA),
261            ),
262        ),
263        vec![Node::assign("record_kind", Expr::u32(1))],
264    ));
265    body.push(Node::if_then(
266        Expr::ne(Expr::var("record_kind"), Expr::u32(0)),
267        vec![
268            Node::let_bind("name_end", t.clone()),
269            Node::let_bind("cursor", t.clone()),
270            Node::let_bind("dot_pos", Expr::u32(INVALID_POS)),
271            Node::let_bind("after_dot", Expr::u32(INVALID_POS)),
272            Node::loop_for(
273                "seg",
274                Expr::u32(0),
275                Expr::u32(crate::parsing::python::MAX_DOTTED_SEGMENTS),
276                vec![
277                    // Reset per iteration via assign  -  the outer
278                    // let_bind lives BEFORE the loop_for so the
279                    // validator doesn't see a re-declaration each
280                    // pass (V008). search_next_token_into is the
281                    // assign-only variant for the same reason.
282                    Node::assign("dot_pos", Expr::u32(INVALID_POS)),
283                    Node::assign("after_dot", Expr::u32(INVALID_POS)),
284                    Node::if_then(
285                        Expr::ne(Expr::var("cursor"), Expr::u32(INVALID_POS)),
286                        search_next_token_into(
287                            "dot_pos",
288                            Expr::add(Expr::var("cursor"), Expr::u32(1)),
289                            tok_types,
290                            haystack_len,
291                        ),
292                    ),
293                    Node::if_then(
294                        Expr::eq(
295                            load_u32(tok_types, Expr::var("dot_pos")),
296                            Expr::u32(TOK_DOT),
297                        ),
298                        search_next_token_into(
299                            "after_dot",
300                            Expr::add(Expr::var("dot_pos"), Expr::u32(1)),
301                            tok_types,
302                            haystack_len,
303                        ),
304                    ),
305                    Node::if_then(
306                        Expr::eq(
307                            load_u32(tok_types, Expr::var("after_dot")),
308                            Expr::u32(TOK_IDENTIFIER),
309                        ),
310                        vec![
311                            Node::assign("name_end", Expr::var("after_dot")),
312                            Node::assign("cursor", Expr::var("after_dot")),
313                        ],
314                    ),
315                    Node::if_then(
316                        Expr::ne(
317                            load_u32(tok_types, Expr::var("after_dot")),
318                            Expr::u32(TOK_IDENTIFIER),
319                        ),
320                        vec![Node::assign("cursor", Expr::u32(INVALID_POS))],
321                    ),
322                ],
323            ),
324            Node::let_bind(
325                "slot",
326                Expr::atomic_add(out_counts, Expr::u32(0), Expr::u32(IMPORT_RECORD_WORDS)),
327            ),
328        ]
329        .into_iter()
330        .chain(store_words(
331            out_records,
332            "slot",
333            &[
334                Expr::var("record_kind"),
335                load_u32(tok_starts, t.clone()),
336                Expr::add(
337                    Expr::sub(
338                        load_u32(tok_starts, Expr::var("name_end")),
339                        load_u32(tok_starts, t.clone()),
340                    ),
341                    load_u32(tok_lens, Expr::var("name_end")),
342                ),
343                Expr::var("prev_tok"),
344                Expr::var("name_end"),
345                Expr::var("next_tok"),
346            ],
347        ))
348        .collect(),
349    ));
350
351    Program::wrapped(
352        vec![
353            BufferDecl::storage(tok_types, 0, BufferAccess::ReadOnly, DataType::U32)
354                .with_count(haystack_len),
355            BufferDecl::storage(tok_starts, 1, BufferAccess::ReadOnly, DataType::U32)
356                .with_count(haystack_len),
357            BufferDecl::storage(tok_lens, 2, BufferAccess::ReadOnly, DataType::U32)
358                .with_count(haystack_len),
359            BufferDecl::storage(out_records, 3, BufferAccess::ReadWrite, DataType::U32)
360                .with_count(haystack_len.saturating_mul(IMPORT_RECORD_WORDS)),
361            BufferDecl::storage(out_counts, 4, BufferAccess::ReadWrite, DataType::U32)
362                .with_count(1),
363        ],
364        [256, 1, 1],
365        vec![wrap_anonymous(
366            "vyre-libs::parsing::python312_extract_imports",
367            vec![child_phase(
368                "vyre-libs::parsing::python312_extract_imports",
369                vyre_primitives::text::line_index::OP_ID,
370                vec![Node::if_then(
371                    Expr::lt(t.clone(), Expr::u32(haystack_len)),
372                    body,
373                )],
374            )],
375        )],
376    )
377    .with_entry_op_id("vyre-libs::parsing::python312_extract_imports")
378    .with_non_composable_with_self(true)
379}
380
381/// Extract `with` / `async with` headers.
382#[must_use]
383pub fn python312_extract_with_blocks(
384    tok_types: &str,
385    tok_starts: &str,
386    tok_lens: &str,
387    out_records: &str,
388    out_counts: &str,
389    haystack_len: u32,
390) -> Program {
391    let t = Expr::InvocationId { axis: 0 };
392    let mut body = vec![
393        Node::let_bind("tok", load_u32(tok_types, t.clone())),
394        Node::let_bind("with_pos", Expr::u32(INVALID_POS)),
395        Node::let_bind("flags", Expr::u32(0)),
396    ];
397    body.extend(search_prev_token("prev_tok", t.clone(), tok_types));
398    body.push(Node::if_then(
399        Expr::and(
400            Expr::eq(Expr::var("tok"), Expr::u32(TOK_WITH)),
401            Expr::ne(
402                load_u32(tok_types, Expr::var("prev_tok")),
403                Expr::u32(TOK_ASYNC),
404            ),
405        ),
406        vec![Node::assign("with_pos", t.clone())],
407    ));
408    body.extend(search_next_token(
409        "async_next",
410        Expr::add(t.clone(), Expr::u32(1)),
411        tok_types,
412        haystack_len,
413    ));
414    body.push(Node::if_then(
415        Expr::and(
416            Expr::eq(Expr::var("tok"), Expr::u32(TOK_ASYNC)),
417            Expr::eq(
418                load_u32(tok_types, Expr::var("async_next")),
419                Expr::u32(TOK_WITH),
420            ),
421        ),
422        vec![
423            Node::assign("with_pos", Expr::var("async_next")),
424            Node::assign("flags", Expr::u32(1)),
425        ],
426    ));
427    body.extend(search_next_token(
428        "manager_pos",
429        Expr::add(Expr::var("with_pos"), Expr::u32(1)),
430        tok_types,
431        haystack_len,
432    ));
433    body.extend(search_next_token(
434        "after_manager",
435        Expr::add(Expr::var("manager_pos"), Expr::u32(1)),
436        tok_types,
437        haystack_len,
438    ));
439    body.push(Node::if_then(
440        Expr::and(
441            Expr::ne(Expr::var("with_pos"), Expr::u32(INVALID_POS)),
442            Expr::eq(
443                load_u32(tok_types, Expr::var("manager_pos")),
444                Expr::u32(TOK_IDENTIFIER),
445            ),
446        ),
447        vec![
448            Node::let_bind("manager_end", Expr::var("manager_pos")),
449            Node::let_bind("cursor", Expr::var("manager_pos")),
450            Node::let_bind("dot_pos", Expr::u32(INVALID_POS)),
451            Node::let_bind("after_dot", Expr::u32(INVALID_POS)),
452            Node::loop_for(
453                "seg",
454                Expr::u32(0),
455                Expr::u32(crate::parsing::python::MAX_DOTTED_SEGMENTS),
456                vec![
457                    // Reset per iteration via assign  -  the outer
458                    // let_bind lives BEFORE the loop_for so the
459                    // validator doesn't see a re-declaration each
460                    // pass (V008). search_next_token_into is the
461                    // assign-only variant for the same reason.
462                    Node::assign("dot_pos", Expr::u32(INVALID_POS)),
463                    Node::assign("after_dot", Expr::u32(INVALID_POS)),
464                    Node::if_then(
465                        Expr::ne(Expr::var("cursor"), Expr::u32(INVALID_POS)),
466                        search_next_token_into(
467                            "dot_pos",
468                            Expr::add(Expr::var("cursor"), Expr::u32(1)),
469                            tok_types,
470                            haystack_len,
471                        ),
472                    ),
473                    Node::if_then(
474                        Expr::eq(
475                            load_u32(tok_types, Expr::var("dot_pos")),
476                            Expr::u32(TOK_DOT),
477                        ),
478                        search_next_token_into(
479                            "after_dot",
480                            Expr::add(Expr::var("dot_pos"), Expr::u32(1)),
481                            tok_types,
482                            haystack_len,
483                        ),
484                    ),
485                    Node::if_then(
486                        Expr::eq(
487                            load_u32(tok_types, Expr::var("after_dot")),
488                            Expr::u32(TOK_IDENTIFIER),
489                        ),
490                        vec![
491                            Node::assign("manager_end", Expr::var("after_dot")),
492                            Node::assign("cursor", Expr::var("after_dot")),
493                        ],
494                    ),
495                    Node::if_then(
496                        Expr::ne(
497                            load_u32(tok_types, Expr::var("after_dot")),
498                            Expr::u32(TOK_IDENTIFIER),
499                        ),
500                        vec![Node::assign("cursor", Expr::u32(INVALID_POS))],
501                    ),
502                ],
503            ),
504            Node::let_bind("colon_pos", Expr::u32(INVALID_POS)),
505            Node::loop_for(
506                "scan",
507                Expr::add(Expr::var("manager_end"), Expr::u32(1)),
508                Expr::u32(haystack_len),
509                vec![Node::if_then(
510                    Expr::and(
511                        Expr::eq(Expr::var("colon_pos"), Expr::u32(INVALID_POS)),
512                        Expr::eq(load_u32(tok_types, Expr::var("scan")), Expr::u32(TOK_COLON)),
513                    ),
514                    vec![Node::assign("colon_pos", Expr::var("scan"))],
515                )],
516            ),
517            Node::let_bind(
518                "slot",
519                Expr::atomic_add(out_counts, Expr::u32(0), Expr::u32(WITH_RECORD_WORDS)),
520            ),
521        ]
522        .into_iter()
523        .chain(store_words(
524            out_records,
525            "slot",
526            &[
527                load_u32(tok_starts, Expr::var("manager_pos")),
528                Expr::add(
529                    Expr::sub(
530                        load_u32(tok_starts, Expr::var("manager_end")),
531                        load_u32(tok_starts, Expr::var("manager_pos")),
532                    ),
533                    load_u32(tok_lens, Expr::var("manager_end")),
534                ),
535                Expr::var("with_pos"),
536                Expr::var("colon_pos"),
537                Expr::var("flags"),
538                Expr::u32(0),
539            ],
540        ))
541        .collect(),
542    ));
543
544    Program::wrapped(
545        vec![
546            BufferDecl::storage(tok_types, 0, BufferAccess::ReadOnly, DataType::U32)
547                .with_count(haystack_len),
548            BufferDecl::storage(tok_starts, 1, BufferAccess::ReadOnly, DataType::U32)
549                .with_count(haystack_len),
550            BufferDecl::storage(tok_lens, 2, BufferAccess::ReadOnly, DataType::U32)
551                .with_count(haystack_len),
552            BufferDecl::storage(out_records, 3, BufferAccess::ReadWrite, DataType::U32)
553                .with_count(haystack_len.saturating_mul(WITH_RECORD_WORDS)),
554            BufferDecl::storage(out_counts, 4, BufferAccess::ReadWrite, DataType::U32)
555                .with_count(1),
556        ],
557        [256, 1, 1],
558        vec![wrap_anonymous(
559            "vyre-libs::parsing::python312_extract_with_blocks",
560            vec![child_phase(
561                "vyre-libs::parsing::python312_extract_with_blocks",
562                vyre_primitives::text::line_index::OP_ID,
563                vec![Node::if_then(
564                    Expr::lt(t.clone(), Expr::u32(haystack_len)),
565                    body,
566                )],
567            )],
568        )],
569    )
570    .with_entry_op_id("vyre-libs::parsing::python312_extract_with_blocks")
571    .with_non_composable_with_self(true)
572}
573
574inventory::submit! {
575    crate::harness::OpEntry {
576        id: "vyre-libs::parsing::python312_extract_structure",
577        build: || python312_extract_structure("tok_types", "tok_starts", "tok_lens", "out_records", "out_counts", 16),
578        test_inputs: Some(structure_fixture_inputs),
579        expected_output: Some(structure_fixture_expected),
580        category: Some("parsing"),
581    }
582}
583
584inventory::submit! {
585    crate::harness::OpEntry {
586        id: "vyre-libs::parsing::python312_extract_imports",
587        build: || python312_extract_imports("tok_types", "tok_starts", "tok_lens", "out_records", "out_counts", 16),
588        test_inputs: Some(import_fixture_inputs),
589        expected_output: Some(import_fixture_expected),
590        category: Some("parsing"),
591    }
592}
593
594inventory::submit! {
595    crate::harness::OpEntry {
596        id: "vyre-libs::parsing::python312_extract_with_blocks",
597        build: || python312_extract_with_blocks("tok_types", "tok_starts", "tok_lens", "out_records", "out_counts", 16),
598        test_inputs: Some(with_fixture_inputs),
599        expected_output: Some(with_fixture_expected),
600        category: Some("parsing"),
601    }
602}
603
604fn pack_sparse_tokens(tokens: &[(usize, u32, u32)]) -> (Vec<u8>, Vec<u8>, Vec<u8>) {
605    let mut tok_types = vec![0u8; 16 * 4];
606    let mut tok_starts = vec![0u8; 16 * 4];
607    let mut tok_lens = vec![0u8; 16 * 4];
608    for &(pos, tok, len) in tokens {
609        let base = pos * 4;
610        tok_types[base..base + 4].copy_from_slice(&tok.to_le_bytes());
611        tok_starts[base..base + 4].copy_from_slice(&(pos as u32).to_le_bytes());
612        tok_lens[base..base + 4].copy_from_slice(&len.to_le_bytes());
613    }
614    (tok_types, tok_starts, tok_lens)
615}
616
617fn structure_fixture_inputs() -> Vec<Vec<Vec<u8>>> {
618    let (tok_types, tok_starts, tok_lens) = pack_sparse_tokens(&[
619        (0, TOK_DEF, 3),
620        (4, TOK_IDENTIFIER, 1),
621        (5, TOK_LPAREN, 1),
622        (6, crate::parsing::python::lex::TOK_RPAREN, 1),
623        (7, TOK_COLON, 1),
624    ]);
625    vec![vec![
626        tok_types,
627        tok_starts,
628        tok_lens,
629        vec![0u8; 16 * DEF_RECORD_WORDS as usize * 4],
630        vec![0u8; 4],
631    ]]
632}
633
634fn structure_fixture_expected() -> Vec<Vec<Vec<u8>>> {
635    let mut records = vec![0u8; 16 * DEF_RECORD_WORDS as usize * 4];
636    write_words(&mut records, &[1, 4, 1, 5, 6, 7]);
637    vec![vec![records, DEF_RECORD_WORDS.to_le_bytes().to_vec()]]
638}
639
640fn import_fixture_inputs() -> Vec<Vec<Vec<u8>>> {
641    let (tok_types, tok_starts, tok_lens) =
642        pack_sparse_tokens(&[(0, TOK_IMPORT, 6), (7, TOK_IDENTIFIER, 2)]);
643    vec![vec![
644        tok_types,
645        tok_starts,
646        tok_lens,
647        vec![0u8; 16 * IMPORT_RECORD_WORDS as usize * 4],
648        vec![0u8; 4],
649    ]]
650}
651
652fn import_fixture_expected() -> Vec<Vec<Vec<u8>>> {
653    let mut records = vec![0u8; 16 * IMPORT_RECORD_WORDS as usize * 4];
654    write_words(
655        &mut records,
656        &[1, 7, 2, 0, 7, crate::parsing::python::INVALID_POS],
657    );
658    vec![vec![records, IMPORT_RECORD_WORDS.to_le_bytes().to_vec()]]
659}
660
661fn with_fixture_inputs() -> Vec<Vec<Vec<u8>>> {
662    let (tok_types, tok_starts, tok_lens) = pack_sparse_tokens(&[
663        (0, TOK_ASYNC, 5),
664        (6, TOK_WITH, 4),
665        (11, TOK_IDENTIFIER, 3),
666        (14, TOK_COLON, 1),
667    ]);
668    vec![vec![
669        tok_types,
670        tok_starts,
671        tok_lens,
672        vec![0u8; 16 * WITH_RECORD_WORDS as usize * 4],
673        vec![0u8; 4],
674    ]]
675}
676
677fn with_fixture_expected() -> Vec<Vec<Vec<u8>>> {
678    let mut records = vec![0u8; 16 * WITH_RECORD_WORDS as usize * 4];
679    write_words(&mut records, &[11, 3, 6, 14, 1, 0]);
680    vec![vec![records, WITH_RECORD_WORDS.to_le_bytes().to_vec()]]
681}
vyre_libs/parsing/python/parse/structure.rs

vyre_libs/parsing/python/parse/
structure.rs
