vox-lang 0.3.7

A systems level compiler for Vox (sentence based code)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
use super::*;

impl Parser {
    /// After a callee name (bare or quoted identifier) has been advanced past,
    /// parse a call connector (`of`/`with`/`on`, and `to` when `allow_to`)
    /// and its argument list into a `FunctionCall` (plan 270 G1). Returns
    /// `None` when the next token is not a call connector, so the caller falls
    /// through to other postfix forms (property access, a bare identifier) or,
    /// in append-value position, the `to` separator. `allow_to` is false
    /// there: `to` is the append separator and must not read as a connector.
    /// `of` is similarly reserved (via `suppress_of_connector`) while parsing
    /// an index in `element N of .../byte N of ...`, where `of` is that
    /// statement's own separator, not this primary's connector.
    pub(crate) fn parse_call_tail(&mut self, name: String, allow_to: bool) -> Result<Option<Expr>, Box<CompileError>> {
        let is_conn = match self.current() {
            Token::Of => !self.suppress_of_connector,
            Token::With | Token::On => true,
            Token::To => allow_to && !self.suppress_to_connector,
            _ => false,
        };
        if !is_conn {
            return Ok(None);
        }
        self.advance();
        self.skip_noise();
        let mut args = Vec::new();
        let mut first_arg = true;
        loop {
            // A function call is a primary and must bind tighter than the
            // additive operators (`add`/`subtract`): `'state pos' of state
            // add by` is `('state pos' of state) add by`, not `'state pos' of
            // (state add by)`. The FIRST argument is therefore parsed at the
            // `cast` level (below additive) so a trailing `add`/`subtract` is
            // left for the caller to apply to the call result, while an
            // argument's own `as a <type>` cast is still kept (`f of x as a
            // number`).
            //
            // Once an `and` marks this as a multi-argument call, the argument
            // boundary is explicit, so LATER arguments parse at the full
            // `parse_expression` level: `gcd of b and aa modulo b` keeps
            // `aa modulo b` as the second argument, and `walk of v and n
            // subtract 1` keeps `n subtract 1`. A boolean `and` inside one
            // argument must be braced (`f of {x and y}`), as before. This keeps
            // comparison parsing intact: `'some call' of x and y is false
            // and ...` still reads `f(x, y) is false and ...`.
            let arg = if first_arg {
                first_arg = false;
                self.parse_cast()?
            } else {
                self.parse_expression()?
            };
            args.push(arg);
            self.skip_noise();
            if *self.current() == Token::Comma {
                // Comma belongs to the enclosing sentence.
                break;
            }
            if *self.current() == Token::And {
                self.advance();
                self.skip_noise();
            } else {
                break;
            }
        }
        Ok(Some(Expr::FunctionCall { name, args }))
    }

    /// Parse a primary expression with `to` and/or `of` reserved for an
    /// enclosing statement grammar rather than available as this primary's
    /// own call connector. Use this wherever a value/index/bound is parsed
    /// immediately before code that then checks for a literal `to`/`of` of
    /// its own (a range bound's `to`, an index's `of`) - otherwise a bare
    /// identifier there greedily reads that following word as its call
    /// connector via `parse_call_tail`'s generic lookahead, leaving nothing
    /// for the enclosing check (plan 270 G1 regression). Restores the prior
    /// suppression state unconditionally, including on error, so a caller
    /// higher up the stack that also suppressed a connector is unaffected.
    pub(crate) fn parse_primary_reserving(&mut self, to: bool, of: bool) -> Result<Expr, Box<CompileError>> {
        let saved_to = self.suppress_to_connector;
        let saved_of = self.suppress_of_connector;
        if to {
            self.suppress_to_connector = true;
        }
        if of {
            self.suppress_of_connector = true;
        }
        let result = self.parse_primary();
        self.suppress_to_connector = saved_to;
        self.suppress_of_connector = saved_of;
        result
    }

    pub(crate) fn parse_library_decl(&mut self) -> Result<Statement, Box<CompileError>> {
        // Library 'name' version "1.0".
        // Plan 270 §6: the library *name* is an identifier (bare or quoted);
        // the *version* is a string literal (data, not a name).
        self.advance(); // consume 'library'
        self.skip_noise();

        // Record that this translation unit declares itself a library. A
        // library file has no top-level entry by design, so its last
        // function body legitimately runs to EOF — the BUGS_FOUND #5
        // "function still open at end of file" warning is suppressed for
        // the rest of this parse (see `parse_function_def`).
        self.saw_library_decl = true;

        // Get library name (a bare or quoted identifier, never a string).
        let name = self.parse_name()?;

        self.skip_noise();

        // Parse version — a string literal.
        let version = if *self.current() == Token::Version {
            self.advance();
            self.skip_noise();
            match self.current().clone() {
                Token::StringLiteral(v) => { self.advance(); v }
                _ => return Err(self.err("Expected version string")),
            }
        } else {
            "1.0".to_string() // Default version
        };

        Ok(Statement::LibraryDecl { name, version })
    }

    pub(crate) fn parse_see(&mut self) -> Result<Statement, Box<CompileError>> {
        // Stage A5 retired the abandoned direct-`.so` syntax. The one library
        // import that survives is the canonical form:
        //   see '<lib>' version "<ver>" from "<path>.lib".
        // A bare `see "<path>.vox".` is a source include — spliced in by the
        // frontend before compilation, never part of the library system — and
        // is unchanged here. Every other `see` form is retired: it gets a
        // diagnostic showing the canonical form, not a bare parse error, so a
        // user who wrote a form that used to be documented learns what to
        // write instead.
        self.advance(); // consume 'see'
        self.skip_noise();

        let mut path = String::new();
        let mut lib_name: Option<String> = None;
        let mut lib_version: Option<String> = None;

        // Helper to get string or identifier value
        let get_name_or_string = |token: &Token| -> Option<String> {
            match token {
                Token::StringLiteral(s) => Some(s.clone()),
                Token::Identifier(s) => Some(s.clone()),
                _ => None,
            }
        };

        // Helper to get version (string, identifier, or number)
        let get_version = |token: &Token| -> Option<String> {
            match token {
                Token::StringLiteral(s) => Some(s.clone()),
                Token::Identifier(s) => Some(s.clone()),
                Token::IntegerLiteral(n) => Some(n.to_string()),
                _ => None,
            }
        };

        // First token is the library name (canonical form: a bare/quoted
        // identifier followed by `version`) or the path (a string literal —
        // the `see "<path>.vox"` source include). Plan 270 §S1.5: a string
        // literal where a *name* is expected is rejected with the teaching
        // diagnostic, so the old `see '<lib>' version ...` form now points
        // the user at `see '<lib>' version "..."`. Detect this *before*
        // advancing so the underline lands on the offending string.
        let first_tok = self.current().clone();
        if let Token::StringLiteral(s) = &first_tok {
            // Look ahead past noise (newlines) for `version`.
            let mut k = 1;
            while matches!(self.peek(k), Token::Newline) {
                k += 1;
            }
            if matches!(self.peek(k), Token::Version) {
                return Err(self.err_string_as_name(s));
            }
        }
        let first = get_name_or_string(&first_tok)
            .ok_or_else(|| self.err(
                "Missing path or library name after 'see'\n  \
                 Canonical form: see '<lib>' version \"<x.y>\" from \"<path>.lib\".\n  \
                 (A source include is: see \"<path>.vox\".)"
            ))?;
        self.advance();
        self.skip_noise();

        if *self.current() == Token::Version {
            // see '<lib>' version "<ver>" from "<path>.lib".
            // `first` is the library name (an identifier in canonical form).
            lib_name = Some(first);
            self.advance();
            self.skip_noise();

            lib_version = get_version(self.current());
            if lib_version.is_some() {
                self.advance();
                self.skip_noise();
            }

            if *self.current() == Token::From {
                self.advance();
                self.skip_noise();
                path = get_name_or_string(self.current()).unwrap_or_default();
                if !path.is_empty() {
                    self.advance();
                }
            }
        } else if *self.current() == Token::From || *self.current() == Token::For {
            // Retired `.so`-era forms: `see "<lib>" from "<path>"` (no version)
            // and `see "<path>" for "<lib>" version "<ver>"`. Both used to
            // compile; both now direct the writer to the canonical `.lib`
            // form rather than failing silently. The keyword is named so the
            // message echoes the shape the user actually wrote.
            let form = if *self.current() == Token::From { "from" } else { "for" };
            return Err(self.err(&format!(
                "The `see ... {} ...` form is no longer supported.\n  \
                 Canonical form: see '<lib>' version \"<x.y>\" from \"<path>.lib\".",
                form
            )));
        } else {
            // Simple `see "<path>"` — a .vox source include.
            path = first;
        }

        // A `.so` is a binary. The abandoned model imported it directly, which
        // compiled silently with the library call simply missing — the trap
        // that made the stale documentation hazardous rather than merely
        // untidy. It now errors, directing the user to the `.lib` interface
        // file that is the canonical way to consume a library. This catches a
        // bare `see "x.so"` and a `see 'lib' version "1" from "x.so"` alike.
        if path.ends_with(".so") {
            return Err(self.err(
                "see of a .so is not supported. A .so is a binary; consume it \
                 through its .lib interface file.\n  \
                 Canonical form: see '<lib>' version \"<x.y>\" from \"<path>.lib\"."
            ));
        }

        Ok(Statement::See { path, lib_name, lib_version })
    }

    pub(crate) fn parse_function_def(&mut self) -> Result<Statement, Box<CompileError>> {
        // Location of the `To` keyword, used by the "function still open at
        // end of file" warning (BUGS_FOUND #5) to point at the definition.
        let def_loc = self.current_location();
        self.advance(); // consume 'To'
        self.skip_noise();
        
        // Get function name: a bare or quoted identifier (plan 270). A string
        // literal here is rejected with the §S1.5 diagnostic.
        let name = self.parse_name().or_else(|e| {
            // Distinguish "missing name entirely" from "used a string literal":
            // parse_name already gives the teaching diagnostic for a string;
            // for anything else (e.g. a keyword or `with`) produce the
            // syntax-hint message.
            if matches!(self.current(), Token::StringLiteral(_)) {
                Err(e)
            } else {
                Err(self.err(
                    "Missing function name after 'To'\n  \
                     Syntax: To 'function name' with parameters. Return a type, expression.\n  \
                     Example: To 'add' with a number called x and a number called y. Return a number, x add y."
                ))
            }
        })?;
        
        self.skip_noise();
        
        // Parse parameters: "with <name>" or "with a <type> called <name> and ..."
        let mut params = Vec::new();
        if *self.current() == Token::With || *self.current() == Token::Of {
            self.advance();
            self.skip_noise();
            
            loop {
                self.skip_noise();

                // A string literal is never a parameter name (plan 270 §S1.5).
                if let Token::StringLiteral(s) = self.current().clone() {
                    return Err(self.err_string_as_name(&s));
                }

                // Check for simple parameter: just an identifier
                if let Token::Identifier(n) = self.current().clone() {
                    // Simple parameter without type
                    self.advance();
                    params.push((n, Type::Unknown));
                } else {
                    // Full syntax: "a <type> called <name>"
                    // Skip optional article before type
                    if matches!(self.current(), Token::A | Token::An) {
                        self.advance();
                        self.skip_noise();
                    }
                    
                    let param_type = match self.declaration_type_token() {
                        Some(t) => { self.advance(); t }
                        None => Type::Unknown,
                    };
                    
                    self.skip_noise();
                    if *self.current() == Token::Called {
                        self.advance();
                        self.skip_noise();
                    }
                    
                    let param_name = self.parse_name()?;

                    params.push((param_name, param_type));
                }
                
                self.skip_noise();
                if *self.current() == Token::And {
                    self.advance();
                    self.skip_noise();
                } else {
                    break;
                }
            }
        }
        
        self.skip_noise();
        // Period or comma after function signature are optional.
        if matches!(self.current(), Token::Period | Token::Comma) {
            self.advance();
            self.skip_noise();
        }
        
        // Parse return type: "Return a <type>, <body>"
        let mut return_type = Type::Void;
        let mut body = Vec::new();
        
        if *self.current() == Token::Return {
            self.advance();
            self.skip_noise();
            
            // Check for return type declaration: "Return a number," or "Return number,"
            // Skip optional article
            if matches!(self.current(), Token::A | Token::An) {
                self.advance();
                self.skip_noise();
            }
            
            let mut declared_type = None;
            if let Some(t) = self.declaration_type_token() {
                self.advance();
                return_type = t;
                declared_type = Some(return_type.clone());
                self.skip_noise();
                self.expect(&Token::Comma);
                self.skip_noise();
            }

            // Parse the return expression
            let expr = self.parse_condition()?;
            body.push(Statement::Return { value: Some(expr), declared_type });
        }

        // A top-level Return ends the function body. LANGUAGE.md states
        // that blank lines are optional and have no effect on program
        // execution, so a function whose body ends in `Return ... .` must
        // not keep consuming following sentences when the author omits the
        // separating blank line. Without this, the next top-level
        // statement was silently absorbed into the function body as dead
        // code (emitted after the epilogue `ret`), producing empty or
        // wrong output. Multi-statement bodies that do not end in a
        // top-level Return still terminate at the paragraph break below.
        let body_ended_at_return =
            matches!(body.last(), Some(Statement::Return { .. }));
        if body_ended_at_return {
            self.skip_noise();
            if matches!(self.current(), Token::Period | Token::Comma) {
                self.advance();
                self.skip_noise();
            }
        }

        // Continue parsing body until paragraph break. A function body never
        // contains another function definition or a Library declaration —
        // `Token::To` and `Token::Library` always begin a NEW top-level
        // construct, so they terminate the body just like a paragraph break.
        // Without this, a bodyless function (`To greet.` with no Return and
        // no separating blank line) silently absorbed the following `To f.`
        // as a *nested* FunctionDef: the nested function was still emitted (so
        // it appeared in `nm -D`) but was invisible to any walk of top-level
        // statements — notably the Stage A3 `.lib` signature collector, which
        // then dropped it from the table of contents while the `.so` still
        // exported it. Terminating on `To`/`Library` keeps the successor
        // top-level where it belongs.
        let mut body_ended_early: Option<SourceLocation> = None;
        // Set when the body terminated because a Gate B `Return` (a Return
        // that is not the function's first statement) closed it — distinct
        // from `body_ended_at_return` (inline first-statement Return) and
        // used to suppress the "still open at EOF" warning for a function
        // that legitimately ends in a Return with no trailing blank line.
        let mut ended_via_return = false;
        while !body_ended_at_return
            && !matches!(self.current(), Token::ParagraphBreak | Token::EOF | Token::To | Token::Library)
        {
            self.skip_noise();
            if matches!(self.current(), Token::Comma) {
                self.advance();
                self.skip_noise();
                continue;
            }
            if matches!(self.current(), Token::Period) {
                self.advance();
                self.skip_noise();
            }
            if matches!(self.current(), Token::ParagraphBreak | Token::EOF | Token::To | Token::Library) {
                if matches!(self.current(), Token::ParagraphBreak) {
                    body_ended_early = self.current_location();
                }
                break;
            }
            let stmt = self.parse_statement()?;
            let is_return = matches!(stmt, Statement::Return { .. });
            // Gate B: `Return` isn't the function's first statement, so its
            // type annotation (if any) was parsed by `parse_return` rather
            // than inline above. Feed it back into the function's declared
            // return type the same way the inline path above does, or a
            // `Return a number, ...` that isn't the first statement would
            // silently leave `return_type` at `Type::Void`.
            if let Statement::Return { declared_type: Some(ref t), .. } = stmt {
                return_type = t.clone();
            }
            body.push(stmt);

            // A top-level Return parsed as a body statement terminates the
            // body; consume its trailing period and stop.
            if is_return {
                ended_via_return = true;
                self.skip_noise();
                if matches!(self.current(), Token::Period | Token::Comma) {
                    self.advance();
                    self.skip_noise();
                }
                break;
            }

            self.skip_noise();
            if *self.current() == Token::Comma {
                self.advance();
                self.skip_noise();
            }
        }

        // BUGS_FOUND #5: a function definition whose body ran all the way to
        // end of file — no closing blank line, no Return, no following `To`/
        // `Library` — has no closing blank line, so everything after the
        // signature is read as part of the body. When the author meant the
        // trailing statements as top-level entry code, that code is silently
        // swallowed and the program typically does nothing (exit 0, no
        // output). A blank line is the ONLY thing that closes a function body
        // (LANGUAGE.md "The termination rule" rule 2), so warn the author
        // rather than compiling a do-nothing program.
        //
        // Suppressed when the unit declares itself a `Library` (or is built
        // `--shared`): a library file legitimately consists only of function
        // definitions with no top-level entry, so its last function body
        // ending at EOF is correct by construction, not an absorption.
        //
        // The parser cannot tell, from structure alone, whether the trailing
        // body statements were *intended* as the body (a function that is
        // simply last in the file) or as top-level entry code that got
        // swallowed. The message therefore states only the structural fact
        // (the body reached EOF with no closing blank line) and gives the
        // blank-line fix as *conditional* advice, so it stays truthful in
        // both shapes — it never asserts that statements were absorbed when
        // none were.
        let body_ended_at_eof = !body_ended_at_return
            && !ended_via_return
            && matches!(self.current(), Token::EOF);
        if body_ended_at_eof && !body.is_empty() && !self.shared_mode && !self.saw_library_decl {
            let mut warn = CompileError::new(&format!(
                "Function '{}' is still open at end of file: its body reached \
                 EOF with no closing blank line. A function body is closed by a \
                 blank line (paragraph break), not by EOF, so without one \
                 everything after the signature is read as part of the body. If \
                 statements after the body were meant to run at the top level, \
                 add a blank line after the function body to close it.",
                name
            ));
            if let Some(loc) = def_loc {
                warn = warn.with_location(loc);
            }
            self.warnings.push(warn.as_warning());
        }

        // Consume paragraph break
        if *self.current() == Token::ParagraphBreak {
            self.advance();
        }
        
        Ok(Statement::FunctionDef {
            name,
            params,
            return_type,
            body,
            body_ended_early,
        })
    }

}