stet_core/
binary_token.rs

1// stet - A PostScript Interpreter
2// Copyright (c) 2026 Scott Bowman
3// SPDX-License-Identifier: Apache-2.0 OR MIT
4
5//! Binary token and binary object sequence parser (PLRM §3.14).
6//!
7//! Handles bytes 128-159 in the PostScript input stream. Tags 128-131 are
8//! binary object sequences (compact arrays of typed objects); tags 132-149
9//! are individual binary tokens.
10
11use crate::context::Context;
12use crate::dict::DictKey;
13use crate::error::PsError;
14use crate::file_store::FileStore;
15use crate::object::{EntityId, ObjFlags, PsObject, PsValue};
16
17/// Result of parsing a binary token.
18pub enum BinaryTokenResult {
19    /// Individual token (types 132-149) — a single PsObject.
20    Single(PsObject),
21    /// Binary object sequence (types 128-131) — an executable array.
22    Sequence(PsObject),
23}
24
25// ─── System name table (PLRM Appendix F, indices 0-480) ────────────────────
26
27/// System name table. Indices 226-255 are reserved (None).
28static SYSTEM_NAME_TABLE: [Option<&[u8]>; 481] = [
29    // 0-9
30    Some(b"abs"),
31    Some(b"add"),
32    Some(b"aload"),
33    Some(b"anchorsearch"),
34    Some(b"and"),
35    Some(b"arc"),
36    Some(b"arcn"),
37    Some(b"arct"),
38    Some(b"arcto"),
39    Some(b"array"),
40    // 10-19
41    Some(b"ashow"),
42    Some(b"astore"),
43    Some(b"awidthshow"),
44    Some(b"begin"),
45    Some(b"bind"),
46    Some(b"bitshift"),
47    Some(b"ceiling"),
48    Some(b"charpath"),
49    Some(b"clear"),
50    Some(b"cleartomark"),
51    // 20-29
52    Some(b"clip"),
53    Some(b"clippath"),
54    Some(b"closepath"),
55    Some(b"concat"),
56    Some(b"concatmatrix"),
57    Some(b"copy"),
58    Some(b"count"),
59    Some(b"counttomark"),
60    Some(b"currentcmykcolor"),
61    Some(b"currentdash"),
62    // 30-39
63    Some(b"currentdict"),
64    Some(b"currentfile"),
65    Some(b"currentfont"),
66    Some(b"currentgray"),
67    Some(b"currentgstate"),
68    Some(b"currenthsbcolor"),
69    Some(b"currentlinecap"),
70    Some(b"currentlinejoin"),
71    Some(b"currentlinewidth"),
72    Some(b"currentmatrix"),
73    // 40-49
74    Some(b"currentpoint"),
75    Some(b"currentrgbcolor"),
76    Some(b"currentshared"),
77    Some(b"curveto"),
78    Some(b"cvi"),
79    Some(b"cvlit"),
80    Some(b"cvn"),
81    Some(b"cvr"),
82    Some(b"cvrs"),
83    Some(b"cvs"),
84    // 50-59
85    Some(b"cvx"),
86    Some(b"def"),
87    Some(b"defineusername"),
88    Some(b"dict"),
89    Some(b"div"),
90    Some(b"dtransform"),
91    Some(b"dup"),
92    Some(b"end"),
93    Some(b"eoclip"),
94    Some(b"eofill"),
95    // 60-69
96    Some(b"eoviewclip"),
97    Some(b"eq"),
98    Some(b"exch"),
99    Some(b"exec"),
100    Some(b"exit"),
101    Some(b"file"),
102    Some(b"fill"),
103    Some(b"findfont"),
104    Some(b"flattenpath"),
105    Some(b"floor"),
106    // 70-79
107    Some(b"flush"),
108    Some(b"flushfile"),
109    Some(b"for"),
110    Some(b"forall"),
111    Some(b"ge"),
112    Some(b"get"),
113    Some(b"getinterval"),
114    Some(b"grestore"),
115    Some(b"gsave"),
116    Some(b"gstate"),
117    // 80-89
118    Some(b"gt"),
119    Some(b"identmatrix"),
120    Some(b"idiv"),
121    Some(b"idtransform"),
122    Some(b"if"),
123    Some(b"ifelse"),
124    Some(b"image"),
125    Some(b"imagemask"),
126    Some(b"index"),
127    Some(b"ineofill"),
128    // 90-99
129    Some(b"infill"),
130    Some(b"initviewclip"),
131    Some(b"inueofill"),
132    Some(b"inufill"),
133    Some(b"invertmatrix"),
134    Some(b"itransform"),
135    Some(b"known"),
136    Some(b"le"),
137    Some(b"length"),
138    Some(b"lineto"),
139    // 100-109
140    Some(b"load"),
141    Some(b"loop"),
142    Some(b"lt"),
143    Some(b"makefont"),
144    Some(b"matrix"),
145    Some(b"maxlength"),
146    Some(b"mod"),
147    Some(b"moveto"),
148    Some(b"mul"),
149    Some(b"ne"),
150    // 110-119
151    Some(b"neg"),
152    Some(b"newpath"),
153    Some(b"not"),
154    Some(b"null"),
155    Some(b"or"),
156    Some(b"pathbbox"),
157    Some(b"pathforall"),
158    Some(b"pop"),
159    Some(b"print"),
160    Some(b"printobject"),
161    // 120-129
162    Some(b"put"),
163    Some(b"putinterval"),
164    Some(b"rcurveto"),
165    Some(b"read"),
166    Some(b"readhexstring"),
167    Some(b"readline"),
168    Some(b"readstring"),
169    Some(b"rectclip"),
170    Some(b"rectfill"),
171    Some(b"rectstroke"),
172    // 130-139
173    Some(b"rectviewclip"),
174    Some(b"repeat"),
175    Some(b"restore"),
176    Some(b"rlineto"),
177    Some(b"rmoveto"),
178    Some(b"roll"),
179    Some(b"rotate"),
180    Some(b"round"),
181    Some(b"save"),
182    Some(b"scale"),
183    // 140-149
184    Some(b"scalefont"),
185    Some(b"search"),
186    Some(b"selectfont"),
187    Some(b"setbbox"),
188    Some(b"setcachedevice"),
189    Some(b"setcachedevice2"),
190    Some(b"setcharwidth"),
191    Some(b"setcmykcolor"),
192    Some(b"setdash"),
193    Some(b"setfont"),
194    // 150-159
195    Some(b"setgray"),
196    Some(b"setgstate"),
197    Some(b"sethsbcolor"),
198    Some(b"setlinecap"),
199    Some(b"setlinejoin"),
200    Some(b"setlinewidth"),
201    Some(b"setmatrix"),
202    Some(b"setrgbcolor"),
203    Some(b"setshared"),
204    Some(b"shareddict"),
205    // 160-169
206    Some(b"show"),
207    Some(b"showpage"),
208    Some(b"stop"),
209    Some(b"stopped"),
210    Some(b"store"),
211    Some(b"string"),
212    Some(b"stringwidth"),
213    Some(b"stroke"),
214    Some(b"strokepath"),
215    Some(b"sub"),
216    // 170-179
217    Some(b"systemdict"),
218    Some(b"token"),
219    Some(b"transform"),
220    Some(b"translate"),
221    Some(b"truncate"),
222    Some(b"type"),
223    Some(b"uappend"),
224    Some(b"ucache"),
225    Some(b"ueofill"),
226    Some(b"ufill"),
227    // 180-189
228    Some(b"undef"),
229    Some(b"upath"),
230    Some(b"userdict"),
231    Some(b"ustroke"),
232    Some(b"viewclip"),
233    Some(b"viewclippath"),
234    Some(b"where"),
235    Some(b"widthshow"),
236    Some(b"write"),
237    Some(b"writehexstring"),
238    // 190-199
239    Some(b"writeobject"),
240    Some(b"writestring"),
241    Some(b"wtranslation"),
242    Some(b"xor"),
243    Some(b"xshow"),
244    Some(b"xyshow"),
245    Some(b"yshow"),
246    Some(b"FontDirectory"),
247    Some(b"SharedFontDirectory"),
248    Some(b"Courier"),
249    // 200-209
250    Some(b"Courier-Bold"),
251    Some(b"Courier-BoldOblique"),
252    Some(b"Courier-Oblique"),
253    Some(b"Helvetica"),
254    Some(b"Helvetica-Bold"),
255    Some(b"Helvetica-BoldOblique"),
256    Some(b"Helvetica-Oblique"),
257    Some(b"Symbol"),
258    Some(b"Times-Bold"),
259    Some(b"Times-BoldItalic"),
260    // 210-219
261    Some(b"Times-Italic"),
262    Some(b"Times-Roman"),
263    Some(b"execuserobject"),
264    Some(b"currentcolor"),
265    Some(b"currentcolorspace"),
266    Some(b"currentglobal"),
267    Some(b"execform"),
268    Some(b"filter"),
269    Some(b"findresource"),
270    Some(b"globaldict"),
271    // 220-225
272    Some(b"makepattern"),
273    Some(b"setcolor"),
274    Some(b"setcolorspace"),
275    Some(b"setglobal"),
276    Some(b"setpagedevice"),
277    Some(b"setpattern"),
278    // 226-255: reserved (30 entries)
279    None,
280    None,
281    None,
282    None,
283    None,
284    None,
285    None,
286    None,
287    None,
288    None,
289    None,
290    None,
291    None,
292    None,
293    None,
294    None,
295    None,
296    None,
297    None,
298    None,
299    None,
300    None,
301    None,
302    None,
303    None,
304    None,
305    None,
306    None,
307    None,
308    None,
309    // 256-259
310    Some(b"="),
311    Some(b"=="),
312    Some(b"ISOLatin1Encoding"),
313    Some(b"StandardEncoding"),
314    // 260-269
315    Some(b"["),
316    Some(b"]"),
317    Some(b"atan"),
318    Some(b"banddevice"),
319    Some(b"bytesavailable"),
320    Some(b"cachestatus"),
321    Some(b"closefile"),
322    Some(b"colorimage"),
323    Some(b"condition"),
324    Some(b"copypage"),
325    // 270-279
326    Some(b"cos"),
327    Some(b"countdictstack"),
328    Some(b"countexecstack"),
329    Some(b"cshow"),
330    Some(b"currentblackgeneration"),
331    Some(b"currentcacheparams"),
332    Some(b"currentcolorscreen"),
333    Some(b"currentcolortransfer"),
334    Some(b"currentcontext"),
335    Some(b"currentflat"),
336    // 280-289
337    Some(b"currenthalftone"),
338    Some(b"currenthalftonephase"),
339    Some(b"currentmiterlimit"),
340    Some(b"currentobjectformat"),
341    Some(b"currentpacking"),
342    Some(b"currentscreen"),
343    Some(b"currentstrokeadjust"),
344    Some(b"currenttransfer"),
345    Some(b"currentundercolorremoval"),
346    Some(b"defaultmatrix"),
347    // 290-299
348    Some(b"definefont"),
349    Some(b"deletefile"),
350    Some(b"detach"),
351    Some(b"deviceinfo"),
352    Some(b"dictstack"),
353    Some(b"echo"),
354    Some(b"erasepage"),
355    Some(b"errordict"),
356    Some(b"execstack"),
357    Some(b"executeonly"),
358    // 300-309
359    Some(b"exp"),
360    Some(b"false"),
361    Some(b"filenameforall"),
362    Some(b"fileposition"),
363    Some(b"fork"),
364    Some(b"framedevice"),
365    Some(b"grestoreall"),
366    Some(b"handleerror"),
367    Some(b"initclip"),
368    Some(b"initgraphics"),
369    // 310-319
370    Some(b"initmatrix"),
371    Some(b"instroke"),
372    Some(b"inustroke"),
373    Some(b"join"),
374    Some(b"kshow"),
375    Some(b"ln"),
376    Some(b"lock"),
377    Some(b"log"),
378    Some(b"mark"),
379    Some(b"monitor"),
380    // 320-329
381    Some(b"noaccess"),
382    Some(b"notify"),
383    Some(b"nulldevice"),
384    Some(b"packedarray"),
385    Some(b"quit"),
386    Some(b"rand"),
387    Some(b"rcheck"),
388    Some(b"readonly"),
389    Some(b"realtime"),
390    Some(b"renamefile"),
391    // 330-339
392    Some(b"renderbands"),
393    Some(b"resetfile"),
394    Some(b"reversepath"),
395    Some(b"rootfont"),
396    Some(b"rrand"),
397    Some(b"run"),
398    Some(b"scheck"),
399    Some(b"setblackgeneration"),
400    Some(b"setcachelimit"),
401    Some(b"setcacheparams"),
402    // 340-349
403    Some(b"setcolorscreen"),
404    Some(b"setcolortransfer"),
405    Some(b"setfileposition"),
406    Some(b"setflat"),
407    Some(b"sethalftone"),
408    Some(b"sethalftonephase"),
409    Some(b"setmiterlimit"),
410    Some(b"setobjectformat"),
411    Some(b"setpacking"),
412    Some(b"setscreen"),
413    // 350-359
414    Some(b"setstrokeadjust"),
415    Some(b"settransfer"),
416    Some(b"setucacheparams"),
417    Some(b"setundercolorremoval"),
418    Some(b"sin"),
419    Some(b"sqrt"),
420    Some(b"srand"),
421    Some(b"stack"),
422    Some(b"status"),
423    Some(b"statusdict"),
424    // 360-369
425    Some(b"true"),
426    Some(b"ucachestatus"),
427    Some(b"undefinefont"),
428    Some(b"usertime"),
429    Some(b"ustrokepath"),
430    Some(b"version"),
431    Some(b"vmreclaim"),
432    Some(b"vmstatus"),
433    Some(b"wait"),
434    Some(b"wcheck"),
435    // 370-379
436    Some(b"xcheck"),
437    Some(b"yield"),
438    Some(b"defineuserobject"),
439    Some(b"undefineuserobject"),
440    Some(b"UserObjects"),
441    Some(b"cleardictstack"),
442    Some(b"A"),
443    Some(b"B"),
444    Some(b"C"),
445    Some(b"D"),
446    // 380-389
447    Some(b"E"),
448    Some(b"F"),
449    Some(b"G"),
450    Some(b"H"),
451    Some(b"I"),
452    Some(b"J"),
453    Some(b"K"),
454    Some(b"L"),
455    Some(b"M"),
456    Some(b"N"),
457    // 390-399
458    Some(b"O"),
459    Some(b"P"),
460    Some(b"Q"),
461    Some(b"R"),
462    Some(b"S"),
463    Some(b"T"),
464    Some(b"U"),
465    Some(b"V"),
466    Some(b"W"),
467    Some(b"X"),
468    // 400-409
469    Some(b"Y"),
470    Some(b"Z"),
471    Some(b"a"),
472    Some(b"b"),
473    Some(b"c"),
474    Some(b"d"),
475    Some(b"e"),
476    Some(b"f"),
477    Some(b"g"),
478    Some(b"h"),
479    // 410-419
480    Some(b"i"),
481    Some(b"j"),
482    Some(b"k"),
483    Some(b"l"),
484    Some(b"m"),
485    Some(b"n"),
486    Some(b"o"),
487    Some(b"p"),
488    Some(b"q"),
489    Some(b"r"),
490    // 420-429
491    Some(b"s"),
492    Some(b"t"),
493    Some(b"u"),
494    Some(b"v"),
495    Some(b"w"),
496    Some(b"x"),
497    Some(b"y"),
498    Some(b"z"),
499    Some(b"setvmthreshold"),
500    Some(b"<<"),
501    // 430-439
502    Some(b">>"),
503    Some(b"currentcolorrendering"),
504    Some(b"currentdevparams"),
505    Some(b"currentoverprint"),
506    Some(b"currentpagedevice"),
507    Some(b"currentsystemparams"),
508    Some(b"currentuserparams"),
509    Some(b"defineresource"),
510    Some(b"findencoding"),
511    Some(b"gcheck"),
512    // 440-449
513    Some(b"glyphshow"),
514    Some(b"languagelevel"),
515    Some(b"product"),
516    Some(b"pstack"),
517    Some(b"resourceforall"),
518    Some(b"resourcestatus"),
519    Some(b"revision"),
520    Some(b"serialnumber"),
521    Some(b"setcolorrendering"),
522    Some(b"setdevparams"),
523    // 450-459
524    Some(b"setoverprint"),
525    Some(b"setsystemparams"),
526    Some(b"setuserparams"),
527    Some(b"startjob"),
528    Some(b"undefineresource"),
529    Some(b"GlobalFontDirectory"),
530    Some(b"ASCII85Decode"),
531    Some(b"ASCII85Encode"),
532    Some(b"ASCIIHexDecode"),
533    Some(b"ASCIIHexEncode"),
534    // 460-469
535    Some(b"CCITTFaxDecode"),
536    Some(b"CCITTFaxEncode"),
537    Some(b"DCTDecode"),
538    Some(b"DCTEncode"),
539    Some(b"LZWDecode"),
540    Some(b"LZWEncode"),
541    Some(b"NullEncode"),
542    Some(b"RunLengthDecode"),
543    Some(b"RunLengthEncode"),
544    Some(b"SubFileDecode"),
545    // 470-479
546    Some(b"CIEBasedA"),
547    Some(b"CIEBasedABC"),
548    Some(b"DeviceCMYK"),
549    Some(b"DeviceGray"),
550    Some(b"DeviceRGB"),
551    Some(b"Indexed"),
552    Some(b"Pattern"),
553    Some(b"Separation"),
554    Some(b"CIEBasedDEF"),
555    Some(b"CIEBasedDEFG"),
556    // 480
557    Some(b"DeviceN"),
558];
559
560// ─── Slice-based parsing (for StringSource fast path) ───────────────────────
561
562/// Parse a binary token from a byte slice. Returns the result and bytes consumed
563/// (not counting the initial tag byte, which was already consumed by the tokenizer).
564pub fn parse_from_slice(
565    ctx: &mut Context,
566    tag: u8,
567    data: &[u8],
568) -> Result<(BinaryTokenResult, usize), PsError> {
569    match tag {
570        128..=131 => parse_bos_from_slice(ctx, tag, data),
571        132..=136 => parse_int_from_slice(ctx, tag, data),
572        137 => parse_fixed_from_slice(ctx, data),
573        138..=140 => parse_real_from_slice(tag, data),
574        141 => parse_bool_from_slice(data),
575        142..=144 => parse_string_from_slice(ctx, tag, data),
576        145..=146 => parse_system_name_from_slice(ctx, tag, data),
577        149 => parse_number_array_from_slice(ctx, data),
578        147 | 148 | 150..=159 => Err(PsError::SyntaxError),
579        _ => Err(PsError::SyntaxError),
580    }
581}
582
583/// Parse a binary token from a file stream. The tag byte has already been read.
584pub fn parse_from_stream(
585    ctx: &mut Context,
586    tag: u8,
587    file_entity: EntityId,
588) -> Result<BinaryTokenResult, PsError> {
589    match tag {
590        128..=131 => parse_bos_from_stream(ctx, tag, file_entity),
591        132..=136 => parse_int_from_stream(ctx, tag, file_entity),
592        137 => parse_fixed_from_stream(ctx, file_entity),
593        138..=140 => parse_real_from_stream(tag, &mut ctx.files, file_entity),
594        141 => parse_bool_from_stream(&mut ctx.files, file_entity),
595        142..=144 => parse_string_from_stream(ctx, tag, file_entity),
596        145..=146 => parse_system_name_from_stream(ctx, tag, file_entity),
597        149 => parse_number_array_from_stream(ctx, file_entity),
598        147 | 148 | 150..=159 => Err(PsError::SyntaxError),
599        _ => Err(PsError::SyntaxError),
600    }
601}
602
603// ─── Helper: require N bytes from slice ─────────────────────────────────────
604
605fn need(data: &[u8], n: usize) -> Result<(), PsError> {
606    if data.len() < n {
607        Err(PsError::SyntaxError)
608    } else {
609        Ok(())
610    }
611}
612
613fn read_n_bytes(files: &mut FileStore, entity: EntityId, n: usize) -> Result<Vec<u8>, PsError> {
614    files
615        .read_n_bytes(entity, n)
616        .map_err(|_| PsError::SyntaxError)
617}
618
619// ─── Integer parsers (132-136) ──────────────────────────────────────────────
620
621fn parse_int_from_slice(
622    _ctx: &mut Context,
623    tag: u8,
624    data: &[u8],
625) -> Result<(BinaryTokenResult, usize), PsError> {
626    match tag {
627        132 => {
628            need(data, 4)?;
629            let v = i32::from_be_bytes([data[0], data[1], data[2], data[3]]);
630            Ok((BinaryTokenResult::Single(PsObject::int(v)), 4))
631        }
632        133 => {
633            need(data, 4)?;
634            let v = i32::from_le_bytes([data[0], data[1], data[2], data[3]]);
635            Ok((BinaryTokenResult::Single(PsObject::int(v)), 4))
636        }
637        134 => {
638            need(data, 2)?;
639            let v = i16::from_be_bytes([data[0], data[1]]) as i32;
640            Ok((BinaryTokenResult::Single(PsObject::int(v)), 2))
641        }
642        135 => {
643            need(data, 2)?;
644            let v = i16::from_le_bytes([data[0], data[1]]) as i32;
645            Ok((BinaryTokenResult::Single(PsObject::int(v)), 2))
646        }
647        136 => {
648            need(data, 1)?;
649            let v = data[0] as i8 as i32;
650            Ok((BinaryTokenResult::Single(PsObject::int(v)), 1))
651        }
652        _ => unreachable!(),
653    }
654}
655
656fn parse_int_from_stream(
657    ctx: &mut Context,
658    tag: u8,
659    file_entity: EntityId,
660) -> Result<BinaryTokenResult, PsError> {
661    match tag {
662        132 => {
663            let b = read_n_bytes(&mut ctx.files, file_entity, 4)?;
664            let v = i32::from_be_bytes([b[0], b[1], b[2], b[3]]);
665            Ok(BinaryTokenResult::Single(PsObject::int(v)))
666        }
667        133 => {
668            let b = read_n_bytes(&mut ctx.files, file_entity, 4)?;
669            let v = i32::from_le_bytes([b[0], b[1], b[2], b[3]]);
670            Ok(BinaryTokenResult::Single(PsObject::int(v)))
671        }
672        134 => {
673            let b = read_n_bytes(&mut ctx.files, file_entity, 2)?;
674            let v = i16::from_be_bytes([b[0], b[1]]) as i32;
675            Ok(BinaryTokenResult::Single(PsObject::int(v)))
676        }
677        135 => {
678            let b = read_n_bytes(&mut ctx.files, file_entity, 2)?;
679            let v = i16::from_le_bytes([b[0], b[1]]) as i32;
680            Ok(BinaryTokenResult::Single(PsObject::int(v)))
681        }
682        136 => {
683            let b = read_n_bytes(&mut ctx.files, file_entity, 1)?;
684            let v = b[0] as i8 as i32;
685            Ok(BinaryTokenResult::Single(PsObject::int(v)))
686        }
687        _ => unreachable!(),
688    }
689}
690
691// ─── Fixed-point parser (137) ───────────────────────────────────────────────
692
693fn parse_fixed_from_slice(
694    _ctx: &mut Context,
695    data: &[u8],
696) -> Result<(BinaryTokenResult, usize), PsError> {
697    need(data, 1)?;
698    let r = data[0];
699    let (nbytes, raw, scale) = decode_fixed_repr(r, &data[1..])?;
700    let obj = if scale == 0 {
701        PsObject::int(raw)
702    } else {
703        PsObject::real(raw as f64 / (1u64 << scale) as f64)
704    };
705    Ok((BinaryTokenResult::Single(obj), 1 + nbytes))
706}
707
708fn parse_fixed_from_stream(
709    ctx: &mut Context,
710    file_entity: EntityId,
711) -> Result<BinaryTokenResult, PsError> {
712    let rb = read_n_bytes(&mut ctx.files, file_entity, 1)?;
713    let r = rb[0];
714    let (nbytes, _) = fixed_repr_info(r)?;
715    let b = read_n_bytes(&mut ctx.files, file_entity, nbytes)?;
716    let (_, raw, scale) = decode_fixed_repr(r, &b)?;
717    let obj = if scale == 0 {
718        PsObject::int(raw)
719    } else {
720        PsObject::real(raw as f64 / (1u64 << scale) as f64)
721    };
722    Ok(BinaryTokenResult::Single(obj))
723}
724
725/// Returns (nbytes, raw_value, scale) from repr byte + data.
726fn decode_fixed_repr(r: u8, data: &[u8]) -> Result<(usize, i32, u8), PsError> {
727    let (nbytes, scale) = fixed_repr_info(r)?;
728    need(data, nbytes)?;
729    let raw = match (nbytes, r < 128) {
730        (4, true) => i32::from_be_bytes([data[0], data[1], data[2], data[3]]),
731        (4, false) => i32::from_le_bytes([data[0], data[1], data[2], data[3]]),
732        (2, true) => i16::from_be_bytes([data[0], data[1]]) as i32,
733        (2, false) => i16::from_le_bytes([data[0], data[1]]) as i32,
734        _ => return Err(PsError::SyntaxError),
735    };
736    Ok((nbytes, raw, scale))
737}
738
739/// Returns (nbytes, scale) for a fixed-point repr byte.
740fn fixed_repr_info(r: u8) -> Result<(usize, u8), PsError> {
741    match r {
742        0..=31 => Ok((4, r)),
743        32..=47 => Ok((2, r - 32)),
744        128..=159 => Ok((4, r - 128)),
745        160..=175 => Ok((2, r - 160)),
746        _ => Err(PsError::SyntaxError),
747    }
748}
749
750// ─── Real parsers (138-140) ─────────────────────────────────────────────────
751
752fn parse_real_from_slice(tag: u8, data: &[u8]) -> Result<(BinaryTokenResult, usize), PsError> {
753    need(data, 4)?;
754    let v = match tag {
755        138 => f32::from_be_bytes([data[0], data[1], data[2], data[3]]),
756        _ => f32::from_le_bytes([data[0], data[1], data[2], data[3]]),
757    };
758    Ok((BinaryTokenResult::Single(PsObject::real(v as f64)), 4))
759}
760
761fn parse_real_from_stream(
762    tag: u8,
763    files: &mut FileStore,
764    file_entity: EntityId,
765) -> Result<BinaryTokenResult, PsError> {
766    let b = files
767        .read_n_bytes(file_entity, 4)
768        .map_err(|_| PsError::SyntaxError)?;
769    let v = match tag {
770        138 => f32::from_be_bytes([b[0], b[1], b[2], b[3]]),
771        _ => f32::from_le_bytes([b[0], b[1], b[2], b[3]]),
772    };
773    Ok(BinaryTokenResult::Single(PsObject::real(v as f64)))
774}
775
776// ─── Boolean parser (141) ───────────────────────────────────────────────────
777
778fn parse_bool_from_slice(data: &[u8]) -> Result<(BinaryTokenResult, usize), PsError> {
779    need(data, 1)?;
780    Ok((BinaryTokenResult::Single(PsObject::bool(data[0] != 0)), 1))
781}
782
783fn parse_bool_from_stream(
784    files: &mut FileStore,
785    file_entity: EntityId,
786) -> Result<BinaryTokenResult, PsError> {
787    let b = files
788        .read_n_bytes(file_entity, 1)
789        .map_err(|_| PsError::SyntaxError)?;
790    Ok(BinaryTokenResult::Single(PsObject::bool(b[0] != 0)))
791}
792
793// ─── String parsers (142-144) ───────────────────────────────────────────────
794
795fn parse_string_from_slice(
796    ctx: &mut Context,
797    tag: u8,
798    data: &[u8],
799) -> Result<(BinaryTokenResult, usize), PsError> {
800    let (str_len, hdr_size) = match tag {
801        142 => {
802            need(data, 1)?;
803            (data[0] as usize, 1)
804        }
805        143 => {
806            need(data, 2)?;
807            (u16::from_be_bytes([data[0], data[1]]) as usize, 2)
808        }
809        144 => {
810            need(data, 2)?;
811            (u16::from_le_bytes([data[0], data[1]]) as usize, 2)
812        }
813        _ => unreachable!(),
814    };
815    need(data, hdr_size + str_len)?;
816    let str_data = &data[hdr_size..hdr_size + str_len];
817    let obj = alloc_string(ctx, str_data);
818    Ok((BinaryTokenResult::Single(obj), hdr_size + str_len))
819}
820
821fn parse_string_from_stream(
822    ctx: &mut Context,
823    tag: u8,
824    file_entity: EntityId,
825) -> Result<BinaryTokenResult, PsError> {
826    let str_len = match tag {
827        142 => {
828            let b = read_n_bytes(&mut ctx.files, file_entity, 1)?;
829            b[0] as usize
830        }
831        143 => {
832            let b = read_n_bytes(&mut ctx.files, file_entity, 2)?;
833            u16::from_be_bytes([b[0], b[1]]) as usize
834        }
835        144 => {
836            let b = read_n_bytes(&mut ctx.files, file_entity, 2)?;
837            u16::from_le_bytes([b[0], b[1]]) as usize
838        }
839        _ => unreachable!(),
840    };
841    let str_data = read_n_bytes(&mut ctx.files, file_entity, str_len)?;
842    let obj = alloc_string(ctx, &str_data);
843    Ok(BinaryTokenResult::Single(obj))
844}
845
846fn alloc_string(ctx: &mut Context, bytes: &[u8]) -> PsObject {
847    let save_level = ctx.save_stack.current_level();
848    let global = ctx.vm_alloc_mode;
849    let created = ctx.save_stack.last_save_id();
850    let entity = ctx
851        .strings
852        .allocate_with(bytes.len(), save_level, global, created);
853    ctx.strings
854        .get_mut(entity, 0, bytes.len() as u32)
855        .copy_from_slice(bytes);
856    let mut obj = PsObject::string(entity, bytes.len() as u32);
857    if global {
858        obj.flags = ObjFlags::new(ObjFlags::ACCESS_UNLIMITED, false, true, true);
859    }
860    obj
861}
862
863// ─── System name parsers (145-146) ──────────────────────────────────────────
864
865fn parse_system_name_from_slice(
866    ctx: &mut Context,
867    tag: u8,
868    data: &[u8],
869) -> Result<(BinaryTokenResult, usize), PsError> {
870    need(data, 1)?;
871    let idx = data[0] as usize;
872    let obj = resolve_system_name(ctx, idx, tag == 146)?;
873    Ok((BinaryTokenResult::Single(obj), 1))
874}
875
876fn parse_system_name_from_stream(
877    ctx: &mut Context,
878    tag: u8,
879    file_entity: EntityId,
880) -> Result<BinaryTokenResult, PsError> {
881    let b = read_n_bytes(&mut ctx.files, file_entity, 1)?;
882    let idx = b[0] as usize;
883    let obj = resolve_system_name(ctx, idx, tag == 146)?;
884    Ok(BinaryTokenResult::Single(obj))
885}
886
887fn resolve_system_name(
888    ctx: &mut Context,
889    idx: usize,
890    executable: bool,
891) -> Result<PsObject, PsError> {
892    if idx >= SYSTEM_NAME_TABLE.len() {
893        return Err(PsError::Undefined);
894    }
895    let name_bytes = SYSTEM_NAME_TABLE[idx].ok_or(PsError::Undefined)?;
896    let name_id = ctx.names.intern(name_bytes);
897    if executable {
898        Ok(PsObject::name_exec(name_id))
899    } else {
900        Ok(PsObject::name_lit(name_id))
901    }
902}
903
904// ─── Homogeneous number array parser (149) ──────────────────────────────────
905
906fn parse_number_array_from_slice(
907    ctx: &mut Context,
908    data: &[u8],
909) -> Result<(BinaryTokenResult, usize), PsError> {
910    need(data, 3)?; // repr + 2-byte count
911    let r = data[0];
912    let (elem_size, is_real, scale, big_endian) = number_array_repr(r)?;
913    let count = if big_endian {
914        u16::from_be_bytes([data[1], data[2]]) as usize
915    } else {
916        u16::from_le_bytes([data[1], data[2]]) as usize
917    };
918    let body_start = 3;
919    let body_len = count * elem_size;
920    need(data, body_start + body_len)?;
921    let elements = decode_number_elements(
922        &data[body_start..],
923        count,
924        elem_size,
925        is_real,
926        scale,
927        big_endian,
928    );
929    let obj = alloc_array(ctx, &elements);
930    Ok((BinaryTokenResult::Single(obj), body_start + body_len))
931}
932
933fn parse_number_array_from_stream(
934    ctx: &mut Context,
935    file_entity: EntityId,
936) -> Result<BinaryTokenResult, PsError> {
937    let rb = read_n_bytes(&mut ctx.files, file_entity, 1)?;
938    let r = rb[0];
939    let (elem_size, is_real, scale, big_endian) = number_array_repr(r)?;
940    let count_bytes = read_n_bytes(&mut ctx.files, file_entity, 2)?;
941    let count = if big_endian {
942        u16::from_be_bytes([count_bytes[0], count_bytes[1]]) as usize
943    } else {
944        u16::from_le_bytes([count_bytes[0], count_bytes[1]]) as usize
945    };
946    let body = read_n_bytes(&mut ctx.files, file_entity, count * elem_size)?;
947    let elements = decode_number_elements(&body, count, elem_size, is_real, scale, big_endian);
948    let obj = alloc_array(ctx, &elements);
949    Ok(BinaryTokenResult::Single(obj))
950}
951
952/// Returns (elem_size, is_real, scale, big_endian) for a number array repr byte.
953fn number_array_repr(r: u8) -> Result<(usize, bool, u8, bool), PsError> {
954    match r {
955        0..=31 => Ok((4, false, r, true)),
956        32..=47 => Ok((2, false, r - 32, true)),
957        48 => Ok((4, true, 0, true)),
958        49 => Ok((4, true, 0, true)), // native real — treat as LE on x86 but length/count is BE
959        128..=159 => Ok((4, false, r - 128, false)),
960        160..=175 => Ok((2, false, r - 160, false)),
961        176 => Ok((4, true, 0, false)),
962        177 => Ok((4, true, 0, false)), // native real LE
963        _ => Err(PsError::SyntaxError),
964    }
965}
966
967fn decode_number_elements(
968    data: &[u8],
969    count: usize,
970    elem_size: usize,
971    is_real: bool,
972    scale: u8,
973    big_endian: bool,
974) -> Vec<PsObject> {
975    let mut elements = Vec::with_capacity(count);
976    for i in 0..count {
977        let off = i * elem_size;
978        if is_real {
979            let v = if big_endian {
980                f32::from_be_bytes([data[off], data[off + 1], data[off + 2], data[off + 3]])
981            } else {
982                f32::from_le_bytes([data[off], data[off + 1], data[off + 2], data[off + 3]])
983            };
984            elements.push(PsObject::real(v as f64));
985        } else if elem_size == 4 {
986            let raw = if big_endian {
987                i32::from_be_bytes([data[off], data[off + 1], data[off + 2], data[off + 3]])
988            } else {
989                i32::from_le_bytes([data[off], data[off + 1], data[off + 2], data[off + 3]])
990            };
991            if scale == 0 {
992                elements.push(PsObject::int(raw));
993            } else {
994                elements.push(PsObject::real(raw as f64 / (1u64 << scale) as f64));
995            }
996        } else {
997            // 2-byte
998            let raw = if big_endian {
999                i16::from_be_bytes([data[off], data[off + 1]]) as i32
1000            } else {
1001                i16::from_le_bytes([data[off], data[off + 1]]) as i32
1002            };
1003            if scale == 0 {
1004                elements.push(PsObject::int(raw));
1005            } else {
1006                elements.push(PsObject::real(raw as f64 / (1u64 << scale) as f64));
1007            }
1008        }
1009    }
1010    elements
1011}
1012
1013fn alloc_array(ctx: &mut Context, elements: &[PsObject]) -> PsObject {
1014    let save_level = ctx.save_stack.current_level();
1015    let global = ctx.vm_alloc_mode;
1016    let created = ctx.save_stack.last_save_id();
1017    let entity = ctx
1018        .arrays
1019        .allocate_with(elements.len(), save_level, global, created);
1020    let dest = ctx.arrays.get_mut(entity, 0, elements.len() as u32);
1021    dest.copy_from_slice(elements);
1022    let mut obj = PsObject {
1023        value: PsValue::Array {
1024            entity,
1025            start: 0,
1026            len: elements.len() as u32,
1027        },
1028        flags: ObjFlags::literal_composite(),
1029    };
1030    if global {
1031        obj.flags = ObjFlags::new(ObjFlags::ACCESS_UNLIMITED, false, true, true);
1032    }
1033    obj
1034}
1035
1036// ─── Binary object sequence parsers (128-131) ───────────────────────────────
1037
1038const MAX_BOS_DEPTH: u32 = 100;
1039
1040fn parse_bos_from_slice(
1041    ctx: &mut Context,
1042    tag: u8,
1043    data: &[u8],
1044) -> Result<(BinaryTokenResult, usize), PsError> {
1045    let big_endian = tag == 128 || tag == 130;
1046
1047    // Read header (first 3 bytes: byte1 + 2-byte count)
1048    need(data, 3)?;
1049    let byte1 = data[0];
1050    let top_level_count = if big_endian {
1051        u16::from_be_bytes([data[1], data[2]]) as usize
1052    } else {
1053        u16::from_le_bytes([data[1], data[2]]) as usize
1054    };
1055
1056    let (overall_length, header_size) = if byte1 > 0 {
1057        (byte1 as usize, 4usize)
1058    } else {
1059        need(data, 7)?;
1060        let ext = if big_endian {
1061            u32::from_be_bytes([data[3], data[4], data[5], data[6]]) as usize
1062        } else {
1063            u32::from_le_bytes([data[3], data[4], data[5], data[6]]) as usize
1064        };
1065        (ext, 8usize)
1066    };
1067
1068    // Note: overall_length includes the tag byte (already consumed) + rest of header + body.
1069    // The data slice starts AFTER the tag byte. So body starts at (header_size - 1) from
1070    // the start of data, and total data consumed = overall_length - 1.
1071    let data_size = overall_length
1072        .checked_sub(header_size)
1073        .ok_or(PsError::SyntaxError)?;
1074    let body_offset = header_size - 1; // -1 because tag byte not in data
1075    need(data, body_offset + data_size)?;
1076    let body = &data[body_offset..body_offset + data_size];
1077
1078    let obj = parse_bos_body(ctx, body, data_size, top_level_count, big_endian)?;
1079    Ok((BinaryTokenResult::Sequence(obj), body_offset + data_size))
1080}
1081
1082fn parse_bos_from_stream(
1083    ctx: &mut Context,
1084    tag: u8,
1085    file_entity: EntityId,
1086) -> Result<BinaryTokenResult, PsError> {
1087    let big_endian = tag == 128 || tag == 130;
1088
1089    // Read header bytes after the tag
1090    let hdr = read_n_bytes(&mut ctx.files, file_entity, 3)?;
1091    let byte1 = hdr[0];
1092    let top_level_count = if big_endian {
1093        u16::from_be_bytes([hdr[1], hdr[2]]) as usize
1094    } else {
1095        u16::from_le_bytes([hdr[1], hdr[2]]) as usize
1096    };
1097
1098    let (overall_length, header_size) = if byte1 > 0 {
1099        (byte1 as usize, 4usize)
1100    } else {
1101        let ext_bytes = read_n_bytes(&mut ctx.files, file_entity, 4)?;
1102        let ext = if big_endian {
1103            u32::from_be_bytes([ext_bytes[0], ext_bytes[1], ext_bytes[2], ext_bytes[3]]) as usize
1104        } else {
1105            u32::from_le_bytes([ext_bytes[0], ext_bytes[1], ext_bytes[2], ext_bytes[3]]) as usize
1106        };
1107        (ext, 8usize)
1108    };
1109
1110    let data_size = overall_length
1111        .checked_sub(header_size)
1112        .ok_or(PsError::SyntaxError)?;
1113    let body = read_n_bytes(&mut ctx.files, file_entity, data_size)?;
1114
1115    let obj = parse_bos_body(ctx, &body, data_size, top_level_count, big_endian)?;
1116    Ok(BinaryTokenResult::Sequence(obj))
1117}
1118
1119/// Parse BOS body (shared by slice and stream paths).
1120fn parse_bos_body(
1121    ctx: &mut Context,
1122    data: &[u8],
1123    data_size: usize,
1124    top_level_count: usize,
1125    big_endian: bool,
1126) -> Result<PsObject, PsError> {
1127    if data_size < top_level_count * 8 {
1128        return Err(PsError::SyntaxError);
1129    }
1130
1131    let mut results = Vec::with_capacity(top_level_count);
1132    for i in 0..top_level_count {
1133        let obj = build_bos_object(ctx, data, data_size, i * 8, big_endian, 0)?;
1134        results.push(obj);
1135    }
1136
1137    // Wrap in executable array
1138    let save_level = ctx.save_stack.current_level();
1139    let global = ctx.vm_alloc_mode;
1140    let created = ctx.save_stack.last_save_id();
1141    let entity = ctx
1142        .arrays
1143        .allocate_with(results.len(), save_level, global, created);
1144    let dest = ctx.arrays.get_mut(entity, 0, results.len() as u32);
1145    dest.copy_from_slice(&results);
1146
1147    let mut obj = PsObject::procedure(entity, results.len() as u32);
1148    if global {
1149        obj.flags = ObjFlags::new(ObjFlags::ACCESS_UNLIMITED, true, true, true);
1150    }
1151    Ok(obj)
1152}
1153
1154/// Recursively build a PsObject from an 8-byte BOS entry.
1155fn build_bos_object(
1156    ctx: &mut Context,
1157    data: &[u8],
1158    data_size: usize,
1159    pos: usize,
1160    big_endian: bool,
1161    depth: u32,
1162) -> Result<PsObject, PsError> {
1163    if depth > MAX_BOS_DEPTH {
1164        return Err(PsError::SyntaxError);
1165    }
1166    if pos + 8 > data_size {
1167        return Err(PsError::SyntaxError);
1168    }
1169
1170    let type_byte = data[pos];
1171    let type_code = type_byte & 0x7F;
1172    let is_exec = type_byte & 0x80 != 0;
1173    let length_u16 = if big_endian {
1174        u16::from_be_bytes([data[pos + 2], data[pos + 3]])
1175    } else {
1176        u16::from_le_bytes([data[pos + 2], data[pos + 3]])
1177    };
1178    let value_u32 = if big_endian {
1179        u32::from_be_bytes([data[pos + 4], data[pos + 5], data[pos + 6], data[pos + 7]])
1180    } else {
1181        u32::from_le_bytes([data[pos + 4], data[pos + 5], data[pos + 6], data[pos + 7]])
1182    };
1183
1184    match type_code {
1185        // null
1186        0 => Ok(PsObject::null()),
1187
1188        // integer
1189        1 => {
1190            let v = if big_endian {
1191                i32::from_be_bytes([data[pos + 4], data[pos + 5], data[pos + 6], data[pos + 7]])
1192            } else {
1193                i32::from_le_bytes([data[pos + 4], data[pos + 5], data[pos + 6], data[pos + 7]])
1194            };
1195            Ok(PsObject::int(v))
1196        }
1197
1198        // real
1199        2 => {
1200            let v = if length_u16 == 0 {
1201                // IEEE float
1202                (if big_endian {
1203                    f32::from_be_bytes([data[pos + 4], data[pos + 5], data[pos + 6], data[pos + 7]])
1204                } else {
1205                    f32::from_le_bytes([data[pos + 4], data[pos + 5], data[pos + 6], data[pos + 7]])
1206                }) as f64
1207            } else {
1208                // Fixed-point: raw / 2^length
1209                let raw = if big_endian {
1210                    i32::from_be_bytes([data[pos + 4], data[pos + 5], data[pos + 6], data[pos + 7]])
1211                } else {
1212                    i32::from_le_bytes([data[pos + 4], data[pos + 5], data[pos + 6], data[pos + 7]])
1213                };
1214                raw as f64 / (1u64 << length_u16) as f64
1215            };
1216            Ok(PsObject::real(v))
1217        }
1218
1219        // name (3) / immediately evaluated name (6)
1220        3 | 6 => {
1221            let signed_len = if big_endian {
1222                i16::from_be_bytes([data[pos + 2], data[pos + 3]])
1223            } else {
1224                i16::from_le_bytes([data[pos + 2], data[pos + 3]])
1225            };
1226            let name_bytes = if signed_len == -1 {
1227                // System name table lookup
1228                let idx = value_u32 as usize;
1229                if idx >= SYSTEM_NAME_TABLE.len() {
1230                    return Err(PsError::SyntaxError);
1231                }
1232                SYSTEM_NAME_TABLE[idx].ok_or(PsError::SyntaxError)?
1233            } else if signed_len > 0 {
1234                let offset = value_u32 as usize;
1235                let len = signed_len as usize;
1236                if offset + len > data_size {
1237                    return Err(PsError::SyntaxError);
1238                }
1239                &data[offset..offset + len]
1240            } else {
1241                return Err(PsError::SyntaxError);
1242            };
1243
1244            if type_code == 6 {
1245                // Immediately evaluated name: look up in dict stack
1246                let name_id = ctx.names.intern(name_bytes);
1247                let key = DictKey::Name(name_id);
1248                ctx.dict_load(&key).ok_or(PsError::Undefined)
1249            } else {
1250                let name_id = ctx.names.intern(name_bytes);
1251                let mut obj = if is_exec {
1252                    PsObject::name_exec(name_id)
1253                } else {
1254                    PsObject::name_lit(name_id)
1255                };
1256                if !is_exec {
1257                    obj.flags.set_literal();
1258                }
1259                Ok(obj)
1260            }
1261        }
1262
1263        // boolean
1264        4 => Ok(PsObject::bool(value_u32 != 0)),
1265
1266        // string
1267        5 => {
1268            let len = length_u16 as usize;
1269            if len == 0 {
1270                let obj = alloc_string(ctx, &[]);
1271                return Ok(obj);
1272            }
1273            let offset = value_u32 as usize;
1274            if offset + len > data_size {
1275                return Err(PsError::SyntaxError);
1276            }
1277            let mut obj = alloc_string(ctx, &data[offset..offset + len]);
1278            if is_exec {
1279                obj.flags.set_executable();
1280            }
1281            Ok(obj)
1282        }
1283
1284        // array
1285        9 => {
1286            let count = length_u16 as usize;
1287            if count == 0 {
1288                let obj = alloc_array(ctx, &[]);
1289                return Ok(obj);
1290            }
1291            let offset = value_u32 as usize;
1292            let mut elements = Vec::with_capacity(count);
1293            for i in 0..count {
1294                let child =
1295                    build_bos_object(ctx, data, data_size, offset + i * 8, big_endian, depth + 1)?;
1296                elements.push(child);
1297            }
1298            let mut obj = alloc_array(ctx, &elements);
1299            if is_exec {
1300                obj.flags.set_executable();
1301            }
1302            Ok(obj)
1303        }
1304
1305        // mark
1306        10 => Ok(PsObject::mark()),
1307
1308        _ => Err(PsError::SyntaxError),
1309    }
1310}
stet_core/binary_token.rs

stet_core/
binary_token.rs
