rust-forth-tokenizer 0.2.1

A Forth tokenizer written in Rust.
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
\ @(#) trace.fth 98/01/28 1.2
\ TRACE ( <name> -- , trace pForth word )
\
\ Single step debugger.
\   TRACE  ( i*x <name> -- , setup trace for Forth word )
\   S      ( -- , step over )
\   SM     ( many -- , step over many times )
\   SD     ( -- , step down )
\   G      ( -- , go to end of word )
\   GD     ( n -- , go down N levels from current level, stop at end of this level )
\
\ This debugger works by emulating the inner interpreter of pForth.
\ It executes code and maintains a separate return stack for the
\ program under test.  Thus all primitives that operate on the return
\ stack, such as DO and R> must be trapped.  Local variables must
\ also be handled specially.  Several state variables are also
\ saved and restored to establish the context for the program being
\ tested.
\
\ Copyright 1997 Phil Burk
\
\ Modifications:
\      19990930 John Providenza - Fixed stack bugs in GD

anew task-trace.fth

: SPACE.TO.COLUMN  ( col -- )
    out @ - spaces
;

: IS.PRIMITIVE? ( xt -- flag , true if kernel primitive )
    ['] first_colon <
;

0 value TRACE_IP         \ instruction pointer
0 value TRACE_LEVEL      \ level of descent for inner interpreter
0 value TRACE_LEVEL_MAX  \ maximum level of descent

private{

\ use fake return stack
128 cells constant TRACE_RETURN_SIZE \ size of return stack in bytes
create TRACE-RETURN-STACK TRACE_RETURN_SIZE 16 + allot
variable TRACE-RSP
: TRACE.>R     ( n -- ) trace-rsp @ cell- dup trace-rsp ! ! ;  \ *(--rsp) = n
: TRACE.R>     ( -- n ) trace-rsp @ dup @ swap cell+ trace-rsp ! ;  \ n = *rsp++
: TRACE.R@     ( -- n ) trace-rsp @ @ ; ; \ n = *rsp
: TRACE.RPICK  ( index -- n ) cells trace-rsp @ + @ ; ; \ n = rsp[index]
: TRACE.0RP    ( -- n ) trace-return-stack trace_return_size + 8 + trace-rsp ! ;
: TRACE.RDROP  ( --  ) cell trace-rsp +! ;
: TRACE.RCHECK ( -- , abort if return stack out of range )
    trace-rsp @ trace-return-stack u<
        abort" TRACE return stack OVERFLOW!"
    trace-rsp @ trace-return-stack trace_return_size + 12 + u>
        abort" TRACE return stack UNDERFLOW!"
;

\ save and restore several state variables
10 cells constant TRACE_STATE_SIZE
create TRACE-STATE-1 TRACE_STATE_SIZE allot
create TRACE-STATE-2 TRACE_STATE_SIZE allot

variable TRACE-STATE-PTR
: TRACE.SAVE++ ( addr -- , save next thing )
    @ trace-state-ptr @ !
    cell trace-state-ptr +!
;

: TRACE.SAVE.STATE  ( -- )
    state trace.save++
    hld   trace.save++
    base  trace.save++
;

: TRACE.SAVE.STATE1  ( -- , save normal state )
    trace-state-1 trace-state-ptr !
    trace.save.state
;
: TRACE.SAVE.STATE2  ( -- , save state of word being debugged )
    trace-state-2 trace-state-ptr !
    trace.save.state
;


: TRACE.RESTORE++ ( addr -- , restore next thing )
    trace-state-ptr @ @ swap !
    cell trace-state-ptr +!
;

: TRACE.RESTORE.STATE  ( -- )
    state trace.restore++
    hld   trace.restore++
    base  trace.restore++
;

: TRACE.RESTORE.STATE1  ( -- )
    trace-state-1 trace-state-ptr !
    trace.restore.state
;
: TRACE.RESTORE.STATE2  ( -- )
    trace-state-2 trace-state-ptr !
    trace.restore.state
;

\ The implementation of these pForth primitives is specific to pForth.

variable TRACE-LOCALS-PTR  \ point to top of local frame

\ create a return stack frame for NUM local variables
: TRACE.(LOCAL.ENTRY)  ( x0 x1 ... xn n -- )  { num | lp -- }
    trace-locals-ptr @ trace.>r
    trace-rsp @ trace-locals-ptr !
    trace-rsp @  num cells - trace-rsp !  \ make room for locals
    trace-rsp @ -> lp
    num 0
    DO
        lp !
        cell +-> lp  \ move data into locals frame on return stack
    LOOP
;

: TRACE.(LOCAL.EXIT) ( -- )
    trace-locals-ptr @  trace-rsp !
    trace.r> trace-locals-ptr !
;
: TRACE.(LOCAL@) ( l# -- n , fetch from local frame )
    trace-locals-ptr @  swap cells - @
;
: TRACE.(1_LOCAL@) ( -- n ) 1 trace.(local@) ;
: TRACE.(2_LOCAL@) ( -- n ) 2 trace.(local@) ;
: TRACE.(3_LOCAL@) ( -- n ) 3 trace.(local@) ;
: TRACE.(4_LOCAL@) ( -- n ) 4 trace.(local@) ;
: TRACE.(5_LOCAL@) ( -- n ) 5 trace.(local@) ;
: TRACE.(6_LOCAL@) ( -- n ) 6 trace.(local@) ;
: TRACE.(7_LOCAL@) ( -- n ) 7 trace.(local@) ;
: TRACE.(8_LOCAL@) ( -- n ) 8 trace.(local@) ;

: TRACE.(LOCAL!) ( n l# -- , store into local frame )
    trace-locals-ptr @  swap cells - !
;
: TRACE.(1_LOCAL!) ( -- n ) 1 trace.(local!) ;
: TRACE.(2_LOCAL!) ( -- n ) 2 trace.(local!) ;
: TRACE.(3_LOCAL!) ( -- n ) 3 trace.(local!) ;
: TRACE.(4_LOCAL!) ( -- n ) 4 trace.(local!) ;
: TRACE.(5_LOCAL!) ( -- n ) 5 trace.(local!) ;
: TRACE.(6_LOCAL!) ( -- n ) 6 trace.(local!) ;
: TRACE.(7_LOCAL!) ( -- n ) 7 trace.(local!) ;
: TRACE.(8_LOCAL!) ( -- n ) 8 trace.(local!) ;

: TRACE.(LOCAL+!) ( n l# -- , store into local frame )
    trace-locals-ptr @  swap cells - +!
;
: TRACE.(?DO)  { limit start ip -- ip' }
    limit start =
    IF
        ip @ +-> ip \ BRANCH
    ELSE
        start trace.>r
        limit trace.>r
        cell +-> ip
    THEN
    ip
;

: TRACE.(LOOP)  { ip | limit indx -- ip' }
    trace.r> -> limit
    trace.r> 1+ -> indx
    limit indx =
    IF
        cell +-> ip
    ELSE
        indx trace.>r
        limit trace.>r
        ip @ +-> ip
    THEN
    ip
;

: TRACE.(+LOOP)  { delta ip | limit indx oldindx -- ip' }
    trace.r> -> limit
    trace.r> -> oldindx
    oldindx delta + -> indx
\ /* Do indices cross boundary between LIMIT-1 and LIMIT ? */
\  if( ( (OldIndex - Limit) & ((Limit-1) - NewIndex) & 0x80000000 ) ||
\    ( (NewIndex - Limit) & ((Limit-1) - OldIndex) & 0x80000000 ) )
    oldindx limit -    limit 1-    indx -  AND $ 80000000 AND
       indx limit -    limit 1- oldindx -  AND $ 80000000 AND OR
    IF
        cell +-> ip
    ELSE
        indx trace.>r
        limit trace.>r
        ip @ +-> ip
    THEN
    ip
;

: TRACE.CHECK.IP  {  ip -- }
    ip ['] first_colon u<
    ip here u> OR
    IF
        ." TRACE - IP out of range = " ip .hex cr
        abort
    THEN
;

: TRACE.SHOW.IP { ip -- , print name and offset }
    ip code> >name dup id.
    name> >code ip swap - ."  +" .
;

: TRACE.SHOW.STACK { | mdepth -- }
    base @ >r
    ." <" base @ decimal 1 .r ." :"
    depth 1 .r ." > "
    r> base !
    depth 5 min -> mdepth
    depth mdepth  -
    IF
        ." ... "  \ if we don't show entire stack
    THEN
    mdepth 0
    ?DO
        mdepth i 1+ - pick .  \ show numbers in current base
    LOOP
;

: TRACE.SHOW.NEXT { ip -- }
    >newline
    ip trace.check.ip
\ show word name and offset
    ." << "
    ip trace.show.ip
    16 space.to.column
\ show data stack
    trace.show.stack
    40 space.to.column ."  ||"
    trace_level 2* spaces
    ip code@
    cell +-> ip
\ show primitive about to be executed
    dup .xt space
\ trap any primitives that are followed by inline data
    CASE
        ['] (LITERAL)  OF ip @  . ENDOF
        ['] (ALITERAL) OF ip a@ . ENDOF
[ exists? (FLITERAL) [IF] ]
        ['] (FLITERAL) OF ip f@ f. ENDOF
[ [THEN] ]
        ['] BRANCH     OF ip @  . ENDOF
        ['] 0BRANCH    OF ip @  . ENDOF
        ['] (.")       OF ip count type .' "' ENDOF
        ['] (C")       OF ip count type .' "' ENDOF
        ['] (S")       OF ip count type .' "' ENDOF
    ENDCASE
    65 space.to.column ." >> "
;

: TRACE.DO.PRIMITIVE  { ip xt | oldhere --  ip' , perform code at ip }
    xt
    CASE
        0 OF -1 +-> trace_level  trace.r> -> ip ENDOF \ EXIT
        ['] (CREATE)   OF ip cell- body_offset + ENDOF
        ['] (LITERAL)  OF ip @ cell +-> ip ENDOF
        ['] (ALITERAL) OF ip a@ cell +-> ip ENDOF
[ exists? (FLITERAL) [IF] ]
        ['] (FLITERAL) OF ip f@ 1 floats +-> ip ENDOF
[ [THEN] ]
        ['] BRANCH     OF ip @ +-> ip ENDOF
        ['] 0BRANCH    OF 0= IF ip @ +-> ip ELSE cell +-> ip THEN ENDOF
        ['] >R         OF trace.>r ENDOF
        ['] R>         OF trace.r> ENDOF
        ['] R@         OF trace.r@ ENDOF
        ['] RDROP      OF trace.rdrop ENDOF
        ['] 2>R        OF trace.>r trace.>r ENDOF
        ['] 2R>        OF trace.r> trace.r> ENDOF
        ['] 2R@        OF trace.r@ 1 trace.rpick ENDOF
        ['] i          OF 1 trace.rpick ENDOF
        ['] j          OF 3 trace.rpick ENDOF
        ['] (LEAVE)    OF trace.rdrop trace.rdrop  ip @ +-> ip ENDOF
        ['] (LOOP)     OF ip trace.(loop) -> ip  ENDOF
        ['] (+LOOP)    OF ip trace.(+loop) -> ip  ENDOF
        ['] (DO)       OF trace.>r trace.>r ENDOF
        ['] (?DO)      OF ip trace.(?do) -> ip ENDOF
        ['] (.")       OF ip count type  ip count + aligned -> ip ENDOF
        ['] (C")       OF ip  ip count + aligned -> ip ENDOF
        ['] (S")       OF ip count  ip count + aligned -> ip ENDOF
        ['] (LOCAL.ENTRY) OF trace.(local.entry) ENDOF
        ['] (LOCAL.EXIT) OF trace.(local.exit) ENDOF
        ['] (LOCAL@)   OF trace.(local@)   ENDOF
        ['] (1_LOCAL@) OF trace.(1_local@) ENDOF
        ['] (2_LOCAL@) OF trace.(2_local@) ENDOF
        ['] (3_LOCAL@) OF trace.(3_local@) ENDOF
        ['] (4_LOCAL@) OF trace.(4_local@) ENDOF
        ['] (5_LOCAL@) OF trace.(5_local@) ENDOF
        ['] (6_LOCAL@) OF trace.(6_local@) ENDOF
        ['] (7_LOCAL@) OF trace.(7_local@) ENDOF
        ['] (8_LOCAL@) OF trace.(8_local@) ENDOF
        ['] (LOCAL!)   OF trace.(local!)   ENDOF
        ['] (1_LOCAL!) OF trace.(1_local!) ENDOF
        ['] (2_LOCAL!) OF trace.(2_local!) ENDOF
        ['] (3_LOCAL!) OF trace.(3_local!) ENDOF
        ['] (4_LOCAL!) OF trace.(4_local!) ENDOF
        ['] (5_LOCAL!) OF trace.(5_local!) ENDOF
        ['] (6_LOCAL!) OF trace.(6_local!) ENDOF
        ['] (7_LOCAL!) OF trace.(7_local!) ENDOF
        ['] (8_LOCAL!) OF trace.(8_local!) ENDOF
        ['] (LOCAL+!)  OF trace.(local+!)  ENDOF
        >r xt EXECUTE r>
    ENDCASE
    ip
;

: TRACE.DO.NEXT  { ip | xt oldhere --  ip' , perform code at ip }
    ip trace.check.ip
\ set context for word under test
    trace.save.state1
    here -> oldhere
    trace.restore.state2
    oldhere 256 + dp !
\ get execution token
    ip code@ -> xt
    cell +-> ip
\ execute token
    xt is.primitive?
    IF  \ primitive
        ip xt trace.do.primitive -> ip
    ELSE \ secondary
        trace_level trace_level_max <
        IF
            ip trace.>r         \ threaded execution
            1 +-> trace_level
            xt codebase + -> ip
        ELSE
            \ treat it as a primitive
            ip xt trace.do.primitive -> ip
        THEN
    THEN
\ restore original context
    trace.rcheck
    trace.save.state2
    trace.restore.state1
    oldhere dp !
    ip
;

: TRACE.NEXT { ip | xt -- ip' }
    trace_level 0>
    IF
        ip trace.do.next -> ip
    THEN
    trace_level 0>
    IF
        ip trace.show.next
    ELSE
        trace-stack on
        ." Finished." cr
    THEN
    ip
;

}private

: TRACE ( i*x <name> -- i*x , setup trace environment )
    ' dup is.primitive?
    IF
        drop ." Sorry. You can't trace a primitive." cr
    ELSE
        1 -> trace_level
        trace_level -> trace_level_max
        trace.0rp
        >code -> trace_ip
        trace_ip trace.show.next
        trace-stack off
        trace.save.state2
    THEN
;

: s ( -- , step over )
    trace_level -> trace_level_max
    trace_ip trace.next -> trace_ip
;

: sd ( -- , step down )
    trace_level 1+ -> trace_level_max
    trace_ip trace.next -> trace_ip
;

: sm ( many -- , step many times )
    trace_level -> trace_level_max
    0
    ?DO
        trace_ip trace.next -> trace_ip
    LOOP
;

defer trace.user   ( IP -- stop?  )
' 0= is trace.user

: gd { more_levels | stop_level -- }
    here   what's trace.user   u<  \ has it been forgotten?
    IF
        ." Resetting TRACE.USER !!!" cr
        ['] 0= is trace.user
    THEN

    more_levels 0<
    more_levels 10 >
    or  \ 19990930 - OR was missing
    IF
        ." GD level out of range (0-10), = " more_levels . cr
    ELSE
        trace_level more_levels + -> trace_level_max
        trace_level 1- -> stop_level
        BEGIN
            trace_ip trace.user \ call deferred user word
            ?dup \ leave flag for UNTIL \ 19990930 - was DUP
            IF
                ." TRACE.USER returned " dup . ." so stopping execution." cr
            ELSE
                trace_ip trace.next -> trace_ip
                trace_level stop_level > not
            THEN
        UNTIL
    THEN
;

: g ( -- , execute until end of word )
    0 gd
;

: TRACE.HELP ( -- )
    ."   TRACE  ( i*x <name> -- , setup trace for Forth word )" cr
    ."   S      ( -- , step over )" cr
    ."   SM     ( many -- , step over many times )" cr
    ."   SD     ( -- , step down )" cr
    ."   G      ( -- , go to end of word )" cr
    ."   GD     ( n -- , go down N levels from current level," cr
    ."                   stop at end of this level )" cr
;

privatize

0 [IF]
variable var1
100 var1 !
: FOO  dup IF 1 + . THEN 77 var1 @ + . ;
: ZOO 29 foo 99 22 + . ;
: ROO 92 >r 1 r@ + . r> . ;
: MOO  c" hello" count type
    ." This is a message." cr
    s" another message" type cr
;
: KOO 7 FOO ." DONE" ;
: TR.DO  4 0 DO i . LOOP ;
: TR.?DO  0 ?DO i . LOOP ;
: TR.LOC1 { aa bb } aa bb + . ;
: TR.LOC2 789 >r 4 5 tr.loc1 r> . ;

[THEN]