sofos 0.2.11

An interactive AI coding agent for your terminal
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
use crate::ui::UI;
use colored::Colorize;
use std::io::{self, Write, stdout};

/// SGR code for bold-on. Shared between the markdown Strong Start handler
/// and the ambient-style restorer so the two never drift apart.
const SGR_BOLD: &str = "\x1b[1m";
/// SGR code for italic-on. Shared between Emphasis Start and the restorer.
const SGR_ITALIC: &str = "\x1b[3m";
/// SGR code for the markdown heading style (bold + cyan). Shared between
/// Heading Start and the restorer; restoring just `\x1b[36m` would silently
/// drop the bold half.
const SGR_HEADING: &str = "\x1b[1;36m";
/// SGR code for the blockquote dim. Strong End (`\x1b[22m`) clears bold
/// *and* faint, and inline Code/Link close with `\x1b[0m` which clears
/// every attribute โ€” so the restorer re-applies this when a tag closes
/// inside a blockquote.
const SGR_BLOCKQUOTE: &str = "\x1b[2m";

impl UI {
    pub fn print_markdown_highlighted(&self, md: &str) -> io::Result<()> {
        let mut out = stdout().lock();
        self.render_markdown_to(&mut out, md)?;
        out.flush()
    }

    pub(super) fn render_markdown_to(&self, out: &mut impl io::Write, md: &str) -> io::Result<()> {
        use pulldown_cmark::{CodeBlockKind, Event, Options, Parser, Tag, TagEnd};

        // Re-emit any ambient inline styles after a full SGR reset, so nested inline
        // tags (Code/Link) and Strong don't leave the outer heading/strong/emphasis bare.
        fn restore_ambient(
            out: &mut impl io::Write,
            bold: bool,
            italic: bool,
            in_heading: bool,
            in_blockquote: bool,
        ) -> io::Result<()> {
            if bold {
                write!(out, "{}", SGR_BOLD)?;
            }
            if italic {
                write!(out, "{}", SGR_ITALIC)?;
            }
            if in_heading {
                write!(out, "{}", SGR_HEADING)?;
            }
            if in_blockquote {
                write!(out, "{}", SGR_BLOCKQUOTE)?;
            }
            Ok(())
        }

        let parser = Parser::new_ext(md, Options::all());

        let mut in_code_block = false;
        let mut code_lang = String::new();
        let mut code_buf = String::new();
        let mut bold = false;
        let mut italic = false;
        let mut in_heading = false;
        let mut in_blockquote = false;

        for event in parser {
            match event {
                Event::Start(Tag::Heading { .. }) => {
                    in_heading = true;
                    write!(out, "{}", SGR_HEADING)?;
                }
                Event::End(TagEnd::Heading(_)) => {
                    in_heading = false;
                    writeln!(out, "\x1b[0m")?;
                }
                Event::Start(Tag::Strong) => {
                    bold = true;
                    write!(out, "{}", SGR_BOLD)?;
                }
                Event::End(TagEnd::Strong) => {
                    bold = false;
                    write!(out, "\x1b[22m")?;
                    restore_ambient(out, bold, italic, in_heading, in_blockquote)?;
                }
                Event::Start(Tag::Emphasis) => {
                    italic = true;
                    write!(out, "{}", SGR_ITALIC)?;
                }
                Event::End(TagEnd::Emphasis) => {
                    italic = false;
                    write!(out, "\x1b[23m")?;
                }
                Event::Start(Tag::CodeBlock(kind)) => {
                    in_code_block = true;
                    code_buf.clear();
                    code_lang = match kind {
                        CodeBlockKind::Fenced(lang) => lang.to_string(),
                        _ => String::new(),
                    };
                }
                Event::End(TagEnd::CodeBlock) => {
                    in_code_block = false;
                    let highlighted = self.highlighter.highlight_code(&code_buf, &code_lang);
                    writeln!(out, "{}", highlighted)?;
                }
                Event::Code(code) => {
                    write!(out, "\x1b[38;2;175;215;255m{}\x1b[0m", code)?;
                    restore_ambient(out, bold, italic, in_heading, in_blockquote)?;
                }
                Event::Text(text) => {
                    if in_code_block {
                        code_buf.push_str(&text);
                    } else {
                        write!(out, "{}", text)?;
                    }
                }
                Event::SoftBreak => {
                    if !in_code_block {
                        writeln!(out)?;
                    }
                }
                Event::HardBreak => {
                    writeln!(out)?;
                }
                Event::Start(Tag::Paragraph) => {}
                Event::End(TagEnd::Paragraph) => {
                    writeln!(out)?;
                    writeln!(out)?;
                }
                Event::Start(Tag::List(_)) => {}
                Event::End(TagEnd::List(_)) => {}
                Event::Start(Tag::Item) => {
                    write!(out, "  {} ", "โ€ข".dimmed())?;
                }
                Event::End(TagEnd::Item) => {
                    writeln!(out)?;
                }
                Event::Start(Tag::BlockQuote(_)) => {
                    in_blockquote = true;
                    write!(out, "{}> ", SGR_BLOCKQUOTE)?;
                }
                Event::End(TagEnd::BlockQuote(_)) => {
                    in_blockquote = false;
                    writeln!(out, "\x1b[0m")?;
                }
                Event::Start(Tag::Link { dest_url, .. }) => {
                    // OSC 8 URI terminates on BEL/ESC; bypass the wrapper if dest_url has any control byte.
                    if dest_url.chars().any(|c| c.is_ascii_control()) {
                        write!(out, "\x1b[4;34m")?;
                    } else {
                        write!(out, "\x1b]8;;{}\x07\x1b[4;34m", dest_url)?;
                    }
                }
                Event::End(TagEnd::Link) => {
                    write!(out, "\x1b[0m\x1b]8;;\x07")?;
                    restore_ambient(out, bold, italic, in_heading, in_blockquote)?;
                }
                Event::Rule => {
                    writeln!(out, "{}", "โ”€".repeat(40).dimmed())?;
                }
                _ => {}
            }
        }

        Ok(())
    }
}

/// Return the byte offset of the last newline in `buf` that is **not**
/// inside an open fenced code block. Lines opening with ``` toggle the
/// fence state; the trailing newline of an open-fence line is therefore
/// not a safe commit point.
///
/// Returns 0 when no safe newline exists yet (caller commits nothing).
///
/// **Known limitation:** the toggle treats any line whose first
/// non-space character is ``` as a fence boundary regardless of the
/// opening fence length. CommonMark allows nesting via longer fences
/// (e.g., a 4-backtick block containing a 3-backtick line), and this
/// algorithm miscounts those โ€” pulldown still renders correctly, but
/// the safe-commit point may be conservative or skewed inside such
/// blocks. Rare in assistant output; accept the limitation.
fn safe_commit_end(buf: &str) -> usize {
    let mut fence_open = false;
    let mut last_safe = 0usize;
    let mut pos = 0usize;
    for line in buf.split_inclusive('\n') {
        if is_fence_line(line) {
            fence_open = !fence_open;
        }
        pos += line.len();
        if line.ends_with('\n') && !fence_open {
            last_safe = pos;
        }
    }
    last_safe
}

/// True when `line` opens or closes a fenced code block under CommonMark
/// indentation rules: at most three leading spaces, then ```. A line
/// with four or more leading spaces is part of an indented code block
/// instead, even if its first non-space content is ```.
fn is_fence_line(line: &str) -> bool {
    let after_spaces = line.trim_start_matches(' ');
    let indent = line.len() - after_spaces.len();
    indent <= 3 && after_spaces.starts_with("```")
}

/// Newline-gated markdown renderer for streaming output. Accumulates
/// deltas, and on every commit re-renders the full buffer prefix up to
/// the last newline as markdown, emitting only the lines past what's
/// already been printed. The partial last line is held back until the
/// next newline or [`finalize`] arrives.
///
/// Modelled after `codex-rs/tui/src/markdown_stream.rs` โ€” same invariant
/// (no commit until newline), same convergence property (the sum of
/// streamed emissions equals what a single non-streaming render of the
/// full buffer would produce). The cost is one full re-render per
/// committed chunk, which is acceptable because pulldown_cmark is fast
/// and assistant turns rarely exceed a few KB.
pub(super) struct MarkdownStreamRenderer {
    buffer: String,
    committed_lines: usize,
}

impl MarkdownStreamRenderer {
    pub(super) fn new() -> Self {
        Self {
            buffer: String::new(),
            committed_lines: 0,
        }
    }

    pub(super) fn push_delta(&mut self, delta: &str) {
        self.buffer.push_str(delta);
    }

    /// Render the buffer prefix up to the last "safe" newline โ€” one
    /// that's outside any open fenced code block โ€” and return only the
    /// lines past `committed_lines`. Returns an empty string when there
    /// is no safe commit point yet.
    ///
    /// Fence-awareness matters because the markdown renderer synthesises
    /// a closing border for an unclosed code fence; committing that
    /// premature border leaves a wrong line on screen that we can't
    /// unprint when the real closing fence arrives.
    pub(super) fn commit(&mut self) -> io::Result<String> {
        let safe_end = safe_commit_end(&self.buffer);
        if safe_end == 0 {
            return Ok(String::new());
        }
        // Drop the trailing blank during commit: pulldown's render is
        // not monotonic when a paragraph stays open across deltas (the
        // End-of-Paragraph blank "moves" further down as more text
        // arrives). Holding that trailing blank back means continuing
        // text gets emitted on the next commit instead of being lost
        // behind a prematurely-committed paragraph terminator.
        let (new, total) = self.render_new_lines(&self.buffer[..safe_end], true)?;
        self.committed_lines = total;
        Ok(new)
    }

    /// Emit the residual: anything past `committed_lines`, including the
    /// partial last line and any trailing blank line that `commit`
    /// deliberately held back. Resets internal state so the renderer
    /// can be reused for the next stream.
    pub(super) fn finalize(&mut self) -> io::Result<String> {
        let mut source = self.buffer.clone();
        // pulldown_cmark needs the trailing newline to close the last
        // paragraph; without it the final line renders as if it were
        // still being built.
        if !source.ends_with('\n') {
            source.push('\n');
        }
        let (new, _) = self.render_new_lines(&source, false)?;
        self.buffer.clear();
        self.committed_lines = 0;
        Ok(new)
    }

    /// Render `source` and return the slice of rendered lines past
    /// `committed_lines` plus the post-drop total line count. The total
    /// is what `commit` writes back to `self.committed_lines`; `finalize`
    /// drops it because it resets the counter anyway. When
    /// `drop_trailing_blank` is true a trailing whitespace-only line is
    /// excluded from both the emitted slice and the returned total โ€”
    /// see [`commit`] for why.
    fn render_new_lines(
        &self,
        source: &str,
        drop_trailing_blank: bool,
    ) -> io::Result<(String, usize)> {
        let mut buf: Vec<u8> = Vec::new();
        UI::shared().render_markdown_to(&mut buf, source)?;
        let rendered = String::from_utf8_lossy(&buf).into_owned();
        let lines: Vec<&str> = rendered.split_inclusive('\n').collect();
        let mut effective_len = lines.len();
        if drop_trailing_blank && effective_len > 0 && lines[effective_len - 1].trim().is_empty() {
            effective_len -= 1;
        }
        let new = if self.committed_lines >= effective_len {
            String::new()
        } else {
            lines[self.committed_lines..effective_len].concat()
        };
        Ok((new, effective_len))
    }
}

impl Default for MarkdownStreamRenderer {
    fn default() -> Self {
        Self::new()
    }
}

#[cfg(test)]
mod render_tests {
    use super::*;

    fn render(md: &str) -> String {
        let ui = UI::new();
        let mut buf = Vec::new();
        ui.render_markdown_to(&mut buf, md).unwrap();
        String::from_utf8(buf).unwrap()
    }

    #[test]
    fn link_emits_osc8_hyperlink_for_normal_url() {
        let out = render("[example](https://example.com)");
        assert!(
            out.contains("\x1b]8;;https://example.com\x07"),
            "OSC 8 opener with URL not found in: {:?}",
            out
        );
        assert!(out.contains("example"), "link text not found");
        assert!(
            out.contains("\x1b]8;;\x07"),
            "OSC 8 closer not found in: {:?}",
            out
        );
    }

    #[test]
    fn strong_in_heading_restores_heading_style() {
        let out = render("# title with **bold** rest");
        let after_strong_end = out
            .split("\x1b[22m")
            .nth(1)
            .expect("Strong End must emit \\x1b[22m");
        let rest_idx = after_strong_end
            .find(" rest")
            .expect("trailing text must be present");
        assert!(
            after_strong_end[..rest_idx].contains(SGR_HEADING),
            "heading style not restored between Strong End and trailing text; segment={:?}",
            &after_strong_end[..rest_idx]
        );
    }

    #[test]
    fn code_in_heading_restores_heading_style() {
        let out = render("# title with `code` rest");
        // Inline Code closes with \x1b[0m before restore_ambient runs.
        let after_code_reset = out
            .split("\x1b[0m")
            .nth(1)
            .expect("inline Code emits \\x1b[0m");
        let rest_idx = after_code_reset
            .find(" rest")
            .expect("trailing text must be present");
        assert!(
            after_code_reset[..rest_idx].contains(SGR_HEADING),
            "heading style not restored between inline Code and trailing text; segment={:?}",
            &after_code_reset[..rest_idx]
        );
    }

    #[test]
    fn link_in_heading_restores_heading_style() {
        let out = render("# title with [link](https://example.com) rest");
        let after_link_close = out
            .split("\x1b]8;;\x07")
            .nth(1)
            .expect("Link End must emit OSC 8 close");
        let rest_idx = after_link_close
            .find(" rest")
            .expect("trailing text must be present");
        assert!(
            after_link_close[..rest_idx].contains(SGR_HEADING),
            "heading style not restored between Link End and trailing text; segment={:?}",
            &after_link_close[..rest_idx]
        );
    }

    #[test]
    fn strong_in_blockquote_restores_dim() {
        let out = render("> **bold** rest");
        let after_strong_end = out
            .split("\x1b[22m")
            .nth(1)
            .expect("Strong End must emit \\x1b[22m");
        let rest_idx = after_strong_end
            .find(" rest")
            .expect("trailing text must be present");
        assert!(
            after_strong_end[..rest_idx].contains(SGR_BLOCKQUOTE),
            "blockquote dim not restored between Strong End and trailing text; segment={:?}",
            &after_strong_end[..rest_idx]
        );
    }

    #[test]
    fn code_in_blockquote_restores_dim() {
        let out = render("> `code` rest");
        // The blockquote's own dim opens with \x1b[2m before the inline Code event;
        // skip past that prefix so we land between Code's \x1b[0m and the trailing text.
        let after_code_reset = out
            .split("\x1b[0m")
            .nth(1)
            .expect("inline Code emits \\x1b[0m");
        let rest_idx = after_code_reset
            .find(" rest")
            .expect("trailing text must be present");
        assert!(
            after_code_reset[..rest_idx].contains(SGR_BLOCKQUOTE),
            "blockquote dim not restored between inline Code and trailing text; segment={:?}",
            &after_code_reset[..rest_idx]
        );
    }

    #[test]
    fn link_in_blockquote_restores_dim() {
        let out = render("> [link](https://example.com) rest");
        let after_link_close = out
            .split("\x1b]8;;\x07")
            .nth(1)
            .expect("Link End must emit OSC 8 close");
        let rest_idx = after_link_close
            .find(" rest")
            .expect("trailing text must be present");
        assert!(
            after_link_close[..rest_idx].contains(SGR_BLOCKQUOTE),
            "blockquote dim not restored between Link End and trailing text; segment={:?}",
            &after_link_close[..rest_idx]
        );
    }

    #[test]
    fn link_in_emphasis_restores_italic() {
        let out = render("*italic [link](https://example.com) rest*");
        let after_link_close = out
            .split("\x1b]8;;\x07")
            .nth(1)
            .expect("Link End must emit OSC 8 close");
        let rest_idx = after_link_close
            .find(" rest")
            .expect("trailing text must be present");
        assert!(
            after_link_close[..rest_idx].contains(SGR_ITALIC),
            "italic not restored between Link End and trailing text; segment={:?}",
            &after_link_close[..rest_idx]
        );
    }
}

#[cfg(test)]
mod stream_tests {
    use super::*;

    fn full_render(md: &str) -> String {
        let ui = UI::new();
        let mut buf = Vec::new();
        ui.render_markdown_to(&mut buf, md).unwrap();
        String::from_utf8(buf).unwrap()
    }

    /// Drive the streaming renderer with the given delta sequence and
    /// return the concatenation of every commit + finalize. The
    /// streaming output must match what a one-shot render of the
    /// concatenated deltas produces, otherwise the stream visibly drifts
    /// from the canonical view.
    fn stream(deltas: &[&str]) -> String {
        let mut renderer = MarkdownStreamRenderer::new();
        let mut out = String::new();
        for d in deltas {
            renderer.push_delta(d);
            out.push_str(&renderer.commit().unwrap());
        }
        out.push_str(&renderer.finalize().unwrap());
        out
    }

    #[test]
    fn no_commit_until_newline() {
        let mut r = MarkdownStreamRenderer::new();
        r.push_delta("Hello, world");
        assert!(
            r.commit().unwrap().is_empty(),
            "commit before newline must hold the line back"
        );
        r.push_delta("!\n");
        let out = r.commit().unwrap();
        assert!(out.contains("Hello, world!"), "got: {out:?}");
    }

    #[test]
    fn finalize_emits_partial_last_line() {
        let mut r = MarkdownStreamRenderer::new();
        r.push_delta("partial without trailing newline");
        assert!(r.commit().unwrap().is_empty());
        let out = r.finalize().unwrap();
        assert!(out.contains("partial"), "got: {out:?}");
    }

    #[test]
    fn streamed_heading_matches_full_render() {
        let md = "### Core loop\n";
        let streamed = stream(&["### Core ", "loop\n"]);
        let full = full_render(md);
        assert_eq!(streamed, full, "streamed heading must match full render");
    }

    #[test]
    fn streamed_numbered_list_matches_full_render() {
        // The exact case the user reported: numbered list with bold
        // inside an item must come out styled, not as raw markdown.
        let md = "1. **You type a prompt**. then more text.\n2. **Second**.\n";
        let streamed = stream(&[
            "1. **You type a prompt**.",
            " then more text.\n2. ",
            "**Second**.\n",
        ]);
        let full = full_render(md);
        assert_eq!(
            streamed, full,
            "streamed numbered list with bold must match full render"
        );
    }

    #[test]
    fn streamed_paragraphs_match_full_render() {
        let md = "First paragraph.\n\nSecond paragraph with **bold**.\n";
        let streamed = stream(&[
            "First paragraph.\n",
            "\nSecond paragraph",
            " with **bold**.\n",
        ]);
        let full = full_render(md);
        assert_eq!(streamed, full);
    }

    #[test]
    fn streamed_fenced_code_matches_full_render() {
        let md = "Here:\n\n```rust\nlet x = 1;\nlet y = 2;\n```\n";
        let streamed = stream(&["Here:\n\n```", "rust\nlet x = 1;\n", "let y = 2;\n```\n"]);
        let full = full_render(md);
        assert_eq!(
            streamed, full,
            "streamed fenced code must match full render"
        );
    }

    #[test]
    fn indented_backticks_are_not_treated_as_fence() {
        // CommonMark ยง4.5: a fenced code block opener allows at most
        // 3 leading spaces. A line with 4+ leading spaces followed by
        // ``` is part of an indented code block and must not toggle
        // fence state โ€” otherwise the streaming renderer stalls
        // commits inside the surrounding indented block.
        assert!(is_fence_line("```\n"));
        assert!(is_fence_line(" ```\n"));
        assert!(is_fence_line("  ```\n"));
        assert!(is_fence_line("   ```\n"));
        assert!(!is_fence_line("    ```\n"));
        assert!(!is_fence_line("\t```\n"));
        assert!(!is_fence_line("text```\n"));
    }

    #[test]
    fn streamed_soft_break_paragraph_matches_full_render() {
        // Two lines belonging to one paragraph (soft break, no blank
        // line between). pulldown emits End-of-Paragraph blank only
        // after the LAST text in the paragraph, so the rendered line
        // count for the partial-paragraph prefix is shifted relative
        // to the full-paragraph render โ€” the regression this test
        // pins is that the second line would get lost behind a
        // prematurely-committed End-of-Paragraph blank.
        let md = "line one\nline two\n";
        let streamed = stream(&["line one\n", "line two\n"]);
        let full = full_render(md);
        assert_eq!(
            streamed, full,
            "second line of a soft-break paragraph must not be lost"
        );
    }

    #[test]
    fn finalize_resets_state_for_reuse() {
        let mut r = MarkdownStreamRenderer::new();
        r.push_delta("first turn\n");
        let _ = r.commit().unwrap();
        let _ = r.finalize().unwrap();

        r.push_delta("second turn\n");
        let out = r.commit().unwrap();
        assert!(
            out.contains("second turn"),
            "renderer must be reusable after finalize; got {out:?}"
        );
    }
}