rtf_parser_tt/
parser.rs

1use std::collections::HashMap;
2use std::{fmt, mem};
3
4use serde::{Deserialize, Serialize};
5#[cfg(feature = "jsbindings")]
6use wasm_bindgen::prelude::wasm_bindgen;
7
8use crate::document::RtfDocument;
9use crate::header::{CharacterSet, Color, ColorRef, ColorTable, Font, FontFamily, FontRef, FontTable, RtfHeader, StyleSheet};
10use crate::paragraph::{Alignment, Paragraph, SpaceBetweenLine};
11use crate::tokens::{ControlWord, Property, Token};
12
13// Use to specify control word in parse_header
14macro_rules! header_control_word {
15    ($cw:ident) => {
16        &Token::ControlSymbol((ControlWord::$cw, _))
17    };
18    ($cw:ident, $prop:ident) => {
19        &Token::ControlSymbol((ControlWord::$cw, Property::$prop))
20    };
21}
22
23#[derive(Debug, Default, PartialEq, Clone, Deserialize, Serialize)]
24#[cfg_attr(feature = "jsbindings", wasm_bindgen(getter_with_clone))]
25pub struct StyleBlock {
26    pub painter: Painter,
27    pub paragraph: Paragraph,
28    pub text: String,
29}
30
31#[derive(Debug, Clone, PartialEq, Hash, Deserialize, Serialize)]
32#[cfg_attr(feature = "jsbindings", wasm_bindgen)]
33pub struct Painter {
34    pub color_ref: ColorRef,
35    pub font_ref: FontRef,
36    pub font_size: u16,
37    pub bold: bool,
38    pub italic: bool,
39    pub underline: bool,
40    pub superscript: bool,
41    pub subscript: bool,
42    pub smallcaps: bool,
43    pub strike: bool,
44}
45
46impl Default for Painter {
47    fn default() -> Self {
48        Self {
49            color_ref: Default::default(),
50            font_ref: Default::default(),
51            font_size: 12,
52            bold: Default::default(),
53            italic: Default::default(),
54            underline: Default::default(),
55            superscript: Default::default(),
56            subscript: Default::default(),
57            smallcaps: Default::default(),
58            strike: Default::default(),
59        }
60    }
61}
62
63#[derive(Debug, Clone)]
64pub enum ParserError {
65    InvalidToken(String),
66    IgnorableDestinationParsingError,
67    MalformedPainterStack,
68    InvalidFontIdentifier(Property),
69    InvalidColorIdentifier(Property),
70    NoMoreToken,
71    ValueCastError(String),
72    UnicodeParsingError(i32),
73    ParseEmptyToken,
74}
75
76impl std::error::Error for ParserError {}
77
78impl fmt::Display for ParserError {
79    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
80        let _ = write!(f, "[RTF Parser] : ");
81        return match self {
82            ParserError::InvalidToken(msg) => write!(f, "{}", msg),
83            ParserError::IgnorableDestinationParsingError => write!(f, "No ignorable destination should be left"),
84            ParserError::MalformedPainterStack => write!(f, "Malformed painter stack : Unbalanced number of brackets"),
85            ParserError::InvalidFontIdentifier(property) => write!(f, "Invalid font identifier : {:?}", property),
86            ParserError::InvalidColorIdentifier(property) => write!(f, "Invalid color identifier : {:?}", property),
87            ParserError::NoMoreToken => write!(f, "No more token to parse"),
88            ParserError::ValueCastError(_type) => write!(f, "Unable to cast i32 to {_type}"),
89            ParserError::UnicodeParsingError(value) => write!(f, "Unable to parse {value} value to unicode"),
90            ParserError::ParseEmptyToken => write!(f, "Try to parse an empty token, this should never happen. If so, please open an issue in the github repository"),
91        };
92    }
93}
94
95// This state keeps track of each value that depends on the scope nesting
96#[derive(Debug, Clone, PartialEq, Hash)]
97struct ParserState {
98    pub painter: Painter,
99    pub paragraph: Paragraph,
100    pub unicode_ignore_count: i32,
101}
102
103impl Default for ParserState {
104    fn default() -> Self {
105        Self {
106            painter: Default::default(),
107            paragraph: Default::default(),
108            unicode_ignore_count: 1,
109        }
110    }
111}
112
113pub struct Parser<'a> {
114    tokens: Vec<Token<'a>>,
115    parsed_item: Vec<bool>,
116    cursor: usize,
117}
118
119impl<'a> Parser<'a> {
120    pub fn new(tokens: Vec<Token<'a>>) -> Self {
121        return Self {
122            parsed_item: vec![false; tokens.len()],
123            tokens,
124            cursor: 0,
125        };
126    }
127
128    pub fn get_tokens(&self) -> Vec<&Token> {
129        // It ignores the empty tokens, that replaced already parsed tokens istead of deleting them for performance reasons
130        return self.tokens.iter().filter(|t| *t != &Token::Empty).collect();
131    }
132
133    fn check_document_validity(&self) -> Result<(), ParserError> {
134        // Check the document boundaries
135        if let Some(token) = self.tokens.first() {
136            if token != &Token::OpeningBracket {
137                return Err(ParserError::InvalidToken(format!("Invalid first token : {:?} not a '{{'", token)));
138            }
139        } else {
140            return Err(ParserError::NoMoreToken);
141        }
142        if let Some(token) = self.tokens.last() {
143            if token != &Token::ClosingBracket {
144                return Err(ParserError::InvalidToken(format!("Invalid last token : {:?} not a '}}'", token)));
145            }
146        } else {
147            return Err(ParserError::NoMoreToken);
148        }
149        return Ok(());
150    }
151
152    pub fn parse(&mut self) -> Result<RtfDocument, ParserError> {
153        self.check_document_validity()?;
154        let mut document = RtfDocument::default(); // Init empty document
155                                                   // Traverse the document and consume the header groups (FontTable, StyleSheet, etc ...)
156        document.header = self.parse_header()?;
157        // Init the state of the docuement. the stack is used to keep track of the different scope changes.
158        let mut state_stack: Vec<ParserState> = vec![ParserState::default()];
159        // Parse the body
160        let len = self.tokens.len();
161        let mut i = 0;
162        let mut pending_high_surrogate: Option<u16> = None;
163
164        while i < len {
165            if self.parsed_item[i] {
166                // The item already has been parsed
167                i += 1;
168                continue;
169            }
170            let token = &self.tokens[i];
171
172            if pending_high_surrogate.is_some() && !matches!(token, Token::ControlSymbol((ControlWord::Unicode | ControlWord::UnicodeIgnoreCount, _))) {
173                pending_high_surrogate = None;
174                Self::add_text_to_document("\u{FFFD}", &state_stack, &mut document)?;
175            }
176
177            match token {
178                Token::OpeningBracket => {
179                    if let Some(last_state) = state_stack.last() {
180                        state_stack.push(last_state.clone()); // Inherit from the last state properties
181                    } else {
182                        state_stack.push(ParserState::default());
183                    }
184                }
185                Token::ClosingBracket => {
186                    let state = state_stack.pop();
187                    if state.is_none() {
188                        return Err(ParserError::MalformedPainterStack);
189                    }
190                }
191                Token::ControlSymbol((control_word, property)) => {
192                    let Some(current_state) = state_stack.last_mut() else {
193                        return Err(ParserError::MalformedPainterStack);
194                    };
195                    let current_painter = &mut current_state.painter;
196                    let paragraph = &mut current_state.paragraph;
197                    #[rustfmt::skip]  // For now, rustfmt does not support this kind of alignement
198                    match control_word {
199                        ControlWord::ColorNumber        => current_painter.color_ref = property.get_value_as::<ColorRef>()?,
200                        ControlWord::FontNumber         => current_painter.font_ref = property.get_value_as::<FontRef>()?,
201                        ControlWord::FontSize           => current_painter.font_size = property.get_value_as::<u16>()?,
202                        ControlWord::Bold               => current_painter.bold = property.as_bool(),
203                        ControlWord::Italic             => current_painter.italic = property.as_bool(),
204                        ControlWord::Underline          => current_painter.underline = property.as_bool(),
205                        ControlWord::UnderlineNone      => current_painter.underline = false,
206                        ControlWord::Superscript        => current_painter.superscript = property.as_bool(),
207                        ControlWord::Subscript          => current_painter.subscript = property.as_bool(),
208                        ControlWord::Smallcaps          => current_painter.smallcaps = property.as_bool(),
209                        ControlWord::Strikethrough      => current_painter.strike = property.as_bool(),
210                        // Paragraph
211                        ControlWord::Pard               => *paragraph = Paragraph::default(), // Reset the par
212                        ControlWord::Plain              => *current_painter = Painter::default(), // Reset the painter
213                        ControlWord::ParDefTab          => paragraph.tab_width = property.get_value(),
214                        ControlWord::LeftAligned
215                            | ControlWord::RightAligned
216                            | ControlWord::Center
217                            | ControlWord::Justify      => paragraph.alignment = Alignment::from(control_word),
218                        ControlWord::SpaceBefore        => paragraph.spacing.before = property.get_value(),
219                        ControlWord::SpaceAfter         => paragraph.spacing.after = property.get_value(),
220                        ControlWord::SpaceBetweenLine   => paragraph.spacing.between_line = SpaceBetweenLine::from(property.get_value()),
221                        ControlWord::SpaceLineMul       => paragraph.spacing.line_multiplier = property.get_value(),
222                        ControlWord::UnicodeIgnoreCount => current_state.unicode_ignore_count = property.get_value(),
223                        ControlWord::Unicode            => {
224                            let mut unicodes = Vec::new();
225                            if let Ok(unicode) = property.get_unicode_value() {
226                                unicodes.push(unicode);
227                            }
228                            // Get the following unicode in case of compounds characters
229                            while i + 1 < len {
230                                // We should not check if the tokens has already been parsed, because we are looking for the following token in the document
231                                if let Token::ControlSymbol((ControlWord::Unicode, property)) = &self.tokens[i + 1] {
232                                    if let Ok(unicode) = property.get_unicode_value() {
233                                        unicodes.push(unicode);
234                                    }
235                                    i += 1;
236                                } else {
237                                    break;
238                                }
239                            }
240                            if unicodes.len() > 0 {
241                                // Handle the fallback unicode (\uc2 \u0000 'FA 'FB)
242                                let mut ignore_mask = vec![true; unicodes.len()];
243                                let mut ignore_counter = 0;
244                                for i in 1..unicodes.len() {
245                                    if unicodes[i] <= 255 && ignore_counter < current_state.unicode_ignore_count {
246                                        ignore_counter += 1;
247                                        ignore_mask[i] = false;
248                                    } else {
249                                        ignore_counter = 0;
250                                    }
251                                }
252                                let mut ignore_mask_iter = ignore_mask.iter();
253                                unicodes.retain(|_| *ignore_mask_iter.next().unwrap());
254                                if let Some(high) = pending_high_surrogate.take() {
255                                    let pairs = unicodes.first().is_some_and(|u| (0xDC00..=0xDFFF).contains(u));
256                                    unicodes.insert(0, if pairs { high } else { 0xFFFD });
257                                }
258                                if unicodes.last().is_some_and(|u| (0xD800..=0xDBFF).contains(u)) {
259                                    pending_high_surrogate = unicodes.pop();
260                                }
261                                if !unicodes.is_empty() {
262                                    let str = String::from_utf16_lossy(unicodes.as_slice());
263                                    Self::add_text_to_document(&str, &state_stack, &mut document)?;
264                                }
265                            }
266                        }
267                        // Special characters - output as text
268                        ControlWord::Emdash           => Self::add_text_to_document("\u{2014}", &state_stack, &mut document)?,
269                        ControlWord::Endash           => Self::add_text_to_document("\u{2013}", &state_stack, &mut document)?,
270                        ControlWord::Bullet           => Self::add_text_to_document("\u{2022}", &state_stack, &mut document)?,
271                        ControlWord::LeftSingleQuote  => Self::add_text_to_document("\u{2018}", &state_stack, &mut document)?,
272                        ControlWord::RightSingleQuote => Self::add_text_to_document("\u{2019}", &state_stack, &mut document)?,
273                        ControlWord::LeftDoubleQuote  => Self::add_text_to_document("\u{201C}", &state_stack, &mut document)?,
274                        ControlWord::RightDoubleQuote => Self::add_text_to_document("\u{201D}", &state_stack, &mut document)?,
275                        ControlWord::Tab              => Self::add_text_to_document("\t", &state_stack, &mut document)?,
276                        ControlWord::Line             => Self::add_text_to_document("\n", &state_stack, &mut document)?,
277                        // Others tokens
278                        _ => {}
279                    };
280                }
281                Token::PlainText(text) => Self::add_text_to_document(*text, &state_stack, &mut document)?,
282                Token::CRLF => Self::add_text_to_document("\n", &state_stack, &mut document)?,
283                Token::IgnorableDestination => {
284                    return Err(ParserError::IgnorableDestinationParsingError);
285                }
286                Token::Empty => return Err(ParserError::ParseEmptyToken),
287            };
288            i += 1;
289        }
290        if pending_high_surrogate.is_some() && !state_stack.is_empty() {
291            Self::add_text_to_document("\u{FFFD}", &state_stack, &mut document)?;
292        }
293        return Ok(document);
294    }
295
296    fn add_text_to_document(text: &str, state_stack: &Vec<ParserState>, document: &mut RtfDocument) -> Result<(), ParserError> {
297        let Some(current_state) = state_stack.last() else {
298            return Err(ParserError::MalformedPainterStack);
299        };
300        let current_painter = &current_state.painter;
301        let paragraph = &current_state.paragraph;
302        let last_style_group = document.body.last_mut();
303        // If the painter is the same as the previous one, merge the two block.
304        if let Some(group) = last_style_group {
305            if group.painter.eq(current_painter) && group.paragraph.eq(&paragraph) {
306                group.text.push_str(text);
307                return Ok(());
308            }
309        }
310        // Else, push another StyleBlock on the stack with its own painter
311        document.body.push(StyleBlock {
312            painter: current_painter.clone(),
313            paragraph: paragraph.clone(),
314            text: String::from(text),
315        });
316        return Ok(());
317    }
318
319    fn get_token_at(&'a self, index: usize) -> Option<&'a Token<'a>> {
320        return self.tokens.get(index);
321    }
322
323    // Get a view of the next token after cursor
324    fn get_next_token(&'a self) -> Option<&'a Token<'a>> {
325        return self.get_token_at(self.cursor);
326    }
327
328    #[inline]
329    fn consume_token_at(&mut self, index: usize) -> Option<Token<'a>> {
330        if self.tokens.is_empty() || index >= self.tokens.len() {
331            return None;
332        }
333        // PERF : vec.remove can require reallocation unlike this method
334        self.cursor += 1;
335        self.parsed_item[index] = true;
336        return Some(mem::replace(&mut self.tokens[index], Token::Empty));
337    }
338
339    fn consume_next_token(&mut self) -> Option<Token<'a>> {
340        return self.consume_token_at(self.cursor);
341    }
342
343    // Consume token from cursor to <reference-token>
344    fn _consume_tokens_until(&mut self, reference_token: &Token<'a>) -> Vec<Token<'a>> {
345        let mut ret = vec![];
346        let token_type_id = mem::discriminant(reference_token);
347        while let Some(token) = self.consume_next_token() {
348            let type_id = mem::discriminant(&token);
349            ret.push(token);
350            if type_id == token_type_id {
351                break;
352            }
353        }
354        return ret;
355    }
356
357    // The opening bracket should already be consumed
358    fn consume_tokens_until_matching_bracket(&mut self) -> Vec<Token<'a>> {
359        let mut ret = vec![];
360        let mut count = 0;
361        while let Some(token) = self.consume_next_token() {
362            match token {
363                Token::OpeningBracket => count += 1,
364                Token::ClosingBracket => count -= 1,
365                _ => {}
366            }
367            ret.push(token);
368            if count < 0 {
369                break;
370            }
371        }
372        return ret;
373    }
374
375    // Consume all the tokens inside a group ({ ... }) and returns the includes ones
376    fn consume_group(&mut self) -> Vec<Token<'a>> {
377        // TODO: check the the token at cursor is indeed an OpeningBracket
378        self.consume_token_at(self.cursor); // Consume the opening bracket
379        return self.consume_tokens_until_matching_bracket();
380    }
381
382    // Consume all tokens until the header is read
383    fn parse_header(&mut self) -> Result<RtfHeader, ParserError> {
384        self.cursor = 0; // Reset the cursor
385        let mut header = RtfHeader::default();
386        while let (Some(token), Some(mut next_token)) = (self.get_token_at(self.cursor), self.get_token_at(self.cursor + 1)) {
387            // Manage the case where there is CRLF between { and control_word
388            // {\n /*/ignoregroup }
389            let mut i = 0;
390            while *next_token == Token::CRLF {
391                if let Some(next_token_not_crlf) = self.get_token_at(self.cursor + 1 + i) {
392                    next_token = next_token_not_crlf;
393                    i += 1;
394                } else {
395                    break;
396                }
397            }
398            match (token, next_token) {
399                (Token::OpeningBracket, Token::IgnorableDestination) => {
400                    let ignore_group_tokens = self.consume_group();
401                    Self::parse_ignore_groups(&ignore_group_tokens);
402                }
403                (Token::OpeningBracket, header_control_word!(FontTable, None)) => {
404                    let font_table_tokens = self.consume_group();
405                    header.font_table = Self::parse_font_table(&font_table_tokens)?;
406                }
407                (Token::OpeningBracket, header_control_word!(ColorTable, None)) => {
408                    let color_table_tokens = self.consume_group();
409                    header.color_table = Self::parse_color_table(&color_table_tokens)?;
410                }
411                (Token::OpeningBracket, header_control_word!(StyleSheet, None)) => {
412                    let stylesheet_tokens = self.consume_group();
413                    header.stylesheet = Self::parse_stylesheet(&stylesheet_tokens)?;
414                }
415                // Check and consume token
416                (token, _) => {
417                    if let Some(charset) = CharacterSet::from(token) {
418                        header.character_set = charset;
419                    }
420                    self.cursor += 1;
421                }
422            }
423        }
424        return Ok(header);
425    }
426
427    fn parse_font_table(font_tables_tokens: &Vec<Token<'a>>) -> Result<FontTable, ParserError> {
428        let Some(font_table_first_token) = font_tables_tokens.get(0) else {
429            return Err(ParserError::NoMoreToken);
430        };
431        if font_table_first_token != header_control_word!(FontTable, None) {
432            return Err(ParserError::InvalidToken(format!("{:?} is not a FontTable token", font_table_first_token)));
433        }
434        let mut table = HashMap::new();
435        let mut current_key = 0;
436        let mut current_font = Font::default();
437        for token in font_tables_tokens.iter() {
438            match token {
439                Token::ControlSymbol((control_word, property)) => match control_word {
440                    ControlWord::FontNumber => {
441                        // Insert previous font
442                        table.insert(current_key, current_font.clone());
443                        if let Property::Value(key) = property {
444                            current_key = *key as FontRef;
445                        } else {
446                            return Err(ParserError::InvalidFontIdentifier(*property));
447                        }
448                    }
449                    ControlWord::Unknown(name) => {
450                        if let Some(font_family) = FontFamily::from(name) {
451                            current_font.font_family = font_family;
452                        }
453                    }
454                    _ => {}
455                },
456                Token::PlainText(name) => {
457                    current_font.name = name.trim_end_matches(';').to_string();
458                }
459                Token::ClosingBracket => {
460                    table.insert(current_key, current_font.clone());
461                } // Insert previous font
462                _ => {}
463            }
464        }
465        return Ok(table);
466    }
467
468    fn parse_color_table(color_table_tokens: &Vec<Token<'a>>) -> Result<ColorTable, ParserError> {
469        let Some(color_table_first_token) = color_table_tokens.get(0) else {
470            return Err(ParserError::NoMoreToken);
471        };
472        if color_table_first_token != header_control_word!(ColorTable, None) {
473            return Err(ParserError::InvalidToken(format!("ParserError: {:?} is not a ColorTable token", color_table_first_token)));
474        }
475        let mut table = HashMap::new();
476        let mut current_key = 1;
477        let mut current_color = Color::default();
478        for token in color_table_tokens.iter() {
479            match token {
480                Token::ControlSymbol((control_word, property)) => match control_word {
481                    ControlWord::ColorRed => current_color.red = property.get_value_as::<u8>()?,
482                    ControlWord::ColorGreen => current_color.green = property.get_value_as::<u8>()?,
483                    ControlWord::ColorBlue => {
484                        current_color.blue = property.get_value_as::<u8>()?;
485                        table.insert(current_key, current_color.clone());
486                        current_key += 1;
487                    }
488                    _ => {}
489                },
490                _ => {}
491            }
492        }
493        return Ok(table);
494    }
495
496    fn parse_stylesheet(_stylesheet_tokens: &Vec<Token<'a>>) -> Result<StyleSheet, ParserError> {
497        // TODO : parse the stylesheet
498        return Ok(StyleSheet::from([]));
499    }
500
501    fn parse_ignore_groups(_tokens: &Vec<Token<'a>>) {
502        // Do nothing for now
503    }
504}
505
506#[cfg(test)]
507pub mod tests {
508    use super::*;
509    use crate::header::CharacterSet::*;
510    use crate::header::FontFamily::*;
511    use crate::header::RtfHeader;
512    use crate::include_test_file;
513    use crate::lexer::Lexer;
514
515    #[test]
516    fn parser_header() {
517        let tokens = Lexer::scan(r#"{ \rtf1\ansi{\fonttbl\f0\fswiss Helvetica;}\f0\pard Voici du texte en {\b gras}.\par }"#).unwrap();
518        let doc = Parser::new(tokens).parse().unwrap();
519        assert_eq!(
520            doc.header,
521            RtfHeader {
522                character_set: Ansi,
523                font_table: FontTable::from([(
524                    0,
525                    Font {
526                        name: "Helvetica".into(),
527                        character_set: 0,
528                        font_family: Swiss
529                    }
530                )]),
531                ..RtfHeader::default()
532            }
533        );
534        assert_eq!(
535            doc.body,
536            [
537                StyleBlock {
538                    painter: Painter::default(),
539                    paragraph: Default::default(),
540                    text: "Voici du texte en ".into(),
541                },
542                StyleBlock {
543                    painter: Painter { bold: true, ..Painter::default() },
544                    paragraph: Default::default(),
545                    text: "gras".into(),
546                },
547                StyleBlock {
548                    painter: Painter::default(),
549                    paragraph: Default::default(),
550                    text: ".".into(),
551                },
552            ]
553        );
554    }
555
556    #[test]
557    fn parse_multiline_document() {
558        let document = r"{\rtf1\ansi\deff0 {\fonttbl {\f0 Courier;}{\f1 ProFontWindows;}}
559            {\colortbl;\red0\green0\blue0;\red255\green0\blue0;\red255\green255\blue0;}
560            This line is font 0 which is courier\line
561            \f1
562            This line is font 1\line
563            \f0
564            This line is font 0 again\line
565            This line has a \cf2 red \cf1 word\line
566            \highlight3 while this line has a \cf2 red \cf1 word and is highlighted in yellow\highlight0\line
567            Finally, back to the default color.\line
568            }";
569        let tokens = Lexer::scan(document).unwrap();
570        let _doc = Parser::new(tokens).parse().unwrap();
571    }
572
573    #[test]
574    fn parse_entire_file_header() {
575        let file_content = include_test_file!("test-file.rtf");
576        let tokens = Lexer::scan(file_content).unwrap();
577        let doc = Parser::new(tokens).parse().unwrap();
578        assert_eq!(
579            doc.header,
580            RtfHeader {
581                character_set: Ansi,
582                font_table: FontTable::from([
583                    (
584                        0,
585                        Font {
586                            name: "Helvetica".into(),
587                            character_set: 0,
588                            font_family: Swiss,
589                        }
590                    ),
591                    (
592                        1,
593                        Font {
594                            name: "Helvetica-Bold".into(),
595                            character_set: 0,
596                            font_family: Swiss,
597                        }
598                    )
599                ]),
600                color_table: ColorTable::from([(1, Color { red: 255, green: 255, blue: 255 }),]),
601                ..RtfHeader::default()
602            }
603        );
604    }
605
606    #[test]
607    fn parse_ignore_group() {
608        let rtf = r"{\*\expandedcolortbl;;}";
609        let tokens = Lexer::scan(rtf).unwrap();
610        let mut parser = Parser::new(tokens);
611        let document = parser.parse().unwrap();
612        assert_eq!(parser.get_tokens(), Vec::<&Token>::new()); // Should have consumed all the tokens
613        assert_eq!(document.header, RtfHeader::default());
614    }
615
616    #[test]
617    fn parse_ignore_group_with_crlf() {
618        let rtf = r"{\
619        \
620        \*\expandedcolortbl;;}";
621        let tokens = Lexer::scan(rtf).unwrap();
622        let mut parser = Parser::new(tokens);
623        let document = parser.parse().unwrap();
624        assert_eq!(parser.get_tokens(), Vec::<&Token>::new()); // Should have consumed all the tokens
625        assert_eq!(document.header, RtfHeader::default());
626    }
627
628    #[test]
629    #[ignore] // Pre-existing test failure from upstream - backslash line breaks not handled correctly
630    fn parse_whitespaces() {
631        let file_content = include_test_file!("list-item.rtf");
632        let tokens = Lexer::scan(file_content).unwrap();
633        let mut parser = Parser::new(tokens);
634        let document = parser.parse().unwrap();
635        assert_eq!(
636            document.body,
637            vec![StyleBlock {
638                painter: Painter { font_size: 24, ..Painter::default() },
639                paragraph: Default::default(),
640                text: "\nEmpty start\n\nList test : \n - item 1\n - item 2\n - item 3\n - item 4".into(),
641            },]
642        );
643    }
644
645    #[test]
646    fn parse_image_data() {
647        // Try to parse without error
648        let rtf_content = include_test_file!("file-with-image.rtf");
649        let tokens = Lexer::scan(rtf_content).unwrap();
650        let _document = Parser::new(tokens).parse();
651    }
652
653    #[test]
654    fn parse_header_and_body() {
655        let rtf = r#"{\rtf1\ansi\ansicpg1252\cocoartf2639
656\cocoatextscaling0\cocoaplatform0{\fonttbl\f0\froman\fcharset0 Times-Bold;\f1\froman\fcharset0 Times-Roman;\f2\froman\fcharset0 Times-Italic;
657\f3\fswiss\fcharset0 Helvetica;}
658{\colortbl;\red255\green255\blue255;\red0\green0\blue10;\red0\green0\blue1;\red191\green191\blue191;
659}
660\f0\b\fs21 \cf2 Lorem ipsum
661\fs56 \
662\pard\pardeftab709\sl288\slmult1\sa225\qj\partightenfactor0
663
664\f1\b0\fs21 \cf0 \
665\pard\pardeftab709\fi-432\ri-1\sb240\sa120\partightenfactor0
666\ls1\ilvl0
667\f0\b\fs36\cf2\plain Lorem ipsum dolor sit amet, consectetur adipiscing elit. Nunc ac faucibus odio. \
668\pard\pardeftab709\sl288\slmult1\sa225\qj\partightenfactor0
669}"#;
670        let tokens = Lexer::scan(rtf).unwrap();
671        let document = Parser::new(tokens).parse().unwrap();
672        assert_eq!(document.body[0].text, "Lorem ipsum");
673        assert_eq!(document.body[1].text, "\n");
674        assert_eq!(document.body[2].text, "\n");
675        assert_eq!(document.body[3].text, "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Nunc ac faucibus odio. \n");
676    }
677
678    #[test]
679    fn parse_paragraph_aligment() {
680        let rtf = r#"{\rtf1\ansi\deff0 {\fonttbl {\f0 Times;}}
681        \fs34
682        {\pard \qc \fs60 Annalium Romae\par}
683        {\pard \qj
684            Urbem Romam a principio reges habuere; libertatem et
685            \par}
686        {\pard \ql
687            Non Cinnae, non Sullae longa dominatio; et Pompei Crassique potentia
688            \par}"#;
689        let tokens = Lexer::scan(rtf).unwrap();
690        let document = Parser::new(tokens).parse().unwrap();
691        assert_eq!(document.body[0].paragraph.alignment, Alignment::Center);
692        assert_eq!(document.body[1].paragraph.alignment, Alignment::Justify);
693        assert_eq!(document.body[2].paragraph.alignment, Alignment::LeftAligned);
694    }
695
696    #[test]
697    fn should_parse_escaped_char() {
698        let rtf = r"{\rtf1\ansi\deff0 {\fonttbl {\f0 Times;}}je suis une b\'eate}";
699        let tokens = Lexer::scan(rtf).unwrap();
700        let document = Parser::new(tokens).parse().unwrap();
701        assert_eq!(document.body[0].text, "je suis une bête");
702    }
703
704    #[test]
705    fn parse_plain_directive() {
706        let rtf = r"{\rtf1{\fonttbl {\f0 Times;}}\f0\b\fs36\u\cf2\plain Plain text}";
707        let tokens = Lexer::scan(rtf).unwrap();
708        let document = Parser::new(tokens).parse().unwrap();
709        assert_eq!(document.body[0].painter, Painter::default());
710    }
711
712    #[test]
713    fn parse_color_table() {
714        // cf0 is unset color, cf1 is first color, cf2 is second color, etc ...
715        let rtf = r#"{\rtf1\ansi\ansicpg936\cocoartf2761
716            \cocoatextscaling0\cocoaplatform0{\fonttbl\f0\fswiss\fcharset0 Helvetica;\f1\fnil\fcharset134 PingFangSC-Regular;}
717            {\colortbl;\red255\green255\blue255;\red251\green2\blue7;\red114\green44\blue253;}
718            {\*\expandedcolortbl;;\cssrgb\c100000\c14913\c0;\cssrgb\c52799\c30710\c99498;}
719            \f0\fs24 \cf2 A
720            \f1 \cf3 B}"#;
721        let tokens = Lexer::scan(rtf).unwrap();
722        let document = Parser::new(tokens).parse().unwrap();
723        assert_eq!(document.header.color_table.get(&document.body[0].painter.color_ref).unwrap(), &Color { red: 251, green: 2, blue: 7 });
724    }
725
726    #[test]
727    fn parse_underline() {
728        // \\ul underline true
729        // \\ulnone underline false
730        let rtf = r#"{\rtf1\ansi\ansicpg936\cocoartf2761
731            \cocoatextscaling0\cocoaplatform0{\fonttbl\f0\fswiss\fcharset0 Helvetica;}
732            {\colortbl;\red255\green255\blue255;}
733            {\*\expandedcolortbl;;}
734            \paperw11900\paperh16840\margl1440\margr1440\vieww11520\viewh8400\viewkind0
735            \pard\tx720\tx1440\tx2160\tx2880\tx3600\tx4320\tx5040\tx5760\tx6480\tx7200\tx7920\tx8640\pardirnatural\partightenfactor0
736
737            \f0\fs24 \cf0 \ul \ulc0 a\ulnone A}"#;
738        let tokens = Lexer::scan(rtf).unwrap();
739        let document = Parser::new(tokens).parse().unwrap();
740        assert_eq!(&document.body[0].painter.underline, &true);
741        assert_eq!(&document.body[1].painter.underline, &false);
742    }
743
744    #[test]
745    fn parse_unicode() {
746        // start with \\uc0
747        // \u21834 => 啊
748        let rtf = r#"{\rtf1\ansi\ansicpg936\cocoartf2761
749            \cocoatextscaling0\cocoaplatform0{\fonttbl\f0\fswiss\fcharset0 Helvetica;}
750            \f0\fs24 \cf0 \uc0\u21834  \u21834 }"#;
751        let tokens = Lexer::scan(rtf).unwrap();
752        let document = Parser::new(tokens).parse().unwrap();
753        assert_eq!(&document.body[0].text, "啊 啊");
754    }
755
756    #[test]
757    fn parse_two_characters_compound_unicode() {
758        let rtf = r#"{\rtf1\ansi
759            \f0 a\u55357 \u56447 1 \u21834}"#;
760        let tokens = Lexer::scan(rtf).unwrap();
761        let document = Parser::new(tokens).parse().unwrap();
762        assert_eq!(&document.body[0].text, "a👿1 啊");
763    }
764
765    #[test]
766    fn parse_unicode_with_fallback() {
767        // Should only consider the first unicode, not the two fallback chars
768        let rtf = r#"{\rtf1\ansi
769            {\f0 \u-10179\'5f\u-9089\'5f}
770            {\f1 \uc2\u32767\'c2\'52}
771            {\f2 \uc2\u26789\'97\'73}
772            {\f3 b\'eate}
773            {\f4 \uc0 b\'ea\'eate}
774           }"#;
775        let tokens = Lexer::scan(rtf).unwrap();
776        let document = Parser::new(tokens).parse().unwrap();
777        assert_eq!(&document.body[0].text, "👿");
778        assert_eq!(&document.body[1].text, "翿");
779        assert_eq!(&document.body[2].text, "梥");
780        assert_eq!(&document.body[3].text, "bête");
781        assert_eq!(&document.body[4].text, "bêête");
782    }
783
784    #[test]
785    fn body_starts_with_a_group() {
786        let rtf = r"{\rtf1\ansi\deff0{\fonttbl {\f0\fnil\fcharset0 Calibri;}{\f1\fnil\fcharset2 Symbol;}}{\colortbl ;}{\pard \u21435  \sb70\par}}";
787        let tokens = Lexer::scan(rtf).unwrap();
788        let _document = Parser::new(tokens).parse().unwrap();
789    }
790
791    #[test]
792    fn rtf_different_semantic() {
793        let rtf1 = r"{\rtf1 \b bold \i Bold Italic \i0 Bold again}";
794        let rtf2 = r"{\rtf1 \b bold {\i Bold Italic }Bold again}";
795        let rtf3 = r"{\rtf1 \b bold \i Bold Italic \plain\b Bold again}";
796        let doc1 = RtfDocument::try_from(rtf1).unwrap();
797        let doc2 = RtfDocument::try_from(rtf2).unwrap();
798        let doc3 = RtfDocument::try_from(rtf3).unwrap();
799        assert_eq!(doc1.body, doc2.body);
800        assert_eq!(doc3.body, doc2.body);
801    }
802
803    #[test]
804    fn parse_emdash() {
805        let rtf = r"{\rtf1\ansi hello\emdash world}";
806        let doc = RtfDocument::try_from(rtf).unwrap();
807        let text: String = doc.body.iter().map(|b| b.text.as_str()).collect();
808        assert!(text.contains("\u{2014}"), "Em-dash not found in: {}", text);
809        assert!(text.contains("hello\u{2014}world"), "Expected 'hello—world', got: {}", text);
810    }
811
812    #[test]
813    fn parse_endash() {
814        let rtf = r"{\rtf1\ansi 2020\endash 2025}";
815        let doc = RtfDocument::try_from(rtf).unwrap();
816        let text: String = doc.body.iter().map(|b| b.text.as_str()).collect();
817        assert!(text.contains("\u{2013}"), "En-dash not found in: {}", text);
818    }
819
820    #[test]
821    fn parse_smart_quotes() {
822        let rtf = r"{\rtf1\ansi \ldblquote Hello\rdblquote  and \lquote hi\rquote}";
823        let doc = RtfDocument::try_from(rtf).unwrap();
824        let text: String = doc.body.iter().map(|b| b.text.as_str()).collect();
825        assert!(text.contains("\u{201C}"), "Left double quote not found");
826        assert!(text.contains("\u{201D}"), "Right double quote not found");
827        assert!(text.contains("\u{2018}"), "Left single quote not found");
828        assert!(text.contains("\u{2019}"), "Right single quote not found");
829    }
830
831    #[test]
832    fn parse_bullet() {
833        let rtf = r"{\rtf1\ansi \bullet Item one}";
834        let doc = RtfDocument::try_from(rtf).unwrap();
835        let text: String = doc.body.iter().map(|b| b.text.as_str()).collect();
836        assert!(text.contains("\u{2022}"), "Bullet not found in: {}", text);
837    }
838
839    #[test]
840    fn parse_tab_and_line() {
841        let rtf = r"{\rtf1\ansi col1\tab col2\line next}";
842        let doc = RtfDocument::try_from(rtf).unwrap();
843        let text: String = doc.body.iter().map(|b| b.text.as_str()).collect();
844        assert!(text.contains("\t"), "Tab not found in: {}", text);
845        assert!(text.contains("\n"), "Line break not found in: {}", text);
846    }
847
848    #[test]
849    fn parse_special_chars_in_scrivener_style() {
850        // Simulates Scrivener RTF output
851        let rtf = r"{\rtf1\ansi\ansicpg1252\deff0
852{\fonttbl{\f0\fnil\fcharset0 TimesNewRomanPSMT;}}
853\f0\fs24 The transformation in reverse\emdash confident expert to tired father.\par
854He said, \ldblquote Hello.\rdblquote\par}";
855        let doc = RtfDocument::try_from(rtf).unwrap();
856        let text: String = doc.body.iter().map(|b| b.text.as_str()).collect();
857        assert!(text.contains("reverse\u{2014}confident"),
858            "Em-dash not properly placed in: {}", text);
859        assert!(text.contains("\u{201C}Hello.\u{201D}"),
860            "Smart quotes not properly placed in: {}", text);
861    }
862}
rtf_parser_tt/parser.rs

rtf_parser_tt/
parser.rs
