reconcile_text/tokenizer/
token.rs

1use std::fmt::Debug;
2
3#[cfg(feature = "serde")]
4use serde::{Deserialize, Serialize};
5
6/// A token with a normalized form (used for diffing) and an original form
7/// (used when applying operations). Joinability flags control whether
8/// adjacent insertions interleave or group.
9///
10/// UTF-8 compatible.
11#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
12#[derive(Debug, Clone)]
13pub struct Token<T>
14where
15    T: PartialEq + Clone + Debug,
16{
17    /// The normalized form of the token used deriving the diff
18    normalized: T,
19
20    /// The original string, that should be inserted or deleted in the document
21    original: String,
22
23    /// Whether the token is semantically joinable with the previous token
24    pub is_left_joinable: bool,
25
26    /// Whether the token is semantically joinable with the next token
27    pub is_right_joinable: bool,
28}
29
30/// Trivial implementation of Token when the normalized form is the same as the
31/// original string
32impl From<&str> for Token<String> {
33    fn from(text: &str) -> Self {
34        Token::new(text.to_owned(), text.to_owned(), true, true)
35    }
36}
37
38impl<T> Token<T>
39where
40    T: PartialEq + Clone + Debug,
41{
42    pub fn new(
43        normalized: T,
44        original: String,
45        is_left_joinable: bool,
46        is_right_joinable: bool,
47    ) -> Self {
48        Token {
49            normalized,
50            original,
51            is_left_joinable,
52            is_right_joinable,
53        }
54    }
55
56    pub fn original(&self) -> &str {
57        &self.original
58    }
59
60    pub fn set_normalized(&mut self, normalized: T) {
61        self.normalized = normalized;
62    }
63
64    pub fn normalized(&self) -> &T {
65        &self.normalized
66    }
67
68    pub fn get_original_length(&self) -> usize {
69        self.original.chars().count()
70    }
71}
72
73impl<T> PartialEq for Token<T>
74where
75    T: PartialEq + Clone + Debug,
76{
77    fn eq(&self, other: &Self) -> bool {
78        self.normalized == other.normalized
79    }
80}
reconcile_text/tokenizer/token.rs

reconcile_text/tokenizer/
token.rs
