ftml 1.42.1

Foundation Text Markup Language - a library to render Wikidot text as HTML
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121

/*
 * wasm/tokenizer.rs
 *
 * ftml - Library to parse Wikidot text
 * Copyright (C) 2019-2026 Wikijump Team
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 */

use super::prelude::*;
use crate::Tokenization as RustTokenization;
use crate::parsing::ExtractedToken as RustExtractedToken;
use crate::utf16::Utf16IndexMap;
use self_cell::self_cell;
use std::sync::Arc;

// Typescript declarations

#[wasm_bindgen(typescript_custom_section)]
const TS_APPEND_CONTENT: &str = r#"
export interface IToken {
    token: string;
    slice: string;
    span: {
        start: number;
        end: number;
    };
}
"#;

#[wasm_bindgen]
extern "C" {
    #[wasm_bindgen(typescript_type = "IToken[]")]
    pub type ITokenArray;
}

// Wrapper structures

self_cell!(
    struct TokenizationInner {
        owner: String,

        #[covariant]
        dependent: RustTokenization,
    }

    impl {Debug}
);

#[wasm_bindgen]
#[derive(Debug)]
pub struct Tokenization {
    inner: Arc<TokenizationInner>,
}

#[wasm_bindgen]
impl Tokenization {
    #[inline]
    pub(crate) fn get(&self) -> &RustTokenization<'_> {
        self.inner.borrow_dependent()
    }

    #[wasm_bindgen]
    pub fn copy(&self) -> Tokenization {
        Tokenization {
            inner: Arc::clone(&self.inner),
        }
    }

    #[wasm_bindgen]
    pub fn text(&self) -> String {
        self.inner.borrow_owner().clone()
    }

    #[wasm_bindgen]
    pub fn tokens(&self) -> Result<ITokenArray, JsValue> {
        self.inner.with_dependent(|_, inner| {
            rust_to_js!(convert_tokens_utf16(inner)).map(|tokens| tokens.into())
        })
    }
}

// Exported functions

#[wasm_bindgen]
pub fn tokenize(text: String) -> Tokenization {
    let inner = TokenizationInner::new(text, |text: &String| crate::tokenize(text));

    Tokenization {
        inner: Arc::new(inner),
    }
}

// Utility functions

fn convert_tokens_utf16<'a>(
    tokenization: &'a RustTokenization,
) -> Vec<RustExtractedToken<'a>> {
    // Because the list of tokens is almost certainly not empty,
    // we don't perform the same check here that we do for errors.

    let full_text = tokenization.full_text().inner();
    let utf16_map = Utf16IndexMap::new(full_text);

    tokenization
        .tokens()
        .iter()
        .map(|token| token.to_utf16_indices(&utf16_map))
        .collect()
}