---
source: src/tokenizer/markdown_tokenizer.rs
expression: "markdown_tokenizer(\"# \\u{1F600} Héllo\\n- \\u{00E9}lément\\n> \\u{4F60}\\u{597D} world\")"
---
[
Token {
normalized: "# 😀",
original: "# 😀",
is_left_joinable: false,
is_right_joinable: true,
},
Token {
normalized: " Héllo",
original: " ",
is_left_joinable: true,
is_right_joinable: true,
},
Token {
normalized: "Héllo",
original: "Héllo",
is_left_joinable: true,
is_right_joinable: true,
},
Token {
normalized: "\n",
original: "\n",
is_left_joinable: false,
is_right_joinable: false,
},
Token {
normalized: "- élément",
original: "- élément",
is_left_joinable: false,
is_right_joinable: true,
},
Token {
normalized: "\n",
original: "\n",
is_left_joinable: false,
is_right_joinable: false,
},
Token {
normalized: "> 你好",
original: "> 你好",
is_left_joinable: false,
is_right_joinable: true,
},
Token {
normalized: " world",
original: " ",
is_left_joinable: true,
is_right_joinable: true,
},
Token {
normalized: "world",
original: "world",
is_left_joinable: true,
is_right_joinable: true,
},
]