rama_http/protocols/html/tokenizer/
token.rs

1//! Borrowed token views handed to a [`TokenSink`](super::TokenSink).
2//!
3//! Each view borrows the input buffer for the duration of the sink call and
4//! exposes byte slices into it — no per-token allocation. Every view also
5//! exposes its full `raw()` span (the exact input bytes it covers); the raw
6//! spans of all tokens partition the input contiguously, which is what makes
7//! verbatim re-serialization (the identity property) possible.
8
9use std::borrow::Cow;
10
11use super::super::decode_entities;
12use super::name::LocalNameHash;
13use super::tag::HtmlTag;
14
15/// UTF-8-lossy decode `bytes` and resolve HTML entities, borrowing the input
16/// when both are no-ops.
17fn decode_lossy(bytes: &[u8]) -> Cow<'_, str> {
18    match String::from_utf8_lossy(bytes) {
19        Cow::Borrowed(s) => decode_entities(s),
20        Cow::Owned(s) => Cow::Owned(decode_entities(&s).into_owned()),
21    }
22}
23
24/// Half-open `[start, end)` byte span into the input buffer.
25#[derive(Debug, Clone, Copy, PartialEq, Eq)]
26pub(crate) struct Span {
27    pub(crate) start: usize,
28    pub(crate) end: usize,
29}
30
31impl Span {
32    pub(crate) const fn new(start: usize, end: usize) -> Self {
33        Self { start, end }
34    }
35
36    pub(crate) const fn empty(at: usize) -> Self {
37        Self { start: at, end: at }
38    }
39
40    fn slice(self, input: &[u8]) -> &[u8] {
41        input.get(self.start..self.end).unwrap_or(&[])
42    }
43}
44
45/// A single attribute's name and value spans (relative to the input).
46#[derive(Debug, Clone, Copy)]
47pub(crate) struct AttrRange {
48    pub(crate) name: Span,
49    pub(crate) value: Span,
50    pub(crate) has_value: bool,
51}
52
53/// A start tag, e.g. `<a href="/x">` or `<br/>`.
54#[derive(Debug)]
55pub struct StartTag<'i> {
56    pub(crate) input: &'i [u8],
57    pub(crate) raw: Span,
58    pub(crate) name: Span,
59    pub(crate) name_hash: LocalNameHash,
60    pub(crate) attributes: &'i [AttrRange],
61    pub(crate) self_closing: bool,
62}
63
64impl<'i> StartTag<'i> {
65    /// The element's [`HtmlTag`] — a known element variant, or
66    /// [`HtmlTag::Other`] (borrowing the original-case name) for a custom tag.
67    #[must_use]
68    pub fn tag(&self) -> HtmlTag<'i> {
69        HtmlTag::classify(self.name_hash, self.name())
70    }
71
72    /// The tag name bytes (original case).
73    pub(crate) fn name(&self) -> &'i [u8] {
74        self.name.slice(self.input)
75    }
76
77    /// The hash of the (ASCII-lowercased) tag name.
78    #[must_use]
79    pub fn name_hash(&self) -> LocalNameHash {
80        self.name_hash
81    }
82
83    /// Whether the tag was written self-closing (`<br/>`).
84    #[must_use]
85    pub fn is_self_closing(&self) -> bool {
86        self.self_closing
87    }
88
89    /// Iterator over the tag's attributes, in source order.
90    #[must_use]
91    pub fn attributes(&self) -> Attributes<'i> {
92        Attributes {
93            input: self.input,
94            ranges: self.attributes.iter(),
95        }
96    }
97
98    /// The full source bytes of the tag, including `<` and `>`.
99    #[must_use]
100    pub fn raw(&self) -> &'i [u8] {
101        self.raw.slice(self.input)
102    }
103}
104
105/// Iterator over a [`StartTag`]'s attributes.
106#[derive(Debug, Clone)]
107pub struct Attributes<'i> {
108    input: &'i [u8],
109    ranges: std::slice::Iter<'i, AttrRange>,
110}
111
112impl<'i> Iterator for Attributes<'i> {
113    type Item = Attribute<'i>;
114
115    fn next(&mut self) -> Option<Self::Item> {
116        let range = self.ranges.next()?;
117        Some(Attribute {
118            name: range.name.slice(self.input),
119            value: range.value.slice(self.input),
120            has_value: range.has_value,
121        })
122    }
123
124    fn size_hint(&self) -> (usize, Option<usize>) {
125        self.ranges.size_hint()
126    }
127}
128
129impl ExactSizeIterator for Attributes<'_> {}
130
131/// A single attribute view.
132#[derive(Debug, Clone, Copy)]
133pub struct Attribute<'i> {
134    name: &'i [u8],
135    value: &'i [u8],
136    has_value: bool,
137}
138
139impl<'i> Attribute<'i> {
140    /// The attribute name bytes (original case).
141    #[must_use]
142    pub fn name(&self) -> &'i [u8] {
143        self.name
144    }
145
146    /// The attribute value bytes (raw, not entity-decoded). Empty for a
147    /// valueless attribute or an empty value; use [`Attribute::has_value`]
148    /// to distinguish them.
149    #[must_use]
150    pub fn value(&self) -> &'i [u8] {
151        self.value
152    }
153
154    /// The attribute value as display text: UTF-8-lossy decoded with HTML
155    /// entities resolved. Borrows [`value`](Self::value) when it is already
156    /// valid UTF-8 with nothing to decode.
157    #[must_use]
158    pub fn value_decoded(&self) -> Cow<'i, str> {
159        decode_lossy(self.value)
160    }
161
162    /// Whether the attribute had an explicit `=value`.
163    #[must_use]
164    pub fn has_value(&self) -> bool {
165        self.has_value
166    }
167}
168
169/// An end tag, e.g. `</a>`.
170#[derive(Debug)]
171pub struct EndTag<'i> {
172    pub(crate) input: &'i [u8],
173    pub(crate) raw: Span,
174    pub(crate) name: Span,
175    pub(crate) name_hash: LocalNameHash,
176}
177
178impl<'i> EndTag<'i> {
179    /// The element's [`HtmlTag`] — a known element variant, or
180    /// [`HtmlTag::Other`] (borrowing the original-case name) for a custom tag.
181    #[must_use]
182    pub fn tag(&self) -> HtmlTag<'i> {
183        HtmlTag::classify(self.name_hash, self.name())
184    }
185
186    /// The tag name bytes (original case).
187    pub(crate) fn name(&self) -> &'i [u8] {
188        self.name.slice(self.input)
189    }
190
191    /// The hash of the (ASCII-lowercased) tag name.
192    #[must_use]
193    pub fn name_hash(&self) -> LocalNameHash {
194        self.name_hash
195    }
196
197    /// The full source bytes of the tag, including `</` and `>`.
198    #[must_use]
199    pub fn raw(&self) -> &'i [u8] {
200        self.raw.slice(self.input)
201    }
202}
203
204/// A run of character data (text). Raw bytes, not entity-decoded.
205#[derive(Debug)]
206pub struct Text<'i> {
207    pub(crate) input: &'i [u8],
208    pub(crate) raw: Span,
209}
210
211impl<'i> Text<'i> {
212    /// The text bytes.
213    #[must_use]
214    pub fn as_bytes(&self) -> &'i [u8] {
215        self.raw.slice(self.input)
216    }
217
218    /// The text as display text: UTF-8-lossy decoded with HTML entities
219    /// resolved. Borrows [`as_bytes`](Self::as_bytes) when it is already valid
220    /// UTF-8 with nothing to decode.
221    #[must_use]
222    pub fn decoded(&self) -> Cow<'i, str> {
223        decode_lossy(self.raw.slice(self.input))
224    }
225
226    /// The full source bytes of the text (same as [`Text::as_bytes`]).
227    #[must_use]
228    pub fn raw(&self) -> &'i [u8] {
229        self.raw.slice(self.input)
230    }
231}
232
233/// A comment, e.g. `<!-- hi -->`.
234#[derive(Debug)]
235pub struct Comment<'i> {
236    pub(crate) input: &'i [u8],
237    pub(crate) raw: Span,
238    pub(crate) data: Span,
239}
240
241impl<'i> Comment<'i> {
242    /// The comment's inner data bytes (without `<!--` / `-->`).
243    #[must_use]
244    pub fn data(&self) -> &'i [u8] {
245        self.data.slice(self.input)
246    }
247
248    /// The full source bytes of the comment.
249    #[must_use]
250    pub fn raw(&self) -> &'i [u8] {
251        self.raw.slice(self.input)
252    }
253}
254
255/// A CDATA section, e.g. `<![CDATA[ x ]]>` (only emitted inside foreign
256/// content — SVG/MathML; elsewhere `<![CDATA[` is a bogus comment).
257#[derive(Debug)]
258pub struct Cdata<'i> {
259    pub(crate) input: &'i [u8],
260    pub(crate) raw: Span,
261    pub(crate) data: Span,
262}
263
264impl<'i> Cdata<'i> {
265    /// The section's inner data bytes (without `<![CDATA[` / `]]>`).
266    #[must_use]
267    pub fn data(&self) -> &'i [u8] {
268        self.data.slice(self.input)
269    }
270
271    /// The full source bytes of the CDATA section.
272    #[must_use]
273    pub fn raw(&self) -> &'i [u8] {
274        self.raw.slice(self.input)
275    }
276}
277
278/// A document type declaration, e.g. `<!DOCTYPE html>`.
279#[derive(Debug)]
280pub struct Doctype<'i> {
281    pub(crate) input: &'i [u8],
282    pub(crate) raw: Span,
283    pub(crate) name: Option<Span>,
284}
285
286impl<'i> Doctype<'i> {
287    /// The doctype name bytes (original case), if present.
288    #[must_use]
289    pub fn name(&self) -> Option<&'i [u8]> {
290        self.name.map(|span| span.slice(self.input))
291    }
292
293    /// The full source bytes of the doctype declaration.
294    #[must_use]
295    pub fn raw(&self) -> &'i [u8] {
296        self.raw.slice(self.input)
297    }
298}
rama_http/protocols/html/tokenizer/token.rs

rama_http/protocols/html/tokenizer/
token.rs
