nquads_syntax/
lexing.rs

1use decoded_char::DecodedChar;
2use locspan::{Span, Spanned};
3use rdf_syntax::{BlankIdBuf, IriBuf, LangTagBuf};
4use std::{fmt, iter::Peekable};
5
6/// Lexing error.
7#[derive(Debug)]
8pub enum NquadsLexingError<E = std::convert::Infallible> {
9	InvalidLangTag(Span),
10	InvalidCodepoint(u32, Span),
11	InvalidIriRef(String, Span),
12	Unexpected(Option<char>, Span),
13	Stream(E),
14}
15
16impl<E: fmt::Display> fmt::Display for NquadsLexingError<E> {
17	fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
18		match self {
19			Self::InvalidLangTag(_) => write!(f, "invalid language tag"),
20			Self::InvalidCodepoint(c, _) => write!(f, "invalid character code point {c:x}"),
21			Self::InvalidIriRef(iri_ref, _) => {
22				write!(f, "invalid IRI reference <{iri_ref}>")
23			}
24			Self::Unexpected(None, _) => write!(f, "unexpected end of file"),
25			Self::Unexpected(Some(c), _) => write!(f, "unexpected character `{c}`"),
26			Self::Stream(e) => e.fmt(f),
27		}
28	}
29}
30
31impl<E: 'static + std::error::Error> std::error::Error for NquadsLexingError<E> {
32	fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
33		match self {
34			Self::Stream(e) => Some(e),
35			_ => None,
36		}
37	}
38}
39
40/// Token.
41#[derive(Debug)]
42pub enum Token {
43	LangTag(LangTagBuf, Span),
44	Iri(IriBuf, Span),
45	StringLiteral(String, Span),
46	BlankNodeLabel(BlankIdBuf, Span),
47	Dot(Span),
48	Carets(Span),
49}
50
51impl Spanned for Token {
52	fn span(&self) -> Span {
53		match self {
54			Self::LangTag(_, s) => *s,
55			Self::Iri(_, s) => *s,
56			Self::StringLiteral(_, s) => *s,
57			Self::BlankNodeLabel(_, s) => *s,
58			Self::Dot(s) => *s,
59			Self::Carets(s) => *s,
60		}
61	}
62}
63
64impl fmt::Display for Token {
65	fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
66		match self {
67			Self::LangTag(tag, _) => write!(f, "language tag `{tag}`"),
68			Self::Iri(iri, _) => write!(f, "IRI <{iri}>"),
69			Self::StringLiteral(string, _) => {
70				write!(f, "string literal \"{}\"", DisplayStringLiteral(string))
71			}
72			Self::BlankNodeLabel(label, _) => write!(f, "blank node label `{label}`"),
73			Self::Dot(_) => write!(f, "dot `.`"),
74			Self::Carets(_) => write!(f, "carets `^^`"),
75		}
76	}
77}
78
79/// Wrapper to display string literals.
80pub struct DisplayStringLiteral<'a>(pub &'a str);
81
82impl fmt::Display for DisplayStringLiteral<'_> {
83	fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
84		for c in self.0.chars() {
85			match c {
86				'"' => write!(f, "\\u0022"),
87				'\\' => write!(f, "\\u005c"),
88				'\n' => write!(f, "\\n"),
89				'\r' => write!(f, "\\r"),
90				'\t' => write!(f, "\\t"),
91				'\u{08}' => write!(f, "\\b"),
92				'\u{0c}' => write!(f, "\\f"),
93				c => c.fmt(f),
94			}?
95		}
96
97		Ok(())
98	}
99}
100
101/// Characters iterator.
102struct Chars<C: Iterator>(Peekable<C>);
103
104impl<E, C: Iterator<Item = Result<DecodedChar, E>>> Chars<C> {
105	fn peek(&mut self) -> Result<Option<DecodedChar>, NquadsLexingError<E>> {
106		match self.0.peek() {
107			None => Ok(None),
108			Some(Ok(c)) => Ok(Some(*c)),
109			Some(Err(_)) => self.next(),
110		}
111	}
112
113	fn next(&mut self) -> Result<Option<DecodedChar>, NquadsLexingError<E>> {
114		self.0.next().transpose().map_err(NquadsLexingError::Stream)
115	}
116}
117
118/// Lexer position.
119#[derive(Default)]
120struct Position {
121	span: Span,
122	last_span: Span,
123}
124
125impl Position {
126	fn current(&self) -> Span {
127		self.span
128	}
129
130	fn end(&self) -> Span {
131		self.span.end.into()
132	}
133
134	fn last(&self) -> Span {
135		self.last_span
136	}
137}
138
139/// Lexer.
140///
141/// Changes a character iterator into a `Token` iterator.
142pub struct Lexer<C: Iterator<Item = Result<DecodedChar, E>>, E> {
143	chars: Chars<C>,
144	pos: Position,
145	// lookahead: Option<Meta<Token, Span>>,
146}
147
148impl<C: Iterator<Item = Result<DecodedChar, E>>, E> Lexer<C, E> {
149	pub fn new(chars: C) -> Self {
150		Self {
151			chars: Chars(chars.peekable()),
152			pos: Position::default(),
153		}
154	}
155}
156
157impl<E, C: Iterator<Item = Result<DecodedChar, E>>> Lexer<C, E> {
158	fn peek_decoded_char(&mut self) -> Result<Option<DecodedChar>, NquadsLexingError<E>> {
159		self.chars.peek()
160	}
161
162	fn peek_char(&mut self) -> Result<Option<char>, NquadsLexingError<E>> {
163		self.peek_decoded_char()
164			.map(|c| c.map(DecodedChar::into_char))
165	}
166
167	fn next_char(&mut self) -> Result<Option<char>, NquadsLexingError<E>> {
168		match self.chars.next()? {
169			Some(c) => {
170				self.pos.span.push(c.len());
171				self.pos.last_span.clear();
172				self.pos.last_span.push(c.len());
173				Ok(Some(*c))
174			}
175			None => Ok(None),
176		}
177	}
178
179	fn expect_char(&mut self) -> Result<char, NquadsLexingError<E>> {
180		self.next_char()?
181			.ok_or_else(|| NquadsLexingError::Unexpected(None, self.pos.last()))
182	}
183
184	fn skip_whitespaces(&mut self) -> Result<(), NquadsLexingError<E>> {
185		while let Some(c) = self.peek_char()? {
186			if c.is_whitespace() {
187				self.next_char()?;
188			} else if c == '#' {
189				self.next_comment()?;
190			} else {
191				break;
192			}
193		}
194
195		self.pos.span.clear();
196		Ok(())
197	}
198
199	/// Parses the rest of a comment, after the first `#` character.
200	///
201	/// Comments in N-Quads take the form of `#`,
202	/// outside an IRIREF or STRING_LITERAL_QUOTE,
203	/// and continue to the end of line (EOL) or end of file
204	/// if there is no end of line after the comment marker.
205	fn next_comment(&mut self) -> Result<(), NquadsLexingError<E>> {
206		loop {
207			if matches!(self.next_char()?, None | Some('\n')) {
208				break Ok(());
209			}
210		}
211	}
212
213	/// Parses the rest of a lang tag, after the first `@` character.
214	fn next_langtag(&mut self) -> Result<(LangTagBuf, Span), NquadsLexingError<E>> {
215		let mut tag = String::new();
216
217		loop {
218			match self.peek_char()? {
219				None => {
220					if tag.is_empty() {
221						return Err(NquadsLexingError::InvalidLangTag(self.pos.current()));
222					} else {
223						break;
224					}
225				}
226				Some(c) => {
227					if c.is_ascii_alphabetic() {
228						tag.push(self.expect_char()?);
229					} else if c.is_whitespace() || c == '-' {
230						if tag.is_empty() {
231							return Err(NquadsLexingError::InvalidLangTag(self.pos.current()));
232						} else {
233							break;
234						}
235					} else {
236						self.next_char()?;
237						return Err(NquadsLexingError::Unexpected(Some(c), self.pos.last()));
238					}
239				}
240			}
241		}
242
243		let mut empty_subtag = true;
244		if let Some('-') = self.peek_char()? {
245			tag.push(self.expect_char()?);
246			loop {
247				match self.peek_char()? {
248					Some('-') if !empty_subtag => tag.push(self.expect_char()?),
249					Some(c) if c.is_ascii_alphanumeric() => {
250						empty_subtag = false;
251						tag.push(self.expect_char()?)
252					}
253					Some(c) => {
254						if c.is_whitespace() {
255							if empty_subtag {
256								return Err(NquadsLexingError::InvalidLangTag(self.pos.current()));
257							} else {
258								break;
259							}
260						} else {
261							self.next_char()?;
262							return Err(NquadsLexingError::Unexpected(Some(c), self.pos.last()));
263						}
264					}
265					None => {
266						if empty_subtag {
267							return Err(NquadsLexingError::InvalidLangTag(self.pos.current()));
268						} else {
269							break;
270						}
271					}
272				}
273			}
274		}
275
276		match LangTagBuf::new(tag) {
277			Ok(tag) => Ok((tag, self.pos.current())),
278			Err(_) => Err(NquadsLexingError::InvalidLangTag(self.pos.current())),
279		}
280	}
281
282	/// Parses an IRI, starting after the first `<` until the closing `>`.
283	fn next_iri(&mut self) -> Result<(IriBuf, Span), NquadsLexingError<E>> {
284		let mut iri = String::new();
285
286		loop {
287			match self.next_char()? {
288				Some('>') => break,
289				Some('\\') => {
290					let span = self.pos.last();
291					let c = match self.next_char()? {
292						Some('u') => self.next_uchar(span, 4)?,
293						Some('U') => self.next_uchar(span, 8)?,
294						unexpected => {
295							return Err(NquadsLexingError::Unexpected(unexpected, self.pos.last()));
296						}
297					};
298
299					iri.push(c)
300				}
301				Some(c) => {
302					if matches!(
303						c,
304						'\u{00}'..='\u{20}' | '<' | '>' | '"' | '{' | '}' | '|' | '^' | '`' | '\\'
305					) {
306						return Err(NquadsLexingError::Unexpected(Some(c), self.pos.last()));
307					}
308
309					iri.push(c)
310				}
311				None => return Err(NquadsLexingError::Unexpected(None, self.pos.end())),
312			}
313		}
314
315		match IriBuf::new(iri) {
316			Ok(iri) => Ok((iri, self.pos.current())),
317			Err(e) => Err(NquadsLexingError::InvalidIriRef(e.0, self.pos.current())),
318		}
319	}
320
321	fn next_uchar(&mut self, mut span: Span, len: u8) -> Result<char, NquadsLexingError<E>> {
322		let mut codepoint = 0;
323
324		for _ in 0..len {
325			let c = self.expect_char()?;
326			match c.to_digit(16) {
327				Some(d) => codepoint = codepoint << 4 | d,
328				None => return Err(NquadsLexingError::Unexpected(Some(c), self.pos.last())),
329			}
330		}
331
332		span.end = self.pos.current().end;
333
334		match char::try_from(codepoint) {
335			Ok(c) => Ok(c),
336			Err(_) => Err(NquadsLexingError::InvalidCodepoint(codepoint, span)),
337		}
338	}
339
340	/// Parses a string literal, starting after the first `"` until the closing `"`.
341	fn next_string_literal(&mut self) -> Result<(String, Span), NquadsLexingError<E>> {
342		let mut string = String::new();
343
344		loop {
345			match self.next_char()? {
346				Some('"') => break,
347				Some('\\') => {
348					let span = self.pos.last();
349					let c = match self.next_char()? {
350						Some('u') => self.next_uchar(span, 4)?,
351						Some('U') => self.next_uchar(span, 8)?,
352						Some('t') => '\t',
353						Some('b') => '\u{08}',
354						Some('n') => '\n',
355						Some('r') => '\r',
356						Some('f') => '\u{0c}',
357						Some('\'') => '\'',
358						Some('"') => '"',
359						Some('\\') => '\\',
360						unexpected => {
361							return Err(NquadsLexingError::Unexpected(unexpected, self.pos.last()));
362						}
363					};
364
365					string.push(c)
366				}
367				Some(c) => {
368					if matches!(c, '\n' | '\r') {
369						return Err(NquadsLexingError::Unexpected(Some(c), self.pos.last()));
370					}
371
372					string.push(c)
373				}
374				None => return Err(NquadsLexingError::Unexpected(None, self.pos.end())),
375			}
376		}
377
378		Ok((string, self.pos.current()))
379	}
380
381	/// Parses a blank node label, starting after the first `_`.
382	fn next_blank_node_label(&mut self) -> Result<(BlankIdBuf, Span), NquadsLexingError<E>> {
383		match self.next_char()? {
384			Some(':') => {
385				let mut label = String::new();
386				label.push('_');
387				label.push(':');
388				match self.next_char()? {
389					Some(c) if c.is_ascii_digit() || is_pn_chars_u(c) => {
390						label.push(c);
391						let mut last_is_pn_chars = true;
392						loop {
393							match self.peek_char()? {
394								Some(c) if is_pn_chars(c) => {
395									label.push(self.expect_char()?);
396									last_is_pn_chars = true
397								}
398								Some('.') => {
399									label.push(self.expect_char()?);
400									last_is_pn_chars = false;
401								}
402								_ if last_is_pn_chars => break,
403								unexpected => {
404									return Err(NquadsLexingError::Unexpected(
405										unexpected,
406										self.pos.last(),
407									));
408								}
409							}
410						}
411
412						Ok((
413							unsafe { BlankIdBuf::new_unchecked(label) },
414							self.pos.current(),
415						))
416					}
417					unexpected => Err(NquadsLexingError::Unexpected(unexpected, self.pos.last())),
418				}
419			}
420			unexpected => Err(NquadsLexingError::Unexpected(unexpected, self.pos.last())),
421		}
422	}
423}
424
425impl<E, C: Iterator<Item = Result<DecodedChar, E>>> Iterator for Lexer<C, E> {
426	type Item = Result<Token, NquadsLexingError<E>>;
427
428	fn next(&mut self) -> Option<Self::Item> {
429		if let Err(e) = self.skip_whitespaces() {
430			return Some(Err(e));
431		}
432
433		match self.next_char() {
434			Ok(Some('@')) => match self.next_langtag() {
435				Ok((tag, span)) => Some(Ok(Token::LangTag(tag, span))),
436				Err(e) => Some(Err(e)),
437			},
438			Ok(Some('<')) => match self.next_iri() {
439				Ok((iri, span)) => Some(Ok(Token::Iri(iri, span))),
440				Err(e) => Some(Err(e)),
441			},
442			Ok(Some('"')) => match self.next_string_literal() {
443				Ok((string, span)) => Some(Ok(Token::StringLiteral(string, span))),
444				Err(e) => Some(Err(e)),
445			},
446			Ok(Some('_')) => match self.next_blank_node_label() {
447				Ok((blank_id, span)) => Some(Ok(Token::BlankNodeLabel(blank_id, span))),
448				Err(e) => Some(Err(e)),
449			},
450			Ok(Some('^')) => match self.next_char() {
451				Ok(Some('^')) => Some(Ok(Token::Carets(self.pos.current()))),
452				Ok(unexpected) => Some(Err(NquadsLexingError::Unexpected(
453					unexpected,
454					self.pos.last(),
455				))),
456				Err(e) => Some(Err(e)),
457			},
458			Ok(Some('.')) => Some(Ok(Token::Dot(self.pos.current()))),
459			Ok(Some(c)) => Some(Err(NquadsLexingError::Unexpected(Some(c), self.pos.last()))),
460			Ok(None) => None,
461			Err(e) => Some(Err(e)),
462		}
463	}
464}
465
466fn is_pn_chars_base(c: char) -> bool {
467	matches!(c, 'A'..='Z' | 'a'..='z' | '\u{00c0}'..='\u{00d6}' | '\u{00d8}'..='\u{00f6}' | '\u{00f8}'..='\u{02ff}' | '\u{0370}'..='\u{037d}' | '\u{037f}'..='\u{1fff}' | '\u{200c}'..='\u{200d}' | '\u{2070}'..='\u{218f}' | '\u{2c00}'..='\u{2fef}' | '\u{3001}'..='\u{d7ff}' | '\u{f900}'..='\u{fdcf}' | '\u{fdf0}'..='\u{fffd}' | '\u{10000}'..='\u{effff}')
468}
469
470fn is_pn_chars_u(c: char) -> bool {
471	is_pn_chars_base(c) || matches!(c, '_' | ':')
472}
473
474fn is_pn_chars(c: char) -> bool {
475	is_pn_chars_u(c)
476		|| matches!(c, '-' | '0'..='9' | '\u{00b7}' | '\u{0300}'..='\u{036f}' | '\u{203f}'..='\u{2040}')
477}
nquads_syntax/lexing.rs

nquads_syntax/
lexing.rs
