pub(crate) fn copy_decode(s: &[u8]) -> String {
let end = s.len();
let mut out = Vec::with_capacity(end);
let mut i = 0;
while i < end {
if i == end - 1 && s[i] != b'_' {
out.push(s[i]);
break;
}
let (byte, consumed) = match s[i] {
b'_' => (b' ', i),
b'\\' => (s[i + 1], i + 1),
other => (other, i),
};
out.push(byte);
i = consumed + 1;
}
String::from_utf8_lossy(&out).into_owned()
}
#[must_use]
pub fn fields(line: &str) -> Vec<String> {
let s = line.as_bytes();
let mut out = Vec::new();
let mut start = 0;
for i in 0..s.len() {
if s[i] == b' ' || s[i] == b'\t' {
if start != i {
out.push(copy_decode(&s[start..i]));
}
start = i + 1;
}
}
if start != s.len() {
out.push(copy_decode(&s[start..]));
}
out
}
#[must_use]
pub fn cut_space(x: &str) -> &str {
let bytes = x.as_bytes();
let len = bytes.len();
if len == 0 {
return x;
}
if x == " " {
return "";
}
let start = usize::from(bytes[0] == b' ');
let stop = if bytes[len - 1] == b' ' { len - 1 } else { len };
&x[start..stop]
}
#[must_use]
pub fn rest_after<'a>(raw: &'a str, keyword: &str) -> &'a str {
raw.strip_prefix(keyword)
.map_or("", |rest| rest.strip_prefix(' ').unwrap_or(rest))
}
#[derive(Debug, Clone)]
pub struct Cursor<'a> {
tokens: &'a [String],
pos: usize,
}
impl<'a> Cursor<'a> {
#[must_use]
pub fn new(tokens: &'a [String]) -> Self {
Self { tokens, pos: 0 }
}
#[must_use]
pub fn is_empty(&self) -> bool {
self.pos >= self.tokens.len()
}
#[must_use]
pub fn peek(&self) -> Option<&'a str> {
self.tokens.get(self.pos).map(String::as_str)
}
#[must_use]
pub fn peek_at(&self, n: usize) -> Option<&'a str> {
self.tokens.get(self.pos + n).map(String::as_str)
}
pub fn advance(&mut self) -> Option<&'a str> {
let token = self.tokens.get(self.pos).map(String::as_str);
if token.is_some() {
self.pos += 1;
}
token
}
pub fn eat(&mut self, token: &str) -> bool {
if self.peek() == Some(token) {
self.pos += 1;
true
} else {
false
}
}
pub fn eat_any(&mut self, tokens: &[&'static str]) -> Option<&'static str> {
let next = self.peek()?;
let matched = tokens.iter().copied().find(|t| *t == next)?;
self.pos += 1;
Some(matched)
}
pub fn field(&mut self, tag: &str) -> String {
match (self.peek(), self.peek_at(1)) {
(Some(t), Some(value)) if t == tag => {
self.pos += 2;
cut_space(value).to_owned()
}
_ => String::new(),
}
}
#[must_use]
pub fn remaining(&self) -> &'a [String] {
&self.tokens[self.pos.min(self.tokens.len())..]
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn underscores_become_spaces() {
assert_eq!(fields("Jean_Pierre"), vec!["Jean Pierre"]);
assert_eq!(fields("a b\tc"), vec!["a", "b", "c"]);
assert_eq!(fields(" a b "), vec!["a", "b"]);
assert_eq!(fields("").len(), 0);
}
#[test]
fn backslash_escapes_the_next_byte() {
assert_eq!(fields(r"a\_b"), vec!["a_b"]);
assert_eq!(fields(r"a\\b"), vec![r"a\b"]);
}
#[test]
fn a_trailing_backslash_is_literal() {
assert_eq!(fields(r"ab\"), vec![r"ab\"]);
}
#[test]
fn a_trailing_underscore_still_becomes_a_space() {
assert_eq!(fields("ab_"), vec!["ab "]);
}
#[test]
fn cut_space_removes_one_space_not_all() {
assert_eq!(cut_space(" Foo"), " Foo");
assert_eq!(cut_space(" Foo "), "Foo");
assert_eq!(cut_space("Foo"), "Foo");
assert_eq!(cut_space(" "), "");
assert_eq!(cut_space(" "), "");
assert_eq!(cut_space(""), "");
}
#[test]
fn leading_space_survives_the_full_pipeline() {
let toks = fields("#bp _[Châlons-sur-Marne]_-_Châlons-en-Champagne,51");
assert_eq!(toks[0], "#bp");
assert_eq!(toks[1], " [Châlons-sur-Marne] - Châlons-en-Champagne,51");
assert_eq!(
cut_space(&toks[1]),
"[Châlons-sur-Marne] - Châlons-en-Champagne,51"
);
}
#[test]
fn multibyte_characters_survive_tokenization() {
assert_eq!(fields("déjà_décédés"), vec!["déjà décédés"]);
assert_eq!(fields("Thérèse_Eugénie"), vec!["Thérèse Eugénie"]);
}
#[test]
fn field_consumes_only_on_match() {
let toks = fields("#bp Paris #bs acte");
let mut cur = Cursor::new(&toks);
assert_eq!(cur.field("#dp"), "");
assert_eq!(cur.peek(), Some("#bp"));
assert_eq!(cur.field("#bp"), "Paris");
assert_eq!(cur.field("#bs"), "acte");
assert!(cur.is_empty());
}
#[test]
fn field_without_a_value_is_not_consumed() {
let toks = fields("#bp");
let mut cur = Cursor::new(&toks);
assert_eq!(cur.field("#bp"), "");
assert_eq!(cur.peek(), Some("#bp"));
}
#[test]
fn rest_after_keeps_free_text_verbatim() {
assert_eq!(rest_after("comm a_b \\c", "comm"), "a_b \\c");
assert_eq!(rest_after("note", "note"), "");
assert_eq!(rest_after("note <br>1880 - x", "note"), "<br>1880 - x");
}
}