use itertools::Itertools;
use unicode_segmentation::UnicodeSegmentation;
use crate::IgnoreWhitespace;
pub fn split_lines(content: &[u8]) -> Vec<&[u8]> {
if content.is_empty() {
return vec![];
}
let content = content
.strip_suffix(b"\r\n")
.or_else(|| content.strip_suffix(b"\n"))
.or_else(|| content.strip_suffix(b"\r"))
.unwrap_or(content);
let mut lines = Vec::with_capacity(content.len() / 80);
let mut start = 0;
let mut i = 0;
while i < content.len() {
match (content.get(i), content.get(i + 1)) {
(Some(b'\r'), Some(b'\n')) => {
lines.push(&content[start..i]);
i += 2;
start = i;
}
(Some(b'\r') | Some(b'\n'), _) => {
lines.push(&content[start..i]);
i += 1;
start = i;
}
_ => {
i += 1;
}
}
}
lines.push(&content[start..]);
lines
}
pub fn lines_to_bytes<T: AsRef<[u8]>>(lines: &[T]) -> Vec<u8> {
let mut out =
Vec::with_capacity(lines.iter().map(|l| l.as_ref().len() + 1).sum());
for l in lines {
out.extend_from_slice(l.as_ref());
out.push(b'\n');
}
out
}
pub fn is_whitespace_token(token: &[u8]) -> bool {
!token.is_empty() && token.iter().all(|b| (*b as char).is_ascii_whitespace())
}
pub fn tokenize_lines<'a>(lines: &[&'a [u8]]) -> Vec<&'a [u8]> {
let mut v: Vec<_> = lines
.iter()
.flat_map(|l| ByteTokenIter::new(l).chain([b"\n" as &[u8]]))
.collect();
v.pop();
v
}
#[derive(Clone, Copy, Debug)]
pub struct NormalizedBytes<'a> {
pub bytes: &'a [u8],
pub mode: IgnoreWhitespace,
}
impl<'a> NormalizedBytes<'a> {
pub fn new(bytes: &'a [u8], mode: IgnoreWhitespace) -> Self {
NormalizedBytes { bytes, mode }
}
}
impl<'a> PartialEq for NormalizedBytes<'a> {
fn eq(&self, other: &Self) -> bool {
if self.mode != other.mode {
return self.bytes == other.bytes;
}
match self.mode {
IgnoreWhitespace::No => self.bytes == other.bytes,
IgnoreWhitespace::LengthChanges => {
let it1 = ByteTokenIter::new(self.bytes);
let it2 = ByteTokenIter::new(other.bytes);
it1.zip_longest(it2).all(|pair| match pair {
itertools::EitherOrBoth::Both(t1, t2) => {
if is_whitespace_token(t1) && is_whitespace_token(t2) {
true
} else {
t1 == t2
}
}
_ => false,
})
}
IgnoreWhitespace::All => {
let it1 = self
.bytes
.iter()
.filter(|&&b| !(b as char).is_ascii_whitespace());
let it2 = other
.bytes
.iter()
.filter(|&&b| !(b as char).is_ascii_whitespace());
it1.eq(it2)
}
}
}
}
impl<'a> Eq for NormalizedBytes<'a> {}
impl<'a> std::hash::Hash for NormalizedBytes<'a> {
fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
match self.mode {
IgnoreWhitespace::No => self.bytes.hash(state),
IgnoreWhitespace::LengthChanges => {
for t in ByteTokenIter::new(self.bytes) {
if is_whitespace_token(t) {
state.write(b" ");
} else {
state.write(t);
}
}
}
IgnoreWhitespace::All => {
for &b in self
.bytes
.iter()
.filter(|&&b| !(b as char).is_ascii_whitespace())
{
state.write_u8(b);
}
}
}
}
}
struct ByteTokenIter<'a> {
content: &'a [u8],
valid: &'a str,
}
impl<'a> ByteTokenIter<'a> {
fn new(content: &'a [u8]) -> Self {
ByteTokenIter { content, valid: valid_prefix(content) }
}
}
fn valid_prefix(content: &[u8]) -> &str {
match std::str::from_utf8(content) {
Ok(s) => s,
Err(e) => unsafe {
std::str::from_utf8_unchecked(&content[..e.valid_up_to()])
},
}
}
impl<'a> Iterator for ByteTokenIter<'a> {
type Item = &'a [u8];
fn next(&mut self) -> Option<&'a [u8]> {
if self.content.is_empty() {
return None;
}
if self.valid.is_empty() {
let n = std::str::from_utf8(self.content)
.err()
.and_then(|e| e.error_len())
.unwrap_or(1);
let (token, rest) = self.content.split_at(n);
self.content = rest;
self.valid = valid_prefix(self.content);
return Some(token);
}
let mut graphemes = self.valid.grapheme_indices(true);
let (_, first) = graphemes.next().unwrap();
let first_char = first.chars().next().unwrap();
let scan = |pred: &dyn Fn(char) -> bool| {
let mut end = first.len();
for (off, g) in graphemes {
if g.chars().next().is_some_and(pred) {
end = off + g.len();
} else {
break;
}
}
end
};
let end = if first_char.is_ascii_whitespace() {
scan(&|c| c.is_ascii_whitespace())
} else if first_char.is_ascii_digit() {
scan(&|c| c.is_ascii_digit())
} else if first_char.is_ascii_alphabetic() || first_char == '_' {
scan(&|c| c.is_ascii_alphanumeric() || c == '_')
} else {
first.len()
};
let (token, rest) = self.content.split_at(end);
self.content = rest;
self.valid = &self.valid[end..];
Some(token)
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn split_lines_lf() {
assert_eq!(split_lines(b"a\nb\nc\n"), &[b"a", b"b", b"c"]);
assert_eq!(split_lines(b"a\nb\nc"), &[b"a", b"b", b"c"]);
}
#[test]
fn split_lines_crlf() {
assert_eq!(split_lines(b"a\r\nb\r\nc\r\n"), &[b"a", b"b", b"c"]);
assert_eq!(split_lines(b"a\r\nb\r\nc"), &[b"a", b"b", b"c"]);
}
#[test]
fn split_lines_cr() {
assert_eq!(split_lines(b"a\rb\rc\r"), &[b"a", b"b", b"c"]);
assert_eq!(split_lines(b"a\rb\rc"), &[b"a", b"b", b"c"]);
}
#[test]
fn split_lines_mixed() {
assert_eq!(split_lines(b"a\r\nb\nc\rd"), &[b"a", b"b", b"c", b"d"]);
}
#[test]
fn tokenize_invalid_utf8() {
assert_eq!(
tokenize_lines(&[b"foo\xffbar"]),
&[b"foo" as &[u8], b"\xff", b"bar"],
);
}
#[test]
fn tokenize_combining_diacritics() {
assert_eq!(tokenize_lines(&[b"cafe\xcc\x81"]), &[b"cafe\xcc\x81" as &[u8]],);
assert_eq!(
tokenize_lines(&[b"caf\xc3\xae"]),
&[b"caf" as &[u8], b"\xc3\xae" as &[u8]],
);
}
#[test]
fn tokenize() {
assert_eq!(
tokenize_lines(&[b"void func1() {", b" x += 1"]),
&[
b"void" as &[u8],
b" ",
b"func1",
b"(",
b")",
b" ",
b"{",
b"\n",
b" ",
b"x",
b" ",
b"+",
b"=",
b" ",
b"1"
],
);
}
use std::collections::hash_map::DefaultHasher;
use std::hash::Hash;
use std::hash::Hasher;
fn hash<T: Hash>(t: &T) -> u64 {
let mut s = DefaultHasher::new();
t.hash(&mut s);
s.finish()
}
#[test]
fn normalized_bytes_no() {
let b1 = NormalizedBytes::new(b"foo bar", IgnoreWhitespace::No);
let b2 = NormalizedBytes::new(b"foo bar", IgnoreWhitespace::No);
assert_ne!(b1, b2);
assert_ne!(hash(&b1), hash(&b2));
}
#[test]
fn normalized_bytes_length_changes() {
let b1 = NormalizedBytes::new(b"foo bar", IgnoreWhitespace::LengthChanges);
let b2 = NormalizedBytes::new(b"foo bar", IgnoreWhitespace::LengthChanges);
let b3 = NormalizedBytes::new(b"foobar", IgnoreWhitespace::LengthChanges);
assert_eq!(b1, b2);
assert_eq!(hash(&b1), hash(&b2));
assert_ne!(b1, b3);
}
#[test]
fn normalized_bytes_all() {
let b1 = NormalizedBytes::new(b"foo bar", IgnoreWhitespace::All);
let b2 = NormalizedBytes::new(b"foobar", IgnoreWhitespace::All);
let b3 = NormalizedBytes::new(b"foo bar ", IgnoreWhitespace::All);
assert_eq!(b1, b2);
assert_eq!(hash(&b1), hash(&b2));
assert_eq!(b1, b3);
assert_eq!(hash(&b1), hash(&b3));
}
}