use super::model::{Anchor, ParseWarning, ParsedRange};
const HASH_LEN: usize = 4;
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) enum BlockTarget {
Replace { range: ParsedRange },
Block { anchor: Anchor },
Delete { range: ParsedRange },
DeleteBlock { anchor: Anchor },
InsertBefore { anchor: Anchor },
InsertAfter { anchor: Anchor },
InsertAfterBlock { anchor: Anchor },
Remove,
Move { dest: String },
Bof,
Eof,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) enum TokenKind {
Blank,
EnvelopeBegin,
EnvelopeEnd,
Abort,
Header {
path: String,
file_hash: Option<String>,
},
OpBlock {
target: BlockTarget,
},
PayloadLiteral {
text: String,
},
Comment {
text: String,
},
Raw {
text: String,
},
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) struct Token {
pub(crate) line_num: usize,
pub(crate) kind: TokenKind,
pub(crate) warnings: Vec<ParseWarning>,
}
#[derive(Default)]
pub(crate) struct Tokenizer {
buffer: String,
next_line_num: usize,
closed: bool,
}
impl Tokenizer {
pub(crate) fn new() -> Self {
Self {
buffer: String::new(),
next_line_num: 1,
closed: false,
}
}
pub(crate) fn feed(&mut self, chunk: &str) -> Vec<Token> {
assert!(
!self.closed,
"tokenizer is closed; call reset before reusing"
);
if chunk.is_empty() {
return Vec::new();
}
self.buffer.push_str(chunk);
self.drain_complete_lines()
}
pub(crate) fn end(&mut self) -> Vec<Token> {
if self.closed {
return Vec::new();
}
self.closed = true;
if self.buffer.is_empty() {
return Vec::new();
}
let line = self.buffer.trim_end_matches('\r').to_string();
self.buffer.clear();
let token = classify_line(&line, self.next_line_num);
self.next_line_num += 1;
vec![token]
}
pub(crate) fn tokenize(&self, line: &str, line_num: usize) -> Token {
classify_line(line, line_num)
}
pub(crate) fn is_op(&self, line: &str) -> bool {
try_parse_hunk_header(line).is_some()
}
pub(crate) fn is_envelope_marker(&self, line: &str) -> bool {
marker_line_equals(line, "*** Begin Patch")
|| marker_line_equals(line, "*** End Patch")
|| marker_line_equals(line, "*** ABORT")
}
fn drain_complete_lines(&mut self) -> Vec<Token> {
let mut tokens = Vec::new();
let mut start = 0usize;
for (index, byte) in self.buffer.as_bytes().iter().enumerate() {
if *byte != b'\n' {
continue;
}
let mut stop = index;
if stop > start && self.buffer.as_bytes()[stop - 1] == b'\r' {
stop -= 1;
}
tokens.push(classify_line(&self.buffer[start..stop], self.next_line_num));
self.next_line_num += 1;
start = index + 1;
}
if start > 0 {
self.buffer = self.buffer[start..].to_string();
}
tokens
}
}
pub(crate) fn split_hashline_lines(text: &str) -> Vec<String> {
if text.is_empty() {
return vec![String::new()];
}
let mut lines = Vec::new();
let mut start = 0usize;
for (index, byte) in text.as_bytes().iter().enumerate() {
if *byte != b'\n' {
continue;
}
let mut end = index;
if end > start && text.as_bytes()[end - 1] == b'\r' {
end -= 1;
}
lines.push(text[start..end].to_string());
start = index + 1;
}
if start < text.len() {
let mut end = text.len();
if end > start && text.as_bytes()[end - 1] == b'\r' {
end -= 1;
}
lines.push(text[start..end].to_string());
}
lines
}
fn classify_line(line: &str, line_num: usize) -> Token {
if line.is_empty() {
return token(line_num, TokenKind::Blank, Vec::new());
}
if marker_line_equals(line, "*** Begin Patch") {
return token(line_num, TokenKind::EnvelopeBegin, Vec::new());
}
if marker_line_equals(line, "*** End Patch") {
return token(line_num, TokenKind::EnvelopeEnd, Vec::new());
}
if marker_line_equals(line, "*** ABORT") {
return token(line_num, TokenKind::Abort, Vec::new());
}
if line.starts_with('[')
&& let Some(header) = try_parse_header(line)
{
return token(
line_num,
TokenKind::Header {
path: header.0,
file_hash: header.1,
},
Vec::new(),
);
}
if let Some((target, warnings)) = try_parse_hunk_header(line) {
return token(line_num, TokenKind::OpBlock { target }, warnings);
}
if let Some(text) = line.strip_prefix('+') {
return token(
line_num,
TokenKind::PayloadLiteral {
text: text.to_string(),
},
Vec::new(),
);
}
if line.trim_start().starts_with('#') {
return token(
line_num,
TokenKind::Comment {
text: line.to_string(),
},
Vec::new(),
);
}
token(
line_num,
TokenKind::Raw {
text: line.to_string(),
},
Vec::new(),
)
}
fn token(line_num: usize, kind: TokenKind, warnings: Vec<ParseWarning>) -> Token {
Token {
line_num,
kind,
warnings,
}
}
fn marker_line_equals(line: &str, marker: &str) -> bool {
let trimmed = line.trim_end_matches(is_whitespace_char);
trimmed == marker
}
pub(crate) fn try_parse_header(line: &str) -> Option<(String, Option<String>)> {
let trimmed = line.trim_end_matches(is_whitespace_char);
let body = trimmed.strip_prefix('[')?.strip_suffix(']')?;
if body.is_empty() {
return None;
}
let mut path = body;
let mut hash = None;
if body.len() > HASH_LEN {
let split = body.len().saturating_sub(HASH_LEN + 1);
if body.as_bytes().get(split) == Some(&b'#') {
let candidate = &body[split + 1..];
if candidate.len() == HASH_LEN && candidate.chars().all(|ch| ch.is_ascii_hexdigit()) {
path = &body[..split];
hash = Some(candidate.to_ascii_uppercase());
}
}
}
if path.is_empty() || path.contains('#') {
return None;
}
Some((path.to_string(), hash))
}
fn try_parse_hunk_header(line: &str) -> Option<(BlockTarget, Vec<ParseWarning>)> {
let end = line.trim_end_matches(is_whitespace_char).len();
let cursor = skip_ws(line, 0, end);
if cursor >= end {
return None;
}
let mut warnings = Vec::new();
let (target, next) = scan_hunk_anchor(line, cursor, end, &mut warnings)?;
if next == end {
Some((target, warnings))
} else {
None
}
}
fn scan_hunk_anchor(
line: &str,
cursor: usize,
end: usize,
warnings: &mut Vec<ParseWarning>,
) -> Option<(BlockTarget, usize)> {
if let Some(next) = scan_keyword(line, cursor, end, "REM") {
let next = skip_ws(line, next, end);
return (next == end).then_some((BlockTarget::Remove, next));
}
if let Some(next) = scan_keyword(line, cursor, end, "MV") {
let dest = scan_move_dest(line, next, end)?;
return (!dest.is_empty()).then_some((BlockTarget::Move { dest }, end));
}
if let Some(next) = scan_keyword(line, cursor, end, "SWAP.BLK") {
let anchor = scan_line_number(line, skip_ws(line, next, end), end)?;
let next = consume_optional_colon(line, anchor.next_index, end, warnings);
return Some((
BlockTarget::Block {
anchor: Anchor { line: anchor.line },
},
next,
));
}
if let Some(next) = scan_keyword(line, cursor, end, "SWAP") {
let range = scan_header_range(line, next, end, true)?;
let next = consume_optional_colon(line, range.next_index, end, warnings);
return Some((BlockTarget::Replace { range: range.range }, next));
}
if let Some(next) = scan_keyword(line, cursor, end, "DEL.BLK") {
let anchor = scan_line_number(line, skip_ws(line, next, end), end)?;
let next = skip_stray_dot(line, skip_ws(line, anchor.next_index, end), end, warnings);
if next < end && line.as_bytes()[next] == b':' {
return None;
}
return Some((
BlockTarget::DeleteBlock {
anchor: Anchor { line: anchor.line },
},
next,
));
}
if let Some(next) = scan_keyword(line, cursor, end, "DEL") {
let range = scan_header_range(line, next, end, true)?;
let next = skip_stray_dot(line, skip_ws(line, range.next_index, end), end, warnings);
if next < end && line.as_bytes()[next] == b':' {
return None;
}
return Some((BlockTarget::Delete { range: range.range }, next));
}
if let Some(next) = scan_keyword(line, cursor, end, "INS.BLK.POST") {
let anchor = scan_line_number(line, skip_ws(line, next, end), end)?;
let next = consume_optional_colon(line, anchor.next_index, end, warnings);
return Some((
BlockTarget::InsertAfterBlock {
anchor: Anchor { line: anchor.line },
},
next,
));
}
if let Some(next) = scan_keyword(line, cursor, end, "INS") {
return scan_insert_target(line, next, end, warnings);
}
None
}
fn scan_insert_target(
line: &str,
index: usize,
end: usize,
warnings: &mut Vec<ParseWarning>,
) -> Option<(BlockTarget, usize)> {
if line.as_bytes().get(index) != Some(&b'.') {
return None;
}
let cursor = skip_ws(line, index + 1, end);
if let Some(next) = scan_keyword(line, cursor, end, "PRE") {
let anchor = scan_line_number(line, skip_ws(line, next, end), end)?;
let next = consume_optional_colon(line, anchor.next_index, end, warnings);
return Some((
BlockTarget::InsertBefore {
anchor: Anchor { line: anchor.line },
},
next,
));
}
if let Some(next) = scan_keyword(line, cursor, end, "POST") {
let anchor = scan_line_number(line, skip_ws(line, next, end), end)?;
let next = consume_optional_colon(line, anchor.next_index, end, warnings);
return Some((
BlockTarget::InsertAfter {
anchor: Anchor { line: anchor.line },
},
next,
));
}
if let Some(next) = scan_keyword(line, cursor, end, "HEAD") {
let next = consume_optional_colon(line, next, end, warnings);
return Some((BlockTarget::Bof, next));
}
if let Some(next) = scan_keyword(line, cursor, end, "TAIL") {
let next = consume_optional_colon(line, next, end, warnings);
return Some((BlockTarget::Eof, next));
}
None
}
fn scan_keyword(line: &str, index: usize, end: usize, keyword: &str) -> Option<usize> {
if !line.get(index..)?.starts_with(keyword) {
return None;
}
let next = index + keyword.len();
if next < end {
let code = line.as_bytes()[next];
if !is_ws_byte(code) && code != b':' && code != b'.' {
return None;
}
}
Some(next)
}
#[derive(Debug)]
struct NumberScan {
line: usize,
next_index: usize,
}
fn scan_line_number(line: &str, index: usize, end: usize) -> Option<NumberScan> {
let bytes = line.as_bytes();
if index >= end || !matches!(bytes[index], b'1'..=b'9') {
return None;
}
let mut number = 0usize;
let mut next = index;
while next < end && bytes[next].is_ascii_digit() {
number = number
.saturating_mul(10)
.saturating_add((bytes[next] - b'0') as usize);
next += 1;
}
Some(NumberScan {
line: number,
next_index: next,
})
}
#[derive(Debug)]
struct RangeScan {
range: ParsedRange,
next_index: usize,
}
fn scan_header_range(
line: &str,
index: usize,
end: usize,
allow_single: bool,
) -> Option<RangeScan> {
let start = scan_line_number(line, skip_ws(line, index, end), end)?;
let Some(after_first) = scan_range_separator(line, start.next_index, end) else {
return allow_single.then_some(RangeScan {
range: ParsedRange {
start: Anchor { line: start.line },
end: Anchor { line: start.line },
},
next_index: skip_ws(line, start.next_index, end),
});
};
let end_number = scan_line_number(line, after_first, end)?;
Some(RangeScan {
range: ParsedRange {
start: Anchor { line: start.line },
end: Anchor {
line: end_number.line,
},
},
next_index: skip_ws(line, end_number.next_index, end),
})
}
fn scan_range_separator(line: &str, index: usize, end: usize) -> Option<usize> {
let mut cursor = index;
let mut consumed = false;
while cursor < end {
let rest = &line[cursor..end];
let byte = line.as_bytes()[cursor];
if is_ws_byte(byte) || byte == b'-' {
cursor += 1;
consumed = true;
} else if rest.starts_with('…') {
cursor += '…'.len_utf8();
consumed = true;
} else if rest.starts_with("..") || rest.starts_with(".=") {
cursor += 2;
consumed = true;
} else {
break;
}
}
(consumed && cursor < end && matches!(line.as_bytes()[cursor], b'1'..=b'9')).then_some(cursor)
}
fn consume_optional_colon(
line: &str,
index: usize,
end: usize,
warnings: &mut Vec<ParseWarning>,
) -> usize {
let mut cursor = skip_ws(line, index, end);
cursor = skip_stray_dot(line, cursor, end, warnings);
if cursor < end && line.as_bytes()[cursor] == b':' {
skip_ws(line, cursor + 1, end)
} else {
cursor
}
}
fn skip_stray_dot(line: &str, index: usize, end: usize, warnings: &mut Vec<ParseWarning>) -> usize {
if index < end && line.as_bytes()[index] == b'.' {
let after = skip_ws(line, index + 1, end);
if after == end || line.as_bytes()[after] == b':' {
warnings.push(ParseWarning::StrayDotSkipped { line_num: 0 });
return after;
}
}
index
}
fn scan_move_dest(line: &str, index: usize, end: usize) -> Option<String> {
let cursor = skip_ws(line, index, end);
if cursor >= end {
return None;
}
let bytes = line.as_bytes();
if matches!(bytes[cursor], b'\'' | b'\"') {
let quote = bytes[cursor];
let mut next = cursor + 1;
while next < end {
if bytes[next] == b'\\' && next + 1 < end {
next += 2;
} else if bytes[next] == quote {
let after = skip_ws(line, next + 1, end);
return (after == end).then(|| line[cursor + 1..next].to_string());
} else {
next += 1;
}
}
return None;
}
Some(line[cursor..end].trim().to_string())
}
fn skip_ws(line: &str, mut index: usize, end: usize) -> usize {
while index < end && is_ws_byte(line.as_bytes()[index]) {
index += 1;
}
index
}
fn is_ws_byte(byte: u8) -> bool {
byte == b' ' || (b'\t'..=b'\r').contains(&byte)
}
fn is_whitespace_char(ch: char) -> bool {
ch == ' ' || ('\t'..='\r').contains(&ch)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn parses_header_and_ops() {
assert_eq!(
try_parse_header("[src/lib.rs#ab12]"),
Some(("src/lib.rs".into(), Some("AB12".into())))
);
assert!(matches!(
classify_line("SWAP 2.=3:", 1).kind,
TokenKind::OpBlock {
target: BlockTarget::Replace { .. }
}
));
assert!(matches!(
classify_line("INS.BLK.POST 2:", 1).kind,
TokenKind::OpBlock {
target: BlockTarget::InsertAfterBlock { .. }
}
));
}
#[test]
fn tolerates_stray_dot_before_colon() {
let token = classify_line("SWAP 2.=3.:", 9);
assert!(matches!(
token.kind,
TokenKind::OpBlock {
target: BlockTarget::Replace { .. }
}
));
assert_eq!(
token.warnings,
vec![ParseWarning::StrayDotSkipped { line_num: 0 }]
);
}
#[test]
fn classifies_literal_plus_content() {
assert_eq!(
classify_line("++literal", 1).kind,
TokenKind::PayloadLiteral {
text: "+literal".to_string()
}
);
assert_eq!(
classify_line("+-literal", 1).kind,
TokenKind::PayloadLiteral {
text: "-literal".to_string()
}
);
}
}