use crate::dfa::DfaStateId;
use crate::error::{Error, ErrorKind, Result};
use crate::jit::codegen::{MaterializedDfa, MaterializedState};
use dynasm::dynasm;
use dynasmrt::{
x64::Assembler, AssemblyOffset, DynamicLabel, DynasmApi, DynasmLabelApi, ExecutableBuffer,
};
pub fn compile_states(
dfa: &MaterializedDfa,
) -> Result<(ExecutableBuffer, AssemblyOffset, Option<AssemblyOffset>)> {
let mut asm = Assembler::new().map_err(|e| {
Error::new(
ErrorKind::Jit(format!("Failed to create assembler: {}", e)),
"",
)
})?;
let max_state_id = dfa.states.iter().map(|s| s.id).max().unwrap_or(0) as usize;
let mut state_labels: Vec<Option<DynamicLabel>> = vec![None; max_state_id + 1];
for state in &dfa.states {
state_labels[state.id as usize] = Some(asm.new_dynamic_label());
}
let dead_label = asm.new_dynamic_label();
let no_match_label = asm.new_dynamic_label();
let restart_label = if !dfa.has_start_anchor {
Some(asm.new_dynamic_label())
} else {
None
};
let dispatch_label = if dfa.has_word_boundary && dfa.start_word.is_some() {
Some(asm.new_dynamic_label())
} else {
None
};
let entry_point = asm.offset();
emit_prologue(
&mut asm,
dfa.start,
&state_labels,
restart_label,
dispatch_label,
dfa.has_word_boundary,
true,
)?;
let entry_point_word = if let Some(start_word) = dfa.start_word {
let offset = asm.offset();
emit_prologue(
&mut asm,
start_word,
&state_labels,
restart_label,
dispatch_label,
dfa.has_word_boundary,
false,
)?;
Some(offset)
} else {
None
};
if let (Some(dispatch), Some(start_word)) = (dispatch_label, dfa.start_word) {
emit_dispatch(&mut asm, dispatch, dfa.start, start_word, &state_labels)?;
}
for state in &dfa.states {
emit_state(&mut asm, state, &state_labels, dead_label, no_match_label)?;
}
emit_dead_state(
&mut asm,
dead_label,
no_match_label,
restart_label,
dispatch_label,
dfa.has_word_boundary,
)?;
emit_no_match(&mut asm, no_match_label, dfa.has_word_boundary)?;
let code = asm.finalize().map_err(|_| {
Error::new(
ErrorKind::Jit("Failed to finalize assembly".to_string()),
"",
)
})?;
Ok((code, entry_point, entry_point_word))
}
fn emit_prologue(
asm: &mut Assembler,
start_state: DfaStateId,
state_labels: &[Option<DynamicLabel>],
restart_label: Option<DynamicLabel>,
dispatch_label: Option<DynamicLabel>,
has_word_boundary: bool,
emit_restart_label: bool,
) -> Result<()> {
let start_label = state_labels
.get(start_state as usize)
.and_then(|opt| opt.as_ref())
.ok_or_else(|| {
Error::new(
ErrorKind::Jit("Start state label not found".to_string()),
"",
)
})?;
#[cfg(target_os = "windows")]
{
dynasm!(asm
; .arch x64
; push rdi
; push rsi
);
if has_word_boundary {
dynasm!(asm
; .arch x64
; push r13
);
}
dynasm!(asm
; .arch x64
; mov r8, rcx ; lea rdx, [rcx + rdx] ; mov rsi, r8 ; xor rdi, rdi ; mov r10, -1i32 as _ ; xor r11, r11 );
}
#[cfg(not(target_os = "windows"))]
{
if has_word_boundary {
dynasm!(asm
; .arch x64
; push r13
);
}
dynasm!(asm
; .arch x64
; mov r8, rdi
; lea rdx, [rdi + rsi]
; mov rsi, r8
; xor rdi, rdi
; mov r10, -1i32 as _
; xor r11, r11
);
}
if has_word_boundary {
dynasm!(asm
; .arch x64
; xor r13d, r13d );
}
if let Some(restart) = restart_label {
if emit_restart_label {
dynasm!(asm
; .arch x64
; =>restart
);
}
}
if dispatch_label.is_some() && !emit_restart_label {
dynasm!(asm
; .arch x64
; mov r13d, 1 );
}
dynasm!(asm
; .arch x64
; jmp =>*start_label
);
Ok(())
}
fn emit_dispatch(
asm: &mut Assembler,
dispatch_label: DynamicLabel,
start: DfaStateId,
start_word: DfaStateId,
state_labels: &[Option<DynamicLabel>],
) -> Result<()> {
let start_label = state_labels
.get(start as usize)
.and_then(|opt| opt.as_ref())
.ok_or_else(|| {
Error::new(
ErrorKind::Jit("Start state label not found".to_string()),
"",
)
})?;
let start_word_label = state_labels
.get(start_word as usize)
.and_then(|opt| opt.as_ref())
.ok_or_else(|| {
Error::new(
ErrorKind::Jit("Start word state label not found".to_string()),
"",
)
})?;
dynasm!(asm
; .arch x64
; .align 16
; =>dispatch_label
; test r13d, r13d
; jnz =>*start_word_label ; jmp =>*start_label );
Ok(())
}
#[allow(clippy::type_complexity)]
fn analyze_self_loop(
state: &MaterializedState,
) -> Option<(Vec<(u8, u8)>, Vec<(u8, u8, DfaStateId)>)> {
let mut self_loop_bytes = Vec::new();
let mut other_transitions = Vec::new();
for byte in 0..=255u8 {
if let Some(target) = state.transitions[byte as usize] {
if target == state.id {
self_loop_bytes.push(byte);
} else {
other_transitions.push((byte, target));
}
}
}
if self_loop_bytes.len() < 3 {
return None;
}
let mut self_loop_ranges = Vec::new();
if !self_loop_bytes.is_empty() {
let mut start = self_loop_bytes[0];
let mut end = self_loop_bytes[0];
for &byte in &self_loop_bytes[1..] {
if byte == end + 1 {
end = byte;
} else {
self_loop_ranges.push((start, end));
start = byte;
end = byte;
}
}
self_loop_ranges.push((start, end));
}
let mut other_ranges = Vec::new();
if !other_transitions.is_empty() {
let mut sorted = other_transitions.clone();
sorted.sort_by_key(|(b, _)| *b);
let mut start = sorted[0].0;
let mut end = sorted[0].0;
let mut target = sorted[0].1;
for &(byte, t) in &sorted[1..] {
if byte == end + 1 && t == target {
end = byte;
} else {
other_ranges.push((start, end, target));
start = byte;
end = byte;
target = t;
}
}
other_ranges.push((start, end, target));
}
Some((self_loop_ranges, other_ranges))
}
fn emit_state(
asm: &mut Assembler,
state: &MaterializedState,
state_labels: &[Option<DynamicLabel>],
dead_label: DynamicLabel,
no_match_label: DynamicLabel,
) -> Result<()> {
let state_label = state_labels
.get(state.id as usize)
.and_then(|opt| opt.as_ref())
.ok_or_else(|| {
Error::new(
ErrorKind::Jit(format!("Label for state {} not found", state.id)),
"",
)
})?;
dynasm!(asm
; .arch x64
; .align 16
; =>*state_label
);
if state.is_match {
dynasm!(asm
; .arch x64
; mov r10, rdi
);
dynasm!(asm
; .arch x64
; lea rax, [rsi + rdi]
; cmp rax, rdx
; jge >match_return
);
} else {
dynasm!(asm
; .arch x64
; lea rax, [rsi + rdi]
; cmp rax, rdx
; jge =>no_match_label
);
}
if let Some((self_loop_ranges, other_transitions)) = analyze_self_loop(state) {
emit_fast_forward_loop(
asm,
state,
&self_loop_ranges,
&other_transitions,
state_labels,
dead_label,
no_match_label,
)?;
} else {
dynasm!(asm
; .arch x64
; movzx eax, BYTE [rsi + rdi]
; inc rdi );
if state.should_use_jump_table() {
emit_dense_transitions(asm, state, state_labels, dead_label)?;
} else {
emit_sparse_transitions(asm, state, state_labels, dead_label)?;
}
}
if state.is_match {
dynasm!(asm
; .arch x64
; match_return:
; jmp =>no_match_label
);
}
Ok(())
}
fn build_self_loop_bitmap(self_loop_ranges: &[(u8, u8)]) -> [u8; 32] {
let mut bitmap = [0u8; 32];
for &(start, end) in self_loop_ranges {
for byte in start..=end {
bitmap[byte as usize / 8] |= 1 << (byte % 8);
}
}
bitmap
}
fn emit_fast_forward_loop(
asm: &mut Assembler,
state: &MaterializedState,
self_loop_ranges: &[(u8, u8)],
other_transitions: &[(u8, u8, DfaStateId)],
state_labels: &[Option<DynamicLabel>],
dead_label: DynamicLabel,
no_match_label: DynamicLabel,
) -> Result<()> {
let use_bitmap = self_loop_ranges.len() >= 3;
if use_bitmap {
let bitmap = build_self_loop_bitmap(self_loop_ranges);
dynasm!(asm
; .arch x64
; jmp >fast_forward_start
; .align 32
; bitmap_data:
; .bytes bitmap.as_slice()
; fast_forward_start:
);
dynasm!(asm
; .arch x64
; lea r9, [<bitmap_data]
);
dynasm!(asm
; .arch x64
; fast_forward_loop:
; lea rax, [rsi + rdi]
; cmp rax, rdx
; jge >exhausted
; movzx eax, BYTE [rsi + rdi]
; mov ecx, eax
; shr ecx, 3
; and eax, 7
; bt DWORD [r9 + rcx], eax
; jnc >check_other_transitions
; inc rdi
);
if state.is_match {
dynasm!(asm
; .arch x64
; mov r10, rdi
);
}
dynasm!(asm
; .arch x64
; jmp <fast_forward_loop
);
} else {
dynasm!(asm
; .arch x64
; fast_forward_loop:
; lea rax, [rsi + rdi]
; cmp rax, rdx
; jge >exhausted
; movzx eax, BYTE [rsi + rdi]
);
for (i, &(start, end)) in self_loop_ranges.iter().enumerate() {
let is_last = i == self_loop_ranges.len() - 1;
if start == end {
dynasm!(asm
; .arch x64
; cmp al, BYTE start as _
; je >consume_byte
);
} else {
dynasm!(asm
; .arch x64
; cmp al, BYTE start as _
; jb >next_range
; cmp al, BYTE end as _
; jbe >consume_byte
; next_range:
);
}
if is_last {
dynasm!(asm
; .arch x64
; jmp >check_other_transitions
);
}
}
dynasm!(asm
; .arch x64
; consume_byte:
; inc rdi
);
if state.is_match {
dynasm!(asm
; .arch x64
; mov r10, rdi
);
}
dynasm!(asm
; .arch x64
; jmp <fast_forward_loop
);
}
dynasm!(asm
; .arch x64
; exhausted:
);
dynasm!(asm
; .arch x64
; jmp =>no_match_label
);
dynasm!(asm
; .arch x64
; check_other_transitions:
);
if !other_transitions.is_empty() {
dynasm!(asm
; .arch x64
; movzx eax, BYTE [rsi + rdi]
; inc rdi
);
for &(start, end, target) in other_transitions {
let target_label = state_labels
.get(target as usize)
.and_then(|opt| opt.as_ref())
.ok_or_else(|| {
Error::new(
ErrorKind::Jit(format!("Label for state {} not found", target)),
"",
)
})?;
if start == end {
dynasm!(asm
; .arch x64
; cmp al, BYTE start as _
; je =>*target_label
);
} else {
dynasm!(asm
; .arch x64
; cmp al, BYTE start as _
; jb >next_other
; cmp al, BYTE end as _
; jbe =>*target_label
; next_other:
);
}
}
}
dynasm!(asm
; .arch x64
; jmp =>dead_label
);
Ok(())
}
fn emit_sparse_transitions(
asm: &mut Assembler,
state: &MaterializedState,
state_labels: &[Option<DynamicLabel>],
dead_label: DynamicLabel,
) -> Result<()> {
let ranges = compute_byte_ranges(state);
for (start, end, target) in ranges {
let target_label = state_labels
.get(target as usize)
.and_then(|opt| opt.as_ref())
.ok_or_else(|| {
Error::new(
ErrorKind::Jit(format!("Label for state {} not found", target)),
"",
)
})?;
if start == end {
dynasm!(asm
; .arch x64
; cmp al, BYTE start as _
; je =>*target_label
);
} else {
dynasm!(asm
; .arch x64
; cmp al, BYTE start as _
; jb >next_check
; cmp al, BYTE end as _
; jbe =>*target_label
; next_check:
);
}
}
dynasm!(asm
; .arch x64
; jmp =>dead_label
);
Ok(())
}
fn emit_dense_transitions(
asm: &mut Assembler,
state: &MaterializedState,
state_labels: &[Option<DynamicLabel>],
dead_label: DynamicLabel,
) -> Result<()> {
let ranges = compute_byte_ranges(state);
for (start, end, target) in ranges {
let target_label = state_labels
.get(target as usize)
.and_then(|opt| opt.as_ref())
.ok_or_else(|| {
Error::new(
ErrorKind::Jit(format!("Label for state {} not found", target)),
"",
)
})?;
if start == end {
dynasm!(asm
; .arch x64
; cmp al, BYTE start as _
; je =>*target_label
);
} else {
dynasm!(asm
; .arch x64
; cmp al, BYTE start as _
; jb >next
; cmp al, BYTE end as _
; jbe =>*target_label
; next:
);
}
}
dynasm!(asm
; .arch x64
; jmp =>dead_label
);
Ok(())
}
fn compute_byte_ranges(state: &MaterializedState) -> Vec<(u8, u8, DfaStateId)> {
let mut ranges = Vec::new();
let mut current_target: Option<DfaStateId> = None;
let mut range_start = 0u8;
for byte in 0..=255u8 {
let target = state.transitions[byte as usize];
match (current_target, target) {
(None, Some(t)) => {
current_target = Some(t);
range_start = byte;
}
(Some(curr), Some(t)) if curr == t => {
}
(Some(curr), _) => {
ranges.push((range_start, byte - 1, curr));
current_target = target;
range_start = byte;
}
(None, None) => {
}
}
if byte == 255 {
if let Some(t) = current_target {
ranges.push((range_start, byte, t));
}
}
}
ranges
}
fn emit_dead_state(
asm: &mut Assembler,
dead_label: DynamicLabel,
no_match_label: DynamicLabel,
restart_label: Option<DynamicLabel>,
dispatch_label: Option<DynamicLabel>,
has_word_boundary: bool,
) -> Result<()> {
dynasm!(asm
; .arch x64
; .align 16
; =>dead_label
);
if let Some(restart) = restart_label {
dynasm!(asm
; .arch x64
; cmp r10, 0
; jge =>no_match_label ; inc r11
; lea rax, [rsi + r11]
; cmp rax, rdx
; jge =>no_match_label ; mov rdi, r11
);
if has_word_boundary {
if let Some(dispatch) = dispatch_label {
dynasm!(asm
; .arch x64
; lea rax, [r11 - 1] ; movzx eax, BYTE [rsi + rax]
; xor r13d, r13d
; cmp al, 0x30
; jb >not_word
; cmp al, 0x39
; jbe >is_word
; cmp al, 0x41
; jb >not_word
; cmp al, 0x5A
; jbe >is_word
; cmp al, 0x5F
; je >is_word
; cmp al, 0x61
; jb >not_word
; cmp al, 0x7A
; jbe >is_word
; jmp >not_word
; is_word:
; mov r13d, 1
; not_word:
; jmp =>dispatch
);
} else {
dynasm!(asm
; .arch x64
; jmp =>restart
);
}
} else {
dynasm!(asm
; .arch x64
; jmp =>restart
);
}
} else {
dynasm!(asm
; .arch x64
; jmp =>no_match_label
);
}
Ok(())
}
fn emit_no_match(
asm: &mut Assembler,
no_match_label: DynamicLabel,
has_word_boundary: bool,
) -> Result<()> {
dynasm!(asm
; .arch x64
; =>no_match_label
; cmp r10, 0
; jl >truly_no_match
; mov rax, r11
; shl rax, 32
; or rax, r10
);
#[cfg(target_os = "windows")]
{
if has_word_boundary {
dynasm!(asm ; .arch x64 ; pop r13);
}
dynasm!(asm
; .arch x64
; pop rsi
; pop rdi
);
}
#[cfg(not(target_os = "windows"))]
{
if has_word_boundary {
dynasm!(asm ; .arch x64 ; pop r13);
}
}
dynasm!(asm
; .arch x64
; ret
; truly_no_match:
; mov rax, -1i32 as _
);
#[cfg(target_os = "windows")]
{
if has_word_boundary {
dynasm!(asm ; .arch x64 ; pop r13);
}
dynasm!(asm
; .arch x64
; pop rsi
; pop rdi
);
}
#[cfg(not(target_os = "windows"))]
{
if has_word_boundary {
dynasm!(asm ; .arch x64 ; pop r13);
}
}
dynasm!(asm
; .arch x64
; ret
);
Ok(())
}
#[cfg(test)]
mod tests {
fn is_contiguous_range(bytes: &[u8]) -> bool {
if bytes.len() <= 1 {
return true;
}
let mut sorted = bytes.to_vec();
sorted.sort_unstable();
for i in 1..sorted.len() {
if sorted[i] != sorted[i - 1] + 1 {
return false;
}
}
true
}
#[test]
fn test_contiguous_range() {
assert!(is_contiguous_range(&[1, 2, 3, 4]));
assert!(is_contiguous_range(b"abc"));
assert!(!is_contiguous_range(&[1, 2, 4, 5]));
assert!(!is_contiguous_range(b"ac"));
assert!(is_contiguous_range(&[42]));
assert!(is_contiguous_range(&[]));
}
#[test]
fn test_unsorted_contiguous() {
assert!(is_contiguous_range(&[3, 1, 2, 4]));
assert!(is_contiguous_range(b"cab"));
}
}