use crate::error::{Error, ErrorKind, Result};
use crate::hir::{Hir, HirAnchor, HirClass, HirExpr};
use dynasmrt::{dynasm, DynasmApi, DynasmLabelApi};
use super::jit::BacktrackingJit;
const ARM64_BACKTRACKING_JIT_ENABLED: bool = true;
pub(super) struct BacktrackingCompiler {
asm: dynasmrt::aarch64::Assembler,
hir: Hir,
backtrack_label: dynasmrt::DynamicLabel,
match_success_label: dynasmrt::DynamicLabel,
no_match_label: dynasmrt::DynamicLabel,
next_start_label: dynasmrt::DynamicLabel,
capture_count: u32,
current_capture: Option<u32>,
}
impl BacktrackingCompiler {
pub(super) fn new(hir: &Hir) -> Result<Self> {
let mut asm = dynasmrt::aarch64::Assembler::new().map_err(|e| {
Error::new(
ErrorKind::Jit(format!("Failed to create assembler: {:?}", e)),
"",
)
})?;
let backtrack_label = asm.new_dynamic_label();
let match_success_label = asm.new_dynamic_label();
let no_match_label = asm.new_dynamic_label();
let next_start_label = asm.new_dynamic_label();
Ok(Self {
asm,
hir: hir.clone(),
backtrack_label,
match_success_label,
no_match_label,
next_start_label,
capture_count: hir.props.capture_count,
current_capture: None,
})
}
pub(super) fn compile(mut self) -> Result<BacktrackingJit> {
if !ARM64_BACKTRACKING_JIT_ENABLED {
return Err(Error::new(
ErrorKind::Jit("ARM64 backtracking JIT temporarily disabled".to_string()),
"",
));
}
let entry_offset = self.asm.offset();
self.emit_prologue();
self.emit_main_loop()?;
self.emit_pattern(&self.hir.expr.clone())?;
dynasm!(self.asm
; .arch aarch64
; b =>self.match_success_label
);
self.emit_backtrack_handler();
self.emit_success_handler();
let code = self
.asm
.finalize()
.map_err(|e| Error::new(ErrorKind::Jit(format!("Failed to finalize: {:?}", e)), ""))?;
let match_fn: unsafe extern "C" fn(*const u8, usize, *mut i64) -> i64 =
unsafe { std::mem::transmute(code.ptr(entry_offset)) };
Ok(BacktrackingJit {
code,
match_fn,
capture_count: self.capture_count,
})
}
fn emit_prologue(&mut self) {
dynasm!(self.asm
; .arch aarch64
; stp x29, x30, [sp, #-16]!
; mov x29, sp
; stp x19, x20, [sp, #-16]!
; stp x21, x22, [sp, #-16]!
; stp x23, x24, [sp, #-16]!
; stp x25, x26, [sp, #-16]!
; stp x27, x28, [sp, #-16]!
; mov x9, 0x1000
; sub sp, sp, x9
; mov x19, x0 ; mov x20, x1 ; mov x22, x2 ; mov x23, #0 ; mov x26, sp
; movn x0, 0
);
let num_slots = (self.capture_count as usize + 1) * 2;
for slot in 0..num_slots {
let offset = (slot * 8) as u32;
if offset < 4096 {
dynasm!(self.asm
; .arch aarch64
; str x0, [x22, offset]
);
} else {
let offset64 = offset as u64;
dynasm!(self.asm
; .arch aarch64
; mov x1, offset64
; str x0, [x22, x1]
);
}
}
}
fn emit_main_loop(&mut self) -> Result<()> {
dynasm!(self.asm
; .arch aarch64
; =>self.next_start_label
; movn x0, 0
);
let num_slots = (self.capture_count as usize + 1) * 2;
for slot in 0..num_slots {
let offset = (slot * 8) as u32;
if offset < 4096 {
dynasm!(self.asm
; .arch aarch64
; str x0, [x22, offset]
);
} else {
let offset64 = offset as u64;
dynasm!(self.asm
; .arch aarch64
; mov x1, offset64
; str x0, [x22, x1]
);
}
}
dynasm!(self.asm
; .arch aarch64
; mov x21, x23
; str x21, [x22]
; mov x9, 0x1000
; sub x26, x29, x9
; sub x26, x26, #0x50 );
Ok(())
}
fn emit_pattern(&mut self, expr: &HirExpr) -> Result<()> {
match expr {
HirExpr::Empty => Ok(()),
HirExpr::Literal(bytes) => self.emit_literal(bytes),
HirExpr::Class(class) => self.emit_class(class),
HirExpr::UnicodeCpClass(_) => Err(Error::new(
ErrorKind::Jit(
"Unicode codepoint classes not supported in backtracking JIT".to_string(),
),
"",
)),
HirExpr::Concat(parts) => {
for part in parts {
self.emit_pattern(part)?;
}
Ok(())
}
HirExpr::Alt(alternatives) => self.emit_alternation(alternatives),
HirExpr::Repeat(repeat) => {
self.emit_repetition(&repeat.expr, repeat.min, repeat.max, repeat.greedy)
}
HirExpr::Capture(capture) => self.emit_capture(capture.index, &capture.expr),
HirExpr::Backref(group) => self.emit_backref(*group),
HirExpr::Anchor(anchor) => self.emit_anchor(*anchor),
HirExpr::Lookaround(_) => Err(Error::new(
ErrorKind::Jit("Lookarounds not supported in backtracking JIT".to_string()),
"",
)),
}
}
fn emit_literal(&mut self, bytes: &[u8]) -> Result<()> {
for &byte in bytes {
dynasm!(self.asm
; .arch aarch64
; cmp x21, x20
; b.hs =>self.backtrack_label
; ldrb w0, [x19, x21]
; cmp w0, #(byte as u32)
; b.ne =>self.backtrack_label
; add x21, x21, #1
);
}
Ok(())
}
fn emit_class(&mut self, class: &HirClass) -> Result<()> {
let match_ok = self.asm.new_dynamic_label();
let no_match = self.asm.new_dynamic_label();
dynasm!(self.asm
; .arch aarch64
; cmp x21, x20
; b.hs =>self.backtrack_label
; ldrb w0, [x19, x21]
);
for &(start, end) in &class.ranges {
if start == end {
dynasm!(self.asm
; .arch aarch64
; cmp w0, #(start as u32)
; b.eq =>match_ok
);
} else {
let next_range = self.asm.new_dynamic_label();
dynasm!(self.asm
; .arch aarch64
; cmp w0, #(start as u32)
; b.lo =>next_range
; cmp w0, #(end as u32)
; b.ls =>match_ok
; =>next_range
);
}
}
dynasm!(self.asm
; .arch aarch64
; b =>no_match
);
dynasm!(self.asm
; .arch aarch64
; =>match_ok
);
if class.negated {
let done = self.asm.new_dynamic_label();
dynasm!(self.asm
; .arch aarch64
; b =>self.backtrack_label
; =>no_match
; add x21, x21, #1
; b =>done
; =>done
);
} else {
let done = self.asm.new_dynamic_label();
dynasm!(self.asm
; .arch aarch64
; add x21, x21, #1
; b =>done
; =>no_match
; b =>self.backtrack_label
; =>done
);
}
Ok(())
}
fn emit_alternation(&mut self, alternatives: &[HirExpr]) -> Result<()> {
if alternatives.is_empty() {
return Ok(());
}
let after_alt = self.asm.new_dynamic_label();
for (i, alt) in alternatives.iter().enumerate() {
let is_last = i == alternatives.len() - 1;
if !is_last {
let try_next = self.asm.new_dynamic_label();
dynasm!(self.asm
; .arch aarch64
; str x21, [x26] ; adr x0, =>try_next
; str x0, [x26, #8] ; str x23, [x26, #16] ; str xzr, [x26, #24] ; add x26, x26, #32 );
self.emit_pattern(alt)?;
dynasm!(self.asm
; .arch aarch64
; sub x26, x26, #32
; b =>after_alt
);
dynasm!(self.asm
; .arch aarch64
; =>try_next
);
} else {
self.emit_pattern(alt)?;
}
}
dynasm!(self.asm
; .arch aarch64
; =>after_alt
);
Ok(())
}
fn emit_repetition(
&mut self,
expr: &HirExpr,
min: u32,
max: Option<u32>,
greedy: bool,
) -> Result<()> {
let loop_done = self.asm.new_dynamic_label();
if let Some(max_val) = max {
if min == max_val && min > 0 {
return self.emit_exact_repetition(expr, min);
}
}
dynasm!(self.asm
; .arch aarch64
; mov x25, #0
);
if greedy {
let loop_start = self.asm.new_dynamic_label();
let try_backtrack = self.asm.new_dynamic_label();
dynasm!(self.asm
; .arch aarch64
; =>loop_start
);
if let Some(max_val) = max {
dynasm!(self.asm
; .arch aarch64
; cmp x25, #(max_val as u32)
; b.hs =>loop_done
);
}
dynasm!(self.asm
; .arch aarch64
; str x21, [x26]
; adr x0, =>try_backtrack
; str x0, [x26, #8]
; str x23, [x26, #16]
; str x25, [x26, #24]
; add x26, x26, #32
);
let iteration_matched = self.asm.new_dynamic_label();
let iteration_backtrack = self.asm.new_dynamic_label();
let old_backtrack = self.backtrack_label;
self.backtrack_label = iteration_backtrack;
self.emit_pattern(expr)?;
self.backtrack_label = old_backtrack;
dynasm!(self.asm
; .arch aarch64
; b =>iteration_matched
; =>iteration_backtrack
; mov x0, 0x1000
; sub x0, x29, x0
; sub x0, x0, #0x50
; cmp x26, x0
; b.ls >empty_stack
; sub x26, x26, #32
; ldr x0, [x26, #8]
; adr x1, =>try_backtrack
; cmp x0, x1
; b.ne >not_our_entry
; ldr x21, [x26]
; ldr x23, [x26, #16]
; ldr x25, [x26, #24]
; b =>loop_done
; not_our_entry:
; ldr x21, [x26]
; ldr x23, [x26, #16]
; ldr x25, [x26, #24]
; br x0
; empty_stack:
; b =>loop_done
; =>iteration_matched
; add x25, x25, #1
; b =>loop_start
; =>try_backtrack
);
if let Some(cap_idx) = self.current_capture {
let end_offset = (cap_idx as u32) * 16 + 8;
if end_offset < 4096 {
dynasm!(self.asm
; .arch aarch64
; str x21, [x22, #end_offset]
);
}
}
dynasm!(self.asm
; .arch aarch64
; cmp x25, #(min as u32)
; b.lo =>self.backtrack_label
; b =>loop_done
);
} else {
for _ in 0..min {
self.emit_pattern(expr)?;
dynasm!(self.asm
; .arch aarch64
; add x25, x25, #1
);
}
if max.map_or(true, |m| m > min) {
let loop_start = self.asm.new_dynamic_label();
let try_more = self.asm.new_dynamic_label();
dynasm!(self.asm
; .arch aarch64
; =>loop_start
);
if let Some(max_val) = max {
dynasm!(self.asm
; .arch aarch64
; cmp x25, #(max_val as u32)
; b.hs =>loop_done
);
}
dynasm!(self.asm
; .arch aarch64
; str x21, [x26]
; adr x0, =>try_more
; str x0, [x26, #8]
; str x23, [x26, #16]
; str x25, [x26, #24]
; add x26, x26, #32
; b =>loop_done
; =>try_more
);
self.emit_pattern(expr)?;
dynasm!(self.asm
; .arch aarch64
; add x25, x25, #1
; b =>loop_start
);
}
}
dynasm!(self.asm
; .arch aarch64
; =>loop_done
; cmp x25, #(min as u32)
; b.lo =>self.backtrack_label
);
Ok(())
}
fn emit_exact_repetition(&mut self, expr: &HirExpr, count: u32) -> Result<()> {
let loop_start = self.asm.new_dynamic_label();
let count64 = count as u64;
dynasm!(self.asm
; .arch aarch64
; mov x25, count64
; =>loop_start
);
self.emit_pattern(expr)?;
dynasm!(self.asm
; .arch aarch64
; subs w25, w25, #1
; b.ne =>loop_start
);
Ok(())
}
fn emit_capture(&mut self, index: u32, expr: &HirExpr) -> Result<()> {
let start_offset = (index as u32) * 16;
let end_offset = start_offset + 8;
if start_offset < 4096 {
dynasm!(self.asm
; .arch aarch64
; str x21, [x22, #start_offset]
);
}
let old_capture = self.current_capture;
self.current_capture = Some(index);
self.emit_pattern(expr)?;
self.current_capture = old_capture;
if end_offset < 4096 {
dynasm!(self.asm
; .arch aarch64
; str x21, [x22, #end_offset]
);
}
Ok(())
}
fn emit_backref(&mut self, group: u32) -> Result<()> {
let start_offset = (group as u32) * 16;
let end_offset = start_offset + 8;
let backref_ok = self.asm.new_dynamic_label();
dynasm!(self.asm
; .arch aarch64
; ldr x8, [x22, #start_offset] ; ldr x9, [x22, #end_offset]
; cmn x8, #1
; b.eq =>self.backtrack_label
; sub x10, x9, x8
; cbz x10, =>backref_ok
; sub x11, x20, x21 ; cmp x10, x11
; b.hi =>self.backtrack_label
; add x8, x8, x19 ; add x9, x19, x21
; mov x24, #0 ; cmp_loop:
; cmp x24, x10
; b.hs =>backref_ok
; ldrb w0, [x8, x24]
; ldrb w1, [x9, x24]
; cmp w0, w1
; b.ne =>self.backtrack_label
; add x24, x24, #1
; b <cmp_loop
; =>backref_ok
; add x21, x21, x10
);
Ok(())
}
fn emit_anchor(&mut self, anchor: HirAnchor) -> Result<()> {
match anchor {
HirAnchor::Start => {
dynasm!(self.asm
; .arch aarch64
; cbnz x21, =>self.backtrack_label
);
}
HirAnchor::End => {
dynasm!(self.asm
; .arch aarch64
; cmp x21, x20
; b.ne =>self.backtrack_label
);
}
HirAnchor::StartLine => {
let ok = self.asm.new_dynamic_label();
dynasm!(self.asm
; .arch aarch64
; cbz x21, =>ok
; sub x0, x21, #1
; ldrb w0, [x19, x0]
; cmp w0, #0x0a
; b.ne =>self.backtrack_label
; =>ok
);
}
HirAnchor::EndLine => {
let ok = self.asm.new_dynamic_label();
dynasm!(self.asm
; .arch aarch64
; cmp x21, x20
; b.eq =>ok
; ldrb w0, [x19, x21]
; cmp w0, #0x0a
; b.ne =>self.backtrack_label
; =>ok
);
}
HirAnchor::WordBoundary | HirAnchor::NotWordBoundary => {
return Err(Error::new(
ErrorKind::Jit(
"Word boundaries not yet supported in backtracking JIT".to_string(),
),
"",
));
}
}
Ok(())
}
fn emit_backtrack_handler(&mut self) {
dynasm!(self.asm
; .arch aarch64
; =>self.backtrack_label
; mov x0, 0x1000
; sub x0, x29, x0
; sub x0, x0, #0x50
; cmp x26, x0
; b.ls >try_next_pos
; sub x26, x26, #32
; ldr x21, [x26] ; ldr x0, [x26, #8] ; ldr x23, [x26, #16] ; ldr x25, [x26, #24]
; br x0
; try_next_pos:
; add x23, x23, #1
; cmp x23, x20
; b.hi =>self.no_match_label
; b =>self.next_start_label
);
}
fn emit_success_handler(&mut self) {
dynasm!(self.asm
; .arch aarch64
; =>self.match_success_label
; str x21, [x22, #8]
; mov x0, x21
; b >epilogue
; =>self.no_match_label
; movn x0, 0
; epilogue:
; mov x9, 0x1000
; add sp, sp, x9
; ldp x27, x28, [sp], #16
; ldp x25, x26, [sp], #16
; ldp x23, x24, [sp], #16
; ldp x21, x22, [sp], #16
; ldp x19, x20, [sp], #16
; ldp x29, x30, [sp], #16
; ret
);
}
}