pub mod build;
pub mod marks;
use crate::token::TokenKind;
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum InferError {
TooFew(usize),
Empty(usize),
Unparsed { pattern: String, msg: String },
Unverified { example: usize, pattern: String },
Matched { counter: usize, pattern: String },
}
impl std::fmt::Display for InferError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
InferError::TooFew(n) => {
write!(f, "inference needs at least two examples; {n} given")
}
InferError::Empty(i) => write!(f, "example {} has no token to read", i + 1),
InferError::Unparsed { pattern, msg } => {
write!(f, "the inferred pattern `{pattern}` does not parse: {msg}")
}
InferError::Unverified { example, pattern } => write!(
f,
"the inferred pattern `{pattern}` does not match example {} whole",
example + 1
),
InferError::Matched { counter, pattern } => write!(
f,
"the inferred pattern `{pattern}` still matches counter-example {}; \
nothing the examples have in common tells them apart",
counter + 1
),
}
}
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum Narrowing {
Folded(crate::orbit::OrbitGroup, String),
Range(String),
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct Slot {
pub kinds: Vec<TokenKind>,
pub text: Option<String>,
pub narrowing: Option<Narrowing>,
pub min: usize,
pub max: usize,
}
impl Slot {
#[must_use]
pub fn spelled(&self) -> String {
let atom = match (&self.text, &self.narrowing) {
(Some(text), _) => crate::templates::quote(text),
(None, Some(Narrowing::Folded(group, text))) => {
format!("(?orbit:{} {})", group.label(), crate::templates::quote(text))
}
(None, narrowing) => {
let atoms: Option<Vec<String>> = self.kinds.iter().map(|k| atom_of(*k)).collect();
match atoms {
Some(atoms) if atoms.len() == 1 => match narrowing {
Some(Narrowing::Range(range)) => format!("{}{{{range}}}", atoms[0]),
_ => atoms[0].clone(),
},
Some(atoms) => format!("[{}]", atoms.join(" ")),
None => ".".to_string(),
}
}
};
match (self.min, self.max) {
(1, 1) => atom,
(0, 1) => format!("{atom}?"),
(a, b) if a == b => format!("{atom}{{{a}}}"),
(a, b) => format!("{atom}{{{a},{b}}}"),
}
}
}
fn atom_of(kind: TokenKind) -> Option<String> {
match kind {
TokenKind::Custom(_) | TokenKind::Open(_) | TokenKind::Close(_) | TokenKind::Other => None,
k => Some(match k.escape() {
Some(c) => format!("\\{c}"),
None => format!("\\{{{}}}", k.name()),
}),
}
}
const FOLDING_RUNGS: [crate::orbit::OrbitGroup; 8] = [
crate::orbit::OrbitGroup::Case,
crate::orbit::OrbitGroup::Notation,
crate::orbit::OrbitGroup::Numeric,
crate::orbit::OrbitGroup::Ip,
crate::orbit::OrbitGroup::Url,
crate::orbit::OrbitGroup::Time,
crate::orbit::OrbitGroup::Path,
crate::orbit::OrbitGroup::Fold,
];
fn narrowing_of(column: &Column) -> Option<Narrowing> {
let texts: Option<Vec<&str>> = column.texts.iter().map(|t| t.as_deref()).collect();
let texts = texts?;
for &group in &FOLDING_RUNGS {
let mut folded = texts.iter().map(|t| crate::orbit::canonical(t.as_bytes(), group));
let first = folded.next()?;
if folded.all(|f| f == first) {
return Some(Narrowing::Folded(group, first));
}
}
let [kind] = column.kinds[..] else { return None };
range_of(kind, &texts).map(Narrowing::Range)
}
fn range_of(kind: TokenKind, texts: &[&str]) -> Option<String> {
match kind {
TokenKind::Number => numeric_band(texts),
TokenKind::Ip => address_block(texts),
TokenKind::Timestamp => calendar_unit(texts),
TokenKind::Version => version_line(texts),
_ => None,
}
}
fn numeric_band(texts: &[&str]) -> Option<String> {
let mut digits: Vec<String> = Vec::with_capacity(texts.len());
for text in texts {
let value = crate::typed::Decimal::parse(text)?.to_text();
if value.contains('.') || value.starts_with('-') {
return None;
}
digits.push(value);
}
let by_value = |a: &&String, b: &&String| a.len().cmp(&b.len()).then_with(|| a.cmp(b));
let low = digits.iter().min_by(by_value)?;
let high = digits.iter().max_by(by_value)?;
let width = high.len();
let lead = |s: &String| if s.len() < width { '0' } else { s.as_bytes()[0] as char };
let band = |first: char, rest: char| -> String {
std::iter::once(first).chain(std::iter::repeat_n(rest, width - 1)).collect()
};
let floor = band(lead(low), '0');
let floor = floor.trim_start_matches('0');
Some(format!("{}..{}", if floor.is_empty() { "0" } else { floor }, band(lead(high), '9')))
}
fn address_block(texts: &[&str]) -> Option<String> {
use std::net::IpAddr;
let mut addresses: Vec<IpAddr> = Vec::with_capacity(texts.len());
for text in texts {
match crate::typed::value_of(TokenKind::Ip, text)? {
crate::typed::TypedValue::Ip(address) => addresses.push(address),
_ => return None,
}
}
let first = *addresses.first()?;
let octets = |a: &IpAddr| -> Vec<u8> {
match a {
IpAddr::V4(v4) => v4.octets().to_vec(),
IpAddr::V6(v6) => v6.octets().to_vec(),
}
};
let head = octets(&first);
let rest: Vec<Vec<u8>> = addresses.iter().map(octets).collect();
if rest.iter().any(|o| o.len() != head.len()) {
return None;
}
let mut shared = 0usize;
'bits: for (i, byte) in head.iter().enumerate() {
for bit in (0..8).rev() {
let mask = 1u8 << bit;
if rest.iter().any(|o| (o[i] & mask) != (byte & mask)) {
break 'bits;
}
shared += 1;
}
}
let mut network = head;
for (i, byte) in network.iter_mut().enumerate() {
let kept = u32::try_from(shared.saturating_sub(i * 8).min(8)).unwrap_or(8);
*byte &= u8::MAX.checked_shl(8 - kept).unwrap_or(0);
}
let address = match first {
IpAddr::V4(_) => {
let mut bytes = [0u8; 4];
bytes.copy_from_slice(&network);
IpAddr::from(bytes)
}
IpAddr::V6(_) => {
let mut bytes = [0u8; 16];
bytes.copy_from_slice(&network);
IpAddr::from(bytes)
}
};
Some(format!("in:{address}/{shared}"))
}
fn calendar_unit(texts: &[&str]) -> Option<String> {
let mut stamps: Vec<crate::typed::Civil> = Vec::with_capacity(texts.len());
for text in texts {
match crate::typed::value_of(TokenKind::Timestamp, text)? {
crate::typed::TypedValue::Instant(civil) if civil.has_date => stamps.push(civil),
_ => return None,
}
}
let first = stamps.first()?;
let year = first.year?;
let agree = |read: &dyn Fn(&crate::typed::Civil) -> Option<i32>| {
let head = read(first);
head.is_some() && stamps.iter().all(|s| read(s) == head)
};
if !agree(&|s| s.year) {
return None;
}
if !agree(&|s| Some(i32::from(s.month))) {
return Some(format!("year={year}"));
}
let mut clauses = format!("year={year},month={}", first.month);
if !agree(&|s| Some(i32::from(s.day))) {
return Some(clauses);
}
clauses.push_str(&format!(",day={}", first.day));
if first.has_time && agree(&|s| Some(i32::from(s.hour))) {
clauses.push_str(&format!(",hour={}", first.hour));
}
Some(clauses)
}
fn version_line(texts: &[&str]) -> Option<String> {
let mut parts: Vec<Vec<String>> = Vec::with_capacity(texts.len());
for text in texts {
match crate::typed::value_of(TokenKind::Version, text)? {
crate::typed::TypedValue::Version(v) => {
parts.push(v.parts().iter().map(crate::typed::Decimal::to_text).collect());
}
_ => return None,
}
}
let first = parts.first()?;
let shared = |n: usize| first.len() > n && parts.iter().all(|p| p.len() > n && p[n] == first[n]);
if !shared(0) {
return None;
}
let mut clauses = format!("major={}", first[0]);
if shared(1) {
clauses.push_str(&format!(",minor={}", first[1]));
}
Some(clauses)
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct Inferred {
pub slots: Vec<Slot>,
pattern: String,
}
impl Inferred {
#[must_use]
pub fn pattern(&self) -> &str {
&self.pattern
}
#[must_use]
pub fn anchored(&self) -> String {
anchor(&self.pattern)
}
}
fn anchor(pattern: &str) -> String {
match pattern.rsplit_once(' ') {
Some((head, last)) => format!("^ {head} $ {last}"),
None => format!("^ $ {pattern}"),
}
}
struct Tok {
kind: TokenKind,
text: String,
start: usize,
end: usize,
}
struct Column {
kinds: Vec<TokenKind>,
text: Option<String>,
texts: Vec<Option<String>>,
}
impl Column {
fn new(examples: usize, example: usize, tok: &Tok) -> Column {
let mut texts = vec![None; examples];
texts[example] = Some(tok.text.clone());
Column { kinds: vec![tok.kind], text: Some(tok.text.clone()), texts }
}
fn absorb(&mut self, example: usize, tok: &Tok) {
if !self.kinds.contains(&tok.kind) {
self.kinds.push(tok.kind);
}
if self.text.as_deref() != Some(tok.text.as_str()) {
self.text = None;
}
self.texts[example] = Some(tok.text.clone());
}
fn present(&self) -> impl Iterator<Item = bool> + '_ {
self.texts.iter().map(Option::is_some)
}
}
enum Op {
Match(usize, usize),
Column(usize),
Token(usize),
}
fn align(columns: &[Column], seq: &[Tok]) -> Vec<Op> {
let (m, k) = (columns.len(), seq.len());
let gain = |i: usize, j: usize| -> Option<(u32, u32)> {
let column = &columns[i];
let tok = &seq[j];
if !column.kinds.contains(&tok.kind) {
return None;
}
Some((1, u32::from(column.text.as_deref() == Some(tok.text.as_str()))))
};
let mut best = vec![vec![(0u32, 0u32); k + 1]; m + 1];
for i in 1..=m {
for j in 1..=k {
let mut here = best[i - 1][j].max(best[i][j - 1]);
if let Some((kind, text)) = gain(i - 1, j - 1) {
let paired = (best[i - 1][j - 1].0 + kind, best[i - 1][j - 1].1 + text);
here = here.max(paired);
}
best[i][j] = here;
}
}
let mut ops = Vec::new();
let (mut i, mut j) = (m, k);
while i > 0 || j > 0 {
let paired = if i > 0 && j > 0 {
gain(i - 1, j - 1).map(|(kind, text)| (best[i - 1][j - 1].0 + kind, best[i - 1][j - 1].1 + text))
} else {
None
};
if paired == Some(best[i][j]) {
ops.push(Op::Match(i - 1, j - 1));
i -= 1;
j -= 1;
} else if i > 0 && (j == 0 || best[i - 1][j] >= best[i][j - 1]) {
ops.push(Op::Column(i - 1));
i -= 1;
} else {
ops.push(Op::Token(j - 1));
j -= 1;
}
}
ops.reverse();
ops
}
fn fold(columns: Vec<Column>, example: usize, seq: &[Tok], examples: usize) -> Vec<Column> {
let ops = align(&columns, seq);
let mut taken: Vec<Option<Column>> = columns.into_iter().map(Some).collect();
let mut out: Vec<Column> = Vec::with_capacity(taken.len() + seq.len());
let mut gap_columns: Vec<Column> = Vec::new();
let mut gap_tokens: Vec<usize> = Vec::new();
let flush = |gap_columns: &mut Vec<Column>, gap_tokens: &mut Vec<usize>, out: &mut Vec<Column>| {
let pairs = gap_columns.len().min(gap_tokens.len());
for (p, mut column) in gap_columns.drain(..).enumerate() {
if p < pairs {
column.absorb(example, &seq[gap_tokens[p]]);
}
out.push(column);
}
for &j in gap_tokens.iter().skip(pairs) {
out.push(Column::new(examples, example, &seq[j]));
}
gap_tokens.clear();
};
for op in ops {
match op {
Op::Match(i, j) => {
flush(&mut gap_columns, &mut gap_tokens, &mut out);
let mut column = taken[i].take().expect("each column is walked once");
column.absorb(example, &seq[j]);
out.push(column);
}
Op::Column(i) => gap_columns.push(taken[i].take().expect("each column is walked once")),
Op::Token(j) => gap_tokens.push(j),
}
}
flush(&mut gap_columns, &mut gap_tokens, &mut out);
out
}
fn slots_of(columns: &[Column]) -> Vec<Slot> {
let mut slots: Vec<Slot> = Vec::new();
let mut run: Option<Vec<usize>> = None;
for column in columns {
let here: Vec<usize> = column.present().map(usize::from).collect();
let all = column.present().all(|p| p);
if let (true, Some(text)) = (all, &column.text) {
slots.push(Slot {
kinds: column.kinds.clone(),
text: Some(text.clone()),
narrowing: None,
min: 1,
max: 1,
});
run = None;
continue;
}
let merges = match (&run, slots.last()) {
(Some(_), Some(last)) => last.text.is_none() && last.kinds == column.kinds,
_ => false,
};
if merges {
let counts = run.as_mut().expect("a run is open when a slot merges");
for (c, h) in counts.iter_mut().zip(&here) {
*c += h;
}
let last = slots.last_mut().expect("a slot is open when a slot merges");
last.min = counts.iter().copied().min().unwrap_or(0);
last.max = counts.iter().copied().max().unwrap_or(0);
last.narrowing = None;
} else {
slots.push(Slot {
kinds: column.kinds.clone(),
text: None,
narrowing: narrowing_of(column),
min: here.iter().copied().min().unwrap_or(0),
max: here.iter().copied().max().unwrap_or(0),
});
run = Some(here);
}
}
slots
}
pub fn infer(examples: &[&[u8]]) -> Result<Inferred, InferError> {
infer_against(examples, &[])
}
pub fn infer_against(examples: &[&[u8]], counters: &[&[u8]]) -> Result<Inferred, InferError> {
if examples.len() < 2 {
return Err(InferError::TooFew(examples.len()));
}
let mut sequences: Vec<Vec<Tok>> = Vec::with_capacity(examples.len());
for (i, example) in examples.iter().enumerate() {
let seq: Vec<Tok> = crate::lexer::lex(example)
.iter()
.filter(|t| t.is_significant())
.map(|t| Tok {
kind: t.kind,
text: String::from_utf8_lossy(&example[t.span()]).into_owned(),
start: t.start(),
end: t.end(),
})
.collect();
if seq.is_empty() {
return Err(InferError::Empty(i));
}
sequences.push(seq);
}
let n = examples.len();
let mut columns: Vec<Column> = sequences[0].iter().map(|t| Column::new(n, 0, t)).collect();
for (e, seq) in sequences.iter().enumerate().skip(1) {
columns = fold(columns, e, seq, n);
}
let mut slots = slots_of(&columns);
let candidates: Vec<usize> = slots
.iter()
.enumerate()
.filter(|(_, s)| matches!(s.narrowing, Some(Narrowing::Range(_))))
.map(|(i, _)| i)
.collect();
let mut ranges: Vec<Option<Narrowing>> = candidates.iter().map(|&i| slots[i].narrowing.take()).collect();
let spell = |slots: &[Slot]| slots.iter().map(Slot::spelled).collect::<Vec<_>>().join(" ");
let missed = |slots: &[Slot]| -> Result<Vec<usize>, InferError> {
let pattern = spell(slots);
let parsed = match crate::parser::parse(&pattern) {
Ok(p) => p,
Err(e) => return Err(InferError::Unparsed { pattern, msg: e.msg }),
};
Ok(counters
.iter()
.enumerate()
.filter(|(_, c)| !crate::engine::scan(&parsed, c).is_empty())
.map(|(i, _)| i)
.collect())
};
let mut hit = missed(&slots)?;
while !hit.is_empty() {
let mut best: Option<(usize, usize)> = None;
for (c, slot) in candidates.iter().enumerate() {
if ranges[c].is_none() {
continue;
}
slots[*slot].narrowing = ranges[c].clone();
let left = missed(&slots)?.len();
slots[*slot].narrowing = None;
if best.is_none_or(|(_, fewest)| left < fewest) {
best = Some((c, left));
}
}
let Some((c, left)) = best else { break };
if left >= hit.len() {
break;
}
slots[candidates[c]].narrowing = ranges[c].take();
hit = missed(&slots)?;
}
let pattern = spell(&slots);
if let Some(&counter) = hit.first() {
return Err(InferError::Matched { counter, pattern });
}
let parsed = match crate::parser::parse(&pattern) {
Ok(p) => p,
Err(e) => return Err(InferError::Unparsed { pattern, msg: e.msg }),
};
for (i, (example, seq)) in examples.iter().zip(&sequences).enumerate() {
let whole = seq[0].start..seq[seq.len() - 1].end;
if !crate::engine::scan(&parsed, example).iter().any(|s| s.range() == whole) {
return Err(InferError::Unverified { example: i, pattern });
}
}
Ok(Inferred { slots, pattern })
}