use std::collections::BTreeSet;
use crate::event::{Event, EventValue};
use crate::schema_discovery::FieldProfile;
use super::{DraftConfig, Stability};
#[derive(Debug, Clone, PartialEq)]
pub(crate) enum DraftValue {
Str(String),
Int(i64),
Float(f64),
Bool(bool),
}
impl DraftValue {
pub(crate) fn from_event_value(v: &EventValue<'_>) -> Option<Self> {
match v {
EventValue::Str(s) => Some(DraftValue::Str(s.to_string())),
EventValue::Int(n) => Some(DraftValue::Int(*n)),
EventValue::Float(f) => Some(DraftValue::Float(*f)),
EventValue::Bool(b) => Some(DraftValue::Bool(*b)),
EventValue::Null | EventValue::Array(_) | EventValue::Map(_) => None,
}
}
pub(crate) fn as_display(&self) -> String {
match self {
DraftValue::Str(s) => s.clone(),
DraftValue::Int(n) => n.to_string(),
DraftValue::Float(f) => f.to_string(),
DraftValue::Bool(b) => b.to_string(),
}
}
pub(crate) fn as_match_str(&self) -> String {
self.as_display()
}
}
#[derive(Debug, Clone, PartialEq)]
pub(crate) enum ValueForm {
Exact(DraftValue),
OneOf(Vec<DraftValue>),
EndsWith(String),
StartsWith(String),
Contains(String),
ContainsAll(Vec<String>),
}
impl ValueForm {
pub(crate) fn modifier(&self) -> &'static str {
match self {
ValueForm::Exact(_) | ValueForm::OneOf(_) => "",
ValueForm::EndsWith(_) => "|endswith",
ValueForm::StartsWith(_) => "|startswith",
ValueForm::Contains(_) => "|contains",
ValueForm::ContainsAll(_) => "|contains|all",
}
}
pub(crate) fn display_values(&self) -> Vec<String> {
match self {
ValueForm::Exact(v) => vec![v.as_display()],
ValueForm::OneOf(vs) => vs.iter().map(|v| v.as_display()).collect(),
ValueForm::EndsWith(s) => vec![format!("*{s}")],
ValueForm::StartsWith(s) => vec![format!("{s}*")],
ValueForm::Contains(s) => vec![format!("*{s}*")],
ValueForm::ContainsAll(ts) => ts.iter().map(|t| format!("*{t}*")).collect(),
}
}
pub(crate) fn matches_lower(&self, lv: &str) -> bool {
match self {
ValueForm::Exact(v) => lv == v.as_match_str().to_lowercase(),
ValueForm::OneOf(vs) => vs.iter().any(|v| lv == v.as_match_str().to_lowercase()),
ValueForm::EndsWith(s) => lv.ends_with(&s.to_lowercase()),
ValueForm::StartsWith(s) => lv.starts_with(&s.to_lowercase()),
ValueForm::Contains(t) => lv.contains(&t.to_lowercase()),
ValueForm::ContainsAll(ts) => ts.iter().all(|t| lv.contains(&t.to_lowercase())),
}
}
}
#[derive(Debug, Clone)]
pub(crate) struct DraftFieldProfile {
pub(crate) stats: FieldProfile,
pub(crate) values: Vec<Option<DraftValue>>,
pub(crate) stability: Stability,
pub(crate) form: Option<ValueForm>,
pub(crate) score: f64,
pub(crate) baseline_prevalence: Option<f64>,
pub(crate) forced: bool,
}
impl DraftFieldProfile {
pub(crate) fn field(&self) -> &str {
&self.stats.field
}
pub(crate) fn distinct(&self) -> Vec<&DraftValue> {
let mut seen: Vec<&DraftValue> = Vec::new();
for v in self.values.iter().flatten() {
if !seen.contains(&v) {
seen.push(v);
}
}
seen
}
}
pub(crate) fn profile_fields<E: Event>(
exemplars: &[E],
config: &DraftConfig,
warnings: &mut Vec<String>,
) -> Vec<DraftFieldProfile> {
let mut all_fields: BTreeSet<String> = BTreeSet::new();
for e in exemplars {
for k in e.field_keys() {
all_fields.insert(k.into_owned());
}
}
let excluded = |f: &str| {
config
.exclude_fields
.iter()
.any(|x| x.eq_ignore_ascii_case(f))
};
let forced = |f: &str| {
config
.include_fields
.iter()
.any(|x| x.eq_ignore_ascii_case(f))
};
for inc in &config.include_fields {
if !all_fields.iter().any(|f| f.eq_ignore_ascii_case(inc)) {
warnings.push(format!(
"--include-field '{inc}' does not appear in any exemplar; ignored"
));
}
}
let total = exemplars.len();
let mut out = Vec::new();
for field in all_fields {
if excluded(&field) {
continue;
}
let values: Vec<Option<DraftValue>> = exemplars
.iter()
.map(|e| {
e.get_field(&field)
.and_then(|v| DraftValue::from_event_value(&v))
})
.collect();
let present = exemplars
.iter()
.filter(|e| e.get_field(&field).is_some())
.count();
let prevalence = present as f64 / total as f64;
let is_forced = forced(&field);
if prevalence < config.min_prevalence && !is_forced {
continue;
}
if is_forced && prevalence < 1.0 {
warnings.push(format!(
"--include-field '{field}' is absent from some exemplars \
({present}/{total}); the draft may not match them"
));
}
let mut distinct_values: Vec<String> = values
.iter()
.flatten()
.map(|v| v.as_display())
.collect::<BTreeSet<_>>()
.into_iter()
.collect();
distinct_values.sort();
let stats = FieldProfile {
field: field.clone(),
present: present as u64,
total: total as u64,
distinct_values,
value_overflow: false,
};
let stability = classify_stability(&field, &values, present, config);
out.push(DraftFieldProfile {
stats,
values,
stability,
form: None,
score: 0.0,
baseline_prevalence: None,
forced: is_forced,
});
}
out
}
pub(crate) fn classify_stability(
field: &str,
values: &[Option<DraftValue>],
present: usize,
config: &DraftConfig,
) -> Stability {
let scalars: Vec<&DraftValue> = values.iter().flatten().collect();
if scalars.is_empty() || scalars.len() < present {
return Stability::Volatile;
}
if is_volatile_name(field) {
return Stability::Volatile;
}
if scalars.iter().any(|v| is_volatile_value(v)) {
return Stability::Volatile;
}
let mut distinct: Vec<&DraftValue> = Vec::new();
for v in &scalars {
if !distinct.contains(v) {
distinct.push(v);
}
}
if distinct.len() == 1 {
return Stability::Constant;
}
if distinct.len() <= config.max_value_cardinality && distinct.len() < scalars.len() {
return Stability::Enumerable;
}
let strings: Vec<&str> = distinct
.iter()
.filter_map(|v| match v {
DraftValue::Str(s) => Some(s.as_str()),
_ => None,
})
.collect();
if strings.len() == distinct.len() {
if distinct.len() == scalars.len() && strings.iter().all(|s| is_random_string(s)) {
return Stability::Volatile;
}
if shared_suffix(&strings, config.min_token_len).is_some()
|| shared_prefix(&strings, config.min_token_len).is_some()
|| !shared_tokens(&strings, config.min_token_len).is_empty()
{
return Stability::Patterned;
}
if distinct.len() <= config.max_value_cardinality {
return Stability::Enumerable;
}
} else if distinct.len() <= config.max_value_cardinality {
return Stability::Enumerable;
}
Stability::Volatile
}
pub(crate) fn is_volatile_name(field: &str) -> bool {
let segment = field.rsplit('.').next().unwrap_or(field);
let last = segment.to_lowercase();
let normalized: String = last.chars().filter(|c| *c != '_' && *c != '-').collect();
if last == "@timestamp" || normalized == "ts" {
return true;
}
if segment_words(segment)
.iter()
.any(|w| matches!(w.as_str(), "time" | "date" | "datetime" | "timestamp"))
{
return true;
}
if normalized.contains("timestamp")
|| normalized.contains("guid")
|| normalized.contains("uuid")
{
return true;
}
matches!(
normalized.as_str(),
"recordid"
| "recordnumber"
| "eventrecordid"
| "sequence"
| "seq"
| "seqno"
| "processid"
| "pid"
| "parentprocessid"
| "ppid"
| "threadid"
| "tid"
| "logonid"
| "sessionid"
| "executionprocessid"
| "executionthreadid"
)
}
pub(crate) fn segment_words(segment: &str) -> Vec<String> {
let mut words: Vec<String> = Vec::new();
let mut cur = String::new();
let mut prev: Option<char> = None;
for c in segment.chars() {
if !c.is_ascii_alphanumeric() {
if !cur.is_empty() {
words.push(std::mem::take(&mut cur));
}
prev = None;
continue;
}
if let Some(p) = prev
&& c.is_ascii_uppercase()
&& (p.is_ascii_lowercase() || p.is_ascii_digit())
&& !cur.is_empty()
{
words.push(std::mem::take(&mut cur));
}
cur.push(c.to_ascii_lowercase());
prev = Some(c);
}
if !cur.is_empty() {
words.push(cur);
}
words
}
pub(crate) fn is_volatile_value(value: &DraftValue) -> bool {
match value {
DraftValue::Str(s) => is_timestamp_string(s) || is_uuid_string(s),
DraftValue::Int(n) => is_epoch_number(*n as f64),
DraftValue::Float(f) => is_epoch_number(*f),
DraftValue::Bool(_) => false,
}
}
pub(crate) fn is_timestamp_string(s: &str) -> bool {
let b = s.as_bytes();
if b.len() < 10 {
return false;
}
let date = b[0].is_ascii_digit()
&& b[1].is_ascii_digit()
&& b[2].is_ascii_digit()
&& b[3].is_ascii_digit()
&& b[4] == b'-'
&& b[5].is_ascii_digit()
&& b[6].is_ascii_digit()
&& b[7] == b'-'
&& b[8].is_ascii_digit()
&& b[9].is_ascii_digit();
if !date {
return false;
}
b.len() == 10 || b[10] == b'T' || b[10] == b' '
}
pub(crate) fn is_uuid_string(s: &str) -> bool {
let s = s.strip_prefix('{').unwrap_or(s);
let s = s.strip_suffix('}').unwrap_or(s);
if s.len() != 36 {
return false;
}
s.char_indices().all(|(i, c)| match i {
8 | 13 | 18 | 23 => c == '-',
_ => c.is_ascii_hexdigit(),
})
}
pub(crate) fn is_epoch_number(n: f64) -> bool {
const RANGES: [(f64, f64); 4] = [
(1e9, 1e10), (1e12, 1e13), (1e15, 1e16), (1e18, 1e19), ];
RANGES.iter().any(|(lo, hi)| n >= *lo && n < *hi)
}
pub(crate) fn is_random_string(s: &str) -> bool {
s.len() >= 16
&& s.chars().all(|c| c.is_ascii_alphanumeric())
&& s.chars().any(|c| c.is_ascii_digit())
&& s.chars().any(|c| c.is_ascii_alphabetic())
}
pub(crate) fn is_structural_name(field: &str) -> bool {
let last = field.rsplit('.').next().unwrap_or(field).to_lowercase();
matches!(
last.as_str(),
"host" | "hostname" | "computer" | "computername" | "domain" | "level" | "severity"
)
}
pub(crate) fn shared_prefix(values: &[&str], min_len: usize) -> Option<String> {
let first = values.first()?;
let mut len = first.len();
for v in &values[1..] {
len = len.min(common_prefix_len(first, v));
}
while len > 0 && !first.is_char_boundary(len) {
len -= 1;
}
if len >= min_len && values.iter().any(|v| v.len() > len) {
Some(first[..len].to_string())
} else {
None
}
}
pub(crate) fn shared_suffix(values: &[&str], min_len: usize) -> Option<String> {
let first = values.first()?;
let mut len = first.len();
for v in &values[1..] {
len = len.min(common_suffix_len(first, v));
}
let mut start = first.len() - len;
while start < first.len() && !first.is_char_boundary(start) {
start += 1;
}
let len = first.len() - start;
if len >= min_len && values.iter().any(|v| v.len() > len) {
Some(first[start..].to_string())
} else {
None
}
}
pub(crate) fn common_prefix_len(a: &str, b: &str) -> usize {
a.bytes().zip(b.bytes()).take_while(|(x, y)| x == y).count()
}
pub(crate) fn common_suffix_len(a: &str, b: &str) -> usize {
a.bytes()
.rev()
.zip(b.bytes().rev())
.take_while(|(x, y)| x == y)
.count()
}
pub(crate) fn shared_tokens(values: &[&str], min_len: usize) -> Vec<String> {
let Some(first) = values.first() else {
return Vec::new();
};
let lowers: Vec<String> = values.iter().map(|v| v.to_lowercase()).collect();
let mut tokens: Vec<String> = tokenize(first, min_len)
.into_iter()
.filter(|t| {
let lt = t.to_lowercase();
lowers.iter().all(|v| v.contains(<))
})
.collect();
tokens.sort_by(|a, b| b.len().cmp(&a.len()).then_with(|| a.cmp(b)));
tokens.dedup();
tokens.truncate(3);
tokens
}
pub(crate) fn tokenize(s: &str, min_len: usize) -> Vec<String> {
let mut out: Vec<String> = Vec::new();
for token in s.split(|c: char| !c.is_ascii_alphanumeric()) {
if token.len() >= min_len && !out.iter().any(|t| t == token) {
out.push(token.to_string());
}
}
out
}
pub(crate) fn infer_form(profile: &mut DraftFieldProfile, config: &DraftConfig) {
if profile.stability == Stability::Volatile {
return;
}
let distinct: Vec<DraftValue> = profile.distinct().into_iter().cloned().collect();
if profile.stability == Stability::Patterned {
let strings: Vec<&str> = distinct
.iter()
.filter_map(|v| match v {
DraftValue::Str(s) => Some(s.as_str()),
_ => None,
})
.collect();
if strings.len() == distinct.len() {
profile.form = derive_pattern_form(&strings, config);
}
}
if profile.form.is_none() {
profile.form = derive_form(&distinct, config);
}
if profile.form.is_none() {
profile.stability = Stability::Volatile;
}
}
pub(crate) fn derive_form(distinct: &[DraftValue], config: &DraftConfig) -> Option<ValueForm> {
match distinct {
[] => None,
[one] => Some(ValueForm::Exact(one.clone())),
many if many.len() <= config.max_value_cardinality => Some(ValueForm::OneOf(many.to_vec())),
many => {
let strings: Vec<&str> = many
.iter()
.filter_map(|v| match v {
DraftValue::Str(s) => Some(s.as_str()),
_ => None,
})
.collect();
if strings.len() != many.len() {
return None;
}
derive_pattern_form(&strings, config)
}
}
}
pub(crate) fn derive_pattern_form(strings: &[&str], config: &DraftConfig) -> Option<ValueForm> {
if let Some(suffix) = shared_suffix(strings, config.min_token_len) {
return Some(ValueForm::EndsWith(suffix));
}
if let Some(prefix) = shared_prefix(strings, config.min_token_len) {
return Some(ValueForm::StartsWith(prefix));
}
let tokens = shared_tokens(strings, config.min_token_len);
match tokens.len() {
0 => None,
1 => Some(ValueForm::Contains(tokens.into_iter().next().unwrap())),
_ => Some(ValueForm::ContainsAll(tokens)),
}
}
pub(crate) fn apply_baseline<E: Event>(
profile: &mut DraftFieldProfile,
baseline: &[E],
config: &DraftConfig,
) {
let Some(form) = profile.form.clone() else {
return;
};
let field = profile.field().to_string();
let values: Vec<String> = baseline
.iter()
.filter_map(|e| {
e.get_field(&field)
.and_then(|v| v.as_str().map(|s| s.to_lowercase()))
})
.collect();
let match_count = |f: &ValueForm| values.iter().filter(|lv| f.matches_lower(lv)).count();
let token_is_generic = |t: &str| {
let lt = t.to_lowercase();
let hits = values.iter().filter(|lv| lv.contains(<)).count();
hits as f64 / baseline.len() as f64 > config.max_baseline_token_prevalence
};
let guarded = match form {
ValueForm::Contains(ref t) => {
if token_is_generic(t) {
profile.form = None;
profile.stability = Stability::Volatile;
return;
}
form
}
ValueForm::ContainsAll(ref ts) => {
let kept: Vec<String> = ts
.iter()
.filter(|t| !token_is_generic(t))
.cloned()
.collect();
match kept.len() {
0 => {
profile.form = None;
profile.stability = Stability::Volatile;
return;
}
1 => ValueForm::Contains(kept.into_iter().next().unwrap()),
_ => ValueForm::ContainsAll(kept),
}
}
other => other,
};
let hits = match_count(&guarded);
profile.form = Some(guarded);
profile.baseline_prevalence = Some(hits as f64 / baseline.len() as f64);
}
pub(crate) fn score_field(profile: &DraftFieldProfile, has_baseline: bool) -> f64 {
if profile.form.is_none() || profile.stability == Stability::Volatile {
return f64::MIN;
}
let stability_base = match profile.stability {
Stability::Constant => 3.0,
Stability::Enumerable => 2.0,
Stability::Patterned => 1.0,
Stability::Volatile => 0.0,
};
let prevalence = profile.stats.prevalence();
match profile.baseline_prevalence {
Some(bp) => stability_base * prevalence * (1.0 - bp),
None => {
let demotion = if !has_baseline && is_structural_name(profile.field()) {
0.5
} else {
0.0
};
stability_base * prevalence - demotion
}
}
}
pub(crate) fn escape_sigma_value(s: &str) -> String {
let chars: Vec<char> = s.chars().collect();
let mut out = String::with_capacity(s.len());
let mut i = 0;
while i < chars.len() {
match chars[i] {
'*' => out.push_str("\\*"),
'?' => out.push_str("\\?"),
'\\' => {
let mut j = i;
while j < chars.len() && chars[j] == '\\' {
j += 1;
}
let run = j - i;
let next = chars.get(j);
let must_escape = run > 1 || matches!(next, Some('*') | Some('?') | None);
for _ in 0..run {
if must_escape {
out.push_str("\\\\");
} else {
out.push('\\');
}
}
i = j;
continue;
}
c => out.push(c),
}
i += 1;
}
out
}
pub(crate) fn yaml_str(s: &str) -> String {
let bare_safe = !s.is_empty()
&& s.chars()
.all(|c| c.is_ascii_alphanumeric() || matches!(c, '_' | '-' | '.'))
&& !s.starts_with('-')
&& s.parse::<f64>().is_err()
&& !matches!(
s.to_ascii_lowercase().as_str(),
"true" | "false" | "null" | "yes" | "no" | "on" | "off"
);
if bare_safe {
s.to_string()
} else {
format!("'{}'", s.replace('\'', "''"))
}
}
pub(crate) fn yaml_title_str(s: &str) -> String {
let bare_safe = !s.is_empty()
&& s.chars().next().is_some_and(|c| c.is_ascii_alphanumeric())
&& !s.ends_with(' ')
&& !s.contains(": ")
&& !s.contains(" #")
&& s.chars().all(|c| {
c.is_ascii_alphanumeric() || matches!(c, ' ' | '_' | '-' | '.' | ',' | '(' | ')')
});
if bare_safe {
s.to_string()
} else {
yaml_str(s)
}
}
pub(crate) fn emit_value(v: &DraftValue) -> String {
match v {
DraftValue::Str(s) => yaml_str(&escape_sigma_value(s)),
DraftValue::Int(n) => n.to_string(),
DraftValue::Float(f) => f.to_string(),
DraftValue::Bool(b) => b.to_string(),
}
}
pub(crate) fn emit_form(out: &mut String, field: &str, form: &ValueForm, indent: &str) {
let key = format!("{field}{}", form.modifier());
match form {
ValueForm::Exact(v) => {
out.push_str(&format!("{indent}{key}: {}\n", emit_value(v)));
}
ValueForm::OneOf(vs) => {
out.push_str(&format!("{indent}{key}:\n"));
for v in vs {
out.push_str(&format!("{indent} - {}\n", emit_value(v)));
}
}
ValueForm::EndsWith(s) | ValueForm::StartsWith(s) | ValueForm::Contains(s) => {
out.push_str(&format!(
"{indent}{key}: {}\n",
yaml_str(&escape_sigma_value(s))
));
}
ValueForm::ContainsAll(ts) => {
out.push_str(&format!("{indent}{key}:\n"));
for t in ts {
out.push_str(&format!(
"{indent} - {}\n",
yaml_str(&escape_sigma_value(t))
));
}
}
}
}