#![allow(dead_code)]
use clap::ValueEnum;
#[derive(Debug, Clone)]
pub struct KeloraConfig {
pub input: InputConfig,
pub output: OutputConfig,
pub processing: ProcessingConfig,
pub performance: PerformanceConfig,
}
#[derive(Debug, Clone)]
pub struct InputConfig {
pub files: Vec<String>,
pub format: InputFormat,
pub file_order: FileOrder,
pub skip_lines: usize,
pub ignore_lines: Option<regex::Regex>,
pub multiline: Option<MultilineConfig>,
#[allow(dead_code)]
pub ts_field: Option<String>,
pub ts_format: Option<String>,
pub default_timezone: Option<String>,
pub extract_prefix: Option<String>,
pub prefix_sep: String,
}
#[derive(Debug, Clone)]
pub struct OutputConfig {
pub format: OutputFormat,
pub keys: Vec<String>,
pub exclude_keys: Vec<String>,
pub core: bool,
pub brief: bool,
pub wrap: bool,
pub color: ColorMode,
pub no_emoji: bool,
pub stats: bool,
pub metrics: bool,
pub metrics_file: Option<String>,
pub timestamp_formatting: TimestampFormatConfig,
}
#[derive(Debug, Clone)]
pub enum ScriptStageType {
Filter(String),
Exec(String),
}
#[derive(Debug, Clone)]
pub struct ErrorReportConfig {
pub style: ErrorReportStyle,
}
#[derive(Debug, Clone)]
pub enum ErrorReportStyle {
Off,
Summary,
Print,
}
#[derive(Debug, Clone)]
pub struct ProcessingConfig {
pub begin: Option<String>,
pub stages: Vec<ScriptStageType>,
pub end: Option<String>,
pub error_report: ErrorReportConfig,
pub levels: Vec<String>,
pub exclude_levels: Vec<String>,
pub window_size: usize,
pub timestamp_filter: Option<TimestampFilterConfig>,
pub take_limit: Option<usize>,
pub strict: bool,
pub verbose: u8,
pub quiet_level: u8,
}
#[derive(Debug, Clone)]
pub struct PerformanceConfig {
pub parallel: bool,
pub threads: usize,
pub batch_size: Option<usize>,
pub batch_timeout: u64,
pub no_preserve_order: bool,
}
#[derive(ValueEnum, Clone, Debug, PartialEq)]
pub enum InputFormat {
Auto,
Json,
Line,
Logfmt,
Syslog,
Cef,
Csv,
Tsv,
Csvnh,
Tsvnh,
Combined,
}
#[derive(ValueEnum, Clone, Debug, Default)]
pub enum OutputFormat {
Json,
#[default]
Default,
Logfmt,
Csv,
Tsv,
Csvnh,
Tsvnh,
None,
}
#[derive(ValueEnum, Clone, Debug)]
pub enum FileOrder {
Cli,
Name,
Mtime,
}
#[derive(ValueEnum, Clone, Debug)]
pub enum ColorMode {
Auto,
Always,
Never,
}
#[derive(Debug, Clone)]
pub struct TimestampFilterConfig {
pub since: Option<chrono::DateTime<chrono::Utc>>,
pub until: Option<chrono::DateTime<chrono::Utc>>,
}
#[derive(Debug, Clone, Default)]
pub struct TimestampFormatConfig {
pub format_fields: Vec<String>,
pub auto_format_all: bool,
pub format_as_utc: bool,
}
#[derive(Debug, Clone)]
pub struct MultilineConfig {
pub strategy: MultilineStrategy,
}
#[derive(Debug, Clone)]
pub enum MultilineStrategy {
Timestamp { pattern: String },
Indent {
spaces: Option<u32>,
tabs: bool,
mixed: bool,
},
Backslash { char: char },
Start { pattern: String },
End { pattern: String },
Boundary { start: String, end: String },
Whole,
}
impl MultilineConfig {
pub fn parse(value: &str) -> Result<Self, String> {
let parts: Vec<&str> = value.split(':').collect();
if parts.is_empty() {
return Err("Empty multiline configuration".to_string());
}
let strategy_name = parts[0];
let strategy = match strategy_name {
"timestamp" => {
let pattern = if parts.len() > 1 {
Self::parse_pattern_option(parts[1])?
} else {
r"^(\d{4}-\d{2}-\d{2}|\w{3}\s+\d{1,2})".to_string()
};
MultilineStrategy::Timestamp { pattern }
}
"indent" => {
let (spaces, tabs, mixed) = if parts.len() > 1 {
Self::parse_indent_options(parts[1])?
} else {
(None, false, true) };
MultilineStrategy::Indent {
spaces,
tabs,
mixed,
}
}
"backslash" => {
let char = if parts.len() > 1 {
Self::parse_char_option(parts[1])?
} else {
'\\' };
MultilineStrategy::Backslash { char }
}
"start" => {
if parts.len() < 2 {
return Err("Start strategy requires pattern: start:REGEX".to_string());
}
let pattern = parts[1].to_string();
MultilineStrategy::Start { pattern }
}
"end" => {
if parts.len() < 2 {
return Err("End strategy requires pattern: end:REGEX".to_string());
}
let pattern = parts[1].to_string();
MultilineStrategy::End { pattern }
}
"boundary" => {
if parts.len() < 2 {
return Err(
"Boundary strategy requires start and end: boundary:start=REGEX:end=REGEX"
.to_string(),
);
}
let (start, end) = Self::parse_boundary_options(&parts[1..])?;
MultilineStrategy::Boundary { start, end }
}
"whole" => MultilineStrategy::Whole,
_ => return Err(format!("Unknown multiline strategy: {}", strategy_name)),
};
Ok(MultilineConfig { strategy })
}
fn parse_pattern_option(option: &str) -> Result<String, String> {
if let Some(pattern) = option.strip_prefix("pattern=") {
Ok(pattern.to_string())
} else {
Err(format!("Invalid timestamp option: {}", option))
}
}
fn parse_indent_options(option: &str) -> Result<(Option<u32>, bool, bool), String> {
match option {
"tabs" => Ok((None, true, false)),
"mixed" => Ok((None, false, true)),
option if option.starts_with("spaces=") => {
let spaces_str = &option[7..];
match spaces_str.parse::<u32>() {
Ok(n) => Ok((Some(n), false, false)),
Err(_) => Err(format!("Invalid spaces value: {}", spaces_str)),
}
}
_ => Err(format!("Invalid indent option: {}", option)),
}
}
fn parse_char_option(option: &str) -> Result<char, String> {
if let Some(char_str) = option.strip_prefix("char=") {
if char_str.len() == 1 {
Ok(char_str.chars().next().unwrap())
} else {
Err(format!(
"Continuation character must be single character: {}",
char_str
))
}
} else {
Err(format!("Invalid backslash option: {}", option))
}
}
fn parse_boundary_options(parts: &[&str]) -> Result<(String, String), String> {
let mut start = None;
let mut end = None;
for part in parts {
if let Some(start_pattern) = part.strip_prefix("start=") {
start = Some(start_pattern.to_string());
} else if let Some(end_pattern) = part.strip_prefix("end=") {
end = Some(end_pattern.to_string());
} else {
return Err(format!("Invalid boundary option: {}", part));
}
}
match (start, end) {
(Some(s), Some(e)) => Ok((s, e)),
_ => Err("Boundary strategy requires both start=REGEX and end=REGEX".to_string()),
}
}
}
impl Default for MultilineConfig {
fn default() -> Self {
Self {
strategy: MultilineStrategy::Timestamp {
pattern: r"^(\d{4}-\d{2}-\d{2}|\w{3}\s+\d{1,2})".to_string(),
},
}
}
}
impl InputFormat {
pub fn default_multiline(&self) -> Option<MultilineConfig> {
None
}
}
impl KeloraConfig {
pub fn get_core_field_names() -> Vec<String> {
let mut core_fields = Vec::new();
core_fields.extend(
crate::event::TIMESTAMP_FIELD_NAMES
.iter()
.map(|s| s.to_string()),
);
core_fields.extend(
crate::event::LEVEL_FIELD_NAMES
.iter()
.map(|s| s.to_string()),
);
core_fields.extend(
crate::event::MESSAGE_FIELD_NAMES
.iter()
.map(|s| s.to_string()),
);
core_fields
}
pub fn format_error_message(&self, message: &str) -> String {
let use_colors = crate::tty::should_use_colors_with_mode(&self.output.color);
let use_emoji = use_colors && !self.output.no_emoji;
if use_emoji {
format!("🔸{}", message)
} else {
format!("kelora: {}", message)
}
}
pub fn format_stats_message(&self, message: &str) -> String {
let use_colors = crate::tty::should_use_colors_with_mode(&self.output.color);
let use_emoji = use_colors && !self.output.no_emoji;
if use_emoji {
format!("🔹Kelora Stats\n{}", message)
} else {
format!("kelora: Stats\n{}", message)
}
}
pub fn format_metrics_message(&self, message: &str) -> String {
let use_colors = crate::tty::should_use_colors_with_mode(&self.output.color);
let use_emoji = use_colors && !self.output.no_emoji;
if use_emoji {
format!("🔹Tracked metrics\n{}", message)
} else {
format!("kelora: Tracked metrics\n{}", message)
}
}
}
pub fn format_error_message_auto(message: &str) -> String {
let use_colors = crate::tty::should_use_colors_with_mode(&ColorMode::Auto);
let no_emoji = std::env::var("NO_EMOJI").is_ok();
let use_emoji = use_colors && !no_emoji;
if use_emoji {
format!("🔸{}", message)
} else {
format!("kelora: {}", message)
}
}
pub fn format_verbose_error(line_num: Option<usize>, error_type: &str, message: &str) -> String {
format_verbose_error_with_config(line_num, error_type, message, None)
}
pub fn format_verbose_error_with_config(
line_num: Option<usize>,
error_type: &str,
message: &str,
config: Option<&KeloraConfig>,
) -> String {
let use_colors = crate::tty::should_use_colors_with_mode(&ColorMode::Auto);
let no_emoji = if let Some(cfg) = config {
cfg.output.no_emoji || std::env::var("NO_EMOJI").is_ok()
} else {
std::env::var("NO_EMOJI").is_ok()
};
let use_emoji = use_colors && !no_emoji;
let prefix = if use_emoji { "🔸" } else { "kelora: " };
if let Some(line) = line_num {
format!("{}line {}: {} - {}", prefix, line, error_type, message)
} else {
format!("{}{} - {}", prefix, error_type, message)
}
}
pub fn print_verbose_error_to_stderr(
line_num: Option<usize>,
error_type: &str,
message: &str,
config: Option<&KeloraConfig>,
) {
if let Some(cfg) = config {
if cfg.processing.quiet_level > 0 {
return;
}
}
let formatted = format_verbose_error_with_config(line_num, error_type, message, config);
eprintln!("{}", formatted);
}
pub fn print_verbose_error_to_stderr_pipeline(
line_num: Option<usize>,
error_type: &str,
message: &str,
config: Option<&crate::pipeline::PipelineConfig>,
) {
if let Some(cfg) = config {
if cfg.quiet_level > 0 {
return;
}
}
let formatted =
format_verbose_error_with_pipeline_config(line_num, error_type, message, config);
eprintln!("{}", formatted);
}
pub fn format_verbose_error_with_pipeline_config(
line_num: Option<usize>,
error_type: &str,
message: &str,
config: Option<&crate::pipeline::PipelineConfig>,
) -> String {
let color_mode = config.map(|c| &c.color_mode).unwrap_or(&ColorMode::Auto);
let use_colors = crate::tty::should_use_colors_with_mode(color_mode);
let no_emoji = if let Some(cfg) = config {
cfg.no_emoji || std::env::var("NO_EMOJI").is_ok()
} else {
std::env::var("NO_EMOJI").is_ok()
};
let use_emoji = use_colors && !no_emoji;
let prefix = if use_emoji { "🔸" } else { "kelora: " };
if let Some(line) = line_num {
format!("{}line {}: {} - {}", prefix, line, error_type, message)
} else {
format!("{}{} - {}", prefix, error_type, message)
}
}
pub fn format_error_line(line: &str) -> String {
if line.chars().any(|c| c.is_control() && c != '\n') {
format!("{:?}", line) } else if line.ends_with('\n') {
line.trim_end().to_string() } else {
line.to_string() }
}
impl OutputConfig {
pub fn get_effective_keys(&self) -> Vec<String> {
if self.core {
let mut keys = KeloraConfig::get_core_field_names();
for key in &self.keys {
if !keys.contains(key) {
keys.push(key.clone());
}
}
keys
} else {
self.keys.clone()
}
}
}
impl KeloraConfig {
pub fn from_cli(cli: &crate::Cli) -> Self {
let color_mode = if cli.no_color {
ColorMode::Never
} else if cli.force_color {
ColorMode::Always
} else {
ColorMode::Auto
};
Self {
input: InputConfig {
files: cli.files.clone(),
format: if cli.json_input {
InputFormat::Json
} else {
cli.format.clone().into()
},
file_order: cli.file_order.clone().into(),
skip_lines: cli.skip_lines.unwrap_or(0),
ignore_lines: None, multiline: None, ts_field: cli.ts_field.clone(),
ts_format: cli.ts_format.clone(),
default_timezone: determine_default_timezone(cli),
extract_prefix: cli.extract_prefix.clone(),
prefix_sep: cli.prefix_sep.clone(),
},
output: OutputConfig {
format: if cli.stats_only {
OutputFormat::None
} else if cli.quiet >= 2 {
OutputFormat::None
} else if cli.json_output {
OutputFormat::Json
} else {
cli.output_format.clone().into()
},
keys: cli.keys.clone(),
exclude_keys: cli.exclude_keys.clone(),
core: cli.core,
brief: cli.brief,
wrap: !cli.no_wrap, color: color_mode,
no_emoji: cli.no_emoji,
stats: cli.stats || cli.stats_only,
metrics: cli.metrics,
metrics_file: cli.metrics_file.clone(),
timestamp_formatting: create_timestamp_format_config(cli),
},
processing: ProcessingConfig {
begin: cli.begin.clone(),
stages: Vec::new(), end: cli.end.clone(),
error_report: parse_error_report_config(cli),
levels: cli.levels.clone(),
exclude_levels: cli.exclude_levels.clone(),
window_size: cli.window_size.unwrap_or(0),
timestamp_filter: None, take_limit: cli.take,
strict: cli.strict,
verbose: cli.verbose,
quiet_level: cli.quiet,
},
performance: PerformanceConfig {
parallel: cli.parallel,
threads: cli.threads,
batch_size: cli.batch_size,
batch_timeout: cli.batch_timeout,
no_preserve_order: cli.no_preserve_order,
},
}
}
pub fn should_use_parallel(&self) -> bool {
self.performance.parallel
|| self.performance.threads > 0
|| self.performance.batch_size.is_some()
}
pub fn effective_batch_size(&self) -> usize {
self.performance.batch_size.unwrap_or(1000)
}
pub fn effective_threads(&self) -> usize {
if self.performance.threads == 0 {
num_cpus::get()
} else {
self.performance.threads
}
}
}
impl Default for KeloraConfig {
fn default() -> Self {
Self {
input: InputConfig {
files: Vec::new(),
format: InputFormat::Json,
file_order: FileOrder::Cli,
skip_lines: 0,
ignore_lines: None,
multiline: None,
ts_field: None,
ts_format: None,
default_timezone: None,
extract_prefix: None,
prefix_sep: "|".to_string(),
},
output: OutputConfig {
format: OutputFormat::Default,
keys: Vec::new(),
exclude_keys: Vec::new(),
core: false,
brief: false,
wrap: true, color: ColorMode::Auto,
no_emoji: false,
stats: false,
metrics: false,
metrics_file: None,
timestamp_formatting: TimestampFormatConfig::default(),
},
processing: ProcessingConfig {
begin: None,
stages: Vec::new(),
end: None,
error_report: ErrorReportConfig {
style: ErrorReportStyle::Summary,
},
levels: Vec::new(),
exclude_levels: Vec::new(),
window_size: 0,
timestamp_filter: None,
take_limit: None,
strict: false,
verbose: 0,
quiet_level: 0,
},
performance: PerformanceConfig {
parallel: false,
threads: 0,
batch_size: None,
batch_timeout: 200,
no_preserve_order: false,
},
}
}
}
fn create_timestamp_format_config(cli: &crate::Cli) -> TimestampFormatConfig {
let format_fields = if let Some(ref pretty_ts) = cli.pretty_ts {
pretty_ts.split(',').map(|s| s.trim().to_string()).collect()
} else {
Vec::new()
};
let auto_format_all = cli.format_timestamps_local || cli.format_timestamps_utc;
let format_as_utc = cli.format_timestamps_utc;
TimestampFormatConfig {
format_fields,
auto_format_all,
format_as_utc,
}
}
fn parse_error_report_config(cli: &crate::Cli) -> ErrorReportConfig {
let style = if cli.strict {
ErrorReportStyle::Print } else {
ErrorReportStyle::Summary };
ErrorReportConfig { style }
}
fn determine_default_timezone(cli: &crate::Cli) -> Option<String> {
if let Some(ref input_tz) = cli.input_tz {
if input_tz == "local" {
return None; } else {
return Some(input_tz.clone());
}
}
if let Ok(tz) = std::env::var("TZ") {
if !tz.is_empty() {
return Some(tz);
}
}
Some("UTC".to_string())
}
impl From<crate::InputFormat> for InputFormat {
fn from(format: crate::InputFormat) -> Self {
match format {
crate::InputFormat::Auto => InputFormat::Auto,
crate::InputFormat::Json => InputFormat::Json,
crate::InputFormat::Line => InputFormat::Line,
crate::InputFormat::Logfmt => InputFormat::Logfmt,
crate::InputFormat::Syslog => InputFormat::Syslog,
crate::InputFormat::Cef => InputFormat::Cef,
crate::InputFormat::Csv => InputFormat::Csv,
crate::InputFormat::Tsv => InputFormat::Tsv,
crate::InputFormat::Csvnh => InputFormat::Csvnh,
crate::InputFormat::Tsvnh => InputFormat::Tsvnh,
crate::InputFormat::Combined => InputFormat::Combined,
}
}
}
impl From<InputFormat> for crate::InputFormat {
fn from(format: InputFormat) -> Self {
match format {
InputFormat::Auto => crate::InputFormat::Auto,
InputFormat::Json => crate::InputFormat::Json,
InputFormat::Line => crate::InputFormat::Line,
InputFormat::Logfmt => crate::InputFormat::Logfmt,
InputFormat::Syslog => crate::InputFormat::Syslog,
InputFormat::Cef => crate::InputFormat::Cef,
InputFormat::Csv => crate::InputFormat::Csv,
InputFormat::Tsv => crate::InputFormat::Tsv,
InputFormat::Csvnh => crate::InputFormat::Csvnh,
InputFormat::Tsvnh => crate::InputFormat::Tsvnh,
InputFormat::Combined => crate::InputFormat::Combined,
}
}
}
impl From<crate::OutputFormat> for OutputFormat {
fn from(format: crate::OutputFormat) -> Self {
match format {
crate::OutputFormat::Json => OutputFormat::Json,
crate::OutputFormat::Default => OutputFormat::Default,
crate::OutputFormat::Logfmt => OutputFormat::Logfmt,
crate::OutputFormat::Csv => OutputFormat::Csv,
crate::OutputFormat::Tsv => OutputFormat::Tsv,
crate::OutputFormat::Csvnh => OutputFormat::Csvnh,
crate::OutputFormat::Tsvnh => OutputFormat::Tsvnh,
crate::OutputFormat::None => OutputFormat::None,
}
}
}
impl From<OutputFormat> for crate::OutputFormat {
fn from(format: OutputFormat) -> Self {
match format {
OutputFormat::Json => crate::OutputFormat::Json,
OutputFormat::Default => crate::OutputFormat::Default,
OutputFormat::Logfmt => crate::OutputFormat::Logfmt,
OutputFormat::Csv => crate::OutputFormat::Csv,
OutputFormat::Tsv => crate::OutputFormat::Tsv,
OutputFormat::Csvnh => crate::OutputFormat::Csvnh,
OutputFormat::Tsvnh => crate::OutputFormat::Tsvnh,
OutputFormat::None => crate::OutputFormat::None,
}
}
}
impl From<crate::FileOrder> for FileOrder {
fn from(order: crate::FileOrder) -> Self {
match order {
crate::FileOrder::Cli => FileOrder::Cli,
crate::FileOrder::Name => FileOrder::Name,
crate::FileOrder::Mtime => FileOrder::Mtime,
}
}
}
impl From<FileOrder> for crate::FileOrder {
fn from(order: FileOrder) -> Self {
match order {
FileOrder::Cli => crate::FileOrder::Cli,
FileOrder::Name => crate::FileOrder::Name,
FileOrder::Mtime => crate::FileOrder::Mtime,
}
}
}