use regex_lite::Regex;
use std::collections::BTreeMap;
use crate::{
Environment, Expression, Int, RuntimeError, RuntimeErrorKind,
expression::table::TableData,
libs::{
BuiltinInfo,
bin::{filesize_lib, time_lib},
helper::{
check_args_len, check_exact_args_len, convert_list_map_to_table, get_integer_arg,
get_string_ref,
},
lazy_module::LazyModule,
pprint::pretty_formatter,
},
reg_info, reg_lazy,
};
use crate::{
runtime::{IFS_CSV, ifs_contains},
syntax::highlight_dark_theme,
};
pub fn regist_lazy() -> LazyModule {
reg_lazy!({
string, int, float, boolean, filesize,
time,
table,
toml, json, csv, pretty,
highlight, strip,
safe, caesar
})
}
pub fn regist_info() -> BTreeMap<&'static str, BuiltinInfo> {
reg_info!({
string => "to string", "<value>"
int => "to int. radix ok(0x/0o/0b), _ as sep. e.g. 0xff_80", "<str|num|bool>"
float => "to float. % as /100, _ as sep. e.g. 12.5% -> 0.125", "<str|num|bool>"
boolean => "to bool", "<value>"
filesize => "to filesize. e.g. 1.5GB, 500K", "<size_str|int>"
time => "to datetime", "<str> [fmt]"
table => "parse cmd output to table", "<output> [split_regex] [headers...]"
toml => "to TOML", "<expr>"
json => "to JSON", "<expr>"
csv => "to CSV", "<expr>"
pretty => "to pretty string", "<expr>"
highlight => "ANSI highlight script", "<script>"
strip => "remove ANSI codes", "<string>"
safe => "wrap str, never eval", "<str>"
caesar => "caesar cipher", "<string> [shift=13]"
})
}
pub fn time(
args: Vec<Expression>,
env: &mut Environment,
ctx: &Expression,
) -> Result<Expression, RuntimeError> {
time_lib::parse(args, env, ctx)
}
pub fn table(
mut args: Vec<Expression>,
_env: &mut Environment,
ctx: &Expression,
) -> Result<Expression, RuntimeError> {
check_args_len("table", &args, 1.., ctx)?;
let opts = args.split_off(1);
let data = match args.into_iter().next().unwrap() {
Expression::String(s) => s,
Expression::List(list) => {
return Ok(Expression::Table(convert_list_map_to_table(&list)));
}
Expression::Table(t) => return Ok(Expression::Table(t)),
e => {
return Err(RuntimeError::new(
RuntimeErrorKind::TypeError {
expected: "String/List<Map>".into(),
found: e.type_name(),
sym: e.to_string(),
},
ctx.clone(),
0,
));
}
};
let mut lines: Vec<&str> = data.lines().collect();
if lines.is_empty() {
return Ok(Expression::None);
} else {
let mut c = lines.first().unwrap().chars();
if let Some(first) = c.next()
&& first.is_ascii_punctuation()
&& Some(first) == c.last()
{
return Ok(Expression::String(data));
}
}
let (headers, splitter): (Vec<String>, Option<Regex>) = match opts {
s if s.is_empty() => (Vec::new(), None),
s if s.len() == 1 => match s.first().unwrap() {
Expression::List(list) => (list.as_ref().iter().map(|x| x.to_string()).collect(), None),
Expression::BSet(list) => (list.as_ref().iter().map(|x| x.to_string()).collect(), None),
Expression::Regex(r) => (Vec::new(), Some(r.regex.clone())),
o => (vec![o.to_string()], None),
},
s if s.len() == 2 && matches!(s.first(), Some(Expression::Regex(_))) => {
match (s.first().unwrap(), s.last().unwrap()) {
(Expression::Regex(r), Expression::List(list)) => (
list.as_ref().iter().map(|x| x.to_string()).collect(),
Some(r.regex.clone()),
),
(Expression::Regex(r), Expression::BSet(list)) => (
list.as_ref().iter().map(|x| x.to_string()).collect(),
Some(r.regex.clone()),
),
(Expression::Regex(r), o) => (vec![o.to_string()], Some(r.regex.clone())),
_ => (s.iter().map(|x| x.to_string()).collect(), None),
}
}
s if matches!(s.first(), Some(Expression::Regex(_))) => (
s.iter().skip(1).map(|x| x.to_string()).collect(),
if let Some(Expression::Regex(r)) = s.first() {
Some(r.regex.clone())
} else {
None
},
),
s => (s.iter().map(|x| x.to_string()).collect(), None),
};
if lines.len() > 2 {
let last_line_cols = split_line(lines.last().unwrap(), &splitter);
let second_last_line_cols = split_line(lines[lines.len() - 2], &splitter);
if last_line_cols.len() < second_last_line_cols.len() {
lines.pop();
}
}
let (data_lines, detected_headers) = if headers.is_empty() {
let maybe_header = lines[0];
let mut maybe_header_cols = split_line(maybe_header, &splitter);
let looks_like_header = maybe_header_cols
.iter()
.all(|s| s.chars().any(|c| c.is_uppercase() || !c.is_ascii()));
if !looks_like_header && lines.len() > 2 {
let second_line_cols = split_line(lines[1], &splitter);
if maybe_header_cols.len() + maybe_header_cols.len() < second_line_cols.len() {
lines.remove(0);
maybe_header_cols = split_line(lines[0], &splitter);
}
}
if looks_like_header {
let detected = maybe_header_cols
.iter()
.map(|s| {
s.replace(":", "_")
.replace("\"", "")
.replace("%", "")
.replace("(", "_")
.replace(")", "")
.replace("$", "")
})
.collect();
(lines.split_off(1), detected)
} else {
let cols = maybe_header_cols
.iter()
.enumerate()
.map(|(i, _)| format!("C{i}"))
.collect();
(lines, cols)
}
} else {
(lines, headers)
};
let max_col = detected_headers.len();
let mut rows = Vec::with_capacity(max_col);
for line in data_lines {
if line.trim().is_empty() {
continue;
}
let slist = split_line_limited(line, &splitter, max_col);
let mut row = Vec::with_capacity(max_col);
for cell in slist {
row.push(Expression::String(cell));
}
if !row.is_empty() {
rows.push(row);
}
}
Ok(Expression::Table(TableData::new(detected_headers, rows)))
}
fn split_line<'a>(line: &'a str, regex: &Option<Regex>) -> Vec<&'a str> {
match regex {
Some(re) => re.split(line).collect(),
None => line.split_whitespace().collect(),
}
}
fn split_line_limited<'a>(line: &'a str, regex: &Option<Regex>, max_cols: usize) -> Vec<String> {
match regex {
Some(re) => re.splitn(line, max_cols).map(|x| x.to_string()).collect(),
None => {
let mut out = Vec::new();
let mut iter = line.split_whitespace();
for _ in 0..max_cols.saturating_sub(1) {
if let Some(v) = iter.next() {
out.push(v.to_string());
}
}
let rest: String = iter.collect::<Vec<_>>().join(" ");
if !rest.is_empty() {
out.push(rest);
}
out
}
}
}
fn boolean(
args: Vec<Expression>,
_env: &mut Environment,
ctx: &Expression,
) -> Result<Expression, RuntimeError> {
check_exact_args_len("boolean", &args, 1, ctx)?;
Ok(Expression::Boolean(args[0].is_truthy()))
}
pub fn string(
args: Vec<Expression>,
_env: &mut Environment,
ctx: &Expression,
) -> Result<Expression, RuntimeError> {
check_exact_args_len("string", &args, 1, ctx)?;
Ok(Expression::String(args[0].to_string()))
}
pub fn int(
args: Vec<Expression>,
_env: &mut Environment,
ctx: &Expression,
) -> Result<Expression, RuntimeError> {
check_exact_args_len("int", &args, 1, ctx)?;
match &args[0] {
Expression::Integer(x) => Ok(Expression::Integer(*x)),
Expression::Float(x) => Ok(Expression::Integer(*x as Int)),
Expression::Boolean(b) => Ok(Expression::Integer(if *b { 1 } else { 0 })),
Expression::String(x) => {
let x = x.replace('_', "");
let int = if x.starts_with("0x") {
Int::from_str_radix(&x, 16).map_err(|_| {
RuntimeError::common(format!("invalid Hex number").into(), ctx.clone(), 0)
})
} else if x.starts_with("0o") {
Int::from_str_radix(&x, 8).map_err(|_| {
RuntimeError::common(format!("invalid Oct number").into(), ctx.clone(), 0)
})
} else if x.starts_with("0b") {
Int::from_str_radix(&x, 2).map_err(|_| {
RuntimeError::common(format!("invalid Bin number").into(), ctx.clone(), 0)
})
} else {
x.parse::<Int>().map_err(|_| {
RuntimeError::common(
format!("could not convert {x:?} to an integer").into(),
ctx.clone(),
0,
)
})
};
Ok(Expression::Integer(int?))
}
otherwise => Err(RuntimeError::common(
format!("could not convert {otherwise:?} to an integer").into(),
ctx.clone(),
0,
)),
}
}
pub fn float(
args: Vec<Expression>,
_env: &mut Environment,
ctx: &Expression,
) -> Result<Expression, RuntimeError> {
check_exact_args_len("float", &args, 1, ctx)?;
match &args[0] {
Expression::Integer(x) => Ok(Expression::Float(*x as f64)),
Expression::Float(x) => Ok(Expression::Float(*x)),
Expression::Boolean(b) => Ok(Expression::Float(if *b { 1.0 } else { 0.0 })),
Expression::String(x) => {
let x = x.replace('_', "");
let xt = x.trim();
let r = match xt.ends_with("%") {
true => xt.trim_end_matches('%').parse::<f64>().map(|f| f * 0.01),
false => xt.parse::<f64>(),
};
if let Ok(n) = r {
Ok(Expression::Float(n))
} else {
Err(RuntimeError::common(
format!("could not convert {x:?} to a float").into(),
ctx.clone(),
0,
))
}
}
otherwise => Err(RuntimeError::common(
format!("could not convert {otherwise:?} to a float").into(),
ctx.clone(),
0,
)),
}
}
pub fn filesize(
args: Vec<Expression>,
env: &mut Environment,
ctx: &Expression,
) -> Result<Expression, RuntimeError> {
check_exact_args_len("filesize", &args, 1, ctx)?;
filesize_lib::from(args, env, ctx)
}
fn escape_string_common(s: &str) -> String {
let mut escaped = String::with_capacity(s.len());
for ch in s.chars() {
match ch {
'"' => escaped.push_str("\\\""),
'\\' => escaped.push_str("\\\\"),
'\n' => escaped.push_str("\\n"),
'\r' => escaped.push_str("\\r"),
'\t' => escaped.push_str("\\t"),
'\u{0008}' => escaped.push_str("\\b"),
'\u{000C}' => escaped.push_str("\\f"),
_ if ch.is_control() => escaped.push_str(&format!("\\u{:04x}", ch as u32)),
_ => escaped.push(ch),
}
}
escaped
}
fn escape_toml_string(s: &str) -> String {
escape_string_common(s)
}
fn escape_json_string(s: &str) -> String {
escape_string_common(s)
}
pub fn toml(
args: Vec<Expression>,
_env: &mut Environment,
ctx: &Expression,
) -> Result<Expression, RuntimeError> {
check_exact_args_len("toml", &args, 1, ctx)?;
let expr = &args[0];
let toml_str = expr_to_toml_string(expr, None);
Ok(Expression::String(toml_str))
}
fn needs_quotes(key: &str) -> bool {
if key.is_empty() {
return true;
}
for ch in key.chars() {
if !ch.is_alphanumeric() && ch != '_' && ch != '-' {
return true;
}
}
matches!(key, "true" | "false" | "null" | "inf" | "nan")
}
fn expr_to_toml_string(expr: &Expression, table_prefix: Option<&str>) -> String {
match expr {
Expression::None => "".to_string(),
Expression::Boolean(b) => b.to_string(),
Expression::Integer(i) => i.to_string(),
Expression::Float(f) => {
if f.is_infinite() {
if f.is_sign_positive() {
"inf".to_string()
} else {
"-inf".to_string()
}
} else if f.is_nan() {
"nan".to_string()
} else {
f.to_string()
}
}
Expression::DateTime(dt) => dt.format("%Y-%m-%dT%H:%M:%S").to_string(),
Expression::String(s) => format!("\"{}\"", escape_toml_string(s)),
Expression::List(list) => {
if list
.iter()
.all(|item| matches!(item, Expression::Map(_) | Expression::HMap(_)))
{
let mut output = Vec::new();
for item in list.iter() {
output.push(format!("[[{}]]", table_prefix.unwrap_or("item")));
let table_content = expr_to_toml_string(item, table_prefix);
output.push(table_content);
}
output.join("\n")
} else {
let items: Vec<String> =
list.iter().map(|e| expr_to_toml_string(e, None)).collect();
format!("[{}]", items.join(", "))
}
}
Expression::BSet(set) => {
let items: Vec<String> = set.iter().map(|e| expr_to_toml_string(e, None)).collect();
format!("[{}]", items.join(", "))
}
Expression::Map(map) => {
let mut output = Vec::new();
let mut tables = BTreeMap::new();
let mut simple_keys = BTreeMap::new();
for (key, value) in map.as_ref() {
if matches!(value, Expression::Map(_) | Expression::HMap(_)) {
tables.insert(key.clone(), value);
} else {
simple_keys.insert(key.clone(), value);
}
}
for (key, value) in &simple_keys {
let formatted_key = if needs_quotes(key) {
format!("\"{}\"", escape_toml_string(key))
} else {
key.clone()
};
output.push(format!(
"{} = {}",
formatted_key,
expr_to_toml_string(value, None)
));
}
for (table_name, table_expr) in &tables {
let formatted_table_name = if needs_quotes(table_name) {
format!("\"{}\"", escape_toml_string(table_name))
} else {
table_name.clone()
};
let full_table_name = match table_prefix {
Some(prefix) => format!("{prefix}.{formatted_table_name}"),
None => formatted_table_name,
};
output.push(format!("\n[{full_table_name}]"));
let table_content = expr_to_toml_string(table_expr, Some(&full_table_name));
for line in table_content.lines() {
output.push(line.to_string());
}
}
output.join("\n")
}
Expression::HMap(map) => {
let mut output = Vec::new();
let mut tables = BTreeMap::new();
let mut simple_keys = BTreeMap::new();
for (key, value) in map.as_ref() {
if matches!(value, Expression::Map(_) | Expression::HMap(_)) {
tables.insert(key.clone(), value);
} else {
simple_keys.insert(key.clone(), value);
}
}
for (key, value) in &simple_keys {
let formatted_key = if needs_quotes(key) {
format!("\"{}\"", escape_toml_string(key))
} else {
key.clone()
};
output.push(format!(
"{} = {}",
formatted_key,
expr_to_toml_string(value, None)
));
}
for (table_name, table_expr) in &tables {
let formatted_table_name = if needs_quotes(table_name) {
format!("\"{}\"", escape_toml_string(table_name))
} else {
table_name.clone()
};
let full_table_name = match table_prefix {
Some(prefix) => format!("{prefix}.{formatted_table_name}"),
None => formatted_table_name,
};
output.push(format!("\n[{full_table_name}]"));
let table_content = expr_to_toml_string(table_expr, Some(&full_table_name));
for line in table_content.lines() {
output.push(line.to_string());
}
}
output.join("\n")
}
Expression::Table(table_data) => {
let mut output = Vec::new();
for row in table_data.rows() {
output.push(format!("[[{}]]", table_prefix.unwrap_or("item")));
for (j, header) in table_data.headers().iter().enumerate() {
let value = row.get(j).cloned().unwrap_or(Expression::None);
output.push(format!(
"{} = {}",
header,
expr_to_toml_string(&value, None)
));
}
}
output.join("\n")
}
other => format!("\"{}\"", escape_toml_string(&other.to_string())),
}
}
pub fn json(
args: Vec<Expression>,
_env: &mut Environment,
ctx: &Expression,
) -> Result<Expression, RuntimeError> {
check_exact_args_len("json", &args, 1, ctx)?;
let expr = &args[0];
let json_str = expr_to_json_string(expr);
Ok(Expression::String(json_str))
}
fn expr_to_json_string(expr: &Expression) -> String {
match expr {
Expression::None => "null".to_string(),
Expression::Boolean(b) => b.to_string(),
Expression::Integer(i) => i.to_string(),
Expression::Float(f) => {
if f.is_infinite() || f.is_nan() {
"null".to_string()
} else {
f.to_string()
}
}
Expression::DateTime(dt) => {
format!("\"{}\"", dt.format("%Y-%m-%dT%H:%M:%S%.fZ"))
}
Expression::String(s) => format!("\"{}\"", escape_json_string(s)),
Expression::List(list) => {
let items: Vec<String> = list.iter().map(expr_to_json_string).collect();
format!("[{}]", items.join(","))
}
Expression::BSet(set) => {
let items: Vec<String> = set.iter().map(expr_to_json_string).collect();
format!("[{}]", items.join(","))
}
Expression::Map(map) => {
let pairs: Vec<String> = map
.iter()
.map(|(k, v)| format!("\"{}\":{}", escape_json_string(k), expr_to_json_string(v)))
.collect();
format!("{{{}}}", pairs.join(","))
}
Expression::HMap(map) => {
let pairs: Vec<String> = map
.iter()
.map(|(k, v)| format!("\"{}\":{}", escape_json_string(k), expr_to_json_string(v)))
.collect();
format!("{{{}}}", pairs.join(","))
}
Expression::Table(table_data) => {
let mut items = Vec::new();
for row in table_data.rows() {
let mut pairs = Vec::new();
for (j, header) in table_data.headers().iter().enumerate() {
let value = row.get(j).cloned().unwrap_or(Expression::None);
pairs.push(format!(
"\"{}\":{}",
escape_json_string(header),
expr_to_json_string(&value)
));
}
items.push(format!("{{{}}}", pairs.join(",")));
}
format!("[{}]", items.join(","))
}
other => format!("\"{}\"", escape_json_string(&other.to_string())),
}
}
pub fn csv(
mut args: Vec<Expression>,
env: &mut Environment,
ctx: &Expression,
) -> Result<Expression, RuntimeError> {
check_exact_args_len("csv", &args, 1, ctx)?;
let expr = args.pop().unwrap();
let ifs = env.get("IFS");
let delimiter = match (ifs_contains(IFS_CSV, env), &ifs) {
(true, Some(Expression::String(fs))) if !fs.is_empty() && fs != "\n" => fs.as_bytes()[0],
_ => b',',
};
let csv_err = |msg: String| RuntimeError::common(msg.into(), ctx.clone(), 0);
let result = match expr {
Expression::List(rows) => {
let mut writer = csv::WriterBuilder::new()
.delimiter(delimiter)
.from_writer(vec![]);
let mut all_keys = BTreeMap::new();
for row in rows.as_ref() {
if let Expression::Map(map) = row {
for key in map.keys() {
all_keys.insert(key.clone(), ());
}
}
}
let sorted_keys: Vec<_> = all_keys.keys().collect();
writer
.write_record(&sorted_keys)
.map_err(|e| csv_err(format!("CSV write failed: {e}")))?;
for row in rows.as_ref() {
if let Expression::Map(map) = row {
let mut record = Vec::new();
for key in &sorted_keys {
let value = map.get(*key).map(expr_to_json_string).unwrap_or_default();
record.push(value);
}
writer
.write_record(&record)
.map_err(|e| csv_err(format!("CSV write failed: {e}")))?;
}
}
let inner = writer
.into_inner()
.map_err(|e| csv_err(format!("CSV flush failed: {e}")))?;
String::from_utf8(inner).map_err(|e| csv_err(format!("CSV write failed: {e}")))
}
Expression::Map(map) => {
let mut writer = csv::WriterBuilder::new()
.delimiter(delimiter)
.from_writer(vec![]);
let sorted_keys: Vec<_> = map.keys().collect();
writer
.write_record(&sorted_keys)
.map_err(|e| csv_err(format!("CSV write failed: {e}")))?;
let record: Vec<_> = sorted_keys
.iter()
.map(|k| expr_to_json_string(map.get(*k).unwrap()))
.collect();
writer
.write_record(&record)
.map_err(|e| csv_err(format!("CSV write failed: {e}")))?;
let inner = writer
.into_inner()
.map_err(|e| csv_err(format!("CSV flush failed: {e}")))?;
String::from_utf8(inner).map_err(|e| csv_err(format!("CSV write failed: {e}")))
}
Expression::HMap(map) => {
let mut writer = csv::WriterBuilder::new()
.delimiter(delimiter)
.from_writer(vec![]);
let sorted_keys: Vec<_> = map.keys().collect();
writer
.write_record(&sorted_keys)
.map_err(|e| csv_err(format!("CSV write failed: {e}")))?;
let record: Vec<_> = sorted_keys
.iter()
.map(|k| expr_to_json_string(map.get(*k).unwrap()))
.collect();
writer
.write_record(&record)
.map_err(|e| csv_err(format!("CSV write failed: {e}")))?;
let inner = writer
.into_inner()
.map_err(|e| csv_err(format!("CSV flush failed: {e}")))?;
String::from_utf8(inner).map_err(|e| csv_err(format!("CSV write failed: {e}")))
}
Expression::String(ct) => Ok(ct),
Expression::Table(table_data) => {
let mut writer = csv::WriterBuilder::new()
.delimiter(delimiter)
.from_writer(vec![]);
writer
.write_record(table_data.headers())
.map_err(|e| csv_err(format!("CSV write failed: {e}")))?;
for row in table_data.rows() {
let record: Vec<String> = row.iter().map(|v| v.to_string()).collect();
writer
.write_record(&record)
.map_err(|e| csv_err(format!("CSV write failed: {e}")))?;
}
let inner = writer
.into_inner()
.map_err(|e| csv_err(format!("CSV flush failed: {e}")))?;
String::from_utf8(inner).map_err(|e| csv_err(format!("CSV write failed: {e}")))
}
o => Ok(o.to_string()),
};
result.map(Expression::from)
}
pub fn pretty(
args: Vec<Expression>,
_env: &mut Environment,
ctx: &Expression,
) -> Result<Expression, RuntimeError> {
check_exact_args_len("pretty", &args, 1, ctx)?;
Ok(Expression::String(pretty_formatter(&args[0])))
}
fn highlight(
args: Vec<Expression>,
_env: &mut Environment,
ctx: &Expression,
) -> Result<Expression, RuntimeError> {
check_exact_args_len("highlight", &args, 1, ctx)?;
let script = get_string_ref(&args[0], ctx)?;
if script.is_empty() {
return Ok(Expression::None);
}
let hi = highlight_dark_theme(script);
Ok(Expression::String(hi))
}
pub fn strip(
args: Vec<Expression>,
_env: &mut Environment,
ctx: &Expression,
) -> Result<Expression, RuntimeError> {
check_exact_args_len("strip", &args, 1, ctx)?;
let p = get_string_ref(&args[0], ctx)?;
Ok(strip_ansi_escapes(p).into())
}
fn safe(
args: Vec<Expression>,
_env: &mut Environment,
_ctx: &Expression,
) -> Result<Expression, RuntimeError> {
let str = args
.into_iter()
.next()
.map_or("".to_string(), |exp| exp.to_string());
Ok(Expression::StringSafe(str))
}
fn caesar(
args: Vec<Expression>,
env: &mut Environment,
ctx: &Expression,
) -> Result<Expression, RuntimeError> {
check_args_len("caesar", &args, 1..=2, ctx)?;
let text = get_string_ref(&args[0], ctx)?;
let shift = if args.len() > 1 {
get_integer_arg(args[1].eval(env)?, ctx)?
} else {
13
};
let mut result = String::with_capacity(text.len());
for c in text.chars() {
if c.is_ascii_alphabetic() {
let base = if c.is_ascii_lowercase() { b'a' } else { b'A' };
let offset = (c as u8 - base) as i64;
let shifted = ((offset + shift).rem_euclid(26) as u8 + base) as char;
result.push(shifted);
} else {
result.push(c);
}
}
Ok(Expression::String(result))
}
pub fn strip_ansi_escapes(text: &str) -> String {
use std::sync::OnceLock;
static ANSI_RE: OnceLock<Regex> = OnceLock::new();
let re = ANSI_RE.get_or_init(|| {
Regex::new(
r"\x1b\[[0-?]*[ -/]*[@-~]|\x1b\][^\x07\x1b]*(?:\x07|\x1b\\)|\x1b[@-_]|[\x80-\x9F]",
)
.unwrap()
});
re.replace_all(text, "").into_owned()
}