//src/security/detect/bash/deobf.rs
//
// 在达到命令边界（CommittedBlock 生成）之后调用，对块内容做两阶段清洗：
//
// Phase A - 词法级还原（不产生新的可执行代码，只还原字面量）
// - 反斜杠转义: c\a\t /etc\/pas\s\w\d -> cat /etc/passwd
// - 垃圾变量胶水: c$9a$1t $7/etc/p$8asswd -> cat /etc/passwd
// - 引号拆分拼接: c"$9"at /etc/passw"$1"d -> cat /etc/passwd
//
// Phase B - 执行汇聚点抽取（识别"字符串在运行时会被当作代码执行"的位置，
// 解码/展开后作为新的 CommittedBlock 递归投喂回 Phase A）
// - 编码管道: echo Y2F0IC9ldGMvcGFzc3dk | base64 -d | bash
// - 十六进制转义管道: echo -e '\x63\x61\x74' | bash / printf '\x63\x61\x74'
// - 反转管道: echo 'dwssap/cte/ tac' | rev | bash
// - herestring 直喂解码器: base64 -d <<< BASE64DATA | bash
// - 裸解释器汇聚: echo '<script>' | bash（无专用解码器时，字面量本身即脚本）
// - eval / bash -c / source 等嵌套解释器
// - $(...) 命令替换内部脚本
//
use std::sync::LazyLock;
use std::sync::atomic::{AtomicUsize, Ordering};
use tree_sitter::{Node, Query, QueryCursor, StreamingIterator, Tree};
use crate::security::detect::ShellContext;
use crate::security::detect::bash::ast::{
BashAstState, CommittedBlock, capture_by_name, get_command_name, language,
};
use crate::security::detect::bash::utils::clean_bash_string;
use crate::security::detect::utils::{name_normalize, node_extract_text};
/// 反混淆递归展开的最大深度（防止 base64(base64(base64(...))) 式套娃）
pub const MAX_DEOBF_DEPTH: usize = 64;
/// 单次 on_detect 调用中，Phase B 允许递归处理的总字节预算
pub const MAX_DEOBF_TOTAL_BYTES: usize = 512 * 1024;
// =============================================================================
// 元数据：记录一个块经历过哪些反混淆处理，供审计 / 规则消费
// =============================================================================
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub enum ObfuscationTechnique {
BackslashEscape,
QuoteSplitConcatenation,
JunkVariableExpansion,
IfsGlue,
AnsiCEscape,
ParameterDefaultValue,
Base64Pipe,
HexPipe,
RotCipherPipe,
/// `echo -e '\xHH...'` 风格的十六进制/转义序列解码
EchoDashEHex,
/// `printf '\xHH...'` 风格的十六进制/转义序列解码
PrintfHex,
/// `| rev` 字符串反转管道
RevPipe,
EvalWrapping,
NestedShellInvocation,
CommandSubstitutionExec,
/// 存在无法静态求值的动态内容（如内层命令替换），仅打标不展开
UnresolvedDynamic,
}
#[derive(Debug, Clone, Default)]
pub struct DeobfMeta {
/// 本块自身（Phase A）命中的还原手法
pub techniques: Vec<ObfuscationTechnique>,
/// 若本块是从父块解码/展开而来，记录完整来源链，便于溯源
pub decode_chain: Vec<ObfuscationTechnique>,
/// 仅在发生了实质性重写时才保留原始文本，用于审计日志
pub raw_source: Option<String>,
}
// =============================================================================
// Phase A 中间表示：与 tree_sitter::Node 生命周期解耦的纯数据结构
// =============================================================================
#[derive(Debug, Clone)]
enum WordPart {
/// 静态字面量，直接拼接，不参与后续任何处理
Literal(String),
/// 出现在非引号上下文中的变量引用：解析结果若含空白，会触发 IFS 分词
UnquotedVar {
name: String,
default: Option<String>,
},
/// 出现在双引号内的变量引用：解析结果整体拼接，不分词
QuotedVar {
name: String,
default: Option<String>,
},
/// 命令替换 / 进程替换等动态内容：Phase A 不解析，原样保留字节，
/// 交给 Phase B 单独抽取处理。写回时不会被加引号，以保留可执行语法。
Raw(String),
}
/// 一个"shell 词"的中间表示。
///
/// `start_byte` / `end_byte` 取的是该词在源码中对应的**完整节点区间**
/// （例如 `command_name` 节点的整体区间，或某个参数节点的区间），
/// Phase A 的重写以此区间为最小编辑粒度，而不是整条 command。
#[derive(Debug, Clone, Default)]
struct WordSpec {
start_byte: usize,
end_byte: usize,
parts: Vec<WordPart>,
}
struct CommandSpec {
/// 按顺序排列：command_name + 各参数（不含 redirect / 赋值前缀等未识别节点）
words: Vec<WordSpec>,
/// 遍历过程中天然发现的手法（转义、拼接等，与变量解析无关的部分）
techs: Vec<ObfuscationTechnique>,
}
// =============================================================================
// Phase A - 同步阶段：AST -> CommandSpec（不访问 ShellContext）
// =============================================================================
fn unescape_word_text(raw: &str) -> String {
let mut out = String::with_capacity(raw.len());
let mut chars = raw.chars();
while let Some(c) = chars.next() {
if c == '\\' {
match chars.next() {
Some('\n') => { /* 行内续行符：吞掉，不产生任何字符 */ }
Some(next) => out.push(next),
None => out.push('\\'),
}
} else {
out.push(c);
}
}
out
}
/// 双引号字符串内部：反斜杠只在 $ ` " \ 换行 前面才有转义意义，其余场景保留原样
fn unescape_double_quoted_content(raw: &str) -> String {
let mut out = String::with_capacity(raw.len());
let mut chars = raw.chars().peekable();
while let Some(c) = chars.next() {
if c == '\\'
&& let Some(&next) = chars.peek()
&& matches!(next, '$' | '`' | '"' | '\\' | '\n')
{
out.push(next);
chars.next();
continue;
}
out.push(c);
}
out
}
/// $'...' ANSI-C 字符串转义（\n \t \r \\ \xHH \oOOO ...）
fn unescape_ansi_c_string(raw: &str) -> String {
let mut out = String::new();
let mut it = raw.chars().peekable();
while let Some(c) = it.next() {
if c != '\\' {
out.push(c);
continue;
}
match it.next() {
Some('n') => out.push('\n'),
Some('t') => out.push('\t'),
Some('r') => out.push('\r'),
Some('e') => out.push('\x1b'),
Some('\\') => out.push('\\'),
Some('\'') => out.push('\''),
Some('0') => out.push('\0'),
Some('x') => {
let hex: String = it.by_ref().take(2).collect();
if let Ok(v) = u8::from_str_radix(&hex, 16) {
out.push(v as char);
}
}
Some(o) if o.is_digit(8) => {
let mut oct = String::from(o);
oct.extend(it.by_ref().take(2).filter(|c| c.is_digit(8)));
if let Ok(v) = u8::from_str_radix(&oct, 8) {
out.push(v as char);
}
}
Some(other) => out.push(other),
None => {}
}
}
out
}
/// echo -e / printf 风格的反斜杠转义（\n \t \r \a \b \f \v \e \xHH \0NNN 等）
///
/// 与 `unescape_ansi_c_string` 的区别在于：这里处理的是不带外层 `$'...'`
/// 包裹的普通字符串（来自 `echo -e '...'` 或 `printf '...'` 的参数），
/// 且对未知转义序列采取保守策略（原样保留 `\X`），避免破坏语义。
fn unescape_c_style_escapes(input: &str) -> String {
let mut out = String::with_capacity(input.len());
let mut chars = input.chars().peekable();
while let Some(c) = chars.next() {
if c != '\\' {
out.push(c);
continue;
}
match chars.next() {
Some('n') => out.push('\n'),
Some('t') => out.push('\t'),
Some('r') => out.push('\r'),
Some('a') => out.push('\x07'),
Some('b') => out.push('\x08'),
Some('f') => out.push('\x0c'),
Some('v') => out.push('\x0b'),
Some('e') => out.push('\x1b'),
Some('\\') => out.push('\\'),
Some('x') => {
let hex: String = chars.by_ref().take(2).collect();
match u8::from_str_radix(&hex, 16) {
Ok(v) => out.push(v as char),
Err(_) => {
out.push_str("\\x");
out.push_str(&hex);
}
}
}
Some(d0) if d0.is_digit(8) => {
let mut oct = String::from(d0);
oct.extend(chars.by_ref().take(2).filter(|c| c.is_digit(8)));
if let Ok(v) = u8::from_str_radix(&oct, 8) {
out.push(v as char);
}
}
Some(other) => {
out.push('\\');
out.push(other);
}
None => out.push('\\'),
}
}
out
}
/// 从 `${name}` / `${name:-default}` / `${name-default}` / `$name` 中提取名字与默认值
fn parse_expansion_text(text: &str) -> (String, Option<String>) {
let inner = text
.trim_start_matches("${")
.trim_end_matches('}')
.trim_start_matches('$');
if let Some((name, default)) = inner.split_once(":-") {
(name.to_string(), Some(default.to_string()))
} else if let Some((name, default)) = inner.split_once('-') {
(name.to_string(), Some(default.to_string()))
} else {
(inner.to_string(), None)
}
}
fn collect_expansion_part(node: Node, source: &[u8], quoted: bool, parts: &mut Vec<WordPart>) {
let text = node_extract_text(&node, source).unwrap_or("");
if text.starts_with("${") {
let (name, default) = parse_expansion_text(text);
if quoted {
parts.push(WordPart::QuotedVar { name, default });
} else {
parts.push(WordPart::UnquotedVar { name, default });
}
} else if text.starts_with('$') {
let inner = text.trim_start_matches('$');
let mut chars = inner.chars();
let mut var_name = String::new();
let mut trailing = String::new();
if let Some(first) = chars.next() {
if first.is_ascii_digit() || matches!(first, '@' | '*' | '?' | '-' | '$' | '!' | '#') {
var_name.push(first);
trailing = inner[first.len_utf8()..].to_string();
} else if first.is_ascii_alphabetic() || first == '_' {
var_name.push(first);
for c in chars {
if c.is_ascii_alphanumeric() || c == '_' {
var_name.push(c);
} else {
break;
}
}
trailing = inner[var_name.len()..].to_string();
} else {
var_name = inner.to_string();
}
}
if quoted {
parts.push(WordPart::QuotedVar {
name: var_name,
default: None,
});
} else {
parts.push(WordPart::UnquotedVar {
name: var_name,
default: None,
});
}
if !trailing.is_empty() {
parts.push(WordPart::Literal(trailing));
}
} else {
let (name, default) = parse_expansion_text(text);
if quoted {
parts.push(WordPart::QuotedVar { name, default });
} else {
parts.push(WordPart::UnquotedVar { name, default });
}
}
}
fn collect_word_parts(
node: Node,
source: &[u8],
force_quoted: bool,
parts: &mut Vec<WordPart>,
techs: &mut Vec<ObfuscationTechnique>,
) {
match node.kind() {
"word" => {
let raw = node_extract_text(&node, source).unwrap_or("");
let cleaned = unescape_word_text(raw);
if cleaned != raw {
techs.push(ObfuscationTechnique::BackslashEscape);
}
parts.push(WordPart::Literal(cleaned));
}
"raw_string" => {
let raw = node_extract_text(&node, source).unwrap_or("");
parts.push(WordPart::Literal(clean_bash_string(raw)));
}
"ansi_c_string" => {
techs.push(ObfuscationTechnique::AnsiCEscape);
let raw = node_extract_text(&node, source).unwrap_or("");
let inner = raw.trim_start_matches("$'").trim_end_matches('\'');
parts.push(WordPart::Literal(unescape_ansi_c_string(inner)));
}
"string" => {
let mut cursor = node.walk();
for child in node.children(&mut cursor) {
match child.kind() {
"string_content" => {
let raw = node_extract_text(&child, source).unwrap_or("");
parts.push(WordPart::Literal(unescape_double_quoted_content(raw)));
}
"simple_expansion" | "expansion" => {
collect_expansion_part(child, source, true, parts);
}
"command_substitution" => {
techs.push(ObfuscationTechnique::UnresolvedDynamic);
let raw = node_extract_text(&child, source).unwrap_or("");
parts.push(WordPart::Raw(raw.to_string()));
}
"\"" => { /* 引号本身跳过 */ }
_ => {
if let Some(t) = node_extract_text(&child, source) {
parts.push(WordPart::Literal(t.to_string()));
}
}
}
}
}
"simple_expansion" | "expansion" => {
collect_expansion_part(node, source, force_quoted, parts);
}
"concatenation" => {
let mut cursor = node.walk();
for child in node.children(&mut cursor) {
collect_word_parts(child, source, force_quoted, parts, techs);
}
techs.push(ObfuscationTechnique::QuoteSplitConcatenation);
}
"command_substitution" | "process_substitution" => {
techs.push(ObfuscationTechnique::UnresolvedDynamic);
let raw = node_extract_text(&node, source).unwrap_or("");
parts.push(WordPart::Raw(raw.to_string()));
}
_ => {
if let Some(t) = node_extract_text(&node, source) {
parts.push(WordPart::Literal(t.to_string()));
}
}
}
}
fn build_word_spec(node: Node, source: &[u8], techs: &mut Vec<ObfuscationTechnique>) -> WordSpec {
let mut parts = Vec::new();
collect_word_parts(node, source, false, &mut parts, techs);
WordSpec {
start_byte: node.start_byte(),
end_byte: node.end_byte(),
parts,
}
}
const WORD_LIKE_KINDS: &[&str] = &[
"word",
"string",
"raw_string",
"ansi_c_string",
"concatenation",
"simple_expansion",
"expansion",
"command_substitution",
"process_substitution",
];
fn build_command_spec(node: Node, source: &[u8]) -> CommandSpec {
let mut words = Vec::new();
let mut techs = Vec::new();
let mut cursor = node.walk();
for child in node.children(&mut cursor) {
match child.kind() {
"command_name" => {
let mut parts = Vec::new();
let mut wc = child.walk();
let mut has_named_child = false;
for w in child.children(&mut wc) {
has_named_child = true;
collect_word_parts(w, source, false, &mut parts, &mut techs);
}
if !has_named_child {
collect_word_parts(child, source, false, &mut parts, &mut techs);
}
words.push(WordSpec {
start_byte: child.start_byte(),
end_byte: child.end_byte(),
parts,
});
}
k if WORD_LIKE_KINDS.contains(&k) => {
words.push(build_word_spec(child, source, &mut techs));
}
_ => {}
}
}
CommandSpec { words, techs }
}
fn has_ancestor_kind(node: &Node, kind: &str) -> bool {
let mut cur = node.parent();
while let Some(p) = cur {
if p.kind() == kind {
return true;
}
cur = p.parent();
}
false
}
fn collect_command_specs(tree: &Tree, source: &[u8]) -> Vec<CommandSpec> {
static QUERY: LazyLock<Query> =
LazyLock::new(|| Query::new(&language(), "(command) @cmd").expect("invalid query"));
let mut cursor = QueryCursor::new();
let mut specs = Vec::new();
let mut matches = cursor.matches(&QUERY, tree.root_node(), source);
while let Some(m) = matches.next() {
if let Some(cmd) = capture_by_name(&QUERY, m, "cmd") {
if has_ancestor_kind(&cmd, "command_substitution") {
continue;
}
specs.push(build_command_spec(cmd, source));
}
}
specs
}
// =============================================================================
// Phase A - 异步阶段：CommandSpec -> 规范化文本（此时才访问 ShellContext）
// =============================================================================
async fn lookup_variable(name: &str, ctx: &ShellContext) -> String {
if let Some(v) = ctx.var.get(name).await
&& let Some(s) = v.as_str()
{
return s.to_string();
}
if let Some(v) = ctx.env_get(name).await {
return v;
}
if name == "IFS" {
return " ".to_string();
}
String::new()
}
async fn resolve_var(
name: &str,
default: &Option<String>,
ctx: &ShellContext,
techs: &mut Vec<ObfuscationTechnique>,
) -> String {
let val = lookup_variable(name, ctx).await;
if val.is_empty() {
if let Some(d) = default {
techs.push(ObfuscationTechnique::ParameterDefaultValue);
return d.clone();
}
techs.push(ObfuscationTechnique::JunkVariableExpansion);
return String::new();
}
val
}
async fn resolve_word_spec(
spec: &WordSpec,
ctx: &ShellContext,
techs: &mut Vec<ObfuscationTechnique>,
) -> Vec<(String, bool)> {
let mut tokens: Vec<(String, bool)> = vec![(String::new(), false)];
for part in &spec.parts {
match part {
WordPart::Literal(s) => {
tokens.last_mut().unwrap().0.push_str(s);
}
WordPart::QuotedVar { name, default } => {
let val = resolve_var(name, default, ctx, techs).await;
tokens.last_mut().unwrap().0.push_str(&val);
}
WordPart::UnquotedVar { name, default } => {
let val = resolve_var(name, default, ctx, techs).await;
if val.chars().any(|c| c.is_whitespace()) {
techs.push(ObfuscationTechnique::IfsGlue);
let mut segs = val.split_whitespace();
if let Some(first) = segs.next() {
tokens.last_mut().unwrap().0.push_str(first);
}
for seg in segs {
tokens.push((seg.to_string(), false));
}
} else {
tokens.last_mut().unwrap().0.push_str(&val);
}
}
WordPart::Raw(s) => {
let cur = tokens.last_mut().unwrap();
cur.0.push_str(s);
cur.1 = true;
}
}
}
tokens
}
fn is_shell_safe_char(c: char) -> bool {
c.is_ascii_alphanumeric()
|| matches!(c, '_' | '-' | '.' | '/' | ':' | '=' | '@' | '%' | '+' | ',')
}
fn quote_token_if_needed(token: &str) -> String {
if !token.is_empty() && token.chars().all(is_shell_safe_char) {
return token.to_string();
}
let mut out = String::with_capacity(token.len() + 2);
out.push('\'');
for c in token.chars() {
if c == '\'' {
out.push_str("'\\''");
} else {
out.push(c);
}
}
out.push('\'');
out
}
async fn build_normalized_source(
tree: &Tree,
source: &[u8],
ctx: &ShellContext,
) -> (String, Vec<ObfuscationTechnique>) {
let specs = collect_command_specs(tree, source);
let mut edits: Vec<(usize, usize, String)> = Vec::new();
let mut all_techs = Vec::new();
for spec in specs {
let mut techs = spec.techs.clone();
let mut spec_changed = false;
for word in &spec.words {
let tokens = resolve_word_spec(word, ctx, &mut techs).await;
let joined = tokens
.iter()
.map(|(t, is_raw)| {
if *is_raw {
t.clone()
} else {
quote_token_if_needed(t)
}
})
.collect::<Vec<_>>()
.join(" ");
let original =
std::str::from_utf8(&source[word.start_byte..word.end_byte]).unwrap_or("");
if joined != original {
edits.push((word.start_byte, word.end_byte, joined));
spec_changed = true;
}
}
if spec_changed {
all_techs.extend(techs);
}
}
if edits.is_empty() {
return (String::from_utf8_lossy(source).into_owned(), Vec::new());
}
edits.sort_by_key(|e| e.0);
let mut out = String::with_capacity(source.len());
let mut last = 0usize;
for (start, end, repl) in edits {
if start < last {
continue;
}
out.push_str(std::str::from_utf8(&source[last..start]).unwrap_or(""));
out.push_str(&repl);
last = end;
}
out.push_str(std::str::from_utf8(&source[last..]).unwrap_or(""));
(out, all_techs)
}
// =============================================================================
// Phase B - 执行汇聚点抽取（全同步，无需访问 ShellContext）
// =============================================================================
fn base64_decode_bytes(input: &str) -> Option<Vec<u8>> {
fn val(c: u8) -> Option<u8> {
match c {
b'A'..=b'Z' => Some(c - b'A'),
b'a'..=b'z' => Some(c - b'a' + 26),
b'0'..=b'9' => Some(c - b'0' + 52),
b'+' => Some(62),
b'/' => Some(63),
_ => None,
}
}
let bytes: Vec<u8> = input.bytes().filter(|&b| b != b'=').collect();
if bytes.is_empty() {
return None;
}
let mut out = Vec::with_capacity(bytes.len() * 3 / 4 + 3);
let mut buf: u32 = 0;
let mut bits: u32 = 0;
for b in bytes {
let v = val(b)?;
buf = (buf << 6) | v as u32;
bits += 6;
if bits >= 8 {
bits -= 8;
out.push(((buf >> bits) & 0xFF) as u8);
}
}
Some(out)
}
fn try_base64_decode(s: &str) -> Option<String> {
let clean: String = s.chars().filter(|c| !c.is_whitespace()).collect();
let bytes = base64_decode_bytes(&clean)?;
String::from_utf8(bytes).ok()
}
fn try_hex_decode(s: &str) -> Option<String> {
let clean: String = s.chars().filter(|c| c.is_ascii_hexdigit()).collect();
if clean.is_empty() || clean.len() % 2 != 0 {
return None;
}
let bytes: Option<Vec<u8>> = (0..clean.len())
.step_by(2)
.map(|i| u8::from_str_radix(&clean[i..i + 2], 16).ok())
.collect();
bytes.and_then(|b| String::from_utf8(b).ok())
}
fn rot13(s: &str) -> String {
s.chars()
.map(|c| match c {
'a'..='z' => (((c as u8 - b'a' + 13) % 26) + b'a') as char,
'A'..='Z' => (((c as u8 - b'A' + 13) % 26) + b'A') as char,
_ => c,
})
.collect()
}
/// 提取 command 节点中，跳过 command_name 之后的第一个字符串型参数
///
/// 注意：这里**不会**跳过形如 `-e` / `-c` 的短选项参数，仅用于
/// exec sink（`bash -c` / `eval` 等）场景，那些场景的选项已由调用方单独处理。
fn extract_first_string_arg(cmd: &Node, source: &[u8]) -> Option<String> {
let mut cursor = cmd.walk();
let mut seen_name = false;
for child in cmd.children(&mut cursor) {
if child.kind() == "command_name" {
seen_name = true;
continue;
}
if !seen_name {
continue;
}
if matches!(
child.kind(),
"word" | "string" | "raw_string" | "concatenation"
) && let Some(t) = node_extract_text(&child, source)
{
return Some(clean_bash_string(t));
}
}
None
}
/// 提取 echo/printf 的"载荷参数"，自动跳过前置短选项（如 `-e` / `-n` / `-ne`）。
///
/// 遇到第一个不是短选项形态（`-` 开头、且不是 `--`）的 word / string /
/// raw_string / concatenation 节点即视为真正的载荷参数并返回。
fn extract_payload_arg(inner: &Node, source: &[u8]) -> Option<String> {
let mut cursor = inner.walk();
let mut seen_name = false;
for child in inner.children(&mut cursor) {
if child.kind() == "command_name" {
seen_name = true;
continue;
}
if !seen_name {
continue;
}
match child.kind() {
"word" => {
let t = node_extract_text(&child, source)?;
if t.starts_with('-') && t.len() > 1 && !t.starts_with("--") {
// 短选项（如 -e / -n / -ne），跳过继续找真正的载荷
continue;
}
return Some(clean_bash_string(t));
}
"string" | "raw_string" | "concatenation" => {
let t = node_extract_text(&child, source)?;
return Some(clean_bash_string(t));
}
_ => {}
}
}
None
}
/// 判断命令是否携带指定的短选项字符（如 `-e` / `-ne` 中的 `e`）。
/// 扫描到第一个非短选项参数即停止（即只看前导的选项串）。
fn command_has_flag_char(inner: &Node, source: &[u8], flag: char) -> bool {
let mut cursor = inner.walk();
let mut seen_name = false;
for child in inner.children(&mut cursor) {
if child.kind() == "command_name" {
seen_name = true;
continue;
}
if !seen_name {
continue;
}
if child.kind() != "word" {
break;
}
let Some(t) = node_extract_text(&child, source) else {
break;
};
if t.starts_with('-') && t.len() > 1 && !t.starts_with("--") {
if t.chars().skip(1).any(|c| c == flag) {
return true;
}
continue;
}
break;
}
false
}
/// 识别 eval / bash -c / sh -c / source 等执行汇聚点，返回其"待执行参数"节点
fn find_exec_sink_argument<'a>(
cmd: &Node<'a>,
source: &[u8],
) -> Option<(Node<'a>, ObfuscationTechnique)> {
let name = get_command_name(cmd, source)?;
let normalized = name_normalize(name).ok()?;
let tech = match normalized.as_str() {
"eval" => ObfuscationTechnique::EvalWrapping,
"bash" | "sh" | "zsh" | "ksh" | "source" => ObfuscationTechnique::NestedShellInvocation,
_ => return None,
};
let is_eval_like = matches!(normalized.as_str(), "eval" | "source");
let mut saw_flag_c = is_eval_like;
let mut seen_name = false;
let mut cursor = cmd.walk();
for child in cmd.children(&mut cursor) {
if child.kind() == "command_name" {
seen_name = true;
continue;
}
if !seen_name {
continue;
}
if !saw_flag_c
&& child.kind() == "word"
&& let Some(t) = node_extract_text(&child, source)
&& t == "-c"
{
saw_flag_c = true;
continue;
}
if saw_flag_c
&& matches!(
child.kind(),
"word" | "string" | "raw_string" | "concatenation" | "ansi_c_string"
)
{
return Some((child, tech));
}
}
None
}
/// 将 pipeline 中出现的 `command` / `redirected_statement` 节点解包为
/// `(外层节点, 内层 command 节点)` 二元组。
///
/// 之所以需要区分外层/内层：当命令携带重定向（如 `base64 -d <<< DATA`）时，
/// tree-sitter-bash 会用 `redirected_statement` 包裹 `command` 节点，
/// 此时重定向（herestring_redirect 等）是外层节点的子节点、而不是
/// 内层 command 节点的子节点。命令名 / 参数解析要用内层节点，
/// 重定向内容解析要用外层节点。
fn unwrap_command(n: Node<'_>) -> Option<(Node<'_>, Node<'_>)> {
match n.kind() {
"command" => Some((n, n)),
"redirected_statement" => {
let mut cursor = n.walk();
for child in n.children(&mut cursor) {
if child.kind() == "command" {
return Some((n, child));
}
}
None
}
_ => None,
}
}
/// 按管道从左到右的语法顺序，展平出所有 `(外层节点, 内层 command 节点)`。
fn flatten_pipeline_commands<'a>(n: Node<'a>, out: &mut Vec<(Node<'a>, Node<'a>)>) {
if let Some(pair) = unwrap_command(n) {
out.push(pair);
return;
}
if n.kind() == "pipeline" {
let mut cursor = n.walk();
for child in n.children(&mut cursor) {
if matches!(child.kind(), "command" | "pipeline" | "redirected_statement") {
flatten_pipeline_commands(child, out);
}
}
}
}
/// 提取 herestring（`<<<`）重定向携带的字面量内容。
///
/// `outer` 应传入 `unwrap_command` 返回的外层节点（可能与内层
/// command 节点相同，也可能是包裹它的 `redirected_statement`），
/// 因为重定向节点通常挂在外层节点下。
fn extract_herestring_content(outer: &Node, source: &[u8]) -> Option<String> {
static QUERY: LazyLock<Query> = LazyLock::new(|| {
Query::new(&language(), "(herestring_redirect) @hs").expect("invalid query")
});
let mut cursor = QueryCursor::new();
let mut matches = cursor.matches(&QUERY, *outer, source);
while let Some(m) = matches.next() {
let Some(n) = m.captures.first().map(|c| c.node) else {
continue;
};
let mut wc = n.walk();
for child in n.children(&mut wc) {
if matches!(
child.kind(),
"word" | "string" | "raw_string" | "concatenation" | "simple_expansion"
| "expansion"
) && let Some(t) = node_extract_text(&child, source)
{
return Some(clean_bash_string(t));
}
}
// 兜底：无法按子节点结构解析时，直接去掉 "<<<" 前缀取剩余文本
if let Some(t) = node_extract_text(&n, source) {
let stripped = t.trim_start_matches("<<<").trim();
if !stripped.is_empty() {
return Some(clean_bash_string(stripped));
}
}
}
None
}
/// 识别编码管道，支持以下几类模式（可组合）：
///
/// 1. `echo/printf <payload> | base64|xxd|tr|rev | ...`
/// —— 字面量来自 echo/printf 参数，解码器在管道后续命令中查找
///
/// 2. `<decoder> <<< <payload> | ...`
/// —— 字面量来自某条命令自身携带的 herestring 重定向
/// （典型如 `base64 -d <<< BASE64DATA | bash`）
///
/// 3. `echo -e '\xHH...' | bash` / `printf '\xHH...' | bash`
/// —— echo -e / printf 自身的转义解码即完成"解码"，
/// 若管道末端是裸解释器（无 `-c`，直接消费 stdin），
/// 则解码后的字面量本身就是待执行脚本，直接抽取
fn detect_decode_pipeline(node: &Node, source: &[u8]) -> Option<(String, ObfuscationTechnique)> {
if node.kind() != "pipeline" {
return None;
}
// 只处理最顶层的 pipeline，避免 tree-sitter 嵌套 pipeline 被重复处理
if let Some(parent) = node.parent()
&& parent.kind() == "pipeline"
{
return None;
}
let mut commands: Vec<(Node, Node)> = Vec::new();
flatten_pipeline_commands(*node, &mut commands);
if commands.len() < 2 {
return None;
}
// ---------------- Step 1: 寻找字面量载荷来源 ----------------
// herestring 优先（因为它往往直接挂在解码器命令本身上），
// 否则退化为在 echo/printf 命令中查找参数。
let mut literal: Option<String> = None;
let mut source_tech: Option<ObfuscationTechnique> = None;
for (outer, inner) in &commands {
if literal.is_some() {
break;
}
if let Some(hs) = extract_herestring_content(outer, source) {
literal = Some(hs);
continue;
}
let Some(name) = get_command_name(inner, source) else {
continue;
};
let Ok(normalized) = name_normalize(name) else {
continue;
};
match normalized.as_str() {
"echo" => {
if let Some(arg) = extract_payload_arg(inner, source) {
if command_has_flag_char(inner, source, 'e') {
let decoded = unescape_c_style_escapes(&arg);
if decoded != arg {
source_tech = Some(ObfuscationTechnique::EchoDashEHex);
}
literal = Some(decoded);
} else {
literal = Some(arg);
}
}
}
"printf" => {
if let Some(arg) = extract_payload_arg(inner, source) {
let decoded = unescape_c_style_escapes(&arg);
if decoded != arg {
source_tech = Some(ObfuscationTechnique::PrintfHex);
}
literal = Some(decoded);
}
}
_ => {}
}
}
let literal = literal?;
// ---------------- Step 2: 在管道命令中寻找专用解码器 ----------------
for (_, inner) in &commands {
let Some(name) = get_command_name(inner, source) else {
continue;
};
let Ok(normalized) = name_normalize(name) else {
continue;
};
match normalized.as_str() {
"base64" => {
return try_base64_decode(&literal).map(|d| (d, ObfuscationTechnique::Base64Pipe));
}
"xxd" => {
return try_hex_decode(&literal).map(|d| (d, ObfuscationTechnique::HexPipe));
}
"tr" => {
return Some((rot13(&literal), ObfuscationTechnique::RotCipherPipe));
}
"rev" => {
return Some((literal.chars().rev().collect(), ObfuscationTechnique::RevPipe));
}
_ => {}
}
}
// ---------------- Step 3: 裸解释器汇聚兜底 ----------------
// 没有命中专用解码器（base64/xxd/tr/rev），但管道中存在直接消费 stdin
// 的解释器命令（如 `... | bash`，没有 `-c`），此时该字面量
// （可能已经过 echo -e / printf 的转义解码）本身就是待执行脚本。
let has_bare_interpreter = commands.iter().any(|(_, inner)| {
get_command_name(inner, source)
.and_then(|n| name_normalize(n).ok())
.map(|n| matches!(n.as_str(), "bash" | "sh" | "zsh" | "ksh" | "dash"))
.unwrap_or(false)
});
if has_bare_interpreter {
let tech = source_tech.unwrap_or(ObfuscationTechnique::NestedShellInvocation);
return Some((literal, tech));
}
None
}
fn extract_command_substitution_scripts(tree: &Tree, source: &[u8]) -> Vec<String> {
static QUERY: LazyLock<Query> = LazyLock::new(|| {
Query::new(&language(), "(command_substitution) @cs").expect("invalid query")
});
let mut cursor = QueryCursor::new();
let mut matches = cursor.matches(&QUERY, tree.root_node(), source);
let mut out = Vec::new();
while let Some(m) = matches.next() {
if let Some(n) = m.captures.first().map(|c| c.node)
&& let Some(text) = node_extract_text(&n, source)
{
out.push(
text.trim_start_matches("$(")
.trim_start_matches('`')
.trim_end_matches(')')
.trim_end_matches('`')
.to_string(),
);
}
}
out
}
fn walk_for_sinks(tree: &Tree, source: &[u8], out: &mut Vec<(String, ObfuscationTechnique)>) {
let mut stack = vec![tree.root_node()];
while let Some(n) = stack.pop() {
match n.kind() {
"command" => {
if let Some((arg_node, tech)) = find_exec_sink_argument(&n, source)
&& let Some(text) = node_extract_text(&arg_node, source)
{
let cleaned = clean_bash_string(text);
if !cleaned.trim().is_empty() {
out.push((cleaned, tech));
}
}
}
"pipeline" => {
if let Some((decoded, tech)) = detect_decode_pipeline(&n, source)
&& !decoded.trim().is_empty()
{
out.push((decoded, tech));
}
}
_ => {}
}
let mut cursor = n.walk();
for child in n.children(&mut cursor) {
stack.push(child);
}
}
for script in extract_command_substitution_scripts(tree, source) {
if !script.trim().is_empty() {
out.push((script, ObfuscationTechnique::CommandSubstitutionExec));
}
}
}
// =============================================================================
// 主函数：整合调度 + 递归深度 / 字节预算保护
// =============================================================================
fn consume_budget(budget: &AtomicUsize, amount: usize) -> bool {
loop {
let cur = budget.load(Ordering::Relaxed);
if amount > cur {
return false;
}
if budget
.compare_exchange_weak(cur, cur - amount, Ordering::Relaxed, Ordering::Relaxed)
.is_ok()
{
return true;
}
}
}
/// 对单个 CommittedBlock 做完整的反混淆处理：
/// 1. Phase A 词法还原（就地替换 block.source / block.tree）
/// 2. Phase B 执行汇聚点抽取，递归展开为若干子块
///
/// 返回值中第一个元素恒为"清洗后的原块"，之后是所有递归展开出的子块。
pub async fn deobfuscate_block(
mut block: CommittedBlock,
ctx: &ShellContext,
ast_state: &BashAstState,
depth: usize,
budget: &AtomicUsize,
) -> Vec<CommittedBlock> {
if depth > MAX_DEOBF_DEPTH {
tracing::warn!(
target: "security::deobf",
depth,
"max deobfuscation depth exceeded, stop expanding further"
);
return vec![block];
}
// ---------------- Phase A ----------------
let (normalized_src, mut techs) =
build_normalized_source(&block.tree, block.source.as_bytes(), ctx).await;
if normalized_src != block.source {
match ast_state.reparse(&normalized_src).await {
Some(new_tree) if !new_tree.root_node().has_error() => {
block
.deobf
.raw_source
.get_or_insert_with(|| block.source.clone());
block.source = normalized_src;
block.tree = new_tree;
}
_ => {
// 规范化后语法非法：回退到原文，仅打标记不生效，避免破坏后续解析
techs.push(ObfuscationTechnique::UnresolvedDynamic);
tracing::debug!(
target: "security::deobf",
"normalized source failed to reparse cleanly, fallback to original"
);
}
}
}
block.deobf.techniques.extend(techs);
// ---------------- Phase B ----------------
let mut payloads: Vec<(String, ObfuscationTechnique)> = Vec::new();
walk_for_sinks(&block.tree, block.source.as_bytes(), &mut payloads);
tracing::debug!(
target: "security::deobf",
depth,
sink_count = payloads.len(),
"phase B sink extraction complete"
);
let parent_chain = block.deobf.decode_chain.clone();
let mut result = vec![block];
for (payload_text, tech) in payloads {
if !consume_budget(budget, payload_text.len()) {
tracing::warn!(
target: "security::deobf",
payload_len = payload_text.len(),
"deobf byte budget exceeded, dropping remaining payload"
);
continue;
}
let Some(tree) = ast_state.reparse(&payload_text).await else {
continue;
};
if tree.root_node().has_error() {
// 解码出的内容本身不是合法 shell 脚本（例如只是普通数据），跳过即可
tracing::debug!(
target: "security::deobf",
depth,
technique = ?tech,
payload_preview = %payload_text.chars().take(80).collect::<String>(),
"decoded payload is not valid shell syntax, skip"
);
continue;
}
let mut chain = parent_chain.clone();
chain.push(tech.clone());
tracing::debug!(
target: "security::deobf",
depth = depth + 1,
technique = ?tech,
payload_len = payload_text.len(),
"extracted execution sink, recursing"
);
let child_block = CommittedBlock {
source: payload_text,
tree,
is_heredoc_body: false,
is_decoded_payload: true,
fragment_count: 0,
deobf: DeobfMeta {
techniques: Vec::new(),
decode_chain: chain,
raw_source: None,
},
};
let expanded = Box::pin(deobfuscate_block(
child_block,
ctx,
ast_state,
depth + 1,
budget,
))
.await;
result.extend(expanded);
}
result
}
#[cfg(test)]
mod test {
use super::*;
use crate::security::detect::bash::ast::CurrentAst;
use crate::security::detect::bash::{BashDetector, deobf};
use anyhow::{Result, ensure};
use std::collections::HashMap;
use tree_sitter::Parser;
/// 测试辅助函数：初始化沙箱环境并执行反混淆
async fn deobf<S: Into<String>>(data: S, append_enter: bool) -> Result<Vec<CommittedBlock>> {
let data = if append_enter {
let mut x = data.into();
x.push('\n');
x
} else {
data.into()
};
// 初始化空白的模拟环境变量环境
let mut ctx = ShellContext::new("/bin/bash", HashMap::new(), 100);
ctx.extensions.insert(BashAstState::new(4096));
ctx.extensions.insert(CurrentAst::new());
let state = ctx
.extensions
.get::<BashAstState>()
.ok_or_else(|| anyhow::anyhow!("BashAstState missing"))?;
let blocks = state.push_and_commit(data.as_ref()).await;
let budget = AtomicUsize::new(MAX_DEOBF_TOTAL_BYTES);
let mut all_blocks = Vec::new();
for block in blocks {
let expanded = deobfuscate_block(block, &ctx, state, 0, &budget).await;
all_blocks.extend(expanded);
}
println!("{all_blocks:#?}");
ensure!(!all_blocks.is_empty(), "No blocks returned from deobfuscator");
Ok(all_blocks)
}
// =========================================================================
// Phase A 测试：词法级还原 (Lexical Deobfuscation)
// =========================================================================
#[tokio::test]
async fn test_phase_a_backslash_and_junk_vars() {
// 反斜杠转义
let bs = deobf(r#"c\a\t /etc\/pas\s\w\d"#, true).await.unwrap();
assert_eq!(bs.first().unwrap().source, "cat /etc/passwd\n");
assert!(bs[0].deobf.techniques.contains(&ObfuscationTechnique::BackslashEscape));
// 垃圾变量穿插 (由于测试上下文中未定义 $9, $1, $7 等，自动还原为空)
let bs = deobf(r#"c$9a$1t $7/etc/p$8asswd"#, true).await.unwrap();
assert_eq!(bs.first().unwrap().source, "cat /etc/passwd\n");
assert!(bs[0].deobf.techniques.contains(&ObfuscationTechnique::JunkVariableExpansion));
// 垃圾变量与引号混淆
let bs = deobf(r#"c"$9"at /etc/pa"$9"ssw"$1"d"#, true).await.unwrap();
assert_eq!(bs.first().unwrap().source, "cat /etc/passwd\n");
// 大杂烩混淆 (由于外层的 \t 不受双引号包裹影响，所以它仍会被还原成字面量 t)
let bs = deobf(r#"c"$9"a\t /e\t$1c\/pa"$9"s\sw"$1"d"#, true).await.unwrap();
assert_eq!(bs.first().unwrap().source, "cat /etc/passwd\n");
}
#[tokio::test]
async fn test_phase_a_ansi_c_and_defaults() {
// ANSI-C 字符串 $'\x2f' -> '/'
let payload = r#"c\a\t $'\x2f\x65\x74\x63\x2f\x70\x61\x73\x73\x77\x64'"#;
let bs = deobf(payload, true).await.unwrap();
assert_eq!(bs.first().unwrap().source, "cat /etc/passwd\n");
assert!(bs[0].deobf.techniques.contains(&ObfuscationTechnique::AnsiCEscape));
// 变量默认值 ${NOT_EXIST:-/etc/passwd}
let payload = r#"cat ${NOT_EXIST:-/etc/passwd}"#;
let bs = deobf(payload, true).await.unwrap();
assert_eq!(bs.first().unwrap().source, "cat /etc/passwd\n");
assert!(bs[0].deobf.techniques.contains(&ObfuscationTechnique::ParameterDefaultValue));
}
#[tokio::test]
async fn test_phase_a_ifs_splitting_and_quoting() {
// 测试包含空格的默认值展开是否正确触发了分词，并且安全字符不需要被单引号包裹
let payload = r#"echo ${NOT_EXIST:-hello world}"#;
let bs = deobf(payload, true).await.unwrap();
assert_eq!(bs.first().unwrap().source, "echo hello world\n");
assert!(bs[0].deobf.techniques.contains(&ObfuscationTechnique::IfsGlue));
}
// =========================================================================
// Phase B 测试：执行汇聚点与嵌套解码 (Execution Sinks & Decoding Pipes)
// =========================================================================
#[tokio::test]
async fn test_phase_b_nested_shell_sinks() {
let payload = r#"bash -c "c\a\t /etc/passwd""#;
let bs = deobf(payload, true).await.unwrap();
// 解释：Phase A 对双引号内部的 c\a\t 实际上只会当成字面量的 c\a\t 而不是转义。
// 由于有空格，规范化重建时会正确加上安全单引号包裹防注入，保留原真实语义。
assert_eq!(bs[0].source, "bash -c 'c\\a\\t /etc/passwd'\n");
assert!(bs.len() >= 2, "Expected sink extraction to produce a second block");
let extracted = &bs[1];
// 由于这已经是抽取的第二层级无引号执行，这里 \a 会被精确计算并擦除。
assert_eq!(extracted.source, "cat /etc/passwd");
assert!(extracted.deobf.decode_chain.contains(&ObfuscationTechnique::NestedShellInvocation));
}
#[tokio::test]
async fn test_phase_b_base64_pipeline() {
let payload = r#"echo "Y2F0IC9ldGMvcGFzc3dk" | base64 -d | sh"#;
let bs = deobf(payload, true).await.unwrap();
let extracted = bs.iter().find(|b| b.source == "cat /etc/passwd").expect("Base64 payload not extracted");
let chain = &extracted.deobf.decode_chain;
assert!(chain.contains(&ObfuscationTechnique::Base64Pipe));
}
#[tokio::test]
async fn test_phase_b_hex_pipeline() {
// 编码管道: 636174202f6574632f706173737764 (cat /etc/passwd in hex)
let payload = r#"echo "636174202f6574632f706173737764" | xxd -r -p | sh"#;
let bs = deobf(payload, true).await.unwrap();
let extracted = bs.iter().find(|b| b.source == "cat /etc/passwd").expect("Hex payload not extracted");
assert!(extracted.deobf.decode_chain.contains(&ObfuscationTechnique::HexPipe));
}
#[tokio::test]
async fn test_phase_b_echo_dash_e_hex_escape() {
// echo -e '\x63\x61\x74 ...' | bash -> cat /etc/passwd
let payload = r#"echo -e '\x63\x61\x74\x20\x2f\x65\x74\x63\x2f\x70\x61\x73\x73\x77\x64' | bash"#;
let bs = deobf(payload, true).await.unwrap();
let extracted = bs
.iter()
.find(|b| b.source == "cat /etc/passwd")
.expect("echo -e hex escape payload not extracted");
assert!(extracted.deobf.decode_chain.contains(&ObfuscationTechnique::EchoDashEHex));
}
#[tokio::test]
async fn test_phase_b_printf_hex_escape() {
let payload = r#"printf '\x63\x61\x74\x20\x2f\x65\x74\x63\x2f\x70\x61\x73\x73\x77\x64' | bash"#;
let bs = deobf(payload, true).await.unwrap();
let extracted = bs
.iter()
.find(|b| b.source == "cat /etc/passwd")
.expect("printf hex escape payload not extracted");
assert!(extracted.deobf.decode_chain.contains(&ObfuscationTechnique::PrintfHex));
}
#[tokio::test]
async fn test_phase_b_rev_pipeline() {
// "cat /etc/passwd" 反转后是 "dwssap/cte/ tac"
let reversed: String = "cat /etc/passwd".chars().rev().collect();
let payload = format!("echo '{reversed}' | rev | bash");
let bs = deobf(payload, true).await.unwrap();
let extracted = bs
.iter()
.find(|b| b.source == "cat /etc/passwd")
.expect("rev pipeline payload not extracted");
assert!(extracted.deobf.decode_chain.contains(&ObfuscationTechnique::RevPipe));
}
#[tokio::test]
async fn test_phase_b_herestring_base64_pipeline() {
// base64 -d <<< DATA | bash 形式：字面量来自 herestring 而非 echo
let payload = r#"base64 -d <<< cHJpbnRmICdjYXQgL2V0Yy9wYXNzd2QnCg== | bash"#;
let bs = deobf(payload, true).await.unwrap();
// base64 解出来是 `printf 'cat /etc/passwd'\n`，再往下一层是纯 printf（无转义字符），
// Phase A 会把它规范化成 printf 'cat /etc/passwd'
let extracted = bs
.iter()
.find(|b| b.deobf.decode_chain.contains(&ObfuscationTechnique::Base64Pipe));
assert!(extracted.is_some(), "herestring base64 payload not extracted");
}
#[tokio::test]
async fn test_phase_b_bare_interpreter_plain_literal() {
// 没有专用解码器、没有转义，管道末端是裸解释器：字面量本身即脚本
let payload = r#"echo "cat /etc/passwd" | bash"#;
let bs = deobf(payload, true).await.unwrap();
let extracted = bs.iter().find(|b| b.source == "cat /etc/passwd");
assert!(extracted.is_some(), "bare interpreter plain literal payload not extracted");
}
// =========================================================================
// 综合测试：多层嵌套混淆完整链路还原
// =========================================================================
#[tokio::test]
async fn test_deobf_multi_layer_chain() {
// 4 层嵌套混淆：
// L0: echo -e '\xHH...' | bash （十六进制转义 -> L1）
// L1: echo '<反转字符串>' | rev | bash （反转 -> L2）
// L2: bash -c "base64 -d <<< ... | bash" （NestedShellInvocation -> L3）
// L3: base64 -d <<< DATA | bash （herestring + base64 -> L4）
// L4: printf '\xHH...' | bash （十六进制转义 -> 最终载荷）
// 最终: cat /etc/passwd
let payload = r#"echo -e '\x65\x63\x68\x6f\x20\x27\x22\x68\x73\x61\x62\x20\x7c\x20\x3d\x3d\x41\x61\x7a\x46\x6d\x59\x67\x77\x48\x49\x6e\x51\x6a\x4e\x34\x78\x31\x4e\x33\x67\x48\x58\x7a\x63\x44\x65\x63\x4e\x7a\x4e\x34\x78\x56\x4d\x32\x67\x48\x58\x77\x63\x44\x65\x63\x5a\x6d\x4d\x34\x78\x31\x4d\x32\x67\x48\x58\x30\x63\x44\x65\x63\x56\x6a\x4e\x34\x78\x6c\x5a\x79\x67\x48\x58\x77\x49\x44\x65\x63\x52\x7a\x4e\x34\x78\x56\x4d\x32\x67\x48\x58\x7a\x59\x44\x65\x63\x64\x43\x49\x6d\x52\x6e\x62\x70\x4a\x48\x63\x20\x3c\x3c\x3c\x20\x64\x2d\x20\x34\x36\x65\x73\x61\x62\x22\x20\x63\x2d\x20\x68\x73\x61\x62\x27\x20\x7c\x20\x72\x65\x76\x20\x7c\x20\x62\x61\x73\x68' | bash"#;
let bs = deobf(payload, true).await.unwrap();
// 最终应当能找到明文 cat /etc/passwd
let extracted = bs.iter().find(|b| b.source.trim() == "cat /etc/passwd");
assert!(
extracted.is_some(),
"Failed to fully unwrap the 4-layer obfuscation chain.\nAll blocks:\n{:#?}",
bs.iter().map(|b| &b.source).collect::<Vec<_>>()
);
let chain = &extracted.unwrap().deobf.decode_chain;
// println!("Final decode chain: {chain:?}");
// 链路中应当依次出现这些手法（顺序不强制校验，只校验存在性，
// 避免因 Phase A 规范化细节调整导致测试过于脆弱）
assert!(chain.contains(&ObfuscationTechnique::EchoDashEHex));
assert!(chain.contains(&ObfuscationTechnique::RevPipe));
assert!(chain.contains(&ObfuscationTechnique::NestedShellInvocation));
assert!(chain.contains(&ObfuscationTechnique::Base64Pipe));
}
}