use std::{
collections::HashMap,
fs::File,
io::{prelude::*, BufReader},
time::Instant,
};
use anyhow::Result;
use log::debug;
use rayon::prelude::*;
use crate::{
config::Config,
data::{Command, Detection},
masker::Masker,
shell,
shell::Shell,
state::ShellContext,
};
pub const SENSITIVE_COMMANDS: &str = include_str!("sensitive-patterns.yaml");
pub struct PatternsEngine {
commands: Vec<Detection>,
masker: Masker,
}
#[derive(Default, Debug)]
pub struct ShellCommands {
engine_kind: HashMap<Shell, Vec<Command>>,
}
impl ShellCommands {
pub fn add_commands(&mut self, shell_type: &Shell, commands: Vec<Command>) {
if let Some(vec) = self.engine_kind.get_mut(shell_type) {
vec.extend(commands);
} else {
self.engine_kind.insert(shell_type.clone(), commands);
}
}
#[must_use]
pub fn get_commands_per_shell(&self, shell_type: &Shell) -> Option<&Vec<Command>> {
self.engine_kind.get(shell_type)
}
#[must_use]
pub fn get_commands_with_secrets(&self) -> Vec<Command> {
self.engine_kind
.values()
.flatten()
.filter(|&f| !f.detections.is_empty())
.cloned()
.collect::<Vec<_>>()
}
}
impl PatternsEngine {
pub fn with_config(config: &Config) -> Result<Self> {
let sensitive_patterns = {
let mut patterns: Vec<Detection> = serde_yaml::from_str(SENSITIVE_COMMANDS)?;
patterns = if config.is_app_path_exists() {
match config.load_patterns_from_default_path() {
Ok(p) => patterns.extend(p),
Err(e) => debug!("could not load external pattern. {:?}", e),
};
match config.get_ignore_patterns() {
Ok(ignores) => patterns
.iter()
.filter(|p| !ignores.contains(&p.id))
.cloned()
.collect::<Vec<_>>(),
Err(e) => {
debug!("could not load ignore pattern. {:?}", e);
patterns
}
}
} else {
debug!(
"app config folder not found in path: {}",
&config.app_path.display()
);
patterns
};
patterns
};
Ok(Self {
commands: sensitive_patterns,
masker: Masker::new(),
})
}
pub fn find_history_commands_from_shell_list(
&self,
shells_context: &Vec<ShellContext>,
) -> Result<ShellCommands> {
let mut commands = ShellCommands::default();
for shell_context in shells_context {
let history = self.find_history_commands(shell_context)?;
commands.add_commands(&shell_context.history.shell, history);
}
Ok(commands)
}
pub fn find_history_commands(&self, state_context: &ShellContext) -> Result<Vec<Command>> {
debug!(
"clear history commands from path: {}",
state_context.history.path
);
match state_context.history.shell {
Shell::Fish => self.find_fish(state_context, &self.commands),
_ => self.find_by_lines(state_context, &self.commands),
}
}
fn find_by_lines(
&self,
state_context: &ShellContext,
sensitive_commands: &[Detection],
) -> Result<Vec<Command>> {
let file = File::open(&state_context.history.path)?;
let reader = BufReader::new(file);
let start = Instant::now();
let lines = reader.lines().flatten().collect::<Vec<_>>();
debug!(
"time elapsed to read history file: {:?}. found {} commands",
start.elapsed(),
lines.len()
);
let start = Instant::now();
let mut results = lines
.par_iter()
.map(|command| {
let (secrets, sensitive_findings) = Self::find_secrets(command, sensitive_commands);
let only_command = match command.split_once(';') {
Some((_x, y)) => y.to_string(),
_ => command.clone(),
};
Command {
shell_type: state_context.history.shell.clone(),
detections: sensitive_findings,
command: only_command,
data: command.clone(),
secrets,
}
})
.collect::<Vec<_>>();
self.masker.mask_sensitive_findings(results.as_mut());
debug!(
"time elapsed for detect sensitive commands: {:?}",
start.elapsed()
);
Ok(results)
}
fn find_fish(
&self,
state_context: &ShellContext,
sensitive_commands: &[Detection],
) -> Result<Vec<Command>> {
let start = Instant::now();
let history: Vec<shell::FishHistory> =
serde_yaml::from_reader(File::open(&state_context.history.path)?)?;
let mut results = history
.par_iter()
.map(|h| {
let (secrets, sensitive_findings) = Self::find_secrets(&h.cmd, sensitive_commands);
Command {
shell_type: state_context.history.shell.clone(),
detections: sensitive_findings,
command: h.cmd.clone(),
data: serde_yaml::to_string(&h).unwrap(),
secrets,
}
})
.collect::<Vec<_>>();
self.masker.mask_sensitive_findings(results.as_mut());
debug!(
"time elapsed to read history file: {:?}. found {} commands",
start.elapsed(),
history.len()
);
Ok(results)
}
fn find_secrets(
command: &str,
sensitive_commands: &[Detection],
) -> (Vec<String>, Vec<Detection>) {
let (secrets, sensitive_findings): (Vec<String>, Vec<Detection>) = sensitive_commands
.par_iter()
.filter_map(|v| {
Some((
v.test
.captures(command)?
.get(v.secret_group as usize)?
.as_str()
.to_string(),
v.clone(),
))
})
.unzip();
(secrets, sensitive_findings)
}
}
#[cfg(test)]
mod test_engine {
use std::{fs, fs::File, io::Write};
use insta::assert_debug_snapshot;
use tempdir::TempDir;
use super::*;
const TEST_SENSITIVE_COMMANDS: &str = r###"
- name: Find me
secret_group: 0
test: FIND_ME=
"###;
const TEMP_HISTORY_LINES_CONTENT: &str = "history
ls
echo 'hello you'
rm -f ./file.txt
export FIND_ME=token
";
const TEMP_HISTORY_FISH: &str = r#"---
- cmd: history
when: "1656438759"
- cmd: ls
when: "1656438760"
- cmd: echo 'hello you'
when: "1656438760"
- cmd: rm -f ./file.txt
when: "1656438760"
- cmd: export FIND_ME=token
when: "1656438760"
"#;
fn create_mock_state(temp_dir: &TempDir, content: &str, shell_type: Shell) -> ShellContext {
let app_folder = temp_dir.path().join("app");
let history_file_name = "history";
let history_file_path = app_folder.join(history_file_name);
fs::create_dir_all(&app_folder).unwrap();
let mut f = File::create(&history_file_path).unwrap();
f.write_all(content.as_bytes()).unwrap();
f.sync_all().unwrap();
ShellContext {
app_folder_path: app_folder.display().to_string(),
history: shell::History {
shell: shell_type,
path: history_file_path.display().to_string(),
file_name: history_file_name.to_string(),
},
}
}
#[test]
fn can_find_history_commands_line() {
let temp_dir = TempDir::new("engine").unwrap();
let en = PatternsEngine {
commands: serde_yaml::from_str(TEST_SENSITIVE_COMMANDS).unwrap(),
masker: Masker::new(),
};
let state_context = create_mock_state(&temp_dir, TEMP_HISTORY_LINES_CONTENT, Shell::Bash);
let result = en.find_history_commands_from_shell_list(&vec![state_context]);
assert_debug_snapshot!(result);
temp_dir.close().unwrap();
}
#[test]
fn can_find_history_commands_fish() {
let temp_dir = TempDir::new("engine").unwrap();
let en = PatternsEngine {
commands: serde_yaml::from_str(TEST_SENSITIVE_COMMANDS).unwrap(),
masker: Masker::new(),
};
let state_context = create_mock_state(&temp_dir, TEMP_HISTORY_FISH, Shell::Fish);
let result = en.find_history_commands_from_shell_list(&vec![state_context]);
assert_debug_snapshot!(result);
temp_dir.close().unwrap();
}
#[test]
fn can_find_custom_patterns() {
let temp_dir = TempDir::new("engine").unwrap();
let config = Config::with_custom_path(&temp_dir.path().join("app"));
config.init().unwrap();
let custom_pattern = r###"
- name: Pattern Name
test: (FIND_ME)
secret_group: 1
id: safeshell_ignore
"###;
fs::write(&config.sensitive_commands_path, custom_pattern).unwrap();
let en = PatternsEngine::with_config(&config).unwrap();
let state_context = create_mock_state(&temp_dir, TEMP_HISTORY_LINES_CONTENT, Shell::Bash);
let result = en.find_history_commands_from_shell_list(&vec![state_context]);
assert_debug_snapshot!(result);
temp_dir.close().unwrap();
}
#[test]
fn can_ignore_patterns() {
let temp_dir = TempDir::new("engine").unwrap();
let config = Config::with_custom_path(&temp_dir.path().join("app"));
config.init().unwrap();
let custom_pattern = r###"
- safeshell_ignore
"###;
fs::write(&config.sensitive_commands_path, custom_pattern).unwrap();
let en = PatternsEngine::with_config(&config).unwrap();
let state_context = create_mock_state(&temp_dir, TEMP_HISTORY_LINES_CONTENT, Shell::Bash);
let result = en.find_history_commands_from_shell_list(&vec![state_context]);
assert_debug_snapshot!(result);
temp_dir.close().unwrap();
}
}