use serde::Serialize;
use std::fs::File;
use std::io::{BufReader, Read};
pub const MIN_STRING_LEN: usize = 4;
pub const MAX_STRINGS: usize = 200_000;
const NOISE_REPEAT1_MIN_LEN: usize = 6;
const NOISE_REPEAT2_MIN_LEN: usize = 8;
const CHUNK_SIZE: usize = 32 * 1024 * 1024;
const MAX_RUN_CHARS: usize = 1_000_000;
const AVG_BYTES_PER_STRING: u64 = 48; const MIN_CAPACITY_HINT: usize = 64;
#[derive(Debug, Clone, Serialize)]
pub struct ExtractedString {
pub offset: u64,
pub encoding: &'static str,
pub text: String,
}
const fn build_printable_table() -> [bool; 256] {
let mut table = [false; 256];
let mut i = 0usize;
while i < 256 {
let b = i as u8;
table[i] = match b {
b'0'..=b'9' => true,
b'A'..=b'Z' => true,
b'a'..=b'z' => true,
b' ' | b'\t' | b'\n' | b'\r' => true,
0x21..=0x2F | 0x3A..=0x40 | 0x5B..=0x60 | 0x7B..=0x7E => true,
_ => false,
};
i += 1;
}
table
}
static PRINTABLE_TABLE: [bool; 256] = build_printable_table();
#[inline(always)]
fn is_printable(b: u8) -> bool {
PRINTABLE_TABLE[b as usize]
}
struct StringRunScanner<'a> {
min_len: usize,
on_found: &'a mut dyn FnMut(ExtractedString),
ascii_start: i64,
ascii_buf: Vec<u8>,
u16_start: i64,
u16_next_pos: i64,
u16_buf: Vec<u16>,
}
impl<'a> StringRunScanner<'a> {
fn new(min_len: usize, on_found: &'a mut dyn FnMut(ExtractedString)) -> Self {
Self {
min_len,
on_found,
ascii_start: -1,
ascii_buf: Vec::with_capacity(64),
u16_start: -1,
u16_next_pos: -1,
u16_buf: Vec::with_capacity(64),
}
}
fn feed(&mut self, buffer: &[u8], offset: usize, process_count: usize, buffer_valid_len: usize, global_base: u64) {
let mut k = 0usize;
while k < process_count {
if self.ascii_start < 0 && self.u16_start < 0 {
while k < process_count && !is_printable(buffer[offset + k]) {
k += 1;
}
if k >= process_count {
break;
}
}
let i = offset + k;
let g = global_base + k as u64;
let b = buffer[i];
let printable_b = is_printable(b);
if printable_b {
if self.ascii_start < 0 {
self.ascii_start = g as i64;
}
self.ascii_buf.push(b);
if self.ascii_buf.len() >= MAX_RUN_CHARS {
self.flush_ascii();
}
} else {
self.flush_ascii();
}
let has_next = (i + 1) < (offset + buffer_valid_len);
if has_next {
let aligned = self.u16_start < 0 || g as i64 == self.u16_next_pos;
if aligned {
let hi = buffer[i + 1];
if hi == 0x00 && printable_b {
if self.u16_start < 0 {
self.u16_start = g as i64;
self.u16_buf.clear();
}
self.u16_buf.push(b as u16);
self.u16_next_pos = g as i64 + 2;
if self.u16_buf.len() >= MAX_RUN_CHARS {
self.flush_u16();
}
} else {
self.flush_u16();
}
}
}
k += 1;
}
}
fn flush_all(&mut self) {
self.flush_ascii();
self.flush_u16();
}
fn flush_ascii(&mut self) {
if self.ascii_start >= 0 && self.ascii_buf.len() >= self.min_len && !is_low_information_repeat(&self.ascii_buf) {
let text = String::from_utf8_lossy(&self.ascii_buf).into_owned();
(self.on_found)(ExtractedString {
offset: self.ascii_start as u64,
encoding: "ascii",
text,
});
}
self.ascii_start = -1;
self.ascii_buf.clear();
}
fn flush_u16(&mut self) {
if self.u16_start >= 0 && self.u16_buf.len() >= self.min_len {
let as_bytes: Vec<u8> = self.u16_buf.iter().map(|&c| c as u8).collect();
if !is_low_information_repeat(&as_bytes) {
let text = String::from_utf16_lossy(&self.u16_buf);
(self.on_found)(ExtractedString {
offset: self.u16_start as u64,
encoding: "utf16le",
text,
});
}
}
self.u16_start = -1;
self.u16_next_pos = -1;
self.u16_buf.clear();
}
}
fn is_low_information_repeat(buf: &[u8]) -> bool {
let n = buf.len();
if n == 0 {
return false;
}
if n >= NOISE_REPEAT1_MIN_LEN && buf.iter().all(|&b| b == buf[0]) {
return true;
}
if n >= NOISE_REPEAT2_MIN_LEN {
let (p0, p1) = (buf[0], buf[1]);
if p0 != p1 && buf.iter().enumerate().all(|(i, &b)| b == if i % 2 == 0 { p0 } else { p1 }) {
return true;
}
}
false
}
fn estimate_capacity(input_len: u64, max_strings: usize) -> usize {
let guess = if input_len > 0 {
std::cmp::max(MIN_CAPACITY_HINT as u64, input_len / AVG_BYTES_PER_STRING)
} else {
MIN_CAPACITY_HINT as u64
};
std::cmp::min(max_strings as u64, guess) as usize
}
pub fn extract_strings(data: &[u8], min_len: usize, max_strings: usize) -> Vec<ExtractedString> {
let mut results: Vec<ExtractedString> = Vec::with_capacity(estimate_capacity(data.len() as u64, max_strings));
{
let mut on_found = |s: ExtractedString| {
if results.len() < max_strings {
results.push(s);
}
};
let mut scanner = StringRunScanner::new(min_len, &mut on_found);
scanner.feed(data, 0, data.len(), data.len(), 0);
scanner.flush_all();
}
results.sort_by_key(|s| s.offset);
results
}
pub fn extract_strings_from_reader<R: Read>(
mut reader: R,
min_len: usize,
max_strings: usize,
estimated_len: u64,
) -> std::io::Result<Vec<ExtractedString>> {
let mut results: Vec<ExtractedString> = Vec::with_capacity(estimate_capacity(estimated_len, max_strings));
let count = std::cell::Cell::new(0usize); {
let mut on_found = |s: ExtractedString| {
if count.get() < max_strings {
results.push(s);
count.set(count.get() + 1);
}
};
let mut scanner = StringRunScanner::new(min_len, &mut on_found);
let mut buffer = vec![0u8; CHUNK_SIZE];
let mut global_base: u64 = 0;
let mut carry_len: usize = 0;
while count.get() < max_strings {
let to_read_into = buffer.len() - carry_len;
let mut read_total = 0usize;
while read_total < to_read_into {
let n = reader.read(&mut buffer[carry_len + read_total..carry_len + to_read_into])?;
if n == 0 {
break; }
read_total += n;
}
let len = carry_len + read_total;
if len == 0 {
break; }
let is_last_chunk = read_total < to_read_into;
if is_last_chunk {
scanner.feed(&buffer, 0, len, len, global_base);
scanner.flush_all();
break;
} else {
scanner.feed(&buffer, 0, len - 1, len, global_base);
buffer[0] = buffer[len - 1];
carry_len = 1;
global_base += (len - 1) as u64;
}
}
}
results.sort_by_key(|s| s.offset);
Ok(results)
}
pub fn extract_strings_from_file(dump_path: &str, min_len: usize, max_strings: usize) -> std::io::Result<Vec<ExtractedString>> {
let file = File::open(dump_path)?;
let estimated_len = file.metadata().map(|m| m.len()).unwrap_or(0);
let reader = BufReader::with_capacity(1 << 16, file);
extract_strings_from_reader(reader, min_len, max_strings, estimated_len)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn ascii_and_utf16_basic() {
let mut data = Vec::new();
data.extend_from_slice(b"\x00\x01hello world\x00");
data.extend_from_slice(&[b'h', 0x00, b'i', 0x00, 0x00, 0x00]);
let out = extract_strings(&data, 4, 200_000);
assert!(out.iter().any(|s| s.encoding == "ascii" && s.text == "hello world"));
}
#[test]
fn min_len_filters_short_runs() {
let data = b"ab\x00cdef\x00".to_vec();
let out = extract_strings(&data, 4, 200_000);
assert!(!out.iter().any(|s| s.text == "ab"));
assert!(out.iter().any(|s| s.text == "cdef"));
}
#[test]
fn chunk_boundary_carries_utf16_pair() {
let full = {
let mut v = Vec::new();
v.extend_from_slice(&[b'a', 0x00, b'b', 0x00, b'c', 0x00, b'd', 0x00]);
v
};
let mut results = Vec::new();
{
let mut on_found = |s: ExtractedString| results.push(s);
let mut scanner = StringRunScanner::new(4, &mut on_found);
scanner.feed(&full[0..6], 0, 5, 6, 0);
let mut block2 = vec![full[5]];
block2.extend_from_slice(&full[6..]);
scanner.feed(&block2, 0, block2.len(), block2.len(), 5);
scanner.flush_all();
}
assert!(results.iter().any(|s| s.encoding == "utf16le" && s.text == "abcd"));
}
#[test]
fn round_trip_via_reader_matches_in_memory() {
let mut data = Vec::new();
for i in 0..5000u32 {
data.extend_from_slice(format!("token_{i:04}_", i = i).as_bytes());
}
let expected = extract_strings(&data, 4, 200_000);
let via_reader = extract_strings_from_reader(std::io::Cursor::new(&data), 4, 200_000, data.len() as u64).unwrap();
assert_eq!(expected.len(), via_reader.len());
assert_eq!(expected[0].text, via_reader[0].text);
}
#[test]
fn noise_period1_ascii_is_filtered() {
let data = b"\x00aaaaaaaa\x00".to_vec();
let out = extract_strings(&data, 4, 200_000);
assert!(!out.iter().any(|s| s.text == "aaaaaaaa"));
}
#[test]
fn noise_period2_ascii_is_filtered() {
let data = b"\x00ABABABAB\x00".to_vec();
let out = extract_strings(&data, 4, 200_000);
assert!(!out.iter().any(|s| s.text == "ABABABAB"));
}
#[test]
fn short_repeat_below_threshold_is_kept() {
let data = b"\x000000\x00".to_vec();
let out = extract_strings(&data, 4, 200_000);
assert!(out.iter().any(|s| s.text == "0000"));
}
#[test]
fn real_looking_string_not_filtered() {
let data = b"\x00password123\x00".to_vec();
let out = extract_strings(&data, 4, 200_000);
assert!(out.iter().any(|s| s.text == "password123"));
}
#[test]
fn noise_period1_utf16_is_filtered() {
let mut data = vec![0x00, 0x00];
for _ in 0..8 {
data.push(b'a');
data.push(0x00);
}
data.extend_from_slice(&[0x00, 0x00]);
let out = extract_strings(&data, 4, 200_000);
assert!(!out.iter().any(|s| s.encoding == "utf16le" && s.text == "aaaaaaaa"));
}
}