nexus_shield/endpoint/
memory_scanner.rs

1// ============================================================================
2// File: endpoint/memory_scanner.rs
3// Description: Shellcode and injected code detection via /proc/[pid]/maps
4// Author: Andrew Jewell Sr. - AutomataNexus
5// Updated: March 24, 2026
6// ============================================================================
7//! Memory Scanner — detects shellcode, RWX memory regions, NOP sleds, and
8//! injected code by reading /proc/[pid]/maps and /proc/[pid]/mem.
9
10use super::{DetectionCategory, RecommendedAction, ScanResult, Severity};
11use serde::{Deserialize, Serialize};
12use std::sync::atomic::{AtomicBool, Ordering};
13use std::sync::Arc;
14
15/// Configuration for the memory scanner.
16#[derive(Debug, Clone, Serialize, Deserialize)]
17pub struct MemoryScanConfig {
18    pub scan_interval_secs: u64,
19    pub scan_suspicious_only: bool,
20    pub max_region_size: u64,
21}
22
23impl Default for MemoryScanConfig {
24    fn default() -> Self {
25        Self {
26            scan_interval_secs: 30,
27            scan_suspicious_only: true,
28            max_region_size: 10_485_760, // 10 MB
29        }
30    }
31}
32
33/// A shellcode byte pattern with optional wildcard mask.
34#[derive(Debug, Clone)]
35pub struct ShellcodePattern {
36    pub name: String,
37    pub pattern: Vec<u8>,
38    pub mask: Vec<u8>, // 0xFF = must match, 0x00 = wildcard
39    pub severity: Severity,
40}
41
42/// A parsed entry from /proc/[pid]/maps.
43#[derive(Debug, Clone)]
44pub struct MapsEntry {
45    pub start_addr: u64,
46    pub end_addr: u64,
47    pub perms: String,
48    pub offset: u64,
49    pub path: String,
50    pub is_rwx: bool,
51    pub is_anonymous: bool,
52}
53
54/// Memory scanner for detecting shellcode and code injection.
55pub struct MemoryScanner {
56    config: MemoryScanConfig,
57    shellcode_patterns: Vec<ShellcodePattern>,
58    running: Arc<AtomicBool>,
59}
60
61impl MemoryScanner {
62    pub fn new(config: MemoryScanConfig) -> Self {
63        Self {
64            config,
65            shellcode_patterns: builtin_patterns(),
66            running: Arc::new(AtomicBool::new(true)),
67        }
68    }
69
70    /// Parse a single line from /proc/[pid]/maps.
71    pub fn parse_maps_line(line: &str) -> Option<MapsEntry> {
72        let line = line.trim();
73        if line.is_empty() {
74            return None;
75        }
76
77        let fields: Vec<&str> = line.splitn(6, char::is_whitespace).collect();
78        if fields.len() < 5 {
79            return None;
80        }
81
82        // Field 0: address range "7f1234000000-7f1234001000"
83        let addr_parts: Vec<&str> = fields[0].split('-').collect();
84        if addr_parts.len() != 2 {
85            return None;
86        }
87        let start_addr = u64::from_str_radix(addr_parts[0], 16).ok()?;
88        let end_addr = u64::from_str_radix(addr_parts[1], 16).ok()?;
89
90        // Field 1: permissions "rwxp"
91        let perms = fields[1].to_string();
92        let is_rwx = perms.contains('r') && perms.contains('w') && perms.contains('x');
93
94        // Field 2: offset
95        let offset = u64::from_str_radix(fields[2], 16).unwrap_or(0);
96
97        // Field 5: pathname (may not exist for anonymous mappings)
98        let path = if fields.len() >= 6 {
99            fields[5].trim().to_string()
100        } else {
101            String::new()
102        };
103
104        let is_anonymous = path.is_empty() || path.starts_with('[');
105
106        Some(MapsEntry {
107            start_addr,
108            end_addr,
109            perms,
110            offset,
111            path,
112            is_rwx,
113            is_anonymous,
114        })
115    }
116
117    /// Parse all lines from /proc/[pid]/maps content.
118    pub fn parse_maps(content: &str) -> Vec<MapsEntry> {
119        content
120            .lines()
121            .filter_map(|line| Self::parse_maps_line(line))
122            .collect()
123    }
124
125    /// Find RWX (read-write-execute) memory regions for a process.
126    pub fn find_rwx_regions(pid: u32) -> Vec<MapsEntry> {
127        let maps_path = format!("/proc/{}/maps", pid);
128        let content = match std::fs::read_to_string(&maps_path) {
129            Ok(c) => c,
130            Err(_) => return Vec::new(),
131        };
132
133        Self::parse_maps(&content)
134            .into_iter()
135            .filter(|e| e.is_rwx)
136            .collect()
137    }
138
139    /// Match a shellcode pattern against data using a byte mask.
140    /// Returns offsets of all matches.
141    pub fn pattern_match(data: &[u8], pattern: &[u8], mask: &[u8]) -> Vec<usize> {
142        if pattern.is_empty() || data.len() < pattern.len() || mask.len() != pattern.len() {
143            return Vec::new();
144        }
145
146        let mut matches = Vec::new();
147        for i in 0..=(data.len() - pattern.len()) {
148            let mut matched = true;
149            for j in 0..pattern.len() {
150                if (data[i + j] & mask[j]) != (pattern[j] & mask[j]) {
151                    matched = false;
152                    break;
153                }
154            }
155            if matched {
156                matches.push(i);
157            }
158        }
159        matches
160    }
161
162    /// Scan a process's memory for shellcode and suspicious regions.
163    pub fn scan_process_memory(&self, pid: u32) -> Vec<ScanResult> {
164        let mut results = Vec::new();
165
166        let rwx_regions = Self::find_rwx_regions(pid);
167
168        // Each RWX region is already suspicious
169        for region in &rwx_regions {
170            if region.is_anonymous {
171                results.push(ScanResult::new(
172                    "memory_scanner",
173                    format!("pid:{} region:0x{:x}-0x{:x}", pid, region.start_addr, region.end_addr),
174                    Severity::Medium,
175                    DetectionCategory::MemoryAnomaly {
176                        pid,
177                        region: format!("0x{:x}-0x{:x}", region.start_addr, region.end_addr),
178                    },
179                    format!(
180                        "Anonymous RWX memory region at 0x{:x}-0x{:x} ({} bytes) — uncommon in legitimate processes",
181                        region.start_addr,
182                        region.end_addr,
183                        region.end_addr - region.start_addr
184                    ),
185                    0.6,
186                    RecommendedAction::Alert,
187                ));
188            }
189
190            // Try to read memory and scan for shellcode
191            let region_size = region.end_addr - region.start_addr;
192            if region_size > self.config.max_region_size {
193                continue;
194            }
195
196            let mem_path = format!("/proc/{}/mem", pid);
197            let data = match read_proc_mem(&mem_path, region.start_addr, region_size as usize) {
198                Some(d) => d,
199                None => continue, // Permission denied is normal
200            };
201
202            // Scan for shellcode patterns
203            for pattern in &self.shellcode_patterns {
204                let offsets = Self::pattern_match(&data, &pattern.pattern, &pattern.mask);
205                if !offsets.is_empty() {
206                    results.push(ScanResult::new(
207                        "memory_scanner",
208                        format!("pid:{} region:0x{:x}", pid, region.start_addr + offsets[0] as u64),
209                        pattern.severity,
210                        DetectionCategory::FilelessMalware {
211                            technique: pattern.name.clone(),
212                        },
213                        format!(
214                            "Shellcode pattern '{}' found at {} offsets in RWX memory of PID {} — possible code injection",
215                            pattern.name, offsets.len(), pid
216                        ),
217                        0.85,
218                        RecommendedAction::KillProcess { pid },
219                    ));
220                }
221            }
222        }
223
224        results
225    }
226
227    /// Scan all processes for suspicious memory.
228    pub fn scan_all_processes(&self) -> Vec<ScanResult> {
229        let mut results = Vec::new();
230
231        let entries = match std::fs::read_dir("/proc") {
232            Ok(e) => e,
233            Err(_) => return results,
234        };
235
236        for entry in entries.flatten() {
237            let name = entry.file_name();
238            let pid: u32 = match name.to_string_lossy().parse() {
239                Ok(p) => p,
240                Err(_) => continue,
241            };
242
243            let mut r = self.scan_process_memory(pid);
244            results.append(&mut r);
245        }
246
247        results
248    }
249
250    /// Start periodic memory scanning in a background task.
251    pub fn start(
252        self: Arc<Self>,
253        detection_tx: tokio::sync::mpsc::UnboundedSender<ScanResult>,
254    ) -> tokio::task::JoinHandle<()> {
255        let running = Arc::clone(&self.running);
256        let interval_secs = self.config.scan_interval_secs;
257
258        tokio::spawn(async move {
259            let mut interval =
260                tokio::time::interval(std::time::Duration::from_secs(interval_secs));
261
262            while running.load(Ordering::Relaxed) {
263                interval.tick().await;
264                let results = self.scan_all_processes();
265                for result in results {
266                    if detection_tx.send(result).is_err() {
267                        return;
268                    }
269                }
270            }
271        })
272    }
273
274    pub fn stop(&self) {
275        self.running.store(false, Ordering::Relaxed);
276    }
277}
278
279/// Built-in shellcode detection patterns.
280fn builtin_patterns() -> Vec<ShellcodePattern> {
281    vec![
282        ShellcodePattern {
283            name: "x86_64_syscall_preamble".to_string(),
284            pattern: vec![0x48, 0x31, 0xc0, 0x48, 0x31, 0xff],
285            mask: vec![0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF],
286            severity: Severity::High,
287        },
288        ShellcodePattern {
289            name: "x86_int80_shellcode".to_string(),
290            pattern: vec![0x31, 0xc0, 0x50, 0x68],
291            mask: vec![0xFF, 0xFF, 0xFF, 0xFF],
292            severity: Severity::High,
293        },
294        ShellcodePattern {
295            name: "nop_sled_16".to_string(),
296            pattern: vec![0x90; 16],
297            mask: vec![0xFF; 16],
298            severity: Severity::Medium,
299        },
300        ShellcodePattern {
301            name: "reverse_tcp_socket".to_string(),
302            pattern: vec![0x6a, 0x29, 0x58, 0x6a, 0x02],
303            mask: vec![0xFF, 0xFF, 0xFF, 0xFF, 0xFF],
304            severity: Severity::Critical,
305        },
306        ShellcodePattern {
307            name: "meterpreter_marker".to_string(),
308            pattern: b"meterpreter".to_vec(),
309            mask: vec![0xFF; 11],
310            severity: Severity::Critical,
311        },
312        ShellcodePattern {
313            name: "metasploit_marker".to_string(),
314            pattern: b"metasploit".to_vec(),
315            mask: vec![0xFF; 10],
316            severity: Severity::Critical,
317        },
318        ShellcodePattern {
319            name: "cobalt_strike_beacon".to_string(),
320            pattern: b"beacon.dll".to_vec(),
321            mask: vec![0xFF; 10],
322            severity: Severity::Critical,
323        },
324    ]
325}
326
327/// Read a region of /proc/[pid]/mem. Returns None on error (permission denied, etc.).
328fn read_proc_mem(path: &str, offset: u64, size: usize) -> Option<Vec<u8>> {
329    use std::io::{Read, Seek, SeekFrom};
330    let mut file = std::fs::File::open(path).ok()?;
331    file.seek(SeekFrom::Start(offset)).ok()?;
332    let mut buf = vec![0u8; size];
333    let n = file.read(&mut buf).ok()?;
334    buf.truncate(n);
335    Some(buf)
336}
337
338#[cfg(test)]
339mod tests {
340    use super::*;
341
342    #[test]
343    fn parse_maps_line_normal() {
344        let line = "7f1234000000-7f1234001000 r-xp 00000000 08:01 12345  /usr/bin/cat";
345        let entry = MemoryScanner::parse_maps_line(line).unwrap();
346        assert_eq!(entry.start_addr, 0x7f1234000000);
347        assert_eq!(entry.end_addr, 0x7f1234001000);
348        assert_eq!(entry.perms, "r-xp");
349        assert!(!entry.is_rwx);
350        assert!(!entry.is_anonymous);
351        assert_eq!(entry.path, "/usr/bin/cat");
352    }
353
354    #[test]
355    fn parse_maps_line_rwx_anonymous() {
356        let line = "7ffc00000000-7ffc00010000 rwxp 00000000 00:00 0";
357        let entry = MemoryScanner::parse_maps_line(line).unwrap();
358        assert!(entry.is_rwx);
359        assert!(entry.is_anonymous);
360    }
361
362    #[test]
363    fn parse_maps_line_heap() {
364        let line = "55a000000000-55a000100000 rw-p 00000000 00:00 0  [heap]";
365        let entry = MemoryScanner::parse_maps_line(line).unwrap();
366        assert!(!entry.is_rwx); // heap is rw- not rwx
367        assert!(entry.is_anonymous); // [heap] starts with [
368    }
369
370    #[test]
371    fn pattern_match_exact() {
372        let data = vec![0x00, 0x48, 0x31, 0xc0, 0x48, 0x31, 0xff, 0x00];
373        let pattern = vec![0x48, 0x31, 0xc0, 0x48, 0x31, 0xff];
374        let mask = vec![0xFF; 6];
375        let matches = MemoryScanner::pattern_match(&data, &pattern, &mask);
376        assert_eq!(matches, vec![1]);
377    }
378
379    #[test]
380    fn pattern_match_with_wildcard() {
381        let data = vec![0x48, 0x31, 0xAA, 0x48, 0x31, 0xBB];
382        let pattern = vec![0x48, 0x31, 0x00, 0x48, 0x31, 0x00];
383        let mask = vec![0xFF, 0xFF, 0x00, 0xFF, 0xFF, 0x00]; // wildcards at positions 2,5
384        let matches = MemoryScanner::pattern_match(&data, &pattern, &mask);
385        assert_eq!(matches, vec![0]);
386    }
387
388    #[test]
389    fn nop_sled_detection() {
390        let mut data = vec![0x00; 100];
391        // Insert 16 NOPs at offset 20
392        for i in 20..36 {
393            data[i] = 0x90;
394        }
395        let pattern = vec![0x90; 16];
396        let mask = vec![0xFF; 16];
397        let matches = MemoryScanner::pattern_match(&data, &pattern, &mask);
398        assert_eq!(matches, vec![20]);
399    }
400
401    #[test]
402    fn meterpreter_detection() {
403        let data = b"some data meterpreter session more data";
404        let pattern = b"meterpreter".to_vec();
405        let mask = vec![0xFF; 11];
406        let matches = MemoryScanner::pattern_match(data, &pattern, &mask);
407        assert!(!matches.is_empty());
408    }
409
410    #[test]
411    fn no_false_positive_on_clean_data() {
412        let data = b"This is perfectly normal program text without any shellcode.";
413        let scanner = MemoryScanner::new(MemoryScanConfig::default());
414        for pattern in &scanner.shellcode_patterns {
415            let matches =
416                MemoryScanner::pattern_match(data, &pattern.pattern, &pattern.mask);
417            assert!(
418                matches.is_empty(),
419                "False positive for pattern '{}'",
420                pattern.name
421            );
422        }
423    }
424
425    #[test]
426    fn pattern_match_empty() {
427        assert!(MemoryScanner::pattern_match(&[], &[0x90], &[0xFF]).is_empty());
428        assert!(MemoryScanner::pattern_match(&[0x90], &[], &[]).is_empty());
429    }
430
431    #[test]
432    fn config_defaults() {
433        let config = MemoryScanConfig::default();
434        assert_eq!(config.scan_interval_secs, 30);
435        assert!(config.scan_suspicious_only);
436        assert!(config.max_region_size > 0);
437    }
438}
nexus_shield/endpoint/memory_scanner.rs

nexus_shield/endpoint/
memory_scanner.rs
