use super::{
analyzer_list::AnalyzerList, extractor_list::ExtractorList, AnalysisOutcome, Content, ContentAnalyzer, ContentExtractor, ContentIdentifier,
ContentType, Filter,
};
use crate::utils;
use crate::ExtractionContext;
use crate::ExtractionRequestMetadata;
use crate::FindingMetadata;
use crate::IdentifierSet;
use crate::NoMetadata;
use crate::Object;
use crate::OwnedContentPtr;
use crate::ScanObserver;
use crate::StopCondition;
use crate::{ContentPtr, Context, ScanResult};
use std::collections::HashSet;
use std::marker::PhantomData;
#[derive(Debug, Clone, Copy, PartialEq, Eq, Ord, PartialOrd)]
enum ScanOutcome {
Continue,
Exit,
}
enum RunExtractorsMethod {
All,
OnlyRequested,
}
pub struct Scanner<T: ContentType, M: FindingMetadata> {
filter: Option<Filter>,
identifiers: IdentifierSet<T>,
analyzers: AnalyzerList<Box<dyn ContentAnalyzer<T, M>>>,
extractors: ExtractorList<Box<dyn ContentExtractor<T>>, T>,
context: Context<T, M>,
stop_condition: Option<Box<dyn StopCondition>>,
max_depth: u32,
max_change_type: u32,
}
impl<T: ContentType, M: FindingMetadata> Scanner<T, M> {
pub fn scan<'a>(&'a mut self, content: &mut dyn Content<T>, filter_root: bool) -> ScanResult<'a, T, M> {
self.context.clear();
if let Some(observer) = self.context.observer.as_mut() {
observer.on_begin(content.path().as_printable_string());
}
if filter_root {
if let Some(filter) = self.filter.as_mut() {
if !filter.should_process(content.path(), content.size()) {
if let Some(observer) = self.context.observer.as_mut() {
observer.on_filtered(content.path().as_printable_string());
observer.on_end();
}
return ScanResult::new(&self.context);
}
}
}
let cptr = ContentPtr::new(content);
self.inner_scan(cptr, 1, Object::INVALID_INDEX);
if let Some(observer) = self.context.observer.as_mut() {
observer.on_end();
}
ScanResult::new(&self.context)
}
fn inner_scan(&mut self, content: ContentPtr<T>, depth: u32, parent_index: u32) -> ScanOutcome {
self.context.local_varmap_handle = None; self.context.current_object_index = None;
let start_req_count = self.context.extraction_requests_stack.len();
if let Some(stop_condition) = self.stop_condition.as_mut() {
if stop_condition.should_stop() {
return ScanOutcome::Exit;
}
}
let (ty, my_index) = self.create_object(content, parent_index);
let result = self.scan_object(content, ty, depth, my_index, start_req_count, 0);
self.restore_extraction_request_stack(start_req_count);
result
}
fn scan_object(
&mut self,
mut content: ContentPtr<T>,
ty: Option<T>,
depth: u32,
my_index: u32,
start_req_count: usize,
change_type_count: u32,
) -> ScanOutcome {
if let Some(observer) = self.context.observer.as_mut() {
observer.on_scan_object(content.as_ref().path().as_printable_string(), ty);
}
let analysis_outcome = self.run_analyzers(content, ty);
let result = match analysis_outcome {
AnalysisOutcome::Continue => self.run_extractors(content, ty, depth, my_index, start_req_count, RunExtractorsMethod::All),
AnalysisOutcome::Skip => ScanOutcome::Continue,
AnalysisOutcome::Exit => ScanOutcome::Exit,
AnalysisOutcome::ReprocessAsType(new_ty) => {
let extr_response = self.run_extractors(content, ty, depth, my_index, start_req_count, RunExtractorsMethod::OnlyRequested);
self.restore_extraction_request_stack(start_req_count);
match extr_response {
ScanOutcome::Exit => ScanOutcome::Exit,
ScanOutcome::Continue => {
if change_type_count < self.max_change_type {
content.as_mut().set_content_type(new_ty);
let parent_local = if let Some(obj) = self.context.objects.get_mut(my_index as usize) {
obj.type_id = new_ty.as_u16();
obj.varmap_handle
} else {
None
};
self.context.current_object_index = Some(my_index);
self.context.local_varmap_handle = parent_local;
self.scan_object(content, Some(new_ty), depth, my_index, start_req_count, change_type_count + 1)
} else {
ScanOutcome::Continue
}
}
}
}
};
result
}
fn create_object(&mut self, content: ContentPtr<T>, parent_index: u32) -> (Option<T>, u32) {
let ty = self.retrieve_content_type(content);
let path_index = self.context.path_arena.alloc(content.as_ref().path().as_printable_string().as_bytes());
let obj = Object {
path: path_index,
parent_index,
next_sibling_index: Object::INVALID_INDEX,
varmap_handle: None,
first_child_index: Object::INVALID_INDEX,
last_child_index: Object::INVALID_INDEX,
type_id: if let Some(ty) = ty { ty.as_u16() } else { u16::MAX },
};
let my_index = self.context.objects.len() as u32;
self.context.objects.push(obj);
self.context.current_object_index = Some(my_index);
let mut last_sibling_index = Object::INVALID_INDEX;
if parent_index != Object::INVALID_INDEX {
if let Some(parent) = self.context.objects.get_mut(parent_index as usize) {
if parent.first_child_index == Object::INVALID_INDEX {
parent.first_child_index = my_index;
} else {
last_sibling_index = parent.last_child_index;
}
parent.last_child_index = my_index;
}
}
if last_sibling_index != Object::INVALID_INDEX {
if let Some(last_sibling) = self.context.objects.get_mut(last_sibling_index as usize) {
last_sibling.next_sibling_index = my_index;
}
}
(ty, my_index)
}
fn run_analyzers(&mut self, content: ContentPtr<T>, content_type: Option<T>) -> AnalysisOutcome<T> {
if let Some(ty) = content_type {
if let Some((start, end)) = self.analyzers.range(ty) {
let res = self.scan_range(content, start, end);
if matches!(res, AnalysisOutcome::Skip | AnalysisOutcome::Exit | AnalysisOutcome::ReprocessAsType(_)) {
return res;
}
}
}
if let Some((start, end)) = self.analyzers.generic_range() {
let res = self.scan_range(content, start, end);
if matches!(res, AnalysisOutcome::Skip | AnalysisOutcome::Exit) {
return res;
}
}
AnalysisOutcome::Continue
}
fn run_extractors(
&mut self,
content: ContentPtr<T>,
content_type: Option<T>,
depth: u32,
my_index: u32,
start_req_index: usize,
method: RunExtractorsMethod,
) -> ScanOutcome {
if matches!(method, RunExtractorsMethod::All) {
if let Some(ty) = content_type {
if let Some((start, end)) = self.extractors.range(ty) {
let res = self.extract_range(content, start, end, depth, my_index, None);
if matches!(res, ScanOutcome::Exit) {
return res;
}
}
}
}
let req_count = self.context.extraction_requests_stack.len();
for i in start_req_index..req_count {
let ty = self.context.extraction_requests_stack[i].content_type;
if let Some((start, end)) = self.extractors.range(ty) {
let res = self.extract_range(content, start, end, depth, my_index, Some(i as u32));
if matches!(res, ScanOutcome::Exit) {
return res;
}
} else {
if let Some(handle) = self.context.extraction_requests_stack[i].params_handle {
self.context.varmap_pool.release(handle);
self.context.extraction_requests_stack[i].params_handle = None;
}
}
}
ScanOutcome::Continue
}
fn restore_extraction_request_stack(&mut self, start_req_index: usize) {
for req in self.context.extraction_requests_stack.iter_mut().skip(start_req_index) {
if let Some(handle) = req.params_handle {
self.context.varmap_pool.release(handle);
}
}
self.context.extraction_requests_stack.truncate(start_req_index);
}
fn scan_range(&mut self, mut content: ContentPtr<T>, start: usize, end: usize) -> AnalysisOutcome<T> {
if (end <= start) || (end > self.analyzers.len()) {
return AnalysisOutcome::Continue;
}
for i in start..end {
let result = unsafe { self.analyzers.get(i).analyze(content.as_mut(), &mut self.context) };
match result {
AnalysisOutcome::Continue => continue,
AnalysisOutcome::Exit => return AnalysisOutcome::Exit,
AnalysisOutcome::Skip => return AnalysisOutcome::Skip,
AnalysisOutcome::ReprocessAsType(ty) => return AnalysisOutcome::ReprocessAsType(ty),
}
}
AnalysisOutcome::Continue
}
fn extract_range(
&mut self,
content: ContentPtr<T>,
start: usize,
end: usize,
depth: u32,
parent_index: u32,
req_index: Option<u32>,
) -> ScanOutcome {
if (end <= start) || (end > self.extractors.len()) {
return ScanOutcome::Continue;
}
let ec_metadata = if let Some(req_index) = req_index {
let request = &self.context.extraction_requests_stack[req_index as usize];
ExtractionRequestMetadata {
start: request.start,
len: request.len,
params_handle: request.params_handle,
}
} else {
ExtractionRequestMetadata {
start: 0,
len: Some(content.as_ref().size()),
params_handle: None,
}
};
let next_action = {
let mut next_action = ScanOutcome::Continue;
for i in start..end {
let result = self.extract_content(content, i, depth, parent_index, &ec_metadata);
match result {
ScanOutcome::Continue => continue,
ScanOutcome::Exit => {
next_action = result;
break;
}
}
}
next_action
};
if let Some(handle) = ec_metadata.params_handle {
self.context.varmap_pool.release(handle);
if let Some(req_index) = req_index {
self.context.extraction_requests_stack[req_index as usize].params_handle = None;
}
}
next_action
}
fn extract_content(
&mut self,
content: ContentPtr<T>,
index: usize,
depth: u32,
parent_index: u32,
ec_metadata: &ExtractionRequestMetadata,
) -> ScanOutcome {
if depth >= self.max_depth {
return ScanOutcome::Continue;
}
let len = self.extractors.len();
if index >= len {
return ScanOutcome::Continue;
}
let extractor = unsafe { self.extractors.get(index) };
let ec = ExtractionContext {
offset: ec_metadata.start,
length: ec_metadata.len,
params: ec_metadata.params_handle.and_then(|h| self.context.varmap_pool.get(h)),
};
let session = extractor.create_session(OwnedContentPtr::new(content), &ec);
if let Some(mut session) = session {
while let Some(entry) = session.advance() {
if !entry.skip_from_filtering {
if let Some(filter) = self.filter.as_mut() {
if !filter.should_process(&entry.path, entry.size) {
if let Some(observer) = self.context.observer.as_mut() {
observer.on_filtered(entry.path.as_printable_string());
}
continue;
}
}
}
if let Some(observer) = self.context.observer.as_mut() {
observer.on_extraction(content.as_ref().path().as_printable_string(), entry);
}
if let Some(mut extracted_content) = session.extract() {
let c_ptr = ContentPtr::new(extracted_content.as_mut());
let result = self.inner_scan(c_ptr, depth + 1, parent_index);
match result {
ScanOutcome::Continue => continue,
ScanOutcome::Exit => return ScanOutcome::Exit,
}
}
}
}
ScanOutcome::Continue
}
fn retrieve_content_type(&mut self, mut content: ContentPtr<T>) -> Option<T> {
if let Some(ty) = content.as_ref().content_type() {
return Some(ty);
}
let p = content.as_ref().path().as_bytes();
let file_name = utils::get_file_name(p);
let type_from_file_name = self.identifiers.type_from_file_name(file_name);
let extension = utils::get_extension(file_name);
let type_from_extension = self.identifiers.type_from_extension(extension);
let type_from_magic = if self.identifiers.has_magic() {
if let Some(buf) = content.as_mut().read(0, 16) {
self.identifiers.type_from_magic(buf)
} else {
None
}
} else {
None
};
if let Some(ty) = type_from_magic {
if self.validate_content_type(content, ty) {
return Some(ty);
}
}
if let Some(ty) = type_from_file_name {
if self.validate_content_type(content, ty) {
return Some(ty);
}
}
if let Some(ty) = type_from_extension {
if self.validate_content_type(content, ty) {
return Some(ty);
}
}
self.identifiers
.identifiers_without_prefilter()
.iter()
.find(|&&ty| self.validate_content_type(content, ty))
.copied()
}
#[inline(always)]
fn validate_content_type(&self, mut content: ContentPtr<T>, content_type: T) -> bool {
self.identifiers
.get(content_type)
.map(|identifier| identifier.validate(content.as_mut()))
.unwrap_or(false)
}
}
pub struct ScannerBuilder<T: ContentType, M: FindingMetadata = NoMetadata> {
filter: Option<Filter>,
analyzers: Vec<(u32, Box<dyn ContentAnalyzer<T, M>>)>,
extractors: Vec<(T, Box<dyn ContentExtractor<T>>)>,
identifiers: Vec<(T, Box<dyn ContentIdentifier<T>>)>,
stop_condition: Option<Box<dyn StopCondition>>,
observer: Option<Box<dyn ScanObserver<T, M>>>,
max_depth: u32,
max_change_type: u32,
store_findings: bool,
_metadata: PhantomData<M>,
}
impl<T: ContentType, M: FindingMetadata> ScannerBuilder<T, M> {
fn empty() -> Self {
Self {
filter: None,
analyzers: Vec::with_capacity(16),
extractors: Vec::with_capacity(4),
identifiers: Vec::with_capacity(4),
stop_condition: None,
observer: None,
max_depth: 8,
max_change_type: 2,
store_findings: true,
_metadata: PhantomData,
}
}
pub fn filter(mut self, filter: Filter) -> Self {
self.filter = Some(filter);
self
}
pub fn add_analyzer<A>(mut self, content_type: T, priority: u8, analyzer: A) -> Self
where
A: ContentAnalyzer<T, M> + 'static,
{
let hash = (content_type.as_u16() as u32) << 16 | priority as u32;
self.analyzers.push((hash, Box::new(analyzer)));
self
}
pub fn add_generic_analyzer<A>(mut self, priority: u8, analyzer: A) -> Self
where
A: ContentAnalyzer<T, M> + 'static,
{
let hash = 0xFFFF0000 | priority as u32;
self.analyzers.push((hash, Box::new(analyzer)));
self
}
pub fn add_extractor<E>(mut self, content_type: T, extractor: E) -> Self
where
E: ContentExtractor<T> + 'static,
{
self.extractors.push((content_type, Box::new(extractor)));
self
}
pub fn add_identifier<I>(mut self, content_type: T, identifier: I) -> Self
where
I: ContentIdentifier<T> + 'static,
{
self.identifiers.push((content_type, Box::new(identifier)));
self
}
pub fn max_depth(mut self, max_depth: u32) -> Self {
self.max_depth = max_depth.clamp(1, u32::MAX - 2);
self
}
pub fn max_change_type(mut self, max_change_type: u32) -> Self {
self.max_change_type = max_change_type.clamp(1, T::COUNT as u32);
self
}
pub fn observer(mut self, observer: impl ScanObserver<T, M> + 'static) -> Self {
self.observer = Some(Box::new(observer));
self
}
pub fn stop_condition(mut self, stop_condition: impl StopCondition + 'static) -> Self {
self.stop_condition = Some(Box::new(stop_condition));
self
}
pub fn store_findings(mut self, store_findings: bool) -> Self {
self.store_findings = store_findings;
self
}
fn check_unique_identifiers(&self) {
let mut m = HashSet::new();
for (content_type, _) in &self.identifiers {
if m.contains(&content_type.as_u16()) {
panic!(
"There can only be one identifier for type ! Type {:?} has multiple identifiers !",
content_type
);
}
m.insert(content_type.as_u16());
}
}
#[cfg(debug_assertions)]
fn check_dependencies(&self) {
use std::collections::HashMap;
let mut map = HashMap::new();
for (hash, analyzer) in &self.analyzers {
let priority = hash & 0xFFFF;
let name = analyzer.name();
map.insert(name, priority);
}
for (hash, analyzer) in &self.analyzers {
let priority = hash & 0xFFFF;
let name = analyzer.name();
let dep = analyzer.dependencies();
for d in dep {
if let Some(dep_priority) = map.get(d) {
if *dep_priority >= priority {
panic!("Analizer {} requiers dependecy {} to have a priority smaller than his. ", name, *d);
}
} else {
panic!("Dependency {} not found for analyzer {}", d, name);
}
}
}
}
pub fn build(self) -> Scanner<T, M> {
self.check_unique_identifiers();
#[cfg(debug_assertions)]
self.check_dependencies();
let analyzers = AnalyzerList::new(self.analyzers, T::COUNT);
let extractors = ExtractorList::new(self.extractors);
let identifiers = IdentifierSet::new(self.identifiers);
Scanner {
filter: self.filter,
identifiers,
analyzers,
extractors,
context: Context::new(self.observer, self.store_findings),
max_depth: self.max_depth,
stop_condition: self.stop_condition,
max_change_type: self.max_change_type,
}
}
}
impl<T: ContentType> ScannerBuilder<T, NoMetadata> {
pub fn new() -> Self {
Self::empty()
}
pub fn with_metadata<M: FindingMetadata>() -> ScannerBuilder<T, M> {
ScannerBuilder::empty()
}
}
impl<T: ContentType, M: FindingMetadata> Default for ScannerBuilder<T, M> {
fn default() -> Self {
Self::empty()
}
}