use super::{
analyzer_list::AnalyzerList, extractor_list::ExtractorList, Content, ContentAnalyzer, ContentExtractor, ContentIdentifier, ContentType, Filter,
NextAction,
};
use crate::utils;
use crate::ExtractionContext;
use crate::IdentifierSet;
use crate::Object;
use crate::{Context, ScanResult};
use std::collections::HashSet;
pub struct Scanner<T: ContentType> {
filter: Option<Filter>,
identifiers: IdentifierSet<T>,
analyzers: AnalyzerList<Box<dyn ContentAnalyzer<T>>>,
extractors: ExtractorList<Box<dyn ContentExtractor<T>>, T>,
context: Context<T>,
max_depth: u32,
}
impl<T: ContentType> Scanner<T> {
const EMPTY_VAR_MAP: varmap::VarMap = varmap::VarMap::new();
pub fn scan<'a>(&'a mut self, content: &mut dyn Content<T>, filter_root: bool) -> ScanResult<'a, T> {
self.context.clear();
if filter_root {
if let Some(filter) = self.filter.as_mut() {
if !filter.should_process(content.path(), content.size()) {
return ScanResult::new(&self.context);
}
}
}
self.inner_scan(content, 1, Object::INVALID_INDEX);
ScanResult::new(&self.context)
}
fn inner_scan(&mut self, content: &mut dyn Content<T>, depth: u32, parent_index: u32) -> NextAction {
self.context.local_varmap_handle = None; self.context.current_object_index = None;
let start_req_count = self.context.extraction_requests_stack.len();
let (ty, my_index) = self.create_object(content, parent_index);
let mut response = self.run_analyzers(content, ty);
if response == NextAction::Continue {
response = self.run_extractors(content, ty, depth, my_index, start_req_count);
}
self.restore_extraction_request_stack(start_req_count);
match response {
NextAction::Continue | NextAction::Skip => NextAction::Continue,
NextAction::Exit => NextAction::Exit,
}
}
fn create_object(&mut self, content: &mut dyn Content<T>, parent_index: u32) -> (Option<T>, u32) {
let ty = self.retrieve_content_type(content);
let path_index = self.context.path_arena.alloc(content.path().as_printable_string().as_bytes());
let obj = Object {
path: path_index,
parent_index,
next_sibling_index: Object::INVALID_INDEX,
varmap_handle: None,
first_child_index: Object::INVALID_INDEX,
last_child_index: Object::INVALID_INDEX,
type_id: if let Some(ty) = ty { ty.as_u16() } else { u16::MAX },
};
let my_index = self.context.objects.len() as u32;
self.context.objects.push(obj);
self.context.current_object_index = Some(my_index);
let mut last_sibling_index = Object::INVALID_INDEX;
if parent_index != Object::INVALID_INDEX {
if let Some(parent) = self.context.objects.get_mut(parent_index as usize) {
if parent.first_child_index == Object::INVALID_INDEX {
parent.first_child_index = my_index;
} else {
last_sibling_index = parent.last_child_index;
}
parent.last_child_index = my_index;
}
}
if last_sibling_index != Object::INVALID_INDEX {
if let Some(last_sibling) = self.context.objects.get_mut(last_sibling_index as usize) {
last_sibling.next_sibling_index = my_index;
}
}
(ty, my_index)
}
fn run_analyzers(&mut self, content: &mut dyn Content<T>, content_type: Option<T>) -> NextAction {
if let Some(ty) = content_type {
if let Some((start, end)) = self.analyzers.range(ty) {
let res = self.scan_range(content, start, end);
if matches!(res, NextAction::Skip | NextAction::Exit) {
return res;
}
}
}
if let Some((start, end)) = self.analyzers.generic_range() {
let res = self.scan_range(content, start, end);
if matches!(res, NextAction::Skip | NextAction::Exit) {
return res;
}
}
NextAction::Continue
}
fn run_extractors(
&mut self,
content: &mut dyn Content<T>,
content_type: Option<T>,
depth: u32,
my_index: u32,
start_req_index: usize,
) -> NextAction {
if let Some(ty) = content_type {
if let Some((start, end)) = self.extractors.range(ty) {
let res = self.extract_range(content, start, end, depth, my_index, None);
if matches!(res, NextAction::Skip | NextAction::Exit) {
return res;
}
}
}
let req_count = self.context.extraction_requests_stack.len();
for i in start_req_index..req_count {
let ty = self.context.extraction_requests_stack[i].content_type;
if let Some((start, end)) = self.extractors.range(ty) {
let res = self.extract_range(content, start, end, depth, my_index, Some(i as u32));
if matches!(res, NextAction::Skip | NextAction::Exit) {
return res;
}
} else {
if let Some(handle) = self.context.extraction_requests_stack[i].params_handle {
self.context.varmap_pool.release(handle);
self.context.extraction_requests_stack[i].params_handle = None;
}
}
}
NextAction::Continue
}
fn restore_extraction_request_stack(&mut self, start_req_index: usize) {
for req in self.context.extraction_requests_stack.iter_mut().skip(start_req_index) {
if let Some(handle) = req.params_handle {
self.context.varmap_pool.release(handle);
}
}
self.context.extraction_requests_stack.truncate(start_req_index);
}
fn scan_range(&mut self, content: &mut dyn Content<T>, start: usize, end: usize) -> NextAction {
if (end <= start) || (end > self.analyzers.len()) {
return NextAction::Continue;
}
for i in start..end {
let result = unsafe { self.analyzers.get(i).analyze(content, &mut self.context) };
match result {
NextAction::Continue => continue,
NextAction::Exit => return NextAction::Exit,
NextAction::Skip => return NextAction::Skip,
}
}
NextAction::Continue
}
fn extract_range(
&mut self,
content: &mut dyn Content<T>,
start: usize,
end: usize,
depth: u32,
parent_index: u32,
req_index: Option<u32>,
) -> NextAction {
if (end <= start) || (end > self.extractors.len()) {
return NextAction::Continue;
}
let (ec, param_handle) = if let Some(req_index) = req_index {
let request = &self.context.extraction_requests_stack[req_index as usize];
let params = if let Some(handle) = request.params_handle {
if self.context.varmap_pool.get(handle).is_some() {
Some(handle)
} else {
None
}
} else {
None
};
(
ExtractionContext {
offset: request.start,
length: request.len,
params: &Self::EMPTY_VAR_MAP,
},
params,
)
} else {
(
ExtractionContext {
offset: 0,
length: Some(content.size()),
params: &Self::EMPTY_VAR_MAP,
},
None,
)
};
let next_action = {
let mut next_action = NextAction::Continue;
for i in start..end {
let result = self.extract_content(content, i, depth, parent_index, &ec, param_handle);
match result {
NextAction::Continue => continue,
NextAction::Exit | NextAction::Skip => {
next_action = result;
break;
}
}
}
next_action
};
if let Some(handle) = param_handle {
self.context.varmap_pool.release(handle);
if let Some(req_index) = req_index {
self.context.extraction_requests_stack[req_index as usize].params_handle = None;
}
}
next_action
}
fn extract_content(
&mut self,
content: &mut dyn Content<T>,
index: usize,
depth: u32,
parent_index: u32,
ec: &ExtractionContext,
ph: Option<varmap::PoolHandle>,
) -> NextAction {
if depth >= self.max_depth {
return NextAction::Continue;
}
let len = self.extractors.len();
if index >= len {
return NextAction::Continue;
}
let mut extractor = unsafe { self.extractors.get(index) };
let acquired = {
let params = ph.and_then(|h| self.context.varmap_pool.get(h)).unwrap_or(ec.params);
let acquire_ec = ExtractionContext {
offset: ec.offset,
length: ec.length,
params,
};
extractor.acquire(content, &acquire_ec)
};
if let Some(handle) = acquired {
while let Some(entry) = unsafe { self.extractors.get(index).advance(handle, content) } {
if !entry.skip_from_filtering {
if let Some(filter) = self.filter.as_mut() {
if !filter.should_process(&entry.path, entry.size) {
continue;
}
}
}
extractor = unsafe { self.extractors.get(index) };
if let Some(mut extracted_content) = extractor.extract(handle, content) {
let result = self.inner_scan(&mut *extracted_content, depth + 1, parent_index);
match result {
NextAction::Continue => continue,
NextAction::Exit => {
extractor = unsafe { self.extractors.get(index) };
extractor.release(handle);
return NextAction::Exit;
}
NextAction::Skip => {
extractor = unsafe { self.extractors.get(index) };
extractor.release(handle);
return NextAction::Continue;
}
}
}
}
extractor = unsafe { self.extractors.get(index) };
extractor.release(handle);
}
NextAction::Continue
}
fn retrieve_content_type(&mut self, content: &mut dyn Content<T>) -> Option<T> {
if let Some(ty) = content.content_type() {
return Some(ty);
}
let p = content.path().as_bytes();
let file_name = utils::get_file_name(p);
let type_from_file_name = self.identifiers.type_from_file_name(file_name);
let extension = utils::get_extension(file_name);
let type_from_extension = self.identifiers.type_from_extension(extension);
let type_from_magic = {
if let Some(buf) = content.read(0, 16) {
self.identifiers.type_from_magic(buf)
} else {
None
}
};
if let Some(ty) = type_from_magic {
if self.validate_content_type(content, ty) {
return Some(ty);
}
}
if let Some(ty) = type_from_file_name {
if self.validate_content_type(content, ty) {
return Some(ty);
}
}
if let Some(ty) = type_from_extension {
if self.validate_content_type(content, ty) {
return Some(ty);
}
}
for &ty in self.identifiers.identifiers_without_prefilter() {
if self.validate_content_type(content, ty) {
return Some(ty);
}
}
None
}
#[inline(always)]
fn validate_content_type(&self, content: &mut dyn Content<T>, content_type: T) -> bool {
self.identifiers
.get(content_type)
.map(|identifier| identifier.validate(content))
.unwrap_or(false)
}
}
pub struct ScannerBuilder<T: ContentType> {
filter: Option<Filter>,
analyzers: Vec<(u32, Box<dyn ContentAnalyzer<T>>)>,
extractors: Vec<(T, Box<dyn ContentExtractor<T>>)>,
identifiers: Vec<(T, Box<dyn ContentIdentifier<T>>)>,
max_depth: u32,
}
impl<T: ContentType> ScannerBuilder<T> {
pub fn new() -> Self {
Self {
filter: None,
analyzers: Vec::with_capacity(16),
extractors: Vec::with_capacity(4),
identifiers: Vec::with_capacity(4),
max_depth: 8,
}
}
pub fn filter(mut self, filter: Filter) -> Self {
self.filter = Some(filter);
self
}
pub fn add_analyzer<A>(mut self, content_type: T, priority: u8, analyzer: A) -> Self
where
A: ContentAnalyzer<T> + 'static,
{
let hash = (content_type.as_u16() as u32) << 16 | priority as u32;
self.analyzers.push((hash, Box::new(analyzer)));
self
}
pub fn add_generic_analyzer<A>(mut self, priority: u8, analyzer: A) -> Self
where
A: ContentAnalyzer<T> + 'static,
{
let hash = 0xFFFF0000 | priority as u32;
self.analyzers.push((hash, Box::new(analyzer)));
self
}
pub fn add_extractor<E>(mut self, content_type: T, extractor: E) -> Self
where
E: ContentExtractor<T> + 'static,
{
self.extractors.push((content_type, Box::new(extractor)));
self
}
pub fn add_identifier<I>(mut self, content_type: T, identifier: I) -> Self
where
I: ContentIdentifier<T> + 'static,
{
self.identifiers.push((content_type, Box::new(identifier)));
self
}
pub fn max_depth(mut self, max_depth: u32) -> Self {
self.max_depth = max_depth.clamp(1, u32::MAX - 2);
self
}
fn check_unique_identifiers(&self) {
let mut m = HashSet::new();
for (content_type, _) in &self.identifiers {
if m.contains(&content_type.as_u16()) {
panic!(
"There can only be one identifier for type ! Type {:?} has multiple identifiers !",
content_type
);
}
m.insert(content_type.as_u16());
}
}
pub fn build(self) -> Scanner<T> {
self.check_unique_identifiers();
let analyzers = AnalyzerList::new(self.analyzers, T::COUNT);
let extractors = ExtractorList::new(self.extractors);
let identifiers = IdentifierSet::new(self.identifiers);
Scanner {
filter: self.filter,
identifiers,
analyzers,
extractors,
context: Context::new(),
max_depth: self.max_depth,
}
}
}