use super::{
analyzer_list::AnalyzerList, extractor_list::ExtractorList, Content, ContentAnalyzer, ContentExtractor, ContentIdentifier, ContentType, Filter,
NextAction,
};
use crate::utils;
use crate::ExtractionContext;
use crate::ExtractionRequestMetadata;
use crate::IdentifierSet;
use crate::Object;
use crate::OwnedContentPtr;
use crate::{ContentPtr, Context, ScanResult};
use std::collections::HashSet;
pub struct Scanner<T: ContentType> {
filter: Option<Filter>,
identifiers: IdentifierSet<T>,
analyzers: AnalyzerList<Box<dyn ContentAnalyzer<T>>>,
extractors: ExtractorList<Box<dyn ContentExtractor<T>>, T>,
context: Context<T>,
max_depth: u32,
}
impl<T: ContentType> Scanner<T> {
pub fn scan<'a>(&'a mut self, content: &mut dyn Content<T>, filter_root: bool) -> ScanResult<'a, T> {
self.context.clear();
if filter_root {
if let Some(filter) = self.filter.as_mut() {
if !filter.should_process(content.path(), content.size()) {
return ScanResult::new(&self.context);
}
}
}
let cptr = ContentPtr::new(content);
self.inner_scan(cptr, 1, Object::INVALID_INDEX);
ScanResult::new(&self.context)
}
fn inner_scan(&mut self, content: ContentPtr<T>, depth: u32, parent_index: u32) -> NextAction {
self.context.local_varmap_handle = None; self.context.current_object_index = None;
let start_req_count = self.context.extraction_requests_stack.len();
let (ty, my_index) = self.create_object(content, parent_index);
let mut response = self.run_analyzers(content, ty);
if response == NextAction::Continue {
response = self.run_extractors(content, ty, depth, my_index, start_req_count);
}
self.restore_extraction_request_stack(start_req_count);
match response {
NextAction::Continue | NextAction::Skip => NextAction::Continue,
NextAction::Exit => NextAction::Exit,
}
}
fn create_object(&mut self, content: ContentPtr<T>, parent_index: u32) -> (Option<T>, u32) {
let ty = self.retrieve_content_type(content);
let path_index = self.context.path_arena.alloc(content.as_ref().path().as_printable_string().as_bytes());
let obj = Object {
path: path_index,
parent_index,
next_sibling_index: Object::INVALID_INDEX,
varmap_handle: None,
first_child_index: Object::INVALID_INDEX,
last_child_index: Object::INVALID_INDEX,
type_id: if let Some(ty) = ty { ty.as_u16() } else { u16::MAX },
};
let my_index = self.context.objects.len() as u32;
self.context.objects.push(obj);
self.context.current_object_index = Some(my_index);
let mut last_sibling_index = Object::INVALID_INDEX;
if parent_index != Object::INVALID_INDEX {
if let Some(parent) = self.context.objects.get_mut(parent_index as usize) {
if parent.first_child_index == Object::INVALID_INDEX {
parent.first_child_index = my_index;
} else {
last_sibling_index = parent.last_child_index;
}
parent.last_child_index = my_index;
}
}
if last_sibling_index != Object::INVALID_INDEX {
if let Some(last_sibling) = self.context.objects.get_mut(last_sibling_index as usize) {
last_sibling.next_sibling_index = my_index;
}
}
(ty, my_index)
}
fn run_analyzers(&mut self, content: ContentPtr<T>, content_type: Option<T>) -> NextAction {
if let Some(ty) = content_type {
if let Some((start, end)) = self.analyzers.range(ty) {
let res = self.scan_range(content, start, end);
if matches!(res, NextAction::Skip | NextAction::Exit) {
return res;
}
}
}
if let Some((start, end)) = self.analyzers.generic_range() {
let res = self.scan_range(content, start, end);
if matches!(res, NextAction::Skip | NextAction::Exit) {
return res;
}
}
NextAction::Continue
}
fn run_extractors(&mut self, content: ContentPtr<T>, content_type: Option<T>, depth: u32, my_index: u32, start_req_index: usize) -> NextAction {
if let Some(ty) = content_type {
if let Some((start, end)) = self.extractors.range(ty) {
let res = self.extract_range(content, start, end, depth, my_index, None);
if matches!(res, NextAction::Skip | NextAction::Exit) {
return res;
}
}
}
let req_count = self.context.extraction_requests_stack.len();
for i in start_req_index..req_count {
let ty = self.context.extraction_requests_stack[i].content_type;
if let Some((start, end)) = self.extractors.range(ty) {
let res = self.extract_range(content, start, end, depth, my_index, Some(i as u32));
if matches!(res, NextAction::Skip | NextAction::Exit) {
return res;
}
} else {
if let Some(handle) = self.context.extraction_requests_stack[i].params_handle {
self.context.varmap_pool.release(handle);
self.context.extraction_requests_stack[i].params_handle = None;
}
}
}
NextAction::Continue
}
fn restore_extraction_request_stack(&mut self, start_req_index: usize) {
for req in self.context.extraction_requests_stack.iter_mut().skip(start_req_index) {
if let Some(handle) = req.params_handle {
self.context.varmap_pool.release(handle);
}
}
self.context.extraction_requests_stack.truncate(start_req_index);
}
fn scan_range(&mut self, mut content: ContentPtr<T>, start: usize, end: usize) -> NextAction {
if (end <= start) || (end > self.analyzers.len()) {
return NextAction::Continue;
}
for i in start..end {
let result = unsafe { self.analyzers.get(i).analyze(content.as_mut(), &mut self.context) };
match result {
NextAction::Continue => continue,
NextAction::Exit => return NextAction::Exit,
NextAction::Skip => return NextAction::Skip,
}
}
NextAction::Continue
}
fn extract_range(
&mut self,
content: ContentPtr<T>,
start: usize,
end: usize,
depth: u32,
parent_index: u32,
req_index: Option<u32>,
) -> NextAction {
if (end <= start) || (end > self.extractors.len()) {
return NextAction::Continue;
}
let ec_metadata = if let Some(req_index) = req_index {
let request = &self.context.extraction_requests_stack[req_index as usize];
ExtractionRequestMetadata {
start: request.start,
len: request.len,
params_handle: request.params_handle,
}
} else {
ExtractionRequestMetadata {
start: 0,
len: Some(content.as_ref().size()),
params_handle: None,
}
};
let next_action = {
let mut next_action = NextAction::Continue;
for i in start..end {
let result = self.extract_content(content, i, depth, parent_index, &ec_metadata);
match result {
NextAction::Continue => continue,
NextAction::Exit | NextAction::Skip => {
next_action = result;
break;
}
}
}
next_action
};
if let Some(handle) = ec_metadata.params_handle {
self.context.varmap_pool.release(handle);
if let Some(req_index) = req_index {
self.context.extraction_requests_stack[req_index as usize].params_handle = None;
}
}
next_action
}
fn extract_content(
&mut self,
content: ContentPtr<T>,
index: usize,
depth: u32,
parent_index: u32,
ec_metadata: &ExtractionRequestMetadata,
) -> NextAction {
if depth >= self.max_depth {
return NextAction::Continue;
}
let len = self.extractors.len();
if index >= len {
return NextAction::Continue;
}
let extractor = unsafe { self.extractors.get(index) };
let ec = ExtractionContext {
offset: ec_metadata.start,
length: ec_metadata.len,
params: ec_metadata.params_handle.and_then(|h| self.context.varmap_pool.get(h)),
};
let session = extractor.create_session(OwnedContentPtr::new(content), &ec);
if let Some(mut session) = session {
while let Some(entry) = session.advance() {
if !entry.skip_from_filtering {
if let Some(filter) = self.filter.as_mut() {
if !filter.should_process(&entry.path, entry.size) {
continue;
}
}
}
if let Some(mut extracted_content) = session.extract() {
let c_ptr = ContentPtr::new(extracted_content.as_mut());
let result = self.inner_scan(c_ptr, depth + 1, parent_index);
match result {
NextAction::Continue => continue,
NextAction::Exit => return NextAction::Exit,
NextAction::Skip => return NextAction::Continue,
}
}
}
}
NextAction::Continue
}
fn retrieve_content_type(&mut self, mut content: ContentPtr<T>) -> Option<T> {
if let Some(ty) = content.as_ref().content_type() {
return Some(ty);
}
let p = content.as_ref().path().as_bytes();
let file_name = utils::get_file_name(p);
let type_from_file_name = self.identifiers.type_from_file_name(file_name);
let extension = utils::get_extension(file_name);
let type_from_extension = self.identifiers.type_from_extension(extension);
let type_from_magic = {
if let Some(buf) = content.as_mut().read(0, 16) {
self.identifiers.type_from_magic(buf)
} else {
None
}
};
if let Some(ty) = type_from_magic {
if self.validate_content_type(content, ty) {
return Some(ty);
}
}
if let Some(ty) = type_from_file_name {
if self.validate_content_type(content, ty) {
return Some(ty);
}
}
if let Some(ty) = type_from_extension {
if self.validate_content_type(content, ty) {
return Some(ty);
}
}
self.identifiers.identifiers_without_prefilter().iter().find(|&&ty| self.validate_content_type(content, ty)).copied()
}
#[inline(always)]
fn validate_content_type(&self, mut content: ContentPtr<T>, content_type: T) -> bool {
self.identifiers
.get(content_type)
.map(|identifier| identifier.validate(content.as_mut()))
.unwrap_or(false)
}
}
pub struct ScannerBuilder<T: ContentType> {
filter: Option<Filter>,
analyzers: Vec<(u32, Box<dyn ContentAnalyzer<T>>)>,
extractors: Vec<(T, Box<dyn ContentExtractor<T>>)>,
identifiers: Vec<(T, Box<dyn ContentIdentifier<T>>)>,
max_depth: u32,
}
impl<T: ContentType> ScannerBuilder<T> {
pub fn new() -> Self {
Self {
filter: None,
analyzers: Vec::with_capacity(16),
extractors: Vec::with_capacity(4),
identifiers: Vec::with_capacity(4),
max_depth: 8,
}
}
pub fn filter(mut self, filter: Filter) -> Self {
self.filter = Some(filter);
self
}
pub fn add_analyzer<A>(mut self, content_type: T, priority: u8, analyzer: A) -> Self
where
A: ContentAnalyzer<T> + 'static,
{
let hash = (content_type.as_u16() as u32) << 16 | priority as u32;
self.analyzers.push((hash, Box::new(analyzer)));
self
}
pub fn add_generic_analyzer<A>(mut self, priority: u8, analyzer: A) -> Self
where
A: ContentAnalyzer<T> + 'static,
{
let hash = 0xFFFF0000 | priority as u32;
self.analyzers.push((hash, Box::new(analyzer)));
self
}
pub fn add_extractor<E>(mut self, content_type: T, extractor: E) -> Self
where
E: ContentExtractor<T> + 'static,
{
self.extractors.push((content_type, Box::new(extractor)));
self
}
pub fn add_identifier<I>(mut self, content_type: T, identifier: I) -> Self
where
I: ContentIdentifier<T> + 'static,
{
self.identifiers.push((content_type, Box::new(identifier)));
self
}
pub fn max_depth(mut self, max_depth: u32) -> Self {
self.max_depth = max_depth.clamp(1, u32::MAX - 2);
self
}
fn check_unique_identifiers(&self) {
let mut m = HashSet::new();
for (content_type, _) in &self.identifiers {
if m.contains(&content_type.as_u16()) {
panic!(
"There can only be one identifier for type ! Type {:?} has multiple identifiers !",
content_type
);
}
m.insert(content_type.as_u16());
}
}
pub fn build(self) -> Scanner<T> {
self.check_unique_identifiers();
let analyzers = AnalyzerList::new(self.analyzers, T::COUNT);
let extractors = ExtractorList::new(self.extractors);
let identifiers = IdentifierSet::new(self.identifiers);
Scanner {
filter: self.filter,
identifiers,
analyzers,
extractors,
context: Context::new(),
max_depth: self.max_depth,
}
}
}
impl<T: ContentType> Default for ScannerBuilder<T> {
fn default() -> Self {
Self::new()
}
}