1use super::{Environment, Source, split, within_input_bytes};
8use crate::budget::{DecodeBuf, Reservation};
9use crate::detect::{DetectError, FileType, check_entry_type_with_reader};
10use crate::error::{Error, LimitError, LimitKind};
11use crate::files::File;
12use crate::limits::{Meter, MeteredVec, heap_cost};
13use crate::raw::RawStr;
14use crate::stream::ReaderKind;
15use crate::zip::ZipArchive;
16use bytes::Bytes;
17use std::path::Path;
18use std::sync::Arc;
19
20mod local;
21pub(super) use local::confined_path;
22pub(super) use local::local_path;
23mod state;
24use state::Seen;
25
26fn walk_file(root: &[u8], basename: &[u8], meter: &Arc<Meter>) -> Result<RawStr, Error> {
29 Ok(RawStr::charged_concat(&[root, b"/", basename], meter)?)
30}
31
32impl Environment {
33 fn input_meter(&self) -> Arc<Meter> {
34 Meter::for_input(
35 &self.ctx().limits,
36 self.ctx().budget.clone(),
37 Reservation::Input,
38 )
39 }
40
41 #[allow(
45 clippy::needless_pass_by_value,
46 reason = "consistent with Environment::load"
47 )]
48 pub fn load_file(&self, source: Source) -> Result<Option<File>, Error> {
49 let meter = self.input_meter();
50 if let Source::Path(p) | Source::ConfinedPath { path: p, .. } = &source
54 && split::base(p.as_os_str().as_encoded_bytes()).is_none()
55 && !p.exists()
56 {
57 let first = split::part(p.as_os_str().as_encoded_bytes(), 0, &meter)?;
58 if !split::path(first.as_bytes())?.exists() {
59 return Ok(None);
60 }
61 }
62 let raw = match &source {
63 Source::Path(p) | Source::ConfinedPath { path: p, .. } => {
64 p.as_os_str().as_encoded_bytes()
65 }
66 Source::Bytes { name, .. } | Source::Fetch { name, .. } => name.as_bytes(),
67 };
68 let raw = if matches!(source, Source::Path(_) | Source::ConfinedPath { .. }) {
69 split::base(raw).unwrap_or(raw)
70 } else {
71 raw
72 };
73 let name = RawStr::charged(raw, &meter)?;
74 let (data, reader) = super::registry::read_source(&source, &meter, self.ctx())?;
75 self.load_data(name, data, reader, &meter)
76 }
77
78 #[allow(
81 clippy::needless_pass_by_value,
82 reason = "consistent with Environment::load"
83 )]
84 pub fn load_zip_file(&self, source: Source) -> Result<(), Error> {
85 let meter = self.input_meter();
86 let data = if let Source::Path(path) | Source::ConfinedPath { path, .. } = &source {
87 let local = local_path(path, &meter)?;
90 let physical = local.path()?;
91 let confined;
92 let physical = if let Source::ConfinedPath { root, .. } = &source {
93 confined = local::confined_path(physical, root)?;
94 confined.as_path()
95 } else {
96 physical
97 };
98 super::read_path(physical, &self.ctx().limits, &meter)?
99 } else {
100 super::registry::read_source(&source, &meter, self.ctx())?.0
101 };
102 self.load_zip_data(data, &meter)
103 }
104
105 pub fn load_path(&self, path: &Path) -> Result<(), Error> {
109 let meter = self.input_meter();
110 let local = local_path(path, &meter)?;
113 let physical = local.path()?;
114 let _path_storage =
115 meter.charge(heap_cost(path.as_os_str().as_encoded_bytes().len() as u64))?;
116 if physical.is_dir() {
117 return self.load_folder(path);
118 }
119 let bytes = path.as_os_str().as_encoded_bytes();
120 if !physical.is_file() {
121 return Ok(());
122 }
123 let (_, ext) = crate::pystr::ntpath_splitext(bytes);
124 if matches!(ext, b".zip" | b".apk") {
125 self.load_zip_file(Source::Path(path.to_path_buf()))
126 } else {
127 self.load_file(Source::Path(path.to_path_buf())).map(|_| ())
128 }
129 }
130
131 pub fn load_files<P: AsRef<Path>>(
134 &self,
135 paths: impl IntoIterator<Item = P>,
136 ) -> Result<(), Error> {
137 let meter = self.input_meter();
138 let mut seen = Seen::new(&meter);
139 for p in paths {
140 let p = p.as_ref();
144 if let Some(base) = split::base(p.as_os_str().as_encoded_bytes())
145 && seen.contains_or_insert(base)?
146 {
147 continue;
148 }
149 let _path_storage =
150 meter.charge(heap_cost(p.as_os_str().as_encoded_bytes().len() as u64))?;
151 self.load_asset_path(p)?;
152 }
153 Ok(())
154 }
155
156 fn load_asset_path(&self, p: &Path) -> Result<(), Error> {
157 if split::base(p.as_os_str().as_encoded_bytes()).is_some() {
160 self.load_file(Source::Path(p.to_path_buf()))?;
161 } else {
162 let meter = self.input_meter();
165 let name = RawStr::charged(p.as_os_str().as_encoded_bytes(), &meter)?;
166 let data = super::read_path(p, &self.ctx().limits, &meter)?;
167 self.load_data(name, data, ReaderKind::Stream, &meter)?;
168 }
169 Ok(())
170 }
171
172 pub fn load_folder(&self, path: &Path) -> Result<(), Error> {
177 let meter = self.input_meter();
178 let _os = meter.charge(heap_cost(32_768))?;
185 let root = local::walk_root(path, &meter)?;
186 if !split::path(root.as_bytes())?.exists() {
187 return Ok(());
188 }
189 let mut pending = MeteredVec::new(&meter);
190 pending.push(root)?;
191 let mut seen = Seen::new(&meter);
192 while let Some(dir) = pending.pop() {
193 let mut children = MeteredVec::new(&meter);
194 let walk = match std::fs::read_dir(split::path(dir.as_bytes())?) {
195 Ok(w) => w,
196 Err(e) if e.kind() == std::io::ErrorKind::PermissionDenied => continue,
197 Err(e) => return Err(e.into()),
198 };
199 for entry in walk {
200 let entry = entry?;
201 if entry.file_type()?.is_dir() {
204 children.push(local::walk_root(&entry.path(), &meter)?)?;
209 } else {
210 let name =
211 walk_file(dir.as_bytes(), entry.file_name().as_encoded_bytes(), &meter)?;
212 if let Some(base) = split::base(name.as_bytes())
213 && seen.contains_or_insert(base)?
214 {
215 continue;
216 }
217 self.load_asset_path(split::path(name.as_bytes())?)?;
218 }
219 }
220 for child in children.as_mut_slice().iter().rev() {
221 pending.push(child.clone())?;
222 }
223 }
224 Ok(())
225 }
226
227 pub(super) fn load_zip_data(&self, data: Bytes, meter: &Arc<Meter>) -> Result<(), Error> {
228 let mut stack = MeteredVec::new(meter);
229 stack.push(Frame::new(data, None, meter)?)?;
230 while !stack.is_empty() {
231 let depth = stack.len().saturating_sub(1) as u64;
232 if depth > u64::from(self.ctx().limits.max_container_depth) {
233 return Err(Error::Limit(LimitError::refused(
234 LimitKind::ContainerDepth,
235 depth,
236 u64::from(self.ctx().limits.max_container_depth),
237 )));
238 }
239 let Some(frame) = stack.as_mut_slice().last_mut() else {
240 break;
241 };
242 let Some(member) = frame.zip.members().get(frame.at) else {
243 if let Some(done) = stack.pop()
244 && let Some(name) = done.name
245 {
246 self.list_zip_none(name, meter)?;
247 }
248 continue;
249 };
250 let index = frame.at;
251 frame.at += 1;
252 let member_name = member.name.clone();
253 let (name, data, reader) = if let Some(base) = split::base(member_name.as_bytes()) {
254 if frame.seen.contains_or_insert(base)? {
255 continue;
256 }
257 let name = RawStr::charged(base, meter)?;
258 let mut out = DecodeBuf::new(
259 meter,
260 0,
261 self.ctx().limits.max_input_bytes,
262 LimitKind::InputBytes,
263 )?;
264 for i in 0..999 {
265 let part = split::part(base, i, meter)?;
266 if let Some(k) = frame.zip.find(part.as_bytes())? {
267 let bytes = frame.read(k, &self.ctx().limits)?;
268 within_input_bytes(
269 (out.len() as u64).saturating_add(bytes.len() as u64),
270 &self.ctx().limits,
271 )?;
272 out.extend_from_slice(&bytes)?;
273 } else if i != 0 {
274 break;
275 }
276 }
277 (name, out.into_bytes()?, ReaderKind::Memory)
278 } else {
279 let name = member_name.clone();
281 let k = frame.zip.find(name.as_bytes())?.unwrap_or(index);
282 (name, frame.read(k, &self.ctx().limits)?, ReaderKind::Stream)
283 };
284 within_input_bytes(data.len() as u64, &self.ctx().limits)?;
285 let ty = check_entry_type_with_reader(&data, &self.ctx().limits, meter, reader)
286 .map_err(|e| match e {
287 DetectError::Work(limit) => Error::Limit(limit),
288 e => Error::Detect(e),
289 })?;
290 if ty == FileType::Zip {
291 stack.push(Frame::new(data, Some(name), meter)?)?;
292 } else {
293 self.load_detected(
294 name,
295 data,
296 reader,
297 ty,
298 meter,
299 u32::try_from(stack.len()).unwrap_or(u32::MAX),
300 )?;
301 }
302 }
303 Ok(())
304 }
305}
306
307struct Cached {
308 remaining: usize,
309 bytes: Option<Bytes>,
310}
311
312struct Frame {
313 zip: ZipArchive,
314 at: usize,
315 seen: Seen,
316 cache: MeteredVec<Cached>,
317 name: Option<RawStr>,
318}
319impl Frame {
320 fn new(data: Bytes, name: Option<RawStr>, meter: &Arc<Meter>) -> Result<Self, Error> {
321 let zip = ZipArchive::open(data, meter)?;
322 let mut cache = MeteredVec::new(meter);
323 for _ in zip.members() {
324 meter.count_decode(1)?;
325 cache.push(Cached {
326 remaining: 0,
327 bytes: None,
328 })?;
329 }
330 for member in zip.members() {
333 if split::base(member.name.as_bytes()).is_none() {
334 let index = zip
335 .find(member.name.as_bytes())?
336 .ok_or_else(|| crate::zip::bad("cache name"))?;
337 let slot = cache
338 .as_mut_slice()
339 .get_mut(index)
340 .ok_or_else(|| crate::zip::bad("cache index"))?;
341 slot.remaining = slot
342 .remaining
343 .checked_add(1)
344 .ok_or_else(|| crate::zip::bad("cache references overflow"))?;
345 }
346 }
347 Ok(Self {
348 zip,
349 at: 0,
350 seen: Seen::new(meter),
351 cache,
352 name,
353 })
354 }
355 fn read(&mut self, index: usize, limits: &crate::limits::Limits) -> Result<Bytes, Error> {
356 let slot = self
357 .cache
358 .as_mut_slice()
359 .get_mut(index)
360 .ok_or_else(|| crate::zip::bad("cache index"))?;
361 slot.remaining = slot.remaining.saturating_sub(1);
362 if let Some(bytes) = slot.bytes.take() {
363 if slot.remaining != 0 {
364 slot.bytes = Some(bytes.clone());
365 }
366 return Ok(bytes);
367 }
368 let bytes = self.zip.read_for_load(index, limits)?;
369 if slot.remaining != 0 {
370 slot.bytes = Some(bytes.clone());
371 }
372 Ok(bytes)
373 }
374}
375
376#[cfg(test)]
377mod tests {
378 use super::*;
379
380 #[test]
381 fn walk_file_keys_preserve_the_yielded_root_and_separator() {
382 let env = Environment::new(super::super::EnvOptions::unitypy());
383 let meter = env.input_meter();
384 for (root, expected) in [
385 ("/", "//x"),
386 ("/a//./b", "/a//./b/x"),
387 ("relative", "relative/x"),
388 ] {
389 assert_eq!(
390 walk_file(root.as_bytes(), b"x", &meter).unwrap().as_bytes(),
391 expected.as_bytes()
392 );
393 }
394 }
395}