1use std::collections::{BTreeMap, BTreeSet};
30use std::fmt::Write as _;
31use std::io::Read as _;
32use std::path::{Component, Path, PathBuf};
33
34use serde::{Deserialize, Serialize};
35use serde_norway::Value;
36use sha2::{Digest, Sha256};
37
38use crate::parser;
39use crate::store::Store;
40
41pub const MANIFEST_FILE: &str = "assets.jsonl";
43
44pub const SUPERSEDES_ASSET_KEY: &str = "supersedes-asset";
47
48#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
55pub struct AssetRecord {
56 pub path: String,
59 pub sha256: String,
62 pub bytes: u64,
64 pub media_type: String,
66 pub wrappers: Vec<String>,
69 pub required: bool,
72}
73
74#[derive(Debug, Clone, PartialEq, Eq)]
76pub struct Declaration {
77 pub path: String,
79 pub required: bool,
82}
83
84#[derive(Debug, Clone, PartialEq, Eq)]
86pub struct AssetSupersession {
87 pub original: String,
89 pub replacement: String,
91}
92
93#[derive(Debug, Serialize)]
99pub struct ScanReport {
100 pub manifest: String,
101 pub cataloged: usize,
102 pub hashed: usize,
103 pub preserved: usize,
104 pub bytes: u64,
105 pub wrote: bool,
106 pub dry_run: bool,
107 pub warnings: Vec<String>,
108 pub untracked: Vec<String>,
109}
110
111#[derive(Debug, Serialize)]
115pub struct RefreshReport {
116 pub manifest: String,
117 pub path: String,
118 pub sha256: String,
119 pub bytes: u64,
120 pub wrappers: Vec<String>,
121 pub required: bool,
122 pub superseded_assets: Vec<String>,
124 pub wrote: bool,
125}
126
127#[derive(Debug, Serialize)]
129pub struct AssetState {
130 pub path: String,
131 pub sha256: String,
132 pub bytes: u64,
133 pub required: bool,
134 pub state: String,
136}
137
138#[derive(Debug, Serialize)]
140pub struct StatusReport {
141 pub total: usize,
142 pub present: usize,
143 pub missing: usize,
144 pub required_missing: usize,
145 pub optional_missing: usize,
146 pub bytes_total: u64,
147 pub bytes_missing: u64,
148 pub assets: Vec<AssetState>,
149}
150
151#[derive(Debug, Serialize)]
153pub struct VerifyReport {
154 pub mode: String,
155 pub checked: usize,
156 pub ok: usize,
157 pub missing: Vec<String>,
158 pub corrupt: Vec<String>,
159 pub complete: bool,
160}
161
162pub fn read_manifest(store: &Store) -> crate::Result<Vec<AssetRecord>> {
171 let text = match store
172 .read_text_bounded(Path::new(MANIFEST_FILE), crate::parser::MAX_DBMD_FILE_BYTES)
173 {
174 Ok(text) => text,
175 Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()),
176 Err(error) => return Err(error.into()),
177 };
178 let mut by_path: BTreeMap<String, AssetRecord> = BTreeMap::new();
179 for (i, line) in text.lines().enumerate() {
180 if line.trim().is_empty() {
181 continue;
182 }
183 let rec: AssetRecord = serde_json::from_str(line).map_err(|e| {
184 std::io::Error::new(
185 std::io::ErrorKind::InvalidData,
186 format!("{MANIFEST_FILE} line {}: {e}", i + 1),
187 )
188 })?;
189 by_path.insert(rec.path.clone(), rec);
190 }
191 Ok(by_path.into_values().collect())
192}
193
194fn serialize_manifest(records: &[AssetRecord]) -> String {
201 if records.is_empty() {
202 return String::new();
203 }
204 let mut sorted = records.to_vec();
205 sorted.sort_by(|a, b| a.path.cmp(&b.path));
206 let mut out = String::new();
207 for rec in &sorted {
208 let line = serde_json::to_string(rec).expect("AssetRecord serializes");
209 out.push_str(&line);
210 out.push('\n');
211 }
212 out
213}
214
215pub fn write_manifest(store: &Store, records: &[AssetRecord]) -> crate::Result<()> {
219 let abs = Path::new(MANIFEST_FILE);
220 let out = serialize_manifest(records);
221 if out.is_empty() {
222 match store.remove_file(abs) {
223 Ok(()) => {}
224 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
225 Err(error) => return Err(error.into()),
226 }
227 return Ok(());
228 }
229 store.write_atomic(abs, out.as_bytes())?;
230 Ok(())
231}
232
233pub fn scan(store: &Store, dry_run: bool, untracked: bool) -> crate::Result<ScanReport> {
246 let existing_by_path: BTreeMap<String, AssetRecord> = read_manifest(store)
250 .unwrap_or_default()
251 .into_iter()
252 .map(|r| (r.path.clone(), r))
253 .collect();
254
255 let mut wrappers_by_path: BTreeMap<String, BTreeSet<String>> = BTreeMap::new();
257 let mut required_by_path: BTreeMap<String, bool> = BTreeMap::new();
258 let mut declared_paths: BTreeSet<String> = BTreeSet::new();
259 let mut supersessions: BTreeMap<String, (String, String)> = BTreeMap::new();
260 let mut ambiguous_supersessions: BTreeSet<String> = BTreeSet::new();
261 let mut warnings: Vec<String> = Vec::new();
262
263 for rel in store.walk()? {
264 let text = match store.read_text_bounded(&rel, parser::MAX_DBMD_FILE_BYTES) {
265 Ok(text) => text,
266 Err(_) => continue,
267 };
268 let parsed = match parser::split_frontmatter(&text, &rel) {
269 Ok(parsed) => parsed,
270 Err(_) => continue,
271 };
272 let fm = match parser::Frontmatter::parse(&parsed.frontmatter_yaml, &rel) {
273 Ok(frontmatter) => frontmatter,
274 Err(_) => continue, };
276 let wrapper = rel_to_string(&rel);
277 for decl in declared_assets(&fm) {
278 let norm = match normalize_asset_path(&decl.path) {
279 Ok(n) => n,
280 Err(e) => {
281 warnings.push(format!("{wrapper}: {e}"));
282 continue;
283 }
284 };
285 if is_markdown(&norm) {
286 warnings.push(format!(
287 "{wrapper}: asset path points at a markdown content file ({norm}); skipped"
288 ));
289 continue;
290 }
291 wrappers_by_path
292 .entry(norm.clone())
293 .or_default()
294 .insert(wrapper.clone());
295 let req = required_by_path.entry(norm.clone()).or_insert(false);
296 *req = *req || decl.required;
297 declared_paths.insert(norm);
298 }
299 match asset_supersession(&fm) {
300 Ok(Some(supersession)) => {
301 if let Some((prior, prior_wrapper)) = supersessions.get(&supersession.original) {
302 if prior != &supersession.replacement {
303 ambiguous_supersessions.insert(supersession.original.clone());
304 warnings.push(format!(
305 "{wrapper}: `{SUPERSEDES_ASSET_KEY}` conflicts with {prior_wrapper} for {}",
306 supersession.original
307 ));
308 }
309 } else {
310 supersessions.insert(
311 supersession.original,
312 (supersession.replacement, wrapper.clone()),
313 );
314 }
315 }
316 Ok(None) => {}
317 Err(error) => warnings.push(format!("{wrapper}: {error}")),
318 }
319 }
320
321 let cyclic_supersessions = supersession_cycle_members(&supersessions);
322 for original in &cyclic_supersessions {
323 if let Some((_, wrapper)) = supersessions.get(original) {
324 warnings.push(format!(
325 "{wrapper}: `{SUPERSEDES_ASSET_KEY}` participates in a replacement cycle at {original}"
326 ));
327 }
328 }
329 for (original, (replacement, wrapper)) in supersessions {
330 if ambiguous_supersessions.contains(&original) {
331 continue;
332 }
333 if cyclic_supersessions.contains(&original) {
334 continue;
335 }
336 if !wrappers_by_path.contains_key(&replacement) {
337 warnings.push(format!(
338 "{wrapper}: replacement asset `{replacement}` is not declared"
339 ));
340 continue;
341 }
342 if !wrappers_by_path.contains_key(&original) && !existing_by_path.contains_key(&original) {
343 warnings.push(format!(
344 "{wrapper}: superseded asset `{original}` is neither declared nor cataloged"
345 ));
346 continue;
347 }
348 wrappers_by_path
349 .entry(original.clone())
350 .or_default()
351 .insert(wrapper);
352 required_by_path.insert(original.clone(), false);
353 declared_paths.insert(original);
354 }
355
356 let mut records: Vec<AssetRecord> = Vec::new();
358 let mut hashed = 0usize;
359 let mut preserved = 0usize;
360 for (path, wrappers) in &wrappers_by_path {
361 let required = *required_by_path.get(path).unwrap_or(&true);
362 let wrappers: Vec<String> = wrappers.iter().cloned().collect();
363
364 let abs = match store.capability_relative(Path::new(path)) {
366 Ok(p) => p,
367 Err(_) => {
368 warnings.push(format!("{path}: escapes the store root; skipped"));
369 continue;
370 }
371 };
372
373 match store.open_regular(abs) {
374 Ok(file) => {
375 let (sha256, bytes) = sha256_file(file)?;
376 records.push(AssetRecord {
377 path: path.clone(),
378 sha256,
379 bytes,
380 media_type: media_type_for(path),
381 wrappers,
382 required,
383 });
384 hashed += 1;
385 }
386 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
387 if let Some(prev) = existing_by_path.get(path) {
388 records.push(AssetRecord {
391 path: path.clone(),
392 sha256: prev.sha256.clone(),
393 bytes: prev.bytes,
394 media_type: media_type_for(path),
395 wrappers,
396 required,
397 });
398 preserved += 1;
399 } else {
400 warnings.push(format!(
401 "{path}: declared but absent and never cataloged; cannot hash (skipped)"
402 ));
403 }
404 }
405 Err(error) => {
406 warnings.push(format!(
407 "{path}: is not a readable regular in-store file: {error}"
408 ));
409 }
410 }
411 }
412 records.sort_by(|a, b| a.path.cmp(&b.path));
413
414 let bytes: u64 = records.iter().fold(0u64, |a, r| a.saturating_add(r.bytes));
417 let cataloged = records.len();
418
419 let untracked_list = if untracked {
420 find_untracked(store, &declared_paths)?
421 } else {
422 Vec::new()
423 };
424
425 let mut wrote = false;
435 if !dry_run {
436 let canonical = serialize_manifest(&records);
437 let on_disk = match store
438 .read_bounded(Path::new(MANIFEST_FILE), crate::parser::MAX_DBMD_FILE_BYTES)
439 {
440 Ok(bytes) => bytes,
441 Err(error) if error.kind() == std::io::ErrorKind::NotFound => Vec::new(),
442 Err(error) => return Err(error.into()),
443 };
444 if on_disk != canonical.as_bytes() {
445 write_manifest(store, &records)?;
446 wrote = true;
447 }
448 }
449
450 Ok(ScanReport {
451 manifest: MANIFEST_FILE.to_string(),
452 cataloged,
453 hashed,
454 preserved,
455 bytes,
456 wrote,
457 dry_run,
458 warnings,
459 untracked: untracked_list,
460 })
461}
462
463fn supersession_cycle_members(
464 supersessions: &BTreeMap<String, (String, String)>,
465) -> BTreeSet<String> {
466 let mut cyclic = BTreeSet::new();
467 for origin in supersessions.keys() {
468 let mut order = Vec::new();
469 let mut positions = BTreeMap::new();
470 let mut current = origin.as_str();
471 while let Some((next, _)) = supersessions.get(current) {
472 if let Some(start) = positions.get(current).copied() {
473 cyclic.extend(order[start..].iter().cloned());
474 break;
475 }
476 positions.insert(current.to_string(), order.len());
477 order.push(current.to_string());
478 current = next;
479 }
480 }
481 cyclic
482}
483
484pub fn refresh(store: &Store, raw_path: &str, raw_wrapper: &str) -> crate::Result<RefreshReport> {
492 let path = normalize_asset_path(raw_path)
493 .map_err(|message| std::io::Error::new(std::io::ErrorKind::InvalidInput, message))?;
494 if is_markdown(&path) {
495 return Err(std::io::Error::new(
496 std::io::ErrorKind::InvalidInput,
497 "asset path points at a markdown content file",
498 )
499 .into());
500 }
501
502 let wrapper_path = normalize_asset_path(raw_wrapper)
503 .map_err(|message| std::io::Error::new(std::io::ErrorKind::InvalidInput, message))?;
504 if !is_markdown(&wrapper_path)
505 || !(wrapper_path.starts_with("sources/") || wrapper_path.starts_with("records/"))
506 {
507 return Err(std::io::Error::new(
508 std::io::ErrorKind::InvalidInput,
509 "wrapper must be a sources/ or records/ markdown content path",
510 )
511 .into());
512 }
513
514 let declaration = |wrapper: &str| -> crate::Result<Option<bool>> {
515 let text =
516 store.read_text_bounded(Path::new(wrapper), crate::parser::MAX_DBMD_FILE_BYTES)?;
517 let parsed = parser::split_frontmatter(&text, Path::new(wrapper))?;
518 let fm = parser::Frontmatter::parse(&parsed.frontmatter_yaml, Path::new(wrapper))?;
519 let mut found = false;
520 let mut required = false;
521 for declaration in declared_assets(&fm) {
522 let declared = normalize_asset_path(&declaration.path).map_err(|message| {
523 std::io::Error::new(std::io::ErrorKind::InvalidInput, message)
524 })?;
525 if declared == path {
526 found = true;
527 required |= declaration.required;
528 }
529 }
530 Ok(found.then_some(required))
531 };
532
533 let Some(requested_required) = declaration(&wrapper_path)? else {
534 return Err(std::io::Error::new(
535 std::io::ErrorKind::InvalidInput,
536 format!("wrapper `{wrapper_path}` does not declare asset `{path}`"),
537 )
538 .into());
539 };
540 let requested_supersession = {
541 let text = store
542 .read_text_bounded(Path::new(&wrapper_path), crate::parser::MAX_DBMD_FILE_BYTES)?;
543 let parsed = parser::split_frontmatter(&text, Path::new(&wrapper_path))?;
544 let fm = parser::Frontmatter::parse(&parsed.frontmatter_yaml, Path::new(&wrapper_path))?;
545 asset_supersession(&fm)
546 .map_err(|message| std::io::Error::new(std::io::ErrorKind::InvalidInput, message))?
547 };
548 if requested_supersession
549 .as_ref()
550 .is_some_and(|supersession| supersession.replacement != path)
551 {
552 return Err(std::io::Error::new(
553 std::io::ErrorKind::InvalidInput,
554 format!("wrapper `{wrapper_path}` supersedes an asset with a different replacement"),
555 )
556 .into());
557 }
558
559 let existing = read_manifest(store)?;
560 let mut wrappers = BTreeSet::from([wrapper_path.clone()]);
561 if let Some(record) = existing.iter().find(|record| record.path == path) {
562 wrappers.extend(record.wrappers.iter().cloned());
563 }
564 let mut live_wrappers = Vec::new();
565 let mut required = requested_required;
566 for wrapper in wrappers {
567 if wrapper != wrapper_path && !store.regular_file_exists(Path::new(&wrapper))? {
568 continue;
572 }
573 match declaration(&wrapper) {
574 Ok(Some(wrapper_required)) => {
575 required |= wrapper_required;
576 live_wrappers.push(wrapper);
577 }
578 Ok(None) => {}
579 Err(error) => return Err(error),
580 }
581 }
582 live_wrappers.sort();
583
584 let asset_path = store.capability_relative(Path::new(&path))?;
585 let file = store.open_regular(asset_path)?;
586 let (sha256, bytes) = sha256_file(file)?;
587 let record = AssetRecord {
588 path: path.clone(),
589 sha256: sha256.clone(),
590 bytes,
591 media_type: media_type_for(&path),
592 wrappers: live_wrappers.clone(),
593 required,
594 };
595 let mut next = existing;
596 next.retain(|candidate| candidate.path != path);
597 next.push(record);
598 let mut superseded_assets = Vec::new();
599 if let Some(supersession) = requested_supersession {
600 let original = next
601 .iter_mut()
602 .find(|candidate| candidate.path == supersession.original)
603 .ok_or_else(|| {
604 std::io::Error::new(
605 std::io::ErrorKind::InvalidInput,
606 format!(
607 "superseded asset `{}` has no existing manifest row; run `dbmd assets scan` first",
608 supersession.original
609 ),
610 )
611 })?;
612 original.required = false;
613 if !original.wrappers.contains(&wrapper_path) {
614 original.wrappers.push(wrapper_path.clone());
615 original.wrappers.sort();
616 }
617 superseded_assets.push(supersession.original);
618 }
619 next.sort_by(|left, right| left.path.cmp(&right.path));
620
621 let canonical = serialize_manifest(&next);
622 let on_disk =
623 match store.read_bounded(Path::new(MANIFEST_FILE), crate::parser::MAX_DBMD_FILE_BYTES) {
624 Ok(bytes) => bytes,
625 Err(error) if error.kind() == std::io::ErrorKind::NotFound => Vec::new(),
626 Err(error) => return Err(error.into()),
627 };
628 let wrote = on_disk != canonical.as_bytes();
629 if wrote {
630 write_manifest(store, &next)?;
631 }
632
633 Ok(RefreshReport {
634 manifest: MANIFEST_FILE.to_string(),
635 path,
636 sha256,
637 bytes,
638 wrappers: live_wrappers,
639 required,
640 superseded_assets,
641 wrote,
642 })
643}
644
645pub fn verify(store: &Store, include_optional: bool, quick: bool) -> crate::Result<VerifyReport> {
655 let records = read_manifest(store)?;
656 let mut missing = Vec::new();
657 let mut corrupt = Vec::new();
658 let mut checked = 0usize;
659
660 for rec in &records {
661 if !rec.required && !include_optional {
662 continue;
663 }
664 checked += 1;
665 let abs = match store.capability_relative(Path::new(&rec.path)) {
666 Ok(p) => p,
667 Err(_) => {
668 corrupt.push(rec.path.clone());
670 continue;
671 }
672 };
673 let file = match store.open_regular(abs) {
674 Ok(file) => file,
675 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
676 missing.push(rec.path.clone());
677 continue;
678 }
679 Err(_) => {
680 corrupt.push(rec.path.clone());
681 continue;
682 }
683 };
684 if quick {
685 let len = file.metadata()?.len();
686 if len != rec.bytes {
687 corrupt.push(rec.path.clone());
688 }
689 } else {
690 let (sha, bytes) = sha256_file(file)?;
691 if sha != rec.sha256 || bytes != rec.bytes {
692 corrupt.push(rec.path.clone());
693 }
694 }
695 }
696
697 let ok = checked - missing.len() - corrupt.len();
698 let complete = missing.is_empty() && corrupt.is_empty();
699 Ok(VerifyReport {
700 mode: if quick { "quick" } else { "deep" }.to_string(),
701 checked,
702 ok,
703 missing,
704 corrupt,
705 complete,
706 })
707}
708
709pub fn status(store: &Store) -> crate::Result<StatusReport> {
717 let records = read_manifest(store)?;
718 let mut present = 0usize;
719 let mut missing = 0usize;
720 let mut required_missing = 0usize;
721 let mut optional_missing = 0usize;
722 let mut bytes_total = 0u64;
723 let mut bytes_missing = 0u64;
724 let mut assets = Vec::with_capacity(records.len());
725
726 for rec in &records {
727 bytes_total = bytes_total.saturating_add(rec.bytes);
732 let is_present = store.open_regular(Path::new(&rec.path)).is_ok();
740 let state = if is_present {
741 present += 1;
742 "present"
743 } else {
744 missing += 1;
745 bytes_missing = bytes_missing.saturating_add(rec.bytes);
746 if rec.required {
747 required_missing += 1;
748 } else {
749 optional_missing += 1;
750 }
751 "missing"
752 };
753 assets.push(AssetState {
754 path: rec.path.clone(),
755 sha256: rec.sha256.clone(),
756 bytes: rec.bytes,
757 required: rec.required,
758 state: state.to_string(),
759 });
760 }
761
762 Ok(StatusReport {
763 total: records.len(),
764 present,
765 missing,
766 required_missing,
767 optional_missing,
768 bytes_total,
769 bytes_missing,
770 assets,
771 })
772}
773
774pub fn paths(store: &Store) -> crate::Result<Vec<String>> {
792 Ok(read_manifest(store)?
793 .into_iter()
794 .filter(|r| store.capability_relative(Path::new(&r.path)).is_ok())
795 .map(|r| r.path)
796 .collect())
797}
798
799pub fn declared_assets(fm: &parser::Frontmatter) -> Vec<Declaration> {
809 let mut out = Vec::new();
810 if let Some(v) = fm.get("asset") {
811 collect_declarations(&v, &mut out);
812 }
813 if let Some(v) = fm.get("assets") {
814 collect_declarations(&v, &mut out);
815 }
816 out
817}
818
819pub fn declarations_from_yaml_map(map: &BTreeMap<String, Value>) -> Vec<Declaration> {
823 let mut out = Vec::new();
824 if let Some(v) = map.get("asset") {
825 collect_declarations(v, &mut out);
826 }
827 if let Some(v) = map.get("assets") {
828 collect_declarations(v, &mut out);
829 }
830 out
831}
832
833pub fn asset_supersession(fm: &parser::Frontmatter) -> Result<Option<AssetSupersession>, String> {
837 asset_supersession_from_parts(fm.get(SUPERSEDES_ASSET_KEY).as_ref(), declared_assets(fm))
838}
839
840pub fn asset_supersession_from_yaml_map(
842 map: &BTreeMap<String, Value>,
843) -> Result<Option<AssetSupersession>, String> {
844 asset_supersession_from_parts(
845 map.get(SUPERSEDES_ASSET_KEY),
846 declarations_from_yaml_map(map),
847 )
848}
849
850fn asset_supersession_from_parts(
851 value: Option<&Value>,
852 declarations: Vec<Declaration>,
853) -> Result<Option<AssetSupersession>, String> {
854 let Some(value) = value else {
855 return Ok(None);
856 };
857 let Value::String(original) = value else {
858 return Err(format!("`{SUPERSEDES_ASSET_KEY}` must be one asset path"));
859 };
860 if declarations.len() != 1 || !declarations[0].required {
861 return Err(format!(
862 "a `{SUPERSEDES_ASSET_KEY}` wrapper must declare exactly one required replacement asset"
863 ));
864 }
865 let original = normalize_asset_path(original)?;
866 let replacement = normalize_asset_path(&declarations[0].path)?;
867 if original == replacement {
868 return Err(format!(
869 "`{SUPERSEDES_ASSET_KEY}` cannot name the wrapper's replacement asset"
870 ));
871 }
872 Ok(Some(AssetSupersession {
873 original,
874 replacement,
875 }))
876}
877
878fn collect_declarations(v: &Value, out: &mut Vec<Declaration>) {
879 match v {
880 Value::String(s) => out.push(Declaration {
881 path: s.clone(),
882 required: true,
883 }),
884 Value::Sequence(items) => {
885 for item in items {
886 match item {
887 Value::String(s) => out.push(Declaration {
888 path: s.clone(),
889 required: true,
890 }),
891 Value::Mapping(m) => {
892 let path = m
893 .get(Value::String("path".to_string()))
894 .and_then(|x| x.as_str())
895 .map(|s| s.to_string());
896 if let Some(path) = path {
897 let required = m
898 .get(Value::String("required".to_string()))
899 .and_then(|x| x.as_bool())
900 .unwrap_or(true);
901 out.push(Declaration { path, required });
902 }
903 }
904 _ => {}
905 }
906 }
907 }
908 _ => {}
909 }
910}
911
912pub fn normalize_asset_path(raw: &str) -> Result<String, String> {
928 let trimmed = raw.trim();
929 if trimmed.is_empty() {
930 return Err("empty asset path".to_string());
931 }
932 let p = Path::new(trimmed);
933 if p.is_absolute() {
934 return Err(format!("absolute asset path not allowed: {raw}"));
935 }
936 let mut normal: Vec<&std::ffi::OsStr> = Vec::new();
937 for c in p.components() {
938 match c {
939 Component::ParentDir => return Err(format!("`..` not allowed in asset path: {raw}")),
940 Component::Prefix(_) | Component::RootDir => {
941 return Err(format!("asset path escapes the store: {raw}"))
942 }
943 Component::CurDir => {}
946 Component::Normal(seg) => normal.push(seg),
947 }
948 }
949 if normal.is_empty() {
950 return Err(format!("asset path names no file: {raw}"));
952 }
953 let joined: PathBuf = normal.into_iter().collect();
954 Ok(joined.to_string_lossy().replace('\\', "/"))
955}
956
957fn is_markdown(path: &str) -> bool {
958 Path::new(path)
959 .extension()
960 .and_then(|e| e.to_str())
961 .map(|e| e.eq_ignore_ascii_case("md"))
962 .unwrap_or(false)
963}
964
965fn rel_to_string(p: &Path) -> String {
966 p.to_string_lossy().replace('\\', "/")
967}
968
969fn sha256_file(mut f: std::fs::File) -> std::io::Result<(String, u64)> {
972 let mut hasher = Sha256::new();
973 let mut buf = [0u8; 65536];
974 let mut total: u64 = 0;
975 loop {
976 let n = f.read(&mut buf)?;
977 if n == 0 {
978 break;
979 }
980 hasher.update(&buf[..n]);
981 total += n as u64;
982 }
983 let digest = hasher.finalize();
984 let mut hex = String::with_capacity(64);
985 for b in digest.iter() {
986 let _ = write!(hex, "{b:02x}");
987 }
988 Ok((hex, total))
989}
990
991fn media_type_for(path: &str) -> String {
995 let ext = Path::new(path)
996 .extension()
997 .and_then(|e| e.to_str())
998 .unwrap_or("")
999 .to_ascii_lowercase();
1000 let mt = match ext.as_str() {
1001 "pdf" => "application/pdf",
1002 "png" => "image/png",
1003 "jpg" | "jpeg" => "image/jpeg",
1004 "gif" => "image/gif",
1005 "webp" => "image/webp",
1006 "svg" => "image/svg+xml",
1007 "tiff" | "tif" => "image/tiff",
1008 "mp4" => "video/mp4",
1009 "mov" => "video/quicktime",
1010 "webm" => "video/webm",
1011 "mkv" => "video/x-matroska",
1012 "mp3" => "audio/mpeg",
1013 "wav" => "audio/wav",
1014 "m4a" => "audio/mp4",
1015 "flac" => "audio/flac",
1016 "zip" => "application/zip",
1017 "gz" | "tgz" => "application/gzip",
1018 "tar" => "application/x-tar",
1019 "csv" => "text/csv",
1020 "tsv" => "text/tab-separated-values",
1021 "json" => "application/json",
1022 "xml" => "application/xml",
1023 "txt" => "text/plain",
1024 "vtt" => "text/vtt",
1025 "srt" => "application/x-subrip",
1026 "html" | "htm" => "text/html",
1027 "epub" => "application/epub+zip",
1028 "docx" => "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
1029 "xlsx" => "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
1030 "pptx" => "application/vnd.openxmlformats-officedocument.presentationml.presentation",
1031 "doc" => "application/msword",
1032 "xls" => "application/vnd.ms-excel",
1033 "ppt" => "application/vnd.ms-powerpoint",
1034 _ => "application/octet-stream",
1035 };
1036 mt.to_string()
1037}
1038
1039fn find_untracked(store: &Store, declared: &BTreeSet<String>) -> crate::Result<Vec<String>> {
1043 let mut out = Vec::new();
1044 let paths = match store.walk_regular_files(Path::new("sources")) {
1045 Ok(paths) => paths,
1046 Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(out),
1047 Err(error) => return Err(error.into()),
1048 };
1049 for path in paths {
1050 let name = match path.file_name().and_then(|name| name.to_str()) {
1051 Some(name) => name,
1052 None => continue,
1053 };
1054 if is_markdown(name) || name == "index.jsonl" {
1055 continue;
1056 }
1057 let rel = rel_to_string(&path);
1058 if !declared.contains(&rel) {
1059 out.push(rel);
1060 }
1061 }
1062 out.sort();
1063 Ok(out)
1064}
1065
1066#[cfg(test)]
1067mod tests {
1068 use super::*;
1069
1070 #[test]
1071 fn supersession_cycles_exclude_only_cycle_members() {
1072 let supersessions = BTreeMap::from([
1073 ("a".to_string(), ("b".to_string(), "a.md".to_string())),
1074 ("b".to_string(), ("a".to_string(), "b.md".to_string())),
1075 (
1076 "before".to_string(),
1077 ("a".to_string(), "before.md".to_string()),
1078 ),
1079 (
1080 "clean".to_string(),
1081 ("next".to_string(), "clean.md".to_string()),
1082 ),
1083 ]);
1084 assert_eq!(
1085 supersession_cycle_members(&supersessions),
1086 BTreeSet::from(["a".to_string(), "b".to_string()])
1087 );
1088 }
1089
1090 #[test]
1096 fn normalize_asset_path_folds_curdir_and_rejects_traversal() {
1097 assert_eq!(
1098 normalize_asset_path("./sources/x.pdf").unwrap(),
1099 "sources/x.pdf"
1100 );
1101 assert_eq!(
1102 normalize_asset_path("sources/x.pdf").unwrap(),
1103 "sources/x.pdf"
1104 );
1105 assert_eq!(
1106 normalize_asset_path("sources/./x.pdf").unwrap(),
1107 "sources/x.pdf"
1108 );
1109 assert_eq!(
1110 normalize_asset_path("sources/x.pdf/").unwrap(),
1111 "sources/x.pdf"
1112 );
1113
1114 assert!(normalize_asset_path("../outside.txt").is_err());
1116 assert!(normalize_asset_path("sources/../../etc/passwd").is_err());
1117 assert!(normalize_asset_path("/abs/x.pdf").is_err());
1118 assert!(normalize_asset_path(".").is_err());
1120 assert!(normalize_asset_path("./").is_err());
1121 assert!(normalize_asset_path("").is_err());
1122 }
1123
1124 #[test]
1129 fn status_and_scan_saturate_on_overflowing_manifest_bytes() {
1130 let tmp = tempfile::TempDir::new().unwrap();
1131 let root = tmp.path();
1132 std::fs::write(root.join("DB.md"), "---\ntype: db-md\n---\n# store\n").unwrap();
1133 std::fs::write(
1135 root.join("assets.jsonl"),
1136 "{\"path\":\"records/a.bin\",\"sha256\":\"x\",\"bytes\":18446744073709551615,\
1137\"media_type\":\"application/octet-stream\",\"wrappers\":[\"records/w.md\"],\"required\":true}\n\
1138{\"path\":\"records/b.bin\",\"sha256\":\"y\",\"bytes\":1,\
1139\"media_type\":\"application/octet-stream\",\"wrappers\":[\"records/w.md\"],\"required\":true}\n",
1140 )
1141 .unwrap();
1142 let store = Store::from_root_and_config(root, crate::parser::Config::default()).unwrap();
1143
1144 let report = status(&store).expect("status is non-failing on a poisoned manifest");
1147 assert_eq!(
1148 report.bytes_total,
1149 u64::MAX,
1150 "byte total must saturate, not wrap"
1151 );
1152 assert_eq!(
1153 report.bytes_missing,
1154 u64::MAX,
1155 "missing bytes must saturate too"
1156 );
1157 assert_eq!(report.total, 2);
1158
1159 scan(&store, true, false).expect("scan must not overflow on a poisoned manifest");
1161 }
1162
1163 fn store_with_one_asset() -> (tempfile::TempDir, Store, String) {
1166 let tmp = tempfile::TempDir::new().unwrap();
1167 let root = tmp.path();
1168 std::fs::create_dir_all(root.join("sources")).unwrap();
1169 std::fs::write(root.join("DB.md"), "---\ntype: db-md\n---\n# store\n").unwrap();
1170 std::fs::write(
1171 root.join("sources/a.pdf.md"),
1172 "---\ntype: pdf-source\nsummary: x\nasset: sources/a.pdf\n---\nbody\n",
1173 )
1174 .unwrap();
1175 std::fs::write(root.join("sources/a.pdf"), b"PDFBYTES").unwrap();
1176 let store = Store::from_root_and_config(root, crate::parser::Config::default()).unwrap();
1177 let report = scan(&store, false, false).unwrap();
1178 assert!(
1179 report.wrote,
1180 "first scan writes the manifest; report: {report:?}"
1181 );
1182 let canonical = std::fs::read_to_string(root.join(MANIFEST_FILE)).unwrap();
1183 (tmp, store, canonical)
1184 }
1185
1186 #[test]
1195 fn scan_recompacts_duplicate_line_manifest() {
1196 let (_tmp, store, canonical) = store_with_one_asset();
1197 let abs = store.root.join(MANIFEST_FILE);
1198
1199 std::fs::write(&abs, format!("{canonical}{canonical}")).unwrap();
1201 assert_eq!(std::fs::read_to_string(&abs).unwrap().lines().count(), 2);
1202
1203 let report = scan(&store, false, false).unwrap();
1204 assert!(
1205 report.wrote,
1206 "a non-canonical (duplicate-line) manifest must be recompacted and reported as updated"
1207 );
1208 let after = std::fs::read_to_string(&abs).unwrap();
1209 assert_eq!(
1210 after.lines().count(),
1211 1,
1212 "duplicate lines must collapse to the single canonical line"
1213 );
1214 assert_eq!(
1215 after, canonical,
1216 "scan must restore the exact canonical bytes"
1217 );
1218 }
1219
1220 #[test]
1224 fn scan_recompacts_noncanonical_byte_layout() {
1225 let (_tmp, store, canonical) = store_with_one_asset();
1226 let abs = store.root.join(MANIFEST_FILE);
1227
1228 std::fs::write(&abs, canonical.trim_end_matches('\n')).unwrap();
1230 let report = scan(&store, false, false).unwrap();
1231 assert!(
1232 report.wrote,
1233 "a manifest missing its trailing newline must be recompacted"
1234 );
1235 assert_eq!(
1236 std::fs::read_to_string(&abs).unwrap(),
1237 canonical,
1238 "scan must restore the canonical trailing newline"
1239 );
1240 }
1241
1242 #[test]
1252 fn paths_omits_store_escaping_records() {
1253 let tmp = tempfile::TempDir::new().unwrap();
1254 let root = tmp.path();
1255 std::fs::write(root.join("DB.md"), "---\ntype: db-md\n---\n# store\n").unwrap();
1256 std::fs::write(
1258 root.join("assets.jsonl"),
1259 "{\"path\":\"sources/legit.pdf\",\"sha256\":\"a\",\"bytes\":9,\
1260\"media_type\":\"application/pdf\",\"wrappers\":[\"sources/legit.pdf.md\"],\"required\":true}\n\
1261{\"path\":\"../../../../../../etc/passwd\",\"sha256\":\"b\",\"bytes\":4096,\
1262\"media_type\":\"text/plain\",\"wrappers\":[\"sources/legit.pdf.md\"],\"required\":false}\n\
1263{\"path\":\"/etc/hosts\",\"sha256\":\"c\",\"bytes\":4096,\
1264\"media_type\":\"text/plain\",\"wrappers\":[\"sources/legit.pdf.md\"],\"required\":false}\n",
1265 )
1266 .unwrap();
1267 let store = Store::from_root_and_config(root, crate::parser::Config::default()).unwrap();
1268
1269 let out = paths(&store).expect("paths is non-failing on a poisoned manifest");
1270 assert_eq!(
1271 out,
1272 vec!["sources/legit.pdf".to_string()],
1273 "only the safe in-store path is emitted; escaping paths are omitted"
1274 );
1275 assert!(
1276 !out.iter().any(|p| p.starts_with('/') || p.contains("..")),
1277 "no absolute or `..` path may ever leak from `paths`: {out:?}"
1278 );
1279 }
1280
1281 #[test]
1284 fn paths_passes_a_clean_manifest_through_unchanged() {
1285 let (_tmp, store, _canonical) = store_with_one_asset();
1286 let out = paths(&store).expect("paths over a clean manifest");
1287 assert_eq!(out, vec!["sources/a.pdf".to_string()]);
1288 }
1289
1290 #[test]
1294 fn scan_canonical_manifest_is_left_untouched() {
1295 let (_tmp, store, canonical) = store_with_one_asset();
1296 let abs = store.root.join(MANIFEST_FILE);
1297
1298 let report = scan(&store, false, false).unwrap();
1299 assert!(
1300 !report.wrote,
1301 "a canonical, unchanged manifest must not be rewritten"
1302 );
1303 assert_eq!(
1304 std::fs::read_to_string(&abs).unwrap(),
1305 canonical,
1306 "a no-op rescan must leave the manifest byte-identical"
1307 );
1308 }
1309
1310 #[cfg(unix)]
1311 #[test]
1312 fn manifest_membership_reads_opened_root_after_path_replacement() {
1313 use std::os::unix::fs::symlink;
1314
1315 let sandbox = tempfile::tempdir().unwrap();
1316 let root = sandbox.path().join("store");
1317 std::fs::create_dir_all(&root).unwrap();
1318 std::fs::write(root.join("DB.md"), "---\ntype: db-md\n---\n").unwrap();
1319 std::fs::write(
1320 root.join(MANIFEST_FILE),
1321 "{\"path\":\"sources/owned.pdf\",\"sha256\":\"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\"bytes\":1,\"media_type\":\"application/pdf\",\"wrappers\":[],\"required\":true}\n",
1322 )
1323 .unwrap();
1324 let store = Store::open_strict(&root).unwrap();
1325 let detached = sandbox.path().join("detached");
1326 std::fs::rename(&root, &detached).unwrap();
1327
1328 let replacement = sandbox.path().join("replacement");
1329 std::fs::create_dir_all(&replacement).unwrap();
1330 std::fs::write(replacement.join("DB.md"), "---\ntype: db-md\n---\n").unwrap();
1331 std::fs::write(
1332 replacement.join(MANIFEST_FILE),
1333 "{\"path\":\"sources/replacement-secret.pdf\",\"sha256\":\"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\",\"bytes\":1,\"media_type\":\"application/pdf\",\"wrappers\":[],\"required\":true}\n",
1334 )
1335 .unwrap();
1336 symlink(&replacement, &root).unwrap();
1337
1338 assert_eq!(paths(&store).unwrap(), vec!["sources/owned.pdf"]);
1339 }
1340}