Skip to main content

openakb_validate/
content.rs

1//! Opt-in content checks that fetch descriptor-related resources.
2
3use std::{
4    collections::{BTreeMap, BTreeSet, HashMap, HashSet},
5    path::{Component, Path, PathBuf},
6    string::FromUtf8Error,
7};
8
9use async_trait::async_trait;
10use base64::{
11    Engine as _, alphabet,
12    engine::{DecodePaddingMode, GeneralPurpose, GeneralPurposeConfig, general_purpose::STANDARD},
13};
14use fluent_uri::{Uri, UriRef};
15use serde::Serialize;
16use serde_json::{Map, Value};
17use sha2::{Digest, Sha256};
18use thiserror::Error;
19use tokio::fs;
20
21use crate::{
22    Advisory, Code, Finding, Segment, json_pointer,
23    schema::provenance_schema_findings,
24    shape::{
25        EntityIndex, EntityKind, Object, casefolded_duplicate_indices, id_kind, indexed_objects,
26        normalize_id, reference_code_id, typed_id_value,
27    },
28};
29
30const SHA256_ALGORITHM: &str = "sha256";
31const SHA256_LENGTH: usize = 32;
32const STRICT_STANDARD_BASE64: GeneralPurpose = GeneralPurpose::new(
33    &alphabet::STANDARD,
34    GeneralPurposeConfig::new()
35        .with_decode_padding_mode(DecodePaddingMode::RequireCanonical)
36        .with_decode_allow_trailing_bits(false),
37);
38
39/// Fetches descriptor-related content bytes.
40///
41/// `check_content` calls [`Resolver::fetch`] with effective references:
42/// descriptor URI references are resolved against `base_uri` when present and
43/// fragments are stripped before fetch.
44#[async_trait]
45pub trait Resolver: Send + Sync {
46    /// Returns bytes for `uri`, or an [`Unfetchable`] error when the resource
47    /// cannot be fetched.
48    async fn fetch(&self, uri: &str) -> Result<Vec<u8>, Unfetchable>;
49
50    /// Returns true when `check_content` should pre-screen local path references.
51    ///
52    /// Resolvers that serve scheme-less local filesystem paths should opt into
53    /// this. The screening rejects raw descriptor references and `base_uri`
54    /// values containing queries, semicolon path parameters, backslashes, or
55    /// `..` path traversal before any fetch happens. Custom URI resolvers should
56    /// keep the default `false` and enforce their own safety policy in `fetch`.
57    fn screens_local_paths(&self) -> bool {
58        false
59    }
60}
61
62/// A resolver-level fetch failure.
63#[derive(Debug, Error)]
64#[error("unfetchable: {reason}")]
65pub struct Unfetchable {
66    /// Human-readable reason the resource could not be fetched.
67    pub reason: String,
68}
69
70impl Unfetchable {
71    fn new(reason: impl Into<String>) -> Self {
72        Self {
73            reason: reason.into(),
74        }
75    }
76
77    fn outside_local_base(uri: &str) -> Self {
78        Self::new(format!("outside local base: {uri}"))
79    }
80}
81
82/// Resolves scheme-less relative paths under a local base directory.
83///
84/// Fragments are ignored. Schemes, authorities, queries, semicolon path
85/// parameters, backslashes, absolute paths, `..` traversal, and symlink escapes
86/// are rejected as outside the base.
87pub struct LocalFileResolver {
88    base_dir: PathBuf,
89}
90
91impl LocalFileResolver {
92    /// Creates a resolver rooted at `base_dir`.
93    #[must_use]
94    pub fn new(base_dir: impl Into<PathBuf>) -> Self {
95        Self {
96            base_dir: base_dir.into(),
97        }
98    }
99
100    async fn local_path(&self, uri: &str) -> Result<PathBuf, Unfetchable> {
101        let raw_reference = strip_fragment(uri);
102        let raw_path = raw_reference
103            .split_once('?')
104            .map_or(raw_reference, |(path, _)| path);
105        if rejects_local_reference(raw_reference, raw_path) {
106            return Err(Unfetchable::outside_local_base(uri));
107        }
108
109        let base = fs::canonicalize(&self.base_dir)
110            .await
111            .map_err(|error| Unfetchable::new(error.to_string()))?;
112        let path = fs::canonicalize(base.join(raw_path))
113            .await
114            .map_err(|error| Unfetchable::new(error.to_string()))?;
115        if path != base && !path.starts_with(&base) {
116            return Err(Unfetchable::outside_local_base(uri));
117        }
118        Ok(path)
119    }
120}
121
122#[async_trait]
123impl Resolver for LocalFileResolver {
124    async fn fetch(&self, uri: &str) -> Result<Vec<u8>, Unfetchable> {
125        let path = self.local_path(uri).await?;
126        fs::read(path)
127            .await
128            .map_err(|error| Unfetchable::new(error.to_string()))
129    }
130
131    fn screens_local_paths(&self) -> bool {
132        true
133    }
134}
135
136/// Three-state outcome for a content check.
137#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
138#[serde(rename_all = "lowercase")]
139pub enum Outcome {
140    /// The check proved the content did not match its descriptor claim.
141    Failed,
142    /// The check could not prove or disprove the descriptor claim.
143    Unverifiable,
144    /// The check proved the descriptor claim.
145    Verified,
146}
147
148/// Content check category.
149#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
150#[serde(rename_all = "kebab-case")]
151pub enum CheckKind {
152    /// Source capture bytes and `content_hash`.
153    Capture,
154    /// Citation markers in fetched Markdown content.
155    Citations,
156    /// Section content bytes and `content_hash`.
157    ContentHash,
158    /// Descriptor guide bytes and `guide_hash`.
159    GuideHash,
160    /// Quoted claim text in source captures.
161    Quote,
162    /// Provenance sidecar bytes and binding.
163    Sidecar,
164}
165
166/// One content check and its three-state outcome.
167#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
168pub struct ContentCheck {
169    /// Kind of content check performed.
170    pub kind: CheckKind,
171    /// JSON Pointer path to the descriptor field or object the check covers.
172    pub path: String,
173    /// Three-state check result.
174    pub outcome: Outcome,
175    /// Human-readable explanation of the result.
176    pub detail: String,
177    /// Fatal validation findings raised by the check.
178    pub findings: Vec<Finding>,
179    /// Non-fatal advisories raised by the check.
180    pub warnings: Vec<Advisory>,
181}
182
183/// Aggregate result for opt-in content checks.
184#[derive(Debug, Clone, Default, Serialize)]
185pub struct ContentReport {
186    /// Individual content checks in execution order.
187    pub checks: Vec<ContentCheck>,
188}
189
190impl ContentReport {
191    /// Returns every fatal finding raised by the report's checks.
192    pub fn findings(&self) -> impl Iterator<Item = &Finding> {
193        self.checks.iter().flat_map(|check| check.findings.iter())
194    }
195
196    /// Returns every non-fatal advisory raised by the report's checks.
197    pub fn warnings(&self) -> impl Iterator<Item = &Advisory> {
198        self.checks.iter().flat_map(|check| check.warnings.iter())
199    }
200
201    /// Returns checks whose outcome equals `outcome`.
202    pub fn with_outcome(&self, outcome: Outcome) -> impl Iterator<Item = &ContentCheck> {
203        self.checks
204            .iter()
205            .filter(move |check| check.outcome == outcome)
206    }
207
208    /// Returns true when no check failed and no fatal findings were raised.
209    #[must_use]
210    pub fn ok(&self) -> bool {
211        self.with_outcome(Outcome::Failed).next().is_none() && self.findings().next().is_none()
212    }
213}
214
215/// Runs opt-in content checks against fetched descriptor resources.
216pub async fn check_content(descriptor: &Value, resolver: &dyn Resolver) -> ContentReport {
217    let Some(object) = descriptor.as_object() else {
218        return ContentReport::default();
219    };
220
221    let graph = Graph::from_descriptor(object);
222    let mut checks = Vec::new();
223    checks.extend(guide_checks(object, resolver).await);
224    checks.extend(section_checks(&graph, object, resolver).await);
225    let sidecars = sidecar_checks(&graph, object, resolver).await;
226    checks.extend(sidecars.checks);
227    let mut quote_claims = inline_quote_claims(&graph);
228    quote_claims.extend(sidecars.claims);
229    let captures = capture_checks(&graph, object, resolver).await;
230    let quote_checks = quote_checks(&graph, &captures, &quote_claims);
231    checks.extend(captures.checks);
232    checks.extend(quote_checks);
233    ContentReport { checks }
234}
235
236#[derive(Debug, Clone)]
237struct Graph<'descriptor> {
238    sources: Vec<(usize, &'descriptor Object)>,
239    sections: Vec<(usize, &'descriptor Object)>,
240    entities: EntityIndex,
241}
242
243impl<'descriptor> Graph<'descriptor> {
244    fn from_descriptor(descriptor: &'descriptor Object) -> Self {
245        let sources: Vec<_> = indexed_objects(descriptor.get("sources")).collect();
246        let sections: Vec<_> = indexed_objects(descriptor.get("sections")).collect();
247        let source_ids = ids(&sources);
248        let section_ids = ids(&sections);
249        Self {
250            sources,
251            sections,
252            entities: EntityIndex::new(source_ids, section_ids),
253        }
254    }
255}
256
257#[derive(Debug, Clone)]
258struct ResolvedContent<'descriptor> {
259    index: usize,
260    payload: Vec<u8>,
261    _section: &'descriptor Object,
262}
263
264#[derive(Debug, Clone)]
265struct UnfetchedContent {
266    index: usize,
267    error: String,
268}
269
270#[derive(Debug, Clone)]
271enum SectionFetch<'descriptor> {
272    Resolved(ResolvedContent<'descriptor>),
273    Unfetched(UnfetchedContent),
274}
275
276#[derive(Debug, Clone)]
277struct ResolvedCapture {
278    payload: Vec<u8>,
279}
280
281#[derive(Debug, Clone)]
282enum CaptureFetch {
283    Resolved(ResolvedCapture),
284    Unfetched(UnfetchedContent),
285}
286
287#[derive(Debug, Clone, Default)]
288struct CaptureResult {
289    payloads: HashMap<String, Vec<u8>>,
290    checks: Vec<ContentCheck>,
291    hash_failed: HashSet<String>,
292}
293
294#[derive(Debug, Clone)]
295struct QuoteClaim {
296    path: Vec<PathSegment>,
297    quote: String,
298    source_ids: Vec<String>,
299}
300
301#[derive(Debug, Clone, PartialEq, Eq)]
302enum PathSegment {
303    Key(&'static str),
304    Index(usize),
305}
306
307#[derive(Debug, Clone, Default)]
308struct SidecarResult {
309    claims: Vec<QuoteClaim>,
310    checks: Vec<ContentCheck>,
311}
312
313async fn guide_checks(object: &Map<String, Value>, resolver: &dyn Resolver) -> Vec<ContentCheck> {
314    let Some(guide_hash) = object.get("guide_hash").and_then(Value::as_str) else {
315        return Vec::new();
316    };
317    let path = guide_hash_path();
318    let expected = match parse_sri(CheckKind::GuideHash, path.clone(), guide_hash) {
319        Ok(expected) => expected,
320        Err(check) => return vec![check],
321    };
322    let Some(guide_uri) = object.get("guide_uri").and_then(Value::as_str) else {
323        return vec![check(
324            Outcome::Unverifiable,
325            CheckKind::GuideHash,
326            path,
327            "missing guide_uri",
328        )];
329    };
330
331    let uri = match effective_reference(object, guide_uri, resolver.screens_local_paths()) {
332        Ok(uri) => uri,
333        Err(error) => {
334            return vec![check(
335                Outcome::Unverifiable,
336                CheckKind::GuideHash,
337                path,
338                error.to_string(),
339            )];
340        }
341    };
342    let payload = match resolver.fetch(&uri).await {
343        Ok(payload) => payload,
344        Err(error) => {
345            return vec![check(
346                Outcome::Unverifiable,
347                CheckKind::GuideHash,
348                path,
349                error.to_string(),
350            )];
351        }
352    };
353    vec![compare_sri(CheckKind::GuideHash, path, &payload, &expected)]
354}
355
356async fn section_checks(
357    graph: &Graph<'_>,
358    descriptor: &Map<String, Value>,
359    resolver: &dyn Resolver,
360) -> Vec<ContentCheck> {
361    let mut checks = Vec::new();
362
363    for (index, section) in &graph.sections {
364        let Some(reference) = section.get("content_uri").and_then(Value::as_str) else {
365            continue;
366        };
367        let content_hash = section.get("content_hash").and_then(Value::as_str);
368        let checks_citations = !section.contains_key("content_type")
369            || section.get("content_type").is_some_and(is_markdown);
370        if content_hash.is_none() && !checks_citations {
371            continue;
372        }
373        let hash_check = content_hash
374            .map(|hash| parse_sri(CheckKind::ContentHash, section_hash_path(*index), hash));
375        if !checks_citations && matches!(hash_check, Some(Err(_))) {
376            if let Some(Err(check)) = hash_check {
377                checks.push(check);
378            }
379            continue;
380        }
381
382        let resolved = fetch_section(*index, section, descriptor, reference, resolver).await;
383        if let Some(hash_check) = hash_check {
384            match hash_check {
385                Ok(expected) => checks.push(check_sri(
386                    CheckKind::ContentHash,
387                    section_hash_path(*index),
388                    &resolved,
389                    &expected,
390                )),
391                Err(check) => checks.push(check),
392            }
393        }
394        if checks_citations {
395            checks.push(citation_check(graph, &resolved));
396        }
397    }
398
399    checks
400}
401
402async fn fetch_section<'descriptor>(
403    index: usize,
404    section: &'descriptor Object,
405    descriptor: &Map<String, Value>,
406    reference: &str,
407    resolver: &dyn Resolver,
408) -> SectionFetch<'descriptor> {
409    let uri = match effective_reference(descriptor, reference, resolver.screens_local_paths()) {
410        Ok(uri) => uri,
411        Err(error) => {
412            return SectionFetch::Unfetched(UnfetchedContent {
413                index,
414                error: error.to_string(),
415            });
416        }
417    };
418    match resolver.fetch(&uri).await {
419        Ok(payload) => SectionFetch::Resolved(ResolvedContent {
420            index,
421            payload,
422            _section: section,
423        }),
424        Err(error) => SectionFetch::Unfetched(UnfetchedContent {
425            index,
426            error: error.to_string(),
427        }),
428    }
429}
430
431fn citation_check(graph: &Graph<'_>, resolved: &SectionFetch<'_>) -> ContentCheck {
432    let resolved = match resolved {
433        SectionFetch::Resolved(resolved) => resolved,
434        SectionFetch::Unfetched(unfetched) => {
435            return check(
436                Outcome::Unverifiable,
437                CheckKind::Citations,
438                section_content_uri_path(unfetched.index),
439                unfetched.error.clone(),
440            );
441        }
442    };
443    let markdown = match String::from_utf8(resolved.payload.clone()) {
444        Ok(markdown) => markdown,
445        Err(error) => {
446            return check(
447                Outcome::Failed,
448                CheckKind::Citations,
449                section_content_uri_path(resolved.index),
450                utf8_error_detail(error),
451            );
452        }
453    };
454    let (findings, warnings) = citation_results(graph, resolved.index, &markdown);
455    ContentCheck {
456        kind: CheckKind::Citations,
457        path: section_content_uri_path(resolved.index),
458        outcome: if findings.is_empty() {
459            Outcome::Verified
460        } else {
461            Outcome::Failed
462        },
463        detail: "citation markers checked".to_owned(),
464        findings,
465        warnings,
466    }
467}
468
469fn citation_results(
470    graph: &Graph<'_>,
471    section_index: usize,
472    markdown: &str,
473) -> (Vec<Finding>, Vec<Advisory>) {
474    let mut findings = Vec::new();
475    let mut warnings = Vec::new();
476    let base_path = vec![
477        PathSegment::Key("sections"),
478        PathSegment::Index(section_index),
479        PathSegment::Key("content_uri"),
480    ];
481    for (marker_index, citation) in crate::extract_citations(markdown).iter().enumerate() {
482        append_duplicate_warnings(&mut warnings, &base_path, &citation.ids);
483        for (id_index, source_id) in citation.ids.iter().enumerate() {
484            let mut path = base_path.clone();
485            path.extend([
486                PathSegment::Key("citations"),
487                PathSegment::Index(marker_index),
488                PathSegment::Index(id_index),
489            ]);
490            let code = if id_kind(source_id).is_none() {
491                // Fetched Markdown is invisible to the descriptor schema, so a
492                // non-typed token was not pre-reported there as AKB011 -- report
493                // it here; reference_code_id skips such tokens by contract, so
494                // this branch is also what keeps them from silently verifying.
495                Code::Akb011
496            } else if let Some(code) =
497                reference_code_id(source_id, EntityKind::Source, &graph.entities)
498            {
499                code
500            } else {
501                continue;
502            };
503            findings.push(Finding {
504                code,
505                path: pointer(path),
506                message: if code == Code::Akb011 {
507                    format!("citation source id {source_id:?} is not a typed source id")
508                } else {
509                    format!("citation source id {source_id:?} does not resolve to a source")
510                },
511            });
512        }
513    }
514    (findings, warnings)
515}
516
517fn append_duplicate_warnings(warnings: &mut Vec<Advisory>, path: &[PathSegment], ids: &[String]) {
518    // One pass counts occurrences; the BTreeMap keeps ids sorted and unique, so the
519    // reported list stays the same sorted, deduped, comma-joined form as a nested
520    // per-id rescan produced, without its O(k^2) cost.
521    let mut counts: BTreeMap<&str, usize> = BTreeMap::new();
522    for id in ids {
523        *counts.entry(id).or_default() += 1;
524    }
525    let duplicates: Vec<&str> = counts
526        .into_iter()
527        .filter(|&(_, count)| count > 1)
528        .map(|(id, _)| id)
529        .collect();
530    if duplicates.is_empty() {
531        return;
532    }
533    warnings.push(Advisory {
534        path: pointer(path.iter().cloned()),
535        message: format!("duplicate citation id in marker: {}", duplicates.join(", ")),
536    });
537}
538
539async fn capture_checks(
540    graph: &Graph<'_>,
541    descriptor: &Map<String, Value>,
542    resolver: &dyn Resolver,
543) -> CaptureResult {
544    let mut result = CaptureResult::default();
545
546    for (index, source) in &graph.sources {
547        append_capture_checks(&mut result, descriptor, *index, source, resolver).await;
548    }
549
550    result
551}
552
553async fn append_capture_checks(
554    result: &mut CaptureResult,
555    descriptor: &Map<String, Value>,
556    index: usize,
557    source: &Map<String, Value>,
558    resolver: &dyn Resolver,
559) {
560    let content_hash_path = source_content_hash_path(index);
561    let capture_uri = source.get("capture_uri").and_then(Value::as_str);
562    let content_hash = source.get("content_hash").and_then(Value::as_str);
563
564    if source.get("type").and_then(Value::as_str) == Some("redacted") {
565        if capture_uri.is_some() || content_hash.is_some() {
566            result.checks.push(check(
567                Outcome::Unverifiable,
568                CheckKind::Capture,
569                source_path(index),
570                "redacted source",
571            ));
572        }
573        return;
574    }
575
576    let hash_check =
577        content_hash.map(|hash| parse_sri(CheckKind::Capture, content_hash_path.clone(), hash));
578    let Some(capture_uri) = capture_uri else {
579        if let Some(hash_check) = hash_check {
580            match hash_check {
581                Ok(_) => result.checks.push(check(
582                    Outcome::Unverifiable,
583                    CheckKind::Capture,
584                    content_hash_path,
585                    "missing capture_uri",
586                )),
587                Err(check) => result.checks.push(check),
588            }
589        }
590        return;
591    };
592
593    let resolved = fetch_capture(index, descriptor, capture_uri, resolver).await;
594    if let CaptureFetch::Resolved(resolved_capture) = &resolved
595        && let Some(source_id) = source.get("id").and_then(Value::as_str)
596        && id_kind(source_id) == Some(EntityKind::Source)
597    {
598        result
599            .payloads
600            .insert(normalize_id(source_id), resolved_capture.payload.clone());
601    }
602
603    if let Some(hash_check) = hash_check {
604        match hash_check {
605            Ok(expected) => match &resolved {
606                CaptureFetch::Resolved(resolved_capture) => {
607                    let comparison = compare_sri(
608                        CheckKind::Capture,
609                        content_hash_path,
610                        &resolved_capture.payload,
611                        &expected,
612                    );
613                    if comparison.outcome == Outcome::Failed
614                        && let Some(source_id) = source.get("id").and_then(Value::as_str)
615                        && id_kind(source_id) == Some(EntityKind::Source)
616                    {
617                        result.hash_failed.insert(normalize_id(source_id));
618                    }
619                    result.checks.push(comparison);
620                }
621                CaptureFetch::Unfetched(error) => result.checks.push(check(
622                    Outcome::Unverifiable,
623                    CheckKind::Capture,
624                    content_hash_path,
625                    error.error.clone(),
626                )),
627            },
628            Err(warning) => {
629                result.checks.push(warning);
630                if let CaptureFetch::Unfetched(error) = resolved {
631                    result.checks.push(check(
632                        Outcome::Unverifiable,
633                        CheckKind::Capture,
634                        source_capture_uri_path(index),
635                        error.error,
636                    ));
637                }
638            }
639        }
640    } else if let CaptureFetch::Unfetched(error) = resolved {
641        result.checks.push(check(
642            Outcome::Unverifiable,
643            CheckKind::Capture,
644            source_capture_uri_path(index),
645            error.error,
646        ));
647    }
648}
649
650async fn fetch_capture(
651    index: usize,
652    descriptor: &Map<String, Value>,
653    reference: &str,
654    resolver: &dyn Resolver,
655) -> CaptureFetch {
656    let uri = match effective_reference(descriptor, reference, resolver.screens_local_paths()) {
657        Ok(uri) => uri,
658        Err(error) => {
659            return CaptureFetch::Unfetched(UnfetchedContent {
660                index,
661                error: error.to_string(),
662            });
663        }
664    };
665    match resolver.fetch(&uri).await {
666        Ok(payload) => CaptureFetch::Resolved(ResolvedCapture { payload }),
667        Err(error) => CaptureFetch::Unfetched(UnfetchedContent {
668            index,
669            error: error.to_string(),
670        }),
671    }
672}
673
674async fn sidecar_checks(
675    graph: &Graph<'_>,
676    descriptor: &Map<String, Value>,
677    resolver: &dyn Resolver,
678) -> SidecarResult {
679    let mut result = SidecarResult::default();
680    for (index, section) in &graph.sections {
681        let Some(reference) = section.get("provenance_uri").and_then(Value::as_str) else {
682            continue;
683        };
684        let resolved = fetch_section(*index, section, descriptor, reference, resolver).await;
685        if let Some(hash) = section.get("provenance_hash").and_then(Value::as_str) {
686            let path = section_provenance_hash_path(*index);
687            match parse_sri(CheckKind::Sidecar, path.clone(), hash) {
688                Ok(expected) => {
689                    result
690                        .checks
691                        .push(check_sri(CheckKind::Sidecar, path, &resolved, &expected));
692                }
693                Err(check) => result.checks.push(check),
694            }
695        }
696        let SectionFetch::Resolved(resolved) = resolved else {
697            if let SectionFetch::Unfetched(unfetched) = resolved {
698                result.checks.push(check(
699                    Outcome::Unverifiable,
700                    CheckKind::Sidecar,
701                    section_provenance_uri_path(unfetched.index),
702                    unfetched.error,
703                ));
704            }
705            continue;
706        };
707        let sidecar = match parse_sidecar(&resolved.payload) {
708            Ok(sidecar) => sidecar,
709            Err(error) => {
710                result.checks.push(check(
711                    Outcome::Failed,
712                    CheckKind::Sidecar,
713                    section_provenance_uri_path(*index),
714                    error,
715                ));
716                continue;
717            }
718        };
719        let (findings, binding_mismatch) = sidecar_findings(graph, *index, section, &sidecar);
720        // Match the reference validator: a failed provenance_hash remains a
721        // failed Sidecar check, but the fetched sidecar still contributes
722        // quote claims so callers can see every observable content result.
723        result.claims.extend(sidecar_quote_claims(*index, &sidecar));
724        let detail = sidecar_detail(section, &sidecar, binding_mismatch);
725        result.checks.push(ContentCheck {
726            kind: CheckKind::Sidecar,
727            path: section_provenance_uri_path(*index),
728            outcome: if findings.is_empty() && !binding_mismatch {
729                Outcome::Verified
730            } else {
731                Outcome::Failed
732            },
733            detail,
734            findings,
735            warnings: Vec::new(),
736        });
737    }
738    result
739}
740
741fn parse_sidecar(payload: &[u8]) -> Result<Value, String> {
742    let text = String::from_utf8(payload.to_vec()).map_err(utf8_error_detail)?;
743    serde_json::from_str(&text).map_err(|error| error.to_string())
744}
745
746fn sidecar_findings(
747    graph: &Graph<'_>,
748    section_index: usize,
749    section: &Object,
750    sidecar: &Value,
751) -> (Vec<Finding>, bool) {
752    let mut findings: Vec<_> = provenance_schema_findings(sidecar)
753        .into_iter()
754        .map(|finding| Finding {
755            code: finding.code,
756            path: format!(
757                "{}{}",
758                section_provenance_uri_path(section_index),
759                finding.path
760            ),
761            message: finding.message,
762        })
763        .collect();
764
765    let Some(sidecar_object) = sidecar.as_object() else {
766        findings.sort();
767        return (findings, false);
768    };
769
770    let section_id = sidecar_object.get("section_id").and_then(Value::as_str);
771    let mut binding_mismatch = false;
772    if let Some(section_id) = section_id {
773        if let Some(code) = reference_code_id(section_id, EntityKind::Section, &graph.entities) {
774            findings.push(Finding {
775                code,
776                path: pointer([
777                    PathSegment::Key("sections"),
778                    PathSegment::Index(section_index),
779                    PathSegment::Key("provenance_uri"),
780                    PathSegment::Key("section_id"),
781                ]),
782                message: format!("sidecar section_id {section_id:?} does not resolve to a section"),
783            });
784        } else if section
785            .get("id")
786            .and_then(Value::as_str)
787            .is_none_or(|id| normalize_id(id) != normalize_id(section_id))
788        {
789            binding_mismatch = true;
790        }
791    }
792
793    for (claim_index, claim) in indexed_objects(sidecar_object.get("claims")) {
794        append_sidecar_source_findings(&mut findings, graph, section_index, claim_index, claim);
795    }
796    findings.sort();
797    (findings, binding_mismatch)
798}
799
800fn append_sidecar_source_findings(
801    findings: &mut Vec<Finding>,
802    graph: &Graph<'_>,
803    section_index: usize,
804    claim_index: usize,
805    claim: &Object,
806) {
807    let Some(source_ids) = claim.get("source_ids").and_then(Value::as_array) else {
808        return;
809    };
810    for (source_index, source_id) in source_ids.iter().enumerate() {
811        let Some(source_id) = source_id.as_str() else {
812            continue;
813        };
814        if let Some(code) = reference_code_id(source_id, EntityKind::Source, &graph.entities) {
815            findings.push(Finding {
816                code,
817                path: pointer(sidecar_claim_source_path(
818                    section_index,
819                    claim_index,
820                    source_index,
821                )),
822                message: format!(
823                    "sidecar claim source id {source_id:?} does not resolve to a source"
824                ),
825            });
826        }
827    }
828
829    for source_index in casefolded_duplicate_indices(source_ids) {
830        let source_id = source_ids[source_index].as_str().unwrap_or_default();
831        findings.push(Finding {
832            code: Code::Akb011,
833            path: pointer(sidecar_claim_source_path(
834                section_index,
835                claim_index,
836                source_index,
837            )),
838            message: format!(
839                "sidecar claim source id {source_id:?} duplicates another entry in this array (case-insensitive)"
840            ),
841        });
842    }
843}
844
845fn sidecar_claim_source_path(
846    section_index: usize,
847    claim_index: usize,
848    source_index: usize,
849) -> [PathSegment; 7] {
850    [
851        PathSegment::Key("sections"),
852        PathSegment::Index(section_index),
853        PathSegment::Key("provenance_uri"),
854        PathSegment::Key("claims"),
855        PathSegment::Index(claim_index),
856        PathSegment::Key("source_ids"),
857        PathSegment::Index(source_index),
858    ]
859}
860
861fn sidecar_detail(section: &Object, sidecar: &Value, binding_mismatch: bool) -> String {
862    if !binding_mismatch {
863        return "provenance sidecar checked".to_owned();
864    }
865    let section_id = sidecar
866        .as_object()
867        .and_then(|object| object.get("section_id"))
868        .and_then(Value::as_str);
869    let expected = section.get("id").and_then(Value::as_str);
870    format!("sidecar section_id {section_id:?} names a different section; expected {expected:?}")
871}
872
873fn inline_quote_claims(graph: &Graph<'_>) -> Vec<QuoteClaim> {
874    let mut claims = Vec::new();
875    for (section_index, section) in &graph.sections {
876        for (claim_index, claim) in indexed_objects(section.get("provenance")) {
877            if let Some(claim) = quote_claim(
878                vec![
879                    PathSegment::Key("sections"),
880                    PathSegment::Index(*section_index),
881                    PathSegment::Key("provenance"),
882                    PathSegment::Index(claim_index),
883                ],
884                claim,
885            ) {
886                claims.push(claim);
887            }
888        }
889    }
890    claims
891}
892
893fn sidecar_quote_claims(section_index: usize, sidecar: &Value) -> Vec<QuoteClaim> {
894    let Some(sidecar_object) = sidecar.as_object() else {
895        return Vec::new();
896    };
897    let mut claims = Vec::new();
898    for (claim_index, claim) in indexed_objects(sidecar_object.get("claims")) {
899        if let Some(claim) = quote_claim(
900            vec![
901                PathSegment::Key("sections"),
902                PathSegment::Index(section_index),
903                PathSegment::Key("provenance_uri"),
904                PathSegment::Key("claims"),
905                PathSegment::Index(claim_index),
906            ],
907            claim,
908        ) {
909            claims.push(claim);
910        }
911    }
912    claims
913}
914
915fn quote_claim(path: Vec<PathSegment>, claim: &Object) -> Option<QuoteClaim> {
916    let quote = claim
917        .get("locator")
918        .and_then(Value::as_object)
919        .and_then(|locator| locator.get("quote"))
920        .and_then(Value::as_str)?;
921    if quote.is_empty() {
922        return None;
923    }
924    let source_ids = claim
925        .get("source_ids")
926        .and_then(Value::as_array)?
927        .iter()
928        .filter_map(Value::as_str)
929        .map(ToOwned::to_owned)
930        .collect::<Vec<_>>();
931    if source_ids.is_empty() {
932        return None;
933    }
934    Some(QuoteClaim {
935        path,
936        quote: quote.to_owned(),
937        source_ids,
938    })
939}
940
941fn quote_checks(
942    graph: &Graph<'_>,
943    captures: &CaptureResult,
944    quote_claims: &[QuoteClaim],
945) -> Vec<ContentCheck> {
946    quote_claims
947        .iter()
948        .map(|claim| {
949            let (outcome, detail) = quote_outcome(claim, captures);
950            let mut path = claim.path.clone();
951            path.extend([PathSegment::Key("locator"), PathSegment::Key("quote")]);
952            let path = pointer(path);
953            ContentCheck {
954                kind: CheckKind::Quote,
955                path: path.clone(),
956                outcome,
957                detail,
958                findings: Vec::new(),
959                warnings: redacted_warnings(graph, &path, &claim.source_ids),
960            }
961        })
962        .collect()
963}
964
965fn quote_outcome(claim: &QuoteClaim, captures: &CaptureResult) -> (Outcome, String) {
966    let keys: Vec<_> = claim
967        .source_ids
968        .iter()
969        .map(|source_id| normalize_id(source_id))
970        .collect();
971    let usable: Vec<_> = keys
972        .iter()
973        .filter(|key| !captures.hash_failed.contains(*key))
974        .filter_map(|key| captures.payloads.get(key))
975        .collect();
976    let needle = claim.quote.as_bytes();
977    if usable
978        .iter()
979        .any(|payload| memchr::memmem::find(payload, needle).is_some())
980    {
981        return (Outcome::Verified, "quote found in capture".to_owned());
982    }
983    if keys
984        .iter()
985        .all(|key| captures.payloads.contains_key(key) && !captures.hash_failed.contains(key))
986    {
987        return (
988            Outcome::Failed,
989            "quote absent from fetched captures".to_owned(),
990        );
991    }
992    if usable.is_empty() {
993        if keys.iter().any(|key| captures.hash_failed.contains(key)) {
994            return (
995                Outcome::Unverifiable,
996                "a cited source's capture failed its content_hash".to_owned(),
997            );
998        }
999        return (
1000            Outcome::Unverifiable,
1001            "no cited source capture fetched".to_owned(),
1002        );
1003    }
1004    if keys.iter().any(|key| captures.hash_failed.contains(key)) {
1005        return (
1006            Outcome::Unverifiable,
1007            "a cited source's capture failed its content_hash".to_owned(),
1008        );
1009    }
1010    (
1011        Outcome::Unverifiable,
1012        "some cited source captures were not fetched".to_owned(),
1013    )
1014}
1015
1016fn redacted_warnings(graph: &Graph<'_>, path: &str, source_ids: &[String]) -> Vec<Advisory> {
1017    let redacted: Vec<_> = source_ids
1018        .iter()
1019        .filter(|source_id| {
1020            graph.sources.iter().any(|(_, source)| {
1021                source
1022                    .get("id")
1023                    .and_then(Value::as_str)
1024                    .is_some_and(|id| normalize_id(id) == normalize_id(source_id))
1025                    && source.get("type").and_then(Value::as_str) == Some("redacted")
1026            })
1027        })
1028        .collect::<BTreeSet<_>>()
1029        .into_iter()
1030        .cloned()
1031        .collect();
1032    if redacted.is_empty() {
1033        return Vec::new();
1034    }
1035    vec![Advisory {
1036        path: path.to_owned(),
1037        message: format!("quote cites redacted source(s): {}", redacted.join(", ")),
1038    }]
1039}
1040
1041fn check_sri(
1042    kind: CheckKind,
1043    path: String,
1044    resolved: &SectionFetch<'_>,
1045    expected: &[u8],
1046) -> ContentCheck {
1047    match resolved {
1048        SectionFetch::Resolved(resolved) => compare_sri(kind, path, &resolved.payload, expected),
1049        SectionFetch::Unfetched(unfetched) => {
1050            check(Outcome::Unverifiable, kind, path, unfetched.error.clone())
1051        }
1052    }
1053}
1054
1055fn parse_sri(kind: CheckKind, path: String, sri: &str) -> Result<Vec<u8>, ContentCheck> {
1056    let Some((algorithm, encoded)) = sri.split_once('-') else {
1057        return Err(warning_check(
1058            kind,
1059            path,
1060            "malformed SRI: expected '<algorithm>-<base64>'",
1061        ));
1062    };
1063    if algorithm != SHA256_ALGORITHM {
1064        return Err(warning_check(
1065            kind,
1066            path,
1067            format!("unsupported hash algorithm: {algorithm}"),
1068        ));
1069    }
1070    let Ok(decoded) = STRICT_STANDARD_BASE64.decode(encoded.as_bytes()) else {
1071        return Err(warning_check(kind, path, "malformed sha256 digest"));
1072    };
1073    if decoded.len() != SHA256_LENGTH {
1074        return Err(warning_check(kind, path, "sha256 digest has wrong length"));
1075    }
1076    if STANDARD.encode(&decoded) != encoded {
1077        return Err(warning_check(kind, path, "non-canonical sha256 digest"));
1078    }
1079    Ok(decoded)
1080}
1081
1082fn compare_sri(kind: CheckKind, path: String, payload: &[u8], expected: &[u8]) -> ContentCheck {
1083    let actual = Sha256::digest(payload);
1084    if actual.as_slice() == expected {
1085        check(Outcome::Verified, kind, path, "sha256 digest matches")
1086    } else {
1087        check(Outcome::Failed, kind, path, "sha256 digest mismatch")
1088    }
1089}
1090
1091fn check(
1092    outcome: Outcome,
1093    kind: CheckKind,
1094    path: String,
1095    detail: impl Into<String>,
1096) -> ContentCheck {
1097    ContentCheck {
1098        kind,
1099        path,
1100        outcome,
1101        detail: detail.into(),
1102        findings: Vec::new(),
1103        warnings: Vec::new(),
1104    }
1105}
1106
1107fn warning_check(kind: CheckKind, path: String, detail: impl Into<String>) -> ContentCheck {
1108    let detail = detail.into();
1109    ContentCheck {
1110        kind,
1111        path: path.clone(),
1112        outcome: Outcome::Unverifiable,
1113        detail: detail.clone(),
1114        findings: Vec::new(),
1115        warnings: vec![Advisory {
1116            path,
1117            message: detail,
1118        }],
1119    }
1120}
1121
1122fn effective_reference(
1123    descriptor: &Map<String, Value>,
1124    reference: &str,
1125    screen_local_paths: bool,
1126) -> Result<String, Unfetchable> {
1127    if screen_local_paths {
1128        local_raw_reference_error(reference)?;
1129        if let Some(base_uri) = descriptor.get("base_uri").and_then(Value::as_str) {
1130            local_raw_reference_error(base_uri)?;
1131        }
1132    }
1133    let joined = descriptor
1134        .get("base_uri")
1135        .and_then(Value::as_str)
1136        .map_or_else(
1137            || reference.to_owned(),
1138            |base_uri| join_reference(base_uri, reference),
1139        );
1140    if screen_local_paths {
1141        local_raw_reference_error(&joined)?;
1142    }
1143    Ok(strip_fragment(&joined).to_owned())
1144}
1145
1146fn local_raw_reference_error(reference: &str) -> Result<(), Unfetchable> {
1147    let raw_reference = strip_fragment(reference);
1148    let raw_path = raw_reference
1149        .split_once('?')
1150        .map_or(raw_reference, |(path, _)| path);
1151    if raw_reference.contains('?')
1152        || raw_path.contains(';')
1153        || raw_path.contains('\\')
1154        || raw_path.split('/').any(|segment| segment == "..")
1155    {
1156        return Err(Unfetchable::outside_local_base(reference));
1157    }
1158    Ok(())
1159}
1160
1161fn join_reference(base_uri: &str, reference: &str) -> String {
1162    let Ok(reference) = UriRef::parse(reference) else {
1163        return reference.to_owned();
1164    };
1165    let base_without_fragment = strip_fragment(base_uri);
1166    if let Ok(base) = Uri::parse(base_without_fragment) {
1167        return reference
1168            .resolve_against(&base)
1169            .map(|uri| uri.to_string())
1170            .unwrap_or_else(|_| reference.to_string());
1171    }
1172    join_relative_reference(base_without_fragment, &reference)
1173}
1174
1175fn join_relative_reference(base_uri: &str, reference: &UriRef<&str>) -> String {
1176    if reference.scheme().is_some() || reference.authority().is_some() {
1177        return reference.to_string();
1178    }
1179    let (base_path, base_query) = split_query(base_uri);
1180    let reference_path = reference.path().as_str();
1181    let path = if reference_path.is_empty() {
1182        base_path.to_owned()
1183    } else if reference_path.starts_with('/') {
1184        remove_dot_segments(reference_path)
1185    } else {
1186        let directory = base_path
1187            .rfind('/')
1188            .map_or("", |index| &base_path[..=index]);
1189        remove_dot_segments(&format!("{directory}{reference_path}"))
1190    };
1191    let query = if reference_path.is_empty() {
1192        reference
1193            .query()
1194            .map_or(base_query, |query| Some(query.as_str()))
1195    } else {
1196        reference.query().map(|query| query.as_str())
1197    };
1198    match query {
1199        Some(query) => format!("{path}?{query}"),
1200        None => path,
1201    }
1202}
1203
1204fn split_query(reference: &str) -> (&str, Option<&str>) {
1205    reference
1206        .split_once('?')
1207        .map_or((reference, None), |(path, query)| (path, Some(query)))
1208}
1209
1210fn remove_dot_segments(path: &str) -> String {
1211    let mut output = Vec::new();
1212    let absolute = path.starts_with('/');
1213    let trailing_slash = path.ends_with('/') || path.ends_with("/.") || path.ends_with("/..");
1214    for segment in path.split('/') {
1215        match segment {
1216            "" | "." => {}
1217            ".." => {
1218                output.pop();
1219            }
1220            value => output.push(value),
1221        }
1222    }
1223    let mut normalized = String::new();
1224    if absolute {
1225        normalized.push('/');
1226    }
1227    normalized.push_str(&output.join("/"));
1228    if trailing_slash && !normalized.ends_with('/') {
1229        normalized.push('/');
1230    }
1231    if normalized.is_empty() && absolute {
1232        normalized.push('/');
1233    }
1234    normalized
1235}
1236
1237fn rejects_local_reference(raw_reference: &str, raw_path: &str) -> bool {
1238    raw_reference.contains('?')
1239        || raw_path.contains(';')
1240        || raw_path.contains('\\')
1241        || raw_path.split('/').any(|segment| segment == "..")
1242        || is_absolute_path(raw_path)
1243        || raw_reference.starts_with("//")
1244        || has_uri_scheme(raw_reference)
1245}
1246
1247fn is_absolute_path(raw_path: &str) -> bool {
1248    raw_path.starts_with('/')
1249        || Path::new(raw_path)
1250            .components()
1251            .next()
1252            .is_some_and(|component| matches!(component, Component::Prefix(_) | Component::RootDir))
1253}
1254
1255fn has_uri_scheme(reference: &str) -> bool {
1256    let Some((scheme, _)) = reference.split_once(':') else {
1257        return false;
1258    };
1259    if scheme.contains('/') || scheme.contains('?') {
1260        return false;
1261    }
1262    let mut bytes = scheme.bytes();
1263    let Some(first) = bytes.next() else {
1264        return false;
1265    };
1266    first.is_ascii_alphabetic()
1267        && bytes.all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'+' | b'-' | b'.'))
1268}
1269
1270fn strip_fragment(uri: &str) -> &str {
1271    uri.split_once('#')
1272        .map_or(uri, |(without_fragment, _)| without_fragment)
1273}
1274
1275fn guide_hash_path() -> String {
1276    json_pointer([Segment::Key("guide_hash")])
1277}
1278
1279fn source_path(index: usize) -> String {
1280    json_pointer([Segment::Key("sources"), Segment::Index(index)])
1281}
1282
1283fn source_capture_uri_path(index: usize) -> String {
1284    json_pointer([
1285        Segment::Key("sources"),
1286        Segment::Index(index),
1287        Segment::Key("capture_uri"),
1288    ])
1289}
1290
1291fn source_content_hash_path(index: usize) -> String {
1292    json_pointer([
1293        Segment::Key("sources"),
1294        Segment::Index(index),
1295        Segment::Key("content_hash"),
1296    ])
1297}
1298
1299fn section_content_uri_path(index: usize) -> String {
1300    json_pointer([
1301        Segment::Key("sections"),
1302        Segment::Index(index),
1303        Segment::Key("content_uri"),
1304    ])
1305}
1306
1307fn section_hash_path(index: usize) -> String {
1308    json_pointer([
1309        Segment::Key("sections"),
1310        Segment::Index(index),
1311        Segment::Key("content_hash"),
1312    ])
1313}
1314
1315fn section_provenance_uri_path(index: usize) -> String {
1316    json_pointer([
1317        Segment::Key("sections"),
1318        Segment::Index(index),
1319        Segment::Key("provenance_uri"),
1320    ])
1321}
1322
1323fn section_provenance_hash_path(index: usize) -> String {
1324    json_pointer([
1325        Segment::Key("sections"),
1326        Segment::Index(index),
1327        Segment::Key("provenance_hash"),
1328    ])
1329}
1330
1331fn pointer(segments: impl IntoIterator<Item = PathSegment>) -> String {
1332    json_pointer(segments.into_iter().map(|segment| match segment {
1333        PathSegment::Key(key) => Segment::Key(key),
1334        PathSegment::Index(index) => Segment::Index(index),
1335    }))
1336}
1337
1338fn is_markdown(content_type: &Value) -> bool {
1339    content_type.as_str().is_some_and(|value| {
1340        value
1341            .split_once(';')
1342            .map_or(value, |(essence, _)| essence)
1343            .trim()
1344            .eq_ignore_ascii_case("text/markdown")
1345    })
1346}
1347
1348fn utf8_error_detail(error: FromUtf8Error) -> String {
1349    error.utf8_error().to_string()
1350}
1351
1352fn ids(items: &[(usize, &Object)]) -> BTreeSet<String> {
1353    items
1354        .iter()
1355        .filter_map(|(_, item)| Some(normalize_id(typed_id_value(item.get("id"))?)))
1356        .collect()
1357}