knowledge_base_validation/
lib.rs

1mod diagnostic;
2
3pub use diagnostic::{Diagnostic, ValidationLayer};
4
5use chrono::{DateTime, NaiveDate};
6use knowledge_base_models::{Cardinality, Entity, EntityId, EntityType, EntityTypeId, IdAllocation, LocalizedMap, Property, PropertyId, Reference, ReferenceId, Value, ValueType};
7use language_tags::LanguageTag;
8use pulldown_cmark::{Event, Options, Parser, Tag};
9use regex::Regex;
10use serde::de::DeserializeOwned;
11use std::collections::{BTreeMap, BTreeSet};
12use std::fs;
13use std::path::{Path, PathBuf};
14use std::sync::LazyLock;
15use url::Url;
16
17static DECIMAL: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"^-?(0|[1-9][0-9]*)(\.[0-9]+)?$").expect("valid regex"));
18static DATE: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"^[0-9]{4}-[0-9]{2}-[0-9]{2}$").expect("valid regex"));
19
20struct Loaded<T> {
21    path: PathBuf,
22    value: T,
23}
24
25struct ContextDocument {
26    path: PathBuf,
27    entity_id: EntityId,
28    source: String,
29}
30
31type Diagnostics = Vec<Diagnostic>;
32
33pub fn validate_repository(root: impl AsRef<Path>) -> Vec<Diagnostic> {
34    let root = root.as_ref();
35    let mut report = Vec::new();
36
37    if !root.is_dir() {
38        push(
39            &mut report,
40            ValidationLayer::Schema,
41            PathBuf::from("."),
42            None,
43            None,
44            format!("knowledge-base root is not a readable directory: {}", root.display()),
45        );
46        sort_diagnostics(&mut report);
47        return report;
48    }
49
50    let entities = load_yaml_directory::<Entity>(root, "entities", &mut report);
51    let entity_types = load_yaml_directory::<EntityType>(root, "entity_types", &mut report);
52    let properties = load_yaml_directory::<Property>(root, "properties", &mut report);
53    let references = load_yaml_directory::<Reference>(root, "references", &mut report);
54    let allocation = load_yaml_file::<IdAllocation>(root, Path::new("id_allocation.yaml"), &mut report);
55    let contexts = load_contexts(root, &mut report);
56
57    let entity_index = build_index(&entities, |item| item.id.clone(), "entity", &mut report);
58    let type_index = build_index(&entity_types, |item| item.id.clone(), "entity type", &mut report);
59    let property_index = build_index(&properties, |item| item.id.clone(), "property", &mut report);
60    let reference_index = build_index(&references, |item| item.id.clone(), "reference", &mut report);
61
62    validate_filenames(&entities, "entity", |item| item.id.as_str(), &mut report);
63    validate_filenames(&entity_types, "entity type", |item| item.id.as_str(), &mut report);
64    validate_filenames(&properties, "property", |item| item.id.as_str(), &mut report);
65    validate_filenames(&references, "reference", |item| item.id.as_str(), &mut report);
66
67    validate_entity_types(&entity_types, &reference_index, &mut report);
68    validate_properties(&properties, &type_index, &property_index, &reference_index, &mut report);
69    validate_references(&references, &mut report);
70    validate_entities(&entities, &entity_index, &type_index, &property_index, &reference_index, &mut report);
71
72    if let Some(allocation) = allocation.as_ref() {
73        validate_allocation(
74            allocation,
75            entity_index.keys().map(EntityId::number).max(),
76            property_index.keys().map(PropertyId::number).max(),
77            reference_index.keys().map(ReferenceId::number).max(),
78            type_index.keys().map(EntityTypeId::number).max(),
79            &mut report,
80        );
81    }
82
83    validate_contexts(&contexts, &entity_index, &reference_index, &mut report);
84    sort_diagnostics(&mut report);
85    report
86}
87
88fn load_yaml_directory<T: DeserializeOwned>(root: &Path, directory: &str, report: &mut Diagnostics) -> Vec<Loaded<T>> {
89    let relative = PathBuf::from(directory);
90    let path = root.join(&relative);
91    let entries = match fs::read_dir(&path) {
92        Ok(entries) => entries,
93        Err(error) => {
94            push(report, ValidationLayer::Schema, relative, None, None, format!("required directory cannot be read: {error}"));
95            return Vec::new();
96        }
97    };
98
99    let mut paths = Vec::new();
100    for entry in entries {
101        match entry {
102            Ok(entry) => paths.push(entry.path()),
103            Err(error) => push(
104                report,
105                ValidationLayer::Schema,
106                PathBuf::from(directory),
107                None,
108                None,
109                format!("directory entry cannot be read: {error}"),
110            ),
111        }
112    }
113    paths.sort();
114
115    let mut loaded = Vec::new();
116    for path in paths {
117        let relative_path = relative_path(root, &path);
118        let is_yaml_file = path.is_file() && path.extension().and_then(|extension| extension.to_str()) == Some("yaml");
119        if !is_yaml_file {
120            push(
121                report,
122                ValidationLayer::Schema,
123                relative_path,
124                None,
125                None,
126                "unexpected entry; managed directories may contain only .yaml files",
127            );
128            continue;
129        }
130        if let Some(value) = load_yaml_at::<T>(&path, relative_path.clone(), report) {
131            loaded.push(Loaded { path: relative_path, value });
132        }
133    }
134    loaded
135}
136
137fn load_yaml_file<T: DeserializeOwned>(root: &Path, relative: &Path, report: &mut Diagnostics) -> Option<Loaded<T>> {
138    let value = load_yaml_at::<T>(&root.join(relative), relative.to_path_buf(), report)?;
139    Some(Loaded {
140        path: relative.to_path_buf(),
141        value,
142    })
143}
144
145fn load_yaml_at<T: DeserializeOwned>(path: &Path, relative: PathBuf, report: &mut Diagnostics) -> Option<T> {
146    let source = match fs::read_to_string(path) {
147        Ok(source) => source,
148        Err(error) => {
149            push(report, ValidationLayer::Schema, relative, None, None, format!("file cannot be read: {error}"));
150            return None;
151        }
152    };
153
154    let value = match serde_yaml::from_str::<serde_yaml::Value>(&source) {
155        Ok(value) => value,
156        Err(error) => {
157            push(
158                report,
159                ValidationLayer::Schema,
160                relative,
161                error.location().map(|location| location.line()),
162                None,
163                format!("invalid YAML: {error}"),
164            );
165            return None;
166        }
167    };
168    match serde_yaml::from_value(value) {
169        Ok(value) => Some(value),
170        Err(error) => {
171            push(
172                report,
173                ValidationLayer::Schema,
174                relative,
175                error.location().map(|location| location.line()),
176                None,
177                format!("invalid file shape: {error}"),
178            );
179            None
180        }
181    }
182}
183
184fn load_contexts(root: &Path, report: &mut Diagnostics) -> Vec<ContextDocument> {
185    let directory = root.join("entity_context");
186    if !directory.exists() {
187        return Vec::new();
188    }
189    let entries = match fs::read_dir(&directory) {
190        Ok(entries) => entries,
191        Err(error) => {
192            push(
193                report,
194                ValidationLayer::Schema,
195                PathBuf::from("entity_context"),
196                None,
197                None,
198                format!("optional context directory cannot be read: {error}"),
199            );
200            return Vec::new();
201        }
202    };
203
204    let mut paths = entries.filter_map(Result::ok).map(|entry| entry.path()).collect::<Vec<_>>();
205    paths.sort();
206    let mut contexts = Vec::new();
207    for path in paths {
208        let relative = relative_path(root, &path);
209        if !path.is_file() || path.extension().and_then(|value| value.to_str()) != Some("md") {
210            push(
211                report,
212                ValidationLayer::Schema,
213                relative,
214                None,
215                None,
216                "unexpected entry; entity_context may contain only .md files",
217            );
218            continue;
219        }
220        let Some(stem) = path.file_stem().and_then(|value| value.to_str()) else {
221            push(report, ValidationLayer::Schema, relative, None, None, "context filename is not valid UTF-8");
222            continue;
223        };
224        let entity_id = match serde_yaml::from_value::<EntityId>(stem.into()) {
225            Ok(identifier) => identifier,
226            Err(_) => {
227                push(
228                    report,
229                    ValidationLayer::Schema,
230                    relative,
231                    None,
232                    Some(stem.to_owned()),
233                    "context filename must be a canonical entity identifier",
234                );
235                continue;
236            }
237        };
238        match fs::read_to_string(&path) {
239            Ok(source) => contexts.push(ContextDocument {
240                path: relative,
241                entity_id,
242                source,
243            }),
244            Err(error) => push(
245                report,
246                ValidationLayer::Schema,
247                relative,
248                None,
249                Some(entity_id.to_string()),
250                format!("context document cannot be read: {error}"),
251            ),
252        }
253    }
254    contexts
255}
256
257fn build_index<'a, T, I, F>(items: &'a [Loaded<T>], identifier: F, kind: &str, report: &mut Diagnostics) -> BTreeMap<I, &'a Loaded<T>>
258where
259    I: Clone + Ord + ToString,
260    F: Fn(&T) -> I,
261{
262    let mut index = BTreeMap::new();
263    for item in items {
264        let id = identifier(&item.value);
265        if let Some(previous) = index.insert(id.clone(), item) {
266            push(
267                report,
268                ValidationLayer::Schema,
269                item.path.clone(),
270                None,
271                Some(id.to_string()),
272                format!("duplicate {kind} identifier; also declared in {}", previous.path.display()),
273            );
274        }
275    }
276    index
277}
278
279fn validate_filenames<T, F>(items: &[Loaded<T>], kind: &str, identifier: F, report: &mut Diagnostics)
280where
281    F: Fn(&T) -> &str,
282{
283    for item in items {
284        let stem = item.path.file_stem().and_then(|value| value.to_str());
285        let id = identifier(&item.value);
286        if stem != Some(id) {
287            push(
288                report,
289                ValidationLayer::Schema,
290                item.path.clone(),
291                None,
292                Some(id.to_owned()),
293                format!("{kind} filename must be exactly {id}.yaml"),
294            );
295        }
296    }
297}
298
299fn validate_entity_types(items: &[Loaded<EntityType>], references: &BTreeMap<ReferenceId, &Loaded<Reference>>, report: &mut Diagnostics) {
300    for item in items {
301        let id = item.value.id.to_string();
302        validate_localized_map(&item.path, &id, "labels", &item.value.labels, true, references, report);
303        validate_localized_map(&item.path, &id, "descriptions", &item.value.descriptions, false, references, report);
304    }
305}
306
307fn validate_properties(
308    items: &[Loaded<Property>],
309    types: &BTreeMap<EntityTypeId, &Loaded<EntityType>>,
310    properties: &BTreeMap<PropertyId, &Loaded<Property>>,
311    references: &BTreeMap<ReferenceId, &Loaded<Reference>>,
312    report: &mut Diagnostics,
313) {
314    for item in items {
315        let property = &item.value;
316        let id = property.id.to_string();
317        validate_localized_map(&item.path, &id, "labels", &property.labels, true, references, report);
318        validate_localized_map(&item.path, &id, "descriptions", &property.descriptions, false, references, report);
319
320        if property.subject_types.is_empty() {
321            schema(report, item, &id, "subject_types must not be empty");
322        }
323        for type_id in &property.subject_types {
324            if !types.contains_key(type_id) {
325                ontology(report, item, &id, format!("subject type {type_id} does not exist"));
326            }
327        }
328
329        match (&property.value_type, &property.target_types) {
330            (ValueType::Entity, Some(targets)) if targets.is_empty() => {
331                schema(report, item, &id, "target_types must not be empty");
332            }
333            (ValueType::Entity, None) => {
334                schema(report, item, &id, "entity-valued property requires target_types");
335            }
336            (ValueType::Entity, Some(_)) | (_, None) => {}
337            (_, Some(_)) => schema(report, item, &id, "target_types is allowed only for entity-valued properties"),
338        }
339        if let Some(targets) = &property.target_types {
340            for type_id in targets {
341                if !types.contains_key(type_id) {
342                    ontology(report, item, &id, format!("target type {type_id} does not exist"));
343                }
344            }
345        }
346        for qualifier in &property.allowed_qualifiers {
347            if !properties.contains_key(qualifier) {
348                ontology(report, item, &id, format!("allowed qualifier property {qualifier} does not exist"));
349            }
350        }
351    }
352}
353
354fn validate_references(items: &[Loaded<Reference>], report: &mut Diagnostics) {
355    for item in items {
356        let reference = &item.value;
357        let id = reference.id.to_string();
358        validate_url(&item.path, &id, "url", &reference.url, report);
359        if let Some(url) = &reference.archive_url {
360            validate_url(&item.path, &id, "archive_url", url, report);
361        }
362        if DateTime::parse_from_rfc3339(&reference.retrieved_at).is_err() {
363            schema(report, item, &id, "retrieved_at must be an RFC 3339 timestamp");
364        }
365    }
366}
367
368fn validate_entities(
369    items: &[Loaded<Entity>],
370    entities: &BTreeMap<EntityId, &Loaded<Entity>>,
371    types: &BTreeMap<EntityTypeId, &Loaded<EntityType>>,
372    properties: &BTreeMap<PropertyId, &Loaded<Property>>,
373    references: &BTreeMap<ReferenceId, &Loaded<Reference>>,
374    report: &mut Diagnostics,
375) {
376    for item in items {
377        let entity = &item.value;
378        let id = entity.id.to_string();
379        validate_localized_map(&item.path, &id, "labels", &entity.labels, true, references, report);
380        validate_localized_map(&item.path, &id, "descriptions", &entity.descriptions, false, references, report);
381
382        if entity.entity_types.is_empty() {
383            schema(report, item, &id, "entity_types must not be empty");
384        }
385        for classification in &entity.entity_types {
386            if !types.contains_key(&classification.value) {
387                ontology(report, item, &id, format!("classified entity type {} does not exist", classification.value));
388            }
389            validate_provenance(&item.path, &id, "classification", &classification.references, references, report);
390        }
391        for image in &entity.images {
392            validate_url(&item.path, &id, "image url", &image.url, report);
393            if let Some(url) = &image.attribution_url {
394                validate_url(&item.path, &id, "image attribution_url", url, report);
395            }
396            if image.attribution.trim().is_empty() {
397                schema(report, item, &id, "image attribution must not be empty");
398            }
399            validate_provenance(&item.path, &id, "image", &image.references, references, report);
400        }
401
402        let mut statement_ids = BTreeSet::new();
403        let mut property_counts = BTreeMap::<PropertyId, usize>::new();
404        for statement in &entity.statements {
405            if !statement_ids.insert(statement.id.clone()) {
406                ontology(report, item, &id, format!("statement identifier {} is duplicated", statement.id));
407            }
408            *property_counts.entry(statement.property.clone()).or_default() += 1;
409            validate_provenance(&item.path, &format!("{id}/{}", statement.id), "statement", &statement.references, references, report);
410            validate_value(&item.path, &format!("{id}/{}", statement.id), &statement.value, report);
411
412            let main_property = properties.get(&statement.property).map(|item| &item.value);
413            if let Some(main_property) = main_property {
414                validate_property_use(item, entity, &format!("{id}/{}", statement.id), main_property, &statement.value, entities, report);
415            } else {
416                ontology(report, item, &id, format!("statement property {} does not exist", statement.property));
417            }
418
419            for qualifier in &statement.qualifiers {
420                validate_value(&item.path, &format!("{id}/{}/{}", statement.id, qualifier.property), &qualifier.value, report);
421                if main_property.is_some_and(|property| !property.allowed_qualifiers.contains(&qualifier.property)) {
422                    ontology(
423                        report,
424                        item,
425                        &id,
426                        format!(
427                            "qualifier {} is not allowed by property {}",
428                            qualifier.property,
429                            main_property.expect("checked as present").id
430                        ),
431                    );
432                }
433                let Some(qualifier_property) = properties.get(&qualifier.property).map(|item| &item.value) else {
434                    ontology(report, item, &id, format!("qualifier property {} does not exist", qualifier.property));
435                    continue;
436                };
437                validate_property_use(
438                    item,
439                    entity,
440                    &format!("{id}/{}/{}", statement.id, qualifier.property),
441                    qualifier_property,
442                    &qualifier.value,
443                    entities,
444                    report,
445                );
446            }
447        }
448        for (property_id, count) in property_counts {
449            if count > 1 && properties.get(&property_id).is_some_and(|property| property.value.cardinality == Cardinality::One) {
450                ontology(report, item, &id, format!("property {property_id} has cardinality one but occurs {count} times"));
451            }
452        }
453    }
454}
455
456fn validate_property_use(
457    item: &Loaded<Entity>,
458    entity: &Entity,
459    owner: &str,
460    property: &Property,
461    value: &Value,
462    entities: &BTreeMap<EntityId, &Loaded<Entity>>,
463    report: &mut Diagnostics,
464) {
465    let entity_types = entity.entity_types.iter().map(|classification| &classification.value).collect::<BTreeSet<_>>();
466    if !property.subject_types.iter().any(|subject_type| entity_types.contains(subject_type)) {
467        ontology(report, item, owner, format!("property {} is not applicable to this entity", property.id));
468    }
469    if value.value_type() != property.value_type {
470        ontology(
471            report,
472            item,
473            owner,
474            format!(
475                "property {} requires {} values but statement uses {}",
476                property.id,
477                value_type_name(property.value_type),
478                value_type_name(value.value_type())
479            ),
480        );
481        return;
482    }
483    if let Value::Entity { value: target_id } = value {
484        let Some(target) = entities.get(target_id) else {
485            ontology(report, item, owner, format!("target entity {target_id} does not exist"));
486            return;
487        };
488        let target_types = target.value.entity_types.iter().map(|classification| &classification.value).collect::<BTreeSet<_>>();
489        if !property
490            .target_types
491            .as_ref()
492            .is_some_and(|allowed| allowed.iter().any(|type_id| target_types.contains(type_id)))
493        {
494            ontology(
495                report,
496                item,
497                owner,
498                format!("target entity {target_id} has none of property {}'s permitted target types", property.id),
499            );
500        }
501    }
502}
503
504fn validate_value(path: &Path, owner: &str, value: &Value, report: &mut Diagnostics) {
505    let message = match value {
506        Value::Decimal { value } if !DECIMAL.is_match(value) => Some("decimal value must use canonical quoted base-10 syntax"),
507        Value::Date { value } if !DATE.is_match(value) || NaiveDate::parse_from_str(value, "%Y-%m-%d").is_err() => Some("date value must be a real ISO 8601 calendar date"),
508        Value::Datetime { value } if DateTime::parse_from_rfc3339(value).is_err() => Some("datetime value must be an RFC 3339 timestamp"),
509        Value::Url { value } if Url::parse(value).is_err() => Some("url value must be an absolute URL"),
510        Value::Coordinate { latitude, longitude } => {
511            if !DECIMAL.is_match(latitude) || !within_absolute_bound(latitude, 90) {
512                Some("coordinate latitude must be canonical decimal text between -90 and 90")
513            } else if !DECIMAL.is_match(longitude) || !within_absolute_bound(longitude, 180) {
514                Some("coordinate longitude must be canonical decimal text between -180 and 180")
515            } else {
516                None
517            }
518        }
519        _ => None,
520    };
521    if let Some(message) = message {
522        push(report, ValidationLayer::Schema, path.to_path_buf(), None, Some(owner.to_owned()), message);
523    }
524}
525
526fn within_absolute_bound(value: &str, bound: u64) -> bool {
527    let unsigned = value.strip_prefix('-').unwrap_or(value);
528    let (integer, fraction) = unsigned.split_once('.').unwrap_or((unsigned, ""));
529    match integer.len().cmp(&bound.to_string().len()) {
530        std::cmp::Ordering::Less => true,
531        std::cmp::Ordering::Greater => false,
532        std::cmp::Ordering::Equal => match integer.parse::<u64>() {
533            Ok(integer) if integer < bound => true,
534            Ok(integer) if integer == bound => fraction.bytes().all(|byte| byte == b'0'),
535            _ => false,
536        },
537    }
538}
539
540fn value_type_name(value_type: ValueType) -> &'static str {
541    match value_type {
542        ValueType::Entity => "entity",
543        ValueType::String => "string",
544        ValueType::Integer => "integer",
545        ValueType::Decimal => "decimal",
546        ValueType::Boolean => "boolean",
547        ValueType::Date => "date",
548        ValueType::Datetime => "datetime",
549        ValueType::Url => "url",
550        ValueType::Coordinate => "coordinate",
551    }
552}
553
554fn validate_localized_map(
555    path: &Path,
556    owner: &str,
557    field: &str,
558    values: &LocalizedMap,
559    required: bool,
560    references: &BTreeMap<ReferenceId, &Loaded<Reference>>,
561    report: &mut Diagnostics,
562) {
563    if required && values.is_empty() {
564        push(
565            report,
566            ValidationLayer::Schema,
567            path.to_path_buf(),
568            None,
569            Some(owner.to_owned()),
570            format!("{field} must not be empty"),
571        );
572    }
573    let mut normalized = BTreeSet::new();
574    for (locale, value) in values {
575        if locale.parse::<LanguageTag>().is_err() {
576            push(
577                report,
578                ValidationLayer::Schema,
579                path.to_path_buf(),
580                None,
581                Some(owner.to_owned()),
582                format!("{field} locale {locale:?} is not a well-formed BCP 47 tag"),
583            );
584        }
585        if !normalized.insert(locale.to_ascii_lowercase()) {
586            push(
587                report,
588                ValidationLayer::Schema,
589                path.to_path_buf(),
590                None,
591                Some(owner.to_owned()),
592                format!("{field} contains locale {locale:?} more than once ignoring case"),
593            );
594        }
595        validate_provenance(path, owner, &format!("{field}.{locale}"), &value.references, references, report);
596    }
597}
598
599fn validate_provenance(path: &Path, owner: &str, field: &str, reference_ids: &[ReferenceId], references: &BTreeMap<ReferenceId, &Loaded<Reference>>, report: &mut Diagnostics) {
600    if reference_ids.is_empty() {
601        push(
602            report,
603            ValidationLayer::Schema,
604            path.to_path_buf(),
605            None,
606            Some(owner.to_owned()),
607            format!("{field} references must not be empty"),
608        );
609    }
610    for reference_id in reference_ids {
611        if !references.contains_key(reference_id) {
612            push(
613                report,
614                ValidationLayer::Provenance,
615                path.to_path_buf(),
616                None,
617                Some(owner.to_owned()),
618                format!("{field} cites missing reference {reference_id}"),
619            );
620        }
621    }
622}
623
624fn validate_url(path: &Path, owner: &str, field: &str, value: &str, report: &mut Diagnostics) {
625    if Url::parse(value).is_err() {
626        push(
627            report,
628            ValidationLayer::Schema,
629            path.to_path_buf(),
630            None,
631            Some(owner.to_owned()),
632            format!("{field} must be an absolute URL"),
633        );
634    }
635}
636
637fn validate_allocation(
638    allocation: &Loaded<IdAllocation>,
639    max_entity: Option<u64>,
640    max_property: Option<u64>,
641    max_reference: Option<u64>,
642    max_type: Option<u64>,
643    report: &mut Diagnostics,
644) {
645    if allocation.value.version != 1 {
646        schema(report, allocation, "id_allocation", "version must be 1");
647    }
648    for (field, next, maximum) in [
649        ("entity", allocation.value.next.entity, max_entity),
650        ("property", allocation.value.next.property, max_property),
651        ("reference", allocation.value.next.reference, max_reference),
652        ("entity_type", allocation.value.next.entity_type, max_type),
653    ] {
654        if next == 0 {
655            schema(report, allocation, "id_allocation", format!("next.{field} must be positive"));
656        } else if maximum.is_some_and(|maximum| next <= maximum) {
657            schema(
658                report,
659                allocation,
660                "id_allocation",
661                format!("next.{field} must be greater than the greatest used identifier number ({})", maximum.unwrap_or_default()),
662            );
663        }
664    }
665}
666
667#[derive(Default)]
668struct FootnoteDefinition {
669    line: usize,
670    targets: Vec<String>,
671}
672
673fn validate_contexts(
674    contexts: &[ContextDocument],
675    entities: &BTreeMap<EntityId, &Loaded<Entity>>,
676    references: &BTreeMap<ReferenceId, &Loaded<Reference>>,
677    report: &mut Diagnostics,
678) {
679    for context in contexts {
680        let owner = context.entity_id.to_string();
681        if !entities.contains_key(&context.entity_id) {
682            push(
683                report,
684                ValidationLayer::Provenance,
685                context.path.clone(),
686                None,
687                Some(owner.clone()),
688                "context document names an entity that does not exist",
689            );
690        }
691
692        let mut definitions = BTreeMap::<String, Vec<FootnoteDefinition>>::new();
693        let mut references_used = BTreeMap::<String, usize>::new();
694        let mut current_definition: Option<(String, FootnoteDefinition)> = None;
695        let options = Options::ENABLE_FOOTNOTES;
696        for (event, range) in Parser::new_ext(&context.source, options).into_offset_iter() {
697            let line = line_at(&context.source, range.start);
698            match event {
699                Event::Start(Tag::FootnoteDefinition(label)) => {
700                    current_definition = Some((label.to_string(), FootnoteDefinition { line, targets: Vec::new() }));
701                }
702                Event::End(pulldown_cmark::TagEnd::FootnoteDefinition) => {
703                    if let Some((label, definition)) = current_definition.take() {
704                        definitions.entry(label).or_default().push(definition);
705                    }
706                }
707                Event::Start(Tag::Link { dest_url, .. }) => {
708                    if let Some((_, definition)) = current_definition.as_mut() {
709                        definition.targets.push(dest_url.to_string());
710                    }
711                }
712                Event::FootnoteReference(label) => {
713                    references_used.entry(label.to_string()).or_insert(line);
714                }
715                _ => {}
716            }
717        }
718
719        for (label, line) in &references_used {
720            parse_reference_label(label, &context.path, *line, &owner, references, report);
721            match definitions.get(label).map(Vec::len).unwrap_or_default() {
722                0 => push(
723                    report,
724                    ValidationLayer::Provenance,
725                    context.path.clone(),
726                    Some(*line),
727                    Some(owner.clone()),
728                    format!("footnote {label:?} has no definition"),
729                ),
730                1 => {}
731                count => push(
732                    report,
733                    ValidationLayer::Provenance,
734                    context.path.clone(),
735                    Some(*line),
736                    Some(owner.clone()),
737                    format!("footnote {label:?} has {count} definitions"),
738                ),
739            }
740        }
741
742        for (label, entries) in definitions {
743            if entries.len() > 1 && !references_used.contains_key(&label) {
744                push(
745                    report,
746                    ValidationLayer::Provenance,
747                    context.path.clone(),
748                    Some(entries[0].line),
749                    Some(owner.clone()),
750                    format!("footnote {label:?} has {} definitions", entries.len()),
751                );
752            }
753            for definition in &entries {
754                let reference_id = parse_reference_label(&label, &context.path, definition.line, &owner, references, report);
755                if let Some(reference_id) = reference_id {
756                    let expected = format!("../references/{reference_id}.yaml");
757                    if definition.targets.as_slice() != [expected.as_str()] {
758                        push(
759                            report,
760                            ValidationLayer::Provenance,
761                            context.path.clone(),
762                            Some(definition.line),
763                            Some(owner.clone()),
764                            format!("footnote {label:?} must contain exactly one link to {expected}"),
765                        );
766                    }
767                }
768            }
769        }
770    }
771}
772
773fn parse_reference_label(
774    label: &str,
775    path: &Path,
776    line: usize,
777    owner: &str,
778    references: &BTreeMap<ReferenceId, &Loaded<Reference>>,
779    report: &mut Diagnostics,
780) -> Option<ReferenceId> {
781    let reference_id = match serde_yaml::from_value::<ReferenceId>(label.into()) {
782        Ok(reference_id) => reference_id,
783        Err(_) => {
784            push(
785                report,
786                ValidationLayer::Provenance,
787                path.to_path_buf(),
788                Some(line),
789                Some(owner.to_owned()),
790                format!("footnote label {label:?} is not a canonical reference identifier"),
791            );
792            return None;
793        }
794    };
795    if !references.contains_key(&reference_id) {
796        push(
797            report,
798            ValidationLayer::Provenance,
799            path.to_path_buf(),
800            Some(line),
801            Some(owner.to_owned()),
802            format!("footnote {label:?} cites a reference that does not exist"),
803        );
804    }
805    Some(reference_id)
806}
807
808fn schema<T>(report: &mut Diagnostics, item: &Loaded<T>, identifier: &str, message: impl Into<String>) {
809    push(report, ValidationLayer::Schema, item.path.clone(), None, Some(identifier.to_owned()), message);
810}
811
812fn ontology<T>(report: &mut Diagnostics, item: &Loaded<T>, identifier: &str, message: impl Into<String>) {
813    push(report, ValidationLayer::Ontology, item.path.clone(), None, Some(identifier.to_owned()), message);
814}
815
816fn push(report: &mut Diagnostics, layer: ValidationLayer, path: PathBuf, line: Option<usize>, identifier: Option<String>, message: impl Into<String>) {
817    report.push(Diagnostic {
818        layer,
819        path,
820        line,
821        identifier,
822        message: message.into(),
823    });
824}
825
826fn sort_diagnostics(diagnostics: &mut [Diagnostic]) {
827    diagnostics.sort_by(|left, right| {
828        (&left.path, left.line.unwrap_or(usize::MAX), &left.identifier, &left.message, left.layer).cmp(&(
829            &right.path,
830            right.line.unwrap_or(usize::MAX),
831            &right.identifier,
832            &right.message,
833            right.layer,
834        ))
835    });
836}
837
838fn relative_path(root: &Path, path: &Path) -> PathBuf {
839    path.strip_prefix(root).map(Path::to_path_buf).unwrap_or_else(|_| path.to_path_buf())
840}
841
842fn line_at(source: &str, offset: usize) -> usize {
843    source[..offset.min(source.len())].bytes().filter(|byte| *byte == b'\n').count() + 1
844}
knowledge_base_validation/lib.rs

knowledge_base_validation/
lib.rs
