citum_schema_style/locale/
mod.rs

1/*
2SPDX-License-Identifier: MIT OR Apache-2.0
3SPDX-FileCopyrightText: © 2023-2026 Bruce D'Arcus and Citum contributors
4*/
5
6//! Locale definitions for Citum.
7//!
8//! Locales provide language-specific terms, date formats, and punctuation rules
9//! for citation formatting.
10
11mod date_patterns;
12/// Locator text normalization.
13pub mod locator;
14/// Message evaluation for parameterized locale strings.
15pub mod message;
16mod message_ids;
17/// Raw locale types used during locale file parsing.
18pub mod raw;
19mod raw_conversion;
20mod sort;
21mod terms;
22/// Structured locale types used by the processor.
23pub mod types;
24mod vocab;
25
26use crate::citation::LocatorType;
27use crate::template::ContributorRole;
28pub use message::{MessageArgs, MessageEvaluator, Mf2MessageEvaluator};
29pub use raw::{RawLocale, RawTermValue};
30#[cfg(feature = "schema")]
31use schemars::JsonSchema;
32use serde::{Deserialize, Serialize};
33use std::collections::HashMap;
34use std::fmt;
35use std::sync::Arc;
36pub use terms::ArchiveHierarchyField;
37pub use types::*;
38
39/// A list of month names (12 elements for Jan-Dec).
40pub type MonthList = Vec<String>;
41
42/// A locale definition containing language-specific terms and formatting rules.
43///
44/// The `evaluator` field holds the message evaluation engine, selected based on
45/// `evaluation.message_syntax`. This allows for trait-based swapping to ICU4X
46/// implementations in the future without changing call sites.
47#[derive(Clone, Deserialize, Serialize)]
48#[cfg_attr(feature = "schema", derive(JsonSchema))]
49#[serde(rename_all = "kebab-case")]
50pub struct Locale {
51    /// The locale identifier (e.g., "en-US", "de-DE").
52    #[cfg_attr(feature = "schema", schemars(skip))]
53    pub locale: String,
54    /// Date-related terms (months, seasons).
55    #[serde(default)]
56    pub dates: DateTerms,
57    /// Contributor role terms (editor, translator, etc.).
58    #[serde(default)]
59    #[cfg_attr(feature = "schema", schemars(skip))]
60    pub roles: HashMap<ContributorRole, ContributorTerm>,
61    /// Authored terms for combinations such as `writer-director`.
62    #[serde(default, skip_serializing_if = "HashMap::is_empty")]
63    #[cfg_attr(feature = "schema", schemars(skip))]
64    pub role_combinations: HashMap<String, ContributorTerm>,
65    /// Locator terms (page, chapter, etc.).
66    #[serde(default)]
67    #[cfg_attr(feature = "schema", schemars(skip))]
68    pub locators: HashMap<LocatorType, LocatorTerm>,
69    /// General terms (and, et al., etc.).
70    #[serde(default)]
71    pub terms: Terms,
72    /// Whether to place periods/commas inside quotation marks.
73    /// true = American style ("text."), false = British style ("text".)
74    #[serde(default)]
75    pub punctuation_in_quote: bool,
76    /// Articles to strip from titles when sorting (e.g., "the", "a", "an" for English).
77    /// These should be lowercase and will be matched case-insensitively.
78    #[serde(default, skip_serializing_if = "Vec::is_empty")]
79    pub sort_articles: Vec<String>,
80    /// Schema version from the source locale file (None = legacy v1).
81    #[serde(default, skip_serializing_if = "Option::is_none")]
82    pub locale_schema_version: Option<String>,
83    /// Runtime evaluation configuration.
84    #[serde(default)]
85    pub evaluation: EvaluationConfig,
86    /// ICU MF1 messages keyed by message ID (populated for v2 locales).
87    #[serde(default, skip_serializing_if = "HashMap::is_empty")]
88    pub messages: HashMap<String, String>,
89    /// Named date format presets: symbolic name → CLDR pattern.
90    #[serde(default, skip_serializing_if = "HashMap::is_empty")]
91    pub date_formats: HashMap<String, String>,
92    /// Number formatting options.
93    #[serde(default)]
94    pub number_formats: NumberFormats,
95    /// Grammar options.
96    #[serde(default)]
97    pub grammar_options: GrammarOptions,
98    /// Backwards-compatibility aliases: old term key → new message ID.
99    #[serde(default, skip_serializing_if = "HashMap::is_empty")]
100    pub legacy_term_aliases: HashMap<String, String>,
101    /// Vocabulary maps for genre and medium display text.
102    #[serde(default, skip_serializing_if = "VocabMap::is_empty")]
103    pub vocab: VocabMap,
104    /// Reference-type description terms, keyed by CSL-style `ref_type`
105    /// spelling (e.g. `"dataset"`, `"article-journal"`). Used by the
106    /// `type-label` template component to resolve a localized fallback
107    /// label when a reference has no `genre`/`medium` override. See
108    /// `docs/specs/TYPE_CLASSIFICATION_CENTRALIZATION.md`.
109    #[serde(default, skip_serializing_if = "HashMap::is_empty")]
110    pub type_terms: HashMap<String, SimpleTerm>,
111    /// Message evaluator implementation (not serialized; set during load).
112    #[serde(skip, default = "default_evaluator")]
113    #[cfg_attr(feature = "schema", schemars(skip))]
114    pub evaluator: Arc<dyn MessageEvaluator>,
115}
116
117/// Default message evaluator (MF2).
118fn default_evaluator() -> Arc<dyn MessageEvaluator> {
119    Arc::new(Mf2MessageEvaluator)
120}
121
122impl Default for Locale {
123    fn default() -> Self {
124        Self {
125            locale: String::default(),
126            dates: DateTerms::default(),
127            roles: HashMap::default(),
128            role_combinations: HashMap::default(),
129            locators: HashMap::default(),
130            terms: Terms::default(),
131            punctuation_in_quote: false,
132            sort_articles: Vec::default(),
133            locale_schema_version: None,
134            evaluation: EvaluationConfig::default(),
135            messages: HashMap::default(),
136            date_formats: HashMap::default(),
137            number_formats: NumberFormats::default(),
138            grammar_options: GrammarOptions::default(),
139            legacy_term_aliases: HashMap::default(),
140            vocab: VocabMap::default(),
141            type_terms: HashMap::default(),
142            evaluator: default_evaluator(),
143        }
144    }
145}
146
147impl fmt::Debug for Locale {
148    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
149        f.debug_struct("Locale")
150            .field("locale", &self.locale)
151            .field("dates", &self.dates)
152            .field("roles", &self.roles)
153            .field("role_combinations", &self.role_combinations)
154            .field("locators", &self.locators)
155            .field("terms", &self.terms)
156            .field("punctuation_in_quote", &self.punctuation_in_quote)
157            .field("sort_articles", &self.sort_articles)
158            .field("locale_schema_version", &self.locale_schema_version)
159            .field("evaluation", &self.evaluation)
160            .field("messages", &self.messages)
161            .field("date_formats", &self.date_formats)
162            .field("number_formats", &self.number_formats)
163            .field("grammar_options", &self.grammar_options)
164            .field("legacy_term_aliases", &self.legacy_term_aliases)
165            .field("vocab", &self.vocab)
166            .field("type_terms", &self.type_terms)
167            .field("evaluator", &"<MessageEvaluator>")
168            .finish()
169    }
170}
171
172impl Locale {
173    /// Create the English (US) locale, the fallback baseline every other
174    /// locale inherits from and the default for the majority of embedded
175    /// styles (which declare no `info.default-locale`).
176    ///
177    /// This parses the embedded canonical asset
178    /// (`embedded/locales/en-US.yaml`) so the YAML is the single source of
179    /// truth — there is no separate hand-maintained Rust copy to drift out
180    /// of sync with it. The parse is memoized in a `std::sync::OnceLock`
181    /// since it is pure and immutable; callers get a `clone()` of the cached
182    /// result (still a deep copy of its maps/vecs, but far cheaper than
183    /// re-parsing the YAML) rather than re-parsing on every call.
184    ///
185    /// Seeds from [`Locale::default()`] (not `from_raw`'s usual
186    /// `Locale::en_us()` seed) via `from_raw_with_base` to avoid infinite
187    /// recursion through this very function.
188    ///
189    /// # Panics
190    ///
191    /// Panics if the embedded `en-US.yaml` asset is missing, not valid
192    /// UTF-8, or fails to parse. This cannot happen at runtime: the asset is
193    /// embedded at compile time and covered by
194    /// `bundled_ar_ar_and_eu_es_locales_are_embedded_and_parseable`-style
195    /// tests, so a failure here indicates a broken build, not bad input.
196    #[allow(
197        clippy::expect_used,
198        reason = "Embedded en-US.yaml locale must parse; failure indicates a broken build, not bad input"
199    )]
200    pub fn en_us() -> Self {
201        static EN_US: std::sync::OnceLock<Locale> = std::sync::OnceLock::new();
202        EN_US
203            .get_or_init(|| {
204                let bytes = crate::embedded::get_locale_bytes("en-US")
205                    .expect("en-US is a compile-time embedded locale");
206                let yaml = std::str::from_utf8(bytes).expect("embedded en-US.yaml is valid UTF-8");
207                let raw: RawLocale =
208                    serde_yaml::from_str(yaml).expect("embedded en-US.yaml parses");
209                Self::from_raw_with_base(raw, Locale::default())
210            })
211            .clone()
212    }
213}
214
215#[cfg(test)]
216#[allow(
217    clippy::unwrap_used,
218    clippy::expect_used,
219    clippy::panic,
220    clippy::indexing_slicing,
221    clippy::todo,
222    clippy::unimplemented,
223    clippy::unreachable,
224    clippy::get_unwrap,
225    reason = "Panicking is acceptable and often desired in tests."
226)]
227mod tests {
228    use super::*;
229
230    #[test]
231    fn test_en_us_locale_model_defaults() {
232        let locale = Locale::en_us();
233        assert_eq!(locale.locale, "en-US");
234        assert!(locale.punctuation_in_quote);
235        assert_eq!(locale.sort_articles, ["the", "a", "an"]);
236        assert!(locale.roles.contains_key(&ContributorRole::Editor));
237        assert!(locale.locators.contains_key(&LocatorType::Page));
238    }
239
240    #[test]
241    fn test_locale_deserialization() {
242        let json = r#"{
243            "locale": "en-US",
244            "dates": {
245                "months": {
246                    "long": ["January", "February", "March", "April", "May", "June",
247                             "July", "August", "September", "October", "November", "December"],
248                    "short": ["Jan", "Feb", "Mar", "Apr", "May", "Jun",
249                              "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"]
250                },
251                "seasons": ["Spring", "Summer", "Autumn", "Winter"]
252            },
253            "roles": {},
254            "terms": {
255                "and": "and",
256                "et-al": "et al."
257            }
258        }"#;
259
260        let locale: Locale = serde_json::from_str(json).unwrap();
261        assert_eq!(locale.locale, "en-US");
262        assert_eq!(locale.dates.months.long[0], "January");
263        assert_eq!(locale.terms.and.as_ref().unwrap(), "and");
264    }
265
266    #[test]
267    fn test_yaml_locale_loading() {
268        let yaml = r#"
269locale: de-DE
270dates:
271  months:
272    long:
273      - Januar
274      - Februar
275      - März
276      - April
277      - Mai
278      - Juni
279      - Juli
280      - August
281      - September
282      - Oktober
283      - November
284      - Dezember
285    short:
286      - Jan.
287      - Feb.
288      - März
289      - Apr.
290      - Mai
291      - Juni
292      - Juli
293      - Aug.
294      - Sep.
295      - Okt.
296      - Nov.
297      - Dez.
298  seasons:
299    - Frühling
300    - Sommer
301    - Herbst
302    - Winter
303terms:
304  and:
305    long: und
306    symbol: "&"
307  et_al:
308    long: "u. a."
309"#;
310
311        let locale = Locale::from_yaml_str(yaml).unwrap();
312        assert_eq!(locale.locale, "de-DE");
313        assert_eq!(locale.terms.and.as_deref(), Some("und"));
314        assert_eq!(locale.terms.et_al.as_deref(), Some("u. a."));
315        assert_eq!(locale.dates.months.long[0], "Januar");
316        assert_eq!(locale.dates.months.long[2], "März");
317    }
318
319    /// v2 locale with grammar-options overrides punctuation_in_quote correctly.
320    #[test]
321    fn test_v2_grammar_options_sync_punctuation_in_quote() {
322        let yaml = r#"
323locale-schema-version: "2"
324locale: en-GB
325grammar-options:
326  punctuation-in-quote: false
327"#;
328        let locale = Locale::from_yaml_str(yaml).unwrap();
329        // grammar_options is the authoritative source for v2 locales
330        assert!(!locale.grammar_options.punctuation_in_quote);
331        // legacy field is synced from grammar_options
332        assert!(!locale.punctuation_in_quote);
333    }
334
335    /// v1 locale (no grammar-options) derives punctuation_in_quote from locale ID.
336    #[test]
337    fn test_v1_locale_derives_punctuation_from_locale_id() {
338        let yaml = r#"
339locale: en-US
340"#;
341        let locale = Locale::from_yaml_str(yaml).unwrap();
342        // en-US uses American style (inside)
343        assert!(locale.punctuation_in_quote);
344        assert!(locale.grammar_options.punctuation_in_quote);
345    }
346
347    /// Partial locales inherit base messages, date formats, and aliases.
348    #[test]
349    fn test_partial_locale_merges_raw_maps_with_base() {
350        let yaml = r#"
351locale-schema-version: "2"
352locale: zz-ZZ
353messages:
354  pattern.in-container: "inside {$container}"
355date-formats:
356  numeric-short: "dd/MM/y"
357locators:
358  page:
359    long:
360      singular: page-localized
361      plural: pages-localized
362legacy-term-aliases:
363  page: term.page-label-long
364"#;
365        let locale = Locale::from_yaml_str(yaml).unwrap();
366
367        assert_eq!(
368            locale
369                .messages
370                .get("pattern.originally-published-as")
371                .map(String::as_str),
372            Some("originally published as {$title}")
373        );
374        assert_eq!(
375            locale
376                .messages
377                .get("pattern.in-container")
378                .map(String::as_str),
379            Some("inside {$container}")
380        );
381        assert_eq!(
382            locale.date_formats.get("textual-full").map(String::as_str),
383            Some("MMMM d, yyyy")
384        );
385        assert_eq!(
386            locale.date_formats.get("numeric-short").map(String::as_str),
387            Some("dd/MM/y")
388        );
389        assert_eq!(
390            locale.legacy_term_aliases.get("and").map(String::as_str),
391            Some("term.and")
392        );
393        assert_eq!(
394            locale.legacy_term_aliases.get("page").map(String::as_str),
395            Some("term.page-label-long")
396        );
397        assert_eq!(
398            locale.resolved_locator_term(&LocatorType::Page, false, &TermForm::Long, None),
399            Some("page-localized".to_string())
400        );
401    }
402
403    /// apply_override merges messages key-by-key into the base locale.
404    #[test]
405    fn test_apply_override_merges_messages() {
406        let mut locale = Locale::en_us();
407        locale
408            .messages
409            .insert("term.page-label".into(), "p.".into());
410        let ov = LocaleOverride {
411            messages: [("term.page-label".into(), "pg.".into())].into(),
412            ..Default::default()
413        };
414        locale.apply_override(&ov);
415        assert_eq!(
416            locale.messages.get("term.page-label").map(|s| s.as_str()),
417            Some("pg.")
418        );
419    }
420
421    /// The hardcoded en-US locale includes phrase messages used by style
422    /// `message:` components, not only legacy term compatibility messages.
423    #[test]
424    fn test_en_us_locale_resolves_phrase_messages() {
425        let locale = Locale::en_us();
426        let args = MessageArgs {
427            named: [("container".to_string(), "Book Title".to_string())].into(),
428            ..Default::default()
429        };
430
431        assert_eq!(
432            locale.resolve_message("pattern.in-container", &args),
433            Some("in Book Title".to_string())
434        );
435    }
436
437    /// apply_override with grammar_options replaces block and syncs punctuation_in_quote.
438    #[test]
439    fn test_apply_override_grammar_options_syncs_punctuation() {
440        let mut locale = Locale::en_us();
441        locale.punctuation_in_quote = false;
442        let ov = LocaleOverride {
443            grammar_options: Some(GrammarOptions {
444                punctuation_in_quote: true,
445                ..Default::default()
446            }),
447            ..Default::default()
448        };
449        locale.apply_override(&ov);
450        assert!(locale.punctuation_in_quote);
451        assert!(locale.grammar_options.punctuation_in_quote);
452    }
453
454    #[test]
455    fn embedded_locale_ids_include_all_bundled_locale_files() {
456        for id in [
457            "en-US", "ar-AR", "de-DE", "es-ES", "eu-ES", "fr-FR", "tr-TR", "zh-CN", "ja-JP",
458            "ko-KR", "ru-RU",
459        ] {
460            assert!(
461                crate::embedded::EMBEDDED_LOCALE_IDS.contains(&id),
462                "{id} should be listed as an embedded locale"
463            );
464        }
465    }
466
467    #[test]
468    fn bundled_ar_ar_and_eu_es_locales_are_embedded_and_parseable() {
469        for id in ["ar-AR", "eu-ES"] {
470            let bytes = crate::embedded::get_locale_bytes(id).expect("locale should be embedded");
471            let yaml = std::str::from_utf8(bytes).expect("embedded locale should be utf-8");
472            let locale = Locale::from_yaml_str(yaml).expect("embedded locale should parse");
473
474            assert_eq!(locale.locale, id);
475        }
476    }
477
478    /// Round-trip regression guard for the new ja-JP/ko-KR/ru-RU locales
479    /// (`csl26-tfi8`): parses each embedded file and spot-checks a handful
480    /// of the values a future edit to that YAML could silently regress.
481    #[test]
482    fn bundled_ja_jp_ko_kr_ru_ru_locales_are_embedded_and_parseable() {
483        for (id, editor_short, and_term) in [
484            ("ja-JP", "編", "と"),
485            ("ko-KR", "편", "및"),
486            ("ru-RU", "ред.", "и"),
487        ] {
488            let bytes = crate::embedded::get_locale_bytes(id).expect("locale should be embedded");
489            let yaml = std::str::from_utf8(bytes).expect("embedded locale should be utf-8");
490            let locale = Locale::from_yaml_str(yaml).expect("embedded locale should parse");
491
492            assert_eq!(locale.locale, id);
493            assert_eq!(
494                locale.resolved_role_term(&ContributorRole::Editor, false, &TermForm::Short, None),
495                Some(editor_short.to_string()),
496                "{id} editor short-form role term"
497            );
498            assert_eq!(
499                locale.resolved_general_term(&GeneralTerm::And, &TermForm::Long, None),
500                Some(and_term.to_string()),
501                "{id} 'and' term"
502            );
503            assert!(
504                locale.date_formats.contains_key("iso"),
505                "{id} should carry date-formats"
506            );
507        }
508    }
509
510    /// CI enforcement for the locale-completeness lint (`csl26-itri`): every
511    /// embedded v2 locale must ship `grammar-options` and `date-formats`, or
512    /// its typography/dates silently fall back to English. Scoped to just
513    /// the two completeness findings (not general lint errors) so this test
514    /// doesn't couple to unrelated pre-existing lint issues in other
515    /// embedded locales.
516    #[test]
517    fn embedded_v2_locales_pass_completeness_lint() {
518        for &id in crate::embedded::EMBEDDED_LOCALE_IDS {
519            let bytes = crate::embedded::get_locale_bytes(id).expect("locale should be embedded");
520            let raw: RawLocale =
521                serde_yaml::from_slice(bytes).expect("embedded locale should parse as RawLocale");
522
523            if raw.locale_schema_version.as_deref() != Some("2") {
524                continue;
525            }
526
527            let report = crate::lint::lint_raw_locale(&raw);
528            assert!(
529                !report
530                    .findings
531                    .iter()
532                    .any(|finding| finding.path == "grammar-options"),
533                "{id} is missing grammar-options"
534            );
535            assert!(
536                !report
537                    .findings
538                    .iter()
539                    .any(|finding| finding.path == "date-formats"),
540                "{id} is missing date-formats"
541            );
542        }
543    }
544
545    /// Round-trip regression guard for `Locale::en_us()` parsing the
546    /// embedded `en-US.yaml` asset: asserts the critical values a future
547    /// edit to that YAML could silently regress, since `en_us()` is the
548    /// fallback baseline for the large majority of embedded styles.
549    #[test]
550    fn en_us_locale_round_trip_carries_critical_values() {
551        let locale = Locale::en_us();
552
553        // Role labels (CSL reference: scripts/locales-en-US.xml).
554        assert_eq!(
555            locale.resolved_role_term(&ContributorRole::Translator, false, &TermForm::Short, None),
556            Some("trans.".to_string())
557        );
558
559        // Locator labels (CSL reference: chap./chaps.).
560        assert_eq!(
561            locale.locator_term(&LocatorType::Chapter, false, &TermForm::Short, None),
562            Some("chap.")
563        );
564        assert_eq!(
565            locale.locator_term(&LocatorType::Chapter, true, &TermForm::Short, None),
566            Some("chaps.")
567        );
568
569        // No-date term is form-aware (see general_term fix).
570        assert_eq!(
571            locale.general_term(&GeneralTerm::NoDate, &TermForm::Long, None),
572            Some("no date")
573        );
574        assert_eq!(
575            locale.general_term(&GeneralTerm::NoDate, &TermForm::Short, None),
576            Some("n.d.")
577        );
578
579        // Core general terms.
580        assert_eq!(locale.terms.and.as_deref(), Some("and"));
581        assert_eq!(locale.terms.et_al.as_deref(), Some("et al."));
582
583        // Month names.
584        assert_eq!(
585            locale.dates.months.long.first().map(String::as_str),
586            Some("January")
587        );
588
589        // Number formats (single-sourced explicitly in the YAML, Step 4).
590        assert_eq!(locale.number_formats.decimal_separator, ".");
591        assert_eq!(locale.number_formats.thousands_separator, ",");
592        assert_eq!(locale.number_formats.minimum_digits, 1);
593
594        // Sort articles.
595        assert_eq!(locale.sort_articles, ["the", "a", "an"]);
596    }
597}
citum_schema_style/locale/mod.rs

citum_schema_style/locale/
mod.rs
