ferrocat_po/
lib.rs

1#![warn(missing_docs, rustdoc::broken_intra_doc_links)]
2//! Performance-first PO parsing and serialization.
3//!
4//! The crate exposes both owned and borrowed parsers for gettext PO files,
5//! a byte-oriented UTF-8 parser entry point, plus helpers for serialization
6//! and higher-level catalog update workflows.
7//!
8//! # Examples
9//!
10//! ```rust
11//! use ferrocat_po::{PoFile, SerializeOptions, parse_po, stringify_po};
12//!
13//! let input = "msgid \"Hello\"\nmsgstr \"Hallo\"\n";
14//! let file = parse_po(input)?;
15//! assert_eq!(file.items[0].msgid, "Hello");
16//!
17//! let output = stringify_po(&file, &SerializeOptions::default());
18//! assert!(output.contains("msgid \"Hello\""));
19//! # Ok::<(), ferrocat_po::ParseError>(())
20//! ```
21//!
22//! ```rust
23//! use ferrocat_po::parse_po_bytes;
24//!
25//! let input = b"msgid \"Hello\"\nmsgstr \"Hallo\"\n";
26//! let file = parse_po_bytes(input)?;
27//! assert_eq!(file.items[0].msgstr[0], "Hallo");
28//! # Ok::<(), ferrocat_po::ParseError>(())
29//! ```
30//!
31//! ```rust
32//! use ferrocat_po::{
33//!     CompileCatalogArtifactOptions, CompileSelectedCatalogArtifactOptions,
34//!     CompiledCatalogIdIndex, ParseCatalogOptions, compile_catalog_artifact_selected,
35//!     parse_catalog,
36//! };
37//!
38//! let source = parse_catalog(ParseCatalogOptions {
39//!     locale: Some("en"),
40//!     ..ParseCatalogOptions::new("msgid \"Hello\"\nmsgstr \"Hello\"\n", "en")
41//! })?
42//! .into_normalized_view()?;
43//! let requested = parse_catalog(ParseCatalogOptions {
44//!     locale: Some("de"),
45//!     ..ParseCatalogOptions::new("msgid \"Hello\"\nmsgstr \"Hallo\"\n", "en")
46//! })?
47//! .into_normalized_view()?;
48//! let index = CompiledCatalogIdIndex::new(&[&requested, &source], ferrocat_po::CompiledKeyStrategy::FerrocatV1)?;
49//! let compiled_ids = index.iter().map(|(id, _)| id.to_owned()).collect::<Vec<_>>();
50//! let compiled = compile_catalog_artifact_selected(
51//!     &[&requested, &source],
52//!     &index,
53//!     &CompileSelectedCatalogArtifactOptions::new("de", "en", &compiled_ids),
54//! )?;
55//!
56//! assert_eq!(compiled.messages.len(), 1);
57//! # Ok::<(), Box<dyn std::error::Error>>(())
58//! ```
59//!
60//! ```rust
61//! use ferrocat_po::{CatalogAuditOptions, ParseCatalogOptions, audit_catalogs, parse_catalog};
62//!
63//! let source = parse_catalog(ParseCatalogOptions {
64//!     locale: Some("en"),
65//!     ..ParseCatalogOptions::new("msgid \"Hello {name}\"\nmsgstr \"Hello {name}\"\n", "en")
66//! })?
67//! .into_normalized_view()?;
68//! let target = parse_catalog(ParseCatalogOptions {
69//!     locale: Some("de"),
70//!     ..ParseCatalogOptions::new("msgid \"Hello {name}\"\nmsgstr \"Hallo\"\n", "en")
71//! })?
72//! .into_normalized_view()?;
73//! let report = audit_catalogs(&[&source, &target], &CatalogAuditOptions::new("en"))?;
74//!
75//! assert!(report.has_errors());
76//! # Ok::<(), Box<dyn std::error::Error>>(())
77//! ```
78
79#[cfg(feature = "catalog")]
80mod api;
81mod borrowed;
82pub mod diagnostic_codes;
83mod line_state;
84mod merge;
85mod parse;
86mod scan;
87mod serialize;
88mod text;
89mod utf8;
90
91#[cfg(feature = "catalog")]
92pub use api::{
93    ApiError, COMPILED_CATALOG_ARTIFACT_SCHEMA_VERSION, CatalogAuditChecks, CatalogAuditDiagnostic,
94    CatalogAuditIcuOptions, CatalogAuditMessageRef, CatalogAuditOptions, CatalogAuditReport,
95    CatalogAuditSummary, CatalogCombineInput, CatalogCombineResult, CatalogCombineSelection,
96    CatalogCombineStats, CatalogConflictStrategy, CatalogCoverageMessage, CatalogCoverageOptions,
97    CatalogCoverageReport, CatalogFileCombineResult, CatalogFileFormat, CatalogLocaleCoverage,
98    CatalogLocaleReview, CatalogMachineTranslationMessage, CatalogMachineTranslationReview,
99    CatalogMachineTranslationStatus, CatalogMessage, CatalogMessageExtra, CatalogMessageKey,
100    CatalogMessageStatus, CatalogMode, CatalogOrigin, CatalogReviewOptions, CatalogReviewReport,
101    CatalogReviewSummary, CatalogReviewTranslation, CatalogSemantics, CatalogSourceChange,
102    CatalogSourceChangeKind, CatalogSourceChangeReport, CatalogStats, CatalogStorageFormat,
103    CatalogTranslationChange, CatalogTranslationChangeReport, CatalogUpdateInput,
104    CatalogUpdateResult, CombineCatalogFilesOptions, CombineCatalogOptions,
105    CompileCatalogArtifactIcuOptions, CompileCatalogArtifactOptions,
106    CompileCatalogArtifactReportOptions, CompileCatalogArtifactReportSelection,
107    CompileCatalogOptions, CompileSelectedCatalogArtifactOptions, CompiledCatalog,
108    CompiledCatalogArtifact, CompiledCatalogArtifactReport, CompiledCatalogDiagnostic,
109    CompiledCatalogIdDescription, CompiledCatalogIdIndex, CompiledCatalogMissingMessage,
110    CompiledCatalogProvenanceReport, CompiledCatalogResolution, CompiledCatalogResolutionKind,
111    CompiledCatalogTranslationKind, CompiledCatalogUnavailableId, CompiledKeyStrategy,
112    CompiledMessage, CompiledTranslation, DescribeCompiledIdsReport, Diagnostic,
113    DiagnosticSeverity, EffectiveTranslation, EffectiveTranslationRef, ExtractedMessage,
114    ExtractedPluralMessage, ExtractedSingularMessage, IcuFormatterSupportPolicy,
115    IcuPseudolocalizationOptions, IcuSyntaxPolicy, MachineTranslationMetadata, NdjsonCatalogReader,
116    NdjsonCatalogReaderOptions, NdjsonCatalogWriter, NdjsonCatalogWriterOptions,
117    NormalizedParsedCatalog, ObsoleteStrategy, OrderBy, ParseCatalogOptions, ParsedCatalog,
118    PlaceholderCommentMode, PluralEncoding, PluralSource, RenderOptions, SourceExtractedMessage,
119    TranslationShape, UpdateCatalogFileOptions, UpdateCatalogOptions, audit_catalogs,
120    audit_catalogs_with_icu_options, catalog_coverage, catalog_review, combine_catalog_files,
121    combine_catalogs, compile_catalog_artifact, compile_catalog_artifact_report,
122    compile_catalog_artifact_selected, compile_catalog_artifact_selected_with_icu_options,
123    compile_catalog_artifact_with_icu_options, compiled_key, machine_translation_hash,
124    parse_catalog, pseudolocalize_compiled_catalog_artifact,
125    pseudolocalize_compiled_catalog_artifact_with_syntax_policy, update_catalog,
126    update_catalog_file,
127};
128pub use borrowed::{
129    BorrowedHeader, BorrowedMsgStr, BorrowedPoFile, BorrowedPoItem, parse_po_borrowed,
130};
131pub use merge::{ExtractedMessage as MergeExtractedMessage, merge_catalog};
132pub use parse::{parse_po, parse_po_bytes};
133pub use serialize::stringify_po;
134pub use text::{escape_string, extract_quoted, extract_quoted_cow, unescape_string};
135
136use core::{fmt, ops::Index};
137
138#[cfg(feature = "serde")]
139use serde::{Deserialize, Serialize};
140
141/// An owned PO document.
142#[derive(Debug, Clone, PartialEq, Eq, Default)]
143#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
144#[cfg_attr(feature = "serde", serde(deny_unknown_fields))]
145pub struct PoFile {
146    /// File-level translator comments that appear before the header block.
147    pub comments: Vec<String>,
148    /// File-level extracted comments that appear before the header block.
149    pub extracted_comments: Vec<String>,
150    /// Parsed header entries from the leading empty `msgid` block.
151    pub headers: Vec<Header>,
152    /// Regular catalog items in source order.
153    pub items: Vec<PoItem>,
154}
155
156/// A single header entry from the PO header block.
157#[derive(Debug, Clone, PartialEq, Eq, Default)]
158#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
159#[cfg_attr(feature = "serde", serde(deny_unknown_fields))]
160pub struct Header {
161    /// Header name such as `Language` or `Plural-Forms`.
162    pub key: String,
163    /// Header value without the trailing newline.
164    pub value: String,
165}
166
167/// A single gettext message entry.
168#[derive(Debug, Clone, PartialEq, Eq, Default)]
169#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
170#[cfg_attr(feature = "serde", serde(deny_unknown_fields))]
171pub struct PoItem {
172    /// Source message identifier.
173    pub msgid: String,
174    /// Optional gettext message context.
175    pub msgctxt: Option<String>,
176    /// Source references such as `src/app.rs:10`.
177    pub references: Vec<String>,
178    /// Optional plural source identifier.
179    pub msgid_plural: Option<String>,
180    /// Translation payload for the message.
181    pub msgstr: MsgStr,
182    /// Translator comments attached to the item.
183    pub comments: Vec<String>,
184    /// Extracted comments attached to the item.
185    pub extracted_comments: Vec<String>,
186    /// Flags such as `fuzzy`.
187    pub flags: Vec<String>,
188    /// Raw metadata lines that do not fit the dedicated fields.
189    pub metadata: Vec<(String, String)>,
190    /// Whether the item is marked obsolete.
191    pub obsolete: bool,
192    /// Number of plural slots expected when the item is serialized.
193    pub nplurals: usize,
194}
195
196impl PoItem {
197    /// Creates an empty message entry with space for `nplurals` plural slots.
198    #[must_use]
199    pub fn new(nplurals: usize) -> Self {
200        Self {
201            nplurals,
202            ..Self::default()
203        }
204    }
205
206    pub(crate) fn clear_for_reuse(&mut self, nplurals: usize) {
207        self.msgid.clear();
208        self.msgctxt = None;
209        self.references.clear();
210        self.msgid_plural = None;
211        self.msgstr = MsgStr::None;
212        self.comments.clear();
213        self.extracted_comments.clear();
214        self.flags.clear();
215        self.metadata.clear();
216        self.obsolete = false;
217        self.nplurals = nplurals;
218    }
219}
220
221/// Message translation payload for a PO item.
222#[derive(Debug, Clone, PartialEq, Eq, Default)]
223#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
224#[cfg_attr(
225    feature = "serde",
226    serde(tag = "kind", content = "value", rename_all = "snake_case")
227)]
228pub enum MsgStr {
229    /// No translation values are present.
230    #[default]
231    None,
232    /// Single translation string.
233    Singular(String),
234    /// Plural translation strings indexed by plural slot.
235    Plural(Vec<String>),
236}
237
238impl MsgStr {
239    /// Returns `true` when no translation values are present.
240    #[must_use]
241    pub const fn is_empty(&self) -> bool {
242        matches!(self, Self::None)
243    }
244
245    /// Returns the number of translation values present.
246    #[must_use]
247    pub fn len(&self) -> usize {
248        match self {
249            Self::None => 0,
250            Self::Singular(_) => 1,
251            Self::Plural(values) => values.len(),
252        }
253    }
254
255    /// Returns the first translation value, if present.
256    #[must_use]
257    pub fn first(&self) -> Option<&String> {
258        match self {
259            Self::None => None,
260            Self::Singular(value) => Some(value),
261            Self::Plural(values) => values.first(),
262        }
263    }
264
265    /// Returns the first translation value as `&str`, if present.
266    #[must_use]
267    pub fn first_str(&self) -> Option<&str> {
268        self.first().map(String::as_str)
269    }
270
271    /// Returns the translation at `index` without panicking.
272    #[must_use]
273    pub fn get(&self, index: usize) -> Option<&str> {
274        match self {
275            Self::Singular(value) if index == 0 => Some(value.as_str()),
276            Self::None | Self::Singular(_) => None,
277            Self::Plural(values) => values.get(index).map(String::as_str),
278        }
279    }
280
281    /// Iterates over all translation values in order.
282    #[must_use]
283    pub fn iter(&self) -> MsgStrIter<'_> {
284        match self {
285            Self::None => MsgStrIter::empty(),
286            Self::Singular(value) => MsgStrIter::single(value),
287            Self::Plural(values) => MsgStrIter::many(values.iter()),
288        }
289    }
290
291    /// Converts the translation payload into an owned vector.
292    #[must_use]
293    pub fn into_vec(self) -> Vec<String> {
294        match self {
295            Self::None => Vec::new(),
296            Self::Singular(value) => vec![value],
297            Self::Plural(values) => values,
298        }
299    }
300}
301
302impl From<String> for MsgStr {
303    fn from(value: String) -> Self {
304        Self::Singular(value)
305    }
306}
307
308impl From<Vec<String>> for MsgStr {
309    fn from(values: Vec<String>) -> Self {
310        match values.len() {
311            0 => Self::None,
312            1 => Self::Singular(values.into_iter().next().expect("single msgstr value")),
313            _ => Self::Plural(values),
314        }
315    }
316}
317
318impl<'a> IntoIterator for &'a MsgStr {
319    type Item = &'a String;
320    type IntoIter = MsgStrIter<'a>;
321
322    fn into_iter(self) -> Self::IntoIter {
323        self.iter()
324    }
325}
326
327impl Index<usize> for MsgStr {
328    type Output = String;
329
330    fn index(&self, index: usize) -> &Self::Output {
331        match self {
332            Self::None => panic!("msgstr index out of bounds: no translations present"),
333            Self::Singular(value) if index == 0 => value,
334            Self::Singular(_) => panic!("msgstr index out of bounds: singular translation"),
335            Self::Plural(values) => &values[index],
336        }
337    }
338}
339
340/// Iterator over [`MsgStr`] values.
341pub struct MsgStrIter<'a> {
342    inner: MsgStrIterInner<'a>,
343}
344
345enum MsgStrIterInner<'a> {
346    Empty,
347    Single(Option<&'a String>),
348    Many(std::slice::Iter<'a, String>),
349}
350
351impl<'a> MsgStrIter<'a> {
352    const fn empty() -> Self {
353        Self {
354            inner: MsgStrIterInner::Empty,
355        }
356    }
357
358    const fn single(value: &'a String) -> Self {
359        Self {
360            inner: MsgStrIterInner::Single(Some(value)),
361        }
362    }
363
364    const fn many(iter: std::slice::Iter<'a, String>) -> Self {
365        Self {
366            inner: MsgStrIterInner::Many(iter),
367        }
368    }
369}
370
371impl<'a> Iterator for MsgStrIter<'a> {
372    type Item = &'a String;
373
374    fn next(&mut self) -> Option<Self::Item> {
375        match &mut self.inner {
376            MsgStrIterInner::Empty => None,
377            MsgStrIterInner::Single(value) => value.take(),
378            MsgStrIterInner::Many(iter) => iter.next(),
379        }
380    }
381}
382
383/// Options controlling PO serialization.
384#[derive(Debug, Clone, PartialEq, Eq)]
385pub struct SerializeOptions {
386    /// Preferred soft line-wrap limit for long string literals.
387    pub fold_length: usize,
388    /// When `true`, one-line values stay compact instead of always expanding.
389    pub compact_multiline: bool,
390}
391
392impl Default for SerializeOptions {
393    fn default() -> Self {
394        Self {
395            fold_length: 80,
396            compact_multiline: true,
397        }
398    }
399}
400
401/// One-based line/column context plus the byte offset for a parse error.
402#[derive(Debug, Clone, Copy, PartialEq, Eq)]
403pub struct ParsePosition {
404    offset: usize,
405    line: usize,
406    column: usize,
407}
408
409impl ParsePosition {
410    /// Creates a new parse position.
411    ///
412    /// `offset` is zero-based and counts bytes from the parsed input after any
413    /// parser-specific pre-processing, while `line` and `column` are one-based.
414    #[must_use]
415    pub const fn new(offset: usize, line: usize, column: usize) -> Self {
416        Self {
417            offset,
418            line,
419            column,
420        }
421    }
422
423    /// Returns the zero-based byte offset in the parsed input.
424    #[must_use]
425    pub const fn offset(self) -> usize {
426        self.offset
427    }
428
429    /// Returns the one-based line number.
430    #[must_use]
431    pub const fn line(self) -> usize {
432        self.line
433    }
434
435    /// Returns the one-based column number.
436    #[must_use]
437    pub const fn column(self) -> usize {
438        self.column
439    }
440}
441
442/// Error returned when parsing or unescaping PO content fails.
443#[derive(Debug, Clone, PartialEq, Eq)]
444pub struct ParseError {
445    message: String,
446    position: Option<ParsePosition>,
447}
448
449impl ParseError {
450    /// Creates a new parse error with the provided message.
451    #[must_use]
452    pub fn new(message: impl Into<String>) -> Self {
453        Self {
454            message: message.into(),
455            position: None,
456        }
457    }
458
459    /// Creates a new parse error with source position metadata.
460    #[must_use]
461    pub fn with_position(message: impl Into<String>, position: ParsePosition) -> Self {
462        Self {
463            message: message.into(),
464            position: Some(position),
465        }
466    }
467
468    /// Returns the human-readable error message.
469    #[must_use]
470    pub fn message(&self) -> &str {
471        &self.message
472    }
473
474    /// Returns source position metadata when the parser could attach it.
475    #[must_use]
476    pub const fn position(&self) -> Option<ParsePosition> {
477        self.position
478    }
479
480    pub(crate) fn with_position_if_missing(mut self, position: ParsePosition) -> Self {
481        self.position.get_or_insert(position);
482        self
483    }
484}
485
486impl fmt::Display for ParseError {
487    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
488        f.write_str(&self.message)
489    }
490}
491
492impl std::error::Error for ParseError {}
493
494#[cfg(test)]
495mod tests {
496    use super::{MsgStr, ParseError, ParsePosition};
497
498    #[cfg(feature = "serde")]
499    use super::{Header, PoFile, PoItem};
500
501    #[test]
502    fn parse_error_accessors_preserve_message_and_optional_position() {
503        let error = ParseError::new("invalid PO string");
504        assert_eq!(error.message(), "invalid PO string");
505        assert_eq!(error.position(), None);
506        assert_eq!(error.to_string(), "invalid PO string");
507
508        let position = ParsePosition::new(12, 2, 3);
509        let positioned = ParseError::with_position("invalid PO string", position);
510        assert_eq!(positioned.message(), "invalid PO string");
511        assert_eq!(positioned.position(), Some(position));
512        assert_eq!(positioned.position().map(ParsePosition::offset), Some(12));
513        assert_eq!(positioned.position().map(ParsePosition::line), Some(2));
514        assert_eq!(positioned.position().map(ParsePosition::column), Some(3));
515        assert_eq!(positioned.to_string(), "invalid PO string");
516    }
517
518    #[test]
519    fn msgstr_get_returns_none_for_empty_values() {
520        let msgstr = MsgStr::None;
521
522        assert_eq!(msgstr.get(0), None);
523    }
524
525    #[test]
526    fn msgstr_get_returns_singular_value_at_zero() {
527        let msgstr = MsgStr::from("Hallo".to_owned());
528
529        assert_eq!(msgstr.get(0), Some("Hallo"));
530        assert_eq!(msgstr.get(1), None);
531    }
532
533    #[test]
534    fn msgstr_get_returns_plural_values_by_index() {
535        let msgstr = MsgStr::from(vec!["eins".to_owned(), "viele".to_owned()]);
536
537        assert_eq!(msgstr.get(0), Some("eins"));
538        assert_eq!(msgstr.get(1), Some("viele"));
539        assert_eq!(msgstr.get(2), None);
540    }
541
542    #[test]
543    fn msgstr_helpers_cover_empty_singular_and_plural_shapes() {
544        let empty = MsgStr::from(Vec::<String>::new());
545        assert!(empty.is_empty());
546        assert_eq!(empty.len(), 0);
547        assert_eq!(empty.first(), None);
548        assert_eq!(empty.first_str(), None);
549        assert_eq!(empty.iter().count(), 0);
550        assert_eq!(empty.into_vec(), Vec::<String>::new());
551
552        let singular = MsgStr::from(vec!["Hallo".to_owned()]);
553        assert!(!singular.is_empty());
554        assert_eq!(singular.len(), 1);
555        assert_eq!(singular.first().map(String::as_str), Some("Hallo"));
556        assert_eq!(singular.first_str(), Some("Hallo"));
557        assert_eq!((&singular).into_iter().collect::<Vec<_>>(), vec!["Hallo"]);
558        assert_eq!(singular[0], "Hallo");
559        assert_eq!(singular.into_vec(), vec!["Hallo"]);
560
561        let plural = MsgStr::from(vec!["eins".to_owned(), "viele".to_owned()]);
562        assert_eq!(plural.len(), 2);
563        assert_eq!(plural.first_str(), Some("eins"));
564        assert_eq!(plural.iter().collect::<Vec<_>>(), vec!["eins", "viele"]);
565        assert_eq!(plural[1], "viele");
566        assert_eq!(plural.into_vec(), vec!["eins", "viele"]);
567    }
568
569    #[cfg(feature = "serde")]
570    #[test]
571    fn po_file_serde_round_trips_owned_document_shape() {
572        let file = PoFile {
573            comments: vec!["translator note".to_owned()],
574            headers: vec![Header {
575                key: "Language".to_owned(),
576                value: "de".to_owned(),
577            }],
578            items: vec![PoItem {
579                msgid: "Hello".to_owned(),
580                msgstr: MsgStr::from("Hallo".to_owned()),
581                references: vec!["src/app.rs:10".to_owned()],
582                nplurals: 1,
583                ..PoItem::default()
584            }],
585            ..PoFile::default()
586        };
587
588        let json = serde_json::to_value(&file).expect("PO file serialization must succeed");
589        assert_eq!(json["items"][0]["msgstr"]["kind"], "singular");
590        assert_eq!(json["items"][0]["msgstr"]["value"], "Hallo");
591
592        let roundtrip: PoFile =
593            serde_json::from_value(json).expect("PO file deserialization must succeed");
594        assert_eq!(roundtrip, file);
595    }
596}
ferrocat_po/lib.rs

ferrocat_po/
lib.rs
