ferrocat_po/api/
ndjson.rs

1use std::collections::{BTreeMap, BTreeSet};
2use std::io::{BufRead, Write};
3
4use serde::{Deserialize, Serialize};
5
6use super::catalog::{
7    CanonicalMessage, CanonicalTranslation, Catalog, public_message_from_canonical,
8    split_placeholder_comments,
9};
10use super::export::{append_placeholder_comments, plural_source_branches};
11use super::mt::{
12    MachineTranslationMetadata, machine_translation_hash, validate_machine_translation_metadata,
13};
14use super::plural::synthesize_icu_plural;
15use super::{
16    ApiError, CatalogMessage, CatalogMessageExtra, CatalogOrigin, CatalogSemantics,
17    EffectiveTranslationRef, PlaceholderCommentMode, TranslationShape,
18};
19
20const FRONTMATTER_DELIMITER: &str = "---";
21const NDJSON_FORMAT_V1: &str = "ferrocat.ndjson.v1";
22
23/// Options for constructing an [`NdjsonCatalogReader`].
24#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
25pub struct NdjsonCatalogReaderOptions<'a> {
26    /// Optional explicit locale override. When `None`, the reader uses the
27    /// locale declared in NDJSON frontmatter.
28    pub locale: Option<&'a str>,
29    /// Source locale expected by the caller.
30    pub source_locale: &'a str,
31}
32
33impl<'a> NdjsonCatalogReaderOptions<'a> {
34    /// Creates reader options with the required source locale set.
35    #[must_use]
36    pub const fn new(source_locale: &'a str) -> Self {
37        Self {
38            locale: None,
39            source_locale,
40        }
41    }
42}
43
44/// Streaming reader for Ferrocat NDJSON catalog records.
45///
46/// The reader consumes and validates frontmatter during construction, then
47/// yields one [`CatalogMessage`] per non-empty NDJSON body line.
48pub struct NdjsonCatalogReader<R> {
49    inner: NdjsonCanonicalReader<R>,
50}
51
52impl<R: BufRead> NdjsonCatalogReader<R> {
53    /// Creates a streaming reader with the required source locale.
54    ///
55    /// # Errors
56    ///
57    /// Returns [`ApiError`] when frontmatter cannot be read, the source locale
58    /// is empty, or a declared NDJSON `source_locale` does not match.
59    pub fn new(reader: R, source_locale: &str) -> Result<Self, ApiError> {
60        Self::with_options(reader, NdjsonCatalogReaderOptions::new(source_locale))
61    }
62
63    /// Creates a streaming reader with explicit options.
64    ///
65    /// # Errors
66    ///
67    /// Returns [`ApiError`] when frontmatter cannot be read, the source locale
68    /// is empty, or a declared NDJSON `source_locale` does not match.
69    pub fn with_options(
70        reader: R,
71        options: NdjsonCatalogReaderOptions<'_>,
72    ) -> Result<Self, ApiError> {
73        Ok(Self {
74            inner: NdjsonCanonicalReader::with_options(reader, options)?,
75        })
76    }
77
78    /// Returns the effective catalog locale after applying the optional
79    /// override.
80    #[must_use]
81    pub fn locale(&self) -> Option<&str> {
82        self.inner.locale.as_deref()
83    }
84
85    /// Returns a shared reference to the wrapped reader.
86    #[must_use]
87    pub const fn get_ref(&self) -> &R {
88        &self.inner.reader
89    }
90
91    /// Returns a mutable reference to the wrapped reader.
92    pub const fn get_mut(&mut self) -> &mut R {
93        &mut self.inner.reader
94    }
95
96    /// Consumes the streaming reader and returns the wrapped reader.
97    #[must_use]
98    pub fn into_inner(self) -> R {
99        self.inner.reader
100    }
101}
102
103impl<R: BufRead> Iterator for NdjsonCatalogReader<R> {
104    type Item = Result<CatalogMessage, ApiError>;
105
106    fn next(&mut self) -> Option<Self::Item> {
107        self.inner
108            .next()
109            .map(|message| message.map(public_message_from_canonical))
110    }
111}
112
113/// Options for constructing an [`NdjsonCatalogWriter`].
114#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
115pub struct NdjsonCatalogWriterOptions<'a> {
116    /// Locale to write into NDJSON frontmatter.
117    pub locale: Option<&'a str>,
118    /// Source locale to write into NDJSON frontmatter.
119    pub source_locale: &'a str,
120}
121
122impl<'a> NdjsonCatalogWriterOptions<'a> {
123    /// Creates writer options with the required source locale set.
124    #[must_use]
125    pub const fn new(source_locale: &'a str) -> Self {
126        Self {
127            locale: None,
128            source_locale,
129        }
130    }
131}
132
133/// Streaming writer for Ferrocat NDJSON catalog records.
134///
135/// The writer emits frontmatter during construction and then appends one JSON
136/// record per [`CatalogMessage`] written.
137pub struct NdjsonCatalogWriter<W> {
138    writer: W,
139}
140
141impl<W: Write> NdjsonCatalogWriter<W> {
142    /// Creates a streaming writer with the required source locale.
143    ///
144    /// # Errors
145    ///
146    /// Returns [`ApiError`] when the source locale is empty or the frontmatter
147    /// cannot be written.
148    pub fn new(writer: W, source_locale: &str) -> Result<Self, ApiError> {
149        Self::with_options(writer, NdjsonCatalogWriterOptions::new(source_locale))
150    }
151
152    /// Creates a streaming writer with explicit options.
153    ///
154    /// # Errors
155    ///
156    /// Returns [`ApiError`] when the source locale is empty or the frontmatter
157    /// cannot be written.
158    pub fn with_options(
159        mut writer: W,
160        options: NdjsonCatalogWriterOptions<'_>,
161    ) -> Result<Self, ApiError> {
162        super::validate_source_locale(options.source_locale)?;
163        write_frontmatter(&mut writer, options.locale, options.source_locale)?;
164        Ok(Self { writer })
165    }
166
167    /// Writes one catalog message as an NDJSON body record.
168    ///
169    /// # Errors
170    ///
171    /// Returns [`ApiError`] when the underlying writer fails.
172    pub fn write_message(&mut self, message: &CatalogMessage) -> Result<(), ApiError> {
173        let record = ndjson_record_from_public_message(message);
174        write_record(&mut self.writer, &record)
175    }
176
177    /// Returns a shared reference to the wrapped writer.
178    #[must_use]
179    pub const fn get_ref(&self) -> &W {
180        &self.writer
181    }
182
183    /// Returns a mutable reference to the wrapped writer.
184    pub const fn get_mut(&mut self) -> &mut W {
185        &mut self.writer
186    }
187
188    /// Flushes and returns the wrapped writer.
189    ///
190    /// # Errors
191    ///
192    /// Returns [`ApiError`] when flushing the underlying writer fails.
193    pub fn finish(mut self) -> Result<W, ApiError> {
194        self.writer.flush()?;
195        Ok(self.writer)
196    }
197}
198
199#[derive(Debug, Default)]
200struct Frontmatter {
201    locale: Option<String>,
202    source_locale: Option<String>,
203}
204
205struct NdjsonCanonicalReader<R> {
206    reader: R,
207    locale: Option<String>,
208    line_number: usize,
209    seen: BTreeSet<(String, Option<String>)>,
210}
211
212impl<R: BufRead> NdjsonCanonicalReader<R> {
213    fn with_options(
214        mut reader: R,
215        options: NdjsonCatalogReaderOptions<'_>,
216    ) -> Result<Self, ApiError> {
217        super::validate_source_locale(options.source_locale)?;
218        let mut line_number = 0;
219        let frontmatter = read_frontmatter(&mut reader, &mut line_number)?;
220        if let Some(header_source_locale) = &frontmatter.source_locale
221            && header_source_locale != options.source_locale
222        {
223            return Err(ApiError::InvalidArguments(format!(
224                "NDJSON source_locale {:?} did not match requested source_locale {:?}",
225                header_source_locale, options.source_locale
226            )));
227        }
228
229        Ok(Self {
230            reader,
231            locale: options.locale.map(str::to_owned).or(frontmatter.locale),
232            line_number,
233            seen: BTreeSet::new(),
234        })
235    }
236}
237
238impl<R: BufRead> Iterator for NdjsonCanonicalReader<R> {
239    type Item = Result<CanonicalMessage, ApiError>;
240
241    fn next(&mut self) -> Option<Self::Item> {
242        loop {
243            let line = match read_line(&mut self.reader, &mut self.line_number) {
244                Ok(Some(line)) => line,
245                Ok(None) => return None,
246                Err(error) => return Some(Err(error)),
247            };
248            let trimmed = line.trim();
249            if trimmed.is_empty() {
250                continue;
251            }
252
253            let record = match serde_json::from_str::<NdjsonRecord>(trimmed) {
254                Ok(record) => record,
255                Err(error) => {
256                    return Some(Err(ApiError::InvalidArguments(format!(
257                        "invalid NDJSON record on line {}: {error}",
258                        self.line_number
259                    ))));
260                }
261            };
262            let key = (record.id.clone(), record.ctx.clone());
263            if !self.seen.insert(key.clone()) {
264                return Some(Err(ApiError::Conflict(format!(
265                    "duplicate NDJSON message for id {:?} and context {:?}",
266                    key.0, key.1
267                ))));
268            }
269
270            return Some(canonical_message_from_record(record));
271        }
272    }
273}
274
275#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
276#[serde(deny_unknown_fields)]
277struct NdjsonRecord {
278    id: String,
279    str: String,
280    #[serde(default, skip_serializing_if = "Option::is_none")]
281    ctx: Option<String>,
282    #[serde(default, skip_serializing_if = "Vec::is_empty")]
283    comments: Vec<String>,
284    #[serde(default, skip_serializing_if = "Vec::is_empty")]
285    origin: Vec<NdjsonOrigin>,
286    #[serde(default, skip_serializing_if = "is_false")]
287    obsolete: bool,
288    #[serde(default, skip_serializing_if = "Option::is_none")]
289    extra: Option<NdjsonExtra>,
290    #[serde(default, skip_serializing_if = "Option::is_none")]
291    mt: Option<MachineTranslationMetadata>,
292}
293
294#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
295#[serde(deny_unknown_fields)]
296struct NdjsonOrigin {
297    file: String,
298    #[serde(default, skip_serializing_if = "Option::is_none")]
299    line: Option<u32>,
300}
301
302#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default)]
303#[serde(deny_unknown_fields)]
304struct NdjsonExtra {
305    #[serde(default, skip_serializing_if = "Vec::is_empty")]
306    translator_comments: Vec<String>,
307    #[serde(default, skip_serializing_if = "Vec::is_empty")]
308    flags: Vec<String>,
309}
310
311pub(super) fn parse_catalog_to_internal_ndjson(
312    content: &str,
313    locale_override: Option<&str>,
314    source_locale: &str,
315    semantics: CatalogSemantics,
316    _strict: bool,
317) -> Result<Catalog, ApiError> {
318    if semantics != CatalogSemantics::IcuNative {
319        return Err(ApiError::Unsupported(
320            "CatalogSemantics::GettextCompat is not supported for NDJSON catalogs".to_owned(),
321        ));
322    }
323
324    let normalized = normalize_input(content);
325    let mut reader = NdjsonCanonicalReader::with_options(
326        std::io::Cursor::new(normalized.as_bytes()),
327        NdjsonCatalogReaderOptions {
328            locale: locale_override,
329            source_locale,
330        },
331    )?;
332    let locale = reader.locale.clone();
333    let mut messages = Vec::new();
334    for message in &mut reader {
335        messages.push(message?);
336    }
337
338    Ok(Catalog {
339        locale,
340        headers: BTreeMap::new(),
341        file_comments: Vec::new(),
342        file_extracted_comments: Vec::new(),
343        messages,
344        diagnostics: Vec::new(),
345    })
346}
347
348pub(super) fn stringify_catalog_ndjson(
349    catalog: &Catalog,
350    locale: Option<&str>,
351    source_locale: &str,
352    placeholder_comment_mode: &PlaceholderCommentMode,
353) -> String {
354    let mut rendered = Vec::new();
355    write_frontmatter(&mut rendered, locale, source_locale)
356        .expect("writing NDJSON frontmatter into a Vec must succeed");
357    for message in &catalog.messages {
358        let record = ndjson_record_from_canonical(message, placeholder_comment_mode);
359        write_record(&mut rendered, &record)
360            .expect("writing NDJSON record into a Vec must succeed");
361    }
362    String::from_utf8(rendered).expect("NDJSON renderer writes UTF-8")
363}
364
365fn canonical_message_from_record(record: NdjsonRecord) -> Result<CanonicalMessage, ApiError> {
366    let (comments, placeholders) = split_placeholder_comments(record.comments);
367    let extra = record.extra.unwrap_or_default();
368    if let Some(metadata) = &record.mt {
369        validate_machine_translation_metadata(metadata)?;
370    }
371    Ok(CanonicalMessage {
372        msgid: record.id,
373        msgctxt: record.ctx,
374        translation: CanonicalTranslation::Singular { value: record.str },
375        comments,
376        origins: record
377            .origin
378            .into_iter()
379            .map(|origin| CatalogOrigin {
380                file: origin.file,
381                line: origin.line,
382            })
383            .collect(),
384        placeholders,
385        obsolete: record.obsolete,
386        machine_translation: record.mt,
387        translator_comments: extra.translator_comments,
388        flags: extra.flags,
389    })
390}
391
392fn ndjson_record_from_canonical(
393    message: &CanonicalMessage,
394    placeholder_comment_mode: &PlaceholderCommentMode,
395) -> NdjsonRecord {
396    let mut comments = message.comments.clone();
397    append_placeholder_comments(
398        &mut comments,
399        &message.placeholders,
400        placeholder_comment_mode,
401    );
402
403    NdjsonRecord {
404        id: ndjson_id(message),
405        str: ndjson_translation(message),
406        ctx: message.msgctxt.clone(),
407        comments,
408        origin: message
409            .origins
410            .iter()
411            .map(|origin| NdjsonOrigin {
412                file: origin.file.clone(),
413                line: origin.line,
414            })
415            .collect(),
416        obsolete: message.obsolete,
417        extra: ndjson_extra(message),
418        mt: ndjson_machine_translation(message),
419    }
420}
421
422fn ndjson_record_from_public_message(message: &CatalogMessage) -> NdjsonRecord {
423    NdjsonRecord {
424        id: ndjson_public_id(message),
425        str: ndjson_public_translation(message),
426        ctx: message.msgctxt.clone(),
427        comments: message.comments.clone(),
428        origin: message
429            .origin
430            .iter()
431            .map(|origin| NdjsonOrigin {
432                file: origin.file.clone(),
433                line: origin.line,
434            })
435            .collect(),
436        obsolete: message.obsolete,
437        extra: ndjson_public_extra(message.extra.as_ref()),
438        mt: ndjson_public_machine_translation(message),
439    }
440}
441
442fn ndjson_machine_translation(message: &CanonicalMessage) -> Option<MachineTranslationMetadata> {
443    let metadata = message.machine_translation.as_ref()?;
444    if validate_machine_translation_metadata(metadata).is_err() {
445        return None;
446    }
447    (metadata.hash == machine_translation_hash(ndjson_translation_ref(message)))
448        .then(|| metadata.clone())
449}
450
451fn ndjson_translation_ref(message: &CanonicalMessage) -> EffectiveTranslationRef<'_> {
452    match &message.translation {
453        CanonicalTranslation::Singular { value } => EffectiveTranslationRef::Singular(value),
454        CanonicalTranslation::Plural {
455            translation_by_category,
456            ..
457        } => EffectiveTranslationRef::Plural(translation_by_category),
458    }
459}
460
461fn ndjson_id(message: &CanonicalMessage) -> String {
462    match &message.translation {
463        CanonicalTranslation::Singular { .. } => message.msgid.clone(),
464        CanonicalTranslation::Plural {
465            source, variable, ..
466        } => synthesize_icu_plural(variable, &plural_source_branches(source)),
467    }
468}
469
470fn ndjson_translation(message: &CanonicalMessage) -> String {
471    match &message.translation {
472        CanonicalTranslation::Singular { value } => value.clone(),
473        CanonicalTranslation::Plural {
474            translation_by_category,
475            variable,
476            ..
477        } => synthesize_icu_plural(variable, translation_by_category),
478    }
479}
480
481fn ndjson_extra(message: &CanonicalMessage) -> Option<NdjsonExtra> {
482    if message.translator_comments.is_empty() && message.flags.is_empty() {
483        None
484    } else {
485        Some(NdjsonExtra {
486            translator_comments: message.translator_comments.clone(),
487            flags: message.flags.clone(),
488        })
489    }
490}
491
492fn ndjson_public_machine_translation(
493    message: &CatalogMessage,
494) -> Option<MachineTranslationMetadata> {
495    let metadata = message.machine_translation.as_ref()?;
496    if validate_machine_translation_metadata(metadata).is_err() {
497        return None;
498    }
499    (metadata.hash == machine_translation_hash(message.effective_translation()))
500        .then(|| metadata.clone())
501}
502
503fn ndjson_public_id(message: &CatalogMessage) -> String {
504    match &message.translation {
505        TranslationShape::Singular { .. } => message.msgid.clone(),
506        TranslationShape::Plural {
507            source, variable, ..
508        } => synthesize_icu_plural(variable, &plural_source_branches(source)),
509    }
510}
511
512fn ndjson_public_translation(message: &CatalogMessage) -> String {
513    match &message.translation {
514        TranslationShape::Singular { value } => value.clone(),
515        TranslationShape::Plural {
516            translation,
517            variable,
518            ..
519        } => synthesize_icu_plural(variable, translation),
520    }
521}
522
523fn ndjson_public_extra(extra: Option<&CatalogMessageExtra>) -> Option<NdjsonExtra> {
524    let extra = extra?;
525    if extra.translator_comments.is_empty() && extra.flags.is_empty() {
526        None
527    } else {
528        Some(NdjsonExtra {
529            translator_comments: extra.translator_comments.clone(),
530            flags: extra.flags.clone(),
531        })
532    }
533}
534
535fn write_frontmatter<W: Write>(
536    writer: &mut W,
537    locale: Option<&str>,
538    source_locale: &str,
539) -> Result<(), ApiError> {
540    writer.write_all(FRONTMATTER_DELIMITER.as_bytes())?;
541    writer.write_all(b"\nformat: ")?;
542    writer.write_all(NDJSON_FORMAT_V1.as_bytes())?;
543    writer.write_all(b"\n")?;
544    if let Some(locale) = locale {
545        writer.write_all(b"locale: ")?;
546        writer.write_all(locale.as_bytes())?;
547        writer.write_all(b"\n")?;
548    }
549    writer.write_all(b"source_locale: ")?;
550    writer.write_all(source_locale.as_bytes())?;
551    writer.write_all(b"\n")?;
552    writer.write_all(FRONTMATTER_DELIMITER.as_bytes())?;
553    writer.write_all(b"\n")?;
554    Ok(())
555}
556
557fn write_record<W: Write>(writer: &mut W, record: &NdjsonRecord) -> Result<(), ApiError> {
558    let line = serde_json::to_string(record).expect("NDJSON record serialization must succeed");
559    writer.write_all(line.as_bytes())?;
560    writer.write_all(b"\n")?;
561    Ok(())
562}
563
564fn read_frontmatter<R: BufRead>(
565    reader: &mut R,
566    line_number: &mut usize,
567) -> Result<Frontmatter, ApiError> {
568    let Some(first_line) = read_line(reader, line_number)? else {
569        return Err(ApiError::InvalidArguments(
570            "NDJSON catalog must start with a frontmatter block".to_owned(),
571        ));
572    };
573    if first_line.trim() != FRONTMATTER_DELIMITER {
574        return Err(ApiError::InvalidArguments(
575            "NDJSON catalog must start with `---`".to_owned(),
576        ));
577    }
578
579    let mut header = Frontmatter::default();
580    let mut seen = BTreeSet::new();
581
582    while let Some(line) = read_line(reader, line_number)? {
583        if line.trim() == FRONTMATTER_DELIMITER {
584            if !seen.contains("format") {
585                return Err(ApiError::InvalidArguments(
586                    "NDJSON frontmatter is missing required `format`".to_owned(),
587                ));
588            }
589            return Ok(header);
590        }
591        let trimmed = line.trim();
592        if trimmed.is_empty() {
593            continue;
594        }
595        let (key, value) = trimmed.split_once(':').ok_or_else(|| {
596            ApiError::InvalidArguments(format!("invalid NDJSON frontmatter line: {trimmed:?}"))
597        })?;
598        let key = key.trim();
599        let value = value.trim();
600        if !seen.insert(key.to_owned()) {
601            return Err(ApiError::InvalidArguments(format!(
602                "duplicate NDJSON frontmatter key {key:?}"
603            )));
604        }
605
606        match key {
607            "format" => {
608                if value != NDJSON_FORMAT_V1 {
609                    return Err(ApiError::InvalidArguments(format!(
610                        "unsupported NDJSON format {:?}; expected {:?}",
611                        value, NDJSON_FORMAT_V1
612                    )));
613                }
614            }
615            "locale" => header.locale = Some(value.to_owned()),
616            "source_locale" => header.source_locale = Some(value.to_owned()),
617            other => {
618                return Err(ApiError::InvalidArguments(format!(
619                    "unknown NDJSON frontmatter key {other:?}"
620                )));
621            }
622        }
623    }
624
625    Err(ApiError::InvalidArguments(
626        "NDJSON frontmatter was not closed with `---`".to_owned(),
627    ))
628}
629
630fn read_line<R: BufRead>(
631    reader: &mut R,
632    line_number: &mut usize,
633) -> Result<Option<String>, ApiError> {
634    let mut line = String::new();
635    if reader.read_line(&mut line)? == 0 {
636        return Ok(None);
637    }
638    *line_number += 1;
639    if *line_number == 1 {
640        line = line.strip_prefix('\u{feff}').unwrap_or(&line).to_owned();
641    }
642    if line.ends_with('\n') {
643        line.pop();
644    }
645    if line.ends_with('\r') {
646        line.pop();
647    }
648    Ok(Some(line))
649}
650
651fn normalize_input(input: &str) -> std::borrow::Cow<'_, str> {
652    let input = input.strip_prefix('\u{feff}').unwrap_or(input);
653    if input.as_bytes().contains(&b'\r') {
654        std::borrow::Cow::Owned(input.replace("\r\n", "\n").replace('\r', "\n"))
655    } else {
656        std::borrow::Cow::Borrowed(input)
657    }
658}
659
660const fn is_false(value: &bool) -> bool {
661    !*value
662}
663
664#[cfg(test)]
665mod tests {
666    use std::{
667        collections::BTreeMap,
668        io::{Cursor, Read},
669    };
670
671    use super::{
672        CanonicalMessage, CanonicalTranslation, Catalog, CatalogMessage, CatalogMessageExtra,
673        CatalogOrigin, CatalogSemantics, EffectiveTranslationRef, MachineTranslationMetadata,
674        NDJSON_FORMAT_V1, NdjsonCatalogReader, NdjsonCatalogReaderOptions, NdjsonCatalogWriter,
675        NdjsonCatalogWriterOptions, PlaceholderCommentMode, TranslationShape,
676        machine_translation_hash, normalize_input, parse_catalog_to_internal_ndjson,
677        read_frontmatter, read_line, stringify_catalog_ndjson,
678    };
679
680    fn sample_catalog() -> Catalog {
681        Catalog {
682            locale: Some("de".to_owned()),
683            headers: BTreeMap::new(),
684            file_comments: Vec::new(),
685            file_extracted_comments: Vec::new(),
686            messages: vec![
687                CanonicalMessage {
688                    msgid: "About us".to_owned(),
689                    msgctxt: Some("nav".to_owned()),
690                    translation: CanonicalTranslation::Singular {
691                        value: "Ueber uns".to_owned(),
692                    },
693                    comments: vec!["Shown in nav".to_owned()],
694                    origins: vec![CatalogOrigin {
695                        file: "src/nav.rs".to_owned(),
696                        line: Some(4),
697                    }],
698                    placeholders: BTreeMap::new(),
699                    obsolete: false,
700                    machine_translation: None,
701                    translator_comments: vec!["Keep short".to_owned()],
702                    flags: vec!["fuzzy".to_owned()],
703                },
704                CanonicalMessage {
705                    msgid: "files".to_owned(),
706                    msgctxt: None,
707                    translation: CanonicalTranslation::Plural {
708                        source: super::super::PluralSource {
709                            one: Some("# file".to_owned()),
710                            other: "# files".to_owned(),
711                        },
712                        translation_by_category: BTreeMap::from([
713                            ("one".to_owned(), "# Datei".to_owned()),
714                            ("other".to_owned(), "# Dateien".to_owned()),
715                        ]),
716                        variable: "count".to_owned(),
717                    },
718                    comments: Vec::new(),
719                    origins: Vec::new(),
720                    placeholders: BTreeMap::new(),
721                    obsolete: true,
722                    machine_translation: None,
723                    translator_comments: Vec::new(),
724                    flags: Vec::new(),
725                },
726            ],
727            diagnostics: Vec::new(),
728        }
729    }
730
731    #[test]
732    fn frontmatter_parser_accepts_valid_blocks_and_rejects_invalid_ones() {
733        let mut reader = Cursor::new(concat!(
734            "---\n",
735            "format: ferrocat.ndjson.v1\n",
736            "locale: de\n",
737            "source_locale: en\n",
738            "---\n",
739            "{\"id\":\"About us\",\"str\":\"Ueber uns\"}\n",
740        ));
741        let mut line_number = 0;
742        let frontmatter = read_frontmatter(&mut reader, &mut line_number).expect("frontmatter");
743        assert_eq!(frontmatter.locale.as_deref(), Some("de"));
744        assert_eq!(frontmatter.source_locale.as_deref(), Some("en"));
745        assert_eq!(line_number, 5);
746        let mut body = String::new();
747        reader.read_to_string(&mut body).expect("body");
748        assert!(body.contains("\"About us\""));
749
750        for invalid in [
751            "format: ferrocat.ndjson.v1\n---\n",
752            "---\nlocale: de\n---\n",
753            "---\nformat: wrong\n---\n",
754            "---\nformat: ferrocat.ndjson.v1\nformat: ferrocat.ndjson.v1\n---\n",
755            "---\nformat: ferrocat.ndjson.v1\nunknown: value\n---\n",
756            "---\nformat: ferrocat.ndjson.v1\n",
757        ] {
758            let mut reader = Cursor::new(invalid);
759            let mut line_number = 0;
760            assert!(
761                read_frontmatter(&mut reader, &mut line_number).is_err(),
762                "{invalid:?}"
763            );
764        }
765    }
766
767    #[test]
768    fn normalize_input_and_streaming_line_reader_handle_bom_and_crlf() {
769        assert_eq!(normalize_input("\u{feff}a\r\nb\r").as_ref(), "a\nb\n");
770
771        let mut reader = Cursor::new("\u{feff}alpha\r\nbeta\n");
772        let mut line_number = 0;
773        assert_eq!(
774            read_line(&mut reader, &mut line_number).expect("line"),
775            Some("alpha".to_owned())
776        );
777        assert_eq!(
778            read_line(&mut reader, &mut line_number).expect("line"),
779            Some("beta".to_owned())
780        );
781        assert_eq!(read_line(&mut reader, &mut line_number).expect("eof"), None);
782        assert_eq!(line_number, 2);
783    }
784
785    #[test]
786    fn ndjson_roundtrip_keeps_comments_metadata_and_plural_rendering() {
787        let rendered = stringify_catalog_ndjson(
788            &sample_catalog(),
789            Some("de"),
790            "en",
791            &PlaceholderCommentMode::Disabled,
792        );
793        assert!(rendered.contains(&format!("format: {NDJSON_FORMAT_V1}")));
794        assert!(rendered.contains("\"ctx\":\"nav\""));
795        assert!(rendered.contains("\"translator_comments\":[\"Keep short\"]"));
796        assert!(rendered.contains("{count, plural, one {# Datei} other {# Dateien}}"));
797
798        let parsed = parse_catalog_to_internal_ndjson(
799            &rendered,
800            None,
801            "en",
802            CatalogSemantics::IcuNative,
803            false,
804        )
805        .expect("roundtrip parse");
806
807        assert_eq!(parsed.locale.as_deref(), Some("de"));
808        assert_eq!(parsed.messages.len(), 2);
809        assert_eq!(parsed.messages[0].msgctxt.as_deref(), Some("nav"));
810        assert_eq!(parsed.messages[0].origins[0].file, "src/nav.rs");
811        assert_eq!(
812            parsed.messages[0].translator_comments,
813            vec!["Keep short".to_owned()]
814        );
815        assert_eq!(parsed.messages[0].flags, vec!["fuzzy".to_owned()]);
816        assert!(parsed.messages[1].obsolete);
817    }
818
819    #[test]
820    fn ndjson_streaming_reader_yields_public_catalog_messages() {
821        let rendered = stringify_catalog_ndjson(
822            &sample_catalog(),
823            Some("de"),
824            "en",
825            &PlaceholderCommentMode::Disabled,
826        );
827        let mut reader = NdjsonCatalogReader::with_options(
828            Cursor::new(rendered.as_bytes()),
829            NdjsonCatalogReaderOptions {
830                locale: None,
831                source_locale: "en",
832            },
833        )
834        .expect("streaming reader");
835
836        assert_eq!(reader.locale(), Some("de"));
837        let messages = reader
838            .by_ref()
839            .collect::<Result<Vec<_>, _>>()
840            .expect("stream messages");
841
842        assert_eq!(messages.len(), 2);
843        assert_eq!(messages[0].msgid, "About us");
844        assert_eq!(messages[0].msgctxt.as_deref(), Some("nav"));
845        assert!(matches!(
846            messages[0].translation,
847            TranslationShape::Singular { ref value } if value == "Ueber uns"
848        ));
849        assert_eq!(messages[0].origin[0].file, "src/nav.rs");
850        assert_eq!(
851            messages[0]
852                .extra
853                .as_ref()
854                .expect("extra")
855                .translator_comments,
856            vec!["Keep short".to_owned()]
857        );
858    }
859
860    #[test]
861    fn ndjson_streaming_writer_emits_parseable_records() {
862        let mut writer = NdjsonCatalogWriter::with_options(
863            Vec::new(),
864            NdjsonCatalogWriterOptions {
865                locale: Some("de"),
866                source_locale: "en",
867            },
868        )
869        .expect("streaming writer");
870
871        writer
872            .write_message(&CatalogMessage {
873                msgid: "Checkout".to_owned(),
874                msgctxt: Some("button".to_owned()),
875                translation: TranslationShape::Singular {
876                    value: "Zur Kasse".to_owned(),
877                },
878                comments: vec!["Short button label".to_owned()],
879                origin: vec![CatalogOrigin {
880                    file: "src/checkout.rs".to_owned(),
881                    line: Some(12),
882                }],
883                obsolete: false,
884                machine_translation: None,
885                extra: Some(CatalogMessageExtra {
886                    translator_comments: vec!["Keep concise".to_owned()],
887                    flags: vec!["rust-format".to_owned()],
888                }),
889            })
890            .expect("write message");
891
892        let rendered = String::from_utf8(writer.finish().expect("finish")).expect("utf8");
893        assert!(rendered.starts_with("---\nformat: ferrocat.ndjson.v1\n"));
894        assert!(rendered.contains("\"ctx\":\"button\""));
895        assert!(rendered.contains("\"translator_comments\":[\"Keep concise\"]"));
896
897        let parsed = parse_catalog_to_internal_ndjson(
898            &rendered,
899            None,
900            "en",
901            CatalogSemantics::IcuNative,
902            false,
903        )
904        .expect("parse streamed output");
905        assert_eq!(parsed.locale.as_deref(), Some("de"));
906        assert_eq!(parsed.messages[0].msgid, "Checkout");
907        assert_eq!(parsed.messages[0].flags, vec!["rust-format".to_owned()]);
908    }
909
910    #[test]
911    fn ndjson_streaming_reader_convenience_methods_expose_inner_reader() {
912        let input = concat!(
913            "---\n",
914            "format: ferrocat.ndjson.v1\n",
915            "source_locale: en\n",
916            "---\n",
917            "{\"id\":\"Checkout\",\"str\":\"Zur Kasse\"}\n",
918        );
919        let mut reader = NdjsonCatalogReader::new(Cursor::new(input.as_bytes()), "en")
920            .expect("streaming reader");
921
922        assert!(reader.get_ref().position() > 0);
923        assert!(reader.get_mut().position() > 0);
924        let message = reader
925            .next()
926            .expect("record")
927            .expect("streamed catalog message");
928        assert_eq!(message.msgid, "Checkout");
929        assert_eq!(reader.into_inner().position() as usize, input.len());
930    }
931
932    #[test]
933    fn ndjson_streaming_writer_convenience_methods_and_plural_records_work() {
934        let mut writer = NdjsonCatalogWriter::new(Vec::new(), "en").expect("streaming writer");
935        assert!(
936            writer
937                .get_ref()
938                .starts_with(b"---\nformat: ferrocat.ndjson.v1")
939        );
940        assert!(!writer.get_mut().is_empty());
941
942        let translation = BTreeMap::from([
943            ("one".to_owned(), "# Datei".to_owned()),
944            ("other".to_owned(), "# Dateien".to_owned()),
945        ]);
946        let hash = machine_translation_hash(EffectiveTranslationRef::Plural(&translation));
947        writer
948            .write_message(&CatalogMessage {
949                msgid: "files".to_owned(),
950                msgctxt: None,
951                translation: TranslationShape::Plural {
952                    source: super::super::PluralSource {
953                        one: Some("# file".to_owned()),
954                        other: "# files".to_owned(),
955                    },
956                    translation,
957                    variable: "count".to_owned(),
958                },
959                comments: Vec::new(),
960                origin: Vec::new(),
961                obsolete: false,
962                machine_translation: Some(MachineTranslationMetadata {
963                    model: "test/model".to_owned(),
964                    modified: None,
965                    confidence: Some(90),
966                    hash,
967                }),
968                extra: None,
969            })
970            .expect("write plural");
971
972        let rendered = String::from_utf8(writer.finish().expect("finish")).expect("utf8");
973        assert!(rendered.contains("{count, plural, one {# file} other {# files}}"));
974        assert!(rendered.contains("{count, plural, one {# Datei} other {# Dateien}}"));
975        assert!(rendered.contains("\"mt\":{\"model\":\"test/model\""));
976    }
977
978    #[test]
979    fn ndjson_parser_rejects_invalid_semantics_duplicates_and_bad_json() {
980        let duplicate = concat!(
981            "---\n",
982            "format: ferrocat.ndjson.v1\n",
983            "source_locale: en\n",
984            "---\n",
985            "{\"id\":\"About us\",\"str\":\"A\"}\n",
986            "{\"id\":\"About us\",\"str\":\"B\"}\n",
987        );
988        assert!(
989            parse_catalog_to_internal_ndjson(
990                duplicate,
991                None,
992                "en",
993                CatalogSemantics::IcuNative,
994                false
995            )
996            .is_err()
997        );
998
999        assert!(
1000            parse_catalog_to_internal_ndjson(
1001                concat!(
1002                    "---\n",
1003                    "format: ferrocat.ndjson.v1\n",
1004                    "source_locale: en\n",
1005                    "---\n",
1006                    "{\"id\":\"About us\",\"str\":\"A\",\n",
1007                ),
1008                None,
1009                "en",
1010                CatalogSemantics::IcuNative,
1011                false
1012            )
1013            .is_err()
1014        );
1015
1016        assert!(
1017            parse_catalog_to_internal_ndjson(
1018                concat!(
1019                    "---\n",
1020                    "format: ferrocat.ndjson.v1\n",
1021                    "source_locale: en\n",
1022                    "---\n",
1023                    "{\"id\":\"About us\",\"str\":\"A\"}\n",
1024                ),
1025                None,
1026                "en",
1027                CatalogSemantics::GettextCompat,
1028                false
1029            )
1030            .is_err()
1031        );
1032    }
1033}
ferrocat_po/api/ndjson.rs

ferrocat_po/api/
ndjson.rs
