1use std::borrow::Cow;
2
3use rustc_hash::FxHashMap;
4
5use crate::line_state::{PoLineContext, PoLineState};
6use crate::scan::{
7 CommentKind, Keyword, LineKind, LineScanner, classify_line, find_byte, find_quoted_bounds,
8 has_byte, parse_plural_index, split_once_byte, trim_ascii, unrecognized_po_line,
9};
10use crate::serialize::{write_keyword, write_prefixed_line};
11use crate::text::{escape_string_into, unescape_string_known, validate_quoted_content};
12use crate::utf8::input_slice_as_str;
13use crate::{BorrowedMsgStr, ParseError, ParsePosition, SerializeOptions};
14
15#[derive(Debug, Clone, PartialEq, Eq, Default)]
17pub struct MergeMessageInput<'a> {
18 pub msgctxt: Option<Cow<'a, str>>,
20 pub msgid: Cow<'a, str>,
22 pub msgid_plural: Option<Cow<'a, str>>,
24 pub references: Vec<Cow<'a, str>>,
26 pub extracted_comments: Vec<Cow<'a, str>>,
28 pub flags: Vec<Cow<'a, str>>,
30}
31
32#[derive(Debug, Clone, PartialEq, Eq, Default)]
33struct MergeBorrowedFile<'a> {
34 comments: Vec<&'a str>,
35 extracted_comments: Vec<&'a str>,
36 headers: Vec<MergeHeader<'a>>,
37 items: Vec<MergeBorrowedItem<'a>>,
38}
39
40#[derive(Debug, Clone, PartialEq, Eq, Default)]
41struct MergeHeader<'a> {
42 key: Cow<'a, str>,
43 value: Cow<'a, str>,
44}
45
46#[derive(Debug, Clone, PartialEq, Eq, Default)]
47struct MergeBorrowedItem<'a> {
48 msgid: Cow<'a, str>,
49 msgctxt: Option<Cow<'a, str>>,
50 references: Vec<&'a str>,
51 msgid_plural: Option<Cow<'a, str>>,
52 msgstr: BorrowedMsgStr<'a>,
53 comments: Vec<&'a str>,
54 extracted_comments: Vec<&'a str>,
55 flags: Vec<&'a str>,
56 metadata: Vec<(&'a str, &'a str)>,
57 obsolete: bool,
58 nplurals: usize,
59}
60
61impl MergeBorrowedItem<'_> {
62 fn new(nplurals: usize) -> Self {
63 Self {
64 nplurals,
65 ..Self::default()
66 }
67 }
68}
69
70#[derive(Debug)]
71struct ParserState<'a> {
72 item: MergeBorrowedItem<'a>,
73 header_entries: Vec<MergeHeader<'a>>,
74 msgstr: BorrowedMsgStr<'a>,
75 line: PoLineState,
76}
77
78impl<'a> ParserState<'a> {
79 fn new(nplurals: usize) -> Self {
80 Self {
81 item: MergeBorrowedItem::new(nplurals),
82 header_entries: Vec::new(),
83 msgstr: BorrowedMsgStr::None,
84 line: PoLineState::default(),
85 }
86 }
87
88 fn reset(&mut self, nplurals: usize) {
89 *self = Self::new(nplurals);
90 }
91
92 #[inline]
93 fn reset_after_take(&mut self, nplurals: usize) {
94 self.item.nplurals = nplurals;
95 self.header_entries.clear();
96 self.msgstr = BorrowedMsgStr::None;
97 self.line.reset();
98 }
99
100 fn set_msgstr(&mut self, plural_index: usize, value: Cow<'a, str>) {
101 self.msgstr.set_slot(plural_index, value);
102 }
103
104 fn append_msgstr(&mut self, plural_index: usize, value: Cow<'a, str>) {
105 self.msgstr.append_slot(plural_index, value);
106 }
107
108 fn materialize_msgstr(&mut self) {
109 self.item.msgstr = std::mem::take(&mut self.msgstr);
110 }
111}
112
113#[derive(Debug, Clone, Copy)]
114struct MergeLine<'a> {
115 trimmed: &'a [u8],
116 obsolete: bool,
117 position: ParsePosition,
118}
119
120pub fn merge_catalog<'a>(
147 existing_po: &'a str,
148 extracted_messages: &[MergeMessageInput<'a>],
149) -> Result<String, ParseError> {
150 let existing = parse_merge_po(existing_po)?;
151 let nplurals = parse_nplurals(&existing.headers).unwrap_or(2);
152 let options = SerializeOptions::default();
153 let mut out = String::with_capacity(estimate_merge_capacity(existing_po, extracted_messages));
154 let mut scratch = String::new();
155
156 write_file_preamble(&mut out, &existing);
157
158 let mut existing_index =
159 FxHashMap::<&str, Vec<(Option<&str>, usize)>>::with_capacity_and_hasher(
160 existing.items.len(),
161 Default::default(),
162 );
163 for (index, item) in existing.items.iter().enumerate() {
164 existing_index
165 .entry(item.msgid.as_ref())
166 .or_default()
167 .push((item.msgctxt.as_deref(), index));
168 }
169
170 let mut matched = vec![false; existing.items.len()];
171 let mut wrote_item = false;
172
173 for extracted in extracted_messages {
174 if wrote_item {
175 out.push('\n');
176 }
177 let existing_index = find_existing_index(
178 &existing_index,
179 extracted.msgctxt.as_deref(),
180 extracted.msgid.as_ref(),
181 );
182
183 match existing_index {
184 Some(index) => {
185 matched[index] = true;
186 write_merged_existing_item(
187 &mut out,
188 &mut scratch,
189 &existing.items[index],
190 extracted,
191 nplurals,
192 &options,
193 );
194 }
195 None => write_new_item(&mut out, &mut scratch, extracted, nplurals, &options),
196 }
197 out.push('\n');
198 wrote_item = true;
199 }
200
201 for (index, item) in existing.items.iter().enumerate() {
202 if matched[index] {
203 continue;
204 }
205 if wrote_item {
206 out.push('\n');
207 }
208 write_existing_item(&mut out, &mut scratch, item, true, &options);
209 out.push('\n');
210 wrote_item = true;
211 }
212
213 Ok(out)
214}
215
216fn parse_merge_po(input: &str) -> Result<MergeBorrowedFile<'_>, ParseError> {
217 let mut file = MergeBorrowedFile::default();
218 file.items.reserve((input.len() / 96).max(1));
219 let mut current_nplurals = 2usize;
220 let mut state = ParserState::new(current_nplurals);
221
222 for line in LineScanner::new(input.as_bytes()) {
223 parse_line(
224 MergeLine {
225 trimmed: line.trimmed,
226 obsolete: line.obsolete,
227 position: line.position,
228 },
229 &mut state,
230 &mut file,
231 &mut current_nplurals,
232 )?;
233 }
234
235 finish_item(&mut state, &mut file, &mut current_nplurals);
236 Ok(file)
237}
238
239fn parse_line<'a>(
240 line: MergeLine<'a>,
241 state: &mut ParserState<'a>,
242 file: &mut MergeBorrowedFile<'a>,
243 current_nplurals: &mut usize,
244) -> Result<(), ParseError> {
245 match classify_line(line.trimmed) {
246 LineKind::Continuation => {
247 append_continuation(line.trimmed, line.obsolete, line.position, state)?;
248 Ok(())
249 }
250 LineKind::Comment(kind) => {
251 parse_comment_line(line.trimmed, kind, state, file, current_nplurals);
252 Ok(())
253 }
254 LineKind::Keyword(keyword) => parse_keyword_line(
255 line.trimmed,
256 line.obsolete,
257 line.position,
258 keyword,
259 state,
260 file,
261 current_nplurals,
262 ),
263 LineKind::Other => Err(unrecognized_po_line(line.position)),
264 }
265}
266
267fn parse_comment_line<'a>(
268 line_bytes: &'a [u8],
269 kind: CommentKind,
270 state: &mut ParserState<'a>,
271 file: &mut MergeBorrowedFile<'a>,
272 current_nplurals: &mut usize,
273) {
274 finish_item(state, file, current_nplurals);
275
276 match kind {
277 CommentKind::Reference => state.item.references.push(trimmed_str(&line_bytes[2..])),
278 CommentKind::Flags => {
279 for flag in trimmed_str(&line_bytes[2..]).split(',') {
280 state.item.flags.push(flag.trim());
281 }
282 }
283 CommentKind::Extracted => state
284 .item
285 .extracted_comments
286 .push(trimmed_str(&line_bytes[2..])),
287 CommentKind::Metadata => {
288 let trimmed = trim_ascii(&line_bytes[2..]);
289 if let Some(value_bytes) = ferrocat_mt_metadata_value(trimmed) {
290 state
291 .item
292 .metadata
293 .push(("ferrocat-mt", trimmed_str(value_bytes)));
294 } else if let Some((key_bytes, value_bytes)) = split_once_byte(trimmed, b':') {
295 let key = trimmed_str(key_bytes);
296 if !key.is_empty() {
297 state.item.metadata.push((key, trimmed_str(value_bytes)));
298 }
299 }
300 }
301 CommentKind::Translator => state.item.comments.push(trimmed_str(&line_bytes[1..])),
302 CommentKind::Other => {}
303 }
304}
305
306fn ferrocat_mt_metadata_value(trimmed: &[u8]) -> Option<&[u8]> {
307 const KEY: &[u8] = b"ferrocat-mt";
308 let rest = trimmed.strip_prefix(KEY)?;
309 rest.first()
310 .is_some_and(u8::is_ascii_whitespace)
311 .then(|| trim_ascii(rest))
312}
313
314fn parse_keyword_line<'a>(
315 line_bytes: &'a [u8],
316 obsolete: bool,
317 position: ParsePosition,
318 keyword: Keyword,
319 state: &mut ParserState<'a>,
320 file: &mut MergeBorrowedFile<'a>,
321 current_nplurals: &mut usize,
322) -> Result<(), ParseError> {
323 match keyword {
324 Keyword::IdPlural => {
325 state
326 .line
327 .mark_keyword(PoLineContext::IdPlural, 0, obsolete);
328 state.item.msgid_plural = Some(at_line_position(
329 extract_merge_quoted_cow(line_bytes),
330 position,
331 )?);
332 }
333 Keyword::Id => {
334 finish_item(state, file, current_nplurals);
335 state.line.mark_keyword(PoLineContext::Id, 0, obsolete);
336 state.item.msgid = at_line_position(extract_merge_quoted_cow(line_bytes), position)?;
337 }
338 Keyword::Str => {
339 let plural_index = parse_plural_index(line_bytes).unwrap_or(0);
340 state
341 .line
342 .mark_keyword(PoLineContext::Str, plural_index, obsolete);
343 state.set_msgstr(
344 plural_index,
345 at_line_position(extract_merge_quoted_cow(line_bytes), position)?,
346 );
347 if is_header_candidate(state) {
348 let headers = at_line_position(parse_header_fragment(line_bytes), position)?;
349 state.header_entries.extend(headers);
350 }
351 }
352 Keyword::Ctxt => {
353 finish_item(state, file, current_nplurals);
354 state.line.mark_keyword(PoLineContext::Ctxt, 0, obsolete);
355 state.item.msgctxt = Some(at_line_position(
356 extract_merge_quoted_cow(line_bytes),
357 position,
358 )?);
359 }
360 }
361
362 Ok(())
363}
364
365fn append_continuation<'a>(
366 line_bytes: &'a [u8],
367 obsolete: bool,
368 position: ParsePosition,
369 state: &mut ParserState<'a>,
370) -> Result<(), ParseError> {
371 state.line.mark_continuation(obsolete);
372 let value = at_line_position(extract_merge_quoted_cow(line_bytes), position)?;
373
374 match state.line.context() {
375 Some(PoLineContext::Str) => {
376 state.append_msgstr(state.line.plural_index(), value);
377 if is_header_candidate(state) {
378 let headers = at_line_position(parse_header_fragment(line_bytes), position)?;
379 state.header_entries.extend(headers);
380 }
381 }
382 Some(PoLineContext::Id) => state.item.msgid.to_mut().push_str(value.as_ref()),
383 Some(PoLineContext::IdPlural) => {
384 let target = state.item.msgid_plural.get_or_insert(Cow::Borrowed(""));
385 target.to_mut().push_str(value.as_ref());
386 }
387 Some(PoLineContext::Ctxt) => {
388 let target = state.item.msgctxt.get_or_insert(Cow::Borrowed(""));
389 target.to_mut().push_str(value.as_ref());
390 }
391 None => {}
392 }
393
394 Ok(())
395}
396
397#[inline]
398fn at_line_position<T>(
399 result: Result<T, ParseError>,
400 position: ParsePosition,
401) -> Result<T, ParseError> {
402 result.map_err(|error| error.with_position_if_missing(position))
403}
404
405fn finish_item<'a>(
406 state: &mut ParserState<'a>,
407 file: &mut MergeBorrowedFile<'a>,
408 current_nplurals: &mut usize,
409) {
410 if !state.line.has_keyword() {
411 return;
412 }
413
414 if state.item.msgid.is_empty() && !is_header_state(state) {
415 return;
416 }
417
418 if state.line.is_obsolete_item() {
419 state.item.obsolete = true;
420 }
421
422 if is_header_state(state) && file.headers.is_empty() && file.items.is_empty() {
423 file.comments = std::mem::take(&mut state.item.comments);
424 file.extracted_comments = std::mem::take(&mut state.item.extracted_comments);
425 file.headers = std::mem::take(&mut state.header_entries);
426 *current_nplurals = parse_nplurals(&file.headers).unwrap_or(2);
427 state.reset(*current_nplurals);
428 return;
429 }
430
431 state.materialize_msgstr();
432
433 state.item.msgstr.ensure_singular();
434 if state.item.msgid_plural.is_some() {
435 state
436 .item
437 .msgstr
438 .expand_singular_to_plural_width(state.item.nplurals);
439 }
440
441 state.item.nplurals = *current_nplurals;
442 file.items.push(std::mem::take(&mut state.item));
443 state.reset_after_take(*current_nplurals);
444}
445
446fn is_header_state(state: &ParserState<'_>) -> bool {
447 state.item.msgid.is_empty()
448 && state.item.msgctxt.is_none()
449 && state.item.msgid_plural.is_none()
450 && !state.msgstr.is_empty()
451}
452
453fn is_header_candidate(state: &ParserState<'_>) -> bool {
454 state.item.msgid.is_empty()
455 && state.item.msgctxt.is_none()
456 && state.item.msgid_plural.is_none()
457 && state.line.plural_index() == 0
458}
459
460fn parse_header_fragment(line_bytes: &[u8]) -> Result<Vec<MergeHeader<'_>>, ParseError> {
461 let Some(raw) = merge_quoted_raw(line_bytes) else {
462 return Ok(Vec::new());
463 };
464
465 if header_fragment_is_borrowable(raw) {
466 return Ok(parse_header_fragment_borrowed(raw));
467 }
468
469 parse_header_fragment_owned(line_bytes)
470}
471
472fn parse_header_fragment_borrowed(raw: &[u8]) -> Vec<MergeHeader<'_>> {
473 let mut headers = Vec::new();
474 let mut start = 0usize;
475 let mut index = 0usize;
476
477 while index < raw.len() {
478 if raw[index] == b'\\' && raw.get(index + 1) == Some(&b'n') {
479 push_borrowed_header_segment(&raw[start..index], &mut headers);
480 index += 2;
481 start = index;
482 continue;
483 }
484 index += 1;
485 }
486
487 push_borrowed_header_segment(&raw[start..], &mut headers);
488 headers
489}
490
491fn push_borrowed_header_segment<'a>(segment: &'a [u8], out: &mut Vec<MergeHeader<'a>>) {
492 if segment.is_empty() {
493 return;
494 }
495 if let Some((key_bytes, value_bytes)) = split_once_byte(segment, b':') {
496 out.push(MergeHeader {
497 key: Cow::Borrowed(trimmed_str(key_bytes)),
498 value: Cow::Borrowed(trimmed_str(value_bytes)),
499 });
500 }
501}
502
503fn parse_header_fragment_owned(line_bytes: &[u8]) -> Result<Vec<MergeHeader<'_>>, ParseError> {
504 let decoded = extract_merge_quoted_cow(line_bytes)?;
505 let mut headers = Vec::new();
506 for segment in decoded.split('\n') {
507 if segment.is_empty() {
508 continue;
509 }
510 if let Some((key, value)) = segment.split_once(':') {
511 headers.push(MergeHeader {
512 key: Cow::Owned(key.trim().to_owned()),
513 value: Cow::Owned(value.trim().to_owned()),
514 });
515 }
516 }
517 Ok(headers)
518}
519
520fn header_fragment_is_borrowable(raw: &[u8]) -> bool {
521 let mut index = 0usize;
522 while index < raw.len() {
523 if raw[index] == b'\\' {
524 if raw.get(index + 1) != Some(&b'n') {
525 return false;
526 }
527 index += 2;
528 continue;
529 }
530 index += 1;
531 }
532 !has_byte(b'"', raw)
533}
534
535#[inline]
536fn extract_merge_quoted_cow(line_bytes: &[u8]) -> Result<Cow<'_, str>, ParseError> {
537 let Some(raw) = merge_quoted_raw(line_bytes) else {
538 return Ok(Cow::Borrowed(""));
539 };
540
541 if !validate_quoted_content(raw)? {
542 return Ok(Cow::Borrowed(bytes_to_str(raw)));
543 }
544
545 Ok(Cow::Owned(unescape_string_known(bytes_to_str(raw))?))
546}
547
548#[inline]
549fn merge_quoted_raw(line_bytes: &[u8]) -> Option<&[u8]> {
550 let start = match line_bytes.first() {
551 Some(b'"') => 1,
552 _ => find_byte(b'"', line_bytes)? + 1,
553 };
554
555 if start > line_bytes.len() {
556 return None;
557 }
558
559 if line_bytes.len() > start && line_bytes.last() == Some(&b'"') {
560 return Some(&line_bytes[start..line_bytes.len() - 1]);
561 }
562
563 let (quoted_start, quoted_end) = find_quoted_bounds(line_bytes)?;
564 Some(&line_bytes[quoted_start..quoted_end])
565}
566
567fn find_existing_index(
568 existing_index: &FxHashMap<&str, Vec<(Option<&str>, usize)>>,
569 msgctxt: Option<&str>,
570 msgid: &str,
571) -> Option<usize> {
572 let candidates = existing_index.get(msgid)?;
573 candidates
574 .iter()
575 .find_map(|(candidate_ctxt, index)| (*candidate_ctxt == msgctxt).then_some(*index))
576}
577
578fn estimate_merge_capacity(input: &str, extracted_messages: &[MergeMessageInput<'_>]) -> usize {
579 let extracted_bytes: usize = extracted_messages
580 .iter()
581 .map(|message| {
582 message.msgid.len()
583 + message.msgctxt.as_ref().map_or(0, |value| value.len())
584 + message.msgid_plural.as_ref().map_or(0, |value| value.len())
585 + message
586 .references
587 .iter()
588 .map(|value| value.len())
589 .sum::<usize>()
590 + message
591 .extracted_comments
592 .iter()
593 .map(|value| value.len())
594 .sum::<usize>()
595 + message.flags.iter().map(|value| value.len()).sum::<usize>()
596 })
597 .sum();
598
599 input.len() + extracted_bytes + 256
600}
601
602fn write_file_preamble(out: &mut String, file: &MergeBorrowedFile<'_>) {
603 write_prefixed_lines(out, "", "#", &file.comments);
604 write_prefixed_lines(out, "", "#.", &file.extracted_comments);
605
606 out.push_str("msgid \"\"\n");
607 out.push_str("msgstr \"\"\n");
608 for header in &file.headers {
609 out.push('"');
610 escape_string_into(out, header.key.as_ref());
611 out.push_str(": ");
612 escape_string_into(out, header.value.as_ref());
613 out.push_str("\\n\"\n");
614 }
615 out.push('\n');
616}
617
618fn write_merged_existing_item(
619 out: &mut String,
620 scratch: &mut String,
621 existing: &MergeBorrowedItem<'_>,
622 extracted: &MergeMessageInput<'_>,
623 nplurals: usize,
624 options: &SerializeOptions,
625) {
626 let obsolete_prefix = "";
627
628 write_prefixed_lines(out, obsolete_prefix, "#", &existing.comments);
629 write_prefixed_lines(out, obsolete_prefix, "#.", &extracted.extracted_comments);
630 write_metadata_lines(out, obsolete_prefix, &existing.metadata);
631 write_prefixed_lines(out, obsolete_prefix, "#:", &extracted.references);
632 write_merged_flags_line(out, obsolete_prefix, &existing.flags, &extracted.flags);
633
634 if let Some(context) = extracted.msgctxt.as_deref() {
635 write_keyword(
636 out,
637 scratch,
638 obsolete_prefix,
639 "msgctxt",
640 context,
641 None,
642 options,
643 );
644 }
645 write_keyword(
646 out,
647 scratch,
648 obsolete_prefix,
649 "msgid",
650 extracted.msgid.as_ref(),
651 None,
652 options,
653 );
654 if let Some(plural) = extracted.msgid_plural.as_deref() {
655 write_keyword(
656 out,
657 scratch,
658 obsolete_prefix,
659 "msgid_plural",
660 plural,
661 None,
662 options,
663 );
664 }
665
666 write_normalized_msgstr(
667 out,
668 scratch,
669 obsolete_prefix,
670 &existing.msgstr,
671 MsgstrShape {
672 preserve_existing: existing.msgid_plural.is_some() == extracted.msgid_plural.is_some(),
673 plural: extracted.msgid_plural.is_some(),
674 },
675 nplurals,
676 options,
677 );
678}
679
680fn write_new_item(
681 out: &mut String,
682 scratch: &mut String,
683 extracted: &MergeMessageInput<'_>,
684 nplurals: usize,
685 options: &SerializeOptions,
686) {
687 let obsolete_prefix = "";
688
689 write_prefixed_lines(out, obsolete_prefix, "#.", &extracted.extracted_comments);
690 write_prefixed_lines(out, obsolete_prefix, "#:", &extracted.references);
691 write_flags_line(out, obsolete_prefix, &extracted.flags);
692
693 if let Some(context) = extracted.msgctxt.as_deref() {
694 write_keyword(
695 out,
696 scratch,
697 obsolete_prefix,
698 "msgctxt",
699 context,
700 None,
701 options,
702 );
703 }
704 write_keyword(
705 out,
706 scratch,
707 obsolete_prefix,
708 "msgid",
709 extracted.msgid.as_ref(),
710 None,
711 options,
712 );
713 if let Some(plural) = extracted.msgid_plural.as_deref() {
714 write_keyword(
715 out,
716 scratch,
717 obsolete_prefix,
718 "msgid_plural",
719 plural,
720 None,
721 options,
722 );
723 }
724
725 write_default_msgstr(
726 out,
727 scratch,
728 obsolete_prefix,
729 extracted.msgid_plural.is_some(),
730 nplurals,
731 options,
732 );
733}
734
735fn write_existing_item(
736 out: &mut String,
737 scratch: &mut String,
738 item: &MergeBorrowedItem<'_>,
739 obsolete: bool,
740 options: &SerializeOptions,
741) {
742 let obsolete_prefix = if obsolete { "#~ " } else { "" };
743
744 write_prefixed_lines(out, obsolete_prefix, "#", &item.comments);
745 write_prefixed_lines(out, obsolete_prefix, "#.", &item.extracted_comments);
746 write_metadata_lines(out, obsolete_prefix, &item.metadata);
747 write_prefixed_lines(out, obsolete_prefix, "#:", &item.references);
748 write_flags_line(out, obsolete_prefix, &item.flags);
749
750 if let Some(context) = item.msgctxt.as_deref() {
751 write_keyword(
752 out,
753 scratch,
754 obsolete_prefix,
755 "msgctxt",
756 context,
757 None,
758 options,
759 );
760 }
761 write_keyword(
762 out,
763 scratch,
764 obsolete_prefix,
765 "msgid",
766 item.msgid.as_ref(),
767 None,
768 options,
769 );
770 if let Some(plural) = item.msgid_plural.as_deref() {
771 write_keyword(
772 out,
773 scratch,
774 obsolete_prefix,
775 "msgid_plural",
776 plural,
777 None,
778 options,
779 );
780 }
781
782 write_existing_msgstr(
783 out,
784 scratch,
785 obsolete_prefix,
786 &item.msgstr,
787 item.msgid_plural.is_some(),
788 item.nplurals,
789 options,
790 );
791}
792
793fn write_prefixed_lines<T: AsRef<str>>(
794 out: &mut String,
795 obsolete_prefix: &str,
796 prefix: &str,
797 values: &[T],
798) {
799 for value in values {
800 write_prefixed_line(out, obsolete_prefix, prefix, value.as_ref());
801 }
802}
803
804fn write_metadata_lines(out: &mut String, obsolete_prefix: &str, values: &[(&str, &str)]) {
805 for (key, value) in values {
806 out.push_str(obsolete_prefix);
807 write_metadata_line(out, key, value);
808 out.push('\n');
809 }
810}
811
812fn write_metadata_line(out: &mut String, key: &str, value: &str) {
813 out.push_str("#@ ");
814 out.push_str(key);
815 if key == "ferrocat-mt" {
816 out.push(' ');
817 } else {
818 out.push_str(": ");
819 }
820 out.push_str(value);
821}
822
823fn write_flags_line<T: AsRef<str>>(out: &mut String, obsolete_prefix: &str, values: &[T]) {
824 if values.is_empty() {
825 return;
826 }
827
828 out.push_str(obsolete_prefix);
829 out.push_str("#, ");
830 for (index, value) in values.iter().enumerate() {
831 if index > 0 {
832 out.push(',');
833 }
834 out.push_str(value.as_ref());
835 }
836 out.push('\n');
837}
838
839fn write_merged_flags_line(
840 out: &mut String,
841 obsolete_prefix: &str,
842 existing: &[&str],
843 extracted: &[Cow<'_, str>],
844) {
845 if existing.is_empty() && extracted.is_empty() {
846 return;
847 }
848
849 out.push_str(obsolete_prefix);
850 out.push_str("#, ");
851
852 let mut wrote_any = false;
853 let mut seen = Vec::with_capacity(existing.len() + extracted.len());
854 for flag in existing
855 .iter()
856 .copied()
857 .chain(extracted.iter().map(AsRef::as_ref))
858 {
859 if seen.contains(&flag) {
860 continue;
861 }
862 if wrote_any {
863 out.push(',');
864 }
865 out.push_str(flag);
866 wrote_any = true;
867 seen.push(flag);
868 }
869 out.push('\n');
870}
871
872fn write_existing_msgstr(
873 out: &mut String,
874 scratch: &mut String,
875 obsolete_prefix: &str,
876 msgstr: &BorrowedMsgStr<'_>,
877 is_plural: bool,
878 nplurals: usize,
879 options: &SerializeOptions,
880) {
881 if is_plural {
882 for index in 0..nplurals.max(1) {
883 let value = match msgstr {
884 BorrowedMsgStr::Singular(value) if index == 0 => value.as_ref(),
885 BorrowedMsgStr::None | BorrowedMsgStr::Singular(_) => "",
886 BorrowedMsgStr::Plural(values) => {
887 values.get(index).map_or("", |value| value.as_ref())
888 }
889 };
890 write_keyword(
891 out,
892 scratch,
893 obsolete_prefix,
894 "msgstr",
895 value,
896 Some(index),
897 options,
898 );
899 }
900 return;
901 }
902
903 let value = match msgstr {
904 BorrowedMsgStr::None => "",
905 BorrowedMsgStr::Singular(value) => value.as_ref(),
906 BorrowedMsgStr::Plural(values) => values.first().map_or("", |value| value.as_ref()),
907 };
908 write_keyword(
909 out,
910 scratch,
911 obsolete_prefix,
912 "msgstr",
913 value,
914 None,
915 options,
916 );
917}
918
919#[derive(Debug, Clone, Copy, PartialEq, Eq)]
920struct MsgstrShape {
921 preserve_existing: bool,
922 plural: bool,
923}
924
925fn write_normalized_msgstr(
926 out: &mut String,
927 scratch: &mut String,
928 obsolete_prefix: &str,
929 msgstr: &BorrowedMsgStr<'_>,
930 shape: MsgstrShape,
931 nplurals: usize,
932 options: &SerializeOptions,
933) {
934 if !shape.preserve_existing {
935 write_default_msgstr(
936 out,
937 scratch,
938 obsolete_prefix,
939 shape.plural,
940 nplurals,
941 options,
942 );
943 return;
944 }
945
946 write_existing_msgstr(
947 out,
948 scratch,
949 obsolete_prefix,
950 msgstr,
951 shape.plural,
952 nplurals,
953 options,
954 );
955}
956
957fn write_default_msgstr(
958 out: &mut String,
959 scratch: &mut String,
960 obsolete_prefix: &str,
961 is_plural: bool,
962 nplurals: usize,
963 options: &SerializeOptions,
964) {
965 if is_plural {
966 for index in 0..nplurals.max(1) {
967 write_keyword(
968 out,
969 scratch,
970 obsolete_prefix,
971 "msgstr",
972 "",
973 Some(index),
974 options,
975 );
976 }
977 return;
978 }
979
980 write_keyword(out, scratch, obsolete_prefix, "msgstr", "", None, options);
981}
982
983fn parse_nplurals(headers: &[MergeHeader<'_>]) -> Option<usize> {
984 let plural_forms = headers
985 .iter()
986 .find(|header| header.key.as_ref() == "Plural-Forms")?
987 .value
988 .as_bytes();
989 let mut rest = plural_forms;
990
991 while !rest.is_empty() {
992 let (part, next) = match split_once_byte(rest, b';') {
993 Some((part, tail)) => (part, tail),
994 None => (rest, &b""[..]),
995 };
996 let trimmed = trim_ascii(part);
997 if let Some((key, value)) = split_once_byte(trimmed, b'=')
998 && trim_ascii(key) == b"nplurals"
999 && let value = bytes_to_str(trim_ascii(value))
1000 && let Ok(parsed) = value.parse::<usize>()
1001 {
1002 return Some(parsed);
1003 }
1004 rest = next;
1005 }
1006
1007 None
1008}
1009
1010fn bytes_to_str(bytes: &[u8]) -> &str {
1011 input_slice_as_str(bytes)
1012}
1013
1014fn trimmed_str(bytes: &[u8]) -> &str {
1015 bytes_to_str(trim_ascii(bytes))
1016}
1017
1018#[cfg(test)]
1019mod tests {
1020 use std::borrow::Cow;
1021
1022 use rustc_hash::FxHashMap;
1023
1024 use super::{
1025 MergeHeader, MergeMessageInput, estimate_merge_capacity, extract_merge_quoted_cow,
1026 find_existing_index, header_fragment_is_borrowable, merge_catalog, parse_header_fragment,
1027 parse_nplurals,
1028 };
1029 use crate::{BorrowedMsgStr, parse_po};
1030
1031 #[test]
1032 fn preserves_existing_translations_and_updates_references() {
1033 let existing = concat!(
1034 "msgid \"hello\"\n",
1035 "msgstr \"world\"\n\n",
1036 "msgid \"old\"\n",
1037 "msgstr \"alt\"\n",
1038 );
1039 let extracted = vec![MergeMessageInput {
1040 msgid: Cow::Borrowed("hello"),
1041 references: vec![Cow::Borrowed("src/new.rs:10")],
1042 ..MergeMessageInput::default()
1043 }];
1044
1045 let merged = merge_catalog(existing, &extracted).expect("merge");
1046 let reparsed = parse_po(&merged).expect("reparse");
1047 let old_items: Vec<_> = reparsed
1048 .items
1049 .iter()
1050 .filter(|item| item.msgid == "old")
1051 .map(|item| (item.obsolete, item.msgstr[0].clone()))
1052 .collect();
1053 assert_eq!(old_items, vec![(true, "alt".to_owned())]);
1054
1055 let hello = reparsed
1056 .items
1057 .iter()
1058 .find(|item| item.msgid == "hello")
1059 .expect("merged hello item");
1060 assert_eq!(hello.msgstr[0], "world");
1061 assert_eq!(
1062 hello.references.as_slice(),
1063 vec!["src/new.rs:10".to_owned()].as_slice()
1064 );
1065 }
1066
1067 #[test]
1068 fn creates_new_items_for_new_extracted_messages() {
1069 let merged = merge_catalog(
1070 "",
1071 &[MergeMessageInput {
1072 msgid: Cow::Borrowed("fresh"),
1073 extracted_comments: vec![Cow::Borrowed("from extractor")],
1074 ..MergeMessageInput::default()
1075 }],
1076 )
1077 .expect("merge");
1078 let reparsed = parse_po(&merged).expect("reparse");
1079
1080 assert_eq!(reparsed.items[0].msgid, "fresh");
1081 assert_eq!(reparsed.items[0].msgstr[0], "");
1082 assert_eq!(
1083 reparsed.items[0].extracted_comments.as_slice(),
1084 vec!["from extractor".to_owned()].as_slice()
1085 );
1086 }
1087
1088 #[test]
1089 fn resets_msgstr_when_switching_between_singular_and_plural() {
1090 let existing = concat!("msgid \"count\"\n", "msgstr \"Anzahl\"\n",);
1091 let extracted = vec![MergeMessageInput {
1092 msgid: Cow::Borrowed("count"),
1093 msgid_plural: Some(Cow::Borrowed("counts")),
1094 ..MergeMessageInput::default()
1095 }];
1096
1097 let merged = merge_catalog(existing, &extracted).expect("merge");
1098 let reparsed = parse_po(&merged).expect("reparse");
1099
1100 assert!(reparsed.items[0].msgid_plural.is_some());
1101 assert_eq!(reparsed.items[0].msgstr.len(), 2);
1102 assert_eq!(reparsed.items[0].msgstr[0], "");
1103 assert_eq!(reparsed.items[0].msgstr[1], "");
1104 }
1105
1106 #[test]
1107 fn merge_helpers_cover_header_and_lookup_paths() {
1108 assert_eq!(
1109 extract_merge_quoted_cow(br#"msgid "plain""#),
1110 Ok(Cow::Borrowed("plain"))
1111 );
1112 assert_eq!(
1113 extract_merge_quoted_cow(br#"msgid "line\nbreak""#),
1114 Ok(Cow::Owned("line\nbreak".to_owned()))
1115 );
1116 assert!(header_fragment_is_borrowable(
1117 br#"Language: de\nPlural-Forms: nplurals=2;\n"#
1118 ));
1119 assert!(!header_fragment_is_borrowable(br#"Language: \"de\"\n"#));
1120 assert_eq!(
1121 parse_header_fragment(br#""Language: de\nPlural-Forms: nplurals=3; plural=(n>1);\n""#)
1122 .expect("header fragment"),
1123 vec![
1124 MergeHeader {
1125 key: Cow::Borrowed("Language"),
1126 value: Cow::Borrowed("de"),
1127 },
1128 MergeHeader {
1129 key: Cow::Borrowed("Plural-Forms"),
1130 value: Cow::Borrowed("nplurals=3; plural=(n>1);"),
1131 },
1132 ]
1133 );
1134 assert_eq!(
1135 parse_nplurals(&[
1136 MergeHeader {
1137 key: Cow::Borrowed("Language"),
1138 value: Cow::Borrowed("de"),
1139 },
1140 MergeHeader {
1141 key: Cow::Borrowed("Plural-Forms"),
1142 value: Cow::Borrowed(" nplurals = 4 ; plural = (n > 1); "),
1143 },
1144 ]),
1145 Some(4)
1146 );
1147 assert_eq!(
1148 find_existing_index(
1149 &FxHashMap::from_iter([("hello", vec![(Some("menu"), 3usize), (None, 1usize)],)]),
1150 Some("menu"),
1151 "hello",
1152 ),
1153 Some(3)
1154 );
1155 assert!(
1156 estimate_merge_capacity(
1157 "msgid \"a\"\nmsgstr \"b\"\n",
1158 &[MergeMessageInput {
1159 msgid: Cow::Borrowed("hello"),
1160 references: vec![Cow::Borrowed("src/app.rs:1")],
1161 ..MergeMessageInput::default()
1162 }],
1163 ) > 24
1164 );
1165 }
1166
1167 #[test]
1168 fn merge_preserves_existing_plural_values_and_dedupes_flags() {
1169 let existing = concat!(
1170 "msgid \"\"\n",
1171 "msgstr \"\"\n",
1172 "\"Plural-Forms: nplurals=3; plural=(n > 1);\\n\"\n",
1173 "\n",
1174 "#, fuzzy\n",
1175 "msgid \"count\"\n",
1176 "msgid_plural \"counts\"\n",
1177 "msgstr[0] \"eins\"\n",
1178 "msgstr[1] \"zwei\"\n",
1179 "msgstr[2] \"viele\"\n",
1180 );
1181 let extracted = vec![MergeMessageInput {
1182 msgid: Cow::Borrowed("count"),
1183 msgid_plural: Some(Cow::Borrowed("counts")),
1184 flags: vec![Cow::Borrowed("fuzzy"), Cow::Borrowed("rust-format")],
1185 ..MergeMessageInput::default()
1186 }];
1187
1188 let merged = merge_catalog(existing, &extracted).expect("merge plural");
1189 assert!(merged.contains("#, fuzzy,rust-format"));
1190 let reparsed = parse_po(&merged).expect("reparse merged plural");
1191 assert_eq!(reparsed.items[0].msgstr.len(), 3);
1192 assert_eq!(reparsed.items[0].msgstr[0], "eins");
1193 assert_eq!(reparsed.items[0].msgstr[2], "viele");
1194 }
1195
1196 #[test]
1197 fn merge_normalizes_crlf_input_and_keeps_existing_single_value() {
1198 let existing = "msgid \"hello\"\r\nmsgstr \"world\"\r\n";
1199 let merged = merge_catalog(
1200 existing,
1201 &[MergeMessageInput {
1202 msgid: Cow::Borrowed("hello"),
1203 ..MergeMessageInput::default()
1204 }],
1205 )
1206 .expect("merge normalized crlf");
1207
1208 let reparsed = parse_po(&merged).expect("reparse normalized");
1209 assert_eq!(reparsed.items[0].msgstr[0], "world");
1210 assert!(!matches!(
1211 BorrowedMsgStr::Singular(Cow::Borrowed("world")),
1212 BorrowedMsgStr::None
1213 ));
1214 }
1215
1216 #[test]
1217 fn merge_preserves_context_comments_metadata_and_marks_removed_items_obsolete() {
1218 let existing = concat!(
1219 "# Header translator\n",
1220 "#. Header extracted\n",
1221 "msgid \"\"\n",
1222 "msgstr \"\"\n",
1223 "\"Language: de\\n\"\n",
1224 "\"Plural-Forms: nplurals=2; plural=(n != 1);\\n\"\n",
1225 "\n",
1226 "# Translator note\n",
1227 "#. Old extracted note\n",
1228 "#@ owner: checkout\n",
1229 "#: old.rs:1\n",
1230 "#, fuzzy,php-format\n",
1231 "msgctxt \"button\"\n",
1232 "msgid \"Save\"\n",
1233 "\" now\"\n",
1234 "msgstr \"Speichern\"\n",
1235 "\n",
1236 "# Translator old only\n",
1237 "msgid \"Remove me\"\n",
1238 "msgstr \"Entferne mich\"\n",
1239 );
1240 let extracted = vec![MergeMessageInput {
1241 msgctxt: Some(Cow::Borrowed("button")),
1242 msgid: Cow::Borrowed("Save now"),
1243 references: vec![Cow::Borrowed("new.rs:2")],
1244 extracted_comments: vec![Cow::Borrowed("Fresh extracted note")],
1245 flags: vec![Cow::Borrowed("fuzzy"), Cow::Borrowed("rust-format")],
1246 ..MergeMessageInput::default()
1247 }];
1248
1249 let merged = merge_catalog(existing, &extracted).expect("merge context item");
1250
1251 assert!(merged.contains("# Header translator\n#. Header extracted\nmsgid \"\""));
1252 assert!(merged.contains("# Translator note\n#. Fresh extracted note\n#@ owner: checkout"));
1253 assert!(merged.contains("#: new.rs:2\n#, fuzzy,php-format,rust-format"));
1254 assert!(merged.contains("msgctxt \"button\"\nmsgid \"Save now\"\nmsgstr \"Speichern\""));
1255 assert!(merged.contains("#~ # Translator old only\n#~ msgid \"Remove me\""));
1256 }
1257
1258 #[test]
1259 fn merge_materializes_plural_slots_from_header_and_resets_shape_changes() {
1260 let existing = concat!(
1261 "msgid \"\"\n",
1262 "msgstr \"\"\n",
1263 "\"Plural-Forms: nplurals=3; plural=(n > 1);\\n\"\n",
1264 "\n",
1265 "msgid \"One file\"\n",
1266 "msgid_plural \"{count} files\"\n",
1267 "msgstr \"Eine Datei\"\n",
1268 "\n",
1269 "msgid \"Plain\"\n",
1270 "msgid_plural \"Plains\"\n",
1271 "msgstr[0] \"Singular\"\n",
1272 "msgstr[1] \"Plural\"\n",
1273 "msgstr[2] \"Viele\"\n",
1274 );
1275 let extracted = vec![
1276 MergeMessageInput {
1277 msgid: Cow::Borrowed("One file"),
1278 msgid_plural: Some(Cow::Borrowed("{count} files")),
1279 ..MergeMessageInput::default()
1280 },
1281 MergeMessageInput {
1282 msgid: Cow::Borrowed("Plain"),
1283 ..MergeMessageInput::default()
1284 },
1285 ];
1286
1287 let merged = merge_catalog(existing, &extracted).expect("merge shape changes");
1288 let reparsed = parse_po(&merged).expect("reparse");
1289
1290 let files = reparsed
1291 .items
1292 .iter()
1293 .find(|item| item.msgid == "One file")
1294 .expect("files item");
1295 assert_eq!(files.msgstr.len(), 3);
1296 assert_eq!(files.msgstr[0], "Eine Datei");
1297 assert_eq!(files.msgstr[1], "");
1298 assert_eq!(files.msgstr[2], "");
1299 let plain = reparsed
1300 .items
1301 .iter()
1302 .find(|item| item.msgid == "Plain")
1303 .expect("plain item");
1304 assert!(plain.msgid_plural.is_none());
1305 assert_eq!(plain.msgstr.len(), 1);
1306 assert_eq!(plain.msgstr[0], "");
1307 }
1308
1309 #[test]
1310 fn merge_writes_new_context_plural_and_obsoletes_unmatched_context_plural() {
1311 let existing = concat!(
1312 "msgid \"\"\n",
1313 "msgstr \"\"\n",
1314 "\"Plural-Forms: nplurals=2; plural=(n != 1);\\n\"\n",
1315 "\n",
1316 "# Translator plural\n",
1317 "#. Extracted plural\n",
1318 "#@ owner: files\n",
1319 "#: old.rs:4\n",
1320 "#, fuzzy\n",
1321 "msgctxt \"files\"\n",
1322 "msgid \"Old file\"\n",
1323 "msgid_plural \"Old files\"\n",
1324 "msgstr[0] \"Alte Datei\"\n",
1325 "msgstr[1] \"Alte Dateien\"\n",
1326 );
1327 let extracted = vec![MergeMessageInput {
1328 msgctxt: Some(Cow::Borrowed("files")),
1329 msgid: Cow::Borrowed("New file"),
1330 msgid_plural: Some(Cow::Borrowed("New files")),
1331 references: vec![Cow::Borrowed("new.rs:8")],
1332 extracted_comments: vec![Cow::Borrowed("Fresh plural")],
1333 flags: vec![Cow::Borrowed("rust-format")],
1334 }];
1335
1336 let merged = merge_catalog(existing, &extracted).expect("merge context plural");
1337
1338 assert!(merged.contains("#. Fresh plural\n#: new.rs:8\n#, rust-format\nmsgctxt \"files\"\nmsgid \"New file\"\nmsgid_plural \"New files\"\nmsgstr[0] \"\"\nmsgstr[1] \"\""));
1339 assert!(merged.contains("#~ # Translator plural\n#~ #. Extracted plural\n#~ #@ owner: files\n#~ #: old.rs:4\n#~ #, fuzzy\n#~ msgctxt \"files\"\n#~ msgid \"Old file\"\n#~ msgid_plural \"Old files\"\n#~ msgstr[0] \"Alte Datei\"\n#~ msgstr[1] \"Alte Dateien\""));
1340 }
1341
1342 #[test]
1343 fn merge_handles_owned_header_fragments_and_missing_quotes() {
1344 assert_eq!(
1345 parse_header_fragment(br#""Language: de\\nPlural-Forms: nplurals=2;\\n""#)
1346 .expect("owned escaped header fragment"),
1347 vec![MergeHeader {
1348 key: Cow::Owned("Language".to_owned()),
1349 value: Cow::Owned("de\\nPlural-Forms: nplurals=2;\\n".to_owned()),
1350 }]
1351 );
1352 assert_eq!(
1353 parse_header_fragment(b"msgstr").expect("missing quotes"),
1354 Vec::new()
1355 );
1356 assert_eq!(
1357 extract_merge_quoted_cow(b"msgid").expect("missing quote"),
1358 ""
1359 );
1360 assert_eq!(parse_nplurals(&[]), None);
1361 assert_eq!(
1362 parse_nplurals(&[MergeHeader {
1363 key: Cow::Borrowed("Plural-Forms"),
1364 value: Cow::Borrowed("plural=(n != 1); nplurals=two;"),
1365 }]),
1366 None
1367 );
1368 }
1369
1370 #[test]
1371 fn merge_parse_errors_include_line_position_for_plural_and_context_keywords() {
1372 let plural_error = merge_catalog(
1373 "msgid \"file\"\nmsgid_plural \"bad\"quote\"\nmsgstr[0] \"\"\n",
1374 &[],
1375 )
1376 .expect_err("unescaped plural quote should fail");
1377 let plural_position = plural_error.position().expect("plural position metadata");
1378 assert_eq!(plural_error.message(), "unescaped quote in string literal");
1379 assert_eq!(plural_position.line(), 2);
1380 assert_eq!(plural_position.column(), 1);
1381
1382 let context_error =
1383 merge_catalog("msgctxt \"bad\"quote\"\nmsgid \"x\"\nmsgstr \"\"\n", &[])
1384 .expect_err("unescaped context quote should fail");
1385 let context_position = context_error.position().expect("context position metadata");
1386 assert_eq!(context_error.message(), "unescaped quote in string literal");
1387 assert_eq!(context_position.line(), 1);
1388 assert_eq!(context_position.column(), 1);
1389 }
1390
1391 #[test]
1392 fn merge_rejects_unrecognized_existing_lines() {
1393 let extracted = [MergeMessageInput {
1394 msgid: Cow::Borrowed("ok"),
1395 ..MergeMessageInput::default()
1396 }];
1397 let error = merge_catalog("msgid \"ok\"\nmsgstr_ \"typo\"\n", &extracted)
1398 .expect_err("unknown existing PO line should fail");
1399 let position = error.position().expect("position metadata");
1400
1401 assert_eq!(error.message(), "unrecognized PO syntax");
1402 assert_eq!(position.line(), 2);
1403 assert_eq!(position.column(), 1);
1404 }
1405
1406 #[test]
1407 fn merge_parses_owned_header_fragments_in_keyword_and_continuation_lines() {
1408 let existing = concat!(
1409 "msgid \"\"\n",
1410 "msgstr \"Project-Id-Version: ferrocat\\t1\\n\"\n",
1411 "\"Language: de\\t\\n\"\n",
1412 "\n",
1413 "msgid \"hello\"\n",
1414 "msgstr \"world\"\n",
1415 );
1416 let merged = merge_catalog(
1417 existing,
1418 &[MergeMessageInput {
1419 msgid: Cow::Borrowed("hello"),
1420 ..MergeMessageInput::default()
1421 }],
1422 )
1423 .expect("merge with owned header fragments");
1424
1425 assert!(merged.contains("Project-Id-Version: ferrocat\\t1\\n"));
1426 assert!(merged.contains("Language: de\\n"));
1427 }
1428}