1use std::borrow::Cow;
2
3use crate::line_state::{PoLineContext, PoLineState};
4use crate::scan::{
5 CommentKind, Keyword, LineKind, LineScanner, classify_line, find_byte, find_quoted_bounds,
6 has_byte, parse_plural_index, split_once_byte, trim_ascii, unrecognized_po_line,
7};
8use crate::serialize::{write_keyword, write_prefixed_line};
9use crate::text::{escape_string_into, unescape_string, validate_quoted_content};
10use crate::utf8::input_slice_as_str;
11use crate::{BorrowedMsgStr, ParseError, ParsePosition, SerializeOptions};
12
13#[derive(Debug, Clone, PartialEq, Eq, Default)]
15pub struct ExtractedMessage<'a> {
16 pub msgctxt: Option<Cow<'a, str>>,
18 pub msgid: Cow<'a, str>,
20 pub msgid_plural: Option<Cow<'a, str>>,
22 pub references: Vec<Cow<'a, str>>,
24 pub extracted_comments: Vec<Cow<'a, str>>,
26 pub flags: Vec<Cow<'a, str>>,
28}
29
30#[derive(Debug, Clone, PartialEq, Eq, Default)]
31struct MergeBorrowedFile<'a> {
32 comments: Vec<&'a str>,
33 extracted_comments: Vec<&'a str>,
34 headers: Vec<MergeHeader<'a>>,
35 items: Vec<MergeBorrowedItem<'a>>,
36}
37
38#[derive(Debug, Clone, PartialEq, Eq, Default)]
39struct MergeHeader<'a> {
40 key: Cow<'a, str>,
41 value: Cow<'a, str>,
42}
43
44#[derive(Debug, Clone, PartialEq, Eq, Default)]
45struct MergeBorrowedItem<'a> {
46 msgid: Cow<'a, str>,
47 msgctxt: Option<Cow<'a, str>>,
48 references: Vec<&'a str>,
49 msgid_plural: Option<Cow<'a, str>>,
50 msgstr: BorrowedMsgStr<'a>,
51 comments: Vec<&'a str>,
52 extracted_comments: Vec<&'a str>,
53 flags: Vec<&'a str>,
54 metadata: Vec<(&'a str, &'a str)>,
55 obsolete: bool,
56 nplurals: usize,
57}
58
59impl MergeBorrowedItem<'_> {
60 fn new(nplurals: usize) -> Self {
61 Self {
62 nplurals,
63 ..Self::default()
64 }
65 }
66}
67
68#[derive(Debug)]
69struct ParserState<'a> {
70 item: MergeBorrowedItem<'a>,
71 header_entries: Vec<MergeHeader<'a>>,
72 msgstr: BorrowedMsgStr<'a>,
73 line: PoLineState,
74}
75
76impl<'a> ParserState<'a> {
77 fn new(nplurals: usize) -> Self {
78 Self {
79 item: MergeBorrowedItem::new(nplurals),
80 header_entries: Vec::new(),
81 msgstr: BorrowedMsgStr::None,
82 line: PoLineState::default(),
83 }
84 }
85
86 fn reset(&mut self, nplurals: usize) {
87 *self = Self::new(nplurals);
88 }
89
90 #[inline]
91 fn reset_after_take(&mut self, nplurals: usize) {
92 self.item.nplurals = nplurals;
93 self.header_entries.clear();
94 self.msgstr = BorrowedMsgStr::None;
95 self.line.reset();
96 }
97
98 fn set_msgstr(&mut self, plural_index: usize, value: Cow<'a, str>) {
99 self.msgstr.set_slot(plural_index, value);
100 }
101
102 fn append_msgstr(&mut self, plural_index: usize, value: Cow<'a, str>) {
103 self.msgstr.append_slot(plural_index, value);
104 }
105
106 fn materialize_msgstr(&mut self) {
107 self.item.msgstr = std::mem::take(&mut self.msgstr);
108 }
109}
110
111#[derive(Debug, Clone, Copy)]
112struct MergeLine<'a> {
113 trimmed: &'a [u8],
114 obsolete: bool,
115 position: ParsePosition,
116}
117
118pub fn merge_catalog<'a>(
145 existing_po: &'a str,
146 extracted_messages: &[ExtractedMessage<'a>],
147) -> Result<String, ParseError> {
148 let existing = parse_merge_po(existing_po)?;
149 let nplurals = parse_nplurals(&existing.headers).unwrap_or(2);
150 let options = SerializeOptions::default();
151 let mut out = String::with_capacity(estimate_merge_capacity(existing_po, extracted_messages));
152 let mut scratch = String::new();
153
154 write_file_preamble(&mut out, &existing);
155
156 let mut existing_index =
157 std::collections::HashMap::<&str, Vec<(Option<&str>, usize)>>::with_capacity(
158 existing.items.len(),
159 );
160 for (index, item) in existing.items.iter().enumerate() {
161 existing_index
162 .entry(item.msgid.as_ref())
163 .or_default()
164 .push((item.msgctxt.as_deref(), index));
165 }
166
167 let mut matched = vec![false; existing.items.len()];
168 let mut wrote_item = false;
169
170 for extracted in extracted_messages {
171 if wrote_item {
172 out.push('\n');
173 }
174 let existing_index = find_existing_index(
175 &existing_index,
176 extracted.msgctxt.as_deref(),
177 extracted.msgid.as_ref(),
178 );
179
180 match existing_index {
181 Some(index) => {
182 matched[index] = true;
183 write_merged_existing_item(
184 &mut out,
185 &mut scratch,
186 &existing.items[index],
187 extracted,
188 nplurals,
189 &options,
190 );
191 }
192 None => write_new_item(&mut out, &mut scratch, extracted, nplurals, &options),
193 }
194 out.push('\n');
195 wrote_item = true;
196 }
197
198 for (index, item) in existing.items.iter().enumerate() {
199 if matched[index] {
200 continue;
201 }
202 if wrote_item {
203 out.push('\n');
204 }
205 write_existing_item(&mut out, &mut scratch, item, true, &options);
206 out.push('\n');
207 wrote_item = true;
208 }
209
210 Ok(out)
211}
212
213fn parse_merge_po(input: &str) -> Result<MergeBorrowedFile<'_>, ParseError> {
214 let mut file = MergeBorrowedFile::default();
215 file.items.reserve((input.len() / 96).max(1));
216 let mut current_nplurals = 2usize;
217 let mut state = ParserState::new(current_nplurals);
218
219 for line in LineScanner::new(input.as_bytes()) {
220 parse_line(
221 MergeLine {
222 trimmed: line.trimmed,
223 obsolete: line.obsolete,
224 position: line.position,
225 },
226 &mut state,
227 &mut file,
228 &mut current_nplurals,
229 )?;
230 }
231
232 finish_item(&mut state, &mut file, &mut current_nplurals);
233 Ok(file)
234}
235
236fn parse_line<'a>(
237 line: MergeLine<'a>,
238 state: &mut ParserState<'a>,
239 file: &mut MergeBorrowedFile<'a>,
240 current_nplurals: &mut usize,
241) -> Result<(), ParseError> {
242 match classify_line(line.trimmed) {
243 LineKind::Continuation => {
244 append_continuation(line.trimmed, line.obsolete, line.position, state)?;
245 Ok(())
246 }
247 LineKind::Comment(kind) => {
248 parse_comment_line(line.trimmed, kind, state, file, current_nplurals);
249 Ok(())
250 }
251 LineKind::Keyword(keyword) => parse_keyword_line(
252 line.trimmed,
253 line.obsolete,
254 line.position,
255 keyword,
256 state,
257 file,
258 current_nplurals,
259 ),
260 LineKind::Other => Err(unrecognized_po_line(line.position)),
261 }
262}
263
264fn parse_comment_line<'a>(
265 line_bytes: &'a [u8],
266 kind: CommentKind,
267 state: &mut ParserState<'a>,
268 file: &mut MergeBorrowedFile<'a>,
269 current_nplurals: &mut usize,
270) {
271 finish_item(state, file, current_nplurals);
272
273 match kind {
274 CommentKind::Reference => state.item.references.push(trimmed_str(&line_bytes[2..])),
275 CommentKind::Flags => {
276 for flag in trimmed_str(&line_bytes[2..]).split(',') {
277 state.item.flags.push(flag.trim());
278 }
279 }
280 CommentKind::Extracted => state
281 .item
282 .extracted_comments
283 .push(trimmed_str(&line_bytes[2..])),
284 CommentKind::Metadata => {
285 let trimmed = trim_ascii(&line_bytes[2..]);
286 if let Some(value_bytes) = ferrocat_mt_metadata_value(trimmed) {
287 state
288 .item
289 .metadata
290 .push(("ferrocat-mt", trimmed_str(value_bytes)));
291 } else if let Some((key_bytes, value_bytes)) = split_once_byte(trimmed, b':') {
292 let key = trimmed_str(key_bytes);
293 if !key.is_empty() {
294 state.item.metadata.push((key, trimmed_str(value_bytes)));
295 }
296 }
297 }
298 CommentKind::Translator => state.item.comments.push(trimmed_str(&line_bytes[1..])),
299 CommentKind::Other => {}
300 }
301}
302
303fn ferrocat_mt_metadata_value(trimmed: &[u8]) -> Option<&[u8]> {
304 const KEY: &[u8] = b"ferrocat-mt";
305 let rest = trimmed.strip_prefix(KEY)?;
306 rest.first()
307 .is_some_and(u8::is_ascii_whitespace)
308 .then(|| trim_ascii(rest))
309}
310
311fn parse_keyword_line<'a>(
312 line_bytes: &'a [u8],
313 obsolete: bool,
314 position: ParsePosition,
315 keyword: Keyword,
316 state: &mut ParserState<'a>,
317 file: &mut MergeBorrowedFile<'a>,
318 current_nplurals: &mut usize,
319) -> Result<(), ParseError> {
320 match keyword {
321 Keyword::IdPlural => {
322 state
323 .line
324 .mark_keyword(PoLineContext::IdPlural, 0, obsolete);
325 state.item.msgid_plural = Some(at_line_position(
326 extract_merge_quoted_cow(line_bytes),
327 position,
328 )?);
329 }
330 Keyword::Id => {
331 finish_item(state, file, current_nplurals);
332 state.line.mark_keyword(PoLineContext::Id, 0, obsolete);
333 state.item.msgid = at_line_position(extract_merge_quoted_cow(line_bytes), position)?;
334 }
335 Keyword::Str => {
336 let plural_index = parse_plural_index(line_bytes).unwrap_or(0);
337 state
338 .line
339 .mark_keyword(PoLineContext::Str, plural_index, obsolete);
340 state.set_msgstr(
341 plural_index,
342 at_line_position(extract_merge_quoted_cow(line_bytes), position)?,
343 );
344 if is_header_candidate(state) {
345 let headers = at_line_position(parse_header_fragment(line_bytes), position)?;
346 state.header_entries.extend(headers);
347 }
348 }
349 Keyword::Ctxt => {
350 finish_item(state, file, current_nplurals);
351 state.line.mark_keyword(PoLineContext::Ctxt, 0, obsolete);
352 state.item.msgctxt = Some(at_line_position(
353 extract_merge_quoted_cow(line_bytes),
354 position,
355 )?);
356 }
357 }
358
359 Ok(())
360}
361
362fn append_continuation<'a>(
363 line_bytes: &'a [u8],
364 obsolete: bool,
365 position: ParsePosition,
366 state: &mut ParserState<'a>,
367) -> Result<(), ParseError> {
368 state.line.mark_continuation(obsolete);
369 let value = at_line_position(extract_merge_quoted_cow(line_bytes), position)?;
370
371 match state.line.context() {
372 Some(PoLineContext::Str) => {
373 state.append_msgstr(state.line.plural_index(), value);
374 if is_header_candidate(state) {
375 let headers = at_line_position(parse_header_fragment(line_bytes), position)?;
376 state.header_entries.extend(headers);
377 }
378 }
379 Some(PoLineContext::Id) => state.item.msgid.to_mut().push_str(value.as_ref()),
380 Some(PoLineContext::IdPlural) => {
381 let target = state.item.msgid_plural.get_or_insert(Cow::Borrowed(""));
382 target.to_mut().push_str(value.as_ref());
383 }
384 Some(PoLineContext::Ctxt) => {
385 let target = state.item.msgctxt.get_or_insert(Cow::Borrowed(""));
386 target.to_mut().push_str(value.as_ref());
387 }
388 None => {}
389 }
390
391 Ok(())
392}
393
394#[inline]
395fn at_line_position<T>(
396 result: Result<T, ParseError>,
397 position: ParsePosition,
398) -> Result<T, ParseError> {
399 result.map_err(|error| error.with_position_if_missing(position))
400}
401
402fn finish_item<'a>(
403 state: &mut ParserState<'a>,
404 file: &mut MergeBorrowedFile<'a>,
405 current_nplurals: &mut usize,
406) {
407 if !state.line.has_keyword() {
408 return;
409 }
410
411 if state.item.msgid.is_empty() && !is_header_state(state) {
412 return;
413 }
414
415 if state.line.is_obsolete_item() {
416 state.item.obsolete = true;
417 }
418
419 if is_header_state(state) && file.headers.is_empty() && file.items.is_empty() {
420 file.comments = std::mem::take(&mut state.item.comments);
421 file.extracted_comments = std::mem::take(&mut state.item.extracted_comments);
422 file.headers = std::mem::take(&mut state.header_entries);
423 *current_nplurals = parse_nplurals(&file.headers).unwrap_or(2);
424 state.reset(*current_nplurals);
425 return;
426 }
427
428 state.materialize_msgstr();
429
430 state.item.msgstr.ensure_singular();
431 if state.item.msgid_plural.is_some() {
432 state
433 .item
434 .msgstr
435 .expand_singular_to_plural_width(state.item.nplurals);
436 }
437
438 state.item.nplurals = *current_nplurals;
439 file.items.push(std::mem::take(&mut state.item));
440 state.reset_after_take(*current_nplurals);
441}
442
443fn is_header_state(state: &ParserState<'_>) -> bool {
444 state.item.msgid.is_empty()
445 && state.item.msgctxt.is_none()
446 && state.item.msgid_plural.is_none()
447 && !state.msgstr.is_empty()
448}
449
450fn is_header_candidate(state: &ParserState<'_>) -> bool {
451 state.item.msgid.is_empty()
452 && state.item.msgctxt.is_none()
453 && state.item.msgid_plural.is_none()
454 && state.line.plural_index() == 0
455}
456
457fn parse_header_fragment(line_bytes: &[u8]) -> Result<Vec<MergeHeader<'_>>, ParseError> {
458 let Some(raw) = merge_quoted_raw(line_bytes) else {
459 return Ok(Vec::new());
460 };
461
462 if header_fragment_is_borrowable(raw) {
463 return Ok(parse_header_fragment_borrowed(raw));
464 }
465
466 parse_header_fragment_owned(line_bytes)
467}
468
469fn parse_header_fragment_borrowed(raw: &[u8]) -> Vec<MergeHeader<'_>> {
470 let mut headers = Vec::new();
471 let mut start = 0usize;
472 let mut index = 0usize;
473
474 while index < raw.len() {
475 if raw[index] == b'\\' && raw.get(index + 1) == Some(&b'n') {
476 push_borrowed_header_segment(&raw[start..index], &mut headers);
477 index += 2;
478 start = index;
479 continue;
480 }
481 index += 1;
482 }
483
484 push_borrowed_header_segment(&raw[start..], &mut headers);
485 headers
486}
487
488fn push_borrowed_header_segment<'a>(segment: &'a [u8], out: &mut Vec<MergeHeader<'a>>) {
489 if segment.is_empty() {
490 return;
491 }
492 if let Some((key_bytes, value_bytes)) = split_once_byte(segment, b':') {
493 out.push(MergeHeader {
494 key: Cow::Borrowed(trimmed_str(key_bytes)),
495 value: Cow::Borrowed(trimmed_str(value_bytes)),
496 });
497 }
498}
499
500fn parse_header_fragment_owned(line_bytes: &[u8]) -> Result<Vec<MergeHeader<'_>>, ParseError> {
501 let decoded = extract_merge_quoted_cow(line_bytes)?;
502 let mut headers = Vec::new();
503 for segment in decoded.split('\n') {
504 if segment.is_empty() {
505 continue;
506 }
507 if let Some((key, value)) = segment.split_once(':') {
508 headers.push(MergeHeader {
509 key: Cow::Owned(key.trim().to_owned()),
510 value: Cow::Owned(value.trim().to_owned()),
511 });
512 }
513 }
514 Ok(headers)
515}
516
517fn header_fragment_is_borrowable(raw: &[u8]) -> bool {
518 let mut index = 0usize;
519 while index < raw.len() {
520 if raw[index] == b'\\' {
521 if raw.get(index + 1) != Some(&b'n') {
522 return false;
523 }
524 index += 2;
525 continue;
526 }
527 index += 1;
528 }
529 !has_byte(b'"', raw)
530}
531
532#[inline]
533fn extract_merge_quoted_cow(line_bytes: &[u8]) -> Result<Cow<'_, str>, ParseError> {
534 let Some(raw) = merge_quoted_raw(line_bytes) else {
535 return Ok(Cow::Borrowed(""));
536 };
537
538 validate_quoted_content(raw)?;
539 if !has_byte(b'\\', raw) {
540 return Ok(Cow::Borrowed(bytes_to_str(raw)));
541 }
542
543 Ok(Cow::Owned(unescape_string(bytes_to_str(raw))?))
544}
545
546#[inline]
547fn merge_quoted_raw(line_bytes: &[u8]) -> Option<&[u8]> {
548 let start = match line_bytes.first() {
549 Some(b'"') => 1,
550 _ => find_byte(b'"', line_bytes)? + 1,
551 };
552
553 if start > line_bytes.len() {
554 return None;
555 }
556
557 if line_bytes.len() > start && line_bytes.last() == Some(&b'"') {
558 return Some(&line_bytes[start..line_bytes.len() - 1]);
559 }
560
561 let (quoted_start, quoted_end) = find_quoted_bounds(line_bytes)?;
562 Some(&line_bytes[quoted_start..quoted_end])
563}
564
565fn find_existing_index(
566 existing_index: &std::collections::HashMap<&str, Vec<(Option<&str>, usize)>>,
567 msgctxt: Option<&str>,
568 msgid: &str,
569) -> Option<usize> {
570 let candidates = existing_index.get(msgid)?;
571 candidates
572 .iter()
573 .find_map(|(candidate_ctxt, index)| (*candidate_ctxt == msgctxt).then_some(*index))
574}
575
576fn estimate_merge_capacity(input: &str, extracted_messages: &[ExtractedMessage<'_>]) -> usize {
577 let extracted_bytes: usize = extracted_messages
578 .iter()
579 .map(|message| {
580 message.msgid.len()
581 + message.msgctxt.as_ref().map_or(0, |value| value.len())
582 + message.msgid_plural.as_ref().map_or(0, |value| value.len())
583 + message
584 .references
585 .iter()
586 .map(|value| value.len())
587 .sum::<usize>()
588 + message
589 .extracted_comments
590 .iter()
591 .map(|value| value.len())
592 .sum::<usize>()
593 + message.flags.iter().map(|value| value.len()).sum::<usize>()
594 })
595 .sum();
596
597 input.len() + extracted_bytes + 256
598}
599
600fn write_file_preamble(out: &mut String, file: &MergeBorrowedFile<'_>) {
601 write_prefixed_lines(out, "", "#", &file.comments);
602 write_prefixed_lines(out, "", "#.", &file.extracted_comments);
603
604 out.push_str("msgid \"\"\n");
605 out.push_str("msgstr \"\"\n");
606 for header in &file.headers {
607 out.push('"');
608 escape_string_into(out, header.key.as_ref());
609 out.push_str(": ");
610 escape_string_into(out, header.value.as_ref());
611 out.push_str("\\n\"\n");
612 }
613 out.push('\n');
614}
615
616fn write_merged_existing_item(
617 out: &mut String,
618 scratch: &mut String,
619 existing: &MergeBorrowedItem<'_>,
620 extracted: &ExtractedMessage<'_>,
621 nplurals: usize,
622 options: &SerializeOptions,
623) {
624 let obsolete_prefix = "";
625
626 write_prefixed_lines(out, obsolete_prefix, "#", &existing.comments);
627 write_prefixed_lines(out, obsolete_prefix, "#.", &extracted.extracted_comments);
628 write_metadata_lines(out, obsolete_prefix, &existing.metadata);
629 write_prefixed_lines(out, obsolete_prefix, "#:", &extracted.references);
630 write_merged_flags_line(out, obsolete_prefix, &existing.flags, &extracted.flags);
631
632 if let Some(context) = extracted.msgctxt.as_deref() {
633 write_keyword(
634 out,
635 scratch,
636 obsolete_prefix,
637 "msgctxt",
638 context,
639 None,
640 options,
641 );
642 }
643 write_keyword(
644 out,
645 scratch,
646 obsolete_prefix,
647 "msgid",
648 extracted.msgid.as_ref(),
649 None,
650 options,
651 );
652 if let Some(plural) = extracted.msgid_plural.as_deref() {
653 write_keyword(
654 out,
655 scratch,
656 obsolete_prefix,
657 "msgid_plural",
658 plural,
659 None,
660 options,
661 );
662 }
663
664 write_normalized_msgstr(
665 out,
666 scratch,
667 obsolete_prefix,
668 &existing.msgstr,
669 MsgstrShape {
670 preserve_existing: existing.msgid_plural.is_some() == extracted.msgid_plural.is_some(),
671 plural: extracted.msgid_plural.is_some(),
672 },
673 nplurals,
674 options,
675 );
676}
677
678fn write_new_item(
679 out: &mut String,
680 scratch: &mut String,
681 extracted: &ExtractedMessage<'_>,
682 nplurals: usize,
683 options: &SerializeOptions,
684) {
685 let obsolete_prefix = "";
686
687 write_prefixed_lines(out, obsolete_prefix, "#.", &extracted.extracted_comments);
688 write_prefixed_lines(out, obsolete_prefix, "#:", &extracted.references);
689 write_flags_line(out, obsolete_prefix, &extracted.flags);
690
691 if let Some(context) = extracted.msgctxt.as_deref() {
692 write_keyword(
693 out,
694 scratch,
695 obsolete_prefix,
696 "msgctxt",
697 context,
698 None,
699 options,
700 );
701 }
702 write_keyword(
703 out,
704 scratch,
705 obsolete_prefix,
706 "msgid",
707 extracted.msgid.as_ref(),
708 None,
709 options,
710 );
711 if let Some(plural) = extracted.msgid_plural.as_deref() {
712 write_keyword(
713 out,
714 scratch,
715 obsolete_prefix,
716 "msgid_plural",
717 plural,
718 None,
719 options,
720 );
721 }
722
723 write_default_msgstr(
724 out,
725 scratch,
726 obsolete_prefix,
727 extracted.msgid_plural.is_some(),
728 nplurals,
729 options,
730 );
731}
732
733fn write_existing_item(
734 out: &mut String,
735 scratch: &mut String,
736 item: &MergeBorrowedItem<'_>,
737 obsolete: bool,
738 options: &SerializeOptions,
739) {
740 let obsolete_prefix = if obsolete { "#~ " } else { "" };
741
742 write_prefixed_lines(out, obsolete_prefix, "#", &item.comments);
743 write_prefixed_lines(out, obsolete_prefix, "#.", &item.extracted_comments);
744 write_metadata_lines(out, obsolete_prefix, &item.metadata);
745 write_prefixed_lines(out, obsolete_prefix, "#:", &item.references);
746 write_flags_line(out, obsolete_prefix, &item.flags);
747
748 if let Some(context) = item.msgctxt.as_deref() {
749 write_keyword(
750 out,
751 scratch,
752 obsolete_prefix,
753 "msgctxt",
754 context,
755 None,
756 options,
757 );
758 }
759 write_keyword(
760 out,
761 scratch,
762 obsolete_prefix,
763 "msgid",
764 item.msgid.as_ref(),
765 None,
766 options,
767 );
768 if let Some(plural) = item.msgid_plural.as_deref() {
769 write_keyword(
770 out,
771 scratch,
772 obsolete_prefix,
773 "msgid_plural",
774 plural,
775 None,
776 options,
777 );
778 }
779
780 write_existing_msgstr(
781 out,
782 scratch,
783 obsolete_prefix,
784 &item.msgstr,
785 item.msgid_plural.is_some(),
786 item.nplurals,
787 options,
788 );
789}
790
791fn write_prefixed_lines<T: AsRef<str>>(
792 out: &mut String,
793 obsolete_prefix: &str,
794 prefix: &str,
795 values: &[T],
796) {
797 for value in values {
798 write_prefixed_line(out, obsolete_prefix, prefix, value.as_ref());
799 }
800}
801
802fn write_metadata_lines(out: &mut String, obsolete_prefix: &str, values: &[(&str, &str)]) {
803 for (key, value) in values {
804 out.push_str(obsolete_prefix);
805 write_metadata_line(out, key, value);
806 out.push('\n');
807 }
808}
809
810fn write_metadata_line(out: &mut String, key: &str, value: &str) {
811 out.push_str("#@ ");
812 out.push_str(key);
813 if key == "ferrocat-mt" {
814 out.push(' ');
815 } else {
816 out.push_str(": ");
817 }
818 out.push_str(value);
819}
820
821fn write_flags_line<T: AsRef<str>>(out: &mut String, obsolete_prefix: &str, values: &[T]) {
822 if values.is_empty() {
823 return;
824 }
825
826 out.push_str(obsolete_prefix);
827 out.push_str("#, ");
828 for (index, value) in values.iter().enumerate() {
829 if index > 0 {
830 out.push(',');
831 }
832 out.push_str(value.as_ref());
833 }
834 out.push('\n');
835}
836
837fn write_merged_flags_line(
838 out: &mut String,
839 obsolete_prefix: &str,
840 existing: &[&str],
841 extracted: &[Cow<'_, str>],
842) {
843 if existing.is_empty() && extracted.is_empty() {
844 return;
845 }
846
847 out.push_str(obsolete_prefix);
848 out.push_str("#, ");
849
850 let mut wrote_any = false;
851 let mut seen = Vec::with_capacity(existing.len() + extracted.len());
852 for flag in existing
853 .iter()
854 .copied()
855 .chain(extracted.iter().map(AsRef::as_ref))
856 {
857 if seen.contains(&flag) {
858 continue;
859 }
860 if wrote_any {
861 out.push(',');
862 }
863 out.push_str(flag);
864 wrote_any = true;
865 seen.push(flag);
866 }
867 out.push('\n');
868}
869
870fn write_existing_msgstr(
871 out: &mut String,
872 scratch: &mut String,
873 obsolete_prefix: &str,
874 msgstr: &BorrowedMsgStr<'_>,
875 is_plural: bool,
876 nplurals: usize,
877 options: &SerializeOptions,
878) {
879 if is_plural {
880 for index in 0..nplurals.max(1) {
881 let value = match msgstr {
882 BorrowedMsgStr::Singular(value) if index == 0 => value.as_ref(),
883 BorrowedMsgStr::None | BorrowedMsgStr::Singular(_) => "",
884 BorrowedMsgStr::Plural(values) => {
885 values.get(index).map_or("", |value| value.as_ref())
886 }
887 };
888 write_keyword(
889 out,
890 scratch,
891 obsolete_prefix,
892 "msgstr",
893 value,
894 Some(index),
895 options,
896 );
897 }
898 return;
899 }
900
901 let value = match msgstr {
902 BorrowedMsgStr::None => "",
903 BorrowedMsgStr::Singular(value) => value.as_ref(),
904 BorrowedMsgStr::Plural(values) => values.first().map_or("", |value| value.as_ref()),
905 };
906 write_keyword(
907 out,
908 scratch,
909 obsolete_prefix,
910 "msgstr",
911 value,
912 None,
913 options,
914 );
915}
916
917#[derive(Debug, Clone, Copy, PartialEq, Eq)]
918struct MsgstrShape {
919 preserve_existing: bool,
920 plural: bool,
921}
922
923fn write_normalized_msgstr(
924 out: &mut String,
925 scratch: &mut String,
926 obsolete_prefix: &str,
927 msgstr: &BorrowedMsgStr<'_>,
928 shape: MsgstrShape,
929 nplurals: usize,
930 options: &SerializeOptions,
931) {
932 if !shape.preserve_existing {
933 write_default_msgstr(
934 out,
935 scratch,
936 obsolete_prefix,
937 shape.plural,
938 nplurals,
939 options,
940 );
941 return;
942 }
943
944 write_existing_msgstr(
945 out,
946 scratch,
947 obsolete_prefix,
948 msgstr,
949 shape.plural,
950 nplurals,
951 options,
952 );
953}
954
955fn write_default_msgstr(
956 out: &mut String,
957 scratch: &mut String,
958 obsolete_prefix: &str,
959 is_plural: bool,
960 nplurals: usize,
961 options: &SerializeOptions,
962) {
963 if is_plural {
964 for index in 0..nplurals.max(1) {
965 write_keyword(
966 out,
967 scratch,
968 obsolete_prefix,
969 "msgstr",
970 "",
971 Some(index),
972 options,
973 );
974 }
975 return;
976 }
977
978 write_keyword(out, scratch, obsolete_prefix, "msgstr", "", None, options);
979}
980
981fn parse_nplurals(headers: &[MergeHeader<'_>]) -> Option<usize> {
982 let plural_forms = headers
983 .iter()
984 .find(|header| header.key.as_ref() == "Plural-Forms")?
985 .value
986 .as_bytes();
987 let mut rest = plural_forms;
988
989 while !rest.is_empty() {
990 let (part, next) = match split_once_byte(rest, b';') {
991 Some((part, tail)) => (part, tail),
992 None => (rest, &b""[..]),
993 };
994 let trimmed = trim_ascii(part);
995 if let Some((key, value)) = split_once_byte(trimmed, b'=')
996 && trim_ascii(key) == b"nplurals"
997 && let value = bytes_to_str(trim_ascii(value))
998 && let Ok(parsed) = value.parse::<usize>()
999 {
1000 return Some(parsed);
1001 }
1002 rest = next;
1003 }
1004
1005 None
1006}
1007
1008fn bytes_to_str(bytes: &[u8]) -> &str {
1009 input_slice_as_str(bytes)
1010}
1011
1012fn trimmed_str(bytes: &[u8]) -> &str {
1013 bytes_to_str(trim_ascii(bytes))
1014}
1015
1016#[cfg(test)]
1017mod tests {
1018 use std::borrow::Cow;
1019
1020 use super::{
1021 ExtractedMessage, MergeHeader, estimate_merge_capacity, extract_merge_quoted_cow,
1022 find_existing_index, header_fragment_is_borrowable, merge_catalog, parse_header_fragment,
1023 parse_nplurals,
1024 };
1025 use crate::{BorrowedMsgStr, parse_po};
1026
1027 #[test]
1028 fn preserves_existing_translations_and_updates_references() {
1029 let existing = concat!(
1030 "msgid \"hello\"\n",
1031 "msgstr \"world\"\n\n",
1032 "msgid \"old\"\n",
1033 "msgstr \"alt\"\n",
1034 );
1035 let extracted = vec![ExtractedMessage {
1036 msgid: Cow::Borrowed("hello"),
1037 references: vec![Cow::Borrowed("src/new.rs:10")],
1038 ..ExtractedMessage::default()
1039 }];
1040
1041 let merged = merge_catalog(existing, &extracted).expect("merge");
1042 let reparsed = parse_po(&merged).expect("reparse");
1043 let old_items: Vec<_> = reparsed
1044 .items
1045 .iter()
1046 .filter(|item| item.msgid == "old")
1047 .map(|item| (item.obsolete, item.msgstr[0].clone()))
1048 .collect();
1049 assert_eq!(old_items, vec![(true, "alt".to_owned())]);
1050
1051 let hello = reparsed
1052 .items
1053 .iter()
1054 .find(|item| item.msgid == "hello")
1055 .expect("merged hello item");
1056 assert_eq!(hello.msgstr[0], "world");
1057 assert_eq!(hello.references, vec!["src/new.rs:10".to_owned()]);
1058 }
1059
1060 #[test]
1061 fn creates_new_items_for_new_extracted_messages() {
1062 let merged = merge_catalog(
1063 "",
1064 &[ExtractedMessage {
1065 msgid: Cow::Borrowed("fresh"),
1066 extracted_comments: vec![Cow::Borrowed("from extractor")],
1067 ..ExtractedMessage::default()
1068 }],
1069 )
1070 .expect("merge");
1071 let reparsed = parse_po(&merged).expect("reparse");
1072
1073 assert_eq!(reparsed.items[0].msgid, "fresh");
1074 assert_eq!(reparsed.items[0].msgstr[0], "");
1075 assert_eq!(
1076 reparsed.items[0].extracted_comments,
1077 vec!["from extractor".to_owned()]
1078 );
1079 }
1080
1081 #[test]
1082 fn resets_msgstr_when_switching_between_singular_and_plural() {
1083 let existing = concat!("msgid \"count\"\n", "msgstr \"Anzahl\"\n",);
1084 let extracted = vec![ExtractedMessage {
1085 msgid: Cow::Borrowed("count"),
1086 msgid_plural: Some(Cow::Borrowed("counts")),
1087 ..ExtractedMessage::default()
1088 }];
1089
1090 let merged = merge_catalog(existing, &extracted).expect("merge");
1091 let reparsed = parse_po(&merged).expect("reparse");
1092
1093 assert!(reparsed.items[0].msgid_plural.is_some());
1094 assert_eq!(reparsed.items[0].msgstr.len(), 2);
1095 assert_eq!(reparsed.items[0].msgstr[0], "");
1096 assert_eq!(reparsed.items[0].msgstr[1], "");
1097 }
1098
1099 #[test]
1100 fn merge_helpers_cover_header_and_lookup_paths() {
1101 assert_eq!(
1102 extract_merge_quoted_cow(br#"msgid "plain""#),
1103 Ok(Cow::Borrowed("plain"))
1104 );
1105 assert_eq!(
1106 extract_merge_quoted_cow(br#"msgid "line\nbreak""#),
1107 Ok(Cow::Owned("line\nbreak".to_owned()))
1108 );
1109 assert!(header_fragment_is_borrowable(
1110 br#"Language: de\nPlural-Forms: nplurals=2;\n"#
1111 ));
1112 assert!(!header_fragment_is_borrowable(br#"Language: \"de\"\n"#));
1113 assert_eq!(
1114 parse_header_fragment(br#""Language: de\nPlural-Forms: nplurals=3; plural=(n>1);\n""#)
1115 .expect("header fragment"),
1116 vec![
1117 MergeHeader {
1118 key: Cow::Borrowed("Language"),
1119 value: Cow::Borrowed("de"),
1120 },
1121 MergeHeader {
1122 key: Cow::Borrowed("Plural-Forms"),
1123 value: Cow::Borrowed("nplurals=3; plural=(n>1);"),
1124 },
1125 ]
1126 );
1127 assert_eq!(
1128 parse_nplurals(&[
1129 MergeHeader {
1130 key: Cow::Borrowed("Language"),
1131 value: Cow::Borrowed("de"),
1132 },
1133 MergeHeader {
1134 key: Cow::Borrowed("Plural-Forms"),
1135 value: Cow::Borrowed(" nplurals = 4 ; plural = (n > 1); "),
1136 },
1137 ]),
1138 Some(4)
1139 );
1140 assert_eq!(
1141 find_existing_index(
1142 &std::collections::HashMap::from([(
1143 "hello",
1144 vec![(Some("menu"), 3usize), (None, 1usize)],
1145 )]),
1146 Some("menu"),
1147 "hello",
1148 ),
1149 Some(3)
1150 );
1151 assert!(
1152 estimate_merge_capacity(
1153 "msgid \"a\"\nmsgstr \"b\"\n",
1154 &[ExtractedMessage {
1155 msgid: Cow::Borrowed("hello"),
1156 references: vec![Cow::Borrowed("src/app.rs:1")],
1157 ..ExtractedMessage::default()
1158 }],
1159 ) > 24
1160 );
1161 }
1162
1163 #[test]
1164 fn merge_preserves_existing_plural_values_and_dedupes_flags() {
1165 let existing = concat!(
1166 "msgid \"\"\n",
1167 "msgstr \"\"\n",
1168 "\"Plural-Forms: nplurals=3; plural=(n > 1);\\n\"\n",
1169 "\n",
1170 "#, fuzzy\n",
1171 "msgid \"count\"\n",
1172 "msgid_plural \"counts\"\n",
1173 "msgstr[0] \"eins\"\n",
1174 "msgstr[1] \"zwei\"\n",
1175 "msgstr[2] \"viele\"\n",
1176 );
1177 let extracted = vec![ExtractedMessage {
1178 msgid: Cow::Borrowed("count"),
1179 msgid_plural: Some(Cow::Borrowed("counts")),
1180 flags: vec![Cow::Borrowed("fuzzy"), Cow::Borrowed("rust-format")],
1181 ..ExtractedMessage::default()
1182 }];
1183
1184 let merged = merge_catalog(existing, &extracted).expect("merge plural");
1185 assert!(merged.contains("#, fuzzy,rust-format"));
1186 let reparsed = parse_po(&merged).expect("reparse merged plural");
1187 assert_eq!(reparsed.items[0].msgstr.len(), 3);
1188 assert_eq!(reparsed.items[0].msgstr[0], "eins");
1189 assert_eq!(reparsed.items[0].msgstr[2], "viele");
1190 }
1191
1192 #[test]
1193 fn merge_normalizes_crlf_input_and_keeps_existing_single_value() {
1194 let existing = "msgid \"hello\"\r\nmsgstr \"world\"\r\n";
1195 let merged = merge_catalog(
1196 existing,
1197 &[ExtractedMessage {
1198 msgid: Cow::Borrowed("hello"),
1199 ..ExtractedMessage::default()
1200 }],
1201 )
1202 .expect("merge normalized crlf");
1203
1204 let reparsed = parse_po(&merged).expect("reparse normalized");
1205 assert_eq!(reparsed.items[0].msgstr[0], "world");
1206 assert!(!matches!(
1207 BorrowedMsgStr::Singular(Cow::Borrowed("world")),
1208 BorrowedMsgStr::None
1209 ));
1210 }
1211
1212 #[test]
1213 fn merge_preserves_context_comments_metadata_and_marks_removed_items_obsolete() {
1214 let existing = concat!(
1215 "# Header translator\n",
1216 "#. Header extracted\n",
1217 "msgid \"\"\n",
1218 "msgstr \"\"\n",
1219 "\"Language: de\\n\"\n",
1220 "\"Plural-Forms: nplurals=2; plural=(n != 1);\\n\"\n",
1221 "\n",
1222 "# Translator note\n",
1223 "#. Old extracted note\n",
1224 "#@ owner: checkout\n",
1225 "#: old.rs:1\n",
1226 "#, fuzzy,php-format\n",
1227 "msgctxt \"button\"\n",
1228 "msgid \"Save\"\n",
1229 "\" now\"\n",
1230 "msgstr \"Speichern\"\n",
1231 "\n",
1232 "# Translator old only\n",
1233 "msgid \"Remove me\"\n",
1234 "msgstr \"Entferne mich\"\n",
1235 );
1236 let extracted = vec![ExtractedMessage {
1237 msgctxt: Some(Cow::Borrowed("button")),
1238 msgid: Cow::Borrowed("Save now"),
1239 references: vec![Cow::Borrowed("new.rs:2")],
1240 extracted_comments: vec![Cow::Borrowed("Fresh extracted note")],
1241 flags: vec![Cow::Borrowed("fuzzy"), Cow::Borrowed("rust-format")],
1242 ..ExtractedMessage::default()
1243 }];
1244
1245 let merged = merge_catalog(existing, &extracted).expect("merge context item");
1246
1247 assert!(merged.contains("# Header translator\n#. Header extracted\nmsgid \"\""));
1248 assert!(merged.contains("# Translator note\n#. Fresh extracted note\n#@ owner: checkout"));
1249 assert!(merged.contains("#: new.rs:2\n#, fuzzy,php-format,rust-format"));
1250 assert!(merged.contains("msgctxt \"button\"\nmsgid \"Save now\"\nmsgstr \"Speichern\""));
1251 assert!(merged.contains("#~ # Translator old only\n#~ msgid \"Remove me\""));
1252 }
1253
1254 #[test]
1255 fn merge_materializes_plural_slots_from_header_and_resets_shape_changes() {
1256 let existing = concat!(
1257 "msgid \"\"\n",
1258 "msgstr \"\"\n",
1259 "\"Plural-Forms: nplurals=3; plural=(n > 1);\\n\"\n",
1260 "\n",
1261 "msgid \"One file\"\n",
1262 "msgid_plural \"{count} files\"\n",
1263 "msgstr \"Eine Datei\"\n",
1264 "\n",
1265 "msgid \"Plain\"\n",
1266 "msgid_plural \"Plains\"\n",
1267 "msgstr[0] \"Singular\"\n",
1268 "msgstr[1] \"Plural\"\n",
1269 "msgstr[2] \"Viele\"\n",
1270 );
1271 let extracted = vec![
1272 ExtractedMessage {
1273 msgid: Cow::Borrowed("One file"),
1274 msgid_plural: Some(Cow::Borrowed("{count} files")),
1275 ..ExtractedMessage::default()
1276 },
1277 ExtractedMessage {
1278 msgid: Cow::Borrowed("Plain"),
1279 ..ExtractedMessage::default()
1280 },
1281 ];
1282
1283 let merged = merge_catalog(existing, &extracted).expect("merge shape changes");
1284 let reparsed = parse_po(&merged).expect("reparse");
1285
1286 let files = reparsed
1287 .items
1288 .iter()
1289 .find(|item| item.msgid == "One file")
1290 .expect("files item");
1291 assert_eq!(files.msgstr.len(), 3);
1292 assert_eq!(files.msgstr[0], "Eine Datei");
1293 assert_eq!(files.msgstr[1], "");
1294 assert_eq!(files.msgstr[2], "");
1295 let plain = reparsed
1296 .items
1297 .iter()
1298 .find(|item| item.msgid == "Plain")
1299 .expect("plain item");
1300 assert!(plain.msgid_plural.is_none());
1301 assert_eq!(plain.msgstr.len(), 1);
1302 assert_eq!(plain.msgstr[0], "");
1303 }
1304
1305 #[test]
1306 fn merge_writes_new_context_plural_and_obsoletes_unmatched_context_plural() {
1307 let existing = concat!(
1308 "msgid \"\"\n",
1309 "msgstr \"\"\n",
1310 "\"Plural-Forms: nplurals=2; plural=(n != 1);\\n\"\n",
1311 "\n",
1312 "# Translator plural\n",
1313 "#. Extracted plural\n",
1314 "#@ owner: files\n",
1315 "#: old.rs:4\n",
1316 "#, fuzzy\n",
1317 "msgctxt \"files\"\n",
1318 "msgid \"Old file\"\n",
1319 "msgid_plural \"Old files\"\n",
1320 "msgstr[0] \"Alte Datei\"\n",
1321 "msgstr[1] \"Alte Dateien\"\n",
1322 );
1323 let extracted = vec![ExtractedMessage {
1324 msgctxt: Some(Cow::Borrowed("files")),
1325 msgid: Cow::Borrowed("New file"),
1326 msgid_plural: Some(Cow::Borrowed("New files")),
1327 references: vec![Cow::Borrowed("new.rs:8")],
1328 extracted_comments: vec![Cow::Borrowed("Fresh plural")],
1329 flags: vec![Cow::Borrowed("rust-format")],
1330 }];
1331
1332 let merged = merge_catalog(existing, &extracted).expect("merge context plural");
1333
1334 assert!(merged.contains("#. Fresh plural\n#: new.rs:8\n#, rust-format\nmsgctxt \"files\"\nmsgid \"New file\"\nmsgid_plural \"New files\"\nmsgstr[0] \"\"\nmsgstr[1] \"\""));
1335 assert!(merged.contains("#~ # Translator plural\n#~ #. Extracted plural\n#~ #@ owner: files\n#~ #: old.rs:4\n#~ #, fuzzy\n#~ msgctxt \"files\"\n#~ msgid \"Old file\"\n#~ msgid_plural \"Old files\"\n#~ msgstr[0] \"Alte Datei\"\n#~ msgstr[1] \"Alte Dateien\""));
1336 }
1337
1338 #[test]
1339 fn merge_handles_owned_header_fragments_and_missing_quotes() {
1340 assert_eq!(
1341 parse_header_fragment(br#""Language: de\\nPlural-Forms: nplurals=2;\\n""#)
1342 .expect("owned escaped header fragment"),
1343 vec![MergeHeader {
1344 key: Cow::Owned("Language".to_owned()),
1345 value: Cow::Owned("de\\nPlural-Forms: nplurals=2;\\n".to_owned()),
1346 }]
1347 );
1348 assert_eq!(
1349 parse_header_fragment(b"msgstr").expect("missing quotes"),
1350 Vec::new()
1351 );
1352 assert_eq!(
1353 extract_merge_quoted_cow(b"msgid").expect("missing quote"),
1354 ""
1355 );
1356 assert_eq!(parse_nplurals(&[]), None);
1357 assert_eq!(
1358 parse_nplurals(&[MergeHeader {
1359 key: Cow::Borrowed("Plural-Forms"),
1360 value: Cow::Borrowed("plural=(n != 1); nplurals=two;"),
1361 }]),
1362 None
1363 );
1364 }
1365
1366 #[test]
1367 fn merge_parse_errors_include_line_position_for_plural_and_context_keywords() {
1368 let plural_error = merge_catalog(
1369 "msgid \"file\"\nmsgid_plural \"bad\"quote\"\nmsgstr[0] \"\"\n",
1370 &[],
1371 )
1372 .expect_err("unescaped plural quote should fail");
1373 let plural_position = plural_error.position().expect("plural position metadata");
1374 assert_eq!(plural_error.message(), "unescaped quote in string literal");
1375 assert_eq!(plural_position.line(), 2);
1376 assert_eq!(plural_position.column(), 1);
1377
1378 let context_error =
1379 merge_catalog("msgctxt \"bad\"quote\"\nmsgid \"x\"\nmsgstr \"\"\n", &[])
1380 .expect_err("unescaped context quote should fail");
1381 let context_position = context_error.position().expect("context position metadata");
1382 assert_eq!(context_error.message(), "unescaped quote in string literal");
1383 assert_eq!(context_position.line(), 1);
1384 assert_eq!(context_position.column(), 1);
1385 }
1386
1387 #[test]
1388 fn merge_rejects_unrecognized_existing_lines() {
1389 let extracted = [ExtractedMessage {
1390 msgid: Cow::Borrowed("ok"),
1391 ..ExtractedMessage::default()
1392 }];
1393 let error = merge_catalog("msgid \"ok\"\nmsgstr_ \"typo\"\n", &extracted)
1394 .expect_err("unknown existing PO line should fail");
1395 let position = error.position().expect("position metadata");
1396
1397 assert_eq!(error.message(), "unrecognized PO syntax");
1398 assert_eq!(position.line(), 2);
1399 assert_eq!(position.column(), 1);
1400 }
1401
1402 #[test]
1403 fn merge_parses_owned_header_fragments_in_keyword_and_continuation_lines() {
1404 let existing = concat!(
1405 "msgid \"\"\n",
1406 "msgstr \"Project-Id-Version: ferrocat\\t1\\n\"\n",
1407 "\"Language: de\\t\\n\"\n",
1408 "\n",
1409 "msgid \"hello\"\n",
1410 "msgstr \"world\"\n",
1411 );
1412 let merged = merge_catalog(
1413 existing,
1414 &[ExtractedMessage {
1415 msgid: Cow::Borrowed("hello"),
1416 ..ExtractedMessage::default()
1417 }],
1418 )
1419 .expect("merge with owned header fragments");
1420
1421 assert!(merged.contains("Project-Id-Version: ferrocat\\t1\\n"));
1422 assert!(merged.contains("Language: de\\n"));
1423 }
1424}