use std::borrow::Cow;
use crate::line_state::{PoLineContext, PoLineState};
use crate::scan::{
CommentKind, Keyword, LineKind, LineScanner, classify_line, find_byte, find_quoted_bounds,
has_byte, parse_plural_index, split_once_byte, trim_ascii, unrecognized_po_line,
};
use crate::serialize::{write_keyword, write_prefixed_line};
use crate::text::{escape_string_into, unescape_string, validate_quoted_content};
use crate::utf8::input_slice_as_str;
use crate::{BorrowedMsgStr, ParseError, ParsePosition, SerializeOptions};
#[derive(Debug, Clone, PartialEq, Eq, Default)]
pub struct ExtractedMessage<'a> {
pub msgctxt: Option<Cow<'a, str>>,
pub msgid: Cow<'a, str>,
pub msgid_plural: Option<Cow<'a, str>>,
pub references: Vec<Cow<'a, str>>,
pub extracted_comments: Vec<Cow<'a, str>>,
pub flags: Vec<Cow<'a, str>>,
}
#[derive(Debug, Clone, PartialEq, Eq, Default)]
struct MergeBorrowedFile<'a> {
comments: Vec<&'a str>,
extracted_comments: Vec<&'a str>,
headers: Vec<MergeHeader<'a>>,
items: Vec<MergeBorrowedItem<'a>>,
}
#[derive(Debug, Clone, PartialEq, Eq, Default)]
struct MergeHeader<'a> {
key: Cow<'a, str>,
value: Cow<'a, str>,
}
#[derive(Debug, Clone, PartialEq, Eq, Default)]
struct MergeBorrowedItem<'a> {
msgid: Cow<'a, str>,
msgctxt: Option<Cow<'a, str>>,
references: Vec<&'a str>,
msgid_plural: Option<Cow<'a, str>>,
msgstr: BorrowedMsgStr<'a>,
comments: Vec<&'a str>,
extracted_comments: Vec<&'a str>,
flags: Vec<&'a str>,
metadata: Vec<(&'a str, &'a str)>,
obsolete: bool,
nplurals: usize,
}
impl MergeBorrowedItem<'_> {
fn new(nplurals: usize) -> Self {
Self {
nplurals,
..Self::default()
}
}
}
#[derive(Debug)]
struct ParserState<'a> {
item: MergeBorrowedItem<'a>,
header_entries: Vec<MergeHeader<'a>>,
msgstr: BorrowedMsgStr<'a>,
line: PoLineState,
}
impl<'a> ParserState<'a> {
fn new(nplurals: usize) -> Self {
Self {
item: MergeBorrowedItem::new(nplurals),
header_entries: Vec::new(),
msgstr: BorrowedMsgStr::None,
line: PoLineState::default(),
}
}
fn reset(&mut self, nplurals: usize) {
*self = Self::new(nplurals);
}
#[inline]
fn reset_after_take(&mut self, nplurals: usize) {
self.item.nplurals = nplurals;
self.header_entries.clear();
self.msgstr = BorrowedMsgStr::None;
self.line.reset();
}
fn set_msgstr(&mut self, plural_index: usize, value: Cow<'a, str>) {
self.msgstr.set_slot(plural_index, value);
}
fn append_msgstr(&mut self, plural_index: usize, value: Cow<'a, str>) {
self.msgstr.append_slot(plural_index, value);
}
fn materialize_msgstr(&mut self) {
self.item.msgstr = std::mem::take(&mut self.msgstr);
}
}
#[derive(Debug, Clone, Copy)]
struct MergeLine<'a> {
trimmed: &'a [u8],
obsolete: bool,
position: ParsePosition,
}
pub fn merge_catalog<'a>(
existing_po: &'a str,
extracted_messages: &[ExtractedMessage<'a>],
) -> Result<String, ParseError> {
let existing = parse_merge_po(existing_po)?;
let nplurals = parse_nplurals(&existing.headers).unwrap_or(2);
let options = SerializeOptions::default();
let mut out = String::with_capacity(estimate_merge_capacity(existing_po, extracted_messages));
let mut scratch = String::new();
write_file_preamble(&mut out, &existing);
let mut existing_index =
std::collections::HashMap::<&str, Vec<(Option<&str>, usize)>>::with_capacity(
existing.items.len(),
);
for (index, item) in existing.items.iter().enumerate() {
existing_index
.entry(item.msgid.as_ref())
.or_default()
.push((item.msgctxt.as_deref(), index));
}
let mut matched = vec![false; existing.items.len()];
let mut wrote_item = false;
for extracted in extracted_messages {
if wrote_item {
out.push('\n');
}
let existing_index = find_existing_index(
&existing_index,
extracted.msgctxt.as_deref(),
extracted.msgid.as_ref(),
);
match existing_index {
Some(index) => {
matched[index] = true;
write_merged_existing_item(
&mut out,
&mut scratch,
&existing.items[index],
extracted,
nplurals,
&options,
);
}
None => write_new_item(&mut out, &mut scratch, extracted, nplurals, &options),
}
out.push('\n');
wrote_item = true;
}
for (index, item) in existing.items.iter().enumerate() {
if matched[index] {
continue;
}
if wrote_item {
out.push('\n');
}
write_existing_item(&mut out, &mut scratch, item, true, &options);
out.push('\n');
wrote_item = true;
}
Ok(out)
}
fn parse_merge_po(input: &str) -> Result<MergeBorrowedFile<'_>, ParseError> {
let mut file = MergeBorrowedFile::default();
file.items.reserve((input.len() / 96).max(1));
let mut current_nplurals = 2usize;
let mut state = ParserState::new(current_nplurals);
for line in LineScanner::new(input.as_bytes()) {
parse_line(
MergeLine {
trimmed: line.trimmed,
obsolete: line.obsolete,
position: line.position,
},
&mut state,
&mut file,
&mut current_nplurals,
)?;
}
finish_item(&mut state, &mut file, &mut current_nplurals);
Ok(file)
}
fn parse_line<'a>(
line: MergeLine<'a>,
state: &mut ParserState<'a>,
file: &mut MergeBorrowedFile<'a>,
current_nplurals: &mut usize,
) -> Result<(), ParseError> {
match classify_line(line.trimmed) {
LineKind::Continuation => {
append_continuation(line.trimmed, line.obsolete, line.position, state)?;
Ok(())
}
LineKind::Comment(kind) => {
parse_comment_line(line.trimmed, kind, state, file, current_nplurals);
Ok(())
}
LineKind::Keyword(keyword) => parse_keyword_line(
line.trimmed,
line.obsolete,
line.position,
keyword,
state,
file,
current_nplurals,
),
LineKind::Other => Err(unrecognized_po_line(line.position)),
}
}
fn parse_comment_line<'a>(
line_bytes: &'a [u8],
kind: CommentKind,
state: &mut ParserState<'a>,
file: &mut MergeBorrowedFile<'a>,
current_nplurals: &mut usize,
) {
finish_item(state, file, current_nplurals);
match kind {
CommentKind::Reference => state.item.references.push(trimmed_str(&line_bytes[2..])),
CommentKind::Flags => {
for flag in trimmed_str(&line_bytes[2..]).split(',') {
state.item.flags.push(flag.trim());
}
}
CommentKind::Extracted => state
.item
.extracted_comments
.push(trimmed_str(&line_bytes[2..])),
CommentKind::Metadata => {
let trimmed = trim_ascii(&line_bytes[2..]);
if let Some(value_bytes) = ferrocat_mt_metadata_value(trimmed) {
state
.item
.metadata
.push(("ferrocat-mt", trimmed_str(value_bytes)));
} else if let Some((key_bytes, value_bytes)) = split_once_byte(trimmed, b':') {
let key = trimmed_str(key_bytes);
if !key.is_empty() {
state.item.metadata.push((key, trimmed_str(value_bytes)));
}
}
}
CommentKind::Translator => state.item.comments.push(trimmed_str(&line_bytes[1..])),
CommentKind::Other => {}
}
}
fn ferrocat_mt_metadata_value(trimmed: &[u8]) -> Option<&[u8]> {
const KEY: &[u8] = b"ferrocat-mt";
let rest = trimmed.strip_prefix(KEY)?;
rest.first()
.is_some_and(u8::is_ascii_whitespace)
.then(|| trim_ascii(rest))
}
fn parse_keyword_line<'a>(
line_bytes: &'a [u8],
obsolete: bool,
position: ParsePosition,
keyword: Keyword,
state: &mut ParserState<'a>,
file: &mut MergeBorrowedFile<'a>,
current_nplurals: &mut usize,
) -> Result<(), ParseError> {
match keyword {
Keyword::IdPlural => {
state
.line
.mark_keyword(PoLineContext::IdPlural, 0, obsolete);
state.item.msgid_plural = Some(at_line_position(
extract_merge_quoted_cow(line_bytes),
position,
)?);
}
Keyword::Id => {
finish_item(state, file, current_nplurals);
state.line.mark_keyword(PoLineContext::Id, 0, obsolete);
state.item.msgid = at_line_position(extract_merge_quoted_cow(line_bytes), position)?;
}
Keyword::Str => {
let plural_index = parse_plural_index(line_bytes).unwrap_or(0);
state
.line
.mark_keyword(PoLineContext::Str, plural_index, obsolete);
state.set_msgstr(
plural_index,
at_line_position(extract_merge_quoted_cow(line_bytes), position)?,
);
if is_header_candidate(state) {
let headers = at_line_position(parse_header_fragment(line_bytes), position)?;
state.header_entries.extend(headers);
}
}
Keyword::Ctxt => {
finish_item(state, file, current_nplurals);
state.line.mark_keyword(PoLineContext::Ctxt, 0, obsolete);
state.item.msgctxt = Some(at_line_position(
extract_merge_quoted_cow(line_bytes),
position,
)?);
}
}
Ok(())
}
fn append_continuation<'a>(
line_bytes: &'a [u8],
obsolete: bool,
position: ParsePosition,
state: &mut ParserState<'a>,
) -> Result<(), ParseError> {
state.line.mark_continuation(obsolete);
let value = at_line_position(extract_merge_quoted_cow(line_bytes), position)?;
match state.line.context() {
Some(PoLineContext::Str) => {
state.append_msgstr(state.line.plural_index(), value);
if is_header_candidate(state) {
let headers = at_line_position(parse_header_fragment(line_bytes), position)?;
state.header_entries.extend(headers);
}
}
Some(PoLineContext::Id) => state.item.msgid.to_mut().push_str(value.as_ref()),
Some(PoLineContext::IdPlural) => {
let target = state.item.msgid_plural.get_or_insert(Cow::Borrowed(""));
target.to_mut().push_str(value.as_ref());
}
Some(PoLineContext::Ctxt) => {
let target = state.item.msgctxt.get_or_insert(Cow::Borrowed(""));
target.to_mut().push_str(value.as_ref());
}
None => {}
}
Ok(())
}
#[inline]
fn at_line_position<T>(
result: Result<T, ParseError>,
position: ParsePosition,
) -> Result<T, ParseError> {
result.map_err(|error| error.with_position_if_missing(position))
}
fn finish_item<'a>(
state: &mut ParserState<'a>,
file: &mut MergeBorrowedFile<'a>,
current_nplurals: &mut usize,
) {
if !state.line.has_keyword() {
return;
}
if state.item.msgid.is_empty() && !is_header_state(state) {
return;
}
if state.line.is_obsolete_item() {
state.item.obsolete = true;
}
if is_header_state(state) && file.headers.is_empty() && file.items.is_empty() {
file.comments = std::mem::take(&mut state.item.comments);
file.extracted_comments = std::mem::take(&mut state.item.extracted_comments);
file.headers = std::mem::take(&mut state.header_entries);
*current_nplurals = parse_nplurals(&file.headers).unwrap_or(2);
state.reset(*current_nplurals);
return;
}
state.materialize_msgstr();
state.item.msgstr.ensure_singular();
if state.item.msgid_plural.is_some() {
state
.item
.msgstr
.expand_singular_to_plural_width(state.item.nplurals);
}
state.item.nplurals = *current_nplurals;
file.items.push(std::mem::take(&mut state.item));
state.reset_after_take(*current_nplurals);
}
fn is_header_state(state: &ParserState<'_>) -> bool {
state.item.msgid.is_empty()
&& state.item.msgctxt.is_none()
&& state.item.msgid_plural.is_none()
&& !state.msgstr.is_empty()
}
fn is_header_candidate(state: &ParserState<'_>) -> bool {
state.item.msgid.is_empty()
&& state.item.msgctxt.is_none()
&& state.item.msgid_plural.is_none()
&& state.line.plural_index() == 0
}
fn parse_header_fragment(line_bytes: &[u8]) -> Result<Vec<MergeHeader<'_>>, ParseError> {
let Some(raw) = merge_quoted_raw(line_bytes) else {
return Ok(Vec::new());
};
if header_fragment_is_borrowable(raw) {
return Ok(parse_header_fragment_borrowed(raw));
}
parse_header_fragment_owned(line_bytes)
}
fn parse_header_fragment_borrowed(raw: &[u8]) -> Vec<MergeHeader<'_>> {
let mut headers = Vec::new();
let mut start = 0usize;
let mut index = 0usize;
while index < raw.len() {
if raw[index] == b'\\' && raw.get(index + 1) == Some(&b'n') {
push_borrowed_header_segment(&raw[start..index], &mut headers);
index += 2;
start = index;
continue;
}
index += 1;
}
push_borrowed_header_segment(&raw[start..], &mut headers);
headers
}
fn push_borrowed_header_segment<'a>(segment: &'a [u8], out: &mut Vec<MergeHeader<'a>>) {
if segment.is_empty() {
return;
}
if let Some((key_bytes, value_bytes)) = split_once_byte(segment, b':') {
out.push(MergeHeader {
key: Cow::Borrowed(trimmed_str(key_bytes)),
value: Cow::Borrowed(trimmed_str(value_bytes)),
});
}
}
fn parse_header_fragment_owned(line_bytes: &[u8]) -> Result<Vec<MergeHeader<'_>>, ParseError> {
let decoded = extract_merge_quoted_cow(line_bytes)?;
let mut headers = Vec::new();
for segment in decoded.split('\n') {
if segment.is_empty() {
continue;
}
if let Some((key, value)) = segment.split_once(':') {
headers.push(MergeHeader {
key: Cow::Owned(key.trim().to_owned()),
value: Cow::Owned(value.trim().to_owned()),
});
}
}
Ok(headers)
}
fn header_fragment_is_borrowable(raw: &[u8]) -> bool {
let mut index = 0usize;
while index < raw.len() {
if raw[index] == b'\\' {
if raw.get(index + 1) != Some(&b'n') {
return false;
}
index += 2;
continue;
}
index += 1;
}
!has_byte(b'"', raw)
}
#[inline]
fn extract_merge_quoted_cow(line_bytes: &[u8]) -> Result<Cow<'_, str>, ParseError> {
let Some(raw) = merge_quoted_raw(line_bytes) else {
return Ok(Cow::Borrowed(""));
};
validate_quoted_content(raw)?;
if !has_byte(b'\\', raw) {
return Ok(Cow::Borrowed(bytes_to_str(raw)));
}
Ok(Cow::Owned(unescape_string(bytes_to_str(raw))?))
}
#[inline]
fn merge_quoted_raw(line_bytes: &[u8]) -> Option<&[u8]> {
let start = match line_bytes.first() {
Some(b'"') => 1,
_ => find_byte(b'"', line_bytes)? + 1,
};
if start > line_bytes.len() {
return None;
}
if line_bytes.len() > start && line_bytes.last() == Some(&b'"') {
return Some(&line_bytes[start..line_bytes.len() - 1]);
}
let (quoted_start, quoted_end) = find_quoted_bounds(line_bytes)?;
Some(&line_bytes[quoted_start..quoted_end])
}
fn find_existing_index(
existing_index: &std::collections::HashMap<&str, Vec<(Option<&str>, usize)>>,
msgctxt: Option<&str>,
msgid: &str,
) -> Option<usize> {
let candidates = existing_index.get(msgid)?;
candidates
.iter()
.find_map(|(candidate_ctxt, index)| (*candidate_ctxt == msgctxt).then_some(*index))
}
fn estimate_merge_capacity(input: &str, extracted_messages: &[ExtractedMessage<'_>]) -> usize {
let extracted_bytes: usize = extracted_messages
.iter()
.map(|message| {
message.msgid.len()
+ message.msgctxt.as_ref().map_or(0, |value| value.len())
+ message.msgid_plural.as_ref().map_or(0, |value| value.len())
+ message
.references
.iter()
.map(|value| value.len())
.sum::<usize>()
+ message
.extracted_comments
.iter()
.map(|value| value.len())
.sum::<usize>()
+ message.flags.iter().map(|value| value.len()).sum::<usize>()
})
.sum();
input.len() + extracted_bytes + 256
}
fn write_file_preamble(out: &mut String, file: &MergeBorrowedFile<'_>) {
write_prefixed_lines(out, "", "#", &file.comments);
write_prefixed_lines(out, "", "#.", &file.extracted_comments);
out.push_str("msgid \"\"\n");
out.push_str("msgstr \"\"\n");
for header in &file.headers {
out.push('"');
escape_string_into(out, header.key.as_ref());
out.push_str(": ");
escape_string_into(out, header.value.as_ref());
out.push_str("\\n\"\n");
}
out.push('\n');
}
fn write_merged_existing_item(
out: &mut String,
scratch: &mut String,
existing: &MergeBorrowedItem<'_>,
extracted: &ExtractedMessage<'_>,
nplurals: usize,
options: &SerializeOptions,
) {
let obsolete_prefix = "";
write_prefixed_lines(out, obsolete_prefix, "#", &existing.comments);
write_prefixed_lines(out, obsolete_prefix, "#.", &extracted.extracted_comments);
write_metadata_lines(out, obsolete_prefix, &existing.metadata);
write_prefixed_lines(out, obsolete_prefix, "#:", &extracted.references);
write_merged_flags_line(out, obsolete_prefix, &existing.flags, &extracted.flags);
if let Some(context) = extracted.msgctxt.as_deref() {
write_keyword(
out,
scratch,
obsolete_prefix,
"msgctxt",
context,
None,
options,
);
}
write_keyword(
out,
scratch,
obsolete_prefix,
"msgid",
extracted.msgid.as_ref(),
None,
options,
);
if let Some(plural) = extracted.msgid_plural.as_deref() {
write_keyword(
out,
scratch,
obsolete_prefix,
"msgid_plural",
plural,
None,
options,
);
}
write_normalized_msgstr(
out,
scratch,
obsolete_prefix,
&existing.msgstr,
MsgstrShape {
preserve_existing: existing.msgid_plural.is_some() == extracted.msgid_plural.is_some(),
plural: extracted.msgid_plural.is_some(),
},
nplurals,
options,
);
}
fn write_new_item(
out: &mut String,
scratch: &mut String,
extracted: &ExtractedMessage<'_>,
nplurals: usize,
options: &SerializeOptions,
) {
let obsolete_prefix = "";
write_prefixed_lines(out, obsolete_prefix, "#.", &extracted.extracted_comments);
write_prefixed_lines(out, obsolete_prefix, "#:", &extracted.references);
write_flags_line(out, obsolete_prefix, &extracted.flags);
if let Some(context) = extracted.msgctxt.as_deref() {
write_keyword(
out,
scratch,
obsolete_prefix,
"msgctxt",
context,
None,
options,
);
}
write_keyword(
out,
scratch,
obsolete_prefix,
"msgid",
extracted.msgid.as_ref(),
None,
options,
);
if let Some(plural) = extracted.msgid_plural.as_deref() {
write_keyword(
out,
scratch,
obsolete_prefix,
"msgid_plural",
plural,
None,
options,
);
}
write_default_msgstr(
out,
scratch,
obsolete_prefix,
extracted.msgid_plural.is_some(),
nplurals,
options,
);
}
fn write_existing_item(
out: &mut String,
scratch: &mut String,
item: &MergeBorrowedItem<'_>,
obsolete: bool,
options: &SerializeOptions,
) {
let obsolete_prefix = if obsolete { "#~ " } else { "" };
write_prefixed_lines(out, obsolete_prefix, "#", &item.comments);
write_prefixed_lines(out, obsolete_prefix, "#.", &item.extracted_comments);
write_metadata_lines(out, obsolete_prefix, &item.metadata);
write_prefixed_lines(out, obsolete_prefix, "#:", &item.references);
write_flags_line(out, obsolete_prefix, &item.flags);
if let Some(context) = item.msgctxt.as_deref() {
write_keyword(
out,
scratch,
obsolete_prefix,
"msgctxt",
context,
None,
options,
);
}
write_keyword(
out,
scratch,
obsolete_prefix,
"msgid",
item.msgid.as_ref(),
None,
options,
);
if let Some(plural) = item.msgid_plural.as_deref() {
write_keyword(
out,
scratch,
obsolete_prefix,
"msgid_plural",
plural,
None,
options,
);
}
write_existing_msgstr(
out,
scratch,
obsolete_prefix,
&item.msgstr,
item.msgid_plural.is_some(),
item.nplurals,
options,
);
}
fn write_prefixed_lines<T: AsRef<str>>(
out: &mut String,
obsolete_prefix: &str,
prefix: &str,
values: &[T],
) {
for value in values {
write_prefixed_line(out, obsolete_prefix, prefix, value.as_ref());
}
}
fn write_metadata_lines(out: &mut String, obsolete_prefix: &str, values: &[(&str, &str)]) {
for (key, value) in values {
out.push_str(obsolete_prefix);
write_metadata_line(out, key, value);
out.push('\n');
}
}
fn write_metadata_line(out: &mut String, key: &str, value: &str) {
out.push_str("#@ ");
out.push_str(key);
if key == "ferrocat-mt" {
out.push(' ');
} else {
out.push_str(": ");
}
out.push_str(value);
}
fn write_flags_line<T: AsRef<str>>(out: &mut String, obsolete_prefix: &str, values: &[T]) {
if values.is_empty() {
return;
}
out.push_str(obsolete_prefix);
out.push_str("#, ");
for (index, value) in values.iter().enumerate() {
if index > 0 {
out.push(',');
}
out.push_str(value.as_ref());
}
out.push('\n');
}
fn write_merged_flags_line(
out: &mut String,
obsolete_prefix: &str,
existing: &[&str],
extracted: &[Cow<'_, str>],
) {
if existing.is_empty() && extracted.is_empty() {
return;
}
out.push_str(obsolete_prefix);
out.push_str("#, ");
let mut wrote_any = false;
let mut seen = Vec::with_capacity(existing.len() + extracted.len());
for flag in existing
.iter()
.copied()
.chain(extracted.iter().map(AsRef::as_ref))
{
if seen.contains(&flag) {
continue;
}
if wrote_any {
out.push(',');
}
out.push_str(flag);
wrote_any = true;
seen.push(flag);
}
out.push('\n');
}
fn write_existing_msgstr(
out: &mut String,
scratch: &mut String,
obsolete_prefix: &str,
msgstr: &BorrowedMsgStr<'_>,
is_plural: bool,
nplurals: usize,
options: &SerializeOptions,
) {
if is_plural {
for index in 0..nplurals.max(1) {
let value = match msgstr {
BorrowedMsgStr::Singular(value) if index == 0 => value.as_ref(),
BorrowedMsgStr::None | BorrowedMsgStr::Singular(_) => "",
BorrowedMsgStr::Plural(values) => {
values.get(index).map_or("", |value| value.as_ref())
}
};
write_keyword(
out,
scratch,
obsolete_prefix,
"msgstr",
value,
Some(index),
options,
);
}
return;
}
let value = match msgstr {
BorrowedMsgStr::None => "",
BorrowedMsgStr::Singular(value) => value.as_ref(),
BorrowedMsgStr::Plural(values) => values.first().map_or("", |value| value.as_ref()),
};
write_keyword(
out,
scratch,
obsolete_prefix,
"msgstr",
value,
None,
options,
);
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
struct MsgstrShape {
preserve_existing: bool,
plural: bool,
}
fn write_normalized_msgstr(
out: &mut String,
scratch: &mut String,
obsolete_prefix: &str,
msgstr: &BorrowedMsgStr<'_>,
shape: MsgstrShape,
nplurals: usize,
options: &SerializeOptions,
) {
if !shape.preserve_existing {
write_default_msgstr(
out,
scratch,
obsolete_prefix,
shape.plural,
nplurals,
options,
);
return;
}
write_existing_msgstr(
out,
scratch,
obsolete_prefix,
msgstr,
shape.plural,
nplurals,
options,
);
}
fn write_default_msgstr(
out: &mut String,
scratch: &mut String,
obsolete_prefix: &str,
is_plural: bool,
nplurals: usize,
options: &SerializeOptions,
) {
if is_plural {
for index in 0..nplurals.max(1) {
write_keyword(
out,
scratch,
obsolete_prefix,
"msgstr",
"",
Some(index),
options,
);
}
return;
}
write_keyword(out, scratch, obsolete_prefix, "msgstr", "", None, options);
}
fn parse_nplurals(headers: &[MergeHeader<'_>]) -> Option<usize> {
let plural_forms = headers
.iter()
.find(|header| header.key.as_ref() == "Plural-Forms")?
.value
.as_bytes();
let mut rest = plural_forms;
while !rest.is_empty() {
let (part, next) = match split_once_byte(rest, b';') {
Some((part, tail)) => (part, tail),
None => (rest, &b""[..]),
};
let trimmed = trim_ascii(part);
if let Some((key, value)) = split_once_byte(trimmed, b'=')
&& trim_ascii(key) == b"nplurals"
&& let value = bytes_to_str(trim_ascii(value))
&& let Ok(parsed) = value.parse::<usize>()
{
return Some(parsed);
}
rest = next;
}
None
}
fn bytes_to_str(bytes: &[u8]) -> &str {
input_slice_as_str(bytes)
}
fn trimmed_str(bytes: &[u8]) -> &str {
bytes_to_str(trim_ascii(bytes))
}
#[cfg(test)]
mod tests {
use std::borrow::Cow;
use super::{
ExtractedMessage, MergeHeader, estimate_merge_capacity, extract_merge_quoted_cow,
find_existing_index, header_fragment_is_borrowable, merge_catalog, parse_header_fragment,
parse_nplurals,
};
use crate::{BorrowedMsgStr, parse_po};
#[test]
fn preserves_existing_translations_and_updates_references() {
let existing = concat!(
"msgid \"hello\"\n",
"msgstr \"world\"\n\n",
"msgid \"old\"\n",
"msgstr \"alt\"\n",
);
let extracted = vec![ExtractedMessage {
msgid: Cow::Borrowed("hello"),
references: vec![Cow::Borrowed("src/new.rs:10")],
..ExtractedMessage::default()
}];
let merged = merge_catalog(existing, &extracted).expect("merge");
let reparsed = parse_po(&merged).expect("reparse");
let old_items: Vec<_> = reparsed
.items
.iter()
.filter(|item| item.msgid == "old")
.map(|item| (item.obsolete, item.msgstr[0].clone()))
.collect();
assert_eq!(old_items, vec![(true, "alt".to_owned())]);
let hello = reparsed
.items
.iter()
.find(|item| item.msgid == "hello")
.expect("merged hello item");
assert_eq!(hello.msgstr[0], "world");
assert_eq!(hello.references, vec!["src/new.rs:10".to_owned()]);
}
#[test]
fn creates_new_items_for_new_extracted_messages() {
let merged = merge_catalog(
"",
&[ExtractedMessage {
msgid: Cow::Borrowed("fresh"),
extracted_comments: vec![Cow::Borrowed("from extractor")],
..ExtractedMessage::default()
}],
)
.expect("merge");
let reparsed = parse_po(&merged).expect("reparse");
assert_eq!(reparsed.items[0].msgid, "fresh");
assert_eq!(reparsed.items[0].msgstr[0], "");
assert_eq!(
reparsed.items[0].extracted_comments,
vec!["from extractor".to_owned()]
);
}
#[test]
fn resets_msgstr_when_switching_between_singular_and_plural() {
let existing = concat!("msgid \"count\"\n", "msgstr \"Anzahl\"\n",);
let extracted = vec![ExtractedMessage {
msgid: Cow::Borrowed("count"),
msgid_plural: Some(Cow::Borrowed("counts")),
..ExtractedMessage::default()
}];
let merged = merge_catalog(existing, &extracted).expect("merge");
let reparsed = parse_po(&merged).expect("reparse");
assert!(reparsed.items[0].msgid_plural.is_some());
assert_eq!(reparsed.items[0].msgstr.len(), 2);
assert_eq!(reparsed.items[0].msgstr[0], "");
assert_eq!(reparsed.items[0].msgstr[1], "");
}
#[test]
fn merge_helpers_cover_header_and_lookup_paths() {
assert_eq!(
extract_merge_quoted_cow(br#"msgid "plain""#),
Ok(Cow::Borrowed("plain"))
);
assert_eq!(
extract_merge_quoted_cow(br#"msgid "line\nbreak""#),
Ok(Cow::Owned("line\nbreak".to_owned()))
);
assert!(header_fragment_is_borrowable(
br#"Language: de\nPlural-Forms: nplurals=2;\n"#
));
assert!(!header_fragment_is_borrowable(br#"Language: \"de\"\n"#));
assert_eq!(
parse_header_fragment(br#""Language: de\nPlural-Forms: nplurals=3; plural=(n>1);\n""#)
.expect("header fragment"),
vec![
MergeHeader {
key: Cow::Borrowed("Language"),
value: Cow::Borrowed("de"),
},
MergeHeader {
key: Cow::Borrowed("Plural-Forms"),
value: Cow::Borrowed("nplurals=3; plural=(n>1);"),
},
]
);
assert_eq!(
parse_nplurals(&[
MergeHeader {
key: Cow::Borrowed("Language"),
value: Cow::Borrowed("de"),
},
MergeHeader {
key: Cow::Borrowed("Plural-Forms"),
value: Cow::Borrowed(" nplurals = 4 ; plural = (n > 1); "),
},
]),
Some(4)
);
assert_eq!(
find_existing_index(
&std::collections::HashMap::from([(
"hello",
vec![(Some("menu"), 3usize), (None, 1usize)],
)]),
Some("menu"),
"hello",
),
Some(3)
);
assert!(
estimate_merge_capacity(
"msgid \"a\"\nmsgstr \"b\"\n",
&[ExtractedMessage {
msgid: Cow::Borrowed("hello"),
references: vec![Cow::Borrowed("src/app.rs:1")],
..ExtractedMessage::default()
}],
) > 24
);
}
#[test]
fn merge_preserves_existing_plural_values_and_dedupes_flags() {
let existing = concat!(
"msgid \"\"\n",
"msgstr \"\"\n",
"\"Plural-Forms: nplurals=3; plural=(n > 1);\\n\"\n",
"\n",
"#, fuzzy\n",
"msgid \"count\"\n",
"msgid_plural \"counts\"\n",
"msgstr[0] \"eins\"\n",
"msgstr[1] \"zwei\"\n",
"msgstr[2] \"viele\"\n",
);
let extracted = vec![ExtractedMessage {
msgid: Cow::Borrowed("count"),
msgid_plural: Some(Cow::Borrowed("counts")),
flags: vec![Cow::Borrowed("fuzzy"), Cow::Borrowed("rust-format")],
..ExtractedMessage::default()
}];
let merged = merge_catalog(existing, &extracted).expect("merge plural");
assert!(merged.contains("#, fuzzy,rust-format"));
let reparsed = parse_po(&merged).expect("reparse merged plural");
assert_eq!(reparsed.items[0].msgstr.len(), 3);
assert_eq!(reparsed.items[0].msgstr[0], "eins");
assert_eq!(reparsed.items[0].msgstr[2], "viele");
}
#[test]
fn merge_normalizes_crlf_input_and_keeps_existing_single_value() {
let existing = "msgid \"hello\"\r\nmsgstr \"world\"\r\n";
let merged = merge_catalog(
existing,
&[ExtractedMessage {
msgid: Cow::Borrowed("hello"),
..ExtractedMessage::default()
}],
)
.expect("merge normalized crlf");
let reparsed = parse_po(&merged).expect("reparse normalized");
assert_eq!(reparsed.items[0].msgstr[0], "world");
assert!(!matches!(
BorrowedMsgStr::Singular(Cow::Borrowed("world")),
BorrowedMsgStr::None
));
}
#[test]
fn merge_preserves_context_comments_metadata_and_marks_removed_items_obsolete() {
let existing = concat!(
"# Header translator\n",
"#. Header extracted\n",
"msgid \"\"\n",
"msgstr \"\"\n",
"\"Language: de\\n\"\n",
"\"Plural-Forms: nplurals=2; plural=(n != 1);\\n\"\n",
"\n",
"# Translator note\n",
"#. Old extracted note\n",
"#@ owner: checkout\n",
"#: old.rs:1\n",
"#, fuzzy,php-format\n",
"msgctxt \"button\"\n",
"msgid \"Save\"\n",
"\" now\"\n",
"msgstr \"Speichern\"\n",
"\n",
"# Translator old only\n",
"msgid \"Remove me\"\n",
"msgstr \"Entferne mich\"\n",
);
let extracted = vec![ExtractedMessage {
msgctxt: Some(Cow::Borrowed("button")),
msgid: Cow::Borrowed("Save now"),
references: vec![Cow::Borrowed("new.rs:2")],
extracted_comments: vec![Cow::Borrowed("Fresh extracted note")],
flags: vec![Cow::Borrowed("fuzzy"), Cow::Borrowed("rust-format")],
..ExtractedMessage::default()
}];
let merged = merge_catalog(existing, &extracted).expect("merge context item");
assert!(merged.contains("# Header translator\n#. Header extracted\nmsgid \"\""));
assert!(merged.contains("# Translator note\n#. Fresh extracted note\n#@ owner: checkout"));
assert!(merged.contains("#: new.rs:2\n#, fuzzy,php-format,rust-format"));
assert!(merged.contains("msgctxt \"button\"\nmsgid \"Save now\"\nmsgstr \"Speichern\""));
assert!(merged.contains("#~ # Translator old only\n#~ msgid \"Remove me\""));
}
#[test]
fn merge_materializes_plural_slots_from_header_and_resets_shape_changes() {
let existing = concat!(
"msgid \"\"\n",
"msgstr \"\"\n",
"\"Plural-Forms: nplurals=3; plural=(n > 1);\\n\"\n",
"\n",
"msgid \"One file\"\n",
"msgid_plural \"{count} files\"\n",
"msgstr \"Eine Datei\"\n",
"\n",
"msgid \"Plain\"\n",
"msgid_plural \"Plains\"\n",
"msgstr[0] \"Singular\"\n",
"msgstr[1] \"Plural\"\n",
"msgstr[2] \"Viele\"\n",
);
let extracted = vec![
ExtractedMessage {
msgid: Cow::Borrowed("One file"),
msgid_plural: Some(Cow::Borrowed("{count} files")),
..ExtractedMessage::default()
},
ExtractedMessage {
msgid: Cow::Borrowed("Plain"),
..ExtractedMessage::default()
},
];
let merged = merge_catalog(existing, &extracted).expect("merge shape changes");
let reparsed = parse_po(&merged).expect("reparse");
let files = reparsed
.items
.iter()
.find(|item| item.msgid == "One file")
.expect("files item");
assert_eq!(files.msgstr.len(), 3);
assert_eq!(files.msgstr[0], "Eine Datei");
assert_eq!(files.msgstr[1], "");
assert_eq!(files.msgstr[2], "");
let plain = reparsed
.items
.iter()
.find(|item| item.msgid == "Plain")
.expect("plain item");
assert!(plain.msgid_plural.is_none());
assert_eq!(plain.msgstr.len(), 1);
assert_eq!(plain.msgstr[0], "");
}
#[test]
fn merge_writes_new_context_plural_and_obsoletes_unmatched_context_plural() {
let existing = concat!(
"msgid \"\"\n",
"msgstr \"\"\n",
"\"Plural-Forms: nplurals=2; plural=(n != 1);\\n\"\n",
"\n",
"# Translator plural\n",
"#. Extracted plural\n",
"#@ owner: files\n",
"#: old.rs:4\n",
"#, fuzzy\n",
"msgctxt \"files\"\n",
"msgid \"Old file\"\n",
"msgid_plural \"Old files\"\n",
"msgstr[0] \"Alte Datei\"\n",
"msgstr[1] \"Alte Dateien\"\n",
);
let extracted = vec![ExtractedMessage {
msgctxt: Some(Cow::Borrowed("files")),
msgid: Cow::Borrowed("New file"),
msgid_plural: Some(Cow::Borrowed("New files")),
references: vec![Cow::Borrowed("new.rs:8")],
extracted_comments: vec![Cow::Borrowed("Fresh plural")],
flags: vec![Cow::Borrowed("rust-format")],
}];
let merged = merge_catalog(existing, &extracted).expect("merge context plural");
assert!(merged.contains("#. Fresh plural\n#: new.rs:8\n#, rust-format\nmsgctxt \"files\"\nmsgid \"New file\"\nmsgid_plural \"New files\"\nmsgstr[0] \"\"\nmsgstr[1] \"\""));
assert!(merged.contains("#~ # Translator plural\n#~ #. Extracted plural\n#~ #@ owner: files\n#~ #: old.rs:4\n#~ #, fuzzy\n#~ msgctxt \"files\"\n#~ msgid \"Old file\"\n#~ msgid_plural \"Old files\"\n#~ msgstr[0] \"Alte Datei\"\n#~ msgstr[1] \"Alte Dateien\""));
}
#[test]
fn merge_handles_owned_header_fragments_and_missing_quotes() {
assert_eq!(
parse_header_fragment(br#""Language: de\\nPlural-Forms: nplurals=2;\\n""#)
.expect("owned escaped header fragment"),
vec![MergeHeader {
key: Cow::Owned("Language".to_owned()),
value: Cow::Owned("de\\nPlural-Forms: nplurals=2;\\n".to_owned()),
}]
);
assert_eq!(
parse_header_fragment(b"msgstr").expect("missing quotes"),
Vec::new()
);
assert_eq!(
extract_merge_quoted_cow(b"msgid").expect("missing quote"),
""
);
assert_eq!(parse_nplurals(&[]), None);
assert_eq!(
parse_nplurals(&[MergeHeader {
key: Cow::Borrowed("Plural-Forms"),
value: Cow::Borrowed("plural=(n != 1); nplurals=two;"),
}]),
None
);
}
#[test]
fn merge_parse_errors_include_line_position_for_plural_and_context_keywords() {
let plural_error = merge_catalog(
"msgid \"file\"\nmsgid_plural \"bad\"quote\"\nmsgstr[0] \"\"\n",
&[],
)
.expect_err("unescaped plural quote should fail");
let plural_position = plural_error.position().expect("plural position metadata");
assert_eq!(plural_error.message(), "unescaped quote in string literal");
assert_eq!(plural_position.line(), 2);
assert_eq!(plural_position.column(), 1);
let context_error =
merge_catalog("msgctxt \"bad\"quote\"\nmsgid \"x\"\nmsgstr \"\"\n", &[])
.expect_err("unescaped context quote should fail");
let context_position = context_error.position().expect("context position metadata");
assert_eq!(context_error.message(), "unescaped quote in string literal");
assert_eq!(context_position.line(), 1);
assert_eq!(context_position.column(), 1);
}
#[test]
fn merge_rejects_unrecognized_existing_lines() {
let extracted = [ExtractedMessage {
msgid: Cow::Borrowed("ok"),
..ExtractedMessage::default()
}];
let error = merge_catalog("msgid \"ok\"\nmsgstr_ \"typo\"\n", &extracted)
.expect_err("unknown existing PO line should fail");
let position = error.position().expect("position metadata");
assert_eq!(error.message(), "unrecognized PO syntax");
assert_eq!(position.line(), 2);
assert_eq!(position.column(), 1);
}
#[test]
fn merge_parses_owned_header_fragments_in_keyword_and_continuation_lines() {
let existing = concat!(
"msgid \"\"\n",
"msgstr \"Project-Id-Version: ferrocat\\t1\\n\"\n",
"\"Language: de\\t\\n\"\n",
"\n",
"msgid \"hello\"\n",
"msgstr \"world\"\n",
);
let merged = merge_catalog(
existing,
&[ExtractedMessage {
msgid: Cow::Borrowed("hello"),
..ExtractedMessage::default()
}],
)
.expect("merge with owned header fragments");
assert!(merged.contains("Project-Id-Version: ferrocat\\t1\\n"));
assert!(merged.contains("Language: de\\n"));
}
}