use std::borrow::Cow;
use std::fs;
use std::{collections::BTreeMap, mem};
use rustc_hash::FxHashMap;
use crate::diagnostic_codes;
use super::collation::{apply_order, collate_indices, sort_messages_collated};
use super::export::export_catalog_content;
use super::file_io::atomic_write;
use super::helpers::{
dedupe_origins, dedupe_placeholders, dedupe_strings, merge_placeholders, merge_unique_origins,
merge_unique_strings,
};
use super::mt::{
MachineMetadata, PO_AI_KEY, PO_LOCK_KEY, parse_ai_descriptor, validate_machine_metadata,
};
use super::plural::{
GettextPluralProfiles, PluralProfile, derive_plural_variable,
expected_gettext_nplurals_for_locale,
};
use super::{
ApiError, CatalogMessage, CatalogOrigin, CatalogSemantics, CatalogStats, CatalogStorageFormat,
CatalogUpdateInput, CatalogUpdateResult, Diagnostic, DiagnosticSeverity, ExtractedMessage,
ObsoleteInfo, ObsoleteStrategy, OrderBy, ParseCatalogOptions, ParsedCatalog, PluralEncoding,
PluralSource, TranslationShape, UpdateCatalogFileOptions, UpdateCatalogOptions,
};
use crate::{BorrowedMsgStr, BorrowedPoFile, BorrowedPoItem, PoVec, parse_po_borrowed};
#[derive(Debug, Clone, PartialEq, Default)]
pub(super) struct Catalog {
pub(super) locale: Option<String>,
pub(super) headers: BTreeMap<String, String>,
pub(super) file_comments: Vec<String>,
pub(super) file_extracted_comments: Vec<String>,
pub(super) messages: Vec<CanonicalMessage>,
pub(super) diagnostics: Vec<Diagnostic>,
}
#[derive(Debug, Clone, PartialEq)]
pub(super) struct CanonicalMessage {
pub(super) msgid: String,
pub(super) msgctxt: Option<String>,
pub(super) translation: CanonicalTranslation,
pub(super) comments: Vec<String>,
pub(super) opaque: Option<Box<OpaqueMetadata>>,
pub(super) origins: PoVec<CatalogOrigin>,
pub(super) placeholders: BTreeMap<String, Vec<String>>,
pub(super) obsolete: Option<ObsoleteInfo>,
pub(super) machine: Option<MachineMetadata>,
}
#[derive(Debug, Clone, PartialEq, Eq, Default)]
pub(super) struct OpaqueMetadata {
pub(super) translator_comments: Vec<String>,
pub(super) flags: Vec<String>,
}
impl CanonicalMessage {
pub(super) fn translator_comments(&self) -> &[String] {
self.opaque
.as_deref()
.map_or(&[], |opaque| opaque.translator_comments.as_slice())
}
pub(super) fn flags(&self) -> &[String] {
self.opaque
.as_deref()
.map_or(&[], |opaque| opaque.flags.as_slice())
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(super) enum OpaqueCapture {
Keep,
ReviewState,
Discard,
}
impl OpaqueMetadata {
pub(super) fn from_parts(
translator_comments: Vec<String>,
flags: Vec<String>,
) -> Option<Box<Self>> {
if translator_comments.is_empty() && flags.is_empty() {
None
} else {
Some(Box::new(Self {
translator_comments,
flags,
}))
}
}
}
pub(super) const PO_OBSOLETE_SINCE_KEY: &str = "obsolete-since";
#[derive(Debug, Clone, PartialEq, Eq)]
pub(super) enum CanonicalTranslation {
Singular {
value: String,
},
Plural {
source: PluralSource,
translation_by_category: BTreeMap<String, String>,
variable: String,
},
}
#[derive(Debug, Clone, PartialEq, Eq)]
struct NormalizedMessage {
msgid: String,
msgctxt: Option<String>,
kind: NormalizedKind,
comments: Vec<String>,
origins: PoVec<CatalogOrigin>,
placeholders: BTreeMap<String, Vec<String>>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
enum NormalizedKind {
Singular,
Plural { source: PluralSource },
}
#[derive(Debug, Clone, PartialEq, Eq, Default)]
struct ParsedPluralFormsHeader {
raw: Option<String>,
nplurals: Option<usize>,
plural: Option<String>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
struct MergeCatalogContext<'a> {
locale: Option<&'a str>,
source_locale: &'a str,
semantics: CatalogSemantics,
overwrite_source_translations: bool,
obsolete_strategy: ObsoleteStrategy,
now: Option<&'a str>,
}
pub fn update_catalog(
mut options: UpdateCatalogOptions<'_>,
) -> Result<CatalogUpdateResult, ApiError> {
super::validate_source_locale(options.source_locale)?;
let input = mem::take(&mut options.input);
let created = options.existing.is_none();
let original = options.existing.unwrap_or("");
let mut existing = match options.existing {
Some(content) if !content.is_empty() => parse_catalog_to_internal(
content,
options.locale,
options.source_locale,
options.mode.semantics(),
options.mode.plural_encoding(),
false,
options.mode.storage_format(),
OpaqueCapture::Keep,
)?,
Some(_) | None => Catalog {
locale: options.locale.map(str::to_owned),
headers: BTreeMap::new(),
file_comments: Vec::new(),
file_extracted_comments: Vec::new(),
messages: Vec::new(),
diagnostics: Vec::new(),
},
};
let locale = options
.locale
.map(str::to_owned)
.or_else(|| existing.locale.clone())
.or_else(|| existing.headers.get("Language").cloned());
let mut diagnostics = mem::take(&mut existing.diagnostics);
let normalized = normalize_update_input(input)?;
let merge_context = MergeCatalogContext {
locale: locale.as_deref(),
source_locale: options.source_locale,
semantics: options.mode.semantics(),
overwrite_source_translations: options.overwrite_source_translations,
obsolete_strategy: options.obsolete_strategy.clone(),
now: options.now,
};
let (mut merged, stats) =
merge_catalogs(existing, normalized, &merge_context, &mut diagnostics);
merged.locale.clone_from(&locale);
apply_storage_defaults(&mut merged, &options, locale.as_deref(), &mut diagnostics)?;
sort_messages(&mut merged.messages, options.render.order_by);
let content = export_catalog_content(&merged, &options, locale.as_deref(), &mut diagnostics)?;
Ok(CatalogUpdateResult {
updated: content != original,
content,
created,
stats,
diagnostics,
})
}
pub fn update_catalog_file(
options: UpdateCatalogFileOptions<'_>,
) -> Result<CatalogUpdateResult, ApiError> {
super::validate_source_locale(options.options.source_locale)?;
if options.target_path.as_os_str().is_empty() {
return Err(ApiError::InvalidArguments(
"target_path must not be empty".to_owned(),
));
}
let existing = match fs::read_to_string(options.target_path) {
Ok(content) => Some(content),
Err(error) if error.kind() == std::io::ErrorKind::NotFound => None,
Err(error) => return Err(ApiError::io_with_path(options.target_path, error)),
};
let mut update_options = options.options;
update_options.existing = existing.as_deref();
let result = update_catalog(update_options)?;
if result.created || result.updated {
atomic_write(options.target_path, &result.content)?;
}
Ok(result)
}
pub fn parse_catalog(options: ParseCatalogOptions<'_>) -> Result<ParsedCatalog, ApiError> {
parse_catalog_projection(options, OpaqueCapture::Discard).map(|(catalog, _)| catalog)
}
pub fn parse_catalog_for_review(
options: ParseCatalogOptions<'_>,
) -> Result<super::NormalizedParsedCatalog, ApiError> {
let (catalog, fuzzy_keys) = parse_catalog_projection(options, OpaqueCapture::ReviewState)?;
super::NormalizedParsedCatalog::new_with_review_state(catalog, fuzzy_keys)
}
fn parse_catalog_projection(
options: ParseCatalogOptions<'_>,
opaque_capture: OpaqueCapture,
) -> Result<
(
ParsedCatalog,
std::collections::BTreeSet<super::CatalogMessageKey>,
),
ApiError,
> {
super::validate_source_locale(options.source_locale)?;
let catalog = parse_catalog_to_internal(
options.content,
options.locale,
options.source_locale,
options.mode.semantics(),
options.mode.plural_encoding(),
options.strict,
options.mode.storage_format(),
opaque_capture,
)?;
let fuzzy_keys = if opaque_capture == OpaqueCapture::ReviewState {
catalog
.messages
.iter()
.filter(|message| message.flags().iter().any(|flag| flag == "fuzzy"))
.map(|message| {
super::CatalogMessageKey::new(message.msgid.clone(), message.msgctxt.clone())
})
.collect()
} else {
std::collections::BTreeSet::new()
};
let messages = catalog
.messages
.into_iter()
.map(public_message_from_canonical)
.collect();
Ok((
ParsedCatalog {
locale: catalog.locale,
semantics: options.mode.semantics(),
headers: catalog.headers,
messages,
diagnostics: catalog.diagnostics,
},
fuzzy_keys,
))
}
fn normalize_update_input(input: CatalogUpdateInput) -> Result<Vec<NormalizedMessage>, ApiError> {
let mut messages = Vec::<NormalizedMessage>::new();
match input {
CatalogUpdateInput::Structured(extracted) => {
for message in extracted {
let (msgid, msgctxt, kind, comments, origins, placeholders) = match message {
ExtractedMessage::Singular(message) => (
message.msgid,
message.msgctxt,
NormalizedKind::Singular,
message.comments,
message.origin,
message.placeholders,
),
ExtractedMessage::Plural(message) => (
message.msgid,
message.msgctxt,
NormalizedKind::Plural {
source: message.source,
},
message.comments,
message.origin,
message.placeholders,
),
};
messages.push(NormalizedMessage {
msgid,
msgctxt,
kind,
comments: dedupe_strings(comments),
origins: dedupe_origins(origins),
placeholders: dedupe_placeholders(placeholders),
});
}
}
CatalogUpdateInput::SourceFirst(source_messages) => {
for message in source_messages {
messages.push(NormalizedMessage {
msgid: message.msgid,
msgctxt: message.msgctxt,
kind: NormalizedKind::Singular,
comments: dedupe_strings(message.comments),
origins: dedupe_origins(message.origin),
placeholders: dedupe_placeholders(message.placeholders),
});
}
}
}
merge_normalized_messages(messages)
}
fn merge_normalized_messages(
messages: Vec<NormalizedMessage>,
) -> Result<Vec<NormalizedMessage>, ApiError> {
let mut index = FxHashMap::<(&str, Option<&str>), usize>::with_capacity_and_hasher(
messages.len(),
Default::default(),
);
let mut selected_indexes = Vec::with_capacity(messages.len());
let mut duplicate_sources = vec![None; messages.len()];
for (message_index, message) in messages.iter().enumerate() {
if message.msgid.is_empty() {
return Err(ApiError::InvalidArguments(
"extracted msgid must not be empty".to_owned(),
));
}
let key = (message.msgid.as_str(), message.msgctxt.as_deref());
if let Some(existing_index) = index.get(&key).copied() {
let existing = &messages[existing_index];
if existing.kind != message.kind {
return Err(ApiError::Conflict(format!(
"conflicting duplicate extracted message for msgid {:?}",
message.msgid
)));
}
duplicate_sources[message_index] = Some(existing_index);
} else {
index.insert(key, message_index);
selected_indexes.push(message_index);
}
}
let mut output_index_by_input = vec![None; messages.len()];
let mut out = Vec::with_capacity(selected_indexes.len());
let mut selected_indexes = selected_indexes.into_iter();
let mut next_selected = selected_indexes.next();
for (message_index, message) in messages.into_iter().enumerate() {
if next_selected == Some(message_index) {
output_index_by_input[message_index] = Some(out.len());
out.push(message);
next_selected = selected_indexes.next();
continue;
}
if let Some(existing_input_index) = duplicate_sources[message_index] {
let existing_output_index = output_index_by_input[existing_input_index]
.expect("duplicate source should already be emitted");
let existing = &mut out[existing_output_index];
merge_unique_strings(&mut existing.comments, message.comments);
merge_unique_origins(&mut existing.origins, message.origins);
merge_placeholders(&mut existing.placeholders, message.placeholders);
}
}
Ok(out)
}
fn merge_catalogs(
existing: Catalog,
normalized: Vec<NormalizedMessage>,
context: &MergeCatalogContext<'_>,
diagnostics: &mut Vec<Diagnostic>,
) -> (Catalog, CatalogStats) {
let is_source_locale = context
.locale
.is_none_or(|value| value == context.source_locale);
let mut stats = CatalogStats::default();
let mut messages = Vec::with_capacity(normalized.len() + existing.messages.len());
let plural_profile = if context.semantics == CatalogSemantics::GettextCompat {
PluralProfile::for_gettext_locale(context.locale)
} else {
PluralProfile::for_locale(context.locale)
};
let mut matched_indexes = Vec::with_capacity(normalized.len());
{
let mut existing_index = FxHashMap::<(&str, Option<&str>), usize>::with_capacity_and_hasher(
existing.messages.len(),
Default::default(),
);
for (index, message) in existing.messages.iter().enumerate() {
existing_index.insert((message.msgid.as_str(), message.msgctxt.as_deref()), index);
}
for next in &normalized {
matched_indexes.push(
existing_index
.get(&(next.msgid.as_str(), next.msgctxt.as_deref()))
.copied(),
);
}
}
let mut previous_messages = existing
.messages
.into_iter()
.map(Some)
.collect::<Vec<Option<CanonicalMessage>>>();
for (next, matched_index) in normalized.into_iter().zip(matched_indexes) {
let previous = matched_index.map(|index| {
previous_messages[index]
.take()
.expect("each existing message is claimed by at most one extracted message")
});
let (merged, outcome) = merge_message(
previous,
next,
is_source_locale,
&plural_profile,
context.overwrite_source_translations,
diagnostics,
);
match outcome {
MergeOutcome::Added => stats.added += 1,
MergeOutcome::Unchanged => stats.unchanged += 1,
MergeOutcome::Changed => stats.changed += 1,
}
messages.push(merged);
}
for message in previous_messages.into_iter().flatten() {
match &context.obsolete_strategy {
ObsoleteStrategy::Delete => {
stats.obsolete_removed += 1;
}
ObsoleteStrategy::Mark => {
let mut message = message;
mark_obsolete(&mut message, context.now, &mut stats);
messages.push(message);
}
ObsoleteStrategy::Keep => {
let mut message = message;
message.obsolete = None;
messages.push(message);
}
ObsoleteStrategy::DropObsoleteBefore(cutoff) => {
let mut message = message;
mark_obsolete(&mut message, context.now, &mut stats);
if let Some(ObsoleteInfo { since: Some(since) }) = &message.obsolete
&& since.as_str() < cutoff.as_str()
{
stats.obsolete_removed += 1;
continue;
}
messages.push(message);
}
}
}
stats.total = messages.len();
(
Catalog {
locale: existing.locale,
headers: existing.headers,
file_comments: existing.file_comments,
file_extracted_comments: existing.file_extracted_comments,
messages,
diagnostics: existing.diagnostics,
},
stats,
)
}
fn mark_obsolete(message: &mut CanonicalMessage, now: Option<&str>, stats: &mut CatalogStats) {
if message.obsolete.is_none() {
message.obsolete = Some(ObsoleteInfo {
since: now.map(str::to_owned),
});
stats.obsolete_marked += 1;
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum MergeOutcome {
Added,
Unchanged,
Changed,
}
fn merge_message(
previous: Option<CanonicalMessage>,
next: NormalizedMessage,
is_source_locale: bool,
plural_profile: &PluralProfile,
overwrite_source_translations: bool,
diagnostics: &mut Vec<Diagnostic>,
) -> (CanonicalMessage, MergeOutcome) {
let NormalizedMessage {
msgid,
msgctxt,
kind,
comments,
origins,
placeholders,
} = next;
let payload_changed = previous.as_ref().is_some_and(|previous| {
previous.obsolete.is_some()
|| previous.comments != comments
|| previous.origins != origins
|| previous.placeholders != placeholders
});
let (previous_translation, machine, opaque) = match previous {
Some(CanonicalMessage {
translation,
machine,
opaque,
..
}) => (Some(translation), machine, opaque),
None => (None, None, None),
};
let had_previous = previous_translation.is_some();
let (translation, translation_changed) = match (kind, previous_translation) {
(NormalizedKind::Singular, Some(previous @ CanonicalTranslation::Singular { .. }))
if !(is_source_locale && overwrite_source_translations) =>
{
(previous, false)
}
(NormalizedKind::Singular, previous) => {
let translation = CanonicalTranslation::Singular {
value: if is_source_locale {
msgid.clone()
} else {
String::new()
},
};
let changed = previous.is_none_or(|previous| previous != translation);
(translation, changed)
}
(
NormalizedKind::Plural { source },
Some(CanonicalTranslation::Plural {
source: previous_source,
translation_by_category: mut previous_translation_by_category,
variable: previous_variable,
}),
) if !(is_source_locale && overwrite_source_translations) => {
let categories_changed = plural_profile
.materialize_translation_in_place(&mut previous_translation_by_category);
let source_changed = source != previous_source;
(
CanonicalTranslation::Plural {
source,
translation_by_category: previous_translation_by_category,
variable: previous_variable,
},
categories_changed || source_changed,
)
}
(NormalizedKind::Plural { source }, previous) => {
let variable = extract_plural_variable(previous.as_ref())
.or_else(|| derive_plural_variable(&placeholders))
.unwrap_or_else(|| {
diagnostics.push(
Diagnostic::new(
DiagnosticSeverity::Warning,
diagnostic_codes::plural::ASSUMED_VARIABLE,
"Unable to determine plural placeholder name, assuming \"count\".",
)
.with_identity(&msgid, msgctxt.as_deref()),
);
"count".to_owned()
});
let translation_by_category = if is_source_locale {
plural_profile.source_locale_translation(&source)
} else {
plural_profile.empty_translation()
};
let translation = CanonicalTranslation::Plural {
source,
translation_by_category,
variable,
};
let changed = previous.is_none_or(|previous| previous != translation);
(translation, changed)
}
};
let outcome = if !had_previous {
MergeOutcome::Added
} else if payload_changed || translation_changed {
MergeOutcome::Changed
} else {
MergeOutcome::Unchanged
};
(
CanonicalMessage {
msgid,
msgctxt,
translation,
comments,
opaque,
origins,
placeholders,
obsolete: None,
machine,
},
outcome,
)
}
fn extract_plural_variable(translation: Option<&CanonicalTranslation>) -> Option<String> {
match translation {
Some(CanonicalTranslation::Plural { variable, .. }) => Some(variable.clone()),
Some(CanonicalTranslation::Singular { .. }) | None => None,
}
}
pub(super) fn apply_header_defaults(
headers: &mut BTreeMap<String, String>,
locale: Option<&str>,
semantics: CatalogSemantics,
diagnostics: &mut Vec<Diagnostic>,
custom: &BTreeMap<String, String>,
) {
headers
.entry("MIME-Version".to_owned())
.or_insert_with(|| "1.0".to_owned());
headers
.entry("Content-Type".to_owned())
.or_insert_with(|| "text/plain; charset=utf-8".to_owned());
headers
.entry("Content-Transfer-Encoding".to_owned())
.or_insert_with(|| "8bit".to_owned());
headers
.entry("X-Generator".to_owned())
.or_insert_with(|| "ferrocat".to_owned());
if let Some(locale) = locale {
headers.insert("Language".to_owned(), locale.to_owned());
}
if semantics == CatalogSemantics::GettextCompat && !custom.contains_key("Plural-Forms") {
let profile = PluralProfile::for_gettext_locale(locale);
let parsed_header = parse_plural_forms_from_headers(headers);
match (parsed_header.raw.as_deref(), profile.gettext_header()) {
(None, Some(header)) => {
headers.insert("Plural-Forms".to_owned(), header);
}
(None, None) => diagnostics.push(Diagnostic::new(
DiagnosticSeverity::Info,
diagnostic_codes::plural::MISSING_PLURAL_FORMS_HEADER,
"No safe default Plural-Forms header is known for this locale; keeping the header unset.",
)),
(Some(_), Some(header))
if parsed_header.nplurals == Some(profile.nplurals())
&& parsed_header.plural.is_none() =>
{
headers.insert("Plural-Forms".to_owned(), header);
diagnostics.push(Diagnostic::new(
DiagnosticSeverity::Info,
diagnostic_codes::plural::COMPLETED_PLURAL_FORMS_HEADER,
"Plural-Forms header was missing the plural expression and has been completed using a safe locale default.",
));
}
_ => {}
}
}
for (key, value) in custom {
headers.insert(key.clone(), value.clone());
}
}
pub(super) fn sort_messages(messages: &mut Vec<CanonicalMessage>, order_by: OrderBy) {
match order_by {
OrderBy::Msgid => sort_messages_collated(messages),
OrderBy::Origin => {
messages.sort_unstable_by(|left, right| {
first_origin_sort_key(&left.origins).cmp(first_origin_sort_key(&right.origins))
});
let mut order: Vec<usize> = (0..messages.len()).collect();
let mut start = 0;
while start < messages.len() {
let mut end = start + 1;
while end < messages.len()
&& first_origin_sort_key(&messages[end].origins)
== first_origin_sort_key(&messages[start].origins)
{
end += 1;
}
collate_indices(messages, &mut order[start..end]);
start = end;
}
apply_order(messages, &order);
}
}
}
fn first_origin_sort_key(origins: &[CatalogOrigin]) -> &str {
origins.first().map_or("", |origin| origin.file.as_str())
}
fn apply_storage_defaults(
catalog: &mut Catalog,
options: &UpdateCatalogOptions<'_>,
locale: Option<&str>,
diagnostics: &mut Vec<Diagnostic>,
) -> Result<(), ApiError> {
match options.mode.storage_format() {
CatalogStorageFormat::Po => {
let empty_custom_headers = BTreeMap::new();
apply_header_defaults(
&mut catalog.headers,
locale,
options.mode.semantics(),
diagnostics,
options
.render
.custom_header_attributes
.unwrap_or(&empty_custom_headers),
);
Ok(())
}
CatalogStorageFormat::Fcl => {
if options
.render
.custom_header_attributes
.is_some_and(|headers| !headers.is_empty())
{
return Err(ApiError::Unsupported(
"custom_header_attributes are not supported for FCL catalogs".to_owned(),
));
}
catalog.headers.clear();
Ok(())
}
}
}
#[expect(
clippy::too_many_arguments,
reason = "Internal fan-in shared by every catalog read path; a config struct would only relabel the same eight decisions."
)]
pub(super) fn parse_catalog_to_internal(
content: &str,
locale_override: Option<&str>,
source_locale: &str,
semantics: CatalogSemantics,
plural_encoding: PluralEncoding,
strict: bool,
storage_format: CatalogStorageFormat,
opaque_capture: OpaqueCapture,
) -> Result<Catalog, ApiError> {
match storage_format {
CatalogStorageFormat::Po => parse_catalog_to_internal_po(
content,
locale_override,
semantics,
plural_encoding,
strict,
opaque_capture,
),
CatalogStorageFormat::Fcl => super::fcl::parse_catalog_to_internal_fcl(
content,
locale_override,
source_locale,
semantics,
strict,
opaque_capture,
),
}
}
fn parse_catalog_to_internal_po(
content: &str,
locale_override: Option<&str>,
semantics: CatalogSemantics,
_plural_encoding: PluralEncoding,
strict: bool,
opaque_capture: OpaqueCapture,
) -> Result<Catalog, ApiError> {
let BorrowedPoFile {
headers: po_headers,
items: po_items,
comments: po_comments,
extracted_comments: po_extracted_comments,
} = parse_po_borrowed(content)?;
let headers = po_headers
.into_iter()
.map(|header| (header.key.into_owned(), header.value.into_owned()))
.collect::<BTreeMap<_, _>>();
let locale = locale_override
.map(str::to_owned)
.or_else(|| headers.get("Language").cloned());
let plural_forms = parse_plural_forms_from_headers(&headers);
let nplurals = plural_forms.nplurals;
let mut diagnostics = Vec::new();
validate_plural_forms_header(
locale.as_deref(),
&plural_forms,
semantics,
&mut diagnostics,
);
let mut messages = Vec::with_capacity(po_items.len());
let mut plural_profiles = GettextPluralProfiles::new(locale.as_deref());
for item in po_items {
let message = import_message_from_po(
item,
&mut plural_profiles,
nplurals,
semantics,
opaque_capture,
strict,
&mut diagnostics,
)?;
messages.push(message);
}
Ok(Catalog {
locale,
headers,
file_comments: po_comments.into_iter().map(Cow::into_owned).collect(),
file_extracted_comments: po_extracted_comments
.into_iter()
.map(Cow::into_owned)
.collect(),
messages,
diagnostics,
})
}
fn import_message_from_po(
item: BorrowedPoItem<'_>,
plural_profiles: &mut GettextPluralProfiles<'_>,
nplurals: Option<usize>,
semantics: CatalogSemantics,
opaque_capture: OpaqueCapture,
_strict: bool,
_diagnostics: &mut Vec<Diagnostic>,
) -> Result<CanonicalMessage, ApiError> {
let (comments, placeholders) = split_placeholder_comments(item.extracted_comments);
let opaque = match opaque_capture {
OpaqueCapture::Keep => OpaqueMetadata::from_parts(
item.comments.into_iter().map(Cow::into_owned).collect(),
item.flags.into_iter().map(Cow::into_owned).collect(),
),
OpaqueCapture::ReviewState => OpaqueMetadata::from_parts(
Vec::new(),
item.flags
.into_iter()
.filter(|flag| flag == "fuzzy")
.map(Cow::into_owned)
.collect(),
),
OpaqueCapture::Discard => None,
};
let origins = item.references.into_iter().map(parse_origin).collect();
let translation = if let Some(msgid_plural) = item.msgid_plural {
if semantics == CatalogSemantics::IcuNative {
return Err(ApiError::Unsupported(
"classic gettext plural requires compat mode".to_owned(),
));
}
let plural_profile = plural_profiles.for_slots(nplurals.or(Some(item.msgstr.len())));
CanonicalTranslation::Plural {
source: PluralSource {
one: Some(item.msgid.as_ref().to_owned()),
other: msgid_plural.into_owned(),
},
translation_by_category: plural_profile
.categories()
.iter()
.zip(
item.msgstr
.into_values()
.map(Cow::into_owned)
.chain(std::iter::repeat_with(String::new)),
)
.map(|(category, value)| ((*category).to_owned(), value))
.collect(),
variable: "count".to_owned(),
}
} else {
if semantics == CatalogSemantics::IcuNative
&& matches!(item.msgstr, BorrowedMsgStr::Plural(_))
{
return Err(ApiError::Unsupported(
"classic gettext plural requires compat mode".to_owned(),
));
}
CanonicalTranslation::Singular {
value: take_first_msgstr(item.msgstr),
}
};
Ok(CanonicalMessage {
msgid: item.msgid.into_owned(),
msgctxt: item.msgctxt.map(Cow::into_owned),
translation,
comments,
opaque,
origins,
placeholders,
obsolete: import_obsolete(item.obsolete, &item.metadata),
machine: import_machine_metadata(&item.metadata)?,
})
}
fn import_obsolete(
obsolete: bool,
metadata: &[(Cow<'_, str>, Cow<'_, str>)],
) -> Option<ObsoleteInfo> {
if !obsolete {
return None;
}
let since = metadata
.iter()
.find(|(key, _)| key == PO_OBSOLETE_SINCE_KEY)
.map(|(_, value)| value.as_ref().to_owned());
Some(ObsoleteInfo { since })
}
fn import_machine_metadata(
metadata: &[(Cow<'_, str>, Cow<'_, str>)],
) -> Result<Option<MachineMetadata>, ApiError> {
let mut lock = None;
let mut ai = None;
for (key, value) in metadata {
if key == PO_LOCK_KEY {
if lock.replace(value).is_some() {
return Err(ApiError::InvalidArguments(
"duplicate `lock` metadata for PO item".to_owned(),
));
}
} else if key == PO_AI_KEY && ai.replace(value).is_some() {
return Err(ApiError::InvalidArguments(
"duplicate `ai` metadata for PO item".to_owned(),
));
}
}
let Some(lock) = lock else {
if ai.is_some() {
return Err(ApiError::InvalidArguments(
"PO `ai` metadata requires a `lock`".to_owned(),
));
}
return Ok(None);
};
let metadata = MachineMetadata {
lock: lock.as_ref().to_owned(),
ai: ai.map(|value| parse_ai_descriptor(value)),
};
validate_machine_metadata(&metadata)?;
Ok(Some(metadata))
}
pub(super) fn split_placeholder_comments<'a>(
extracted_comments: impl IntoIterator<Item = Cow<'a, str>>,
) -> (Vec<String>, BTreeMap<String, Vec<String>>) {
let mut comments = Vec::new();
let mut placeholders = BTreeMap::<String, Vec<String>>::new();
for comment in extracted_comments {
if let Some((name, value)) = parse_placeholder_comment(&comment) {
placeholders
.entry(name.to_owned())
.or_default()
.push(value.to_owned());
} else {
comments.push(comment.into_owned());
}
}
(comments, dedupe_placeholders(placeholders))
}
fn parse_placeholder_comment(comment: &str) -> Option<(&str, &str)> {
let rest = comment.strip_prefix("placeholder {")?;
let end = rest.find("}: ")?;
Some((&rest[..end], &rest[end + 3..]))
}
pub(super) fn parse_origin(reference: Cow<'_, str>) -> CatalogOrigin {
match reference {
Cow::Borrowed(reference) => {
let (file, scope) = split_origin(reference);
CatalogOrigin {
file: file.to_owned(),
scope: scope.map(str::to_owned),
}
}
Cow::Owned(mut reference) => {
let (file, scope) = split_origin(&reference);
let file_len = file.len();
let scope = scope.map(str::to_owned);
reference.truncate(file_len);
CatalogOrigin {
file: reference,
scope,
}
}
}
}
fn split_origin(reference: &str) -> (&str, Option<&str>) {
let (file, scope) = match reference.rsplit_once('#') {
Some((file, scope)) if !scope.is_empty() => (file, Some(scope)),
_ => (reference, None),
};
if let Some((trimmed, line)) = file.rsplit_once(':')
&& !line.is_empty()
&& line.bytes().all(|byte| byte.is_ascii_digit())
{
return (trimmed, scope);
}
(file, scope)
}
pub(super) fn write_origin(out: &mut String, origin: &CatalogOrigin) {
out.push_str(&origin.file);
if let Some(scope) = &origin.scope {
out.push('#');
out.push_str(scope);
}
}
fn take_first_msgstr(msgstr: BorrowedMsgStr<'_>) -> String {
match msgstr {
BorrowedMsgStr::Singular(value) => value.into_owned(),
BorrowedMsgStr::Plural(values) => values
.into_iter()
.next()
.map(Cow::into_owned)
.unwrap_or_default(),
BorrowedMsgStr::None => String::new(),
}
}
fn parse_plural_forms_from_headers(headers: &BTreeMap<String, String>) -> ParsedPluralFormsHeader {
let Some(plural_forms) = headers.get("Plural-Forms") else {
return ParsedPluralFormsHeader::default();
};
let mut parsed = ParsedPluralFormsHeader {
raw: Some(plural_forms.clone()),
..ParsedPluralFormsHeader::default()
};
for part in plural_forms.split(';') {
let trimmed = part.trim();
if let Some(value) = trimmed.strip_prefix("nplurals=") {
parsed.nplurals = value.trim().parse().ok();
} else if let Some(value) = trimmed.strip_prefix("plural=") {
let value = value.trim();
if !value.is_empty() {
parsed.plural = Some(value.to_owned());
}
}
}
parsed
}
fn validate_plural_forms_header(
locale: Option<&str>,
plural_forms: &ParsedPluralFormsHeader,
semantics: CatalogSemantics,
diagnostics: &mut Vec<Diagnostic>,
) {
if semantics != CatalogSemantics::GettextCompat {
return;
}
if let Some(nplurals) = plural_forms.nplurals {
match expected_gettext_nplurals_for_locale(locale) {
Some(expected) if nplurals != expected => diagnostics.push(Diagnostic::new(
DiagnosticSeverity::Warning,
diagnostic_codes::plural::NPLURALS_LOCALE_MISMATCH,
format!(
"Plural-Forms declares nplurals={nplurals}, but locale-derived categories expect {expected}."
),
)),
_ => {}
}
} else if plural_forms.plural.is_some() {
diagnostics.push(Diagnostic::new(
DiagnosticSeverity::Warning,
diagnostic_codes::parse::INVALID_PLURAL_FORMS_HEADER,
"Plural-Forms header contains a plural expression but no parseable nplurals value.",
));
}
if plural_forms.nplurals.is_some() && plural_forms.plural.is_none() {
diagnostics.push(Diagnostic::new(
DiagnosticSeverity::Info,
diagnostic_codes::plural::MISSING_PLURAL_EXPRESSION,
"Plural-Forms header declares nplurals but omits the plural expression.",
));
}
}
pub(super) fn public_message_from_canonical(message: CanonicalMessage) -> CatalogMessage {
let translation = match message.translation {
CanonicalTranslation::Singular { value } => TranslationShape::Singular { value },
CanonicalTranslation::Plural {
source,
translation_by_category,
variable,
..
} => TranslationShape::Plural {
source,
translation: translation_by_category,
variable,
},
};
CatalogMessage {
msgid: message.msgid,
msgctxt: message.msgctxt,
translation,
comments: message.comments,
origin: message.origins,
obsolete: message.obsolete,
machine: message.machine,
}
}
#[cfg(test)]
mod tests {
use std::borrow::Cow;
use super::{
CanonicalMessage, CanonicalTranslation, first_origin_sort_key, parse_origin, sort_messages,
take_first_msgstr,
};
use crate::api::{CatalogOrigin, ObsoleteInfo, OrderBy};
use crate::{BorrowedMsgStr, PoVec};
#[test]
fn parse_origin_strips_line_numbers_and_keeps_file() {
assert_eq!(
parse_origin(Cow::Borrowed("src/app.rs:42")).file,
"src/app.rs"
);
assert_eq!(
parse_origin(Cow::Owned("src/app.rs:42".to_owned())).file,
"src/app.rs"
);
assert_eq!(parse_origin(Cow::Borrowed("README")).file, "README");
assert_eq!(parse_origin(Cow::Borrowed("a:b:rev")).file, "a:b:rev");
let origin = parse_origin(Cow::Owned("src/app.rs:42#Checkout".to_owned()));
assert_eq!(origin.file, "src/app.rs");
assert_eq!(origin.scope.as_deref(), Some("Checkout"));
}
#[test]
fn take_first_msgstr_moves_first_available_value() {
assert_eq!(
take_first_msgstr(BorrowedMsgStr::Singular(Cow::Borrowed("one"))),
"one"
);
assert_eq!(
take_first_msgstr(BorrowedMsgStr::Plural(vec![
Cow::Borrowed("a"),
Cow::Borrowed("b"),
])),
"a"
);
assert_eq!(take_first_msgstr(BorrowedMsgStr::Plural(Vec::new())), "");
assert_eq!(take_first_msgstr(BorrowedMsgStr::None), "");
}
#[test]
fn first_origin_sort_key_borrows_origin_file() {
let origins = PoVec::from(vec![CatalogOrigin {
file: "src/app.rs".to_owned(),
scope: None,
}]);
assert_eq!(first_origin_sort_key(&origins), "src/app.rs");
}
#[test]
fn sort_messages_by_origin_uses_msgid_and_context_tiebreakers() {
let mut messages = vec![
test_message("src/app.rs", "beta", Some("dialog")),
test_message("src/app.rs", "alpha", Some("dialog")),
test_message("src/app.rs", "alpha", None),
test_message("src/collation.rs", "<0>Continue</0>", None),
test_message(
"src/collation.rs",
"{count, plural, one {#} other {#}}",
None,
),
test_message("src/lib.rs", "alpha", None),
];
sort_messages(&mut messages, OrderBy::Origin);
assert_eq!(
messages
.iter()
.map(|message| {
(
message.origins[0].file.as_str(),
message.msgid.as_str(),
message.msgctxt.as_deref(),
)
})
.collect::<Vec<_>>(),
vec![
("src/app.rs", "alpha", None),
("src/app.rs", "alpha", Some("dialog")),
("src/app.rs", "beta", Some("dialog")),
(
"src/collation.rs",
"{count, plural, one {#} other {#}}",
None
),
("src/collation.rs", "<0>Continue</0>", None),
("src/lib.rs", "alpha", None)
]
);
}
#[test]
fn sort_messages_by_origin_keeps_active_identity_before_obsolete_identity() {
let active = test_message("src/app.rs", "same", Some("dialog"));
let mut obsolete = active.clone();
obsolete.obsolete = Some(ObsoleteInfo::default());
let mut messages = vec![obsolete, active];
sort_messages(&mut messages, OrderBy::Origin);
assert!(messages[0].obsolete.is_none());
assert!(messages[1].obsolete.is_some());
}
fn test_message(file: &str, msgid: &str, msgctxt: Option<&str>) -> CanonicalMessage {
CanonicalMessage {
msgid: msgid.to_owned(),
msgctxt: msgctxt.map(str::to_owned),
translation: CanonicalTranslation::Singular {
value: String::new(),
},
comments: Vec::new(),
opaque: None,
origins: PoVec::from(vec![CatalogOrigin {
file: file.to_owned(),
scope: None,
}]),
placeholders: Default::default(),
obsolete: None,
machine: None,
}
}
}