Skip to main content

resopt/
localization.rs

1//! Translation coverage and placeholder consistency of localization files.
2//!
3//! Inspection only: every file is parsed from the bytes that were hashed and
4//! nothing is written back. Languages are compared within one *table*: an
5//! `.xcstrings` catalog on its own, the `<lang>.lproj/<Table>.strings` (or
6//! `.stringsdict`) files that share a directory, or the
7//! `values[-<locale>]/<name>.xml` files of one Android resource directory.
8use crate::{
9    Resource,
10    analysis::ResourceAnalysis,
11    filesystem::{contained_file, hash},
12    resources::bounded_read,
13};
14use anyhow::{Result, anyhow};
15use langcodec::{
16    FormatType,
17    formats::{AndroidStringsFormat, StringsFormat, StringsdictFormat, XcstringsFormat},
18    traits::Parser,
19};
20use serde::{Deserialize, Serialize};
21use std::{
22    collections::BTreeMap,
23    path::{Path, PathBuf},
24};
25
26pub(crate) use crate::localization_checks::MAX_ISSUES;
27
28/// Language label of an Android `values/` directory without a locale qualifier.
29pub const ANDROID_DEFAULT_LANGUAGE: &str = "default";
30
31#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
32pub struct LocalizationInfo {
33    /// Table name: the catalog or `.strings` file stem, or the Android file name.
34    pub table: String,
35    /// `xcstrings`, `strings`, `stringsdict` or `android_strings`.
36    pub format: String,
37    /// Language every other language is compared with.
38    pub source_language: String,
39    /// Keys that should be translated (excludes "do not translate" entries).
40    pub keys: usize,
41    /// Keys Xcode marked stale: no longer found in source code.
42    #[serde(default)]
43    pub stale_keys: usize,
44    /// Source language first, then by language tag.
45    pub languages: Vec<LanguageCoverage>,
46    /// Every issue in the table, by kind, including issues not listed below.
47    pub issue_counts: BTreeMap<String, usize>,
48    /// Issues relevant to this file, at most [`MAX_ISSUES`], by key then language.
49    pub issues: Vec<LocalizationIssue>,
50    /// Files of a multi-file table and the language each one holds.
51    #[serde(default, skip_serializing_if = "Vec::is_empty")]
52    pub files: Vec<LocalizationFile>,
53}
54
55#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
56pub struct LanguageCoverage {
57    pub language: String,
58    /// Keys with a non-empty value.
59    pub translated: usize,
60    /// Keys without a value in this language.
61    pub missing: usize,
62    /// Keys whose value is an empty string.
63    pub empty: usize,
64    /// Translated keys marked "needs review" or "stale".
65    pub needs_review: usize,
66    /// Keys present only in this language, not in the source language.
67    pub extra: usize,
68    /// Issues found in this language.
69    #[serde(default)]
70    pub issues: usize,
71}
72
73#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
74pub struct LocalizationIssue {
75    /// `placeholder_type`, `placeholder_count` or `empty_value`.
76    pub kind: String,
77    pub key: String,
78    pub language: String,
79    /// Source-language arguments, as `%1$d`-style canonical placeholders.
80    #[serde(default, skip_serializing_if = "Vec::is_empty")]
81    pub expected: Vec<String>,
82    /// Arguments of this language's value.
83    #[serde(default, skip_serializing_if = "Vec::is_empty")]
84    pub found: Vec<String>,
85    /// This language's value, shortened for display.
86    #[serde(default, skip_serializing_if = "String::is_empty")]
87    pub text: String,
88}
89
90#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
91pub struct LocalizationFile {
92    pub path: PathBuf,
93    pub language: String,
94}
95
96/// Files this module inspects. Android `values` XML qualifies only if it
97/// turns out to hold `<string>` or `<plurals>` entries.
98pub(crate) fn is_candidate(resource: &Resource) -> bool {
99    resource.conversion_exclusion.is_none()
100        && (matches!(
101            resource.format.as_str(),
102            "xcstrings" | "strings" | "stringsdict"
103        ) || (resource.extension == "xml"
104            && resource.android.as_ref().is_some_and(|android| {
105                android.area == "res" && android.res_type.as_deref() == Some("values")
106            })))
107}
108
109/// One parsed file, keyed by the table it belongs to.
110pub(crate) struct ParsedFile {
111    pub index: usize,
112    pub digest: String,
113    pub format: &'static str,
114    pub table: String,
115    /// Files with equal keys are the languages of one table.
116    pub group: String,
117    /// Language of a single-language file; `None` for `.xcstrings`.
118    pub language: Option<String>,
119    pub catalogs: Vec<langcodec::Resource>,
120}
121
122/// What reading one candidate produced.
123pub(crate) enum Parsed {
124    File(ParsedFile),
125    /// Not a localization file after all (Android XML without strings,
126    /// compiled binary `.strings`); the ordinary inventory row applies.
127    NotLocalization,
128    /// A valid file the parser does not model (nested `.stringsdict` rules).
129    Unsupported(String),
130    Failed(String),
131}
132
133pub(crate) fn parse(root: &Path, resource: &Resource, index: usize) -> Parsed {
134    let bytes = match contained_file(root, &resource.path).and_then(|path| bounded_read(&path)) {
135        Ok(bytes) => bytes,
136        Err(error) => return Parsed::Failed(format!("{error:#}")),
137    };
138    if bytes.starts_with(b"bplist") && resource.format == "strings" {
139        return Parsed::NotLocalization;
140    }
141    let digest = hash(&bytes);
142    let android = resource.format == "xml";
143    match read_catalogs(resource, &bytes) {
144        Ok(catalogs) if android && catalogs.iter().all(|c| c.entries.is_empty()) => {
145            Parsed::NotLocalization
146        }
147        Ok(catalogs) => {
148            let (format, table, group, language) = identity(resource);
149            Parsed::File(ParsedFile {
150                index,
151                digest,
152                format,
153                table,
154                group,
155                language,
156                catalogs,
157            })
158        }
159        // `values` XML may hold any resource type; only Apple formats are
160        // known to be localization files when they fail to parse.
161        Err(_) if android => Parsed::NotLocalization,
162        Err(error) if resource.format == "stringsdict" => {
163            Parsed::Unsupported(format!("localization_parse_failed: {error}"))
164        }
165        Err(error) => Parsed::Failed(format!("localization_parse_failed: {error}")),
166    }
167}
168
169fn read_catalogs(resource: &Resource, bytes: &[u8]) -> Result<Vec<langcodec::Resource>> {
170    Ok(match resource.format.as_str() {
171        "xcstrings" => XcstringsFormat::from_reader(bytes)?.try_into()?,
172        "strings" => vec![StringsFormat::from_reader(bytes)?.into()],
173        "stringsdict" => vec![StringsdictFormat::from_reader(bytes)?.into()],
174        "xml" => vec![AndroidStringsFormat::from_reader(bytes)?.into()],
175        other => return Err(anyhow!("not a localization format: {other}")),
176    })
177}
178
179/// Format, table name, grouping key and language of one file.
180fn identity(resource: &Resource) -> (&'static str, String, String, Option<String>) {
181    let path = &resource.path;
182    let name = path
183        .file_name()
184        .map(|n| n.to_string_lossy().into_owned())
185        .unwrap_or_default();
186    let stem = path
187        .file_stem()
188        .map(|n| n.to_string_lossy().into_owned())
189        .unwrap_or_default();
190    let parent = path.parent().unwrap_or(Path::new(""));
191    match resource.format.as_str() {
192        "xcstrings" => ("xcstrings", stem, key(&["xcstrings", &text(path)]), None),
193        format @ ("strings" | "stringsdict") => {
194            let format = if format == "strings" {
195                "strings"
196            } else {
197                "stringsdict"
198            };
199            let lproj = parent
200                .file_name()
201                .map(|n| n.to_string_lossy().into_owned())
202                .and_then(|n| n.strip_suffix(".lproj").map(str::to_string));
203            match lproj {
204                Some(language) => {
205                    let bundle = text(parent.parent().unwrap_or(Path::new("")));
206                    (format, stem, key(&[format, &bundle, &name]), Some(language))
207                }
208                None => (format, stem, key(&[format, &text(path)]), None),
209            }
210        }
211        _ => {
212            let values = parent
213                .file_name()
214                .map(|n| n.to_string_lossy().into_owned())
215                .unwrap_or_default();
216            let res = text(parent.parent().unwrap_or(Path::new("")));
217            let other_qualifiers = non_locale_qualifiers(&values);
218            let language =
219                langcodec::infer_language_from_path(path, &FormatType::AndroidStrings(None))
220                    .ok()
221                    .flatten()
222                    .filter(|_| values != "values")
223                    .unwrap_or_else(|| ANDROID_DEFAULT_LANGUAGE.to_string());
224            (
225                "android_strings",
226                name.clone(),
227                key(&["android", &res, &other_qualifiers, &name]),
228                Some(language),
229            )
230        }
231    }
232}
233
234fn text(path: &Path) -> String {
235    path.to_string_lossy().replace('\\', "/")
236}
237
238fn key(parts: &[&str]) -> String {
239    parts.join("\u{0}")
240}
241
242/// Qualifiers of a `values-…` directory other than the locale, so that
243/// `values-night` and `values-v21` stay separate tables from `values`.
244fn non_locale_qualifiers(directory: &str) -> String {
245    let mut locale_seen = false;
246    let mut region_allowed = false;
247    directory
248        .split('-')
249        .skip(1)
250        .filter(|token| {
251            let language = !locale_seen
252                && (2..=3).contains(&token.len())
253                && token.chars().all(|c| c.is_ascii_lowercase())
254                && *token != "car";
255            let region = region_allowed
256                && token.len() >= 3
257                && token.starts_with('r')
258                && token[1..]
259                    .chars()
260                    .all(|c| c.is_ascii_uppercase() || c.is_ascii_digit());
261            let bcp47 = token.starts_with("b+");
262            region_allowed = language;
263            locale_seen |= language || bcp47;
264            !(language || region || bcp47)
265        })
266        .collect::<Vec<_>>()
267        .join("-")
268}
269
270/// Groups parsed files into tables and returns one report row per file.
271pub(crate) fn rows(
272    resources: &[Resource],
273    files: Vec<ParsedFile>,
274) -> Vec<(usize, ResourceAnalysis)> {
275    let mut tables: BTreeMap<String, Vec<ParsedFile>> = BTreeMap::new();
276    for file in files {
277        tables.entry(file.group.clone()).or_default().push(file);
278    }
279    tables
280        .into_values()
281        .flat_map(|files| table_rows(resources, files))
282        .collect()
283}
284
285fn table_rows(
286    resources: &[Resource],
287    mut files: Vec<ParsedFile>,
288) -> Vec<(usize, ResourceAnalysis)> {
289    files.sort_by(|a, b| resources[a.index].path.cmp(&resources[b.index].path));
290    distinguish_shared_languages(resources, &mut files);
291    let mut languages: Vec<(String, &langcodec::Resource)> = vec![];
292    for file in &files {
293        for catalog in &file.catalogs {
294            let language = file
295                .language
296                .clone()
297                .unwrap_or_else(|| catalog.metadata.language.clone());
298            languages.push((language, catalog));
299        }
300    }
301    let source = source_language(&files, &languages);
302    let key_is_source_text = files.iter().any(|file| file.format == "xcstrings");
303    let table = crate::localization_checks::check(&source, &languages, key_is_source_text);
304    let listed: Vec<LocalizationFile> = if files.len() > 1 {
305        files
306            .iter()
307            .map(|file| LocalizationFile {
308                path: resources[file.index].path.clone(),
309                language: file.language.clone().unwrap_or_default(),
310            })
311            .collect()
312    } else {
313        vec![]
314    };
315    files
316        .iter()
317        .map(|file| {
318            let resource = &resources[file.index];
319            let mut row = ResourceAnalysis::new(resource, "inspected");
320            row.sha256 = Some(file.digest.clone());
321            row.resource.kind = "localization".into();
322            // The source file (or a single catalog) lists every language's
323            // issues; a translation file lists its own.
324            let own = file
325                .language
326                .as_deref()
327                .filter(|language| *language != source);
328            row.localization = Some(LocalizationInfo {
329                table: file.table.clone(),
330                format: file.format.into(),
331                source_language: source.clone(),
332                keys: table.keys,
333                stale_keys: table.stale_keys,
334                languages: table.languages.clone(),
335                issue_counts: table.issue_counts.clone(),
336                issues: table
337                    .issues
338                    .iter()
339                    .filter(|issue| own.is_none_or(|language| issue.language == language))
340                    .take(MAX_ISSUES)
341                    .cloned()
342                    .collect(),
343                files: listed.clone(),
344            });
345            (file.index, row)
346        })
347        .collect()
348}
349
350/// Android folders such as `values-in` and `values-id` normalize to one
351/// language tag. Merging them would hide either file's gaps, so each keeps
352/// its own coverage row, labelled with its folder.
353fn distinguish_shared_languages(resources: &[Resource], files: &mut [ParsedFile]) {
354    let mut seen: BTreeMap<String, usize> = BTreeMap::new();
355    for file in files.iter() {
356        if let Some(language) = &file.language {
357            *seen.entry(language.clone()).or_insert(0) += 1;
358        }
359    }
360    for file in files.iter_mut() {
361        if let Some(language) = &file.language
362            && seen.get(language).is_some_and(|count| *count > 1)
363        {
364            let folder = resources[file.index]
365                .path
366                .parent()
367                .and_then(Path::file_name)
368                .map(|n| n.to_string_lossy().into_owned())
369                .unwrap_or_default();
370            file.language = Some(format!("{language} ({folder})"));
371        }
372    }
373}
374
375/// A catalog names its source language. Across files, the Apple `Base`
376/// localization or the Android default `values/` is the source; otherwise
377/// English, then the language with the most entries.
378fn source_language(files: &[ParsedFile], languages: &[(String, &langcodec::Resource)]) -> String {
379    if let Some(declared) = files
380        .iter()
381        .filter(|file| file.format == "xcstrings")
382        .flat_map(|file| &file.catalogs)
383        .find_map(|catalog| catalog.metadata.custom.get("source_language"))
384        .filter(|language| !language.is_empty())
385    {
386        return declared.clone();
387    }
388    for preferred in ["Base", ANDROID_DEFAULT_LANGUAGE, "en"] {
389        if languages.iter().any(|(language, _)| language == preferred) {
390            return preferred.into();
391        }
392    }
393    languages
394        .iter()
395        .max_by(|a, b| {
396            a.1.entries
397                .len()
398                .cmp(&b.1.entries.len())
399                .then_with(|| b.0.cmp(&a.0))
400        })
401        .map_or_else(String::new, |(language, _)| language.clone())
402}
403
404#[cfg(test)]
405mod tests {
406    use super::*;
407
408    #[test]
409    fn android_locale_qualifiers_are_not_part_of_the_table() {
410        assert_eq!(non_locale_qualifiers("values"), "");
411        assert_eq!(non_locale_qualifiers("values-zh-rCN"), "");
412        assert_eq!(non_locale_qualifiers("values-b+sr+Latn"), "");
413        assert_eq!(non_locale_qualifiers("values-es-night"), "night");
414        assert_eq!(non_locale_qualifiers("values-v21"), "v21");
415        assert_eq!(non_locale_qualifiers("values-night-v31"), "night-v31");
416        assert_eq!(non_locale_qualifiers("values-car"), "car");
417    }
418}