1use crate::{
9 Resource,
10 analysis::ResourceAnalysis,
11 filesystem::{contained_file, hash},
12 resources::bounded_read,
13};
14use anyhow::{Result, anyhow};
15use langcodec::{
16 FormatType,
17 formats::{AndroidStringsFormat, StringsFormat, StringsdictFormat, XcstringsFormat},
18 traits::Parser,
19};
20use serde::{Deserialize, Serialize};
21use std::{
22 collections::BTreeMap,
23 path::{Path, PathBuf},
24};
25
26pub(crate) use crate::localization_checks::MAX_ISSUES;
27
28pub const ANDROID_DEFAULT_LANGUAGE: &str = "default";
30
31#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
32pub struct LocalizationInfo {
33 pub table: String,
35 pub format: String,
37 pub source_language: String,
39 pub keys: usize,
41 #[serde(default)]
43 pub stale_keys: usize,
44 pub languages: Vec<LanguageCoverage>,
46 pub issue_counts: BTreeMap<String, usize>,
48 pub issues: Vec<LocalizationIssue>,
50 #[serde(default, skip_serializing_if = "Vec::is_empty")]
52 pub files: Vec<LocalizationFile>,
53}
54
55#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
56pub struct LanguageCoverage {
57 pub language: String,
58 pub translated: usize,
60 pub missing: usize,
62 pub empty: usize,
64 pub needs_review: usize,
66 pub extra: usize,
68 #[serde(default)]
70 pub issues: usize,
71}
72
73#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
74pub struct LocalizationIssue {
75 pub kind: String,
77 pub key: String,
78 pub language: String,
79 #[serde(default, skip_serializing_if = "Vec::is_empty")]
81 pub expected: Vec<String>,
82 #[serde(default, skip_serializing_if = "Vec::is_empty")]
84 pub found: Vec<String>,
85 #[serde(default, skip_serializing_if = "String::is_empty")]
87 pub text: String,
88}
89
90#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
91pub struct LocalizationFile {
92 pub path: PathBuf,
93 pub language: String,
94}
95
96pub(crate) fn is_candidate(resource: &Resource) -> bool {
99 resource.conversion_exclusion.is_none()
100 && (matches!(
101 resource.format.as_str(),
102 "xcstrings" | "strings" | "stringsdict"
103 ) || (resource.extension == "xml"
104 && resource.android.as_ref().is_some_and(|android| {
105 android.area == "res" && android.res_type.as_deref() == Some("values")
106 })))
107}
108
109pub(crate) struct ParsedFile {
111 pub index: usize,
112 pub digest: String,
113 pub format: &'static str,
114 pub table: String,
115 pub group: String,
117 pub language: Option<String>,
119 pub catalogs: Vec<langcodec::Resource>,
120}
121
122pub(crate) enum Parsed {
124 File(ParsedFile),
125 NotLocalization,
128 Unsupported(String),
130 Failed(String),
131}
132
133pub(crate) fn parse(root: &Path, resource: &Resource, index: usize) -> Parsed {
134 let bytes = match contained_file(root, &resource.path).and_then(|path| bounded_read(&path)) {
135 Ok(bytes) => bytes,
136 Err(error) => return Parsed::Failed(format!("{error:#}")),
137 };
138 if bytes.starts_with(b"bplist") && resource.format == "strings" {
139 return Parsed::NotLocalization;
140 }
141 let digest = hash(&bytes);
142 let android = resource.format == "xml";
143 match read_catalogs(resource, &bytes) {
144 Ok(catalogs) if android && catalogs.iter().all(|c| c.entries.is_empty()) => {
145 Parsed::NotLocalization
146 }
147 Ok(catalogs) => {
148 let (format, table, group, language) = identity(resource);
149 Parsed::File(ParsedFile {
150 index,
151 digest,
152 format,
153 table,
154 group,
155 language,
156 catalogs,
157 })
158 }
159 Err(_) if android => Parsed::NotLocalization,
162 Err(error) if resource.format == "stringsdict" => {
163 Parsed::Unsupported(format!("localization_parse_failed: {error}"))
164 }
165 Err(error) => Parsed::Failed(format!("localization_parse_failed: {error}")),
166 }
167}
168
169fn read_catalogs(resource: &Resource, bytes: &[u8]) -> Result<Vec<langcodec::Resource>> {
170 Ok(match resource.format.as_str() {
171 "xcstrings" => XcstringsFormat::from_reader(bytes)?.try_into()?,
172 "strings" => vec![StringsFormat::from_reader(bytes)?.into()],
173 "stringsdict" => vec![StringsdictFormat::from_reader(bytes)?.into()],
174 "xml" => vec![AndroidStringsFormat::from_reader(bytes)?.into()],
175 other => return Err(anyhow!("not a localization format: {other}")),
176 })
177}
178
179fn identity(resource: &Resource) -> (&'static str, String, String, Option<String>) {
181 let path = &resource.path;
182 let name = path
183 .file_name()
184 .map(|n| n.to_string_lossy().into_owned())
185 .unwrap_or_default();
186 let stem = path
187 .file_stem()
188 .map(|n| n.to_string_lossy().into_owned())
189 .unwrap_or_default();
190 let parent = path.parent().unwrap_or(Path::new(""));
191 match resource.format.as_str() {
192 "xcstrings" => ("xcstrings", stem, key(&["xcstrings", &text(path)]), None),
193 format @ ("strings" | "stringsdict") => {
194 let format = if format == "strings" {
195 "strings"
196 } else {
197 "stringsdict"
198 };
199 let lproj = parent
200 .file_name()
201 .map(|n| n.to_string_lossy().into_owned())
202 .and_then(|n| n.strip_suffix(".lproj").map(str::to_string));
203 match lproj {
204 Some(language) => {
205 let bundle = text(parent.parent().unwrap_or(Path::new("")));
206 (format, stem, key(&[format, &bundle, &name]), Some(language))
207 }
208 None => (format, stem, key(&[format, &text(path)]), None),
209 }
210 }
211 _ => {
212 let values = parent
213 .file_name()
214 .map(|n| n.to_string_lossy().into_owned())
215 .unwrap_or_default();
216 let res = text(parent.parent().unwrap_or(Path::new("")));
217 let other_qualifiers = non_locale_qualifiers(&values);
218 let language =
219 langcodec::infer_language_from_path(path, &FormatType::AndroidStrings(None))
220 .ok()
221 .flatten()
222 .filter(|_| values != "values")
223 .unwrap_or_else(|| ANDROID_DEFAULT_LANGUAGE.to_string());
224 (
225 "android_strings",
226 name.clone(),
227 key(&["android", &res, &other_qualifiers, &name]),
228 Some(language),
229 )
230 }
231 }
232}
233
234fn text(path: &Path) -> String {
235 path.to_string_lossy().replace('\\', "/")
236}
237
238fn key(parts: &[&str]) -> String {
239 parts.join("\u{0}")
240}
241
242fn non_locale_qualifiers(directory: &str) -> String {
245 let mut locale_seen = false;
246 let mut region_allowed = false;
247 directory
248 .split('-')
249 .skip(1)
250 .filter(|token| {
251 let language = !locale_seen
252 && (2..=3).contains(&token.len())
253 && token.chars().all(|c| c.is_ascii_lowercase())
254 && *token != "car";
255 let region = region_allowed
256 && token.len() >= 3
257 && token.starts_with('r')
258 && token[1..]
259 .chars()
260 .all(|c| c.is_ascii_uppercase() || c.is_ascii_digit());
261 let bcp47 = token.starts_with("b+");
262 region_allowed = language;
263 locale_seen |= language || bcp47;
264 !(language || region || bcp47)
265 })
266 .collect::<Vec<_>>()
267 .join("-")
268}
269
270pub(crate) fn rows(
272 resources: &[Resource],
273 files: Vec<ParsedFile>,
274) -> Vec<(usize, ResourceAnalysis)> {
275 let mut tables: BTreeMap<String, Vec<ParsedFile>> = BTreeMap::new();
276 for file in files {
277 tables.entry(file.group.clone()).or_default().push(file);
278 }
279 tables
280 .into_values()
281 .flat_map(|files| table_rows(resources, files))
282 .collect()
283}
284
285fn table_rows(
286 resources: &[Resource],
287 mut files: Vec<ParsedFile>,
288) -> Vec<(usize, ResourceAnalysis)> {
289 files.sort_by(|a, b| resources[a.index].path.cmp(&resources[b.index].path));
290 distinguish_shared_languages(resources, &mut files);
291 let mut languages: Vec<(String, &langcodec::Resource)> = vec![];
292 for file in &files {
293 for catalog in &file.catalogs {
294 let language = file
295 .language
296 .clone()
297 .unwrap_or_else(|| catalog.metadata.language.clone());
298 languages.push((language, catalog));
299 }
300 }
301 let source = source_language(&files, &languages);
302 let key_is_source_text = files.iter().any(|file| file.format == "xcstrings");
303 let table = crate::localization_checks::check(&source, &languages, key_is_source_text);
304 let listed: Vec<LocalizationFile> = if files.len() > 1 {
305 files
306 .iter()
307 .map(|file| LocalizationFile {
308 path: resources[file.index].path.clone(),
309 language: file.language.clone().unwrap_or_default(),
310 })
311 .collect()
312 } else {
313 vec![]
314 };
315 files
316 .iter()
317 .map(|file| {
318 let resource = &resources[file.index];
319 let mut row = ResourceAnalysis::new(resource, "inspected");
320 row.sha256 = Some(file.digest.clone());
321 row.resource.kind = "localization".into();
322 let own = file
325 .language
326 .as_deref()
327 .filter(|language| *language != source);
328 row.localization = Some(LocalizationInfo {
329 table: file.table.clone(),
330 format: file.format.into(),
331 source_language: source.clone(),
332 keys: table.keys,
333 stale_keys: table.stale_keys,
334 languages: table.languages.clone(),
335 issue_counts: table.issue_counts.clone(),
336 issues: table
337 .issues
338 .iter()
339 .filter(|issue| own.is_none_or(|language| issue.language == language))
340 .take(MAX_ISSUES)
341 .cloned()
342 .collect(),
343 files: listed.clone(),
344 });
345 (file.index, row)
346 })
347 .collect()
348}
349
350fn distinguish_shared_languages(resources: &[Resource], files: &mut [ParsedFile]) {
354 let mut seen: BTreeMap<String, usize> = BTreeMap::new();
355 for file in files.iter() {
356 if let Some(language) = &file.language {
357 *seen.entry(language.clone()).or_insert(0) += 1;
358 }
359 }
360 for file in files.iter_mut() {
361 if let Some(language) = &file.language
362 && seen.get(language).is_some_and(|count| *count > 1)
363 {
364 let folder = resources[file.index]
365 .path
366 .parent()
367 .and_then(Path::file_name)
368 .map(|n| n.to_string_lossy().into_owned())
369 .unwrap_or_default();
370 file.language = Some(format!("{language} ({folder})"));
371 }
372 }
373}
374
375fn source_language(files: &[ParsedFile], languages: &[(String, &langcodec::Resource)]) -> String {
379 if let Some(declared) = files
380 .iter()
381 .filter(|file| file.format == "xcstrings")
382 .flat_map(|file| &file.catalogs)
383 .find_map(|catalog| catalog.metadata.custom.get("source_language"))
384 .filter(|language| !language.is_empty())
385 {
386 return declared.clone();
387 }
388 for preferred in ["Base", ANDROID_DEFAULT_LANGUAGE, "en"] {
389 if languages.iter().any(|(language, _)| language == preferred) {
390 return preferred.into();
391 }
392 }
393 languages
394 .iter()
395 .max_by(|a, b| {
396 a.1.entries
397 .len()
398 .cmp(&b.1.entries.len())
399 .then_with(|| b.0.cmp(&a.0))
400 })
401 .map_or_else(String::new, |(language, _)| language.clone())
402}
403
404#[cfg(test)]
405mod tests {
406 use super::*;
407
408 #[test]
409 fn android_locale_qualifiers_are_not_part_of_the_table() {
410 assert_eq!(non_locale_qualifiers("values"), "");
411 assert_eq!(non_locale_qualifiers("values-zh-rCN"), "");
412 assert_eq!(non_locale_qualifiers("values-b+sr+Latn"), "");
413 assert_eq!(non_locale_qualifiers("values-es-night"), "night");
414 assert_eq!(non_locale_qualifiers("values-v21"), "v21");
415 assert_eq!(non_locale_qualifiers("values-night-v31"), "night-v31");
416 assert_eq!(non_locale_qualifiers("values-car"), "car");
417 }
418}