from __future__ import print_function
import argparse
import json
import os
import re
import shutil
import subprocess
import sys
import unittest
_SCRIPT_DIR = os.path.dirname(__file__)
_SCRIPT_NAME = os.path.join(_SCRIPT_DIR, os.path.basename(__file__))
_TOP_SRC_DIR = os.path.join(_SCRIPT_DIR, '..')
sys.path.insert(0, os.path.join(_SCRIPT_DIR, 'android/gyp'))
from util import build_utils
from util import resource_utils
_DEFAULT_LOCALE = 'en-US'
_CONSOLE_CODE_MOVE_CURSOR_TO_COLUMN_0 = '\x1b[0G'
_CONSOLE_CODE_ERASE_LINE = '\x1b[K'
_CONSOLE_START_LINE = (
_CONSOLE_CODE_MOVE_CURSOR_TO_COLUMN_0 + _CONSOLE_CODE_ERASE_LINE)
def _FixChromiumLangAttribute(lang):
_CHROMIUM_LANG_FIXES = {
'en': 'en-US', 'iw': 'he', 'no': 'nb', }
return _CHROMIUM_LANG_FIXES.get(lang, lang)
def _FixTranslationConsoleLocaleName(locale):
_FIXES = {
'nb': 'no', 'he': 'iw', }
return _FIXES.get(locale, locale)
def _CompareLocaleLists(list_a, list_expected, list_name):
errors = []
missing_locales = sorted(set(list_a) - set(list_expected))
if missing_locales:
errors.append('Missing locales: %s' % missing_locales)
extra_locales = sorted(set(list_expected) - set(list_a))
if extra_locales:
errors.append('Unexpected locales: %s' % extra_locales)
if errors:
print('Errors in %s definition:' % list_name)
for error in errors:
print(' %s\n' % error)
return True
return False
def _BuildIntervalList(input_list, predicate):
result = []
size = len(input_list)
start = 0
while True:
while start < size and not predicate(input_list[start]):
start += 1
if start >= size:
return result
end = start + 1
while end < size and predicate(input_list[end]):
end += 1
result.append((start, end))
start = end + 1
def _SortListSubRange(input_list, start, end, key_func):
result = input_list[:start]
inputs = []
for pos in xrange(start, end):
line = input_list[pos]
key = key_func(line)
inputs.append((key, line))
for _, line in sorted(inputs):
result.append(line)
result += input_list[end:]
return result
def _SortElementsRanges(lines, element_predicate, element_key):
intervals = _BuildIntervalList(lines, element_predicate)
for start, end in intervals:
lines = _SortListSubRange(lines, start, end, element_key)
return lines
def _ProcessFile(input_file, locales, check_func, fix_func):
print('%sProcessing %s...' % (_CONSOLE_START_LINE, input_file), end=' ')
sys.stdout.flush()
with open(input_file) as f:
input_lines = f.readlines()
errors = check_func(input_file, input_lines, locales)
if errors:
print('\n%s%s' % (_CONSOLE_START_LINE, '\n'.join(errors)))
if fix_func:
try:
input_lines = fix_func(input_file, input_lines, locales)
output = ''.join(input_lines)
with open(input_file, 'wt') as f:
f.write(output)
print('Fixed %s.' % input_file)
except Exception as e: print('Skipped %s: %s' % (input_file, e))
return True
def _ScanDirectoriesForFiles(scan_dirs, file_predicate):
result = []
for src_dir in scan_dirs:
for root, _, files in os.walk(src_dir):
result.extend(os.path.join(root, f) for f in files if file_predicate(f))
return result
def _WriteFile(file_path, file_data):
with open(file_path, 'w') as f:
f.write(file_data)
def _FindGnExecutable():
buildtools_dir = os.path.join(_TOP_SRC_DIR, 'buildtools')
for subdir in os.listdir(buildtools_dir):
subdir_path = os.path.join(buildtools_dir, subdir)
if not os.path.isdir(subdir_path):
continue
gn_path = os.path.join(subdir_path, 'gn')
if os.path.exists(gn_path):
return gn_path
return None
def _PrettyPrintListAsLines(input_list, available_width, trailing_comma=False):
result = []
input_str = ', '.join(input_list)
while len(input_str) > available_width:
pos = input_str.rfind(',', 0, available_width)
result.append(input_str[:pos + 1])
input_str = input_str[pos + 1:].lstrip()
if trailing_comma and input_str:
input_str += ','
result.append(input_str)
return result
class _PrettyPrintListAsLinesTest(unittest.TestCase):
def test_empty_list(self):
self.assertListEqual([''], _PrettyPrintListAsLines([], 10))
def test_wrapping(self):
input_list = ['foo', 'bar', 'zoo', 'tool']
self.assertListEqual(
_PrettyPrintListAsLines(input_list, 8),
['foo,', 'bar,', 'zoo,', 'tool'])
self.assertListEqual(
_PrettyPrintListAsLines(input_list, 12), ['foo, bar,', 'zoo, tool'])
self.assertListEqual(
_PrettyPrintListAsLines(input_list, 79), ['foo, bar, zoo, tool'])
def test_trailing_comma(self):
input_list = ['foo', 'bar', 'zoo', 'tool']
self.assertListEqual(
_PrettyPrintListAsLines(input_list, 8, trailing_comma=True),
['foo,', 'bar,', 'zoo,', 'tool,'])
self.assertListEqual(
_PrettyPrintListAsLines(input_list, 12, trailing_comma=True),
['foo, bar,', 'zoo, tool,'])
self.assertListEqual(
_PrettyPrintListAsLines(input_list, 79, trailing_comma=True),
['foo, bar, zoo, tool,'])
_INTERNAL_CHROME_LOCALES = []
_INTERNAL_ANDROID_APK_OMITTED_LOCALES = []
_INTERNAL_IOS_UNSUPPORTED_LOCALES = []
def ChromeLocales():
if not _INTERNAL_CHROME_LOCALES:
_ExtractAllChromeLocalesLists()
return _INTERNAL_CHROME_LOCALES
def AndroidAPKOmittedLocales():
if not _INTERNAL_ANDROID_APK_OMITTED_LOCALES:
_ExtractAllChromeLocalesLists()
return _INTERNAL_ANDROID_APK_OMITTED_LOCALES
def IosUnsupportedLocales():
if not _INTERNAL_IOS_UNSUPPORTED_LOCALES:
_ExtractAllChromeLocalesLists()
return _INTERNAL_IOS_UNSUPPORTED_LOCALES
def _PrepareTinyGnWorkspace(work_dir, out_subdir_name='out'):
_WriteFile(os.path.join(work_dir, '.gn'),
'buildconfig = "//BUILDCONFIG.gn"\n')
_WriteFile(
os.path.join(work_dir, 'BUILDCONFIG.gn'),
r'''set_default_toolchain("toolchain")
declare_args () {
is_ios = false
is_android = true
}
''')
os.mkdir(os.path.join(work_dir, 'toolchain'))
_WriteFile(os.path.join(work_dir, 'toolchain', 'BUILD.gn'),
r'''toolchain("toolchain") {
tool("stamp") {
command = "touch {{output}}" # Required by action()
}
}
''')
_WriteFile(
os.path.join(work_dir, 'BUILD.gn'), r'''import("//locales.gni")
action("create_foo") { script = "//build/create_foo.py"
inputs = []
outputs = [ "$target_out_dir/$target_name" ]
}
_filename = root_build_dir + "/foo"
write_file(_filename + ".locales", locales, "json")
write_file(_filename + ".android_apk_omitted_locales",
android_apk_omitted_locales,
"json")
write_file(_filename + ".ios_unsupported_locales",
ios_unsupported_locales,
"json")
''')
shutil.copyfile(os.path.join(_TOP_SRC_DIR, 'build', 'config', 'locales.gni'),
os.path.join(work_dir, 'locales.gni'))
out_path = os.path.join(work_dir, out_subdir_name)
os.mkdir(out_path)
return out_path
_DEBUG_LOCALES_WORK_DIR = None
def _ReadJsonList(file_path):
with open(file_path) as f:
data = json.load(f)
assert isinstance(data, list), "JSON file %s is not a list!" % file_path
return [item.encode('utf8') for item in data]
def _ExtractAllChromeLocalesLists():
with build_utils.TempDir() as tmp_path:
if _DEBUG_LOCALES_WORK_DIR:
tmp_path = _DEBUG_LOCALES_WORK_DIR
build_utils.DeleteDirectory(tmp_path)
build_utils.MakeDirectory(tmp_path)
out_path = _PrepareTinyGnWorkspace(tmp_path, 'out')
gn_executable = _FindGnExecutable()
try:
subprocess.check_output(
[gn_executable, 'gen', out_path, '--root=' + tmp_path])
except subprocess.CalledProcessError as e:
print(e.output)
raise e
global _INTERNAL_CHROME_LOCALES
_INTERNAL_CHROME_LOCALES = _ReadJsonList(
os.path.join(out_path, 'foo.locales'))
global _INTERNAL_ANDROID_APK_OMITTED_LOCALES
_INTERNAL_ANDROID_APK_OMITTED_LOCALES = _ReadJsonList(
os.path.join(out_path, 'foo.android_apk_omitted_locales'))
global _INTERNAL_IOS_UNSUPPORTED_LOCALES
_INTERNAL_IOS_UNSUPPORTED_LOCALES = _ReadJsonList(
os.path.join(out_path, 'foo.ios_unsupported_locales'))
_RE_OUTPUT_ELEMENT = re.compile(r'<output (.*)\s*/>')
_RE_TRANSLATION_ELEMENT = re.compile(r'<file( | .* )path="(.*\.xtb)".*/>')
_RE_FILENAME_ATTRIBUTE = re.compile(r'filename="([^"]*)"')
_RE_LANG_ATTRIBUTE = re.compile(r'lang="([^"]*)"')
_RE_PATH_ATTRIBUTE = re.compile(r'path="([^"]*)"')
_RE_TYPE_ANDROID_ATTRIBUTE = re.compile(r'type="android"')
def _IsGritInputFile(input_file):
return input_file.endswith('.grd')
def _GetXmlLangAttribute(xml_line):
m = _RE_LANG_ATTRIBUTE.search(xml_line)
if not m:
return None
return m.group(1)
class _GetXmlLangAttributeTest(unittest.TestCase):
TEST_DATA = {
'': None,
'foo': None,
'lang=foo': None,
'lang="foo"': 'foo',
'<something lang="foo bar" />': 'foo bar',
'<file lang="fr-CA" path="path/to/strings_fr-CA.xtb" />': 'fr-CA',
}
def test_GetXmlLangAttribute(self):
for test_line, expected in self.TEST_DATA.iteritems():
self.assertEquals(_GetXmlLangAttribute(test_line), expected)
def _SortGrdElementsRanges(grd_lines, element_predicate):
return _SortElementsRanges(grd_lines, element_predicate, _GetXmlLangAttribute)
def _CheckGrdElementRangeLang(grd_lines, start, end, wanted_locales):
errors = []
locales = set()
for pos in xrange(start, end):
line = grd_lines[pos]
lang = _GetXmlLangAttribute(line)
if not lang:
errors.append('%d: Missing "lang" attribute in <output> element' % pos +
1)
continue
cr_locale = _FixChromiumLangAttribute(lang)
if cr_locale in locales:
errors.append(
'%d: Redefinition of <output> for "%s" locale' % (pos + 1, lang))
locales.add(cr_locale)
extra_locales = locales.difference(wanted_locales)
if extra_locales:
errors.append('%d-%d: Extra locales found: %s' % (start + 1, end + 1,
sorted(extra_locales)))
missing_locales = wanted_locales.difference(locales)
if missing_locales:
errors.append('%d-%d: Missing locales: %s' % (start + 1, end + 1,
sorted(missing_locales)))
return errors
def _IsGrdAndroidOutputLine(line):
m = _RE_OUTPUT_ELEMENT.search(line)
if m:
return 'type="android"' in m.group(1)
return False
assert _IsGrdAndroidOutputLine(' <output type="android"/>')
def _CheckGrdElementRangeAndroidOutputFilename(grd_lines, start, end,
wanted_locales):
errors = []
for pos in xrange(start, end):
line = grd_lines[pos]
lang = _GetXmlLangAttribute(line)
if not lang:
continue
cr_locale = _FixChromiumLangAttribute(lang)
m = _RE_FILENAME_ATTRIBUTE.search(line)
if not m:
errors.append('%d: Missing filename attribute in <output> element' % pos +
1)
else:
filename = m.group(1)
if not filename.endswith('.xml'):
errors.append(
'%d: Filename should end with ".xml": %s' % (pos + 1, filename))
dirname = os.path.basename(os.path.dirname(filename))
prefix = ('values-%s' % resource_utils.ToAndroidLocaleName(cr_locale)
if cr_locale != _DEFAULT_LOCALE else 'values')
if dirname != prefix:
errors.append(
'%s: Directory name should be %s: %s' % (pos + 1, prefix, filename))
return errors
def _CheckGrdAndroidOutputElements(grd_file, grd_lines, wanted_locales):
intervals = _BuildIntervalList(grd_lines, _IsGrdAndroidOutputLine)
errors = []
for start, end in intervals:
errors += _CheckGrdElementRangeLang(grd_lines, start, end, wanted_locales)
errors += _CheckGrdElementRangeAndroidOutputFilename(grd_lines, start, end,
wanted_locales)
return errors
def _AddMissingLocalesInGrdAndroidOutputs(grd_file, grd_lines, wanted_locales):
intervals = _BuildIntervalList(grd_lines, _IsGrdAndroidOutputLine)
for start, end in reversed(intervals):
locales = set()
for pos in xrange(start, end):
lang = _GetXmlLangAttribute(grd_lines[pos])
locale = _FixChromiumLangAttribute(lang)
locales.add(locale)
missing_locales = wanted_locales.difference(locales)
if not missing_locales:
continue
src_locale = 'bg'
src_lang_attribute = 'lang="%s"' % src_locale
src_line = None
for pos in xrange(start, end):
if src_lang_attribute in grd_lines[pos]:
src_line = grd_lines[pos]
break
if not src_line:
raise Exception(
'Cannot find <output> element with "%s" lang attribute' % src_locale)
line_count = end - 1
for locale in missing_locales:
android_locale = resource_utils.ToAndroidLocaleName(locale)
dst_line = src_line.replace(
'lang="%s"' % src_locale, 'lang="%s"' % locale).replace(
'values-%s/' % src_locale, 'values-%s/' % android_locale)
grd_lines.insert(line_count, dst_line)
line_count += 1
return _SortGrdElementsRanges(grd_lines, _IsGrdAndroidOutputLine)
def _IsTranslationGrdOutputLine(line):
m = _RE_TRANSLATION_ELEMENT.search(line)
return m is not None
class _IsTranslationGrdOutputLineTest(unittest.TestCase):
def test_GrdTranslationOutputLines(self):
_VALID_INPUT_LINES = [
'<file path="foo/bar.xtb" />',
'<file path="foo/bar.xtb"/>',
'<file lang="fr-CA" path="translations/aw_strings_fr-CA.xtb"/>',
'<file lang="fr-CA" path="translations/aw_strings_fr-CA.xtb" />',
' <file path="translations/aw_strings_ar.xtb" lang="ar" />',
]
_INVALID_INPUT_LINES = ['<file path="foo/bar.xml" />']
for line in _VALID_INPUT_LINES:
self.assertTrue(
_IsTranslationGrdOutputLine(line),
'_IsTranslationGrdOutputLine() returned False for [%s]' % line)
for line in _INVALID_INPUT_LINES:
self.assertFalse(
_IsTranslationGrdOutputLine(line),
'_IsTranslationGrdOutputLine() returned True for [%s]' % line)
def _CheckGrdTranslationElementRange(grd_lines, start, end,
wanted_locales):
errors = []
for pos in xrange(start, end):
line = grd_lines[pos]
lang = _GetXmlLangAttribute(line)
if not lang:
continue
m = _RE_PATH_ATTRIBUTE.search(line)
if not m:
errors.append('%d: Missing path attribute in <file> element' % pos +
1)
else:
filename = m.group(1)
if not filename.endswith('.xtb'):
errors.append(
'%d: Path should end with ".xtb": %s' % (pos + 1, filename))
return errors
def _CheckGrdTranslations(grd_file, grd_lines, wanted_locales):
wanted_locales = wanted_locales - set([_DEFAULT_LOCALE])
intervals = _BuildIntervalList(grd_lines, _IsTranslationGrdOutputLine)
errors = []
for start, end in intervals:
errors += _CheckGrdElementRangeLang(grd_lines, start, end, wanted_locales)
errors += _CheckGrdTranslationElementRange(grd_lines, start, end,
wanted_locales)
return errors
_RE_TRANSLATIONBUNDLE = re.compile('<translationbundle lang="(.*)">')
def _CreateFakeXtbFileFrom(src_xtb_path, dst_xtb_path, dst_locale):
with open(src_xtb_path) as f:
src_xtb_lines = f.readlines()
def replace_xtb_lang_attribute(line):
m = _RE_TRANSLATIONBUNDLE.search(line)
if not m:
return line
return line[:m.start(1)] + dst_locale + line[m.end(1):]
dst_xtb_lines = [replace_xtb_lang_attribute(line) for line in src_xtb_lines]
with build_utils.AtomicOutput(dst_xtb_path) as tmp:
tmp.writelines(dst_xtb_lines)
def _AddMissingLocalesInGrdTranslations(grd_file, grd_lines, wanted_locales):
wanted_locales = wanted_locales - set([_DEFAULT_LOCALE])
intervals = _BuildIntervalList(grd_lines, _IsTranslationGrdOutputLine)
for start, end in reversed(intervals):
locales = set()
for pos in xrange(start, end):
lang = _GetXmlLangAttribute(grd_lines[pos])
locale = _FixChromiumLangAttribute(lang)
locales.add(locale)
missing_locales = wanted_locales.difference(locales)
if not missing_locales:
continue
src_locale = 'en-GB'
src_lang_attribute = 'lang="%s"' % src_locale
src_line = None
for pos in xrange(start, end):
if src_lang_attribute in grd_lines[pos]:
src_line = grd_lines[pos]
break
if not src_line:
raise Exception(
'Cannot find <file> element with "%s" lang attribute' % src_locale)
src_path = os.path.join(
os.path.dirname(grd_file),
_RE_PATH_ATTRIBUTE.search(src_line).group(1))
line_count = end - 1
for locale in missing_locales:
dst_line = src_line.replace(
'lang="%s"' % src_locale, 'lang="%s"' % locale).replace(
'_%s.xtb' % src_locale, '_%s.xtb' % locale)
grd_lines.insert(line_count, dst_line)
line_count += 1
dst_path = src_path.replace('_%s.xtb' % src_locale, '_%s.xtb' % locale)
_CreateFakeXtbFileFrom(src_path, dst_path, locale)
return _SortGrdElementsRanges(grd_lines, _IsTranslationGrdOutputLine)
_RE_GN_VALUES_LIST_LINE = re.compile(
r'^\s*".*values(\-([A-Za-z0-9-]+))?/.*\.xml",\s*$')
def _IsBuildGnInputFile(input_file):
return os.path.basename(input_file) == 'BUILD.gn'
def _GetAndroidGnOutputLocale(line):
m = _RE_GN_VALUES_LIST_LINE.match(line)
if not m:
return None
if m.group(1): return m.group(2)
return resource_utils.ToAndroidLocaleName(_DEFAULT_LOCALE)
def _IsAndroidGnOutputLine(line):
return _GetAndroidGnOutputLocale(line) != None
def _CheckGnOutputsRangeForLocalizedStrings(gn_lines, start, end):
for pos in xrange(start, end):
if not 'values/' in gn_lines[pos]:
return True
return False
def _CheckGnOutputsRange(gn_lines, start, end, wanted_locales):
if not _CheckGnOutputsRangeForLocalizedStrings(gn_lines, start, end):
return []
errors = []
locales = set()
for pos in xrange(start, end):
line = gn_lines[pos]
android_locale = _GetAndroidGnOutputLocale(line)
assert android_locale != None
cr_locale = resource_utils.ToChromiumLocaleName(android_locale)
if cr_locale in locales:
errors.append('%s: Redefinition of output for "%s" locale' %
(pos + 1, android_locale))
locales.add(cr_locale)
extra_locales = locales.difference(wanted_locales)
if extra_locales:
errors.append('%d-%d: Extra locales: %s' % (start + 1, end + 1,
sorted(extra_locales)))
missing_locales = wanted_locales.difference(locales)
if missing_locales:
errors.append('%d-%d: Missing locales: %s' % (start + 1, end + 1,
sorted(missing_locales)))
return errors
def _CheckGnAndroidOutputs(gn_file, gn_lines, wanted_locales):
intervals = _BuildIntervalList(gn_lines, _IsAndroidGnOutputLine)
errors = []
for start, end in intervals:
errors += _CheckGnOutputsRange(gn_lines, start, end, wanted_locales)
return errors
def _AddMissingLocalesInGnAndroidOutputs(gn_file, gn_lines, wanted_locales):
intervals = _BuildIntervalList(gn_lines, _IsAndroidGnOutputLine)
for start, end in reversed(intervals):
if not _CheckGnOutputsRangeForLocalizedStrings(gn_lines, start, end):
continue
locales = set()
for pos in xrange(start, end):
lang = _GetAndroidGnOutputLocale(gn_lines[pos])
locale = resource_utils.ToChromiumLocaleName(lang)
locales.add(locale)
missing_locales = wanted_locales.difference(locales)
if not missing_locales:
continue
src_locale = 'bg'
src_values = 'values-%s/' % resource_utils.ToAndroidLocaleName(src_locale)
src_line = None
for pos in xrange(start, end):
if src_values in gn_lines[pos]:
src_line = gn_lines[pos]
break
if not src_line:
raise Exception(
'Cannot find output list item with "%s" locale' % src_locale)
line_count = end - 1
for locale in missing_locales:
if locale == _DEFAULT_LOCALE:
dst_line = src_line.replace('values-%s/' % src_locale, 'values/')
else:
dst_line = src_line.replace(
'values-%s/' % src_locale,
'values-%s/' % resource_utils.ToAndroidLocaleName(locale))
gn_lines.insert(line_count, dst_line)
line_count += 1
gn_lines = _SortListSubRange(
gn_lines, start, line_count,
lambda line: _RE_GN_VALUES_LIST_LINE.match(line).group(1))
return gn_lines
_EXPECTATIONS_FILENAME = 'translation_expectations.pyl'
def _ReadPythonLiteralFile(pyl_path):
with open(pyl_path) as f:
pyl_content = f.read()
return eval(pyl_content, dict(), dict())
def _UpdateLocalesInExpectationLines(pyl_lines,
wanted_locales,
available_width=79):
locales_list = ['"%s"' % loc for loc in sorted(wanted_locales)]
result = []
line_count = len(pyl_lines)
line_num = 0
DICT_START = '"languages": ['
while line_num < line_count:
line = pyl_lines[line_num]
line_num += 1
result.append(line)
pos = line.find(DICT_START)
if pos < 0:
continue
start_margin = pos
start_line = line_num
while (line_num < line_count and
not pyl_lines[line_num].rstrip().endswith('],')):
line_num += 1
continue
if line_num == line_count:
raise Exception('%d: Missing list termination!' % start_line)
locale_width = available_width - (start_margin + 2)
locale_lines = _PrettyPrintListAsLines(
locales_list, locale_width, trailing_comma=True)
for locale_line in locale_lines:
result.append(' ' * (start_margin + 2) + locale_line)
result.append(' ' * start_margin + '],')
line_num += 1
return result
class _UpdateLocalesInExpectationLinesTest(unittest.TestCase):
def test_simple(self):
self.maxDiff = 1000
input_text = r'''
{
"android_grd": {
"languages": [
"aa", "bb", "cc", "dd", "ee",
"ff", "gg", "hh", "ii", "jj",
"kk"],
},
"another_grd": {
"languages": [
"aa", "bb", "cc", "dd", "ee", "ff", "gg", "hh", "ii", "jj", "kk",
],
},
}
'''
expected_text = r'''
{
"android_grd": {
"languages": [
"A2", "AA", "BB", "CC", "DD",
"E2", "EE", "FF", "GG", "HH",
"I2", "II", "JJ", "KK",
],
},
"another_grd": {
"languages": [
"A2", "AA", "BB", "CC", "DD",
"E2", "EE", "FF", "GG", "HH",
"I2", "II", "JJ", "KK",
],
},
}
'''
input_lines = input_text.splitlines()
test_locales = ([
'AA', 'BB', 'CC', 'DD', 'EE', 'FF', 'GG', 'HH', 'II', 'JJ', 'KK', 'A2',
'E2', 'I2'
])
expected_lines = expected_text.splitlines()
self.assertListEqual(
_UpdateLocalesInExpectationLines(input_lines, test_locales, 40),
expected_lines)
def test_missing_list_termination(self):
input_lines = r'''
"languages": ['
"aa", "bb", "cc", "dd"
'''.splitlines()
with self.assertRaises(Exception) as cm:
_UpdateLocalesInExpectationLines(input_lines, ['a', 'b'], 40)
self.assertEqual(str(cm.exception), '2: Missing list termination!')
def _UpdateLocalesInExpectationFile(pyl_path, wanted_locales):
tc_locales = {
_FixTranslationConsoleLocaleName(locale)
for locale in set(wanted_locales) - set([_DEFAULT_LOCALE])
}
with open(pyl_path) as f:
input_lines = [l.rstrip() for l in f.readlines()]
updated_lines = _UpdateLocalesInExpectationLines(input_lines, tc_locales)
with build_utils.AtomicOutput(pyl_path) as f:
f.writelines('\n'.join(updated_lines) + '\n')
def _IsAllInputFile(input_file):
return _IsGritInputFile(input_file) or _IsBuildGnInputFile(input_file)
def _CheckAllFiles(input_file, input_lines, wanted_locales):
errors = []
if _IsGritInputFile(input_file):
errors += _CheckGrdTranslations(input_file, input_lines, wanted_locales)
errors += _CheckGrdAndroidOutputElements(
input_file, input_lines, wanted_locales)
elif _IsBuildGnInputFile(input_file):
errors += _CheckGnAndroidOutputs(input_file, input_lines, wanted_locales)
return errors
def _AddMissingLocalesInAllFiles(input_file, input_lines, wanted_locales):
if _IsGritInputFile(input_file):
lines = _AddMissingLocalesInGrdTranslations(
input_file, input_lines, wanted_locales)
lines = _AddMissingLocalesInGrdAndroidOutputs(
input_file, lines, wanted_locales)
elif _IsBuildGnInputFile(input_file):
lines = _AddMissingLocalesInGnAndroidOutputs(
input_file, input_lines, wanted_locales)
return lines
class _Command(object):
name = None
description = None
long_description = None
def __init__(self):
self._parser = None
self.args = None
def RegisterExtraArgs(self, subparser):
pass
def RegisterArgs(self, parser):
subp = parser.add_parser(
self.name, help=self.description,
description=self.long_description or self.description,
formatter_class=argparse.RawDescriptionHelpFormatter)
self._parser = subp
subp.set_defaults(command=self)
group = subp.add_argument_group('%s arguments' % self.name)
self.RegisterExtraArgs(group)
def ProcessArgs(self, args):
self.args = args
class _ListLocalesCommand(_Command):
name = 'list-locales'
description = 'List supported Chrome locales'
long_description = r'''
List locales of interest, by default this prints all locales supported by
Chrome, but `--type=android_apk_omitted` can be used to print the list of
locales omitted from Android APKs (but not app bundles), and
`--type=ios_unsupported` for the list of locales unsupported on iOS.
These values are extracted directly from build/config/locales.gni.
Additionally, use the --as-json argument to print the list as a JSON list,
instead of the default format (which is a space-separated list of locale names).
'''
TYPE_MAP = {
'all': ChromeLocales,
'android_apk_omitted': AndroidAPKOmittedLocales,
'ios_unsupported': IosUnsupportedLocales,
}
def RegisterExtraArgs(self, group):
group.add_argument(
'--as-json',
action='store_true',
help='Output as JSON list.')
group.add_argument(
'--type',
choices=tuple(self.TYPE_MAP.viewkeys()),
default='all',
help='Select type of locale list to print.')
def Run(self):
locale_list = self.TYPE_MAP[self.args.type]()
if self.args.as_json:
print('[%s]' % ", ".join("'%s'" % loc for loc in locale_list))
else:
print(' '.join(locale_list))
class _CheckInputFileBaseCommand(_Command):
select_file_func = None
check_func = None
fix_func = None
def RegisterExtraArgs(self, group):
group.add_argument(
'--scan-dir',
action='append',
help='Optional directory to scan for input files recursively.')
group.add_argument(
'input',
nargs='*',
help='Input file(s) to check.')
group.add_argument(
'--fix-inplace',
action='store_true',
help='Try to fix the files in-place too.')
group.add_argument(
'--add-locales',
help='Space-separated list of additional locales to use')
def Run(self):
args = self.args
input_files = []
if args.input:
input_files = args.input
if args.scan_dir:
input_files.extend(_ScanDirectoriesForFiles(
args.scan_dir, self.select_file_func.__func__))
locales = ChromeLocales()
if args.add_locales:
locales.extend(args.add_locales.split(' '))
locales = set(locales)
for input_file in input_files:
_ProcessFile(input_file,
locales,
self.check_func.__func__,
self.fix_func.__func__ if args.fix_inplace else None)
print('%sDone.' % (_CONSOLE_START_LINE))
class _CheckGrdAndroidOutputsCommand(_CheckInputFileBaseCommand):
name = 'check-grd-android-outputs'
description = (
'Check the Android resource (.xml) files outputs in GRIT input files.')
long_description = r'''
Check the Android .xml files outputs in one or more input GRIT (.grd) files
for the following conditions:
- Each item has a correct 'lang' attribute.
- There are no duplicated lines for the same 'lang' attribute.
- That there are no extra locales that Chromium doesn't want.
- That no wanted locale is missing.
- Filenames exist for each listed locale.
- Filenames are well-formed.
'''
select_file_func = _IsGritInputFile
check_func = _CheckGrdAndroidOutputElements
fix_func = _AddMissingLocalesInGrdAndroidOutputs
class _CheckGrdTranslationsCommand(_CheckInputFileBaseCommand):
name = 'check-grd-translations'
description = (
'Check the translation (.xtb) files outputted by .grd input files.')
long_description = r'''
Check the translation (.xtb) file outputs in one or more input GRIT (.grd) files
for the following conditions:
- Each item has a correct 'lang' attribute.
- There are no duplicated lines for the same 'lang' attribute.
- That there are no extra locales that Chromium doesn't want.
- That no wanted locale is missing.
- Each item has a 'path' attribute.
- Each such path value ends up with '.xtb'.
'''
select_file_func = _IsGritInputFile
check_func = _CheckGrdTranslations
fix_func = _AddMissingLocalesInGrdTranslations
class _CheckGnAndroidOutputsCommand(_CheckInputFileBaseCommand):
name = 'check-gn-android-outputs'
description = 'Check the Android .xml file lists in GN build files.'
long_description = r'''
Check one or more BUILD.gn file, looking for lists of Android resource .xml
files, and checking that:
- There are no duplicated output files in the list.
- Each output file belongs to a wanted Chromium locale.
- There are no output files for unwanted Chromium locales.
'''
select_file_func = _IsBuildGnInputFile
check_func = _CheckGnAndroidOutputs
fix_func = _AddMissingLocalesInGnAndroidOutputs
class _CheckAllCommand(_CheckInputFileBaseCommand):
name = 'check-all'
description = 'Check everything.'
long_description = 'Equivalent to calling all other check-xxx commands.'
select_file_func = _IsAllInputFile
check_func = _CheckAllFiles
fix_func = _AddMissingLocalesInAllFiles
class _UpdateExpectationsCommand(_Command):
name = 'update-expectations'
description = 'Update translation expectations file.'
long_description = r'''
Update %s files to match the current list of locales supported by Chromium.
This is especially useful to add new locales before updating any GRIT or GN
input file with the --add-locales option.
''' % _EXPECTATIONS_FILENAME
def RegisterExtraArgs(self, group):
group.add_argument(
'--add-locales',
help='Space-separated list of additional locales to use.')
def Run(self):
locales = ChromeLocales()
add_locales = self.args.add_locales
if add_locales:
locales.extend(add_locales.split(' '))
expectation_paths = [
'tools/gritsettings/translation_expectations.pyl',
'clank/tools/translation_expectations.pyl',
]
missing_expectation_files = []
for path in enumerate(expectation_paths):
file_path = os.path.join(_TOP_SRC_DIR, path)
if not os.path.exists(file_path):
missing_expectation_files.append(file_path)
continue
_UpdateLocalesInExpectationFile(file_path, locales)
if missing_expectation_files:
sys.stderr.write('WARNING: Missing file(s): %s\n' %
(', '.join(missing_expectation_files)))
class _UnitTestsCommand(_Command):
name = 'unit-tests'
description = 'Run internal unit-tests for this script'
def RegisterExtraArgs(self, group):
group.add_argument(
'-v', '--verbose', action='count', help='Increase test verbosity.')
group.add_argument('args', nargs=argparse.REMAINDER)
def Run(self):
argv = [_SCRIPT_NAME] + self.args.args
unittest.main(argv=argv, verbosity=self.args.verbose)
_COMMANDS = [
_ListLocalesCommand,
_CheckGrdAndroidOutputsCommand,
_CheckGrdTranslationsCommand,
_CheckGnAndroidOutputsCommand,
_CheckAllCommand,
_UpdateExpectationsCommand,
_UnitTestsCommand,
]
def main(argv):
parser = argparse.ArgumentParser(
description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
subparsers = parser.add_subparsers()
commands = [clazz() for clazz in _COMMANDS]
for command in commands:
command.RegisterArgs(subparsers)
if not argv:
argv = ['--help']
args = parser.parse_args(argv)
args.command.ProcessArgs(args)
args.command.Run()
if __name__ == "__main__":
main(sys.argv[1:])