i_slint_compiler/

translations.rs

// Copyright © SixtyFPS GmbH <info@slint.dev>
// SPDX-License-Identifier: GPL-3.0-only OR LicenseRef-Slint-Royalty-free-2.0 OR LicenseRef-Slint-Software-3.0

use crate::llr::Expression;
use rspolib::TranslatedEntry;
use smol_str::{SmolStr, ToSmolStr};
use std::collections::HashMap;
use std::collections::hash_map::Entry;
use std::path::Path;
use std::rc::Rc;

#[derive(Clone, Debug)]
pub struct Translations {
    /// An array with all the array of string
    /// The first vector index is stored in the LLR.
    /// The inner vector index is the language id. (The first is the original)
    /// Only contains the string that are not having plural forms
    pub strings: Vec<Vec<Option<SmolStr>>>,
    /// An array with all the strings that are used in a plural form.
    /// The first vector index is stored in the LLR.
    /// The inner vector index is the language. (The first is the original string)
    /// The last vector contains each form
    pub plurals: Vec<Vec<Option<Vec<SmolStr>>>>,

    /// Expression is a function that maps its first and only argument (an integer)
    /// to the plural form index (an integer)
    /// It can only do basic mathematical operations.
    /// The expression cannot reference properties or variable.
    /// Only builtin math functions, and its first argument
    pub plural_rules: Vec<Option<Expression>>,

    /// The "names" of the languages and the decimal separator
    pub languages: Vec<(SmolStr, char)>,
}

#[derive(Clone)]
pub struct TranslationsBuilder {
    result: Translations,
    /// Maps (msgid, msgid_plural, msgctx) to the index in the result
    /// (the index is in strings or plurals depending if there is a plural)
    map: HashMap<(SmolStr, SmolStr, SmolStr), usize>,

    /// The catalog containing the translations
    catalogs: Rc<Vec<rspolib::POFile>>,
}

impl TranslationsBuilder {
    pub fn load_translations(
        path: &Path,
        domain: &str,
        all_loaded_files: &mut std::collections::BTreeSet<std::path::PathBuf>,
    ) -> std::io::Result<Self> {
        let mut languages = vec![("".into(), i_slint_common::DEFAULT_DECIMAL_SEPARATOR)];
        let mut catalogs = Vec::new();
        let mut plural_rules =
            vec![Some(plural_rule_parser::parse_rule_expression("n!=1").unwrap())];
        // Sort the entries so the bundled language order doesn't depend on the
        // filesystem's directory order.
        // Otherwise the same sources produce different string tables on
        // different machines, which breaks reproducible builds.
        let mut entries = std::fs::read_dir(path)
            .map_err(|e| std::io::Error::other(format!("Error reading directory {path:?}: {e}")))?
            .collect::<Result<Vec<_>, _>>()?;
        entries.sort_by_key(|l| l.file_name());
        for l in entries {
            let path = l.path().join("LC_MESSAGES").join(format!("{domain}.po"));
            if path.exists() {
                all_loaded_files.insert(path.clone());
                let catalog = rspolib::pofile(path.as_path()).map_err(|e| {
                    std::io::Error::other(format!("Error parsing {}: {e}", path.display()))
                })?;
                let language_name = l.file_name().to_string_lossy().to_smolstr();
                languages.push((
                    language_name.clone(),
                    i_slint_common::decimal_separator_for_locale(language_name.as_str()),
                ));

                let expr = if let Some(header) = catalog.metadata.get("Plural-Forms") {
                    let plural_expr = header.split(';').find_map(|sub_entry| {
                        let (key, expression) = sub_entry.split_once('=')?;
                        (key.trim() == "plural").then_some(expression)
                    });
                    plural_expr.ok_or_else(|| {
                        std::io::Error::other(format!(
                            "Error parsing plural rules in {}",
                            path.display()
                        ))
                    })?
                } else {
                    "n != 1"
                };
                plural_rules.push(Some(plural_rule_parser::parse_rule_expression(expr).map_err(
                    |_| {
                        std::io::Error::other(format!(
                            "Error parsing plural rules in {}",
                            path.display()
                        ))
                    },
                )?));

                catalogs.push(catalog);
            }
        }
        if catalogs.is_empty() {
            return Err(std::io::Error::other(format!(
                "No translations found. We look for files in '{}/<lang>/LC_MESSAGES/{domain}.po",
                path.display()
            )));
        }
        Ok(Self {
            result: Translations {
                strings: Vec::new(),
                plurals: Vec::new(),
                plural_rules,
                languages,
            },
            map: HashMap::new(),
            catalogs: Rc::new(catalogs),
        })
    }

    pub fn lower_translate_call(&mut self, args: Vec<Expression>) -> Expression {
        let [original, contextid, _domain, format_args, n, plural] = args
            .try_into()
            .expect("The resolving pass should have ensured that the arguments are correct");
        let original = get_string(original).expect("original must be a string");
        let contextid = get_string(contextid).expect("contextid must be a string");
        let plural = get_string(plural).expect("plural must be a string");

        let is_plural =
            !plural.is_empty() || !matches!(n, Expression::NumberLiteral(f) if f == 1.0);

        match self.map.entry((original.clone(), plural.clone(), contextid.clone())) {
            Entry::Occupied(entry) => Expression::TranslationReference {
                format_args: format_args.into(),
                string_index: *entry.get(),
                plural: is_plural.then(|| n.into()),
            },
            Entry::Vacant(entry) => {
                let messages = self.catalogs.iter().map(|catalog| {
                    catalog
                        .find_by_msgid_msgctxt(original.as_str(), contextid.as_str())
                        .filter(|entry| entry.translated())
                });
                let idx = if is_plural {
                    let messages = std::iter::once(Some(vec![original.clone(), plural.clone()]))
                        .chain(messages.map(|opt_entry| {
                            opt_entry.and_then(|entry| {
                                if entry.msgstr_plural.is_empty() {
                                    None
                                } else {
                                    Some(
                                        entry
                                            .msgstr_plural
                                            .iter()
                                            .map(|s| s.to_smolstr())
                                            .collect(),
                                    )
                                }
                            })
                        }))
                        .collect();
                    self.result.plurals.push(messages);
                    self.result.plurals.len() - 1
                } else {
                    let messages = std::iter::once(Some(original.clone()))
                        .chain(messages.map(|opt_entry| {
                            opt_entry.and_then(|entry| entry.msgstr.map(|s| s.to_smolstr()))
                        }))
                        .collect::<Vec<_>>();
                    self.result.strings.push(messages);
                    self.result.strings.len() - 1
                };
                Expression::TranslationReference {
                    format_args: format_args.into(),
                    string_index: *entry.insert(idx),
                    plural: is_plural.then(|| n.into()),
                }
            }
        }
    }

    pub fn result(self) -> Translations {
        self.result
    }

    /// Add all characters in any po file to `characters_seen` if they are not yet there
    pub fn collect_characters_seen(&self, characters_seen: &mut impl Extend<char>) {
        characters_seen.extend(
            self.catalogs
                .iter()
                .flat_map(|catalog| {
                    catalog.entries.iter().flat_map(|entry| {
                        entry
                            .msgstr
                            .iter()
                            .map(|s| s.as_str())
                            .chain(entry.msgstr_plural.iter().map(|s| s.as_str()))
                    })
                })
                .flat_map(|str| str.chars()),
        );
    }
}

fn get_string(plural: Expression) -> Option<SmolStr> {
    match plural {
        Expression::StringLiteral(s) => Some(s),
        _ => None,
    }
}

mod plural_rule_parser {
    use super::Expression;
    pub struct ParseError<'a>(&'static str, &'a [u8]);
    impl std::fmt::Debug for ParseError<'_> {
        fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
            write!(f, "ParseError({}, rest={:?})", self.0, std::str::from_utf8(self.1).unwrap())
        }
    }
    pub fn parse_rule_expression(string: &str) -> Result<Expression, ParseError<'_>> {
        let ascii = string.as_bytes();
        let s = parse_expression(ascii)?;
        if !s.rest.is_empty() {
            return Err(ParseError("extra character in string", s.rest));
        }
        match s.ty {
            Ty::Number => Ok(s.expr),
            Ty::Boolean => Ok(Expression::Condition {
                condition: s.expr.into(),
                true_expr: Expression::NumberLiteral(1.).into(),
                false_expr: Expression::NumberLiteral(0.).into(),
            }),
        }
    }

    #[derive(Copy, Clone, Debug, PartialEq, Eq)]
    enum Ty {
        Number,
        Boolean,
    }

    struct ParsingState<'a> {
        expr: Expression,
        rest: &'a [u8],
        ty: Ty,
    }

    impl ParsingState<'_> {
        fn skip_whitespace(self) -> Self {
            let rest = self.rest.trim_ascii_start();
            Self { rest, ..self }
        }
    }

    /// `<condition> ('?' <expr> : <expr> )?`
    fn parse_expression(string: &[u8]) -> Result<ParsingState<'_>, ParseError<'_>> {
        let string = string.trim_ascii_start();
        let state = parse_condition(string)?.skip_whitespace();
        if state.ty != Ty::Boolean {
            return Ok(state);
        }
        if let Some(rest) = state.rest.strip_prefix(b"?") {
            let s1 = parse_expression(rest)?.skip_whitespace();
            let rest = s1.rest.strip_prefix(b":").ok_or(ParseError("expected ':'", s1.rest))?;
            let s2 = parse_expression(rest)?;
            if s1.ty != s2.ty {
                return Err(ParseError("incompatible types in ternary operator", s2.rest));
            }
            Ok(ParsingState {
                expr: Expression::Condition {
                    condition: state.expr.into(),
                    true_expr: s1.expr.into(),
                    false_expr: s2.expr.into(),
                },
                rest: s2.rest.trim_ascii_start(),
                ty: s2.ty,
            })
        } else {
            Ok(state)
        }
    }

    /// `<and_expr> ("||" <condition>)?`
    fn parse_condition(string: &[u8]) -> Result<ParsingState<'_>, ParseError<'_>> {
        let string = string.trim_ascii_start();
        let state = parse_and_expr(string)?.skip_whitespace();
        if state.rest.is_empty() {
            return Ok(state);
        }
        if let Some(rest) = state.rest.strip_prefix(b"||") {
            let state2 = parse_condition(rest)?;
            if state.ty != Ty::Boolean || state2.ty != Ty::Boolean {
                return Err(ParseError("incompatible types in || operator", state2.rest));
            }
            Ok(ParsingState {
                expr: Expression::BinaryExpression {
                    lhs: state.expr.into(),
                    rhs: state2.expr.into(),
                    op: '|',
                },
                ty: Ty::Boolean,
                rest: state2.rest.trim_ascii_start(),
            })
        } else {
            Ok(state)
        }
    }

    /// `<cmp_expr> ("&&" <and_expr>)?`
    fn parse_and_expr(string: &[u8]) -> Result<ParsingState<'_>, ParseError<'_>> {
        let string = string.trim_ascii_start();
        let state = parse_cmp_expr(string)?.skip_whitespace();
        if state.rest.is_empty() {
            return Ok(state);
        }
        if let Some(rest) = state.rest.strip_prefix(b"&&") {
            let state2 = parse_and_expr(rest)?;
            if state.ty != Ty::Boolean || state2.ty != Ty::Boolean {
                return Err(ParseError("incompatible types in || operator", state2.rest));
            }
            Ok(ParsingState {
                expr: Expression::BinaryExpression {
                    lhs: state.expr.into(),
                    rhs: state2.expr.into(),
                    op: '&',
                },
                ty: Ty::Boolean,
                rest: state2.rest.trim_ascii_start(),
            })
        } else {
            Ok(state)
        }
    }

    /// `<value> ('=='|'!='|'<'|'>'|'<='|'>=' <cmp_expr>)?`
    fn parse_cmp_expr(string: &[u8]) -> Result<ParsingState<'_>, ParseError<'_>> {
        let string = string.trim_ascii_start();
        let mut state = parse_value(string)?;
        state.rest = state.rest.trim_ascii_start();
        if state.rest.is_empty() {
            return Ok(state);
        }
        for (token, op) in [
            (b"==" as &[u8], '='),
            (b"!=", '!'),
            (b"<=", '≤'),
            (b">=", '≥'),
            (b"<", '<'),
            (b">", '>'),
        ] {
            if let Some(rest) = state.rest.strip_prefix(token) {
                let state2 = parse_cmp_expr(rest)?;
                if state.ty != Ty::Number || state2.ty != Ty::Number {
                    return Err(ParseError("incompatible types in comparison", state2.rest));
                }
                return Ok(ParsingState {
                    expr: Expression::BinaryExpression {
                        lhs: state.expr.into(),
                        rhs: state2.expr.into(),
                        op,
                    },
                    ty: Ty::Boolean,
                    rest: state2.rest.trim_ascii_start(),
                });
            }
        }
        Ok(state)
    }

    /// `<term> ('%' <term>)?`
    fn parse_value(string: &[u8]) -> Result<ParsingState<'_>, ParseError<'_>> {
        let string = string.trim_ascii_start();
        let mut state = parse_term(string)?;
        state.rest = state.rest.trim_ascii_start();
        if state.rest.is_empty() {
            return Ok(state);
        }
        if let Some(rest) = state.rest.strip_prefix(b"%") {
            let state2 = parse_term(rest)?;
            if state.ty != Ty::Number || state2.ty != Ty::Number {
                return Err(ParseError("incompatible types in % operator", state2.rest));
            }
            Ok(ParsingState {
                expr: Expression::BuiltinFunctionCall {
                    source_location: None,
                    function: crate::expression_tree::BuiltinFunction::Mod,
                    arguments: vec![state.expr, state2.expr],
                },
                ty: Ty::Number,
                rest: state2.rest.trim_ascii_start(),
            })
        } else {
            Ok(state)
        }
    }

    fn parse_term(string: &[u8]) -> Result<ParsingState<'_>, ParseError<'_>> {
        let string = string.trim_ascii_start();
        let state = match string.first().ok_or(ParseError("unexpected end of string", string))? {
            b'n' => ParsingState {
                expr: Expression::FunctionParameterReference { index: 0 },
                rest: &string[1..],
                ty: Ty::Number,
            },
            b'(' => {
                let mut s = parse_expression(&string[1..])?;
                s.rest = s.rest.strip_prefix(b")").ok_or(ParseError("expected ')'", s.rest))?;
                s
            }
            x if x.is_ascii_digit() => {
                let (n, rest) = parse_number(string)?;
                ParsingState { expr: Expression::NumberLiteral(n as _), rest, ty: Ty::Number }
            }
            _ => return Err(ParseError("unexpected token", string)),
        };
        Ok(state)
    }
    fn parse_number(string: &[u8]) -> Result<(i32, &[u8]), ParseError<'_>> {
        let end = string.iter().position(|&c| !c.is_ascii_digit()).unwrap_or(string.len());
        let n = std::str::from_utf8(&string[..end])
            .expect("string is valid utf-8")
            .parse()
            .map_err(|_| ParseError("can't parse number", string))?;
        Ok((n, &string[end..]))
    }
    #[test]
    fn test_parse_rule_expression() {
        #[track_caller]
        fn p(string: &str) -> String {
            let ctx = crate::llr::EvaluationContext {
                compilation_unit: &crate::llr::CompilationUnit {
                    public_components: Default::default(),
                    sub_components: Default::default(),
                    used_sub_components: Default::default(),
                    globals: Default::default(),
                    has_debug_info: false,
                    translations: None,
                    popup_menu: None,
                    type_exports: Default::default(),
                },
                current_scope: crate::llr::EvaluationScope::Global(0.into()),
                generator_state: (),
                argument_types: &[crate::langtype::Type::Int32],
            };
            crate::llr::pretty_print::DisplayExpression(
                &parse_rule_expression(string).expect("parse error"),
                &ctx,
            )
            .to_string()
        }

        // en
        assert_eq!(p("n != 1"), "((arg_0 ! 1.0) ? 1.0 : 0.0)");
        // fr
        assert_eq!(p("n > 1"), "((arg_0 > 1.0) ? 1.0 : 0.0)");
        // ar
        assert_eq!(
            p("(n==0 ? 0 : n==1 ? 1 : n==2 ? 2 : n%100>=3 && n%100<=10 ? 3 : n%100>=11 ? 4 : 5)"),
            "((arg_0 = 0.0) ? 0.0 : ((arg_0 = 1.0) ? 1.0 : ((arg_0 = 2.0) ? 2.0 : (((Mod(arg_0, 100.0) ≥ 3.0) & (Mod(arg_0, 100.0) ≤ 10.0)) ? 3.0 : ((Mod(arg_0, 100.0) ≥ 11.0) ? 4.0 : 5.0)))))"
        );
        // ga
        assert_eq!(
            p("n==1 ? 0 : n==2 ? 1 : (n>2 && n<7) ? 2 :(n>6 && n<11) ? 3 : 4"),
            "((arg_0 = 1.0) ? 0.0 : ((arg_0 = 2.0) ? 1.0 : (((arg_0 > 2.0) & (arg_0 < 7.0)) ? 2.0 : (((arg_0 > 6.0) & (arg_0 < 11.0)) ? 3.0 : 4.0))))"
        );
        // ja
        assert_eq!(p("0"), "0.0");
        // pl
        assert_eq!(
            p("(n==1 ? 0 : n%10>=2 && n%10<=4 && (n%100<10 || n%100>=20) ? 1 : 2)"),
            "((arg_0 = 1.0) ? 0.0 : (((Mod(arg_0, 10.0) ≥ 2.0) & ((Mod(arg_0, 10.0) ≤ 4.0) & ((Mod(arg_0, 100.0) < 10.0) | (Mod(arg_0, 100.0) ≥ 20.0)))) ? 1.0 : 2.0))",
        );

        // ru
        assert_eq!(
            p("(n%10==1 && n%100!=11 ? 0 : n%10>=2 && n%10<=4 && (n%100<10 || n%100>=20) ? 1 : 2)"),
            "(((Mod(arg_0, 10.0) = 1.0) & (Mod(arg_0, 100.0) ! 11.0)) ? 0.0 : (((Mod(arg_0, 10.0) ≥ 2.0) & ((Mod(arg_0, 10.0) ≤ 4.0) & ((Mod(arg_0, 100.0) < 10.0) | (Mod(arg_0, 100.0) ≥ 20.0)))) ? 1.0 : 2.0))",
        );
    }
}