unic-langid-impl 0.9.6

mod errors;
mod layout_table;
#[cfg(feature = "likelysubtags")]
pub mod likelysubtags;
#[doc(hidden)]
pub mod parser;
#[cfg(feature = "serde")]
mod serde;
pub mod subtags;

pub use crate::errors::LanguageIdentifierError;
use std::fmt::Write;
use std::iter::Peekable;
use std::str::FromStr;

/// Enum representing available character direction orientations.
#[derive(Clone, Copy, Debug, PartialEq)]
pub enum CharacterDirection {
    /// Right To Left
    ///
    /// Used in languages such as Arabic, Hebrew, Fula, Kurdish etc.
    RTL,
    /// Left To Right
    ///
    /// Used in languages such as French, Spanish, English, German etc.
    LTR,
    /// Top To Bottom
    ///
    /// Used in Traditional Mongolian
    TTB,
}

type PartsTuple = (
    subtags::Language,
    Option<subtags::Script>,
    Option<subtags::Region>,
    Vec<subtags::Variant>,
);

/// `LanguageIdentifier` is a core struct representing a Unicode Language Identifier.
///
/// # Examples
///
/// ```
/// use unic_langid_impl::LanguageIdentifier;
///
/// let li: LanguageIdentifier = "en-US".parse()
///     .expect("Failed to parse.");
///
/// assert_eq!(li.language, "en");
/// assert_eq!(li.script, None);
/// assert_eq!(li.region.as_ref().map(Into::into), Some("US"));
/// assert_eq!(li.variants().len(), 0);
/// ```
///
/// # Parsing
///
/// Unicode recognizes three levels of standard conformance for any language identifier:
///
///  * *well-formed* - syntactically correct
///  * *valid* - well-formed and only uses registered language subtags, extensions, keywords, types...
///  * *canonical* - valid and no deprecated codes or structure.
///
/// At the moment parsing normalizes a well-formed language identifier converting
/// `_` separators to `-` and adjusting casing to conform to the Unicode standard.
///
/// Any bogus subtags will cause the parsing to fail with an error.
/// No subtag validation is performed.
///
/// # Examples:
///
/// ```
/// use unic_langid_impl::LanguageIdentifier;
///
/// let li: LanguageIdentifier = "eN_latn_Us-Valencia".parse()
///     .expect("Failed to parse.");
///
/// assert_eq!(li.language, "en");
/// assert_eq!(li.script.as_ref().map(Into::into), Some("Latn"));
/// assert_eq!(li.region.as_ref().map(Into::into), Some("US"));
/// assert_eq!(li.variants().map(|v| v.as_str()).collect::<Vec<_>>(), &["valencia"]);
/// ```
#[derive(Default, Debug, PartialEq, Eq, Clone, Hash, PartialOrd, Ord)]
pub struct LanguageIdentifier {
    pub language: subtags::Language,
    pub script: Option<subtags::Script>,
    pub region: Option<subtags::Region>,
    variants: Option<Box<[subtags::Variant]>>,
}

impl LanguageIdentifier {
    /// A constructor which takes a utf8 slice, parses it and
    /// produces a well-formed `LanguageIdentifier`.
    ///
    /// # Examples
    ///
    /// ```
    /// use unic_langid_impl::LanguageIdentifier;
    ///
    /// let li = LanguageIdentifier::from_bytes("en-US".as_bytes())
    ///     .expect("Parsing failed.");
    ///
    /// assert_eq!(li.to_string(), "en-US");
    /// ```
    pub fn from_bytes(v: &[u8]) -> Result<Self, LanguageIdentifierError> {
        Ok(parser::parse_language_identifier(v)?)
    }

    /// A constructor which takes optional subtags as `AsRef<[u8]>`, parses them and
    /// produces a well-formed `LanguageIdentifier`.
    ///
    /// # Examples
    ///
    /// ```
    /// use unic_langid_impl::LanguageIdentifier;
    ///
    /// let li = LanguageIdentifier::from_parts(
    ///     "fr".parse().expect("Parsing failed."),
    ///     None,
    ///     Some("CA".parse().expect("Parsing failed.")),
    ///     &[]
    /// );
    ///
    /// assert_eq!(li.to_string(), "fr-CA");
    /// ```
    pub fn from_parts(
        language: subtags::Language,
        script: Option<subtags::Script>,
        region: Option<subtags::Region>,
        variants: &[subtags::Variant],
    ) -> Self {
        let variants = if !variants.is_empty() {
            let mut v = variants.to_vec();
            v.sort_unstable();
            v.dedup();
            Some(v.into_boxed_slice())
        } else {
            None
        };

        Self {
            language,
            script,
            region,
            variants,
        }
    }

    /// # Unchecked
    ///
    /// This function accepts subtags expecting variants
    /// to be deduplicated and ordered.
    pub const fn from_raw_parts_unchecked(
        language: subtags::Language,
        script: Option<subtags::Script>,
        region: Option<subtags::Region>,
        variants: Option<Box<[subtags::Variant]>>,
    ) -> Self {
        Self {
            language,
            script,
            region,
            variants,
        }
    }

    #[doc(hidden)]
    /// This method is used by `unic-locale` to handle partial
    /// subtag iterator.
    ///
    /// Not stable.
    pub fn try_from_iter<'a>(
        iter: &mut Peekable<impl Iterator<Item = &'a [u8]>>,
        allow_extension: bool,
    ) -> Result<LanguageIdentifier, LanguageIdentifierError> {
        Ok(parser::parse_language_identifier_from_iter(
            iter,
            allow_extension,
        )?)
    }

    /// Consumes `LanguageIdentifier` and produces raw internal representations
    /// of all subtags in form of `u64`/`u32`.
    ///
    /// Primarily used for storing internal representation and restoring via
    /// `from_raw_parts_unchecked`.
    ///
    /// # Examples
    ///
    /// ```
    /// use unic_langid_impl::LanguageIdentifier;
    /// use tinystr::{TinyStr8, TinyStr4};
    ///
    /// let li: LanguageIdentifier = "en-US".parse()
    ///     .expect("Parsing failed.");
    ///
    /// let (lang, script, region, variants) = li.into_parts();
    ///
    /// // let li2 = LanguageIdentifier::from_raw_parts_unchecked(
    /// //     lang.map(|l| unsafe { TinyStr8::new_unchecked(l) }),
    /// //    script.map(|s| unsafe { TinyStr4::new_unchecked(s) }),
    /// //    region.map(|r| unsafe { TinyStr4::new_unchecked(r) }),
    /// //    variants.map(|v| v.into_iter().map(|v| unsafe { TinyStr8::new_unchecked(*v) }).collect()),
    /// //);
    ///
    /// //assert_eq!(li2.to_string(), "en-US");
    /// ```
    pub fn into_parts(self) -> PartsTuple {
        (
            self.language,
            self.script,
            self.region,
            self.variants.map_or_else(Vec::new, |v| v.to_vec()),
        )
    }

    /// Compares a `LanguageIdentifier` to another `AsRef<LanguageIdentifier`
    /// allowing for either side to use the missing fields as wildcards.
    ///
    /// This allows for matching between `en` (treated as `en-*-*-*`) and `en-US`.
    ///
    /// # Examples
    ///
    /// ```
    /// use unic_langid_impl::LanguageIdentifier;
    ///
    /// let li1: LanguageIdentifier = "en".parse()
    ///     .expect("Parsing failed.");
    ///
    /// let li2: LanguageIdentifier = "en-US".parse()
    ///     .expect("Parsing failed.");
    ///
    /// assert_ne!(li1, li2); // "en" != "en-US"
    /// assert_ne!(li1.to_string(), li2.to_string()); // "en" != "en-US"
    ///
    /// assert_eq!(li1.matches(&li2, false, false), false); // "en" != "en-US"
    /// assert_eq!(li1.matches(&li2, true, false), true); // "en-*-*-*" == "en-US"
    /// assert_eq!(li1.matches(&li2, false, true), false); // "en" != "en-*-US-*"
    /// assert_eq!(li1.matches(&li2, true, true), true); // "en-*-*-*" == "en-*-US-*"
    /// ```
    pub fn matches<O: AsRef<Self>>(
        &self,
        other: &O,
        self_as_range: bool,
        other_as_range: bool,
    ) -> bool {
        let other = other.as_ref();
        self.language
            .matches(other.language, self_as_range, other_as_range)
            && subtag_matches(&self.script, &other.script, self_as_range, other_as_range)
            && subtag_matches(&self.region, &other.region, self_as_range, other_as_range)
            && subtags_match(
                &self.variants,
                &other.variants,
                self_as_range,
                other_as_range,
            )
    }

    /// Returns a vector of variants subtags of the `LanguageIdentifier`.
    ///
    /// # Examples
    ///
    /// ```
    /// use unic_langid_impl::LanguageIdentifier;
    ///
    /// let li1: LanguageIdentifier = "ca-ES-valencia".parse()
    ///     .expect("Parsing failed.");
    ///
    /// assert_eq!(li1.variants().map(|v| v.as_str()).collect::<Vec<_>>(), &["valencia"]);
    ///
    /// let li2: LanguageIdentifier = "de".parse()
    ///     .expect("Parsing failed.");
    ///
    /// assert_eq!(li2.variants().len(), 0);
    /// ```
    pub fn variants(&self) -> impl ExactSizeIterator<Item = &subtags::Variant> {
        let variants: &[_] = match self.variants {
            Some(ref v) => v,
            None => &[],
        };

        variants.iter()
    }

    /// Sets variant subtags of the `LanguageIdentifier`.
    ///
    /// # Examples
    ///
    /// ```
    /// use unic_langid_impl::LanguageIdentifier;
    ///
    /// let mut li: LanguageIdentifier = "ca-ES".parse()
    ///     .expect("Parsing failed.");
    ///
    /// li.set_variants(&["valencia".parse().expect("Parsing failed.")]);
    ///
    /// assert_eq!(li.to_string(), "ca-ES-valencia");
    /// ```
    pub fn set_variants(&mut self, variants: &[subtags::Variant]) {
        let mut v = variants.to_vec();

        if v.is_empty() {
            self.variants = None;
        } else {
            v.sort_unstable();
            v.dedup();
            self.variants = Some(v.into_boxed_slice());
        }
    }

    /// Tests if a variant subtag is present in the `LanguageIdentifier`.
    ///
    /// # Examples
    ///
    /// ```
    /// use unic_langid_impl::LanguageIdentifier;
    ///
    /// let mut li: LanguageIdentifier = "ca-ES-macos".parse()
    ///     .expect("Parsing failed.");
    ///
    /// assert_eq!(li.has_variant("valencia".parse().unwrap()), false);
    /// assert_eq!(li.has_variant("macos".parse().unwrap()), true);
    /// ```
    pub fn has_variant(&self, variant: subtags::Variant) -> bool {
        if let Some(variants) = &self.variants {
            variants.contains(&variant)
        } else {
            false
        }
    }

    /// Clears variant subtags of the `LanguageIdentifier`.
    ///
    /// # Examples
    ///
    /// ```
    /// use unic_langid_impl::LanguageIdentifier;
    ///
    /// let mut li: LanguageIdentifier = "ca-ES-valencia".parse()
    ///     .expect("Parsing failed.");
    ///
    /// li.clear_variants();
    ///
    /// assert_eq!(li.to_string(), "ca-ES");
    /// ```
    pub fn clear_variants(&mut self) {
        self.variants = None;
    }

    /// Extends the `LanguageIdentifier` adding likely subtags based
    /// on tables provided by CLDR.
    ///
    /// # Examples
    ///
    /// ```
    /// use unic_langid_impl::LanguageIdentifier;
    ///
    /// let mut li: LanguageIdentifier = "en-US".parse()
    ///     .expect("Parsing failed.");
    ///
    /// assert_eq!(li.maximize(), true);
    /// assert_eq!(li.to_string(), "en-Latn-US");
    /// ```
    #[cfg(feature = "likelysubtags")]
    pub fn maximize(&mut self) -> bool {
        if let Some(new_li) = likelysubtags::maximize(self.language, self.script, self.region) {
            self.language = new_li.0;
            self.script = new_li.1;
            self.region = new_li.2;
            true
        } else {
            false
        }
    }

    /// Extends the `LanguageIdentifier` removing likely subtags based
    /// on tables provided by CLDR.
    ///
    /// # Examples
    ///
    /// ```
    /// use unic_langid_impl::LanguageIdentifier;
    ///
    /// let mut li: LanguageIdentifier = "en-Latn-US".parse()
    ///     .expect("Parsing failed.");
    ///
    /// assert_eq!(li.minimize(), true);
    /// assert_eq!(li.to_string(), "en");
    /// ```
    #[cfg(feature = "likelysubtags")]
    pub fn minimize(&mut self) -> bool {
        if let Some(new_li) = likelysubtags::minimize(self.language, self.script, self.region) {
            self.language = new_li.0;
            self.script = new_li.1;
            self.region = new_li.2;
            true
        } else {
            false
        }
    }

    /// Returns character direction of the `LanguageIdentifier`.
    ///
    /// # Examples
    ///
    /// ```
    /// use unic_langid_impl::{LanguageIdentifier, CharacterDirection};
    ///
    /// let li1: LanguageIdentifier = "es-AR".parse()
    ///     .expect("Parsing failed.");
    /// let li2: LanguageIdentifier = "fa".parse()
    ///     .expect("Parsing failed.");
    ///
    /// assert_eq!(li1.character_direction(), CharacterDirection::LTR);
    /// assert_eq!(li2.character_direction(), CharacterDirection::RTL);
    /// ```
    pub fn character_direction(&self) -> CharacterDirection {
        match (self.language.into(), self.script) {
            (_, Some(script))
                if layout_table::SCRIPTS_CHARACTER_DIRECTION_LTR.contains(&script.into()) =>
            {
                CharacterDirection::LTR
            }
            (_, Some(script))
                if layout_table::SCRIPTS_CHARACTER_DIRECTION_RTL.contains(&script.into()) =>
            {
                CharacterDirection::RTL
            }
            (_, Some(script))
                if layout_table::SCRIPTS_CHARACTER_DIRECTION_TTB.contains(&script.into()) =>
            {
                CharacterDirection::TTB
            }
            (Some(lang), _) if layout_table::LANGS_CHARACTER_DIRECTION_RTL.contains(&lang) => {
                #[cfg(feature = "likelysubtags")]
                if let Some((_, Some(script), _)) =
                    likelysubtags::maximize(self.language, None, self.region)
                {
                    if layout_table::SCRIPTS_CHARACTER_DIRECTION_LTR.contains(&script.into()) {
                        return CharacterDirection::LTR;
                    }
                }
                CharacterDirection::RTL
            }
            _ => CharacterDirection::LTR,
        }
    }
}

impl FromStr for LanguageIdentifier {
    type Err = LanguageIdentifierError;

    fn from_str(source: &str) -> Result<Self, Self::Err> {
        Self::from_bytes(source.as_bytes())
    }
}

impl AsRef<LanguageIdentifier> for LanguageIdentifier {
    #[inline(always)]
    fn as_ref(&self) -> &LanguageIdentifier {
        self
    }
}

impl std::fmt::Display for LanguageIdentifier {
    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
        self.language.fmt(f)?;
        if let Some(ref script) = self.script {
            f.write_char('-')?;
            script.fmt(f)?;
        }
        if let Some(ref region) = self.region {
            f.write_char('-')?;
            region.fmt(f)?;
        }
        if let Some(variants) = &self.variants {
            for variant in variants.iter() {
                f.write_char('-')?;
                variant.fmt(f)?;
            }
        }
        Ok(())
    }
}

impl PartialEq<&str> for LanguageIdentifier {
    fn eq(&self, other: &&str) -> bool {
        self.to_string().as_str() == *other
    }
}

fn subtag_matches<P: PartialEq>(
    subtag1: &Option<P>,
    subtag2: &Option<P>,
    as_range1: bool,
    as_range2: bool,
) -> bool {
    (as_range1 && subtag1.is_none()) || (as_range2 && subtag2.is_none()) || subtag1 == subtag2
}

fn is_option_empty<P: PartialEq>(subtag: &Option<Box<[P]>>) -> bool {
    subtag.as_ref().is_none_or(|t| t.is_empty())
}

fn subtags_match<P: PartialEq>(
    subtag1: &Option<Box<[P]>>,
    subtag2: &Option<Box<[P]>>,
    as_range1: bool,
    as_range2: bool,
) -> bool {
    // or is some and is empty!
    (as_range1 && is_option_empty(subtag1))
        || (as_range2 && is_option_empty(subtag2))
        || subtag1 == subtag2
}

/// This is a best-effort operation that performs all available levels of canonicalization.
///
/// At the moment the operation will normalize casing and the separator, but in the future
/// it may also validate and update from deprecated subtags to canonical ones.
///
/// # Examples
///
/// ```
/// use unic_langid_impl::canonicalize;
///
/// assert_eq!(canonicalize("pL_latn_pl"), Ok("pl-Latn-PL".to_string()));
/// ```
pub fn canonicalize<S: AsRef<[u8]>>(input: S) -> Result<String, LanguageIdentifierError> {
    let lang_id = LanguageIdentifier::from_bytes(input.as_ref())?;
    Ok(lang_id.to_string())
}

#[test]
fn invalid_subtag() {
    assert!(LanguageIdentifier::from_bytes("en-ÁÁÁÁ".as_bytes()).is_err());
}