uqa-sql 0.6.0

PostgreSQL-compatible SQL compiler built on libpg_query
Documentation
//
// Unified Query Algebra
//
// Copyright (c) 2023-2026 Cognica, Inc.
//

//! Word boundaries of ICU 76's root break rules (`icu4c/source/data/brkitr/rules/word.txt`), which ICU's `u_strToTitle` uses when `PostgreSQL` titlecases under an ICU collation.
//!
//! The rules are UAX #29 with ICU's tailoring: colons are not `MidLetter`, Han characters are never `Extend`, complex-context letters join `ALetter`, and Hangul syllables, Han, Hiragana and Katakana chain only with their immediate neighbours of the same group. ICU additionally subdivides runs of the dictionary scripts with its dictionaries. A subdivision starts a segment at an uncased character, so it leaves the titlecased result unchanged unless it falls before uncased modifier letters, such as Thai MAIYAMOK, that a cased letter follows directly in the same segment; these rules do not reproduce that subdivision.

use icu_properties::{
    props::{ExtendedPictographic, GraphemeClusterBreak, LineBreak, Script, WordBreak as Property},
    CodePointMapData, CodePointSetData,
};
use uqa_core::{
    memory::{Produced, ProductionControl, ProductionVec},
    ValueRetentionError,
};

/// A character's role in the rules. Complex-context letters are `ALetter`, and `ALetter` excludes the dictionary CJK scripts.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
enum Class {
    Cr,
    Lf,
    Newline,
    Extend,
    Format,
    Zwj,
    RegionalIndicator,
    Katakana,
    HebrewLetter,
    ALetter,
    SingleQuote,
    DoubleQuote,
    MidNumLet,
    MidLetter,
    MidNum,
    Numeric,
    ExtendNumLet,
    WSegSpace,
    Other,
}

#[derive(Clone, Copy)]
struct Character {
    class: Class,
    pictographic: bool,
    hangul_syllable: bool,
    kana_kanji: bool,
}

/// Characters classified between cancellation checks.
const BATCH: usize = 256;

impl Character {
    const UNCLASSIFIED: Self = Self {
        class: Class::Other,
        pictographic: false,
        hangul_syllable: false,
        kana_kanji: false,
    };

    /// The classification of `character`; ASCII characters read a table built once from the same properties.
    fn of(character: char) -> Self {
        static ASCII: std::sync::OnceLock<[Character; 128]> = std::sync::OnceLock::new();
        if !character.is_ascii() {
            return Self::new(character);
        }
        ASCII.get_or_init(|| {
            std::array::from_fn(|code| {
                Self::new(char::from(
                    u8::try_from(code).expect("an ASCII code point fits in a byte"),
                ))
            })
        })[usize::from(u8::try_from(character).expect("ASCII character"))]
    }

    fn new(character: char) -> Self {
        let word_break = CodePointMapData::<Property>::new().get(character);
        let script = CodePointMapData::<Script>::new().get(character);
        let han = script == Script::Han;
        let kana_kanji = han || script == Script::Hiragana || word_break == Property::Katakana;
        let hangul_syllable = ('\u{AC00}'..='\u{D7A3}').contains(&character);
        let mut class = match word_break {
            Property::CR => Class::Cr,
            Property::LF => Class::Lf,
            Property::Newline => Class::Newline,
            Property::Extend if !han => Class::Extend,
            Property::Format => Class::Format,
            Property::ZWJ => Class::Zwj,
            Property::RegionalIndicator => Class::RegionalIndicator,
            Property::Katakana => Class::Katakana,
            Property::HebrewLetter => Class::HebrewLetter,
            Property::ALetter if !kana_kanji && !hangul_syllable => Class::ALetter,
            Property::SingleQuote => Class::SingleQuote,
            Property::DoubleQuote => Class::DoubleQuote,
            Property::MidNumLet => Class::MidNumLet,
            Property::MidLetter if !matches!(character, ':' | '\u{FE55}' | '\u{FF1A}') => {
                Class::MidLetter
            }
            Property::MidNum => Class::MidNum,
            Property::Numeric => Class::Numeric,
            Property::ExtendNumLet => Class::ExtendNumLet,
            Property::WSegSpace => Class::WSegSpace,
            _ => Class::Other,
        };
        if class == Class::Other
            && CodePointMapData::<LineBreak>::new().get(character) == LineBreak::ComplexContext
            && CodePointMapData::<GraphemeClusterBreak>::new().get(character)
                != GraphemeClusterBreak::Control
        {
            class = Class::ALetter;
        }
        Self {
            class,
            pictographic: CodePointSetData::new::<ExtendedPictographic>().contains(character),
            hangul_syllable,
            kana_kanji,
        }
    }

    /// `$ExFm`: characters that attach to the preceding character.
    fn attaches(self) -> bool {
        matches!(self.class, Class::Extend | Class::Format | Class::Zwj)
    }

    fn letter(self) -> bool {
        matches!(self.class, Class::ALetter | Class::HebrewLetter)
    }

    fn mid_letter(self) -> bool {
        matches!(
            self.class,
            Class::MidLetter | Class::MidNumLet | Class::SingleQuote
        )
    }

    fn mid_number(self) -> bool {
        matches!(
            self.class,
            Class::MidNum | Class::MidNumLet | Class::SingleQuote
        )
    }

    fn line_end(self) -> bool {
        matches!(self.class, Class::Cr | Class::Lf | Class::Newline)
    }
}

/// The ordered byte offsets where `text` has a word boundary, including its start and end.
pub(super) fn boundaries(
    text: &str,
    control: &ProductionControl<'_>,
) -> Result<Produced<Vec<usize>>, ValueRetentionError> {
    let mut characters = ProductionVec::new(*control);
    let mut pending = [Character::UNCLASSIFIED; BATCH];
    let mut length = 0;
    for character in text.chars() {
        pending[length] = Character::of(character);
        length += 1;
        if length == BATCH {
            characters.extend_from_slice(&pending)?;
            length = 0;
        }
    }
    characters.extend_from_slice(&pending[..length])?;
    let mut offsets = ProductionVec::new(*control);
    offsets.push_copy(0)?;
    let mut offset = 0;
    for (index, character) in text.chars().enumerate() {
        if index.is_multiple_of(BATCH) {
            control.check()?;
        }
        if index > 0 && !joins(&characters, index) {
            offsets.push_copy(offset)?;
        }
        offset += character.len_utf8();
    }
    if !text.is_empty() {
        offsets.push_copy(text.len())?;
    }
    offsets.finish()
}

/// Whether the characters before and at `index` stay in one segment.
fn joins(characters: &[Character], index: usize) -> bool {
    let previous = characters[index - 1];
    let next = characters[index];
    if previous.class == Class::Cr && next.class == Class::Lf {
        return true;
    }
    if previous.line_end() || next.line_end() {
        return false;
    }
    if (previous.class == Class::Zwj && next.pictographic)
        || (previous.class == Class::WSegSpace && next.class == Class::WSegSpace)
        || next.attaches()
        || (previous.hangul_syllable && next.hangul_syllable)
        || (previous.kana_kanji && next.kana_kanji)
    {
        return true;
    }
    let Some(left) = base_before(characters, index) else {
        return false;
    };
    let base = characters[left];
    let before = || base_before(characters, left).map(|index| characters[index]);
    let after = || base_after(characters, index).map(|index| characters[index]);
    if base.letter() && next.letter()
        || base.letter() && next.mid_letter() && after().is_some_and(Character::letter)
        || base.mid_letter() && next.letter() && before().is_some_and(Character::letter)
        || base.class == Class::HebrewLetter && next.class == Class::SingleQuote
    {
        return true;
    }
    let hebrew = |character: Character| character.class == Class::HebrewLetter;
    let numeric = |character: Character| character.class == Class::Numeric;
    if base.class == Class::HebrewLetter
        && next.class == Class::DoubleQuote
        && after().is_some_and(hebrew)
        || base.class == Class::DoubleQuote && hebrew(next) && before().is_some_and(hebrew)
        || numeric(base) && numeric(next)
        || base.letter() && numeric(next)
        || numeric(base) && next.letter()
        || base.mid_number() && numeric(next) && before().is_some_and(numeric)
        || numeric(base) && next.mid_number() && after().is_some_and(numeric)
        || base.class == Class::Katakana && next.class == Class::Katakana
    {
        return true;
    }
    if next.class == Class::ExtendNumLet
        && (base.letter()
            || matches!(
                base.class,
                Class::Numeric | Class::Katakana | Class::ExtendNumLet
            ))
        || base.class == Class::ExtendNumLet
            && (next.letter() || matches!(next.class, Class::Numeric | Class::Katakana))
    {
        return true;
    }
    if base.class == Class::RegionalIndicator && next.class == Class::RegionalIndicator {
        // Indicators pair from the start of their run; an odd count before this one leaves it unpaired.
        let mut count = 0_usize;
        let mut cursor = Some(left);
        while let Some(position) = cursor {
            if characters[position].class != Class::RegionalIndicator {
                break;
            }
            count += 1;
            cursor = base_before(characters, position);
        }
        return count % 2 == 1;
    }
    false
}

/// The nearest character before `index` that does not attach to its predecessor. Attached characters after the start of text or a line end are not part of a preceding base.
fn base_before(characters: &[Character], index: usize) -> Option<usize> {
    let mut position = index;
    while position > 0 {
        position -= 1;
        let character = characters[position];
        if !character.attaches() {
            return (!(character.line_end() && position + 1 < index)).then_some(position);
        }
    }
    None
}

/// The nearest character after `index` that does not attach to its predecessor.
fn base_after(characters: &[Character], index: usize) -> Option<usize> {
    (index + 1..characters.len()).find(|position| !characters[*position].attaches())
}

#[cfg(test)]
mod tests;