use std::borrow::Cow;
use std::collections::HashMap;
use squonk_ast::dialect::FeatureSet;
use squonk_ast::{FoldedSymbol, Keyword, Symbol, lookup_keyword};
mod fast_hash;
use fast_hash::FxBuildHasher;
const KEYWORD_COUNT: u32 = Keyword::ALL.len() as u32;
#[derive(Debug)]
pub struct Interner {
map: HashMap<Box<str>, Symbol, FxBuildHasher>,
strings: Vec<Box<str>>,
}
impl Default for Interner {
fn default() -> Self {
Self::new()
}
}
impl Interner {
pub fn new() -> Self {
Self {
map: HashMap::default(),
strings: Vec::new(),
}
}
pub fn intern(&mut self, s: &str) -> Symbol {
if let Some(keyword) = lookup_keyword(s) {
if keyword.as_str() == s {
return keyword.symbol();
}
}
self.intern_dynamic(s)
}
pub(crate) fn intern_nonkeyword(&mut self, s: &str) -> Symbol {
self.intern_dynamic(s)
}
pub(crate) fn intern_keyword_ident(&mut self, kw: Keyword, s: &str) -> Symbol {
if s == kw.as_str() {
return kw.symbol();
}
self.intern_dynamic(s)
}
fn intern_dynamic(&mut self, s: &str) -> Symbol {
if let Some(&existing) = self.map.get(s) {
return existing;
}
let dynamic_key =
u32::try_from(self.strings.len() + 1).expect("interner exceeded u32::MAX");
let key = KEYWORD_COUNT
.checked_add(dynamic_key)
.expect("interner exceeded u32::MAX distinct symbols");
let symbol = Symbol::new(key).expect("interner keys are one-based and non-zero");
let text: Box<str> = Box::from(s);
self.strings.push(text.clone());
self.map.insert(text, symbol);
symbol
}
pub fn freeze(self) -> FrozenResolver {
FrozenResolver {
strings: self.strings.into_boxed_slice(),
}
}
}
#[derive(Debug)]
pub struct FrozenResolver {
strings: Box<[Box<str>]>,
}
impl squonk_ast::Resolver for FrozenResolver {
fn try_resolve(&self, sym: Symbol) -> Option<&str> {
resolve_symbol(&self.strings, sym)
}
}
impl FrozenResolver {
#[cfg(any(feature = "serde-serialize", feature = "serde-deserialize"))]
pub(crate) fn dynamic_strings(&self) -> &[Box<str>] {
&self.strings
}
pub fn fold_symbols(&self, features: &FeatureSet) -> FoldedSymbols {
FoldedSymbols::build(&self.strings, features)
}
}
#[derive(Debug)]
pub struct FoldedSymbols {
folded: Box<[FoldedSymbol]>,
}
impl FoldedSymbols {
fn build(strings: &[Box<str>], features: &FeatureSet) -> Self {
let dynamic = u32::try_from(strings.len()).expect("dynamic symbol count fits u32");
let total = KEYWORD_COUNT
.checked_add(dynamic)
.expect("folded symbol space exceeds u32::MAX");
let mut folded = Vec::with_capacity(total as usize);
let mut ids: HashMap<Cow<'_, str>, FoldedSymbol, FxBuildHasher> = HashMap::default();
let mut next_id: u32 = 1;
for raw in 1..=total {
let sym = Symbol::new(raw).expect("folded keys are one-based and non-zero");
let text = resolve_symbol(strings, sym).expect("every symbol in range resolves");
let key = features.fold_unquoted_identifier(text);
let id = *ids.entry(key).or_insert_with(|| {
let id = FoldedSymbol::new(next_id).expect("folded ids are one-based and non-zero");
next_id += 1;
id
});
folded.push(id);
}
Self {
folded: folded.into_boxed_slice(),
}
}
pub fn fold(&self, sym: Symbol) -> Option<FoldedSymbol> {
self.folded.get(sym.index()).copied()
}
}
fn resolve_symbol(strings: &[Box<str>], sym: Symbol) -> Option<&str> {
let raw = sym.as_u32();
if raw <= KEYWORD_COUNT {
let index = usize::try_from(raw - 1).expect("keyword symbol index fits usize");
return Some(Keyword::ALL[index].as_str());
}
let dynamic_index =
usize::try_from(raw - KEYWORD_COUNT - 1).expect("dynamic symbol index fits usize");
strings.get(dynamic_index).map(|stored| &**stored)
}
impl squonk_ast::Resolver for Interner {
fn try_resolve(&self, sym: Symbol) -> Option<&str> {
resolve_symbol(&self.strings, sym)
}
}
#[cfg(test)]
mod tests {
use super::*;
use squonk_ast::Resolver as _;
use squonk_ast::dialect::{Casing, FeatureDelta};
#[test]
fn interning_same_text_returns_same_symbol() {
let mut interner = Interner::new();
let first = interner.intern("users");
let second = interner.intern("users");
assert_eq!(first, second);
assert_eq!(first.as_u32(), second.as_u32());
}
#[test]
fn fast_hash_map_dedups_and_folds_a_pile_of_identifiers() {
use std::collections::HashMap;
let pile = [
"users",
"Users",
"USERS",
"id",
"id",
"users",
"i_item_id",
"ss_quantity",
"ss_list_price",
"ss_coupon_amt",
"store_sales",
"customer_demographics",
"date_dim",
"item",
"promotion",
"ss_sold_date_sk",
"d_date_sk",
"ss_item_sk",
"i_item_sk",
"cd_gender",
"Cd_Gender",
"CD_GENDER",
"id",
"agg1",
"agg2",
"agg3",
];
let mut interner = Interner::new();
let mut seen: HashMap<&str, Symbol> = HashMap::new();
let mut symbol_to_text: HashMap<Symbol, &str> = HashMap::new();
for &text in &pile {
let sym = interner.intern(text);
match seen.get(&text) {
Some(&first) => assert_eq!(sym, first, "re-interning {text:?} must dedup"),
None => {
assert!(
symbol_to_text.insert(sym, text).is_none(),
"{text:?} collided onto another text's symbol"
);
seen.insert(text, sym);
}
}
}
let distinct_texts: std::collections::HashSet<&str> = pile.iter().copied().collect();
assert_eq!(seen.len(), distinct_texts.len());
let resolver = interner.freeze();
let folded = resolver.fold_symbols(&FeatureSet::POSTGRES);
let fold = |text| folded.fold(seen[text]).expect("interned symbol folds");
assert_eq!(fold("users"), fold("Users"));
assert_eq!(fold("users"), fold("USERS"));
assert_eq!(fold("cd_gender"), fold("Cd_Gender"));
assert_eq!(fold("cd_gender"), fold("CD_GENDER"));
assert_ne!(fold("users"), fold("cd_gender"));
assert_ne!(fold("users"), fold("id"));
}
#[test]
fn keywords_are_preinterned_in_fixed_low_slots() {
let mut interner = Interner::new();
for keyword in Keyword::ALL {
assert_eq!(interner.intern(keyword.as_str()), keyword.symbol());
}
assert_ne!(interner.intern("SELECT"), Keyword::Select.symbol());
}
#[test]
fn keyword_identifiers_preserve_non_canonical_source_case() {
let mut interner = Interner::new();
let canonical = interner.intern("asc");
let mixed = interner.intern("Asc");
let upper = interner.intern("ASC");
assert_eq!(canonical, Keyword::Asc.symbol());
assert_ne!(mixed, Keyword::Asc.symbol());
assert_ne!(upper, Keyword::Asc.symbol());
assert_ne!(mixed, upper);
let resolver = interner.freeze();
assert_eq!(resolver.resolve(canonical), "asc");
assert_eq!(resolver.resolve(mixed), "Asc");
assert_eq!(resolver.resolve(upper), "ASC");
}
#[test]
fn nonkeyword_skip_matches_full_intern_identity() {
for text in ["users", "id", "_c1", "selecting", "fromage", "order_by"] {
assert_eq!(lookup_keyword(text), None, "test precondition: {text:?}");
let skipped = Interner::new().intern_nonkeyword(text);
let full = Interner::new().intern(text);
assert_eq!(
skipped, full,
"intern_nonkeyword({text:?}) must match intern({text:?})",
);
}
}
#[test]
fn keyword_ident_skip_matches_full_intern_identity() {
let cases = [
(Keyword::Select, "select"), (Keyword::Asc, "asc"), (Keyword::Asc, "Asc"), (Keyword::Asc, "ASC"), ];
for (kw, text) in cases {
assert_eq!(
lookup_keyword(text),
Some(kw),
"test precondition: {text:?}"
);
let skipped = Interner::new().intern_keyword_ident(kw, text);
let full = Interner::new().intern(text);
assert_eq!(
skipped, full,
"intern_keyword_ident({kw:?}, {text:?}) must match intern({text:?})",
);
}
assert_eq!(
Interner::new().intern_keyword_ident(Keyword::Select, "select"),
Keyword::Select.symbol(),
);
}
#[test]
fn distinct_text_returns_distinct_symbols() {
let mut interner = Interner::new();
let users = interner.intern("users");
let id = interner.intern("id");
assert_ne!(users, id);
}
#[test]
fn freeze_then_resolve_returns_exact_original_case() {
let mut interner = Interner::new();
let lower = interner.intern("users");
let upper = interner.intern("USERS");
let mixed = interner.intern("Users");
assert_ne!(lower, upper);
assert_ne!(lower, mixed);
assert_ne!(upper, mixed);
let resolver = interner.freeze();
assert_eq!(resolver.try_resolve(lower), Some("users"));
assert_eq!(resolver.try_resolve(upper), Some("USERS"));
assert_eq!(resolver.try_resolve(mixed), Some("Users"));
assert_eq!(resolver.resolve(mixed), "Users");
}
#[test]
fn resolving_out_of_range_symbol_returns_none() {
let mut interner = Interner::new();
let only = interner.intern("solo");
let resolver = interner.freeze();
assert_eq!(resolver.try_resolve(only), Some("solo"));
let beyond = Symbol::new(only.as_u32() + 1).expect("non-zero symbol");
assert_eq!(resolver.try_resolve(beyond), None);
let fresh = Interner::new().freeze();
assert_eq!(fresh.try_resolve(Keyword::All.symbol()), Some("all"));
let beyond_keywords = Symbol::new(Keyword::ALL.len() as u32 + 1).expect("non-zero symbol");
assert_eq!(fresh.try_resolve(beyond_keywords), None);
}
#[test]
fn frozen_resolver_is_send_and_sync() {
fn assert_send_sync<T: Send + Sync>() {}
assert_send_sync::<FrozenResolver>();
}
#[test]
fn mixed_case_unquoted_identifiers_share_folded_identity() {
let mut interner = Interner::new();
let lower = interner.intern("users");
let mixed = interner.intern("Users");
let upper = interner.intern("USERS");
let other = interner.intern("id");
assert_ne!(lower, mixed);
assert_ne!(lower, upper);
let resolver = interner.freeze();
let folded = resolver.fold_symbols(&FeatureSet::POSTGRES);
let lower_id = folded.fold(lower).expect("folded id for interned symbol");
assert_eq!(folded.fold(mixed), Some(lower_id));
assert_eq!(folded.fold(upper), Some(lower_id));
assert_ne!(folded.fold(other), Some(lower_id));
assert_eq!(resolver.resolve(mixed), "Users");
assert_eq!(resolver.resolve(upper), "USERS");
}
#[test]
fn quoted_identifiers_keep_case_sensitive_exact_identity() {
let mut interner = Interner::new();
let upper = interner.intern("Col");
let lower = interner.intern("col");
assert_ne!(upper, lower);
let resolver = interner.freeze();
let folded = resolver.fold_symbols(&FeatureSet::POSTGRES);
assert_eq!(folded.fold(upper), folded.fold(lower));
assert_ne!(upper, lower);
assert_eq!(resolver.resolve(upper), "Col");
assert_eq!(resolver.resolve(lower), "col");
}
#[test]
fn dialect_casing_changes_folded_identity() {
let mut interner = Interner::new();
let lower = interner.intern("users");
let upper = interner.intern("USERS");
let resolver = interner.freeze();
let upper_fold = resolver.fold_symbols(&FeatureSet::ANSI);
assert_eq!(upper_fold.fold(lower), upper_fold.fold(upper));
let lower_fold = resolver.fold_symbols(&FeatureSet::POSTGRES);
assert_eq!(lower_fold.fold(lower), lower_fold.fold(upper));
let preserve =
FeatureSet::ANSI.with(FeatureDelta::EMPTY.identifier_casing(Casing::Preserve));
let preserve_fold = resolver.fold_symbols(&preserve);
assert_ne!(preserve_fold.fold(lower), preserve_fold.fold(upper));
}
#[test]
fn keyword_slots_fold_alongside_dynamic_identifiers() {
let mut interner = Interner::new();
let upper_select = interner.intern("SELECT");
assert_ne!(upper_select, Keyword::Select.symbol());
let resolver = interner.freeze();
let upper_fold = resolver.fold_symbols(&FeatureSet::ANSI);
assert_eq!(
upper_fold.fold(Keyword::Select.symbol()),
upper_fold.fold(upper_select)
);
let lower_fold = resolver.fold_symbols(&FeatureSet::POSTGRES);
assert_eq!(
lower_fold.fold(Keyword::Select.symbol()),
lower_fold.fold(upper_select)
);
}
#[test]
fn fold_returns_none_for_foreign_symbol() {
let mut interner = Interner::new();
let only = interner.intern("solo");
let resolver = interner.freeze();
let folded = resolver.fold_symbols(&FeatureSet::POSTGRES);
assert!(folded.fold(only).is_some());
let beyond = Symbol::new(only.as_u32() + 1).expect("non-zero symbol");
assert_eq!(folded.fold(beyond), None);
}
}