use std::collections::HashMap;
use std::collections::hash_map::Entry;
use std::ops::Range;
use std::sync::Arc;
use std::time::Duration;
use anyhow::Result;
use rand::Rng;
use revision::revisioned;
use serde::{Deserialize, Serialize};
use tokio::sync::{Mutex, RwLock};
use tokio::time::sleep;
use uuid::Uuid;
use web_time::Instant;
use crate::catalog::providers::{DatabaseProvider, NamespaceProvider, TableProvider};
use crate::catalog::{DatabaseId, IndexId, NamespaceId, TableId};
use crate::ctx::Context;
use crate::err::Error;
use crate::idx::IndexKeyBase;
use crate::idx::seqdocids::DocId;
use crate::key::database::th::TableIdGeneratorBatchKey;
use crate::key::database::ti::TableIdGeneratorStateKey;
use crate::key::namespace::dh::DatabaseIdGeneratorBatchKey;
use crate::key::namespace::di::DatabaseIdGeneratorStateKey;
use crate::key::root::nh::NamespaceIdGeneratorBatchKey;
use crate::key::root::ni::NamespaceIdGeneratorStateKey;
use crate::key::sequence::Prefix;
use crate::key::sequence::ba::Ba;
use crate::key::sequence::st::St;
use crate::key::table::ih::IndexIdGeneratorBatchKey;
use crate::key::table::is::IndexIdGeneratorStateKey;
use crate::kvs::ds::TransactionFactory;
use crate::kvs::{KVKey, LockType, Transaction, TransactionType, impl_kv_value_revisioned};
use crate::val::TableName;
type SequencesMap = Arc<RwLock<HashMap<Arc<SequenceDomain>, Arc<Mutex<Sequence>>>>>;
#[derive(Clone)]
pub struct Sequences {
tf: TransactionFactory,
nid: Uuid,
sequences: SequencesMap,
}
#[derive(Hash, PartialEq, Eq)]
enum SequenceDomain {
UserName(NamespaceId, DatabaseId, String),
FullTextDocIds(IndexKeyBase),
NameSpacesIds,
DatabasesIds(NamespaceId),
TablesIds(NamespaceId, DatabaseId),
IndexIds(NamespaceId, DatabaseId, TableName),
}
impl SequenceDomain {
fn new_user(ns: NamespaceId, db: DatabaseId, sq: &str) -> Self {
Self::UserName(ns, db, sq.to_string())
}
pub(crate) fn new_ft_doc_ids(ikb: IndexKeyBase) -> Self {
Self::FullTextDocIds(ikb)
}
pub(crate) fn new_namespace_ids() -> Self {
Self::NameSpacesIds
}
pub(crate) fn new_database_ids(ns: NamespaceId) -> Self {
Self::DatabasesIds(ns)
}
pub(crate) fn new_table_ids(ns: NamespaceId, db: DatabaseId) -> Self {
Self::TablesIds(ns, db)
}
pub(crate) fn new_index_ids(ns: NamespaceId, db: DatabaseId, tb: TableName) -> Self {
Self::IndexIds(ns, db, tb)
}
fn new_batch_range_keys(&self) -> Result<Range<Vec<u8>>> {
match self {
Self::UserName(ns, db, sq) => Prefix::new_ba_range(*ns, *db, sq),
Self::FullTextDocIds(ibk) => ibk.new_ib_range(),
Self::NameSpacesIds => NamespaceIdGeneratorBatchKey::range(),
Self::DatabasesIds(ns) => DatabaseIdGeneratorBatchKey::range(*ns),
Self::TablesIds(ns, db) => TableIdGeneratorBatchKey::range(*ns, *db),
Self::IndexIds(ns, db, tb) => IndexIdGeneratorBatchKey::range(*ns, *db, tb),
}
}
fn new_batch_key(&self, start: i64) -> Result<Vec<u8>> {
match &self {
Self::UserName(ns, db, sq) => Ba::new(*ns, *db, sq, start).encode_key(),
Self::FullTextDocIds(ikb) => ikb.new_ib_key(start).encode_key(),
Self::NameSpacesIds => NamespaceIdGeneratorBatchKey::new(start).encode_key(),
Self::DatabasesIds(ns) => DatabaseIdGeneratorBatchKey::new(*ns, start).encode_key(),
Self::TablesIds(ns, db) => TableIdGeneratorBatchKey::new(*ns, *db, start).encode_key(),
Self::IndexIds(ns, db, tb) => {
IndexIdGeneratorBatchKey::new(*ns, *db, tb, start).encode_key()
}
}
}
fn new_state_key(&self, nid: Uuid) -> Result<Vec<u8>> {
match &self {
Self::UserName(ns, db, sq) => St::new(*ns, *db, sq, nid).encode_key(),
Self::FullTextDocIds(ikb) => ikb.new_is_key(nid).encode_key(),
Self::NameSpacesIds => NamespaceIdGeneratorStateKey::new(nid).encode_key(),
Self::DatabasesIds(ns) => DatabaseIdGeneratorStateKey::new(*ns, nid).encode_key(),
Self::TablesIds(ns, db) => TableIdGeneratorStateKey::new(*ns, *db, nid).encode_key(),
Self::IndexIds(ns, db, tb) => {
IndexIdGeneratorStateKey::new(*ns, *db, tb, nid).encode_key()
}
}
}
}
#[revisioned(revision = 1)]
#[derive(Clone, Debug, Eq, PartialEq, PartialOrd, Serialize, Deserialize, Hash)]
pub(crate) struct BatchValue {
to: i64,
owner: Uuid,
}
impl_kv_value_revisioned!(BatchValue);
impl BatchValue {
#[cfg(test)]
pub(crate) fn new(to: i64, owner: Uuid) -> Self {
Self {
to,
owner,
}
}
}
#[revisioned(revision = 1)]
#[derive(Clone, Debug, Eq, PartialEq, PartialOrd, Serialize, Deserialize, Hash)]
pub(crate) struct SequenceState {
next: i64,
}
impl_kv_value_revisioned!(SequenceState);
impl SequenceState {
#[cfg(test)]
pub(crate) fn new(next: i64) -> Self {
Self {
next,
}
}
}
impl Sequences {
pub(super) fn new(tf: TransactionFactory, nid: Uuid) -> Self {
Self {
tf,
sequences: Arc::new(Default::default()),
nid,
}
}
pub(crate) async fn namespace_removed(&self, tx: &Transaction, ns: NamespaceId) -> Result<()> {
for db in tx.all_db(ns, None).await?.iter() {
self.database_removed(tx, ns, db.database_id).await?;
}
Ok(())
}
pub(crate) async fn database_removed(
&self,
tx: &Transaction,
ns: NamespaceId,
db: DatabaseId,
) -> Result<()> {
for sqs in tx.all_db_sequences(ns, db, None).await?.iter() {
self.sequence_removed(ns, db, &sqs.name).await;
}
Ok(())
}
pub(crate) async fn sequence_removed(&self, ns: NamespaceId, db: DatabaseId, sq: &str) {
let key = SequenceDomain::new_user(ns, db, sq);
self.sequences.write().await.remove(&key);
}
async fn next_val(
&self,
ctx: Option<&Context>,
seq: Arc<SequenceDomain>,
start: i64,
batch: u32,
timeout: Option<Duration>,
) -> Result<i64> {
let sequence = self.sequences.read().await.get(&seq).cloned();
if let Some(s) = sequence {
return s.lock().await.next(self, ctx, &seq, batch).await;
}
let s = match self.sequences.write().await.entry(Arc::clone(&seq)) {
Entry::Occupied(e) => Arc::clone(e.get()),
Entry::Vacant(e) => {
let s = Arc::new(Mutex::new(
Sequence::load(ctx, self, &seq, start, batch, timeout).await?,
));
Arc::clone(e.insert(s))
}
};
s.lock().await.next(self, ctx, &seq, batch).await
}
pub(crate) async fn next_namespace_id(&self, ctx: Option<&Context>) -> Result<NamespaceId> {
let domain = Arc::new(SequenceDomain::new_namespace_ids());
let id = self.next_val(ctx, domain, 0, 100, None).await?;
Ok(NamespaceId(id as u32))
}
pub(crate) async fn next_database_id(
&self,
ctx: Option<&Context>,
ns: NamespaceId,
) -> Result<DatabaseId> {
let domain = Arc::new(SequenceDomain::new_database_ids(ns));
let id = self.next_val(ctx, domain, 0, 100, None).await?;
Ok(DatabaseId(id as u32))
}
pub(crate) async fn next_table_id(
&self,
ctx: Option<&Context>,
ns: NamespaceId,
db: DatabaseId,
) -> Result<TableId> {
let domain = Arc::new(SequenceDomain::new_table_ids(ns, db));
let id = self.next_val(ctx, domain, 0, 100, None).await?;
Ok(TableId(id as u32))
}
pub(crate) async fn next_index_id(
&self,
ctx: Option<&Context>,
ns: NamespaceId,
db: DatabaseId,
tb: TableName,
) -> Result<IndexId> {
let domain = Arc::new(SequenceDomain::new_index_ids(ns, db, tb));
let id = self.next_val(ctx, domain, 0, 100, None).await?;
Ok(IndexId(id as u32))
}
pub(crate) async fn next_user_sequence_id(
&self,
ctx: Option<&Context>,
tx: &Transaction,
ns: NamespaceId,
db: DatabaseId,
sq: &str,
) -> Result<i64> {
let seq = tx.get_db_sequence(ns, db, sq, None).await?;
let domain = Arc::new(SequenceDomain::new_user(ns, db, sq));
self.next_val(ctx, domain, seq.start, seq.batch, seq.timeout).await
}
pub(crate) async fn next_fts_doc_id(
&self,
ctx: Option<&Context>,
ikb: IndexKeyBase,
batch: u32,
) -> Result<DocId> {
let domain = Arc::new(SequenceDomain::new_ft_doc_ids(ikb));
let id = self.next_val(ctx, domain, 0, batch, None).await?;
Ok(id as DocId)
}
}
struct Sequence {
tf: TransactionFactory,
st: SequenceState,
timeout: Option<Duration>,
to: i64,
state_key: Vec<u8>,
}
impl Sequence {
async fn load(
ctx: Option<&Context>,
sqs: &Sequences,
seq: &SequenceDomain,
start: i64,
batch: u32,
timeout: Option<Duration>,
) -> Result<Self> {
let state_key = seq.new_state_key(sqs.nid)?;
let tx =
sqs.tf.transaction(TransactionType::Read, LockType::Optimistic, sqs.clone()).await?;
let mut st: SequenceState = if let Some(v) = tx.get(&state_key, None).await? {
revision::from_slice(&v)?
} else {
let start = Self::seed_start_from_catalog(&tx, seq, start).await?;
SequenceState {
next: start,
}
};
tx.cancel().await?;
let (from, to) =
Self::find_batch_allocation(sqs, ctx, seq, st.next, batch, timeout).await?;
st.next = from;
Ok(Self {
tf: sqs.tf.clone(),
state_key,
to,
st,
timeout,
})
}
async fn seed_start_from_catalog(
tx: &Transaction,
seq: &SequenceDomain,
start: i64,
) -> Result<i64> {
let mut seeded = start;
match seq {
SequenceDomain::NameSpacesIds => {
for ns in tx.all_ns(None).await?.iter() {
seeded = seeded.max(ns.namespace_id.0 as i64 + 1);
}
}
SequenceDomain::DatabasesIds(ns) => {
for db in tx.all_db(*ns, None).await?.iter() {
seeded = seeded.max(db.database_id.0 as i64 + 1);
}
}
SequenceDomain::TablesIds(ns, db) => {
for tb in tx.all_tb(*ns, *db, None).await?.iter() {
seeded = seeded.max(tb.table_id.0 as i64 + 1);
}
}
SequenceDomain::IndexIds(ns, db, tb) => {
for ix in tx.all_tb_indexes(*ns, *db, tb, None).await?.iter() {
seeded = seeded.max(ix.index_id.0 as i64 + 1);
}
}
SequenceDomain::FullTextDocIds(_) | SequenceDomain::UserName(..) => {}
}
Ok(seeded)
}
async fn next(
&mut self,
sqs: &Sequences,
ctx: Option<&Context>,
seq: &SequenceDomain,
batch: u32,
) -> Result<i64> {
if self.st.next >= self.to {
(self.st.next, self.to) =
Self::find_batch_allocation(sqs, ctx, seq, self.st.next, batch, self.timeout)
.await?;
}
let v = self.st.next;
self.st.next += 1;
let tx =
self.tf.transaction(TransactionType::Write, LockType::Optimistic, sqs.clone()).await?;
match tx.set(&self.state_key, &revision::to_vec(&self.st)?).await {
Ok(_) => {
tx.commit().await?;
Ok(v)
}
Err(e) => {
tx.cancel().await?;
Err(e)
}
}
}
async fn find_batch_allocation(
sqs: &Sequences,
ctx: Option<&Context>,
seq: &SequenceDomain,
next: i64,
batch: u32,
to: Option<Duration>,
) -> Result<(i64, i64)> {
let mut tempo = 4;
const MAX_BACKOFF: u64 = 32_768;
let start = if to.is_some() {
Some(Instant::now())
} else {
None
};
loop {
if let Some(ctx) = ctx {
ctx.expect_not_timedout().await?;
} else {
yield_now!();
}
if let (Some(ref start), Some(ref to)) = (start, to) {
if start.elapsed().ge(to) {
let timeout = (*to).into();
return Err(anyhow::Error::new(Error::QueryTimedout(timeout)));
}
}
if let Ok(r) = Self::check_batch_allocation(sqs, seq, next, batch).await {
return Ok(r);
}
let sleep_ms = rand::rng().random_range(1..=tempo);
sleep(Duration::from_millis(sleep_ms)).await;
if tempo < MAX_BACKOFF {
tempo *= 2;
}
}
}
async fn check_batch_allocation(
sqs: &Sequences,
seq: &SequenceDomain,
next: i64,
batch: u32,
) -> Result<(i64, i64)> {
let tx =
sqs.tf.transaction(TransactionType::Write, LockType::Optimistic, sqs.clone()).await?;
let result = async {
let batch_range = seq.new_batch_range_keys()?;
let val = tx.getr(batch_range, None).await?;
let mut next_start = next;
for (key, val) in val.iter() {
let ba: BatchValue = revision::from_slice(val)?;
next_start = next_start.max(ba.to);
if ba.owner == sqs.nid {
if next < ba.to {
return Ok((next, ba.to));
}
tx.del(key).await?;
}
}
let next_to = next_start + batch as i64;
let bv = revision::to_vec(&BatchValue {
to: next_to,
owner: sqs.nid,
})?;
let batch_key = seq.new_batch_key(next_start)?;
tx.set(&batch_key, &bv).await?;
Ok::<(i64, i64), anyhow::Error>((next_start, next_to))
}
.await;
match result {
Ok(res) => {
tx.commit().await?;
Ok(res)
}
Err(e) => {
tx.cancel().await?;
Err(e)
}
}
}
}
#[cfg(test)]
mod tests {
use crate::catalog::providers::{DatabaseProvider, NamespaceProvider, TableProvider};
use crate::catalog::{
DatabaseDefinition, DatabaseId, Index, IndexDefinition, IndexId, NamespaceDefinition,
NamespaceId, TableDefinition, TableId,
};
use crate::kvs::sequences::{Sequence, SequenceDomain};
use crate::kvs::{Datastore, LockType, TransactionType};
use crate::val::TableName;
#[tokio::test]
async fn seed_start_from_catalog_uses_max_existing_id() {
let ds = Datastore::new("memory").await.unwrap();
let ns_id = NamespaceId(7);
let db_id = DatabaseId(11);
let tb_name: TableName = "tb".into();
let tx = ds.transaction(TransactionType::Write, LockType::Optimistic).await.unwrap();
tx.put_ns(NamespaceDefinition {
namespace_id: ns_id,
name: "ns".into(),
comment: None,
})
.await
.unwrap();
tx.put_db(
"ns",
DatabaseDefinition {
namespace_id: ns_id,
database_id: db_id,
name: "db".into(),
comment: None,
changefeed: None,
strict: false,
},
)
.await
.unwrap();
tx.put_tb("ns", "db", &TableDefinition::new(ns_id, db_id, TableId(13), tb_name.clone()))
.await
.unwrap();
tx.put_tb_index(
ns_id,
db_id,
&tb_name,
&IndexDefinition {
index_id: IndexId(17),
name: "ix".into(),
table_name: tb_name.clone(),
cols: vec![],
index: Index::Idx,
comment: None,
prepare_remove: false,
},
)
.await
.unwrap();
tx.commit().await.unwrap();
let tx = ds.transaction(TransactionType::Read, LockType::Optimistic).await.unwrap();
assert_eq!(
Sequence::seed_start_from_catalog(&tx, &SequenceDomain::NameSpacesIds, 0)
.await
.unwrap(),
8
);
assert_eq!(
Sequence::seed_start_from_catalog(&tx, &SequenceDomain::DatabasesIds(ns_id), 0)
.await
.unwrap(),
12
);
assert_eq!(
Sequence::seed_start_from_catalog(&tx, &SequenceDomain::TablesIds(ns_id, db_id), 0)
.await
.unwrap(),
14
);
assert_eq!(
Sequence::seed_start_from_catalog(
&tx,
&SequenceDomain::IndexIds(ns_id, db_id, tb_name.clone()),
0,
)
.await
.unwrap(),
18
);
assert_eq!(
Sequence::seed_start_from_catalog(&tx, &SequenceDomain::NameSpacesIds, 100)
.await
.unwrap(),
100
);
assert_eq!(
Sequence::seed_start_from_catalog(
&tx,
&SequenceDomain::DatabasesIds(NamespaceId(999)),
3,
)
.await
.unwrap(),
3
);
tx.cancel().await.unwrap();
}
#[tokio::test]
async fn seed_start_from_catalog_returns_start_on_empty_store() {
let ds = Datastore::new("memory").await.unwrap();
let tx = ds.transaction(TransactionType::Read, LockType::Optimistic).await.unwrap();
assert_eq!(
Sequence::seed_start_from_catalog(&tx, &SequenceDomain::NameSpacesIds, 0)
.await
.unwrap(),
0
);
assert_eq!(
Sequence::seed_start_from_catalog(
&tx,
&SequenceDomain::DatabasesIds(NamespaceId(0)),
0,
)
.await
.unwrap(),
0
);
tx.cancel().await.unwrap();
}
}