use std::sync::Arc;
use base64::Engine;
use crate::client::RedisClient;
use crate::error::{NasaRedisError, Result};
use crate::job::config::JobConfig;
use crate::job::definition::JobDefinition;
use crate::job::identifiers::shard_run_id;
use crate::job::keyspace::JobKeyspace;
use crate::job::model::{JobResultCode, JobState, JobWireCodec};
use crate::job::registry::{ClusterSnapshot, ExecutorMember};
use crate::job::repository::{
interpret_finish, interpret_recover, interpret_renew, interpret_start, FinishOutcome,
RecoverOutcome, RenewOutcome, StartOutcome,
};
use crate::job::script::{eval, value_to_string, JobScript};
const INBOX_GROUP: &str = "redis-job-fanout";
static FANOUT_BEGIN: JobScript = JobScript::new(include_str!("lua/fanout_begin.lua"));
static FANOUT_ADD_SHARDS: JobScript = JobScript::new(include_str!("lua/fanout_add_shards.lua"));
static FANOUT_COMMIT: JobScript = JobScript::new(include_str!("lua/fanout_commit.lua"));
static READ_FANOUT_ROOT: JobScript = JobScript::new(include_str!("lua/read_fanout_root.lua"));
static READ_FANOUT_SHARD: JobScript = JobScript::new(include_str!("lua/read_fanout_shard.lua"));
static FANOUT_DELIVER_BATCH: JobScript =
JobScript::new(include_str!("lua/fanout_deliver_batch.lua"));
static FANOUT_ACCEPT_SHARD: JobScript = JobScript::new(include_str!("lua/fanout_accept_shard.lua"));
static FANOUT_RETRY_RECEIPT: JobScript =
JobScript::new(include_str!("lua/fanout_retry_receipt.lua"));
static FANOUT_PROMOTE_READY: JobScript =
JobScript::new(include_str!("lua/fanout_promote_ready.lua"));
static FANOUT_DEFER_READY: JobScript = JobScript::new(include_str!("lua/fanout_defer_ready.lua"));
static START_RUN: JobScript = JobScript::new(include_str!("lua/start_run.lua"));
static RENEW_RUN: JobScript = JobScript::new(include_str!("lua/renew_run.lua"));
static FINISH_RUN: JobScript = JobScript::new(include_str!("lua/finish_run.lua"));
static FANOUT_WATCH_ROOT: JobScript = JobScript::new(include_str!("lua/fanout_watch_root.lua"));
static FANOUT_MARK_RECONCILED: JobScript =
JobScript::new(include_str!("lua/fanout_mark_reconciled.lua"));
static FANOUT_ADVANCE_CAPABILITY_CURSOR: JobScript =
JobScript::new(include_str!("lua/fanout_advance_capability_cursor.lua"));
static FANOUT_FAIL_CREATING: JobScript =
JobScript::new(include_str!("lua/fanout_fail_creating.lua"));
static FANOUT_AGGREGATE: JobScript = JobScript::new(include_str!("lua/fanout_aggregate.lua"));
static FANOUT_REASSIGN_SHARD: JobScript =
JobScript::new(include_str!("lua/fanout_reassign_shard.lua"));
static FANOUT_CLEANUP: JobScript = JobScript::new(include_str!("lua/fanout_cleanup.lua"));
static FANOUT_CANCEL_BATCH: JobScript = JobScript::new(include_str!("lua/fanout_cancel_batch.lua"));
static RECOVER_EXPIRED: JobScript = JobScript::new(include_str!("lua/recover_expired.lua"));
static FANOUT_SCAN_DUE: JobScript = JobScript::new(include_str!("lua/fanout_scan_due.lua"));
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum FanoutBeginOutcome {
Ok {
redis_now: i64,
},
Adopted,
Conflict,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum FanoutAddShardsOutcome {
Ok {
added: i64,
created_shard_count: i64,
},
StateMismatch,
Conflict,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum FanoutCommitOutcome {
Ok {
redis_now: i64,
},
Adopted,
StateMismatch,
Invalid,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum FanoutDeliverOutcome {
Ok {
delivered: i64,
delivery_cursor: i64,
},
NotCommitted,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum FanoutAcceptOutcome {
Ok {
redis_now: i64,
start_visible_at: i64,
},
NotCommitted,
Adopted,
StateMismatch,
StaleAssignment,
SourceMismatch,
ProtocolUnsupported,
ContractMismatch,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum FanoutReceiptRetryOutcome {
Ok {
count: i64,
next_deadline: i64,
},
Stale,
StaleAssignment,
NotDue {
deadline: i64,
},
RetryExhausted {
count: i64,
},
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum FanoutReadyPromoteOutcome {
Ok {
count: i64,
next_visible_at: i64,
},
Stale,
StaleAssignment,
NotDue {
visible_at: i64,
},
CapacityExhausted {
count: i64,
},
WakeupExhausted {
count: i64,
},
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum FanoutReadyDeferOutcome {
Deferred {
next_visible_at: i64,
},
CapacityExhausted {
first_deferred_at: i64,
elapsed_ms: i64,
},
Stale,
StaleAssignment,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum FanoutWatchRootOutcome {
Ok {
state: String,
root_run_id: String,
root_attempt: i64,
root_job_name: String,
root_schedule_shard: i64,
cancel_reason: String,
error_type: String,
},
NotFound,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum FanoutMarkReconciledOutcome {
Ok {
reconciled_at: i64,
},
StateMismatch,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum FanoutCapabilityCursorOutcome {
Ok {
next_cursor: i64,
},
NotFound,
Stale {
current_cursor: i64,
},
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum FanoutFailCreatingOutcome {
Ok {
state: String,
expire_at: i64,
},
StateMismatch,
NotDue {
deadline: i64,
},
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum FanoutAggregateOutcome {
Ok {
root_state_or_waiting: String,
expire_at_or_terminal_count: i64,
},
AlreadyCompleted,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum FanoutReassignOutcome {
Ok {
next_epoch: i64,
},
Busy,
StaleAssignment,
NoCapableExecutor,
CapacityRouteExhausted,
StaleCapacity,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum FanoutCleanupOutcome {
Ok {
deleted: i64,
next_cursor: i64,
},
Completed {
deleted: i64,
cursor: i64,
},
StateMismatch,
NotDue {
expire_at: i64,
},
NotReconciled,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum FanoutCancelBatchOutcome {
Cancelling {
next_cursor: i64,
},
Cancelled {
expire_at: i64,
},
AlreadyCompleted {
state: String,
},
Stale {
cursor: i64,
},
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct DueIndexScan {
pub next_score: Option<i64>,
pub members: Vec<(String, i64)>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct FanoutDueScans {
pub redis_now: i64,
pub root_count: i64,
pub receipts: DueIndexScan,
pub ready: DueIndexScan,
pub leases: DueIndexScan,
pub gc: DueIndexScan,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct FanoutContract {
pub worker_name: String,
pub contract_revision: i64,
pub schema_id: String,
pub wire_codec: JobWireCodec,
pub failure_policy: crate::job::model::JobFanoutFailurePolicy,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct FanoutRoot {
pub fanout_id: String,
pub state: String,
pub root_run_id: String,
pub root_attempt: i64,
pub root_job_name: String,
pub root_schedule_shard: i64,
pub worker_name: String,
pub contract_revision: i64,
pub schema_id: String,
pub wire_codec: String,
pub shard_total: i64,
pub delivery_cursor: i64,
pub receipt_timeout_ms: i64,
pub receipt_max_retries: i64,
pub failure_policy: String,
pub expire_at: i64,
pub cleanup_cursor: i64,
pub reconciled_at: i64,
pub cancel_reason: String,
pub capability_cursor: i64,
pub cancel_cursor: i64,
pub create_deadline_at: i64,
pub scheduler_qualifier: String,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct FanoutShard {
pub fanout_id: String,
pub root_run_id: String,
pub snapshot_id: String,
pub worker_name: String,
pub contract_revision: i64,
pub schema_id: String,
pub wire_codec: String,
pub shard_index: i64,
pub shard_total: i64,
pub seq: i64,
pub execution_key: String,
pub target_node_identity: String,
pub target_startup_id: String,
pub assignment_epoch: i64,
pub assignment_count: i64,
pub state: String,
pub attempt: i64,
pub attempt_token: String,
pub owner: String,
pub lease_until: i64,
pub parameter_payload: String,
pub inbox_message_id: String,
pub receipt_retry_count: i64,
pub ready_wakeup_count: i64,
pub origin_executor_id: String,
pub receipt_deadline_at: i64,
pub start_visible_at: i64,
pub target_heartbeat_revision: i64,
}
#[derive(Clone)]
pub struct FanoutRepository {
client: Arc<RedisClient>,
keyspace: JobKeyspace,
config: Arc<JobConfig>,
}
impl FanoutRepository {
pub fn new(client: Arc<RedisClient>, keyspace: JobKeyspace, config: Arc<JobConfig>) -> Self {
Self {
client,
keyspace,
config,
}
}
#[allow(clippy::too_many_arguments)]
pub async fn begin(
&self,
fanout_id: &str,
root_run_id: &str,
root_attempt: i64,
snapshot: &ClusterSnapshot,
root_definition: &JobDefinition,
contract: &FanoutContract,
shard_total: i64,
) -> Result<FanoutBeginOutcome> {
let root_shard = self.keyspace.schedule_shard(root_definition.name());
let keys: Vec<Vec<u8>> = vec![
self.keyspace.fanout_root(fanout_id).into_bytes(),
self.keyspace.fanout_roots(fanout_id).into_bytes(),
];
let snapshot_payload =
base64::engine::general_purpose::STANDARD.encode(snapshot.wire_json());
let argv: Vec<Vec<u8>> = vec![
fanout_id.as_bytes().to_vec(),
root_run_id.as_bytes().to_vec(),
root_attempt.to_string().into_bytes(),
snapshot.snapshot_id.clone().into_bytes(),
snapshot_payload.into_bytes(),
contract.worker_name.as_bytes().to_vec(),
contract.contract_revision.to_string().into_bytes(),
contract.schema_id.as_bytes().to_vec(),
contract.wire_codec.wire_name().as_bytes().to_vec(),
shard_total.to_string().into_bytes(),
root_definition
.fanout_receipt_timeout_ms()
.to_string()
.into_bytes(),
root_definition
.fanout_receipt_max_retries()
.to_string()
.into_bytes(),
contract.failure_policy.wire_name().as_bytes().to_vec(),
self.config
.fanout_create_timeout_ms
.to_string()
.into_bytes(),
root_definition.name().as_bytes().to_vec(),
root_shard.to_string().into_bytes(),
self.keyspace.qualifier().as_bytes().to_vec(),
];
let raw = eval(&self.client, &FANOUT_BEGIN, &keys, &argv).await?;
let code = raw.first().map(value_to_string).unwrap_or_default();
Ok(match code.as_str() {
"ADOPTED" => FanoutBeginOutcome::Adopted,
"CONFLICT" => FanoutBeginOutcome::Conflict,
"OK" => FanoutBeginOutcome::Ok {
redis_now: parse_num(raw.get(1), "fanout_begin redisNow")?,
},
_ => return Err(protocol("fanout_begin 返回未知码")),
})
}
#[allow(clippy::too_many_arguments)]
pub async fn add_shards(
&self,
fanout_id: &str,
snapshot: &ClusterSnapshot,
contract: &FanoutContract,
root_run_id: &str,
origin_executor_id: &str,
shard_payloads: &[Vec<u8>],
) -> Result<FanoutAddShardsOutcome> {
if shard_payloads.len() != snapshot.members.len() {
return Err(protocol("add_shards 载荷数必须与快照成员数一致"));
}
let shard_total = shard_payloads.len() as i64;
let batch_size = self.config.fanout_delivery_batch_size.max(1) as usize;
let mut added_total = 0_i64;
let mut created_shard_count = 0_i64;
for offset in (0..shard_payloads.len()).step_by(batch_size) {
let end = (offset + batch_size).min(shard_payloads.len());
let mut keys: Vec<Vec<u8>> = vec![self.keyspace.fanout_root(fanout_id).into_bytes()];
let mut argv: Vec<Vec<u8>> = vec![
snapshot.snapshot_id.clone().into_bytes(),
(end - offset).to_string().into_bytes(),
self.keyspace.qualifier().as_bytes().to_vec(),
self.config.protocol_version.to_string().into_bytes(),
];
#[allow(clippy::needless_range_loop)]
for index in offset..end {
let seq = index as i64;
let member = &snapshot.members[index];
keys.push(self.keyspace.fanout_shard(fanout_id, seq).into_bytes());
let execution_key = self.keyspace.execution_key(fanout_id, seq);
let payload =
base64::engine::general_purpose::STANDARD.encode(&shard_payloads[index]);
argv.extend([
seq.to_string().into_bytes(),
fanout_id.as_bytes().to_vec(),
root_run_id.as_bytes().to_vec(),
contract.worker_name.as_bytes().to_vec(),
contract.contract_revision.to_string().into_bytes(),
contract.schema_id.as_bytes().to_vec(),
contract.wire_codec.wire_name().as_bytes().to_vec(),
shard_total.to_string().into_bytes(),
seq.to_string().into_bytes(),
execution_key.into_bytes(),
member.node_identity.clone().into_bytes(),
member.startup_id.clone().into_bytes(),
member.heartbeat_revision.to_string().into_bytes(),
origin_executor_id.as_bytes().to_vec(),
payload.into_bytes(),
]);
}
let raw = eval(&self.client, &FANOUT_ADD_SHARDS, &keys, &argv).await?;
let code = raw.first().map(value_to_string).unwrap_or_default();
match code.as_str() {
"STATE_MISMATCH" => return Ok(FanoutAddShardsOutcome::StateMismatch),
"CONFLICT" => return Ok(FanoutAddShardsOutcome::Conflict),
"OK" => {
added_total += parse_num(raw.get(1), "add_shards added")?;
created_shard_count = parse_num(raw.get(2), "add_shards createdShardCount")?;
}
_ => return Err(protocol("fanout_add_shards 返回未知码")),
}
}
Ok(FanoutAddShardsOutcome::Ok {
added: added_total,
created_shard_count,
})
}
pub async fn commit(&self, fanout_id: &str) -> Result<FanoutCommitOutcome> {
let keys: Vec<Vec<u8>> = vec![
self.keyspace.fanout_root(fanout_id).into_bytes(),
self.keyspace.fanout_roots(fanout_id).into_bytes(),
self.keyspace.fanout_completion(fanout_id).into_bytes(),
];
let raw = eval(
&self.client,
&FANOUT_COMMIT,
&keys,
&[fanout_id.as_bytes().to_vec()],
)
.await?;
let code = raw.first().map(value_to_string).unwrap_or_default();
Ok(match code.as_str() {
"ADOPTED" => FanoutCommitOutcome::Adopted,
"STATE_MISMATCH" => FanoutCommitOutcome::StateMismatch,
"INVALID" => FanoutCommitOutcome::Invalid,
"OK" => FanoutCommitOutcome::Ok {
redis_now: parse_num(raw.get(1), "fanout_commit redisNow")?,
},
_ => return Err(protocol("fanout_commit 返回未知码")),
})
}
pub async fn read_root(&self, fanout_id: &str) -> Result<Option<FanoutRoot>> {
let key = self.keyspace.fanout_root(fanout_id).into_bytes();
let raw = eval(&self.client, &READ_FANOUT_ROOT, &[key], &[]).await?;
if raw.is_empty() {
return Ok(None);
}
if raw.len() != 23 {
return Err(protocol("read_fanout_root 返回字段数不符合合同"));
}
Ok(Some(FanoutRoot {
fanout_id: text(&raw, 0),
state: text(&raw, 1),
root_run_id: text(&raw, 2),
root_attempt: num(&raw, 3)?,
root_job_name: text(&raw, 4),
root_schedule_shard: num(&raw, 5)?,
worker_name: text(&raw, 6),
contract_revision: num(&raw, 7)?,
schema_id: text(&raw, 8),
wire_codec: text(&raw, 9),
shard_total: num(&raw, 10)?,
delivery_cursor: num(&raw, 11)?,
receipt_timeout_ms: num(&raw, 12)?,
receipt_max_retries: num(&raw, 13)?,
failure_policy: text(&raw, 14),
expire_at: num(&raw, 15)?,
cleanup_cursor: num(&raw, 16)?,
reconciled_at: num(&raw, 17)?,
cancel_reason: text(&raw, 18),
capability_cursor: num(&raw, 19)?,
cancel_cursor: num(&raw, 20)?,
create_deadline_at: num(&raw, 21)?,
scheduler_qualifier: text(&raw, 22),
}))
}
pub async fn read_shard(&self, fanout_id: &str, seq: i64) -> Result<Option<FanoutShard>> {
let key = self.keyspace.fanout_shard(fanout_id, seq).into_bytes();
let raw = eval(&self.client, &READ_FANOUT_SHARD, &[key], &[]).await?;
if raw.is_empty() {
return Ok(None);
}
if raw.len() != 28 {
return Err(protocol("read_fanout_shard 返回字段数不符合合同"));
}
Ok(Some(FanoutShard {
fanout_id: text(&raw, 0),
root_run_id: text(&raw, 1),
snapshot_id: text(&raw, 2),
worker_name: text(&raw, 3),
contract_revision: num(&raw, 4)?,
schema_id: text(&raw, 5),
wire_codec: text(&raw, 6),
shard_index: num(&raw, 7)?,
shard_total: num(&raw, 8)?,
seq: num(&raw, 9)?,
execution_key: text(&raw, 10),
target_node_identity: text(&raw, 11),
target_startup_id: text(&raw, 12),
assignment_epoch: num(&raw, 13)?,
assignment_count: num(&raw, 14)?,
state: text(&raw, 15),
attempt: num(&raw, 16)?,
attempt_token: text(&raw, 17),
owner: text(&raw, 18),
lease_until: num(&raw, 19)?,
parameter_payload: text(&raw, 20),
inbox_message_id: text(&raw, 21),
receipt_retry_count: num(&raw, 22)?,
ready_wakeup_count: num(&raw, 23)?,
origin_executor_id: text(&raw, 24),
receipt_deadline_at: num(&raw, 25)?,
start_visible_at: num(&raw, 26)?,
target_heartbeat_revision: num(&raw, 27)?,
}))
}
pub async fn deliver_batch(
&self,
fanout_id: &str,
snapshot: &ClusterSnapshot,
_definition: &JobDefinition,
) -> Result<FanoutDeliverOutcome> {
let batch_size = self.config.fanout_delivery_batch_size.max(1) as usize;
let mut delivered_total = 0_i64;
let mut delivery_cursor = 0_i64;
for offset in (0..snapshot.members.len()).step_by(batch_size) {
let end = (offset + batch_size).min(snapshot.members.len());
let mut shards = Vec::with_capacity(end - offset);
for index in offset..end {
let seq = index as i64;
let shard = self
.read_shard(fanout_id, seq)
.await?
.ok_or_else(|| protocol("首次投递缺少已提交的 shard"))?;
shards.push(shard);
}
match self.deliver_assignments(fanout_id, &shards, true).await? {
FanoutDeliverOutcome::NotCommitted => {
return Ok(FanoutDeliverOutcome::NotCommitted);
}
FanoutDeliverOutcome::Ok {
delivered,
delivery_cursor: cursor,
} => {
delivered_total += delivered;
delivery_cursor = cursor;
}
}
}
Ok(FanoutDeliverOutcome::Ok {
delivered: delivered_total,
delivery_cursor,
})
}
pub async fn deliver_assignments(
&self,
fanout_id: &str,
shards: &[FanoutShard],
advance_cursor: bool,
) -> Result<FanoutDeliverOutcome> {
let root = match self.read_root(fanout_id).await? {
Some(root) if root.state == "COMMITTED" || root.state == "WAITING_CHILDREN" => root,
_ => return Ok(FanoutDeliverOutcome::NotCommitted),
};
if shards.is_empty() {
return Ok(FanoutDeliverOutcome::Ok {
delivered: 0,
delivery_cursor: root.delivery_cursor,
});
}
let publish = self.config.pubsub_mode.publish_command();
let batch_size = self.config.fanout_delivery_batch_size.max(1) as usize;
let mut delivered_total = 0_i64;
let mut delivery_cursor = 0_i64;
for batch in shards.chunks(batch_size) {
let mut keys: Vec<Vec<u8>> = vec![
self.keyspace.fanout_root(fanout_id).into_bytes(),
self.keyspace.fanout_receipts(fanout_id).into_bytes(),
];
let mut argv: Vec<Vec<u8>> = vec![
batch.len().to_string().into_bytes(),
fanout_id.as_bytes().to_vec(),
publish.as_bytes().to_vec(),
if advance_cursor {
b"1".to_vec()
} else {
b"0".to_vec()
},
];
for shard in batch {
if shard.fanout_id != fanout_id {
return Err(protocol("投递 shard 与 fanout_id 不一致"));
}
keys.push(
self.keyspace
.fanout_shard(fanout_id, shard.seq)
.into_bytes(),
);
keys.push(
self.keyspace
.fanout_inbox(fanout_id, &shard.target_node_identity)
.into_bytes(),
);
keys.push(
self.keyspace
.fanout_notify_channel(fanout_id, &shard.target_node_identity)
.into_bytes(),
);
argv.extend([
shard.seq.to_string().into_bytes(),
shard.assignment_epoch.to_string().into_bytes(),
shard_run_id(fanout_id, shard.seq).into_bytes(),
root.receipt_timeout_ms.to_string().into_bytes(),
]);
}
let raw = eval(&self.client, &FANOUT_DELIVER_BATCH, &keys, &argv).await?;
let code = raw.first().map(value_to_string).unwrap_or_default();
match code.as_str() {
"NOT_COMMITTED" => return Ok(FanoutDeliverOutcome::NotCommitted),
"OK" => {
delivered_total += parse_num(raw.get(1), "deliver delivered")?;
delivery_cursor = parse_num(raw.get(2), "deliver deliveryCursor")?;
}
_ => return Err(protocol("fanout_deliver_batch 返回未知码")),
}
}
Ok(FanoutDeliverOutcome::Ok {
delivered: delivered_total,
delivery_cursor,
})
}
#[allow(clippy::too_many_arguments)]
pub async fn accept_shard(
&self,
definition: &JobDefinition,
fanout_id: &str,
seq: i64,
target_node_identity: &str,
executor_id: &str,
assignment_epoch: i64,
origin_executor_id: &str,
) -> Result<FanoutAcceptOutcome> {
let keys: Vec<Vec<u8>> = vec![
self.keyspace.fanout_root(fanout_id).into_bytes(),
self.keyspace.fanout_shard(fanout_id, seq).into_bytes(),
self.keyspace.fanout_receipts(fanout_id).into_bytes(),
self.keyspace.fanout_ready(fanout_id).into_bytes(),
self.keyspace
.fanout_receipt_channel(fanout_id, origin_executor_id)
.into_bytes(),
];
let argv: Vec<Vec<u8>> = vec![
fanout_id.as_bytes().to_vec(),
seq.to_string().into_bytes(),
target_node_identity.as_bytes().to_vec(),
executor_id.as_bytes().to_vec(),
assignment_epoch.to_string().into_bytes(),
self.config.min_scan_interval_ms.to_string().into_bytes(),
self.config
.pubsub_mode
.publish_command()
.as_bytes()
.to_vec(),
self.keyspace.qualifier().as_bytes().to_vec(),
self.config.protocol_version.to_string().into_bytes(),
definition.contract_revision().to_string().into_bytes(),
definition.schema_id().as_bytes().to_vec(),
definition.wire_codecs().as_bytes().to_vec(),
];
let raw = eval(&self.client, &FANOUT_ACCEPT_SHARD, &keys, &argv).await?;
let code = raw.first().map(value_to_string).unwrap_or_default();
Ok(match code.as_str() {
"NOT_COMMITTED" => FanoutAcceptOutcome::NotCommitted,
"ADOPTED" => FanoutAcceptOutcome::Adopted,
"STATE_MISMATCH" => FanoutAcceptOutcome::StateMismatch,
"STALE_ASSIGNMENT" => FanoutAcceptOutcome::StaleAssignment,
"SOURCE_MISMATCH" => FanoutAcceptOutcome::SourceMismatch,
"PROTOCOL_UNSUPPORTED" => FanoutAcceptOutcome::ProtocolUnsupported,
"CONTRACT_MISMATCH" => FanoutAcceptOutcome::ContractMismatch,
"OK" => FanoutAcceptOutcome::Ok {
redis_now: parse_num(raw.get(1), "accept redisNow")?,
start_visible_at: parse_num(raw.get(2), "accept startVisibleAt")?,
},
_ => return Err(protocol("fanout_accept_shard 返回未知码")),
})
}
pub async fn retry_receipt(
&self,
root: &FanoutRoot,
shard: &FanoutShard,
) -> Result<FanoutReceiptRetryOutcome> {
let keys: Vec<Vec<u8>> = vec![
self.keyspace
.fanout_shard(&root.fanout_id, shard.seq)
.into_bytes(),
self.keyspace.fanout_receipts(&root.fanout_id).into_bytes(),
self.keyspace
.fanout_notify_channel(&root.fanout_id, &shard.target_node_identity)
.into_bytes(),
];
let argv: Vec<Vec<u8>> = vec![
root.fanout_id.as_bytes().to_vec(),
shard.seq.to_string().into_bytes(),
shard.assignment_epoch.to_string().into_bytes(),
root.receipt_max_retries.to_string().into_bytes(),
root.receipt_timeout_ms.to_string().into_bytes(),
self.config
.pubsub_mode
.publish_command()
.as_bytes()
.to_vec(),
root.failure_policy.as_bytes().to_vec(),
self.config.max_scan_interval_ms.to_string().into_bytes(),
];
let raw = eval(&self.client, &FANOUT_RETRY_RECEIPT, &keys, &argv).await?;
let code = raw.first().map(value_to_string).unwrap_or_default();
Ok(match code.as_str() {
"STALE" => FanoutReceiptRetryOutcome::Stale,
"STALE_ASSIGNMENT" => FanoutReceiptRetryOutcome::StaleAssignment,
"NOT_DUE" => FanoutReceiptRetryOutcome::NotDue {
deadline: parse_num(raw.get(1), "retry_receipt deadline")?,
},
"RETRY_EXHAUSTED" => FanoutReceiptRetryOutcome::RetryExhausted {
count: parse_num(raw.get(1), "retry_receipt count")?,
},
"OK" => FanoutReceiptRetryOutcome::Ok {
count: parse_num(raw.get(1), "retry_receipt count")?,
next_deadline: parse_num(raw.get(2), "retry_receipt nextDeadline")?,
},
_ => return Err(protocol("fanout_retry_receipt 返回未知码")),
})
}
pub async fn promote_ready(
&self,
root: &FanoutRoot,
shard: &FanoutShard,
) -> Result<FanoutReadyPromoteOutcome> {
let keys: Vec<Vec<u8>> = vec![
self.keyspace
.fanout_shard(&root.fanout_id, shard.seq)
.into_bytes(),
self.keyspace.fanout_ready(&root.fanout_id).into_bytes(),
self.keyspace
.fanout_notify_channel(&root.fanout_id, &shard.target_node_identity)
.into_bytes(),
];
let argv: Vec<Vec<u8>> = vec![
root.fanout_id.as_bytes().to_vec(),
shard.seq.to_string().into_bytes(),
shard.assignment_epoch.to_string().into_bytes(),
self.config.ready_max_wakeups.to_string().into_bytes(),
self.config.min_scan_interval_ms.to_string().into_bytes(),
self.config
.pubsub_mode
.publish_command()
.as_bytes()
.to_vec(),
root.failure_policy.as_bytes().to_vec(),
self.config.max_scan_interval_ms.to_string().into_bytes(),
];
let raw = eval(&self.client, &FANOUT_PROMOTE_READY, &keys, &argv).await?;
let code = raw.first().map(value_to_string).unwrap_or_default();
Ok(match code.as_str() {
"STALE" => FanoutReadyPromoteOutcome::Stale,
"STALE_ASSIGNMENT" => FanoutReadyPromoteOutcome::StaleAssignment,
"NOT_DUE" => FanoutReadyPromoteOutcome::NotDue {
visible_at: parse_num(raw.get(1), "promote_ready visibleAt")?,
},
"WAKEUP_EXHAUSTED" => FanoutReadyPromoteOutcome::WakeupExhausted {
count: parse_num(raw.get(1), "promote_ready count")?,
},
"CAPACITY_EXHAUSTED" => FanoutReadyPromoteOutcome::CapacityExhausted {
count: parse_num(raw.get(1), "promote_ready count")?,
},
"OK" => FanoutReadyPromoteOutcome::Ok {
count: parse_num(raw.get(1), "promote_ready count")?,
next_visible_at: parse_num(raw.get(2), "promote_ready nextVisibleAt")?,
},
_ => return Err(protocol("fanout_promote_ready 返回未知码")),
})
}
pub async fn defer_ready_for_capacity(
&self,
shard: &FanoutShard,
) -> Result<FanoutReadyDeferOutcome> {
self.defer_ready_for_capacity_mode(shard, false).await
}
pub(crate) async fn defer_ready_for_capacity_mode(
&self,
shard: &FanoutShard,
reopen: bool,
) -> Result<FanoutReadyDeferOutcome> {
let keys: Vec<Vec<u8>> = vec![
self.keyspace
.fanout_shard(&shard.fanout_id, shard.seq)
.into_bytes(),
self.keyspace.fanout_ready(&shard.fanout_id).into_bytes(),
self.keyspace.fanout_root(&shard.fanout_id).into_bytes(),
];
let argv: Vec<Vec<u8>> = vec![
shard.fanout_id.as_bytes().to_vec(),
shard.seq.to_string().into_bytes(),
shard.target_node_identity.as_bytes().to_vec(),
shard.assignment_epoch.to_string().into_bytes(),
self.config.min_scan_interval_ms.to_string().into_bytes(),
self.config.fanout_capacity_wait_ms.to_string().into_bytes(),
self.config.ready_max_wakeups.to_string().into_bytes(),
if reopen {
b"REOPEN".to_vec()
} else {
b"CONTINUE".to_vec()
},
];
let raw = eval(&self.client, &FANOUT_DEFER_READY, &keys, &argv).await?;
let code = raw.first().map(value_to_string).unwrap_or_default();
Ok(match code.as_str() {
"STALE" => FanoutReadyDeferOutcome::Stale,
"STALE_ASSIGNMENT" => FanoutReadyDeferOutcome::StaleAssignment,
"DEFERRED" => FanoutReadyDeferOutcome::Deferred {
next_visible_at: parse_num(raw.get(1), "defer_ready nextVisibleAt")?,
},
"CAPACITY_EXHAUSTED" => FanoutReadyDeferOutcome::CapacityExhausted {
first_deferred_at: parse_num(raw.get(1), "defer_ready firstDeferredAt")?,
elapsed_ms: parse_num(raw.get(2), "defer_ready elapsedMs")?,
},
_ => return Err(protocol("fanout_defer_ready 返回未知码")),
})
}
#[allow(clippy::too_many_arguments)]
pub async fn start_shard(
&self,
definition: &JobDefinition,
fanout_id: &str,
seq: i64,
node_identity: &str,
executor_id: &str,
assignment_epoch: i64,
message_id: &str,
) -> Result<StartOutcome> {
let keys: Vec<Vec<u8>> = vec![
self.keyspace.fanout_shard(fanout_id, seq).into_bytes(),
self.keyspace.fanout_leases(fanout_id).into_bytes(),
self.keyspace.fanout_ready(fanout_id).into_bytes(),
self.keyspace.fanout_root(fanout_id).into_bytes(),
self.keyspace.fanout_receipts(fanout_id).into_bytes(),
self.keyspace.fanout_root(fanout_id).into_bytes(),
self.keyspace
.fanout_inbox(fanout_id, node_identity)
.into_bytes(),
self.keyspace.fanout_completion(fanout_id).into_bytes(),
self.keyspace.fanout_root(fanout_id).into_bytes(),
];
let argv: Vec<Vec<u8>> = vec![
definition.worker_name().as_bytes().to_vec(),
shard_run_id(fanout_id, seq).into_bytes(),
executor_id.as_bytes().to_vec(),
message_id.as_bytes().to_vec(),
self.config.lease_ms.to_string().into_bytes(),
b"PARALLEL".to_vec(),
b"0".to_vec(),
self.config.min_scan_interval_ms.to_string().into_bytes(),
INBOX_GROUP.as_bytes().to_vec(),
self.config.fanout_retention_ms.to_string().into_bytes(),
b"FANOUT".to_vec(),
node_identity.as_bytes().to_vec(),
assignment_epoch.to_string().into_bytes(),
fanout_id.as_bytes().to_vec(),
seq.to_string().into_bytes(),
self.keyspace.execution_key(fanout_id, seq).into_bytes(),
Vec::new(),
Vec::new(),
self.keyspace.qualifier().as_bytes().to_vec(),
self.config.protocol_version.to_string().into_bytes(),
definition.contract_revision().to_string().into_bytes(),
definition.schema_id().as_bytes().to_vec(),
definition.wire_codecs().as_bytes().to_vec(),
];
let raw = eval(&self.client, &START_RUN, &keys, &argv).await?;
interpret_start(&raw)
}
pub async fn renew_shard(
&self,
fanout_id: &str,
seq: i64,
executor_id: &str,
attempt_token: i64,
) -> Result<RenewOutcome> {
let keys: Vec<Vec<u8>> = vec![
self.keyspace.fanout_shard(fanout_id, seq).into_bytes(),
self.keyspace.fanout_leases(fanout_id).into_bytes(),
self.keyspace.fanout_roots(fanout_id).into_bytes(),
];
let argv: Vec<Vec<u8>> = vec![
format!("{fanout_id}:{seq}").into_bytes(),
executor_id.as_bytes().to_vec(),
attempt_token.to_string().into_bytes(),
self.config.lease_ms.to_string().into_bytes(),
];
let raw = eval(&self.client, &RENEW_RUN, &keys, &argv).await?;
interpret_renew(&raw)
}
#[allow(clippy::too_many_arguments)]
pub async fn finish_shard(
&self,
fanout_id: &str,
seq: i64,
worker_name: &str,
executor_id: &str,
attempt_token: i64,
assignment_epoch: i64,
definition: &JobDefinition,
result_code: JobResultCode,
summary: &str,
) -> Result<FinishOutcome> {
let keys: Vec<Vec<u8>> = vec![
self.keyspace.fanout_shard(fanout_id, seq).into_bytes(),
self.keyspace.fanout_leases(fanout_id).into_bytes(),
self.keyspace.fanout_ready(fanout_id).into_bytes(),
self.keyspace.fanout_receipts(fanout_id).into_bytes(),
self.keyspace.fanout_root(fanout_id).into_bytes(),
self.keyspace.fanout_roots(fanout_id).into_bytes(),
self.keyspace.fanout_completion(fanout_id).into_bytes(),
self.keyspace.fanout_gc(fanout_id).into_bytes(),
];
let argv: Vec<Vec<u8>> = vec![
shard_run_id(fanout_id, seq).into_bytes(),
executor_id.as_bytes().to_vec(),
attempt_token.to_string().into_bytes(),
worker_name.as_bytes().to_vec(),
result_code.wire_name().as_bytes().to_vec(),
summary.as_bytes().to_vec(),
definition.max_attempts().to_string().into_bytes(),
definition.retry_delay_ms().to_string().into_bytes(),
self.config.fanout_retention_ms.to_string().into_bytes(),
b"FANOUT".to_vec(),
fanout_id.as_bytes().to_vec(),
seq.to_string().into_bytes(),
assignment_epoch.to_string().into_bytes(),
self.keyspace.execution_key(fanout_id, seq).into_bytes(),
];
let raw = eval(&self.client, &FINISH_RUN, &keys, &argv).await?;
interpret_finish(&raw)
}
pub async fn watch_root(&self, fanout_id: &str) -> Result<FanoutWatchRootOutcome> {
let keys: Vec<Vec<u8>> = vec![
self.keyspace.fanout_root(fanout_id).into_bytes(),
self.keyspace.fanout_roots(fanout_id).into_bytes(),
];
let argv: Vec<Vec<u8>> = vec![
fanout_id.as_bytes().to_vec(),
self.config.max_scan_interval_ms.to_string().into_bytes(),
];
let raw = eval(&self.client, &FANOUT_WATCH_ROOT, &keys, &argv).await?;
let code = raw.first().map(value_to_string).unwrap_or_default();
Ok(match code.as_str() {
"NOT_FOUND" => FanoutWatchRootOutcome::NotFound,
"OK" => FanoutWatchRootOutcome::Ok {
state: text(&raw, 1),
root_run_id: text(&raw, 2),
root_attempt: num(&raw, 3)?,
root_job_name: text(&raw, 4),
root_schedule_shard: num(&raw, 5)?,
cancel_reason: text(&raw, 6),
error_type: text(&raw, 7),
},
_ => return Err(protocol("fanout_watch_root 返回未知码")),
})
}
pub async fn mark_reconciled(&self, fanout_id: &str) -> Result<FanoutMarkReconciledOutcome> {
let keys: Vec<Vec<u8>> = vec![
self.keyspace.fanout_root(fanout_id).into_bytes(),
self.keyspace.fanout_roots(fanout_id).into_bytes(),
];
let raw = eval(
&self.client,
&FANOUT_MARK_RECONCILED,
&keys,
&[fanout_id.as_bytes().to_vec()],
)
.await?;
let code = raw.first().map(value_to_string).unwrap_or_default();
Ok(match code.as_str() {
"STATE_MISMATCH" => FanoutMarkReconciledOutcome::StateMismatch,
"OK" => FanoutMarkReconciledOutcome::Ok {
reconciled_at: parse_num(raw.get(1), "mark_reconciled reconciledAt")?,
},
_ => return Err(protocol("fanout_mark_reconciled 返回未知码")),
})
}
pub async fn advance_capability_cursor(
&self,
fanout_id: &str,
current_cursor: i64,
next_cursor: i64,
) -> Result<FanoutCapabilityCursorOutcome> {
let argv: Vec<Vec<u8>> = vec![
current_cursor.to_string().into_bytes(),
next_cursor.to_string().into_bytes(),
];
let raw = eval(
&self.client,
&FANOUT_ADVANCE_CAPABILITY_CURSOR,
&[self.keyspace.fanout_root(fanout_id).into_bytes()],
&argv,
)
.await?;
let code = raw.first().map(value_to_string).unwrap_or_default();
Ok(match code.as_str() {
"NOT_FOUND" => FanoutCapabilityCursorOutcome::NotFound,
"STALE" => FanoutCapabilityCursorOutcome::Stale {
current_cursor: parse_num(raw.get(1), "advance_capability currentCursor")?,
},
"OK" => FanoutCapabilityCursorOutcome::Ok {
next_cursor: parse_num(raw.get(1), "advance_capability nextCursor")?,
},
_ => return Err(protocol("fanout_advance_capability_cursor 返回未知码")),
})
}
pub async fn fail_creating(&self, fanout_id: &str) -> Result<FanoutFailCreatingOutcome> {
let keys: Vec<Vec<u8>> = vec![
self.keyspace.fanout_root(fanout_id).into_bytes(),
self.keyspace.fanout_roots(fanout_id).into_bytes(),
self.keyspace.fanout_completion(fanout_id).into_bytes(),
self.keyspace.fanout_gc(fanout_id).into_bytes(),
];
let argv: Vec<Vec<u8>> = vec![
fanout_id.as_bytes().to_vec(),
self.config.fanout_retention_ms.to_string().into_bytes(),
];
let raw = eval(&self.client, &FANOUT_FAIL_CREATING, &keys, &argv).await?;
let code = raw.first().map(value_to_string).unwrap_or_default();
Ok(match code.as_str() {
"STATE_MISMATCH" => FanoutFailCreatingOutcome::StateMismatch,
"NOT_DUE" => FanoutFailCreatingOutcome::NotDue {
deadline: parse_num(raw.get(1), "fail_creating deadline")?,
},
"OK" => FanoutFailCreatingOutcome::Ok {
state: text(&raw, 1),
expire_at: parse_num(raw.get(2), "fail_creating expireAt")?,
},
_ => return Err(protocol("fanout_fail_creating 返回未知码")),
})
}
pub async fn aggregate(
&self,
fanout_id: &str,
seq: i64,
shard_terminal_state: JobState,
result_code: &str,
result_summary: &str,
) -> Result<FanoutAggregateOutcome> {
let keys: Vec<Vec<u8>> = vec![
self.keyspace.fanout_root(fanout_id).into_bytes(),
self.keyspace.fanout_shard(fanout_id, seq).into_bytes(),
self.keyspace.fanout_receipts(fanout_id).into_bytes(),
self.keyspace.fanout_ready(fanout_id).into_bytes(),
self.keyspace.fanout_leases(fanout_id).into_bytes(),
self.keyspace.fanout_completion(fanout_id).into_bytes(),
self.keyspace.fanout_gc(fanout_id).into_bytes(),
];
let argv: Vec<Vec<u8>> = vec![
fanout_id.as_bytes().to_vec(),
seq.to_string().into_bytes(),
shard_terminal_state.wire_name().as_bytes().to_vec(),
result_code.as_bytes().to_vec(),
result_summary.as_bytes().to_vec(),
self.config.fanout_retention_ms.to_string().into_bytes(),
];
let raw = eval(&self.client, &FANOUT_AGGREGATE, &keys, &argv).await?;
let code = raw.first().map(value_to_string).unwrap_or_default();
Ok(match code.as_str() {
"ALREADY_COMPLETED" => FanoutAggregateOutcome::AlreadyCompleted,
"OK" => FanoutAggregateOutcome::Ok {
root_state_or_waiting: text(&raw, 1),
expire_at_or_terminal_count: parse_num(raw.get(2), "aggregate expireAtOrCount")?,
},
_ => return Err(protocol("fanout_aggregate 返回未知码")),
})
}
pub async fn reassign_shard(
&self,
root: &FanoutRoot,
shard: &FanoutShard,
new_target: Option<&ExecutorMember>,
) -> Result<FanoutReassignOutcome> {
self.reassign_shard_mode(root, shard, new_target, "FAILURE")
.await
}
pub(crate) async fn reassign_shard_mode(
&self,
root: &FanoutRoot,
shard: &FanoutShard,
new_target: Option<&ExecutorMember>,
mode: &str,
) -> Result<FanoutReassignOutcome> {
let keys: Vec<Vec<u8>> = vec![
self.keyspace
.fanout_shard(&root.fanout_id, shard.seq)
.into_bytes(),
self.keyspace.fanout_receipts(&root.fanout_id).into_bytes(),
self.keyspace.fanout_ready(&root.fanout_id).into_bytes(),
self.keyspace.fanout_leases(&root.fanout_id).into_bytes(),
self.keyspace
.fanout_inbox(&root.fanout_id, &shard.target_node_identity)
.into_bytes(),
];
let (node_identity, startup_id, heartbeat_revision) = match new_target {
Some(target) => (
target.node_identity.as_str(),
target.startup_id.as_str(),
target.heartbeat_revision,
),
None => ("", "", 0),
};
let argv: Vec<Vec<u8>> = vec![
root.fanout_id.as_bytes().to_vec(),
shard.seq.to_string().into_bytes(),
shard.target_node_identity.as_bytes().to_vec(),
shard.assignment_epoch.to_string().into_bytes(),
node_identity.as_bytes().to_vec(),
startup_id.as_bytes().to_vec(),
heartbeat_revision.to_string().into_bytes(),
INBOX_GROUP.as_bytes().to_vec(),
self.config.fanout_max_assignments.to_string().into_bytes(),
mode.as_bytes().to_vec(),
self.config.fanout_capacity_wait_ms.to_string().into_bytes(),
];
let raw = eval(&self.client, &FANOUT_REASSIGN_SHARD, &keys, &argv).await?;
let code = raw.first().map(value_to_string).unwrap_or_default();
Ok(match code.as_str() {
"BUSY" => FanoutReassignOutcome::Busy,
"STALE_ASSIGNMENT" => FanoutReassignOutcome::StaleAssignment,
"NO_CAPABLE_EXECUTOR" => FanoutReassignOutcome::NoCapableExecutor,
"CAPACITY_ROUTE_EXHAUSTED" => FanoutReassignOutcome::CapacityRouteExhausted,
"STALE_CAPACITY" => FanoutReassignOutcome::StaleCapacity,
"OK" => FanoutReassignOutcome::Ok {
next_epoch: parse_num(raw.get(1), "reassign nextEpoch")?,
},
_ => return Err(protocol("fanout_reassign_shard 返回未知码")),
})
}
pub async fn cleanup(&self, root: &FanoutRoot) -> Result<FanoutCleanupOutcome> {
if root.cleanup_cursor < 0 || root.shard_total < root.cleanup_cursor {
return Err(protocol("Fanout 清理游标超出分片范围"));
}
let remaining = root.shard_total - root.cleanup_cursor;
let count = remaining.min(self.config.fanout_cleanup_batch_size as i64);
let mut keys: Vec<Vec<u8>> = vec![
self.keyspace.fanout_root(&root.fanout_id).into_bytes(),
self.keyspace.fanout_gc(&root.fanout_id).into_bytes(),
self.keyspace.fanout_roots(&root.fanout_id).into_bytes(),
self.keyspace.fanout_receipts(&root.fanout_id).into_bytes(),
self.keyspace.fanout_ready(&root.fanout_id).into_bytes(),
self.keyspace.fanout_leases(&root.fanout_id).into_bytes(),
];
for index in 0..count {
let seq = root.cleanup_cursor + index;
let target = match self.read_shard(&root.fanout_id, seq).await? {
Some(s) => s.target_node_identity,
None => "cleanup".to_owned(),
};
keys.push(
self.keyspace
.fanout_shard(&root.fanout_id, seq)
.into_bytes(),
);
keys.push(
self.keyspace
.fanout_inbox(&root.fanout_id, &target)
.into_bytes(),
);
}
let argv: Vec<Vec<u8>> = vec![
root.fanout_id.as_bytes().to_vec(),
count.to_string().into_bytes(),
INBOX_GROUP.as_bytes().to_vec(),
root.cleanup_cursor.to_string().into_bytes(),
];
let raw = eval(&self.client, &FANOUT_CLEANUP, &keys, &argv).await?;
let code = raw.first().map(value_to_string).unwrap_or_default();
Ok(match code.as_str() {
"STATE_MISMATCH" => FanoutCleanupOutcome::StateMismatch,
"NOT_RECONCILED" => FanoutCleanupOutcome::NotReconciled,
"NOT_DUE" => FanoutCleanupOutcome::NotDue {
expire_at: parse_num(raw.get(1), "cleanup expireAt")?,
},
"COMPLETED" => FanoutCleanupOutcome::Completed {
deleted: parse_num(raw.get(1), "cleanup deleted")?,
cursor: parse_num(raw.get(2), "cleanup cursor")?,
},
"OK" => FanoutCleanupOutcome::Ok {
deleted: parse_num(raw.get(1), "cleanup deleted")?,
next_cursor: parse_num(raw.get(2), "cleanup nextCursor")?,
},
_ => return Err(protocol("fanout_cleanup 返回未知码")),
})
}
pub async fn cancel_batch(
&self,
root: &FanoutRoot,
reason: &str,
) -> Result<FanoutCancelBatchOutcome> {
if root.cancel_cursor < 0 || root.cancel_cursor > root.shard_total {
return Err(protocol("Fanout 取消游标超出分片范围"));
}
let batch_size = i64::from(self.config.fanout_cleanup_batch_size);
let remaining = root.shard_total - root.cancel_cursor;
let count = remaining.min(batch_size);
let mut keys: Vec<Vec<u8>> = vec![
self.keyspace.fanout_root(&root.fanout_id).into_bytes(),
self.keyspace.fanout_roots(&root.fanout_id).into_bytes(),
self.keyspace
.fanout_completion(&root.fanout_id)
.into_bytes(),
self.keyspace.fanout_gc(&root.fanout_id).into_bytes(),
self.keyspace.fanout_receipts(&root.fanout_id).into_bytes(),
self.keyspace.fanout_ready(&root.fanout_id).into_bytes(),
self.keyspace.fanout_leases(&root.fanout_id).into_bytes(),
];
for index in 0..count {
let seq = root.cancel_cursor + index;
let target = match self.read_shard(&root.fanout_id, seq).await? {
Some(s) => s.target_node_identity,
None => "cancel".to_owned(),
};
keys.push(
self.keyspace
.fanout_shard(&root.fanout_id, seq)
.into_bytes(),
);
keys.push(
self.keyspace
.fanout_inbox(&root.fanout_id, &target)
.into_bytes(),
);
}
let argv: Vec<Vec<u8>> = vec![
root.fanout_id.as_bytes().to_vec(),
reason.as_bytes().to_vec(),
self.config.fanout_retention_ms.to_string().into_bytes(),
root.cancel_cursor.to_string().into_bytes(),
count.to_string().into_bytes(),
INBOX_GROUP.as_bytes().to_vec(),
self.config.min_scan_interval_ms.to_string().into_bytes(),
];
let raw = eval(&self.client, &FANOUT_CANCEL_BATCH, &keys, &argv).await?;
let code = raw.first().map(value_to_string).unwrap_or_default();
let tag = raw.get(1).map(value_to_string).unwrap_or_default();
Ok(match code.as_str() {
"ALREADY_COMPLETED" => FanoutCancelBatchOutcome::AlreadyCompleted {
state: tag,
},
"STALE" => FanoutCancelBatchOutcome::Stale {
cursor: parse_num(raw.get(1), "cancel_batch cursor")?,
},
"OK" if tag == "CANCELLING" => FanoutCancelBatchOutcome::Cancelling {
next_cursor: parse_num(raw.get(2), "cancel_batch nextCursor")?,
},
"OK" if tag == "CANCELLED" => FanoutCancelBatchOutcome::Cancelled {
expire_at: parse_num(raw.get(2), "cancel_batch expireAt")?,
},
_ => return Err(protocol("fanout_cancel_batch 返回未知码")),
})
}
pub async fn recover_shard(
&self,
root: &FanoutRoot,
shard: &FanoutShard,
definition: &JobDefinition,
) -> Result<RecoverOutcome> {
self.recover_shard_with_mode(
root,
shard,
definition.max_attempts(),
definition.retry_delay_ms(),
"FANOUT",
)
.await
}
pub(crate) async fn recover_cancelling_shard(
&self,
root: &FanoutRoot,
shard: &FanoutShard,
) -> Result<RecoverOutcome> {
self.recover_shard_with_mode(root, shard, 0, 0, "FANOUT_CANCEL_ONLY")
.await
}
async fn recover_shard_with_mode(
&self,
root: &FanoutRoot,
shard: &FanoutShard,
max_attempts: u32,
retry_delay_ms: u64,
mode: &str,
) -> Result<RecoverOutcome> {
let keys: Vec<Vec<u8>> = vec![
self.keyspace
.fanout_shard(&root.fanout_id, shard.seq)
.into_bytes(),
self.keyspace.fanout_leases(&root.fanout_id).into_bytes(),
self.keyspace.fanout_ready(&root.fanout_id).into_bytes(),
self.keyspace.fanout_root(&root.fanout_id).into_bytes(),
self.keyspace.fanout_receipts(&root.fanout_id).into_bytes(),
self.keyspace.fanout_roots(&root.fanout_id).into_bytes(),
self.keyspace
.fanout_completion(&root.fanout_id)
.into_bytes(),
self.keyspace.fanout_gc(&root.fanout_id).into_bytes(),
];
let member = format!("{}:{}", root.fanout_id, shard.seq);
let argv: Vec<Vec<u8>> = vec![
member.into_bytes(),
shard.worker_name.as_bytes().to_vec(),
max_attempts.to_string().into_bytes(),
retry_delay_ms.to_string().into_bytes(),
self.config.fanout_retention_ms.to_string().into_bytes(),
mode.as_bytes().to_vec(),
root.fanout_id.as_bytes().to_vec(),
shard.seq.to_string().into_bytes(),
root.failure_policy.as_bytes().to_vec(),
Vec::new(),
];
let raw = eval(&self.client, &RECOVER_EXPIRED, &keys, &argv).await?;
interpret_recover(&raw)
}
pub async fn scan_due(&self, bucket: u32) -> Result<FanoutDueScans> {
let keys: Vec<Vec<u8>> = vec![
self.keyspace.fanout_roots_at(bucket).into_bytes(),
self.keyspace.fanout_receipts_at(bucket).into_bytes(),
self.keyspace.fanout_ready_at(bucket).into_bytes(),
self.keyspace.fanout_leases_at(bucket).into_bytes(),
self.keyspace.fanout_gc_at(bucket).into_bytes(),
];
let argv: Vec<Vec<u8>> = vec![self.config.scan_batch_size.to_string().into_bytes()];
let raw = eval(&self.client, &FANOUT_SCAN_DUE, &keys, &argv).await?;
if raw.len() < 2 {
return Err(protocol("fanout_scan_due 返回缺少时间头"));
}
let redis_now = parse_num(raw.first(), "fanout_scan_due redisNow")?;
let root_count = parse_num(raw.get(1), "fanout_scan_due rootCount")?;
if redis_now < 0 || root_count < 0 {
return Err(protocol("fanout_scan_due 时间或根数量非法"));
}
let mut cursor = 2_usize;
let receipts = parse_due_index(&raw, &mut cursor)?;
let ready = parse_due_index(&raw, &mut cursor)?;
let leases = parse_due_index(&raw, &mut cursor)?;
let gc = parse_due_index(&raw, &mut cursor)?;
if cursor != raw.len()
|| [&receipts, &ready, &leases, &gc]
.iter()
.any(|index| index.members.len() > self.config.scan_batch_size as usize)
{
return Err(protocol("fanout_scan_due 索引数量或批量不符合合同"));
}
Ok(FanoutDueScans {
redis_now,
root_count,
receipts,
ready,
leases,
gc,
})
}
}
fn parse_due_index(raw: &[redis::Value], cursor: &mut usize) -> Result<DueIndexScan> {
let next_text = raw
.get(*cursor)
.map(value_to_string)
.ok_or_else(|| protocol("fanout_scan_due 缺少 nextScore"))?;
*cursor += 1;
let next_score = if next_text.is_empty() {
None
} else {
Some(
next_text
.parse::<i64>()
.map_err(|_| protocol("fanout_scan_due nextScore 非法"))?,
)
};
let count = parse_num(raw.get(*cursor), "fanout_scan_due dueCount")?;
*cursor += 1;
let count =
usize::try_from(count).map_err(|_| protocol("fanout_scan_due dueCount 超出合法范围"))?;
if count > raw.len().saturating_sub(*cursor) / 2 {
return Err(protocol("fanout_scan_due dueCount 超出响应成员数"));
}
let mut members = Vec::with_capacity(count);
for _ in 0..count {
let member = raw
.get(*cursor)
.map(value_to_string)
.ok_or_else(|| protocol("fanout_scan_due 缺少成员"))?;
*cursor += 1;
let score = parse_num(raw.get(*cursor), "fanout_scan_due score")?;
*cursor += 1;
members.push((member, score));
}
Ok(DueIndexScan {
next_score,
members,
})
}
fn text(fields: &[redis::Value], index: usize) -> String {
match fields.get(index) {
Some(redis::Value::Nil) | None => String::new(),
Some(redis::Value::Int(n)) => n.to_string(),
Some(redis::Value::BulkString(bytes)) => String::from_utf8_lossy(bytes).into_owned(),
Some(redis::Value::SimpleString(s)) => s.clone(),
Some(_) => String::new(),
}
}
fn num(fields: &[redis::Value], index: usize) -> Result<i64> {
match fields.get(index) {
Some(redis::Value::Nil) | None => Ok(0),
Some(redis::Value::Int(n)) => Ok(*n),
Some(redis::Value::BulkString(bytes)) => {
let s = String::from_utf8_lossy(bytes);
if s.is_empty() {
Ok(0)
} else {
s.parse::<i64>()
.map_err(|_| protocol("Fanout 投影数值字段非法"))
}
}
Some(redis::Value::SimpleString(s)) if s.is_empty() => Ok(0),
Some(redis::Value::SimpleString(s)) => s
.parse::<i64>()
.map_err(|_| protocol("Fanout 投影数值字段非法")),
Some(_) => Err(protocol("Fanout 投影数值字段类型非法")),
}
}
fn parse_num(value: Option<&redis::Value>, field: &str) -> Result<i64> {
value
.map(value_to_string)
.ok_or_else(|| protocol(&format!("{field} 缺失")))?
.parse::<i64>()
.map_err(|_| protocol(&format!("{field} 非法")))
}
fn protocol(message: &str) -> NasaRedisError {
crate::job::JobError::Protocol(message.to_owned()).into()
}