use std::collections::VecDeque;
use std::sync::{Arc, Mutex};
use shared::{Event, ServiceStatus};
use tokio::sync::{Notify, OwnedSemaphorePermit, Semaphore, watch};
use tracing::{info, warn};
use crate::bootstrap::state::AppState;
use crate::features::deploy::executor::DeployExecutor;
#[derive(Default)]
struct QueueInner {
queued: VecDeque<String>,
running: Option<String>,
paused: bool,
}
pub struct DeployQueue {
inner: Mutex<QueueInner>,
notify: Notify,
slot: Arc<Semaphore>,
paused_tx: watch::Sender<bool>,
}
impl DeployQueue {
pub fn new() -> Arc<Self> {
Arc::new(Self {
inner: Mutex::new(QueueInner::default()),
notify: Notify::new(),
slot: Arc::new(Semaphore::new(1)),
paused_tx: watch::channel(false).0,
})
}
pub fn enqueue(&self, dep_id: String) {
{
let mut g = self.inner.lock().unwrap();
if g.running.as_deref() == Some(dep_id.as_str()) || g.queued.contains(&dep_id) {
return;
}
g.queued.push_back(dep_id);
}
self.notify.notify_one();
}
pub fn remove_queued(&self, dep_id: &str) -> bool {
let mut g = self.inner.lock().unwrap();
if let Some(pos) = g.queued.iter().position(|d| d == dep_id) {
g.queued.remove(pos);
true
} else {
false
}
}
pub fn promote(&self, dep_id: &str) {
let mut g = self.inner.lock().unwrap();
if let Some(pos) = g.queued.iter().position(|d| d == dep_id)
&& let Some(d) = g.queued.remove(pos)
{
g.queued.push_front(d);
}
}
pub fn reorder(&self, order: &[String]) {
let mut g = self.inner.lock().unwrap();
let mut remaining: VecDeque<String> = std::mem::take(&mut g.queued);
let mut next = VecDeque::with_capacity(remaining.len());
for id in order {
if let Some(pos) = remaining.iter().position(|d| d == id)
&& let Some(d) = remaining.remove(pos)
{
next.push_back(d);
}
}
next.extend(remaining);
g.queued = next;
}
pub fn set_paused(&self, paused: bool) {
{
let mut g = self.inner.lock().unwrap();
g.paused = paused;
}
self.paused_tx.send_replace(paused);
if !paused {
self.notify.notify_one();
}
}
pub fn snapshot(&self) -> (Option<String>, Vec<String>, bool) {
let g = self.inner.lock().unwrap();
(
g.running.clone(),
g.queued.iter().cloned().collect(),
g.paused,
)
}
fn has_ready(&self) -> bool {
let g = self.inner.lock().unwrap();
!g.paused && !g.queued.is_empty()
}
pub async fn acquire_next(&self) -> (OwnedSemaphorePermit, String) {
loop {
while !self.has_ready() {
self.wait().await;
}
let permit = self
.slot
.clone()
.acquire_owned()
.await
.expect("semáforo da fila nunca é fechado");
if let Some(id) = self.take_next() {
return (permit, id);
}
drop(permit);
}
}
pub async fn acquire_job_slot(&self) -> OwnedSemaphorePermit {
let mut paused = self.paused_tx.subscribe();
loop {
let _ = paused.wait_for(|p| !*p).await;
let permit = self
.slot
.clone()
.acquire_owned()
.await
.expect("semáforo da fila nunca é fechado");
if !*paused.borrow() {
return permit;
}
drop(permit);
}
}
pub fn try_job_slot_now(&self) -> Option<OwnedSemaphorePermit> {
if *self.paused_tx.borrow() {
return None;
}
self.slot.clone().try_acquire_owned().ok()
}
fn take_next(&self) -> Option<String> {
let mut g = self.inner.lock().unwrap();
if g.paused {
return None;
}
let next = g.queued.pop_front();
if let Some(id) = &next {
g.running = Some(id.clone());
}
next
}
fn clear_running(&self) {
self.inner.lock().unwrap().running = None;
}
async fn wait(&self) {
self.notify.notified().await;
}
}
pub async fn run_worker(state: AppState) {
let queue = state.deploy_queue.clone();
info!("deploy queue worker iniciado");
loop {
let (slot, dep_id) = queue.acquire_next().await;
state.bus.publish(Event::DeployQueueChanged);
run_one(&state, &dep_id).await;
queue.clear_running();
drop(slot);
state.bus.publish(Event::DeployQueueChanged);
}
}
async fn run_one(state: &AppState, dep_id: &str) {
if let Ok(Some(dep)) = crate::store::deployments::get(&state.db, dep_id).await {
let _ = crate::store::services::update_status(
&state.db,
&dep.service_id,
&ServiceStatus::Deploying,
None,
)
.await;
state.bus.publish(Event::ServiceStatusChanged {
service_id: dep.service_id.clone(),
status: ServiceStatus::Deploying,
});
} else {
warn!(deployment_id = %dep_id, "deploy queue: deployment sumiu antes de rodar");
return;
}
let executor = Arc::new(DeployExecutor {
db: state.db.clone(),
docker: state.docker.clone(),
ingress: state.ingress.clone(),
bus: state.bus.clone(),
secrets: state.secrets.clone(),
tls: state.tls.clone(),
db_path: state.db_path.clone(),
drain_secs: state.drain_secs,
registry_internal_token: state.registry_internal_token.clone(),
});
let dep_owned = dep_id.to_string();
let handle = tokio::spawn(async move { executor.run(dep_owned).await });
if let Ok(mut map) = state.active_deploys.lock() {
map.insert(dep_id.to_string(), handle.abort_handle());
}
let _ = handle.await;
if let Ok(mut map) = state.active_deploys.lock() {
map.remove(dep_id);
}
}
#[cfg(test)]
mod tests {
use super::*;
fn queued(q: &DeployQueue) -> Vec<String> {
q.snapshot().1
}
#[test]
fn enqueue_orders_fifo_and_dedups() {
let q = DeployQueue::new();
q.enqueue("a".into());
q.enqueue("b".into());
q.enqueue("a".into()); assert_eq!(queued(&q), vec!["a", "b"]);
}
#[test]
fn promote_moves_to_front() {
let q = DeployQueue::new();
for id in ["a", "b", "c"] {
q.enqueue(id.into());
}
q.promote("c");
assert_eq!(queued(&q), vec!["c", "a", "b"]);
q.promote("zzz");
assert_eq!(queued(&q), vec!["c", "a", "b"]);
}
#[test]
fn reorder_applies_and_appends_omitted() {
let q = DeployQueue::new();
for id in ["a", "b", "c"] {
q.enqueue(id.into());
}
q.reorder(&["c".into(), "zzz".into(), "a".into()]);
assert_eq!(queued(&q), vec!["c", "a", "b"]);
}
#[test]
fn remove_queued_reports_and_removes() {
let q = DeployQueue::new();
q.enqueue("a".into());
q.enqueue("b".into());
assert!(q.remove_queued("a"));
assert!(!q.remove_queued("nope"));
assert_eq!(queued(&q), vec!["b"]);
}
#[test]
fn take_next_respects_pause_and_marks_running() {
let q = DeployQueue::new();
q.enqueue("a".into());
q.set_paused(true);
assert!(q.take_next().is_none(), "pausada não entrega");
q.set_paused(false);
assert_eq!(q.take_next().as_deref(), Some("a"));
let (running, queued_now, paused) = q.snapshot();
assert_eq!(running.as_deref(), Some("a"));
assert!(queued_now.is_empty());
assert!(!paused);
q.clear_running();
assert!(q.snapshot().0.is_none());
}
use std::time::Duration;
use tokio::time::{sleep, timeout};
const ESPERA: Duration = Duration::from_millis(60);
const LIMITE: Duration = Duration::from_secs(2);
#[tokio::test]
async fn job_e_deploy_nao_executam_juntos() {
let q = DeployQueue::new();
q.enqueue("a".into());
let job = q.acquire_job_slot().await;
let q2 = q.clone();
let worker = tokio::spawn(async move { q2.acquire_next().await });
sleep(ESPERA).await;
assert!(
!worker.is_finished(),
"o deploy não pode começar com um job rodando"
);
let (running, queued, _) = q.snapshot();
assert!(running.is_none(), "ainda não está rodando");
assert_eq!(queued, vec!["a"], "segue na fila");
drop(job);
let (permit, id) = timeout(LIMITE, worker).await.unwrap().unwrap();
assert_eq!(id, "a");
assert_eq!(q.snapshot().0.as_deref(), Some("a"));
assert!(q.try_job_slot_now().is_none(), "vaga ocupada pelo deploy");
let q3 = q.clone();
let job2 = tokio::spawn(async move { q3.acquire_job_slot().await });
sleep(ESPERA).await;
assert!(
!job2.is_finished(),
"o job não pode começar com um deploy rodando"
);
q.clear_running();
drop(permit);
let _vaga = timeout(LIMITE, job2)
.await
.expect("o job entra quando o deploy termina")
.unwrap();
}
#[tokio::test]
async fn dois_jobs_nao_executam_juntos_e_entram_em_ordem_de_chegada() {
let q = DeployQueue::new();
let primeiro = q.acquire_job_slot().await;
let ordem = Arc::new(Mutex::new(Vec::new()));
let mut tarefas = Vec::new();
for n in 1..=3 {
let (q, ordem) = (q.clone(), ordem.clone());
tarefas.push(tokio::spawn(async move {
let _vaga = q.acquire_job_slot().await;
ordem.lock().unwrap().push(n);
sleep(Duration::from_millis(20)).await;
}));
sleep(Duration::from_millis(20)).await; }
assert!(
ordem.lock().unwrap().is_empty(),
"nenhum entra com a vaga ocupada"
);
drop(primeiro);
for t in tarefas {
timeout(LIMITE, t).await.unwrap().unwrap();
}
assert_eq!(*ordem.lock().unwrap(), vec![1, 2, 3]);
}
#[tokio::test]
async fn pausa_segura_os_jobs_ate_retomar() {
let q = DeployQueue::new();
q.set_paused(true);
assert!(
q.try_job_slot_now().is_none(),
"pausada: não entrega, mesmo com a vaga livre"
);
let q2 = q.clone();
let job = tokio::spawn(async move { q2.acquire_job_slot().await });
sleep(ESPERA).await;
assert!(!job.is_finished(), "o job espera a fila ser retomada");
q.set_paused(false);
let _vaga = timeout(LIMITE, job)
.await
.expect("retomada libera o job")
.unwrap();
}
#[tokio::test]
async fn pausa_durante_a_espera_da_vaga_devolve_a_vaga() {
let q = DeployQueue::new();
q.enqueue("a".into());
let (permit, _) = q.acquire_next().await;
let q2 = q.clone();
let job = tokio::spawn(async move { q2.acquire_job_slot().await });
sleep(ESPERA).await;
q.set_paused(true);
q.clear_running();
drop(permit);
sleep(ESPERA).await;
assert!(!job.is_finished(), "pausada: não pode começar");
assert!(q.try_job_slot_now().is_none());
q.set_paused(false);
let _vaga = timeout(LIMITE, job).await.expect("retomou").unwrap();
}
#[tokio::test]
async fn item_removido_enquanto_espera_a_vaga_nao_e_entregue() {
let q = DeployQueue::new();
q.enqueue("a".into());
let job = q.acquire_job_slot().await;
let q2 = q.clone();
let worker = tokio::spawn(async move { q2.acquire_next().await });
sleep(ESPERA).await;
assert!(q.remove_queued("a"), "ainda está na fila, dá para remover");
drop(job);
sleep(ESPERA).await;
assert!(!worker.is_finished(), "sem item, o worker segue esperando");
assert!(
q.try_job_slot_now().is_some(),
"e não fica com a vaga presa"
);
q.enqueue("b".into());
let (_, id) = timeout(LIMITE, worker).await.unwrap().unwrap();
assert_eq!(id, "b");
}
#[tokio::test]
async fn worker_espera_pausa_e_fila_vazia_sem_segurar_a_vaga() {
let q = DeployQueue::new();
let q2 = q.clone();
let worker = tokio::spawn(async move { q2.acquire_next().await });
sleep(ESPERA).await;
assert!(!worker.is_finished());
assert!(
q.try_job_slot_now().is_some(),
"vaga livre: jobs não podem ficar presos"
);
worker.abort();
}
}