1use std::collections::VecDeque;
17use std::path::PathBuf;
18use std::sync::atomic::{AtomicU64, Ordering};
19use std::sync::{Arc, Condvar, Mutex};
20
21use codewhale_lane::control::{
22 ControlContext, ControlFailure, ControlFailureKind, ControlOperation, ControlReceipt,
23 ControlSurface, execute_lane_control_in, parse_target,
24};
25
26pub const MAX_PENDING: usize = 16;
32
33#[derive(Debug)]
34struct Submission {
35 ticket: String,
36 operation: ControlOperation,
37 raw_target: Option<String>,
38 registry_root: Option<PathBuf>,
39}
40
41#[derive(Debug, Default)]
42struct Shared {
43 pending: VecDeque<Submission>,
44 completed: Vec<ControlReceipt>,
45 in_flight: Vec<(String, String)>,
48 shutdown: bool,
49 worker_started: bool,
53}
54
55#[derive(Debug, Clone)]
57pub struct LaneControlQueue {
58 shared: Arc<(Mutex<Shared>, Condvar)>,
59 tickets: Arc<AtomicU64>,
60}
61
62impl Default for LaneControlQueue {
63 fn default() -> Self {
64 Self::new()
65 }
66}
67
68impl LaneControlQueue {
69 #[must_use]
73 pub fn new() -> Self {
74 Self {
75 shared: Arc::new((Mutex::new(Shared::default()), Condvar::new())),
76 tickets: Arc::new(AtomicU64::new(1)),
77 }
78 }
79
80 fn ensure_worker(&self, shared: &mut Shared) {
86 if shared.worker_started {
87 return;
88 }
89 shared.worker_started = true;
90 let worker = self.clone();
91 if std::thread::Builder::new()
92 .name("lane-control".to_string())
93 .spawn(move || worker.run())
94 .is_err()
95 {
96 shared.worker_started = false;
99 }
100 }
101
102 #[must_use]
108 pub fn submit(
109 &self,
110 operation: ControlOperation,
111 raw_target: Option<&str>,
112 registry_root: Option<PathBuf>,
113 ) -> ControlReceipt {
114 let descriptor = operation.descriptor();
115 let surface = ControlSurface::Slash;
116
117 let availability = descriptor.availability(surface, ControlContext::new(true, true));
127 if !availability.is_available() {
128 return ControlReceipt::unavailable(descriptor, surface, availability);
129 }
130
131 let target = match parse_target(descriptor, raw_target) {
134 Ok(target) => target,
135 Err(failure) => {
136 return ControlReceipt::rejected(descriptor, surface, None, failure);
137 }
138 };
139
140 let (lock, condvar) = &*self.shared;
141 let mut shared = match lock.lock() {
142 Ok(shared) => shared,
143 Err(_) => {
146 return ControlReceipt::failed(
147 descriptor,
148 surface,
149 target,
150 ControlFailure::backend("Lane control worker is not running"),
151 );
152 }
153 };
154
155 if shared.pending.len() >= MAX_PENDING {
156 return ControlReceipt::rejected(
157 descriptor,
158 surface,
159 target,
160 ControlFailure::new(
161 ControlFailureKind::Saturated,
162 format!(
163 "Lane control queue is full ({MAX_PENDING} pending); \
164 nothing was submitted"
165 ),
166 ),
167 );
168 }
169
170 if let Some(target) = target.as_ref()
173 && let Some((ticket, _)) = shared
174 .in_flight
175 .iter()
176 .find(|(_, id)| *id == target.id)
177 .cloned()
178 {
179 return ControlReceipt::rejected(
180 descriptor,
181 surface,
182 Some(target.clone()),
183 ControlFailure::conflict(format!(
184 "{} is already in flight under ticket {ticket}",
185 target.id
186 )),
187 );
188 }
189
190 let ticket = format!("lane-ctl-{}", self.tickets.fetch_add(1, Ordering::Relaxed));
191 if let Some(target) = target.as_ref() {
192 shared.in_flight.push((ticket.clone(), target.id.clone()));
193 }
194 shared.pending.push_back(Submission {
195 ticket: ticket.clone(),
196 operation,
197 raw_target: raw_target.map(str::to_string),
198 registry_root,
199 });
200 if cfg!(not(test)) {
203 self.ensure_worker(&mut shared);
204 }
205 condvar.notify_one();
206 drop(shared);
207
208 ControlReceipt::queued(descriptor, surface, target, ticket)
209 }
210
211 #[must_use]
213 pub fn drain_completed(&self) -> Vec<ControlReceipt> {
214 let (lock, _) = &*self.shared;
215 match lock.lock() {
216 Ok(mut shared) => std::mem::take(&mut shared.completed),
217 Err(_) => Vec::new(),
218 }
219 }
220
221 #[cfg(test)]
225 #[must_use]
226 pub fn pending_len(&self) -> usize {
227 let (lock, _) = &*self.shared;
228 lock.lock().map(|shared| shared.pending.len()).unwrap_or(0)
229 }
230
231 pub fn run_once(&self) -> bool {
236 let (lock, _) = &*self.shared;
237 let Some(submission) = ({
238 let Ok(mut shared) = lock.lock() else {
239 return false;
240 };
241 shared.pending.pop_front()
242 }) else {
243 return false;
244 };
245
246 let receipt = execute_lane_control_in(
249 ControlSurface::Cli,
250 submission.operation,
251 submission.raw_target.as_deref(),
252 submission.registry_root.as_deref(),
253 );
254 let mut receipt = receipt.with_ticket(submission.ticket.clone());
258 receipt.surface = ControlSurface::Slash;
259
260 if let Ok(mut shared) = lock.lock() {
261 shared
262 .in_flight
263 .retain(|(ticket, _)| *ticket != submission.ticket);
264 shared.completed.push(receipt);
265 }
266 true
267 }
268
269 fn run(&self) {
270 let (lock, condvar) = &*self.shared;
271 loop {
272 {
273 let Ok(mut shared) = lock.lock() else {
274 return;
275 };
276 while shared.pending.is_empty() && !shared.shutdown {
277 let Ok(next) = condvar.wait(shared) else {
278 return;
279 };
280 shared = next;
281 }
282 if shared.shutdown && shared.pending.is_empty() {
283 return;
284 }
285 }
286 self.run_once();
287 }
288 }
289}
290
291#[cfg(test)]
292mod tests {
293 use super::*;
294 use codewhale_lane::{LaneRegistry, LaneStatus, LifecycleOutcome, RuntimeBackendKind};
295
296 fn seeded() -> (tempfile::TempDir, String) {
297 let dir = tempfile::tempdir().unwrap();
298 let registry = LaneRegistry::open(dir.path()).unwrap();
299 let record = registry
300 .create_pending(
301 Some("stopship".into()),
302 Some("stopship".into()),
303 Some("4022".into()),
304 None,
305 RuntimeBackendKind::Inline,
306 None,
307 )
308 .unwrap();
309 let id = record.id.clone();
310 (dir, id)
311 }
312
313 #[test]
316 fn slash_interrupt_returns_queued_then_completes_off_loop() {
317 let (dir, id) = seeded();
318 let queue = LaneControlQueue::new();
319
320 let submitted = queue.submit(
321 ControlOperation::LaneInterrupt,
322 Some(id.as_str()),
323 Some(dir.path().to_path_buf()),
324 );
325 assert_eq!(submitted.outcome, LifecycleOutcome::Queued);
326 assert_eq!(submitted.surface, ControlSurface::Slash);
327 assert!(submitted.ticket.is_some());
328 assert!(
329 !submitted.is_error(),
330 "queued is not an error, but it is also not success"
331 );
332 assert_eq!(
334 LaneRegistry::open(dir.path())
335 .unwrap()
336 .load(&id)
337 .unwrap()
338 .status,
339 LaneStatus::Pending
340 );
341
342 assert!(queue.run_once());
343 let completed = queue.drain_completed();
344 assert_eq!(completed.len(), 1);
345 assert_eq!(completed[0].ticket, submitted.ticket);
346 assert_eq!(completed[0].outcome, LifecycleOutcome::Transitioned);
347 assert_eq!(
348 completed[0].surface,
349 ControlSurface::Slash,
350 "the receipt reports who asked, not which thread ran it"
351 );
352 assert_eq!(
353 LaneRegistry::open(dir.path())
354 .unwrap()
355 .load(&id)
356 .unwrap()
357 .status,
358 LaneStatus::Stopped
359 );
360 }
361
362 #[test]
364 fn invalid_targets_are_refused_synchronously() {
365 let queue = LaneControlQueue::new();
366 for bad in [None, Some(""), Some("../escape"), Some("a b")] {
367 let receipt = queue.submit(ControlOperation::LaneInterrupt, bad, None);
368 assert_eq!(
369 receipt.failure.as_ref().map(|failure| failure.kind),
370 Some(ControlFailureKind::InvalidTarget),
371 "{bad:?}"
372 );
373 assert!(receipt.ticket.is_none());
374 }
375 assert_eq!(queue.pending_len(), 0);
376 }
377
378 #[test]
382 fn declared_but_unbuilt_verbs_are_refused_without_queueing() {
383 let queue = LaneControlQueue::new();
384 for operation in [ControlOperation::LaneRestart, ControlOperation::LaneResume] {
385 let receipt = queue.submit(operation, Some("lane-a1b2c3d4"), None);
386 assert_eq!(receipt.outcome, LifecycleOutcome::Rejected);
387 assert_eq!(
388 receipt.availability.reason(),
389 Some(codewhale_lane::UnavailableReason::BackendNotImplemented),
390 "{operation:?}"
391 );
392 assert!(receipt.ticket.is_none());
393 assert!(!receipt.retryable);
394 }
395 assert_eq!(queue.pending_len(), 0, "nothing may have been enqueued");
396 }
397
398 #[test]
401 fn a_saturated_queue_refuses_without_submitting() {
402 let queue = LaneControlQueue::new();
403 for index in 0..MAX_PENDING {
404 let receipt = queue.submit(
405 ControlOperation::LaneInterrupt,
406 Some(format!("lane-{index:08}").as_str()),
407 None,
408 );
409 assert_eq!(receipt.outcome, LifecycleOutcome::Queued);
410 }
411 assert_eq!(queue.pending_len(), MAX_PENDING);
412
413 let refused = queue.submit(ControlOperation::LaneInterrupt, Some("lane-overflow"), None);
414 assert_eq!(refused.outcome, LifecycleOutcome::Rejected);
415 assert_eq!(
416 refused.failure.as_ref().map(|failure| failure.kind),
417 Some(ControlFailureKind::Saturated)
418 );
419 assert!(refused.retryable, "saturation clears on its own");
420 assert!(refused.ticket.is_none());
421 assert_eq!(
422 queue.pending_len(),
423 MAX_PENDING,
424 "the refused submission must not have been enqueued"
425 );
426 }
427
428 #[test]
431 fn a_duplicate_submission_for_one_lane_is_a_conflict() {
432 let queue = LaneControlQueue::new();
433 let first = queue.submit(ControlOperation::LaneInterrupt, Some("lane-a1b2c3d4"), None);
434 assert_eq!(first.outcome, LifecycleOutcome::Queued);
435
436 let second = queue.submit(ControlOperation::LaneInterrupt, Some("lane-a1b2c3d4"), None);
437 assert_eq!(
438 second.failure.as_ref().map(|failure| failure.kind),
439 Some(ControlFailureKind::Conflict)
440 );
441 assert_eq!(queue.pending_len(), 1);
442 }
443
444 #[test]
448 fn submission_does_not_block_on_runtime_teardown() {
449 let (dir, id) = seeded();
450 let queue = LaneControlQueue::new();
451 let started = std::time::Instant::now();
452 let receipt = queue.submit(
453 ControlOperation::LaneInterrupt,
454 Some(id.as_str()),
455 Some(dir.path().to_path_buf()),
456 );
457 let elapsed = started.elapsed();
458 assert_eq!(receipt.outcome, LifecycleOutcome::Queued);
459 assert!(
460 elapsed < std::time::Duration::from_millis(250),
461 "submission took {elapsed:?}; it must not run teardown inline"
462 );
463 assert_eq!(
466 LaneRegistry::open(dir.path())
467 .unwrap()
468 .load(&id)
469 .unwrap()
470 .status,
471 LaneStatus::Pending
472 );
473 }
474
475 #[test]
478 fn a_stale_fence_conflicts_when_the_queued_work_runs() {
479 let (dir, id) = seeded();
480 let queue = LaneControlQueue::new();
481 let submitted = queue.submit(
482 ControlOperation::LaneInterrupt,
483 Some(format!("{id}@99").as_str()),
484 Some(dir.path().to_path_buf()),
485 );
486 assert_eq!(submitted.outcome, LifecycleOutcome::Queued);
487
488 assert!(queue.run_once());
489 let completed = queue.drain_completed();
490 assert_eq!(completed.len(), 1);
491 assert_eq!(completed[0].outcome, LifecycleOutcome::Rejected);
492 assert_eq!(
493 completed[0].failure.as_ref().map(|failure| failure.kind),
494 Some(ControlFailureKind::Conflict)
495 );
496 assert_eq!(
497 LaneRegistry::open(dir.path())
498 .unwrap()
499 .load(&id)
500 .unwrap()
501 .status,
502 LaneStatus::Pending,
503 "a refused fence must not have torn anything down"
504 );
505 }
506
507 #[test]
509 fn completing_a_submission_frees_its_in_flight_slot() {
510 let (dir, id) = seeded();
511 let queue = LaneControlQueue::new();
512 let _ = queue.submit(
513 ControlOperation::LaneInterrupt,
514 Some(id.as_str()),
515 Some(dir.path().to_path_buf()),
516 );
517 assert!(queue.run_once());
518 let _ = queue.drain_completed();
519
520 let again = queue.submit(
521 ControlOperation::LaneInterrupt,
522 Some(id.as_str()),
523 Some(dir.path().to_path_buf()),
524 );
525 assert_eq!(
526 again.outcome,
527 LifecycleOutcome::Queued,
528 "the slot must be reusable once the ticket completes"
529 );
530 }
531}