use std::cell::{Cell, RefCell};
use std::path::{Path, PathBuf};
use std::time::Duration;
use shep_client::RequestError;
use shep_client::shep_core::config::AppConfig;
use shep_client::shep_core::protocol::{ProcessInfo, RpcError, RpcErrorCode};
use shep_client::shep_core::status::ProcStatus;
use tokio::time::Instant;
use super::prepare::{Prepared, prepare};
use crate::daemon::Daemon;
use crate::error::Error;
use crate::fixtures;
use crate::paths::Tree;
use crate::state::{State, Watch};
pub(super) struct RollOf<'a>(pub(super) &'a [(&'a str, &'a Path)]);
impl Daemon for RollOf<'_> {
async fn save_roll(&self) -> Result<PathBuf, Error> {
let apps: Vec<String> = self
.0
.iter()
.map(|(name, cwd)| {
format!(
"{{\"app\":{{\"name\":{name:?},\"script\":\"./run.sh\",\"cwd\":{:?}}}}}",
cwd.to_str().expect("utf-8 cwd")
)
})
.collect();
let (_, cwd) = self.0.first().expect("a roll names at least one sheep");
let path = cwd.join(".flock-roll.json");
std::fs::write(&path, format!("{{\"apps\":[{}]}}", apps.join(","))).expect("write roll");
Ok(path)
}
crate::fixtures::daemon_methods!(unimplemented;
dog_config, list_flock, describe, start, delete, reload, restart, set_smit,
);
}
pub(super) fn checkout_with_commit() -> tempfile::TempDir {
checkout_declaring("[[app]]\nname = \"bpm\"\nscript = \"./run.sh\"\nwait_ready = true\n")
}
pub(super) fn checkout_declaring(flockfile: &str) -> tempfile::TempDir {
let dir = fixtures::checkout(&[
("Flockfile.toml", flockfile),
("run.sh", "#!/bin/sh\necho hi\n"),
]);
fixtures::run_git(
dir.path(),
&[
"remote",
"add",
"origin",
dir.path().to_str().expect("utf-8 tempdir path"),
],
);
dir
}
pub(super) fn write_target(home: &Path, sheep: &str, watch: Watch, sha: Option<&str>) {
let tree = Tree::for_sheep(home, sheep);
std::fs::create_dir_all(tree.state_file().parent().expect("has a parent"))
.expect("create target dir");
let state = State {
deployed: sha.map(str::to_owned),
watch,
..fixtures::state()
};
state.write(&tree.state_file()).expect("write state");
}
pub(super) const ORIGINAL_PID: u32 = 100;
pub(super) const NEWCOMER_ID: u32 = 99;
pub(super) const NEWCOMER_PID: u32 = 200;
pub(super) const REPAIR_ID: u32 = 500;
pub(super) const REPAIR_PID: u32 = 700;
#[derive(Debug, Clone, Copy)]
pub(super) enum Shape {
Absent,
Up {
status: ProcStatus,
pid: u32,
restarts: u32,
},
}
pub(super) fn shape_at(script: &[(Duration, Shape)], elapsed: Duration) -> Shape {
script
.iter()
.rev()
.find(|(from, _)| *from <= elapsed)
.map_or(Shape::Absent, |(_, shape)| *shape)
}
pub(super) fn originals_stay_online() -> Vec<(Duration, Shape)> {
vec![(
Duration::ZERO,
Shape::Up {
status: ProcStatus::Online,
pid: ORIGINAL_PID,
restarts: 0,
},
)]
}
pub(super) fn originals_respawn_mid_cutover() -> Vec<(Duration, Shape)> {
vec![
(
Duration::ZERO,
Shape::Up {
status: ProcStatus::Online,
pid: ORIGINAL_PID,
restarts: 0,
},
),
(
Duration::from_secs(1),
Shape::Up {
status: ProcStatus::Online,
pid: ORIGINAL_PID + 50,
restarts: 1,
},
),
]
}
pub(super) struct CutOverDouble {
originals: Vec<u32>,
original_script: Vec<(Duration, Shape)>,
script: Vec<(Duration, Shape)>,
refuses: Option<usize>,
refuses_deletes_from: Option<usize>,
deletes_seen: Cell<usize>,
mute_after: Option<Duration>,
starts: RefCell<Vec<AppConfig>>,
deletes: RefCell<Vec<u32>>,
attempts: Cell<usize>,
accepted_at: Cell<Option<Instant>>,
newcomers_at_most: Option<u32>,
repairs: Cell<u32>,
repaired_at: Cell<Option<Instant>>,
repair_stagger: Option<Duration>,
late_newcomer_after: Option<Duration>,
}
pub(super) const SPAWN_LAG: Duration = Duration::from_millis(250);
impl CutOverDouble {
pub(super) fn new(
originals: &[u32],
script: Vec<(Duration, Shape)>,
refuses: Option<usize>,
) -> Self {
Self {
originals: originals.to_vec(),
original_script: originals_stay_online(),
script,
refuses,
refuses_deletes_from: None,
deletes_seen: Cell::new(0),
mute_after: None,
starts: RefCell::new(Vec::new()),
deletes: RefCell::new(Vec::new()),
attempts: Cell::new(0),
accepted_at: Cell::new(None),
newcomers_at_most: None,
repairs: Cell::new(0),
repaired_at: Cell::new(None),
repair_stagger: None,
late_newcomer_after: None,
}
}
pub(super) fn with_a_late_newcomer_after(mut self, after: Duration) -> Self {
self.late_newcomer_after = Some(after);
self
}
pub(super) fn staggering_repairs_by(mut self, gap: Duration) -> Self {
self.repair_stagger = Some(gap);
self
}
pub(super) fn spawning_at_most(mut self, n: u32) -> Self {
self.newcomers_at_most = Some(n);
self
}
pub(super) fn going_quiet_after(mut self, after: Duration) -> Self {
self.mute_after = Some(after);
self
}
pub(super) fn refusing_deletes(self) -> Self {
self.refusing_deletes_from(0)
}
pub(super) fn refusing_deletes_from(mut self, from: usize) -> Self {
self.refuses_deletes_from = Some(from);
self
}
pub(super) fn while_the_originals(mut self, script: Vec<(Duration, Shape)>) -> Self {
self.original_script = script;
self
}
pub(super) fn started(&self) -> Vec<AppConfig> {
self.starts.borrow().clone()
}
pub(super) fn deleted(&self) -> Vec<u32> {
self.deletes.borrow().clone()
}
pub(super) fn is_deleted(&self, id: u32) -> bool {
self.deletes.borrow().contains(&id)
}
fn row(&self, id: u32, status: ProcStatus, pid: u32, restarts: u32) -> ProcessInfo {
let _ = self;
ProcessInfo::builder(id, "bpm", status)
.pid(Some(pid))
.restarts(restarts)
.build()
}
}
impl Daemon for CutOverDouble {
async fn describe(&self, _sheep: &str) -> Result<Vec<ProcessInfo>, Error> {
let mut flock = Vec::new();
let elapsed = self
.accepted_at
.get()
.map_or(Duration::ZERO, |accepted_at| Instant::now() - accepted_at);
if self.accepted_at.get().is_some() && self.mute_after.is_some_and(|after| elapsed >= after)
{
return Err(Error::Request(RequestError::Rpc(RpcError {
code: RpcErrorCode::Internal,
message: "the shepherd is not answering".to_owned(),
daemon_version: None,
})));
}
if let Shape::Up {
status,
pid,
restarts,
} = shape_at(&self.original_script, elapsed)
{
for (offset, id) in self.originals.iter().enumerate() {
let offset = u32::try_from(offset).expect("a handful of instances");
if !self.is_deleted(*id) {
flock.push(self.row(*id, status, pid + offset, restarts));
}
}
}
if self.accepted_at.get().is_some()
&& let Shape::Up {
status,
pid,
restarts,
} = shape_at(&self.script, elapsed)
{
let asked = self
.starts
.borrow()
.first()
.map_or(1, |app| app.instances.max(1));
let spawned = self.newcomers_at_most.map_or(asked, |cap| asked.min(cap));
for offset in 0..spawned {
let id = NEWCOMER_ID - offset;
if !self.is_deleted(id) {
flock.push(self.row(id, status, pid + offset, restarts));
}
}
if let Some(late) = self.late_newcomer_after
&& elapsed >= late
&& spawned < asked
{
let id = NEWCOMER_ID - spawned;
if !self.is_deleted(id) {
flock.push(self.row(id, ProcStatus::Online, pid + spawned, 0));
}
}
}
if let Some(at) = self.repaired_at.get() {
let since = Instant::now() - at;
for offset in 0..self.repairs.get() {
let lands_at = SPAWN_LAG + self.repair_stagger.unwrap_or(Duration::ZERO) * offset;
let id = REPAIR_ID + offset;
if since >= lands_at && !self.is_deleted(id) {
flock.push(self.row(id, ProcStatus::Online, REPAIR_PID + offset, 0));
}
}
}
Ok(flock)
}
async fn start(&self, apps: Vec<AppConfig>) -> Result<Vec<u32>, Error> {
let attempt = self.attempts.get();
self.attempts.set(attempt + 1);
if self.refuses == Some(attempt) {
return Err(Error::Request(RequestError::Rpc(RpcError {
code: RpcErrorCode::Internal,
message: "bpm cannot be started".to_owned(),
daemon_version: None,
})));
}
let asked = apps.first().map_or(1, |app| app.instances.max(1));
self.starts.borrow_mut().extend(apps);
if attempt == 0 {
self.accepted_at.set(Some(Instant::now()));
Ok((0..asked).map(|offset| NEWCOMER_ID - offset).collect())
} else {
let first = self.repairs.get();
self.repairs.set(first + asked);
if self.repaired_at.get().is_none() {
self.repaired_at.set(Some(Instant::now()));
}
Ok((first..first + asked)
.map(|offset| REPAIR_ID + offset)
.collect())
}
}
async fn delete(&self, id: u32) -> Result<(), Error> {
let seen = self.deletes_seen.get();
self.deletes_seen.set(seen + 1);
if self.refuses_deletes_from.is_some_and(|from| seen >= from) {
return Err(Error::Request(RequestError::Rpc(RpcError {
code: RpcErrorCode::Internal,
message: format!("instance {id} cannot be deleted"),
daemon_version: None,
})));
}
self.deletes.borrow_mut().push(id);
Ok(())
}
crate::fixtures::daemon_methods!(unimplemented;
dog_config, list_flock, reload, restart, save_roll, set_smit,
);
}
pub(super) struct Dirs {
_home: tempfile::TempDir,
_checkout: tempfile::TempDir,
}
pub(super) async fn cutover_fixture_of(
originals: &[u32],
script: Vec<(Duration, Shape)>,
refuses: Option<usize>,
) -> (CutOverDouble, Prepared, Dirs) {
cutover_fixture_from(checkout_with_commit(), originals, script, refuses).await
}
pub(super) async fn cutover_fixture_from(
checkout: tempfile::TempDir,
originals: &[u32],
script: Vec<(Duration, Shape)>,
refuses: Option<usize>,
) -> (CutOverDouble, Prepared, Dirs) {
let home = tempfile::tempdir().expect("tempdir");
let entries = [("bpm", checkout.path())];
let roll = RollOf(&entries);
let prepared = prepare(&roll, home.path(), "bpm", &fixtures::dog_config())
.await
.expect("prepares");
(
CutOverDouble::new(originals, script, refuses),
prepared,
Dirs {
_home: home,
_checkout: checkout,
},
)
}
pub(super) fn comes_up() -> Vec<(Duration, Shape)> {
vec![
(Duration::ZERO, Shape::Absent),
(
Duration::from_millis(250),
Shape::Up {
status: ProcStatus::Starting,
pid: NEWCOMER_PID,
restarts: 0,
},
),
(
Duration::from_millis(450),
Shape::Up {
status: ProcStatus::Online,
pid: NEWCOMER_PID,
restarts: 0,
},
),
]
}
pub(super) async fn cutover_fixture() -> (CutOverDouble, Prepared, Dirs) {
cutover_fixture_of(&[7], comes_up(), None).await
}
pub(super) async fn cutover_fixture_with_instances(
originals: &[u32],
) -> (CutOverDouble, Prepared, Dirs) {
cutover_fixture_of(originals, comes_up(), None).await
}
pub(super) async fn cutover_fixture_online_then_gone() -> (CutOverDouble, Prepared, Dirs) {
cutover_fixture_of(
&[7],
vec![
(
Duration::ZERO,
Shape::Up {
status: ProcStatus::Online,
pid: NEWCOMER_PID,
restarts: 0,
},
),
(Duration::from_secs(5), Shape::Absent),
],
None,
)
.await
}
pub(super) async fn cutover_fixture_crash_looping() -> (CutOverDouble, Prepared, Dirs) {
cutover_fixture_of(
&[7],
vec![
(Duration::ZERO, Shape::Absent),
(
Duration::from_millis(250),
Shape::Up {
status: ProcStatus::Online,
pid: NEWCOMER_PID,
restarts: 0,
},
),
(
Duration::from_secs(5),
Shape::Up {
status: ProcStatus::Online,
pid: NEWCOMER_PID + 50,
restarts: 1,
},
),
],
None,
)
.await
}
pub(super) fn dies_during_dwell() -> Vec<(Duration, Shape)> {
vec![
(Duration::ZERO, Shape::Absent),
(
Duration::from_millis(250),
Shape::Up {
status: ProcStatus::Online,
pid: NEWCOMER_PID,
restarts: 0,
},
),
(
Duration::from_secs(5),
Shape::Up {
status: ProcStatus::Errored,
pid: NEWCOMER_PID,
restarts: 0,
},
),
]
}
pub(super) async fn cutover_fixture_dies_during_dwell() -> (CutOverDouble, Prepared, Dirs) {
cutover_fixture_of(&[7], dies_during_dwell(), None).await
}
pub(super) async fn cutover_fixture_dies_and_refuses_repair() -> (CutOverDouble, Prepared, Dirs) {
cutover_fixture_of(&[7], dies_during_dwell(), Some(1)).await
}
pub(super) async fn cutover_fixture_never_appears() -> (CutOverDouble, Prepared, Dirs) {
cutover_fixture_of(&[7], vec![(Duration::ZERO, Shape::Absent)], None).await
}
pub(super) async fn cutover_fixture_comes_up_but_refuses_deletes() -> (CutOverDouble, Prepared, Dirs)
{
let (daemon, prepared, dirs) = cutover_fixture_of(&[7, 8], comes_up(), None).await;
(daemon.refusing_deletes(), prepared, dirs)
}
pub(super) async fn cutover_fixture_shepherd_goes_quiet() -> (CutOverDouble, Prepared, Dirs) {
let (daemon, prepared, dirs) = cutover_fixture_of(&[7], comes_up(), None).await;
(daemon.going_quiet_after(Duration::ZERO), prepared, dirs)
}
pub(super) async fn cutover_fixture_dies_and_refuses_the_second_delete()
-> (CutOverDouble, Prepared, Dirs) {
let (daemon, prepared, dirs) = cutover_fixture_of(&[7], dies_during_dwell(), None).await;
(daemon.refusing_deletes_from(1), prepared, dirs)
}
pub(super) async fn cutover_fixture_dies_and_refuses_deletes() -> (CutOverDouble, Prepared, Dirs) {
let (daemon, prepared, dirs) = cutover_fixture_of(&[7], dies_during_dwell(), None).await;
(daemon.refusing_deletes(), prepared, dirs)
}
pub(super) async fn cutover_fixture_original_respawns() -> (CutOverDouble, Prepared, Dirs) {
let (daemon, prepared, dirs) =
cutover_fixture_of(&[7], vec![(Duration::ZERO, Shape::Absent)], None).await;
(
daemon.while_the_originals(originals_respawn_mid_cutover()),
prepared,
dirs,
)
}
pub(super) async fn cutover_fixture_refusing_start() -> (CutOverDouble, Prepared, Dirs) {
cutover_fixture_of(&[7], comes_up(), Some(0)).await
}