#!/usr/bin/env bash
# Watch a container-backed worker be supervised, for real, on this machine.
#
# Why this exists: every container path in `src/agent/supervisor.rs` is driven
# in CI by a `FakeDocker`, which answers what it was told to answer. That is
# the right way to test argv and decisions, and it cannot tell you that the
# loop works — that a worker killed inside a container that stays up is
# noticed, that the restart lands, that a restarted agent re-adopts the
# container instead of orphaning it or starting a second one. Nothing but a
# real daemon can say that, so this runs one.
#
# What it drives: the real `Supervisor`, against the real image, with a real
# `zc broker` under it, in two phases — two separate OS processes, because the
# gap between them *is* the agent restart being tested. `launchd` is not
# involved at all; the supervisor loop is driven directly, which is also what
# keeps this off the agent you have installed.
#
# What it will not touch: your agent, your broker, your launchd job, your
# containers. Everything here is namespaced — its own state dir, its own two
# ports, and a node fingerprint (`e2e263sv`) that cannot collide with a real
# one, because a real one is hex and this is not. Only containers and networks
# matching that prefix are ever removed, and nothing is ever pruned.
#
# Usage:
#   scripts/verify-container-supervision.sh
#   BROKER_PORT=9161 WORKER_PORT=3997 scripts/verify-container-supervision.sh
#
# Idempotent: it clears its own leftovers on the way in, and cleans up on the
# way out however it exits.

set -euo pipefail
cd "$(dirname "$0")/.."
REPO="$(pwd)"

# --- this run's identity ----------------------------------------------------
# Not hex, so it can never be the first eight characters of a real node
# fingerprint, so `zc-share-e2e263sv-*` can never be a container anything else
# on this machine started.
FP8="${FP8:-e2e263sv}"
BROKER_PORT="${BROKER_PORT:-9160}"
WORKER_PORT="${WORKER_PORT:-3995}"
IMAGE="${IMAGE:-zakuro-image-compute:latest}"
SCRATCH="${SCRATCH:-${TMPDIR:-/tmp}/zc-e2e-263-supervision}"
CONTAINER_PREFIX="zc-share-${FP8}"
NETWORK="zc-worker-${FP8}"
ZC_BIN="${ZC_BIN:-$REPO/target/debug/zc}"

TEST_MOD="agent::supervisor::tests"
PHASE1="${TEST_MOD}::a_container_backed_worker_is_spawned_and_restarted_when_the_worker_dies_inside_its_container"
PHASE2="${TEST_MOD}::a_restarted_agent_re_adopts_the_container_it_left_running_and_then_tears_it_down"

step() { printf '\n\033[1m==> %s\033[0m\n' "$*"; }
info() { printf '    %s\n' "$*"; }
fail() { printf '\n\033[31mFAILED: %s\033[0m\n' "$*" >&2; exit 1; }
ok()   { printf '\033[32m    ok: %s\033[0m\n' "$*"; }

# `assert <description> <command...>` — prints what it is asserting before it
# asserts it, and stops the run on the first one that does not hold.
assert() {
  local what="$1"; shift
  info "assert: $what"
  "$@" >/dev/null 2>&1 || fail "$what"
  ok "$what"
}

# Only ever our own prefix. Never a bare `docker rm`, never a prune.
our_containers() {
  docker ps -a --format '{{.Names}}' 2>/dev/null | grep -E "^${CONTAINER_PREFIX}" || true
}
our_networks() {
  docker network ls --format '{{.Name}}' 2>/dev/null | grep -E "^${NETWORK}$" || true
}
# The broker this harness starts, identified by its exact argv (binary path
# AND port), so a broker anyone else is running cannot match.
our_brokers() {
  pgrep -f "^${ZC_BIN} broker ${BROKER_PORT}$" 2>/dev/null || true
}

cleanup() {
  local code=$?
  step "cleaning up (exit code $code)"
  local c
  for c in $(our_containers); do
    info "docker rm -f $c"
    docker rm -f "$c" >/dev/null 2>&1 || true
  done
  for c in $(our_brokers); do
    info "kill $c (the harness's own broker on ${BROKER_PORT})"
    kill "$c" 2>/dev/null || true
  done
  # Nothing in zc removes this network (see the note printed at teardown), so
  # the harness removes the one it caused to be created.
  for c in $(our_networks); do
    info "docker network rm $c"
    docker network rm "$c" >/dev/null 2>&1 || true
  done
  rm -rf "$SCRATCH"
  info "scratch dir removed: $SCRATCH"
  exit $code
}
trap cleanup EXIT INT TERM

port_busy() { lsof -nP -iTCP:"$1" -sTCP:LISTEN >/dev/null 2>&1; }

# --- preflight --------------------------------------------------------------
step "preflight"
command -v docker >/dev/null || fail "docker is not on PATH"
docker info >/dev/null 2>&1 || fail "the Docker daemon is not reachable — is Docker Desktop running?"
ok "the Docker daemon answers"
docker image inspect "$IMAGE" >/dev/null 2>&1 || fail "image $IMAGE is not present locally (docker pull it first)"
ok "image $IMAGE is present"

info "containers already running on this machine, none of which this run may touch:"
docker ps --format '      {{.Names}}' | sort

for p in "$BROKER_PORT" "$WORKER_PORT"; do
  if port_busy "$p"; then
    fail "port $p is already in use; pass BROKER_PORT=/WORKER_PORT= to move this run"
  fi
done
ok "ports $BROKER_PORT (broker) and $WORKER_PORT (worker) are free"

# Idempotency: clear anything a previous run of THIS harness left behind.
for c in $(our_containers); do info "removing leftover container $c"; docker rm -f "$c" >/dev/null; done
for c in $(our_brokers); do info "killing leftover broker $c"; kill "$c" 2>/dev/null || true; done
rm -rf "$SCRATCH"
mkdir -p "$SCRATCH"
ok "scratch state dir: $SCRATCH"

step "building zc (the broker the supervisor starts) and the test binary"
cargo build --bin zc
[ -x "$ZC_BIN" ] || fail "no zc binary at $ZC_BIN"
cargo test --bin zc --no-run >/dev/null
ok "$ZC_BIN"

export ZC_E2E_DIR="$SCRATCH"
export ZC_E2E_FP8="$FP8"
export ZC_E2E_BROKER_PORT="$BROKER_PORT"
export ZC_E2E_WORKER_PORT="$WORKER_PORT"
export ZC_E2E_IMAGE="$IMAGE"
export ZC_E2E_ZC_BIN="$ZC_BIN"

# --- phase one: spawn, crash inside the container, restart ------------------
step "phase 1/2 — spawn, reach /health, kill the worker INSIDE the container, restart (x3)"
info "asserting: a container-backed worker starts from the argv \`zc share\` uses and answers /health"
info "asserting: killing the worker inside it leaves .State.Running true — the case a naive check misses"
info "asserting: the supervisor still notices, and RestartWorker(0) brings a new container run up healthy"
info "asserting: each crash keeps the worker's OWN /tmp/worker.log, not the image's s6 stream"
info "asserting: three crashes in five minutes report a crash loop that can still say why"
cargo test --bin zc -- --ignored --exact --nocapture "$PHASE1" \
  || fail "phase 1: the supervisor did not spawn/restart the container worker as asserted"
ok "phase 1"

assert "phase 1 left its container running for the restart test" \
  bash -c "[ \"\$(docker inspect -f '{{.State.Running}}' ${CONTAINER_PREFIX}-w0)\" = true ]"
assert "phase 1 left a children.json for the next agent to adopt" \
  test -s "$SCRATCH/children.json"
info "children.json: $(cat "$SCRATCH/children.json")"

# --- phase two: a different process re-adopts it ----------------------------
step "phase 2/2 — a NEW process re-adopts that container and broker, then tears everything down"
info "asserting: a fresh Supervisor adopts the same container run, and the broker with it"
info "asserting: its first reconcile tick starts nothing a second time"
info "asserting: terminate_all leaves no container and nothing on either port"
cargo test --bin zc -- --ignored --exact --nocapture "$PHASE2" \
  || fail "phase 2: the restarted agent did not re-adopt / tear down as asserted"
ok "phase 2"

# --- teardown, asserted from outside the process that did it ----------------
step "teardown, checked from outside"
assert "no container of this run is left behind" bash -c "[ -z \"\$(docker ps -a --format '{{.Names}}' | grep -E '^${CONTAINER_PREFIX}' || true)\" ]"
assert "nothing is left listening on the worker port $WORKER_PORT" bash -c "! lsof -nP -iTCP:$WORKER_PORT -sTCP:LISTEN >/dev/null 2>&1"
assert "nothing is left listening on the broker port $BROKER_PORT" bash -c "! lsof -nP -iTCP:$BROKER_PORT -sTCP:LISTEN >/dev/null 2>&1"
assert "no broker process of this run survives" bash -c "[ -z \"\$(pgrep -f '^${ZC_BIN} broker ${BROKER_PORT}\$' || true)\" ]"

# Stated, not asserted away: nothing in zc removes this network. It is created
# idempotently, once per node, by both `zc share`'s container path (#279) and
# the agent's (#301), and neither has a teardown for it. An empty user-defined
# bridge is harmless and gets reused by the next worker, but it IS a leftover,
# and pretending otherwise would be the kind of weakened assertion this script
# is supposed to avoid. The harness removes its own on the way out.
if [ -n "$(our_networks)" ]; then
  printf '\033[33m    note: the per-node docker network %s survives teardown — nothing in zc ever removes it (by design, since #279). This harness removes its own.\033[0m\n' "$NETWORK"
fi

step "all assertions held"
