#!/usr/bin/env bash
# E2E Integration Test for knot-server
# Spins up Qdrant + Neo4j containers, runs server, tests full lifecycle
# with real git clone + indexing using a local fixture bare repo.

set -e
set -u

RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
CYAN='\033[0;36m'
NC='\033[0m'

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
COMPOSE_FILE="$SCRIPT_DIR/docker-compose.e2e.yml"
FIXTURE_DIR="$SCRIPT_DIR/fixtures"
WORKSPACE_DIR="/tmp/knot-e2e-workspace-$$"
SERVER_PORT=18080
SERVER_PID=""
FIXTURE_REPO=""
SIGTEST_REPO=""
DUPTEST_REPO=""

NEO4J_URI="bolt://localhost:17687"
NEO4J_USER="neo4j"
NEO4J_PASSWORD="e2e_test_password"
QDRANT_URL="http://localhost:16334"

echo -e "${GREEN}========================================${NC}"
echo -e "${GREEN}knot-server E2E Integration Test${NC}"
echo -e "${GREEN}========================================${NC}"

cleanup() {
    local exit_code=$?
    if [ -n "$SERVER_PID" ] && kill -0 "$SERVER_PID" 2>/dev/null; then
        kill "$SERVER_PID" 2>/dev/null || true
        wait "$SERVER_PID" 2>/dev/null || true
    fi
    cd "$SCRIPT_DIR"
    docker compose -f "$COMPOSE_FILE" down -v 2>/dev/null || true
    rm -rf "$WORKSPACE_DIR" 2>/dev/null || true
    rm -rf "$LOCAL_SOURCE_ROOT" 2>/dev/null || true
    cp /tmp/knot-server-e2e.log "$SCRIPT_DIR/.last-e2e-server.log" 2>/dev/null || true
    rm -f /tmp/knot-server-e2e.log
    cp "$RECOVERY_LOG_R" "$SCRIPT_DIR/.last-recovery-r.log" 2>/dev/null || true
    rm -f "$RECOVERY_LOG_Q" "$RECOVERY_LOG_R"
    if [ $exit_code -ne 0 ]; then
        echo -e "\n${RED}Tests failed!${NC}"
    fi
    exit $exit_code
}
trap cleanup EXIT INT TERM

wait_for_port() {
    local port="$1"
    local label="$2"
    local max_wait="${3:-60}"
    echo -n "  Waiting for $label (port $port)..."
    for i in $(seq 1 "$max_wait"); do
        if nc -z localhost "$port" 2>/dev/null; then
            echo -e " ${GREEN}ready${NC}"
            return 0
        fi
        sleep 1
    done
    echo -e " ${RED}timeout${NC}"
    return 1
}

# Create local bare git repos with fixture source files.
# Knot-indexer needs parseable source files to produce results.
# Produces: $FIXTURE_REPO (main), $SIGTEST_REPO, $DUPTEST_REPO
create_fixture_repo() {
    local bare_path="$WORKSPACE_DIR/fixtures/test-repo.git"
    local work_path="$WORKSPACE_DIR/fixtures-tmp"

    rm -rf "$bare_path" "$work_path"
    mkdir -p "$(dirname "$bare_path")"

    git init --bare "$bare_path"
    git clone "$bare_path" "$work_path"

    # Copy fixture source files into the working tree
    cp "$FIXTURE_DIR"/*.java "$work_path/" 2>/dev/null || true

    # Create a minimal README
    echo "# E2E Test Repository" > "$work_path/README.md"

    cd "$work_path"
    git add .
    git commit -m "initial commit with fixture sources"
    git push origin main 2>/dev/null || { git branch -M main && git push origin main; }
    cd "$SCRIPT_DIR"

    rm -rf "$work_path"
    FIXTURE_REPO="$bare_path"

    # Create extra bare repos for webhook and duplicate tests.
    # Clone the existing bare repo so they have real content.
    SIGTEST_REPO="$WORKSPACE_DIR/fixtures/sigtest.git"
    DUPTEST_REPO="$WORKSPACE_DIR/fixtures/dup-test.git"
    rm -rf "$SIGTEST_REPO" "$DUPTEST_REPO"
    git clone --bare "$FIXTURE_REPO" "$SIGTEST_REPO"
    git clone --bare "$FIXTURE_REPO" "$DUPTEST_REPO"
}

# -------------------------------------------------------
# Step 1: Start Docker containers
# -------------------------------------------------------
echo -e "${YELLOW}[1/7] Starting Docker containers...${NC}"
cd "$SCRIPT_DIR"
docker compose -f "$COMPOSE_FILE" down -v 2>/dev/null || true
docker compose -f "$COMPOSE_FILE" up -d

# -------------------------------------------------------
# Step 2: Wait for databases
# -------------------------------------------------------
echo -e "${YELLOW}[2/7] Waiting for databases...${NC}"
wait_for_port 17687 "Neo4j" 60
wait_for_port 16334 "Qdrant" 30

echo -n "  Waiting for Neo4j health check..."
for i in $(seq 1 30); do
    STATUS=$(docker inspect --format='{{.State.Health.Status}}' knot_server_neo4j_e2e 2>/dev/null || echo "unknown")
    if [ "$STATUS" = "healthy" ]; then
        echo -e " ${GREEN}healthy${NC}"
        break
    fi
    if [ "$i" -eq 30 ]; then
        echo -e " ${RED}timeout (status: $STATUS)${NC}"
        exit 1
    fi
    sleep 1
done
sleep 3

# -------------------------------------------------------
# Step 3: Create fixture repo + start server
# -------------------------------------------------------
echo -e "${YELLOW}[3/7] Creating fixture bare repo + building server...${NC}"
cd "$PROJECT_ROOT"

rm -rf "$WORKSPACE_DIR"
mkdir -p "$WORKSPACE_DIR"
create_fixture_repo
echo "  Fixture repo: $FIXTURE_REPO"

# Share fastembed cache across tests and CI runs to avoid HF 429 rate limits
mkdir -p /tmp/fastembed_cache_shared
ln -s /tmp/fastembed_cache_shared "$WORKSPACE_DIR/fastembed_cache"

cargo build 2>&1 | grep -E "(Compiling|Finished|error)" || true

KNOT_SERVER_QDRANT_URL="$QDRANT_URL" \
KNOT_SERVER_NEO4J_URI="$NEO4J_URI" \
KNOT_SERVER_NEO4J_USER="$NEO4J_USER" \
KNOT_NEO4J_PASSWORD="$NEO4J_PASSWORD" \
KNOT_SERVER_PORT="$SERVER_PORT" \
KNOT_WORKSPACE_DIR="$WORKSPACE_DIR" \
KNOT_SERVER_QUEUE_CAPACITY="${KNOT_SERVER_QUEUE_CAPACITY:-4}" \
RUST_LOG="${RUST_LOG:-info}" \
   "$PROJECT_ROOT/target/debug/knot-server" >/tmp/knot-server-e2e.log 2>&1 &
SERVER_PID=$!

echo "Waiting for knot-server on port $SERVER_PORT..."
for i in $(seq 1 30); do
    if curl -sf "http://localhost:$SERVER_PORT/api/repos" > /dev/null 2>&1; then
        echo -e "${GREEN}knot-server is ready${NC}"
        break
    fi
    if [ "$i" -eq 30 ]; then
        echo -e "${RED}ERROR: knot-server did not start within 30s${NC}"
        exit 1
    fi
    sleep 1
done

BASE_URL="http://localhost:$SERVER_PORT"

# -------------------------------------------------------
# Step 4: Register repo + wait for indexing to complete
# -------------------------------------------------------
echo -e "${YELLOW}[4/7] Register repository + wait for indexing...${NC}"

# Test B: Register the repo with the local fixture bare repo URL
echo -e "\n${CYAN}Test B: Register repository${NC}"
REGISTER_CODE=$(curl -sf -w "%{http_code}" -o /tmp/register_body.json \
    -X POST "$BASE_URL/api/repos" \
    -H "Content-Type: application/json" \
    -d "{\"url\": \"$FIXTURE_REPO\", \"auth_type\": \"ssh\"}")
if [ "$REGISTER_CODE" = "202" ]; then
    echo -e "${GREEN}PASS${NC} — 202 Accepted"
else
    echo -e "${RED}FAIL${NC} — expected 202, got $REGISTER_CODE"
    cat /tmp/register_body.json
    exit 1
fi
REPO_ID=$(jq -r '.id' /tmp/register_body.json)
echo "  Repo ID: $REPO_ID"

# Test C: Get repo immediately (should exist, status may be indexing or error)
echo -e "\n${CYAN}Test C: Get repository${NC}"
GET=$(curl -sf "$BASE_URL/api/repos/$REPO_ID")
if echo "$GET" | jq -e ".id == \"$REPO_ID\"" > /dev/null 2>&1; then
    echo -e "${GREEN}PASS${NC} — repo details returned"
else
    echo -e "${RED}FAIL${NC}"
    echo "$GET"
    exit 1
fi

# Test D: Wait for indexing to complete (status=indexed, last_indexed not null)
echo -e "\n${CYAN}Test D: Wait for indexing completion${NC}"
INDEXED_OK=false
for i in $(seq 1 60); do
    REPO_JSON=$(curl -sf "$BASE_URL/api/repos/$REPO_ID")
    STATUS=$(echo "$REPO_JSON" | jq -r '.status')
    LAST_INDEXED=$(echo "$REPO_JSON" | jq -r '.last_indexed')

    if [ "$STATUS" = "indexed" ] && [ "$LAST_INDEXED" != "null" ] && [ -n "$LAST_INDEXED" ]; then
        echo -e "${GREEN}PASS${NC} — indexing complete (last_indexed: $LAST_INDEXED)"
        INDEXED_OK=true
        break
    elif [ "$STATUS" = "error" ]; then
        echo -e "${RED}FAIL${NC} — indexing failed with error status"
        echo "Server log tail:"
        tail -30 /tmp/knot-server-e2e.log 2>/dev/null || true
        exit 1
    fi
    if [ "$i" -eq 60 ]; then
        echo -e "${RED}FAIL${NC} — indexing did not complete within 60s (status: $STATUS)"
        echo "Server log tail:"
        tail -30 /tmp/knot-server-e2e.log 2>/dev/null || true
        exit 1
    fi
    sleep 1
done

# -------------------------------------------------------
# Step 5: Query tests (only run if indexing succeeded)
# -------------------------------------------------------
echo -e "${YELLOW}[5/7] Query tests...${NC}"

if [ "$INDEXED_OK" = "true" ]; then
    # Test E: Semantic search finds UserService
    echo -e "\n${CYAN}Test E: Search finds UserService${NC}"
    SEARCH=$(curl -sf "$BASE_URL/api/repos/$REPO_ID/search?q=UserService")
    if echo "$SEARCH" | jq -e '.' > /dev/null 2>&1; then
        echo -e "${GREEN}PASS${NC} — search returned valid JSON"
    else
        echo -e "${RED}FAIL${NC} — invalid JSON response"
        echo "$SEARCH"
        exit 1
    fi

    # ── Test G1: Graph endpoint with real data ──
    echo -e "\n${CYAN}Test G1: Graph endpoint returns subgraph for indexed entity${NC}"
    RESP=$(curl -s "http://localhost:${SERVER_PORT}/api/repos/${REPO_ID}/graph?entity=UserService")
    ROOT_ID=$(echo "$RESP" | jq -r '.root_id')
    NODE_COUNT=$(echo "$RESP" | jq '.nodes | length')
    HAS_USER_SERVICE=$(echo "$RESP" | jq '[.nodes[] | select(.name == "UserService")] | length')

    if [ "$ROOT_ID" != "null" ] && [ "$NODE_COUNT" -ge 1 ] && [ "$HAS_USER_SERVICE" -ge 1 ]; then
      echo -e "${GREEN}PASS${NC}: root_id=$ROOT_ID, nodes=$NODE_COUNT"
    else
      echo -e "${RED}FAIL${NC}: root_id=$ROOT_ID, nodes=$NODE_COUNT, has_UserService=$HAS_USER_SERVICE"
      echo "Response: $RESP"
      exit 1
    fi

    # ── Test G2: Graph endpoint with depth and relationships params ──
    echo -e "\n${CYAN}Test G2: Graph endpoint with depth and relationships params${NC}"
    CODE=$(curl -s -w "%{http_code}" -o /tmp/g2.json \
      "http://localhost:${SERVER_PORT}/api/repos/${REPO_ID}/graph?entity=UserService&depth=3&relationships=CALLS,CONTAINS")
    NODE_COUNT=$(jq '.nodes | length' /tmp/g2.json)

    if [ "$CODE" = "200" ] && [ "$NODE_COUNT" -ge 1 ]; then
      echo -e "${GREEN}PASS${NC}: status=$CODE, nodes=$NODE_COUNT"
    else
      echo -e "${RED}FAIL${NC}: status=$CODE, nodes=$NODE_COUNT"
      cat /tmp/g2.json
      exit 1
    fi

    # ── Test G3: Graph endpoint with direction=outgoing ──
    echo -e "\n${CYAN}Test G3: Graph endpoint with direction=outgoing${NC}"
    CODE=$(curl -s -w "%{http_code}" -o /tmp/g3.json \
      "http://localhost:${SERVER_PORT}/api/repos/${REPO_ID}/graph?entity=UserService&direction=outgoing")

    if [ "$CODE" = "200" ] && jq -e '.nodes | length >= 1' /tmp/g3.json > /dev/null; then
      echo -e "${GREEN}PASS${NC}: status=$CODE"
    else
      echo -e "${RED}FAIL${NC}: status=$CODE"
      cat /tmp/g3.json
      exit 1
    fi

    # ── Test G4: Graph endpoint with nonexistent entity returns empty ──
    echo -e "\n${CYAN}Test G4: Graph endpoint with nonexistent entity returns empty${NC}"
    RESP=$(curl -s "http://localhost:${SERVER_PORT}/api/repos/${REPO_ID}/graph?entity=NonExistentEntity12345")
    NODE_COUNT=$(echo "$RESP" | jq '.nodes | length')
    TOTAL=$(echo "$RESP" | jq '.total_nodes_found')

    if [ "$NODE_COUNT" = "0" ] && [ "$TOTAL" = "0" ]; then
      echo -e "${GREEN}PASS${NC}: empty result as expected"
    else
      echo -e "${RED}FAIL${NC}: expected empty, got nodes=$NODE_COUNT total=$TOTAL"
      exit 1
    fi

    # ── Test G5: Graph expand endpoint returns data ──
    echo -e "\n${CYAN}Test G5: Graph expand endpoint returns data${NC}"
    CODE=$(curl -s -w "%{http_code}" -o /tmp/g5.json \
      "http://localhost:${SERVER_PORT}/api/repos/${REPO_ID}/graph/expand?entity=UserService")
    NODE_COUNT=$(jq '.nodes | length' /tmp/g5.json)

    if [ "$CODE" = "200" ] && [ "$NODE_COUNT" -ge 1 ]; then
      echo -e "${GREEN}PASS${NC}: status=$CODE, nodes=$NODE_COUNT"
    else
      echo -e "${RED}FAIL${NC}: status=$CODE, nodes=$NODE_COUNT"
      cat /tmp/g5.json
      exit 1
    fi

    # ── Test G6: Graph expand with exclude filters nodes ──
    echo -e "\n${CYAN}Test G6: Graph expand with exclude filters nodes${NC}"
    US_UUID=$(curl -s "http://localhost:${SERVER_PORT}/api/repos/${REPO_ID}/graph?entity=UserService" \
      | jq -r '[.nodes[] | select(.name == "UserService")][0].id')

    RESP=$(curl -s "http://localhost:${SERVER_PORT}/api/repos/${REPO_ID}/graph/expand?entity=UserService&exclude=${US_UUID}")
    EXCLUDED=$(echo "$RESP" | jq --arg uuid "$US_UUID" '[.nodes[] | select(.id == $uuid)] | length')

    if [ "$EXCLUDED" = "0" ]; then
      echo -e "${GREEN}PASS${NC}: excluded node not in result"
    else
      echo -e "${RED}FAIL${NC}: excluded node still present"
      echo "Response: $RESP"
      exit 1
    fi

    # ── Test G7: Graph viewer HTML is served ──
    echo -e "\n${CYAN}Test G7: Graph viewer HTML is served${NC}"
    CODE=$(curl -s -w "%{http_code}" -o /tmp/g7.html "http://localhost:${SERVER_PORT}/graph")
    HAS_DOCTYPE=$(grep -c '<!DOCTYPE html>' /tmp/g7.html || true)
    HAS_FORCE=$(grep -c 'ForceGraph3D' /tmp/g7.html || true)

    if [ "$CODE" = "200" ] && [ "$HAS_DOCTYPE" -ge 1 ] && [ "$HAS_FORCE" -ge 1 ]; then
      echo -e "${GREEN}PASS${NC}: HTML viewer served correctly"
    else
      echo -e "${RED}FAIL${NC}: status=$CODE, doctype=$HAS_DOCTYPE, forcegraph=$HAS_FORCE"
      exit 1
    fi

    # ── Test G8: Graph endpoint rejects invalid relationship type ──
    echo -e "\n${CYAN}Test G8: Graph endpoint rejects invalid relationship type${NC}"
    CODE=$(curl -s -w "%{http_code}" -o /tmp/g8.json \
      "http://localhost:${SERVER_PORT}/api/repos/${REPO_ID}/graph?entity=UserService&relationships=INVALID_REL")

    if [ "$CODE" = "400" ] && jq -e '.error' /tmp/g8.json > /dev/null; then
      echo -e "${GREEN}PASS${NC}: invalid relationship rejected with 400 Bad Request"
    else
      echo -e "${RED}FAIL${NC}: status=$CODE"
      cat /tmp/g8.json
      exit 1
    fi

    # ── Test G9: Graph node response schema validation ──
    echo -e "\n${CYAN}Test G9: Graph node response schema validation${NC}"
    RESP=$(curl -s "http://localhost:${SERVER_PORT}/api/repos/${REPO_ID}/graph?entity=UserService")
    NODE_COUNT=$(echo "$RESP" | jq '.nodes | length')
    FULL_NODES=$(echo "$RESP" | jq '[.nodes[] | select(has("id") and has("name") and has("kind") and has("language") and has("file_path") and has("start_line") and has("signature"))] | length')

    if [ "$FULL_NODES" = "$NODE_COUNT" ] && [ "$NODE_COUNT" -ge 1 ]; then
      echo -e "${GREEN}PASS${NC}: all $NODE_COUNT nodes match schema"
    else
      echo -e "${RED}FAIL${NC}: schema validation failed ($FULL_NODES / $NODE_COUNT)"
      echo "$RESP" | jq '.nodes[0]'
      exit 1
    fi

    # ── Test G10: Graph overview — all entities without entity param ──
    echo -e "\n${CYAN}Test G10: Graph overview returns all entities (no entity param, default relationships)${NC}"
    CODE=$(curl -s -w "%{http_code}" -o /tmp/g10.json \
      "http://localhost:${SERVER_PORT}/api/repos/${REPO_ID}/graph")
    NODE_COUNT=$(jq '.nodes | length' /tmp/g10.json)
    EDGE_COUNT=$(jq '.edges | length' /tmp/g10.json)
    ROOT_ID=$(jq -r '.root_id' /tmp/g10.json)

    # Default overview uses CALLS,EXTENDS,IMPLEMENTS (no CONTAINS traversal).
    # For a tiny fixture repo, only root entities are returned.
    if [ "$CODE" = "200" ] && [ "$NODE_COUNT" -ge 1 ] && [ "$ROOT_ID" = "null" ]; then
      echo -e "${GREEN}PASS${NC}: status=$CODE, nodes=$NODE_COUNT, edges=$EDGE_COUNT, root_id=null"
    else
      echo -e "${RED}FAIL${NC}: status=$CODE, nodes=$NODE_COUNT, edges=$EDGE_COUNT, root_id=$ROOT_ID"
      cat /tmp/g10.json
      exit 1
    fi

    # ── Test G11: Overview with explicit relationships including CONTAINS ──
    echo -e "\n${CYAN}Test G11: Graph overview with explicit relationships including CONTAINS${NC}"
    CODE=$(curl -s -w "%{http_code}" -o /tmp/g11.json \
      "http://localhost:${SERVER_PORT}/api/repos/${REPO_ID}/graph?relationships=CALLS,CONTAINS")
    G11_NODES=$(jq '.nodes | length' /tmp/g11.json)
    G11_EDGES=$(jq '.edges | length' /tmp/g11.json)
    G10_NODES=$(jq '.nodes | length' /tmp/g10.json)
    ROOT_ID=$(jq -r '.root_id' /tmp/g11.json)

    # CONTAINS filters which nodes are considered roots (excludes nodes that are
    # contained by something). For a single-file fixture with no CONTAINS edges,
    # the result may be the same size or different depending on graph structure.
    if [ "$CODE" = "200" ] && [ "$G11_NODES" -ge 1 ] && [ "$ROOT_ID" = "null" ]; then
      echo -e "${GREEN}PASS${NC}: CALLS,CONTAINS (nodes=$G11_NODES, edges=$G11_EDGES) returns valid overview"
    else
      echo -e "${RED}FAIL${NC}: status=$CODE, CALLS+CONTAINS_nodes=$G11_NODES, default_nodes=$G10_NODES, root_id=$ROOT_ID"
      cat /tmp/g11.json
      exit 1
    fi

    # ── Test G12: Overview rejects invalid relationship type ──
    echo -e "\n${CYAN}Test G12: Graph overview rejects invalid relationship type${NC}"
    CODE=$(curl -s -w "%{http_code}" -o /tmp/g12.json \
      "http://localhost:${SERVER_PORT}/api/repos/${REPO_ID}/graph?relationships=INVALID,BAD")

    if [ "$CODE" = "400" ] && jq -e '.error' /tmp/g12.json > /dev/null; then
      echo -e "${GREEN}PASS${NC}: overview invalid relationship rejected with 400"
    else
      echo -e "${RED}FAIL${NC}: status=$CODE"
      cat /tmp/g12.json
      exit 1
    fi

    # ── Test G13: Expand endpoint rejects invalid relationship type ──
    echo -e "\n${CYAN}Test G13: Graph expand rejects invalid relationship type${NC}"
    CODE=$(curl -s -w "%{http_code}" -o /tmp/g13.json \
      "http://localhost:${SERVER_PORT}/api/repos/${REPO_ID}/graph/expand?entity=UserService&relationships=INVALID_REL")

    if [ "$CODE" = "400" ] && jq -e '.error' /tmp/g13.json > /dev/null; then
      echo -e "${GREEN}PASS${NC}: expand invalid relationship rejected with 400"
    else
      echo -e "${RED}FAIL${NC}: status=$CODE"
      cat /tmp/g13.json
      exit 1
    fi

    # Test F: List repos shows status indexed
    echo -e "\n${CYAN}Test F: List contains indexed repo${NC}"
    LIST=$(curl -sf "$BASE_URL/api/repos")
    if echo "$LIST" | jq -e ".repositories[] | select(.id == \"$REPO_ID\" and .status == \"indexed\")" > /dev/null 2>&1; then
        echo -e "${GREEN}PASS${NC} — repo has status indexed"
    else
        echo -e "${RED}FAIL${NC} — repo status is not indexed"
        echo "$LIST" | jq ".repositories[] | {id, status, last_indexed}"
        exit 1
    fi

    # Test G: Health endpoint
    echo -e "\n${CYAN}Test G: Health endpoint${NC}"
    HEALTH=$(curl -sf "$BASE_URL/api/health")
    if echo "$HEALTH" | jq -e '.status == "ok"' > /dev/null 2>&1; then
        echo -e "${GREEN}PASS${NC} — health check returns ok"
        REPO_COUNT=$(echo "$HEALTH" | jq -r '.repositories_total')
        echo "  Repositories: $REPO_COUNT"
    else
        echo -e "${RED}FAIL${NC}"
        echo "$HEALTH"
        exit 1
    fi
else
    echo -e "${YELLOW}  Skipping query tests (indexing did not complete)${NC}"
fi

# -------------------------------------------------------
# Step 6: Error handling tests
# -------------------------------------------------------
echo -e "${YELLOW}[6/7] Error handling tests...${NC}"

# Test H: Search without query → 400
echo -e "\n${CYAN}Test H: Missing query param → 400${NC}"
CODE=$(curl -s -w "%{http_code}" -o /dev/null "$BASE_URL/api/repos/ghost/search")
if [ "$CODE" = "400" ]; then echo -e "${GREEN}PASS${NC}"; else echo -e "${RED}FAIL${NC} — got $CODE"; exit 1; fi

# Test I: Delete nonexistent → 404
echo -e "\n${CYAN}Test I: Delete nonexistent → 404${NC}"
CODE=$(curl -s -w "%{http_code}" -o /dev/null -X DELETE "$BASE_URL/api/repos/ghost")
if [ "$CODE" = "404" ]; then echo -e "${GREEN}PASS${NC}"; else echo -e "${RED}FAIL${NC} — got $CODE"; exit 1; fi

# Test J: Webhook without signature → 401
# test-repo exists but has no webhook_secret configured → 401
echo -e "\n${CYAN}Test J: Unsigned webhook → 401${NC}"
CODE=$(curl -s -w "%{http_code}" -o /dev/null -X POST "$BASE_URL/api/webhook/$REPO_ID" -H "Content-Type: application/json" -d '{}')
if [ "$CODE" = "401" ]; then echo -e "${GREEN}PASS${NC}"; else echo -e "${RED}FAIL${NC} — got $CODE"; exit 1; fi

# Test K: Webhook with valid GitLab token → 202
# Register a dedicated repo with webhook_secret for this test
echo -e "\n${CYAN}Test K: Valid GitLab webhook → 202${NC}"
SIGTEST_ID=$(curl -s -X POST "$BASE_URL/api/repos" \
    -H "Content-Type: application/json" \
    -d "{\"url\": \"$SIGTEST_REPO\", \"auth_type\": \"ssh\", \"webhook_secret\": \"test-secret-123\"}" | jq -r '.id')
CODE=$(curl -s -w "%{http_code}" -o /dev/null -X POST "$BASE_URL/api/webhook/$SIGTEST_ID" \
    -H "Content-Type: application/json" \
    -H "X-Gitlab-Token: test-secret-123" \
    -d '{"ref":"refs/heads/main"}')
if [ "$CODE" = "202" ]; then echo -e "${GREEN}PASS${NC}"; else echo -e "${RED}FAIL${NC} — got $CODE"; exit 1; fi

# Test L: Duplicate registration → 202 (upsert / re-register)
echo -e "\n${CYAN}Test L: Duplicate registration → 202 (upsert)${NC}"
curl -s -o /dev/null -X POST "$BASE_URL/api/repos" \
    -H "Content-Type: application/json" \
    -d "{\"url\": \"$DUPTEST_REPO\", \"auth_type\": \"ssh\"}"
RESP=$(curl -s -w "\n%{http_code}" -X POST "$BASE_URL/api/repos" \
    -H "Content-Type: application/json" \
    -d "{\"url\": \"$DUPTEST_REPO\", \"auth_type\": \"ssh\"}")
CODE=$(echo "$RESP" | tail -1)
BODY=$(echo "$RESP" | sed '$d')
MSG=$(echo "$BODY" | jq -r '.message // ""')
if [ "$CODE" = "202" ] && [ "$MSG" = "Repository re-registered successfully" ]; then
    echo -e "${GREEN}PASS${NC}"
else
    echo -e "${RED}FAIL${NC} — got $CODE, message: $MSG"; exit 1
fi

# Test M: Manual sync → 202
echo -e "\n${CYAN}Test M: Manual sync → 202${NC}"
# Brief pause to let worker drain the queue from previous tests
sleep 1
SYNC_CODE=$(curl -s -w "%{http_code}" -o /dev/null -X POST "$BASE_URL/api/repos/$REPO_ID/sync")
if [ "$SYNC_CODE" = "202" ]; then echo -e "${GREEN}PASS${NC}"; else echo -e "${RED}FAIL${NC} — got $SYNC_CODE"; exit 1; fi

# Wait for the sync job to complete before deleting
echo -e "\n${CYAN}  Waiting for sync to complete...${NC}"
for i in $(seq 1 30); do
    REPO_JSON=$(curl -sf "$BASE_URL/api/repos/$REPO_ID")
    STATUS=$(echo "$REPO_JSON" | jq -r '.status')
    if [ "$STATUS" = "indexed" ]; then
        echo -e "  ${GREEN}sync complete (status: indexed)${NC}"
        break
    elif [ "$STATUS" = "error" ]; then
        echo -e "  ${YELLOW}sync returned error (acceptable for unchanged repo, continuing)${NC}"
        break
    fi
    if [ "$i" -eq 30 ]; then
        echo -e "  ${YELLOW}sync still running after 30s, continuing anyway${NC}"
    fi
    sleep 1
done

# Test N: Delete repo
echo -e "\n${CYAN}Test N: Delete repository${NC}"
DEL_CODE=$(curl -s -w "%{http_code}" -o /dev/null -X DELETE "$BASE_URL/api/repos/$REPO_ID")
if [ "$DEL_CODE" = "200" ]; then
    echo -e "${GREEN}PASS${NC} — 200 OK"
else
    echo -e "${RED}FAIL${NC} — expected 200, got $DEL_CODE"
    exit 1
fi

# Verify deletion
LIST=$(curl -sf "$BASE_URL/api/repos")
if echo "$LIST" | jq -e ".repositories[] | select(.id == \"$REPO_ID\")" > /dev/null 2>&1; then
    echo -e "${RED}FAIL${NC} — repo still present after deletion"
    exit 1
else
    echo -e "${GREEN}PASS${NC} — repo removed from listing"
fi

# Test O: Verify database cleanup after delete
echo -e "\n${CYAN}Test O: Neo4j/Qdrant cleanup after delete${NC}"
NEO4J_PASS="${NEO4J_PASSWORD:-e2e_test_password}"
NEO4J_PORT="${NEO4J_PORT:-17687}"

check_neo4j_cleanup() {
    local container="$1"
    local output
    output=$(docker exec "$container" cypher-shell -u neo4j -p "$NEO4J_PASS" \
        "MATCH (e:Entity {repo_name: '$REPO_ID'}) RETURN count(e) AS cnt" 2>/dev/null) || return 1
    echo "$output" | grep -o '[0-9]\+' | head -1
}

NEO4J_COUNT=""
for container in knot_server_neo4j_e2e knot-server-neo4j-1; do
    if docker ps --format '{{.Names}}' | grep -q "^${container}$"; then
        NEO4J_COUNT=$(check_neo4j_cleanup "$container")
        if [ -n "$NEO4J_COUNT" ]; then
            break
        fi
    fi
done

if [ -z "$NEO4J_COUNT" ]; then
    echo -e "  ${YELLOW}SKIP${NC} — cypher-shell not available or container not running"
elif [ "$NEO4J_COUNT" -eq 0 ]; then
    echo -e "${GREEN}PASS${NC} — Neo4j entities cleaned ($NEO4J_COUNT remaining)"
else
    echo -e "${RED}FAIL${NC} — Neo4j has $NEO4J_COUNT remaining entities for '$REPO_ID'"
    exit 1
fi

# Clean up secondary repos from Test K
curl -s -o /dev/null -X DELETE "$BASE_URL/api/repos/$SIGTEST_ID" 2>/dev/null

# Test P: Queue capacity configured correctly
echo -e "\n${CYAN}Test P: Queue capacity from env var${NC}"
QCAP=$(curl -sf "$BASE_URL/api/health" | jq -r '.queue_capacity')
if [ "$QCAP" = "4" ]; then
    echo -e "${GREEN}PASS${NC} — queue_capacity=$QCAP from KNOT_SERVER_QUEUE_CAPACITY"
else
    echo -e "${RED}FAIL${NC} — expected queue_capacity=4, got $QCAP"
    exit 1
fi

# -------------------------------------------------------
# Step 7: Indexing state recovery regression tests
# -------------------------------------------------------
# Regression tests for two bugs:
#   Q) "Pending status eternally": a repo left in Pending (e.g. server killed
#      before its initial Clone job ran) was never re-enqueued.
#   R) "Indexing status abruptly cut": a repo left mid-indexing (Cloning /
#      Pulling / Indexing) after a crash had no active recovery — it only
#      recovered if a stale .knot.lock file happened to survive the crash.
# Both bugs are fixed by the startup recovery loop in src/main.rs, which
# re-enqueues any non-terminal repo on restart. The tests below simulate
# the crash by stopping the server, rewriting repos.json with a stuck
# entry, and restarting. The server MUST recover the repo to "indexed".
echo -e "${YELLOW}[7/8] Indexing state recovery regression tests...${NC}"

RECOVERY_LOG_Q="/tmp/knot-server-recovery-q-$$.log"
RECOVERY_LOG_R="/tmp/knot-server-recovery-r-$$.log"

stop_current_server() {
    if [ -n "$SERVER_PID" ] && kill -0 "$SERVER_PID" 2>/dev/null; then
        kill "$SERVER_PID" 2>/dev/null || true
        wait "$SERVER_PID" 2>/dev/null || true
    fi
    SERVER_PID=""
}

start_recovery_server() {
    local logfile="$1"
    KNOT_SERVER_QDRANT_URL="$QDRANT_URL" \
    KNOT_SERVER_NEO4J_URI="$NEO4J_URI" \
    KNOT_SERVER_NEO4J_USER="$NEO4J_USER" \
    KNOT_NEO4J_PASSWORD="$NEO4J_PASSWORD" \
    KNOT_SERVER_PORT="$SERVER_PORT" \
    KNOT_WORKSPACE_DIR="$WORKSPACE_DIR" \
    KNOT_SERVER_POLL_INTERVAL_SECS=2 \
    KNOT_SERVER_STALE_LOCK_TIMEOUT_SECS=2 \
    KNOT_SERVER_QUEUE_CAPACITY=4 \
    RUST_LOG=info \
        "$PROJECT_ROOT/target/debug/knot-server" > "$logfile" 2>&1 &
    SERVER_PID=$!

    for i in $(seq 1 30); do
        if curl -sf "http://localhost:$SERVER_PORT/api/repos" > /dev/null 2>&1; then
            return 0
        fi
        sleep 1
    done
    return 1
}

wait_for_recovery_indexed() {
    local repo_id="$1"
    local logfile="$2"
    for i in $(seq 1 60); do
        local status
        status=$(curl -sf "$BASE_URL/api/repos/$repo_id" 2>/dev/null | jq -r '.status' 2>/dev/null || echo "")
        if [ "$status" = "indexed" ]; then
            return 0
        elif [ "$status" = "error" ]; then
            echo -e "${RED}FAIL${NC} — recovery set repo to error"
            tail -30 "$logfile"
            return 2
        fi
        sleep 1
    done
    return 1
}

# Stop the currently running server (left over from Step 6).
stop_current_server

# ── Test Q: Pending status recovery on restart ────────────────
# Simulates a server that was killed before its initial Clone job ran
# (e.g. crash between POST /api/repos and the worker picking up the job).
# After restart, the startup recovery loop must enqueue a Clone job and
# the repo MUST reach "indexed".
echo -e "\n${CYAN}Test Q: Pending status recovery on restart${NC}"

RECOVERY_Q_ID="recovery-pending-bug"
RECOVERY_Q_PATH="$WORKSPACE_DIR/$RECOVERY_Q_ID"
rm -rf "$RECOVERY_Q_PATH"

# Inject a single Pending repo into repos.json. No local_path yet —
# the recovery code will enqueue a Clone (not Pull) for this case.
cat > "$WORKSPACE_DIR/repos.json" <<EOF
{
  "repositories": [
    {
      "id": "$RECOVERY_Q_ID",
      "url": "$FIXTURE_REPO",
      "auth_type": "ssh",
      "local_path": "$RECOVERY_Q_PATH",
      "branch": "main",
      "status": "pending"
    }
  ]
}
EOF

if ! start_recovery_server "$RECOVERY_LOG_Q"; then
    echo -e "${RED}FAIL${NC} — server did not start"
    cat "$RECOVERY_LOG_Q"
    exit 1
fi
echo "  Server restarted, waiting for recovery to index '$RECOVERY_Q_ID'..."

if wait_for_recovery_indexed "$RECOVERY_Q_ID" "$RECOVERY_LOG_Q"; then
    echo -e "${GREEN}PASS${NC} — Pending repo recovered to 'indexed' after restart"
else
    FINAL=$(curl -sf "$BASE_URL/api/repos/$RECOVERY_Q_ID" 2>/dev/null | jq -r '.status' 2>/dev/null || echo "unknown")
    echo -e "${RED}FAIL${NC} — Pending repo stuck (final status: $FINAL)"
    echo "Server log (last 40 lines):"
    tail -40 "$RECOVERY_LOG_Q"
    exit 1
fi

# Verify the startup recovery log message is present
if grep -q "Recovering stuck repo '$RECOVERY_Q_ID' (was pending)" "$RECOVERY_LOG_Q"; then
    echo -e "${GREEN}PASS${NC} — startup recovery log entry present ('was pending')"
else
    echo -e "${RED}FAIL${NC} — startup recovery log entry missing"
    echo "Server log (last 40 lines):"
    tail -40 "$RECOVERY_LOG_Q"
    exit 1
fi

# ── Test R: Indexing status abrupt cut recovery on restart ────
# Simulates a server killed mid-indexing: the .git dir is present (clone
# already succeeded) but the worker was running the indexing pipeline when
# the process died. After restart, the recovery loop must re-enqueue a
# Pull job and the repo MUST reach "indexed".
echo -e "\n${CYAN}Test R: Indexing status recovery on restart${NC}"

stop_current_server

RECOVERY_R_ID="recovery-indexing-bug"
RECOVERY_R_PATH="$WORKSPACE_DIR/$RECOVERY_R_ID"
rm -rf "$RECOVERY_R_PATH"

# Seed the local path with a real .git dir, as if the clone had finished
# before the crash. No .knot.lock — the worker process released it on
# exit. (A surviving lock would be cleaned by the stale-lock scheduler
# path, not the startup recovery loop under test.)
git clone "$FIXTURE_REPO" "$RECOVERY_R_PATH" > /dev/null 2>&1
if [ ! -d "$RECOVERY_R_PATH/.git" ]; then
    echo -e "${RED}FAIL${NC} — could not seed $RECOVERY_R_PATH/.git"
    exit 1
fi

# Inject the stuck repo (status=indexing, .git present) into repos.json.
cat > "$WORKSPACE_DIR/repos.json" <<EOF
{
  "repositories": [
    {
      "id": "$RECOVERY_R_ID",
      "url": "$FIXTURE_REPO",
      "auth_type": "ssh",
      "local_path": "$RECOVERY_R_PATH",
      "branch": "main",
      "status": "indexing"
    }
  ]
}
EOF

if ! start_recovery_server "$RECOVERY_LOG_R"; then
    echo -e "${RED}FAIL${NC} — server did not start"
    cat "$RECOVERY_LOG_R"
    exit 1
fi
echo "  Server restarted, waiting for recovery to index '$RECOVERY_R_ID'..."

if wait_for_recovery_indexed "$RECOVERY_R_ID" "$RECOVERY_LOG_R"; then
    echo -e "${GREEN}PASS${NC} — Indexing repo recovered to 'indexed' after restart"
else
    FINAL=$(curl -sf "$BASE_URL/api/repos/$RECOVERY_R_ID" 2>/dev/null | jq -r '.status' 2>/dev/null || echo "unknown")
    echo -e "${RED}FAIL${NC} — Indexing repo stuck (final status: $FINAL)"
    echo "Server log (last 40 lines):"
    tail -40 "$RECOVERY_LOG_R"
    exit 1
fi

# Verify the startup recovery log message names the original status
if grep -q "Recovering stuck repo '$RECOVERY_R_ID' (was indexing)" "$RECOVERY_LOG_R"; then
    echo -e "${GREEN}PASS${NC} — startup recovery log entry present ('was indexing')"
else
    echo -e "${RED}FAIL${NC} — startup recovery log entry missing"
    echo "Server log (last 40 lines):"
    tail -40 "$RECOVERY_LOG_R"
    exit 1
fi

# Verify the recovery chose the Pull job (.git present → no dir removal)
if grep -q "Recovering stuck repo '$RECOVERY_R_ID' (was indexing): .git exists, enqueuing Pull" "$RECOVERY_LOG_R"; then
    echo -e "${GREEN}PASS${NC} — recovery correctly chose Pull job (not Clone)"
else
    echo -e "${RED}FAIL${NC} — recovery did not log expected Pull decision"
    tail -40 "$RECOVERY_LOG_R"
    exit 1
fi

cp "$RECOVERY_LOG_R" /tmp/last-recovery-r.log 2>/dev/null || true

# (Do not delete RECOVERY_LOG_R yet — Test S5 still needs to grep it
# for the stale-state-removal log line. The cleanup trap at the bottom
# of the script will remove the file on exit.)

# -------------------------------------------------------
# Step 8: Local working-tree sync — uncommitted changes
# -------------------------------------------------------
# Regression test for the bug where `repo.url` is a local filesystem path:
# the worker used to call `git fetch` (which only sees committed objects),
# so uncommitted working-tree changes in the source were never picked up.
# The fix routes local-path URLs through `local_sync::sync_local_working_tree`,
# which mirrors the source tree into `repo.local_path` so the incremental
# indexer can detect file-level changes. The tests below exercise the full
# add / modify / commit-count flow against a real git working tree.
echo -e "${YELLOW}[8/9] Local working-tree sync regression test...${NC}"

# Place the local source repo OUTSIDE the workspace. The registry
# derives `local_path = workspace/<id>` from the URL's basename, so if
# the source lived inside the workspace the mirror destination would
# equal the source — and `fs::copy(file, file)` truncates the file to
# zero bytes. Keeping the source outside the workspace lets the mirror
# be a distinct directory.
LOCAL_LIVE_PATH="/tmp/knot-e2e-source-$$/local-live-repo"
LOCAL_SOURCE_ROOT="/tmp/knot-e2e-source-$$"
rm -rf "$LOCAL_SOURCE_ROOT"
mkdir -p "$LOCAL_LIVE_PATH"
git init -q "$LOCAL_LIVE_PATH"
git -C "$LOCAL_LIVE_PATH" checkout -q -b main
cp "$FIXTURE_DIR"/*.java "$LOCAL_LIVE_PATH/"
git -C "$LOCAL_LIVE_PATH" add . > /dev/null 2>&1
git -C "$LOCAL_LIVE_PATH" -c user.email=test@test.com -c user.name=Test \
    commit -q -m "initial"
echo "  Created local live repo at $LOCAL_LIVE_PATH with initial commit"

# ── Test S1: Register local live repo and wait for initial indexing ──
echo -e "\n${CYAN}Test S1: Register local live repo + wait for indexing${NC}"
REGISTER_BODY=$(mktemp)
REGISTER_CODE=$(curl -sf -w "%{http_code}" -o "$REGISTER_BODY" \
    -X POST "$BASE_URL/api/repos" \
    -H "Content-Type: application/json" \
    -d "{\"url\": \"$LOCAL_LIVE_PATH\", \"auth_type\": \"ssh\"}")
if [ "$REGISTER_CODE" = "202" ]; then
    echo -e "${GREEN}PASS${NC} — 202 Accepted for local-path url"
else
    echo -e "${RED}FAIL${NC} — expected 202, got $REGISTER_CODE"
    cat "$REGISTER_BODY"
    rm -f "$REGISTER_BODY"
    exit 1
fi
LOCAL_REPO_ID=$(jq -r '.id' "$REGISTER_BODY")
rm -f "$REGISTER_BODY"
echo "  Local repo ID: $LOCAL_REPO_ID"

S_INDEXED_OK=false
for i in $(seq 1 60); do
    REPO_JSON=$(curl -sf "$BASE_URL/api/repos/$LOCAL_REPO_ID" 2>/dev/null || echo "")
    STATUS=$(echo "$REPO_JSON" | jq -r '.status' 2>/dev/null || echo "")
    LAST_INDEXED=$(echo "$REPO_JSON" | jq -r '.last_indexed' 2>/dev/null || echo "")

    if [ "$STATUS" = "indexed" ] && [ "$LAST_INDEXED" != "null" ] && [ -n "$LAST_INDEXED" ]; then
        echo -e "${GREEN}PASS${NC} — local repo indexed (last_indexed: $LAST_INDEXED)"
        S_INDEXED_OK=true
        break
    elif [ "$STATUS" = "error" ]; then
        echo -e "${RED}FAIL${NC} — local repo indexing failed with error status"
        echo "Server log tail:"
        tail -30 /tmp/knot-server-e2e.log 2>/dev/null || true
        exit 1
    fi
    if [ "$i" -eq 60 ]; then
        echo -e "${RED}FAIL${NC} — local repo indexing did not complete within 60s (status: $STATUS)"
        echo "Server log tail:"
        tail -30 /tmp/knot-server-e2e.log 2>/dev/null || true
        exit 1
    fi
    sleep 1
done

# ── Test S2: Add a new uncommitted file, trigger sync, verify it appears ──
echo -e "\n${CYAN}Test S2: Uncommitted new file picked up after sync${NC}"
cat > "$LOCAL_LIVE_PATH/SyncTestService.java" <<'JAVA'
public class SyncTestService {
    public String syncTestMethod() {
        return "hello from uncommitted code";
    }
}
JAVA

# Verify it is genuinely uncommitted (git status shows "??" for untracked)
UNCOMMITTED_PROOF=$(git -C "$LOCAL_LIVE_PATH" status --short | grep "SyncTestService.java" || true)
if [ -z "$UNCOMMITTED_PROOF" ]; then
    echo -e "${RED}FAIL${NC} — SyncTestService.java was not actually uncommitted (git status empty)"
    exit 1
fi
echo "  Confirmed uncommitted: $UNCOMMITTED_PROOF"

SYNC_CODE=$(curl -s -w "%{http_code}" -o /dev/null \
    -X POST "$BASE_URL/api/repos/$LOCAL_REPO_ID/sync")
if [ "$SYNC_CODE" = "202" ]; then
    echo -e "  ${GREEN}sync accepted (202)${NC}"
else
    echo -e "${RED}FAIL${NC} — sync returned $SYNC_CODE (expected 202)"
    exit 1
fi

for i in $(seq 1 60); do
    REPO_JSON=$(curl -sf "$BASE_URL/api/repos/$LOCAL_REPO_ID" 2>/dev/null || echo "")
    STATUS=$(echo "$REPO_JSON" | jq -r '.status' 2>/dev/null || echo "")
    if [ "$STATUS" = "indexed" ]; then
        break
    elif [ "$STATUS" = "error" ]; then
        echo -e "${RED}FAIL${NC} — sync re-index failed with error status"
        tail -30 /tmp/knot-server-e2e.log 2>/dev/null || true
        exit 1
    fi
    if [ "$i" -eq 60 ]; then
        echo -e "${RED}FAIL${NC} — sync did not reach indexed within 60s (status: $STATUS)"
        tail -30 /tmp/knot-server-e2e.log 2>/dev/null || true
        exit 1
    fi
    sleep 1
done

# Search for the new class
SEARCH=$(curl -sf "$BASE_URL/api/repos/$LOCAL_REPO_ID/search?q=SyncTestService" 2>/dev/null || echo "")
HAS_RESULT=$(echo "$SEARCH" | jq '[.. | strings | select(contains("SyncTestService"))] | length' 2>/dev/null || echo "0")
if [ -n "$HAS_RESULT" ] && [ "$HAS_RESULT" -ge 1 ]; then
    echo -e "${GREEN}PASS${NC} — new class found in search results"
else
    echo -e "${RED}FAIL${NC} — SyncTestService not found in search"
    echo "Search response: $SEARCH"
    tail -30 /tmp/knot-server-e2e.log 2>/dev/null || true
    exit 1
fi

# Explore the new file directly
EXPLORE=$(curl -s -w "%{http_code}" -o /tmp/s2_explore.json \
    "$BASE_URL/api/repos/$LOCAL_REPO_ID/explore?path=SyncTestService.java")
if [ "$EXPLORE" = "200" ] && \
   jq -e '[.entities[] | select(.name == "SyncTestService")] | length >= 1' /tmp/s2_explore.json > /dev/null 2>&1; then
    echo -e "${GREEN}PASS${NC} — new class visible in /explore"
else
    echo -e "${RED}FAIL${NC} — /explore did not return SyncTestService (status: $EXPLORE)"
    cat /tmp/s2_explore.json
    exit 1
fi

# ── Test S3: Modify an existing class uncommitted, re-sync, verify change ──
echo -e "\n${CYAN}Test S3: Uncommitted modification picked up after sync${NC}"
cat >> "$LOCAL_LIVE_PATH/UserService.java" <<'JAVA'

    public void uncommittedNewMethod() {}
JAVA

UNCOMMITTED_PROOF=$(git -C "$LOCAL_LIVE_PATH" status --short | grep "UserService.java" || true)
if [ -z "$UNCOMMITTED_PROOF" ]; then
    echo -e "${RED}FAIL${NC} — UserService.java modification was not uncommitted"
    exit 1
fi
echo "  Confirmed uncommitted modification: $UNCOMMITTED_PROOF"

curl -sf -X POST "$BASE_URL/api/repos/$LOCAL_REPO_ID/sync" > /dev/null

for i in $(seq 1 60); do
    REPO_JSON=$(curl -sf "$BASE_URL/api/repos/$LOCAL_REPO_ID" 2>/dev/null || echo "")
    STATUS=$(echo "$REPO_JSON" | jq -r '.status' 2>/dev/null || echo "")
    if [ "$STATUS" = "indexed" ]; then
        break
    elif [ "$STATUS" = "error" ]; then
        echo -e "${RED}FAIL${NC} — re-sync failed with error"
        tail -30 /tmp/knot-server-e2e.log 2>/dev/null || true
        exit 1
    fi
    if [ "$i" -eq 60 ]; then
        echo -e "${RED}FAIL${NC} — re-sync did not reach indexed within 60s (status: $STATUS)"
        tail -30 /tmp/knot-server-e2e.log 2>/dev/null || true
        exit 1
    fi
    sleep 1
done

CALLERS=$(curl -sf "$BASE_URL/api/repos/$LOCAL_REPO_ID/callers?entity=uncommittedNewMethod" 2>/dev/null || echo "")
if echo "$CALLERS" | jq -e '.' > /dev/null 2>&1; then
    echo -e "${GREEN}PASS${NC} — new method discoverable via /callers (entity exists in graph)"
else
    echo -e "${RED}FAIL${NC} — /callers did not return valid JSON for uncommittedNewMethod"
    echo "Response: $CALLERS"
    tail -30 /tmp/knot-server-e2e.log 2>/dev/null || true
    exit 1
fi

# ── Test S4: Source git log still has exactly 1 commit (changes never committed) ──
echo -e "\n${CYAN}Test S4: Source git log unchanged (changes were never committed)${NC}"
COMMIT_COUNT=$(git -C "$LOCAL_LIVE_PATH" log --oneline | wc -l | tr -d ' ')
if [ "$COMMIT_COUNT" = "1" ]; then
    echo -e "${GREEN}PASS${NC} — source still has only 1 commit (uncommitted changes never committed)"
else
    echo -e "${RED}FAIL${NC} — unexpected commit count: $COMMIT_COUNT (expected 1)"
    exit 1
fi

# ── Test S5: Stale .knot/index_state.json from an older knot is auto-cleared ──
# Regression test for the version-mismatch bug:
#   "Detected index_state v0; current version is v3. The on-disk index is
#    incompatible."
# When `local_sync` mirrors a source tree into the destination, it preserves
# `.knot/` (indexer state) so future syncs remain incremental. But if the
# destination's state file was written by an older `knot` version (no `version`
# field), the new indexer refuses to load it and every sync would fail.
# The worker must detect the stale format, delete the file, and re-index.
echo -e "\n${CYAN}Test S5: Stale .knot/index_state.json is auto-cleared on sync${NC}"

LOCAL_REPO_DEST="$WORKSPACE_DIR/$LOCAL_REPO_ID"
STALE_STATE_FILE="$LOCAL_REPO_DEST/.knot/index_state.json"

# Inject a state file in the OLD format (no top-level "version" key).
# This mimics exactly the on-disk state left behind by a pre-versioned `knot`.
mkdir -p "$(dirname "$STALE_STATE_FILE")"
cat > "$STALE_STATE_FILE" <<'JSON'
{
  "file_hashes": {
    "/tmp/knot-e2e-workspace-$$/local-live-repo/SomeFile.java": "deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef"
  }
}
JSON
echo "  Injected stale state file at $STALE_STATE_FILE (no 'version' key)"

# Trigger sync — the worker should detect staleness, clear the file,
# and complete indexing successfully.
SYNC_CODE=$(curl -s -w "%{http_code}" -o /dev/null \
    -X POST "$BASE_URL/api/repos/$LOCAL_REPO_ID/sync")
if [ "$SYNC_CODE" = "202" ]; then
    echo -e "  ${GREEN}sync accepted (202)${NC}"
else
    echo -e "${RED}FAIL${NC} — sync returned $SYNC_CODE (expected 202)"
    tail -30 /tmp/knot-server-e2e.log 2>/dev/null || true
    exit 1
fi

# Wait for indexing to reach 'indexed' (NOT 'error')
S5_INDEXED=false
for i in $(seq 1 60); do
    REPO_JSON=$(curl -sf "$BASE_URL/api/repos/$LOCAL_REPO_ID" 2>/dev/null || echo "")
    STATUS=$(echo "$REPO_JSON" | jq -r '.status' 2>/dev/null || echo "")

    if [ "$STATUS" = "indexed" ]; then
        S5_INDEXED=true
        break
    elif [ "$STATUS" = "error" ]; then
        echo -e "${RED}FAIL${NC} — sync failed with error status (stale state was not auto-cleared)"
        tail -30 /tmp/knot-server-e2e.log 2>/dev/null || true
        exit 1
    fi
    if [ "$i" -eq 60 ]; then
        echo -e "${RED}FAIL${NC} — sync did not reach 'indexed' within 60s (status: $STATUS)"
        tail -30 /tmp/knot-server-e2e.log 2>/dev/null || true
        exit 1
    fi
    sleep 1
done
if [ "$S5_INDEXED" = "true" ]; then
    echo -e "${GREEN}PASS${NC} — sync reached 'indexed' (stale state was auto-cleared, not propagated as error)"
else
    echo -e "${RED}FAIL${NC} — sync never reached 'indexed'"
    exit 1
fi

# Verify the indexer rewrote the state file with a current version.
if [ ! -f "$STALE_STATE_FILE" ]; then
    echo -e "${RED}FAIL${NC} — indexer did not recreate $STALE_STATE_FILE after re-index"
    exit 1
fi
NEW_VERSION=$(jq -r '.version // "missing"' "$STALE_STATE_FILE" 2>/dev/null || echo "missing")
if [ "$NEW_VERSION" = "missing" ] || [ "$NEW_VERSION" = "null" ] || [ -z "$NEW_VERSION" ]; then
    echo -e "${RED}FAIL${NC} — rewritten state file has no 'version' key: $NEW_VERSION"
    cat "$STALE_STATE_FILE"
    exit 1
fi
echo -e "${GREEN}PASS${NC} — rewritten state file has version=$NEW_VERSION"

# Also verify the auto-clear log message is present (proves the worker took
# the intended path, not some accidental recovery). The local-live-repo is
# processed by the server that was started in the recovery test phase, so
# its log is RECOVERY_LOG_R, not the original /tmp/knot-server-e2e.log.
if grep -q "Removed stale .knot/index_state.json for local repo '$LOCAL_REPO_ID'" "$RECOVERY_LOG_R"; then
    echo -e "${GREEN}PASS${NC} — worker logged stale-state removal"
else
    echo -e "${RED}FAIL${NC} — expected stale-state log line not found in $RECOVERY_LOG_R"
    tail -40 "$RECOVERY_LOG_R" 2>/dev/null || true
    exit 1
fi

# ── Test S6: Build-artifact directories (target/, node_modules/) are skipped ──
# Regression test for the 30 GB `target/` blow-up: when a user registers their
# own Rust project as a local repo, the local sync used to recursively copy
# `target/`, which contains the entire Cargo build output. Syncs of such
# projects took ~40 s and ballooned the workspace. The skip-list in
# `local_sync` now treats `target/`, `node_modules/`, `build/`, `dist/`, etc.
# as ignored directories: they are never copied and are actively removed from
# the mirror if they survived a previous unfiltered sync.
echo -e "\n${CYAN}Test S6: Build-artifact directories are skipped during local sync${NC}"

# Place the artifact source OUTSIDE the workspace. The registry
# derives `local_path = workspace/<id>` from the URL's basename, so
# any source inside the workspace would collide with its own mirror.
ARTIFACT_LIVE_PATH="$LOCAL_SOURCE_ROOT/artifact-live-repo"
rm -rf "$ARTIFACT_LIVE_PATH"
mkdir -p "$ARTIFACT_LIVE_PATH"
git init -q "$ARTIFACT_LIVE_PATH"
git -C "$ARTIFACT_LIVE_PATH" checkout -q -b main
cp "$FIXTURE_DIR"/*.java "$ARTIFACT_LIVE_PATH/"
git -C "$ARTIFACT_LIVE_PATH" add . > /dev/null 2>&1
git -C "$ARTIFACT_LIVE_PATH" -c user.email=test@test.com -c user.name=Test \
    commit -q -m "initial"

# Plant a fat "build artifact" dir to ensure it is NOT copied. We use
# dd to create ~50 MB so that any actual copy would noticeably slow the
# test down — that is the signal we are testing for.
ARTIFACT_LIVE_TARGET="$ARTIFACT_LIVE_PATH/target"
mkdir -p "$ARTIFACT_LIVE_TARGET"
dd if=/dev/zero of="$ARTIFACT_LIVE_TARGET/big_binary" bs=1M count=50 \
    status=none 2>/dev/null
mkdir -p "$ARTIFACT_LIVE_PATH/node_modules/react"
echo "fake module" > "$ARTIFACT_LIVE_PATH/node_modules/react/index.js"
echo "  Planted 50 MB target/big_binary and a fake node_modules/ in source"

# Register
ARTIFACT_REPO_BODY=$(mktemp)
ARTIFACT_CODE=$(curl -sf -w "%{http_code}" -o "$ARTIFACT_REPO_BODY" \
    -X POST "$BASE_URL/api/repos" \
    -H "Content-Type: application/json" \
    -d "{\"url\": \"$ARTIFACT_LIVE_PATH\", \"auth_type\": \"ssh\"}")
if [ "$ARTIFACT_CODE" = "202" ]; then
    echo -e "  ${GREEN}registered (202)${NC}"
else
    echo -e "${RED}FAIL${NC} — expected 202, got $ARTIFACT_CODE"
    cat "$ARTIFACT_REPO_BODY"
    rm -f "$ARTIFACT_REPO_BODY"
    exit 1
fi
ARTIFACT_REPO_ID=$(jq -r '.id' "$ARTIFACT_REPO_BODY")
rm -f "$ARTIFACT_REPO_BODY"

# Wait for indexing
for i in $(seq 1 60); do
    REPO_JSON=$(curl -sf "$BASE_URL/api/repos/$ARTIFACT_REPO_ID" 2>/dev/null || echo "")
    STATUS=$(echo "$REPO_JSON" | jq -r '.status' 2>/dev/null || echo "")
    if [ "$STATUS" = "indexed" ]; then break; fi
    if [ "$STATUS" = "error" ]; then
        echo -e "${RED}FAIL${NC} — indexing error"
        tail -30 /tmp/knot-server-e2e.log 2>/dev/null || true
        exit 1
    fi
    if [ "$i" -eq 60 ]; then
        echo -e "${RED}FAIL${NC} — indexing timed out (status: $STATUS)"
        tail -30 /tmp/knot-server-e2e.log 2>/dev/null || true
        exit 1
    fi
    sleep 1
done

ARTIFACT_DEST="$WORKSPACE_DIR/$ARTIFACT_REPO_ID"

# Verify target/ is NOT in the mirror
if [ -e "$ARTIFACT_DEST/target" ]; then
    TARGET_SIZE=$(du -sh "$ARTIFACT_DEST/target" 2>/dev/null | cut -f1)
    echo -e "${RED}FAIL${NC} — target/ was copied to mirror ($TARGET_SIZE); skip list is not working"
    exit 1
else
    echo -e "${GREEN}PASS${NC} — target/ not in mirror (50 MB artifact was correctly skipped)"
fi

# Verify node_modules/ is NOT in the mirror
if [ -e "$ARTIFACT_DEST/node_modules" ]; then
    echo -e "${RED}FAIL${NC} — node_modules/ was copied to mirror"
    exit 1
else
    echo -e "${GREEN}PASS${NC} — node_modules/ not in mirror"
fi

# Verify the Java source file WAS copied (sanity check — we did not over-skip)
if [ ! -f "$ARTIFACT_DEST/UserService.java" ]; then
    echo -e "${RED}FAIL${NC} — UserService.java missing from mirror (skip list is over-aggressive)"
    exit 1
else
    echo -e "${GREEN}PASS${NC} — legitimate source files still copied"
fi

# Trigger a second sync. If the prune logic is correct, the dst's
# `.knot/` state is preserved AND no artifact dir accumulates. Total
# size of the mirror should stay small and not double.
curl -sf -X POST "$BASE_URL/api/repos/$ARTIFACT_REPO_ID/sync" > /dev/null
for i in $(seq 1 60); do
    STATUS=$(curl -sf "$BASE_URL/api/repos/$ARTIFACT_REPO_ID" 2>/dev/null \
        | jq -r '.status' 2>/dev/null || echo "")
    if [ "$STATUS" = "indexed" ]; then break; fi
    if [ "$i" -eq 60 ]; then
        echo -e "${RED}FAIL${NC} — re-sync timed out (status: $STATUS)"
        tail -30 /tmp/knot-server-e2e.log 2>/dev/null || true
        exit 1
    fi
    sleep 1
done
if [ -e "$ARTIFACT_DEST/target" ] || [ -e "$ARTIFACT_DEST/node_modules" ]; then
    echo -e "${RED}FAIL${NC} — artifact dir appeared after a second sync"
    exit 1
else
    echo -e "${GREEN}PASS${NC} — second sync still clean (no artifact drift)"
fi

# Clean up
curl -s -o /dev/null -X DELETE "$BASE_URL/api/repos/$ARTIFACT_REPO_ID"
rm -rf "$ARTIFACT_LIVE_PATH"
echo "  Cleaned up artifact live repo"

# Clean up the local live repo
curl -s -o /dev/null -X DELETE "$BASE_URL/api/repos/$LOCAL_REPO_ID"
rm -rf "$LOCAL_LIVE_PATH"
rm -rf "$LOCAL_SOURCE_ROOT"
rm -f /tmp/s2_explore.json
echo "  Cleaned up local live repo"

# -------------------------------------------------------
# Step 9: Summary
# -------------------------------------------------------
echo ""
echo -e "${GREEN}========================================${NC}"
echo -e "${GREEN}All E2E tests passed${NC}"
echo -e "${GREEN}========================================${NC}"
echo ""
echo "Validated: clone + index pipeline, search, callers, explore, deps"
echo "           POST/GET/DELETE /api/repos, /api/repos/:id/sync, /api/health"
echo "           /api/webhook (GitLab validation), error codes 400/401/404/429"
echo "           Database cleanup on DELETE (Neo4j + Qdrant)"
echo "           Queue capacity limit (429 Too Many Requests)"
echo "           Idempotent re-registration (POST upsert returns 202)"
echo "           Recovery of stuck repos on restart (Pending + Indexing)"
echo "           Local-path sync picks up uncommitted working-tree changes"
echo "           Local-path sync auto-clears stale index_state.json (knot version transitions)"
echo "           Local-path sync skips build artifacts (target/, node_modules/, build/, ...)"
echo ""

exit 0