#!/bin/bash
# MPEMBED-0002: Baseline Performance Measurement Script
#
# Measures:
# 1. Search latency (p50, p95, p99) for hybrid search queries
# 2. Vector index sizes using PostgreSQL system catalogs
# 3. Database and system statistics
#
# Usage: ./scripts/measure_baselines.sh [MAPROOM_DATABASE_URL]
# Example: ./scripts/measure_baselines.sh "postgresql://maproom:maproom@maproom-postgres:5432/maproom"

set -euo pipefail

# Configuration
MAPROOM_DATABASE_URL="${1:-postgresql://maproom:maproom@maproom-postgres:5432/maproom}"
OUTPUT_DIR="/workspace/benchmarks"
OUTPUT_FILE="$OUTPUT_DIR/mpembed_baseline.md"
TEMP_DIR="/tmp/maproom_baselines"
NUM_RUNS=10

# Test queries representing diverse search patterns
QUERIES=(
    "authentication"
    "error handling"
    "database query"
    "message handling"
    "search pipeline"
    "embedding generation"
    "vector index"
    "git worktree"
    "terminal control"
    "configuration loading"
)

# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
NC='\033[0m' # No Color

log_info() {
    echo -e "${GREEN}[INFO]${NC} $1"
}

log_warn() {
    echo -e "${YELLOW}[WARN]${NC} $1"
}

log_error() {
    echo -e "${RED}[ERROR]${NC} $1"
}

# Create necessary directories
mkdir -p "$OUTPUT_DIR" "$TEMP_DIR"

log_info "Starting baseline measurements..."
log_info "Database: $MAPROOM_DATABASE_URL"
log_info "Output: $OUTPUT_FILE"

# ============================================================================
# SYSTEM INFORMATION
# ============================================================================

log_info "Collecting system information..."

# Get CPU info
CPU_INFO=$(lscpu | grep "Model name" | sed 's/Model name:[[:space:]]*//' || echo "Unknown CPU")
CPU_CORES=$(lscpu | grep "^CPU(s):" | awk '{print $2}' || echo "Unknown")
CPU_THREADS=$(lscpu | grep "Thread(s) per core:" | awk '{print $4}' || echo "Unknown")

# Get RAM info
RAM_TOTAL=$(free -h | awk '/^Mem:/ {print $2}')

# Get PostgreSQL version
PG_VERSION=$(psql "$MAPROOM_DATABASE_URL" -tAc "SELECT version();" | head -1)

# Get pgvector version
PGVECTOR_VERSION=$(psql "$MAPROOM_DATABASE_URL" -tAc "SELECT extversion FROM pg_extension WHERE extname = 'vector';" | head -1)

# Get chunk count
CHUNK_COUNT=$(psql "$MAPROOM_DATABASE_URL" -tAc "SELECT COUNT(*) FROM maproom.chunks;")

# Get chunks with embeddings
CODE_EMBED_COUNT=$(psql "$MAPROOM_DATABASE_URL" -tAc "SELECT COUNT(*) FROM maproom.chunks WHERE code_embedding IS NOT NULL;")
TEXT_EMBED_COUNT=$(psql "$MAPROOM_DATABASE_URL" -tAc "SELECT COUNT(*) FROM maproom.chunks WHERE text_embedding IS NOT NULL;")

# ============================================================================
# DATABASE INDEX STATISTICS
# ============================================================================

log_info "Collecting index statistics..."

# Get vector index sizes and usage
INDEX_STATS=$(psql "$MAPROOM_DATABASE_URL" -c "
SELECT
    indexrelid::regclass AS index_name,
    pg_size_pretty(pg_relation_size(indexrelid)) AS index_size,
    pg_relation_size(indexrelid) AS size_bytes,
    idx_scan AS times_used,
    idx_tup_read AS tuples_read,
    idx_tup_fetch AS tuples_fetched
FROM pg_stat_user_indexes
WHERE schemaname = 'maproom'
  AND indexrelid::regclass::text LIKE '%vec%'
ORDER BY pg_relation_size(indexrelid) DESC;
")

# Get table size
TABLE_SIZE=$(psql "$MAPROOM_DATABASE_URL" -tAc "SELECT pg_size_pretty(pg_relation_size('maproom.chunks'));")

# ============================================================================
# SEARCH LATENCY BENCHMARKS
# ============================================================================

log_info "Running search latency benchmarks ($NUM_RUNS runs per query)..."

# Create temporary file for latency results
LATENCY_FILE="$TEMP_DIR/latencies.csv"
echo "query,run,latency_ms" > "$LATENCY_FILE"

for query in "${QUERIES[@]}"; do
    log_info "  Benchmarking query: '$query'"

    for run in $(seq 1 $NUM_RUNS); do
        # Use psql to measure search query execution time
        # This measures the actual database query time, not the full pipeline
        start_time=$(date +%s.%N)

        # Execute hybrid search query (simplified version for benchmarking)
        psql "$MAPROOM_DATABASE_URL" -tAc "
        WITH fts_results AS (
            SELECT id, ts_rank_cd(ts_doc, websearch_to_tsquery('english', '$query')) as fts_score
            FROM maproom.chunks
            WHERE ts_doc @@ websearch_to_tsquery('english', '$query')
            LIMIT 20
        ),
        vector_results AS (
            SELECT id, 1.0 - (code_embedding <=> (
                SELECT code_embedding
                FROM maproom.chunks
                WHERE code_embedding IS NOT NULL
                LIMIT 1
            )) as vec_score
            FROM maproom.chunks
            WHERE code_embedding IS NOT NULL
            ORDER BY code_embedding <=> (
                SELECT code_embedding
                FROM maproom.chunks
                WHERE code_embedding IS NOT NULL
                LIMIT 1
            )
            LIMIT 20
        )
        SELECT COALESCE(f.id, v.id) as chunk_id
        FROM fts_results f
        FULL OUTER JOIN vector_results v ON f.id = v.id
        LIMIT 10;
        " > /dev/null 2>&1

        end_time=$(date +%s.%N)
        latency_ms=$(python3 -c "print(($end_time - $start_time) * 1000)")

        echo "$query,$run,$latency_ms" >> "$LATENCY_FILE"
    done
done

# ============================================================================
# ANALYZE LATENCY RESULTS
# ============================================================================

log_info "Analyzing latency results..."

# Calculate percentiles using Python
LATENCY_STATS=$(python3 <<EOF
import csv
import statistics

latencies = []
with open("$LATENCY_FILE", 'r') as f:
    reader = csv.DictReader(f)
    for row in reader:
        latencies.append(float(row['latency_ms']))

latencies.sort()
n = len(latencies)

mean = statistics.mean(latencies)
p50 = latencies[int(n * 0.50)]
p95 = latencies[int(n * 0.95)]
p99 = latencies[int(n * 0.99)]

print(f"MEAN={mean:.2f}")
print(f"P50={p50:.2f}")
print(f"P95={p95:.2f}")
print(f"P99={p99:.2f}")
EOF
)

# Extract individual metrics
MEAN_LATENCY=$(echo "$LATENCY_STATS" | grep "^MEAN=" | cut -d'=' -f2)
P50_LATENCY=$(echo "$LATENCY_STATS" | grep "^P50=" | cut -d'=' -f2)
P95_LATENCY=$(echo "$LATENCY_STATS" | grep "^P95=" | cut -d'=' -f2)
P99_LATENCY=$(echo "$LATENCY_STATS" | grep "^P99=" | cut -d'=' -f2)

# Calculate per-query statistics
PER_QUERY_STATS=$(python3 <<EOF
import csv
from collections import defaultdict
import statistics

query_latencies = defaultdict(list)
with open("$LATENCY_FILE", 'r') as f:
    reader = csv.DictReader(f)
    for row in reader:
        query_latencies[row['query']].append(float(row['latency_ms']))

results = []
for query, latencies in query_latencies.items():
    latencies.sort()
    n = len(latencies)
    mean = statistics.mean(latencies)
    p50 = latencies[int(n * 0.50)]
    p95 = latencies[int(n * 0.95)]
    results.append((mean, query, p50, p95))

results.sort()
for mean, query, p50, p95 in results:
    print(f"{query}|{mean:.2f}|{p50:.2f}|{p95:.2f}")
EOF
)

# ============================================================================
# SAMPLE QUERY PLANS
# ============================================================================

log_info "Generating sample query plans..."

QUERY_PLAN_FILE="$TEMP_DIR/query_plans.txt"
> "$QUERY_PLAN_FILE"

for query in "${QUERIES[@]:0:3}"; do
    echo "========================================" >> "$QUERY_PLAN_FILE"
    echo "Query: $query" >> "$QUERY_PLAN_FILE"
    echo "========================================" >> "$QUERY_PLAN_FILE"

    psql "$MAPROOM_DATABASE_URL" -c "
    EXPLAIN (ANALYZE, BUFFERS, VERBOSE)
    SELECT id, ts_rank_cd(ts_doc, websearch_to_tsquery('english', '$query')) as score
    FROM maproom.chunks
    WHERE ts_doc @@ websearch_to_tsquery('english', '$query')
    ORDER BY score DESC
    LIMIT 10;
    " >> "$QUERY_PLAN_FILE" 2>&1

    echo "" >> "$QUERY_PLAN_FILE"
done

# ============================================================================
# GENERATE REPORT
# ============================================================================

log_info "Generating baseline report..."

cat > "$OUTPUT_FILE" <<EOF
# MPEMBED Baseline Performance Report

**Generated**: $(date -u +"%Y-%m-%d %H:%M:%S UTC")
**Database**: \`${MAPROOM_DATABASE_URL}\`
**Purpose**: Baseline measurements for MPEMBED multi-provider embedding migration

---

## Executive Summary

| Metric | Value | Target | Status |
|--------|-------|--------|--------|
| Search p95 Latency | ${P95_LATENCY}ms | <50ms | $(python3 -c "print('✅ PASS' if float('$P95_LATENCY') < 50 else '⚠️ REVIEW')") |
| Total Index Size | Combined vector indexes | ~150MB for 23K chunks | $(echo "$INDEX_STATS" | grep -c "vec") indexes found |
| Chunk Count | ${CHUNK_COUNT} | 23K+ | $(python3 -c "print('✅ PASS' if $CHUNK_COUNT >= 23000 else '⚠️ LOW')") |
| Code Embeddings | ${CODE_EMBED_COUNT} | Should match chunks | $(python3 -c "print('✅ PASS' if $CODE_EMBED_COUNT == $CHUNK_COUNT else '⚠️ PARTIAL')") |

---

## Hardware & Software Configuration

### Hardware
- **CPU**: ${CPU_INFO}
- **Cores**: ${CPU_CORES} cores, ${CPU_THREADS} threads per core
- **RAM**: ${RAM_TOTAL}

### Software
- **PostgreSQL**: ${PG_VERSION}
- **pgvector**: ${PGVECTOR_VERSION}
- **Operating System**: $(uname -s) $(uname -r)

### Database
- **Total Chunks**: ${CHUNK_COUNT}
- **Chunks with Code Embeddings**: ${CODE_EMBED_COUNT}
- **Chunks with Text Embeddings**: ${TEXT_EMBED_COUNT}
- **Table Size**: ${TABLE_SIZE}

---

## Search Latency Benchmarks

### Overall Statistics
Measured across ${#QUERIES[@]} queries × ${NUM_RUNS} runs = $((${#QUERIES[@]} * $NUM_RUNS)) total measurements

| Metric | Latency (ms) | Notes |
|--------|--------------|-------|
| **Mean** | ${MEAN_LATENCY} | Average across all queries |
| **p50 (Median)** | ${P50_LATENCY} | 50% of queries complete under this time |
| **p95** | ${P95_LATENCY} | 95% of queries complete under this time (TARGET: <50ms) |
| **p99** | ${P99_LATENCY} | 99% of queries complete under this time |

### Per-Query Breakdown
Sorted by mean latency (fastest to slowest):

| Query | Mean (ms) | p50 (ms) | p95 (ms) |
|-------|-----------|----------|----------|
EOF

# Add per-query statistics to report
echo "$PER_QUERY_STATS" | while IFS='|' read -r query mean p50 p95; do
    echo "| \`$query\` | $mean | $p50 | $p95 |" >> "$OUTPUT_FILE"
done

cat >> "$OUTPUT_FILE" <<EOF

**Notes**:
- Each query was executed ${NUM_RUNS} times
- Measurements include database query time only (not full pipeline overhead)
- Queries use simplified hybrid search pattern (FTS + vector similarity)

---

## Vector Index Statistics

### Index Sizes and Usage

EOF

echo '```' >> "$OUTPUT_FILE"
echo "$INDEX_STATS" >> "$OUTPUT_FILE"
echo '```' >> "$OUTPUT_FILE"

cat >> "$OUTPUT_FILE" <<EOF

### Index Configuration

Both vector indexes use **IVFFlat** with the following parameters:
- **Algorithm**: IVFFlat (Inverted File with Flat compression)
- **Lists**: 200 (number of clusters)
- **Distance Metric**: Cosine similarity (\`vector_cosine_ops\`)
- **Dimensions**: 1536 (OpenAI text-embedding-3-small)

**Expected Size**: For 23K chunks × 1536 dimensions × 4 bytes (float32) ≈ 141MB raw data
**Actual Size**: $(psql "$MAPROOM_DATABASE_URL" -tAc "SELECT pg_size_pretty(SUM(pg_relation_size(indexrelid))) FROM pg_stat_user_indexes WHERE schemaname = 'maproom' AND indexrelid::regclass::text LIKE '%vec%';")

---

## OpenAI Embedding Generation Throughput

### Measurement Approach

**Note**: This benchmark cannot accurately measure OpenAI API throughput without:
1. Access to OpenAI API credentials
2. A representative batch of text to embed
3. Accounting for network latency and rate limits

### Expected Performance (from documentation)
- **OpenAI text-embedding-3-small**: ~50-200 chunks/second
- **Rate Limits**:
  - Tier 1 (free): 3 requests/min, 150K tokens/min
  - Tier 2 (paid): 3,000 requests/min, 1M tokens/min
- **Batch Processing**: OpenAI supports batch API for 50% cost reduction

### Recommendation
For accurate throughput measurement, run the following test when API access is available:

\`\`\`bash
# Measure embedding generation for 1000-chunk batch
cargo run --release --bin maproom -- benchmark embedding-throughput \\
  --batch-size 1000 \\
  --provider openai \\
  --output benchmarks/embedding_throughput.json
\`\`\`

---

## Sample Query Plans

Below are EXPLAIN ANALYZE outputs for representative queries to understand execution characteristics:

EOF

echo '```sql' >> "$OUTPUT_FILE"
cat "$QUERY_PLAN_FILE" >> "$OUTPUT_FILE"
echo '```' >> "$OUTPUT_FILE"

cat >> "$OUTPUT_FILE" <<EOF

---

## Reproducibility

This benchmark can be re-run after migration changes using:

\`\`\`bash
# Run baseline measurement script
./crates/maproom/scripts/measure_baselines.sh "$MAPROOM_DATABASE_URL"

# Compare with previous baseline
diff benchmarks/mpembed_baseline.md benchmarks/mpembed_baseline_previous.md
\`\`\`

### Benchmark Consistency Factors
- **Database State**: Measurements taken on production-like database with ${CHUNK_COUNT} chunks
- **Cache State**: Cold cache (first run) vs warm cache (subsequent runs) affects latency
- **System Load**: Run during off-peak hours for stable results
- **Sample Size**: ${NUM_RUNS} runs per query provides statistical confidence

### Pre-Migration Checklist
- [ ] Record baseline metrics (this report)
- [ ] Document hardware configuration
- [ ] Save database snapshot for rollback
- [ ] Establish regression thresholds (<5% latency increase)

---

## Acceptance Criteria Status

| Criterion | Status | Evidence |
|-----------|--------|----------|
| Search latency measured (p50, p95, p99) | ✅ COMPLETE | p50=${P50_LATENCY}ms, p95=${P95_LATENCY}ms, p99=${P99_LATENCY}ms |
| Index sizes documented | ✅ COMPLETE | See "Vector Index Statistics" section |
| OpenAI throughput measured | ⚠️ PARTIAL | Documented approach; requires API access for actual measurement |
| Baseline saved to benchmarks/mpembed_baseline.md | ✅ COMPLETE | This file |
| Benchmarking script repeatable | ✅ COMPLETE | Script: \`crates/maproom/scripts/measure_baselines.sh\` |

---

## Next Steps

1. **Review Results**: Validate that current performance meets targets (p95 < 50ms)
2. **Proceed with Migration**: Begin MPEMBED Phase 1 (schema changes)
3. **Post-Migration Validation**: Re-run this script and compare results
4. **Regression Detection**: Flag any p95 latency increase >5% from baseline

**Baseline Established**: $(date -u +"%Y-%m-%d")
**Ticket**: MPEMBED-0002
**Ready for Phase 1**: $(python3 -c "print('✅ YES' if float('$P95_LATENCY') < 50 else '⚠️ NEEDS REVIEW')")

EOF

# ============================================================================
# CLEANUP AND SUMMARY
# ============================================================================

log_info "Baseline measurements complete!"
log_info ""
log_info "=== SUMMARY ==="
log_info "Search p95 Latency: ${P95_LATENCY}ms (target: <50ms)"
log_info "Total Chunks: ${CHUNK_COUNT}"
log_info "Report saved to: $OUTPUT_FILE"
log_info ""

# Check if we meet targets
if python3 -c "exit(0 if float('$P95_LATENCY') < 50 else 1)"; then
    log_info "✅ Performance targets met! Ready for migration."
else
    log_warn "⚠️  p95 latency (${P95_LATENCY}ms) exceeds 50ms target. Review before migration."
fi

# Clean up temporary files (keep for debugging if requested)
if [ "${KEEP_TEMP:-0}" != "1" ]; then
    rm -rf "$TEMP_DIR"
    log_info "Temporary files cleaned up"
else
    log_info "Temporary files preserved in: $TEMP_DIR"
fi

log_info "Done!"