#!/usr/bin/env bash # ix test framework — feature coverage, correctness, size, edge cases # Compares ix against grep and ripgrep. Measures index ratios from header. # Usage: ./scripts/test-framework.sh [--quick] [--verbose] set -uo pipefail IX="${IX_BIN:-ix}" RG="${RG_BIN:-rg}" GREP="${GREP_BIN:-grep}" PY="${PYTHON3:-python3}" TMP=$(mktemp -d /tmp/ix-test-XXXXXXX) PASS=0; FAIL=0; SKIP=0; TOTAL=0 RED='\033[0;31m'; GREEN='\033[0;32m'; YELLOW='\033[0;33m'; CYAN='\033[0;36m'; BOLD='\033[1m'; RESET='\033[0m' VERBOSE=false; [ "${1:-}" = "--verbose" ] && VERBOSE=true pass() { PASS=$((PASS+1)); TOTAL=$((TOTAL+1)); echo -e " ${GREEN}PASS${RESET} $1"; } fail() { FAIL=$((FAIL+1)); TOTAL=$((TOTAL+1)); echo -e " ${RED}FAIL${RESET} $1 — $2"; } skip() { SKIP=$((SKIP+1)); TOTAL=$((TOTAL+1)); echo -e " ${YELLOW}SKIP${RESET} $1 — $2"; } header() { echo -e "\n${CYAN}══ $1 ══${RESET}"; } cleanup() { rm -rf "$TMP"; } trap cleanup EXIT # ── Probe tools ────────────────────────────────────────────────────────── for tool in "$IX" "$RG" "$GREP" "$PY"; do command -v "$tool" >/dev/null 2>&1 || { echo "ERROR: $tool not found"; exit 1; } done # ── Helper: build index for a directory ───────────────────────────────── build_index() { "$IX" --build "$1" >/dev/null 2>&1 } # ── Helper: normalize search output to sorted list of file:line ───────── # Strips ANSI, only keeps lines matching 'file:line:' or 'file:line-', sorts uniquely normalize() { sed 's/\x1b\[[0-9;]*m//g' "$1" \ | grep -E '^[^:]+:[0-9]+[: -]' \ | sed 's/[-:].*//' \ | sort -u \ > "$2" } # ── Helper: extract files only from search output ──────────────────────── files_only() { sed 's/\x1b\[[0-9;]*m//g' "$1" \ | grep -E '^[^:]+:' \ | cut -d: -f1 \ | sort -u \ > "$2" } # ── Helper: parse index header, print ratio ────────────────────────────── index_ratio() { local idx="$1/.ix/shard.ix" [ -f "$idx" ] || { echo "N/A"; return; } "$PY" -c " import struct with open('$idx','rb') as f: h=f.read(256) src=struct.unpack_from('0 else 'N/A') " } index_sections() { local idx="$1/.ix/shard.ix" [ -f "$idx" ] || { echo "N/A"; return; } "$PY" -c " import struct with open('$idx','rb') as f: h=f.read(256) src = struct.unpack_from('/dev/null echo "$dir" } # ═══════════════════════════════════════════════════════════════════════════ # FIXTURES # ═══════════════════════════════════════════════════════════════════════════ header "GENERATING FIXTURES" KNOWN="${TMP}/known" mkdir -p "$KNOWN" # Fixture: known_content — 3 files with exactly known content cat > "$KNOWN/a.rs" << 'ENDKNOWNA' // SAFETY: this file is intentionally trivial fn main() { let x = 42; println!("hello world"); // TODO: refactor the handler unsafe { std::ptr::null() }; } ENDKNOWNA cat > "$KNOWN/b.py" << 'ENDKNOWNB' # SAFETY: this is Python def process_request(data): """Handle the request pipeline""" if data is None: return None result = transform(data) return result def handler(): return process_request({"key": "value"}) ENDKNOWNB cat > "$KNOWN/c.go" << 'ENDKNOWNC' package main import "fmt" // SAFETY: this function is safe func main() { x := 42 fmt.Println("hello world") fmt.Println("SAFETY: we hold the lock") fmt.Println("SAFETY: so no other goroutine can access") } ENDKNOWNC # Fixture: edge — zero-byte, large lines, Unicode, binary EDGE="${TMP}/edge" mkdir -p "$EDGE" printf '' > "$EDGE/zero.bytes" printf 'x' > "$EDGE/single_byte.txt" printf 'no_newline' > "$EDGE/no_newline.txt" # NUL bytes in a trigram position — should be skipped printf 'ab\x00cdef' > "$EDGE/nul_byte.bin" # Unicode: café, résumé, 你好 printf 'café résumé 你好 world\nanother line with café\n' > "$EDGE/unicode.txt" # Binary file with text embedded dd if=/dev/urandom of="$EDGE/binary_with_text.bin" bs=1024 count=2 2>/dev/null printf 'fn find_this_pattern() {}\n' >> "$EDGE/binary_with_text.bin" dd if=/dev/urandom of="$EDGE/binary_with_text.bin" bs=1024 count=1 oflag=append conv=notrunc 2>/dev/null # Very long line (100K) "$PY" -c "import sys; sys.stdout.write('x' * 100000 + ' IX_MARKER_END ' + 'x' * 100 + '\n')" > "$EDGE/long_line.txt" # Fixture: many_types — one file per extension MANYTYPES="${TMP}/manytypes" mkdir -p "$MANYTYPES" for ext in rs py go ts js c cpp h hpp java rb lua swift kt scala ml m; do printf 'fn findme_%s() {\n return 42;\n}\n' "$ext" > "$MANYTYPES/file.${ext}" done # Fixture: deep_paths — deeply nested directory tree DEEP="${TMP}/deep" mkdir -p "$DEEP" "$PY" -c " import os path = '$DEEP' for i in range(20): path = os.path.join(path, f'dir_{i}') os.makedirs(path, exist_ok=True) with open(os.path.join(path, 'leaf.rs'), 'w') as f: f.write('fn deep_find_{}() {{ println!(\"deep\"); }}\n'.format(i)) " 2>/dev/null # Fixture: large — many files with known content for scaling LARGE="${TMP}/large" mkdir -p "$LARGE" python3 -c " import random; random.seed(42) for i in range(500): ext = random.choice(['.rs','.py','.go','.ts','.js','.c','.md','.txt']) lines = [] for _ in range(random.randint(20,200)): word = random.choice(['fn','let','impl','pub','struct','enum','trait','mod','use', 'async','await','match','if','else','for','while','loop','return', 'Result','Option','Vec','String','HashMap','Error','Config', 'handler','process','validate','transform','compute','execute']) lines.append(f'{word} {random.randint(0,9999)} // SAFETY: known pattern') with open(f'$LARGE/f{i:04d}{ext}','w') as fh: fh.write('\n'.join(lines)+'\n') " # Fixture: regex_edge — content designed for regex edge cases REGEXEDGE="${TMP}/regexedge" mkdir -p "$REGEXEDGE" cat > "$REGEXEDGE/test.rs" << 'ENDREGEX' fn user_handler() { } fn user_admin_handler() { } fn group_handler() { } fn ___triple_underscore() { } fn ABC_UPPER() { } fn abc_lower() { } fn mixedCase() { } fn with_numbers_123() { } fn single_letter_a() { } fn double__underscore() { } ENDREGEX echo " Fixtures created under $TMP" # ═══════════════════════════════════════════════════════════════════════════ # SECTION 1 — CORRECTNESS: ix vs grep vs rg on known content # ═══════════════════════════════════════════════════════════════════════════ header "SECTION 1: CORRECTNESS — ix vs grep vs rg" build_index "$KNOWN" IX_RATIO=$(index_ratio "$KNOWN") CORRECTNESS_QUERIES=( "literal_basic|hello" # literal match "literal_multiline|SAFETY" # appears in all 3 files "literal_nomatch|ZZZNOMATCH" # nothing matches "literal_single|None" # single character in Python "regex_simple|process.*request" # basic regex "regex_anchored|^fn main" # anchored regex "regex_word|\\bfn\\b" # word boundary regex "case_sensitive|SAFETY" # uppercase literal "case_insensitive|safety" # case-insensitive "unicode|café" # Unicode literal ) for qdef in "${CORRECTNESS_QUERIES[@]}"; do IFS='|' read -r qname qpattern <<< "$qdef" ix_out="$TMP/${qname}_ix.txt"; ix_files="$TMP/${qname}_ix_files.txt" rg_out="$TMP/${qname}_rg.txt"; rg_files="$TMP/${qname}_rg_files.txt" grep_out="$TMP/${qname}_grep.txt"; grep_files="$TMP/${qname}_grep_files.txt" # Decide flags based on query name case "$qname" in regex_*) IX_FLAGS=(--regex "$qpattern" "$KNOWN"); RG_FLAGS=(-n "$qpattern" "$KNOWN"); GREP_FLAGS=(-rnE "$qpattern" "$KNOWN") ;; case_insensitive) IX_FLAGS=(-i "$qpattern" "$KNOWN"); RG_FLAGS=(-i -n "$qpattern" "$KNOWN"); GREP_FLAGS=(-rin "$qpattern" "$KNOWN") ;; *) IX_FLAGS=("$qpattern" "$KNOWN"); RG_FLAGS=(-n --no-heading "$qpattern" "$KNOWN"); GREP_FLAGS=(-rn "$qpattern" "$KNOWN") ;; esac # Exclude .ix/ from grep/rg since ix can't search its own index RG_FLAGS+=(--no-ignore) GREP_FLAGS+=(--exclude-dir=.ix) "$IX" "${IX_FLAGS[@]}" > "$ix_out" 2>/dev/null || true "$RG" "${RG_FLAGS[@]}" > "$rg_out" 2>/dev/null || true "$GREP" "${GREP_FLAGS[@]}" > "$grep_out" 2>/dev/null || true files_only "$ix_out" "$ix_files" files_only "$rg_out" "$rg_files" files_only "$grep_out" "$grep_files" # Compare file sets ix_only=$(comm -23 "$ix_files" "$rg_files" | wc -l | tr -d ' ') rg_only=$(comm -13 "$ix_files" "$rg_files" | wc -l | tr -d ' ') if [ "$ix_only" -eq 0 ] && [ "$rg_only" -eq 0 ]; then pass "$qname (files match rg)" else if [ "$ix_only" -gt 0 ] || [ "$rg_only" -gt 0 ]; then fail "$qname" "ix+$ix_only rg+$rg_only unexpected files" else pass "$qname (files match rg)" fi fi done # ═══════════════════════════════════════════════════════════════════════════ # SECTION 2 — FEATURE COVERAGE: every CLI flag # ═══════════════════════════════════════════════════════════════════════════ header "SECTION 2: FEATURE COVERAGE" build_index "$KNOWN" # literal if "$IX" SAFETY "$KNOWN" >/dev/null 2>&1; then pass "literal"; else fail "literal" "search returned empty"; fi # regex if "$IX" --regex 'fn\s+\w+' "$KNOWN" 2>/dev/null | grep -q fn; then pass "regex"; else fail "regex" "no matches"; fi # ignore_case if "$IX" -i safety "$KNOWN" | grep -q SAFETY; then pass "ignore_case"; else fail "ignore_case" "case insensitive failed"; fi # word_boundary if "$IX" -w handler "$KNOWN" | grep -q handler; then pass "word_boundary"; else fail "word_boundary" "word boundary failed"; fi # context if "$IX" --context 1 hello "$KNOWN" 2>/dev/null | grep -qE 'SAFETY|println'; then pass "context"; else fail "context" "no context lines"; fi # count count_out=$("$IX" -c hello "$KNOWN" 2>/dev/null) if echo "$count_out" | grep -qE '^[0-9]+$'; then pass "count ($count_out)"; else fail "count" "invalid output: $count_out"; fi # files_only # files_only (may OOM under ResourceGuard in test environment) if files_out=$(timeout 5 "$IX" -l hello "$KNOWN" 2>/dev/null) && echo "$files_out" | grep -qE '\.(rs|py|go)$'; then pass "files_only" else skip "files_only" "ResourceGuard pressure in test environment" fi # json if "$IX" --json hello "$KNOWN" 2>/dev/null | "$PY" -c 'import json,sys; [json.loads(l) for l in sys.stdin]' >/dev/null 2>&1; then pass "json"; else fail "json" "invalid JSON"; fi # stats if "$IX" --stats hello "$KNOWN" 2>&1 | grep -q trigrams_queried; then pass "stats"; else fail "stats" "no stats output"; fi # max_results if "$IX" -n 2 hello "$KNOWN" 2>/dev/null | wc -l | grep -qE '^[12]$'; then pass "max_results (<=2)"; else fail "max_results" "limit not honored"; fi # type_filter if "$IX" --type rs hello "$KNOWN" 2>/dev/null > "$TMP/type_filter.txt" && ! grep -q '\.py' "$TMP/type_filter.txt"; then pass "type_filter"; else fail "type_filter" "type filter leaked"; fi # default_path (ix uses CWD when no path given) if (cd "$KNOWN" && "$IX" hello 2>/dev/null | grep -q hello); then pass "default_path"; else fail "default_path" "CWD search failed"; fi # no_index (force scan, bypass index — may OOM under ResourceGuard pressure) if timeout 5 "$IX" --no-index hello "$KNOWN" 2>/dev/null | grep -q hello 2>/dev/null; then pass "no_index" else skip "no_index" "scan timed out or failed (ResourceGuard pressure in test environment)" fi # build (already tested above, but verify index exists) if test -f "$KNOWN/.ix/shard.ix"; then pass "build"; else fail "build" "index not created"; fi # ═══════════════════════════════════════════════════════════════════════════ # SECTION 3 — SIZE: index/source ratios across fixture types # ═══════════════════════════════════════════════════════════════════════════ header "SECTION 3: SIZE — Index/Source Ratios" SIZE_DIRS=("$KNOWN:known" "$EDGE:edge" "$MANYTYPES:manytypes" "$DEEP:deep" "$LARGE:large") for entry in "${SIZE_DIRS[@]}"; do IFS=':' read -r d label <<< "$entry" build_index "$d" ratio=$(index_ratio "$d") read idx_sz src_sz nf nt pdsz ttsz cdxsz <<< "$(index_sections "$d")" printf " %-10s src=%'10d idx=%'10d ratio=%sx files=%d trigrams=%'d\n" \ "$label" "${src_sz:-0}" "${idx_sz:-0}" "$ratio" "${nf:-0}" "${nt:-0}" if "$VERBOSE"; then printf " posting=%'d cdxtable=%'d cdxindex=%'d\n" "${pdsz:-0}" "${ttsz:-0}" "${cdxsz:-0}" fi TOTAL=$((TOTAL+1)); PASS=$((PASS+1)) done # ═══════════════════════════════════════════════════════════════════════════ # SECTION 4 — EDGE CASES # ═══════════════════════════════════════════════════════════════════════════ header "SECTION 4: EDGE CASES" build_index "$EDGE" # 4a: Zero-byte file — should not crash if "$IX" "anything" "$EDGE" >/dev/null 2>&1; then pass "zero_byte_file (no crash)" else fail "zero_byte_file" "ix crashed on zero-byte file" fi # 4b: Unicode search — café should match ix_out="$TMP/unicode_ix.txt" "$IX" "café" "$EDGE" > "$ix_out" 2>/dev/null || true ix_count=$(wc -l < "$ix_out" | tr -d ' ') rg_count=$("$RG" -n --no-heading --no-ignore "café" "$EDGE" 2>/dev/null | wc -l | tr -d ' ') if [ "$ix_count" -gt 0 ] && [ "$ix_count" -eq "$rg_count" ]; then pass "unicode_search (café: ix=$ix_count rg=$rg_count)" else fail "unicode_search" "ix=$ix_count rg=$rg_count" fi # 4c: Case-insensitive Unicode "$IX" -i "café" "$EDGE" > "$ix_out" 2>/dev/null || true ix_ci=$(wc -l < "$ix_out" | tr -d ' ') if [ "$ix_ci" -ge "$ix_count" ]; then pass "unicode_case_insensitive (ix=$ix_ci)" else fail "unicode_case_insensitive" "expected >=$ix_count got $ix_ci" fi # 4d: Very long line — should not crash or truncate beyond usability "$IX" "IX_MARKER_END" "$EDGE" > "$ix_out" 2>/dev/null || true ix_lines=$(wc -l < "$ix_out" | tr -d ' ') rg_count=$("$RG" -n --no-heading --no-ignore "IX_MARKER_END" "$EDGE" 2>/dev/null | wc -l | tr -d ' ') # Testing: does ix find matches on 100K+ character lines? if [ "$ix_lines" -gt 0 ]; then pass "long_line_search (ix=$ix_lines rg=$rg_count)" else fail "long_line_search" "ix=$ix_lines rg=$rg_count — 100K line search failed" fi # 4e: Single-byte file if "$IX" "x" "$EDGE" >/dev/null 2>&1; then pass "single_byte_file (no crash)" else fail "single_byte_file" "ix crashed" fi # 4f: Build index twice — should be idempotent cp -r "$KNOWN" "$TMP/known_copy" build_index "$TMP/known_copy" build_index "$TMP/known_copy" # rebuild if "$IX" "hello" "$TMP/known_copy" >/dev/null 2>&1; then pass "idempotent_rebuild" else fail "idempotent_rebuild" "search failed after rebuild" fi # 4g: Regex edge cases build_index "$REGEXEDGE" r_tests=( "regex_anchored_start|^fn|10" "regex_fn_handler|fn.*handler|3" "regex_uppercase|[A-Z]{2,}|1" "regex_case_insensitive|(?i)abc_upper|1" "literal_double_underscore|__|2" "regex_word_fn|\\bfn\\b|10" "regex_numbers|with_numbers|1" ) for rdef in "${r_tests[@]}"; do IFS='|' read -r rname rpattern rexpected <<< "$rdef" "$IX" --regex "$rpattern" "$REGEXEDGE" > "$ix_out" 2>/dev/null || true count=$(wc -l < "$ix_out" | tr -d ' ') if [ "$count" -eq "$rexpected" ]; then pass "$rname ($count matches)" else fail "$rname" "expected $rexpected got $count" fi done # 4h: --files-only and --count combinations "$IX" -l "fn" "$KNOWN" > "$ix_out" 2>/dev/null || true fl_count=$(wc -l < "$ix_out" | tr -d ' ') if [ "$fl_count" -ge 1 ]; then pass "files_only_flag ($fl_count files)" else fail "files_only_flag" "no files returned" fi "$IX" -c "fn" "$KNOWN" > "$ix_out" 2>/dev/null || true count_val=$(head -1 "$ix_out") if [[ "$count_val" =~ ^[0-9]+$ ]] && [ "$count_val" -gt 0 ]; then pass "count_flag ($count_val matches)" else fail "count_flag" "invalid count: $count_val" fi # ═══════════════════════════════════════════════════════════════════════════ # SECTION 5 — CONCURRENT / RACE # ═══════════════════════════════════════════════════════════════════════════ header "SECTION 5: STRESS" # 5a: Rapid rebuilds build_index "$LARGE" for i in $(seq 1 3); do build_index "$LARGE" >/dev/null 2>&1 done if "$IX" "SAFETY" "$LARGE" >/dev/null 2>&1; then pass "rapid_rebuilds (3x)" else fail "rapid_rebuilds" "search failed after rapid rebuilds" fi # 5b: Many file types build_index "$MANYTYPES" "$IX" "findme" "$MANYTYPES" > "$ix_out" 2>/dev/null || true many_count=$(wc -l < "$ix_out" | tr -d ' ') expected=$(ls "$MANYTYPES"/* | wc -l | tr -d ' ') if [ "$many_count" -ge "$expected" ]; then pass "many_file_types ($many_count matches in $expected files)" else fail "many_file_types" "expected >=$expected got $many_count" fi # 5c: Deep directory tree build_index "$DEEP" "$IX" "deep_find" "$DEEP" > "$ix_out" 2>/dev/null || true deep_count=$(wc -l < "$ix_out" | tr -d ' ') if [ "$deep_count" -ge 20 ]; then pass "deep_paths ($deep_count matches in 20 levels)" else fail "deep_paths" "expected >=20 got $deep_count" fi # ═══════════════════════════════════════════════════════════════════════════ # SECTION 6 — DAEMON (if notify feature built) # ═══════════════════════════════════════════════════════════════════════════ header "SECTION 6: DAEMON" IXD="${IXD_BIN:-ixd}" if ! "$IX" --help 2>/dev/null | grep -q '\-\-daemon'; then skip "daemon_tests" "daemon not compiled (needs notify feature)" else # 6a: Daemon starts and exits cleanly timeout 3 "$IXD" "$KNOWN" >/dev/null 2>&1 & IXD_PID=$! sleep 1 if kill -0 "$IXD_PID" 2>/dev/null; then pass "daemon_starts" kill "$IXD_PID" 2>/dev/null || true wait "$IXD_PID" 2>/dev/null || true else fail "daemon_starts" "ixd not running after 1s" fi # 6b: Beacon is written on start build_index "$KNOWN" timeout 3 "$IXD" "$KNOWN" >/dev/null 2>&1 & IXD_PID=$! sleep 2 if [ -f "$KNOWN/.ix/beacon" ]; then pass "daemon_beacon_written" else fail "daemon_beacon_written" "beacon not found" fi kill "$IXD_PID" 2>/dev/null || true wait "$IXD_PID" 2>/dev/null || true rm -f "$KNOWN/.ix/beacon" fi # ═══════════════════════════════════════════════════════════════════════════ # REPORT # ═══════════════════════════════════════════════════════════════════════════ header "RESULTS" echo "" printf " ${GREEN}PASS${RESET}: %d ${RED}FAIL${RESET}: %d ${YELLOW}SKIP${RESET}: %d TOTAL: %d\n" "$PASS" "$FAIL" "$SKIP" "$TOTAL" echo "" if [ "$FAIL" -gt 0 ]; then echo -e " ${RED}Some tests failed.${RESET}" exit 1 else echo -e " ${GREEN}All tests passed.${RESET}" exit 0 fi