infino 0.3.2

name: CI

on:
  push:
    branches: [main]
  pull_request:
    branches: [main]

env:
  CARGO_TERM_COLOR: always
  RUSTFLAGS: -D warnings

jobs:
  check:
    name: Format, Build & Test
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      # The default github-hosted ubuntu runner has ~14 GB free at job
      # start; infino's dep graph (datafusion + lance + tantivy)
      # plus `cargo llvm-cov`'s separate `target/llvm-cov-target/`
      # directory (it instruments a second full build of everything)
      # overflows that, so the coverage step fails with ENOSPC. Free
      # ~30 GB by removing toolchains the build doesn't need (Android
      # SDK, .NET, Haskell, ...) before any compile work starts.
      - uses: jlumbroso/free-disk-space@main
        with:
          tool-cache: false
          android: true
          dotnet: true
          haskell: true
          large-packages: true
          docker-images: true
          swap-storage: true
      - uses: dtolnay/rust-toolchain@stable
        with:
          components: rustfmt
      - uses: Swatinem/rust-cache@v2
        with:
          cache-directories: |
            ~/.cargo/bin
            target/infino-bench/rustfs
      - name: Install cargo-llvm-cov
        run: |
          if ! command -v cargo-llvm-cov &> /dev/null; then
            cargo install cargo-llvm-cov --locked
          fi
      - run: make ci
      - name: RustFS storage wire + smoke tests
        run: cargo test --features test-helpers --test supertable 'storage::rustfs_s3_wire::|storage::smoke_rustfs::' -- --test-threads=1

  public-api:
    name: Public API surface
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      # cargo-public-api builds rustdoc JSON, which requires nightly.
      - uses: dtolnay/rust-toolchain@nightly
      - uses: Swatinem/rust-cache@v2
        with:
          cache-directories: |
            ~/.cargo/bin
      - name: Install cargo-public-api
        run: |
          if ! command -v cargo-public-api &> /dev/null; then
            cargo install cargo-public-api --locked
          fi
      # Fails if the curated public surface drifts from the committed
      # `public-api.txt` snapshot. The snapshot is taken without
      # `test-helpers`, so internal modules stay off the contract.
      - run: make public-api

  python:
    name: Python wheel & tests (${{ matrix.os }})
    runs-on: ${{ matrix.os }}
    # The bindings build `infino` as a normal dependency; don't fail the
    # build on a transitive-dependency deprecation warning. infino's own
    # warnings are gated by the `check` job (against infino's lockfile).
    env:
      RUSTFLAGS: ""
    strategy:
      fail-fast: false
      # Native host build (no cross-compile); one abi3 wheel each, CPython >= 3.9.
      matrix:
        os: [ubuntu-latest, ubuntu-24.04-arm, macos-14]
    steps:
      - uses: actions/checkout@v4
      - name: Free disk space
        if: runner.os == 'Linux'
        uses: jlumbroso/free-disk-space@main
        with:
          tool-cache: false
          android: true
          dotnet: true
          haskell: true
          large-packages: true
          docker-images: true
          swap-storage: true
      - uses: dtolnay/rust-toolchain@stable
      - uses: actions/setup-python@v5
        with:
          python-version: "3.12"
      - uses: Swatinem/rust-cache@v2
        with:
          workspaces: infino-python
      # Release abi3 wheel — the shipped artifact.
      - run: make python-wheel
      # Test the wheel via the advertised `uv pip install` path.
      # duckdb keeps the Parquet-interop test from being skipped.
      - name: Smoke-test wheel with uv
        shell: bash
        run: |
          python3 -m pip install --upgrade uv
          uv venv /tmp/uv-smoke
          uv pip install --python /tmp/uv-smoke infino-python/dist/*.whl pytest pyarrow pandas duckdb
          /tmp/uv-smoke/bin/python -m pytest infino-python/tests/ -v
      # mypy --strict over the stubs; platform-independent, so run once.
      - name: Type-check stubs
        if: matrix.os == 'ubuntu-latest'
        run: make python-typecheck
      - uses: actions/upload-artifact@v4
        with:
          name: wheel-${{ matrix.os }}
          path: infino-python/dist

  python-examples:
    name: Python examples (end to end)
    runs-on: ubuntu-latest
    timeout-minutes: 45
    # Build infino as a normal dep — don't fail on a transitive deprecation warning.
    env:
      RUSTFLAGS: ""
      # Azure LLM creds; absent on fork PRs, where notebooks print context instead.
      AZURE_AI_ENDPOINT: ${{ secrets.AZURE_AI_ENDPOINT }}
      AZURE_AI_API_KEY: ${{ secrets.AZURE_AI_API_KEY }}
      DEFAULT_AZURE_MODEL: gpt-5.4
    steps:
      - uses: actions/checkout@v4
      - name: Free disk space
        uses: jlumbroso/free-disk-space@main
        with:
          tool-cache: false
          android: true
          dotnet: true
          haskell: true
          large-packages: true
          docker-images: true
          swap-storage: true
      - uses: dtolnay/rust-toolchain@stable
      - uses: actions/setup-python@v5
        with:
          python-version: "3.12"
      - uses: Swatinem/rust-cache@v2
        with:
          workspaces: infino-python
      # Cache the model + dataset samples the notebooks download.
      - name: Cache HuggingFace assets
        uses: actions/cache@v4
        with:
          path: ~/.cache/huggingface
          key: hf-${{ runner.os }}-${{ hashFiles('infino-python/examples/_shared/loaders.py', 'infino-python/examples/_shared/embedding.py') }}
          restore-keys: hf-${{ runner.os }}-
      # Run every notebook; the LLM answer step runs when the secrets are present.
      - run: make python-examples-test

  node:
    name: Node bindings (${{ matrix.os }})
    runs-on: ${{ matrix.os }}
    # Same rationale as the Python job: the bindings build `infino` as a
    # normal dependency, so don't fail on a transitive deprecation warning.
    env:
      RUSTFLAGS: ""
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-latest, macos-14]
    steps:
      - uses: actions/checkout@v4
      - name: Free disk space
        if: runner.os == 'Linux'
        uses: jlumbroso/free-disk-space@main
        with:
          tool-cache: false
          android: true
          dotnet: true
          haskell: true
          large-packages: true
          docker-images: true
          swap-storage: true
      - uses: dtolnay/rust-toolchain@stable
      - uses: actions/setup-node@v4
        with:
          node-version: "20"
      - uses: Swatinem/rust-cache@v2
        with:
          workspaces: infino-node
      # Builds the addon (debug) + runs the node:test smoke suite.
      - run: make node-test

  # End-to-end: run the Node examples against a freshly built addon (mirrors the
  # python-examples job). They fetch real public datasets and download a model,
  # so this is its own parallel job — it never slows the `node` smoke job — and is
  # meant to stay NON-required: a Hub outage shouldn't block a merge. The
  # deterministic gate is the `node` job's `make node-test`.
  node-examples:
    name: Node examples (end to end)
    runs-on: ubuntu-latest
    timeout-minutes: 30
    # Build infino as a normal dep — don't fail on a transitive deprecation warning.
    env:
      RUSTFLAGS: ""
    steps:
      - uses: actions/checkout@v4
      - name: Free disk space
        uses: jlumbroso/free-disk-space@main
        with:
          tool-cache: false
          android: true
          dotnet: true
          haskell: true
          large-packages: true
          docker-images: true
          swap-storage: true
      - uses: dtolnay/rust-toolchain@stable
      - uses: actions/setup-node@v4
        with:
          node-version: "20"
      - uses: Swatinem/rust-cache@v2
        with:
          workspaces: infino-node
      # Build the addon (debug); the examples' `file:../..` dependency links it.
      - run: cd infino-node && npm install && npm run build:debug
      # Run each example end to end (self-asserting; non-zero exit fails the job).
      - run: make node-example

  # LOCOMO recall tripwire — runs the locomo-recall example against the committed
  # fixture (frozen vectors → only the engine varies) and reports recall@k on real
  # long-term-memory queries. Complements the brute-force recall gate in `check`
  # (which proves the ANN index returns the true neighbours); this proves
  # ranking/fusion quality holds up on real questions. Cheap — no embedder or
  # network. Keep it out of branch protection's required set so a recall wobble
  # never blocks a merge; tighten --fail-under as the baseline firms up.
  locomo-recall:
    name: LOCOMO recall test
    runs-on: ubuntu-latest
    timeout-minutes: 30
    steps:
      - uses: actions/checkout@v4
      - name: Free disk space
        uses: jlumbroso/free-disk-space@main
        with:
          tool-cache: false
          android: true
          dotnet: true
          haskell: true
          large-packages: true
          docker-images: true
          swap-storage: true
      - uses: dtolnay/rust-toolchain@stable
      - uses: Swatinem/rust-cache@v2
      # Frozen fixture → no embedder, key, or network. Reports recall@k + every
      # hybrid miss in the log, and fails only below the tolerance floor.
      - run: cargo run --example locomo-recall -- --fail-under=0.68

  # Gate for `object-store-e2e`: object-store secrets are absent on fork
  # PRs, so probe for them once and let the e2e job key off the result (a
  # job-level `if` can't read secrets directly).
  object-store-e2e-guard:
    name: Object store e2e (secret check)
    runs-on: ubuntu-latest
    environment: ci
    outputs:
      azure: ${{ steps.check.outputs.azure }}
      gcs: ${{ steps.check.outputs.gcs }}
      any: ${{ steps.check.outputs.any }}
    steps:
      - id: check
        run: |
          azure=false; gcs=false
          if [ -n "${{ secrets.AZURE_STORAGE_ACCOUNT_KEY }}" ] && [ -n "${{ secrets.INFINO_REAL_AZURE_CONTAINER }}" ]; then azure=true; fi
          if [ -n "${{ secrets.GCP_SA_KEY }}" ] && [ -n "${{ secrets.INFINO_REAL_GCS_BUCKET }}" ]; then gcs=true; fi
          any=false; if [ "$azure" = true ] || [ "$gcs" = true ]; then any=true; fi
          { echo "azure=$azure"; echo "gcs=$gcs"; echo "any=$any"; } >> "$GITHUB_OUTPUT"

  # Engine end-to-end against real object stores: the Python and Node bindings
  # run the full storage_options + connect-probe path over the wire, one step
  # per backend (Azure, GCS). Bindings are built once and reused. Each backend
  # step runs only when its secrets are present, so a fork PR (no secrets) or a
  # single-backend outage never blocks a merge; the bindings' smoke suites are
  # the deterministic gate.
  object-store-e2e:
    name: Object store e2e
    needs: object-store-e2e-guard
    if: needs.object-store-e2e-guard.outputs.any == 'true'
    runs-on: ubuntu-latest
    environment: ci
    timeout-minutes: 30
    env:
      RUSTFLAGS: ""
      INFINO_TEST_REAL_AZURE: "1"
      INFINO_TEST_REAL_GCS: "1"
      AZURE_STORAGE_ACCOUNT_NAME: ${{ secrets.AZURE_STORAGE_ACCOUNT_NAME }}
      AZURE_STORAGE_ACCOUNT_KEY: ${{ secrets.AZURE_STORAGE_ACCOUNT_KEY }}
      AZURE_STORAGE_CONTAINER_NAME: ${{ secrets.INFINO_REAL_AZURE_CONTAINER }}
      INFINO_REAL_GCS_BUCKET: ${{ secrets.INFINO_REAL_GCS_BUCKET }}
      GOOGLE_SERVICE_ACCOUNT_KEY: ${{ secrets.GCP_SA_KEY }}
      # Run-scoped so cleanup can't touch a parallel run's objects.
      INFINO_E2E_PREFIX: e2e/${{ github.run_id }}
    steps:
      - uses: actions/checkout@v4
      - name: Free disk space
        uses: jlumbroso/free-disk-space@main
        with:
          tool-cache: false
          android: true
          dotnet: true
          haskell: true
          large-packages: true
          docker-images: true
          swap-storage: true
      - uses: dtolnay/rust-toolchain@stable
      - uses: actions/setup-python@v5
        with:
          python-version: "3.12"
      - uses: actions/setup-node@v4
        with:
          node-version: "20"
      - uses: Swatinem/rust-cache@v2
      # Build both bindings once; every backend step below reuses them.
      - name: Build Python binding
        run: |
          python3 -m pip install --upgrade uv
          uv venv /tmp/uv-object-store-e2e
          source /tmp/uv-object-store-e2e/bin/activate
          uv pip install maturin pytest pyarrow
          maturin develop --manifest-path infino-python/Cargo.toml
      - name: Build Node binding
        working-directory: infino-node
        run: |
          npm install
          npm run build:debug
      # --- Azure ---
      - name: Azure · Python
        if: needs.object-store-e2e-guard.outputs.azure == 'true'
        run: |
          source /tmp/uv-object-store-e2e/bin/activate
          python -m pytest infino-python/tests/test_azure_e2e.py -v
      - name: Azure · Node
        if: needs.object-store-e2e-guard.outputs.azure == 'true'
        working-directory: infino-node
        run: node --test __test__/azure.e2e.test.mjs
      # --- GCS ---
      - name: GCS · Python
        if: needs.object-store-e2e-guard.outputs.gcs == 'true'
        run: |
          source /tmp/uv-object-store-e2e/bin/activate
          python -m pytest infino-python/tests/test_gcs_e2e.py -v
      - name: GCS · Node
        if: needs.object-store-e2e-guard.outputs.gcs == 'true'
        working-directory: infino-node
        run: node --test __test__/gcs.e2e.test.mjs
      # Per-backend cleanup of this run's prefix, incl. the catalog object
      # drop_table leaves. Best-effort: a hiccup must not fail the job.
      - name: Clean up Azure blobs
        if: always() && needs.object-store-e2e-guard.outputs.azure == 'true'
        continue-on-error: true
        run: |
          az storage blob delete-batch \
            --account-name "$AZURE_STORAGE_ACCOUNT_NAME" \
            --account-key "$AZURE_STORAGE_ACCOUNT_KEY" \
            --source "$AZURE_STORAGE_CONTAINER_NAME" \
            --pattern "$INFINO_E2E_PREFIX/*"
      - name: Clean up GCS objects
        if: always() && needs.object-store-e2e-guard.outputs.gcs == 'true'
        continue-on-error: true
        run: |
          printf '%s' "$GOOGLE_SERVICE_ACCOUNT_KEY" > /tmp/gcs-key.json
          gcloud auth activate-service-account --key-file=/tmp/gcs-key.json
          gcloud storage rm -r "gs://$INFINO_REAL_GCS_BUCKET/$INFINO_E2E_PREFIX/" || true
          rm -f /tmp/gcs-key.json