name: CI
on:
push:
branches: [main, master]
pull_request:
branches: [main, master]
workflow_dispatch:
inputs:
run_unit:
description: Unit tests & lint
type: boolean
default: true
run_integration_single:
description: Integration (single-node)
type: boolean
default: false
run_integration_cluster:
description: Integration (3-broker cluster)
type: boolean
default: false
run_integration_sasl:
description: Integration (SASL)
type: boolean
default: false
run_integration_tls:
description: Integration (TLS)
type: boolean
default: false
run_integration_kerberos:
description: Integration (Kerberos)
type: boolean
default: false
env:
CARGO_TERM_COLOR: always
RUST_BACKTRACE: 1
KAFKA_IMAGE: apache/kafka:4.3.0
jobs:
lint:
name: Lint & Format
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: dtolnay/rust-toolchain@stable
with:
components: rustfmt, clippy
- uses: Swatinem/rust-cache@v2
- run: cargo fmt -- --check
- run: cargo clippy --all-targets -- -D warnings
unit-tests:
name: Unit Tests
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: dtolnay/rust-toolchain@stable
- uses: Swatinem/rust-cache@v2
- run: cargo test --lib
msrv-check:
name: MSRV Check (Rust 1.92)
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: dtolnay/rust-toolchain@1.92
- uses: Swatinem/rust-cache@v2
- run: cargo build --all-targets
integration-single:
name: Integration (single-node)
if: ${{ !cancelled() && (github.event_name != 'workflow_dispatch' || github.event.inputs.run_integration_single == 'true') }}
runs-on: ubuntu-latest
services:
kafka:
image: apache/kafka:4.3.0
ports:
- 29092:9092
env:
KAFKA_NODE_ID: 1
KAFKA_PROCESS_ROLES: broker,controller
KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092,CONTROLLER://0.0.0.0:9093
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://127.0.0.1:29092
KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT
KAFKA_CONTROLLER_QUORUM_VOTERS: 1@127.0.0.1:9093
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
KAFKA_GROUP_COORDINATOR_REBALANCE_PROTOCOLS: classic
KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
CLUSTER_ID: MkU3OEVBNTcwNTJENDM2Qk
options: >-
--health-cmd "/opt/kafka/bin/kafka-broker-api-versions.sh --bootstrap-server 127.0.0.1:9092"
--health-interval 5s
--health-timeout 10s
--health-retries 20
steps:
- uses: actions/checkout@v4
- uses: dtolnay/rust-toolchain@stable
- uses: Swatinem/rust-cache@v2
- name: Build tests
run: cargo test --no-run --features integration_tests
- name: Run single-node tests
run: cargo test --test produce_consume --test produce_with_keys --test producer_acks --test large_batch --test multi_topic --features integration_tests -- --nocapture
env:
KAFKA_BOOTSTRAP: 127.0.0.1:29092
KAFKA_CLUSTER_SIZE: 1
RUST_TEST_THREADS: 1
integration-cluster:
name: Integration (3-broker)
if: ${{ !cancelled() && (github.event_name != 'workflow_dispatch' || github.event.inputs.run_integration_cluster == 'true') }}
runs-on: ubuntu-latest
defaults:
run:
working-directory: tests
steps:
- uses: actions/checkout@v4
- uses: dtolnay/rust-toolchain@stable
- uses: Swatinem/rust-cache@v2
- name: Build tests
working-directory: .
run: cargo test --no-run --features integration_tests
- name: Start 3-broker cluster
run: |
docker compose -f docker-compose.yml up -d
for c in kafka-1 kafka-2 kafka-3; do
echo -n " ${c}... "
for i in $(seq 1 60); do
if docker exec "${c}" \
/opt/kafka/bin/kafka-broker-api-versions.sh \
--bootstrap-server 127.0.0.1:9092 &>/dev/null; then
echo "ready (~${i}s)"
break
fi
sleep 1
done
done
- name: Run cluster-dependent tests
working-directory: .
run: cargo test --test offset_commit --test offset_reset --test consumer_group --test consumer_seek --test cluster --features integration_tests -- --nocapture
env:
KAFKA_BOOTSTRAP: 127.0.0.1:29093,127.0.0.1:29095,127.0.0.1:29097
KAFKA_CLUSTER_SIZE: 3
RUST_TEST_THREADS: 1
- name: Dump logs on failure
if: failure()
working-directory: tests
run: docker compose -f docker-compose.yml logs
- name: Stop cluster
if: always()
working-directory: tests
run: docker compose -f docker-compose.yml down -v
integration-sasl:
name: Integration (SASL)
if: ${{ !cancelled() && (github.event_name != 'workflow_dispatch' || github.event.inputs.run_integration_sasl == 'true') }}
runs-on: ubuntu-latest
defaults:
run:
working-directory: tests
steps:
- uses: actions/checkout@v4
- uses: dtolnay/rust-toolchain@stable
- uses: Swatinem/rust-cache@v2
- name: Build tests
working-directory: .
run: cargo test --no-run --features integration_tests
- name: Start SASL broker
run: |
docker compose -f docker-compose.sasl.yml up -d
echo -n " kafka-sasl-broker... "
for i in $(seq 1 60); do
if docker exec kafka-sasl-broker \
bash -c "echo > /dev/tcp/127.0.0.1/9094" 2>/dev/null; then
sleep 5
echo "ready (~${i}s)"
break
fi
sleep 1
done
- name: Run SASL auth tests
working-directory: .
run: cargo test --test auth --features integration_tests -- --nocapture
env:
KAFKA_BOOTSTRAP_SASL: 127.0.0.1:9094
SASL_MECHANISM: PLAIN
SASL_USERNAME: admin
SASL_PASSWORD: admin-secret
RUST_TEST_THREADS: 1
- name: Dump logs on failure
if: failure()
working-directory: tests
run: docker compose -f docker-compose.sasl.yml logs
- name: Stop SASL broker
if: always()
working-directory: tests
run: docker compose -f docker-compose.sasl.yml down -v
integration-tls:
name: Integration (TLS)
if: ${{ !cancelled() && (github.event_name != 'workflow_dispatch' || github.event.inputs.run_integration_tls == 'true') }}
runs-on: ubuntu-latest
defaults:
run:
working-directory: tests
steps:
- uses: actions/checkout@v4
- uses: dtolnay/rust-toolchain@stable
- uses: Swatinem/rust-cache@v2
- name: Build tests
working-directory: .
run: cargo test --no-run --features integration_tests
- name: Generate TLS certificates
run: ./gen-certs.sh
- name: Start TLS broker
run: |
docker compose -f docker-compose.tls.yml up -d
echo -n " kafka-tls-broker... "
for i in $(seq 1 60); do
if docker exec kafka-tls-broker \
bash -c "echo > /dev/tcp/127.0.0.1/9093" 2>/dev/null; then
sleep 8
echo "ready (~${i}s)"
break
fi
sleep 1
done
- name: Run TLS tests
working-directory: .
run: cargo test --test tls --features integration_tests -- --nocapture
env:
KAFKA_BOOTSTRAP_TLS: 127.0.0.1:9093
RUST_TEST_THREADS: 1
- name: Dump logs on failure
if: failure()
working-directory: tests
run: docker compose -f docker-compose.tls.yml logs
- name: Stop TLS broker
if: always()
working-directory: tests
run: docker compose -f docker-compose.tls.yml down -v
integration-kerberos:
name: Integration (Kerberos)
if: ${{ !cancelled() && (github.event_name != 'workflow_dispatch' || github.event.inputs.run_integration_kerberos == 'true') }}
runs-on: ubuntu-latest
defaults:
run:
working-directory: tests
steps:
- uses: actions/checkout@v4
- uses: dtolnay/rust-toolchain@stable
- uses: Swatinem/rust-cache@v2
- name: Build tests
working-directory: .
run: cargo test --no-run --features integration_tests
- name: Start KDC + Kerberos Kafka broker
run: |
docker compose -f docker-compose.kerberos.yml up -d --build
echo -n " kafka-kerberos-broker... "
for i in $(seq 1 120); do
if docker exec kafka-kerberos-broker \
/opt/kafka/bin/kafka-broker-api-versions.sh \
--bootstrap-server 127.0.0.1:9096 &>/dev/null; then
echo "ready (~${i}s)"
break
fi
if [ "$i" -eq 120 ]; then
echo "timeout"
exit 1
fi
sleep 2
done
- name: Wait for keytab
run: |
for i in $(seq 1 30); do
if [ -f fixtures/kerberos/keytabs/client.keytab ]; then
echo "keytab ready (~${i}s)"
break
fi
if [ "$i" -eq 30 ]; then
echo "keytab timeout"
exit 1
fi
sleep 2
done
- name: Run Kerberos tests
working-directory: .
run: >
cargo test --test kerberos --test kerberos_service_ticket
--features integration_tests -- --nocapture
env:
KAFKA_BOOTSTRAP_KERBEROS: 127.0.0.1:9096
KERBEROS_KEYTAB: tests/fixtures/kerberos/keytabs/client.keytab
KERBEROS_KDC_HOST: localhost
KERBEROS_KDC_PORT: "8888"
KAFKA_CLUSTER_SIZE: 1
RUST_TEST_THREADS: 1
- name: Dump logs on failure
if: failure()
working-directory: tests
run: docker compose -f docker-compose.kerberos.yml logs
- name: Stop cluster
if: always()
working-directory: tests
run: docker compose -f docker-compose.kerberos.yml down -v