name: ci
on:
pull_request: {}
push:
tags:
- "v*"
branches:
- main
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
env:
CARGO_TERM_COLOR: always
jobs:
build:
name: build / ${{ matrix.profile }}
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
include:
- profile: debug
make_target: build
cache_suffix: ""
- profile: release
make_target: build-release
cache_suffix: "-release"
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 - uses: dtolnay/rust-toolchain@631a55b12751854ce901bb631d5902ceb48146f7 - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 with:
path: |
~/.cargo/registry
~/.cargo/git
target
key: ${{ runner.os }}-cargo${{ matrix.cache_suffix }}-${{ hashFiles('**/Cargo.lock') }}
- run: make ${{ matrix.make_target }}
build-cross:
name: build / cross / ${{ matrix.arch }}
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
include:
- arch: x86_64
target: x86_64-unknown-linux-gnu
strip: strip
- arch: aarch64
target: aarch64-unknown-linux-gnu
strip: aarch64-linux-gnu-strip
- arch: armv7
target: armv7-unknown-linux-gnueabihf
strip: arm-linux-gnueabihf-strip
- arch: ppc64le
target: powerpc64le-unknown-linux-gnu
strip: powerpc64le-linux-gnu-strip
- arch: s390x
target: s390x-unknown-linux-gnu
strip: s390x-linux-gnu-strip
- arch: riscv64
target: riscv64gc-unknown-linux-gnu
strip: riscv64-linux-gnu-strip
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 - uses: dtolnay/rust-toolchain@631a55b12751854ce901bb631d5902ceb48146f7 - uses: taiki-e/install-action@7f4eb899022d8fe70b20c4f3de697aa85c309026 with:
tool: cross
- uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 with:
path: |
~/.cargo/registry
~/.cargo/git
target
key: ${{ runner.os }}-cargo-${{ matrix.target }}-${{ hashFiles('**/Cargo.lock') }}
- run: make build-cross CROSS_TARGET=${{ matrix.target }}
- name: Strip binary
run: |
docker run --rm \
-v $PWD/target/${{ matrix.target }}/release:/target \
ghcr.io/cross-rs/${{ matrix.target }}:main \
${{ matrix.strip }} -s /target/kubernix
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a with:
name: kubernix-${{ matrix.arch }}
path: target/${{ matrix.target }}/release/kubernix
deploy:
name: deploy / release
runs-on: ubuntu-latest
needs: build-cross
if: startsWith(github.ref, 'refs/tags/v')
permissions:
contents: write
steps:
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c with:
path: artifacts
- name: Prepare release binaries
run: |
mkdir -p release
for dir in artifacts/kubernix-*; do
target=$(basename "$dir" | sed 's/^kubernix-//')
cp "$dir/kubernix" "release/kubernix-$target"
done
ls -la release/
- name: Upload release binaries
uses: softprops/action-gh-release@3d0d9888cb7fd7b750713d6e236d1fcb99157228 with:
files: release/*
docs:
name: docs / build
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 - uses: dtolnay/rust-toolchain@631a55b12751854ce901bb631d5902ceb48146f7 - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 with:
path: |
~/.cargo/registry
~/.cargo/git
target
key: ${{ runner.os }}-cargo-docs-${{ hashFiles('**/Cargo.lock') }}
- run: make docs
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a with:
name: docs
path: target/doc
docs-publish:
name: docs / publish
runs-on: ubuntu-latest
needs: docs
if: github.ref == 'refs/heads/main'
permissions:
contents: write
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c with:
name: docs
path: doc
- uses: peaceiris/actions-gh-pages@84c30a85c19949d7eee79c4ff27748b70285e453 with:
github_token: ${{ secrets.GITHUB_TOKEN }}
publish_dir: ./doc
lint-clippy:
name: lint / clippy
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 - uses: dtolnay/rust-toolchain@631a55b12751854ce901bb631d5902ceb48146f7 with:
components: clippy
- uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 with:
path: |
~/.cargo/registry
~/.cargo/git
target
key: ${{ runner.os }}-cargo-clippy-${{ hashFiles('**/Cargo.lock') }}
- run: make lint-clippy
lint-rustfmt:
name: lint / rustfmt
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 - uses: dtolnay/rust-toolchain@631a55b12751854ce901bb631d5902ceb48146f7 with:
components: rustfmt
- run: make lint-rustfmt
lint-audit:
name: lint / audit
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 - uses: taiki-e/install-action@7f4eb899022d8fe70b20c4f3de697aa85c309026 with:
tool: cargo-audit
- run: make lint-audit
lint-deny:
name: lint / deny
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 - uses: EmbarkStudios/cargo-deny-action@3c6349835b2b7b196a839186cb8b78e02f7b5f25
lint-dependencies:
name: lint / dependencies
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 - run: make lint-dependencies
test-msrv:
name: test / msrv
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 - name: Read MSRV from Cargo.toml
id: msrv
run: echo "version=$(grep '^rust-version' Cargo.toml | sed 's/.*"\(.*\)"/\1/')" >> "$GITHUB_OUTPUT"
- uses: dtolnay/rust-toolchain@631a55b12751854ce901bb631d5902ceb48146f7
with:
toolchain: ${{ steps.msrv.outputs.version }}
- uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 with:
path: |
~/.cargo/registry
~/.cargo/git
target
key: ${{ runner.os }}-cargo-msrv-${{ hashFiles('**/Cargo.lock') }}
- name: Install cfssl
run: contrib/install-cfssl
- run: cargo test --lib
test-unit:
name: test / unit
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 - uses: dtolnay/rust-toolchain@631a55b12751854ce901bb631d5902ceb48146f7 - uses: taiki-e/install-action@7f4eb899022d8fe70b20c4f3de697aa85c309026 with:
tool: cargo-llvm-cov
- uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 with:
path: |
~/.cargo/registry
~/.cargo/git
target
key: ${{ runner.os }}-cargo-test-${{ hashFiles('**/Cargo.lock') }}
- name: Install cfssl
run: contrib/install-cfssl
- name: Generate code coverage
run: cargo llvm-cov --all-features --lib --lcov --output-path lcov.info
- name: Upload Results
uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f with:
files: lcov.info
token: ${{ secrets.CODECOV_TOKEN }}
test-integration:
name: test / integration / ${{ matrix.cri_runtime }} / ${{ matrix.mode }}${{ matrix.rootless && ' / rootless' || '' }}
runs-on: ubuntu-latest
needs: build-cross
strategy:
fail-fast: false
matrix:
include:
- mode: single
nodes: 1
cri_runtime: crio
rootless: false
- mode: multi
nodes: 2
cri_runtime: crio
rootless: false
- mode: single
nodes: 1
cri_runtime: containerd
rootless: false
- mode: multi
nodes: 2
cri_runtime: containerd
rootless: false
- mode: single
nodes: 1
cri_runtime: crio
rootless: true
- mode: single
nodes: 1
cri_runtime: containerd
rootless: true
- mode: multi
nodes: 2
cri_runtime: crio
rootless: true
- mode: multi
nodes: 2
cri_runtime: containerd
rootless: true
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 - uses: cachix/install-nix-action@630ae543ea3a38a9a4166f03376c02c50f408342 with:
extra_nix_config: |
experimental-features = nix-command flakes
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c with:
name: kubernix-x86_64
- run: chmod +x kubernix
- name: Set hostname
if: ${{ !matrix.rootless }}
run: |
echo "127.0.0.1 test" | sudo tee -a /etc/hosts
sudo hostnamectl set-hostname test
- name: Free port 53 for CoreDNS
run: |
echo 'nameserver 8.8.8.8' | sudo tee /etc/resolv.conf
sudo systemctl stop systemd-resolved || true
- name: Prepare the system
if: ${{ !matrix.rootless }}
run: sudo contrib/prepare-system
- name: Prepare rootless environment
if: ${{ matrix.rootless }}
run: |
sudo sh -c 'echo "kernel.unprivileged_userns_clone=1" > /etc/sysctl.d/rootless.conf && sysctl --system'
sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 || true
sudo sysctl -w net.ipv4.ip_unprivileged_port_start=53
sudo systemctl disable --now apparmor || true
sudo sh -c 'mkdir -p /etc/systemd/system/user-.slice.d && printf "[Slice]\nDelegate=yes\n" > /etc/systemd/system/user-.slice.d/delegate.conf && systemctl daemon-reload'
- name: Run integration test
run: |
${{ !matrix.rootless && 'sudo -E env "PATH=$PATH"' || '' }} ./kubernix --log-level=debug --no-shell --nodes=${{ matrix.nodes }} --cri-runtime=${{ matrix.cri_runtime }} &
KUBERNIX_PID=$!
timeout 600 bash -c 'while [ ! -f kubernix-run/kubernix.pid ]; do sleep 1; done'
echo "Cluster is up, running checks"
export KUBECONFIG=$PWD/kubernix-run/kubeconfig/admin.kubeconfig
echo "Asserting all nodes are Ready..."
READY_COUNT=$(kubectl get nodes -o go-template='{{range .items}}{{range .status.conditions}}{{if eq .type "Ready"}}{{if eq .status "True"}}x{{end}}{{end}}{{end}}{{end}}' | wc -c)
if [ "$READY_COUNT" -ne "${{ matrix.nodes }}" ]; then
echo "FAIL: expected ${{ matrix.nodes }} ready nodes, got $READY_COUNT"
exit 1
fi
echo "Asserting all pods are healthy..."
timeout 60 bash -c 'until [ "$(kubectl get pods -A -o go-template="{{range .items}}{{if and (ne .status.phase \"Running\") (ne .status.phase \"Succeeded\")}}x{{end}}{{end}}" | wc -c)" -eq 0 ]; do sleep 2; done'
UNHEALTHY=$(kubectl get pods -A -o go-template='{{range .items}}{{if and (ne .status.phase "Running") (ne .status.phase "Succeeded")}}{{.metadata.name}}({{.status.phase}}) {{end}}{{end}}')
if [ -n "$UNHEALTHY" ]; then
echo "FAIL: unhealthy pods: $UNHEALTHY"
exit 1
fi
echo "Asserting CoreDNS is ready..."
if ! timeout 110 bash -c 'until kubectl -n kube-system wait --for=condition=Ready pod -l k8s-app=coredns --timeout=5s 2>/dev/null; do sleep 2; done'; then
echo "=== CoreDNS pod describe ==="
kubectl describe pod -n kube-system -l k8s-app=coredns || true
echo "=== CoreDNS logs ==="
kubectl logs -n kube-system -l k8s-app=coredns --tail=100 || true
echo "=== kubernetes endpoints ==="
kubectl get endpoints kubernetes || true
exit 1
fi
echo "CoreDNS is ready"
echo "All assertions passed"
${{ !matrix.rootless && 'sudo' || '' }} kill $KUBERNIX_PID
wait $KUBERNIX_PID || true
timeout-minutes: 15