name: Main Catnip
concurrency:
group: catnip
cancel-in-progress: false
on:
pull_request:
types: [opened, synchronize]
push:
branches:
- main
- unstable
- dev
env:
LIBOS: catnip
SERVER: ${{ secrets.CATNIP_HOSTNAME_A }}
CLIENT: ${{ secrets.CATNIP_HOSTNAME_B }}
SERVER_ADDR: 10.3.1.50
CLIENT_ADDR: 10.3.1.51
jobs:
debug-pipeline:
name: Debug Pipeline
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Setup SSH
shell: bash
run: |
mkdir -p $HOME/.ssh/
echo "${{ secrets.SSHKEY }}" > "$HOME/.ssh/id_rsa"
chmod 400 $HOME/.ssh/id_rsa
echo "Host *" > $HOME/.ssh/config
echo -e "\tStrictHostKeyChecking no" >> $HOME/.ssh/config
echo -e "\tIdentityFile $HOME/.ssh/id_rsa" >> $HOME/.ssh/config
echo -e "\tIdentitiesOnly yes" >> $HOME/.ssh/config
echo -e "\tPasswordAuthentication no" >> $HOME/.ssh/config
echo -e "\tUser ${{ secrets.USERNAME }}" >> $HOME/.ssh/config
echo -e "\tPort ${{ secrets.PORTNUM }}" >> $HOME/.ssh/config
- name: Bootstrap VMs
shell: bash
run: |
echo "Running azure.sh on SERVER"
ssh $SERVER "sudo bash ~/demikernel/scripts/setup/azure.sh"
echo "Running hugepages.sh on SERVER"
ssh $SERVER "sudo bash ~/demikernel/scripts/setup/hugepages.sh"
echo "Running ip addr flush on SERVER"
ssh $SERVER "sudo ip -br addr show dev eth1 | grep -q ' UP ' && \
sudo ip addr flush dev eth1"
echo "Running azure.sh on CLIENT"
ssh $CLIENT "sudo bash ~/demikernel/scripts/setup/azure.sh"
echo "Running hugepages.sh on CLIENT"
ssh $CLIENT "sudo bash ~/demikernel/scripts/setup/hugepages.sh"
echo "Running ip addr flush on CLIENT"
ssh $CLIENT "sudo ip -br addr show dev eth1 | grep -q ' UP ' && \
sudo ip addr flush dev eth1"
- name: Run
run: |
if [[ "${{ github.event_name }}" == "pull_request" ]]; then
branch_name="${{ github.head_ref }}"
else
branch_name="${{ github.ref_name }}"
fi
python3 tools/demikernel_ci.py \
--platform linux \
--server $SERVER \
--client $CLIENT \
--repository demikernel \
--branch origin/$branch_name \
--debug \
--libos $LIBOS \
--test-unit --test-integration --test-system all \
--delay 2 \
--server-addr $SERVER_ADDR \
--client-addr $CLIENT_ADDR
- name: Archive Logs
if: always()
uses: actions/upload-artifact@v4
with:
name: debug-pipeline-logs
path: |
**/*.stdout.txt
**/*.stderr.txt
release-pipeline:
name: Release Pipeline
needs: debug-pipeline
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Setup SSH
shell: bash
run: |
mkdir -p $HOME/.ssh/
echo "${{ secrets.SSHKEY }}" > "$HOME/.ssh/id_rsa"
chmod 400 $HOME/.ssh/id_rsa
echo "Host *" > $HOME/.ssh/config
echo -e "\tStrictHostKeyChecking no" >> $HOME/.ssh/config
echo -e "\tIdentityFile $HOME/.ssh/id_rsa" >> $HOME/.ssh/config
echo -e "\tIdentitiesOnly yes" >> $HOME/.ssh/config
echo -e "\tPasswordAuthentication no" >> $HOME/.ssh/config
echo -e "\tUser ${{ secrets.USERNAME }}" >> $HOME/.ssh/config
echo -e "\tPort ${{ secrets.PORTNUM }}" >> $HOME/.ssh/config
- name: Bootstrap VMs
shell: bash
run: |
echo "Running azure.sh on SERVER"
ssh $SERVER "sudo bash ~/demikernel/scripts/setup/azure.sh"
echo "Running hugepages.sh on SERVER"
ssh $SERVER "sudo bash ~/demikernel/scripts/setup/hugepages.sh"
echo "Running ip addr flush on SERVER"
ssh $SERVER "sudo ip -br addr show dev eth1 | grep -q ' UP ' && \
sudo ip addr flush dev eth1"
echo "Running azure.sh on CLIENT"
ssh $CLIENT "sudo bash ~/demikernel/scripts/setup/azure.sh"
echo "Running hugepages.sh on CLIENT"
ssh $CLIENT "sudo bash ~/demikernel/scripts/setup/hugepages.sh"
echo "Running ip addr flush on CLIENT"
ssh $CLIENT "sudo ip -br addr show dev eth1 | grep -q ' UP ' && \
sudo ip addr flush dev eth1"
- name: Run
run: |
if [[ "${{ github.event_name }}" == "pull_request" ]]; then
branch_name="${{ github.head_ref }}"
else
branch_name="${{ github.ref_name }}"
fi
python3 tools/demikernel_ci.py \
--platform linux \
--server $SERVER \
--client $CLIENT \
--repository demikernel \
--branch origin/$branch_name \
--libos $LIBOS \
--test-unit --test-integration --test-system all \
--delay 2 \
--server-addr $SERVER_ADDR \
--client-addr $CLIENT_ADDR
- name: Archive Logs
if: always()
uses: actions/upload-artifact@v4
with:
name: release-pipeline-logs
path: |
**/*.stdout.txt
**/*.stderr.txt
redis-pipeline:
name: Redis Pipeline
needs: release-pipeline
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Setup SSH
shell: bash
run: |
mkdir -p $HOME/.ssh/
echo "${{ secrets.SSHKEY }}" > "$HOME/.ssh/id_rsa"
chmod 400 $HOME/.ssh/id_rsa
echo "Host *" > $HOME/.ssh/config
echo -e "\tStrictHostKeyChecking no" >> $HOME/.ssh/config
echo -e "\tIdentityFile $HOME/.ssh/id_rsa" >> $HOME/.ssh/config
echo -e "\tIdentitiesOnly yes" >> $HOME/.ssh/config
echo -e "\tPasswordAuthentication no" >> $HOME/.ssh/config
echo -e "\tUser ${{ secrets.USERNAME }}" >> $HOME/.ssh/config
echo -e "\tPort ${{ secrets.PORTNUM }}" >> $HOME/.ssh/config
- name: Run
run: |
if [[ "${{ github.event_name }}" == "pull_request" ]]; then
branch_name="${{ github.head_ref }}"
else
branch_name="${{ github.ref_name }}"
fi
python3 tools/demikernel_ci.py \
--platform linux \
--server $SERVER \
--client $CLIENT \
--repository demikernel \
--branch origin/$branch_name \
--libos $LIBOS \
--test-redis --delay 10 \
--server-addr $SERVER_ADDR \
--client-addr $CLIENT_ADDR
- name: Archive Logs
if: always()
uses: actions/upload-artifact@v4
with:
name: redis-pipeline-logs
path: |
**/*.stdout.txt
**/*.stderr.txt
report-performance:
name: Report Performance
needs: [ redis-pipeline, release-pipeline ]
runs-on: ubuntu-latest
permissions:
actions: read
contents: read
deployments: read
packages: none
pull-requests: write
security-events: write
steps:
- name: Checkout
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Download Release Pipeline Logs
uses: actions/download-artifact@v4
with:
name: release-pipeline-logs
path: ./release_pipeline_logs
- name: Setup FlameGraph Repository
run: |
git clone https://github.com/brendangregg/FlameGraph.git /tmp/FlameGraph
- name: Parse Statistics
id: parse-stats
run: |
if [[ "${{ github.event_name }}" == "pull_request" ]]; then
branch_name="${{ github.head_ref }}"
else
branch_name="${{ github.ref_name }}"
fi
pip install pandas tabulate
stats=$(python3 tools/perf.py \
--branch origin/$branch_name \
--libos $LIBOS \
--log-dir ./release_pipeline_logs)
if [[ "${{ github.event_name }}" == "pull_request" ]]; then
EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64)
echo "MESSAGE<<$EOF" >> $GITHUB_OUTPUT
echo "$stats" >> $GITHUB_OUTPUT
echo "$EOF" >> $GITHUB_OUTPUT
fi
- name: Archive Performance Data
uses: actions/upload-artifact@v4
with:
name: performance-data
path: |
**/perf_data.csv
**/flamegraph.svg
- name: Report Statistics
if: github.event_name == 'pull_request'
uses: actions/github-script@v7
with:
github-token: ${{secrets.GITHUB_TOKEN}}
script: |
const message = `${{ steps.parse-stats.outputs.MESSAGE }}`
github.rest.issues.createComment({
issue_number: context.issue.number,
owner: context.repo.owner,
repo: context.repo.repo,
body: message
})