name: Integration Parquet
on: [push, pull_request]
jobs:
docker:
name: Test
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
with:
submodules: true
- name: Setup Rust toolchain
run: |
rustup toolchain install stable
rustup default stable
rustup component add rustfmt clippy
- name: Cache Cargo
uses: actions/cache@v2
with:
path: /home/runner/.cargo
key: cargo-parquet-cache-
- name: Cache Rust dependencies
uses: actions/cache@v2
with:
path: /home/runner/target
key: ${{ runner.os }}-amd64-target-parquet-cache
- uses: actions/setup-python@v2
with:
python-version: "3.10"
- name: Build
run: |
export CARGO_HOME="/home/runner/.cargo"
export CARGO_TARGET_DIR="/home/runner/target"
cd arrow-parquet-integration-testing
cargo build
- name: Run
run: |
export CARGO_HOME="/home/runner/.cargo"
export CARGO_TARGET_DIR="/home/runner/target"
cd arrow-parquet-integration-testing
python -m venv venv
source venv/bin/activate
pip install --upgrade pip
pip install pyarrow==8 pyspark==3
python main.py
# test against spark
python main_spark.py