datafusion-dft 0.3.0

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

name: Tests
on: [push, pull_request]

jobs:
  test:
    name: DB
    runs-on: ubuntu-latest
    strategy:
      matrix:
        arch: [amd64]
    steps:
      - uses: actions/checkout@v2
        with:
          submodules: true
      - name: Cache Cargo registry and git
        uses: actions/cache@v4
        with:
          path: /home/runner/.cargo
          key: cargo-${{ runner.os }}-${{ hashFiles('**/Cargo.lock') }}
          restore-keys: |
            cargo-${{ runner.os }}-
      - name: Cache Rust target directory
        uses: actions/cache@v4
        with:
          path: target
          key: target-${{ runner.os }}-${{ hashFiles('**/Cargo.lock') }}-${{ hashFiles('**/Cargo.toml') }}
          restore-keys: |
            target-${{ runner.os }}-${{ hashFiles('**/Cargo.lock') }}-
            target-${{ runner.os }}-
      - name: Setup Rust Toolchain
        uses: ./.github/actions/setup-rust
      - name: Run core tests
        run: |
          cargo test db
  test-flightsql:
    name: Extension / FlightSQL
    runs-on: ubuntu-latest
    strategy:
      matrix:
        arch: [amd64]
    steps:
      - uses: actions/checkout@v2
        with:
          submodules: true
      - name: Cache Cargo registry and git
        uses: actions/cache@v4
        with:
          path: /home/runner/.cargo
          key: cargo-${{ runner.os }}-${{ hashFiles('**/Cargo.lock') }}
          restore-keys: |
            cargo-${{ runner.os }}-
      - name: Cache Rust target directory
        uses: actions/cache@v4
        with:
          path: target
          key: target-${{ runner.os }}-${{ hashFiles('**/Cargo.lock') }}-${{ hashFiles('**/Cargo.toml') }}
          restore-keys: |
            target-${{ runner.os }}-${{ hashFiles('**/Cargo.lock') }}-
            target-${{ runner.os }}-
      - name: Setup Rust Toolchain
        uses: ./.github/actions/setup-rust
      - name: Run FlightSQL tests
        run: |
          # Single thread needed because we spin up a server that listens on port and we need each
          # test to only be run against the server spun up in that test.  With parallelism tests 
          # can connec to server in different test which breaks determinism.
          cargo test --features=flightsql extension_cases::flightsql -- --test-threads=1
  test-cli:
    name: App / CLI
    runs-on: ubuntu-latest
    strategy:
      matrix:
        arch: [amd64]
    steps:
      - uses: actions/checkout@v3
      - name: Cache Cargo registry and git
        uses: actions/cache@v4
        with:
          path: /home/runner/.cargo
          key: cargo-${{ runner.os }}-${{ hashFiles('**/Cargo.lock') }}
          restore-keys: |
            cargo-${{ runner.os }}-
      - name: Cache Rust target directory
        uses: actions/cache@v4
        with:
          path: target
          key: target-${{ runner.os }}-${{ hashFiles('**/Cargo.lock') }}-${{ hashFiles('**/Cargo.toml') }}
          restore-keys: |
            target-${{ runner.os }}-${{ hashFiles('**/Cargo.lock') }}-
            target-${{ runner.os }}-
      - name: Setup Rust Toolchain
        uses: ./.github/actions/setup-rust
      - name: Start LocalStack
        uses: LocalStack/setup-localstack@v0.2.3
        with:
          image-tag: 'latest'
          install-awslocal: 'true'
          configuration: DEBUG=1
      - name: Run Tests against LocalStack
        run: |
          awslocal s3 mb s3://test
          awslocal s3 mv data/aggregate_test_100.csv s3://test/
          awslocal s3 mb s3://tpch-db
          echo "Test Execution complete!"
      - name: Run CLI tests
        run: |
          cargo test cli_cases
  test-tui:
    name: App / TUI
    runs-on: ubuntu-latest
    strategy:
      matrix:
        arch: [amd64]
    steps:
      - uses: actions/checkout@v3
      - name: Cache Cargo registry and git
        uses: actions/cache@v4
        with:
          path: /home/runner/.cargo
          key: cargo-${{ runner.os }}-${{ hashFiles('**/Cargo.lock') }}
          restore-keys: |
            cargo-${{ runner.os }}-
      - name: Cache Rust target directory
        uses: actions/cache@v4
        with:
          path: target
          key: target-${{ runner.os }}-${{ hashFiles('**/Cargo.lock') }}-${{ hashFiles('**/Cargo.toml') }}
          restore-keys: |
            target-${{ runner.os }}-${{ hashFiles('**/Cargo.lock') }}-
            target-${{ runner.os }}-
      - name: Setup Rust Toolchain
        uses: ./.github/actions/setup-rust
      - name: Start LocalStack
        uses: LocalStack/setup-localstack@v0.2.3
        with:
          image-tag: 'latest'
          install-awslocal: 'true'
          configuration: DEBUG=1
      - name: Run Tests against LocalStack
        run: |
          awslocal s3 mb s3://test
          awslocal s3 mv data/aggregate_test_100.csv s3://test/
          awslocal s3 mb s3://tpch-db
          echo "Test Execution complete!"
      - name: Run TUI tests
        run: |
          cargo test --features=tui tui_cases

  test-s3:
    name: Extension / S3
    runs-on: ubuntu-latest
    strategy:
      matrix:
        arch: [amd64]
    steps:
      - uses: actions/checkout@v3
      - name: Cache Cargo registry and git
        uses: actions/cache@v4
        with:
          path: /home/runner/.cargo
          key: cargo-${{ runner.os }}-${{ hashFiles('**/Cargo.lock') }}
          restore-keys: |
            cargo-${{ runner.os }}-
      - name: Cache Rust target directory
        uses: actions/cache@v4
        with:
          path: target
          key: target-${{ runner.os }}-${{ hashFiles('**/Cargo.lock') }}-${{ hashFiles('**/Cargo.toml') }}
          restore-keys: |
            target-${{ runner.os }}-${{ hashFiles('**/Cargo.lock') }}-
            target-${{ runner.os }}-
      - name: Setup Rust Toolchain
        uses: ./.github/actions/setup-rust
      - name: Start LocalStack
        uses: LocalStack/setup-localstack@v0.2.3
        with:
          image-tag: 'latest'
          install-awslocal: 'true'
          configuration: DEBUG=1
      - name: Run Tests against LocalStack
        run: |
          awslocal s3 mb s3://test
          awslocal s3 mv data/aggregate_test_100.csv s3://test/
          echo "Test Execution complete!"
      - name: Run S3 tests
        run: |
          cargo test --features=s3 extension_cases::s3
  test-functions-json:
    name: Extension / Functions-JSON
    runs-on: ubuntu-latest
    strategy:
      matrix:
        arch: [amd64]
    steps:
      - uses: actions/checkout@v2
        with:
          submodules: true
      - name: Cache Cargo registry and git
        uses: actions/cache@v4
        with:
          path: /home/runner/.cargo
          key: cargo-${{ runner.os }}-${{ hashFiles('**/Cargo.lock') }}
          restore-keys: |
            cargo-${{ runner.os }}-
      - name: Cache Rust target directory
        uses: actions/cache@v4
        with:
          path: target
          key: target-${{ runner.os }}-${{ hashFiles('**/Cargo.lock') }}-${{ hashFiles('**/Cargo.toml') }}
          restore-keys: |
            target-${{ runner.os }}-${{ hashFiles('**/Cargo.lock') }}-
            target-${{ runner.os }}-
      - name: Setup Rust Toolchain
        uses: ./.github/actions/setup-rust
      - name: Run Functions-JSON tests
        run: |
          cargo test --features=functions-json extension_cases::functions_json
  test-deltalake:
    name: Extension / Deltalake
    runs-on: ubuntu-latest
    strategy:
      matrix:
        arch: [amd64]
    steps:
      - uses: actions/checkout@v3
      - name: Cache Cargo registry and git
        uses: actions/cache@v4
        with:
          path: /home/runner/.cargo
          key: cargo-${{ runner.os }}-${{ hashFiles('**/Cargo.lock') }}
          restore-keys: |
            cargo-${{ runner.os }}-
      - name: Cache Rust target directory
        uses: actions/cache@v4
        with:
          path: target
          key: target-${{ runner.os }}-${{ hashFiles('**/Cargo.lock') }}-${{ hashFiles('**/Cargo.toml') }}
          restore-keys: |
            target-${{ runner.os }}-${{ hashFiles('**/Cargo.lock') }}-
            target-${{ runner.os }}-
      - name: Setup Rust Toolchain
        uses: ./.github/actions/setup-rust
      - name: Start LocalStack
        uses: LocalStack/setup-localstack@v0.2.3
        with:
          image-tag: 'latest'
          install-awslocal: 'true'
          configuration: DEBUG=1
      - name: Upload Delta Lake data to LocalStack
        run: |
          awslocal s3 mb s3://test
          awslocal s3 sync data/deltalake/simple_table s3://test/deltalake/simple_table
          echo "Delta Lake data uploaded to LocalStack"
      - name: Run Deltalake tests
        run: |
          cargo test --features="deltalake s3" extension_cases::deltalake
        env:
          AWS_ACCESS_KEY_ID: LSIAQAAAAAAVNCBMPNSG
          AWS_SECRET_ACCESS_KEY: 5555555555555555555555555555555555555555
          AWS_ENDPOINT_URL: http://localhost:4566
          AWS_REGION: us-east-1
          AWS_ALLOW_HTTP: true
  test-udfs-wasm:
    name: Extension / UDFs-WASM
    runs-on: ubuntu-latest
    strategy:
      matrix:
        arch: [amd64]
    steps:
      - uses: actions/checkout@v2
        with:
          submodules: true
      - name: Cache Cargo registry and git
        uses: actions/cache@v4
        with:
          path: /home/runner/.cargo
          key: cargo-${{ runner.os }}-${{ hashFiles('**/Cargo.lock') }}
          restore-keys: |
            cargo-${{ runner.os }}-
      - name: Cache Rust target directory
        uses: actions/cache@v4
        with:
          path: target
          key: target-${{ runner.os }}-${{ hashFiles('**/Cargo.lock') }}-${{ hashFiles('**/Cargo.toml') }}
          restore-keys: |
            target-${{ runner.os }}-${{ hashFiles('**/Cargo.lock') }}-
            target-${{ runner.os }}-
      - name: Setup Rust Toolchain
        uses: ./.github/actions/setup-rust
      - name: Run UDFs WASM tests
        run: |
          cargo test --features=udfs-wasm extension_cases::udfs_WASM
  test-vortex:
    name: Extension / Vortex
    runs-on: ubuntu-latest
    strategy:
      matrix:
        arch: [amd64]
    steps:
      - uses: actions/checkout@v2
        with:
          submodules: true
      - name: Cache Cargo registry and git
        uses: actions/cache@v4
        with:
          path: /home/runner/.cargo
          key: cargo-${{ runner.os }}-${{ hashFiles('**/Cargo.lock') }}
          restore-keys: |
            cargo-${{ runner.os }}-
      - name: Cache Rust target directory
        uses: actions/cache@v4
        with:
          path: target
          key: target-${{ runner.os }}-${{ hashFiles('**/Cargo.lock') }}-${{ hashFiles('**/Cargo.toml') }}
          restore-keys: |
            target-${{ runner.os }}-${{ hashFiles('**/Cargo.lock') }}-
            target-${{ runner.os }}-
      - name: Setup Rust Toolchain
        uses: ./.github/actions/setup-rust
      - name: Run Vortex tests
        run: |
          cargo test --features=vortex extension_cases::vortex
  test-crate-functions-parquet:
    name: Crate / Functions-Parquet
    runs-on: ubuntu-latest
    strategy:
      matrix:
        arch: [amd64]
    steps:
      - uses: actions/checkout@v2
        with:
          submodules: true
      - name: Cache Cargo registry and git
        uses: actions/cache@v4
        with:
          path: /home/runner/.cargo
          key: cargo-${{ runner.os }}-${{ hashFiles('**/Cargo.lock') }}
          restore-keys: |
            cargo-${{ runner.os }}-
      - name: Cache Rust target directory
        uses: actions/cache@v4
        with:
          path: target
          key: target-${{ runner.os }}-${{ hashFiles('**/Cargo.lock') }}-${{ hashFiles('**/Cargo.toml') }}
          restore-keys: |
            target-${{ runner.os }}-${{ hashFiles('**/Cargo.lock') }}-
            target-${{ runner.os }}-
      - name: Setup Rust Toolchain
        uses: ./.github/actions/setup-rust
      - name: Run UDFs WASM tests
        run: |
          cargo test --manifest-path crates/datafusion-functions-parquet/Cargo.toml
  test-crate-datafusion-app:
    name: Crate / DataFusion-App
    runs-on: ubuntu-latest
    strategy:
      matrix:
        arch: [amd64]
    steps:
      - uses: actions/checkout@v2
        with:
          submodules: true
      - name: Cache Cargo registry and git
        uses: actions/cache@v4
        with:
          path: /home/runner/.cargo
          key: cargo-${{ runner.os }}-${{ hashFiles('**/Cargo.lock') }}
          restore-keys: |
            cargo-${{ runner.os }}-
      - name: Cache Rust target directory
        uses: actions/cache@v4
        with:
          path: target
          key: target-${{ runner.os }}-${{ hashFiles('**/Cargo.lock') }}-${{ hashFiles('**/Cargo.toml') }}
          restore-keys: |
            target-${{ runner.os }}-${{ hashFiles('**/Cargo.lock') }}-
            target-${{ runner.os }}-
      - name: Setup Rust Toolchain
        uses: ./.github/actions/setup-rust
      - name: Run UDFs WASM tests
        run: |
          cargo test --all-features --manifest-path crates/datafusion-app/Cargo.toml
  test-crate-udfs-wasm:
    name: Crate / UDFs-WASM
    runs-on: ubuntu-latest
    strategy:
      matrix:
        arch: [amd64]
    steps:
      - uses: actions/checkout@v2
        with:
          submodules: true
      - name: Cache Cargo registry and git
        uses: actions/cache@v4
        with:
          path: /home/runner/.cargo
          key: cargo-${{ runner.os }}-${{ hashFiles('**/Cargo.lock') }}
          restore-keys: |
            cargo-${{ runner.os }}-
      - name: Cache Rust target directory
        uses: actions/cache@v4
        with:
          path: target
          key: target-${{ runner.os }}-${{ hashFiles('**/Cargo.lock') }}-${{ hashFiles('**/Cargo.toml') }}
          restore-keys: |
            target-${{ runner.os }}-${{ hashFiles('**/Cargo.lock') }}-
            target-${{ runner.os }}-
      - name: Setup Rust Toolchain
        uses: ./.github/actions/setup-rust
      - name: Run UDFs WASM tests
        run: |
          cargo test --manifest-path crates/datafusion-udfs-wasm/Cargo.toml
  test-auth:
    name: Extension / Auth
    runs-on: ubuntu-latest
    strategy:
      matrix:
        arch: [amd64]
    steps:
      - uses: actions/checkout@v2
        with:
          submodules: true
      - name: Cache Cargo registry and git
        uses: actions/cache@v4
        with:
          path: /home/runner/.cargo
          key: cargo-${{ runner.os }}-${{ hashFiles('**/Cargo.lock') }}
          restore-keys: |
            cargo-${{ runner.os }}-
      - name: Cache Rust target directory
        uses: actions/cache@v4
        with:
          path: target
          key: target-${{ runner.os }}-${{ hashFiles('**/Cargo.lock') }}-${{ hashFiles('**/Cargo.toml') }}
          restore-keys: |
            target-${{ runner.os }}-${{ hashFiles('**/Cargo.lock') }}-
            target-${{ runner.os }}-
      - name: Setup Rust Toolchain
        uses: ./.github/actions/setup-rust
      # Right now basic and bearer auth can not be used at the same time.  So we serve an instance with one, then kill it, then serve an instance with another.  An alternative, perhaps simpler option, would be configuring the ports used - but this would come at the cost of a more annoying local dev experience as you have to remember to keep updating the ports.
      - name: Start FlightSQL Server with Basic Auth
        run: |
          cargo r --features=flightsql -- serve-flightsql --config data/configs/flightsql_basic.toml & echo $! > server.pid
      - name: Run Basic Auth tests
        run: |
          cargo t --features=flightsql extension_cases::auth_basic
      - name: Kill FlightSQL Server
        run: |
          kill $(cat server.pid)
          rm server.pid
      - name: Start FlightSQL Server with Bearer Auth
        run: |
          cargo r --features=flightsql -- serve-flightsql --config data/configs/flightsql_bearer.toml &
      - name: Run Bearer Auth tests
        run: |
          cargo t --features=flightsql extension_cases::auth_bearer
  test-http-server:
    name: Extension / HTTP Server
    runs-on: ubuntu-latest
    strategy:
      matrix:
        arch: [amd64]
    steps:
      - uses: actions/checkout@v2
        with:
          submodules: true
      - name: Cache Cargo registry and git
        uses: actions/cache@v4
        with:
          path: /home/runner/.cargo
          key: cargo-${{ runner.os }}-${{ hashFiles('**/Cargo.lock') }}
          restore-keys: |
            cargo-${{ runner.os }}-
      - name: Cache Rust target directory
        uses: actions/cache@v4
        with:
          path: target
          key: target-${{ runner.os }}-${{ hashFiles('**/Cargo.lock') }}-${{ hashFiles('**/Cargo.toml') }}
          restore-keys: |
            target-${{ runner.os }}-${{ hashFiles('**/Cargo.lock') }}-
            target-${{ runner.os }}-
      - name: Setup Rust Toolchain
        uses: ./.github/actions/setup-rust
      - name: Start FlightSQL Server
        run: |
          cargo r --features=flightsql -- serve-flightsql &
      - name: Run tests
        run: |
          cargo t --features=http,flightsql server::http::router
      - name: Run CLI cases
        run: |
          cargo t --features=http,flightsql server_cases::http
  test-flightsql-server:
    name: Extension / FlightSQL Server
    runs-on: ubuntu-latest
    strategy:
      matrix:
        arch: [amd64]
    steps:
      - uses: actions/checkout@v2
        with:
          submodules: true
      - name: Cache Cargo registry and git
        uses: actions/cache@v4
        with:
          path: /home/runner/.cargo
          key: cargo-${{ runner.os }}-${{ hashFiles('**/Cargo.lock') }}
          restore-keys: |
            cargo-${{ runner.os }}-
      - name: Cache Rust target directory
        uses: actions/cache@v4
        with:
          path: target
          key: target-${{ runner.os }}-${{ hashFiles('**/Cargo.lock') }}-${{ hashFiles('**/Cargo.toml') }}
          restore-keys: |
            target-${{ runner.os }}-${{ hashFiles('**/Cargo.lock') }}-
            target-${{ runner.os }}-
      - name: Setup Rust Toolchain
        uses: ./.github/actions/setup-rust
      - name: Run CLI cases
        run: |
          cargo t --features=http,flightsql server_cases::http