elinor 0.4.0

Evaluation Library in Information Retrieval
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127

name: build

on:
  push:
    branches:
      - main
    tags:
      - "v*.*.*"
  pull_request:
    branches:
      - main

env:
  CARGO_TERM_COLOR: always

jobs:
  build:
    name: Check on ${{ matrix.rust }}
    runs-on: ubuntu-latest
    strategy:
      matrix:
        rust:
          - 1.65.0 # MSRV
          - stable
          - nightly
    steps:
      - uses: actions/checkout@v4
      - name: Install ${{ matrix.rust }}
        run: |
          rustup toolchain install ${{ matrix.rust }} --profile minimal --component rustfmt,clippy
          rustup default ${{ matrix.rust }}
      - name: Run cargo check
        run: cargo check
      - name: Run cargo check (all)
        # MSRV should be ignored for dev-dependencies.
        continue-on-error: ${{ matrix.rust != 'stable' }}
        run: cargo check --all
      - name: Run cargo check (all, serde)
        # MSRV should be ignored for dev-dependencies.
        continue-on-error: ${{ matrix.rust != 'stable' }}
        run: cargo check --all --features serde
      - name: Run cargo fmt
        # imports_granularity is not stable yet.
        continue-on-error: ${{ matrix.rust != 'nightly' }}
        run: cargo fmt --all -- --check
      - name: Run cargo clippy (all, serde)
        # Run clippy only on stable to ignore unreasonable old warnings.
        continue-on-error: ${{ matrix.rust != 'stable' }}
        run: cargo clippy --all --features serde -- -D warnings -W clippy::nursery
      - name: Run cargo test
        # MSRV should be ignored for dev-dependencies.
        continue-on-error: ${{ matrix.rust != 'stable' }}
        run: cargo test --release --features serde
      - name: Run cargo doc
        run: RUSTDOCFLAGS="--html-in-header katex.html" cargo doc --no-deps --features serde
      - name: Run cargo example
        # MSRV should be ignored for dev-dependencies.
        continue-on-error: ${{ matrix.rust != 'stable' }}
        run: |
          cargo run --release --example from_jsonl --features serde
          cargo run --release --example from_trec
          cargo run --release --example paired_bootstrap_test
          cargo run --release --example randomized_tukey_hsd_test
          cargo run --release --example two_way_anova_without_replication

  run-cli:
    name: Run elinor-cli
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Install stable
        run: |
          rustup toolchain install stable --profile minimal
          rustup default stable
      - name: Build elinor-cli
        run: cargo build --release -p elinor-cli
      - name: Run elinor-convert
        run: |
          ./target/release/elinor-convert --help
          ./target/release/elinor-convert --input-trec test-data/trec-eval/qrels.test --output-jsonl qrels.jsonl --rel-type true
          ./target/release/elinor-convert --input-trec test-data/trec-eval/results.test --output-jsonl results.jsonl --rel-type pred
      - name: Run elinor-evaluate
        run: |
          ./target/release/elinor-evaluate --help
          ./target/release/elinor-evaluate --true-jsonl qrels.jsonl --pred-jsonl results.jsonl --output-csv results.csv
      - name: Run elinor-compare
        run: |
          ./target/release/elinor-compare --help
          ./target/release/elinor-compare --input-csvs test-data/sample/pred_1.csv
          ./target/release/elinor-compare --input-csvs test-data/sample/pred_1.csv --input-csvs test-data/sample/pred_2.csv
          ./target/release/elinor-compare --input-csvs test-data/sample/pred_1.csv --input-csvs test-data/sample/pred_2.csv --input-csvs test-data/sample/pred_3.csv

  correctness-test:
    name: Correctness test against trec_eval
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Install stable
        run: |
          rustup toolchain install stable --profile minimal
          rustup default stable
      - name: Install Python 3.12
        uses: actions/setup-python@v5
        with:
          python-version: 3.12
      - name: Build elinor-cli
        run: cargo build --release -p elinor-cli
      - name: Run correctness test
        run: |
          ./scripts/compare_with_trec_eval.py target/release test-data/trec-eval/qrels.test.jsonl test-data/trec-eval/results.test.jsonl test-data/trec-eval/qrels.test.output.json
          ./scripts/compare_with_trec_eval.py target/release test-data/trec-eval/qrels.rel_level.jsonl test-data/trec-eval/results.test.jsonl test-data/trec-eval/qrels.rel_level.output.json

  publish:
    name: Publish
    runs-on: ubuntu-latest
    if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/')
    needs: [build, correctness-test]
    steps:
      - uses: actions/checkout@v4
      - name: Install latest stable
        run: |
          rustup toolchain install stable --profile minimal
          rustup default stable
      - name: Run cargo publish
        run: cargo publish
        env:
          CARGO_REGISTRY_TOKEN: ${{ secrets.CRATES_TOKEN }}