kwaak 0.18.0

Run a team of autonomous agents on your code, right from your terminal
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89

//! Evaluate with RAGAS
use anyhow::Result;
use std::path::Path;

use swiftide::query::evaluators::ragas;

use crate::{
    indexing::{build_query_pipeline, duckdb_index::get_duckdb},
    repository::Repository,
};

/// Evaluate a query pipeline with RAGAS
///
/// Ragas is a simple model with 4 base metrics to evaluate rag pipelines.
///
/// Takes an input file as a base dataset and will write to the output file as json.
///
/// Example format of the files:
///
/// ```json
/// {
///   "What is the capital of France?": {
///     "question": "What is the capital of France?",
///
///     // The answer given by the pipeline
///     "answer": "Paris",
///
///     // Retrieved documents
///     "contexts": ["The capital of France is Paris"],
///
///     // The expected answer
///     "ground_truth": "Paris",
///     }
/// }
/// ```
///
///
/// See [ragas](https://ragas.io) for more information.
pub async fn evaluate_query_pipeline(
    repository: &Repository,
    input: Option<&Path>,
    output: &Path,
    questions: Option<&[String]>,
    record_ground_truth: bool,
) -> Result<()> {
    // Load dataset

    let dataset: ragas::EvaluationDataSet = if let Some(input) = input {
        println!("Reading dataset from file: {}", input.display());
        std::fs::read_to_string(input)?.parse()?
    } else if let Some(questions) = questions {
        println!("Generation evaluations for {} questions", questions.len());
        questions.into()
    } else {
        anyhow::bail!("Either input or questions must be provided")
    };

    // Setup ragas
    tracing::info!("Setting up evaluator");
    let ragas = ragas::Ragas::from_prepared_questions(dataset);

    // Build query pipeline
    tracing::info!("Building query pipeline");
    let pipeline = build_query_pipeline(
        repository,
        &get_duckdb(repository.config()),
        Some(Box::new(ragas.clone())),
    )?;

    // Query all questions from the dataset
    tracing::info!("Querying all questions");
    pipeline.query_all(ragas.questions().await).await?;

    if record_ground_truth {
        // Record ground truth
        tracing::info!("Recording ground truth");
        ragas.record_answers_as_ground_truth().await;
    }

    tracing::info!("Writing evaluation results to file");
    // Export to file
    let json = ragas.to_json().await;
    if let Some(parent) = output.parent() {
        let _ = std::fs::create_dir_all(parent);
    }
    std::fs::write(output, json)?;

    Ok(())
}