dkp 0.4.3

dkp — Domain Knowledge Pack management CLI
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118

use anyhow::Result;
use clap::Args;
use std::path::PathBuf;
use std::sync::Arc;

use dkp_gen_core::{CliOverrides, GenConfig, OpenAiClient, PipelineContext};

use crate::cli::CmdCtx;

#[derive(Args, Debug)]
pub struct EvalArgs {
    /// Path to the DKP bundle directory
    pub pack: PathBuf,

    #[arg(long, value_name = "PROVIDER")]
    pub provider: Option<String>,

    #[arg(long, value_name = "MODEL")]
    pub model: Option<String>,

    #[arg(long, value_name = "URL")]
    pub base_url: Option<String>,

    #[arg(
        long,
        value_name = "KEY",
        env = "DKP_GEN_API_KEY",
        hide_env_values = true
    )]
    pub api_key: Option<String>,

    /// Run only first N eval pairs (default: all)
    #[arg(long, value_name = "N")]
    pub pairs: Option<usize>,

    /// Score without DKP context (baseline only)
    #[arg(long)]
    pub baseline_only: bool,
}

pub async fn run(args: EvalArgs, ctx: &CmdCtx) -> Result<()> {
    let pack = dkp_core::Pack::open(&args.pack)?;
    let domain = pack.manifest.domain.clone();
    let pack_name = pack.manifest.name.clone();
    let pack_version = pack.manifest.version.clone();
    let min_eval_delta = pack.manifest.min_eval_delta.unwrap_or(0.0);

    let config = GenConfig::load(CliOverrides {
        base_url: args.base_url,
        api_key: args.api_key,
        model: args.model,
        overwrite: true,
        // `dkp eval` measures what the pack adds over an unaided baseline
        // answer; tool access would contaminate that comparison, so it's
        // always disabled here regardless of global config.
        no_tools: true,
        instructions: None,
        no_render_formats: true,
        max_tool_turns: None,
    })?;
    if config.api_key.is_empty() {
        anyhow::bail!(
            "API key required: pass --api-key, set DKP_GEN_API_KEY, \
             or add api_key to ~/.dkp/gen.toml"
        );
    }

    let client = Arc::new(OpenAiClient::new(&config)?);
    let gen_ctx = Arc::new(PipelineContext {
        pack_dir: args.pack.clone(),
        domain,
        pack_name,
        config,
        client,
        progress: None,
        verbose: !ctx.quiet,
    });

    if !ctx.quiet {
        println!(
            "Running eval ({} cases)...",
            if let Some(pairs) = args.pairs {
                format!("up to {}", pairs)
            } else {
                "all".into()
            }
        );
    }

    let report = dkp_gen_core::eval::run(
        gen_ctx,
        &pack_version,
        min_eval_delta,
        args.pairs,
        args.baseline_only,
    )
    .await?;

    let pct = (report.summary.passed * 100)
        .checked_div(report.summary.total)
        .unwrap_or(0);

    println!(
        "Eval complete: {}/{} passed ({}%)",
        report.summary.passed, report.summary.total, pct
    );

    if !report.failures.is_empty() {
        println!("\nFailed cases:");
        for f in &report.failures {
            println!("  ✗ {}", f.query);
            println!("    Reason: {}", f.reason);
        }
        println!("\nRun `dkp fix <pack>` to address failures.");
    }

    Ok(())
}