algocline-engine 0.46.1

algocline Lua execution engine — VM, session, bridge
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139

-- vendored from evalframe v0.4.0 (~/.algocline/packages/evalframe/presets/graders.lua)
-- DO NOT EDIT here — upstream sync via manual copy
--[[
  presets/graders.lua — Built-in grader catalog

  Deterministic graders (Tier 1 — always prefer over LLM-as-Judge).

  Usage:
    local graders = require("evalframe.presets.graders")
    bind { graders.exact_match, scorer_def }
]]

local grader = require("evalframe.model.grader")
local std    = require("evalframe.std")

local M = {}

-- ============================================================
-- Exact match: response.text == any expected value
-- ============================================================

M.exact_match = grader "exact_match" {
  check = function(resp, case)
    if not case.expected then return false end
    local text = resp.text or ""
    for _, exp in ipairs(case.expected) do
      if text == exp then return true end
    end
    return false
  end,

}

-- ============================================================
-- Contains: response.text contains any expected value
-- ============================================================

M.contains = grader "contains" {
  check = function(resp, case)
    if not case.expected then return false end
    local text = resp.text or ""
    for _, exp in ipairs(case.expected) do
      if text:find(exp, 1, true) then return true end
    end
    return false
  end,

}

-- ============================================================
-- Starts with: response.text starts with expected
-- ============================================================

M.starts_with = grader "starts_with" {
  check = function(resp, case)
    if not case.expected then return false end
    local text = resp.text or ""
    for _, exp in ipairs(case.expected) do
      if text:sub(1, #exp) == exp then return true end
    end
    return false
  end,

}

-- ============================================================
-- Regex: response.text matches Lua pattern
-- Expects case.context.pattern or case.expected[1] as pattern
-- ============================================================

M.regex = grader "regex" {
  check = function(resp, case)
    local text = resp.text or ""
    local pattern = (case.context and case.context.pattern)
                    or (case.expected and case.expected[1])
    if not pattern then return false end
    return text:match(pattern) ~= nil
  end,

}

-- ============================================================
-- JSON valid: response.text is valid JSON
-- ============================================================

M.json_valid = grader "json_valid" {
  check = function(resp, _case)
    local text = resp.text or ""
    local ok = pcall(std.json.decode, text)
    return ok
  end,

}

-- ============================================================
-- Length: returns text length as raw grade (pair with linear scorer)
-- ============================================================

M.length = grader "length" {
  check = function(resp, _case)
    return #(resp.text or "")
  end,

}

-- ============================================================
-- Latency: returns response latency in ms
-- ============================================================

M.latency = grader "latency" {
  check = function(resp, _case)
    return resp.latency_ms  -- nil when missing (scorer handles nil → 0)
  end,

}

-- ============================================================
-- Not empty: response.text is non-empty
-- ============================================================

M.not_empty = grader "not_empty" {
  check = function(resp, _case)
    local text = resp.text or ""
    return #text > 0
  end,

}

-- ============================================================
-- Composed graders (combinators)
-- ============================================================

--- Non-empty AND valid JSON. Common precondition for structured output.
M.valid_json_response = grader.all(M.not_empty, M.json_valid)

--- Exact match OR contains. Lenient matching that accepts both.
M.flexible_match = grader.any(M.exact_match, M.contains)

return M