llm-test-bench-datasets 0.1.0

Dataset management and utilities for LLM Test Bench - load, validate, and manage test datasets
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50

{
  "name": "coding-tasks",
  "description": "Basic coding challenges in multiple languages",
  "version": "1.0.0",
  "defaults": {
    "temperature": 0.0,
    "max_tokens": 500
  },
  "test_cases": [
    {
      "id": "fizzbuzz-python",
      "category": "coding",
      "prompt": "Write a Python function that implements FizzBuzz for numbers 1 to {{n}}. Return only the code, no explanation.",
      "variables": {
        "n": "100"
      },
      "expected": "def fizzbuzz",
      "references": ["for i in range", "if i % 15", "FizzBuzz"]
    },
    {
      "id": "reverse-string-rust",
      "category": "coding",
      "prompt": "Write a Rust function to reverse a string in-place. Return only the code.",
      "expected": "fn reverse",
      "references": ["chars()", "rev()", "collect()"]
    },
    {
      "id": "fibonacci-javascript",
      "category": "coding",
      "prompt": "Implement a {{lang}} function to calculate the nth Fibonacci number using recursion. Return only the code.",
      "variables": {
        "lang": "JavaScript"
      },
      "references": ["function", "fibonacci", "return"]
    },
    {
      "id": "palindrome-check",
      "category": "coding",
      "prompt": "Write a function in any language that checks if a string is a palindrome. Include test cases.",
      "references": ["palindrome", "reverse", "toLowerCase"]
    },
    {
      "id": "binary-search",
      "category": "algorithms",
      "prompt": "Implement binary search in Python with proper handling of edge cases. Return code with comments.",
      "expected": "def binary_search",
      "references": ["mid = (left + right) // 2", "while left <= right"]
    }
  ]
}