llm-test-bench-datasets 0.1.0

Dataset management and utilities for LLM Test Bench - load, validate, and manage test datasets
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60

{
  "name": "creative-writing",
  "description": "Creative generation tasks",
  "version": "1.0.0",
  "defaults": {
    "temperature": 0.9,
    "max_tokens": 600
  },
  "test_cases": [
    {
      "id": "story-opening",
      "category": "creative-writing",
      "prompt": "Write an engaging opening paragraph for a science fiction story about {{topic}}.",
      "variables": {
        "topic": "time travel"
      }
    },
    {
      "id": "haiku-nature",
      "category": "creative-writing",
      "prompt": "Write a haiku about {{subject}}. Follow the traditional 5-7-5 syllable pattern.",
      "variables": {
        "subject": "autumn leaves"
      },
      "references": [
        "5 syllables",
        "7 syllables"
      ]
    },
    {
      "id": "product-description",
      "category": "creative-writing",
      "prompt": "Write a creative product description for a {{product}} that emphasizes its unique features.",
      "variables": {
        "product": "smart coffee mug"
      }
    },
    {
      "id": "dialogue-scene",
      "category": "creative-writing",
      "prompt": "Write a short dialogue between two characters meeting at a coffee shop. One character has exciting news to share."
    },
    {
      "id": "metaphor-explanation",
      "category": "creative-writing",
      "prompt": "Explain the concept of '{{concept}}' using a creative metaphor or analogy.",
      "variables": {
        "concept": "cloud computing"
      }
    },
    {
      "id": "limerick",
      "category": "creative-writing",
      "prompt": "Write a limerick about {{subject}}.",
      "variables": {
        "subject": "programming bugs"
      }
    }
  ]
}