kibble 0.1.0

chew through any source into clean datasets — a fast ingestion, RAG & fine-tuning toolkit
Documentation
# kibble run config — example
#
# Copy this to kibble.toml and edit the [[source]] entries, backends, and
# [pack.metadata] for your own corpus. kibble.toml is gitignored so your
# real config (sources, model names, dataset IDs) never lands in git.

# ---- sources ----
[[source]]
path = "data/ingest/docs"                 # local docs/writeups (OCR'd or crawled)
name = "docs"
type = "files"
system_prompt = "Answer accurately and technically, with correct, current details."

[[source]]
path = "data/code-sample"                 # local repos staged for a fast scan
name = "code-repos"
type = "codebase"
system_prompt = "You are an expert programmer. Write correct, idiomatic, secure code."

[[source]]
path = "data/datasets/example/normalized" # a fetched/normalized HF dataset
name = "example-dataset"
type = "dataset"
system_prompt = "You are a precise technical reasoner. Think step by step, then answer accurately."

[index]
# Retrieval/RAG corpus — pick the sources worth grounding answers in.
sources = [
  "data/ingest/docs",
]

[understand.embed]
# Empty = offline: index/search/classify fall soft to lexical-only BM25 (no backend needed).
# To enable semantic embeddings, point at any OpenAI-compatible server, e.g. Ollama:
#   base_url = "http://localhost:11434/v1"   # ollama pull nomic-embed-text
base_url = ""
model = "nomic"

[cluster]
k = 12
enabled = true
rebalance = false           # inline topic rebalancing — caps any topic at max_topic_share
max_topic_share = 0.35
llm_labels = false          # opt-in: LLM-named topics (reuses [ask])

[eval]
semantic = false

[ask]
# Empty = ask/bench disabled. Point at any OpenAI-compatible server to enable grounded RAG:
#   base_url = "http://localhost:11434/v1"   # ollama pull llama3.2
#   model    = "llama3.2"
base_url = ""                              # served LLM (empty → ask errors with guidance)
model = "your-served-model"
max_tokens = 1024
k = 6

[web]
# Perplexity-style web augmentation for `kibble ask --web`: web_search hits a
# SearXNG JSON API, then fetch_page reads a promising result; web passages are
# cited [n] alongside corpus passages. default=false → off unless --web is passed.
base_url = ""               # empty → DuckDuckGo scrape fallback
default = false
max_results = 5

[classify]
enabled = false

[tune]
model = "org/base-model"
smoke_rows = 5

[[tune.phase]]
name = "phase1"
sources = ["docs", "code-repos", "example-dataset"]
max_steps = 700

[extract.ocr]
base_url = "http://localhost:8091/v1"
model = "unlimited-ocr"

[pack]
name = "example-v0.1"
staging = "kaggle/dataset-staging"
splits = ["train", "valid"]

[[pack.include]]
src = "data/tune/sprint"
dest = "sprint"

[pack.metadata]
id = "example-user/example-dataset"
title = "Example dataset — SFT training data"
license = "other"