kibble 0.1.0

use serde::Deserialize;
use std::path::Path;

/// The config file kibble reads from the working directory.
pub const CONFIG_FILE: &str = "kibble.toml";

#[derive(Debug, Deserialize, Clone)]
pub struct SourceConfig {
    pub path: String,
    #[serde(rename = "type")]
    pub r#type: Option<String>,
    pub name: Option<String>,
    pub max_rows: Option<usize>,
    pub system_prompt: Option<String>,
    pub user_prefix: Option<String>,
    #[allow(dead_code)]
    pub role: Option<String>,
    pub preserve_code: Option<bool>,
    pub refresh: Option<bool>,
}

impl SourceConfig {
    pub fn source_name(&self) -> String {
        if let Some(n) = &self.name {
            return n.clone();
        }
        Path::new(&self.path)
            .file_stem()
            .and_then(|s| s.to_str())
            .map(|s| s.to_string())
            .unwrap_or_else(|| self.path.clone())
    }
}

#[derive(Debug, Deserialize, Default)]
pub struct NetworkConfig {
    pub proxy: Option<String>,
}

fn default_true() -> bool { true }
fn default_min_answer_chars() -> usize { 20 }
fn default_near_threshold() -> f64 { 0.85 }
fn default_shingle_size() -> usize { 5 }
fn d_sem_threshold() -> f64 { 0.90 }

#[derive(Debug, Deserialize)]
pub struct CurateConfig {
    #[serde(default = "default_true")] pub dedup: bool,
    #[serde(default = "default_true")] pub leakage_safe_split: bool,
    #[serde(default = "default_true")] pub drop_malformed: bool,
    #[serde(default = "default_true")] pub drop_degenerate: bool,
    #[serde(default = "default_min_answer_chars")] pub min_answer_chars: usize,
    #[serde(default)] pub near_dedup: bool,
    #[serde(default = "default_near_threshold")] pub near_dedup_threshold: f64,
    #[serde(default = "default_shingle_size")] pub shingle_size: usize,
    #[serde(default)] pub semantic_dedup: bool,
    #[serde(default = "d_sem_threshold")] pub semantic_threshold: f64,
}
impl Default for CurateConfig {
    fn default() -> Self {
        CurateConfig { dedup: true, leakage_safe_split: true, drop_malformed: true, drop_degenerate: true, min_answer_chars: 20, near_dedup: false, near_dedup_threshold: 0.85, shingle_size: 5, semantic_dedup: false, semantic_threshold: 0.90 }
    }
}

#[derive(Debug, Deserialize, Clone, Default)]
pub struct ClassifyConfig {
    /// Opt-in: discover hierarchical document topics during build (needs embed + [ask] LLM).
    #[serde(default)] pub enabled: bool,
}

fn d_embed_model() -> String { "nomic".into() }
fn d_embed_batch() -> usize { 64 }
fn d_embed_store() -> String { "data/embeddings".into() }

#[derive(Debug, Deserialize, Clone)]
pub struct EmbedConfig {
    #[serde(default)] pub base_url: String,
    #[serde(default = "d_embed_model")] pub model: String,
    #[serde(default = "d_embed_batch")] pub batch_size: usize,
    #[serde(default = "d_embed_store")] pub store: String,
}
impl Default for EmbedConfig {
    fn default() -> Self {
        EmbedConfig { base_url: String::new(), model: d_embed_model(), batch_size: d_embed_batch(), store: d_embed_store() }
    }
}

#[derive(Debug, Deserialize, Default, Clone)]
pub struct UnderstandConfig {
    #[serde(default)] pub embed: EmbedConfig,
}

fn default_pack_name() -> String { "bundle".to_string() }
fn default_pack_staging() -> String { "kaggle/dataset-staging".to_string() }
fn default_pack_splits() -> Vec<String> { vec!["train".to_string(), "valid".to_string()] }
fn default_pack_zip() -> String { "kaggle/dataset.zip".to_string() }

#[derive(Debug, Deserialize)]
pub struct PackInclude {
    pub src: String,
    pub dest: String,
}

#[derive(Debug, Deserialize, Default)]
pub struct PackMetadata {
    pub id: Option<String>,
    pub title: Option<String>,
    pub license: Option<String>,
}

#[derive(Debug, Deserialize)]
pub struct PackConfig {
    #[serde(default = "default_pack_name")]
    pub name: String,
    #[serde(default = "default_pack_staging")]
    pub staging: String,
    #[serde(default = "default_pack_splits")]
    pub splits: Vec<String>,
    #[serde(default = "default_pack_zip")]
    pub zip: String,
    pub readme: Option<String>,
    #[serde(default)]
    pub include: Vec<PackInclude>,
    #[serde(default)]
    pub metadata: PackMetadata,
}

fn default_eval_out() -> String { "eval/report".to_string() }
fn default_eval_method() -> String { "sft".to_string() }
fn d_min_rows() -> usize { 50 }
fn d_max_malformed() -> f64 { 0.0 }
fn d_max_dup() -> f64 { 0.02 }
fn d_max_leak() -> f64 { 0.0 }
fn d_max_short() -> f64 { 0.10 }
fn d_max_degen() -> f64 { 0.01 }
fn d_max_source_share() -> f64 { 0.60 }
fn d_max_topic_share() -> f64 { 0.60 }
fn d_min_distinct2() -> f64 { 0.30 }
fn d_short_char_min() -> usize { 20 }
fn d_max_near_dup() -> f64 { 0.02 }
fn d_max_near_leak() -> f64 { 0.0 }
fn d_near_threshold() -> f64 { 0.85 }
fn d_near_shingle() -> usize { 5 }
fn d_max_sem_leak() -> f64 { 0.0 }
fn d_max_sem_dup() -> f64 { 0.02 }
fn d_sem_eval_threshold() -> f64 { 0.90 }

#[derive(Debug, Deserialize)]
pub struct EvalThresholds {
    #[serde(default = "d_min_rows")] pub min_rows: usize,
    #[serde(default = "d_max_malformed")] pub max_malformed_rate: f64,
    #[serde(default = "d_max_dup")] pub max_duplicate_rate: f64,
    #[serde(default = "d_max_leak")] pub max_leakage_rate: f64,
    #[serde(default = "d_max_short")] pub max_short_rate: f64,
    #[serde(default = "d_max_degen")] pub max_degenerate_rate: f64,
    #[serde(default = "d_max_source_share")] pub max_source_share: f64,
    #[serde(default = "d_max_topic_share")] pub max_topic_share: f64,
    #[serde(default = "d_min_distinct2")] pub min_distinct2: f64,
    #[serde(default = "d_short_char_min")] pub short_char_min: usize,
    #[serde(default = "d_max_near_dup")] pub max_near_duplicate_rate: f64,
    #[serde(default = "d_max_near_leak")] pub max_near_leakage_rate: f64,
    #[serde(default = "d_near_threshold")] pub near_dup_threshold: f64,
    #[serde(default = "d_near_shingle")] pub near_shingle_size: usize,
    #[serde(default = "d_max_sem_leak")] pub max_semantic_leakage_rate: f64,
    #[serde(default = "d_max_sem_dup")] pub max_semantic_near_dup_rate: f64,
    #[serde(default = "d_sem_eval_threshold")] pub semantic_threshold: f64,
}
impl Default for EvalThresholds {
    fn default() -> Self {
        EvalThresholds {
            min_rows: d_min_rows(), max_malformed_rate: d_max_malformed(), max_duplicate_rate: d_max_dup(),
            max_leakage_rate: d_max_leak(), max_short_rate: d_max_short(), max_degenerate_rate: d_max_degen(),
            max_source_share: d_max_source_share(), max_topic_share: d_max_topic_share(),
            min_distinct2: d_min_distinct2(), short_char_min: d_short_char_min(),
            max_near_duplicate_rate: d_max_near_dup(), max_near_leakage_rate: d_max_near_leak(),
            near_dup_threshold: d_near_threshold(), near_shingle_size: d_near_shingle(),
            max_semantic_leakage_rate: d_max_sem_leak(),
            max_semantic_near_dup_rate: d_max_sem_dup(),
            semantic_threshold: d_sem_eval_threshold(),
        }
    }
}

#[derive(Debug, Deserialize)]
pub struct EvalConfig {
    #[serde(default)] pub dataset_dir: Option<String>,
    #[serde(default = "default_eval_out")] pub out: String,
    #[serde(default = "default_eval_method")] pub method: String,
    #[serde(default)] pub thresholds: EvalThresholds,
    #[serde(default)] pub near_dup: bool,
    #[serde(default)] pub semantic: bool,
}
impl Default for EvalConfig {
    fn default() -> Self {
        EvalConfig { dataset_dir: None, out: default_eval_out(), method: default_eval_method(), thresholds: EvalThresholds::default(), near_dup: false, semantic: false }
    }
}

fn default_pdf_render_cli() -> String { "pdftoppm".to_string() }
fn default_pdf_min_chars() -> usize { 50 }

#[derive(Debug, Deserialize)]
pub struct PdfConfig {
    #[serde(default = "default_pdf_render_cli")]
    pub render_cli: String,
    #[serde(default = "default_pdf_min_chars")]
    pub min_chars: usize,
}

impl Default for PdfConfig {
    fn default() -> Self {
        PdfConfig { render_cli: default_pdf_render_cli(), min_chars: default_pdf_min_chars() }
    }
}

fn default_video_cli() -> String { "ffmpeg".to_string() }
fn default_office_cli() -> String { "pandoc".to_string() }

#[derive(Debug, Deserialize)]
pub struct VideoConfig {
    #[serde(default = "default_video_cli")]
    pub cli: String,
}
impl Default for VideoConfig {
    fn default() -> Self { VideoConfig { cli: default_video_cli() } }
}

#[derive(Debug, Deserialize)]
pub struct OfficeConfig {
    #[serde(default = "default_office_cli")]
    pub cli: String,
}
impl Default for OfficeConfig {
    fn default() -> Self { OfficeConfig { cli: default_office_cli() } }
}

fn default_extract_out() -> String { "data/extracted".to_string() }
fn default_extract_max_bytes() -> u64 { 524_288_000 }

#[derive(Debug, Deserialize, Default)]
pub struct OcrConfig {
    #[serde(default)]
    pub base_url: String,
    #[serde(default)]
    pub model: String,
    #[serde(default)]
    pub cli: String,
}

#[derive(Debug, Deserialize, Default)]
pub struct TranscribeConfig {
    #[serde(default)]
    pub base_url: String,
    #[serde(default)]
    pub model: String,
    #[serde(default)]
    pub cli: String,
}

#[derive(Debug, Deserialize, Default)]
#[allow(dead_code)]
pub struct ExtractWebConfig {
    #[serde(default)]
    pub base_url: String,
}

#[derive(Debug, Deserialize)]
#[allow(dead_code)]
pub struct ExtractConfig {
    #[serde(default = "default_extract_out")]
    pub out: String,
    #[serde(default = "default_extract_max_bytes")]
    pub max_bytes: u64,
    #[serde(default)]
    pub ocr: OcrConfig,
    #[serde(default)]
    pub transcribe: TranscribeConfig,
    #[serde(default)]
    pub web: ExtractWebConfig,
    #[serde(default)]
    pub pdf: PdfConfig,
    #[serde(default)]
    pub video: VideoConfig,
    #[serde(default)]
    pub office: OfficeConfig,
}

impl Default for ExtractConfig {
    fn default() -> Self {
        ExtractConfig {
            out: default_extract_out(),
            max_bytes: default_extract_max_bytes(),
            ocr: OcrConfig::default(),
            transcribe: TranscribeConfig::default(),
            web: ExtractWebConfig::default(),
            pdf: PdfConfig::default(),
            video: VideoConfig::default(),
            office: OfficeConfig::default(),
        }
    }
}

fn default_caps_root() -> String { "~/.kibble/caps".to_string() }
fn default_caps_max_bytes() -> u64 { 524_288_000 }

#[derive(Debug, Deserialize)]
pub struct CapsConfig {
    #[serde(default = "default_caps_root")]
    pub root: String,
    #[serde(default = "default_caps_max_bytes")]
    pub max_bytes: u64,
}
impl Default for CapsConfig {
    fn default() -> Self { CapsConfig { root: default_caps_root(), max_bytes: default_caps_max_bytes() } }
}

fn default_ingest_root() -> String { "data/ingest".to_string() }
fn default_max_bytes() -> u64 { 1_073_741_824 }
fn default_datasets_root() -> String { "data/datasets".to_string() }
fn default_models_root() -> String { "data/models".to_string() }

#[derive(Debug, Deserialize)]
pub struct FetchConfig {
    #[serde(default = "default_ingest_root")]
    pub ingest_root: String,
    #[serde(default = "default_max_bytes")]
    pub max_bytes: u64,
    #[serde(default = "default_datasets_root")]
    pub datasets_root: String,
    #[serde(default = "default_models_root")]
    pub models_root: String,
}

impl Default for FetchConfig {
    fn default() -> Self {
        FetchConfig {
            ingest_root: default_ingest_root(),
            max_bytes: default_max_bytes(),
            datasets_root: default_datasets_root(),
            models_root: default_models_root(),
        }
    }
}

fn default_serve_bind() -> String { "127.0.0.1:7777".to_string() }

#[derive(Debug, Deserialize)]
pub struct ServeConfig {
    #[serde(default = "default_serve_bind")]
    pub bind: String,
    #[serde(default)]
    pub allow_hosts: Vec<String>,
}

impl Default for ServeConfig {
    fn default() -> Self {
        ServeConfig { bind: default_serve_bind(), allow_hosts: Vec::new() }
    }
}

fn default_data_root() -> String { "data".to_string() }
fn default_dataset_dir() -> String { "data/datasets/unsloth".to_string() }

#[derive(Debug, Deserialize)]
pub struct PathsConfig {
    #[serde(default = "default_data_root")]
    pub data_root: String,
    #[serde(default = "default_dataset_dir")]
    pub dataset_dir: String,
}

impl Default for PathsConfig {
    fn default() -> Self {
        PathsConfig { data_root: default_data_root(), dataset_dir: default_dataset_dir() }
    }
}

fn d_crawl_pages() -> usize { 50 }
fn d_crawl_depth() -> usize { 2 }
fn d_crawl_delay() -> u64 { 200 }
fn d_crawl_ua() -> String { "kibble-crawler".to_string() }
fn d_crawl_bytes() -> u64 { 2_000_000 }
fn d_crawl_out() -> String { "data/ingest".to_string() }

#[derive(Debug, Deserialize)]
pub struct CrawlConfig {
    #[serde(default = "d_crawl_pages")] pub max_pages: usize,
    #[serde(default = "d_crawl_depth")] pub max_depth: usize,
    #[serde(default = "default_true")] pub same_host: bool,
    #[serde(default = "d_crawl_delay")] pub delay_ms: u64,
    #[serde(default = "default_true")] pub respect_robots: bool,
    #[serde(default = "d_crawl_ua")] pub user_agent: String,
    #[serde(default = "d_crawl_bytes")] pub max_bytes: u64,
    #[serde(default = "d_crawl_out")] pub out: String,
    #[serde(default)] pub allow_hosts: Vec<String>,
}
impl Default for CrawlConfig {
    fn default() -> Self {
        CrawlConfig { max_pages: d_crawl_pages(), max_depth: d_crawl_depth(), same_host: true,
            delay_ms: d_crawl_delay(), respect_robots: true, user_agent: d_crawl_ua(),
            max_bytes: d_crawl_bytes(), out: d_crawl_out(), allow_hosts: Vec::new() }
    }
}

fn d_index_dir() -> String { "data/index".into() }
fn d_index_sources() -> Vec<String> {
    vec!["data/raw".into(), "data/ingest".into(), "data/extracted".into()]
}
fn d_chunk_chars() -> usize { 800 }
fn d_rrf_k() -> f32 { 60.0 }

#[derive(Debug, Deserialize, Clone)]
pub struct IndexConfig {
    #[serde(default = "d_index_dir")] pub dir: String,
    #[serde(default = "d_index_sources")] pub sources: Vec<String>,
    #[serde(default = "d_chunk_chars")] pub chunk_chars: usize,
    #[serde(default = "d_rrf_k")] pub rrf_k: f32,
}
impl Default for IndexConfig {
    fn default() -> Self {
        IndexConfig { dir: d_index_dir(), sources: d_index_sources(), chunk_chars: d_chunk_chars(), rrf_k: d_rrf_k() }
    }
}

fn default_bench_out() -> String { "bench/report".to_string() }
fn default_bench_temp() -> f64 { 0.0 }
fn default_bench_max_tokens() -> usize { 512 }

#[derive(Debug, Deserialize)]
pub struct BenchModelConfig {
    pub base_url: String,
    pub model: String,
    #[serde(default = "default_bench_temp")] pub temperature: f64,
    #[serde(default = "default_bench_max_tokens")] pub max_tokens: usize,
}

#[derive(Debug, Deserialize)]
pub struct BenchJudgeConfig {
    pub base_url: String,
    pub model: String,
}

#[derive(Debug, Deserialize)]
pub struct BenchmarkEntry {
    pub name: String,
    pub path: String,
    pub method: String,
    pub threshold: f64,
    #[serde(default)] pub system_prompt: Option<String>,
    #[serde(default)] pub min_tokens_per_sec: Option<f64>,
    #[serde(default)] pub scorer: Option<String>,
    #[serde(default)] pub max_turns: Option<usize>,
    #[serde(default)] pub fetch_chars: Option<usize>,
    #[serde(default)] pub fetch_bytes: Option<u64>,
}

fn default_search_max_results() -> usize { 5 }

#[derive(Debug, Deserialize)]
pub struct BenchSearchConfig {
    #[serde(default)] pub base_url: Option<String>,
    #[serde(default = "default_search_max_results")] pub max_results: usize,
    #[serde(default)] pub allow_hosts: Vec<String>,
}

#[derive(Debug, Deserialize)]
pub struct BenchConfig {
    #[serde(default = "default_bench_out")] pub out: String,
    pub model: BenchModelConfig,
    #[serde(default)] pub judge: Option<BenchJudgeConfig>,
    #[serde(default)] pub benchmark: Vec<BenchmarkEntry>,
    #[serde(default)] pub search: Option<BenchSearchConfig>,
}

fn d_ask_model() -> String { "kibble-style".into() }
fn d_ask_temp() -> f64 { 0.2 }
fn d_ask_max_tokens() -> usize { 1024 }
fn d_ask_top_p() -> f64 { 0.95 }
fn d_ask_top_k() -> usize { 20 }
fn d_ask_rep_penalty() -> f64 { 1.05 }
fn d_ask_k() -> usize { 6 }
fn d_ask_rounds() -> usize { 3 }
fn d_ask_stream() -> bool { true }

#[derive(Debug, Deserialize, Clone)]
pub struct AskConfig {
    #[serde(default)] pub base_url: String,
    #[serde(default = "d_ask_model")] pub model: String,
    #[serde(default = "d_ask_temp")] pub temperature: f64,
    #[serde(default = "d_ask_max_tokens")] pub max_tokens: usize,
    #[serde(default = "d_ask_top_p")] pub top_p: f64,
    #[serde(default = "d_ask_top_k")] pub top_k: usize,
    #[serde(default = "d_ask_rep_penalty")] pub repetition_penalty: f64,
    #[serde(default = "d_ask_k")] pub k: usize,
    #[serde(default = "d_ask_rounds")] pub max_rounds: usize,
    #[serde(default = "d_ask_stream")] pub stream: bool,
}
impl Default for AskConfig {
    fn default() -> Self {
        AskConfig { base_url: String::new(), model: d_ask_model(), temperature: d_ask_temp(), max_tokens: d_ask_max_tokens(), top_p: d_ask_top_p(), top_k: d_ask_top_k(), repetition_penalty: d_ask_rep_penalty(), k: d_ask_k(), max_rounds: d_ask_rounds(), stream: d_ask_stream() }
    }
}

fn d_web_max_results() -> usize { 5 }
fn d_web_fetch_bytes() -> u64 { 2_000_000 }
fn d_web_fetch_chars() -> usize { 4000 }

#[derive(Debug, Deserialize, Clone)]
pub struct WebConfig {
    #[serde(default)] pub base_url: String,
    #[serde(default)] pub default: bool,
    #[serde(default = "d_web_max_results")] pub max_results: usize,
    #[serde(default)] pub allow_hosts: Vec<String>,
    #[serde(default = "d_web_fetch_bytes")] pub fetch_bytes: u64,
    #[serde(default = "d_web_fetch_chars")] pub fetch_chars: usize,
}
impl Default for WebConfig {
    fn default() -> Self {
        WebConfig { base_url: String::new(), default: false, max_results: d_web_max_results(), allow_hosts: Vec::new(), fetch_bytes: d_web_fetch_bytes(), fetch_chars: d_web_fetch_chars() }
    }
}

fn d_tune_r() -> usize { 16 }
fn d_tune_alpha() -> usize { 16 }
fn d_tune_dropout() -> f64 { 0.05 }
fn d_tune_seq() -> usize { 4096 }
fn d_tune_lr() -> f64 { 2.0e-4 }
fn d_tune_smoke() -> usize { 20 }
fn d_tune_out() -> String { "data/tune".into() }

#[derive(Debug, Deserialize, Clone)]
pub struct TunePhase {
    pub name: String,
    #[serde(default)] pub sources: Vec<String>,
    #[serde(default)] pub max_steps: usize,
    #[serde(default)] pub resume_from: Option<String>,
}

#[derive(Debug, Deserialize, Clone)]
pub struct TuneConfig {
    #[serde(default)] pub model: String,
    #[serde(default = "d_tune_r")] pub r: usize,
    #[serde(default = "d_tune_alpha")] pub lora_alpha: usize,
    #[serde(default = "d_tune_dropout")] pub lora_dropout: f64,
    #[serde(default = "d_tune_seq")] pub max_seq_length: usize,
    #[serde(default = "d_tune_lr")] pub learning_rate: f64,
    #[serde(default = "d_tune_smoke")] pub smoke_rows: usize,
    #[serde(default = "d_tune_out")] pub out: String,
    #[serde(default)] pub phase: Vec<TunePhase>,
}

#[derive(Debug, Deserialize, Clone, Default)]
pub struct TrainConfig {
    /// argv to run against the tune package (required; empty => error at run time).
    #[serde(default)] pub command: Vec<String>,
    /// working directory for the command; default = the resolved [tune].out dir.
    #[serde(default)] pub cwd: Option<String>,
}

fn d_cluster_k() -> usize { 12 }
fn d_cluster_out() -> String { "data/clusters.json".into() }
fn d_rebalance_share() -> f64 { 0.35 }
fn d_min_cluster_size() -> usize { 2 }

#[derive(Debug, Deserialize, Clone)]
pub struct ClusterConfig {
    #[serde(default = "d_cluster_k")] pub k: usize,
    #[serde(default = "default_true")] pub enabled: bool,
    #[serde(default = "d_cluster_out")] pub out: String,
    #[serde(default)] pub rebalance: bool,
    #[serde(default = "d_rebalance_share")] pub max_topic_share: f64,
    #[serde(default)] pub llm_labels: bool,
    #[serde(default = "d_min_cluster_size")] pub min_cluster_size: usize,
}
impl Default for ClusterConfig {
    fn default() -> Self {
        ClusterConfig { k: d_cluster_k(), enabled: true, out: d_cluster_out(), rebalance: false, max_topic_share: d_rebalance_share(), llm_labels: false, min_cluster_size: d_min_cluster_size() }
    }
}

#[derive(Debug, Deserialize, Default)]
pub struct KibbleConfig {
    #[serde(default)]
    pub source: Vec<SourceConfig>,
    #[serde(default)]
    pub network: NetworkConfig,
    #[serde(default)]
    pub paths: PathsConfig,
    #[serde(default)]
    pub pack: Option<PackConfig>,
    #[serde(default)]
    pub eval: Option<EvalConfig>,
    #[serde(default)]
    pub tune: Option<TuneConfig>,
    #[serde(default)]
    pub train: Option<TrainConfig>,
    #[serde(default)]
    pub fetch: FetchConfig,
    #[serde(default)]
    pub serve: ServeConfig,
    #[serde(default)]
    pub extract: ExtractConfig,
    #[serde(default)]
    pub caps: CapsConfig,
    #[serde(default)]
    pub curate: CurateConfig,
    #[serde(default)]
    pub bench: Option<BenchConfig>,
    #[serde(default)]
    pub crawl: CrawlConfig,
    #[serde(default)]
    pub understand: UnderstandConfig,
    #[serde(default)]
    pub index: IndexConfig,
    #[serde(default)]
    pub ask: AskConfig,
    #[serde(default)]
    pub web: WebConfig,
    #[serde(default)]
    pub cluster: ClusterConfig,
    #[serde(default)]
    pub classify: ClassifyConfig,
}

#[derive(Debug, PartialEq)]
pub enum SourceType {
    Dataset,
    Codebase,
    Web,
    Blog,
    Files,
}

fn url_host(url: &str) -> Option<String> {
    // strip scheme
    let after = url.split_once("://").map(|(_, r)| r).unwrap_or(url);
    // host ends at the first '/', '?', or '#'
    let host = after.split(['/', '?', '#']).next().unwrap_or("");
    // drop userinfo and port
    let host = host.rsplit('@').next().unwrap_or(host);
    let host = host.split(':').next().unwrap_or(host);
    if host.is_empty() { None } else { Some(host.to_lowercase()) }
}

fn is_github_host(url: &str) -> bool {
    matches!(url_host(url).as_deref(), Some("github.com") | Some("www.github.com"))
}

pub fn detect_type(path: &str, explicit: Option<&str>) -> SourceType {
    if let Some(t) = explicit {
        return match t.to_lowercase().as_str() {
            "dataset" => SourceType::Dataset,
            "codebase" => SourceType::Codebase,
            "web" => SourceType::Web,
            "blog" => SourceType::Blog,
            _ => SourceType::Files,
        };
    }
    let lower = path.to_lowercase();
    if lower.starts_with("http://") || lower.starts_with("https://") {
        if is_github_host(path) {
            return SourceType::Codebase;
        }
        return SourceType::Web;
    }
    if lower.ends_with(".jsonl") {
        return SourceType::Dataset;
    }
    if lower.ends_with(".html") || lower.ends_with(".htm") {
        return SourceType::Blog;
    }
    let p = Path::new(path);
    if p.is_dir() {
        if p.join(".git").exists() {
            return SourceType::Codebase;
        }
        if let Ok(entries) = std::fs::read_dir(p) {
            for entry in entries.flatten() {
                let ep = entry.path();
                if ep.is_file() {
                    if let Some(name) = ep.file_name().and_then(|n| n.to_str()) {
                        #[cfg(feature = "full")]
                        let is_code_file = !crate::codebase::lang_for_path(name).is_empty();
                        #[cfg(not(feature = "full"))]
                        let is_code_file = false;
                        if is_code_file || name.eq_ignore_ascii_case("dockerfile") {
                            return SourceType::Codebase;
                        }
                    }
                }
            }
        }
        return SourceType::Files;
    }
    SourceType::Files
}

pub fn load_config(path: &Path) -> KibbleConfig {
    std::fs::read_to_string(path)
        .ok()
        .and_then(|s| toml::from_str(&s).ok())
        .unwrap_or_default()
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn detects_types_from_path() {
        assert!(matches!(detect_type("https://github.com/u/r", None), SourceType::Codebase));
        assert!(matches!(detect_type("https://example.com/post", None), SourceType::Web));
        assert!(matches!(detect_type("data/x.jsonl", None), SourceType::Dataset));
        assert!(matches!(detect_type("page.html", None), SourceType::Blog));
    }

    #[test]
    fn explicit_type_overrides() {
        assert!(matches!(detect_type("https://example.com/x", Some("dataset")), SourceType::Dataset));
    }

    #[test]
    fn parses_toml_sources_and_network() {
        let toml = r#"
            [[source]]
            path = "a.jsonl"
            name = "mythos"
            max_rows = 5

            [[source]]
            path = "https://github.com/u/r"

            [network]
            proxy = "socks5://127.0.0.1:9050"
        "#;
        let cfg: KibbleConfig = toml::from_str(toml).unwrap();
        assert_eq!(cfg.source.len(), 2);
        assert_eq!(cfg.source[0].source_name(), "mythos");
        assert_eq!(cfg.source[0].max_rows, Some(5));
        // default name derives from path stem
        assert_eq!(cfg.source[1].source_name(), "r");
        assert_eq!(cfg.network.proxy.as_deref(), Some("socks5://127.0.0.1:9050"));
    }

    #[test]
    fn missing_config_is_default() {
        let cfg = load_config(Path::new("/no/such/kibble.toml"));
        assert!(cfg.source.is_empty());
    }

    #[test]
    fn github_detection_is_host_anchored() {
        // genuine github → Codebase
        assert!(matches!(detect_type("https://github.com/u/r", None), SourceType::Codebase));
        assert!(matches!(detect_type("https://www.github.com/u/r", None), SourceType::Codebase));
        // a non-github host that merely contains the string must be Web, not Codebase
        assert!(matches!(detect_type("https://evil.com/?x=github.com", None), SourceType::Web));
        assert!(matches!(detect_type("https://notgithub.com.evil.net/x", None), SourceType::Web));
    }

    #[test]
    fn paths_default_and_override() {
        let def: KibbleConfig = toml::from_str("").unwrap();
        assert_eq!(def.paths.data_root, "data");
        assert_eq!(def.paths.dataset_dir, "data/datasets/unsloth");

        let cfg: KibbleConfig = toml::from_str(
            "[paths]\ndata_root = \"corpus\"\ndataset_dir = \"out/ds\"\n",
        )
        .unwrap();
        assert_eq!(cfg.paths.data_root, "corpus");
        assert_eq!(cfg.paths.dataset_dir, "out/ds");
    }

    #[test]
    fn parses_pack_table() {
        let toml = r#"
            [pack]
            name = "style-lora"
            splits = ["train", "valid"]
            zip = "out/bundle.zip"

            [[pack.include]]
            src = "kaggle/train_kaggle.py"
            dest = "scripts/train_kaggle.py"

            [pack.metadata]
            id = "example-user/example-dataset"
            title = "Example Dataset"
            license = "other"
        "#;
        let cfg: KibbleConfig = toml::from_str(toml).unwrap();
        let pack = cfg.pack.unwrap();
        assert_eq!(pack.name, "style-lora");
        assert_eq!(pack.splits, vec!["train", "valid"]);
        assert_eq!(pack.staging, "kaggle/dataset-staging"); // default
        assert_eq!(pack.include.len(), 1);
        assert_eq!(pack.include[0].dest, "scripts/train_kaggle.py");
        assert_eq!(pack.metadata.id.as_deref(), Some("example-user/example-dataset"));
    }

    #[test]
    fn missing_pack_is_none() {
        let cfg: KibbleConfig = toml::from_str("").unwrap();
        assert!(cfg.pack.is_none());
    }

    #[test]
    fn eval_config_defaults_and_override() {
        let cfg: KibbleConfig = toml::from_str("[eval]\n").unwrap();
        let e = cfg.eval.unwrap();
        assert_eq!(e.method, "sft");
        assert_eq!(e.out, "eval/report");
        assert!(e.dataset_dir.is_none());
        assert_eq!(e.thresholds.max_duplicate_rate, 0.02);
        let o: KibbleConfig = toml::from_str("[eval]\ndataset_dir=\"d\"\n[eval.thresholds]\nmax_duplicate_rate=0.05\n").unwrap();
        let e = o.eval.unwrap();
        assert_eq!(e.dataset_dir.as_deref(), Some("d"));
        assert_eq!(e.thresholds.max_duplicate_rate, 0.05);
        assert_eq!(e.thresholds.min_rows, 50);
    }

    #[test]
    fn eval_max_topic_share_default_and_override() {
        assert!((EvalThresholds::default().max_topic_share - 0.60).abs() < 1e-9);
        let cfg: KibbleConfig = toml::from_str("[eval.thresholds]\nmax_topic_share = 0.4\n").unwrap();
        assert!((cfg.eval.unwrap().thresholds.max_topic_share - 0.4).abs() < 1e-9);
    }

    #[test]
    fn bench_config_parses() {
        let toml = r#"
[bench]
out = "bench/report"
[bench.model]
base_url = "http://localhost:8000/v1"
model = "style-lora"
[[bench.benchmark]]
name = "qa"
path = "benchmarks/qa.jsonl"
method = "contains"
threshold = 0.7
"#;
        let cfg: KibbleConfig = toml::from_str(toml).unwrap();
        let b = cfg.bench.unwrap();
        assert_eq!(b.out, "bench/report");
        assert_eq!(b.model.model, "style-lora");
        assert_eq!(b.model.temperature, 0.0);
        assert_eq!(b.model.max_tokens, 512);
        assert_eq!(b.benchmark.len(), 1);
        assert_eq!(b.benchmark[0].method, "contains");
        assert!(b.judge.is_none());
    }

    #[test]
    fn fetch_defaults_and_override() {
        let def: KibbleConfig = toml::from_str("").unwrap();
        assert_eq!(def.fetch.ingest_root, "data/ingest");
        assert_eq!(def.fetch.max_bytes, 1_073_741_824);
        assert_eq!(def.fetch.datasets_root, "data/datasets");
        assert_eq!(def.fetch.models_root, "data/models");

        let cfg: KibbleConfig = toml::from_str("[fetch]\ningest_root = \"dl\"\nmax_bytes = 1000\n").unwrap();
        assert_eq!(cfg.fetch.ingest_root, "dl");
        assert_eq!(cfg.fetch.max_bytes, 1000);
    }

    #[test]
    fn serve_defaults_and_override() {
        let def: KibbleConfig = toml::from_str("").unwrap();
        assert_eq!(def.serve.bind, "127.0.0.1:7777");
        assert!(def.serve.allow_hosts.is_empty());

        let cfg: KibbleConfig = toml::from_str("[serve]\nbind = \"0.0.0.0:9000\"\nallow_hosts = [\"127.0.0.1\"]\n").unwrap();
        assert_eq!(cfg.serve.bind, "0.0.0.0:9000");
        assert_eq!(cfg.serve.allow_hosts, vec!["127.0.0.1".to_string()]);
    }

    #[test]
    fn extract_defaults_and_override() {
        let def: KibbleConfig = toml::from_str("").unwrap();
        assert_eq!(def.extract.out, "data/extracted");
        assert_eq!(def.extract.max_bytes, 524_288_000);
        assert_eq!(def.extract.ocr.cli, "");

        let cfg: KibbleConfig = toml::from_str(
            "[extract]\nout = \"ex\"\n\n[extract.ocr]\nbase_url = \"http://o\"\nmodel = \"m\"\ncli = \"tesseract\"\n",
        )
        .unwrap();
        assert_eq!(cfg.extract.out, "ex");
        assert_eq!(cfg.extract.ocr.base_url, "http://o");
        assert_eq!(cfg.extract.ocr.cli, "tesseract");
    }

    #[test]
    fn extract_pdf_defaults_and_override() {
        let def: KibbleConfig = toml::from_str("").unwrap();
        assert_eq!(def.extract.pdf.render_cli, "pdftoppm");
        assert_eq!(def.extract.pdf.min_chars, 50);

        let cfg: KibbleConfig = toml::from_str("[extract.pdf]\nrender_cli = \"mutool\"\nmin_chars = 10\n").unwrap();
        assert_eq!(cfg.extract.pdf.render_cli, "mutool");
        assert_eq!(cfg.extract.pdf.min_chars, 10);
    }

    #[test]
    fn extract_video_office_defaults_and_override() {
        let def: KibbleConfig = toml::from_str("").unwrap();
        assert_eq!(def.extract.video.cli, "ffmpeg");
        assert_eq!(def.extract.office.cli, "pandoc");

        let cfg: KibbleConfig = toml::from_str("[extract.video]\ncli = \"avconv\"\n\n[extract.office]\ncli = \"soffice\"\n").unwrap();
        assert_eq!(cfg.extract.video.cli, "avconv");
        assert_eq!(cfg.extract.office.cli, "soffice");
    }

    #[test]
    fn caps_defaults_and_override() {
        let def: KibbleConfig = toml::from_str("").unwrap();
        assert_eq!(def.caps.root, "~/.kibble/caps");
        assert_eq!(def.caps.max_bytes, 524_288_000);
        let cfg: KibbleConfig = toml::from_str("[caps]\nroot = \"/tmp/caps\"\n").unwrap();
        assert_eq!(cfg.caps.root, "/tmp/caps");
    }

    #[test]
    fn curate_config_defaults_and_override() {
        let def: KibbleConfig = toml::from_str("").unwrap();
        assert!(def.curate.dedup);
        assert!(def.curate.leakage_safe_split);
        let cfg: KibbleConfig = toml::from_str("[curate]\nleakage_safe_split = false\n").unwrap();
        assert!(cfg.curate.dedup); // unspecified keeps default
        assert!(!cfg.curate.leakage_safe_split);
    }

    #[test]
    fn curate_filter_defaults() {
        let def: KibbleConfig = toml::from_str("").unwrap();
        assert!(def.curate.drop_malformed);
        assert!(def.curate.drop_degenerate);
        assert_eq!(def.curate.min_answer_chars, 20);
        let cfg: KibbleConfig = toml::from_str("[curate]\ndrop_degenerate = false\nmin_answer_chars = 5\n").unwrap();
        assert!(cfg.curate.drop_malformed);     // default kept
        assert!(!cfg.curate.drop_degenerate);
        assert_eq!(cfg.curate.min_answer_chars, 5);
    }

    #[test]
    fn curate_near_dedup_defaults() {
        let def: KibbleConfig = toml::from_str("").unwrap();
        assert!(!def.curate.near_dedup);
        assert_eq!(def.curate.near_dedup_threshold, 0.85);
        assert_eq!(def.curate.shingle_size, 5);
        let cfg: KibbleConfig = toml::from_str("[curate]\nnear_dedup = true\nnear_dedup_threshold = 0.7\n").unwrap();
        assert!(cfg.curate.near_dedup);
        assert_eq!(cfg.curate.near_dedup_threshold, 0.7);
        assert_eq!(cfg.curate.shingle_size, 5);
    }

    #[test]
    fn classify_config_default_and_parse() {
        assert!(!KibbleConfig::default().classify.enabled);
        let cfg: KibbleConfig = toml::from_str("[classify]\nenabled = true\n").unwrap();
        assert!(cfg.classify.enabled);
    }

    #[test]
    fn eval_near_dup_config_defaults() {
        let def: KibbleConfig = toml::from_str("[eval]\n").unwrap();
        let e = def.eval.unwrap();
        assert!(!e.near_dup);
        assert_eq!(e.thresholds.max_near_duplicate_rate, 0.02);
        assert_eq!(e.thresholds.max_near_leakage_rate, 0.0);
        assert_eq!(e.thresholds.near_dup_threshold, 0.85);
        assert_eq!(e.thresholds.near_shingle_size, 5);
    }

    #[test]
    fn bench_search_and_research_fields() {
        let toml = "[bench]\n[bench.model]\nbase_url=\"u\"\nmodel=\"m\"\n[bench.search]\nmax_results=3\nallow_hosts=[\"127.0.0.1\"]\n[[bench.benchmark]]\nname=\"r\"\npath=\"r.jsonl\"\nmethod=\"research\"\nthreshold=0.5\nscorer=\"contains\"\nmax_turns=3\n";
        let cfg: KibbleConfig = toml::from_str(toml).unwrap();
        let b = cfg.bench.unwrap();
        let s = b.search.unwrap();
        assert_eq!(s.max_results, 3);
        assert_eq!(s.allow_hosts, vec!["127.0.0.1".to_string()]);
        assert!(s.base_url.is_none());
        assert_eq!(b.benchmark[0].scorer.as_deref(), Some("contains"));
        assert_eq!(b.benchmark[0].max_turns, Some(3));
    }

    #[test]
    fn crawl_config_defaults() {
        let def: KibbleConfig = toml::from_str("").unwrap();
        assert_eq!(def.crawl.max_pages, 50);
        assert_eq!(def.crawl.max_depth, 2);
        assert!(def.crawl.same_host && def.crawl.respect_robots);
        let cfg: KibbleConfig = toml::from_str("[crawl]\nmax_pages=10\ndelay_ms=0\n").unwrap();
        assert_eq!(cfg.crawl.max_pages, 10);
        assert_eq!(cfg.crawl.delay_ms, 0);
    }

    #[test]
    fn web_config_defaults_and_parse() {
        let def: KibbleConfig = toml::from_str("").unwrap();
        assert_eq!(def.web.base_url, "");
        assert!(!def.web.default);
        assert_eq!(def.web.max_results, 5);
        assert_eq!(def.web.fetch_bytes, 2_000_000);
        assert_eq!(def.web.fetch_chars, 4000);

        let c: KibbleConfig = toml::from_str("[web]\nbase_url=\"https://xng.example\"\ndefault=true\nmax_results=3\n").unwrap();
        assert_eq!(c.web.base_url, "https://xng.example");
        assert!(c.web.default);
        assert_eq!(c.web.max_results, 3);
    }

    #[test]
    fn ask_stream_defaults_true_and_parses() {
        let def: KibbleConfig = toml::from_str("").unwrap();
        assert!(def.ask.stream);
        let c: KibbleConfig = toml::from_str("[ask]\nstream=false\n").unwrap();
        assert!(!c.ask.stream);
    }

    #[test]
    fn detects_code_dir_without_git() {
        use std::fs;
        let dir = std::env::temp_dir().join(format!("kibble_detect_cb_{}", std::process::id()));
        let _ = fs::remove_dir_all(&dir);
        fs::create_dir_all(&dir).unwrap();
        fs::write(dir.join("main.rs"), "fn main() {}").unwrap();
        assert!(matches!(detect_type(dir.to_str().unwrap(), None), SourceType::Codebase));
    }

    #[test]
    fn cluster_rebalance_defaults_and_override() {
        let d = ClusterConfig::default();
        assert!(!d.rebalance);
        assert!((d.max_topic_share - 0.35).abs() < 1e-9);
        let cfg: KibbleConfig = toml::from_str("[cluster]\nrebalance = true\nmax_topic_share = 0.5\n").unwrap();
        assert!(cfg.cluster.rebalance);
        assert!((cfg.cluster.max_topic_share - 0.5).abs() < 1e-9);
    }

    #[test]
    fn cluster_llm_labels_default_and_override() {
        assert!(!ClusterConfig::default().llm_labels);
        let cfg: KibbleConfig = toml::from_str("[cluster]\nllm_labels = true\n").unwrap();
        assert!(cfg.cluster.llm_labels);
    }

    #[test]
    fn cluster_min_cluster_size_default_and_override() {
        assert_eq!(ClusterConfig::default().min_cluster_size, 2);
        let cfg: KibbleConfig = toml::from_str("[cluster]\nmin_cluster_size = 0\n").unwrap();
        assert_eq!(cfg.cluster.min_cluster_size, 0);
    }

    #[test]
    fn tune_config_parses() {
        let cfg: KibbleConfig = toml::from_str(concat!(
            "[tune]\nmodel = \"m\"\nsmoke_rows = 5\nout = \"data/tune\"\n",
            "[[tune.phase]]\nname = \"p1\"\nsources = [\"a\",\"b\"]\nmax_steps = 300\n",
            "[[tune.phase]]\nname = \"p2\"\nsources = [\"c\"]\nmax_steps = 400\nresume_from = \"p1\"\n",
        )).unwrap();
        let t = cfg.tune.unwrap();
        assert_eq!(t.model, "m");
        assert_eq!(t.smoke_rows, 5);
        assert_eq!(t.phase.len(), 2);
        assert_eq!(t.phase[0].name, "p1");
        assert_eq!(t.phase[0].sources, vec!["a".to_string(), "b".to_string()]);
        assert_eq!(t.phase[1].resume_from.as_deref(), Some("p1"));
        // defaults
        assert_eq!(t.r, 16);
        assert_eq!(t.max_seq_length, 4096);
    }

    #[test]
    fn train_config_parses() {
        let cfg: KibbleConfig = toml::from_str(
            "[train]\ncommand = [\"python\", \"t.py\", \"--config\", \"{config}\"]\ncwd = \"data/tune\"\n",
        ).unwrap();
        let t = cfg.train.expect("train section");
        assert_eq!(t.command, vec!["python", "t.py", "--config", "{config}"]);
        assert_eq!(t.cwd.as_deref(), Some("data/tune"));
        // absent => None
        let none: KibbleConfig = toml::from_str("").unwrap();
        assert!(none.train.is_none());
    }
}