{
"metadata": {
"version": "1.0.0",
"description": "Perplexity baselines for RuvLTRA-Small quality validation",
"model": "ruvltra-small",
"quantization_tested": ["Q4_K", "Q5_K", "Q8_0", "F16"],
"last_updated": "2024-01-19"
},
"quality_thresholds": {
"max_acceptable_perplexity": 50.0,
"warning_perplexity": 30.0,
"excellent_perplexity": 15.0,
"notes": "Perplexity values vary by dataset and prompt type"
},
"baselines": {
"wikitext": {
"description": "WikiText-2 test set perplexity",
"dataset_url": "https://huggingface.co/datasets/wikitext",
"values": {
"F16": {
"perplexity": 8.5,
"tokens_evaluated": 250000,
"notes": "Full precision baseline"
},
"Q8_0": {
"perplexity": 8.7,
"degradation_pct": 2.4,
"notes": "8-bit quantization, minimal quality loss"
},
"Q5_K": {
"perplexity": 9.2,
"degradation_pct": 8.2,
"notes": "5-bit k-quant, good balance"
},
"Q4_K": {
"perplexity": 9.8,
"degradation_pct": 15.3,
"notes": "4-bit k-quant, most common deployment format"
},
"Q2_K": {
"perplexity": 14.5,
"degradation_pct": 70.6,
"notes": "2-bit extreme quantization, noticeable degradation"
}
}
},
"lambada": {
"description": "LAMBADA last-word prediction accuracy",
"metric": "accuracy",
"values": {
"F16": {
"accuracy": 0.72,
"notes": "Full precision accuracy"
},
"Q4_K": {
"accuracy": 0.68,
"degradation_pct": 5.6,
"notes": "Slight accuracy drop acceptable"
}
}
},
"hellaswag": {
"description": "HellaSwag commonsense reasoning",
"metric": "accuracy",
"values": {
"F16": {
"accuracy": 0.68
},
"Q4_K": {
"accuracy": 0.65,
"degradation_pct": 4.4
}
}
},
"custom_prompts": {
"description": "Perplexity on custom test prompts",
"values": {
"simple_completion": {
"expected_ppl_range": [5.0, 20.0],
"notes": "Common phrase continuation should have low perplexity"
},
"code_generation": {
"expected_ppl_range": [8.0, 30.0],
"notes": "Code has higher entropy but should still be coherent"
},
"creative_writing": {
"expected_ppl_range": [15.0, 45.0],
"notes": "Creative tasks have higher acceptable perplexity"
},
"factual_qa": {
"expected_ppl_range": [3.0, 15.0],
"notes": "Factual responses should be confident"
}
}
}
},
"degradation_limits": {
"max_perplexity_increase_pct": 20.0,
"max_accuracy_decrease_pct": 10.0,
"notes": "Quantization should not degrade quality beyond these limits"
},
"token_probability_thresholds": {
"min_top1_probability": 0.01,
"min_top5_cumulative": 0.1,
"max_entropy": 10.0,
"notes": "Thresholds for detecting garbled or degenerate output"
},
"repetition_metrics": {
"max_ngram_repetition_ratio": 0.3,
"max_consecutive_repeats": 3,
"ngram_window_sizes": [2, 3, 4],
"notes": "Detect excessive repetition in generated text"
},
"coherence_metrics": {
"min_sentence_length": 3,
"max_sentence_length": 200,
"punctuation_ratio_range": [0.01, 0.15],
"alphanumeric_ratio_min": 0.7,
"notes": "Basic structural coherence checks"
},
"speed_baselines": {
"description": "Token generation speed baselines (tokens/second)",
"device_baselines": {
"m4_pro_ane": {
"prompt_processing": 2000,
"generation": 60,
"notes": "M4 Pro with ANE acceleration"
},
"m4_pro_neon": {
"prompt_processing": 1500,
"generation": 45,
"notes": "M4 Pro NEON-only fallback"
},
"m1_ane": {
"prompt_processing": 1200,
"generation": 40,
"notes": "M1 with ANE"
},
"cpu_x86": {
"prompt_processing": 500,
"generation": 15,
"notes": "x86 CPU baseline (AVX2)"
}
}
},
"memory_baselines": {
"model_sizes_mb": {
"F16": 4000,
"Q8_0": 2200,
"Q4_K": 1200,
"Q2_K": 700
},
"kv_cache_per_token_bytes": {
"F16": 1100,
"Q8_0": 1100,
"notes": "KV cache typically stays in F16 for accuracy"
},
"peak_memory_multiplier": 1.5,
"notes": "Peak memory = model_size * multiplier during inference"
}
}