1use serde::{Deserialize, Serialize};
9use std::collections::HashMap;
10
11pub mod real_evaluator;
16pub use real_evaluator::{
17 generate_milestone_json, generate_milestone_report, optimize_all_gemm_problems,
18 optimize_gemm_problem, standard_gemm_problems, GemmOptResult, GemmProblemConfig,
19 RealGemmEvaluator,
20};
21
22pub mod attention_eval;
23pub use attention_eval::{
24 generate_attention_milestone_json, generate_attention_milestone_report,
25 optimize_all_attention_problems, optimize_attention_problem, standard_attention_problems,
26 AttentionOptResult, AttentionProblemConfig, RealAttentionEvaluator,
27};
28
29pub mod fused_eval;
30pub use fused_eval::{
31 fused_gemm_problems, generate_beat_cublas_json, generate_beat_cublas_report,
32 optimize_fused_problem, run_beat_cublas_campaign, FusedGemmEvaluator, FusedGemmOptResult,
33 FusedGemmProblem,
34};
35
36pub mod conv2d_eval;
37pub use conv2d_eval::{
38 generate_conv2d_milestone_json, generate_conv2d_milestone_report, optimize_all_conv2d_problems,
39 optimize_conv2d_problem, standard_conv2d_problems, Conv2dOptResult, Conv2dProblemConfig,
40 RealConv2dEvaluator,
41};
42
43pub mod normalization_eval;
44pub use normalization_eval::{
45 generate_normalization_milestone_json, generate_normalization_milestone_report,
46 optimize_all_normalization_problems, optimize_normalization_problem,
47 standard_normalization_problems, NormalizationOptResult, NormalizationProblemConfig,
48 RealNormalizationEvaluator,
49};
50
51pub mod vector_add_eval;
52pub use vector_add_eval::{
53 generate_vector_add_milestone_json, generate_vector_add_milestone_report,
54 optimize_all_vector_add_problems, optimize_vector_add_problem, standard_vector_add_problems,
55 RealVectorAddEvaluator, VectorAddOptResult, VectorAddProblemConfig,
56};
57
58pub mod kernel_category;
59pub use kernel_category::KernelCategory;
60
61impl<const N: usize> From<[(&'static str, u32); N]> for TuningParams {
68 fn from(arr: [(&'static str, u32); N]) -> Self {
69 let mut map = HashMap::with_capacity(N);
70 for (k, v) in arr {
71 map.insert(k.to_string(), v);
72 }
73 TuningParams(map)
74 }
75}
76pub type ParamVal = u32;
83
84#[derive(Debug, Clone, PartialEq, Eq, Default, Serialize, Deserialize)]
86pub struct TuningParams(pub HashMap<String, ParamVal>);
87
88impl TuningParams {
89 pub fn get(&self, key: &str) -> Option<ParamVal> {
90 self.0.get(key).copied()
91 }
92
93 pub fn set(&mut self, key: impl Into<String>, val: ParamVal) {
94 self.0.insert(key.into(), val);
95 }
96
97 pub fn display(&self) -> String {
98 let mut pairs: Vec<_> = self.0.iter().collect();
99 pairs.sort_by_key(|(k, _)| k.as_str());
100 pairs
101 .iter()
102 .map(|(k, v)| format!("{}={}", k, v))
103 .collect::<Vec<_>>()
104 .join(", ")
105 }
106}
107
108#[derive(Debug, Clone)]
110pub struct ParamDim {
111 pub name: String,
112 pub values: Vec<ParamVal>,
113}
114
115impl ParamDim {
116 pub fn new(name: impl Into<String>, values: Vec<ParamVal>) -> Self {
117 Self {
118 name: name.into(),
119 values,
120 }
121 }
122
123 pub fn powers_of_two(name: impl Into<String>, min: u32, max: u32) -> Self {
125 let mut vals = Vec::new();
126 let mut v = min;
127 while v <= max {
128 vals.push(v);
129 v *= 2;
130 }
131 Self {
132 name: name.into(),
133 values: vals,
134 }
135 }
136}
137
138#[derive(Debug, Clone)]
140pub struct ParamSpace {
141 pub dims: Vec<ParamDim>,
142}
143
144impl ParamSpace {
145 pub fn new(dims: Vec<ParamDim>) -> Self {
146 Self { dims }
147 }
148
149 pub fn gemm_default() -> Self {
151 Self::new(vec![
152 ParamDim::powers_of_two("tile_m", 16, 128),
153 ParamDim::powers_of_two("tile_n", 16, 128),
154 ParamDim::powers_of_two("tile_k", 8, 64),
155 ParamDim::powers_of_two("vec_width", 1, 8),
156 ParamDim::new("unroll", vec![1, 2, 4, 8]),
157 ])
158 }
159
160 pub fn attention_default() -> Self {
162 Self::new(vec![
163 ParamDim::powers_of_two("tile_seq", 16, 128),
164 ParamDim::powers_of_two("tile_head", 4, 64),
165 ParamDim::powers_of_two("tile_k", 16, 64),
166 ParamDim::powers_of_two("vec_width", 1, 8),
167 ParamDim::new("unroll", vec![1, 2, 4, 8]),
168 ])
169 }
170
171 pub fn total_configs(&self) -> usize {
173 self.dims.iter().map(|d| d.values.len()).product()
174 }
175
176 pub fn all_params(&self) -> Vec<TuningParams> {
178 let mut result: Vec<HashMap<String, ParamVal>> = vec![HashMap::new()];
179 for dim in &self.dims {
180 let mut next = Vec::with_capacity(result.len() * dim.values.len());
181 for existing in &result {
182 for &val in &dim.values {
183 let mut p = existing.clone();
184 p.insert(dim.name.clone(), val);
185 next.push(p);
186 }
187 }
188 result = next;
189 }
190 result.into_iter().map(TuningParams).collect()
191 }
192
193 pub fn neighbors(&self, params: &TuningParams) -> Vec<TuningParams> {
195 let mut neighbors = Vec::new();
196 for dim in &self.dims {
197 let current = match params.0.get(&dim.name) {
198 Some(&v) => v,
199 None => continue,
200 };
201 let idx = match dim.values.iter().position(|&v| v == current) {
202 Some(i) => i,
203 None => continue,
204 };
205 if idx > 0 {
206 let mut n = params.clone();
207 n.0.insert(dim.name.clone(), dim.values[idx - 1]);
208 neighbors.push(n);
209 }
210 if idx + 1 < dim.values.len() {
211 let mut n = params.clone();
212 n.0.insert(dim.name.clone(), dim.values[idx + 1]);
213 neighbors.push(n);
214 }
215 }
216 neighbors
217 }
218
219 pub fn clamp(&self, raw: &HashMap<String, u64>) -> TuningParams {
221 let mut params = HashMap::new();
222 for dim in &self.dims {
223 let target = raw.get(&dim.name).copied().unwrap_or(dim.values[0] as u64) as i64;
224 let closest = dim
225 .values
226 .iter()
227 .min_by_key(|&&v| (v as i64 - target).abs())
228 .copied()
229 .unwrap_or(dim.values[0]);
230 params.insert(dim.name.clone(), closest);
231 }
232 TuningParams(params)
233 }
234}
235
236#[derive(Debug, Clone, Serialize, Deserialize)]
242pub struct OptResult {
243 pub params: TuningParams,
244 pub score: f64,
246 pub eval_count: usize,
248}
249
250pub trait KernelEvaluator: Send {
251 fn evaluate(&self, params: &TuningParams) -> f64;
252}
253
254pub struct SimulatedEvaluator {
258 pub kernel_name: String,
259}
260
261impl SimulatedEvaluator {
262 pub fn new(kernel_name: impl Into<String>) -> Self {
263 Self {
264 kernel_name: kernel_name.into(),
265 }
266 }
267}
268
269impl KernelEvaluator for SimulatedEvaluator {
270 fn evaluate(&self, params: &TuningParams) -> f64 {
271 let category = crate::kernel_category::KernelCategory::from_name(&self.kernel_name);
272 let base = match category {
273 Some(crate::kernel_category::KernelCategory::Gemm) => Self::score_gemm(params),
274 Some(crate::kernel_category::KernelCategory::Attention) => {
275 Self::score_attention(params)
276 }
277 Some(crate::kernel_category::KernelCategory::Conv2d) => Self::score_conv2d(params),
278 Some(crate::kernel_category::KernelCategory::VectorAdd) => {
279 Self::score_vector_add(params)
280 }
281 Some(crate::kernel_category::KernelCategory::Normalization) => {
282 Self::score_normalization(params)
283 }
284 None => Self::score_gemm(params),
285 };
286 use std::collections::hash_map::DefaultHasher;
287 use std::hash::{Hash, Hasher};
288 let mut h = DefaultHasher::new();
289 params.display().hash(&mut h);
290 let noise = ((h.finish() % 200) as f64 / 200.0 - 0.5) * 0.04 * base;
291 (base + noise).max(1.0)
292 }
293}
294
295impl SimulatedEvaluator {
296 pub fn score_gemm(params: &TuningParams) -> f64 {
297 let tile_m = params.get("tile_m").unwrap_or(32) as f64;
298 let tile_n = params.get("tile_n").unwrap_or(32) as f64;
299 let tile_k = params.get("tile_k").unwrap_or(16) as f64;
300 let vec_width = params.get("vec_width").unwrap_or(4) as f64;
301 let unroll = params.get("unroll").unwrap_or(4) as f64;
302 let tile_score = (1.0 - ((tile_m - 64.0).abs() + (tile_n - 64.0).abs()) / 256.0).max(0.2);
303 let k_score = (1.0 - (tile_k - 32.0).abs() / 64.0).max(0.2);
304 let vec_score = (vec_width / 4.0).min(1.0);
305 let unroll_score = (unroll / 4.0).min(1.0);
306 120.0 * tile_score * k_score * vec_score * unroll_score
307 }
308
309 pub fn score_attention(params: &TuningParams) -> f64 {
310 let block_q = params.get("block_q").unwrap_or(32) as f64;
311 let block_kv = params.get("block_kv").unwrap_or(32) as f64;
312 let num_heads = params.get("num_heads").unwrap_or(4) as f64;
313 let head_dim = params.get("head_dim").unwrap_or(64) as f64;
314 let unroll = params.get("unroll").unwrap_or(2) as f64;
315 let bq_score = (1.0 - (block_q - 64.0).abs() / 128.0).max(0.2);
316 let bkv_score = (1.0 - (block_kv - 64.0).abs() / 128.0).max(0.2);
317 let heads_score = (num_heads / 8.0).clamp(0.25, 1.0);
318 let dim_score = (1.0 - (head_dim - 64.0).abs() / 128.0).max(0.3);
319 let unroll_score = (unroll / 4.0).clamp(0.5, 1.0);
320 200.0 * bq_score * bkv_score * heads_score * dim_score * unroll_score
321 }
322
323 pub fn score_conv2d(params: &TuningParams) -> f64 {
324 let tile_oc = params.get("tile_oc").unwrap_or(32) as f64;
325 let tile_ic = params.get("tile_ic").unwrap_or(16) as f64;
326 let tile_oh = params.get("tile_oh").unwrap_or(8) as f64;
327 let tile_ow = params.get("tile_ow").unwrap_or(8) as f64;
328 let kernel_w = params.get("kernel_w").unwrap_or(3) as f64;
329 let kernel_h = params.get("kernel_h").unwrap_or(3) as f64;
330 let stride = params.get("stride").unwrap_or(1) as f64;
331 let unroll = params.get("unroll").unwrap_or(2) as f64;
332 let oc_score = (1.0 - (tile_oc - 64.0).abs() / 128.0).max(0.2);
333 let ic_score = (1.0 - (tile_ic - 32.0).abs() / 64.0).max(0.2);
334 let oh_score = (1.0 - (tile_oh - 16.0).abs() / 32.0).max(0.2);
335 let ow_score = (1.0 - (tile_ow - 16.0).abs() / 32.0).max(0.2);
336 let kw_score = (1.0 - (kernel_w - 3.0).abs() / 8.0).max(0.3);
337 let kh_score = (1.0 - (kernel_h - 3.0).abs() / 8.0).max(0.3);
338 let stride_score = if stride <= 1.0 { 1.0 } else { 0.7 };
339 let unroll_score = (unroll / 4.0).clamp(0.5, 1.0);
340 150.0
341 * oc_score
342 * ic_score
343 * oh_score
344 * ow_score
345 * kw_score
346 * kh_score
347 * stride_score
348 * unroll_score
349 }
350
351 pub fn score_vector_add(params: &TuningParams) -> f64 {
352 let block_size = params.get("block_size").unwrap_or(128) as f64;
353 let vec_width = params.get("vec_width").unwrap_or(2) as f64;
354 let grid_size = params.get("grid_size").unwrap_or(16) as f64;
355 let bs_score = (1.0 - (block_size - 256.0).abs() / 512.0).max(0.2);
356 let vw_score = (vec_width / 4.0).clamp(0.25, 1.0);
357 let gs_score = (grid_size / 32.0).clamp(0.25, 1.0);
358 16.0 * bs_score * vw_score * gs_score
359 }
360
361 pub fn score_normalization(params: &TuningParams) -> f64 {
362 let block_size = params.get("block_size").unwrap_or(128) as f64;
363 let vec_width = params.get("vec_width").unwrap_or(2) as f64;
364 let unroll = params.get("unroll").unwrap_or(2) as f64;
365 let warp_reduce = params.get("warp_reduce").unwrap_or(1) as f64;
366 let bs_score = (1.0 - (block_size - 256.0).abs() / 512.0).max(0.2);
367 let vw_score = (vec_width / 4.0).clamp(0.25, 1.0);
368 let ur_score = (unroll / 4.0).clamp(0.5, 1.0);
369 let wr_score = if warp_reduce >= 1.0 { 1.0 } else { 0.6 };
370 32.0 * bs_score * vw_score * ur_score * wr_score
371 }
372}
373
374pub fn grid_search(space: &ParamSpace, eval: &dyn KernelEvaluator) -> Vec<OptResult> {
380 let all = space.all_params();
381 let total = all.len();
382 let mut results: Vec<OptResult> = all
383 .iter()
384 .enumerate()
385 .map(|(i, params)| {
386 let score = eval.evaluate(params);
387 if i == 0 || (i + 1) % 50 == 0 || i + 1 == total {
388 eprintln!(" grid [{}/{}] best so far: {:.2}", i + 1, total, score);
389 }
390 OptResult {
391 params: params.clone(),
392 score,
393 eval_count: i + 1,
394 }
395 })
396 .collect();
397 results.sort_by(|a, b| {
398 b.score
399 .partial_cmp(&a.score)
400 .unwrap_or(std::cmp::Ordering::Equal)
401 });
402 results
403}
404
405pub fn hill_climb(
411 space: &ParamSpace,
412 start: &TuningParams,
413 eval: &dyn KernelEvaluator,
414 max_iters: usize,
415) -> OptResult {
416 let mut current = start.clone();
417 let mut current_score = eval.evaluate(¤t);
418 let mut eval_count = 1;
419
420 for iter in 0..max_iters {
421 let neighbors = space.neighbors(¤t);
422 let mut improved = false;
423
424 for neighbor in &neighbors {
425 let score = eval.evaluate(neighbor);
426 eval_count += 1;
427 if score > current_score {
428 eprintln!(
429 " hill-climb [iter {}] {} → {:.2}",
430 iter + 1,
431 neighbor.display(),
432 score
433 );
434 current = neighbor.clone();
435 current_score = score;
436 improved = true;
437 break;
438 }
439 }
440
441 if !improved {
442 eprintln!(
443 " hill-climb converged after {} iters ({} evals)",
444 iter + 1,
445 eval_count
446 );
447 break;
448 }
449 }
450
451 OptResult {
452 params: current,
453 score: current_score,
454 eval_count,
455 }
456}
457
458pub fn ai_guided_search(
467 space: &ParamSpace,
468 initial: &TuningParams,
469 eval: &dyn KernelEvaluator,
470 provider: &dyn tpt_gpu_shared::AiProvider,
471 kernel_name: &str,
472 iterations: usize,
473) -> OptResult {
474 let mut best = initial.clone();
475 let mut best_score = eval.evaluate(initial);
476 let mut history: Vec<(TuningParams, f64)> = vec![(best.clone(), best_score)];
477 let mut eval_count = 1;
478
479 for iter in 0..iterations {
480 let space_desc = space
481 .dims
482 .iter()
483 .map(|d| format!(" {}: {:?}", d.name, d.values))
484 .collect::<Vec<_>>()
485 .join("\n");
486
487 let mut sorted_history = history.clone();
489 sorted_history.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal));
490 let history_str = sorted_history
491 .iter()
492 .take(5)
493 .map(|(p, s)| format!(" {} → {:.2} GFLOPS", p.display(), s))
494 .collect::<Vec<_>>()
495 .join("\n");
496
497 let prompt = format!(
498 "You are optimizing GPU kernel tuning parameters. \
499 Suggest the single best next parameter set to try.\n\n\
500 Kernel: {}\n\
501 Parameter space (name: allowed values):\n{}\n\n\
502 Current best: {} → {:.2} GFLOPS\n\
503 Top trials so far:\n{}\n\n\
504 Rules:\n\
505 - Each value MUST be from the allowed list above.\n\
506 - Respond with ONLY a JSON object, nothing else.\n\
507 - Example: {{\"tile_m\": 64, \"tile_n\": 64, \"tile_k\": 32, \"vec_width\": 4, \"unroll\": 4}}",
508 kernel_name, space_desc, best.display(), best_score, history_str
509 );
510
511 eprintln!(
512 " ai-search [iter {}] querying {}...",
513 iter + 1,
514 provider.name()
515 );
516
517 match provider.generate(&prompt) {
518 Ok(response) => {
519 if let Some(candidate) = parse_json_params(&response, space) {
520 let score = eval.evaluate(&candidate);
521 eval_count += 1;
522 eprintln!(
523 " ai-search [iter {}] {} → {:.2}",
524 iter + 1,
525 candidate.display(),
526 score
527 );
528 if score > best_score {
529 best = candidate.clone();
530 best_score = score;
531 }
532 history.push((candidate, score));
533 } else {
534 eprintln!(
535 " ai-search [iter {}] could not parse response, skipping",
536 iter + 1
537 );
538 }
539 }
540 Err(e) => {
541 eprintln!(" ai-search [iter {}] provider error: {}", iter + 1, e);
542 }
543 }
544 }
545
546 OptResult {
547 params: best,
548 score: best_score,
549 eval_count,
550 }
551}
552
553fn parse_json_params(response: &str, space: &ParamSpace) -> Option<TuningParams> {
554 let start = response.find('{')?;
555 let end = response.rfind('}')?;
556 let json_str = &response[start..=end];
557 let val: serde_json::Value = serde_json::from_str(json_str).ok()?;
558 let obj = val.as_object()?;
559
560 let mut raw: HashMap<String, u64> = HashMap::new();
561 for (k, v) in obj {
562 raw.insert(k.clone(), v.as_u64()?);
563 }
564
565 if space.dims.iter().all(|d| raw.contains_key(&d.name)) {
567 Some(space.clamp(&raw))
568 } else {
569 None
570 }
571}
572
573#[cfg(test)]
578mod tests {
579 use super::*;
580
581 fn make_params(tm: u32, tn: u32, tk: u32, vw: u32, ur: u32) -> TuningParams {
582 TuningParams(HashMap::from([
583 ("tile_m".to_string(), tm),
584 ("tile_n".to_string(), tn),
585 ("tile_k".to_string(), tk),
586 ("vec_width".to_string(), vw),
587 ("unroll".to_string(), ur),
588 ]))
589 }
590
591 #[test]
592 fn test_param_space_total() {
593 let space = ParamSpace::gemm_default();
594 assert_eq!(space.total_configs(), 4 * 4 * 4 * 4 * 4);
597 }
598
599 #[test]
600 fn test_neighbors() {
601 let space = ParamSpace::gemm_default();
602 let p = make_params(64, 64, 32, 4, 4);
603 let neighbors = space.neighbors(&p);
604 assert_eq!(neighbors.len(), 10);
607 }
608
609 #[test]
610 fn test_simulated_evaluator_peak() {
611 let eval = SimulatedEvaluator::new("matmul");
612 let best = make_params(64, 64, 32, 4, 4);
613 let score = eval.evaluate(&best);
614 assert!(score > 100.0, "Peak should score above 100 GFLOPS");
615 }
616
617 #[test]
618 fn test_grid_search_returns_sorted() {
619 let space = ParamSpace::gemm_default();
620 let eval = SimulatedEvaluator::new("matmul");
621 let results = grid_search(&space, &eval);
622 assert_eq!(results.len(), space.total_configs());
623 for i in 1..results.len() {
625 assert!(results[i - 1].score >= results[i].score);
626 }
627 }
628
629 #[test]
630 fn test_hill_climb_improves() {
631 let space = ParamSpace::gemm_default();
632 let start = make_params(16, 16, 8, 1, 1); let eval = SimulatedEvaluator::new("matmul");
634 let initial_score = eval.evaluate(&start);
635 let result = hill_climb(&space, &start, &eval, 50);
636 assert!(result.score >= initial_score);
637 }
638
639 #[test]
644 fn test_attention_param_space_total() {
645 let space = ParamSpace::attention_default();
646 let expected: usize = space.dims.iter().map(|d| d.values.len()).product();
647 assert_eq!(space.total_configs(), expected);
648 assert!(expected > 0);
649 }
650
651 #[test]
652 fn test_attention_evaluator_reasonable_efficiency() {
653 let problem = AttentionProblemConfig::new(1024, 8, 64, 0.8);
654 let eval = RealAttentionEvaluator::new(problem);
655 let params = TuningParams(HashMap::from([
656 ("tile_seq".to_string(), 64),
657 ("tile_head".to_string(), 16),
658 ("tile_k".to_string(), 32),
659 ("vec_width".to_string(), 4),
660 ("unroll".to_string(), 4),
661 ]));
662 let eff = eval.evaluate(¶ms);
663 assert!(eff > 0.0, "Efficiency should be positive, got {}", eff);
664 }
665
666 #[test]
667 fn test_attention_problems_standard_count() {
668 let problems = standard_attention_problems();
669 assert_eq!(
670 problems.len(),
671 7,
672 "Should have 7 standard attention problems"
673 );
674 }
675
676 #[test]
677 fn test_attention_problem_gflops() {
678 let p = AttentionProblemConfig::new(1024, 8, 64, 0.8);
679 let expected = 4.0 * 1024.0 * 1024.0 * 64.0 * 8.0;
680 assert_eq!(p.flops(), expected);
681 }
682}