xlog-cuda 0.9.2

// Single source of truth for CUDA kernel modules.
//
// This file is consumed by both `build.rs` (via include!()) and `lib.rs`
// (via `pub mod kernel_manifest_data`) to avoid the kernel list being
// duplicated in two places.
// NOTE: Use regular comments (//), NOT inner doc comments (//!), because
// include!() in build.rs would interpret //! as documenting the wrong item.

/// Module names matching the .cu filenames (without extension).
/// Order matches provider/mod.rs load order. All 24 modules listed.
pub const KERNEL_CU_NAMES: &[&str] = &[
    "join",
    "dedup",
    "groupby",
    "scan",
    "sort",
    "filter",
    "set_ops",
    "pack",
    "pir",
    "cnf",
    "cache",
    "weights",
    "circuit",
    "mc_sample",
    "mc_eval",
    "arith",
    "sat",
    "d4",
    "neural",
    "ilp",
    "ilp_credit",
    "ilp_exact",
    "epistemic",
    "wcoj",
    "mc_resident",
];

/// Describes a single CUDA module: the .cu file name, the runtime module name
/// used by cudarc, and the list of kernel function entry points within.
pub struct KernelModuleSpec {
    pub cu_name: &'static str,
    pub module_name: &'static str,
    pub kernels: &'static [&'static str],
}

/// All kernel modules with their entry-point function names.
/// Order and cu_names match `KERNEL_CU_NAMES`.
pub const KERNEL_MODULES: &[KernelModuleSpec] = &[
    KernelModuleSpec {
        cu_name: "join",
        module_name: "xlog_join",
        kernels: &[
            "hash_join_build",
            "hash_join_probe",
            "compute_composite_hash",
            "hash_join_bucket_count_v2",
            "hash_join_scatter_v2",
            "hash_join_probe_v2",
            "hash_join_probe_v2_count_per_row",
            "hash_join_probe_v2_materialize",
            "hash_join_total_from_scan",
            "hash_join_csm_unmatched_mask",
            "hash_join_semi",
            "hash_join_anti",
            "init_hash_table",
            // W4.2 nested-loop inner join (emit-pairs design).
            "nested_loop_join_inner_u32_1key_pairs",
            // W4.3 sort-merge inner join (emit-pairs design,
            // caller-asserted pre-sorted inputs).
            "sort_merge_join_inner_u32_1key_pairs",
        ],
    },
    KernelModuleSpec {
        cu_name: "dedup",
        module_name: "xlog_dedup",
        kernels: &[
            "mark_duplicates",
            "mark_unique_columnar",
            "mark_unique_and_scan_columnar",
            "compact_rows",
            "mark_unique_full_row_bytewise",
            "mark_diff_full_row_typed_sorted",
            "small_sort_full_row_indices_typed",
        ],
    },
    KernelModuleSpec {
        cu_name: "groupby",
        module_name: "xlog_groupby",
        kernels: &[
            "detect_group_boundaries",
            "detect_boundaries",
            "extract_group_keys",
            "group_ids_from_boundaries",
            "group_start_indices",
            "capture_num_groups",
            "groupby_count",
            "groupby_sum",
            "groupby_min",
            "groupby_max",
            "groupby_logsumexp_max",
            "groupby_logsumexp_sumexp",
            "groupby_logsumexp_final",
        ],
    },
    KernelModuleSpec {
        cu_name: "scan",
        module_name: "xlog_scan",
        kernels: &[
            "block_inclusive_scan",
            "add_block_offsets",
            "exclusive_scan_mask",
            "count_mask",
            "multiblock_scan_phase1",
            "multiblock_scan_u32_phase1",
            "multiblock_scan_phase2",
            "multiblock_scan_phase3",
        ],
    },
    KernelModuleSpec {
        cu_name: "sort",
        module_name: "xlog_sort",
        kernels: &[
            "radix_histogram",
            "radix_scatter",
            "compute_ranks",
            "radix_scatter_stable",
            "compute_digit_prefix_sums",
            "init_indices",
            "apply_permutation_u32",
            "apply_permutation_bytes",
            "gather_keys_i32_ordered_u32",
            "gather_keys_f32_ordered_u32",
            "gather_keys_bool_ordered_u32",
            "gather_keys_u64_lo_u32",
            "gather_keys_u64_hi_u32",
            "gather_keys_i64_lo_u32",
            "gather_keys_i64_hi_u32",
            "gather_keys_f64_lo_u32",
            "gather_keys_f64_hi_u32",
            // W4.3 sortedness-detection kernel (used by the
            // dispatch-site eligibility check before invoking
            // the sort-merge join).
            "check_ascending_sorted_u32",
        ],
    },
    KernelModuleSpec {
        cu_name: "filter",
        module_name: "xlog_filter",
        kernels: &[
            "filter_compare_u32",
            "filter_compare_i64",
            "filter_compare_f64",
            "filter_compare_i32",
            "filter_compare_u64",
            "filter_compare_f32",
            "filter_compare_u8",
            "filter_compare_u32_scan_phase1",
            "filter_compare_f64_scan_phase1",
            "filter_compare_f32_scan_phase1",
            "filter_compare_u32_col",
            "filter_compare_i32_col",
            "filter_compare_i64_col",
            "filter_compare_u64_col",
            "filter_compare_f32_col",
            "filter_compare_f64_col",
            "filter_compare_u8_col",
            "fill_u32_iota",
            "fill_u32_const",
            "mark_random_vars",
            "random_var_to_bit_from_list",
            "check_random_var_count",
            "compact_u32_by_mask",
            "compact_i64_by_mask",
            "compact_f64_by_mask",
            "compact_bytes_by_mask",
            "capture_compact_count",
            "mask_clamp_rows",
            "mask_and",
            "mask_or",
            "mask_not",
        ],
    },
    KernelModuleSpec {
        cu_name: "set_ops",
        module_name: "xlog_set_ops",
        kernels: &["concat_u32", "concat_bytes", "sorted_diff_mark"],
    },
    KernelModuleSpec {
        cu_name: "pack",
        module_name: "xlog_pack",
        kernels: &[
            "pack_keys",
            "hash_packed_keys",
            "pack_and_hash_keys",
            "pack_and_hash_keys_generic",
            "pack_keys_aligned",
            "unpack_column",
            "unpack_column_counted",
            "gather_packed_rows",
            "gather_packed_rows_counted",
            "scatter_packed_rows",
            "compare_packed_keys",
            "pack_bools_to_bitmap",
        ],
    },
    KernelModuleSpec {
        cu_name: "pir",
        module_name: "xlog_pir",
        kernels: &[
            "pir_pack_keys",
            "pir_hash_keys",
            "pir_mark_unique",
            "pir_find_existing",
            "pir_mark_new_groups",
            "pir_build_group_ids",
            "pir_fill_child_parents",
            "pir_mark_unique_pairs",
            "pir_compact_pairs",
            "pir_count_children",
            "pir_write_child_offsets",
            "pir_gather_children",
            "pir_build_graph_child_counts",
            "pir_sum_counts",
            "pir_emit_nodes_and_ids",
            "pir_update_counts",
        ],
    },
    KernelModuleSpec {
        cu_name: "cnf",
        module_name: "xlog_cnf",
        kernels: &[
            "cnf_reachability_init",
            "cnf_reachability_bfs",
            "cnf_mark_leaf_choice",
            "cnf_assign_leaf_var",
            "cnf_assign_choice_var",
            "cnf_mark_node_vars",
            "cnf_count_clauses",
            "cnf_capture_last_counts",
            "cnf_compute_leaf_choice_totals",
            "cnf_compute_totals",
            "cnf_assign_node_var",
            "cnf_emit_clauses",
            "cnf_set_clause_end",
        ],
    },
    KernelModuleSpec {
        cu_name: "cache",
        module_name: "xlog_cache",
        kernels: &[
            "cache_cnf_hash",
            "cache_lookup_or_insert",
            "cache_evict_lru",
            "cache_store_u8",
            "cache_store_u32",
            "cache_store_i32",
            "cache_store_f64",
            "cache_store_meta",
        ],
    },
    KernelModuleSpec {
        cu_name: "weights",
        module_name: "xlog_weights",
        kernels: &[
            "weights_fill_leaf",
            "weights_fill_choice",
            "weights_count_lift_exact",
            "weights_set_evidence_from_nodes",
            "weights_apply_evidence",
            "weights_map_nodes_to_vars",
            "weights_force_var_false",
            "weights_restore_var_false",
            "weights_force_var_true",
            "weights_restore_var_true",
            "weights_copy_slot_to_batch",
            "weights_apply_query_vars",
            "weights_restore_query_vars",
            "weights_apply_query_vars_false_batched",
            "weights_restore_query_vars_false_batched",
            "weights_apply_query_vars_true_batched",
            "weights_restore_query_vars_true_batched",
        ],
    },
    KernelModuleSpec {
        cu_name: "circuit",
        module_name: "xlog_circuit",
        kernels: &[
            "xgcf_forward_level",
            "xgcf_backward_level_propagate",
            "xgcf_backward_level_decision_grad",
            "xgcf_backward_level_lit_grad",
            "xgcf_free_var_apply_grad",
            "xgcf_free_var_reduce_stage",
            "xgcf_add_scalar",
            "xgcf_forward_level_cached",
            "xgcf_eval_all_levels_cached",
            "xgcf_eval_all_levels_cached_batched",
            "xgcf_backward_level_propagate_cached",
            "xgcf_backward_level_decision_grad_cached",
            "xgcf_backward_level_lit_grad_cached",
            "xgcf_backward_all_levels_cached",
            "xgcf_backward_all_levels_cached_batched",
            "xgcf_free_var_apply_grad_cached",
            "xgcf_free_var_reduce_stage_cached",
            "xgcf_add_scalar_cached",
            "xgcf_set_root_adj_cached_batched",
            "xgcf_copy_root_cached",
            "xgcf_copy_root_cached_meta",
            "xgcf_copy_root_cached_meta_batched",
        ],
    },
    KernelModuleSpec {
        cu_name: "mc_sample",
        module_name: "xlog_mc_sample",
        kernels: &["mc_sample_bernoulli"],
    },
    KernelModuleSpec {
        cu_name: "mc_eval",
        module_name: "xlog_mc_eval",
        kernels: &[
            "mc_eval_mask_var",
            "mc_eval_mask_ad_choice",
            "mc_eval_query_evidence_truth",
            "mc_accumulate_counts",
        ],
    },
    KernelModuleSpec {
        cu_name: "arith",
        module_name: "xlog_arith",
        kernels: &[
            "arith_binary_i64",
            "arith_binary_i32",
            "arith_binary_u64",
            "arith_binary_u32",
            "arith_binary_f64",
            "arith_binary_f32",
            "arith_abs_i64",
            "arith_abs_i32",
            "arith_abs_f64",
            "arith_abs_f32",
            "arith_pow_f64",
            "arith_cast",
            "arith_fill_const_u32",
            "arith_fill_const_u64",
            "arith_fill_const_i64",
            "arith_fill_const_i32",
            "arith_fill_const_f64",
            "arith_fill_const_f32",
            "arith_fill_const_u8",
            "arith_select_i64",
            "arith_select_i32",
            "arith_select_u64",
            "arith_select_u32",
            "arith_select_f64",
            "arith_select_f32",
        ],
    },
    KernelModuleSpec {
        cu_name: "sat",
        module_name: "xlog_sat",
        kernels: &[
            "sat_cdcl_solve",
            "sat_check_model",
            "sat_proof_mark_needed",
            "sat_proof_check",
            "sat_assert_status",
            "sat_assert_ok",
            "sat_xgcf_cnf_counts",
            "sat_xgcf_cnf_emit",
            "sat_xgcf_cnf_capture_last_counts",
            "sat_xgcf_cnf_compute_totals",
            "sat_cnf_write_terminator",
            "sat_cnf_copy_into",
            "sat_shift_offsets",
            "sat_xgcf_write_root_unit_clause",
            "sat_not_phi_counts",
            "sat_emit_not_phi",
        ],
    },
    KernelModuleSpec {
        cu_name: "d4",
        module_name: "xlog_d4",
        kernels: &[
            "d4_validate_cnf",
            "d4_levelize_counts",
            "d4_levelize_emit",
            "d4_frontier_prepare",
            "d4_frontier_expand",
            "d4_frontier_prepare_dense",
            "d4_frontier_expand_dense",
            "d4_compile_count",
            "d4_compile_emit",
            "d4_capture_emit_meta",
            "d4_support_level",
            "d4_support_set_root_bits",
            "d4_smooth_count",
            "d4_smooth_wrapper_counts",
            "d4_smooth_wrapper_edge_counts_or",
            "d4_smooth_wrapper_edge_counts_dec",
            "d4_smooth_init_nodes",
            "d4_smooth_emit_level",
            "d4_smooth_check_edge_cap",
            "d4_mark_vars_in_clauses",
            "d4_mark_vars_in_circuit",
            "d4_build_free_var_mask",
            "d4_assert_u32_eq",
            "d4_assert_bitset_var",
            "d4_assert_dense_var",
            "d4_assert_leaf_root_and_degree",
        ],
    },
    KernelModuleSpec {
        cu_name: "neural",
        module_name: "xlog_neural",
        kernels: &[
            "neural_fill_ad_chain_f32",
            "neural_scatter_ad_chain_grads_f32",
        ],
    },
    KernelModuleSpec {
        cu_name: "ilp",
        module_name: "xlog_ilp",
        kernels: &[
            "extract_nonzero_indices",
            "ilp_mark_selected_ids_u32",
            "ilp_mark_selected_ids_i32",
            "ilp_mark_selected_ids_i64",
            "ilp_mark_selected_ids_u64",
            "ilp_validate_selected_ids_u32",
            "ilp_validate_selected_ids_i32",
            "ilp_validate_selected_ids_i64",
            "ilp_validate_selected_ids_u64",
            "ilp_broadcast_candidate_flag",
            "ilp_coo_fill_from_mask",
            "ilp_csr_histogram",
            "ilp_reduce_sum_f32",
            "ilp_reduce_sum_f64",
        ],
    },
    KernelModuleSpec {
        cu_name: "ilp_credit",
        module_name: "xlog_ilp_credit",
        kernels: &[
            "ilp_coo_fill",
            "ilp_credit_forward_f32",
            "ilp_credit_forward_f64",
            "ilp_credit_backward_f32",
            "ilp_credit_backward_f64",
        ],
    },
    KernelModuleSpec {
        cu_name: "ilp_exact",
        module_name: "xlog_ilp_exact",
        kernels: &[
            "ilp_exact_score",
            "ilp_exact_score_u32",
            "ilp_exact_score_chain_smem",
            "ilp_exact_score_chain_smem_u32",
            "ilp_exact_select_topk",
        ],
    },
    KernelModuleSpec {
        cu_name: "epistemic",
        module_name: "xlog_epistemic",
        kernels: &[
            "epistemic_generate_candidate_assumptions_u8",
            "epistemic_propagate_candidates_u8",
            "epistemic_validate_candidate_bits_u8",
            "epistemic_populate_model_membership_u8",
            "epistemic_populate_model_membership_from_tuple_source_u8",
            "epistemic_populate_model_membership_from_tuple_source_arity1_u8",
            "epistemic_populate_model_membership_from_tuple_source_arity2_u8",
            "epistemic_populate_model_membership_from_tuple_source_arity3_u8",
            "epistemic_populate_model_membership_from_tuple_source_arity_n_u8",
            "epistemic_validate_world_views_u8",
            "epistemic_validate_constraints_u8",
            "epistemic_materialize_accepted_candidates_u8",
            "epistemic_materialize_final_result_flags_u8",
            "epistemic_build_final_tuple_row_map_u8",
            "epistemic_close_final_tuple_rejections_u8",
            "epistemic_materialize_final_tuple_column_u8",
        ],
    },
    KernelModuleSpec {
        cu_name: "wcoj",
        module_name: "xlog_wcoj",
        kernels: &[
            "wcoj_build_metadata_mark_boundaries_u32",
            "wcoj_build_metadata_mark_boundaries_u64",
            "wcoj_build_metadata_scatter_u32",
            "wcoj_build_metadata_scatter_u64",
            "wcoj_triangle_build_hg_work_plan_u32",
            "wcoj_triangle_count_hg_u32",
            "wcoj_triangle_materialize_hg_u32",
            "wcoj_triangle_build_hg_work_plan_u64",
            "wcoj_triangle_count_hg_u64",
            "wcoj_triangle_materialize_hg_u64",
            "wcoj_triangle_count_hg_cached_u32",
            "wcoj_triangle_materialize_hg_cached_u32",
            "wcoj_scan_hg_block_counts_u32",
            "wcoj_compute_total",
            "wcoj_layout_check_sorted_unique_u32",
            "wcoj_layout_check_sorted_unique_u64",
            "wcoj_4cycle_build_e2_work_prefix_u32",
            "wcoj_4cycle_build_hg_work_plan_u32",
            "wcoj_4cycle_count_hg_u32",
            "wcoj_4cycle_materialize_hg_u32",
            "wcoj_4cycle_build_e2_work_prefix_u64",
            "wcoj_4cycle_build_hg_work_plan_u64",
            "wcoj_4cycle_count_hg_u64",
            "wcoj_4cycle_materialize_hg_u64",
            // W3.2/W6.4 — General-arity clique kernel (k=5..8 from
            // single C++ template; ABI wrappers below are
            // template-call-only per Tier-1 source-audit).
            "wcoj_clique5_count_hg_u32",
            "wcoj_clique5_materialize_hg_u32",
            "wcoj_clique5_count_hg_u64",
            "wcoj_clique5_materialize_hg_u64",
            "wcoj_clique6_count_hg_u32",
            "wcoj_clique6_materialize_hg_u32",
            "wcoj_clique6_count_hg_u64",
            "wcoj_clique6_materialize_hg_u64",
            "wcoj_clique7_count_hg_u32",
            "wcoj_clique7_materialize_hg_u32",
            "wcoj_clique7_count_hg_u64",
            "wcoj_clique7_materialize_hg_u64",
            "wcoj_clique8_count_hg_u32",
            "wcoj_clique8_materialize_hg_u32",
            "wcoj_clique8_count_hg_u64",
            "wcoj_clique8_materialize_hg_u64",
        ],
    },
    KernelModuleSpec {
        cu_name: "mc_resident",
        module_name: "xlog_mc_resident",
        kernels: &["mc_resident_engine"],
    },
];

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn kernel_modules_matches_cu_names() {
        assert_eq!(
            KERNEL_MODULES.len(),
            KERNEL_CU_NAMES.len(),
            "KERNEL_MODULES length ({}) != KERNEL_CU_NAMES length ({})",
            KERNEL_MODULES.len(),
            KERNEL_CU_NAMES.len(),
        );
        for (i, spec) in KERNEL_MODULES.iter().enumerate() {
            assert_eq!(
                spec.cu_name, KERNEL_CU_NAMES[i],
                "KERNEL_MODULES[{}].cu_name = {:?}, expected {:?}",
                i, spec.cu_name, KERNEL_CU_NAMES[i],
            );
        }
    }

    #[test]
    fn kernel_modules_count_is_25() {
        assert_eq!(KERNEL_MODULES.len(), 25);
    }

    #[test]
    fn all_kernel_entries_are_non_empty() {
        for spec in KERNEL_MODULES {
            assert!(
                !spec.kernels.is_empty(),
                "module {:?} has no kernel entries",
                spec.cu_name,
            );
            assert!(
                !spec.module_name.is_empty(),
                "module {:?} has empty module_name",
                spec.cu_name,
            );
        }
    }
}