Skip to main content

mittens_engine/engine/ecs/system/
openxr_system.rs

1use crate::engine::ecs::component::CameraXRComponent;
2use crate::engine::ecs::component::{
3    ControllerHand, ControllerPoseKind, InputXRComponent, XRHandComponent, XrComponent,
4};
5use crate::engine::ecs::system::System;
6use crate::engine::ecs::system::TransformSystem;
7use crate::engine::ecs::system::vr_types::{XrGamepadState, XrHandGamepadState, XrInputState};
8use crate::engine::ecs::{ComponentId, IntentValue, SignalEmitter, World};
9use crate::engine::graphics::CameraData;
10use crate::engine::graphics::VisualWorld;
11use crate::engine::graphics::VulkanoRenderer;
12use crate::engine::graphics::XRSwapchain;
13use crate::engine::graphics::XrVulkanGraphics;
14use crate::engine::graphics::xr_renderer;
15use crate::engine::user_input::InputState;
16use crate::utils::math;
17
18use ash::vk::Handle as _;
19
20use std::collections::{HashMap, HashSet, VecDeque};
21use std::sync::OnceLock;
22use std::time::Instant;
23
24fn openxr_debug_enabled() -> bool {
25    static ENABLED: OnceLock<bool> = OnceLock::new();
26    *ENABLED.get_or_init(|| {
27        std::env::var("CAT_OPENXR_DEBUG")
28            .ok()
29            .map(|s| {
30                let s = s.trim().to_ascii_lowercase();
31                s == "1" || s == "true" || s == "on" || s == "yes"
32            })
33            .unwrap_or(false)
34    })
35}
36
37fn log_xr_gamepad_changes(prev: XrGamepadState, next: XrGamepadState) {
38    if !openxr_debug_enabled() {
39        return;
40    }
41    if prev.active != next.active {
42        eprintln!(
43            "[OpenXR][gamepad] active -> {}",
44            if next.active { "true" } else { "false" }
45        );
46    }
47    for (index, hand_name) in ["left", "right"].into_iter().enumerate() {
48        let before = prev.hands[index];
49        let after = next.hands[index];
50        if before.thumbstick != after.thumbstick {
51            eprintln!(
52                "[OpenXR][gamepad] {hand_name} thumbstick -> {:?}",
53                after.thumbstick
54            );
55        }
56        if before.trigger_value != after.trigger_value {
57            eprintln!(
58                "[OpenXR][gamepad] {hand_name} trigger_value -> {:?}",
59                after.trigger_value
60            );
61        }
62        if before.trigger_pressed != after.trigger_pressed {
63            eprintln!(
64                "[OpenXR][gamepad] {hand_name} trigger_pressed -> {:?}",
65                after.trigger_pressed
66            );
67        }
68        if before.grip_value != after.grip_value {
69            eprintln!(
70                "[OpenXR][gamepad] {hand_name} grip_value -> {:?}",
71                after.grip_value
72            );
73        }
74        if before.grip_pressed != after.grip_pressed {
75            eprintln!(
76                "[OpenXR][gamepad] {hand_name} grip_pressed -> {:?}",
77                after.grip_pressed
78            );
79        }
80        if before.button_x != after.button_x {
81            eprintln!(
82                "[OpenXR][gamepad] {hand_name} button_x -> {:?}",
83                after.button_x
84            );
85        }
86        if before.button_y != after.button_y {
87            eprintln!(
88                "[OpenXR][gamepad] {hand_name} button_y -> {:?}",
89                after.button_y
90            );
91        }
92        if before.button_a != after.button_a {
93            eprintln!(
94                "[OpenXR][gamepad] {hand_name} button_a -> {:?}",
95                after.button_a
96            );
97        }
98        if before.button_b != after.button_b {
99            eprintln!(
100                "[OpenXR][gamepad] {hand_name} button_b -> {:?}",
101                after.button_b
102            );
103        }
104    }
105}
106
107fn log_active_interaction_profile(
108    instance: &openxr::Instance,
109    session: &openxr::Session<openxr::Vulkan>,
110    user_path: openxr::Path,
111    user_path_str: &str,
112    last_logged: &mut Option<openxr::Path>,
113) {
114    let Ok(profile) = session.current_interaction_profile(user_path) else {
115        return;
116    };
117    if *last_logged == Some(profile) {
118        return;
119    }
120    *last_logged = Some(profile);
121    if profile == openxr::Path::NULL {
122        eprintln!("[OpenXR][profile] {user_path_str}: no active interaction profile");
123        return;
124    }
125    match instance.path_to_string(profile) {
126        Ok(profile_str) => {
127            eprintln!("[OpenXR][profile] {user_path_str}: active profile = {profile_str}");
128        }
129        Err(err) => {
130            eprintln!(
131                "[OpenXR][profile] {user_path_str}: active profile path {:?} (path_to_string failed: {:?})",
132                profile, err
133            );
134        }
135    }
136}
137
138fn profile_string_or_none(instance: &openxr::Instance, profile: Option<openxr::Path>) -> String {
139    match profile {
140        Some(profile) if profile == openxr::Path::NULL => "none".to_string(),
141        Some(profile) => instance
142            .path_to_string(profile)
143            .unwrap_or_else(|_| format!("{profile:?}")),
144        None => "unknown".to_string(),
145    }
146}
147
148fn scalar_stick_value(
149    x: Option<openxr::ActionState<f32>>,
150    y: Option<openxr::ActionState<f32>>,
151) -> Option<[f32; 2]> {
152    match (x, y) {
153        (Some(x), Some(y)) if x.is_active || y.is_active => {
154            Some([x.current_state, y.current_state])
155        }
156        _ => None,
157    }
158}
159
160fn scalar_stick_state(
161    x: Option<openxr::ActionState<f32>>,
162    y: Option<openxr::ActionState<f32>>,
163) -> Option<(bool, [f32; 2])> {
164    match (x, y) {
165        (Some(x), Some(y)) => Some((
166            x.is_active || y.is_active,
167            [x.current_state, y.current_state],
168        )),
169        _ => None,
170    }
171}
172
173const XR_TRIGGER_PRESS_THRESHOLD: f32 = 0.7;
174const XR_GRIP_PRESS_THRESHOLD: f32 = 0.7;
175
176fn action_state_bool(
177    action: &openxr::Action<bool>,
178    session: &openxr::Session<openxr::Vulkan>,
179    hand: openxr::Path,
180) -> Option<openxr::ActionState<bool>> {
181    action.state(session, hand).ok().filter(|s| s.is_active)
182}
183
184fn action_state_f32(
185    action: &openxr::Action<f32>,
186    session: &openxr::Session<openxr::Vulkan>,
187    hand: openxr::Path,
188) -> Option<openxr::ActionState<f32>> {
189    action.state(session, hand).ok().filter(|s| s.is_active)
190}
191
192fn derive_trigger_like_pressed(
193    primary_click: Option<openxr::ActionState<bool>>,
194    analog_value: Option<openxr::ActionState<f32>>,
195    threshold: f32,
196) -> Option<(bool, f32)> {
197    primary_click
198        .map(|state| {
199            (
200                state.current_state,
201                if state.current_state { 1.0 } else { 0.0 },
202            )
203        })
204        .or_else(|| {
205            analog_value.map(|state| (state.current_state >= threshold, state.current_state))
206        })
207}
208
209fn derive_select_down(
210    select: Option<openxr::ActionState<bool>>,
211    trigger_click: Option<openxr::ActionState<bool>>,
212    trigger_value: Option<openxr::ActionState<f32>>,
213) -> bool {
214    select
215        .map(|state| state.current_state)
216        .or_else(|| trigger_click.map(|state| state.current_state))
217        .or_else(|| trigger_value.map(|state| state.current_state >= XR_TRIGGER_PRESS_THRESHOLD))
218        .unwrap_or(false)
219}
220
221fn log_action_bound_sources<T: openxr::ActionTy>(
222    session: &openxr::Session<openxr::Vulkan>,
223    label: &str,
224    action: &openxr::Action<T>,
225) {
226    let flags = openxr::InputSourceLocalizedNameFlags::USER_PATH
227        | openxr::InputSourceLocalizedNameFlags::INTERACTION_PROFILE
228        | openxr::InputSourceLocalizedNameFlags::COMPONENT;
229    match action.bound_sources(session) {
230        Ok(sources) if sources.is_empty() => {
231            eprintln!("[OpenXR][bound_sources] {label}: <unbound>");
232        }
233        Ok(sources) => {
234            for source in sources {
235                let path_str = action
236                    .instance()
237                    .path_to_string(source)
238                    .unwrap_or_else(|_| format!("{source:?}"));
239                let localized = session
240                    .input_source_localized_name(source, flags)
241                    .unwrap_or_else(|err| format!("<localized-name error: {err:?}>"));
242                eprintln!(
243                    "[OpenXR][bound_sources] {label}: path={} localized={}",
244                    path_str, localized
245                );
246            }
247        }
248        Err(err) => {
249            eprintln!("[OpenXR][bound_sources] {label}: error {err:?}");
250        }
251    }
252}
253
254fn log_controller_input_bound_sources(
255    session: &openxr::Session<openxr::Vulkan>,
256    ci: &ControllerInput,
257) {
258    log_action_bound_sources(session, "select", &ci.select);
259    log_action_bound_sources(session, "left_stick_x", &ci.left_stick_x);
260    log_action_bound_sources(session, "left_stick_y", &ci.left_stick_y);
261    log_action_bound_sources(session, "right_stick_x", &ci.right_stick_x);
262    log_action_bound_sources(session, "right_stick_y", &ci.right_stick_y);
263    log_action_bound_sources(session, "trigger_value", &ci.trigger_value);
264    log_action_bound_sources(session, "trigger_click", &ci.trigger_click);
265    log_action_bound_sources(session, "grip_value", &ci.grip_value);
266    log_action_bound_sources(session, "grip_click", &ci.grip_click);
267    log_action_bound_sources(session, "button_x", &ci.button_x);
268    log_action_bound_sources(session, "button_y", &ci.button_y);
269    log_action_bound_sources(session, "button_a", &ci.button_a);
270    log_action_bound_sources(session, "button_b", &ci.button_b);
271}
272
273fn append_action_bindings<'a>(
274    instance: &openxr::Instance,
275    suggested: &mut Vec<(&'static str, &'static str, openxr::Binding<'a>)>,
276    spec: BindingSpec,
277    left_stick_x: &'a openxr::Action<f32>,
278    left_stick_y: &'a openxr::Action<f32>,
279    right_stick_x: &'a openxr::Action<f32>,
280    right_stick_y: &'a openxr::Action<f32>,
281    trigger_value: &'a openxr::Action<f32>,
282    trigger_click: &'a openxr::Action<bool>,
283    grip_value: &'a openxr::Action<f32>,
284    grip_click: &'a openxr::Action<bool>,
285    button_a: &'a openxr::Action<bool>,
286    button_b: &'a openxr::Action<bool>,
287    button_x: &'a openxr::Action<bool>,
288    button_y: &'a openxr::Action<bool>,
289) {
290    for &path_str in spec.paths {
291        let Ok(path) = instance.string_to_path(path_str) else {
292            continue;
293        };
294        match spec.label {
295            "left_stick_x" => suggested.push((
296                spec.label,
297                path_str,
298                openxr::Binding::new(left_stick_x, path),
299            )),
300            "left_stick_y" => suggested.push((
301                spec.label,
302                path_str,
303                openxr::Binding::new(left_stick_y, path),
304            )),
305            "right_stick_x" => suggested.push((
306                spec.label,
307                path_str,
308                openxr::Binding::new(right_stick_x, path),
309            )),
310            "right_stick_y" => suggested.push((
311                spec.label,
312                path_str,
313                openxr::Binding::new(right_stick_y, path),
314            )),
315            "trigger_value" => suggested.push((
316                spec.label,
317                path_str,
318                openxr::Binding::new(trigger_value, path),
319            )),
320            "trigger_click" => suggested.push((
321                spec.label,
322                path_str,
323                openxr::Binding::new(trigger_click, path),
324            )),
325            "grip_value" => {
326                suggested.push((spec.label, path_str, openxr::Binding::new(grip_value, path)))
327            }
328            "grip_click" => {
329                suggested.push((spec.label, path_str, openxr::Binding::new(grip_click, path)))
330            }
331            "a" => suggested.push((spec.label, path_str, openxr::Binding::new(button_a, path))),
332            "b" => suggested.push((spec.label, path_str, openxr::Binding::new(button_b, path))),
333            "x" => suggested.push((spec.label, path_str, openxr::Binding::new(button_x, path))),
334            "y" => suggested.push((spec.label, path_str, openxr::Binding::new(button_y, path))),
335            _ => {}
336        }
337    }
338}
339
340fn suggest_binding_best_effort(
341    instance: &openxr::Instance,
342    profile: openxr::Path,
343    profile_name: &str,
344    binding_label: &str,
345    binding_path: &str,
346    binding: openxr::Binding<'_>,
347) -> Result<BindingSuggestionOutcome, String> {
348    match instance.suggest_interaction_profile_bindings(profile, &[binding]) {
349        Ok(()) => Ok(BindingSuggestionOutcome::Accepted),
350        Err(openxr::sys::Result::ERROR_PATH_UNSUPPORTED) => {
351            eprintln!(
352                "[OpenXR] ignoring unsupported binding profile={} label={} path={}",
353                profile_name, binding_label, binding_path
354            );
355            Ok(BindingSuggestionOutcome::Unsupported)
356        }
357        Err(err) => Err(format!(
358            "suggest_interaction_profile_bindings({}, {} -> {}): {err:?}",
359            profile_name, binding_label, binding_path
360        )),
361    }
362}
363
364fn log_profile_binding_dump(spec: &ProfileBindingSpec) {
365    if !openxr_debug_enabled() {
366        return;
367    }
368    eprintln!("[OpenXR][binding_dump] profile {}", spec.profile);
369    eprintln!(
370        "[OpenXR][binding_dump]   select left={:?} right={:?}",
371        spec.select_left, spec.select_right
372    );
373    for spec in spec.left_specs {
374        eprintln!(
375            "[OpenXR][binding_dump]   left {:<13} -> {}",
376            spec.label,
377            spec.paths.join(", ")
378        );
379    }
380    for spec in spec.right_specs {
381        eprintln!(
382            "[OpenXR][binding_dump]   right {:<12} -> {}",
383            spec.label,
384            spec.paths.join(", ")
385        );
386    }
387}
388
389#[derive(Debug, Clone, Copy, PartialEq, Eq)]
390enum BindingSuggestionOutcome {
391    Accepted,
392    Unsupported,
393}
394
395#[derive(Debug, Clone, PartialEq, Eq)]
396struct BindingSuggestionRecord {
397    label: &'static str,
398    path: String,
399    outcome: BindingSuggestionOutcome,
400}
401
402#[derive(Debug, Clone, PartialEq, Eq)]
403struct ProfileBindingReport {
404    profile_path: String,
405    suggestions: Vec<BindingSuggestionRecord>,
406}
407
408impl ProfileBindingReport {
409    fn accepted_count(&self) -> usize {
410        self.suggestions
411            .iter()
412            .filter(|record| record.outcome == BindingSuggestionOutcome::Accepted)
413            .count()
414    }
415
416    fn unsupported_count(&self) -> usize {
417        self.suggestions
418            .iter()
419            .filter(|record| record.outcome == BindingSuggestionOutcome::Unsupported)
420            .count()
421    }
422}
423
424fn log_runtime_profile_binding_report(
425    active_profile_label: &str,
426    report: &ProfileBindingReport,
427    snapshot: &ControllerDebugSnapshot,
428) {
429    eprintln!(
430        "[OpenXR][binding_report] hand={} profile={} accepted={} unsupported={}",
431        active_profile_label,
432        report.profile_path,
433        report.accepted_count(),
434        report.unsupported_count(),
435    );
436    for record in &report.suggestions {
437        let outcome = match record.outcome {
438            BindingSuggestionOutcome::Accepted => "accepted",
439            BindingSuggestionOutcome::Unsupported => "unsupported",
440        };
441        eprintln!(
442            "[OpenXR][binding_report]   {:<12} {:<11} {}",
443            record.label, outcome, record.path
444        );
445    }
446    eprintln!(
447        "[OpenXR][binding_report]   runtime_state profileL={} profileR={} poseL(aim={},grip={}) poseR(aim={},grip={})",
448        snapshot.left_profile,
449        snapshot.right_profile,
450        snapshot.left_aim_valid,
451        snapshot.left_grip_valid,
452        snapshot.right_aim_valid,
453        snapshot.right_grip_valid,
454    );
455    eprintln!(
456        "[OpenXR][binding_report]   buttons/select left(select={:?},x={:?},y={:?}) right(select={:?},a={:?},b={:?})",
457        snapshot.left_select,
458        snapshot.left_x,
459        snapshot.left_y,
460        snapshot.right_select,
461        snapshot.right_a,
462        snapshot.right_b,
463    );
464    eprintln!(
465        "[OpenXR][binding_report]   analog left(stick={:?},trigger={:?}/{:?},grip={:?}/{:?}) right(stick={:?},trigger={:?}/{:?},grip={:?}/{:?})",
466        snapshot.left_thumbstick,
467        snapshot.left_trigger_value,
468        snapshot.left_trigger_click,
469        snapshot.left_grip_value,
470        snapshot.left_grip_click,
471        snapshot.right_thumbstick,
472        snapshot.right_trigger_value,
473        snapshot.right_trigger_click,
474        snapshot.right_grip_value,
475        snapshot.right_grip_click,
476    );
477}
478
479pub struct OpenXRSystem {
480    state: Option<OpenXRState>,
481    last_init_error: Option<String>,
482    vulkan_graphics: Option<XrVulkanGraphics>,
483    preferred_swapchain_format: Option<u32>,
484
485    // Best-effort XR frame timing diagnostics.
486    last_render_instant: Option<Instant>,
487    last_render_dt_sec: Option<f32>,
488    render_profile: XrRenderProfile,
489
490    input_xr_components: HashSet<ComponentId>,
491    controller_components: HashSet<ComponentId>,
492    controller_pose_source_last_logged: HashMap<ComponentId, &'static str>,
493
494    xr_input_state: XrInputState,
495    xr_gamepad_state: XrGamepadState,
496    xr_gamepad_state_last_logged: XrGamepadState,
497}
498
499#[derive(Debug, Default)]
500struct XrRenderProfile {
501    frames: u64,
502    total: std::time::Duration,
503    wait_frame: std::time::Duration,
504    tracking_input: std::time::Duration,
505    acquire_wait: std::time::Duration,
506    eye_render: std::time::Duration,
507    copy: std::time::Duration,
508    submit: std::time::Duration,
509}
510
511struct OpenXRState {
512    #[allow(dead_code)]
513    entry: openxr::Entry,
514    #[allow(dead_code)]
515    instance: openxr::Instance,
516    #[allow(dead_code)]
517    system: openxr::SystemId,
518    events: openxr::EventDataBuffer,
519
520    session: Option<OpenXRSessionState>,
521    view_type: openxr::ViewConfigurationType,
522    blend_mode: openxr::EnvironmentBlendMode,
523}
524
525struct OpenXRSessionState {
526    session: openxr::Session<openxr::Vulkan>,
527    frame_waiter: openxr::FrameWaiter,
528    frame_stream: openxr::FrameStream<openxr::Vulkan>,
529    reference_space: openxr::Space,
530    running: bool,
531    current_state: openxr::SessionState,
532
533    xr_swapchain: XRSwapchain,
534
535    swapchain_image_initialized: Vec<bool>,
536
537    did_log_format_mismatch: bool,
538
539    vk_device: ash::Device,
540    vk_queue: ash::vk::Queue,
541
542    #[allow(dead_code)]
543    vk_command_pool: ash::vk::CommandPool,
544    vk_command_buffer: ash::vk::CommandBuffer,
545
546    hand_tracking: Option<HandTrackingState>,
547    hand_root_pose_cache: HandRootPoseCache,
548    hand_rotation_debug: HandRotationDebugState,
549    last_hand_debug_snapshot: Option<HandDebugSnapshot>,
550    head_pose_cache: Option<openxr::Posef>,
551    controller_input: Option<ControllerInput>,
552    controller_pose_cache: ControllerPoseCache,
553}
554
555#[derive(Debug, Default, Clone, Copy)]
556struct ControllerPoseCache {
557    left_aim: Option<openxr::Posef>,
558    right_aim: Option<openxr::Posef>,
559    left_grip: Option<openxr::Posef>,
560    right_grip: Option<openxr::Posef>,
561}
562
563struct HandTrackingState {
564    left: openxr::HandTracker,
565    right: openxr::HandTracker,
566}
567
568#[derive(Debug, Default, Clone, Copy)]
569struct HandRootPoseCache {
570    left_root: Option<openxr::Posef>,
571    right_root: Option<openxr::Posef>,
572    left_root_joint: Option<openxr::HandJointEXT>,
573    right_root_joint: Option<openxr::HandJointEXT>,
574}
575
576#[derive(Debug, Default)]
577struct HandRotationDebugState {
578    left: RollingAngleWindow,
579    right: RollingAngleWindow,
580}
581
582#[derive(Debug, Default)]
583struct RollingAngleWindow {
584    previous_quat_xyzw: Option<[f32; 4]>,
585    step_deg: VecDeque<f32>,
586    sample_count: u64,
587}
588
589#[derive(Debug, Clone, Copy, PartialEq, Eq)]
590struct HandDebugState {
591    root_joint: Option<openxr::HandJointEXT>,
592    wrist_palm_delta_deg: Option<u16>,
593    wrist_step_deg: Option<u16>,
594    wrist_step_spike: bool,
595}
596
597#[derive(Debug, Clone, Copy, PartialEq, Eq)]
598struct HandDebugSnapshot {
599    left: HandDebugState,
600    right: HandDebugState,
601}
602
603#[derive(Debug, Clone, PartialEq)]
604struct ControllerDebugSnapshot {
605    left_profile: String,
606    right_profile: String,
607    left_aim_valid: bool,
608    right_aim_valid: bool,
609    left_grip_valid: bool,
610    right_grip_valid: bool,
611    left_select: Option<(bool, bool)>,
612    right_select: Option<(bool, bool)>,
613    left_thumbstick: Option<(bool, [f32; 2])>,
614    right_thumbstick: Option<(bool, [f32; 2])>,
615    left_trigger_value: Option<(bool, f32)>,
616    right_trigger_value: Option<(bool, f32)>,
617    left_trigger_click: Option<(bool, bool)>,
618    right_trigger_click: Option<(bool, bool)>,
619    left_grip_value: Option<(bool, f32)>,
620    right_grip_value: Option<(bool, f32)>,
621    left_grip_click: Option<(bool, bool)>,
622    right_grip_click: Option<(bool, bool)>,
623    left_x: Option<(bool, bool)>,
624    left_y: Option<(bool, bool)>,
625    right_a: Option<(bool, bool)>,
626    right_b: Option<(bool, bool)>,
627}
628
629#[derive(Debug, Clone, Copy)]
630struct BindingSpec {
631    label: &'static str,
632    paths: &'static [&'static str],
633}
634
635#[derive(Debug, Clone, Copy)]
636struct ProfileBindingSpec {
637    profile: &'static str,
638    select_left: Option<&'static str>,
639    select_right: Option<&'static str>,
640    left_specs: &'static [BindingSpec],
641    right_specs: &'static [BindingSpec],
642}
643
644#[allow(dead_code)]
645struct ControllerInput {
646    action_set: openxr::ActionSet,
647    aim_pose: openxr::Action<openxr::Posef>,
648    grip_pose: openxr::Action<openxr::Posef>,
649    select: openxr::Action<bool>,
650    left_stick_x: openxr::Action<f32>,
651    left_stick_y: openxr::Action<f32>,
652    right_stick_x: openxr::Action<f32>,
653    right_stick_y: openxr::Action<f32>,
654    trigger_value: openxr::Action<f32>,
655    trigger_click: openxr::Action<bool>,
656    grip_value: openxr::Action<f32>,
657    grip_click: openxr::Action<bool>,
658    button_a: openxr::Action<bool>,
659    button_b: openxr::Action<bool>,
660    button_x: openxr::Action<bool>,
661    button_y: openxr::Action<bool>,
662
663    left: openxr::Path,
664    right: openxr::Path,
665
666    left_aim_space: openxr::Space,
667    right_aim_space: openxr::Space,
668    left_grip_space: openxr::Space,
669    right_grip_space: openxr::Space,
670
671    profile_poll_counter: u32,
672    last_logged_left_profile: Option<openxr::Path>,
673    last_logged_right_profile: Option<openxr::Path>,
674    last_reported_left_profile: Option<openxr::Path>,
675    last_reported_right_profile: Option<openxr::Path>,
676    last_debug_snapshot: Option<ControllerDebugSnapshot>,
677    did_log_bound_sources: bool,
678    binding_reports: Vec<ProfileBindingReport>,
679}
680
681const PROFILE_SIMPLE_CONTROLLER: &str = "/interaction_profiles/khr/simple_controller";
682const PROFILE_OCULUS_TOUCH: &str = "/interaction_profiles/oculus/touch_controller";
683const PROFILE_HTC_VIVE: &str = "/interaction_profiles/htc/vive_controller";
684const PROFILE_HTC_VIVE_FOCUS3: &str = "/interaction_profiles/htc/vive_focus3_controller";
685const PROFILE_VALVE_INDEX: &str = "/interaction_profiles/valve/index_controller";
686const PROFILE_MICROSOFT_MOTION: &str = "/interaction_profiles/microsoft/motion_controller";
687const PROFILE_EXT_HAND_INTERACTION: &str = "/interaction_profiles/ext/hand_interaction_ext";
688
689const SIMPLE_CONTROLLER_LEFT_SPECS: &[BindingSpec] = &[];
690const SIMPLE_CONTROLLER_RIGHT_SPECS: &[BindingSpec] = &[];
691
692const OCULUS_TOUCH_LEFT_SPECS: &[BindingSpec] = &[
693    BindingSpec {
694        label: "left_stick_x",
695        paths: &["/user/hand/left/input/thumbstick/x"],
696    },
697    BindingSpec {
698        label: "left_stick_y",
699        paths: &["/user/hand/left/input/thumbstick/y"],
700    },
701    BindingSpec {
702        label: "trigger_value",
703        paths: &["/user/hand/left/input/trigger/value"],
704    },
705    BindingSpec {
706        label: "grip_value",
707        paths: &["/user/hand/left/input/squeeze/value"],
708    },
709    BindingSpec {
710        label: "x",
711        paths: &["/user/hand/left/input/x/click"],
712    },
713    BindingSpec {
714        label: "y",
715        paths: &["/user/hand/left/input/y/click"],
716    },
717];
718const OCULUS_TOUCH_RIGHT_SPECS: &[BindingSpec] = &[
719    BindingSpec {
720        label: "right_stick_x",
721        paths: &["/user/hand/right/input/thumbstick/x"],
722    },
723    BindingSpec {
724        label: "right_stick_y",
725        paths: &["/user/hand/right/input/thumbstick/y"],
726    },
727    BindingSpec {
728        label: "trigger_value",
729        paths: &["/user/hand/right/input/trigger/value"],
730    },
731    BindingSpec {
732        label: "grip_value",
733        paths: &["/user/hand/right/input/squeeze/value"],
734    },
735    BindingSpec {
736        label: "a",
737        paths: &["/user/hand/right/input/a/click"],
738    },
739    BindingSpec {
740        label: "b",
741        paths: &["/user/hand/right/input/b/click"],
742    },
743];
744
745const VALVE_INDEX_LEFT_SPECS: &[BindingSpec] = &[
746    BindingSpec {
747        label: "left_stick_x",
748        paths: &[
749            "/user/hand/left/input/thumbstick/x",
750            "/user/hand/left/input/joystick/x",
751        ],
752    },
753    BindingSpec {
754        label: "left_stick_y",
755        paths: &[
756            "/user/hand/left/input/thumbstick/y",
757            "/user/hand/left/input/joystick/y",
758        ],
759    },
760    BindingSpec {
761        label: "trigger_value",
762        paths: &["/user/hand/left/input/trigger/value"],
763    },
764    BindingSpec {
765        label: "grip_value",
766        paths: &[
767            "/user/hand/left/input/squeeze/value",
768            "/user/hand/left/input/grip/value",
769        ],
770    },
771];
772const VALVE_INDEX_RIGHT_SPECS: &[BindingSpec] = &[
773    BindingSpec {
774        label: "right_stick_x",
775        paths: &[
776            "/user/hand/right/input/thumbstick/x",
777            "/user/hand/right/input/joystick/x",
778        ],
779    },
780    BindingSpec {
781        label: "right_stick_y",
782        paths: &[
783            "/user/hand/right/input/thumbstick/y",
784            "/user/hand/right/input/joystick/y",
785        ],
786    },
787    BindingSpec {
788        label: "trigger_value",
789        paths: &["/user/hand/right/input/trigger/value"],
790    },
791    BindingSpec {
792        label: "grip_value",
793        paths: &[
794            "/user/hand/right/input/squeeze/value",
795            "/user/hand/right/input/grip/value",
796        ],
797    },
798    BindingSpec {
799        label: "a",
800        paths: &["/user/hand/right/input/a/click"],
801    },
802    BindingSpec {
803        label: "b",
804        paths: &["/user/hand/right/input/b/click"],
805    },
806];
807
808const MICROSOFT_MOTION_LEFT_SPECS: &[BindingSpec] = &[
809    BindingSpec {
810        label: "left_stick_x",
811        paths: &[
812            "/user/hand/left/input/thumbstick/x",
813            "/user/hand/left/input/joystick/x",
814        ],
815    },
816    BindingSpec {
817        label: "left_stick_y",
818        paths: &[
819            "/user/hand/left/input/thumbstick/y",
820            "/user/hand/left/input/joystick/y",
821        ],
822    },
823    BindingSpec {
824        label: "trigger_value",
825        paths: &["/user/hand/left/input/trigger/value"],
826    },
827    BindingSpec {
828        label: "grip_click",
829        paths: &[
830            "/user/hand/left/input/squeeze/click",
831            "/user/hand/left/input/grip/click",
832        ],
833    },
834];
835const MICROSOFT_MOTION_RIGHT_SPECS: &[BindingSpec] = &[
836    BindingSpec {
837        label: "right_stick_x",
838        paths: &[
839            "/user/hand/right/input/thumbstick/x",
840            "/user/hand/right/input/joystick/x",
841        ],
842    },
843    BindingSpec {
844        label: "right_stick_y",
845        paths: &[
846            "/user/hand/right/input/thumbstick/y",
847            "/user/hand/right/input/joystick/y",
848        ],
849    },
850    BindingSpec {
851        label: "trigger_value",
852        paths: &["/user/hand/right/input/trigger/value"],
853    },
854    BindingSpec {
855        label: "grip_click",
856        paths: &[
857            "/user/hand/right/input/squeeze/click",
858            "/user/hand/right/input/grip/click",
859        ],
860    },
861];
862
863const HTC_VIVE_LEFT_SPECS: &[BindingSpec] = &[
864    BindingSpec {
865        label: "trigger_click",
866        paths: &["/user/hand/left/input/trigger/click"],
867    },
868    BindingSpec {
869        label: "grip_click",
870        paths: &[
871            "/user/hand/left/input/squeeze/click",
872            "/user/hand/left/input/grip/click",
873        ],
874    },
875];
876const HTC_VIVE_RIGHT_SPECS: &[BindingSpec] = &[
877    BindingSpec {
878        label: "trigger_click",
879        paths: &["/user/hand/right/input/trigger/click"],
880    },
881    BindingSpec {
882        label: "grip_click",
883        paths: &[
884            "/user/hand/right/input/squeeze/click",
885            "/user/hand/right/input/grip/click",
886        ],
887    },
888];
889
890const HTC_VIVE_FOCUS3_LEFT_SPECS: &[BindingSpec] = &[
891    BindingSpec {
892        label: "left_stick_x",
893        paths: &["/user/hand/left/input/thumbstick/x"],
894    },
895    BindingSpec {
896        label: "left_stick_y",
897        paths: &["/user/hand/left/input/thumbstick/y"],
898    },
899    BindingSpec {
900        label: "trigger_value",
901        paths: &["/user/hand/left/input/trigger/value"],
902    },
903    BindingSpec {
904        label: "grip_value",
905        paths: &["/user/hand/left/input/squeeze/value"],
906    },
907    BindingSpec {
908        label: "x",
909        paths: &["/user/hand/left/input/x/click"],
910    },
911    BindingSpec {
912        label: "y",
913        paths: &["/user/hand/left/input/y/click"],
914    },
915];
916const HTC_VIVE_FOCUS3_RIGHT_SPECS: &[BindingSpec] = &[
917    BindingSpec {
918        label: "right_stick_x",
919        paths: &["/user/hand/right/input/thumbstick/x"],
920    },
921    BindingSpec {
922        label: "right_stick_y",
923        paths: &["/user/hand/right/input/thumbstick/y"],
924    },
925    BindingSpec {
926        label: "trigger_value",
927        paths: &["/user/hand/right/input/trigger/value"],
928    },
929    BindingSpec {
930        label: "grip_value",
931        paths: &["/user/hand/right/input/squeeze/value"],
932    },
933    BindingSpec {
934        label: "a",
935        paths: &["/user/hand/right/input/a/click"],
936    },
937    BindingSpec {
938        label: "b",
939        paths: &["/user/hand/right/input/b/click"],
940    },
941];
942
943const EXT_HAND_INTERACTION_LEFT_SPECS: &[BindingSpec] = &[];
944const EXT_HAND_INTERACTION_RIGHT_SPECS: &[BindingSpec] = &[];
945
946const PROFILE_BINDING_SPECS: &[ProfileBindingSpec] = &[
947    ProfileBindingSpec {
948        profile: PROFILE_SIMPLE_CONTROLLER,
949        select_left: Some("/user/hand/left/input/select/click"),
950        select_right: Some("/user/hand/right/input/select/click"),
951        left_specs: SIMPLE_CONTROLLER_LEFT_SPECS,
952        right_specs: SIMPLE_CONTROLLER_RIGHT_SPECS,
953    },
954    ProfileBindingSpec {
955        profile: PROFILE_OCULUS_TOUCH,
956        select_left: None,
957        select_right: None,
958        left_specs: OCULUS_TOUCH_LEFT_SPECS,
959        right_specs: OCULUS_TOUCH_RIGHT_SPECS,
960    },
961    ProfileBindingSpec {
962        profile: PROFILE_HTC_VIVE,
963        select_left: Some("/user/hand/left/input/trigger/click"),
964        select_right: Some("/user/hand/right/input/trigger/click"),
965        left_specs: HTC_VIVE_LEFT_SPECS,
966        right_specs: HTC_VIVE_RIGHT_SPECS,
967    },
968    ProfileBindingSpec {
969        profile: PROFILE_HTC_VIVE_FOCUS3,
970        select_left: None,
971        select_right: None,
972        left_specs: HTC_VIVE_FOCUS3_LEFT_SPECS,
973        right_specs: HTC_VIVE_FOCUS3_RIGHT_SPECS,
974    },
975    ProfileBindingSpec {
976        profile: PROFILE_VALVE_INDEX,
977        select_left: Some("/user/hand/left/input/trigger/value"),
978        select_right: Some("/user/hand/right/input/trigger/value"),
979        left_specs: VALVE_INDEX_LEFT_SPECS,
980        right_specs: VALVE_INDEX_RIGHT_SPECS,
981    },
982    ProfileBindingSpec {
983        profile: PROFILE_MICROSOFT_MOTION,
984        select_left: Some("/user/hand/left/input/trigger/value"),
985        select_right: Some("/user/hand/right/input/trigger/value"),
986        left_specs: MICROSOFT_MOTION_LEFT_SPECS,
987        right_specs: MICROSOFT_MOTION_RIGHT_SPECS,
988    },
989    ProfileBindingSpec {
990        profile: PROFILE_EXT_HAND_INTERACTION,
991        select_left: Some("/user/hand/left/input/pinch_ext/value"),
992        select_right: Some("/user/hand/right/input/pinch_ext/value"),
993        left_specs: EXT_HAND_INTERACTION_LEFT_SPECS,
994        right_specs: EXT_HAND_INTERACTION_RIGHT_SPECS,
995    },
996];
997
998impl Default for OpenXRSystem {
999    fn default() -> Self {
1000        Self {
1001            state: None,
1002            last_init_error: None,
1003            vulkan_graphics: None,
1004            preferred_swapchain_format: None,
1005
1006            last_render_instant: None,
1007            last_render_dt_sec: None,
1008            render_profile: XrRenderProfile::default(),
1009
1010            input_xr_components: HashSet::new(),
1011            controller_components: HashSet::new(),
1012            controller_pose_source_last_logged: HashMap::new(),
1013
1014            xr_input_state: XrInputState::default(),
1015            xr_gamepad_state: XrGamepadState::default(),
1016            xr_gamepad_state_last_logged: XrGamepadState::default(),
1017        }
1018    }
1019}
1020
1021impl std::fmt::Debug for OpenXRSystem {
1022    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1023        f.debug_struct("OpenXRSystem")
1024            .field("initialized", &self.state.is_some())
1025            .field("last_init_error", &self.last_init_error)
1026            .field("last_render_dt_sec", &self.last_render_dt_sec)
1027            .finish()
1028    }
1029}
1030
1031impl OpenXRSystem {
1032    fn profile_systems_enabled() -> bool {
1033        static ENABLED: OnceLock<bool> = OnceLock::new();
1034        *ENABLED.get_or_init(|| {
1035            matches!(
1036                std::env::var("CAT_PROFILE_SYSTEMS")
1037                    .unwrap_or_default()
1038                    .trim()
1039                    .to_ascii_lowercase()
1040                    .as_str(),
1041                "1" | "true" | "yes" | "on"
1042            )
1043        })
1044    }
1045
1046    fn finish_xr_render_profile_frame(&mut self, frame_started: Instant) {
1047        self.render_profile.frames += 1;
1048        self.render_profile.total += frame_started.elapsed();
1049        if self.render_profile.frames < 120 {
1050            return;
1051        }
1052        let frames = self.render_profile.frames as f64;
1053        let avg_ms = |duration: std::time::Duration| duration.as_secs_f64() * 1000.0 / frames;
1054        let line = format!(
1055            "[xr-render-profile] frames={} avg_ms total={:.3} wait_frame={:.3} tracking_input={:.3} acquire_wait={:.3} eye_render={:.3} copy={:.3} submit={:.3}",
1056            self.render_profile.frames,
1057            avg_ms(self.render_profile.total),
1058            avg_ms(self.render_profile.wait_frame),
1059            avg_ms(self.render_profile.tracking_input),
1060            avg_ms(self.render_profile.acquire_wait),
1061            avg_ms(self.render_profile.eye_render),
1062            avg_ms(self.render_profile.copy),
1063            avg_ms(self.render_profile.submit),
1064        );
1065        eprintln!("{line}");
1066        crate::utils::profile_log::append("system-profile", &line);
1067        self.render_profile = XrRenderProfile::default();
1068    }
1069
1070    pub fn xr_input_state(&self) -> &XrInputState {
1071        &self.xr_input_state
1072    }
1073
1074    pub fn xr_gamepad_state(&self) -> &XrGamepadState {
1075        &self.xr_gamepad_state
1076    }
1077
1078    pub fn last_init_error(&self) -> Option<&str> {
1079        self.last_init_error.as_deref()
1080    }
1081
1082    pub fn prepare_for_renderer_init(&mut self, world: &World) {
1083        let should_enable = world.all_components().any(|cid| {
1084            world
1085                .get_component_by_id_as::<XrComponent>(cid)
1086                .map(|component| component.enabled)
1087                .unwrap_or(false)
1088        });
1089
1090        if should_enable {
1091            let _ = self.initialize_runtime();
1092        }
1093    }
1094
1095    fn debug_hand_rotation_enabled() -> bool {
1096        static ENABLED: OnceLock<bool> = OnceLock::new();
1097        *ENABLED.get_or_init(|| {
1098            std::env::var("CAT_DEBUG_XR_HAND_ROTATION")
1099                .ok()
1100                .map(|value| {
1101                    let value = value.trim().to_ascii_lowercase();
1102                    matches!(value.as_str(), "1" | "true" | "yes" | "on")
1103                })
1104                .unwrap_or(false)
1105        })
1106    }
1107
1108    fn debug_hand_rotation_window_len() -> usize {
1109        static WINDOW_LEN: OnceLock<usize> = OnceLock::new();
1110        *WINDOW_LEN.get_or_init(|| {
1111            std::env::var("CAT_DEBUG_XR_HAND_ROTATION_WINDOW")
1112                .ok()
1113                .and_then(|value| value.parse::<usize>().ok())
1114                .map(|value| value.clamp(1, 600))
1115                .unwrap_or(60)
1116        })
1117    }
1118
1119    fn quat_from_posef(pose: openxr::Posef) -> [f32; 4] {
1120        [
1121            pose.orientation.x,
1122            pose.orientation.y,
1123            pose.orientation.z,
1124            pose.orientation.w,
1125        ]
1126    }
1127
1128    fn quat_normalize(q: [f32; 4]) -> [f32; 4] {
1129        let len = (q[0] * q[0] + q[1] * q[1] + q[2] * q[2] + q[3] * q[3]).sqrt();
1130        if len > 0.0 {
1131            [q[0] / len, q[1] / len, q[2] / len, q[3] / len]
1132        } else {
1133            [0.0, 0.0, 0.0, 1.0]
1134        }
1135    }
1136
1137    fn quat_nlerp(a: [f32; 4], b: [f32; 4], t: f32) -> [f32; 4] {
1138        let mut end = b;
1139        let dot = a[0] * b[0] + a[1] * b[1] + a[2] * b[2] + a[3] * b[3];
1140        if dot < 0.0 {
1141            end = [-b[0], -b[1], -b[2], -b[3]];
1142        }
1143        let one_minus_t = 1.0 - t;
1144        Self::quat_normalize([
1145            a[0] * one_minus_t + end[0] * t,
1146            a[1] * one_minus_t + end[1] * t,
1147            a[2] * one_minus_t + end[2] * t,
1148            a[3] * one_minus_t + end[3] * t,
1149        ])
1150    }
1151
1152    fn pose_from_quat_translation(
1153        rotation_xyzw: [f32; 4],
1154        translation_xyz: [f32; 3],
1155    ) -> openxr::Posef {
1156        openxr::Posef {
1157            orientation: openxr::Quaternionf {
1158                x: rotation_xyzw[0],
1159                y: rotation_xyzw[1],
1160                z: rotation_xyzw[2],
1161                w: rotation_xyzw[3],
1162            },
1163            position: openxr::Vector3f {
1164                x: translation_xyz[0],
1165                y: translation_xyz[1],
1166                z: translation_xyz[2],
1167            },
1168        }
1169    }
1170
1171    fn derive_head_pose(views: &[openxr::View]) -> Option<openxr::Posef> {
1172        match views {
1173            [] => None,
1174            [view] => Some(view.pose),
1175            [left, right, ..] => {
1176                let left_q = Self::quat_from_posef(left.pose);
1177                let right_q = Self::quat_from_posef(right.pose);
1178                let center_q = Self::quat_nlerp(left_q, right_q, 0.5);
1179                let center_t = [
1180                    0.5 * (left.pose.position.x + right.pose.position.x),
1181                    0.5 * (left.pose.position.y + right.pose.position.y),
1182                    0.5 * (left.pose.position.z + right.pose.position.z),
1183                ];
1184                Some(Self::pose_from_quat_translation(center_q, center_t))
1185            }
1186        }
1187    }
1188
1189    fn quat_angle_degrees(a: [f32; 4], b: [f32; 4]) -> f32 {
1190        let a = Self::quat_normalize(a);
1191        let b = Self::quat_normalize(b);
1192        let dot = (a[0] * b[0] + a[1] * b[1] + a[2] * b[2] + a[3] * b[3])
1193            .abs()
1194            .clamp(0.0, 1.0);
1195        (2.0 * dot.acos()).to_degrees()
1196    }
1197
1198    fn rolling_avg(window: &VecDeque<f32>) -> f32 {
1199        if window.is_empty() {
1200            0.0
1201        } else {
1202            window.iter().copied().sum::<f32>() / window.len() as f32
1203        }
1204    }
1205
1206    fn rolling_max(window: &VecDeque<f32>) -> f32 {
1207        window.iter().copied().fold(0.0, f32::max)
1208    }
1209
1210    fn update_hand_rotation_debug(
1211        debug_state: &mut HandRotationDebugState,
1212        hand: ControllerHand,
1213        _joint: Option<openxr::HandJointEXT>,
1214        pose: openxr::Posef,
1215    ) {
1216        if !Self::debug_hand_rotation_enabled() {
1217            return;
1218        }
1219
1220        let quat = Self::quat_from_posef(pose);
1221        let window_len = Self::debug_hand_rotation_window_len();
1222        let debug = match hand {
1223            ControllerHand::Left => &mut debug_state.left,
1224            ControllerHand::Right => &mut debug_state.right,
1225        };
1226
1227        let step_deg = debug
1228            .previous_quat_xyzw
1229            .map(|previous| Self::quat_angle_degrees(previous, quat))
1230            .unwrap_or(0.0);
1231        debug.previous_quat_xyzw = Some(quat);
1232
1233        if debug.step_deg.len() >= window_len {
1234            let _ = debug.step_deg.pop_front();
1235        }
1236        debug.step_deg.push_back(step_deg);
1237        debug.sample_count += 1;
1238
1239        let _ = window_len;
1240    }
1241
1242    fn preferred_pose(
1243        sess: &OpenXRSessionState,
1244        hand: ControllerHand,
1245        pose: ControllerPoseKind,
1246    ) -> (Option<openxr::Posef>, &'static str) {
1247        let hand_root = match hand {
1248            ControllerHand::Left => sess.hand_root_pose_cache.left_root,
1249            ControllerHand::Right => sess.hand_root_pose_cache.right_root,
1250        };
1251        if let Some(pose) = hand_root {
1252            return (Some(pose), "hand_root");
1253        }
1254
1255        let controller_pose = match (hand, pose) {
1256            (ControllerHand::Left, ControllerPoseKind::Aim) => sess.controller_pose_cache.left_aim,
1257            (ControllerHand::Right, ControllerPoseKind::Aim) => {
1258                sess.controller_pose_cache.right_aim
1259            }
1260            (ControllerHand::Left, ControllerPoseKind::Grip) => {
1261                sess.controller_pose_cache.left_grip
1262            }
1263            (ControllerHand::Right, ControllerPoseKind::Grip) => {
1264                sess.controller_pose_cache.right_grip
1265            }
1266        };
1267
1268        if controller_pose.is_some() {
1269            (controller_pose, "controller_action")
1270        } else {
1271            (None, "none")
1272        }
1273    }
1274
1275    fn transform_child_of(world: &World, component: ComponentId) -> Option<ComponentId> {
1276        world.children_of(component).iter().copied().find(|&cid| {
1277            world
1278                .get_component_by_id_as::<crate::engine::ecs::component::TransformComponent>(cid)
1279                .is_some()
1280        })
1281    }
1282
1283    fn transform_parent_world(world: &World, transform_cid: ComponentId) -> [[f32; 4]; 4] {
1284        world
1285            .parent_of(transform_cid)
1286            .and_then(|parent| TransformSystem::world_model(world, parent))
1287            .unwrap_or([
1288                [1.0, 0.0, 0.0, 0.0],
1289                [0.0, 1.0, 0.0, 0.0],
1290                [0.0, 0.0, 1.0, 0.0],
1291                [0.0, 0.0, 0.0, 1.0],
1292            ])
1293    }
1294
1295    fn input_xr_ancestor(world: &World, cid: ComponentId) -> Option<ComponentId> {
1296        let mut cur = cid;
1297        loop {
1298            if world
1299                .get_component_by_id_as::<InputXRComponent>(cur)
1300                .is_some()
1301            {
1302                return Some(cur);
1303            }
1304            let Some(parent) = world.parent_of(cur) else {
1305                return None;
1306            };
1307            cur = parent;
1308        }
1309    }
1310
1311    fn xr_rig_origin_world(world: &World, visuals: &VisualWorld) -> [[f32; 4]; 4] {
1312        let Some(camera_cid) = visuals
1313            .active_xr_camera()
1314            .or_else(|| Self::first_enabled_camera_xr(world))
1315        else {
1316            return [
1317                [1.0, 0.0, 0.0, 0.0],
1318                [0.0, 1.0, 0.0, 0.0],
1319                [0.0, 0.0, 1.0, 0.0],
1320                [0.0, 0.0, 0.0, 1.0],
1321            ];
1322        };
1323
1324        if let Some(input_xr_cid) = Self::input_xr_ancestor(world, camera_cid) {
1325            if let Some(driven_transform) = Self::transform_child_of(world, input_xr_cid) {
1326                return Self::transform_parent_world(world, driven_transform);
1327            }
1328        }
1329
1330        TransformSystem::world_model(world, camera_cid).unwrap_or([
1331            [1.0, 0.0, 0.0, 0.0],
1332            [0.0, 1.0, 0.0, 0.0],
1333            [0.0, 0.0, 1.0, 0.0],
1334            [0.0, 0.0, 0.0, 1.0],
1335        ])
1336    }
1337
1338    /// Hint: prefer this Vulkan `VkFormat` (as raw `u32`) when creating the OpenXR swapchain.
1339    ///
1340    /// This is used to match the window renderer's color attachment format, so we can copy
1341    /// rendered images into the OpenXR swapchain without format mismatch.
1342    pub fn set_preferred_swapchain_format(&mut self, format: u32) {
1343        self.preferred_swapchain_format = Some(format);
1344    }
1345
1346    /// Returns the Vulkan instance/device extensions required by the active OpenXR runtime.
1347    ///
1348    /// This uses the `XR_KHR_vulkan_enable` query APIs (space-delimited strings). Even when the
1349    /// runtime supports `XR_KHR_vulkan_enable2`, SteamVR commonly still reports the required
1350    /// extension lists via these functions.
1351    pub fn required_vulkan_extensions(&self) -> Option<(Vec<String>, Vec<String>)> {
1352        let state = self.state.as_ref()?;
1353
1354        let instance_exts = state
1355            .instance
1356            .vulkan_legacy_instance_extensions(state.system)
1357            .ok()?;
1358        let device_exts = state
1359            .instance
1360            .vulkan_legacy_device_extensions(state.system)
1361            .ok()?;
1362
1363        let split = |s: String| -> Vec<String> {
1364            s.split_whitespace()
1365                .map(|v| v.trim().to_string())
1366                .filter(|v| !v.is_empty())
1367                .collect()
1368        };
1369
1370        Some((split(instance_exts), split(device_exts)))
1371    }
1372
1373    pub fn set_vulkan_graphics(&mut self, gfx: XrVulkanGraphics) {
1374        self.vulkan_graphics = Some(gfx);
1375
1376        // If OpenXR is already initialized, opportunistically create the session now.
1377        let Some(state) = self.state.as_mut() else {
1378            return;
1379        };
1380
1381        if state.session.is_none() {
1382            if let Err(err) = Self::try_init_session(state, gfx, self.preferred_swapchain_format) {
1383                eprintln!("[OpenXR] Session init failed: {err}");
1384                self.last_init_error = Some(err);
1385            }
1386        }
1387    }
1388
1389    pub fn initialize_runtime(&mut self) -> Result<(), String> {
1390        if self.state.is_some() {
1391            return Ok(());
1392        }
1393
1394        match Self::try_init_openxr() {
1395            Ok(state) => {
1396                println!("[OpenXR] Initialized.");
1397                self.state = Some(state);
1398                self.last_init_error = None;
1399
1400                if let (Some(state), Some(gfx)) = (self.state.as_mut(), self.vulkan_graphics) {
1401                    if state.session.is_none() {
1402                        if let Err(err) =
1403                            Self::try_init_session(state, gfx, self.preferred_swapchain_format)
1404                        {
1405                            eprintln!("[OpenXR] Session init failed: {err}");
1406                            self.last_init_error = Some(err.clone());
1407                            return Err(err);
1408                        }
1409                    }
1410                }
1411
1412                Ok(())
1413            }
1414            Err(err) => {
1415                eprintln!("[OpenXR] Init failed: {err}");
1416                self.last_init_error = Some(err.clone());
1417                Err(err)
1418            }
1419        }
1420    }
1421
1422    pub fn retry_runtime(&mut self) -> Result<(), String> {
1423        if let Some(state) = self.state.as_mut() {
1424            if state.session.is_some() {
1425                return Ok(());
1426            }
1427            let gfx = self
1428                .vulkan_graphics
1429                .ok_or_else(|| "Vulkan graphics are not initialized yet".to_string())?;
1430            return Self::try_init_session(state, gfx, self.preferred_swapchain_format).map_err(
1431                |error| {
1432                    self.last_init_error = Some(error.clone());
1433                    error
1434                },
1435            );
1436        }
1437        self.initialize_runtime()
1438    }
1439
1440    pub fn register_xr(
1441        &mut self,
1442        world: &mut World,
1443        _visuals: &mut VisualWorld,
1444        component: ComponentId,
1445    ) {
1446        let Some(cfg) = world.get_component_by_id_as::<XrComponent>(component) else {
1447            return;
1448        };
1449
1450        if !cfg.enabled {
1451            return;
1452        }
1453
1454        if self.state.is_some() {
1455            return;
1456        }
1457
1458        let _ = self.initialize_runtime();
1459    }
1460
1461    pub fn register_controller_xr(
1462        &mut self,
1463        _world: &mut World,
1464        _visuals: &mut VisualWorld,
1465        component: ComponentId,
1466    ) {
1467        self.controller_components.insert(component);
1468        self.controller_pose_source_last_logged.remove(&component);
1469    }
1470
1471    pub fn register_input_xr(
1472        &mut self,
1473        _world: &mut World,
1474        _visuals: &mut VisualWorld,
1475        component: ComponentId,
1476    ) {
1477        self.input_xr_components.insert(component);
1478    }
1479
1480    pub fn remove_controller_xr(
1481        &mut self,
1482        _world: &mut World,
1483        _visuals: &mut VisualWorld,
1484        component: ComponentId,
1485    ) {
1486        self.controller_components.remove(&component);
1487        self.controller_pose_source_last_logged.remove(&component);
1488    }
1489
1490    pub fn remove_input_xr(
1491        &mut self,
1492        _world: &mut World,
1493        _visuals: &mut VisualWorld,
1494        component: ComponentId,
1495    ) {
1496        self.input_xr_components.remove(&component);
1497    }
1498
1499    fn pump_events(&mut self) {
1500        let Some(state) = self.state.as_mut() else {
1501            return;
1502        };
1503
1504        // Drain events; for now we just print key session state transitions.
1505        loop {
1506            let evt = match state.instance.poll_event(&mut state.events) {
1507                Ok(v) => v,
1508                Err(e) => {
1509                    eprintln!("[OpenXR] poll_event error: {e:?}");
1510                    return;
1511                }
1512            };
1513
1514            let Some(evt) = evt else {
1515                break;
1516            };
1517
1518            match evt {
1519                openxr::Event::InstanceLossPending(_) => {
1520                    eprintln!("[OpenXR] Event: InstanceLossPending");
1521                }
1522                openxr::Event::SessionStateChanged(e) => {
1523                    println!("[OpenXR] Event: SessionStateChanged -> {:?}", e.state());
1524
1525                    if let Some(sess) = state.session.as_mut() {
1526                        sess.current_state = e.state();
1527                        match e.state() {
1528                            openxr::SessionState::READY => {
1529                                if !sess.running {
1530                                    if let Err(err) = sess.session.begin(state.view_type) {
1531                                        eprintln!("[OpenXR] session.begin failed: {err:?}");
1532                                    } else {
1533                                        sess.running = true;
1534                                    }
1535                                }
1536                            }
1537                            openxr::SessionState::STOPPING => {
1538                                if sess.running {
1539                                    if let Err(err) = sess.session.end() {
1540                                        eprintln!("[OpenXR] session.end failed: {err:?}");
1541                                    }
1542                                    sess.running = false;
1543                                }
1544                            }
1545                            openxr::SessionState::EXITING | openxr::SessionState::LOSS_PENDING => {
1546                                sess.running = false;
1547                            }
1548                            _ => {}
1549                        }
1550                    }
1551                }
1552                openxr::Event::EventsLost(e) => {
1553                    eprintln!("[OpenXR] Event: EventsLost ({})", e.lost_event_count());
1554                }
1555                _ => {
1556                    // Too noisy to print everything by default.
1557                }
1558            }
1559        }
1560    }
1561
1562    /// Like `tick`, but also queues transform updates for registered ControllerXRComponents.
1563    ///
1564    /// Controller poses are sourced from the last cache update performed during `render_xr`.
1565    pub fn tick_with_queue(
1566        &mut self,
1567        world: &mut World,
1568        visuals: &mut VisualWorld,
1569        _input: &InputState,
1570        emit: &mut dyn SignalEmitter,
1571        _dt_sec: f32,
1572    ) {
1573        self.pump_events();
1574
1575        // Validity is frame-local. Consumers must not solve against stale/default XR transforms.
1576        for component in self.input_xr_components.iter().copied().collect::<Vec<_>>() {
1577            if let Some(input) = world.get_component_by_id_as_mut::<InputXRComponent>(component) {
1578                input.pose_valid = false;
1579            }
1580        }
1581        for component in self
1582            .controller_components
1583            .iter()
1584            .copied()
1585            .collect::<Vec<_>>()
1586        {
1587            if let Some(controller) = world.get_component_by_id_as_mut::<XRHandComponent>(component)
1588            {
1589                controller.pose_valid = false;
1590            }
1591        }
1592
1593        let Some(state) = self.state.as_ref() else {
1594            return;
1595        };
1596        let Some(sess) = state.session.as_ref() else {
1597            return;
1598        };
1599        if !sess.running {
1600            return;
1601        }
1602        if !matches!(
1603            sess.current_state,
1604            openxr::SessionState::VISIBLE | openxr::SessionState::FOCUSED
1605        ) {
1606            return;
1607        }
1608
1609        // Compose headset/controller poses with the authored XR rig origin.
1610        let rig_world = Self::xr_rig_origin_world(world, visuals);
1611
1612        let input_xr_ids: Vec<ComponentId> = self.input_xr_components.iter().copied().collect();
1613        for input_xr_cid in input_xr_ids {
1614            let Some(cfg) = world.get_component_by_id_as::<InputXRComponent>(input_xr_cid) else {
1615                self.input_xr_components.remove(&input_xr_cid);
1616                continue;
1617            };
1618
1619            if !cfg.enabled {
1620                continue;
1621            }
1622
1623            let Some(head_pose) = sess.head_pose_cache else {
1624                continue;
1625            };
1626
1627            let Some(tcid) = Self::transform_child_of(world, input_xr_cid) else {
1628                continue;
1629            };
1630
1631            let world_from_head = Self::mul_mat4(
1632                &Self::transform_parent_world(world, tcid),
1633                &Self::mat4_from_pose(head_pose),
1634            );
1635            let desired_world_pos = [
1636                world_from_head[3][0],
1637                world_from_head[3][1],
1638                world_from_head[3][2],
1639            ];
1640            let desired_world_rot = math::mat_to_quat(world_from_head);
1641
1642            let local_translation =
1643                Self::world_to_local_translation(world, tcid, desired_world_pos);
1644            let parent_world_rot =
1645                Self::parent_world_rotation_quat(world, tcid).unwrap_or([0.0, 0.0, 0.0, 1.0]);
1646            let local_rotation =
1647                math::quat_mul(math::quat_conjugate(parent_world_rot), desired_world_rot);
1648
1649            let Some(t) = world
1650                .get_component_by_id_as_mut::<crate::engine::ecs::component::TransformComponent>(
1651                    tcid,
1652                )
1653            else {
1654                continue;
1655            };
1656
1657            t.transform.translation = local_translation;
1658            t.transform.rotation = local_rotation;
1659            t.transform.recompute_model();
1660
1661            let transform = t.transform;
1662            emit.push_intent_now(
1663                tcid,
1664                IntentValue::UpdateTransform {
1665                    component_ids: vec![tcid],
1666                    translation: transform.translation,
1667                    rotation_quat_xyzw: transform.rotation,
1668                    scale: transform.scale,
1669                },
1670            );
1671            if let Some(input) = world.get_component_by_id_as_mut::<InputXRComponent>(input_xr_cid)
1672            {
1673                input.pose_valid = true;
1674            }
1675        }
1676
1677        let controller_ids: Vec<ComponentId> = self.controller_components.iter().copied().collect();
1678        for controller_cid in controller_ids {
1679            let Some(cfg) = world.get_component_by_id_as::<XRHandComponent>(controller_cid) else {
1680                self.controller_components.remove(&controller_cid);
1681                continue;
1682            };
1683
1684            if !cfg.enabled {
1685                continue;
1686            }
1687
1688            let (pose, pose_source) = Self::preferred_pose(sess, cfg.hand, cfg.pose);
1689            let last_pose_source = self
1690                .controller_pose_source_last_logged
1691                .get(&controller_cid)
1692                .copied();
1693            if openxr_debug_enabled() && last_pose_source != Some(pose_source) {
1694                eprintln!(
1695                    "[OpenXR][ctlxr] component={controller_cid:?} hand={:?} pose={:?} source={pose_source}",
1696                    cfg.hand, cfg.pose
1697                );
1698                self.controller_pose_source_last_logged
1699                    .insert(controller_cid, pose_source);
1700            }
1701
1702            let Some(pose) = pose else {
1703                continue;
1704            };
1705
1706            // Semantics: ControllerXRComponent drives a TransformComponent child, if present.
1707            let Some(tcid) = Self::transform_child_of(world, controller_cid) else {
1708                continue;
1709            };
1710
1711            // Pose is in OpenXR reference space; convert to engine world by applying rig transform.
1712            let world_from_controller = Self::mul_mat4(&rig_world, &Self::mat4_from_pose(pose));
1713            let desired_world_pos = [
1714                world_from_controller[3][0],
1715                world_from_controller[3][1],
1716                world_from_controller[3][2],
1717            ];
1718            let desired_world_rot = math::mat_to_quat(world_from_controller);
1719
1720            let local_translation =
1721                Self::world_to_local_translation(world, tcid, desired_world_pos);
1722            let parent_world_rot =
1723                Self::parent_world_rotation_quat(world, tcid).unwrap_or([0.0, 0.0, 0.0, 1.0]);
1724            let local_rotation =
1725                math::quat_mul(math::quat_conjugate(parent_world_rot), desired_world_rot);
1726
1727            let Some(t) = world
1728                .get_component_by_id_as_mut::<crate::engine::ecs::component::TransformComponent>(
1729                    tcid,
1730                )
1731            else {
1732                continue;
1733            };
1734
1735            // Convert world-space target into local-space values relative to the nearest parent
1736            // transform above `tcid` (if any), matching how transform chains are composed.
1737            t.transform.translation = local_translation;
1738            t.transform.rotation = local_rotation;
1739            t.transform.recompute_model();
1740
1741            let transform = t.transform;
1742            emit.push_intent_now(
1743                tcid,
1744                IntentValue::UpdateTransform {
1745                    component_ids: vec![tcid],
1746                    translation: transform.translation,
1747                    rotation_quat_xyzw: transform.rotation,
1748                    scale: transform.scale,
1749                },
1750            );
1751            if let Some(controller) =
1752                world.get_component_by_id_as_mut::<XRHandComponent>(controller_cid)
1753            {
1754                controller.pose_valid = true;
1755            }
1756        }
1757    }
1758
1759    fn world_to_local_translation(
1760        world: &World,
1761        transform_cid: ComponentId,
1762        desired_world: [f32; 3],
1763    ) -> [f32; 3] {
1764        let mut cur = transform_cid;
1765        while let Some(parent) = world.parent_of(cur) {
1766            if let Some(t) = world
1767                .get_component_by_id_as::<crate::engine::ecs::component::TransformComponent>(parent)
1768            {
1769                if let Some(inv) = math::mat4_inverse(t.transform.matrix_world) {
1770                    let p_local = math::mat4_mul_vec4(
1771                        inv,
1772                        [desired_world[0], desired_world[1], desired_world[2], 1.0],
1773                    );
1774                    return [p_local[0], p_local[1], p_local[2]];
1775                }
1776                break;
1777            }
1778            cur = parent;
1779        }
1780
1781        desired_world
1782    }
1783
1784    fn parent_world_rotation_quat(world: &World, transform_cid: ComponentId) -> Option<[f32; 4]> {
1785        let mut cur = transform_cid;
1786        while let Some(parent) = world.parent_of(cur) {
1787            if let Some(t) = world
1788                .get_component_by_id_as::<crate::engine::ecs::component::TransformComponent>(parent)
1789            {
1790                return Some(math::mat_to_quat(t.transform.matrix_world));
1791            }
1792            cur = parent;
1793        }
1794        None
1795    }
1796
1797    fn try_init_hand_tracking(
1798        session: &openxr::Session<openxr::Vulkan>,
1799    ) -> Result<HandTrackingState, String> {
1800        let left = session
1801            .create_hand_tracker(openxr::HandEXT::LEFT)
1802            .map_err(|e| format!("create_hand_tracker(left): {e:?}"))?;
1803        let right = session
1804            .create_hand_tracker(openxr::HandEXT::RIGHT)
1805            .map_err(|e| format!("create_hand_tracker(right): {e:?}"))?;
1806
1807        Ok(HandTrackingState { left, right })
1808    }
1809
1810    fn valid_pose_from_hand_joint(joint: &openxr::HandJointLocationEXT) -> Option<openxr::Posef> {
1811        let flags = joint.location_flags;
1812        if flags.contains(openxr::SpaceLocationFlags::POSITION_VALID)
1813            && flags.contains(openxr::SpaceLocationFlags::ORIENTATION_VALID)
1814        {
1815            Some(joint.pose)
1816        } else {
1817            None
1818        }
1819    }
1820
1821    fn select_hand_root_pose(
1822        joints: &openxr::HandJointLocations,
1823    ) -> Option<(openxr::Posef, openxr::HandJointEXT)> {
1824        let wrist = &joints[openxr::HandJointEXT::WRIST];
1825        if let Some(pose) = Self::valid_pose_from_hand_joint(wrist) {
1826            return Some((pose, openxr::HandJointEXT::WRIST));
1827        }
1828
1829        let palm = &joints[openxr::HandJointEXT::PALM];
1830        Self::valid_pose_from_hand_joint(palm).map(|pose| (pose, openxr::HandJointEXT::PALM))
1831    }
1832
1833    fn wrist_palm_delta_deg(joints: &openxr::HandJointLocations) -> Option<u16> {
1834        let wrist = Self::valid_pose_from_hand_joint(&joints[openxr::HandJointEXT::WRIST])?;
1835        let palm = Self::valid_pose_from_hand_joint(&joints[openxr::HandJointEXT::PALM])?;
1836        let deg =
1837            Self::quat_angle_degrees(Self::quat_from_posef(wrist), Self::quat_from_posef(palm));
1838        Some(deg.round().clamp(0.0, u16::MAX as f32) as u16)
1839    }
1840
1841    fn quantize_angle_deg(value: u16, bucket_size: u16) -> u16 {
1842        if bucket_size <= 1 {
1843            value
1844        } else {
1845            (value / bucket_size) * bucket_size
1846        }
1847    }
1848
1849    fn root_pose_for_joint(
1850        joints: &openxr::HandJointLocations,
1851        joint: Option<openxr::HandJointEXT>,
1852    ) -> Option<openxr::Posef> {
1853        let joint = joint?;
1854        Self::valid_pose_from_hand_joint(&joints[joint])
1855    }
1856
1857    fn hand_debug_state(
1858        joints: Option<&openxr::HandJointLocations>,
1859        root_joint: Option<openxr::HandJointEXT>,
1860        previous_root_quat_xyzw: Option<[f32; 4]>,
1861    ) -> HandDebugState {
1862        let wrist_palm_delta_deg = joints
1863            .and_then(Self::wrist_palm_delta_deg)
1864            .map(|deg| Self::quantize_angle_deg(deg, 5));
1865        let root_pose = joints.and_then(|j| Self::root_pose_for_joint(j, root_joint));
1866        let wrist_step_deg = match (previous_root_quat_xyzw, root_pose) {
1867            (Some(previous), Some(pose)) => Some(
1868                Self::quat_angle_degrees(previous, Self::quat_from_posef(pose))
1869                    .round()
1870                    .clamp(0.0, u16::MAX as f32) as u16,
1871            ),
1872            _ => None,
1873        }
1874        .map(|deg| Self::quantize_angle_deg(deg, 2));
1875        HandDebugState {
1876            root_joint,
1877            wrist_palm_delta_deg,
1878            wrist_step_deg,
1879            wrist_step_spike: wrist_step_deg.is_some_and(|deg| deg >= 8),
1880        }
1881    }
1882
1883    fn log_hand_debug_snapshot(
1884        sess: &mut OpenXRSessionState,
1885        left_joints: Option<&openxr::HandJointLocations>,
1886        right_joints: Option<&openxr::HandJointLocations>,
1887        left_root_joint: Option<openxr::HandJointEXT>,
1888        right_root_joint: Option<openxr::HandJointEXT>,
1889    ) {
1890        let snapshot = HandDebugSnapshot {
1891            left: Self::hand_debug_state(
1892                left_joints,
1893                left_root_joint,
1894                sess.hand_rotation_debug.left.previous_quat_xyzw,
1895            ),
1896            right: Self::hand_debug_state(
1897                right_joints,
1898                right_root_joint,
1899                sess.hand_rotation_debug.right.previous_quat_xyzw,
1900            ),
1901        };
1902        if sess.last_hand_debug_snapshot == Some(snapshot) {
1903            return;
1904        }
1905        sess.last_hand_debug_snapshot = Some(snapshot);
1906        if openxr_debug_enabled() {
1907            eprintln!(
1908                "[OpenXR][hand] left root={:?} wrist_palm_delta_deg={:?} wrist_step_deg={:?} spike={} | right root={:?} wrist_palm_delta_deg={:?} wrist_step_deg={:?} spike={}",
1909                snapshot.left.root_joint,
1910                snapshot.left.wrist_palm_delta_deg,
1911                snapshot.left.wrist_step_deg,
1912                snapshot.left.wrist_step_spike,
1913                snapshot.right.root_joint,
1914                snapshot.right.wrist_palm_delta_deg,
1915                snapshot.right.wrist_step_deg,
1916                snapshot.right.wrist_step_spike,
1917            );
1918        }
1919    }
1920
1921    fn try_init_openxr() -> Result<OpenXRState, String> {
1922        // Prefer dynamically loading the OpenXR loader. This keeps us from requiring
1923        // special linker setup and matches typical Linux setups.
1924        let entry = unsafe { openxr::Entry::load().map_err(|e| format!("Entry::load: {e:?}"))? };
1925
1926        let available_extensions = entry
1927            .enumerate_extensions()
1928            .map_err(|e| format!("enumerate_extensions: {e:?}"))?;
1929
1930        let app_info = openxr::ApplicationInfo {
1931            application_name: "cat-engine",
1932            application_version: 1,
1933            engine_name: "cat-engine",
1934            engine_version: 1,
1935            api_version: openxr::Version::new(1, 0, 0),
1936        };
1937
1938        let mut extensions = openxr::ExtensionSet::default();
1939        // Use the legacy Vulkan binding path for now (reuse an already-created VkInstance/VkDevice).
1940        // SteamVR can be stricter with XR_KHR_vulkan_enable2 unless you create Vulkan objects
1941        // via xrCreateVulkanInstanceKHR/xrCreateVulkanDeviceKHR.
1942        extensions.khr_vulkan_enable2 = false;
1943        // Needed for Vulkan session creation.
1944        extensions.khr_vulkan_enable = true;
1945        if available_extensions.ext_hand_tracking {
1946            extensions.ext_hand_tracking = true;
1947        }
1948        if available_extensions.ext_hand_interaction {
1949            extensions.ext_hand_interaction = true;
1950        }
1951        if available_extensions.htc_vive_focus3_controller_interaction {
1952            extensions.htc_vive_focus3_controller_interaction = true;
1953        }
1954        if available_extensions.htc_hand_interaction {
1955            extensions.htc_hand_interaction = true;
1956        }
1957        let layers: [&str; 0] = [];
1958
1959        let instance = entry
1960            .create_instance(&app_info, &extensions, &layers)
1961            .map_err(|e| format!("create_instance: {e:?}"))?;
1962
1963        println!(
1964            "[OpenXR] Enabled extensions: khr_vulkan_enable={}, ext_hand_tracking={}, ext_hand_interaction={}, htc_vive_focus3_controller_interaction={}, htc_hand_interaction={}",
1965            extensions.khr_vulkan_enable,
1966            extensions.ext_hand_tracking,
1967            extensions.ext_hand_interaction,
1968            extensions.htc_vive_focus3_controller_interaction,
1969            extensions.htc_hand_interaction,
1970        );
1971
1972        // Best-effort runtime identification (helps debugging which OpenXR runtime is active).
1973        if let Ok(props) = instance.properties() {
1974            println!(
1975                "[OpenXR] Runtime: {} ({:?})",
1976                props.runtime_name, props.runtime_version
1977            );
1978        }
1979
1980        let system = match instance.system(openxr::FormFactor::HEAD_MOUNTED_DISPLAY) {
1981            Ok(system) => system,
1982            Err(openxr::sys::Result::ERROR_FORM_FACTOR_UNAVAILABLE) => {
1983                return Err(
1984                    "system(HMD): ERROR_FORM_FACTOR_UNAVAILABLE (no HMD detected / runtime not ready).\n\
1985                    Start an OpenXR runtime (e.g. SteamVR/Monado/ALVR) and ensure a headset is connected and the runtime is running before launching cat-engine.\n\
1986                    On Linux you can also check XR_RUNTIME_JSON points to an installed runtime manifest."
1987                        .to_string(),
1988                );
1989            }
1990            Err(e) => return Err(format!("system(HMD): {e:?}")),
1991        };
1992
1993        // Pick a supported blend mode and stash common config.
1994        let view_type = openxr::ViewConfigurationType::PRIMARY_STEREO;
1995        let blend_mode = instance
1996            .enumerate_environment_blend_modes(system, view_type)
1997            .ok()
1998            .and_then(|m| m.first().copied())
1999            .unwrap_or(openxr::EnvironmentBlendMode::OPAQUE);
2000
2001        Ok(OpenXRState {
2002            entry,
2003            instance,
2004            system,
2005            events: openxr::EventDataBuffer::new(),
2006            session: None,
2007            view_type,
2008            blend_mode,
2009        })
2010    }
2011
2012    fn try_init_session(
2013        state: &mut OpenXRState,
2014        gfx: XrVulkanGraphics,
2015        preferred_swapchain_format: Option<u32>,
2016    ) -> Result<(), String> {
2017        // Log Vulkan version requirements (useful debugging).
2018        if let Ok(reqs) = state
2019            .instance
2020            .graphics_requirements::<openxr::Vulkan>(state.system)
2021        {
2022            println!(
2023                "[OpenXR] Vulkan requirements: min {:?}, max {:?}",
2024                reqs.min_api_version_supported, reqs.max_api_version_supported
2025            );
2026        }
2027
2028        // SteamVR may validate that the VkPhysicalDevice matches the one it expects for the HMD.
2029        // If these differ (multi-GPU setups), create_session can fail.
2030        if let Ok(required_pd) = unsafe {
2031            state
2032                .instance
2033                .vulkan_graphics_device(state.system, gfx.vk_instance)
2034        } {
2035            if required_pd != gfx.vk_physical_device {
2036                return Err(format!(
2037                    "Vulkan physical device mismatch for OpenXR system.\n\
2038Engine/Vulkano physical device: {:?}\n\
2039OpenXR-required physical device: {:?}\n\
2040Fix: pick the OpenXR-required VkPhysicalDevice when creating the Vulkan device/queues.",
2041                    gfx.vk_physical_device, required_pd
2042                ));
2043            }
2044        }
2045
2046        let info = openxr::vulkan::SessionCreateInfo {
2047            instance: gfx.vk_instance,
2048            physical_device: gfx.vk_physical_device,
2049            device: gfx.vk_device,
2050            queue_family_index: gfx.queue_family_index,
2051            queue_index: gfx.queue_index,
2052        };
2053
2054        let (session, frame_waiter, frame_stream) = unsafe {
2055            state
2056                .instance
2057                .create_session::<openxr::Vulkan>(state.system, &info)
2058                .map_err(|e| {
2059                    let required_instance = state
2060                        .instance
2061                        .vulkan_legacy_instance_extensions(state.system)
2062                        .ok();
2063                    let required_device = state
2064                        .instance
2065                        .vulkan_legacy_device_extensions(state.system)
2066                        .ok();
2067
2068                    let extra = match (required_instance, required_device) {
2069                        (Some(i), Some(d)) => format!(
2070                            "\n[OpenXR] Runtime-required Vulkan instance extensions: {i}\n[OpenXR] Runtime-required Vulkan device extensions: {d}"
2071                        ),
2072                        _ => String::new(),
2073                    };
2074
2075                    format!(
2076                        "create_session(Vulkan): {e:?}.\n\
2077If this fails with Vulkan extension errors, the Vulkan instance/device created by Vulkano may be missing runtime-required extensions (from XR_KHR_vulkan_enable2)."
2078                    )
2079                    + extra.as_str()
2080                })?
2081        };
2082
2083        let reference_space = session
2084            .create_reference_space(openxr::ReferenceSpaceType::STAGE, openxr::Posef::IDENTITY)
2085            .map_err(|e| format!("create_reference_space(STAGE): {e:?}"))?;
2086
2087        let controller_input = match Self::try_init_controller_input(&state.instance, &session) {
2088            Ok(v) => {
2089                println!("[OpenXR] Controller input initialized.");
2090                Some(v)
2091            }
2092            Err(err) => {
2093                eprintln!("[OpenXR] Controller input init failed: {err}");
2094                None
2095            }
2096        };
2097
2098        let hand_tracking = match state.instance.supports_hand_tracking(state.system) {
2099            Ok(v) => {
2100                println!("[OpenXR] Runtime supports hand tracking: {v}");
2101                if v {
2102                    match Self::try_init_hand_tracking(&session) {
2103                        Ok(trackers) => {
2104                            println!("[OpenXR] Hand tracking initialized.");
2105                            Some(trackers)
2106                        }
2107                        Err(err) => {
2108                            eprintln!("[OpenXR] Hand tracking init failed: {err}");
2109                            None
2110                        }
2111                    }
2112                } else {
2113                    None
2114                }
2115            }
2116            Err(e) => {
2117                eprintln!("[OpenXR] supports_hand_tracking query failed: {e:?}");
2118                None
2119            }
2120        };
2121
2122        let xr_swapchain = XRSwapchain::new(
2123            &state.instance,
2124            &session,
2125            state.system,
2126            state.view_type,
2127            preferred_swapchain_format,
2128        )?;
2129
2130        let swapchain_image_initialized = vec![false; xr_swapchain.images().len()];
2131
2132        // Minimal Vulkan command recording for XR bring-up: wrap the existing VkDevice via ash.
2133        let entry = unsafe { ash::Entry::load().map_err(|e| format!("ash::Entry::load: {e:?}"))? };
2134
2135        let vk_instance = ash::vk::Instance::from_raw(gfx.vk_instance as usize as u64);
2136        let vk_device_handle = ash::vk::Device::from_raw(gfx.vk_device as usize as u64);
2137
2138        let vk_instance = unsafe { ash::Instance::load(entry.static_fn(), vk_instance) };
2139        let vk_device = unsafe { ash::Device::load(vk_instance.fp_v1_0(), vk_device_handle) };
2140        let vk_queue =
2141            unsafe { vk_device.get_device_queue(gfx.queue_family_index, gfx.queue_index) };
2142
2143        let vk_command_pool = unsafe {
2144            vk_device
2145                .create_command_pool(
2146                    &ash::vk::CommandPoolCreateInfo::default()
2147                        .queue_family_index(gfx.queue_family_index)
2148                        .flags(ash::vk::CommandPoolCreateFlags::RESET_COMMAND_BUFFER),
2149                    None,
2150                )
2151                .map_err(|e| format!("create_command_pool: {e:?}"))?
2152        };
2153
2154        let vk_command_buffer = unsafe {
2155            vk_device
2156                .allocate_command_buffers(
2157                    &ash::vk::CommandBufferAllocateInfo::default()
2158                        .command_pool(vk_command_pool)
2159                        .level(ash::vk::CommandBufferLevel::PRIMARY)
2160                        .command_buffer_count(1),
2161                )
2162                .map_err(|e| format!("allocate_command_buffers: {e:?}"))?
2163                .into_iter()
2164                .next()
2165                .ok_or("allocate_command_buffers returned 0 command buffers")?
2166        };
2167
2168        state.session = Some(OpenXRSessionState {
2169            session,
2170            frame_waiter,
2171            frame_stream,
2172            reference_space,
2173            running: false,
2174            current_state: openxr::SessionState::IDLE,
2175
2176            xr_swapchain,
2177
2178            swapchain_image_initialized,
2179
2180            did_log_format_mismatch: false,
2181
2182            vk_device,
2183            vk_queue,
2184            vk_command_pool,
2185            vk_command_buffer,
2186
2187            hand_tracking,
2188            hand_root_pose_cache: HandRootPoseCache::default(),
2189            hand_rotation_debug: HandRotationDebugState::default(),
2190            last_hand_debug_snapshot: None,
2191            head_pose_cache: None,
2192            controller_input,
2193            controller_pose_cache: ControllerPoseCache::default(),
2194        });
2195
2196        println!("[OpenXR] Session created (Vulkan)");
2197        Ok(())
2198    }
2199
2200    fn try_init_controller_input(
2201        instance: &openxr::Instance,
2202        session: &openxr::Session<openxr::Vulkan>,
2203    ) -> Result<ControllerInput, String> {
2204        let left = instance
2205            .string_to_path("/user/hand/left")
2206            .map_err(|e| format!("string_to_path(/user/hand/left): {e:?}"))?;
2207        let right = instance
2208            .string_to_path("/user/hand/right")
2209            .map_err(|e| format!("string_to_path(/user/hand/right): {e:?}"))?;
2210
2211        let action_set = instance
2212            .create_action_set("mittens_engine", "Cat Engine", 0)
2213            .map_err(|e| format!("create_action_set: {e:?}"))?;
2214        let hand_subaction_paths = [left, right];
2215
2216        let aim_pose = action_set
2217            .create_action::<openxr::Posef>("aim_pose", "Aim Pose", &hand_subaction_paths)
2218            .map_err(|e| format!("create_action(aim_pose): {e:?}"))?;
2219        let grip_pose = action_set
2220            .create_action::<openxr::Posef>("grip_pose", "Grip Pose", &hand_subaction_paths)
2221            .map_err(|e| format!("create_action(grip_pose): {e:?}"))?;
2222        let select = action_set
2223            .create_action::<bool>("select", "Select", &hand_subaction_paths)
2224            .map_err(|e| format!("create_action(select): {e:?}"))?;
2225        let left_stick_x = action_set
2226            .create_action::<f32>("left_stick_x", "Left Stick X", &hand_subaction_paths)
2227            .map_err(|e| format!("create_action(left_stick_x): {e:?}"))?;
2228        let left_stick_y = action_set
2229            .create_action::<f32>("left_stick_y", "Left Stick Y", &hand_subaction_paths)
2230            .map_err(|e| format!("create_action(left_stick_y): {e:?}"))?;
2231        let right_stick_x = action_set
2232            .create_action::<f32>("right_stick_x", "Right Stick X", &hand_subaction_paths)
2233            .map_err(|e| format!("create_action(right_stick_x): {e:?}"))?;
2234        let right_stick_y = action_set
2235            .create_action::<f32>("right_stick_y", "Right Stick Y", &hand_subaction_paths)
2236            .map_err(|e| format!("create_action(right_stick_y): {e:?}"))?;
2237        let trigger_value = action_set
2238            .create_action::<f32>("trigger_value", "Trigger Value", &hand_subaction_paths)
2239            .map_err(|e| format!("create_action(trigger_value): {e:?}"))?;
2240        let trigger_click = action_set
2241            .create_action::<bool>("trigger_click", "Trigger Click", &hand_subaction_paths)
2242            .map_err(|e| format!("create_action(trigger_click): {e:?}"))?;
2243        let grip_value = action_set
2244            .create_action::<f32>("grip_value", "Grip Value", &hand_subaction_paths)
2245            .map_err(|e| format!("create_action(grip_value): {e:?}"))?;
2246        let grip_click = action_set
2247            .create_action::<bool>("grip_click", "Grip Click", &hand_subaction_paths)
2248            .map_err(|e| format!("create_action(grip_click): {e:?}"))?;
2249        let button_a = action_set
2250            .create_action::<bool>("button_a", "Button A", &hand_subaction_paths)
2251            .map_err(|e| format!("create_action(button_a): {e:?}"))?;
2252        let button_b = action_set
2253            .create_action::<bool>("button_b", "Button B", &hand_subaction_paths)
2254            .map_err(|e| format!("create_action(button_b): {e:?}"))?;
2255        let button_x = action_set
2256            .create_action::<bool>("button_x", "Button X", &hand_subaction_paths)
2257            .map_err(|e| format!("create_action(button_x): {e:?}"))?;
2258        let button_y = action_set
2259            .create_action::<bool>("button_y", "Button Y", &hand_subaction_paths)
2260            .map_err(|e| format!("create_action(button_y): {e:?}"))?;
2261
2262        let left_aim_space = aim_pose
2263            .create_space(session.clone(), left, openxr::Posef::IDENTITY)
2264            .map_err(|e| format!("aim_pose.create_space(left): {e:?}"))?;
2265        let right_aim_space = aim_pose
2266            .create_space(session.clone(), right, openxr::Posef::IDENTITY)
2267            .map_err(|e| format!("aim_pose.create_space(right): {e:?}"))?;
2268        let left_grip_space = grip_pose
2269            .create_space(session.clone(), left, openxr::Posef::IDENTITY)
2270            .map_err(|e| format!("grip_pose.create_space(left): {e:?}"))?;
2271        let right_grip_space = grip_pose
2272            .create_space(session.clone(), right, openxr::Posef::IDENTITY)
2273            .map_err(|e| format!("grip_pose.create_space(right): {e:?}"))?;
2274
2275        // Suggest bindings before attaching the action set. Once attached, action sets
2276        // become immutable, and hiding suggestion failures makes profile bring-up
2277        // impossible to debug on SteamVR.
2278        let left_aim_path = instance
2279            .string_to_path("/user/hand/left/input/aim/pose")
2280            .map_err(|e| format!("string_to_path(left aim): {e:?}"))?;
2281        let right_aim_path = instance
2282            .string_to_path("/user/hand/right/input/aim/pose")
2283            .map_err(|e| format!("string_to_path(right aim): {e:?}"))?;
2284        let left_grip_path = instance
2285            .string_to_path("/user/hand/left/input/grip/pose")
2286            .map_err(|e| format!("string_to_path(left grip): {e:?}"))?;
2287        let right_grip_path = instance
2288            .string_to_path("/user/hand/right/input/grip/pose")
2289            .map_err(|e| format!("string_to_path(right grip): {e:?}"))?;
2290
2291        // Pose bindings are common across all profiles.
2292        let pose_bindings = [
2293            openxr::Binding::new(&aim_pose, left_aim_path),
2294            openxr::Binding::new(&aim_pose, right_aim_path),
2295            openxr::Binding::new(&grip_pose, left_grip_path),
2296            openxr::Binding::new(&grip_pose, right_grip_path),
2297        ];
2298
2299        let mut binding_reports = Vec::new();
2300
2301        for spec in PROFILE_BINDING_SPECS {
2302            log_profile_binding_dump(spec);
2303            let Ok(profile) = instance.string_to_path(spec.profile) else {
2304                continue;
2305            };
2306            let mut report = ProfileBindingReport {
2307                profile_path: spec.profile.to_string(),
2308                suggestions: Vec::new(),
2309            };
2310            let mut accepted_bindings = Vec::new();
2311            for (binding_label, binding_path, binding) in [
2312                (
2313                    "aim_pose_left",
2314                    "/user/hand/left/input/aim/pose",
2315                    pose_bindings[0],
2316                ),
2317                (
2318                    "aim_pose_right",
2319                    "/user/hand/right/input/aim/pose",
2320                    pose_bindings[1],
2321                ),
2322                (
2323                    "grip_pose_left",
2324                    "/user/hand/left/input/grip/pose",
2325                    pose_bindings[2],
2326                ),
2327                (
2328                    "grip_pose_right",
2329                    "/user/hand/right/input/grip/pose",
2330                    pose_bindings[3],
2331                ),
2332            ] {
2333                let outcome = suggest_binding_best_effort(
2334                    instance,
2335                    profile,
2336                    spec.profile,
2337                    binding_label,
2338                    binding_path,
2339                    binding,
2340                )?;
2341                if outcome == BindingSuggestionOutcome::Accepted {
2342                    accepted_bindings.push(binding);
2343                }
2344                report.suggestions.push(BindingSuggestionRecord {
2345                    label: binding_label,
2346                    path: binding_path.to_string(),
2347                    outcome,
2348                });
2349            }
2350            if let (Some(l), Some(r)) = (spec.select_left, spec.select_right) {
2351                if let (Ok(lp), Ok(rp)) = (instance.string_to_path(l), instance.string_to_path(r)) {
2352                    let outcome = suggest_binding_best_effort(
2353                        instance,
2354                        profile,
2355                        spec.profile,
2356                        "select_left",
2357                        l,
2358                        openxr::Binding::new(&select, lp),
2359                    )?;
2360                    if outcome == BindingSuggestionOutcome::Accepted {
2361                        accepted_bindings.push(openxr::Binding::new(&select, lp));
2362                    }
2363                    report.suggestions.push(BindingSuggestionRecord {
2364                        label: "select_left",
2365                        path: l.to_string(),
2366                        outcome,
2367                    });
2368                    let outcome = suggest_binding_best_effort(
2369                        instance,
2370                        profile,
2371                        spec.profile,
2372                        "select_right",
2373                        r,
2374                        openxr::Binding::new(&select, rp),
2375                    )?;
2376                    if outcome == BindingSuggestionOutcome::Accepted {
2377                        accepted_bindings.push(openxr::Binding::new(&select, rp));
2378                    }
2379                    report.suggestions.push(BindingSuggestionRecord {
2380                        label: "select_right",
2381                        path: r.to_string(),
2382                        outcome,
2383                    });
2384                }
2385            }
2386            let mut suggested = Vec::new();
2387            for hand_spec in spec.left_specs {
2388                append_action_bindings(
2389                    instance,
2390                    &mut suggested,
2391                    *hand_spec,
2392                    &left_stick_x,
2393                    &left_stick_y,
2394                    &right_stick_x,
2395                    &right_stick_y,
2396                    &trigger_value,
2397                    &trigger_click,
2398                    &grip_value,
2399                    &grip_click,
2400                    &button_a,
2401                    &button_b,
2402                    &button_x,
2403                    &button_y,
2404                );
2405            }
2406            for hand_spec in spec.right_specs {
2407                append_action_bindings(
2408                    instance,
2409                    &mut suggested,
2410                    *hand_spec,
2411                    &left_stick_x,
2412                    &left_stick_y,
2413                    &right_stick_x,
2414                    &right_stick_y,
2415                    &trigger_value,
2416                    &trigger_click,
2417                    &grip_value,
2418                    &grip_click,
2419                    &button_a,
2420                    &button_b,
2421                    &button_x,
2422                    &button_y,
2423                );
2424            }
2425            if !suggested.is_empty() {
2426                for (binding_label, binding_path, binding) in suggested {
2427                    let outcome = suggest_binding_best_effort(
2428                        instance,
2429                        profile,
2430                        spec.profile,
2431                        binding_label,
2432                        binding_path,
2433                        binding,
2434                    )?;
2435                    if outcome == BindingSuggestionOutcome::Accepted {
2436                        accepted_bindings.push(binding);
2437                    }
2438                    report.suggestions.push(BindingSuggestionRecord {
2439                        label: binding_label,
2440                        path: binding_path.to_string(),
2441                        outcome,
2442                    });
2443                }
2444            }
2445            if !accepted_bindings.is_empty() {
2446                instance
2447                    .suggest_interaction_profile_bindings(profile, &accepted_bindings)
2448                    .map_err(|err| {
2449                        format!(
2450                            "suggest_interaction_profile_bindings(final batch {}, count={}): {err:?}",
2451                            spec.profile,
2452                            accepted_bindings.len()
2453                        )
2454                    })?;
2455            }
2456            binding_reports.push(report);
2457        }
2458
2459        session
2460            .attach_action_sets(&[&action_set])
2461            .map_err(|e| format!("attach_action_sets: {e:?}"))?;
2462
2463        Ok(ControllerInput {
2464            action_set,
2465            aim_pose,
2466            grip_pose,
2467            select,
2468            left_stick_x,
2469            left_stick_y,
2470            right_stick_x,
2471            right_stick_y,
2472            trigger_value,
2473            trigger_click,
2474            grip_value,
2475            grip_click,
2476            button_a,
2477            button_b,
2478            button_x,
2479            button_y,
2480            left,
2481            right,
2482            left_aim_space,
2483            right_aim_space,
2484            left_grip_space,
2485            right_grip_space,
2486            profile_poll_counter: 0,
2487            last_logged_left_profile: None,
2488            last_logged_right_profile: None,
2489            last_reported_left_profile: None,
2490            last_reported_right_profile: None,
2491            last_debug_snapshot: None,
2492            did_log_bound_sources: false,
2493            binding_reports,
2494        })
2495    }
2496
2497    fn first_enabled_camera_xr(world: &World) -> Option<ComponentId> {
2498        world
2499            .all_components()
2500            .filter_map(|id| {
2501                world
2502                    .get_component_by_id_as::<CameraXRComponent>(id)
2503                    .map(|c| (id, c.enabled))
2504            })
2505            .find(|(_, enabled)| *enabled)
2506            .map(|(id, _)| id)
2507    }
2508
2509    fn mul_mat4(a: &[[f32; 4]; 4], b: &[[f32; 4]; 4]) -> [[f32; 4]; 4] {
2510        let mut c = [[0.0f32; 4]; 4];
2511        for col in 0..4 {
2512            for row in 0..4 {
2513                c[col][row] = a[0][row] * b[col][0]
2514                    + a[1][row] * b[col][1]
2515                    + a[2][row] * b[col][2]
2516                    + a[3][row] * b[col][3];
2517            }
2518        }
2519        c
2520    }
2521
2522    fn transform_from_matrix_world(
2523        m: [[f32; 4]; 4],
2524    ) -> crate::engine::graphics::primitives::Transform {
2525        let mut t = crate::engine::graphics::primitives::Transform::default();
2526        t.model = m;
2527        t.matrix_world = m;
2528        t
2529    }
2530
2531    fn mat4_from_pose(pose: openxr::Posef) -> [[f32; 4]; 4] {
2532        // IMPORTANT: This must match the engine's quaternion convention.
2533        // `Transform::recompute_model` is the canonical implementation.
2534        let q = pose.orientation;
2535        let p = pose.position;
2536
2537        let mut t = crate::engine::graphics::primitives::Transform::default();
2538        t.translation = [p.x, p.y, p.z];
2539        t.rotation = [q.x, q.y, q.z, q.w];
2540        t.scale = [1.0, 1.0, 1.0];
2541        t.recompute_model();
2542
2543        // OpenXR view poses are right-handed with -Z forward, which matches the engine's
2544        // Camera3D convention (forward -Z) and our projection math.
2545        //
2546        // Do not apply additional basis flips here unless we have a proven convention mismatch,
2547        // since flipping X/Z will also flip head translation direction and can break stereo.
2548        t.model
2549    }
2550
2551    fn invert_affine_transform(m: &[[f32; 4]; 4]) -> [[f32; 4]; 4] {
2552        // Upper-left 3x3 in column-major.
2553        let c0 = [m[0][0], m[0][1], m[0][2]];
2554        let c1 = [m[1][0], m[1][1], m[1][2]];
2555        let c2 = [m[2][0], m[2][1], m[2][2]];
2556
2557        // Row-major elements for determinant/cofactors.
2558        let a00 = c0[0];
2559        let a10 = c0[1];
2560        let a20 = c0[2];
2561        let a01 = c1[0];
2562        let a11 = c1[1];
2563        let a21 = c1[2];
2564        let a02 = c2[0];
2565        let a12 = c2[1];
2566        let a22 = c2[2];
2567
2568        let det = a00 * (a11 * a22 - a12 * a21) - a01 * (a10 * a22 - a12 * a20)
2569            + a02 * (a10 * a21 - a11 * a20);
2570
2571        if det.abs() < 1e-8 {
2572            return [
2573                [1.0, 0.0, 0.0, 0.0],
2574                [0.0, 1.0, 0.0, 0.0],
2575                [0.0, 0.0, 1.0, 0.0],
2576                [0.0, 0.0, 0.0, 1.0],
2577            ];
2578        }
2579        let inv_det = 1.0 / det;
2580
2581        // Inverse in row-major.
2582        let inv00 = (a11 * a22 - a12 * a21) * inv_det;
2583        let inv01 = (a02 * a21 - a01 * a22) * inv_det;
2584        let inv02 = (a01 * a12 - a02 * a11) * inv_det;
2585
2586        let inv10 = (a12 * a20 - a10 * a22) * inv_det;
2587        let inv11 = (a00 * a22 - a02 * a20) * inv_det;
2588        let inv12 = (a02 * a10 - a00 * a12) * inv_det;
2589
2590        let inv20 = (a10 * a21 - a11 * a20) * inv_det;
2591        let inv21 = (a01 * a20 - a00 * a21) * inv_det;
2592        let inv22 = (a00 * a11 - a01 * a10) * inv_det;
2593
2594        let tx = m[3][0];
2595        let ty = m[3][1];
2596        let tz = m[3][2];
2597
2598        let itx = -(inv00 * tx + inv01 * ty + inv02 * tz);
2599        let ity = -(inv10 * tx + inv11 * ty + inv12 * tz);
2600        let itz = -(inv20 * tx + inv21 * ty + inv22 * tz);
2601
2602        [
2603            [inv00, inv10, inv20, 0.0],
2604            [inv01, inv11, inv21, 0.0],
2605            [inv02, inv12, inv22, 0.0],
2606            [itx, ity, itz, 1.0],
2607        ]
2608    }
2609
2610    fn proj_from_fov_rh_zo(fov: openxr::Fovf, z_near: f32, z_far: f32) -> [[f32; 4]; 4] {
2611        let l = (fov.angle_left).tan() * z_near;
2612        let r = (fov.angle_right).tan() * z_near;
2613        let d = (fov.angle_down).tan() * z_near;
2614        let u = (fov.angle_up).tan() * z_near;
2615
2616        let w = r - l;
2617        let h = u - d;
2618        let nf = 1.0 / (z_near - z_far);
2619
2620        // Match the engine's column-major RH ZO layout used in Camera3D::perspective_rh_zo.
2621        // Now: +Y is up in clip space, matching OpenXR and GLTF conventions.
2622        [
2623            [2.0 * z_near / w, 0.0, 0.0, 0.0],
2624            [0.0, 2.0 * z_near / h, 0.0, 0.0],
2625            [(r + l) / w, (u + d) / h, z_far * nf, -1.0],
2626            [0.0, 0.0, (z_near * z_far) * nf, 0.0],
2627        ]
2628    }
2629}
2630
2631impl System for OpenXRSystem {
2632    fn tick(
2633        &mut self,
2634        _world: &mut World,
2635        _visuals: &mut VisualWorld,
2636        _input: &InputState,
2637        _dt_sec: f32,
2638    ) {
2639        self.pump_events();
2640    }
2641}
2642
2643impl OpenXRSystem {
2644    pub fn last_render_dt_sec(&self) -> Option<f32> {
2645        self.last_render_dt_sec
2646    }
2647
2648    pub fn render_xr(
2649        &mut self,
2650        world: &World,
2651        visuals: &mut VisualWorld,
2652        renderer: &mut VulkanoRenderer,
2653    ) {
2654        let profile = Self::profile_systems_enabled();
2655        let frame_profile_started = profile.then(Instant::now);
2656        let Some(state) = self.state.as_mut() else {
2657            self.xr_input_state = XrInputState::default();
2658            self.xr_gamepad_state = XrGamepadState::default();
2659            visuals.set_xr_frame_dt_sec(None);
2660            return;
2661        };
2662
2663        let Some(sess) = state.session.as_mut() else {
2664            self.xr_input_state = XrInputState::default();
2665            self.xr_gamepad_state = XrGamepadState::default();
2666            visuals.set_xr_frame_dt_sec(None);
2667            return;
2668        };
2669        if !sess.running {
2670            self.xr_input_state = XrInputState::default();
2671            self.xr_gamepad_state = XrGamepadState::default();
2672            visuals.set_xr_frame_dt_sec(None);
2673            return;
2674        }
2675
2676        let now = Instant::now();
2677        let dt_sec = self
2678            .last_render_instant
2679            .map(|prev| now.saturating_duration_since(prev).as_secs_f32());
2680        self.last_render_instant = Some(now);
2681        if let Some(dt_sec) = dt_sec {
2682            self.last_render_dt_sec = Some(dt_sec);
2683            visuals.set_xr_frame_dt_sec(Some(dt_sec));
2684        }
2685
2686        // If no XR camera is enabled, do not enter the OpenXR frame pacing path at all.
2687        // `wait_frame()` can block to headset cadence, which would otherwise throttle the
2688        // desktop window even though XR scene rendering is effectively disabled.
2689        if visuals
2690            .active_xr_camera()
2691            .or_else(|| Self::first_enabled_camera_xr(world))
2692            .is_none()
2693        {
2694            self.xr_input_state = XrInputState::default();
2695            self.xr_gamepad_state = XrGamepadState::default();
2696            sess.head_pose_cache = None;
2697            sess.hand_root_pose_cache = HandRootPoseCache::default();
2698            sess.controller_pose_cache = ControllerPoseCache::default();
2699            visuals.set_xr_frame_dt_sec(None);
2700            return;
2701        }
2702
2703        let stage_started = profile.then(Instant::now);
2704        let frame_state = match sess.frame_waiter.wait() {
2705            Ok(s) => s,
2706            Err(e) => {
2707                eprintln!("[OpenXR] wait_frame failed: {e:?}");
2708                return;
2709            }
2710        };
2711        if let Some(started) = stage_started {
2712            self.render_profile.wait_frame += started.elapsed();
2713        }
2714
2715        if let Err(e) = sess.frame_stream.begin() {
2716            eprintln!("[OpenXR] begin_frame failed: {e:?}");
2717            return;
2718        }
2719
2720        if !frame_state.should_render {
2721            let _ =
2722                sess.frame_stream
2723                    .end(frame_state.predicted_display_time, state.blend_mode, &[]);
2724            return;
2725        }
2726
2727        let tracking_started = profile.then(Instant::now);
2728
2729        let views = match sess.session.locate_views(
2730            state.view_type,
2731            frame_state.predicted_display_time,
2732            &sess.reference_space,
2733        ) {
2734            Ok((_flags, views)) => views,
2735            Err(e) => {
2736                eprintln!("[OpenXR] locate_views failed: {e:?}");
2737                let _ = sess.frame_stream.end(
2738                    frame_state.predicted_display_time,
2739                    state.blend_mode,
2740                    &[],
2741                );
2742                return;
2743            }
2744        };
2745
2746        sess.head_pose_cache = Self::derive_head_pose(&views);
2747        self.xr_gamepad_state.head_pose_rotation = sess.head_pose_cache.map(Self::quat_from_posef);
2748
2749        let mut left_root_for_debug: Option<(openxr::Posef, openxr::HandJointEXT)> = None;
2750        let mut right_root_for_debug: Option<(openxr::Posef, openxr::HandJointEXT)> = None;
2751        let mut left_joints_for_debug: Option<openxr::HandJointLocations> = None;
2752        let mut right_joints_for_debug: Option<openxr::HandJointLocations> = None;
2753
2754        if let Some(hand_tracking) = sess.hand_tracking.as_ref() {
2755            let left_joints = sess
2756                .reference_space
2757                .locate_hand_joints(&hand_tracking.left, frame_state.predicted_display_time);
2758            let right_joints = sess
2759                .reference_space
2760                .locate_hand_joints(&hand_tracking.right, frame_state.predicted_display_time);
2761
2762            match left_joints {
2763                Ok(Some(joints)) => {
2764                    let root = Self::select_hand_root_pose(&joints);
2765                    sess.hand_root_pose_cache.left_root = root.map(|(pose, _)| pose);
2766                    sess.hand_root_pose_cache.left_root_joint = root.map(|(_, joint)| joint);
2767                    left_root_for_debug = root;
2768                    left_joints_for_debug = Some(joints);
2769                }
2770                Ok(None) => {
2771                    sess.hand_root_pose_cache.left_root = None;
2772                    sess.hand_root_pose_cache.left_root_joint = None;
2773                }
2774                Err(e) => {
2775                    sess.hand_root_pose_cache.left_root = None;
2776                    sess.hand_root_pose_cache.left_root_joint = None;
2777                    eprintln!("[OpenXR] locate_hand_joints(left) failed: {e:?}");
2778                }
2779            }
2780
2781            match right_joints {
2782                Ok(Some(joints)) => {
2783                    let root = Self::select_hand_root_pose(&joints);
2784                    sess.hand_root_pose_cache.right_root = root.map(|(pose, _)| pose);
2785                    sess.hand_root_pose_cache.right_root_joint = root.map(|(_, joint)| joint);
2786                    right_root_for_debug = root;
2787                    right_joints_for_debug = Some(joints);
2788                }
2789                Ok(None) => {
2790                    sess.hand_root_pose_cache.right_root = None;
2791                    sess.hand_root_pose_cache.right_root_joint = None;
2792                }
2793                Err(e) => {
2794                    sess.hand_root_pose_cache.right_root = None;
2795                    sess.hand_root_pose_cache.right_root_joint = None;
2796                    eprintln!("[OpenXR] locate_hand_joints(right) failed: {e:?}");
2797                }
2798            }
2799        } else {
2800            sess.hand_root_pose_cache = HandRootPoseCache::default();
2801        }
2802
2803        Self::log_hand_debug_snapshot(
2804            sess,
2805            left_joints_for_debug.as_ref(),
2806            right_joints_for_debug.as_ref(),
2807            sess.hand_root_pose_cache.left_root_joint,
2808            sess.hand_root_pose_cache.right_root_joint,
2809        );
2810
2811        if let Some((pose, joint)) = left_root_for_debug {
2812            Self::update_hand_rotation_debug(
2813                &mut sess.hand_rotation_debug,
2814                ControllerHand::Left,
2815                Some(joint),
2816                pose,
2817            );
2818        }
2819        if let Some((pose, joint)) = right_root_for_debug {
2820            Self::update_hand_rotation_debug(
2821                &mut sess.hand_rotation_debug,
2822                ControllerHand::Right,
2823                Some(joint),
2824                pose,
2825            );
2826        }
2827
2828        // Update controller pose cache at the same predicted time as views.
2829        if let Some(ci) = sess.controller_input.as_mut() {
2830            // Sync actions (best-effort).
2831            let active = openxr::ActiveActionSet::new(&ci.action_set);
2832            match sess.session.sync_actions(&[active]) {
2833                Ok(()) => {}
2834                Err(e) => {
2835                    eprintln!("[OpenXR] sync_actions failed: {e:?}");
2836                }
2837            }
2838            let should_poll_profiles =
2839                ci.profile_poll_counter == 0 || ci.profile_poll_counter >= 89;
2840            ci.profile_poll_counter = if should_poll_profiles {
2841                1
2842            } else {
2843                ci.profile_poll_counter.saturating_add(1)
2844            };
2845            if should_poll_profiles {
2846                log_active_interaction_profile(
2847                    &state.instance,
2848                    &sess.session,
2849                    ci.left,
2850                    "/user/hand/left",
2851                    &mut ci.last_logged_left_profile,
2852                );
2853                log_active_interaction_profile(
2854                    &state.instance,
2855                    &sess.session,
2856                    ci.right,
2857                    "/user/hand/right",
2858                    &mut ci.last_logged_right_profile,
2859                );
2860            }
2861            let update_pose = |space: &openxr::Space, base: &openxr::Space, t: openxr::Time| {
2862                space.locate(base, t).ok()
2863            };
2864
2865            let left_profile = ci.last_logged_left_profile;
2866            let right_profile = ci.last_logged_right_profile;
2867
2868            sess.controller_pose_cache.left_aim = update_pose(
2869                &ci.left_aim_space,
2870                &sess.reference_space,
2871                frame_state.predicted_display_time,
2872            )
2873            .filter(|loc| {
2874                loc.location_flags
2875                    .contains(openxr::SpaceLocationFlags::POSITION_VALID)
2876                    && loc
2877                        .location_flags
2878                        .contains(openxr::SpaceLocationFlags::ORIENTATION_VALID)
2879            })
2880            .map(|loc| loc.pose);
2881            sess.controller_pose_cache.right_aim = update_pose(
2882                &ci.right_aim_space,
2883                &sess.reference_space,
2884                frame_state.predicted_display_time,
2885            )
2886            .filter(|loc| {
2887                loc.location_flags
2888                    .contains(openxr::SpaceLocationFlags::POSITION_VALID)
2889                    && loc
2890                        .location_flags
2891                        .contains(openxr::SpaceLocationFlags::ORIENTATION_VALID)
2892            })
2893            .map(|loc| loc.pose);
2894
2895            sess.controller_pose_cache.left_grip = update_pose(
2896                &ci.left_grip_space,
2897                &sess.reference_space,
2898                frame_state.predicted_display_time,
2899            )
2900            .filter(|loc| {
2901                loc.location_flags
2902                    .contains(openxr::SpaceLocationFlags::POSITION_VALID)
2903                    && loc
2904                        .location_flags
2905                        .contains(openxr::SpaceLocationFlags::ORIENTATION_VALID)
2906            })
2907            .map(|loc| loc.pose);
2908            sess.controller_pose_cache.right_grip = update_pose(
2909                &ci.right_grip_space,
2910                &sess.reference_space,
2911                frame_state.predicted_display_time,
2912            )
2913            .filter(|loc| {
2914                loc.location_flags
2915                    .contains(openxr::SpaceLocationFlags::POSITION_VALID)
2916                    && loc
2917                        .location_flags
2918                        .contains(openxr::SpaceLocationFlags::ORIENTATION_VALID)
2919            })
2920            .map(|loc| loc.pose);
2921
2922            let left_select = action_state_bool(&ci.select, &sess.session, ci.left);
2923            let right_select = action_state_bool(&ci.select, &sess.session, ci.right);
2924            let left_stick_x = ci.left_stick_x.state(&sess.session, ci.left).ok();
2925            let left_stick_y = ci.left_stick_y.state(&sess.session, ci.left).ok();
2926            let right_stick_x = ci.right_stick_x.state(&sess.session, ci.right).ok();
2927            let right_stick_y = ci.right_stick_y.state(&sess.session, ci.right).ok();
2928            let left_trigger_value = action_state_f32(&ci.trigger_value, &sess.session, ci.left);
2929            let right_trigger_value = action_state_f32(&ci.trigger_value, &sess.session, ci.right);
2930            let left_trigger_click = action_state_bool(&ci.trigger_click, &sess.session, ci.left);
2931            let right_trigger_click = action_state_bool(&ci.trigger_click, &sess.session, ci.right);
2932            let left_grip_value = action_state_f32(&ci.grip_value, &sess.session, ci.left);
2933            let right_grip_value = action_state_f32(&ci.grip_value, &sess.session, ci.right);
2934            let left_grip_click = action_state_bool(&ci.grip_click, &sess.session, ci.left);
2935            let right_grip_click = action_state_bool(&ci.grip_click, &sess.session, ci.right);
2936            let left_x = action_state_bool(&ci.button_x, &sess.session, ci.left);
2937            let left_y = action_state_bool(&ci.button_y, &sess.session, ci.left);
2938            let right_a = action_state_bool(&ci.button_a, &sess.session, ci.right);
2939            let right_b = action_state_bool(&ci.button_b, &sess.session, ci.right);
2940            let snapshot = ControllerDebugSnapshot {
2941                left_profile: profile_string_or_none(&state.instance, left_profile),
2942                right_profile: profile_string_or_none(&state.instance, right_profile),
2943                left_aim_valid: sess.controller_pose_cache.left_aim.is_some(),
2944                right_aim_valid: sess.controller_pose_cache.right_aim.is_some(),
2945                left_grip_valid: sess.controller_pose_cache.left_grip.is_some(),
2946                right_grip_valid: sess.controller_pose_cache.right_grip.is_some(),
2947                left_select: left_select.map(|s| (s.is_active, s.current_state)),
2948                right_select: right_select.map(|s| (s.is_active, s.current_state)),
2949                left_thumbstick: scalar_stick_state(left_stick_x, left_stick_y),
2950                right_thumbstick: scalar_stick_state(right_stick_x, right_stick_y),
2951                left_trigger_value: left_trigger_value.map(|s| (s.is_active, s.current_state)),
2952                right_trigger_value: right_trigger_value.map(|s| (s.is_active, s.current_state)),
2953                left_trigger_click: left_trigger_click.map(|s| (s.is_active, s.current_state)),
2954                right_trigger_click: right_trigger_click.map(|s| (s.is_active, s.current_state)),
2955                left_grip_value: left_grip_value.map(|s| (s.is_active, s.current_state)),
2956                right_grip_value: right_grip_value.map(|s| (s.is_active, s.current_state)),
2957                left_grip_click: left_grip_click.map(|s| (s.is_active, s.current_state)),
2958                right_grip_click: right_grip_click.map(|s| (s.is_active, s.current_state)),
2959                left_x: left_x.map(|s| (s.is_active, s.current_state)),
2960                left_y: left_y.map(|s| (s.is_active, s.current_state)),
2961                right_a: right_a.map(|s| (s.is_active, s.current_state)),
2962                right_b: right_b.map(|s| (s.is_active, s.current_state)),
2963            };
2964            if let Some(left_profile) = left_profile {
2965                if left_profile != openxr::Path::NULL
2966                    && ci.last_reported_left_profile != Some(left_profile)
2967                {
2968                    if let Some(report) = ci
2969                        .binding_reports
2970                        .iter()
2971                        .find(|report| report.profile_path == snapshot.left_profile)
2972                    {
2973                        log_runtime_profile_binding_report("left", report, &snapshot);
2974                    } else {
2975                        eprintln!(
2976                            "[OpenXR][binding_report] left active profile has no stored binding report: {}",
2977                            snapshot.left_profile
2978                        );
2979                    }
2980                    ci.last_reported_left_profile = Some(left_profile);
2981                }
2982            }
2983            if let Some(right_profile) = right_profile {
2984                if right_profile != openxr::Path::NULL
2985                    && ci.last_reported_right_profile != Some(right_profile)
2986                {
2987                    if let Some(report) = ci
2988                        .binding_reports
2989                        .iter()
2990                        .find(|report| report.profile_path == snapshot.right_profile)
2991                    {
2992                        log_runtime_profile_binding_report("right", report, &snapshot);
2993                    } else {
2994                        eprintln!(
2995                            "[OpenXR][binding_report] right active profile has no stored binding report: {}",
2996                            snapshot.right_profile
2997                        );
2998                    }
2999                    ci.last_reported_right_profile = Some(right_profile);
3000                }
3001            }
3002            if !ci.did_log_bound_sources
3003                && (matches!(left_profile, Some(profile) if profile != openxr::Path::NULL)
3004                    || matches!(right_profile, Some(profile) if profile != openxr::Path::NULL))
3005            {
3006                log_controller_input_bound_sources(&sess.session, ci);
3007                ci.did_log_bound_sources = true;
3008            }
3009            if openxr_debug_enabled() && ci.last_debug_snapshot.as_ref() != Some(&snapshot) {
3010                eprintln!(
3011                    "[OpenXR][debug] profiles left={} right={}",
3012                    snapshot.left_profile, snapshot.right_profile,
3013                );
3014                eprintln!(
3015                    "[OpenXR][debug] pose_valid left_aim={} right_aim={} left_grip={} right_grip={}",
3016                    snapshot.left_aim_valid,
3017                    snapshot.right_aim_valid,
3018                    snapshot.left_grip_valid,
3019                    snapshot.right_grip_valid,
3020                );
3021                eprintln!(
3022                    "[OpenXR][debug] select L={:?} R={:?} thumbstick L={:?} R={:?}",
3023                    snapshot.left_select,
3024                    snapshot.right_select,
3025                    snapshot.left_thumbstick,
3026                    snapshot.right_thumbstick,
3027                );
3028                eprintln!(
3029                    "[OpenXR][debug] trigger_value L={:?} R={:?} trigger_click L={:?} R={:?}",
3030                    snapshot.left_trigger_value,
3031                    snapshot.right_trigger_value,
3032                    snapshot.left_trigger_click,
3033                    snapshot.right_trigger_click,
3034                );
3035                eprintln!(
3036                    "[OpenXR][debug] grip_value L={:?} R={:?} grip_click L={:?} R={:?}",
3037                    snapshot.left_grip_value,
3038                    snapshot.right_grip_value,
3039                    snapshot.left_grip_click,
3040                    snapshot.right_grip_click,
3041                );
3042                eprintln!(
3043                    "[OpenXR][debug] face_buttons X={:?} Y={:?} A={:?} B={:?}",
3044                    snapshot.left_x, snapshot.left_y, snapshot.right_a, snapshot.right_b,
3045                );
3046            }
3047            ci.last_debug_snapshot = Some(snapshot);
3048
3049            // Select is a semantic trigger action: prefer the dedicated select action when it is
3050            // live, otherwise fall back to trigger click or trigger analog threshold.
3051            let prev = self.xr_input_state.trigger_down;
3052            self.xr_gamepad_state = XrGamepadState {
3053                active: true,
3054                hands: [XrHandGamepadState::default(), XrHandGamepadState::default()],
3055                head_pose_rotation: self.xr_gamepad_state.head_pose_rotation,
3056            };
3057            for i in 0..2 {
3058                let hand_path = if i == 0 { ci.left } else { ci.right };
3059                let select_state = action_state_bool(&ci.select, &sess.session, hand_path);
3060                let trigger_click_state =
3061                    action_state_bool(&ci.trigger_click, &sess.session, hand_path);
3062                let trigger_value_state =
3063                    action_state_f32(&ci.trigger_value, &sess.session, hand_path);
3064                let grip_click_state = action_state_bool(&ci.grip_click, &sess.session, hand_path);
3065                let grip_value_state = action_state_f32(&ci.grip_value, &sess.session, hand_path);
3066
3067                let cur_down =
3068                    derive_select_down(select_state, trigger_click_state, trigger_value_state);
3069                self.xr_input_state.trigger_down[i] = cur_down;
3070                self.xr_input_state.trigger_pressed[i] = cur_down && !prev[i];
3071                self.xr_input_state.trigger_released[i] = !cur_down && prev[i];
3072
3073                self.xr_gamepad_state.hands[i].thumbstick = match i {
3074                    0 => scalar_stick_value(
3075                        ci.left_stick_x.state(&sess.session, ci.left).ok(),
3076                        ci.left_stick_y.state(&sess.session, ci.left).ok(),
3077                    ),
3078                    1 => scalar_stick_value(
3079                        ci.right_stick_x.state(&sess.session, ci.right).ok(),
3080                        ci.right_stick_y.state(&sess.session, ci.right).ok(),
3081                    ),
3082                    _ => None,
3083                };
3084
3085                let trigger_value = trigger_value_state.map(|s| s.current_state);
3086                self.xr_gamepad_state.hands[i].trigger_value = trigger_value;
3087                self.xr_gamepad_state.hands[i].trigger_pressed = derive_trigger_like_pressed(
3088                    trigger_click_state,
3089                    trigger_value_state,
3090                    XR_TRIGGER_PRESS_THRESHOLD,
3091                );
3092
3093                let grip_value = grip_value_state.map(|s| s.current_state);
3094                self.xr_gamepad_state.hands[i].grip_value = grip_value;
3095                self.xr_gamepad_state.hands[i].grip_pressed = derive_trigger_like_pressed(
3096                    grip_click_state,
3097                    grip_value_state,
3098                    XR_GRIP_PRESS_THRESHOLD,
3099                );
3100            }
3101            self.xr_gamepad_state.hands[0].button_x = ci
3102                .button_x
3103                .state(&sess.session, ci.left)
3104                .ok()
3105                .filter(|s| s.is_active)
3106                .map(|s| (s.current_state, if s.current_state { 1.0 } else { 0.0 }));
3107            self.xr_gamepad_state.hands[0].button_y = ci
3108                .button_y
3109                .state(&sess.session, ci.left)
3110                .ok()
3111                .filter(|s| s.is_active)
3112                .map(|s| (s.current_state, if s.current_state { 1.0 } else { 0.0 }));
3113            self.xr_gamepad_state.hands[1].button_a = ci
3114                .button_a
3115                .state(&sess.session, ci.right)
3116                .ok()
3117                .filter(|s| s.is_active)
3118                .map(|s| (s.current_state, if s.current_state { 1.0 } else { 0.0 }));
3119            self.xr_gamepad_state.hands[1].button_b = ci
3120                .button_b
3121                .state(&sess.session, ci.right)
3122                .ok()
3123                .filter(|s| s.is_active)
3124                .map(|s| (s.current_state, if s.current_state { 1.0 } else { 0.0 }));
3125            log_xr_gamepad_changes(self.xr_gamepad_state_last_logged, self.xr_gamepad_state);
3126            self.xr_gamepad_state_last_logged = self.xr_gamepad_state;
3127        } else {
3128            self.xr_input_state = XrInputState::default();
3129            self.xr_gamepad_state = XrGamepadState::default();
3130            log_xr_gamepad_changes(self.xr_gamepad_state_last_logged, self.xr_gamepad_state);
3131            self.xr_gamepad_state_last_logged = self.xr_gamepad_state;
3132        }
3133
3134        // Publish XR per-eye camera matrices into VisualWorld (CameraTarget::Xr).
3135        let rig_world = Self::xr_rig_origin_world(world, visuals);
3136
3137        let mut eyes = Vec::with_capacity(views.len());
3138        for v in &views {
3139            let eye_from_space = Self::mat4_from_pose(v.pose);
3140            let world_from_eye = Self::mul_mat4(&rig_world, &eye_from_space);
3141            let view = Self::invert_affine_transform(&world_from_eye);
3142            let proj = Self::proj_from_fov_rh_zo(v.fov, 0.1, 100.0);
3143            eyes.push(CameraData {
3144                view,
3145                proj,
3146                transform: Self::transform_from_matrix_world(world_from_eye),
3147            });
3148        }
3149        visuals.set_xr_camera(eyes);
3150        if let Some(started) = tracking_started {
3151            self.render_profile.tracking_input += started.elapsed();
3152        }
3153
3154        // Acquire XR swapchain image.
3155        let acquire_started = profile.then(Instant::now);
3156        let image_index = {
3157            let swapchain = sess.xr_swapchain.swapchain_mut();
3158            match swapchain.acquire_image() {
3159                Ok(i) => i,
3160                Err(e) => {
3161                    eprintln!("[OpenXR] acquire_image failed: {e:?}");
3162                    let _ = sess.frame_stream.end(
3163                        frame_state.predicted_display_time,
3164                        state.blend_mode,
3165                        &[],
3166                    );
3167                    return;
3168                }
3169            }
3170        };
3171
3172        if let Err(e) = sess
3173            .xr_swapchain
3174            .swapchain_mut()
3175            .wait_image(openxr::Duration::INFINITE)
3176        {
3177            eprintln!("[OpenXR] wait_image failed: {e:?}");
3178        } else {
3179            if let Some(started) = acquire_started {
3180                self.render_profile.acquire_wait += started.elapsed();
3181            }
3182            // Render into offscreen Vulkano images (per eye), then copy into the OpenXR swapchain layers.
3183            let extent = sess.xr_swapchain.extent();
3184            let extent_u = [extent.width as u32, extent.height as u32];
3185
3186            let image_index_usize = image_index as usize;
3187            let dst_was_initialized = sess
3188                .swapchain_image_initialized
3189                .get(image_index_usize)
3190                .copied()
3191                .unwrap_or(false);
3192
3193            // Render both eyes first (blocking bring-up path).
3194            let view_count = sess.xr_swapchain.view_count() as usize;
3195
3196            let xr_format = sess.xr_swapchain.format();
3197            let window_format = renderer.window_vk_format_raw();
3198            let format_matches = window_format.map(|f| f == xr_format).unwrap_or(true);
3199
3200            // Copy requires compatible formats. If we couldn't match formats at swapchain creation,
3201            // fall back to a pink clear to prove we're submitting frames.
3202            let dst_image = sess.xr_swapchain.images()[image_index_usize];
3203            if !format_matches {
3204                if !sess.did_log_format_mismatch {
3205                    sess.did_log_format_mismatch = true;
3206                    eprintln!(
3207                        "[OpenXR] XR swapchain format mismatch (xr={} window={:?}); falling back to clear_color",
3208                        xr_format, window_format
3209                    );
3210                }
3211
3212                if let Err(e) = xr_renderer::clear_xr_swapchain_image(
3213                    &sess.vk_device,
3214                    sess.vk_queue,
3215                    sess.vk_command_buffer,
3216                    sess.xr_swapchain.view_count(),
3217                    dst_image,
3218                    visuals.clear_color(),
3219                    dst_was_initialized,
3220                ) {
3221                    eprintln!("[OpenXR] clear XR image failed: {e:?}");
3222                } else if let Some(slot) =
3223                    sess.swapchain_image_initialized.get_mut(image_index_usize)
3224                {
3225                    *slot = true;
3226                }
3227            } else {
3228                let eye_render_started = profile.then(Instant::now);
3229                for eye in 0..view_count.min(views.len()) {
3230                    if let Err(e) = renderer.render_xr_eye_offscreen(visuals, eye, extent_u) {
3231                        eprintln!("[OpenXR] render_xr_eye_offscreen failed: {e}");
3232                    }
3233                }
3234                if let Some(started) = eye_render_started {
3235                    self.render_profile.eye_render += started.elapsed();
3236                }
3237
3238                let copy_started = profile.then(Instant::now);
3239                if let Err(e) = xr_renderer::copy_offscreen_to_xr_layers(
3240                    &sess.vk_device,
3241                    sess.vk_queue,
3242                    sess.vk_command_buffer,
3243                    &sess.xr_swapchain,
3244                    renderer,
3245                    dst_image,
3246                    dst_was_initialized,
3247                    view_count,
3248                ) {
3249                    eprintln!("[OpenXR] copy to XR image failed: {e:?}");
3250                } else if let Some(slot) =
3251                    sess.swapchain_image_initialized.get_mut(image_index_usize)
3252                {
3253                    *slot = true;
3254                }
3255                if let Some(started) = copy_started {
3256                    self.render_profile.copy += started.elapsed();
3257                }
3258            }
3259        }
3260
3261        if let Err(e) = sess.xr_swapchain.swapchain_mut().release_image() {
3262            eprintln!("[OpenXR] release_image failed: {e:?}");
3263        }
3264
3265        // Submit a projection layer.
3266        if views.len() >= 2 {
3267            let rect = openxr::Rect2Di {
3268                offset: openxr::Offset2Di { x: 0, y: 0 },
3269                extent: sess.xr_swapchain.extent(),
3270            };
3271
3272            let pv0 = openxr::CompositionLayerProjectionView::new()
3273                .pose(views[0].pose)
3274                .fov(views[0].fov)
3275                .sub_image(
3276                    openxr::SwapchainSubImage::new()
3277                        .swapchain(sess.xr_swapchain.swapchain())
3278                        .image_array_index(0)
3279                        .image_rect(rect),
3280                );
3281
3282            let pv1 = openxr::CompositionLayerProjectionView::new()
3283                .pose(views[1].pose)
3284                .fov(views[1].fov)
3285                .sub_image(
3286                    openxr::SwapchainSubImage::new()
3287                        .swapchain(sess.xr_swapchain.swapchain())
3288                        .image_array_index(1)
3289                        .image_rect(rect),
3290                );
3291
3292            let projection_views = [pv0, pv1];
3293            let layer = openxr::CompositionLayerProjection::new()
3294                .space(&sess.reference_space)
3295                .views(&projection_views);
3296
3297            let submit_started = profile.then(Instant::now);
3298            if let Err(e) = sess.frame_stream.end(
3299                frame_state.predicted_display_time,
3300                state.blend_mode,
3301                &[&layer],
3302            ) {
3303                eprintln!("[OpenXR] end_frame failed: {e:?}");
3304            }
3305            if let Some(started) = submit_started {
3306                self.render_profile.submit += started.elapsed();
3307            }
3308
3309            if let Some(started) = frame_profile_started {
3310                self.finish_xr_render_profile_frame(started);
3311            }
3312            return;
3313        }
3314
3315        let submit_started = profile.then(Instant::now);
3316        if let Err(e) =
3317            sess.frame_stream
3318                .end(frame_state.predicted_display_time, state.blend_mode, &[])
3319        {
3320            eprintln!("[OpenXR] end_frame failed: {e:?}");
3321        }
3322        if let Some(started) = submit_started {
3323            self.render_profile.submit += started.elapsed();
3324        }
3325        if let Some(started) = frame_profile_started {
3326            self.finish_xr_render_profile_frame(started);
3327        }
3328    }
3329}