1use crate::engine::ecs::component::CameraXRComponent;
2use crate::engine::ecs::component::{
3 ControllerHand, ControllerPoseKind, InputXRComponent, XRHandComponent, XrComponent,
4};
5use crate::engine::ecs::system::System;
6use crate::engine::ecs::system::TransformSystem;
7use crate::engine::ecs::system::vr_types::{XrGamepadState, XrHandGamepadState, XrInputState};
8use crate::engine::ecs::{ComponentId, IntentValue, SignalEmitter, World};
9use crate::engine::graphics::CameraData;
10use crate::engine::graphics::VisualWorld;
11use crate::engine::graphics::VulkanoRenderer;
12use crate::engine::graphics::XRSwapchain;
13use crate::engine::graphics::XrVulkanGraphics;
14use crate::engine::graphics::xr_renderer;
15use crate::engine::user_input::InputState;
16use crate::utils::math;
17
18use ash::vk::Handle as _;
19
20use std::collections::{HashMap, HashSet, VecDeque};
21use std::sync::OnceLock;
22use std::time::Instant;
23
24fn openxr_debug_enabled() -> bool {
25 static ENABLED: OnceLock<bool> = OnceLock::new();
26 *ENABLED.get_or_init(|| {
27 std::env::var("CAT_OPENXR_DEBUG")
28 .ok()
29 .map(|s| {
30 let s = s.trim().to_ascii_lowercase();
31 s == "1" || s == "true" || s == "on" || s == "yes"
32 })
33 .unwrap_or(false)
34 })
35}
36
37fn log_xr_gamepad_changes(prev: XrGamepadState, next: XrGamepadState) {
38 if !openxr_debug_enabled() {
39 return;
40 }
41 if prev.active != next.active {
42 eprintln!(
43 "[OpenXR][gamepad] active -> {}",
44 if next.active { "true" } else { "false" }
45 );
46 }
47 for (index, hand_name) in ["left", "right"].into_iter().enumerate() {
48 let before = prev.hands[index];
49 let after = next.hands[index];
50 if before.thumbstick != after.thumbstick {
51 eprintln!(
52 "[OpenXR][gamepad] {hand_name} thumbstick -> {:?}",
53 after.thumbstick
54 );
55 }
56 if before.trigger_value != after.trigger_value {
57 eprintln!(
58 "[OpenXR][gamepad] {hand_name} trigger_value -> {:?}",
59 after.trigger_value
60 );
61 }
62 if before.trigger_pressed != after.trigger_pressed {
63 eprintln!(
64 "[OpenXR][gamepad] {hand_name} trigger_pressed -> {:?}",
65 after.trigger_pressed
66 );
67 }
68 if before.grip_value != after.grip_value {
69 eprintln!(
70 "[OpenXR][gamepad] {hand_name} grip_value -> {:?}",
71 after.grip_value
72 );
73 }
74 if before.grip_pressed != after.grip_pressed {
75 eprintln!(
76 "[OpenXR][gamepad] {hand_name} grip_pressed -> {:?}",
77 after.grip_pressed
78 );
79 }
80 if before.button_x != after.button_x {
81 eprintln!(
82 "[OpenXR][gamepad] {hand_name} button_x -> {:?}",
83 after.button_x
84 );
85 }
86 if before.button_y != after.button_y {
87 eprintln!(
88 "[OpenXR][gamepad] {hand_name} button_y -> {:?}",
89 after.button_y
90 );
91 }
92 if before.button_a != after.button_a {
93 eprintln!(
94 "[OpenXR][gamepad] {hand_name} button_a -> {:?}",
95 after.button_a
96 );
97 }
98 if before.button_b != after.button_b {
99 eprintln!(
100 "[OpenXR][gamepad] {hand_name} button_b -> {:?}",
101 after.button_b
102 );
103 }
104 }
105}
106
107fn log_active_interaction_profile(
108 instance: &openxr::Instance,
109 session: &openxr::Session<openxr::Vulkan>,
110 user_path: openxr::Path,
111 user_path_str: &str,
112 last_logged: &mut Option<openxr::Path>,
113) {
114 let Ok(profile) = session.current_interaction_profile(user_path) else {
115 return;
116 };
117 if *last_logged == Some(profile) {
118 return;
119 }
120 *last_logged = Some(profile);
121 if profile == openxr::Path::NULL {
122 eprintln!("[OpenXR][profile] {user_path_str}: no active interaction profile");
123 return;
124 }
125 match instance.path_to_string(profile) {
126 Ok(profile_str) => {
127 eprintln!("[OpenXR][profile] {user_path_str}: active profile = {profile_str}");
128 }
129 Err(err) => {
130 eprintln!(
131 "[OpenXR][profile] {user_path_str}: active profile path {:?} (path_to_string failed: {:?})",
132 profile, err
133 );
134 }
135 }
136}
137
138fn profile_string_or_none(instance: &openxr::Instance, profile: Option<openxr::Path>) -> String {
139 match profile {
140 Some(profile) if profile == openxr::Path::NULL => "none".to_string(),
141 Some(profile) => instance
142 .path_to_string(profile)
143 .unwrap_or_else(|_| format!("{profile:?}")),
144 None => "unknown".to_string(),
145 }
146}
147
148fn scalar_stick_value(
149 x: Option<openxr::ActionState<f32>>,
150 y: Option<openxr::ActionState<f32>>,
151) -> Option<[f32; 2]> {
152 match (x, y) {
153 (Some(x), Some(y)) if x.is_active || y.is_active => {
154 Some([x.current_state, y.current_state])
155 }
156 _ => None,
157 }
158}
159
160fn scalar_stick_state(
161 x: Option<openxr::ActionState<f32>>,
162 y: Option<openxr::ActionState<f32>>,
163) -> Option<(bool, [f32; 2])> {
164 match (x, y) {
165 (Some(x), Some(y)) => Some((
166 x.is_active || y.is_active,
167 [x.current_state, y.current_state],
168 )),
169 _ => None,
170 }
171}
172
173const XR_TRIGGER_PRESS_THRESHOLD: f32 = 0.7;
174const XR_GRIP_PRESS_THRESHOLD: f32 = 0.7;
175
176fn action_state_bool(
177 action: &openxr::Action<bool>,
178 session: &openxr::Session<openxr::Vulkan>,
179 hand: openxr::Path,
180) -> Option<openxr::ActionState<bool>> {
181 action.state(session, hand).ok().filter(|s| s.is_active)
182}
183
184fn action_state_f32(
185 action: &openxr::Action<f32>,
186 session: &openxr::Session<openxr::Vulkan>,
187 hand: openxr::Path,
188) -> Option<openxr::ActionState<f32>> {
189 action.state(session, hand).ok().filter(|s| s.is_active)
190}
191
192fn derive_trigger_like_pressed(
193 primary_click: Option<openxr::ActionState<bool>>,
194 analog_value: Option<openxr::ActionState<f32>>,
195 threshold: f32,
196) -> Option<(bool, f32)> {
197 primary_click
198 .map(|state| {
199 (
200 state.current_state,
201 if state.current_state { 1.0 } else { 0.0 },
202 )
203 })
204 .or_else(|| {
205 analog_value.map(|state| (state.current_state >= threshold, state.current_state))
206 })
207}
208
209fn derive_select_down(
210 select: Option<openxr::ActionState<bool>>,
211 trigger_click: Option<openxr::ActionState<bool>>,
212 trigger_value: Option<openxr::ActionState<f32>>,
213) -> bool {
214 select
215 .map(|state| state.current_state)
216 .or_else(|| trigger_click.map(|state| state.current_state))
217 .or_else(|| trigger_value.map(|state| state.current_state >= XR_TRIGGER_PRESS_THRESHOLD))
218 .unwrap_or(false)
219}
220
221fn log_action_bound_sources<T: openxr::ActionTy>(
222 session: &openxr::Session<openxr::Vulkan>,
223 label: &str,
224 action: &openxr::Action<T>,
225) {
226 let flags = openxr::InputSourceLocalizedNameFlags::USER_PATH
227 | openxr::InputSourceLocalizedNameFlags::INTERACTION_PROFILE
228 | openxr::InputSourceLocalizedNameFlags::COMPONENT;
229 match action.bound_sources(session) {
230 Ok(sources) if sources.is_empty() => {
231 eprintln!("[OpenXR][bound_sources] {label}: <unbound>");
232 }
233 Ok(sources) => {
234 for source in sources {
235 let path_str = action
236 .instance()
237 .path_to_string(source)
238 .unwrap_or_else(|_| format!("{source:?}"));
239 let localized = session
240 .input_source_localized_name(source, flags)
241 .unwrap_or_else(|err| format!("<localized-name error: {err:?}>"));
242 eprintln!(
243 "[OpenXR][bound_sources] {label}: path={} localized={}",
244 path_str, localized
245 );
246 }
247 }
248 Err(err) => {
249 eprintln!("[OpenXR][bound_sources] {label}: error {err:?}");
250 }
251 }
252}
253
254fn log_controller_input_bound_sources(
255 session: &openxr::Session<openxr::Vulkan>,
256 ci: &ControllerInput,
257) {
258 log_action_bound_sources(session, "select", &ci.select);
259 log_action_bound_sources(session, "left_stick_x", &ci.left_stick_x);
260 log_action_bound_sources(session, "left_stick_y", &ci.left_stick_y);
261 log_action_bound_sources(session, "right_stick_x", &ci.right_stick_x);
262 log_action_bound_sources(session, "right_stick_y", &ci.right_stick_y);
263 log_action_bound_sources(session, "trigger_value", &ci.trigger_value);
264 log_action_bound_sources(session, "trigger_click", &ci.trigger_click);
265 log_action_bound_sources(session, "grip_value", &ci.grip_value);
266 log_action_bound_sources(session, "grip_click", &ci.grip_click);
267 log_action_bound_sources(session, "button_x", &ci.button_x);
268 log_action_bound_sources(session, "button_y", &ci.button_y);
269 log_action_bound_sources(session, "button_a", &ci.button_a);
270 log_action_bound_sources(session, "button_b", &ci.button_b);
271}
272
273fn append_action_bindings<'a>(
274 instance: &openxr::Instance,
275 suggested: &mut Vec<(&'static str, &'static str, openxr::Binding<'a>)>,
276 spec: BindingSpec,
277 left_stick_x: &'a openxr::Action<f32>,
278 left_stick_y: &'a openxr::Action<f32>,
279 right_stick_x: &'a openxr::Action<f32>,
280 right_stick_y: &'a openxr::Action<f32>,
281 trigger_value: &'a openxr::Action<f32>,
282 trigger_click: &'a openxr::Action<bool>,
283 grip_value: &'a openxr::Action<f32>,
284 grip_click: &'a openxr::Action<bool>,
285 button_a: &'a openxr::Action<bool>,
286 button_b: &'a openxr::Action<bool>,
287 button_x: &'a openxr::Action<bool>,
288 button_y: &'a openxr::Action<bool>,
289) {
290 for &path_str in spec.paths {
291 let Ok(path) = instance.string_to_path(path_str) else {
292 continue;
293 };
294 match spec.label {
295 "left_stick_x" => suggested.push((
296 spec.label,
297 path_str,
298 openxr::Binding::new(left_stick_x, path),
299 )),
300 "left_stick_y" => suggested.push((
301 spec.label,
302 path_str,
303 openxr::Binding::new(left_stick_y, path),
304 )),
305 "right_stick_x" => suggested.push((
306 spec.label,
307 path_str,
308 openxr::Binding::new(right_stick_x, path),
309 )),
310 "right_stick_y" => suggested.push((
311 spec.label,
312 path_str,
313 openxr::Binding::new(right_stick_y, path),
314 )),
315 "trigger_value" => suggested.push((
316 spec.label,
317 path_str,
318 openxr::Binding::new(trigger_value, path),
319 )),
320 "trigger_click" => suggested.push((
321 spec.label,
322 path_str,
323 openxr::Binding::new(trigger_click, path),
324 )),
325 "grip_value" => {
326 suggested.push((spec.label, path_str, openxr::Binding::new(grip_value, path)))
327 }
328 "grip_click" => {
329 suggested.push((spec.label, path_str, openxr::Binding::new(grip_click, path)))
330 }
331 "a" => suggested.push((spec.label, path_str, openxr::Binding::new(button_a, path))),
332 "b" => suggested.push((spec.label, path_str, openxr::Binding::new(button_b, path))),
333 "x" => suggested.push((spec.label, path_str, openxr::Binding::new(button_x, path))),
334 "y" => suggested.push((spec.label, path_str, openxr::Binding::new(button_y, path))),
335 _ => {}
336 }
337 }
338}
339
340fn suggest_binding_best_effort(
341 instance: &openxr::Instance,
342 profile: openxr::Path,
343 profile_name: &str,
344 binding_label: &str,
345 binding_path: &str,
346 binding: openxr::Binding<'_>,
347) -> Result<BindingSuggestionOutcome, String> {
348 match instance.suggest_interaction_profile_bindings(profile, &[binding]) {
349 Ok(()) => Ok(BindingSuggestionOutcome::Accepted),
350 Err(openxr::sys::Result::ERROR_PATH_UNSUPPORTED) => {
351 eprintln!(
352 "[OpenXR] ignoring unsupported binding profile={} label={} path={}",
353 profile_name, binding_label, binding_path
354 );
355 Ok(BindingSuggestionOutcome::Unsupported)
356 }
357 Err(err) => Err(format!(
358 "suggest_interaction_profile_bindings({}, {} -> {}): {err:?}",
359 profile_name, binding_label, binding_path
360 )),
361 }
362}
363
364fn log_profile_binding_dump(spec: &ProfileBindingSpec) {
365 if !openxr_debug_enabled() {
366 return;
367 }
368 eprintln!("[OpenXR][binding_dump] profile {}", spec.profile);
369 eprintln!(
370 "[OpenXR][binding_dump] select left={:?} right={:?}",
371 spec.select_left, spec.select_right
372 );
373 for spec in spec.left_specs {
374 eprintln!(
375 "[OpenXR][binding_dump] left {:<13} -> {}",
376 spec.label,
377 spec.paths.join(", ")
378 );
379 }
380 for spec in spec.right_specs {
381 eprintln!(
382 "[OpenXR][binding_dump] right {:<12} -> {}",
383 spec.label,
384 spec.paths.join(", ")
385 );
386 }
387}
388
389#[derive(Debug, Clone, Copy, PartialEq, Eq)]
390enum BindingSuggestionOutcome {
391 Accepted,
392 Unsupported,
393}
394
395#[derive(Debug, Clone, PartialEq, Eq)]
396struct BindingSuggestionRecord {
397 label: &'static str,
398 path: String,
399 outcome: BindingSuggestionOutcome,
400}
401
402#[derive(Debug, Clone, PartialEq, Eq)]
403struct ProfileBindingReport {
404 profile_path: String,
405 suggestions: Vec<BindingSuggestionRecord>,
406}
407
408impl ProfileBindingReport {
409 fn accepted_count(&self) -> usize {
410 self.suggestions
411 .iter()
412 .filter(|record| record.outcome == BindingSuggestionOutcome::Accepted)
413 .count()
414 }
415
416 fn unsupported_count(&self) -> usize {
417 self.suggestions
418 .iter()
419 .filter(|record| record.outcome == BindingSuggestionOutcome::Unsupported)
420 .count()
421 }
422}
423
424fn log_runtime_profile_binding_report(
425 active_profile_label: &str,
426 report: &ProfileBindingReport,
427 snapshot: &ControllerDebugSnapshot,
428) {
429 eprintln!(
430 "[OpenXR][binding_report] hand={} profile={} accepted={} unsupported={}",
431 active_profile_label,
432 report.profile_path,
433 report.accepted_count(),
434 report.unsupported_count(),
435 );
436 for record in &report.suggestions {
437 let outcome = match record.outcome {
438 BindingSuggestionOutcome::Accepted => "accepted",
439 BindingSuggestionOutcome::Unsupported => "unsupported",
440 };
441 eprintln!(
442 "[OpenXR][binding_report] {:<12} {:<11} {}",
443 record.label, outcome, record.path
444 );
445 }
446 eprintln!(
447 "[OpenXR][binding_report] runtime_state profileL={} profileR={} poseL(aim={},grip={}) poseR(aim={},grip={})",
448 snapshot.left_profile,
449 snapshot.right_profile,
450 snapshot.left_aim_valid,
451 snapshot.left_grip_valid,
452 snapshot.right_aim_valid,
453 snapshot.right_grip_valid,
454 );
455 eprintln!(
456 "[OpenXR][binding_report] buttons/select left(select={:?},x={:?},y={:?}) right(select={:?},a={:?},b={:?})",
457 snapshot.left_select,
458 snapshot.left_x,
459 snapshot.left_y,
460 snapshot.right_select,
461 snapshot.right_a,
462 snapshot.right_b,
463 );
464 eprintln!(
465 "[OpenXR][binding_report] analog left(stick={:?},trigger={:?}/{:?},grip={:?}/{:?}) right(stick={:?},trigger={:?}/{:?},grip={:?}/{:?})",
466 snapshot.left_thumbstick,
467 snapshot.left_trigger_value,
468 snapshot.left_trigger_click,
469 snapshot.left_grip_value,
470 snapshot.left_grip_click,
471 snapshot.right_thumbstick,
472 snapshot.right_trigger_value,
473 snapshot.right_trigger_click,
474 snapshot.right_grip_value,
475 snapshot.right_grip_click,
476 );
477}
478
479pub struct OpenXRSystem {
480 state: Option<OpenXRState>,
481 last_init_error: Option<String>,
482 vulkan_graphics: Option<XrVulkanGraphics>,
483 preferred_swapchain_format: Option<u32>,
484
485 last_render_instant: Option<Instant>,
487 last_render_dt_sec: Option<f32>,
488 render_profile: XrRenderProfile,
489
490 input_xr_components: HashSet<ComponentId>,
491 controller_components: HashSet<ComponentId>,
492 controller_pose_source_last_logged: HashMap<ComponentId, &'static str>,
493
494 xr_input_state: XrInputState,
495 xr_gamepad_state: XrGamepadState,
496 xr_gamepad_state_last_logged: XrGamepadState,
497}
498
499#[derive(Debug, Default)]
500struct XrRenderProfile {
501 frames: u64,
502 total: std::time::Duration,
503 wait_frame: std::time::Duration,
504 tracking_input: std::time::Duration,
505 acquire_wait: std::time::Duration,
506 eye_render: std::time::Duration,
507 copy: std::time::Duration,
508 submit: std::time::Duration,
509}
510
511struct OpenXRState {
512 #[allow(dead_code)]
513 entry: openxr::Entry,
514 #[allow(dead_code)]
515 instance: openxr::Instance,
516 #[allow(dead_code)]
517 system: openxr::SystemId,
518 events: openxr::EventDataBuffer,
519
520 session: Option<OpenXRSessionState>,
521 view_type: openxr::ViewConfigurationType,
522 blend_mode: openxr::EnvironmentBlendMode,
523}
524
525struct OpenXRSessionState {
526 session: openxr::Session<openxr::Vulkan>,
527 frame_waiter: openxr::FrameWaiter,
528 frame_stream: openxr::FrameStream<openxr::Vulkan>,
529 reference_space: openxr::Space,
530 running: bool,
531 current_state: openxr::SessionState,
532
533 xr_swapchain: XRSwapchain,
534
535 swapchain_image_initialized: Vec<bool>,
536
537 did_log_format_mismatch: bool,
538
539 vk_device: ash::Device,
540 vk_queue: ash::vk::Queue,
541
542 #[allow(dead_code)]
543 vk_command_pool: ash::vk::CommandPool,
544 vk_command_buffer: ash::vk::CommandBuffer,
545
546 hand_tracking: Option<HandTrackingState>,
547 hand_root_pose_cache: HandRootPoseCache,
548 hand_rotation_debug: HandRotationDebugState,
549 last_hand_debug_snapshot: Option<HandDebugSnapshot>,
550 head_pose_cache: Option<openxr::Posef>,
551 controller_input: Option<ControllerInput>,
552 controller_pose_cache: ControllerPoseCache,
553}
554
555#[derive(Debug, Default, Clone, Copy)]
556struct ControllerPoseCache {
557 left_aim: Option<openxr::Posef>,
558 right_aim: Option<openxr::Posef>,
559 left_grip: Option<openxr::Posef>,
560 right_grip: Option<openxr::Posef>,
561}
562
563struct HandTrackingState {
564 left: openxr::HandTracker,
565 right: openxr::HandTracker,
566}
567
568#[derive(Debug, Default, Clone, Copy)]
569struct HandRootPoseCache {
570 left_root: Option<openxr::Posef>,
571 right_root: Option<openxr::Posef>,
572 left_root_joint: Option<openxr::HandJointEXT>,
573 right_root_joint: Option<openxr::HandJointEXT>,
574}
575
576#[derive(Debug, Default)]
577struct HandRotationDebugState {
578 left: RollingAngleWindow,
579 right: RollingAngleWindow,
580}
581
582#[derive(Debug, Default)]
583struct RollingAngleWindow {
584 previous_quat_xyzw: Option<[f32; 4]>,
585 step_deg: VecDeque<f32>,
586 sample_count: u64,
587}
588
589#[derive(Debug, Clone, Copy, PartialEq, Eq)]
590struct HandDebugState {
591 root_joint: Option<openxr::HandJointEXT>,
592 wrist_palm_delta_deg: Option<u16>,
593 wrist_step_deg: Option<u16>,
594 wrist_step_spike: bool,
595}
596
597#[derive(Debug, Clone, Copy, PartialEq, Eq)]
598struct HandDebugSnapshot {
599 left: HandDebugState,
600 right: HandDebugState,
601}
602
603#[derive(Debug, Clone, PartialEq)]
604struct ControllerDebugSnapshot {
605 left_profile: String,
606 right_profile: String,
607 left_aim_valid: bool,
608 right_aim_valid: bool,
609 left_grip_valid: bool,
610 right_grip_valid: bool,
611 left_select: Option<(bool, bool)>,
612 right_select: Option<(bool, bool)>,
613 left_thumbstick: Option<(bool, [f32; 2])>,
614 right_thumbstick: Option<(bool, [f32; 2])>,
615 left_trigger_value: Option<(bool, f32)>,
616 right_trigger_value: Option<(bool, f32)>,
617 left_trigger_click: Option<(bool, bool)>,
618 right_trigger_click: Option<(bool, bool)>,
619 left_grip_value: Option<(bool, f32)>,
620 right_grip_value: Option<(bool, f32)>,
621 left_grip_click: Option<(bool, bool)>,
622 right_grip_click: Option<(bool, bool)>,
623 left_x: Option<(bool, bool)>,
624 left_y: Option<(bool, bool)>,
625 right_a: Option<(bool, bool)>,
626 right_b: Option<(bool, bool)>,
627}
628
629#[derive(Debug, Clone, Copy)]
630struct BindingSpec {
631 label: &'static str,
632 paths: &'static [&'static str],
633}
634
635#[derive(Debug, Clone, Copy)]
636struct ProfileBindingSpec {
637 profile: &'static str,
638 select_left: Option<&'static str>,
639 select_right: Option<&'static str>,
640 left_specs: &'static [BindingSpec],
641 right_specs: &'static [BindingSpec],
642}
643
644#[allow(dead_code)]
645struct ControllerInput {
646 action_set: openxr::ActionSet,
647 aim_pose: openxr::Action<openxr::Posef>,
648 grip_pose: openxr::Action<openxr::Posef>,
649 select: openxr::Action<bool>,
650 left_stick_x: openxr::Action<f32>,
651 left_stick_y: openxr::Action<f32>,
652 right_stick_x: openxr::Action<f32>,
653 right_stick_y: openxr::Action<f32>,
654 trigger_value: openxr::Action<f32>,
655 trigger_click: openxr::Action<bool>,
656 grip_value: openxr::Action<f32>,
657 grip_click: openxr::Action<bool>,
658 button_a: openxr::Action<bool>,
659 button_b: openxr::Action<bool>,
660 button_x: openxr::Action<bool>,
661 button_y: openxr::Action<bool>,
662
663 left: openxr::Path,
664 right: openxr::Path,
665
666 left_aim_space: openxr::Space,
667 right_aim_space: openxr::Space,
668 left_grip_space: openxr::Space,
669 right_grip_space: openxr::Space,
670
671 profile_poll_counter: u32,
672 last_logged_left_profile: Option<openxr::Path>,
673 last_logged_right_profile: Option<openxr::Path>,
674 last_reported_left_profile: Option<openxr::Path>,
675 last_reported_right_profile: Option<openxr::Path>,
676 last_debug_snapshot: Option<ControllerDebugSnapshot>,
677 did_log_bound_sources: bool,
678 binding_reports: Vec<ProfileBindingReport>,
679}
680
681const PROFILE_SIMPLE_CONTROLLER: &str = "/interaction_profiles/khr/simple_controller";
682const PROFILE_OCULUS_TOUCH: &str = "/interaction_profiles/oculus/touch_controller";
683const PROFILE_HTC_VIVE: &str = "/interaction_profiles/htc/vive_controller";
684const PROFILE_HTC_VIVE_FOCUS3: &str = "/interaction_profiles/htc/vive_focus3_controller";
685const PROFILE_VALVE_INDEX: &str = "/interaction_profiles/valve/index_controller";
686const PROFILE_MICROSOFT_MOTION: &str = "/interaction_profiles/microsoft/motion_controller";
687const PROFILE_EXT_HAND_INTERACTION: &str = "/interaction_profiles/ext/hand_interaction_ext";
688
689const SIMPLE_CONTROLLER_LEFT_SPECS: &[BindingSpec] = &[];
690const SIMPLE_CONTROLLER_RIGHT_SPECS: &[BindingSpec] = &[];
691
692const OCULUS_TOUCH_LEFT_SPECS: &[BindingSpec] = &[
693 BindingSpec {
694 label: "left_stick_x",
695 paths: &["/user/hand/left/input/thumbstick/x"],
696 },
697 BindingSpec {
698 label: "left_stick_y",
699 paths: &["/user/hand/left/input/thumbstick/y"],
700 },
701 BindingSpec {
702 label: "trigger_value",
703 paths: &["/user/hand/left/input/trigger/value"],
704 },
705 BindingSpec {
706 label: "grip_value",
707 paths: &["/user/hand/left/input/squeeze/value"],
708 },
709 BindingSpec {
710 label: "x",
711 paths: &["/user/hand/left/input/x/click"],
712 },
713 BindingSpec {
714 label: "y",
715 paths: &["/user/hand/left/input/y/click"],
716 },
717];
718const OCULUS_TOUCH_RIGHT_SPECS: &[BindingSpec] = &[
719 BindingSpec {
720 label: "right_stick_x",
721 paths: &["/user/hand/right/input/thumbstick/x"],
722 },
723 BindingSpec {
724 label: "right_stick_y",
725 paths: &["/user/hand/right/input/thumbstick/y"],
726 },
727 BindingSpec {
728 label: "trigger_value",
729 paths: &["/user/hand/right/input/trigger/value"],
730 },
731 BindingSpec {
732 label: "grip_value",
733 paths: &["/user/hand/right/input/squeeze/value"],
734 },
735 BindingSpec {
736 label: "a",
737 paths: &["/user/hand/right/input/a/click"],
738 },
739 BindingSpec {
740 label: "b",
741 paths: &["/user/hand/right/input/b/click"],
742 },
743];
744
745const VALVE_INDEX_LEFT_SPECS: &[BindingSpec] = &[
746 BindingSpec {
747 label: "left_stick_x",
748 paths: &[
749 "/user/hand/left/input/thumbstick/x",
750 "/user/hand/left/input/joystick/x",
751 ],
752 },
753 BindingSpec {
754 label: "left_stick_y",
755 paths: &[
756 "/user/hand/left/input/thumbstick/y",
757 "/user/hand/left/input/joystick/y",
758 ],
759 },
760 BindingSpec {
761 label: "trigger_value",
762 paths: &["/user/hand/left/input/trigger/value"],
763 },
764 BindingSpec {
765 label: "grip_value",
766 paths: &[
767 "/user/hand/left/input/squeeze/value",
768 "/user/hand/left/input/grip/value",
769 ],
770 },
771];
772const VALVE_INDEX_RIGHT_SPECS: &[BindingSpec] = &[
773 BindingSpec {
774 label: "right_stick_x",
775 paths: &[
776 "/user/hand/right/input/thumbstick/x",
777 "/user/hand/right/input/joystick/x",
778 ],
779 },
780 BindingSpec {
781 label: "right_stick_y",
782 paths: &[
783 "/user/hand/right/input/thumbstick/y",
784 "/user/hand/right/input/joystick/y",
785 ],
786 },
787 BindingSpec {
788 label: "trigger_value",
789 paths: &["/user/hand/right/input/trigger/value"],
790 },
791 BindingSpec {
792 label: "grip_value",
793 paths: &[
794 "/user/hand/right/input/squeeze/value",
795 "/user/hand/right/input/grip/value",
796 ],
797 },
798 BindingSpec {
799 label: "a",
800 paths: &["/user/hand/right/input/a/click"],
801 },
802 BindingSpec {
803 label: "b",
804 paths: &["/user/hand/right/input/b/click"],
805 },
806];
807
808const MICROSOFT_MOTION_LEFT_SPECS: &[BindingSpec] = &[
809 BindingSpec {
810 label: "left_stick_x",
811 paths: &[
812 "/user/hand/left/input/thumbstick/x",
813 "/user/hand/left/input/joystick/x",
814 ],
815 },
816 BindingSpec {
817 label: "left_stick_y",
818 paths: &[
819 "/user/hand/left/input/thumbstick/y",
820 "/user/hand/left/input/joystick/y",
821 ],
822 },
823 BindingSpec {
824 label: "trigger_value",
825 paths: &["/user/hand/left/input/trigger/value"],
826 },
827 BindingSpec {
828 label: "grip_click",
829 paths: &[
830 "/user/hand/left/input/squeeze/click",
831 "/user/hand/left/input/grip/click",
832 ],
833 },
834];
835const MICROSOFT_MOTION_RIGHT_SPECS: &[BindingSpec] = &[
836 BindingSpec {
837 label: "right_stick_x",
838 paths: &[
839 "/user/hand/right/input/thumbstick/x",
840 "/user/hand/right/input/joystick/x",
841 ],
842 },
843 BindingSpec {
844 label: "right_stick_y",
845 paths: &[
846 "/user/hand/right/input/thumbstick/y",
847 "/user/hand/right/input/joystick/y",
848 ],
849 },
850 BindingSpec {
851 label: "trigger_value",
852 paths: &["/user/hand/right/input/trigger/value"],
853 },
854 BindingSpec {
855 label: "grip_click",
856 paths: &[
857 "/user/hand/right/input/squeeze/click",
858 "/user/hand/right/input/grip/click",
859 ],
860 },
861];
862
863const HTC_VIVE_LEFT_SPECS: &[BindingSpec] = &[
864 BindingSpec {
865 label: "trigger_click",
866 paths: &["/user/hand/left/input/trigger/click"],
867 },
868 BindingSpec {
869 label: "grip_click",
870 paths: &[
871 "/user/hand/left/input/squeeze/click",
872 "/user/hand/left/input/grip/click",
873 ],
874 },
875];
876const HTC_VIVE_RIGHT_SPECS: &[BindingSpec] = &[
877 BindingSpec {
878 label: "trigger_click",
879 paths: &["/user/hand/right/input/trigger/click"],
880 },
881 BindingSpec {
882 label: "grip_click",
883 paths: &[
884 "/user/hand/right/input/squeeze/click",
885 "/user/hand/right/input/grip/click",
886 ],
887 },
888];
889
890const HTC_VIVE_FOCUS3_LEFT_SPECS: &[BindingSpec] = &[
891 BindingSpec {
892 label: "left_stick_x",
893 paths: &["/user/hand/left/input/thumbstick/x"],
894 },
895 BindingSpec {
896 label: "left_stick_y",
897 paths: &["/user/hand/left/input/thumbstick/y"],
898 },
899 BindingSpec {
900 label: "trigger_value",
901 paths: &["/user/hand/left/input/trigger/value"],
902 },
903 BindingSpec {
904 label: "grip_value",
905 paths: &["/user/hand/left/input/squeeze/value"],
906 },
907 BindingSpec {
908 label: "x",
909 paths: &["/user/hand/left/input/x/click"],
910 },
911 BindingSpec {
912 label: "y",
913 paths: &["/user/hand/left/input/y/click"],
914 },
915];
916const HTC_VIVE_FOCUS3_RIGHT_SPECS: &[BindingSpec] = &[
917 BindingSpec {
918 label: "right_stick_x",
919 paths: &["/user/hand/right/input/thumbstick/x"],
920 },
921 BindingSpec {
922 label: "right_stick_y",
923 paths: &["/user/hand/right/input/thumbstick/y"],
924 },
925 BindingSpec {
926 label: "trigger_value",
927 paths: &["/user/hand/right/input/trigger/value"],
928 },
929 BindingSpec {
930 label: "grip_value",
931 paths: &["/user/hand/right/input/squeeze/value"],
932 },
933 BindingSpec {
934 label: "a",
935 paths: &["/user/hand/right/input/a/click"],
936 },
937 BindingSpec {
938 label: "b",
939 paths: &["/user/hand/right/input/b/click"],
940 },
941];
942
943const EXT_HAND_INTERACTION_LEFT_SPECS: &[BindingSpec] = &[];
944const EXT_HAND_INTERACTION_RIGHT_SPECS: &[BindingSpec] = &[];
945
946const PROFILE_BINDING_SPECS: &[ProfileBindingSpec] = &[
947 ProfileBindingSpec {
948 profile: PROFILE_SIMPLE_CONTROLLER,
949 select_left: Some("/user/hand/left/input/select/click"),
950 select_right: Some("/user/hand/right/input/select/click"),
951 left_specs: SIMPLE_CONTROLLER_LEFT_SPECS,
952 right_specs: SIMPLE_CONTROLLER_RIGHT_SPECS,
953 },
954 ProfileBindingSpec {
955 profile: PROFILE_OCULUS_TOUCH,
956 select_left: None,
957 select_right: None,
958 left_specs: OCULUS_TOUCH_LEFT_SPECS,
959 right_specs: OCULUS_TOUCH_RIGHT_SPECS,
960 },
961 ProfileBindingSpec {
962 profile: PROFILE_HTC_VIVE,
963 select_left: Some("/user/hand/left/input/trigger/click"),
964 select_right: Some("/user/hand/right/input/trigger/click"),
965 left_specs: HTC_VIVE_LEFT_SPECS,
966 right_specs: HTC_VIVE_RIGHT_SPECS,
967 },
968 ProfileBindingSpec {
969 profile: PROFILE_HTC_VIVE_FOCUS3,
970 select_left: None,
971 select_right: None,
972 left_specs: HTC_VIVE_FOCUS3_LEFT_SPECS,
973 right_specs: HTC_VIVE_FOCUS3_RIGHT_SPECS,
974 },
975 ProfileBindingSpec {
976 profile: PROFILE_VALVE_INDEX,
977 select_left: Some("/user/hand/left/input/trigger/value"),
978 select_right: Some("/user/hand/right/input/trigger/value"),
979 left_specs: VALVE_INDEX_LEFT_SPECS,
980 right_specs: VALVE_INDEX_RIGHT_SPECS,
981 },
982 ProfileBindingSpec {
983 profile: PROFILE_MICROSOFT_MOTION,
984 select_left: Some("/user/hand/left/input/trigger/value"),
985 select_right: Some("/user/hand/right/input/trigger/value"),
986 left_specs: MICROSOFT_MOTION_LEFT_SPECS,
987 right_specs: MICROSOFT_MOTION_RIGHT_SPECS,
988 },
989 ProfileBindingSpec {
990 profile: PROFILE_EXT_HAND_INTERACTION,
991 select_left: Some("/user/hand/left/input/pinch_ext/value"),
992 select_right: Some("/user/hand/right/input/pinch_ext/value"),
993 left_specs: EXT_HAND_INTERACTION_LEFT_SPECS,
994 right_specs: EXT_HAND_INTERACTION_RIGHT_SPECS,
995 },
996];
997
998impl Default for OpenXRSystem {
999 fn default() -> Self {
1000 Self {
1001 state: None,
1002 last_init_error: None,
1003 vulkan_graphics: None,
1004 preferred_swapchain_format: None,
1005
1006 last_render_instant: None,
1007 last_render_dt_sec: None,
1008 render_profile: XrRenderProfile::default(),
1009
1010 input_xr_components: HashSet::new(),
1011 controller_components: HashSet::new(),
1012 controller_pose_source_last_logged: HashMap::new(),
1013
1014 xr_input_state: XrInputState::default(),
1015 xr_gamepad_state: XrGamepadState::default(),
1016 xr_gamepad_state_last_logged: XrGamepadState::default(),
1017 }
1018 }
1019}
1020
1021impl std::fmt::Debug for OpenXRSystem {
1022 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1023 f.debug_struct("OpenXRSystem")
1024 .field("initialized", &self.state.is_some())
1025 .field("last_init_error", &self.last_init_error)
1026 .field("last_render_dt_sec", &self.last_render_dt_sec)
1027 .finish()
1028 }
1029}
1030
1031impl OpenXRSystem {
1032 fn profile_systems_enabled() -> bool {
1033 static ENABLED: OnceLock<bool> = OnceLock::new();
1034 *ENABLED.get_or_init(|| {
1035 matches!(
1036 std::env::var("CAT_PROFILE_SYSTEMS")
1037 .unwrap_or_default()
1038 .trim()
1039 .to_ascii_lowercase()
1040 .as_str(),
1041 "1" | "true" | "yes" | "on"
1042 )
1043 })
1044 }
1045
1046 fn finish_xr_render_profile_frame(&mut self, frame_started: Instant) {
1047 self.render_profile.frames += 1;
1048 self.render_profile.total += frame_started.elapsed();
1049 if self.render_profile.frames < 120 {
1050 return;
1051 }
1052 let frames = self.render_profile.frames as f64;
1053 let avg_ms = |duration: std::time::Duration| duration.as_secs_f64() * 1000.0 / frames;
1054 let line = format!(
1055 "[xr-render-profile] frames={} avg_ms total={:.3} wait_frame={:.3} tracking_input={:.3} acquire_wait={:.3} eye_render={:.3} copy={:.3} submit={:.3}",
1056 self.render_profile.frames,
1057 avg_ms(self.render_profile.total),
1058 avg_ms(self.render_profile.wait_frame),
1059 avg_ms(self.render_profile.tracking_input),
1060 avg_ms(self.render_profile.acquire_wait),
1061 avg_ms(self.render_profile.eye_render),
1062 avg_ms(self.render_profile.copy),
1063 avg_ms(self.render_profile.submit),
1064 );
1065 eprintln!("{line}");
1066 crate::utils::profile_log::append("system-profile", &line);
1067 self.render_profile = XrRenderProfile::default();
1068 }
1069
1070 pub fn xr_input_state(&self) -> &XrInputState {
1071 &self.xr_input_state
1072 }
1073
1074 pub fn xr_gamepad_state(&self) -> &XrGamepadState {
1075 &self.xr_gamepad_state
1076 }
1077
1078 pub fn last_init_error(&self) -> Option<&str> {
1079 self.last_init_error.as_deref()
1080 }
1081
1082 pub fn prepare_for_renderer_init(&mut self, world: &World) {
1083 let should_enable = world.all_components().any(|cid| {
1084 world
1085 .get_component_by_id_as::<XrComponent>(cid)
1086 .map(|component| component.enabled)
1087 .unwrap_or(false)
1088 });
1089
1090 if should_enable {
1091 let _ = self.initialize_runtime();
1092 }
1093 }
1094
1095 fn debug_hand_rotation_enabled() -> bool {
1096 static ENABLED: OnceLock<bool> = OnceLock::new();
1097 *ENABLED.get_or_init(|| {
1098 std::env::var("CAT_DEBUG_XR_HAND_ROTATION")
1099 .ok()
1100 .map(|value| {
1101 let value = value.trim().to_ascii_lowercase();
1102 matches!(value.as_str(), "1" | "true" | "yes" | "on")
1103 })
1104 .unwrap_or(false)
1105 })
1106 }
1107
1108 fn debug_hand_rotation_window_len() -> usize {
1109 static WINDOW_LEN: OnceLock<usize> = OnceLock::new();
1110 *WINDOW_LEN.get_or_init(|| {
1111 std::env::var("CAT_DEBUG_XR_HAND_ROTATION_WINDOW")
1112 .ok()
1113 .and_then(|value| value.parse::<usize>().ok())
1114 .map(|value| value.clamp(1, 600))
1115 .unwrap_or(60)
1116 })
1117 }
1118
1119 fn quat_from_posef(pose: openxr::Posef) -> [f32; 4] {
1120 [
1121 pose.orientation.x,
1122 pose.orientation.y,
1123 pose.orientation.z,
1124 pose.orientation.w,
1125 ]
1126 }
1127
1128 fn quat_normalize(q: [f32; 4]) -> [f32; 4] {
1129 let len = (q[0] * q[0] + q[1] * q[1] + q[2] * q[2] + q[3] * q[3]).sqrt();
1130 if len > 0.0 {
1131 [q[0] / len, q[1] / len, q[2] / len, q[3] / len]
1132 } else {
1133 [0.0, 0.0, 0.0, 1.0]
1134 }
1135 }
1136
1137 fn quat_nlerp(a: [f32; 4], b: [f32; 4], t: f32) -> [f32; 4] {
1138 let mut end = b;
1139 let dot = a[0] * b[0] + a[1] * b[1] + a[2] * b[2] + a[3] * b[3];
1140 if dot < 0.0 {
1141 end = [-b[0], -b[1], -b[2], -b[3]];
1142 }
1143 let one_minus_t = 1.0 - t;
1144 Self::quat_normalize([
1145 a[0] * one_minus_t + end[0] * t,
1146 a[1] * one_minus_t + end[1] * t,
1147 a[2] * one_minus_t + end[2] * t,
1148 a[3] * one_minus_t + end[3] * t,
1149 ])
1150 }
1151
1152 fn pose_from_quat_translation(
1153 rotation_xyzw: [f32; 4],
1154 translation_xyz: [f32; 3],
1155 ) -> openxr::Posef {
1156 openxr::Posef {
1157 orientation: openxr::Quaternionf {
1158 x: rotation_xyzw[0],
1159 y: rotation_xyzw[1],
1160 z: rotation_xyzw[2],
1161 w: rotation_xyzw[3],
1162 },
1163 position: openxr::Vector3f {
1164 x: translation_xyz[0],
1165 y: translation_xyz[1],
1166 z: translation_xyz[2],
1167 },
1168 }
1169 }
1170
1171 fn derive_head_pose(views: &[openxr::View]) -> Option<openxr::Posef> {
1172 match views {
1173 [] => None,
1174 [view] => Some(view.pose),
1175 [left, right, ..] => {
1176 let left_q = Self::quat_from_posef(left.pose);
1177 let right_q = Self::quat_from_posef(right.pose);
1178 let center_q = Self::quat_nlerp(left_q, right_q, 0.5);
1179 let center_t = [
1180 0.5 * (left.pose.position.x + right.pose.position.x),
1181 0.5 * (left.pose.position.y + right.pose.position.y),
1182 0.5 * (left.pose.position.z + right.pose.position.z),
1183 ];
1184 Some(Self::pose_from_quat_translation(center_q, center_t))
1185 }
1186 }
1187 }
1188
1189 fn quat_angle_degrees(a: [f32; 4], b: [f32; 4]) -> f32 {
1190 let a = Self::quat_normalize(a);
1191 let b = Self::quat_normalize(b);
1192 let dot = (a[0] * b[0] + a[1] * b[1] + a[2] * b[2] + a[3] * b[3])
1193 .abs()
1194 .clamp(0.0, 1.0);
1195 (2.0 * dot.acos()).to_degrees()
1196 }
1197
1198 fn rolling_avg(window: &VecDeque<f32>) -> f32 {
1199 if window.is_empty() {
1200 0.0
1201 } else {
1202 window.iter().copied().sum::<f32>() / window.len() as f32
1203 }
1204 }
1205
1206 fn rolling_max(window: &VecDeque<f32>) -> f32 {
1207 window.iter().copied().fold(0.0, f32::max)
1208 }
1209
1210 fn update_hand_rotation_debug(
1211 debug_state: &mut HandRotationDebugState,
1212 hand: ControllerHand,
1213 _joint: Option<openxr::HandJointEXT>,
1214 pose: openxr::Posef,
1215 ) {
1216 if !Self::debug_hand_rotation_enabled() {
1217 return;
1218 }
1219
1220 let quat = Self::quat_from_posef(pose);
1221 let window_len = Self::debug_hand_rotation_window_len();
1222 let debug = match hand {
1223 ControllerHand::Left => &mut debug_state.left,
1224 ControllerHand::Right => &mut debug_state.right,
1225 };
1226
1227 let step_deg = debug
1228 .previous_quat_xyzw
1229 .map(|previous| Self::quat_angle_degrees(previous, quat))
1230 .unwrap_or(0.0);
1231 debug.previous_quat_xyzw = Some(quat);
1232
1233 if debug.step_deg.len() >= window_len {
1234 let _ = debug.step_deg.pop_front();
1235 }
1236 debug.step_deg.push_back(step_deg);
1237 debug.sample_count += 1;
1238
1239 let _ = window_len;
1240 }
1241
1242 fn preferred_pose(
1243 sess: &OpenXRSessionState,
1244 hand: ControllerHand,
1245 pose: ControllerPoseKind,
1246 ) -> (Option<openxr::Posef>, &'static str) {
1247 let hand_root = match hand {
1248 ControllerHand::Left => sess.hand_root_pose_cache.left_root,
1249 ControllerHand::Right => sess.hand_root_pose_cache.right_root,
1250 };
1251 if let Some(pose) = hand_root {
1252 return (Some(pose), "hand_root");
1253 }
1254
1255 let controller_pose = match (hand, pose) {
1256 (ControllerHand::Left, ControllerPoseKind::Aim) => sess.controller_pose_cache.left_aim,
1257 (ControllerHand::Right, ControllerPoseKind::Aim) => {
1258 sess.controller_pose_cache.right_aim
1259 }
1260 (ControllerHand::Left, ControllerPoseKind::Grip) => {
1261 sess.controller_pose_cache.left_grip
1262 }
1263 (ControllerHand::Right, ControllerPoseKind::Grip) => {
1264 sess.controller_pose_cache.right_grip
1265 }
1266 };
1267
1268 if controller_pose.is_some() {
1269 (controller_pose, "controller_action")
1270 } else {
1271 (None, "none")
1272 }
1273 }
1274
1275 fn transform_child_of(world: &World, component: ComponentId) -> Option<ComponentId> {
1276 world.children_of(component).iter().copied().find(|&cid| {
1277 world
1278 .get_component_by_id_as::<crate::engine::ecs::component::TransformComponent>(cid)
1279 .is_some()
1280 })
1281 }
1282
1283 fn transform_parent_world(world: &World, transform_cid: ComponentId) -> [[f32; 4]; 4] {
1284 world
1285 .parent_of(transform_cid)
1286 .and_then(|parent| TransformSystem::world_model(world, parent))
1287 .unwrap_or([
1288 [1.0, 0.0, 0.0, 0.0],
1289 [0.0, 1.0, 0.0, 0.0],
1290 [0.0, 0.0, 1.0, 0.0],
1291 [0.0, 0.0, 0.0, 1.0],
1292 ])
1293 }
1294
1295 fn input_xr_ancestor(world: &World, cid: ComponentId) -> Option<ComponentId> {
1296 let mut cur = cid;
1297 loop {
1298 if world
1299 .get_component_by_id_as::<InputXRComponent>(cur)
1300 .is_some()
1301 {
1302 return Some(cur);
1303 }
1304 let Some(parent) = world.parent_of(cur) else {
1305 return None;
1306 };
1307 cur = parent;
1308 }
1309 }
1310
1311 fn xr_rig_origin_world(world: &World, visuals: &VisualWorld) -> [[f32; 4]; 4] {
1312 let Some(camera_cid) = visuals
1313 .active_xr_camera()
1314 .or_else(|| Self::first_enabled_camera_xr(world))
1315 else {
1316 return [
1317 [1.0, 0.0, 0.0, 0.0],
1318 [0.0, 1.0, 0.0, 0.0],
1319 [0.0, 0.0, 1.0, 0.0],
1320 [0.0, 0.0, 0.0, 1.0],
1321 ];
1322 };
1323
1324 if let Some(input_xr_cid) = Self::input_xr_ancestor(world, camera_cid) {
1325 if let Some(driven_transform) = Self::transform_child_of(world, input_xr_cid) {
1326 return Self::transform_parent_world(world, driven_transform);
1327 }
1328 }
1329
1330 TransformSystem::world_model(world, camera_cid).unwrap_or([
1331 [1.0, 0.0, 0.0, 0.0],
1332 [0.0, 1.0, 0.0, 0.0],
1333 [0.0, 0.0, 1.0, 0.0],
1334 [0.0, 0.0, 0.0, 1.0],
1335 ])
1336 }
1337
1338 pub fn set_preferred_swapchain_format(&mut self, format: u32) {
1343 self.preferred_swapchain_format = Some(format);
1344 }
1345
1346 pub fn required_vulkan_extensions(&self) -> Option<(Vec<String>, Vec<String>)> {
1352 let state = self.state.as_ref()?;
1353
1354 let instance_exts = state
1355 .instance
1356 .vulkan_legacy_instance_extensions(state.system)
1357 .ok()?;
1358 let device_exts = state
1359 .instance
1360 .vulkan_legacy_device_extensions(state.system)
1361 .ok()?;
1362
1363 let split = |s: String| -> Vec<String> {
1364 s.split_whitespace()
1365 .map(|v| v.trim().to_string())
1366 .filter(|v| !v.is_empty())
1367 .collect()
1368 };
1369
1370 Some((split(instance_exts), split(device_exts)))
1371 }
1372
1373 pub fn set_vulkan_graphics(&mut self, gfx: XrVulkanGraphics) {
1374 self.vulkan_graphics = Some(gfx);
1375
1376 let Some(state) = self.state.as_mut() else {
1378 return;
1379 };
1380
1381 if state.session.is_none() {
1382 if let Err(err) = Self::try_init_session(state, gfx, self.preferred_swapchain_format) {
1383 eprintln!("[OpenXR] Session init failed: {err}");
1384 self.last_init_error = Some(err);
1385 }
1386 }
1387 }
1388
1389 pub fn initialize_runtime(&mut self) -> Result<(), String> {
1390 if self.state.is_some() {
1391 return Ok(());
1392 }
1393
1394 match Self::try_init_openxr() {
1395 Ok(state) => {
1396 println!("[OpenXR] Initialized.");
1397 self.state = Some(state);
1398 self.last_init_error = None;
1399
1400 if let (Some(state), Some(gfx)) = (self.state.as_mut(), self.vulkan_graphics) {
1401 if state.session.is_none() {
1402 if let Err(err) =
1403 Self::try_init_session(state, gfx, self.preferred_swapchain_format)
1404 {
1405 eprintln!("[OpenXR] Session init failed: {err}");
1406 self.last_init_error = Some(err.clone());
1407 return Err(err);
1408 }
1409 }
1410 }
1411
1412 Ok(())
1413 }
1414 Err(err) => {
1415 eprintln!("[OpenXR] Init failed: {err}");
1416 self.last_init_error = Some(err.clone());
1417 Err(err)
1418 }
1419 }
1420 }
1421
1422 pub fn retry_runtime(&mut self) -> Result<(), String> {
1423 if let Some(state) = self.state.as_mut() {
1424 if state.session.is_some() {
1425 return Ok(());
1426 }
1427 let gfx = self
1428 .vulkan_graphics
1429 .ok_or_else(|| "Vulkan graphics are not initialized yet".to_string())?;
1430 return Self::try_init_session(state, gfx, self.preferred_swapchain_format).map_err(
1431 |error| {
1432 self.last_init_error = Some(error.clone());
1433 error
1434 },
1435 );
1436 }
1437 self.initialize_runtime()
1438 }
1439
1440 pub fn register_xr(
1441 &mut self,
1442 world: &mut World,
1443 _visuals: &mut VisualWorld,
1444 component: ComponentId,
1445 ) {
1446 let Some(cfg) = world.get_component_by_id_as::<XrComponent>(component) else {
1447 return;
1448 };
1449
1450 if !cfg.enabled {
1451 return;
1452 }
1453
1454 if self.state.is_some() {
1455 return;
1456 }
1457
1458 let _ = self.initialize_runtime();
1459 }
1460
1461 pub fn register_controller_xr(
1462 &mut self,
1463 _world: &mut World,
1464 _visuals: &mut VisualWorld,
1465 component: ComponentId,
1466 ) {
1467 self.controller_components.insert(component);
1468 self.controller_pose_source_last_logged.remove(&component);
1469 }
1470
1471 pub fn register_input_xr(
1472 &mut self,
1473 _world: &mut World,
1474 _visuals: &mut VisualWorld,
1475 component: ComponentId,
1476 ) {
1477 self.input_xr_components.insert(component);
1478 }
1479
1480 pub fn remove_controller_xr(
1481 &mut self,
1482 _world: &mut World,
1483 _visuals: &mut VisualWorld,
1484 component: ComponentId,
1485 ) {
1486 self.controller_components.remove(&component);
1487 self.controller_pose_source_last_logged.remove(&component);
1488 }
1489
1490 pub fn remove_input_xr(
1491 &mut self,
1492 _world: &mut World,
1493 _visuals: &mut VisualWorld,
1494 component: ComponentId,
1495 ) {
1496 self.input_xr_components.remove(&component);
1497 }
1498
1499 fn pump_events(&mut self) {
1500 let Some(state) = self.state.as_mut() else {
1501 return;
1502 };
1503
1504 loop {
1506 let evt = match state.instance.poll_event(&mut state.events) {
1507 Ok(v) => v,
1508 Err(e) => {
1509 eprintln!("[OpenXR] poll_event error: {e:?}");
1510 return;
1511 }
1512 };
1513
1514 let Some(evt) = evt else {
1515 break;
1516 };
1517
1518 match evt {
1519 openxr::Event::InstanceLossPending(_) => {
1520 eprintln!("[OpenXR] Event: InstanceLossPending");
1521 }
1522 openxr::Event::SessionStateChanged(e) => {
1523 println!("[OpenXR] Event: SessionStateChanged -> {:?}", e.state());
1524
1525 if let Some(sess) = state.session.as_mut() {
1526 sess.current_state = e.state();
1527 match e.state() {
1528 openxr::SessionState::READY => {
1529 if !sess.running {
1530 if let Err(err) = sess.session.begin(state.view_type) {
1531 eprintln!("[OpenXR] session.begin failed: {err:?}");
1532 } else {
1533 sess.running = true;
1534 }
1535 }
1536 }
1537 openxr::SessionState::STOPPING => {
1538 if sess.running {
1539 if let Err(err) = sess.session.end() {
1540 eprintln!("[OpenXR] session.end failed: {err:?}");
1541 }
1542 sess.running = false;
1543 }
1544 }
1545 openxr::SessionState::EXITING | openxr::SessionState::LOSS_PENDING => {
1546 sess.running = false;
1547 }
1548 _ => {}
1549 }
1550 }
1551 }
1552 openxr::Event::EventsLost(e) => {
1553 eprintln!("[OpenXR] Event: EventsLost ({})", e.lost_event_count());
1554 }
1555 _ => {
1556 }
1558 }
1559 }
1560 }
1561
1562 pub fn tick_with_queue(
1566 &mut self,
1567 world: &mut World,
1568 visuals: &mut VisualWorld,
1569 _input: &InputState,
1570 emit: &mut dyn SignalEmitter,
1571 _dt_sec: f32,
1572 ) {
1573 self.pump_events();
1574
1575 for component in self.input_xr_components.iter().copied().collect::<Vec<_>>() {
1577 if let Some(input) = world.get_component_by_id_as_mut::<InputXRComponent>(component) {
1578 input.pose_valid = false;
1579 }
1580 }
1581 for component in self
1582 .controller_components
1583 .iter()
1584 .copied()
1585 .collect::<Vec<_>>()
1586 {
1587 if let Some(controller) = world.get_component_by_id_as_mut::<XRHandComponent>(component)
1588 {
1589 controller.pose_valid = false;
1590 }
1591 }
1592
1593 let Some(state) = self.state.as_ref() else {
1594 return;
1595 };
1596 let Some(sess) = state.session.as_ref() else {
1597 return;
1598 };
1599 if !sess.running {
1600 return;
1601 }
1602 if !matches!(
1603 sess.current_state,
1604 openxr::SessionState::VISIBLE | openxr::SessionState::FOCUSED
1605 ) {
1606 return;
1607 }
1608
1609 let rig_world = Self::xr_rig_origin_world(world, visuals);
1611
1612 let input_xr_ids: Vec<ComponentId> = self.input_xr_components.iter().copied().collect();
1613 for input_xr_cid in input_xr_ids {
1614 let Some(cfg) = world.get_component_by_id_as::<InputXRComponent>(input_xr_cid) else {
1615 self.input_xr_components.remove(&input_xr_cid);
1616 continue;
1617 };
1618
1619 if !cfg.enabled {
1620 continue;
1621 }
1622
1623 let Some(head_pose) = sess.head_pose_cache else {
1624 continue;
1625 };
1626
1627 let Some(tcid) = Self::transform_child_of(world, input_xr_cid) else {
1628 continue;
1629 };
1630
1631 let world_from_head = Self::mul_mat4(
1632 &Self::transform_parent_world(world, tcid),
1633 &Self::mat4_from_pose(head_pose),
1634 );
1635 let desired_world_pos = [
1636 world_from_head[3][0],
1637 world_from_head[3][1],
1638 world_from_head[3][2],
1639 ];
1640 let desired_world_rot = math::mat_to_quat(world_from_head);
1641
1642 let local_translation =
1643 Self::world_to_local_translation(world, tcid, desired_world_pos);
1644 let parent_world_rot =
1645 Self::parent_world_rotation_quat(world, tcid).unwrap_or([0.0, 0.0, 0.0, 1.0]);
1646 let local_rotation =
1647 math::quat_mul(math::quat_conjugate(parent_world_rot), desired_world_rot);
1648
1649 let Some(t) = world
1650 .get_component_by_id_as_mut::<crate::engine::ecs::component::TransformComponent>(
1651 tcid,
1652 )
1653 else {
1654 continue;
1655 };
1656
1657 t.transform.translation = local_translation;
1658 t.transform.rotation = local_rotation;
1659 t.transform.recompute_model();
1660
1661 let transform = t.transform;
1662 emit.push_intent_now(
1663 tcid,
1664 IntentValue::UpdateTransform {
1665 component_ids: vec![tcid],
1666 translation: transform.translation,
1667 rotation_quat_xyzw: transform.rotation,
1668 scale: transform.scale,
1669 },
1670 );
1671 if let Some(input) = world.get_component_by_id_as_mut::<InputXRComponent>(input_xr_cid)
1672 {
1673 input.pose_valid = true;
1674 }
1675 }
1676
1677 let controller_ids: Vec<ComponentId> = self.controller_components.iter().copied().collect();
1678 for controller_cid in controller_ids {
1679 let Some(cfg) = world.get_component_by_id_as::<XRHandComponent>(controller_cid) else {
1680 self.controller_components.remove(&controller_cid);
1681 continue;
1682 };
1683
1684 if !cfg.enabled {
1685 continue;
1686 }
1687
1688 let (pose, pose_source) = Self::preferred_pose(sess, cfg.hand, cfg.pose);
1689 let last_pose_source = self
1690 .controller_pose_source_last_logged
1691 .get(&controller_cid)
1692 .copied();
1693 if openxr_debug_enabled() && last_pose_source != Some(pose_source) {
1694 eprintln!(
1695 "[OpenXR][ctlxr] component={controller_cid:?} hand={:?} pose={:?} source={pose_source}",
1696 cfg.hand, cfg.pose
1697 );
1698 self.controller_pose_source_last_logged
1699 .insert(controller_cid, pose_source);
1700 }
1701
1702 let Some(pose) = pose else {
1703 continue;
1704 };
1705
1706 let Some(tcid) = Self::transform_child_of(world, controller_cid) else {
1708 continue;
1709 };
1710
1711 let world_from_controller = Self::mul_mat4(&rig_world, &Self::mat4_from_pose(pose));
1713 let desired_world_pos = [
1714 world_from_controller[3][0],
1715 world_from_controller[3][1],
1716 world_from_controller[3][2],
1717 ];
1718 let desired_world_rot = math::mat_to_quat(world_from_controller);
1719
1720 let local_translation =
1721 Self::world_to_local_translation(world, tcid, desired_world_pos);
1722 let parent_world_rot =
1723 Self::parent_world_rotation_quat(world, tcid).unwrap_or([0.0, 0.0, 0.0, 1.0]);
1724 let local_rotation =
1725 math::quat_mul(math::quat_conjugate(parent_world_rot), desired_world_rot);
1726
1727 let Some(t) = world
1728 .get_component_by_id_as_mut::<crate::engine::ecs::component::TransformComponent>(
1729 tcid,
1730 )
1731 else {
1732 continue;
1733 };
1734
1735 t.transform.translation = local_translation;
1738 t.transform.rotation = local_rotation;
1739 t.transform.recompute_model();
1740
1741 let transform = t.transform;
1742 emit.push_intent_now(
1743 tcid,
1744 IntentValue::UpdateTransform {
1745 component_ids: vec![tcid],
1746 translation: transform.translation,
1747 rotation_quat_xyzw: transform.rotation,
1748 scale: transform.scale,
1749 },
1750 );
1751 if let Some(controller) =
1752 world.get_component_by_id_as_mut::<XRHandComponent>(controller_cid)
1753 {
1754 controller.pose_valid = true;
1755 }
1756 }
1757 }
1758
1759 fn world_to_local_translation(
1760 world: &World,
1761 transform_cid: ComponentId,
1762 desired_world: [f32; 3],
1763 ) -> [f32; 3] {
1764 let mut cur = transform_cid;
1765 while let Some(parent) = world.parent_of(cur) {
1766 if let Some(t) = world
1767 .get_component_by_id_as::<crate::engine::ecs::component::TransformComponent>(parent)
1768 {
1769 if let Some(inv) = math::mat4_inverse(t.transform.matrix_world) {
1770 let p_local = math::mat4_mul_vec4(
1771 inv,
1772 [desired_world[0], desired_world[1], desired_world[2], 1.0],
1773 );
1774 return [p_local[0], p_local[1], p_local[2]];
1775 }
1776 break;
1777 }
1778 cur = parent;
1779 }
1780
1781 desired_world
1782 }
1783
1784 fn parent_world_rotation_quat(world: &World, transform_cid: ComponentId) -> Option<[f32; 4]> {
1785 let mut cur = transform_cid;
1786 while let Some(parent) = world.parent_of(cur) {
1787 if let Some(t) = world
1788 .get_component_by_id_as::<crate::engine::ecs::component::TransformComponent>(parent)
1789 {
1790 return Some(math::mat_to_quat(t.transform.matrix_world));
1791 }
1792 cur = parent;
1793 }
1794 None
1795 }
1796
1797 fn try_init_hand_tracking(
1798 session: &openxr::Session<openxr::Vulkan>,
1799 ) -> Result<HandTrackingState, String> {
1800 let left = session
1801 .create_hand_tracker(openxr::HandEXT::LEFT)
1802 .map_err(|e| format!("create_hand_tracker(left): {e:?}"))?;
1803 let right = session
1804 .create_hand_tracker(openxr::HandEXT::RIGHT)
1805 .map_err(|e| format!("create_hand_tracker(right): {e:?}"))?;
1806
1807 Ok(HandTrackingState { left, right })
1808 }
1809
1810 fn valid_pose_from_hand_joint(joint: &openxr::HandJointLocationEXT) -> Option<openxr::Posef> {
1811 let flags = joint.location_flags;
1812 if flags.contains(openxr::SpaceLocationFlags::POSITION_VALID)
1813 && flags.contains(openxr::SpaceLocationFlags::ORIENTATION_VALID)
1814 {
1815 Some(joint.pose)
1816 } else {
1817 None
1818 }
1819 }
1820
1821 fn select_hand_root_pose(
1822 joints: &openxr::HandJointLocations,
1823 ) -> Option<(openxr::Posef, openxr::HandJointEXT)> {
1824 let wrist = &joints[openxr::HandJointEXT::WRIST];
1825 if let Some(pose) = Self::valid_pose_from_hand_joint(wrist) {
1826 return Some((pose, openxr::HandJointEXT::WRIST));
1827 }
1828
1829 let palm = &joints[openxr::HandJointEXT::PALM];
1830 Self::valid_pose_from_hand_joint(palm).map(|pose| (pose, openxr::HandJointEXT::PALM))
1831 }
1832
1833 fn wrist_palm_delta_deg(joints: &openxr::HandJointLocations) -> Option<u16> {
1834 let wrist = Self::valid_pose_from_hand_joint(&joints[openxr::HandJointEXT::WRIST])?;
1835 let palm = Self::valid_pose_from_hand_joint(&joints[openxr::HandJointEXT::PALM])?;
1836 let deg =
1837 Self::quat_angle_degrees(Self::quat_from_posef(wrist), Self::quat_from_posef(palm));
1838 Some(deg.round().clamp(0.0, u16::MAX as f32) as u16)
1839 }
1840
1841 fn quantize_angle_deg(value: u16, bucket_size: u16) -> u16 {
1842 if bucket_size <= 1 {
1843 value
1844 } else {
1845 (value / bucket_size) * bucket_size
1846 }
1847 }
1848
1849 fn root_pose_for_joint(
1850 joints: &openxr::HandJointLocations,
1851 joint: Option<openxr::HandJointEXT>,
1852 ) -> Option<openxr::Posef> {
1853 let joint = joint?;
1854 Self::valid_pose_from_hand_joint(&joints[joint])
1855 }
1856
1857 fn hand_debug_state(
1858 joints: Option<&openxr::HandJointLocations>,
1859 root_joint: Option<openxr::HandJointEXT>,
1860 previous_root_quat_xyzw: Option<[f32; 4]>,
1861 ) -> HandDebugState {
1862 let wrist_palm_delta_deg = joints
1863 .and_then(Self::wrist_palm_delta_deg)
1864 .map(|deg| Self::quantize_angle_deg(deg, 5));
1865 let root_pose = joints.and_then(|j| Self::root_pose_for_joint(j, root_joint));
1866 let wrist_step_deg = match (previous_root_quat_xyzw, root_pose) {
1867 (Some(previous), Some(pose)) => Some(
1868 Self::quat_angle_degrees(previous, Self::quat_from_posef(pose))
1869 .round()
1870 .clamp(0.0, u16::MAX as f32) as u16,
1871 ),
1872 _ => None,
1873 }
1874 .map(|deg| Self::quantize_angle_deg(deg, 2));
1875 HandDebugState {
1876 root_joint,
1877 wrist_palm_delta_deg,
1878 wrist_step_deg,
1879 wrist_step_spike: wrist_step_deg.is_some_and(|deg| deg >= 8),
1880 }
1881 }
1882
1883 fn log_hand_debug_snapshot(
1884 sess: &mut OpenXRSessionState,
1885 left_joints: Option<&openxr::HandJointLocations>,
1886 right_joints: Option<&openxr::HandJointLocations>,
1887 left_root_joint: Option<openxr::HandJointEXT>,
1888 right_root_joint: Option<openxr::HandJointEXT>,
1889 ) {
1890 let snapshot = HandDebugSnapshot {
1891 left: Self::hand_debug_state(
1892 left_joints,
1893 left_root_joint,
1894 sess.hand_rotation_debug.left.previous_quat_xyzw,
1895 ),
1896 right: Self::hand_debug_state(
1897 right_joints,
1898 right_root_joint,
1899 sess.hand_rotation_debug.right.previous_quat_xyzw,
1900 ),
1901 };
1902 if sess.last_hand_debug_snapshot == Some(snapshot) {
1903 return;
1904 }
1905 sess.last_hand_debug_snapshot = Some(snapshot);
1906 if openxr_debug_enabled() {
1907 eprintln!(
1908 "[OpenXR][hand] left root={:?} wrist_palm_delta_deg={:?} wrist_step_deg={:?} spike={} | right root={:?} wrist_palm_delta_deg={:?} wrist_step_deg={:?} spike={}",
1909 snapshot.left.root_joint,
1910 snapshot.left.wrist_palm_delta_deg,
1911 snapshot.left.wrist_step_deg,
1912 snapshot.left.wrist_step_spike,
1913 snapshot.right.root_joint,
1914 snapshot.right.wrist_palm_delta_deg,
1915 snapshot.right.wrist_step_deg,
1916 snapshot.right.wrist_step_spike,
1917 );
1918 }
1919 }
1920
1921 fn try_init_openxr() -> Result<OpenXRState, String> {
1922 let entry = unsafe { openxr::Entry::load().map_err(|e| format!("Entry::load: {e:?}"))? };
1925
1926 let available_extensions = entry
1927 .enumerate_extensions()
1928 .map_err(|e| format!("enumerate_extensions: {e:?}"))?;
1929
1930 let app_info = openxr::ApplicationInfo {
1931 application_name: "cat-engine",
1932 application_version: 1,
1933 engine_name: "cat-engine",
1934 engine_version: 1,
1935 api_version: openxr::Version::new(1, 0, 0),
1936 };
1937
1938 let mut extensions = openxr::ExtensionSet::default();
1939 extensions.khr_vulkan_enable2 = false;
1943 extensions.khr_vulkan_enable = true;
1945 if available_extensions.ext_hand_tracking {
1946 extensions.ext_hand_tracking = true;
1947 }
1948 if available_extensions.ext_hand_interaction {
1949 extensions.ext_hand_interaction = true;
1950 }
1951 if available_extensions.htc_vive_focus3_controller_interaction {
1952 extensions.htc_vive_focus3_controller_interaction = true;
1953 }
1954 if available_extensions.htc_hand_interaction {
1955 extensions.htc_hand_interaction = true;
1956 }
1957 let layers: [&str; 0] = [];
1958
1959 let instance = entry
1960 .create_instance(&app_info, &extensions, &layers)
1961 .map_err(|e| format!("create_instance: {e:?}"))?;
1962
1963 println!(
1964 "[OpenXR] Enabled extensions: khr_vulkan_enable={}, ext_hand_tracking={}, ext_hand_interaction={}, htc_vive_focus3_controller_interaction={}, htc_hand_interaction={}",
1965 extensions.khr_vulkan_enable,
1966 extensions.ext_hand_tracking,
1967 extensions.ext_hand_interaction,
1968 extensions.htc_vive_focus3_controller_interaction,
1969 extensions.htc_hand_interaction,
1970 );
1971
1972 if let Ok(props) = instance.properties() {
1974 println!(
1975 "[OpenXR] Runtime: {} ({:?})",
1976 props.runtime_name, props.runtime_version
1977 );
1978 }
1979
1980 let system = match instance.system(openxr::FormFactor::HEAD_MOUNTED_DISPLAY) {
1981 Ok(system) => system,
1982 Err(openxr::sys::Result::ERROR_FORM_FACTOR_UNAVAILABLE) => {
1983 return Err(
1984 "system(HMD): ERROR_FORM_FACTOR_UNAVAILABLE (no HMD detected / runtime not ready).\n\
1985 Start an OpenXR runtime (e.g. SteamVR/Monado/ALVR) and ensure a headset is connected and the runtime is running before launching cat-engine.\n\
1986 On Linux you can also check XR_RUNTIME_JSON points to an installed runtime manifest."
1987 .to_string(),
1988 );
1989 }
1990 Err(e) => return Err(format!("system(HMD): {e:?}")),
1991 };
1992
1993 let view_type = openxr::ViewConfigurationType::PRIMARY_STEREO;
1995 let blend_mode = instance
1996 .enumerate_environment_blend_modes(system, view_type)
1997 .ok()
1998 .and_then(|m| m.first().copied())
1999 .unwrap_or(openxr::EnvironmentBlendMode::OPAQUE);
2000
2001 Ok(OpenXRState {
2002 entry,
2003 instance,
2004 system,
2005 events: openxr::EventDataBuffer::new(),
2006 session: None,
2007 view_type,
2008 blend_mode,
2009 })
2010 }
2011
2012 fn try_init_session(
2013 state: &mut OpenXRState,
2014 gfx: XrVulkanGraphics,
2015 preferred_swapchain_format: Option<u32>,
2016 ) -> Result<(), String> {
2017 if let Ok(reqs) = state
2019 .instance
2020 .graphics_requirements::<openxr::Vulkan>(state.system)
2021 {
2022 println!(
2023 "[OpenXR] Vulkan requirements: min {:?}, max {:?}",
2024 reqs.min_api_version_supported, reqs.max_api_version_supported
2025 );
2026 }
2027
2028 if let Ok(required_pd) = unsafe {
2031 state
2032 .instance
2033 .vulkan_graphics_device(state.system, gfx.vk_instance)
2034 } {
2035 if required_pd != gfx.vk_physical_device {
2036 return Err(format!(
2037 "Vulkan physical device mismatch for OpenXR system.\n\
2038Engine/Vulkano physical device: {:?}\n\
2039OpenXR-required physical device: {:?}\n\
2040Fix: pick the OpenXR-required VkPhysicalDevice when creating the Vulkan device/queues.",
2041 gfx.vk_physical_device, required_pd
2042 ));
2043 }
2044 }
2045
2046 let info = openxr::vulkan::SessionCreateInfo {
2047 instance: gfx.vk_instance,
2048 physical_device: gfx.vk_physical_device,
2049 device: gfx.vk_device,
2050 queue_family_index: gfx.queue_family_index,
2051 queue_index: gfx.queue_index,
2052 };
2053
2054 let (session, frame_waiter, frame_stream) = unsafe {
2055 state
2056 .instance
2057 .create_session::<openxr::Vulkan>(state.system, &info)
2058 .map_err(|e| {
2059 let required_instance = state
2060 .instance
2061 .vulkan_legacy_instance_extensions(state.system)
2062 .ok();
2063 let required_device = state
2064 .instance
2065 .vulkan_legacy_device_extensions(state.system)
2066 .ok();
2067
2068 let extra = match (required_instance, required_device) {
2069 (Some(i), Some(d)) => format!(
2070 "\n[OpenXR] Runtime-required Vulkan instance extensions: {i}\n[OpenXR] Runtime-required Vulkan device extensions: {d}"
2071 ),
2072 _ => String::new(),
2073 };
2074
2075 format!(
2076 "create_session(Vulkan): {e:?}.\n\
2077If this fails with Vulkan extension errors, the Vulkan instance/device created by Vulkano may be missing runtime-required extensions (from XR_KHR_vulkan_enable2)."
2078 )
2079 + extra.as_str()
2080 })?
2081 };
2082
2083 let reference_space = session
2084 .create_reference_space(openxr::ReferenceSpaceType::STAGE, openxr::Posef::IDENTITY)
2085 .map_err(|e| format!("create_reference_space(STAGE): {e:?}"))?;
2086
2087 let controller_input = match Self::try_init_controller_input(&state.instance, &session) {
2088 Ok(v) => {
2089 println!("[OpenXR] Controller input initialized.");
2090 Some(v)
2091 }
2092 Err(err) => {
2093 eprintln!("[OpenXR] Controller input init failed: {err}");
2094 None
2095 }
2096 };
2097
2098 let hand_tracking = match state.instance.supports_hand_tracking(state.system) {
2099 Ok(v) => {
2100 println!("[OpenXR] Runtime supports hand tracking: {v}");
2101 if v {
2102 match Self::try_init_hand_tracking(&session) {
2103 Ok(trackers) => {
2104 println!("[OpenXR] Hand tracking initialized.");
2105 Some(trackers)
2106 }
2107 Err(err) => {
2108 eprintln!("[OpenXR] Hand tracking init failed: {err}");
2109 None
2110 }
2111 }
2112 } else {
2113 None
2114 }
2115 }
2116 Err(e) => {
2117 eprintln!("[OpenXR] supports_hand_tracking query failed: {e:?}");
2118 None
2119 }
2120 };
2121
2122 let xr_swapchain = XRSwapchain::new(
2123 &state.instance,
2124 &session,
2125 state.system,
2126 state.view_type,
2127 preferred_swapchain_format,
2128 )?;
2129
2130 let swapchain_image_initialized = vec![false; xr_swapchain.images().len()];
2131
2132 let entry = unsafe { ash::Entry::load().map_err(|e| format!("ash::Entry::load: {e:?}"))? };
2134
2135 let vk_instance = ash::vk::Instance::from_raw(gfx.vk_instance as usize as u64);
2136 let vk_device_handle = ash::vk::Device::from_raw(gfx.vk_device as usize as u64);
2137
2138 let vk_instance = unsafe { ash::Instance::load(entry.static_fn(), vk_instance) };
2139 let vk_device = unsafe { ash::Device::load(vk_instance.fp_v1_0(), vk_device_handle) };
2140 let vk_queue =
2141 unsafe { vk_device.get_device_queue(gfx.queue_family_index, gfx.queue_index) };
2142
2143 let vk_command_pool = unsafe {
2144 vk_device
2145 .create_command_pool(
2146 &ash::vk::CommandPoolCreateInfo::default()
2147 .queue_family_index(gfx.queue_family_index)
2148 .flags(ash::vk::CommandPoolCreateFlags::RESET_COMMAND_BUFFER),
2149 None,
2150 )
2151 .map_err(|e| format!("create_command_pool: {e:?}"))?
2152 };
2153
2154 let vk_command_buffer = unsafe {
2155 vk_device
2156 .allocate_command_buffers(
2157 &ash::vk::CommandBufferAllocateInfo::default()
2158 .command_pool(vk_command_pool)
2159 .level(ash::vk::CommandBufferLevel::PRIMARY)
2160 .command_buffer_count(1),
2161 )
2162 .map_err(|e| format!("allocate_command_buffers: {e:?}"))?
2163 .into_iter()
2164 .next()
2165 .ok_or("allocate_command_buffers returned 0 command buffers")?
2166 };
2167
2168 state.session = Some(OpenXRSessionState {
2169 session,
2170 frame_waiter,
2171 frame_stream,
2172 reference_space,
2173 running: false,
2174 current_state: openxr::SessionState::IDLE,
2175
2176 xr_swapchain,
2177
2178 swapchain_image_initialized,
2179
2180 did_log_format_mismatch: false,
2181
2182 vk_device,
2183 vk_queue,
2184 vk_command_pool,
2185 vk_command_buffer,
2186
2187 hand_tracking,
2188 hand_root_pose_cache: HandRootPoseCache::default(),
2189 hand_rotation_debug: HandRotationDebugState::default(),
2190 last_hand_debug_snapshot: None,
2191 head_pose_cache: None,
2192 controller_input,
2193 controller_pose_cache: ControllerPoseCache::default(),
2194 });
2195
2196 println!("[OpenXR] Session created (Vulkan)");
2197 Ok(())
2198 }
2199
2200 fn try_init_controller_input(
2201 instance: &openxr::Instance,
2202 session: &openxr::Session<openxr::Vulkan>,
2203 ) -> Result<ControllerInput, String> {
2204 let left = instance
2205 .string_to_path("/user/hand/left")
2206 .map_err(|e| format!("string_to_path(/user/hand/left): {e:?}"))?;
2207 let right = instance
2208 .string_to_path("/user/hand/right")
2209 .map_err(|e| format!("string_to_path(/user/hand/right): {e:?}"))?;
2210
2211 let action_set = instance
2212 .create_action_set("mittens_engine", "Cat Engine", 0)
2213 .map_err(|e| format!("create_action_set: {e:?}"))?;
2214 let hand_subaction_paths = [left, right];
2215
2216 let aim_pose = action_set
2217 .create_action::<openxr::Posef>("aim_pose", "Aim Pose", &hand_subaction_paths)
2218 .map_err(|e| format!("create_action(aim_pose): {e:?}"))?;
2219 let grip_pose = action_set
2220 .create_action::<openxr::Posef>("grip_pose", "Grip Pose", &hand_subaction_paths)
2221 .map_err(|e| format!("create_action(grip_pose): {e:?}"))?;
2222 let select = action_set
2223 .create_action::<bool>("select", "Select", &hand_subaction_paths)
2224 .map_err(|e| format!("create_action(select): {e:?}"))?;
2225 let left_stick_x = action_set
2226 .create_action::<f32>("left_stick_x", "Left Stick X", &hand_subaction_paths)
2227 .map_err(|e| format!("create_action(left_stick_x): {e:?}"))?;
2228 let left_stick_y = action_set
2229 .create_action::<f32>("left_stick_y", "Left Stick Y", &hand_subaction_paths)
2230 .map_err(|e| format!("create_action(left_stick_y): {e:?}"))?;
2231 let right_stick_x = action_set
2232 .create_action::<f32>("right_stick_x", "Right Stick X", &hand_subaction_paths)
2233 .map_err(|e| format!("create_action(right_stick_x): {e:?}"))?;
2234 let right_stick_y = action_set
2235 .create_action::<f32>("right_stick_y", "Right Stick Y", &hand_subaction_paths)
2236 .map_err(|e| format!("create_action(right_stick_y): {e:?}"))?;
2237 let trigger_value = action_set
2238 .create_action::<f32>("trigger_value", "Trigger Value", &hand_subaction_paths)
2239 .map_err(|e| format!("create_action(trigger_value): {e:?}"))?;
2240 let trigger_click = action_set
2241 .create_action::<bool>("trigger_click", "Trigger Click", &hand_subaction_paths)
2242 .map_err(|e| format!("create_action(trigger_click): {e:?}"))?;
2243 let grip_value = action_set
2244 .create_action::<f32>("grip_value", "Grip Value", &hand_subaction_paths)
2245 .map_err(|e| format!("create_action(grip_value): {e:?}"))?;
2246 let grip_click = action_set
2247 .create_action::<bool>("grip_click", "Grip Click", &hand_subaction_paths)
2248 .map_err(|e| format!("create_action(grip_click): {e:?}"))?;
2249 let button_a = action_set
2250 .create_action::<bool>("button_a", "Button A", &hand_subaction_paths)
2251 .map_err(|e| format!("create_action(button_a): {e:?}"))?;
2252 let button_b = action_set
2253 .create_action::<bool>("button_b", "Button B", &hand_subaction_paths)
2254 .map_err(|e| format!("create_action(button_b): {e:?}"))?;
2255 let button_x = action_set
2256 .create_action::<bool>("button_x", "Button X", &hand_subaction_paths)
2257 .map_err(|e| format!("create_action(button_x): {e:?}"))?;
2258 let button_y = action_set
2259 .create_action::<bool>("button_y", "Button Y", &hand_subaction_paths)
2260 .map_err(|e| format!("create_action(button_y): {e:?}"))?;
2261
2262 let left_aim_space = aim_pose
2263 .create_space(session.clone(), left, openxr::Posef::IDENTITY)
2264 .map_err(|e| format!("aim_pose.create_space(left): {e:?}"))?;
2265 let right_aim_space = aim_pose
2266 .create_space(session.clone(), right, openxr::Posef::IDENTITY)
2267 .map_err(|e| format!("aim_pose.create_space(right): {e:?}"))?;
2268 let left_grip_space = grip_pose
2269 .create_space(session.clone(), left, openxr::Posef::IDENTITY)
2270 .map_err(|e| format!("grip_pose.create_space(left): {e:?}"))?;
2271 let right_grip_space = grip_pose
2272 .create_space(session.clone(), right, openxr::Posef::IDENTITY)
2273 .map_err(|e| format!("grip_pose.create_space(right): {e:?}"))?;
2274
2275 let left_aim_path = instance
2279 .string_to_path("/user/hand/left/input/aim/pose")
2280 .map_err(|e| format!("string_to_path(left aim): {e:?}"))?;
2281 let right_aim_path = instance
2282 .string_to_path("/user/hand/right/input/aim/pose")
2283 .map_err(|e| format!("string_to_path(right aim): {e:?}"))?;
2284 let left_grip_path = instance
2285 .string_to_path("/user/hand/left/input/grip/pose")
2286 .map_err(|e| format!("string_to_path(left grip): {e:?}"))?;
2287 let right_grip_path = instance
2288 .string_to_path("/user/hand/right/input/grip/pose")
2289 .map_err(|e| format!("string_to_path(right grip): {e:?}"))?;
2290
2291 let pose_bindings = [
2293 openxr::Binding::new(&aim_pose, left_aim_path),
2294 openxr::Binding::new(&aim_pose, right_aim_path),
2295 openxr::Binding::new(&grip_pose, left_grip_path),
2296 openxr::Binding::new(&grip_pose, right_grip_path),
2297 ];
2298
2299 let mut binding_reports = Vec::new();
2300
2301 for spec in PROFILE_BINDING_SPECS {
2302 log_profile_binding_dump(spec);
2303 let Ok(profile) = instance.string_to_path(spec.profile) else {
2304 continue;
2305 };
2306 let mut report = ProfileBindingReport {
2307 profile_path: spec.profile.to_string(),
2308 suggestions: Vec::new(),
2309 };
2310 let mut accepted_bindings = Vec::new();
2311 for (binding_label, binding_path, binding) in [
2312 (
2313 "aim_pose_left",
2314 "/user/hand/left/input/aim/pose",
2315 pose_bindings[0],
2316 ),
2317 (
2318 "aim_pose_right",
2319 "/user/hand/right/input/aim/pose",
2320 pose_bindings[1],
2321 ),
2322 (
2323 "grip_pose_left",
2324 "/user/hand/left/input/grip/pose",
2325 pose_bindings[2],
2326 ),
2327 (
2328 "grip_pose_right",
2329 "/user/hand/right/input/grip/pose",
2330 pose_bindings[3],
2331 ),
2332 ] {
2333 let outcome = suggest_binding_best_effort(
2334 instance,
2335 profile,
2336 spec.profile,
2337 binding_label,
2338 binding_path,
2339 binding,
2340 )?;
2341 if outcome == BindingSuggestionOutcome::Accepted {
2342 accepted_bindings.push(binding);
2343 }
2344 report.suggestions.push(BindingSuggestionRecord {
2345 label: binding_label,
2346 path: binding_path.to_string(),
2347 outcome,
2348 });
2349 }
2350 if let (Some(l), Some(r)) = (spec.select_left, spec.select_right) {
2351 if let (Ok(lp), Ok(rp)) = (instance.string_to_path(l), instance.string_to_path(r)) {
2352 let outcome = suggest_binding_best_effort(
2353 instance,
2354 profile,
2355 spec.profile,
2356 "select_left",
2357 l,
2358 openxr::Binding::new(&select, lp),
2359 )?;
2360 if outcome == BindingSuggestionOutcome::Accepted {
2361 accepted_bindings.push(openxr::Binding::new(&select, lp));
2362 }
2363 report.suggestions.push(BindingSuggestionRecord {
2364 label: "select_left",
2365 path: l.to_string(),
2366 outcome,
2367 });
2368 let outcome = suggest_binding_best_effort(
2369 instance,
2370 profile,
2371 spec.profile,
2372 "select_right",
2373 r,
2374 openxr::Binding::new(&select, rp),
2375 )?;
2376 if outcome == BindingSuggestionOutcome::Accepted {
2377 accepted_bindings.push(openxr::Binding::new(&select, rp));
2378 }
2379 report.suggestions.push(BindingSuggestionRecord {
2380 label: "select_right",
2381 path: r.to_string(),
2382 outcome,
2383 });
2384 }
2385 }
2386 let mut suggested = Vec::new();
2387 for hand_spec in spec.left_specs {
2388 append_action_bindings(
2389 instance,
2390 &mut suggested,
2391 *hand_spec,
2392 &left_stick_x,
2393 &left_stick_y,
2394 &right_stick_x,
2395 &right_stick_y,
2396 &trigger_value,
2397 &trigger_click,
2398 &grip_value,
2399 &grip_click,
2400 &button_a,
2401 &button_b,
2402 &button_x,
2403 &button_y,
2404 );
2405 }
2406 for hand_spec in spec.right_specs {
2407 append_action_bindings(
2408 instance,
2409 &mut suggested,
2410 *hand_spec,
2411 &left_stick_x,
2412 &left_stick_y,
2413 &right_stick_x,
2414 &right_stick_y,
2415 &trigger_value,
2416 &trigger_click,
2417 &grip_value,
2418 &grip_click,
2419 &button_a,
2420 &button_b,
2421 &button_x,
2422 &button_y,
2423 );
2424 }
2425 if !suggested.is_empty() {
2426 for (binding_label, binding_path, binding) in suggested {
2427 let outcome = suggest_binding_best_effort(
2428 instance,
2429 profile,
2430 spec.profile,
2431 binding_label,
2432 binding_path,
2433 binding,
2434 )?;
2435 if outcome == BindingSuggestionOutcome::Accepted {
2436 accepted_bindings.push(binding);
2437 }
2438 report.suggestions.push(BindingSuggestionRecord {
2439 label: binding_label,
2440 path: binding_path.to_string(),
2441 outcome,
2442 });
2443 }
2444 }
2445 if !accepted_bindings.is_empty() {
2446 instance
2447 .suggest_interaction_profile_bindings(profile, &accepted_bindings)
2448 .map_err(|err| {
2449 format!(
2450 "suggest_interaction_profile_bindings(final batch {}, count={}): {err:?}",
2451 spec.profile,
2452 accepted_bindings.len()
2453 )
2454 })?;
2455 }
2456 binding_reports.push(report);
2457 }
2458
2459 session
2460 .attach_action_sets(&[&action_set])
2461 .map_err(|e| format!("attach_action_sets: {e:?}"))?;
2462
2463 Ok(ControllerInput {
2464 action_set,
2465 aim_pose,
2466 grip_pose,
2467 select,
2468 left_stick_x,
2469 left_stick_y,
2470 right_stick_x,
2471 right_stick_y,
2472 trigger_value,
2473 trigger_click,
2474 grip_value,
2475 grip_click,
2476 button_a,
2477 button_b,
2478 button_x,
2479 button_y,
2480 left,
2481 right,
2482 left_aim_space,
2483 right_aim_space,
2484 left_grip_space,
2485 right_grip_space,
2486 profile_poll_counter: 0,
2487 last_logged_left_profile: None,
2488 last_logged_right_profile: None,
2489 last_reported_left_profile: None,
2490 last_reported_right_profile: None,
2491 last_debug_snapshot: None,
2492 did_log_bound_sources: false,
2493 binding_reports,
2494 })
2495 }
2496
2497 fn first_enabled_camera_xr(world: &World) -> Option<ComponentId> {
2498 world
2499 .all_components()
2500 .filter_map(|id| {
2501 world
2502 .get_component_by_id_as::<CameraXRComponent>(id)
2503 .map(|c| (id, c.enabled))
2504 })
2505 .find(|(_, enabled)| *enabled)
2506 .map(|(id, _)| id)
2507 }
2508
2509 fn mul_mat4(a: &[[f32; 4]; 4], b: &[[f32; 4]; 4]) -> [[f32; 4]; 4] {
2510 let mut c = [[0.0f32; 4]; 4];
2511 for col in 0..4 {
2512 for row in 0..4 {
2513 c[col][row] = a[0][row] * b[col][0]
2514 + a[1][row] * b[col][1]
2515 + a[2][row] * b[col][2]
2516 + a[3][row] * b[col][3];
2517 }
2518 }
2519 c
2520 }
2521
2522 fn transform_from_matrix_world(
2523 m: [[f32; 4]; 4],
2524 ) -> crate::engine::graphics::primitives::Transform {
2525 let mut t = crate::engine::graphics::primitives::Transform::default();
2526 t.model = m;
2527 t.matrix_world = m;
2528 t
2529 }
2530
2531 fn mat4_from_pose(pose: openxr::Posef) -> [[f32; 4]; 4] {
2532 let q = pose.orientation;
2535 let p = pose.position;
2536
2537 let mut t = crate::engine::graphics::primitives::Transform::default();
2538 t.translation = [p.x, p.y, p.z];
2539 t.rotation = [q.x, q.y, q.z, q.w];
2540 t.scale = [1.0, 1.0, 1.0];
2541 t.recompute_model();
2542
2543 t.model
2549 }
2550
2551 fn invert_affine_transform(m: &[[f32; 4]; 4]) -> [[f32; 4]; 4] {
2552 let c0 = [m[0][0], m[0][1], m[0][2]];
2554 let c1 = [m[1][0], m[1][1], m[1][2]];
2555 let c2 = [m[2][0], m[2][1], m[2][2]];
2556
2557 let a00 = c0[0];
2559 let a10 = c0[1];
2560 let a20 = c0[2];
2561 let a01 = c1[0];
2562 let a11 = c1[1];
2563 let a21 = c1[2];
2564 let a02 = c2[0];
2565 let a12 = c2[1];
2566 let a22 = c2[2];
2567
2568 let det = a00 * (a11 * a22 - a12 * a21) - a01 * (a10 * a22 - a12 * a20)
2569 + a02 * (a10 * a21 - a11 * a20);
2570
2571 if det.abs() < 1e-8 {
2572 return [
2573 [1.0, 0.0, 0.0, 0.0],
2574 [0.0, 1.0, 0.0, 0.0],
2575 [0.0, 0.0, 1.0, 0.0],
2576 [0.0, 0.0, 0.0, 1.0],
2577 ];
2578 }
2579 let inv_det = 1.0 / det;
2580
2581 let inv00 = (a11 * a22 - a12 * a21) * inv_det;
2583 let inv01 = (a02 * a21 - a01 * a22) * inv_det;
2584 let inv02 = (a01 * a12 - a02 * a11) * inv_det;
2585
2586 let inv10 = (a12 * a20 - a10 * a22) * inv_det;
2587 let inv11 = (a00 * a22 - a02 * a20) * inv_det;
2588 let inv12 = (a02 * a10 - a00 * a12) * inv_det;
2589
2590 let inv20 = (a10 * a21 - a11 * a20) * inv_det;
2591 let inv21 = (a01 * a20 - a00 * a21) * inv_det;
2592 let inv22 = (a00 * a11 - a01 * a10) * inv_det;
2593
2594 let tx = m[3][0];
2595 let ty = m[3][1];
2596 let tz = m[3][2];
2597
2598 let itx = -(inv00 * tx + inv01 * ty + inv02 * tz);
2599 let ity = -(inv10 * tx + inv11 * ty + inv12 * tz);
2600 let itz = -(inv20 * tx + inv21 * ty + inv22 * tz);
2601
2602 [
2603 [inv00, inv10, inv20, 0.0],
2604 [inv01, inv11, inv21, 0.0],
2605 [inv02, inv12, inv22, 0.0],
2606 [itx, ity, itz, 1.0],
2607 ]
2608 }
2609
2610 fn proj_from_fov_rh_zo(fov: openxr::Fovf, z_near: f32, z_far: f32) -> [[f32; 4]; 4] {
2611 let l = (fov.angle_left).tan() * z_near;
2612 let r = (fov.angle_right).tan() * z_near;
2613 let d = (fov.angle_down).tan() * z_near;
2614 let u = (fov.angle_up).tan() * z_near;
2615
2616 let w = r - l;
2617 let h = u - d;
2618 let nf = 1.0 / (z_near - z_far);
2619
2620 [
2623 [2.0 * z_near / w, 0.0, 0.0, 0.0],
2624 [0.0, 2.0 * z_near / h, 0.0, 0.0],
2625 [(r + l) / w, (u + d) / h, z_far * nf, -1.0],
2626 [0.0, 0.0, (z_near * z_far) * nf, 0.0],
2627 ]
2628 }
2629}
2630
2631impl System for OpenXRSystem {
2632 fn tick(
2633 &mut self,
2634 _world: &mut World,
2635 _visuals: &mut VisualWorld,
2636 _input: &InputState,
2637 _dt_sec: f32,
2638 ) {
2639 self.pump_events();
2640 }
2641}
2642
2643impl OpenXRSystem {
2644 pub fn last_render_dt_sec(&self) -> Option<f32> {
2645 self.last_render_dt_sec
2646 }
2647
2648 pub fn render_xr(
2649 &mut self,
2650 world: &World,
2651 visuals: &mut VisualWorld,
2652 renderer: &mut VulkanoRenderer,
2653 ) {
2654 let profile = Self::profile_systems_enabled();
2655 let frame_profile_started = profile.then(Instant::now);
2656 let Some(state) = self.state.as_mut() else {
2657 self.xr_input_state = XrInputState::default();
2658 self.xr_gamepad_state = XrGamepadState::default();
2659 visuals.set_xr_frame_dt_sec(None);
2660 return;
2661 };
2662
2663 let Some(sess) = state.session.as_mut() else {
2664 self.xr_input_state = XrInputState::default();
2665 self.xr_gamepad_state = XrGamepadState::default();
2666 visuals.set_xr_frame_dt_sec(None);
2667 return;
2668 };
2669 if !sess.running {
2670 self.xr_input_state = XrInputState::default();
2671 self.xr_gamepad_state = XrGamepadState::default();
2672 visuals.set_xr_frame_dt_sec(None);
2673 return;
2674 }
2675
2676 let now = Instant::now();
2677 let dt_sec = self
2678 .last_render_instant
2679 .map(|prev| now.saturating_duration_since(prev).as_secs_f32());
2680 self.last_render_instant = Some(now);
2681 if let Some(dt_sec) = dt_sec {
2682 self.last_render_dt_sec = Some(dt_sec);
2683 visuals.set_xr_frame_dt_sec(Some(dt_sec));
2684 }
2685
2686 if visuals
2690 .active_xr_camera()
2691 .or_else(|| Self::first_enabled_camera_xr(world))
2692 .is_none()
2693 {
2694 self.xr_input_state = XrInputState::default();
2695 self.xr_gamepad_state = XrGamepadState::default();
2696 sess.head_pose_cache = None;
2697 sess.hand_root_pose_cache = HandRootPoseCache::default();
2698 sess.controller_pose_cache = ControllerPoseCache::default();
2699 visuals.set_xr_frame_dt_sec(None);
2700 return;
2701 }
2702
2703 let stage_started = profile.then(Instant::now);
2704 let frame_state = match sess.frame_waiter.wait() {
2705 Ok(s) => s,
2706 Err(e) => {
2707 eprintln!("[OpenXR] wait_frame failed: {e:?}");
2708 return;
2709 }
2710 };
2711 if let Some(started) = stage_started {
2712 self.render_profile.wait_frame += started.elapsed();
2713 }
2714
2715 if let Err(e) = sess.frame_stream.begin() {
2716 eprintln!("[OpenXR] begin_frame failed: {e:?}");
2717 return;
2718 }
2719
2720 if !frame_state.should_render {
2721 let _ =
2722 sess.frame_stream
2723 .end(frame_state.predicted_display_time, state.blend_mode, &[]);
2724 return;
2725 }
2726
2727 let tracking_started = profile.then(Instant::now);
2728
2729 let views = match sess.session.locate_views(
2730 state.view_type,
2731 frame_state.predicted_display_time,
2732 &sess.reference_space,
2733 ) {
2734 Ok((_flags, views)) => views,
2735 Err(e) => {
2736 eprintln!("[OpenXR] locate_views failed: {e:?}");
2737 let _ = sess.frame_stream.end(
2738 frame_state.predicted_display_time,
2739 state.blend_mode,
2740 &[],
2741 );
2742 return;
2743 }
2744 };
2745
2746 sess.head_pose_cache = Self::derive_head_pose(&views);
2747 self.xr_gamepad_state.head_pose_rotation = sess.head_pose_cache.map(Self::quat_from_posef);
2748
2749 let mut left_root_for_debug: Option<(openxr::Posef, openxr::HandJointEXT)> = None;
2750 let mut right_root_for_debug: Option<(openxr::Posef, openxr::HandJointEXT)> = None;
2751 let mut left_joints_for_debug: Option<openxr::HandJointLocations> = None;
2752 let mut right_joints_for_debug: Option<openxr::HandJointLocations> = None;
2753
2754 if let Some(hand_tracking) = sess.hand_tracking.as_ref() {
2755 let left_joints = sess
2756 .reference_space
2757 .locate_hand_joints(&hand_tracking.left, frame_state.predicted_display_time);
2758 let right_joints = sess
2759 .reference_space
2760 .locate_hand_joints(&hand_tracking.right, frame_state.predicted_display_time);
2761
2762 match left_joints {
2763 Ok(Some(joints)) => {
2764 let root = Self::select_hand_root_pose(&joints);
2765 sess.hand_root_pose_cache.left_root = root.map(|(pose, _)| pose);
2766 sess.hand_root_pose_cache.left_root_joint = root.map(|(_, joint)| joint);
2767 left_root_for_debug = root;
2768 left_joints_for_debug = Some(joints);
2769 }
2770 Ok(None) => {
2771 sess.hand_root_pose_cache.left_root = None;
2772 sess.hand_root_pose_cache.left_root_joint = None;
2773 }
2774 Err(e) => {
2775 sess.hand_root_pose_cache.left_root = None;
2776 sess.hand_root_pose_cache.left_root_joint = None;
2777 eprintln!("[OpenXR] locate_hand_joints(left) failed: {e:?}");
2778 }
2779 }
2780
2781 match right_joints {
2782 Ok(Some(joints)) => {
2783 let root = Self::select_hand_root_pose(&joints);
2784 sess.hand_root_pose_cache.right_root = root.map(|(pose, _)| pose);
2785 sess.hand_root_pose_cache.right_root_joint = root.map(|(_, joint)| joint);
2786 right_root_for_debug = root;
2787 right_joints_for_debug = Some(joints);
2788 }
2789 Ok(None) => {
2790 sess.hand_root_pose_cache.right_root = None;
2791 sess.hand_root_pose_cache.right_root_joint = None;
2792 }
2793 Err(e) => {
2794 sess.hand_root_pose_cache.right_root = None;
2795 sess.hand_root_pose_cache.right_root_joint = None;
2796 eprintln!("[OpenXR] locate_hand_joints(right) failed: {e:?}");
2797 }
2798 }
2799 } else {
2800 sess.hand_root_pose_cache = HandRootPoseCache::default();
2801 }
2802
2803 Self::log_hand_debug_snapshot(
2804 sess,
2805 left_joints_for_debug.as_ref(),
2806 right_joints_for_debug.as_ref(),
2807 sess.hand_root_pose_cache.left_root_joint,
2808 sess.hand_root_pose_cache.right_root_joint,
2809 );
2810
2811 if let Some((pose, joint)) = left_root_for_debug {
2812 Self::update_hand_rotation_debug(
2813 &mut sess.hand_rotation_debug,
2814 ControllerHand::Left,
2815 Some(joint),
2816 pose,
2817 );
2818 }
2819 if let Some((pose, joint)) = right_root_for_debug {
2820 Self::update_hand_rotation_debug(
2821 &mut sess.hand_rotation_debug,
2822 ControllerHand::Right,
2823 Some(joint),
2824 pose,
2825 );
2826 }
2827
2828 if let Some(ci) = sess.controller_input.as_mut() {
2830 let active = openxr::ActiveActionSet::new(&ci.action_set);
2832 match sess.session.sync_actions(&[active]) {
2833 Ok(()) => {}
2834 Err(e) => {
2835 eprintln!("[OpenXR] sync_actions failed: {e:?}");
2836 }
2837 }
2838 let should_poll_profiles =
2839 ci.profile_poll_counter == 0 || ci.profile_poll_counter >= 89;
2840 ci.profile_poll_counter = if should_poll_profiles {
2841 1
2842 } else {
2843 ci.profile_poll_counter.saturating_add(1)
2844 };
2845 if should_poll_profiles {
2846 log_active_interaction_profile(
2847 &state.instance,
2848 &sess.session,
2849 ci.left,
2850 "/user/hand/left",
2851 &mut ci.last_logged_left_profile,
2852 );
2853 log_active_interaction_profile(
2854 &state.instance,
2855 &sess.session,
2856 ci.right,
2857 "/user/hand/right",
2858 &mut ci.last_logged_right_profile,
2859 );
2860 }
2861 let update_pose = |space: &openxr::Space, base: &openxr::Space, t: openxr::Time| {
2862 space.locate(base, t).ok()
2863 };
2864
2865 let left_profile = ci.last_logged_left_profile;
2866 let right_profile = ci.last_logged_right_profile;
2867
2868 sess.controller_pose_cache.left_aim = update_pose(
2869 &ci.left_aim_space,
2870 &sess.reference_space,
2871 frame_state.predicted_display_time,
2872 )
2873 .filter(|loc| {
2874 loc.location_flags
2875 .contains(openxr::SpaceLocationFlags::POSITION_VALID)
2876 && loc
2877 .location_flags
2878 .contains(openxr::SpaceLocationFlags::ORIENTATION_VALID)
2879 })
2880 .map(|loc| loc.pose);
2881 sess.controller_pose_cache.right_aim = update_pose(
2882 &ci.right_aim_space,
2883 &sess.reference_space,
2884 frame_state.predicted_display_time,
2885 )
2886 .filter(|loc| {
2887 loc.location_flags
2888 .contains(openxr::SpaceLocationFlags::POSITION_VALID)
2889 && loc
2890 .location_flags
2891 .contains(openxr::SpaceLocationFlags::ORIENTATION_VALID)
2892 })
2893 .map(|loc| loc.pose);
2894
2895 sess.controller_pose_cache.left_grip = update_pose(
2896 &ci.left_grip_space,
2897 &sess.reference_space,
2898 frame_state.predicted_display_time,
2899 )
2900 .filter(|loc| {
2901 loc.location_flags
2902 .contains(openxr::SpaceLocationFlags::POSITION_VALID)
2903 && loc
2904 .location_flags
2905 .contains(openxr::SpaceLocationFlags::ORIENTATION_VALID)
2906 })
2907 .map(|loc| loc.pose);
2908 sess.controller_pose_cache.right_grip = update_pose(
2909 &ci.right_grip_space,
2910 &sess.reference_space,
2911 frame_state.predicted_display_time,
2912 )
2913 .filter(|loc| {
2914 loc.location_flags
2915 .contains(openxr::SpaceLocationFlags::POSITION_VALID)
2916 && loc
2917 .location_flags
2918 .contains(openxr::SpaceLocationFlags::ORIENTATION_VALID)
2919 })
2920 .map(|loc| loc.pose);
2921
2922 let left_select = action_state_bool(&ci.select, &sess.session, ci.left);
2923 let right_select = action_state_bool(&ci.select, &sess.session, ci.right);
2924 let left_stick_x = ci.left_stick_x.state(&sess.session, ci.left).ok();
2925 let left_stick_y = ci.left_stick_y.state(&sess.session, ci.left).ok();
2926 let right_stick_x = ci.right_stick_x.state(&sess.session, ci.right).ok();
2927 let right_stick_y = ci.right_stick_y.state(&sess.session, ci.right).ok();
2928 let left_trigger_value = action_state_f32(&ci.trigger_value, &sess.session, ci.left);
2929 let right_trigger_value = action_state_f32(&ci.trigger_value, &sess.session, ci.right);
2930 let left_trigger_click = action_state_bool(&ci.trigger_click, &sess.session, ci.left);
2931 let right_trigger_click = action_state_bool(&ci.trigger_click, &sess.session, ci.right);
2932 let left_grip_value = action_state_f32(&ci.grip_value, &sess.session, ci.left);
2933 let right_grip_value = action_state_f32(&ci.grip_value, &sess.session, ci.right);
2934 let left_grip_click = action_state_bool(&ci.grip_click, &sess.session, ci.left);
2935 let right_grip_click = action_state_bool(&ci.grip_click, &sess.session, ci.right);
2936 let left_x = action_state_bool(&ci.button_x, &sess.session, ci.left);
2937 let left_y = action_state_bool(&ci.button_y, &sess.session, ci.left);
2938 let right_a = action_state_bool(&ci.button_a, &sess.session, ci.right);
2939 let right_b = action_state_bool(&ci.button_b, &sess.session, ci.right);
2940 let snapshot = ControllerDebugSnapshot {
2941 left_profile: profile_string_or_none(&state.instance, left_profile),
2942 right_profile: profile_string_or_none(&state.instance, right_profile),
2943 left_aim_valid: sess.controller_pose_cache.left_aim.is_some(),
2944 right_aim_valid: sess.controller_pose_cache.right_aim.is_some(),
2945 left_grip_valid: sess.controller_pose_cache.left_grip.is_some(),
2946 right_grip_valid: sess.controller_pose_cache.right_grip.is_some(),
2947 left_select: left_select.map(|s| (s.is_active, s.current_state)),
2948 right_select: right_select.map(|s| (s.is_active, s.current_state)),
2949 left_thumbstick: scalar_stick_state(left_stick_x, left_stick_y),
2950 right_thumbstick: scalar_stick_state(right_stick_x, right_stick_y),
2951 left_trigger_value: left_trigger_value.map(|s| (s.is_active, s.current_state)),
2952 right_trigger_value: right_trigger_value.map(|s| (s.is_active, s.current_state)),
2953 left_trigger_click: left_trigger_click.map(|s| (s.is_active, s.current_state)),
2954 right_trigger_click: right_trigger_click.map(|s| (s.is_active, s.current_state)),
2955 left_grip_value: left_grip_value.map(|s| (s.is_active, s.current_state)),
2956 right_grip_value: right_grip_value.map(|s| (s.is_active, s.current_state)),
2957 left_grip_click: left_grip_click.map(|s| (s.is_active, s.current_state)),
2958 right_grip_click: right_grip_click.map(|s| (s.is_active, s.current_state)),
2959 left_x: left_x.map(|s| (s.is_active, s.current_state)),
2960 left_y: left_y.map(|s| (s.is_active, s.current_state)),
2961 right_a: right_a.map(|s| (s.is_active, s.current_state)),
2962 right_b: right_b.map(|s| (s.is_active, s.current_state)),
2963 };
2964 if let Some(left_profile) = left_profile {
2965 if left_profile != openxr::Path::NULL
2966 && ci.last_reported_left_profile != Some(left_profile)
2967 {
2968 if let Some(report) = ci
2969 .binding_reports
2970 .iter()
2971 .find(|report| report.profile_path == snapshot.left_profile)
2972 {
2973 log_runtime_profile_binding_report("left", report, &snapshot);
2974 } else {
2975 eprintln!(
2976 "[OpenXR][binding_report] left active profile has no stored binding report: {}",
2977 snapshot.left_profile
2978 );
2979 }
2980 ci.last_reported_left_profile = Some(left_profile);
2981 }
2982 }
2983 if let Some(right_profile) = right_profile {
2984 if right_profile != openxr::Path::NULL
2985 && ci.last_reported_right_profile != Some(right_profile)
2986 {
2987 if let Some(report) = ci
2988 .binding_reports
2989 .iter()
2990 .find(|report| report.profile_path == snapshot.right_profile)
2991 {
2992 log_runtime_profile_binding_report("right", report, &snapshot);
2993 } else {
2994 eprintln!(
2995 "[OpenXR][binding_report] right active profile has no stored binding report: {}",
2996 snapshot.right_profile
2997 );
2998 }
2999 ci.last_reported_right_profile = Some(right_profile);
3000 }
3001 }
3002 if !ci.did_log_bound_sources
3003 && (matches!(left_profile, Some(profile) if profile != openxr::Path::NULL)
3004 || matches!(right_profile, Some(profile) if profile != openxr::Path::NULL))
3005 {
3006 log_controller_input_bound_sources(&sess.session, ci);
3007 ci.did_log_bound_sources = true;
3008 }
3009 if openxr_debug_enabled() && ci.last_debug_snapshot.as_ref() != Some(&snapshot) {
3010 eprintln!(
3011 "[OpenXR][debug] profiles left={} right={}",
3012 snapshot.left_profile, snapshot.right_profile,
3013 );
3014 eprintln!(
3015 "[OpenXR][debug] pose_valid left_aim={} right_aim={} left_grip={} right_grip={}",
3016 snapshot.left_aim_valid,
3017 snapshot.right_aim_valid,
3018 snapshot.left_grip_valid,
3019 snapshot.right_grip_valid,
3020 );
3021 eprintln!(
3022 "[OpenXR][debug] select L={:?} R={:?} thumbstick L={:?} R={:?}",
3023 snapshot.left_select,
3024 snapshot.right_select,
3025 snapshot.left_thumbstick,
3026 snapshot.right_thumbstick,
3027 );
3028 eprintln!(
3029 "[OpenXR][debug] trigger_value L={:?} R={:?} trigger_click L={:?} R={:?}",
3030 snapshot.left_trigger_value,
3031 snapshot.right_trigger_value,
3032 snapshot.left_trigger_click,
3033 snapshot.right_trigger_click,
3034 );
3035 eprintln!(
3036 "[OpenXR][debug] grip_value L={:?} R={:?} grip_click L={:?} R={:?}",
3037 snapshot.left_grip_value,
3038 snapshot.right_grip_value,
3039 snapshot.left_grip_click,
3040 snapshot.right_grip_click,
3041 );
3042 eprintln!(
3043 "[OpenXR][debug] face_buttons X={:?} Y={:?} A={:?} B={:?}",
3044 snapshot.left_x, snapshot.left_y, snapshot.right_a, snapshot.right_b,
3045 );
3046 }
3047 ci.last_debug_snapshot = Some(snapshot);
3048
3049 let prev = self.xr_input_state.trigger_down;
3052 self.xr_gamepad_state = XrGamepadState {
3053 active: true,
3054 hands: [XrHandGamepadState::default(), XrHandGamepadState::default()],
3055 head_pose_rotation: self.xr_gamepad_state.head_pose_rotation,
3056 };
3057 for i in 0..2 {
3058 let hand_path = if i == 0 { ci.left } else { ci.right };
3059 let select_state = action_state_bool(&ci.select, &sess.session, hand_path);
3060 let trigger_click_state =
3061 action_state_bool(&ci.trigger_click, &sess.session, hand_path);
3062 let trigger_value_state =
3063 action_state_f32(&ci.trigger_value, &sess.session, hand_path);
3064 let grip_click_state = action_state_bool(&ci.grip_click, &sess.session, hand_path);
3065 let grip_value_state = action_state_f32(&ci.grip_value, &sess.session, hand_path);
3066
3067 let cur_down =
3068 derive_select_down(select_state, trigger_click_state, trigger_value_state);
3069 self.xr_input_state.trigger_down[i] = cur_down;
3070 self.xr_input_state.trigger_pressed[i] = cur_down && !prev[i];
3071 self.xr_input_state.trigger_released[i] = !cur_down && prev[i];
3072
3073 self.xr_gamepad_state.hands[i].thumbstick = match i {
3074 0 => scalar_stick_value(
3075 ci.left_stick_x.state(&sess.session, ci.left).ok(),
3076 ci.left_stick_y.state(&sess.session, ci.left).ok(),
3077 ),
3078 1 => scalar_stick_value(
3079 ci.right_stick_x.state(&sess.session, ci.right).ok(),
3080 ci.right_stick_y.state(&sess.session, ci.right).ok(),
3081 ),
3082 _ => None,
3083 };
3084
3085 let trigger_value = trigger_value_state.map(|s| s.current_state);
3086 self.xr_gamepad_state.hands[i].trigger_value = trigger_value;
3087 self.xr_gamepad_state.hands[i].trigger_pressed = derive_trigger_like_pressed(
3088 trigger_click_state,
3089 trigger_value_state,
3090 XR_TRIGGER_PRESS_THRESHOLD,
3091 );
3092
3093 let grip_value = grip_value_state.map(|s| s.current_state);
3094 self.xr_gamepad_state.hands[i].grip_value = grip_value;
3095 self.xr_gamepad_state.hands[i].grip_pressed = derive_trigger_like_pressed(
3096 grip_click_state,
3097 grip_value_state,
3098 XR_GRIP_PRESS_THRESHOLD,
3099 );
3100 }
3101 self.xr_gamepad_state.hands[0].button_x = ci
3102 .button_x
3103 .state(&sess.session, ci.left)
3104 .ok()
3105 .filter(|s| s.is_active)
3106 .map(|s| (s.current_state, if s.current_state { 1.0 } else { 0.0 }));
3107 self.xr_gamepad_state.hands[0].button_y = ci
3108 .button_y
3109 .state(&sess.session, ci.left)
3110 .ok()
3111 .filter(|s| s.is_active)
3112 .map(|s| (s.current_state, if s.current_state { 1.0 } else { 0.0 }));
3113 self.xr_gamepad_state.hands[1].button_a = ci
3114 .button_a
3115 .state(&sess.session, ci.right)
3116 .ok()
3117 .filter(|s| s.is_active)
3118 .map(|s| (s.current_state, if s.current_state { 1.0 } else { 0.0 }));
3119 self.xr_gamepad_state.hands[1].button_b = ci
3120 .button_b
3121 .state(&sess.session, ci.right)
3122 .ok()
3123 .filter(|s| s.is_active)
3124 .map(|s| (s.current_state, if s.current_state { 1.0 } else { 0.0 }));
3125 log_xr_gamepad_changes(self.xr_gamepad_state_last_logged, self.xr_gamepad_state);
3126 self.xr_gamepad_state_last_logged = self.xr_gamepad_state;
3127 } else {
3128 self.xr_input_state = XrInputState::default();
3129 self.xr_gamepad_state = XrGamepadState::default();
3130 log_xr_gamepad_changes(self.xr_gamepad_state_last_logged, self.xr_gamepad_state);
3131 self.xr_gamepad_state_last_logged = self.xr_gamepad_state;
3132 }
3133
3134 let rig_world = Self::xr_rig_origin_world(world, visuals);
3136
3137 let mut eyes = Vec::with_capacity(views.len());
3138 for v in &views {
3139 let eye_from_space = Self::mat4_from_pose(v.pose);
3140 let world_from_eye = Self::mul_mat4(&rig_world, &eye_from_space);
3141 let view = Self::invert_affine_transform(&world_from_eye);
3142 let proj = Self::proj_from_fov_rh_zo(v.fov, 0.1, 100.0);
3143 eyes.push(CameraData {
3144 view,
3145 proj,
3146 transform: Self::transform_from_matrix_world(world_from_eye),
3147 });
3148 }
3149 visuals.set_xr_camera(eyes);
3150 if let Some(started) = tracking_started {
3151 self.render_profile.tracking_input += started.elapsed();
3152 }
3153
3154 let acquire_started = profile.then(Instant::now);
3156 let image_index = {
3157 let swapchain = sess.xr_swapchain.swapchain_mut();
3158 match swapchain.acquire_image() {
3159 Ok(i) => i,
3160 Err(e) => {
3161 eprintln!("[OpenXR] acquire_image failed: {e:?}");
3162 let _ = sess.frame_stream.end(
3163 frame_state.predicted_display_time,
3164 state.blend_mode,
3165 &[],
3166 );
3167 return;
3168 }
3169 }
3170 };
3171
3172 if let Err(e) = sess
3173 .xr_swapchain
3174 .swapchain_mut()
3175 .wait_image(openxr::Duration::INFINITE)
3176 {
3177 eprintln!("[OpenXR] wait_image failed: {e:?}");
3178 } else {
3179 if let Some(started) = acquire_started {
3180 self.render_profile.acquire_wait += started.elapsed();
3181 }
3182 let extent = sess.xr_swapchain.extent();
3184 let extent_u = [extent.width as u32, extent.height as u32];
3185
3186 let image_index_usize = image_index as usize;
3187 let dst_was_initialized = sess
3188 .swapchain_image_initialized
3189 .get(image_index_usize)
3190 .copied()
3191 .unwrap_or(false);
3192
3193 let view_count = sess.xr_swapchain.view_count() as usize;
3195
3196 let xr_format = sess.xr_swapchain.format();
3197 let window_format = renderer.window_vk_format_raw();
3198 let format_matches = window_format.map(|f| f == xr_format).unwrap_or(true);
3199
3200 let dst_image = sess.xr_swapchain.images()[image_index_usize];
3203 if !format_matches {
3204 if !sess.did_log_format_mismatch {
3205 sess.did_log_format_mismatch = true;
3206 eprintln!(
3207 "[OpenXR] XR swapchain format mismatch (xr={} window={:?}); falling back to clear_color",
3208 xr_format, window_format
3209 );
3210 }
3211
3212 if let Err(e) = xr_renderer::clear_xr_swapchain_image(
3213 &sess.vk_device,
3214 sess.vk_queue,
3215 sess.vk_command_buffer,
3216 sess.xr_swapchain.view_count(),
3217 dst_image,
3218 visuals.clear_color(),
3219 dst_was_initialized,
3220 ) {
3221 eprintln!("[OpenXR] clear XR image failed: {e:?}");
3222 } else if let Some(slot) =
3223 sess.swapchain_image_initialized.get_mut(image_index_usize)
3224 {
3225 *slot = true;
3226 }
3227 } else {
3228 let eye_render_started = profile.then(Instant::now);
3229 for eye in 0..view_count.min(views.len()) {
3230 if let Err(e) = renderer.render_xr_eye_offscreen(visuals, eye, extent_u) {
3231 eprintln!("[OpenXR] render_xr_eye_offscreen failed: {e}");
3232 }
3233 }
3234 if let Some(started) = eye_render_started {
3235 self.render_profile.eye_render += started.elapsed();
3236 }
3237
3238 let copy_started = profile.then(Instant::now);
3239 if let Err(e) = xr_renderer::copy_offscreen_to_xr_layers(
3240 &sess.vk_device,
3241 sess.vk_queue,
3242 sess.vk_command_buffer,
3243 &sess.xr_swapchain,
3244 renderer,
3245 dst_image,
3246 dst_was_initialized,
3247 view_count,
3248 ) {
3249 eprintln!("[OpenXR] copy to XR image failed: {e:?}");
3250 } else if let Some(slot) =
3251 sess.swapchain_image_initialized.get_mut(image_index_usize)
3252 {
3253 *slot = true;
3254 }
3255 if let Some(started) = copy_started {
3256 self.render_profile.copy += started.elapsed();
3257 }
3258 }
3259 }
3260
3261 if let Err(e) = sess.xr_swapchain.swapchain_mut().release_image() {
3262 eprintln!("[OpenXR] release_image failed: {e:?}");
3263 }
3264
3265 if views.len() >= 2 {
3267 let rect = openxr::Rect2Di {
3268 offset: openxr::Offset2Di { x: 0, y: 0 },
3269 extent: sess.xr_swapchain.extent(),
3270 };
3271
3272 let pv0 = openxr::CompositionLayerProjectionView::new()
3273 .pose(views[0].pose)
3274 .fov(views[0].fov)
3275 .sub_image(
3276 openxr::SwapchainSubImage::new()
3277 .swapchain(sess.xr_swapchain.swapchain())
3278 .image_array_index(0)
3279 .image_rect(rect),
3280 );
3281
3282 let pv1 = openxr::CompositionLayerProjectionView::new()
3283 .pose(views[1].pose)
3284 .fov(views[1].fov)
3285 .sub_image(
3286 openxr::SwapchainSubImage::new()
3287 .swapchain(sess.xr_swapchain.swapchain())
3288 .image_array_index(1)
3289 .image_rect(rect),
3290 );
3291
3292 let projection_views = [pv0, pv1];
3293 let layer = openxr::CompositionLayerProjection::new()
3294 .space(&sess.reference_space)
3295 .views(&projection_views);
3296
3297 let submit_started = profile.then(Instant::now);
3298 if let Err(e) = sess.frame_stream.end(
3299 frame_state.predicted_display_time,
3300 state.blend_mode,
3301 &[&layer],
3302 ) {
3303 eprintln!("[OpenXR] end_frame failed: {e:?}");
3304 }
3305 if let Some(started) = submit_started {
3306 self.render_profile.submit += started.elapsed();
3307 }
3308
3309 if let Some(started) = frame_profile_started {
3310 self.finish_xr_render_profile_frame(started);
3311 }
3312 return;
3313 }
3314
3315 let submit_started = profile.then(Instant::now);
3316 if let Err(e) =
3317 sess.frame_stream
3318 .end(frame_state.predicted_display_time, state.blend_mode, &[])
3319 {
3320 eprintln!("[OpenXR] end_frame failed: {e:?}");
3321 }
3322 if let Some(started) = submit_started {
3323 self.render_profile.submit += started.elapsed();
3324 }
3325 if let Some(started) = frame_profile_started {
3326 self.finish_xr_render_profile_frame(started);
3327 }
3328 }
3329}