Merge pull request #9087 from RaananW/fullScreenGuiAR

[XR] pointerX and pointerY update in XR

dist/preview release/what's new.md

@@ -207,6 +207,7 @@
 - Fixed an issue with moving backwards in XR ([#8854](https://github.com/BabylonJS/Babylon.js/issues/8854)) ([RaananW](https://github.com/RaananW))
 - Hit-Test results can be an empty array ([#8887](https://github.com/BabylonJS/Babylon.js/issues/8887)) ([RaananW](https://github.com/RaananW))
 - XR's main camera uses the first eye's projection matrix ([#8944](https://github.com/BabylonJS/Babylon.js/issues/8944)) ([RaananW](https://github.com/RaananW))
+- pointerX and pointerY of the scene are now updated when using the pointer selection feature ([#8879](https://github.com/BabylonJS/Babylon.js/issues/8879)) ([RaananW](https://github.com/RaananW))
 - XR tracking state was added to the camera ([#9076](https://github.com/BabylonJS/Babylon.js/issues/9076)) ([RaananW](https://github.com/RaananW))
 
 ### Collisions

src/Maths/math.vector.ts

@@ -1837,6 +1837,21 @@ export class Vector3 {
      * @returns the new Vector3
      */
     public static Project(vector: DeepImmutable<Vector3>, world: DeepImmutable<Matrix>, transform: DeepImmutable<Matrix>, viewport: DeepImmutable<Viewport>): Vector3 {
+        const result = new Vector3();
+        Vector3.ProjectToRef(vector, world, transform, viewport, result);
+        return result;
+    }
+
+    /**
+     * Project a Vector3 onto screen space to reference
+     * @param vector defines the Vector3 to project
+     * @param world defines the world matrix to use
+     * @param transform defines the transform (view x projection) matrix to use
+     * @param viewport defines the screen viewport to use
+     * @param result the vector in which the screen space will be stored
+     * @returns the new Vector3
+     */
+    public static ProjectToRef(vector: DeepImmutable<Vector3>, world: DeepImmutable<Matrix>, transform: DeepImmutable<Matrix>, viewport: DeepImmutable<Viewport>, result: DeepImmutable<Vector3>): Vector3 {
         var cw = viewport.width;
         var ch = viewport.height;
         var cx = viewport.x;
@@ -1854,7 +1869,8 @@ export class Vector3 {
         world.multiplyToRef(transform, matrix);
         matrix.multiplyToRef(viewportMatrix, matrix);
 
-        return Vector3.TransformCoordinates(vector, matrix);
+        Vector3.TransformCoordinatesToRef(vector, matrix, result);
+        return result;
     }
 
     /** @hidden */

src/XR/features/WebXRControllerPointerSelection.ts

@@ -7,7 +7,7 @@ import { WebXRInputSource } from "../webXRInputSource";
 import { Scene } from "../../scene";
 import { WebXRControllerComponent } from "../motionController/webXRControllerComponent";
 import { Nullable } from "../../types";
-import { Vector3 } from "../../Maths/math.vector";
+import { Matrix, Vector3 } from "../../Maths/math.vector";
 import { Color3 } from "../../Maths/math.color";
 import { Axis } from "../../Maths/math.axis";
 import { StandardMaterial } from "../../Materials/standardMaterial";
@@ -20,6 +20,7 @@ import { UtilityLayerRenderer } from "../../Rendering/utilityLayerRenderer";
 import { WebXRAbstractMotionController } from "../motionController/webXRAbstractMotionController";
 import { WebXRCamera } from "../webXRCamera";
 import { Node } from "../../node";
+import { Viewport } from "../../Maths/math.viewport";
 
 /**
  * Options interface for the pointer selection module
@@ -72,6 +73,14 @@ export interface IWebXRControllerPointerSelectionOptions {
      * the xr input to use with this pointer selection
      */
     xrInput: WebXRInput;
+
+    /**
+     * Should the scene pointerX and pointerY update be disabled
+     * This is required for fullscreen AR GUI, but might slow down other experiences.
+     * Disable in VR, if not needed.
+     * The first rig camera (left eye) will be used to calculate the projection
+     */
+    disableScenePointerVectorUpdate: boolean;
 }
 
 /**
@@ -281,18 +290,38 @@ export class WebXRControllerPointerSelection extends WebXRAbstractFeature {
         return null;
     }
 
+    private _identityMatrix = Matrix.Identity();
+    private _screenCoordinatesRef = Vector3.Zero();
+    private _viewportRef = new Viewport(0, 0, 0, 0);
+
     protected _onXRFrame(_xrFrame: XRFrame) {
         Object.keys(this._controllers).forEach((id) => {
             const controllerData = this._controllers[id];
 
+            let controllerGlobalPosition: Vector3;
+
             // Every frame check collisions/input
             if (controllerData.xrController) {
+                controllerGlobalPosition = controllerData.xrController.pointer.position;
                 controllerData.xrController.getWorldPointerRayToRef(controllerData.tmpRay);
             } else if (controllerData.webXRCamera) {
+                controllerGlobalPosition = controllerData.webXRCamera.position;
                 controllerData.webXRCamera.getForwardRayToRef(controllerData.tmpRay);
             } else {
                 return;
             }
+            // update pointerX and pointerY of the scene. Only if the flag is set to true!
+            if (!this._options.disableScenePointerVectorUpdate && controllerGlobalPosition) {
+                const scene = this._xrSessionManager.scene;
+                const camera = this._options.xrInput.xrCamera;
+                if (camera) {
+                    camera.viewport.toGlobalToRef(scene.getEngine().getRenderWidth(), scene.getEngine().getRenderHeight(), this._viewportRef);
+                    Vector3.ProjectToRef(controllerGlobalPosition, this._identityMatrix, scene.getTransformMatrix(), this._viewportRef, this._screenCoordinatesRef);
+
+                    scene.pointerX = this._screenCoordinatesRef.x;
+                    scene.pointerY = this._screenCoordinatesRef.y;
+                }
+            }
             controllerData.pick = this._scene.pickWithRay(controllerData.tmpRay, this._scene.pointerMovePredicate || this.raySelectionPredicate);
 
             const pick = controllerData.pick;

src/XR/features/WebXRControllerTeleportation.ts

@@ -764,7 +764,7 @@ export class WebXRMotionControllerTeleportation extends WebXRAbstractFeature {
 
     private _teleportForward(controllerId: string) {
         const controllerData = this._controllers[controllerId];
-        if (!controllerData.teleportationState.forward) {
+        if (!controllerData || !controllerData.teleportationState.forward) {
             return;
         }
         controllerData.teleportationState.forward = false;