Merge pull request #7406 from RaananW/xr-other-ray-modes

WebXR - Other Ray Modes in Select and Trigger teleportation

dist/preview release/what's new.md

@@ -190,6 +190,7 @@
 - Teleportation allows selecting direction before teleporting when using thumbstick/touchpad. ([#7290](https://github.com/BabylonJS/Babylon.js/issues/7290)) ([RaananW](https://github.com/RaananW/))
 - It is now possible to force a certain profile type for the controllers ([#7348](https://github.com/BabylonJS/Babylon.js/issues/7375)) ([RaananW](https://github.com/RaananW/))
 - WebXR camera is initialized on the first frame ([#7389](https://github.com/BabylonJS/Babylon.js/issues/7389)) ([RaananW](https://github.com/RaananW/))
+- Selection has forcable gaze mode and touch-screen support ([#7395](https://github.com/BabylonJS/Babylon.js/issues/7395)) ([RaananW](https://github.com/RaananW/))
 
 ### Ray
 

src/Cameras/XR/features/WebXRControllerPointerSelection.ts

@@ -30,6 +30,32 @@ export interface IWebXRControllerPointerSelectionOptions {
      * Different button type to use instead of the main component
      */
     overrideButtonId?: string;
+    /**
+     * The amount of time in miliseconds it takes between pick found something to a pointer down event.
+     * Used in gaze modes. Tracked pointer uses the trigger, screen uses touch events
+     * 3000 means 3 seconds between pointing at something and selecting it
+     */
+    timeToSelect?: number;
+
+    /**
+     * Disable the pointer up event when the xr controller in screen and gaze mode is disposed (meaning - when the user removed the finger from the screen)
+     * If not disabled, the last picked point will be used to execute a pointer up event
+     * If disabled, pointer up event will be triggered right after the pointer down event.
+     * Used in screen and gaze target ray mode only
+     */
+    disablePointerUpOnTouchOut: boolean;
+
+    /**
+     * For gaze mode (time to select instead of press)
+     */
+    forceGazeMode: boolean;
+
+    /**
+     * Factor to be applied to the pointer-moved function in the gaze mode. How sensitive should the gaze mode be when checking if the pointer moved
+     * to start a new countdown to the pointer down event.
+     * Defaults to 1.
+     */
+    gazeModePointerMovedFactor?: number;
 }
 
 /**
@@ -51,11 +77,11 @@ export class WebXRControllerPointerSelection implements IWebXRFeature {
     /**
      * This color will be set to the laser pointer when selection is triggered
      */
-    public onPickedLaserPointerColor: Color3 = new Color3(0.7, 0.7, 0.7);
+    public laserPointerPickedColor: Color3 = new Color3(0.7, 0.7, 0.7);
     /**
      * This color will be applied to the selection ring when selection is triggered
      */
-    public onPickedSelectionMeshColor: Color3 = new Color3(0.7, 0.7, 0.7);
+    public selectionMeshPickedColor: Color3 = new Color3(0.7, 0.7, 0.7);
     /**
      * default color of the selection ring
      */
@@ -76,6 +102,7 @@ export class WebXRControllerPointerSelection implements IWebXRFeature {
             xrController: WebXRController;
             selectionComponent?: WebXRControllerComponent;
             onButtonChangedObserver?: Nullable<Observer<WebXRControllerComponent>>;
+            onFrameObserver?: Nullable<Observer<XRFrame>>;
             laserPointer: AbstractMesh;
             selectionMesh: AbstractMesh;
             pick: Nullable<PickingInfo>;
@@ -124,14 +151,6 @@ export class WebXRControllerPointerSelection implements IWebXRFeature {
                 controllerData.xrController.getWorldPointerRayToRef(this._tmpRay);
                 controllerData.pick = this._scene.pickWithRay(this._tmpRay);
 
-                if (controllerData.selectionComponent && controllerData.selectionComponent.pressed) {
-                    (<StandardMaterial>controllerData.selectionMesh.material).emissiveColor = this.onPickedSelectionMeshColor;
-                    (<StandardMaterial>controllerData.laserPointer.material).emissiveColor = this.onPickedLaserPointerColor;
-                } else {
-                    (<StandardMaterial>controllerData.selectionMesh.material).emissiveColor = this.selectionMeshDefaultColor;
-                    (<StandardMaterial>controllerData.laserPointer.material).emissiveColor = this.lasterPointerDefaultColor;
-                }
-
                 const pick = controllerData.pick;
 
                 if (pick && pick.pickedPoint && pick.hit) {
@@ -187,17 +206,29 @@ export class WebXRControllerPointerSelection implements IWebXRFeature {
         return true;
     }
 
-    private _attachController = (xrController: WebXRController) => {
-        // only support tracker pointer
-        if (xrController.inputSource.targetRayMode !== "tracked-pointer") {
-            return;
+    /**
+     * Get the xr controller that correlates to the pointer id in the pointer event
+     *
+     * @param id the pointer id to search for
+     * @returns the controller that correlates to this id or null if not found
+     */
+    public getXRControllerByPointerId(id: number): Nullable<WebXRController> {
+        const keys = Object.keys(this._controllers);
+
+        for (let i = 0; i < keys.length; ++i) {
+            if (this._controllers[keys[i]].id === id) {
+                return this._controllers[keys[i]].xrController;
+            }
         }
+        return null;
+    }
 
-        if (this._controllers[xrController.uniqueId] || !xrController.gamepadController) {
+    private _attachController = (xrController: WebXRController) => {
+        if (this._controllers[xrController.uniqueId]) {
             // already attached
             return;
         }
-
+        // only support tracker pointer
         const { laserPointer, selectionMesh } = this._generateNewMeshPair(xrController);
 
         // get two new meshes
@@ -208,6 +239,124 @@ export class WebXRControllerPointerSelection implements IWebXRFeature {
             pick: null,
             id: WebXRControllerPointerSelection._idCounter++
         };
+        switch (xrController.inputSource.targetRayMode) {
+            case "tracked-pointer":
+                return this._attachTrackedPointerRayMode(xrController);
+            case "gaze":
+                return this._attachGazeMode(xrController);
+            case "screen":
+                return this._attachScreenRayMode(xrController);
+        }
+    }
+
+    private _attachScreenRayMode(xrController: WebXRController) {
+        const controllerData = this._controllers[xrController.uniqueId];
+        let downTriggered = false;
+        controllerData.onFrameObserver = this._xrSessionManager.onXRFrameObservable.add(() => {
+            if (!controllerData.pick || (this._options.disablePointerUpOnTouchOut && downTriggered)) { return; }
+            if (!downTriggered) {
+                this._scene.simulatePointerDown(controllerData.pick, { pointerId: controllerData.id });
+                downTriggered = true;
+                if (this._options.disablePointerUpOnTouchOut) {
+                    this._scene.simulatePointerUp(controllerData.pick, { pointerId: controllerData.id });
+                }
+            } else {
+                this._scene.simulatePointerMove(controllerData.pick, { pointerId: controllerData.id });
+            }
+        });
+        xrController.onDisposeObservable.addOnce(() => {
+            if (controllerData.pick && downTriggered && !this._options.disablePointerUpOnTouchOut) {
+                this._scene.simulatePointerUp(controllerData.pick, { pointerId: controllerData.id });
+            }
+        });
+    }
+
+    private _attachGazeMode(xrController: WebXRController) {
+        const controllerData = this._controllers[xrController.uniqueId];
+        // attached when touched, detaches when raised
+        const timeToSelect = this._options.timeToSelect || 3000;
+        let oldPick = new PickingInfo();
+        let discMesh = TorusBuilder.CreateTorus("selection", {
+            diameter: 0.0035 * 15,
+            thickness: 0.0025 * 6,
+            tessellation: 20
+        }, this._scene);
+        discMesh.isVisible = false;
+        discMesh.isPickable = false;
+        discMesh.parent = controllerData.selectionMesh;
+        let timer = 0;
+        let downTriggered = false;
+        controllerData.onFrameObserver = this._xrSessionManager.onXRFrameObservable.add(() => {
+            if (!controllerData.pick) { return; }
+            discMesh.isVisible = false;
+            if (controllerData.pick.hit) {
+                if (!this._pickingMoved(oldPick, controllerData.pick)) {
+                    if (timer > timeToSelect / 10) {
+                        discMesh.isVisible = true;
+                    }
+
+                    timer += this._scene.getEngine().getDeltaTime();
+                    if (timer >= timeToSelect) {
+                        this._scene.simulatePointerDown(controllerData.pick, { pointerId: controllerData.id });
+                        downTriggered = true;
+                        // pointer up right after down, if disable on touch out
+                        if (this._options.disablePointerUpOnTouchOut) {
+                            this._scene.simulatePointerUp(controllerData.pick, { pointerId: controllerData.id });
+                        } else {
+                            this._scene.simulatePointerMove(controllerData.pick, { pointerId: controllerData.id });
+                        }
+                        discMesh.isVisible = false;
+                    } else {
+                        const scaleFactor = 1 - (timer / timeToSelect);
+                        discMesh.scaling.set(scaleFactor, scaleFactor, scaleFactor);
+                    }
+                } else {
+                    if (downTriggered) {
+                        if (!this._options.disablePointerUpOnTouchOut) {
+                            this._scene.simulatePointerUp(controllerData.pick, { pointerId: controllerData.id });
+                        }
+                    }
+                    downTriggered = false;
+                    timer = 0;
+                }
+            } else {
+                downTriggered = false;
+                timer = 0;
+            }
+
+            oldPick = controllerData.pick;
+        });
+        xrController.onDisposeObservable.addOnce(() => {
+            if (controllerData.pick && !this._options.disablePointerUpOnTouchOut && downTriggered) {
+                this._scene.simulatePointerUp(controllerData.pick, { pointerId: controllerData.id });
+            }
+            discMesh.dispose();
+        });
+    }
+    private _tmpVectorForPickCompare = new Vector3();
+
+    private _pickingMoved(oldPick: PickingInfo, newPick: PickingInfo) {
+        if (!oldPick.hit || !newPick.hit) { return true; }
+        if (!oldPick.pickedMesh || !oldPick.pickedPoint || !newPick.pickedMesh || !newPick.pickedPoint) { return true; }
+        if (oldPick.pickedMesh !== newPick.pickedMesh) { return true; }
+        oldPick.pickedPoint?.subtractToRef(newPick.pickedPoint, this._tmpVectorForPickCompare);
+        this._tmpVectorForPickCompare.set(Math.abs(this._tmpVectorForPickCompare.x), Math.abs(this._tmpVectorForPickCompare.y), Math.abs(this._tmpVectorForPickCompare.z));
+        const delta = (this._options.gazeModePointerMovedFactor || 1) * 0.01 / newPick.distance;
+        const length = this._tmpVectorForPickCompare.length();
+        if (length > delta) { return true; }
+        return false;
+
+    }
+
+    private _attachTrackedPointerRayMode(xrController: WebXRController) {
+        if (!xrController.gamepadController) {
+            return;
+        }
+
+        if (this._options.forceGazeMode) {
+            return this._attachGazeMode(xrController);
+        }
+
         const controllerData = this._controllers[xrController.uniqueId];
 
         if (this._options.overrideButtonId) {
@@ -219,6 +368,16 @@ export class WebXRControllerPointerSelection implements IWebXRFeature {
 
         let observer: Nullable<Observer<XRFrame>> = null;
 
+        controllerData.onFrameObserver = this._xrSessionManager.onXRFrameObservable.add(() => {
+            if (controllerData.selectionComponent && controllerData.selectionComponent.pressed) {
+                (<StandardMaterial>controllerData.selectionMesh.material).emissiveColor = this.selectionMeshPickedColor;
+                (<StandardMaterial>controllerData.laserPointer.material).emissiveColor = this.laserPointerPickedColor;
+            } else {
+                (<StandardMaterial>controllerData.selectionMesh.material).emissiveColor = this.selectionMeshDefaultColor;
+                (<StandardMaterial>controllerData.laserPointer.material).emissiveColor = this.lasterPointerDefaultColor;
+            }
+        });
+
         controllerData.onButtonChangedObserver = controllerData.selectionComponent.onButtonStateChanged.add((component) => {
             if (component.changes.pressed) {
                 const pressed = component.changes.pressed.current;
@@ -237,7 +396,6 @@ export class WebXRControllerPointerSelection implements IWebXRFeature {
                 }
             }
         });
-
     }
 
     private _detachController(xrControllerUniqueId: string) {
@@ -248,6 +406,9 @@ export class WebXRControllerPointerSelection implements IWebXRFeature {
                 controllerData.selectionComponent.onButtonStateChanged.remove(controllerData.onButtonChangedObserver);
             }
         }
+        if (controllerData.onFrameObserver) {
+            this._xrSessionManager.onXRFrameObservable.remove(controllerData.onFrameObserver);
+        }
         controllerData.selectionMesh.dispose();
         controllerData.laserPointer.dispose();
         // remove from the map

src/Cameras/XR/features/WebXRControllerTeleportation.ts

@@ -65,6 +65,17 @@ export interface IWebXRTeleportationOptions {
          */
         disableAnimation?: boolean;
     };
+
+    /**
+     * Disable using the thumbstick and use the main component (usuallly trigger) on long press.
+     * This will be automatically true if the controller doesnt have a thumbstick or touchpad.
+     */
+    useMainComponentOnly?: boolean;
+
+    /**
+     * If main component is used (no thumbstick), how long should the "long press" take before teleporting
+     */
+    timeToTeleport?: number;
 }
 
 /**
@@ -325,20 +336,48 @@ export class WebXRMotionControllerTeleportation implements IWebXRFeature {
         // motion controller support
         if (xrController.gamepadController) {
             const movementController = xrController.gamepadController.getComponent(WebXRControllerComponent.THUMBSTICK) || xrController.gamepadController.getComponent(WebXRControllerComponent.TOUCHPAD);
-            if (!movementController) {
+            if (!movementController || this._options.useMainComponentOnly) {
                 // use trigger to move on long press
+                const mainComponent = xrController.gamepadController.getMainComponent();
+                if (!mainComponent) {
+                    return;
+                }
+                controllerData.onButtonChangedObserver = mainComponent.onButtonStateChanged.add(() => {
+                    // did "pressed" changed?
+                    if (mainComponent.changes.pressed) {
+                        if (mainComponent.changes.pressed.current) {
+                            // simulate "forward" thumbstick push
+                            controllerData.teleportationState.forward = true;
+                            this._currentTeleportationControllerId = controllerData.xrController.uniqueId;
+                            controllerData.teleportationState.baseRotation = this._options.xrInput.xrCamera.rotationQuaternion.toEulerAngles().y;
+                            controllerData.teleportationState.currentRotation = 0;
+                            const timeToSelect = this._options.timeToTeleport || 3000;
+                            let timer = 0;
+                            const observer = this._xrSessionManager.onXRFrameObservable.add(() => {
+                                if (!mainComponent.pressed) {
+                                    this._xrSessionManager.onXRFrameObservable.remove(observer);
+                                    return;
+                                }
+                                timer += this._xrSessionManager.scene.getEngine().getDeltaTime();
+                                if (timer >= timeToSelect && this._currentTeleportationControllerId === controllerData.xrController.uniqueId && controllerData.teleportationState.forward) {
+                                    this._teleportForward(xrController.uniqueId);
+                                }
+
+                                // failsafe
+                                if (timer >= timeToSelect) {
+                                    this._xrSessionManager.onXRFrameObservable.remove(observer);
+                                }
+                            });
+                        } else {
+                            controllerData.teleportationState.forward = false;
+                            this._currentTeleportationControllerId = "";
+                        }
+                    }
+                });
             } else {
                 controllerData.onButtonChangedObserver = movementController.onButtonStateChanged.add(() => {
                     if (this._currentTeleportationControllerId === controllerData.xrController.uniqueId && controllerData.teleportationState.forward && !movementController.touched) {
-                        controllerData.teleportationState.forward = false;
-                        this._currentTeleportationControllerId = "";
-                        // do the movement forward here
-                        if (this._options.teleportationTargetMesh && this._options.teleportationTargetMesh.isVisible) {
-                            const height = this._options.xrInput.xrCamera.position.y - this._options.teleportationTargetMesh.position.y;
-                            this._options.xrInput.xrCamera.position.copyFrom(this._options.teleportationTargetMesh.position);
-                            this._options.xrInput.xrCamera.position.y += height;
-                            this._options.xrInput.xrCamera.rotationQuaternion.multiplyInPlace(Quaternion.FromEulerAngles(0, controllerData.teleportationState.currentRotation, 0));
-                        }
+                        this._teleportForward(xrController.uniqueId);
                     }
                 });
                 // use thumbstick (or touchpad if thumbstick not available)
@@ -390,6 +429,8 @@ export class WebXRMotionControllerTeleportation implements IWebXRFeature {
                                     setTimeout(() => {
                                         controllerData.teleportationState.currentRotation = Math.atan2(axesData.x, -axesData.y);
                                     });
+                                } else {
+                                    controllerData.teleportationState.currentRotation = 0;
                                 }
                             }
                         }
@@ -401,6 +442,19 @@ export class WebXRMotionControllerTeleportation implements IWebXRFeature {
         }
     }
 
+    private _teleportForward(controllerId: string) {
+        const controllerData = this._controllers[controllerId];
+        controllerData.teleportationState.forward = false;
+        this._currentTeleportationControllerId = "";
+        // do the movement forward here
+        if (this._options.teleportationTargetMesh && this._options.teleportationTargetMesh.isVisible) {
+            const height = this._options.xrInput.xrCamera.position.y - this._options.teleportationTargetMesh.position.y;
+            this._options.xrInput.xrCamera.position.copyFrom(this._options.teleportationTargetMesh.position);
+            this._options.xrInput.xrCamera.position.y += height;
+            this._options.xrInput.xrCamera.rotationQuaternion.multiplyInPlace(Quaternion.FromEulerAngles(0, controllerData.teleportationState.currentRotation, 0));
+        }
+    }
+
     private _detachController(xrControllerUniqueId: string) {
         const controllerData = this._controllers[xrControllerUniqueId];
         if (!controllerData) { return; }

src/Cameras/XR/webXRFeaturesManager.ts

@@ -183,27 +183,24 @@ export class WebXRFeaturesManager implements IDisposable {
         }
         // check if already initialized
         const feature = this._features[name];
-        if (!feature || !feature.featureImplementation || feature.version !== versionToLoad) {
-            const constructFunction = WebXRFeaturesManager.ConstructFeature(name, versionToLoad, this._xrSessionManager, moduleOptions);
-            if (!constructFunction) {
-                // report error?
-                throw new Error(`feature not found - ${name}`);
-            }
-
-            if (feature) {
-                this.disableFeature(name);
-            }
+        const constructFunction = WebXRFeaturesManager.ConstructFeature(name, versionToLoad, this._xrSessionManager, moduleOptions);
+        if (!constructFunction) {
+            // report error?
+            throw new Error(`feature not found - ${name}`);
+        }
 
-            this._features[name] = {
-                featureImplementation: constructFunction(),
-                enabled: true,
-                version: versionToLoad
-            };
-        } else {
-            // make sure it is enabled now:
-            feature.enabled = true;
+        /* If the feature is already enabled, detach and dispose it, and create a new one */
+        if (feature) {
+            this.disableFeature(name);
+            feature.featureImplementation.dispose();
         }
 
+        this._features[name] = {
+            featureImplementation: constructFunction(),
+            enabled: true,
+            version: versionToLoad
+        };
+
         // if session started already, request and enable
         if (this._xrSessionManager.session && !feature.featureImplementation.attached && attachIfPossible) {
             // enable feature