Merge pull request #7260 from RaananW/ar-module

AR Pointer event simulation on screen target ray mode

dist/preview release/what's new.md

@@ -171,6 +171,8 @@
 - VRExperienceHelper has now an XR fallback to force XR usage (Beta) ([RaananW](https://github.com/RaananW/))
 - Added option to change the teleportation easing function in the VRExperienceHelper class ([https://github.com/LeoRodz](https://github.com/LeoRodz))
 - Windows motion controller mapping corrected to XR ([RaananW](https://github.com/RaananW/))
+- Pointer-Event simulation for screen target ray mode ([RaananW](https://github.com/RaananW/))
+- New observable that triggers when a session was initialized ([RaananW](https://github.com/RaananW/))
 
 ### Ray
 

src/Cameras/XR/webXRControllerPointerSelection.ts

@@ -63,6 +63,10 @@ export class WebXRControllerPointerSelection {
                             pressed = true;
                         }
                     }
+                    // in screen mode - means finger is on the screen
+                    if (controller.inputSource.targetRayMode === 'screen') {
+                        pressed = true;
+                    }
                     if (pressed) {
                         if (!triggerDown) {
                             scene.simulatePointerDown(pick, { pointerId: id });
@@ -106,7 +110,13 @@ export class WebXRControllerPointerSelection {
             controller.onDisposeObservable.addOnce(() => {
                 laserPointer.dispose();
                 cursorMesh.dispose();
-
+                if (controller.inputSource.targetRayMode === 'screen') {
+                    controller.getWorldPointerRayToRef(this._tmpRay);
+                    let pick = scene.pickWithRay(this._tmpRay);
+                    if (pick) {
+                        scene.simulatePointerUp(pick, { pointerId: id });
+                    }
+                }
                 scene.onBeforeRenderObservable.remove(renderObserver);
             });
         });

src/Cameras/XR/webXRDefaultExperience.ts

@@ -5,7 +5,7 @@ import { WebXRControllerModelLoader } from './webXRControllerModelLoader';
 import { WebXRControllerPointerSelection } from './webXRControllerPointerSelection';
 import { WebXRControllerTeleportation } from './webXRControllerTeleportation';
 import { WebXRRenderTarget } from './webXRTypes';
-import { WebXREnterExitUI } from './webXREnterExitUI';
+import { WebXREnterExitUI, WebXREnterExitUIOptions } from './webXREnterExitUI';
 import { AbstractMesh } from '../../Meshes/abstractMesh';
 import { WebXRManagedOutputCanvasOptions } from './webXRManagedOutputCanvas';
 
@@ -26,7 +26,12 @@ export class WebXRDefaultExperienceOptions {
     /**
      * optional configuration for the output canvas
      */
-    public outputCanvasOptions? : WebXRManagedOutputCanvasOptions;
+    public outputCanvasOptions?: WebXRManagedOutputCanvasOptions;
+
+    /**
+     * optional UI options. This can be used among other to change session mode and reference space type
+     */
+    public uiOptions?: WebXREnterExitUIOptions;
 }
 
 /**
@@ -54,7 +59,7 @@ export class WebXRDefaultExperience {
      */
     public teleportation: WebXRControllerTeleportation;
     /**
-     * Enables ui for enetering/exiting xr
+     * Enables ui for entering/exiting xr
      */
     public enterExitUI: WebXREnterExitUI;
     /**
@@ -88,8 +93,11 @@ export class WebXRDefaultExperience {
             result.renderTarget = result.baseExperience.sessionManager.getWebXRRenderTarget(xrHelper.onStateChangedObservable, options.outputCanvasOptions);
 
             if (!options.disableDefaultUI) {
+                if (options.uiOptions) {
+                    options.uiOptions.renderTarget = options.uiOptions.renderTarget || result.renderTarget;
+                }
                 // Create ui for entering/exiting xr
-                return WebXREnterExitUI.CreateAsync(scene, result.baseExperience, { renderTarget: result.renderTarget }).then((ui) => {
+                return WebXREnterExitUI.CreateAsync(scene, result.baseExperience, options.uiOptions || { renderTarget: result.renderTarget }).then((ui) => {
                     result.enterExitUI = ui;
                 });
             } else {

src/Cameras/XR/webXREnterExitUI.ts

@@ -117,7 +117,16 @@ export class WebXREnterExitUI implements IDisposable {
         });
     }
 
-    private constructor(private scene: Scene, options: WebXREnterExitUIOptions) {
+    /**
+     *
+     * @param scene babylon scene object to use
+     * @param options (read-only) version of the options passed to this UI
+     */
+    private constructor(
+        private scene: Scene,
+        /** version of the options passed to this UI */
+        public options: WebXREnterExitUIOptions
+    ) {
         this._overlay = document.createElement("div");
         this._overlay.style.cssText = "z-index:11;position: absolute; right: 20px;bottom: 50px;";
 

src/Cameras/XR/webXRSessionManager.ts

@@ -31,11 +31,15 @@ export class WebXRSessionManager implements IDisposable {
     /**
      * Fires every time a new xrFrame arrives which can be used to update the camera
      */
-    public onXRFrameObservable: Observable<any> = new Observable<any>();
+    public onXRFrameObservable: Observable<XRFrame> = new Observable<XRFrame>();
     /**
      * Fires when the xr session is ended either by the device or manually done
      */
     public onXRSessionEnded: Observable<any> = new Observable<any>();
+    /**
+     * Fires when the xr session is ended either by the device or manually done
+     */
+    public onXRSessionInit: Observable<XRSession> = new Observable<XRSession>();
 
     /**
      * Underlying xr session
@@ -62,7 +66,10 @@ export class WebXRSessionManager implements IDisposable {
      * Constructs a WebXRSessionManager, this must be initialized within a user action before usage
      * @param scene The scene which the session should be created for
      */
-    constructor(private scene: Scene) {
+    constructor(
+        /** The scene which the session should be created for */
+        public scene: Scene
+    ) {
 
     }
 
@@ -90,6 +97,7 @@ export class WebXRSessionManager implements IDisposable {
     public initializeSessionAsync(xrSessionMode: XRSessionMode, optionalFeatures: any = {}): Promise<XRSession> {
         return this._xrNavigator.xr.requestSession(xrSessionMode, optionalFeatures).then((session: XRSession) => {
             this.session = session;
+            this.onXRSessionInit.notifyObservers(session);
             this._sessionEnded = false;
 
             // handle when the session is ended (By calling session.end or device ends its own session eg. pressing home button on phone)
@@ -158,7 +166,9 @@ export class WebXRSessionManager implements IDisposable {
                 }
                 // Store the XR frame in the manager to be consumed by the XR camera to update pose
                 this.currentFrame = xrFrame;
-                this.onXRFrameObservable.notifyObservers(null);
+                if (xrFrame) {
+                    this.onXRFrameObservable.notifyObservers(xrFrame);
+                }
                 this.scene.getEngine()._renderLoop();
             }
         };
@@ -231,6 +241,7 @@ export class WebXRSessionManager implements IDisposable {
      * Converts the render layer of xrSession to a render target
      * @param session session to create render target for
      * @param scene scene the new render target should be created for
+     * @param baseLayer the webgl layer to create the render target for
      */
     public static _CreateRenderTargetTextureFromSession(session: XRSession, scene: Scene, baseLayer: XRWebGLLayer) {
         if (!baseLayer) {

src/LibDeclarations/webxr.d.ts

@@ -123,4 +123,9 @@ interface XRInputSourceChangeEvent {
     session: XRSession;
     removed: Array<XRInputSource>;
     added: Array<XRInputSource>;
+}
+
+interface XRInputSourceEvent extends Event {
+    readonly frame: XRFrame;
+    readonly inputSource: XRInputSource;
 }