Merge pull request #6965 from syntheticmagus/nativeWebXR

WebXR changes for Babylon Native

dist/preview release/what's new.md

@@ -161,7 +161,7 @@
 - Fixed Textblock line spacing evaluation when linespacing > 0 ([Deltakosh](https://github.com/deltakosh/))
 - Fixed Xbox One gamepad controller button schemes ([MackeyK24](https://github.com/MackeyK24/))
 - Added support for `AnimationGroup` serialization ([Drigax](https://github.com/drigax/))
-- Removing assetContainer from scene will also remove gui layers ([TrevorDev](https://github.com/TrevorDev))
+- Removing `assetContainer` from scene will also remove gui layers ([TrevorDev](https://github.com/TrevorDev))
 - A scene's input manager not adding key listeners when the canvas is already focused ([Poolminer](https://github.com/Poolminer))
 - Runtime animation `goToFrame` when going back in time now correctly triggers future events when reached ([zakhenry](https://github.com/zakhenry))
 - Fixed bug in `Ray.intersectsTriangle` where the barycentric coordinates `bu` and `bv` being returned is actually `bv` and `bw`. ([bghgary](https://github.com/bghgary))

src/Cameras/XR/index.ts

@@ -7,4 +7,5 @@ export * from "./webXRControllerPointerSelection";
 export * from "./webXRControllerModelLoader";
 export * from "./webXRController";
 export * from "./webXRManagedOutputCanvas";
+export * from "./webXRTypes";
 export * from "./webXRSessionManager";

src/Cameras/XR/webXRCamera.ts

@@ -119,8 +119,7 @@ export class WebXRCamera extends FreeCamera {
             }
 
             // Set cameras to render to the session's render target
-            currentRig.outputRenderTarget = xrSessionManager._sessionRenderTargetTexture;
-
+            currentRig.outputRenderTarget = xrSessionManager.getRenderTargetTextureForEye(view.eye);
         });
         return true;
     }

src/Cameras/XR/webXRDefaultExperience.ts

@@ -4,9 +4,10 @@ import { WebXRInput } from './webXRInput';
 import { WebXRControllerModelLoader } from './webXRControllerModelLoader';
 import { WebXRControllerPointerSelection } from './webXRControllerPointerSelection';
 import { WebXRControllerTeleportation } from './webXRControllerTeleportation';
-import { WebXRManagedOutputCanvas } from './webXRManagedOutputCanvas';
+import { WebXRRenderTarget } from './webXRTypes';
 import { WebXREnterExitUI } from './webXREnterExitUI';
 import { AbstractMesh } from '../../Meshes/abstractMesh';
+
 /**
  * Options for the default xr helper
  */
@@ -15,6 +16,11 @@ export class WebXRDefaultExperienceOptions {
      * Floor meshes that should be used for teleporting
      */
     public floorMeshes: Array<AbstractMesh>;
+
+    /**
+     * Enable or disable default UI to enter XR
+     */
+    public disableDefaultUI: boolean;
 }
 
 /**
@@ -46,9 +52,9 @@ export class WebXRDefaultExperience {
      */
     public enterExitUI: WebXREnterExitUI;
     /**
-     * Default output canvas xr should render to
+     * Default target xr should render to
      */
-    public outputCanvas: WebXRManagedOutputCanvas;
+    public renderTarget: WebXRRenderTarget;
 
     /**
      * Creates the default xr experience
@@ -67,15 +73,23 @@ export class WebXRDefaultExperience {
             result.input = new WebXRInput(xrHelper);
             result.controllerModelLoader = new WebXRControllerModelLoader(result.input);
             result.pointerSelection = new WebXRControllerPointerSelection(result.input);
-            result.teleportation = new WebXRControllerTeleportation(result.input, options.floorMeshes);
 
-            // Create output canvas manager (this controls where the xr frames will be rendered)
-            result.outputCanvas = new WebXRManagedOutputCanvas(xrHelper, scene.getEngine().getRenderingCanvas() as HTMLCanvasElement);
+            if (options.floorMeshes) {
+                result.teleportation = new WebXRControllerTeleportation(result.input, options.floorMeshes);
+            }
+
+            // Create the WebXR output target
+            result.renderTarget = result.baseExperience.sessionManager.getWebXRRenderTarget(xrHelper.onStateChangedObservable);
 
-            // Create ui for entering/exiting xr
-            return WebXREnterExitUI.CreateAsync(scene, result.baseExperience, {webXRManagedOutputCanvas: result.outputCanvas});
-        }).then((ui) => {
-            result.enterExitUI = ui;
+            if (!options.disableDefaultUI) {
+                // Create ui for entering/exiting xr
+                return WebXREnterExitUI.CreateAsync(scene, result.baseExperience, { renderTarget: result.renderTarget }).then((ui) => {
+                    result.enterExitUI = ui;
+                });
+            } else {
+                return;
+            }
+        }).then(() => {
             return result;
         });
     }
@@ -97,8 +111,8 @@ export class WebXRDefaultExperience {
         if (this.enterExitUI) {
             this.enterExitUI.dispose();
         }
-        if (this.outputCanvas) {
-            this.outputCanvas.dispose();
+        if (this.renderTarget) {
+            this.renderTarget.dispose();
         }
     }
 }

src/Cameras/XR/webXREnterExitUI.ts

@@ -1,8 +1,8 @@
 import { Nullable } from "../../types";
 import { Observable } from "../../Misc/observable";
 import { IDisposable, Scene } from "../../scene";
-import { WebXRExperienceHelper, WebXRState } from "./webXRExperienceHelper";
-import { WebXRManagedOutputCanvas } from '../XR/webXRManagedOutputCanvas';
+import { WebXRExperienceHelper } from "./webXRExperienceHelper";
+import { WebXRState, WebXRRenderTarget } from './webXRTypes';
 /**
  * Button which can be used to enter a different mode of XR
  */
@@ -36,7 +36,7 @@ export class WebXREnterExitUIOptions {
     /**
      * Context to enter xr with
      */
-    webXRManagedOutputCanvas?: Nullable<WebXRManagedOutputCanvas>;
+    renderTarget?: Nullable<WebXRRenderTarget>;
 
     /**
      * User provided buttons to enable/disable WebXR. The system will provide default if not set
@@ -86,8 +86,8 @@ export class WebXREnterExitUI implements IDisposable {
                             return;
                         } else if (helper.state == WebXRState.NOT_IN_XR) {
                             ui._updateButtons(ui._buttons[i]);
-                            if (options.webXRManagedOutputCanvas) {
-                                await helper.enterXRAsync(ui._buttons[i].sessionMode, ui._buttons[i].referenceSpaceType, options.webXRManagedOutputCanvas);
+                            if (options.renderTarget) {
+                                await helper.enterXRAsync(ui._buttons[i].sessionMode, ui._buttons[i].referenceSpaceType, options.renderTarget);
                             }
                         }
                     };

src/Cameras/XR/webXRExperienceHelper.ts

@@ -6,28 +6,8 @@ import { AbstractMesh } from "../../Meshes/abstractMesh";
 import { Camera } from "../../Cameras/camera";
 import { WebXRSessionManager } from "./webXRSessionManager";
 import { WebXRCamera } from "./webXRCamera";
-import { WebXRManagedOutputCanvas } from './webXRManagedOutputCanvas';
-/**
- * States of the webXR experience
- */
-export enum WebXRState {
-    /**
-     * Transitioning to being in XR mode
-     */
-    ENTERING_XR,
-    /**
-     * Transitioning to non XR mode
-     */
-    EXITING_XR,
-    /**
-     * In XR mode and presenting
-     */
-    IN_XR,
-    /**
-     * Not entered XR mode
-     */
-    NOT_IN_XR
-}
+import { WebXRState, WebXRRenderTarget } from './webXRTypes';
+
 /**
  * Base set of functionality needed to create an XR experince (WebXRSessionManager, Camera, StateManagement, etc.)
  * @see https://doc.babylonjs.com/how_to/webxr
@@ -110,10 +90,10 @@ export class WebXRExperienceHelper implements IDisposable {
      * Enters XR mode (This must be done within a user interaction in most browsers eg. button click)
      * @param sessionCreationOptions options for the XR session
      * @param referenceSpaceType frame of reference of the XR session
-     * @param outputCanvas the output canvas that will be used to enter XR mode
+     * @param renderTarget the output canvas that will be used to enter XR mode
      * @returns promise that resolves after xr mode has entered
      */
-    public enterXRAsync(sessionCreationOptions: XRSessionMode, referenceSpaceType: XRReferenceSpaceType, outputCanvas: WebXRManagedOutputCanvas) {
+    public enterXRAsync(sessionCreationOptions: XRSessionMode, referenceSpaceType: XRReferenceSpaceType, renderTarget: WebXRRenderTarget) {
         if (!this._supported) {
             throw "XR session not supported by this browser";
         }
@@ -121,9 +101,9 @@ export class WebXRExperienceHelper implements IDisposable {
         return this.sessionManager.initializeSessionAsync(sessionCreationOptions).then(() => {
             return this.sessionManager.setReferenceSpaceAsync(referenceSpaceType);
         }).then(() => {
-            return outputCanvas.initializeXRLayerAsync(this.sessionManager.session);
+            return renderTarget.initializeXRLayerAsync(this.sessionManager.session);
         }).then(() => {
-            return this.sessionManager.updateRenderStateAsync({ depthFar: this.camera.maxZ, depthNear: this.camera.minZ, baseLayer: outputCanvas.xrLayer! });
+            return this.sessionManager.updateRenderStateAsync({ depthFar: this.camera.maxZ, depthNear: this.camera.minZ, baseLayer: renderTarget.xrLayer! });
         }).then(() => {
             return this.sessionManager.startRenderingToXRAsync();
         }).then(() => {

src/Cameras/XR/webXRInput.ts

@@ -1,8 +1,9 @@
 import { Nullable } from "../../types";
 import { Observer, Observable } from "../../Misc/observable";
 import { IDisposable } from "../../scene";
-import { WebXRExperienceHelper, WebXRState } from "./webXRExperienceHelper";
+import { WebXRExperienceHelper } from "./webXRExperienceHelper";
 import { WebXRController } from './webXRController';
+import { WebXRState } from './webXRTypes';
 
 /**
  * XR input used to track XR inputs such as controllers/rays

src/Cameras/XR/webXRManagedOutputCanvas.ts

@@ -1,11 +1,16 @@
 import { Nullable } from "../../types";
-import { IDisposable } from "../../scene";
-import { WebXRExperienceHelper, WebXRState } from "./webXRExperienceHelper";
+import { Observable } from "../../Misc/observable";
+import { ThinEngine } from '../../Engines/thinEngine';
+import { WebXRState, WebXRRenderTarget } from "./webXRTypes";
+
 /**
  * Creates a canvas that is added/removed from the webpage when entering/exiting XR
  */
-export class WebXRManagedOutputCanvas implements IDisposable {
+export class WebXRManagedOutputCanvas implements WebXRRenderTarget {
+
+    private _engine: ThinEngine;
     private _canvas: Nullable<HTMLCanvasElement> = null;
+
     /**
      * xrpresent context of the canvas which can be used to display/mirror xr content
      */
@@ -34,20 +39,22 @@ export class WebXRManagedOutputCanvas implements IDisposable {
 
     /**
      * Initializes the canvas to be added/removed upon entering/exiting xr
-     * @param helper the xr experience helper used to trigger adding/removing of the canvas
+     * @param engine the Babylon engine
+     * @param onStateChangedObservable the mechanism by which the canvas will be added/removed based on XR state
      * @param canvas The canvas to be added/removed (If not specified a full screen canvas will be created)
      */
-    constructor(private helper: WebXRExperienceHelper, canvas?: HTMLCanvasElement) {
+    constructor(engine: ThinEngine, onStateChangedObservable: Observable<WebXRState>, canvas?: HTMLCanvasElement) {
+        this._engine = engine;
         if (!canvas) {
             canvas = document.createElement('canvas');
             canvas.style.cssText = "position:absolute; bottom:0px;right:0px;z-index:10;width:90%;height:100%;background-color: #000000;";
         }
         this._setManagedOutputCanvas(canvas);
-        helper.onStateChangedObservable.add((stateInfo) => {
+        onStateChangedObservable.add((stateInfo) => {
             if (stateInfo == WebXRState.ENTERING_XR) {
                 // The canvas is added to the screen before entering XR because currently the xr session must be initialized while the canvas is added render properly
                 this._addCanvas();
-            } else if (helper.state == WebXRState.NOT_IN_XR) {
+            } else if (stateInfo == WebXRState.NOT_IN_XR) {
                 this._removeCanvas();
             }
         });
@@ -75,13 +82,13 @@ export class WebXRManagedOutputCanvas implements IDisposable {
     }
 
     private _addCanvas() {
-        if (this._canvas && this._canvas !== this.helper.container.getScene().getEngine().getRenderingCanvas()) {
+        if (this._canvas && this._canvas !== this._engine.getRenderingCanvas()) {
             document.body.appendChild(this._canvas);
         }
     }
 
     private _removeCanvas() {
-        if (this._canvas && document.body.contains(this._canvas) && this._canvas !== this.helper.container.getScene().getEngine().getRenderingCanvas()) {
+        if (this._canvas && document.body.contains(this._canvas) && this._canvas !== this._engine.getRenderingCanvas()) {
             document.body.removeChild(this._canvas);
         }
     }

src/Cameras/XR/webXRSessionManager.ts

@@ -4,6 +4,26 @@ import { Nullable } from "../../types";
 import { IDisposable, Scene } from "../../scene";
 import { InternalTexture, InternalTextureSource } from "../../Materials/Textures/internalTexture";
 import { RenderTargetTexture } from "../../Materials/Textures/renderTargetTexture";
+import { WebXRRenderTarget } from './webXRTypes';
+import { WebXRManagedOutputCanvas } from './webXRManagedOutputCanvas';
+import { WebXRState } from '../../Legacy/legacy';
+
+interface IRenderTargetProvider {
+    getRenderTargetForEye(eye: XREye): RenderTargetTexture;
+}
+
+class RenderTargetProvider implements IRenderTargetProvider {
+    private _texture: RenderTargetTexture;
+
+    public constructor(texture: RenderTargetTexture) {
+        this._texture = texture;
+    }
+
+    public getRenderTargetForEye(eye: XREye): RenderTargetTexture {
+        return this._texture;
+    }
+}
+
 /**
  * Manages an XRSession to work with Babylon's engine
  * @see https://doc.babylonjs.com/how_to/webxr
@@ -28,15 +48,14 @@ export class WebXRSessionManager implements IDisposable {
      */
     public referenceSpace: XRReferenceSpace;
 
-    /** @hidden */
-    public _sessionRenderTargetTexture: Nullable<RenderTargetTexture> = null;
-
     /**
      * Current XR frame
      */
     public currentFrame: Nullable<XRFrame>;
+
     private _xrNavigator: any;
     private baseLayer: Nullable<XRWebGLLayer> = null;
+    private _rttProvider: Nullable<IRenderTargetProvider>;
 
     private _sessionEnded: boolean = false;
 
@@ -77,7 +96,7 @@ export class WebXRSessionManager implements IDisposable {
             this.session.addEventListener("end", () => {
                 this._sessionEnded = true;
                 // Remove render target texture and notify frame obervers
-                this._sessionRenderTargetTexture = null;
+                this._rttProvider = null;
 
                 // Restore frame buffer to avoid clear on xr framebuffer after session end
                 this.scene.getEngine().restoreDefaultFramebuffer();
@@ -131,24 +150,40 @@ export class WebXRSessionManager implements IDisposable {
                 this.scene.getEngine()._renderLoop();
             }
         };
-        // Create render target texture from xr's webgl render target
-        this._sessionRenderTargetTexture = WebXRSessionManager._CreateRenderTargetTextureFromSession(this.session, this.scene, this.baseLayer!);
+
+        if (this._xrNavigator.xr.native) {
+            this._rttProvider = this._xrNavigator.xr.getNativeRenderTargetProvider(this.session, (width: number, height: number) => {
+                return this.scene.getEngine().createRenderTargetTexture({ width: width, height: height }, false);
+            });
+        } else {
+            // Create render target texture from xr's webgl render target
+            this._rttProvider = new RenderTargetProvider(WebXRSessionManager._CreateRenderTargetTextureFromSession(this.session, this.scene, this.baseLayer!));
+        }
 
         // Stop window's animation frame and trigger sessions animation frame
-        window.cancelAnimationFrame(this.scene.getEngine()._frameHandler);
+        if (window.cancelAnimationFrame) { window.cancelAnimationFrame(this.scene.getEngine()._frameHandler); }
         this.scene.getEngine()._renderLoop();
         return Promise.resolve();
     }
 
     /**
+     * Gets the correct render target texture to be rendered this frame for this eye
+     * @param eye the eye for which to get the render target
+     * @returns the render target for the specified eye
+     */
+    public getRenderTargetTextureForEye(eye: XREye) : RenderTargetTexture {
+        return this._rttProvider!.getRenderTargetForEye(eye);
+    }
+
+    /**
      * Stops the xrSession and restores the renderloop
      * @returns Promise which resolves after it exits XR
      */
     public exitXRAsync() {
         if (this.session) {
-            this.session.end();
+            return this.session.end();
         }
-        return new Promise(() => { });
+        return Promise.resolve();
     }
 
     /**
@@ -170,6 +205,20 @@ export class WebXRSessionManager implements IDisposable {
     }
 
     /**
+     * Creates a WebXRRenderTarget object for the XR session
+     * @param onStateChangedObservable optional, mechanism for enabling/disabling XR rendering canvas, used only on Web
+     * @returns a WebXR render target to which the session can render
+     */
+    public getWebXRRenderTarget(onStateChangedObservable?: Observable<WebXRState>) : WebXRRenderTarget {
+        if (this._xrNavigator.xr.native) {
+            return this._xrNavigator.xr.getWebXRRenderTarget(this.scene.getEngine());
+        }
+        else {
+            return new WebXRManagedOutputCanvas(this.scene.getEngine(), onStateChangedObservable!, this.scene.getEngine().getRenderingCanvas() as HTMLCanvasElement);
+        }
+    }
+
+    /**
      * @hidden
      * Converts the render layer of xrSession to a render target
      * @param session session to create render target for

src/Cameras/XR/webXRTypes.ts

@@ -0,0 +1,46 @@
+import { Nullable } from "../../types";
+import { IDisposable } from "../../scene";
+
+/**
+ * States of the webXR experience
+ */
+export enum WebXRState {
+    /**
+     * Transitioning to being in XR mode
+     */
+    ENTERING_XR,
+    /**
+     * Transitioning to non XR mode
+     */
+    EXITING_XR,
+    /**
+     * In XR mode and presenting
+     */
+    IN_XR,
+    /**
+     * Not entered XR mode
+     */
+    NOT_IN_XR
+}
+
+/**
+ * Abstraction of the XR render target
+ */
+export interface WebXRRenderTarget extends IDisposable {
+    /**
+     * xrpresent context of the canvas which can be used to display/mirror xr content
+     */
+    canvasContext: WebGLRenderingContext;
+
+    /**
+     * xr layer for the canvas
+     */
+    xrLayer: Nullable<XRWebGLLayer>;
+
+    /**
+     * Initializes the xr layer for the session
+     * @param xrSession xr session
+     * @returns a promise that will resolve once the XR Layer has been created
+     */
+    initializeXRLayerAsync(xrSession: XRSession) : Promise<void>;
+}

src/Engines/nativeEngine.ts

@@ -81,9 +81,10 @@ interface INativeEngine {
     setTexture(uniform: WebGLUniformLocation, texture: Nullable<WebGLTexture>): void;
     deleteTexture(texture: Nullable<WebGLTexture>): void;
 
-    createFrameBuffer(texture: WebGLTexture, width: number, height: number, format: number, samplingMode: number, generateStencilBuffer: boolean, generateDepthBuffer: boolean, generateMipMaps: boolean): WebGLFramebuffer;
-    bindFrameBuffer(frameBuffer: WebGLFramebuffer): void;
-    unbindFrameBuffer(frameBuffer: WebGLFramebuffer): void;
+    createFramebuffer(texture: WebGLTexture, width: number, height: number, format: number, samplingMode: number, generateStencilBuffer: boolean, generateDepthBuffer: boolean, generateMipMaps: boolean): WebGLFramebuffer;
+    deleteFramebuffer(framebuffer: WebGLFramebuffer): void;
+    bindFramebuffer(framebuffer: WebGLFramebuffer): void;
+    unbindFramebuffer(framebuffer: WebGLFramebuffer): void;
 
     drawIndexed(fillMode: number, indexStart: number, indexCount: number): void;
     draw(fillMode: number, vertexStart: number, vertexCount: number): void;
@@ -158,6 +159,17 @@ class NativeTextureFormat {
 }
 
 /** @hidden */
+class NativeTexture extends InternalTexture {
+    public getInternalTexture(): InternalTexture {
+        return this;
+    }
+
+    public getViewCount(): number {
+        return 1;
+    }
+}
+
+/** @hidden */
 declare var nativeEngine: INativeEngine;
 
 /** @hidden */
@@ -236,11 +248,36 @@ export class NativeEngine extends Engine {
      * Can be used to override the current requestAnimationFrame requester.
      * @hidden
      */
-    protected _queueNewFrame(bindedRenderFunction: any, requester: any): number {
-        this._native.requestAnimationFrame(bindedRenderFunction);
+    protected _queueNewFrame(bindedRenderFunction: any, requester?: any): number {
+        if (requester.requestAnimationFrame) {
+            requester.requestAnimationFrame(bindedRenderFunction);
+        } else {
+            this._native.requestAnimationFrame(bindedRenderFunction);
+        }
         return 0;
     }
 
+    /**
+     * Override default engine behavior.
+     * @param color
+     * @param backBuffer
+     * @param depth
+     * @param stencil
+     */
+    public _bindUnboundFramebuffer(framebuffer: Nullable<WebGLFramebuffer>) {
+        if (this._currentFramebuffer !== framebuffer) {
+            if (this._currentFramebuffer) {
+                this._native.unbindFramebuffer(this._currentFramebuffer!);
+            }
+
+            if (framebuffer) {
+                this._native.bindFramebuffer(framebuffer);
+            }
+
+            this._currentFramebuffer = framebuffer;
+        }
+    }
+
     public clear(color: Color4, backBuffer: boolean, depth: boolean, stencil: boolean = false): void {
         this._native.clear(color.r, color.g, color.b, color.a, backBuffer, depth, stencil);
     }
@@ -1100,7 +1137,7 @@ export class NativeEngine extends Engine {
         }
     }
 
-    public createRenderTargetTexture(size: number | { width: number, height: number }, options: boolean | RenderTargetCreationOptions): InternalTexture {
+    public createRenderTargetTexture(size: number | { width: number, height: number }, options: boolean | RenderTargetCreationOptions): NativeTexture {
         let fullOptions = new RenderTargetCreationOptions();
 
         if (options !== undefined && typeof options === "object") {
@@ -1127,7 +1164,7 @@ export class NativeEngine extends Engine {
             // if floating point linear (HALF_FLOAT) then force to NEAREST_SAMPLINGMODE
             fullOptions.samplingMode = Constants.TEXTURE_NEAREST_SAMPLINGMODE;
         }
-        var texture = new InternalTexture(this, InternalTextureSource.RenderTarget);
+        var texture = new NativeTexture(this, InternalTextureSource.RenderTarget);
 
         var width = (<{ width: number, height: number }>size).width || <number>size;
         var height = (<{ width: number, height: number }>size).height || <number>size;
@@ -1137,7 +1174,7 @@ export class NativeEngine extends Engine {
             Logger.Warn("Float textures are not supported. Render target forced to TEXTURETYPE_UNSIGNED_BYTE type");
         }
 
-        var framebuffer = this._native.createFrameBuffer(
+        var framebuffer = this._native.createFramebuffer(
             texture._webGLTexture!,
             width,
             height,
@@ -1187,7 +1224,7 @@ export class NativeEngine extends Engine {
             throw new Error("forceFullscreenViewport for frame buffers not yet supported in NativeEngine.");
         }
 
-        this._native.bindFrameBuffer(texture._framebuffer!);
+        this._bindUnboundFramebuffer(texture._framebuffer);
     }
 
     public unBindFramebuffer(texture: InternalTexture, disableGenerateMipMaps = false, onBeforeUnbind?: () => void): void {
@@ -1198,7 +1235,8 @@ export class NativeEngine extends Engine {
         if (onBeforeUnbind) {
             onBeforeUnbind();
         }
-        this._native.unbindFrameBuffer(texture._framebuffer!);
+
+        this._bindUnboundFramebuffer(null);
     }
 
     public createDynamicVertexBuffer(data: DataArray): DataBuffer {