Merge branch 'master' of https://github.com/BabylonJS/Babylon.js

Tools/Gulp/config.json

@@ -1298,7 +1298,8 @@
                 "../../src/Cameras/VR/babylon.vrDeviceOrientationArcRotateCamera.js",
                 "../../src/Cameras/VR/babylon.vrDeviceOrientationGamepadCamera.js",
                 "../../src/Cameras/VR/babylon.vrExperienceHelper.js",
-                "../../src/Cameras/XR/babylon.webXRCamera.js"
+                "../../src/Cameras/XR/babylon.webXRCamera.js",
+                "../../src/Cameras/XR/babylon.webXRSessionManager.js"
             ],
             "dependUpon": [
                 "core",

dist/preview release/what's new.md

@@ -9,6 +9,7 @@
   - Add customAnimationFrameRequester to allow sessions to hook into engine's render loop ([TrevorDev](https://github.com/TrevorDev))
   - camera customDefaultRenderTarget to allow cameras to render to a custom render target (eg. xr framebuffer) instead of the canvas ([TrevorDev](https://github.com/TrevorDev))
   - webXR camera which can be updated by a webXRSession ([TrevorDev](https://github.com/TrevorDev))
+  - webXRSessionManager to bridge xrSession to babylon's engine/camera ([TrevorDev](https://github.com/TrevorDev))
 
 ## Updates
 

src/Cameras/XR/babylon.webXRCamera.ts

@@ -4,6 +4,8 @@ module BABYLON {
      * @see https://doc.babylonjs.com/how_to/webxr
      */
     export class WebXRCamera extends FreeCamera {
+        private static _TmpMatrix = new BABYLON.Matrix();
+
         /**
          * Creates a new webXRCamera, this should only be set at the camera after it has been updated by the xrSessionManager
          * @param name the name of the camera
@@ -40,10 +42,59 @@ module BABYLON {
             this._updateNumberOfRigCameras(2);
             this.rigCameras[0].viewport = new BABYLON.Viewport(0, 0, 0.5, 1.0);
             this.rigCameras[0].position.x = -pupilDistance / 2;
-            this.rigCameras[0].customDefaultRenderTarget = null;
+            this.rigCameras[0].outputRenderTarget = null;
             this.rigCameras[1].viewport = new BABYLON.Viewport(0.5, 0, 0.5, 1.0);
             this.rigCameras[1].position.x = pupilDistance / 2;
-            this.rigCameras[1].customDefaultRenderTarget = null;
+            this.rigCameras[1].outputRenderTarget = null;
+        }
+
+        /**
+         * Updates the cameras position from the current pose information of the  XR session
+         * @param xrSessionManager the session containing pose information
+         */
+        public updateFromXRSessionManager(xrSessionManager: WebXRSessionManager) {
+            // Ensure all frame data is available
+            if (!xrSessionManager._currentXRFrame || !xrSessionManager._currentXRFrame.getDevicePose) {
+                return;
+            }
+            var pose = xrSessionManager._currentXRFrame.getDevicePose(xrSessionManager._frameOfReference);
+            if (!pose || !pose.poseModelMatrix) {
+                return;
+            }
+
+            // Update the parent cameras matrix
+            BABYLON.Matrix.FromFloat32ArrayToRefScaled(pose.poseModelMatrix, 0, 1, WebXRCamera._TmpMatrix);
+            if (!this._scene.useRightHandedSystem) {
+                WebXRCamera._TmpMatrix.toggleModelMatrixHandInPlace();
+            }
+            WebXRCamera._TmpMatrix.getTranslationToRef(this.position);
+            WebXRCamera._TmpMatrix.getRotationMatrixToRef(WebXRCamera._TmpMatrix);
+            BABYLON.Quaternion.FromRotationMatrixToRef(WebXRCamera._TmpMatrix, this.rotationQuaternion);
+            this.computeWorldMatrix();
+
+            // Update camera rigs
+            this._updateNumberOfRigCameras(xrSessionManager._currentXRFrame.views.length);
+            xrSessionManager._currentXRFrame.views.forEach((view, i) => {
+                // Update view/projection matrix
+                BABYLON.Matrix.FromFloat32ArrayToRefScaled(pose.getViewMatrix(view), 0, 1, this.rigCameras[i]._computedViewMatrix);
+                BABYLON.Matrix.FromFloat32ArrayToRefScaled(view.projectionMatrix, 0, 1, this.rigCameras[i]._projectionMatrix);
+                if (!this._scene.useRightHandedSystem) {
+                    this.rigCameras[i]._computedViewMatrix.toggleModelMatrixHandInPlace();
+                    this.rigCameras[i]._projectionMatrix.toggleProjectionMatrixHandInPlace();
+                }
+
+                // Update viewport
+                var viewport  = xrSessionManager._xrSession.baseLayer.getViewport(view);
+                var width = xrSessionManager._xrSession.baseLayer.framebufferWidth;
+                var height = xrSessionManager._xrSession.baseLayer.framebufferHeight;
+                this.rigCameras[i].viewport.width = viewport.width / width;
+                this.rigCameras[i].viewport.height = viewport.height / height;
+                this.rigCameras[i].viewport.x = viewport.x / width;
+                this.rigCameras[i].viewport.y = viewport.y / height;
+
+                // Set cameras to render to the session's render target
+                this.rigCameras[i].outputRenderTarget = xrSessionManager._sessionRenderTargetTexture;
+            });
         }
     }
 }

src/Cameras/XR/babylon.webXRSessionManager.ts

@@ -0,0 +1,143 @@
+module BABYLON {
+    /**
+     * Manages an XRSession
+     * @see https://doc.babylonjs.com/how_to/webxr
+     */
+    export class WebXRSessionManager {
+        private _xrNavigator: any;
+        private _xrDevice: XRDevice;
+        private _tmpMatrix = new BABYLON.Matrix();
+        /** @hidden */
+        public _xrSession: XRSession;
+        /** @hidden */
+        public _frameOfReference: XRFrameOfReference;
+        /** @hidden */
+        public _sessionRenderTargetTexture: RenderTargetTexture;
+        /** @hidden */
+        public _currentXRFrame: XRFrame;
+
+        /**
+         * Constructs a WebXRSessionManager, this must be initialized within a user action before usage
+         * @param scene The scene which the session should be created for
+         */
+        constructor(private scene: BABYLON.Scene) {
+
+        }
+
+        /**
+         * Initializes the manager, this must be done with a user action (eg. button click event)
+         * After initialization enterXR can be called to start an XR session
+         * @returns Promise which resolves after it is initialized
+         */
+        public initialize(): Promise<void> {
+             // Check if the browser supports webXR
+            this._xrNavigator = navigator;
+            if (!this._xrNavigator.xr) {
+                return Promise.reject("webXR not supported by this browser");
+            }
+             // Request the webXR device
+            return this._xrNavigator.xr.requestDevice().then((device: XRDevice) => {
+                this._xrDevice = device;
+                return (<any>this.scene.getEngine()._gl).setCompatibleXRDevice(this._xrDevice);
+            });
+        }
+
+        /**
+         * Enters XR with the desired XR session options
+         * @param sessionCreationOptions xr options to create the session with
+         * @param frameOfReferenceType option to configure how the xr pose is expressed
+         * @returns Promise which resolves after it enters XR
+         */
+        public enterXR(sessionCreationOptions: XRSessionCreationOptions, frameOfReferenceType: XRFrameOfReferenceType): Promise<void> {
+            // initialize session
+            return this._xrDevice.requestSession(sessionCreationOptions).then((session: XRSession) => {
+                this._xrSession = session;
+                this._xrSession.baseLayer = new XRWebGLLayer(this._xrSession, this.scene.getEngine()._gl);
+                return this._xrSession.requestFrameOfReference(frameOfReferenceType);
+            }).then((frameOfRef: any) => {
+                this._frameOfReference = frameOfRef;
+                // Tell the engine's render loop to be driven by the xr session's refresh rate and provide xr pose information
+                this.scene.getEngine().customAnimationFrameRequester = {
+                    requestAnimationFrame: this._xrSession.requestAnimationFrame.bind(this._xrSession),
+                    renderFunction: (timestamp: number, xrFrame: XRFrame) => {
+                        // Store the XR frame in the manager to be consumed by the XR camera to update pose
+                        this._currentXRFrame = xrFrame;
+                        this.scene.getEngine()._renderLoop();
+                    }
+                };
+                // Create render target texture from xr's webgl render target
+                this._sessionRenderTargetTexture = WebXRSessionManager._CreateRenderTargetTextureFromSession(this._xrSession, this.scene);
+            });
+        }
+
+        /**
+         * Stops the xrSession and restores the renderloop
+         * @returns Promise which resolves after it exits XR
+         */
+        public exitXR() {
+            return new Promise((res) => {
+                this.scene.getEngine().customAnimationFrameRequester = null;
+                this._xrSession.end();
+                // Restore frame buffer to avoid clear on xr framebuffer after session end
+                this.scene.getEngine().restoreDefaultFramebuffer();
+                // Need to restart render loop as after calling session.end the last request for new frame will never call callback
+                this.scene.getEngine()._renderLoop();
+                res();
+            });
+        }
+
+        /**
+         * Fires a ray and returns the closest hit in the xr sessions enviornment, useful to place objects in AR
+         * @param ray ray to cast into the environment
+         * @returns Promise which resolves with a collision point in the environment if it exists
+         */
+        public environmentPointHitTest(ray: BABYLON.Ray): Promise<Nullable<Vector3>> {
+            return new Promise((res, rej) => {
+                // Compute left handed inputs to request hit test
+                var origin = new Float32Array([ray.origin.x, ray.origin.y, ray.origin.z]);
+                var direction = new Float32Array([ray.direction.x, ray.direction.y, ray.direction.z]);
+                if (!this.scene.useRightHandedSystem) {
+                    origin[2] *= -1;
+                    direction[2] *= -1;
+                }
+
+                // Fire hittest
+                this._xrSession.requestHitTest(origin, direction, this._frameOfReference)
+                .then((hits: any) => {
+                    if (hits.length > 0) {
+                        BABYLON.Matrix.FromFloat32ArrayToRefScaled(hits[0].hitMatrix, 0, 1.0, this._tmpMatrix);
+                        var hitPoint = this._tmpMatrix.getTranslation();
+                        if (!this.scene.useRightHandedSystem) {
+                            hitPoint.z *= -1;
+                        }
+                        res(hitPoint);
+                    }else {
+                        res(null);
+                    }
+                }).catch((e: Error) => {
+                    res(null);
+                });
+            });
+        }
+
+        /**
+         * @hidden
+         * Converts the render layer of xrSession to a render target
+         * @param session session to create render target for
+         * @param scene scene the new render target should be created for
+         */
+        public static _CreateRenderTargetTextureFromSession(session: XRSession, scene: BABYLON.Scene) {
+            // Create internal texture
+            var internalTexture = new BABYLON.InternalTexture(scene.getEngine(), BABYLON.InternalTexture.DATASOURCE_UNKNOWN, true);
+            internalTexture.width = session.baseLayer.framebufferWidth;
+            internalTexture.height = session.baseLayer.framebufferHeight;
+            internalTexture._framebuffer = session.baseLayer.framebuffer;
+
+             // Create render target texture from the internal texture
+            var renderTargetTexture = new BABYLON.RenderTargetTexture("XR renderTargetTexture", {width: internalTexture.width, height: internalTexture.height}, scene, undefined, undefined, undefined, undefined, undefined, undefined, undefined, undefined, undefined, true);
+            renderTargetTexture._texture = internalTexture;
+
+             return renderTargetTexture;
+        }
+    }
+}

src/Cameras/babylon.camera.ts

@@ -199,7 +199,7 @@ module BABYLON {
         public isStereoscopicSideBySide: boolean;
 
         /**
-         * Defines the list of custom render target the camera should render to.
+         * Defines the list of custom render target which are rendered to and then used as the input to this camera's render. Eg. display another camera view on a TV in the main scene
          * This is pretty helpfull if you wish to make a camera render to a texture you could reuse somewhere
          * else in the scene.
          */
@@ -207,7 +207,7 @@ module BABYLON {
         /**
          * When set, the camera will render to this render target instead of the default canvas
          */
-        public customDefaultRenderTarget: Nullable<RenderTargetTexture> = null;
+        public outputRenderTarget: Nullable<RenderTargetTexture> = null;
 
         /**
          * Observable triggered when the camera view matrix has changed.
@@ -250,7 +250,8 @@ module BABYLON {
 
         protected _globalPosition = Vector3.Zero();
 
-        private _computedViewMatrix = Matrix.Identity();
+        /** hidden */
+        public _computedViewMatrix = Matrix.Identity();
         private _doNotComputeProjectionMatrix = false;
         private _transformMatrix = Matrix.Zero();
         private _frustumPlanes: Plane[];

src/Materials/Textures/babylon.internalTexture.ts

@@ -236,12 +236,15 @@ module BABYLON {
          * Creates a new InternalTexture
          * @param engine defines the engine to use
          * @param dataSource defines the type of data that will be used
+         * @param delayAllocation if the texture allocation should be delayed (default: false)
          */
-        constructor(engine: Engine, dataSource: number) {
+        constructor(engine: Engine, dataSource: number, delayAllocation = false) {
             this._engine = engine;
             this._dataSource = dataSource;
 
-            this._webGLTexture = engine._createTexture();
+            if (!delayAllocation) {
+                this._webGLTexture = engine._createTexture();
+            }
         }
 
         /**

src/Materials/Textures/babylon.renderTargetTexture.ts

@@ -260,8 +260,9 @@ module BABYLON {
          * @param generateStencilBuffer True to generate a stencil buffer
          * @param isMulti True if multiple textures need to be created (Draw Buffers)
          * @param format The internal format of the buffer in the RTT (RED, RG, RGB, RGBA, ALPHA...)
+         * @param delayAllocation if the texture allocation should be delayed (default: false)
          */
-        constructor(name: string, size: number | { width: number, height: number } | { ratio: number }, scene: Nullable<Scene>, generateMipMaps?: boolean, doNotChangeAspectRatio: boolean = true, type: number = Engine.TEXTURETYPE_UNSIGNED_INT, public isCube = false, samplingMode = Texture.TRILINEAR_SAMPLINGMODE, generateDepthBuffer = true, generateStencilBuffer = false, isMulti = false, format = Engine.TEXTUREFORMAT_RGBA) {
+        constructor(name: string, size: number | { width: number, height: number } | { ratio: number }, scene: Nullable<Scene>, generateMipMaps?: boolean, doNotChangeAspectRatio: boolean = true, type: number = Engine.TEXTURETYPE_UNSIGNED_INT, public isCube = false, samplingMode = Texture.TRILINEAR_SAMPLINGMODE, generateDepthBuffer = true, generateStencilBuffer = false, isMulti = false, format = Engine.TEXTUREFORMAT_RGBA, delayAllocation = false) {
             super(null, scene, !generateMipMaps);
             scene = this.getScene();
 
@@ -305,14 +306,15 @@ module BABYLON {
                 this.wrapV = Texture.CLAMP_ADDRESSMODE;
             }
 
-            if (isCube) {
-                this._texture = scene.getEngine().createRenderTargetCubeTexture(this.getRenderSize(), this._renderTargetOptions);
-                this.coordinatesMode = Texture.INVCUBIC_MODE;
-                this._textureMatrix = Matrix.Identity();
-            } else {
-                this._texture = scene.getEngine().createRenderTargetTexture(this._size, this._renderTargetOptions);
+            if (!delayAllocation) {
+                if (isCube) {
+                    this._texture = scene.getEngine().createRenderTargetCubeTexture(this.getRenderSize(), this._renderTargetOptions);
+                    this.coordinatesMode = Texture.INVCUBIC_MODE;
+                    this._textureMatrix = Matrix.Identity();
+                } else {
+                    this._texture = scene.getEngine().createRenderTargetTexture(this._size, this._renderTargetOptions);
+                }
             }
-
         }
 
         /**

src/Math/babylon.math.ts

@@ -4880,9 +4880,26 @@ module BABYLON {
             return this;
         }
 
-        // Statics
+        /**
+         * Toggles model matrix from being right handed to left handed in place and vice versa
+         */
+        public toggleModelMatrixHandInPlace() {
+            [2, 6, 8, 9, 14].forEach((num) => {
+                this.m[num] *= -1;
+            });
+        }
 
         /**
+         * Toggles projection matrix from being right handed to left handed in place and vice versa
+         */
+        public toggleProjectionMatrixHandInPlace() {
+            [8, 9, 10, 11].forEach((num) => {
+                this.m[num] *= -1;
+            });
+        }
+
+        // Statics
+        /**
          * Creates a matrix from an array
          * @param array defines the source array
          * @param offset defines an offset in the source array

src/babylon.mixins.ts

@@ -183,3 +183,44 @@ interface EXT_disjoint_timer_query {
 interface WebGLUniformLocation {
     _currentState: any;
 }
+
+// WebXR
+interface XRDevice {
+    requestSession(options: XRSessionCreationOptions): Promise<XRSession>;
+}
+interface XRSession {
+    baseLayer: XRWebGLLayer;
+    requestFrameOfReference(type: XRFrameOfReferenceType): Promise<void>;
+    requestHitTest(origin: Float32Array, direction: Float32Array, frameOfReference: any): any;
+    end(): void;
+    requestAnimationFrame: Function;
+}
+interface XRSessionCreationOptions {
+}
+interface XRLayer {
+    getViewport: Function;
+    framebufferWidth: number;
+    framebufferHeight: number;
+}
+interface XRView {
+    projectionMatrix: Float32Array;
+}
+interface XRFrame {
+    getDevicePose: Function;
+    views: Array<XRView>;
+    baseLayer: XRLayer;
+}
+interface XRFrameOfReference {
+}
+interface XRWebGLLayer extends XRLayer {
+    framebuffer: WebGLFramebuffer;
+}
+declare var XRWebGLLayer: {
+    prototype: XRWebGLLayer;
+    new(session: XRSession, context?: WebGLRenderingContext): XRWebGLLayer;
+};
+enum XRFrameOfReferenceType {
+    "head-model",
+    "eye-level",
+    "stage",
+}

src/babylon.scene.ts

@@ -1805,27 +1805,17 @@ module BABYLON {
 
             let type = PointerEventTypes.POINTERUP;
             if (this.onPointerObservable.hasObservers()) {
-                if (!clickInfo.ignore) {
-                    if (!clickInfo.hasSwiped) {
-                        if (clickInfo.singleClick && this.onPointerObservable.hasSpecificMask(PointerEventTypes.POINTERTAP)) {
-                            let type = PointerEventTypes.POINTERTAP;
-                            let pi = new PointerInfo(type, evt, pickResult);
-                            this._setRayOnPointerInfo(pi);
-                            this.onPointerObservable.notifyObservers(pi, type);
-                        }
-                        if (clickInfo.doubleClick && this.onPointerObservable.hasSpecificMask(PointerEventTypes.POINTERDOUBLETAP)) {
-                            let type = PointerEventTypes.POINTERDOUBLETAP;
-                            let pi = new PointerInfo(type, evt, pickResult);
-                            this._setRayOnPointerInfo(pi);
-                            this.onPointerObservable.notifyObservers(pi, type);
-                        }
+                if (!clickInfo.ignore && !clickInfo.hasSwiped) {
+                    if (clickInfo.singleClick && this.onPointerObservable.hasSpecificMask(PointerEventTypes.POINTERTAP)) {
+                        type = PointerEventTypes.POINTERTAP;
+                    }
+                    else if (clickInfo.doubleClick && this.onPointerObservable.hasSpecificMask(PointerEventTypes.POINTERDOUBLETAP)) {
+                        type = PointerEventTypes.POINTERDOUBLETAP;
                     }
                 }
-                else {
-                    let pi = new PointerInfo(type, evt, pickResult);
-                    this._setRayOnPointerInfo(pi);
-                    this.onPointerObservable.notifyObservers(pi, type);
-                }
+                let pi = new PointerInfo(type, evt, pickResult);
+                this._setRayOnPointerInfo(pi);
+                this.onPointerObservable.notifyObservers(pi, type);
             }
 
             if (this.onPointerUp && !clickInfo.ignore) {
@@ -4402,8 +4392,8 @@ module BABYLON {
 
                 this._intermediateRendering = false;
 
-                if (this.activeCamera.customDefaultRenderTarget) {
-                    var internalTexture = this.activeCamera.customDefaultRenderTarget.getInternalTexture();
+                if (this.activeCamera.outputRenderTarget) {
+                    var internalTexture = this.activeCamera.outputRenderTarget.getInternalTexture();
                     if (internalTexture) {
                         engine.bindFramebuffer(internalTexture);
                     } else {