xr session manager

Tools/Gulp/config.json

@@ -1298,7 +1298,8 @@
                 "../../src/Cameras/VR/babylon.vrDeviceOrientationArcRotateCamera.js",
                 "../../src/Cameras/VR/babylon.vrDeviceOrientationGamepadCamera.js",
                 "../../src/Cameras/VR/babylon.vrExperienceHelper.js",
-                "../../src/Cameras/XR/babylon.webXRCamera.js"
+                "../../src/Cameras/XR/babylon.webXRCamera.js",
+                "../../src/Cameras/XR/babylon.webXRSessionManager.js"
             ],
             "dependUpon": [
                 "core",

src/Cameras/XR/babylon.webXRCamera.ts

@@ -4,6 +4,8 @@ module BABYLON {
      * @see https://doc.babylonjs.com/how_to/webxr
      */
     export class WebXRCamera extends FreeCamera {
+        private _tmpMatrix = new BABYLON.Matrix();
+        
         /**
          * Creates a new webXRCamera, this should only be set at the camera after it has been updated by the xrSessionManager
          * @param name the name of the camera
@@ -40,10 +42,59 @@ module BABYLON {
             this._updateNumberOfRigCameras(2);
             this.rigCameras[0].viewport = new BABYLON.Viewport(0, 0, 0.5, 1.0);
             this.rigCameras[0].position.x = -pupilDistance / 2;
-            this.rigCameras[0].customDefaultRenderTarget = null;
+            this.rigCameras[0].renderTarget = null;
             this.rigCameras[1].viewport = new BABYLON.Viewport(0.5, 0, 0.5, 1.0);
             this.rigCameras[1].position.x = pupilDistance / 2;
-            this.rigCameras[1].customDefaultRenderTarget = null;
+            this.rigCameras[1].renderTarget = null;
+        }
+
+        /**
+         * Updates the cameras position from the current pose information of the  XR session
+         * @param xrSessionManager the session containing pose information 
+         */
+        public updateFromXRSessionManager(xrSessionManager:WebXRSessionManager){
+            // Ensure all frame data is availible
+            if(!xrSessionManager._currentXRFrame || !xrSessionManager._currentXRFrame.getDevicePose){
+                return;
+            }
+            var pose = xrSessionManager._currentXRFrame.getDevicePose(xrSessionManager._frameOfReference);
+            if(!pose || !pose.poseModelMatrix){
+                return;
+            }
+
+            // Update the parent cameras matrix
+            BABYLON.Matrix.FromFloat32ArrayToRefScaled(pose.poseModelMatrix,0,1,this._tmpMatrix);
+            if (!this._scene.useRightHandedSystem) {
+                BABYLON.Matrix._ToggleModelMatrixHandInPlace(this._tmpMatrix);
+            }
+            this._tmpMatrix.getTranslationToRef(this.position);
+            this._tmpMatrix.getRotationMatrixToRef(this._tmpMatrix);
+            BABYLON.Quaternion.FromRotationMatrixToRef(this._tmpMatrix, this.rotationQuaternion);
+            this.computeWorldMatrix();
+            
+            // Update camera rigs
+            this._updateNumberOfRigCameras(xrSessionManager._currentXRFrame.views.length);
+            xrSessionManager._currentXRFrame.views.forEach((view, i)=>{
+                // Update view/projection matrix
+                BABYLON.Matrix.FromFloat32ArrayToRefScaled(pose.getViewMatrix(view), 0, 1, this.rigCameras[i]._computedViewMatrix)                                    
+                BABYLON.Matrix.FromFloat32ArrayToRefScaled(view.projectionMatrix, 0, 1, this.rigCameras[i]._projectionMatrix)
+                if (!this._scene.useRightHandedSystem) {
+                    BABYLON.Matrix._ToggleModelMatrixHandInPlace(this.rigCameras[i]._computedViewMatrix);
+                    BABYLON.Matrix._ToggleProjectionMatrixHandInPlace(this.rigCameras[i]._projectionMatrix);
+                }
+
+                // Update viewport
+                var viewport  = xrSessionManager._xrSession.baseLayer.getViewport(view);
+                var width = xrSessionManager._xrSession.baseLayer.framebufferWidth;
+                var height = xrSessionManager._xrSession.baseLayer.framebufferHeight;
+                this.rigCameras[i].viewport.width = viewport.width/width;
+                this.rigCameras[i].viewport.height = viewport.height/height;
+                this.rigCameras[i].viewport.x = viewport.x/width;
+                this.rigCameras[i].viewport.y = viewport.y/height;
+                
+                // Set cameras to render to the session's render target
+                this.rigCameras[i].renderTarget = xrSessionManager._sessionRenderTargetTexture;
+            });
         }
     }
 }

src/Cameras/XR/babylon.webXRSessionManager.ts

@@ -0,0 +1,137 @@
+module BABYLON {
+    /**
+     * Manages an XRSession
+     * @see https://doc.babylonjs.com/how_to/webxr
+     */
+    export class WebXRSessionManager {
+        private _xrNavigator:any;
+        private _xrDevice:XRDevice;
+        private _tmpMatrix = new BABYLON.Matrix();
+        /** @hidden */
+        public _xrSession:XRSession;
+        /** @hidden */
+        public _frameOfReference:XRFrameOfReference;
+        /** @hidden */
+        public _sessionRenderTargetTexture:RenderTargetTexture;
+        /** @hidden */
+        public _currentXRFrame:XRFrame;
+
+        /**
+         * Constructs a WebXRSessionManager, this must be initialized within a user action before usage
+         * @param scene The scene which the session should be created for
+         */
+        constructor(private scene:BABYLON.Scene){
+
+        }
+
+        /**
+         * Initializes the manager, this must be done with a user action (eg. button click event)
+         * After initialization enterXR can be called to start an XR session
+         * @param scene 
+         */
+        public initialize():Promise<void>{
+             // Check if the browser supports webXR
+            this._xrNavigator = navigator;
+            if(!this._xrNavigator.xr){
+                return Promise.reject("webXR not supported by this browser");
+            }
+             // Request the webXR device
+            return this._xrNavigator.xr.requestDevice().then((device:XRDevice)=>{
+                this._xrDevice = device;
+                return (<any>this.scene.getEngine()._gl).setCompatibleXRDevice(this._xrDevice)
+            })
+        }
+
+        /**
+         * Enters XR with the desired XR session options
+         * @param sessionCreationOptions 
+         */
+        public enterXR(sessionCreationOptions:XRSessionCreationOptions, frameOfReferenceType:XRFrameOfReferenceType):Promise<void>{
+            // initialize session
+            return this._xrDevice.requestSession(sessionCreationOptions).then((session:XRSession)=>{
+                this._xrSession = session;
+                this._xrSession.baseLayer = new XRWebGLLayer(this._xrSession, this.scene.getEngine()._gl);
+                return this._xrSession.requestFrameOfReference(frameOfReferenceType);
+            }).then((frameOfRef:any)=>{
+                this._frameOfReference = frameOfRef;
+                // Tell the engine's render loop to be driven by the xr session's refresh rate and provide xr pose information
+                this.scene.getEngine().customAnimationFrameRequester = {
+                    requestAnimationFrame: this._xrSession.requestAnimationFrame.bind(this._xrSession),
+                    renderFunction: (timestamp:number, xrFrame:XRFrame)=>{
+                        // Store the XR frame in the manager to be consumed by the XR camera to update pose
+                        this._currentXRFrame = xrFrame;
+                        this.scene.getEngine()._renderLoop();
+                    }
+                }
+                // Create render target texture from xr's webgl render target
+                this._sessionRenderTargetTexture = WebXRSessionManager._CreateRenderTargetTextureFromSession(this._xrSession, this.scene);
+            })
+        }
+
+        /**
+         * Stops the xrSession and restores the renderloop
+         */
+        public exitXR(){
+            return new Promise((res)=>{
+                this.scene.getEngine().customAnimationFrameRequester = null;
+                this._xrSession.end()
+                // Restore frame buffer to avoid clear on xr framebuffer after session end
+                this.scene.getEngine().restoreDefaultFramebuffer();
+                // Need to restart render loop as after calling session.end the last request for new frame will never call callback
+                this.scene.getEngine()._renderLoop();
+                res();
+            })
+        }
+
+        /**
+         * Fires a ray and returns the closest hit in the xr sessions enviornment, useful to place objects in AR
+         * @param ray ray to cast into the environment
+         */
+        public environmentPointHitTest(ray:BABYLON.Ray):Promise<Nullable<Vector3>>{
+            return new Promise((res, rej)=>{
+                // Compute left handed inputs to request hit test
+                var origin = new Float32Array([ray.origin.x, ray.origin.y, ray.origin.z]);
+                var direction = new Float32Array([ray.direction.x, ray.direction.y, ray.direction.z]);
+                if (!this.scene.useRightHandedSystem) {
+                    origin[2] *= -1;
+                    direction[2] *= -1;
+                }
+
+                // Fire hittest
+                this._xrSession.requestHitTest(origin, direction, this._frameOfReference)
+                .then((hits:any)=>{
+                    if(hits.length > 0){
+                        BABYLON.Matrix.FromFloat32ArrayToRefScaled(hits[0].hitMatrix, 0, 1.0, this._tmpMatrix);
+                        var hitPoint = this._tmpMatrix.getTranslation();
+                        if (!this.scene.useRightHandedSystem) {
+                            hitPoint.z *= -1;
+                        }
+                        res(hitPoint);
+                    }else{
+                        res(null);
+                    }
+                }).catch((e:Error)=>{
+                    res(null);
+                });
+            });
+        }
+
+        /**
+         * @hidden
+         * Converts the render layer of xrSession to a render target
+         */
+        public static _CreateRenderTargetTextureFromSession(session:XRSession, scene:BABYLON.Scene){
+            // Create internal texture
+            var internalTexture = new BABYLON.InternalTexture(scene.getEngine(), BABYLON.InternalTexture.DATASOURCE_UNKNOWN, true);
+            internalTexture.width = session.baseLayer.framebufferWidth;
+            internalTexture.height = session.baseLayer.framebufferHeight;
+            internalTexture._framebuffer = session.baseLayer.framebuffer;
+
+             // Create render target texture from the internal texture
+            var renderTargetTexture = new BABYLON.RenderTargetTexture("XR renderTargetTexture", {width: internalTexture.width, height: internalTexture.height},scene, undefined, undefined, undefined, undefined, undefined, undefined, undefined, undefined, undefined, true);
+            renderTargetTexture._texture = internalTexture;
+
+             return renderTargetTexture;
+        }
+    }
+}

src/Cameras/babylon.camera.ts

@@ -207,7 +207,7 @@ module BABYLON {
         /**
          * When set, the camera will render to this render target instead of the default canvas
          */
-        public customDefaultRenderTarget: Nullable<RenderTargetTexture> = null;
+        public renderTarget: Nullable<RenderTargetTexture> = null;
 
         /**
          * Observable triggered when the camera view matrix has changed.
@@ -250,7 +250,8 @@ module BABYLON {
 
         protected _globalPosition = Vector3.Zero();
 
-        private _computedViewMatrix = Matrix.Identity();
+        /** hidden */
+        public _computedViewMatrix = Matrix.Identity();
         private _doNotComputeProjectionMatrix = false;
         private _transformMatrix = Matrix.Zero();
         private _frustumPlanes: Plane[];

src/Math/babylon.math.ts

@@ -4881,6 +4881,19 @@ module BABYLON {
         }
 
         // Statics
+        /** @hidden */
+        public static _ToggleModelMatrixHandInPlace(matrix:Matrix){
+            [2, 6, 8, 9, 14].forEach((num) => {
+                matrix.m[num] *= -1;
+            });
+        }
+
+        /** @hidden */
+        public static _ToggleProjectionMatrixHandInPlace(matrix:Matrix){
+            [8, 9, 10, 11].forEach((num) => {
+                matrix.m[num] *= -1;
+            });
+        }
 
         /**
          * Creates a matrix from an array

src/babylon.mixins.ts

@@ -187,3 +187,44 @@ interface EXT_disjoint_timer_query {
 interface WebGLUniformLocation {
     _currentState: any;
 }
+
+// WebXR
+interface XRDevice {
+    requestSession(options:XRSessionCreationOptions):Promise<XRSession>
+}
+interface XRSession {
+    baseLayer: XRWebGLLayer
+    requestFrameOfReference(type:XRFrameOfReferenceType):Promise<void>
+    requestHitTest(origin:Float32Array, direction:Float32Array, frameOfReference:any):any
+    end():void
+    requestAnimationFrame:Function
+}
+interface XRSessionCreationOptions {
+}
+interface XRLayer {
+    getViewport:Function
+    framebufferWidth:number
+    framebufferHeight:number
+}
+interface XRView {
+    projectionMatrix:Float32Array
+}
+interface XRFrame {
+    getDevicePose:Function
+    views:Array<XRView>
+    baseLayer:XRLayer
+}
+interface XRFrameOfReference {
+}
+interface XRWebGLLayer extends XRLayer {
+    framebuffer:WebGLFramebuffer
+}
+declare var XRWebGLLayer: {
+    prototype: XRWebGLLayer;
+    new(session: XRSession, context?: WebGLRenderingContext): XRWebGLLayer;
+}
+enum XRFrameOfReferenceType {
+    "head-model",
+    "eye-level",
+    "stage",
+};

src/babylon.scene.ts

@@ -4341,8 +4341,8 @@ module BABYLON {
 
                 this._intermediateRendering = false;
 
-                if (this.activeCamera.customDefaultRenderTarget) {
-                    var internalTexture = this.activeCamera.customDefaultRenderTarget.getInternalTexture();
+                if (this.activeCamera.renderTarget) {
+                    var internalTexture = this.activeCamera.renderTarget.getInternalTexture();
                     if (internalTexture) {
                         engine.bindFramebuffer(internalTexture);
                     }else {