Merge pull request #5372 from TrevorDev/WebXRInput

Web xr input

Tools/Gulp/config.json

@@ -1302,7 +1302,8 @@
                 "../../src/Cameras/XR/babylon.webXRSessionManager.js",
                 "../../src/Cameras/XR/babylon.webXRExperienceHelper.js",
                 "../../src/Cameras/XR/babylon.webXREnterExitUI.js",
-                "../../src/Cameras/XR/babylon.webXRManagedOutputCanvas.js"
+                "../../src/Cameras/XR/babylon.webXRManagedOutputCanvas.js",
+                "../../src/Cameras/XR/babylon.webXRInput.js"
             ],
             "dependUpon": [
                 "core",

dist/preview release/what's new.md

@@ -12,6 +12,7 @@
   - webXRSessionManager to bridge xrSession to babylon's engine/camera ([TrevorDev](https://github.com/TrevorDev))
   - webXRExperienceHelper to setup a default XR experience ([TrevorDev](https://github.com/TrevorDev))
   - WebXREnterExitUI and WebXRManagedOutputCanvas classes to configure the XR experience ([TrevorDev](https://github.com/TrevorDev))
+  - WebXRInput manage controllers for the XR experience ([TrevorDev](https://github.com/TrevorDev))
 
 ## Updates
 

src/Cameras/XR/babylon.webXREnterExitUI.ts

@@ -1,5 +1,28 @@
 module BABYLON {
     /**
+     * Button which can be used to enter a different mode of XR
+     */
+    export class WebXREnterExitUIButton {
+        /**
+         * Creates a WebXREnterExitUIButton
+         * @param element button element
+         * @param initializationOptions XR initialization options for the button
+         */
+        constructor(
+            /** button element */
+            public element: HTMLElement,
+            /** XR initialization options for the button */
+            public initializationOptions: XRSessionCreationOptions
+        ) {}
+        /**
+         * Overwritable function which can be used to update the button's visuals when the state changes
+         * @param activeButton the current active button in the UI
+         */
+        update(activeButton: Nullable<WebXREnterExitUIButton>) {
+        }
+    }
+
+    /**
      * Options to create the webXR UI
      */
     export class WebXREnterExitUIOptions {
@@ -11,14 +34,15 @@ module BABYLON {
         /**
          * User provided buttons to enable/disable WebXR. The system will provide default if not set
          */
-        customButtons?: Array<{ element: HTMLElement, initializationOptions: XRSessionCreationOptions }>;
+        customButtons?: Array<WebXREnterExitUIButton>;
     }
     /**
      * UI to allow the user to enter/exit XR mode
      */
     export class WebXREnterExitUI implements IDisposable {
         private _overlay: HTMLDivElement;
-        private _buttons: Array<{ element: HTMLElement, initializationOptions: XRSessionCreationOptions }> = [];
+        private _buttons: Array<WebXREnterExitUIButton> = [];
+        private _activeButton: Nullable<WebXREnterExitUIButton> = null;
         /**
          * Creates UI to allow the user to enter/exit XR mode
          * @param scene the scene to add the ui to
@@ -31,15 +55,22 @@ module BABYLON {
             var supportedPromises = ui._buttons.map((btn) => {
                 return helper.supportsSession(btn.initializationOptions);
             });
+            helper.onStateChangedObservable.add((state) => {
+                if (state == WebXRState.NOT_IN_XR) {
+                    ui._updateButtons(null);
+                }
+            });
             return Promise.all(supportedPromises).then((results) => {
                 results.forEach((supported, i) => {
                     if (supported) {
                         ui._overlay.appendChild(ui._buttons[i].element);
                         ui._buttons[i].element.onclick = async() => {
                             if (helper.state == BABYLON.WebXRState.IN_XR) {
+                                ui._updateButtons(null);
                                 await helper.exitXR();
                                 return;
                             } else if (helper.state == BABYLON.WebXRState.NOT_IN_XR) {
+                                ui._updateButtons(ui._buttons[i]);
                                 await helper.enterXR(ui._buttons[i].initializationOptions, "eye-level");
                             }
                         };
@@ -58,12 +89,21 @@ module BABYLON {
                 var hmdBtn = document.createElement("button");
                 hmdBtn.style.cssText = "color: #868686; border-color: #868686; border-style: solid; margin-left: 10px; height: 50px; width: 80px; background-color: rgba(51,51,51,0.7); background-repeat:no-repeat; background-position: center; outline: none;";
                 hmdBtn.innerText = "HMD";
-                this._buttons.push({ element: hmdBtn, initializationOptions: { immersive: true } });
+                this._buttons.push(new WebXREnterExitUIButton(hmdBtn, {immersive: true, outputContext: options.outputCanvasContext}));
+                this._buttons[this._buttons.length - 1].update = function(activeButton: WebXREnterExitUIButton) {
+                    this.element.style.display = (activeButton === null || activeButton === this) ? "" : "none";
+                    this.element.innerText = activeButton === this ? "EXIT" : "HMD";
+                };
 
                 var windowBtn = document.createElement("button");
                 windowBtn.style.cssText = hmdBtn.style.cssText;
                 windowBtn.innerText = "Window";
-                this._buttons.push({ element: windowBtn, initializationOptions: { immersive: false, environmentIntegration: true, outputContext: options.outputCanvasContext } });
+                this._buttons.push(new WebXREnterExitUIButton(windowBtn, { immersive: false, environmentIntegration: true, outputContext: options.outputCanvasContext }));
+                this._buttons[this._buttons.length - 1].update = function(activeButton: WebXREnterExitUIButton) {
+                    this.element.style.display = (activeButton === null || activeButton === this) ? "" : "none";
+                    this.element.innerText = activeButton === this ? "EXIT" : "Window";
+                };
+                this._updateButtons(null);
             }
 
             var renderCanvas = scene.getEngine().getRenderingCanvas();
@@ -74,6 +114,14 @@ module BABYLON {
                 });
             }
         }
+
+        private _updateButtons(activeButton: Nullable<WebXREnterExitUIButton>) {
+            this._activeButton = activeButton;
+            this._buttons.forEach((b) => {
+                b.update(this._activeButton);
+            });
+        }
+
         /**
          * Disposes of the object
          */

src/Cameras/XR/babylon.webXRExperienceHelper.ts

@@ -49,7 +49,8 @@ module BABYLON {
          */
         public onStateChangedObservable = new Observable<WebXRState>();
 
-        private _sessionManager: WebXRSessionManager;
+        /** @hidden */
+        public _sessionManager: WebXRSessionManager;
 
         private _nonVRCamera: Nullable<Camera> = null;
         private _originalSceneAutoClear = true;
@@ -130,6 +131,15 @@ module BABYLON {
         }
 
         /**
+         * Fires a ray and returns the closest hit in the xr sessions enviornment, useful to place objects in AR
+         * @param ray ray to cast into the environment
+         * @returns Promise which resolves with a collision point in the environment if it exists
+         */
+        public environmentPointHitTest(ray: BABYLON.Ray): Promise<Nullable<Vector3>> {
+            return this._sessionManager.environmentPointHitTest(ray);
+        }
+
+        /**
          * Checks if the creation options are supported by the xr session
          * @param options creation options
          * @returns true if supported

src/Cameras/XR/babylon.webXRInput.ts

@@ -0,0 +1,104 @@
+module BABYLON {
+    /**
+     * Represents an XR input
+     */
+    export class WebXRController {
+        /**
+         * Represents the part of the controller that is held. This may not exist if the controller is the head mounted display itself, if thats the case only the pointer from the head will be availible
+         */
+        public grip?: BABYLON.AbstractMesh;
+        /**
+         * Pointer which can be used to select objects or attach a visible laser to
+         */
+        public pointer: BABYLON.AbstractMesh;
+
+        /**
+         * Creates the controller
+         * @see https://doc.babylonjs.com/how_to/webxr
+         * @param scene the scene which the controller should be associated to
+         */
+        constructor(scene: BABYLON.Scene) {
+            this.pointer = new BABYLON.AbstractMesh("controllerPointer", scene);
+        }
+        /**
+         * Disposes of the object
+         */
+        dispose() {
+            if (this.grip) {
+                this.grip.dispose();
+            }
+            this.pointer.dispose();
+        }
+    }
+
+    /**
+     * XR input used to track XR inputs such as controllers/rays
+     */
+    export class WebXRInput implements IDisposable {
+        /**
+         * XR controllers being tracked
+         */
+        public controllers: Array<WebXRController> = [];
+        private _tmpMatrix = new BABYLON.Matrix();
+        private _frameObserver: Nullable<Observer<any>>;
+
+        /**
+         * Initializes the WebXRInput
+         * @param helper experience helper which the input should be created for
+         */
+        public constructor(private helper: WebXRExperienceHelper) {
+            this._frameObserver = helper._sessionManager.onXRFrameObservable.add(() => {
+                if (!helper._sessionManager._currentXRFrame || !helper._sessionManager._currentXRFrame.getDevicePose) {
+                    return false;
+                }
+
+                var xrFrame = helper._sessionManager._currentXRFrame;
+                var inputSources = helper._sessionManager._xrSession.getInputSources();
+
+                inputSources.forEach((input, i) => {
+                    let inputPose = xrFrame.getInputPose(input, helper._sessionManager._frameOfReference);
+                    if (inputPose) {
+                        if (this.controllers.length <= i) {
+                            this.controllers.push(new WebXRController(helper.container.getScene()));
+                        }
+                        var controller = this.controllers[i];
+
+                        // Manage the grip if it exists
+                        if (inputPose.gripMatrix) {
+                            if (!controller.grip) {
+                                controller.grip = new BABYLON.AbstractMesh("controllerGrip", helper.container.getScene());
+                            }
+                            BABYLON.Matrix.FromFloat32ArrayToRefScaled(inputPose.gripMatrix, 0, 1, this._tmpMatrix);
+                            if (!controller.grip.getScene().useRightHandedSystem) {
+                                this._tmpMatrix.toggleModelMatrixHandInPlace();
+                            }
+                            if (!controller.grip.rotationQuaternion) {
+                                controller.grip.rotationQuaternion = new BABYLON.Quaternion();
+                            }
+                            this._tmpMatrix.decompose(controller.grip.scaling, controller.grip.rotationQuaternion, controller.grip.position);
+                        }
+
+                        // Manager pointer of controller
+                        BABYLON.Matrix.FromFloat32ArrayToRefScaled(inputPose.targetRay.transformMatrix, 0, 1, this._tmpMatrix);
+                        if (!controller.pointer.getScene().useRightHandedSystem) {
+                            this._tmpMatrix.toggleModelMatrixHandInPlace();
+                        }
+                        if (!controller.pointer.rotationQuaternion) {
+                            controller.pointer.rotationQuaternion = new BABYLON.Quaternion();
+                        }
+                        this._tmpMatrix.decompose(controller.pointer.scaling, controller.pointer.rotationQuaternion, controller.pointer.position);
+                    }
+                });
+            });
+        }
+        /**
+         * Disposes of the object
+         */
+        public dispose() {
+            this.controllers.forEach((c) => {
+                c.dispose();
+            });
+            this.helper._sessionManager.onXRFrameObservable.remove(this._frameObserver);
+        }
+    }
+}

src/Cameras/XR/babylon.webXRManagedOutputCanvas.ts

@@ -16,7 +16,7 @@ module BABYLON {
         public constructor(helper: WebXRExperienceHelper, canvas?: HTMLCanvasElement) {
             if (!canvas) {
                 canvas = document.createElement('canvas');
-                canvas.style.cssText = "position:absolute; bottom:0px;right:0px;z-index:10;width:100%;height:100%;background-color: #48989e;";
+                canvas.style.cssText = "position:absolute; bottom:0px;right:0px;z-index:10;width:100%;height:100%;background-color: #000000;";
             }
             this._setManagedOutputCanvas(canvas);
             helper.onStateChangedObservable.add((stateInfo) => {

src/Helpers/babylon.sceneHelpers.ts

@@ -184,7 +184,9 @@ module BABYLON {
     Scene.prototype.createDefaultXRExperienceAsync = function(): Promise<BABYLON.WebXRExperienceHelper> {
         return BABYLON.WebXRExperienceHelper.CreateAsync(this).then((helper) => {
             var outputCanvas = new BABYLON.WebXRManagedOutputCanvas(helper);
-            return BABYLON.WebXREnterExitUI.CreateAsync(this, helper, {outputCanvasContext: outputCanvas.canvasContext}).then((ui) => {
+            return BABYLON.WebXREnterExitUI.CreateAsync(this, helper, {outputCanvasContext: outputCanvas.canvasContext})
+            .then((ui) => {
+                new BABYLON.WebXRInput(helper);
                 return helper;
             });
         });

src/babylon.mixins.ts

@@ -184,6 +184,7 @@ interface XRDevice {
     supportsSession(options: XRSessionCreationOptions): Promise<void>;
 }
 interface XRSession {
+    getInputSources(): Array<any>;
     baseLayer: XRWebGLLayer;
     requestFrameOfReference(type: string): Promise<void>;
     requestHitTest(origin: Float32Array, direction: Float32Array, frameOfReference: any): any;
@@ -206,6 +207,7 @@ interface XRView {
 }
 interface XRFrame {
     getDevicePose: Function;
+    getInputPose: Function;
     views: Array<XRView>;
     baseLayer: XRLayer;
 }