Merge branch 'master' of https://github.com/BabylonJS/Babylon.js

dist/preview release/what's new.md

@@ -176,6 +176,8 @@
 - Pointer-Event simulation for screen target ray mode ([RaananW](https://github.com/RaananW/))
 - New observable that triggers when a session was initialized ([RaananW](https://github.com/RaananW/))
 - WebXR teleportation can now be disabled after initialized ([RaananW](https://github.com/RaananW/))
+- New Features Manager for WebXR features ([RaananW](https://github.com/RaananW/))
+- New features - Plane detection, Hit Test, Background remover ([RaananW](https://github.com/RaananW/))
 
 ### Ray
 

src/Cameras/XR/features/WebXRAnchorSystem.ts

@@ -0,0 +1,271 @@
+import { IWebXRFeature, WebXRFeaturesManager } from '../webXRFeaturesManager';
+import { WebXRSessionManager } from '../webXRSessionManager';
+import { Observable, Observer } from '../../../Misc/observable';
+import { Matrix } from '../../../Maths/math.vector';
+import { TransformNode } from '../../../Meshes/transformNode';
+import { WebXRPlaneDetector } from './WebXRPlaneDetector';
+import { Nullable } from '../../../types';
+import { WebXRHitTestLegacy } from './WebXRHitTestLegacy';
+
+const Name = "xr-anchor-system";
+
+/**
+ * Configuration options of the anchor system
+ */
+export interface IWebXRAnchorSystemOptions {
+    /**
+     * a node that will be used to convert local to world coordinates
+     */
+    worldParentNode?: TransformNode;
+    /**
+     * should the anchor system use plane detection.
+     * If set to true, the plane-detection feature should be set using setPlaneDetector
+     */
+    usePlaneDetection?: boolean;
+    /**
+     * Should a new anchor be added every time a select event is triggered
+     */
+    addAnchorOnSelect?: boolean;
+}
+
+/**
+ * A babylon container for an XR Anchor
+ */
+export interface IWebXRAnchor {
+    /**
+     * A babylon-assigned ID for this anchor
+     */
+    id: number;
+    /**
+     * The native anchor object
+     */
+    xrAnchor: XRAnchor;
+    /**
+     * Transformation matrix to apply to an object attached to this anchor
+     */
+    transformationMatrix: Matrix;
+}
+
+let anchorIdProvider = 0;
+
+/**
+ * An implementation of the anchor system of WebXR.
+ * Note that the current documented implementation is not available in any browser. Future implementations
+ * will use the frame to create an anchor and not the session or a detected plane
+ * For further information see https://github.com/immersive-web/anchors/
+ */
+export class WebXRAnchorSystem implements IWebXRFeature {
+
+    /**
+     * The module's name
+     */
+    public static readonly Name = Name;
+    /**
+     * The (Babylon) version of this module.
+     * This is an integer representing the implementation version.
+     * This number does not correspond to the webxr specs version
+     */
+    public static readonly Version = 1;
+
+    /**
+     * Observers registered here will be executed when a new anchor was added to the session
+     */
+    public onAnchorAddedObservable: Observable<IWebXRAnchor> = new Observable();
+    /**
+     * Observers registered here will be executed when an existing anchor updates
+     * This can execute N times every frame
+     */
+    public onAnchorUpdatedObservable: Observable<IWebXRAnchor> = new Observable();
+    /**
+     * Observers registered here will be executed when an anchor was removed from the session
+     */
+    public onAnchorRemovedObservable: Observable<IWebXRAnchor> = new Observable();
+
+    private _planeDetector: WebXRPlaneDetector;
+    private _hitTestModule: WebXRHitTestLegacy;
+
+    private _enabled: boolean = false;
+    private _attached: boolean = false;
+    private _trackedAnchors: Array<IWebXRAnchor> = [];
+    private _lastFrameDetected: XRAnchorSet = new Set();
+    private _observerTracked: Nullable<Observer<XRFrame>>;
+
+    /**
+     * constructs a new anchor system
+     * @param _xrSessionManager an instance of WebXRSessionManager
+     * @param _options configuration object for this feature
+     */
+    constructor(private _xrSessionManager: WebXRSessionManager, private _options: IWebXRAnchorSystemOptions = {}) {
+    }
+
+    /**
+     * set the plane detector to use in order to create anchors from frames
+     * @param planeDetector the plane-detector module to use
+     * @param enable enable plane-anchors. default is true
+     */
+    public setPlaneDetector(planeDetector: WebXRPlaneDetector, enable: boolean = true) {
+        this._planeDetector = planeDetector;
+        this._options.usePlaneDetection = enable;
+    }
+
+    /**
+     * If set, it will improve performance by using the current hit-test results instead of executing a new hit-test
+     * @param hitTestModule the hit-test module to use.
+     */
+    public setHitTestModule(hitTestModule: WebXRHitTestLegacy) {
+        this._hitTestModule = hitTestModule;
+    }
+
+    /**
+     * attach this feature
+     * Will usually be called by the features manager
+     *
+     * @returns true if successful.
+     */
+    attach(): boolean {
+        this._observerTracked = this._xrSessionManager.onXRFrameObservable.add(() => {
+            const frame = this._xrSessionManager.currentFrame;
+            if (!this._attached || !this._enabled || !frame) { return; }
+            // const timestamp = this.xrSessionManager.currentTimestamp;
+
+            const trackedAnchors = frame.trackedAnchors;
+            if (trackedAnchors && trackedAnchors.size) {
+                this._trackedAnchors.filter((anchor) => !trackedAnchors.has(anchor.xrAnchor)).map((anchor) => {
+                    const index = this._trackedAnchors.indexOf(anchor);
+                    this._trackedAnchors.splice(index, 1);
+                    this.onAnchorRemovedObservable.notifyObservers(anchor);
+                });
+                // now check for new ones
+                trackedAnchors.forEach((xrAnchor) => {
+                    if (!this._lastFrameDetected.has(xrAnchor)) {
+                        const newAnchor: Partial<IWebXRAnchor> = {
+                            id: anchorIdProvider++,
+                            xrAnchor: xrAnchor
+                        };
+                        const plane = this._updateAnchorWithXRFrame(xrAnchor, newAnchor, frame);
+                        this._trackedAnchors.push(plane);
+                        this.onAnchorAddedObservable.notifyObservers(plane);
+                    } else {
+                        // updated?
+                        if (xrAnchor.lastChangedTime === this._xrSessionManager.currentTimestamp) {
+                            let index = this._findIndexInAnchorArray(xrAnchor);
+                            const anchor = this._trackedAnchors[index];
+                            this._updateAnchorWithXRFrame(xrAnchor, anchor, frame);
+                            this.onAnchorUpdatedObservable.notifyObservers(anchor);
+                        }
+                    }
+                });
+                this._lastFrameDetected = trackedAnchors;
+            }
+        });
+
+        if (this._options.addAnchorOnSelect) {
+            this._xrSessionManager.session.addEventListener('select', this._onSelect, false);
+        }
+
+        this._attached = true;
+        return true;
+    }
+
+    /**
+     * detach this feature.
+     * Will usually be called by the features manager
+     *
+     * @returns true if successful.
+     */
+    detach(): boolean {
+        this._attached = false;
+
+        this._xrSessionManager.session.removeEventListener('select', this._onSelect);
+
+        if (this._observerTracked) {
+            this._xrSessionManager.onXRFrameObservable.remove(this._observerTracked);
+        }
+
+        return true;
+    }
+
+    /**
+     * Dispose this feature and all of the resources attached
+     */
+    dispose(): void {
+        this.detach();
+        this.onAnchorAddedObservable.clear();
+        this.onAnchorRemovedObservable.clear();
+        this.onAnchorUpdatedObservable.clear();
+    }
+
+    private _onSelect = (event: XRInputSourceEvent) => {
+        if (!this._options.addAnchorOnSelect) {
+            return;
+        }
+        const onResults = (results: XRHitResult[]) => {
+            if (results.length) {
+                const hitResult = results[0];
+                const transform = new XRRigidTransform(hitResult.hitMatrix);
+                // find the plane on which to add.
+                this.addAnchorAtRigidTransformation(transform);
+            }
+        };
+
+        // avoid the hit-test, if the hit-test module is defined
+        if (this._hitTestModule && !this._hitTestModule.options.testOnPointerDownOnly) {
+            onResults(this._hitTestModule.lastNativeXRHitResults);
+        }
+        WebXRHitTestLegacy.XRHitTestWithSelectEvent(event, this._xrSessionManager.referenceSpace).then(onResults);
+
+        // API will soon change, will need to use the plane
+        this._planeDetector;
+    }
+
+    /**
+     * Add anchor at a specific XR point.
+     *
+     * @param xrRigidTransformation xr-coordinates where a new anchor should be added
+     * @param anchorCreator the object o use to create an anchor with. either a session or a plane
+     * @returns a promise the fulfills when the anchor was created
+     */
+    public addAnchorAtRigidTransformation(xrRigidTransformation: XRRigidTransform, anchorCreator?: XRAnchorCreator): Promise<XRAnchor> {
+        const creator = anchorCreator || this._xrSessionManager.session;
+        return creator.createAnchor(xrRigidTransformation, this._xrSessionManager.referenceSpace);
+    }
+
+    private _updateAnchorWithXRFrame(xrAnchor: XRAnchor, anchor: Partial<IWebXRAnchor>, xrFrame: XRFrame): IWebXRAnchor {
+        // matrix
+        const pose = xrFrame.getPose(xrAnchor.anchorSpace, this._xrSessionManager.referenceSpace);
+        if (pose) {
+            const mat = anchor.transformationMatrix || new Matrix();
+            Matrix.FromArrayToRef(pose.transform.matrix, 0, mat);
+            if (!this._xrSessionManager.scene.useRightHandedSystem) {
+                mat.toggleModelMatrixHandInPlace();
+            }
+            anchor.transformationMatrix = mat;
+            if (!this._options.worldParentNode) {
+                // Logger.Warn("Please provide a world parent node to apply world transformation");
+            } else {
+                mat.multiplyToRef(this._options.worldParentNode.getWorldMatrix(), mat);
+            }
+        }
+
+        return <IWebXRAnchor>anchor;
+    }
+
+    /**
+     * avoiding using Array.find for global support.
+     * @param xrAnchor the plane to find in the array
+     */
+    private _findIndexInAnchorArray(xrAnchor: XRAnchor) {
+        for (let i = 0; i < this._trackedAnchors.length; ++i) {
+            if (this._trackedAnchors[i].xrAnchor === xrAnchor) {
+                return i;
+            }
+        }
+        return -1;
+    }
+
+}
+
+//register the plugin
+WebXRFeaturesManager.AddWebXRFeature(WebXRAnchorSystem.Name, (xrSessionManager, options) => {
+    return () => new WebXRAnchorSystem(xrSessionManager, options);
+}, WebXRAnchorSystem.Version);

src/Cameras/XR/features/WebXRBackgroundRemover.ts

@@ -0,0 +1,136 @@
+import { WebXRFeaturesManager, IWebXRFeature } from "../webXRFeaturesManager";
+import { WebXRSessionManager } from '../webXRSessionManager';
+import { AbstractMesh } from '../../../Meshes/abstractMesh';
+import { Observable } from '../../../Misc/observable';
+
+const Name = "xr-background-remover";
+
+/**
+ * Options interface for the background remover plugin
+ */
+export interface IWebXRBackgroundRemoverOptions {
+    /**
+     * don't disable the environment helper
+     */
+    ignoreEnvironmentHelper?: boolean;
+    /**
+     * flags to configure the removal of the environment helper.
+     * If not set, the entire background will be removed. If set, flags should be set as well.
+     */
+    environmentHelperRemovalFlags?: {
+        /**
+         * Should the skybox be removed (default false)
+         */
+        skyBox?: boolean;
+        /**
+         * Should the ground be removed (default false)
+         */
+        ground?: boolean;
+    };
+    /**
+     * Further background meshes to disable when entering AR
+     */
+    backgroundMeshes?: AbstractMesh[];
+}
+
+/**
+ * A module that will automatically disable background meshes when entering AR and will enable them when leaving AR.
+ */
+export class WebXRBackgroundRemover implements IWebXRFeature {
+
+    /**
+     * The module's name
+     */
+    public static readonly Name = Name;
+    /**
+     * The (Babylon) version of this module.
+     * This is an integer representing the implementation version.
+     * This number does not correspond to the webxr specs version
+     */
+    public static readonly Version = 1;
+
+    /**
+     * registered observers will be triggered when the background state changes
+     */
+    public onBackgroundStateChangedObservable: Observable<boolean> = new Observable();
+
+    /**
+     * constructs a new background remover module
+     * @param _xrSessionManager the session manager for this module
+     * @param options read-only options to be used in this module
+     */
+    constructor(private _xrSessionManager: WebXRSessionManager,
+        /**
+         * read-only options to be used in this module
+         */
+        public readonly options: IWebXRBackgroundRemoverOptions = {}) {
+
+    }
+
+    /**
+     * attach this feature
+     * Will usually be called by the features manager
+     *
+     * @returns true if successful.
+     */
+    attach(): boolean {
+        this._setBackgroundState(false);
+
+        return true;
+    }
+
+    /**
+     * detach this feature.
+     * Will usually be called by the features manager
+     *
+     * @returns true if successful.
+     */
+    detach(): boolean {
+        this._setBackgroundState(true);
+
+        return true;
+    }
+
+    private _setBackgroundState(newState: boolean) {
+        const scene = this._xrSessionManager.scene;
+        if (!this.options.ignoreEnvironmentHelper) {
+            if (this.options.environmentHelperRemovalFlags) {
+                if (this.options.environmentHelperRemovalFlags.skyBox) {
+                    const backgroundSkybox = scene.getMeshByName("BackgroundSkybox");
+                    if (backgroundSkybox) {
+                        backgroundSkybox.setEnabled(newState);
+                    }
+                }
+                if (this.options.environmentHelperRemovalFlags.ground) {
+                    const backgroundPlane = scene.getMeshByName("BackgroundPlane");
+                    if (backgroundPlane) {
+                        backgroundPlane.setEnabled(newState);
+                    }
+                }
+            } else {
+                const backgroundHelper = scene.getMeshByName("BackgroundHelper");
+                if (backgroundHelper) {
+                    backgroundHelper.setEnabled(newState);
+                }
+            }
+        }
+
+        if (this.options.backgroundMeshes) {
+            this.options.backgroundMeshes.forEach((mesh) => mesh.setEnabled(newState));
+        }
+
+        this.onBackgroundStateChangedObservable.notifyObservers(newState);
+    }
+
+    /**
+     * Dispose this feature and all of the resources attached
+     */
+    dispose(): void {
+        this.onBackgroundStateChangedObservable.clear();
+    }
+}
+
+//register the plugin
+WebXRFeaturesManager.AddWebXRFeature(WebXRBackgroundRemover.Name, (xrSessionManager, options) => {
+    return () => new WebXRBackgroundRemover(xrSessionManager, options);
+}, WebXRBackgroundRemover.Version, true);

src/Cameras/XR/features/WebXRHitTestLegacy.ts

@@ -0,0 +1,215 @@
+import { IWebXRFeature, WebXRFeaturesManager } from '../webXRFeaturesManager';
+import { WebXRSessionManager } from '../webXRSessionManager';
+import { Observable, Observer } from '../../../Misc/observable';
+import { Vector3, Matrix } from '../../../Maths/math.vector';
+import { TransformNode } from '../../../Meshes/transformNode';
+import { Nullable } from '../../../types';
+
+/**
+ * name of module (can be reused with other versions)
+ */
+const WebXRHitTestModuleName = "xr-hit-test";
+
+// the plugin is registered at the end of the file
+
+/**
+ * Options used for hit testing
+ */
+export interface IWebXRHitTestOptions {
+    /**
+     * Only test when user interacted with the scene. Default - hit test every frame
+     */
+    testOnPointerDownOnly?: boolean;
+    /**
+     * The node to use to transform the local results to world coordinates
+     */
+    worldParentNode?: TransformNode;
+}
+
+/**
+ * Interface defining the babylon result of raycasting/hit-test
+ */
+export interface IWebXRHitResult {
+    /**
+     * The native hit test result
+     */
+    xrHitResult: XRHitResult;
+    /**
+     * Transformation matrix that can be applied to a node that will put it in the hit point location
+     */
+    transformationMatrix: Matrix;
+}
+
+/**
+ * The currently-working hit-test module.
+ * Hit test (or raycasting) is used to interact with the real world.
+ * For further information read here - https://github.com/immersive-web/hit-test
+ */
+export class WebXRHitTestLegacy implements IWebXRFeature {
+
+    /**
+     * The module's name
+     */
+    public static readonly Name = WebXRHitTestModuleName;
+    /**
+     * The (Babylon) version of this module.
+     * This is an integer representing the implementation version.
+     * This number does not correspond to the webxr specs version
+     */
+    public static readonly Version = 1;
+
+    /**
+     * Execute a hit test on the current running session using a select event returned from a transient input (such as touch)
+     * @param event the (select) event to use to select with
+     * @param referenceSpace the reference space to use for this hit test
+     * @returns a promise that resolves with an array of native XR hit result in xr coordinates system
+     */
+    public static XRHitTestWithSelectEvent(event: XRInputSourceEvent, referenceSpace: XRReferenceSpace): Promise<XRHitResult[]> {
+        let targetRayPose = event.frame.getPose(event.inputSource.targetRaySpace, referenceSpace);
+        if (!targetRayPose) {
+            return Promise.resolve([]);
+        }
+        let targetRay = new XRRay(targetRayPose.transform);
+
+        return this.XRHitTestWithRay(event.frame.session, targetRay, referenceSpace);
+    }
+
+    /**
+     * execute a hit test with an XR Ray
+     *
+     * @param xrSession a native xrSession that will execute this hit test
+     * @param xrRay the ray (position and direction) to use for raycasting
+     * @param referenceSpace native XR reference space to use for the hit-test
+     * @param filter filter function that will filter the results
+     * @returns a promise that resolves with an array of native XR hit result in xr coordinates system
+     */
+    public static XRHitTestWithRay(xrSession: XRSession, xrRay: XRRay, referenceSpace: XRReferenceSpace, filter?: (result: XRHitResult) => boolean): Promise<XRHitResult[]> {
+        return xrSession.requestHitTest(xrRay, referenceSpace).then((results) => {
+            const filterFunction = filter || ((result) => !!result.hitMatrix);
+            return results.filter(filterFunction);
+        });
+    }
+
+    /**
+     * Triggered when new babylon (transformed) hit test results are available
+     */
+    public onHitTestResultObservable: Observable<IWebXRHitResult[]> = new Observable();
+
+    /**
+     * Creates a new instance of the (legacy version) hit test feature
+     * @param _xrSessionManager an instance of WebXRSessionManager
+     * @param options options to use when constructing this feature
+     */
+    constructor(private _xrSessionManager: WebXRSessionManager,
+        /**
+         * options to use when constructing this feature
+         */
+        public readonly options: IWebXRHitTestOptions = {}) { }
+
+    private _onSelectEnabled = false;
+    private _xrFrameObserver: Nullable<Observer<XRFrame>>;
+    private _attached: boolean = false;
+
+    /**
+     * Populated with the last native XR Hit Results
+     */
+    public lastNativeXRHitResults: XRHitResult[] = [];
+
+    /**
+     * attach this feature
+     * Will usually be called by the features manager
+     *
+     * @returns true if successful.
+     */
+    attach(): boolean {
+        if (this.options.testOnPointerDownOnly) {
+            this._xrSessionManager.session.addEventListener('select', this._onSelect, false);
+        } else {
+            // we are in XR space!
+            const origin = new Vector3(0, 0, 0);
+            // in XR space z-forward is negative
+            const direction = new Vector3(0, 0, -1);
+            const mat = new Matrix();
+            this._xrFrameObserver = this._xrSessionManager.onXRFrameObservable.add((frame) => {
+                // make sure we do nothing if (async) not attached
+                if (!this._attached) {
+                    return;
+                }
+                let pose = frame.getViewerPose(this._xrSessionManager.referenceSpace);
+                if (!pose) {
+                    return;
+                }
+                Matrix.FromArrayToRef(pose.transform.matrix, 0, mat);
+                Vector3.TransformCoordinatesFromFloatsToRef(0, 0, 0, mat, origin);
+                Vector3.TransformCoordinatesFromFloatsToRef(0, 0, -1, mat, direction);
+                direction.subtractInPlace(origin);
+                direction.normalize();
+                let ray = new XRRay((<DOMPointReadOnly>{ x: origin.x, y: origin.y, z: origin.z, w: 0 }),
+                    (<DOMPointReadOnly>{ x: direction.x, y: direction.y, z: direction.z, w: 0 }));
+                WebXRHitTestLegacy.XRHitTestWithRay(this._xrSessionManager.session, ray, this._xrSessionManager.referenceSpace).then(this._onHitTestResults);
+            });
+        }
+        this._attached = true;
+
+        return true;
+    }
+
+    /**
+     * detach this feature.
+     * Will usually be called by the features manager
+     *
+     * @returns true if successful.
+     */
+    detach(): boolean {
+        // disable select
+        this._onSelectEnabled = false;
+        this._xrSessionManager.session.removeEventListener('select', this._onSelect);
+        if (this._xrFrameObserver) {
+            this._xrSessionManager.onXRFrameObservable.remove(this._xrFrameObserver);
+            this._xrFrameObserver = null;
+        }
+        this._attached = false;
+        return true;
+    }
+
+    private _onHitTestResults = (xrResults: XRHitResult[]) => {
+        const mats = xrResults.map((result) => {
+            let mat = Matrix.FromArray(result.hitMatrix);
+            if (!this._xrSessionManager.scene.useRightHandedSystem) {
+                mat.toggleModelMatrixHandInPlace();
+            }
+            // if (this.options.coordinatesSpace === Space.WORLD) {
+            if (this.options.worldParentNode) {
+                mat.multiplyToRef(this.options.worldParentNode.getWorldMatrix(), mat);
+            }
+            return {
+                xrHitResult: result,
+                transformationMatrix: mat
+            };
+        });
+
+        this.lastNativeXRHitResults = xrResults;
+        this.onHitTestResultObservable.notifyObservers(mats);
+    }
+
+    // can be done using pointerdown event, and xrSessionManager.currentFrame
+    private _onSelect = (event: XRInputSourceEvent) => {
+        if (!this._onSelectEnabled) {
+            return;
+        }
+        WebXRHitTestLegacy.XRHitTestWithSelectEvent(event, this._xrSessionManager.referenceSpace);
+    }
+
+    /**
+     * Dispose this feature and all of the resources attached
+     */
+    dispose(): void {
+        this.detach();
+        this.onHitTestResultObservable.clear();
+    }
+}
+
+//register the plugin versions
+WebXRFeaturesManager.AddWebXRFeature(WebXRHitTestLegacy.Name, (xrSessionManager, options) => {
+    return () => new WebXRHitTestLegacy(xrSessionManager, options);
+}, WebXRHitTestLegacy.Version, true);

src/Cameras/XR/features/WebXRPlaneDetector.ts

@@ -0,0 +1,213 @@
+import { WebXRFeaturesManager, IWebXRFeature } from '../webXRFeaturesManager';
+import { TransformNode } from '../../../Meshes/transformNode';
+import { WebXRSessionManager } from '../webXRSessionManager';
+import { Observable, Observer } from '../../../Misc/observable';
+import { Vector3, Matrix } from '../../../Maths/math.vector';
+import { Nullable } from '../../../types';
+
+const Name = "xr-plane-detector";
+
+/**
+ * Options used in the plane detector module
+ */
+export interface IWebXRPlaneDetectorOptions {
+    /**
+     * The node to use to transform the local results to world coordinates
+     */
+    worldParentNode?: TransformNode;
+}
+
+/**
+ * A babylon interface for a webxr plane.
+ * A Plane is actually a polygon, built from N points in space
+ */
+export interface IWebXRPlane {
+    /**
+     * a babylon-assigned ID for this polygon
+     */
+    id: number;
+    /**
+     * the native xr-plane object
+     */
+    xrPlane: XRPlane;
+    /**
+     * an array of vector3 points in babylon space. right/left hand system is taken into account.
+     */
+    polygonDefinition: Array<Vector3>;
+    /**
+     * A transformation matrix to apply on the mesh that will be built using the polygonDefinition
+     * Local vs. World are decided if worldParentNode was provided or not in the options when constructing the module
+     */
+    transformationMatrix: Matrix;
+}
+
+let planeIdProvider = 0;
+
+/**
+ * The plane detector is used to detect planes in the real world when in AR
+ * For more information see https://github.com/immersive-web/real-world-geometry/
+ */
+export class WebXRPlaneDetector implements IWebXRFeature {
+
+    /**
+     * The module's name
+     */
+    public static readonly Name = Name;
+    /**
+     * The (Babylon) version of this module.
+     * This is an integer representing the implementation version.
+     * This number does not correspond to the webxr specs version
+     */
+    public static readonly Version = 1;
+
+    /**
+     * Observers registered here will be executed when a new plane was added to the session
+     */
+    public onPlaneAddedObservable: Observable<IWebXRPlane> = new Observable();
+    /**
+     * Observers registered here will be executed when a plane is no longer detected in the session
+     */
+    public onPlaneRemovedObservable: Observable<IWebXRPlane> = new Observable();
+    /**
+     * Observers registered here will be executed when an existing plane updates (for example - expanded)
+     * This can execute N times every frame
+     */
+    public onPlaneUpdatedObservable: Observable<IWebXRPlane> = new Observable();
+
+    private _enabled: boolean = false;
+    private _attached: boolean = false;
+    private _detectedPlanes: Array<IWebXRPlane> = [];
+    private _lastFrameDetected: XRPlaneSet = new Set();
+    private _observerTracked: Nullable<Observer<XRFrame>>;
+
+    /**
+     * construct a new Plane Detector
+     * @param _xrSessionManager an instance of xr Session manager
+     * @param _options configuration to use when constructing this feature
+     */
+    constructor(private _xrSessionManager: WebXRSessionManager, private _options: IWebXRPlaneDetectorOptions = {}) {
+        if (this._xrSessionManager.session) {
+            this._xrSessionManager.session.updateWorldTrackingState({ planeDetectionState: { enabled: true } });
+            this._enabled = true;
+        } else {
+            this._xrSessionManager.onXRSessionInit.addOnce(() => {
+                this._xrSessionManager.session.updateWorldTrackingState({ planeDetectionState: { enabled: true } });
+                this._enabled = true;
+            });
+        }
+    }
+
+    /**
+     * attach this feature
+     * Will usually be called by the features manager
+     *
+     * @returns true if successful.
+     */
+    attach(): boolean {
+
+        this._observerTracked = this._xrSessionManager.onXRFrameObservable.add(() => {
+            const frame = this._xrSessionManager.currentFrame;
+            if (!this._attached || !this._enabled || !frame) { return; }
+            // const timestamp = this.xrSessionManager.currentTimestamp;
+
+            const detectedPlanes = frame.worldInformation.detectedPlanes;
+            if (detectedPlanes && detectedPlanes.size) {
+                this._detectedPlanes.filter((plane) => !detectedPlanes.has(plane.xrPlane)).map((plane) => {
+                    const index = this._detectedPlanes.indexOf(plane);
+                    this._detectedPlanes.splice(index, 1);
+                    this.onPlaneRemovedObservable.notifyObservers(plane);
+                });
+                // now check for new ones
+                detectedPlanes.forEach((xrPlane) => {
+                    if (!this._lastFrameDetected.has(xrPlane)) {
+                        const newPlane: Partial<IWebXRPlane> = {
+                            id: planeIdProvider++,
+                            xrPlane: xrPlane,
+                            polygonDefinition: []
+                        };
+                        const plane = this._updatePlaneWithXRPlane(xrPlane, newPlane, frame);
+                        this._detectedPlanes.push(plane);
+                        this.onPlaneAddedObservable.notifyObservers(plane);
+                    } else {
+                        // updated?
+                        if (xrPlane.lastChangedTime === this._xrSessionManager.currentTimestamp) {
+                            let index = this.findIndexInPlaneArray(xrPlane);
+                            const plane = this._detectedPlanes[index];
+                            this._updatePlaneWithXRPlane(xrPlane, plane, frame);
+                            this.onPlaneUpdatedObservable.notifyObservers(plane);
+                        }
+                    }
+                });
+                this._lastFrameDetected = detectedPlanes;
+            }
+        });
+
+        this._attached = true;
+        return true;
+    }
+
+    /**
+     * detach this feature.
+     * Will usually be called by the features manager
+     *
+     * @returns true if successful.
+     */
+    detach(): boolean {
+        this._attached = false;
+
+        if (this._observerTracked) {
+            this._xrSessionManager.onXRFrameObservable.remove(this._observerTracked);
+        }
+
+        return true;
+    }
+
+    /**
+     * Dispose this feature and all of the resources attached
+     */
+    dispose(): void {
+        this.detach();
+        this.onPlaneAddedObservable.clear();
+        this.onPlaneRemovedObservable.clear();
+        this.onPlaneUpdatedObservable.clear();
+    }
+
+    private _updatePlaneWithXRPlane(xrPlane: XRPlane, plane: Partial<IWebXRPlane>, xrFrame: XRFrame): IWebXRPlane {
+        plane.polygonDefinition = xrPlane.polygon.map((xrPoint) => {
+            const rightHandedSystem = this._xrSessionManager.scene.useRightHandedSystem ? 1 : -1;
+            return new Vector3(xrPoint.x, xrPoint.y, xrPoint.z * rightHandedSystem);
+        });
+        // matrix
+        const pose = xrFrame.getPose(xrPlane.planeSpace, this._xrSessionManager.referenceSpace);
+        if (pose) {
+            const mat = plane.transformationMatrix || new Matrix();
+            Matrix.FromArrayToRef(pose.transform.matrix, 0, mat);
+            if (!this._xrSessionManager.scene.useRightHandedSystem) {
+                mat.toggleModelMatrixHandInPlace();
+            }
+            plane.transformationMatrix = mat;
+            if (this._options.worldParentNode) {
+                mat.multiplyToRef(this._options.worldParentNode.getWorldMatrix(), mat);
+            }
+        }
+        return <IWebXRPlane>plane;
+    }
+
+    /**
+     * avoiding using Array.find for global support.
+     * @param xrPlane the plane to find in the array
+     */
+    private findIndexInPlaneArray(xrPlane: XRPlane) {
+        for (let i = 0; i < this._detectedPlanes.length; ++i) {
+            if (this._detectedPlanes[i].xrPlane === xrPlane) {
+                return i;
+            }
+        }
+        return -1;
+    }
+}
+
+//register the plugin
+WebXRFeaturesManager.AddWebXRFeature(WebXRPlaneDetector.Name, (xrSessionManager, options) => {
+    return () => new WebXRPlaneDetector(xrSessionManager, options);
+}, WebXRPlaneDetector.Version);

src/Cameras/XR/features/index.ts

@@ -0,0 +1,4 @@
+export * from "./WebXRHitTestLegacy";
+export * from "./WebXRAnchorSystem";
+export * from "./WebXRPlaneDetector";
+export * from "./WebXRBackgroundRemover";

src/Cameras/XR/index.ts

@@ -9,4 +9,6 @@ export * from "./webXRController";
 export * from "./webXRManagedOutputCanvas";
 export * from "./webXRTypes";
 export * from "./webXRSessionManager";
-export * from "./webXRDefaultExperience";
+export * from "./webXRDefaultExperience";
+export * from "./webXRFeaturesManager";
+export * from "./features/index";

src/Cameras/XR/webXRExperienceHelper.ts

@@ -7,6 +7,7 @@ import { Camera } from "../../Cameras/camera";
 import { WebXRSessionManager } from "./webXRSessionManager";
 import { WebXRCamera } from "./webXRCamera";
 import { WebXRState, WebXRRenderTarget } from './webXRTypes';
+import { WebXRFeaturesManager } from './webXRFeaturesManager';
 
 /**
  * Base set of functionality needed to create an XR experince (WebXRSessionManager, Camera, StateManagement, etc.)
@@ -42,6 +43,9 @@ export class WebXRExperienceHelper implements IDisposable {
     /** Session manager used to keep track of xr session */
     public sessionManager: WebXRSessionManager;
 
+    /** A features manager for this xr session */
+    public featuresManager: WebXRFeaturesManager;
+
     private _nonVRCamera: Nullable<Camera> = null;
     private _originalSceneAutoClear = true;
 
@@ -69,6 +73,7 @@ export class WebXRExperienceHelper implements IDisposable {
     private constructor(private scene: Scene) {
         this.camera = new WebXRCamera("", scene);
         this.sessionManager = new WebXRSessionManager(scene);
+        this.featuresManager = new WebXRFeaturesManager(this.sessionManager);
         this.container = new AbstractMesh("WebXR Container", scene);
         this.camera.parent = this.container;
 

src/Cameras/XR/webXRFeaturesManager.ts

@@ -0,0 +1,279 @@
+import { WebXRSessionManager } from './webXRSessionManager';
+import { IDisposable } from '../../scene';
+
+/**
+ * Defining the interface required for a (webxr) feature
+ */
+export interface IWebXRFeature extends IDisposable {
+    /**
+     * Attach the feature to the session
+     * Will usually be called by the features manager
+     *
+     * @returns true if successful.
+     */
+    attach(): boolean;
+    /**
+     * Detach the feature from the session
+     * Will usually be called by the features manager
+     *
+     * @returns true if successful.
+     */
+    detach(): boolean;
+}
+
+/**
+ * Defining the constructor of a feature. Used to register the modules.
+ */
+export type WebXRFeatureConstructor = (xrSessionManager: WebXRSessionManager, options?: any) => (() => IWebXRFeature);
+
+/**
+ * The WebXR features manager is responsible of enabling or disabling features required for the current XR session.
+ * It is mainly used in AR sessions.
+ *
+ * A feature can have a version that is defined by Babylon (and does not correspond with the webxr version).
+ */
+export class WebXRFeaturesManager implements IDisposable {
+
+    private static readonly _AvailableFeatures: {
+        [name: string]: {
+            stable: number;
+            latest: number;
+            [version: number]: WebXRFeatureConstructor;
+        }
+    } = {};
+
+    /**
+     * Used to register a module. After calling this function a developer can use this feature in the scene.
+     * Mainly used internally.
+     *
+     * @param featureName the name of the feature to register
+     * @param constructorFunction the function used to construct the module
+     * @param version the (babylon) version of the module
+     * @param stable is that a stable version of this module
+     */
+    public static AddWebXRFeature(featureName: string, constructorFunction: WebXRFeatureConstructor, version: number = 1, stable: boolean = false) {
+        this._AvailableFeatures[featureName] = this._AvailableFeatures[featureName] || { latest: version };
+        if (version > this._AvailableFeatures[featureName].latest) {
+            this._AvailableFeatures[featureName].latest = version;
+        }
+        if (stable) {
+            this._AvailableFeatures[featureName].stable = version;
+        }
+        this._AvailableFeatures[featureName][version] = constructorFunction;
+    }
+
+    /**
+     * Returns a constructor of a specific feature.
+     *
+     * @param featureName the name of the feature to construct
+     * @param version the version of the feature to load
+     * @param xrSessionManager the xrSessionManager. Used to construct the module
+     * @param options optional options provided to the module.
+     * @returns a function that, when called, will return a new instance of this feature
+     */
+    public static ConstructFeature(featureName: string, version: number = 1, xrSessionManager: WebXRSessionManager, options?: any): (() => IWebXRFeature) {
+        const constructorFunction = this._AvailableFeatures[featureName][version];
+        if (!constructorFunction) {
+            // throw an error? return nothing?
+            throw new Error('feature not found');
+        }
+
+        return constructorFunction(xrSessionManager, options);
+    }
+
+    /**
+     * Return the latest unstable version of this feature
+     * @param featureName the name of the feature to search
+     * @returns the version number. if not found will return -1
+     */
+    public static GetLatestVersionOfFeature(featureName: string): number {
+        return (this._AvailableFeatures[featureName] && this._AvailableFeatures[featureName].latest) || -1;
+    }
+
+    /**
+     * Return the latest stable version of this feature
+     * @param featureName the name of the feature to search
+     * @returns the version number. if not found will return -1
+     */
+    public static GetStableVersionOfFeature(featureName: string): number {
+        return (this._AvailableFeatures[featureName] && this._AvailableFeatures[featureName].stable) || -1;
+    }
+
+    /**
+     * Can be used to return the list of features currently registered
+     *
+     * @returns an Array of available features
+     */
+    public static GetAvailableFeatures() {
+        return Object.keys(this._AvailableFeatures);
+    }
+
+    /**
+     * Gets the versions available for a specific feature
+     * @param featureName the name of the feature
+     * @returns an array with the available versions
+     */
+    public static GetAvailableVersions(featureName: string) {
+        return Object.keys(this._AvailableFeatures[featureName]);
+    }
+
+    private _features: {
+        [name: string]: {
+            featureImplementation: IWebXRFeature,
+            version: number,
+            enabled: boolean,
+            attached: boolean
+        }
+    } = {};
+
+    /**
+     * constructs a new features manages.
+     *
+     * @param _xrSessionManager an instance of WebXRSessionManager
+     */
+    constructor(private _xrSessionManager: WebXRSessionManager) {
+        // when session starts / initialized - attach
+        this._xrSessionManager.onXRSessionInit.add(() => {
+            this.getEnabledFeatures().forEach((featureName) => {
+                const feature = this._features[featureName];
+                if (feature.enabled && !feature.attached) {
+                    this.attachFeature(featureName);
+                }
+            });
+        });
+
+        // when session ends - detach
+        this._xrSessionManager.onXRSessionEnded.add(() => {
+            this.getEnabledFeatures().forEach((featureName) => {
+                const feature = this._features[featureName];
+                if (feature.enabled && feature.attached) {
+                    // detach, but don't disable!
+                    this.detachFeature(featureName);
+                }
+            });
+        });
+    }
+
+    /**
+     * Enable a feature using its name and a version. This will enable it in the scene, and will be responsible to attach it when the session starts.
+     *
+     * @param featureName the name of the feature to load or the class of the feature
+     * @param version optional version to load. if not provided the latest version will be enabled
+     * @param moduleOptions options provided to the module. Ses the module documentation / constructor
+     * @param attachIfPossible if set to true (default) the feature will be automatically attached, if it is currently possible
+     * @returns a new constructed feature or throws an error if feature not found.
+     */
+    public enableFeature(featureName: string | { Name: string }, version: number | string = 'latest', moduleOptions: any = {}, attachIfPossible: boolean = true): IWebXRFeature {
+        const name = typeof featureName === 'string' ? featureName : featureName.Name;
+        let versionToLoad = 0;
+        if (typeof version === 'string') {
+            if (version === 'stable') {
+                versionToLoad = WebXRFeaturesManager.GetStableVersionOfFeature(name);
+            } else if (version === 'latest') {
+                versionToLoad = WebXRFeaturesManager.GetLatestVersionOfFeature(name);
+            }
+            if (versionToLoad === -1) {
+                throw new Error(`feature not found - ${name} (${version})`);
+            }
+        } else {
+            versionToLoad = version;
+        }
+        // check if already initialized
+        const feature = this._features[name];
+        if (!feature || !feature.featureImplementation || feature.version !== versionToLoad) {
+            const constructFunction = WebXRFeaturesManager.ConstructFeature(name, versionToLoad, this._xrSessionManager, moduleOptions);
+            if (!constructFunction) {
+                // report error?
+                throw new Error(`feature not found - ${name}`);
+            }
+
+            if (feature) {
+                this.disableFeature(name);
+            }
+
+            this._features[name] = {
+                featureImplementation: constructFunction(),
+                attached: false,
+                enabled: true,
+                version: versionToLoad
+            };
+        } else {
+            // make sure it is enabled now:
+            feature.enabled = true;
+        }
+
+        // if session started already, request and enable
+        if (this._xrSessionManager.session && !feature.attached && attachIfPossible) {
+            // enable feature
+            this.attachFeature(name);
+        }
+
+        return this._features[name].featureImplementation;
+    }
+
+    /**
+     * Used to disable an already-enabled feature
+     * @param featureName the feature to disable
+     * @returns true if disable was successful
+     */
+    public disableFeature(featureName: string | { Name: string }): boolean {
+        const name = typeof featureName === 'string' ? featureName : featureName.Name;
+        const feature = this._features[name];
+        if (feature && feature.enabled) {
+            feature.enabled = false;
+            this.detachFeature(name);
+            return true;
+        }
+        return false;
+    }
+
+    /**
+     * Attach a feature to the current session. Mainly used when session started to start the feature effect.
+     * Can be used during a session to start a feature
+     * @param featureName the name of feature to attach
+     */
+    public attachFeature(featureName: string) {
+        const feature = this._features[featureName];
+        if (feature && feature.enabled && !feature.attached) {
+            feature.featureImplementation.attach();
+            feature.attached = true;
+        }
+    }
+
+    /**
+     * Can be used inside a session or when the session ends to detach a specific feature
+     * @param featureName the name of the feature to detach
+     */
+    public detachFeature(featureName: string) {
+        const feature = this._features[featureName];
+        if (feature && feature.attached) {
+            feature.featureImplementation.detach();
+            feature.attached = false;
+        }
+    }
+
+    /**
+     * Get the list of enabled features
+     * @returns an array of enabled features
+     */
+    public getEnabledFeatures() {
+        return Object.keys(this._features);
+    }
+
+    /**
+     * get the implementation of an enabled feature.
+     * @param featureName the name of the feature to load
+     * @returns the feature class, if found
+     */
+    public getEnabledFeature(featureName: string): IWebXRFeature {
+        return this._features[featureName] && this._features[featureName].featureImplementation;
+    }
+
+    /**
+     * dispose this features manager
+     */
+    dispose(): void {
+        this.getEnabledFeatures().forEach((feature) => this._features[feature].featureImplementation.dispose());
+    }
+
+}

src/Cameras/XR/webXRSessionManager.ts

@@ -56,6 +56,9 @@ export class WebXRSessionManager implements IDisposable {
      */
     public currentFrame: Nullable<XRFrame>;
 
+    /** WebXR timestamp updated every frame */
+    public currentTimestamp: number = -1;
+
     private _xrNavigator: any;
     private baseLayer: Nullable<XRWebGLLayer> = null;
     private _rttProvider: Nullable<IRenderTargetProvider>;
@@ -166,6 +169,7 @@ export class WebXRSessionManager implements IDisposable {
                 }
                 // Store the XR frame in the manager to be consumed by the XR camera to update pose
                 this.currentFrame = xrFrame;
+                this.currentTimestamp = timestamp;
                 if (xrFrame) {
                     this.onXRFrameObservable.notifyObservers(xrFrame);
                 }
@@ -202,12 +206,10 @@ export class WebXRSessionManager implements IDisposable {
      * @returns Promise which resolves after it exits XR
      */
     public exitXRAsync() {
-        if (this.session) {
-            try {
-                return this.session.end();
-            } catch (e) {
+        if (this.session && !this._sessionEnded) {
+            return this.session.end().catch((e) => {
                 Logger.Warn("could not end XR session. It has ended already.");
-            }
+            });
         }
         return Promise.resolve();
     }

src/LibDeclarations/webxr.d.ts

@@ -55,8 +55,9 @@ interface XRInputSource {
     profiles: Array<string>;
 }
 
-interface XRSession {
+interface XRSession extends XRAnchorCreator {
     addEventListener: Function;
+    removeEventListener: Function;
     requestReferenceSpace(type: XRReferenceSpaceType): Promise<XRReferenceSpace>;
     updateRenderState(XRRenderStateInit: XRRenderState): Promise<void>;
     requestAnimationFrame: Function;
@@ -64,6 +65,12 @@ interface XRSession {
     renderState: XRRenderState;
     inputSources: Array<XRInputSource>;
 
+    // AR hit test
+    requestHitTest(ray: XRRay, referenceSpace: XRReferenceSpace): Promise<XRHitResult[]>;
+
+    updateWorldTrackingState(options: {
+        planeDetectionState?: { enabled: boolean; }
+    }): void;
 }
 
 interface XRReferenceSpace extends XRSpace {
@@ -71,10 +78,20 @@ interface XRReferenceSpace extends XRSpace {
     onreset: any;
 }
 
+type XRPlaneSet = Set<XRPlane>;
+type XRAnchorSet = Set<XRAnchor>;
+
 interface XRFrame {
     session: XRSession;
     getViewerPose(referenceSpace: XRReferenceSpace): XRViewerPose | undefined;
     getPose(space: XRSpace, baseSpace: XRSpace): XRPose | undefined;
+
+    // Anchors
+    trackedAnchors?: XRAnchorSet;
+    // Planes
+    worldInformation: {
+        detectedPlanes?: XRPlaneSet;
+    };
 }
 
 interface XRViewerPose extends XRPose {
@@ -87,12 +104,12 @@ interface XRPose {
 }
 
 interface XRWebGLLayerOptions {
-    antialias ?: boolean;
-    depth ?: boolean;
-    stencil ?: boolean;
-    alpha ?: boolean;
-    multiview ?: boolean;
-    framebufferScaleFactor ?: number;
+    antialias?: boolean;
+    depth?: boolean;
+    stencil?: boolean;
+    alpha?: boolean;
+    multiview?: boolean;
+    framebufferScaleFactor?: number;
 }
 
 declare var XRWebGLLayer: {
@@ -106,7 +123,8 @@ interface XRWebGLLayer {
     getViewport: Function;
 }
 
-interface XRRigidTransform {
+declare class XRRigidTransform {
+    constructor(matrix: Float32Array);
     position: DOMPointReadOnly;
     orientation: DOMPointReadOnly;
     matrix: Float32Array;
@@ -128,4 +146,36 @@ interface XRInputSourceChangeEvent {
 interface XRInputSourceEvent extends Event {
     readonly frame: XRFrame;
     readonly inputSource: XRInputSource;
+}
+
+// Experimental(er) features
+declare class XRRay {
+    constructor(transformOrOrigin: XRRigidTransform | DOMPointReadOnly, direction?: DOMPointReadOnly);
+    origin: DOMPointReadOnly;
+    direction: DOMPointReadOnly;
+    matrix: Float32Array;
+}
+
+interface XRHitResult {
+    hitMatrix: Float32Array;
+}
+
+interface XRAnchor {
+    // remove?
+    id?: string;
+    anchorSpace: XRSpace;
+    lastChangedTime: number;
+    detach(): void;
+}
+
+interface XRPlane extends XRAnchorCreator {
+    orientation: "Horizontal" | "Vertical";
+    planeSpace: XRSpace;
+    polygon: Array<DOMPointReadOnly>;
+    lastChangedTime: number;
+}
+
+interface XRAnchorCreator {
+    // AR Anchors
+    createAnchor(pose: XRPose | XRRigidTransform, referenceSpace: XRReferenceSpace): Promise<XRAnchor>;
 }