Merge pull request #8677 from Alex-MSFT/feature_points

Expose feature points as an experimental WebXR feature

dist/preview release/what's new.md

@@ -170,6 +170,7 @@
 - All camera view matrices are now calculated by Babylon to support left and right handed systems ([RaananW](https://github.com/RaananW))
 - WebXR Features Manager now has the ability to check if a feature can be enabled, and set native features optional or required ([RaananW](https://github.com/RaananW))
 - Optional camera gaze mode added to the pointer selection feature ([RaananW](https://github.com/RaananW))
+- Exposing feature points when running on top of BabylonNative ([Alex-MSFT](https://github.com/Alex-MSFT))
 - WebXR hit test can now define different entity type for the results ([#8687](https://github.com/BabylonJS/Babylon.js/issues/8687)) ([RaananW](https://github.com/RaananW))
 
 ### Collisions

src/LibDeclarations/webxr.nativeextensions.d.ts

@@ -0,0 +1,11 @@
+// This file contains native only extensions for WebXR  These APIs are not supported in the browser yet.
+// They are intended for use with either Babylon Native https://github.com/BabylonJS/BabylonNative or
+// Babylon React Native: https://github.com/BabylonJS/BabylonReactNative
+
+interface XRSession {
+    trySetFeaturePointCloudEnabled(enabled: boolean): boolean;
+}
+
+interface XRFrame {
+    featurePointCloud? : Array<number>;
+}

src/XR/features/WebXRFeaturePointSystem.ts

@@ -0,0 +1,170 @@
+import { WebXRFeaturesManager, WebXRFeatureName } from "../webXRFeaturesManager";
+import { WebXRSessionManager } from "../webXRSessionManager";
+import { Observable } from "../../Misc/observable";
+import { Vector3 } from "../../Maths/math.vector";
+import { WebXRAbstractFeature } from "./WebXRAbstractFeature";
+
+/**
+ * A babylon interface for a "WebXR" feature point.
+ * Represents the position and confidence value of a given feature point.
+ */
+export interface IWebXRFeaturePoint {
+    /**
+     * Represents the position of the feature point in world space.
+     */
+    position : Vector3;
+    /**
+     * Represents the confidence value of the feature point in world space. 0 being least confident, and 1 being most confident.
+     */
+    confidenceValue : number;
+}
+
+/**
+ * The feature point system is used to detect feature points from real world geometry.
+ * This feature is currently experimental and only supported on BabylonNative, and should not be used in the browser.
+ * The newly introduced API can be seen in webxr.nativeextensions.d.ts and described in FeaturePoints.md.
+ */
+export class WebXRFeaturePointSystem extends WebXRAbstractFeature {
+    private _enabled: boolean = false;
+    private _featurePointCloud: Array<IWebXRFeaturePoint> = [];
+
+    /**
+     * The module's name
+     */
+    public static readonly Name = WebXRFeatureName.FEATURE_POINTS;
+    /**
+     * The (Babylon) version of this module.
+     * This is an integer representing the implementation version.
+     * This number does not correspond to the WebXR specs version
+     */
+    public static readonly Version = 1;
+     /**
+     * Observers registered here will be executed whenever new feature points are added (on XRFrame while the session is tracking).
+     * Will notify the observers about which feature points have been added.
+     */
+    public readonly onFeaturePointsAddedObservable: Observable<number[]> = new Observable();
+    /**
+     * Observers registered here will be executed whenever a feature point has been updated (on XRFrame while the session is tracking).
+     * Will notify the observers about which feature points have been updated.
+     */
+    public readonly onFeaturePointsUpdatedObservable: Observable<number[]> = new Observable();
+    /**
+     * The current feature point cloud maintained across frames.
+     */
+    public get featurePointCloud() : Array<IWebXRFeaturePoint> {
+        return this._featurePointCloud;
+    }
+
+    /**
+     * construct the feature point system
+     * @param _xrSessionManager an instance of xr Session manager
+     */
+    constructor(_xrSessionManager: WebXRSessionManager) {
+        super(_xrSessionManager);
+        this.xrNativeFeatureName = "bjsfeature-points";
+        if (this._xrSessionManager.session) {
+            this._init();
+        } else {
+            this._xrSessionManager.onXRSessionInit.addOnce(() => {
+                this._init();
+            });
+        }
+    }
+
+    /**
+     * Detach this feature.
+     * Will usually be called by the features manager
+     *
+     * @returns true if successful.
+     */
+    public detach(): boolean {
+        if (!super.detach()) {
+            return false;
+        }
+
+        this.featurePointCloud.length = 0;
+        return true;
+    }
+
+    /**
+     * Dispose this feature and all of the resources attached
+     */
+    public dispose(): void {
+        super.dispose();
+
+        this._featurePointCloud.length = 0;
+        this.onFeaturePointsUpdatedObservable.clear();
+        this.onFeaturePointsAddedObservable.clear();
+    }
+
+    /**
+     * On receiving a new XR frame if this feature is attached notify observers new feature point data is available.
+     */
+    protected _onXRFrame(frame: XRFrame) {
+        if (!this.attached || !this._enabled || !frame) {
+            return;
+        }
+
+        const featurePointRawData: number[] | undefined = frame.featurePointCloud;
+        if (!featurePointRawData || featurePointRawData.length === 0) {
+            return;
+        } else {
+            if (featurePointRawData.length % 5 !== 0) {
+                throw new Error("Received malformed feature point cloud of length: " + featurePointRawData.length);
+            }
+
+            const numberOfFeaturePoints : number = featurePointRawData.length / 5;
+            let updatedFeaturePoints = new Array();
+            let addedFeaturePoints = new Array();
+            for (var i = 0; i < numberOfFeaturePoints; i++) {
+                const rawIndex: number = i * 5;
+                const id = featurePointRawData[rawIndex + 4];
+
+                // IDs should be durable across frames and strictly increasing from 0 up, so use them as indexing into the feature point array.
+                if (!this._featurePointCloud[id]) {
+                    this._featurePointCloud[id] = { position: new Vector3(), confidenceValue: 0 };
+                    addedFeaturePoints.push(id);
+                } else {
+                    updatedFeaturePoints.push(id);
+                }
+
+                // Set the feature point values.
+                this._featurePointCloud[id].position.x = featurePointRawData[rawIndex];
+                this._featurePointCloud[id].position.y = featurePointRawData[rawIndex + 1];
+                this._featurePointCloud[id].position.z = featurePointRawData[rawIndex + 2];
+                this._featurePointCloud[id].confidenceValue = featurePointRawData[rawIndex + 3];
+            }
+
+            // Signal observers that feature points have been added if necessary.
+            if (addedFeaturePoints.length > 0) {
+                this.onFeaturePointsAddedObservable.notifyObservers(addedFeaturePoints);
+            }
+
+            // Signal observers that feature points have been updated if necessary.
+            if (updatedFeaturePoints.length > 0) {
+                this.onFeaturePointsUpdatedObservable.notifyObservers(updatedFeaturePoints);
+            }
+        }
+    }
+
+    /**
+     * Initializes the feature. If the feature point feature is not available for this environment do not mark the feature as enabled.
+     */
+    private _init() {
+        if (!this._xrSessionManager.session.trySetFeaturePointCloudEnabled || !this._xrSessionManager.session.trySetFeaturePointCloudEnabled(true)) {
+            // fail silently
+            return;
+        }
+
+        this._enabled = true;
+    }
+}
+
+// register the plugin
+WebXRFeaturesManager.AddWebXRFeature(
+    WebXRFeaturePointSystem.Name,
+    (xrSessionManager) => {
+        return () => new WebXRFeaturePointSystem(xrSessionManager);
+    },
+    WebXRFeaturePointSystem.Version
+);

src/XR/features/index.ts

@@ -6,3 +6,4 @@ export * from "./WebXRControllerTeleportation";
 export * from "./WebXRControllerPointerSelection";
 export * from "./WebXRControllerPhysics";
 export * from "./WebXRHitTest";
+export * from "./WebXRFeaturePointSystem";

src/XR/webXRFeaturesManager.ts

@@ -52,31 +52,35 @@ export class WebXRFeatureName {
     /**
      * The name of the anchor system feature
      */
-    public static ANCHOR_SYSTEM = "xr-anchor-system";
+    public static readonly ANCHOR_SYSTEM = "xr-anchor-system";
     /**
      * The name of the background remover feature
      */
-    public static BACKGROUND_REMOVER = "xr-background-remover";
+    public static readonly BACKGROUND_REMOVER = "xr-background-remover";
     /**
      * The name of the hit test feature
      */
-    public static HIT_TEST = "xr-hit-test";
+    public static readonly HIT_TEST = "xr-hit-test";
     /**
      * physics impostors for xr controllers feature
      */
-    public static PHYSICS_CONTROLLERS = "xr-physics-controller";
+    public static readonly PHYSICS_CONTROLLERS = "xr-physics-controller";
     /**
      * The name of the plane detection feature
      */
-    public static PLANE_DETECTION = "xr-plane-detection";
+    public static readonly PLANE_DETECTION = "xr-plane-detection";
     /**
      * The name of the pointer selection feature
      */
-    public static POINTER_SELECTION = "xr-controller-pointer-selection";
+    public static readonly POINTER_SELECTION = "xr-controller-pointer-selection";
     /**
      * The name of the teleportation feature
      */
-    public static TELEPORTATION = "xr-controller-teleportation";
+    public static readonly TELEPORTATION = "xr-controller-teleportation";
+    /**
+     * The name of the feature points feature.
+     */
+    public static readonly FEATURE_POINTS = "xr-feature-points";
 }
 
 /**