Merge pull request #7790 from RaananW/new-hit-test

[WIP] New hit test

dist/preview release/what's new.md

@@ -34,19 +34,23 @@
 - Recast.js plugin nav mesh and crowd agent to ref performance optimizations. ([MackeyK24](https://github.com/MackeyK24))
 
 ### Loaders
+
 - Added support for glTF mesh instancing extension ([#7521](https://github.com/BabylonJS/Babylon.js/issues/7521)) ([drigax](https://github.com/Drigax))
 
 ### Navigation
 - export/load prebuilt binary navigation mesh ([cedricguillemet](https://github.com/cedricguillemet))
 
 ### Materials
+
 - Added the `roughness` and `albedoScaling` parameters to PBR sheen ([Popov72](https://github.com/Popov72))
 - Updated the energy conservation factor for the clear coat layer in PBR materials ([Popov72](https://github.com/Popov72))
 - Added the `transparencyMode` property to the `StandardMaterial` class ([Popov72](https://github.com/Popov72))
 - Added to `FresnelParameters` constructor options and equals method ([brianzinn](https://github.com/brianzinn))
 
 ### WebXR
+
 - Added optional ray and mesh selection predicates to `WebXRControllerPointerSelection` ([Exolun](https://github.com/Exolun))
+- Implemented the new WebXR HitTest API ([#7364](https://github.com/BabylonJS/Babylon.js/issues/7364)) ([RaananW](https://github.com/RaananW))
 
 ### Collisions
 - Added an option to optimize collision detection performance ([jsdream](https://github.com/jsdream)) - [PR](https://github.com/BabylonJS/Babylon.js/pull/7810)

src/LibDeclarations/webxr.d.ts

@@ -70,9 +70,14 @@ interface XRSession extends XRAnchorCreator {
     renderState: XRRenderState;
     inputSources: Array<XRInputSource>;
 
-    // AR hit test
+    // hit test
+    requestHitTestSource(options: XRHitTestOptionsInit): Promise<XRHitTestSource>;
+    requestHitTestSourceForTransientInput(options: XRTransientInputHitTestOptionsInit): Promise<XRTransientInputHitTestSource>;
+
+    // legacy AR hit test
     requestHitTest(ray: XRRay, referenceSpace: XRReferenceSpace): Promise<XRHitResult[]>;
 
+    // legacy plane detection
     updateWorldTrackingState(options: {
         planeDetectionState?: { enabled: boolean; }
     }): void;
@@ -91,6 +96,9 @@ interface XRFrame {
     getViewerPose(referenceSpace: XRReferenceSpace): XRViewerPose | undefined;
     getPose(space: XRSpace, baseSpace: XRSpace): XRPose | undefined;
 
+    // AR
+    getHitTestResults(hitTestSource: XRHitTestSource): Array<XRHitTestResult> ;
+    getHitTestResultsForTransientInput(hitTestSource: XRTransientInputHitTestSource): Array<XRTransientInputHitTestResult>;
     // Anchors
     trackedAnchors?: XRAnchorSet;
     // Planes
@@ -161,10 +169,44 @@ declare class XRRay {
     matrix: Float32Array;
 }
 
+declare enum XRHitTestTrackableType {
+    "point",
+    "plane"
+}
+
 interface XRHitResult {
     hitMatrix: Float32Array;
 }
 
+interface XRTransientInputHitTestResult {
+    readonly inputSource: XRInputSource;
+    readonly results: Array<XRHitTestResult>;
+}
+
+interface XRHitTestResult {
+    getPose(baseSpace: XRSpace): XRPose | undefined;
+}
+
+interface XRHitTestSource {
+    cancel(): void;
+}
+
+interface XRTransientInputHitTestSource {
+    cancel(): void;
+}
+
+interface XRHitTestOptionsInit {
+    space: XRSpace;
+    entityTypes?: Array<XRHitTestTrackableType>;
+    offsetRay?: XRRay;
+}
+
+interface XRTransientInputHitTestOptionsInit {
+    profile: string;
+    entityTypes?: Array<XRHitTestTrackableType>;
+    offsetRay?: XRRay;
+}
+
 interface XRAnchor {
     // remove?
     id?: string;

src/XR/features/WebXRHitTest.ts

@@ -0,0 +1,246 @@
+import { WebXRFeaturesManager, WebXRFeatureName } from '../webXRFeaturesManager';
+import { WebXRSessionManager } from '../webXRSessionManager';
+import { Observable } from '../../Misc/observable';
+import { Vector3, Matrix, Quaternion } from '../../Maths/math.vector';
+import { WebXRAbstractFeature } from './WebXRAbstractFeature';
+import { IWebXRLegacyHitTestOptions, IWebXRLegacyHitResult } from './WebXRHitTestLegacy';
+
+/**
+ * Options used for hit testing (version 2)
+ */
+export interface IWebXRHitTestOptions extends IWebXRLegacyHitTestOptions {
+    /**
+     * Do not create a permanent hit test. Will usually be used when only
+     * transient inputs are needed.
+     */
+    disablePermanentHitTest?: boolean;
+    /**
+     * Enable transient (for example touch-based) hit test inspections
+     */
+    enableTransientHitTest?: boolean;
+    /**
+     * Offset ray for the permanent hit test
+     */
+    offsetRay?: Vector3;
+    /**
+     * Offset ray for the transient hit test
+     */
+    transientOffsetRay?: Vector3;
+    /**
+     * Instead of using viewer space for hit tests, use the reference space defined in the session manager
+     */
+    useReferenceSpace?: boolean;
+}
+
+/**
+ * Interface defining the babylon result of hit-test
+ */
+export interface IWebXRHitResult extends IWebXRLegacyHitResult {
+    /**
+     * The input source that generated this hit test (if transient)
+     */
+    inputSource?: XRInputSource;
+    /**
+     * Is this a transient hit test
+     */
+    isTransient?: boolean;
+    /**
+     * Position of the hit test result
+     */
+    position: Vector3;
+    /**
+     * Rotation of the hit test result
+     */
+    rotationQuaternion: Quaternion;
+}
+
+/**
+ * The currently-working hit-test module.
+ * Hit test (or Ray-casting) is used to interact with the real world.
+ * For further information read here - https://github.com/immersive-web/hit-test
+ *
+ * Tested on chrome (mobile) 80.
+ */
+export class WebXRHitTest extends WebXRAbstractFeature {
+    private _tmpMat: Matrix = new Matrix();
+    private _tmpPos: Vector3 = new Vector3();
+    private _tmpQuat: Quaternion = new Quaternion();
+    private _transientXrHitTestSource: XRTransientInputHitTestSource;
+    // in XR space z-forward is negative
+    private _xrHitTestSource: XRHitTestSource;
+    private initHitTestSource = () => {
+        const offsetRay = new XRRay(this.options.offsetRay || {});
+        const options: XRHitTestOptionsInit = {
+            space: this.options.useReferenceSpace ? this._xrSessionManager.referenceSpace : this._xrSessionManager.viewerReferenceSpace,
+            offsetRay: offsetRay
+        };
+        this._xrSessionManager.session.requestHitTestSource(options).then((hitTestSource) => {
+            if (this._xrHitTestSource) {
+                this._xrHitTestSource.cancel();
+            }
+            this._xrHitTestSource = hitTestSource;
+        });
+    }
+
+    /**
+     * The module's name
+     */
+    public static readonly Name = WebXRFeatureName.HIT_TEST;
+    /**
+     * The (Babylon) version of this module.
+     * This is an integer representing the implementation version.
+     * This number does not correspond to the WebXR specs version
+     */
+    public static readonly Version = 2;
+
+    /**
+     * When set to true, each hit test will have its own position/rotation objects
+     * When set to false, position and rotation objects will be reused for each hit test. It is expected that
+     * the developers will clone them or copy them as they see fit.
+     */
+    public autoCloneTransformation: boolean = false;
+    /**
+     * Populated with the last native XR Hit Results
+     */
+    public lastNativeXRHitResults: XRHitResult[] = [];
+    /**
+     * Triggered when new babylon (transformed) hit test results are available
+     */
+    public onHitTestResultObservable: Observable<IWebXRHitResult[]> = new Observable();
+    /**
+     * Use this to temporarily pause hit test checks.
+     */
+    public paused: boolean = false;
+
+    /**
+     * Creates a new instance of the hit test feature
+     * @param _xrSessionManager an instance of WebXRSessionManager
+     * @param options options to use when constructing this feature
+     */
+    constructor(_xrSessionManager: WebXRSessionManager,
+        /**
+         * options to use when constructing this feature
+         */
+        public readonly options: IWebXRHitTestOptions = {}) {
+        super(_xrSessionManager);
+    }
+
+    /**
+     * attach this feature
+     * Will usually be called by the features manager
+     *
+     * @returns true if successful.
+     */
+    public attach(): boolean {
+        if (!super.attach()) {
+            return false;
+        }
+
+        if (!this.options.disablePermanentHitTest) {
+            if (this._xrSessionManager.referenceSpace) {
+                this.initHitTestSource();
+            }
+            this._xrSessionManager.onXRReferenceSpaceChanged.add(this.initHitTestSource);
+        }
+        if (this.options.enableTransientHitTest) {
+            const offsetRay = new XRRay(this.options.transientOffsetRay || {});
+            this._xrSessionManager.session.requestHitTestSourceForTransientInput({
+                profile : 'generic-touchscreen',
+                offsetRay
+            }).then((hitSource) => {
+                this._transientXrHitTestSource = hitSource;
+            });
+        }
+
+        return true;
+    }
+
+    /**
+     * detach this feature.
+     * Will usually be called by the features manager
+     *
+     * @returns true if successful.
+     */
+    public detach(): boolean {
+        if (!super.detach()) {
+            return false;
+        }
+        if (this._xrHitTestSource) {
+            this._xrHitTestSource.cancel();
+        }
+        this._xrSessionManager.onXRReferenceSpaceChanged.removeCallback(this.initHitTestSource);
+        if (this._transientXrHitTestSource) {
+            this._transientXrHitTestSource.cancel();
+        }
+        return true;
+    }
+
+    /**
+     * Dispose this feature and all of the resources attached
+     */
+    public dispose(): void {
+        super.dispose();
+        this.onHitTestResultObservable.clear();
+    }
+
+    protected _onXRFrame(frame: XRFrame) {
+        // make sure we do nothing if (async) not attached
+        if (!this.attached || this.paused) {
+            return;
+        }
+
+        if (this._xrHitTestSource) {
+            const results = frame.getHitTestResults(this._xrHitTestSource);
+            if (results.length) {
+                this._processWebXRHitTestResult(results);
+            }
+        }
+        if (this._transientXrHitTestSource) {
+            let hitTestResultsPerInputSource = frame.getHitTestResultsForTransientInput(this._transientXrHitTestSource);
+
+            hitTestResultsPerInputSource.forEach((resultsPerInputSource) => {
+                if (resultsPerInputSource.results.length > 0) {
+                    this._processWebXRHitTestResult(resultsPerInputSource.results, resultsPerInputSource.inputSource);
+                }
+            });
+        }
+    }
+
+    private _processWebXRHitTestResult(hitTestResults: XRHitTestResult[], inputSource?: XRInputSource) {
+        const results : IWebXRHitResult[] = [];
+        hitTestResults.forEach((hitTestResult) => {
+            const pose = hitTestResult.getPose(this._xrSessionManager.referenceSpace);
+            if (!pose) {
+                return;
+            }
+            this._tmpPos.copyFrom(pose.transform.position as unknown as Vector3);
+            this._tmpQuat.copyFrom(pose.transform.orientation as unknown as Quaternion);
+            Matrix.FromFloat32ArrayToRefScaled(pose.transform.matrix, 0, 1, this._tmpMat);
+            if (!this._xrSessionManager.scene.useRightHandedSystem) {
+                this._tmpPos.z *= -1;
+                this._tmpQuat.z *= -1;
+                this._tmpQuat.w *= -1;
+                this._tmpMat.toggleModelMatrixHandInPlace();
+            }
+
+            const result: IWebXRHitResult = {
+                position: this.autoCloneTransformation ? this._tmpPos.clone() : this._tmpPos,
+                rotationQuaternion: this.autoCloneTransformation ? this._tmpQuat.clone() : this._tmpQuat,
+                transformationMatrix: this.autoCloneTransformation ? this._tmpMat.clone() : this._tmpMat,
+                inputSource: inputSource,
+                isTransient: !!inputSource,
+                xrHitResult: hitTestResult
+            };
+            results.push(result);
+        });
+
+        if (results.length) {
+            this.onHitTestResultObservable.notifyObservers(results);
+        }
+    }
+}
+
+//register the plugin versions
+WebXRFeaturesManager.AddWebXRFeature(WebXRHitTest.Name, (xrSessionManager, options) => {
+    return () => new WebXRHitTest(xrSessionManager, options);
+}, WebXRHitTest.Version, false);

src/XR/features/WebXRHitTestLegacy.ts

@@ -10,7 +10,7 @@ import { WebXRAbstractFeature } from './WebXRAbstractFeature';
 /**
  * Options used for hit testing
  */
-export interface IWebXRHitTestOptions {
+export interface IWebXRLegacyHitTestOptions {
     /**
      * Only test when user interacted with the scene. Default - hit test every frame
      */
@@ -24,7 +24,7 @@ export interface IWebXRHitTestOptions {
 /**
  * Interface defining the babylon result of raycasting/hit-test
  */
-export interface IWebXRHitResult {
+export interface IWebXRLegacyHitResult {
     /**
      * Transformation matrix that can be applied to a node that will put it in the hit point location
      */
@@ -32,7 +32,7 @@ export interface IWebXRHitResult {
     /**
      * The native hit test result
      */
-    xrHitResult: XRHitResult;
+    xrHitResult: XRHitResult | XRHitTestResult;
 }
 
 /**
@@ -65,7 +65,7 @@ export class WebXRHitTestLegacy extends WebXRAbstractFeature {
     /**
      * Triggered when new babylon (transformed) hit test results are available
      */
-    public onHitTestResultObservable: Observable<IWebXRHitResult[]> = new Observable();
+    public onHitTestResultObservable: Observable<IWebXRLegacyHitResult[]> = new Observable();
 
     /**
      * Creates a new instance of the (legacy version) hit test feature
@@ -76,7 +76,7 @@ export class WebXRHitTestLegacy extends WebXRAbstractFeature {
         /**
          * options to use when constructing this feature
          */
-        public readonly options: IWebXRHitTestOptions = {}) {
+        public readonly options: IWebXRLegacyHitTestOptions = {}) {
         super(_xrSessionManager);
     }
 
@@ -204,4 +204,4 @@ export class WebXRHitTestLegacy extends WebXRAbstractFeature {
 //register the plugin versions
 WebXRFeaturesManager.AddWebXRFeature(WebXRHitTestLegacy.Name, (xrSessionManager, options) => {
     return () => new WebXRHitTestLegacy(xrSessionManager, options);
-}, WebXRHitTestLegacy.Version, true);
+}, WebXRHitTestLegacy.Version, false);

src/XR/features/index.ts

@@ -4,4 +4,5 @@ export * from "./WebXRPlaneDetector";
 export * from "./WebXRBackgroundRemover";
 export * from "./WebXRControllerTeleportation";
 export * from "./WebXRControllerPointerSelection";
-export * from './WebXRControllerPhysics';
+export * from './WebXRControllerPhysics';
+export * from './WebXRHitTest';