Merge pull request #9512 from RaananW/XRImageTracking

[XR] AR image tracking feature

dist/preview release/what's new.md

@@ -39,6 +39,8 @@
 ### WebXR
 
 - A browser error preventing the emulator to render scene is now correctly dealt with ([RaananW](https://github.com/RaananW))
+- Added a way to extend the XRSessionInit Object from inside of a feature ([RaananW](https://github.com/RaananW))
+- Added image tracking feature ([RaananW](https://github.com/RaananW))
 
 ## Bugs
 

src/LibDeclarations/webxr.d.ts

@@ -44,6 +44,7 @@ interface XRLayer extends EventTarget {}
 interface XRSessionInit {
     optionalFeatures?: string[];
     requiredFeatures?: string[];
+    trackedImages?: XRTrackedImageInit[];
 }
 
 interface XRSessionEvent extends Event {
@@ -143,6 +144,8 @@ interface XRFrame {
     worldInformation?: XRWorldInformation;
     // Hand tracking
     getJointPose?(joint: XRJointSpace, baseSpace: XRSpace): XRJointPose;
+    // Image tracking
+    getImageTrackingResults?(): Array<XRImageTrackingResult>;
 }
 
 interface XRInputSourceEvent extends Event {
@@ -214,6 +217,9 @@ interface XRSession {
 
     // legacy plane detection
     updateWorldTrackingState?(options: { planeDetectionState?: { enabled: boolean } }): void;
+
+    // image tracking
+    getTrackedImageScores?(): XRImageTrackingScore[];
 }
 
 interface XRViewerPose extends XRPose {
@@ -345,3 +351,18 @@ interface XRHand extends Iterable<XRJointSpace> {
     readonly LITTLE_PHALANX_DISTAL: number;
     readonly LITTLE_PHALANX_TIP: number;
 }
+
+type XRImageTrackingState = "tracked" | "emulated";
+type XRImageTrackingScore = "untrackable" | "trackable";
+
+interface XRTrackedImageInit {
+    image: ImageBitmap;
+    widthInMeters: number;
+}
+
+interface XRImageTrackingResult {
+    readonly imageSpace: XRSpace;
+    readonly index: number;
+    readonly trackingState: XRImageTrackingState;
+    readonly measuredWidthInMeters: number;
+}

src/XR/features/WebXRImageTracking.ts

@@ -0,0 +1,303 @@
+import { WebXRFeaturesManager, WebXRFeatureName } from "../webXRFeaturesManager";
+import { WebXRSessionManager } from "../webXRSessionManager";
+import { Observable } from "../../Misc/observable";
+import { WebXRAbstractFeature } from "./WebXRAbstractFeature";
+import { Matrix } from "../../Maths/math.vector";
+import { Tools } from "../../Misc/tools";
+import { Nullable } from "../../types";
+
+declare const XRImageTrackingResult: XRImageTrackingResult;
+
+/**
+ * Options interface for the background remover plugin
+ */
+export interface IWebXRImageTrackingOptions {
+    /**
+     * A required array with images to track
+     */
+    images: {
+        /**
+         * The source of the image. can be a URL or an image bitmap
+         */
+        src: string | ImageBitmap;
+        /**
+         * The estimated width in the real world (in meters)
+         */
+        estimatedRealWorldWidth: number; // In meters!
+    }[];
+}
+
+/**
+ * An object representing an image tracked by the system
+ */
+export interface IWebXRTrackedImage {
+    /**
+     * The ID of this image (which is the same as the position in the array that was used to initialize the feature)
+     */
+    id: number;
+    /**
+     * Is the transformation provided emulated. If it is, the system "guesses" its real position. Otherwise it can be considered as exact position.
+     */
+    emulated?: boolean;
+    /**
+     * Just in case it is needed - the image bitmap that is being tracked
+     */
+    originalBitmap: ImageBitmap;
+    /**
+     * The native XR result image tracking result, untouched
+     */
+    xrTrackingResult?: XRImageTrackingResult;
+    /**
+     * Width in real world (meters)
+     */
+    realWorldWidth?: number;
+    /**
+     * A transformation matrix of this current image in the current reference space.
+     */
+    transformationMatrix: Matrix;
+    /**
+     * The width/height ratio of this image. can be used to calculate the size of the detected object/image
+     */
+    ratio?: number;
+}
+
+/**
+ * Image tracking for immersive AR sessions.
+ * Providing a list of images and their estimated widths will enable tracking those images in the real world.
+ */
+export class WebXRImageTracking extends WebXRAbstractFeature {
+    /**
+     * The module's name
+     */
+    public static readonly Name = WebXRFeatureName.IMAGE_TRACKING;
+    /**
+     * The (Babylon) version of this module.
+     * This is an integer representing the implementation version.
+     * This number does not correspond to the WebXR specs version
+     */
+    public static readonly Version = 1;
+
+    /**
+     * This will be triggered if the underlying system deems an image untrackable.
+     * The index is the index of the image from the array used to initialize the feature.
+     */
+    public onUntrackableImageFoundObservable: Observable<number> = new Observable();
+    /**
+     * An image was deemed trackable, and the system will start tracking it.
+     */
+    public onTrackableImageFoundObservable: Observable<IWebXRTrackedImage> = new Observable();
+    /**
+     * The image was found and its state was updated.
+     */
+    public onTrackedImageUpdatedObservable: Observable<IWebXRTrackedImage> = new Observable();
+
+    private _trackedImages: IWebXRTrackedImage[] = [];
+
+    private _originalTrackingRequest: XRTrackedImageInit[];
+
+    /**
+     * constructs the image tracking feature
+     * @param _xrSessionManager the session manager for this module
+     * @param options read-only options to be used in this module
+     */
+    constructor(
+        _xrSessionManager: WebXRSessionManager,
+        /**
+         * read-only options to be used in this module
+         */
+        public readonly options: IWebXRImageTrackingOptions
+    ) {
+        super(_xrSessionManager);
+        this.xrNativeFeatureName = "image-tracking";
+        if (this.options.images.length === 0) {
+            // no images provided?... return.
+            return;
+        }
+        if (this._xrSessionManager.session) {
+            this._init();
+        } else {
+            this._xrSessionManager.onXRSessionInit.addOnce(() => {
+                this._init();
+            });
+        }
+    }
+
+    /**
+     * attach this feature
+     * Will usually be called by the features manager
+     *
+     * @returns true if successful.
+     */
+    public attach(): boolean {
+        return super.attach();
+    }
+
+    /**
+     * detach this feature.
+     * Will usually be called by the features manager
+     *
+     * @returns true if successful.
+     */
+    public detach(): boolean {
+        return super.detach();
+    }
+
+    /**
+     * Check if the needed objects are defined.
+     * This does not mean that the feature is enabled, but that the objects needed are well defined.
+     */
+    public isCompatible(): boolean {
+        return typeof XRImageTrackingResult !== "undefined";
+    }
+
+    /**
+     * Get a tracked image by its ID.
+     *
+     * @param id the id of the image to load (position in the init array)
+     * @returns a trackable image, if exists in this location
+     */
+    public getTrackedImageById(id: number): Nullable<IWebXRTrackedImage> {
+        return this._trackedImages[id] || null;
+    }
+
+    /**
+     * Dispose this feature and all of the resources attached
+     */
+    public dispose(): void {
+        super.dispose();
+        this._trackedImages.forEach((trackedImage) => {
+            trackedImage.originalBitmap.close();
+        });
+        this._trackedImages.length = 0;
+        this.onTrackableImageFoundObservable.clear();
+        this.onUntrackableImageFoundObservable.clear();
+        this.onTrackedImageUpdatedObservable.clear();
+    }
+
+    /**
+     * Extends the session init object if needed
+     * @returns augmentation object fo the xr session init object.
+     */
+    public async getXRSessionInitExtension(): Promise<Partial<XRSessionInit>> {
+        if (!this.options.images || !this.options.images.length) {
+            return {};
+        }
+        const promises = this.options.images.map((image) => {
+            if (typeof image.src === "string") {
+                const p = new Promise<ImageBitmap>((resolve, reject) => {
+                    if (typeof image.src === "string") {
+                        const img = new Image();
+                        img.src = image.src;
+                        img.onload = () => {
+                            img.decode().then(() => {
+                                createImageBitmap(img).then((imageBitmap) => {
+                                    resolve(imageBitmap);
+                                });
+                            });
+                        };
+                        img.onerror = () => {
+                            Tools.Error(`Error loading image ${image.src}`);
+                            reject(`Error loading image ${image.src}`);
+                        };
+                    }
+                });
+                return p;
+            } else {
+                return Promise.resolve(image.src); // resolve is probably unneeded
+            }
+        });
+
+        const images = await Promise.all(promises);
+
+        this._originalTrackingRequest = images.map((image, idx) => {
+            return {
+                image,
+                widthInMeters: this.options.images[idx].estimatedRealWorldWidth,
+            };
+        });
+
+        return {
+            trackedImages: this._originalTrackingRequest,
+        };
+    }
+
+    protected _onXRFrame(_xrFrame: XRFrame) {
+        if (!_xrFrame.getImageTrackingResults) {
+            return;
+        }
+        const imageTrackedResults = _xrFrame.getImageTrackingResults();
+        for (const result of imageTrackedResults) {
+            let changed = false;
+            const imageIndex = result.index;
+
+            const imageObject = this._trackedImages[imageIndex];
+            if (!imageObject) {
+                // something went wrong!
+                continue;
+            }
+
+            imageObject.xrTrackingResult = result;
+            if (imageObject.realWorldWidth !== result.measuredWidthInMeters) {
+                imageObject.realWorldWidth = result.measuredWidthInMeters;
+                changed = true;
+            }
+
+            // Get the pose of the image relative to a reference space.
+            const pose = _xrFrame.getPose(result.imageSpace, this._xrSessionManager.referenceSpace);
+
+            if (pose) {
+                const mat = imageObject.transformationMatrix;
+                Matrix.FromArrayToRef(pose.transform.matrix, 0, mat);
+                if (!this._xrSessionManager.scene.useRightHandedSystem) {
+                    mat.toggleModelMatrixHandInPlace();
+                }
+                changed = true;
+            }
+
+            const state = result.trackingState;
+            const emulated = state === "emulated";
+
+            if (imageObject.emulated !== emulated) {
+                imageObject.emulated = emulated;
+                changed = true;
+            }
+            if (changed) {
+                this.onTrackedImageUpdatedObservable.notifyObservers(imageObject);
+            }
+        }
+    }
+
+    private async _init() {
+        if (!this._xrSessionManager.session.getTrackedImageScores) {
+            return;
+        }
+        //
+        const imageScores = await this._xrSessionManager.session.getTrackedImageScores();
+        // check the scores for all
+        for (let idx = 0; idx < imageScores.length; ++idx) {
+            if (imageScores[idx] == "untrackable") {
+                this.onUntrackableImageFoundObservable.notifyObservers(idx);
+            } else {
+                const originalBitmap = this._originalTrackingRequest[idx].image;
+                const imageObject: IWebXRTrackedImage = {
+                    id: idx,
+                    originalBitmap,
+                    transformationMatrix: new Matrix(),
+                    ratio: originalBitmap.width / originalBitmap.height,
+                };
+                this._trackedImages[idx] = imageObject;
+                this.onTrackableImageFoundObservable.notifyObservers(imageObject);
+            }
+        }
+    }
+}
+
+//register the plugin
+WebXRFeaturesManager.AddWebXRFeature(
+    WebXRImageTracking.Name,
+    (xrSessionManager, options) => {
+        return () => new WebXRImageTracking(xrSessionManager, options);
+    },
+    WebXRImageTracking.Version,
+    false
+);

src/XR/features/index.ts

@@ -10,3 +10,4 @@ export * from "./WebXRHitTest";
 export * from "./WebXRFeaturePointSystem";
 export * from "./WebXRHandTracking";
 export * from "./WebXRMeshDetector";
+export * from "./WebXRImageTracking";

src/XR/webXRExperienceHelper.ts

@@ -97,7 +97,7 @@ export class WebXRExperienceHelper implements IDisposable {
      * @param sessionCreationOptions optional XRSessionInit object to init the session with
      * @returns promise that resolves after xr mode has entered
      */
-    public enterXRAsync(sessionMode: XRSessionMode, referenceSpaceType: XRReferenceSpaceType, renderTarget: WebXRRenderTarget = this.sessionManager.getWebXRRenderTarget(), sessionCreationOptions: XRSessionInit = {}): Promise<WebXRSessionManager> {
+    public async enterXRAsync(sessionMode: XRSessionMode, referenceSpaceType: XRReferenceSpaceType, renderTarget: WebXRRenderTarget = this.sessionManager.getWebXRRenderTarget(), sessionCreationOptions: XRSessionInit = {}): Promise<WebXRSessionManager> {
         if (!this._supported) {
             throw "WebXR not supported in this browser or environment";
         }
@@ -106,77 +106,68 @@ export class WebXRExperienceHelper implements IDisposable {
             sessionCreationOptions.optionalFeatures = sessionCreationOptions.optionalFeatures || [];
             sessionCreationOptions.optionalFeatures.push(referenceSpaceType);
         }
-        this.featuresManager.extendXRSessionInitObject(sessionCreationOptions);
+        sessionCreationOptions = await this.featuresManager._extendXRSessionInitObject(sessionCreationOptions);
         // we currently recommend "unbounded" space in AR (#7959)
         if (sessionMode === "immersive-ar" && referenceSpaceType !== "unbounded") {
             Logger.Warn("We recommend using 'unbounded' reference space type when using 'immersive-ar' session mode");
         }
         // make sure that the session mode is supported
-        return this.sessionManager
-            .initializeSessionAsync(sessionMode, sessionCreationOptions)
-            .then(() => {
-                return this.sessionManager.setReferenceSpaceTypeAsync(referenceSpaceType);
-            })
-            .then(() => {
-                return renderTarget.initializeXRLayerAsync(this.sessionManager.session);
-            })
-            .then(() => {
-                return this.sessionManager.updateRenderStateAsync({
-                    depthFar: this.camera.maxZ,
-                    depthNear: this.camera.minZ,
-                    baseLayer: renderTarget.xrLayer!,
+        try {
+            await this.sessionManager.initializeSessionAsync(sessionMode, sessionCreationOptions);
+            await this.sessionManager.setReferenceSpaceTypeAsync(referenceSpaceType);
+            await renderTarget.initializeXRLayerAsync(this.sessionManager.session);
+            await this.sessionManager.updateRenderStateAsync({
+                depthFar: this.camera.maxZ,
+                depthNear: this.camera.minZ,
+                baseLayer: renderTarget.xrLayer!,
+            });
+            // run the render loop
+            this.sessionManager.runXRRenderLoop();
+            // Cache pre xr scene settings
+            this._originalSceneAutoClear = this.scene.autoClear;
+            this._nonVRCamera = this.scene.activeCamera;
+
+            this.scene.activeCamera = this.camera;
+            // do not compensate when AR session is used
+            if (sessionMode !== "immersive-ar") {
+                this._nonXRToXRCamera();
+            } else {
+                // Kept here, TODO - check if needed
+                this.scene.autoClear = false;
+                this.camera.compensateOnFirstFrame = false;
+            }
+
+            this.sessionManager.onXRSessionEnded.addOnce(() => {
+                // Reset camera rigs output render target to ensure sessions render target is not drawn after it ends
+                this.camera.rigCameras.forEach((c) => {
+                    c.outputRenderTarget = null;
                 });
-            })
-            .then(() => {
-                // run the render loop
-                this.sessionManager.runXRRenderLoop();
-                // Cache pre xr scene settings
-                this._originalSceneAutoClear = this.scene.autoClear;
-                this._nonVRCamera = this.scene.activeCamera;
-
-                this.scene.activeCamera = this.camera;
-                // do not compensate when AR session is used
-                if (sessionMode !== "immersive-ar") {
-                    this._nonXRToXRCamera();
-                } else {
-                    // Kept here, TODO - check if needed
-                    this.scene.autoClear = false;
-                    this.camera.compensateOnFirstFrame = false;
-                }
-
-                this.sessionManager.onXRSessionEnded.addOnce(() => {
-                    // Reset camera rigs output render target to ensure sessions render target is not drawn after it ends
-                    this.camera.rigCameras.forEach((c) => {
-                        c.outputRenderTarget = null;
-                    });
 
-                    // Restore scene settings
-                    this.scene.autoClear = this._originalSceneAutoClear;
-                    this.scene.activeCamera = this._nonVRCamera;
-                    if (sessionMode !== "immersive-ar" && this.camera.compensateOnFirstFrame) {
-                        if ((<any>this._nonVRCamera).setPosition) {
-                            (<any>this._nonVRCamera).setPosition(this.camera.position);
-                        } else {
-                            this._nonVRCamera!.position.copyFrom(this.camera.position);
-                        }
+                // Restore scene settings
+                this.scene.autoClear = this._originalSceneAutoClear;
+                this.scene.activeCamera = this._nonVRCamera;
+                if (sessionMode !== "immersive-ar" && this.camera.compensateOnFirstFrame) {
+                    if ((<any>this._nonVRCamera).setPosition) {
+                        (<any>this._nonVRCamera).setPosition(this.camera.position);
+                    } else {
+                        this._nonVRCamera!.position.copyFrom(this.camera.position);
                     }
+                }
 
-                    this._setState(WebXRState.NOT_IN_XR);
-                });
-
-                // Wait until the first frame arrives before setting state to in xr
-                this.sessionManager.onXRFrameObservable.addOnce(() => {
-                    this._setState(WebXRState.IN_XR);
-                });
-
-                return this.sessionManager;
-            })
-            .catch((e: any) => {
-                console.log(e);
-                console.log(e.message);
                 this._setState(WebXRState.NOT_IN_XR);
-                throw e;
             });
+
+            // Wait until the first frame arrives before setting state to in xr
+            this.sessionManager.onXRFrameObservable.addOnce(() => {
+                this._setState(WebXRState.IN_XR);
+            });
+            return this.sessionManager;
+        } catch (e) {
+            console.log(e);
+            console.log(e.message);
+            this._setState(WebXRState.NOT_IN_XR);
+            throw e;
+        }
     }
 
     /**

src/XR/webXRFeaturesManager.ts

@@ -53,6 +53,11 @@ export interface IWebXRFeature extends IDisposable {
      * A list of (Babylon WebXR) features this feature depends on
      */
     dependsOn?: string[];
+
+    /**
+     * If this feature requires to extend the XRSessionInit object, this function will return the partial XR session init object
+     */
+    getXRSessionInitExtension?: () => Promise<Partial<XRSessionInit>>;
 }
 
 /**
@@ -99,6 +104,10 @@ export class WebXRFeatureName {
      * The name of the hand tracking feature.
      */
     public static readonly HAND_TRACKING = "xr-hand-tracking";
+    /**
+     * The name of the image tracking feature
+     */
+    public static readonly IMAGE_TRACKING = "xr-image-tracking";
 }
 
 /**
@@ -384,16 +393,16 @@ export class WebXRFeaturesManager implements IDisposable {
     }
 
     /**
-     * This function will exten the session creation configuration object with enabled features.
+     * This function will extend the session creation configuration object with enabled features.
      * If, for example, the anchors feature is enabled, it will be automatically added to the optional or required features list,
      * according to the defined "required" variable, provided during enableFeature call
      * @param xrSessionInit the xr Session init object to extend
      *
      * @returns an extended XRSessionInit object
      */
-    public extendXRSessionInitObject(xrSessionInit: XRSessionInit): XRSessionInit {
+    public async _extendXRSessionInitObject(xrSessionInit: XRSessionInit): Promise<XRSessionInit> {
         const enabledFeatures = this.getEnabledFeatures();
-        enabledFeatures.forEach((featureName) => {
+        for (const featureName of enabledFeatures) {
             const feature = this._features[featureName];
             const nativeName = feature.featureImplementation.xrNativeFeatureName;
             if (nativeName) {
@@ -409,7 +418,14 @@ export class WebXRFeaturesManager implements IDisposable {
                     }
                 }
             }
-        });
+            if (feature.featureImplementation.getXRSessionInitExtension) {
+                const extended = await feature.featureImplementation.getXRSessionInitExtension();
+                xrSessionInit = {
+                    ...xrSessionInit,
+                    ...extended,
+                };
+            }
+        }
         return xrSessionInit;
     }
 }