Merge branch 'master' of https://github.com/BabylonJS/Babylon.js into WebGPU

Playground/src/tools/monacoManager.ts

@@ -234,6 +234,11 @@ class Playground {
             "https://preview.babylonjs.com/inspector/babylon.inspector.d.ts",
         ];
 
+        // Check for Unity Toolkit
+        if (location.href.indexOf("UnityToolkit") !== -1 || Utilities.ReadBoolFromStore("unity-toolkit", false)) {
+            declarations.push("https://playground.babylonjs.com/libs/babylon.manager.d.ts");
+        }
+
         let libContent = "";
         const responses = await Promise.all(declarations.map((declaration) => fetch(declaration)));
         for (const response of responses) {

Tools/Gulp/gulpfile.js

@@ -78,7 +78,7 @@ gulp.task("run", gulp.series("generate-symlinks", "cleanup", "watchCore", "watch
 /**
  * Do it all (Build).
  */
-gulp.task("typescript-all", gulp.series("generate-symlinks", "typescript-libraries", "typescript-es6", "typescript-apps", "netlify-cleanup"));
+gulp.task("typescript-all", gulp.series("typescript-libraries", "typescript-es6", "typescript-apps", "netlify-cleanup"));
 
 /**
  * Do it all (tests).
@@ -88,9 +88,9 @@ gulp.task("tests-all", gulp.series("generate-symlinks", "tests-unit", "tests-mod
 /**
  * Get Ready to test Npm Packages.
  */
-gulp.task("npmPackages", gulp.series("generate-symlinks", "npmPackages-all"));
+gulp.task("npmPackages", gulp.series("npmPackages-all"));
 
 /**
  * The default task, concat and min the main BJS files.
  */
-gulp.task("default", gulp.series("generate-symlinks", "cleanup", "tsLint", "importLint", "circularDependencies", "typescript-all", "documentation", "typedoc-all", "tests-all"));
+gulp.task("default", gulp.series("cleanup", "tsLint", "importLint", "circularDependencies", "typescript-all", "documentation", "typedoc-all", "tests-all"));

Tools/Gulp/tasks/gulpTasks-libraries.js

@@ -6,6 +6,7 @@ var cp = require('child_process');
 var path = require("path");
 var concat = require('gulp-concat');
 var minimist = require("minimist");
+var symlinkDir = require('symlink-dir');
 
 // Gulp Helpers
 var uncommentShaders = require('../helpers/gulp-removeShaderComments');
@@ -164,12 +165,30 @@ var processDTSFiles = function(libraries, settings, cb) {
 }
 
 /**
+ * Generate our required symlinked for the shared components.
+ */
+var generateSharedUiComponents = function(settings, done) {
+    if (!settings.build.sharedUiComponents) {
+        done();
+        return;
+    }
+
+    var sharedUiComponents = config.computed.sharedUiComponentsSrcPath;
+    var umdSharedUiComponents = path.resolve(settings.computed.mainDirectory, settings.build.sharedUiComponents);
+
+    symlinkDir(sharedUiComponents, umdSharedUiComponents).then(() => {
+        done();
+    });
+};
+
+/**
  * Dynamic module creation In Serie for WebPack leaks.
  */
 function buildExternalLibraries(settings, fast) {
     // Creates the required tasks.
     var tasks = [];
 
+    var sharedUiComponents = function(cb) { return generateSharedUiComponents(settings, cb); };
     var cleanup = function() { return cleanShaders(settings); };
     var shaders = function() { return buildShaders(settings); };
     var buildMin = function() { return buildExternalLibrariesMultiEntry(settings.libraries, settings, true) };
@@ -183,9 +202,9 @@ function buildExternalLibraries(settings, fast) {
     }
 
     if (fast) {
-        tasks.push(buildMax);
+        tasks.push(sharedUiComponents, buildMax);
     } else {
-        tasks.push(cleanup, shaders, buildMin, buildMax, buildAMDDTS, processDTS, ...appendLoseDTS);
+        tasks.push(sharedUiComponents, cleanup, shaders, buildMin, buildMax, buildAMDDTS, processDTS, ...appendLoseDTS);
     }
 
     return gulp.series.apply(this, tasks);

src/LibDeclarations/webxr.d.ts

@@ -124,6 +124,10 @@ interface XRPose {
     readonly emulatedPosition: boolean;
 }
 
+interface XRWorldInformation {
+    detectedPlanes?: XRPlaneSet;
+}
+
 interface XRFrame {
     readonly session: XRSession;
     getPose(space: XRSpace, baseSpace: XRSpace): XRPose | undefined;
@@ -135,10 +139,8 @@ interface XRFrame {
     // Anchors
     trackedAnchors?: XRAnchorSet;
     createAnchor?(pose: XRRigidTransform, space: XRSpace): Promise<XRAnchor>;
-    // Planes
-    worldInformation?: {
-        detectedPlanes?: XRPlaneSet;
-    };
+    // World geometries
+    worldInformation?: XRWorldInformation;
     // Hand tracking
     getJointPose?(joint: XRJointSpace, baseSpace: XRSpace): XRJointPose;
 }

src/LibDeclarations/webxr.nativeextensions.d.ts

@@ -1,11 +1,75 @@
-// This file contains native only extensions for WebXR  These APIs are not supported in the browser yet.
+// This file contains native only extensions for WebXR. These APIs are not supported in the browser yet.
 // They are intended for use with either Babylon Native https://github.com/BabylonJS/BabylonNative or
 // Babylon React Native: https://github.com/BabylonJS/BabylonReactNative
 
+type XRSceneObjectType = "unknown" | "background" | "wall" | "floor" | "ceiling" | "platform";
+
+interface XRSceneObject {
+    type: XRSceneObjectType;
+}
+
+interface XRFieldOfView {
+    angleLeft: number;
+    angleRight: number;
+    angleUp: number;
+    angleDown: number;
+}
+
+interface XRFrustum {
+    position: DOMPointReadOnly;
+    orientation: DOMPointReadOnly;
+    fieldOfView: XRFieldOfView;
+    farDistance: number;
+}
+
+interface XRPlane {
+    parentSceneObject?: XRSceneObject;
+}
+
+interface XRMesh {
+    meshSpace: XRSpace;
+    positions: Float32Array;
+    indices: Uint32Array;
+    normals?: Float32Array;
+    lastChangedTime: number;
+    parentSceneObject?: XRSceneObject;
+}
+
+interface XRFrustumDetectionBoundary {
+    type: "frustum";
+    frustum: XRFrustum;
+}
+
+interface XRSphereDetectionBoundary {
+    type: "sphere";
+    radius: number;
+}
+
+interface XRBoxDetectionBoundary {
+    type: "box";
+    extent: DOMPointReadOnly;
+}
+
+type XRDetectionBoundary = XRFrustumDetectionBoundary | XRSphereDetectionBoundary | XRBoxDetectionBoundary;
+
+interface XRGeometryDetectorOptions {
+    detectionBoundary?: XRDetectionBoundary;
+    updateInterval?: number;
+}
+
 interface XRSession {
     trySetFeaturePointCloudEnabled(enabled: boolean): boolean;
+    trySetPreferredPlaneDetectorOptions(preferredOptions: XRGeometryDetectorOptions): boolean;
+    trySetMeshDetectorEnabled(enabled: boolean): boolean;
+    trySetPreferredMeshDetectorOptions(preferredOptions: XRGeometryDetectorOptions): boolean;
 }
 
 interface XRFrame {
-    featurePointCloud? : Array<number>;
+    featurePointCloud?: Array<number>;
+}
+
+type XRMeshSet = Set<XRMesh>;
+
+interface XRWorldInformation {
+    detectedMeshes?: XRMeshSet;
 }

src/XR/features/WebXRMeshDetector.ts

@@ -0,0 +1,274 @@
+import { WebXRFeaturesManager, WebXRFeatureName } from '../webXRFeaturesManager';
+import { WebXRAbstractFeature } from './WebXRAbstractFeature';
+import { WebXRSessionManager } from '../webXRSessionManager';
+import { TransformNode } from '../../Meshes/transformNode';
+import { Matrix } from '../../Maths/math';
+import { Observable } from '../../Misc/observable';
+
+/**
+ * Options used in the mesh detector module
+ */
+export interface IWebXRMeshDetectorOptions {
+    /**
+     * The node to use to transform the local results to world coordinates
+     */
+    worldParentNode?: TransformNode;
+    /**
+     * If set to true a reference of the created meshes will be kept until the next session starts
+     * If not defined, meshes will be removed from the array when the feature is detached or the session ended.
+     */
+    doNotRemoveMeshesOnSessionEnded?: boolean;
+    /**
+     * Preferred detector configuration, not all preferred options will be supported by all platforms.
+     */
+    preferredDetectorOptions?: XRGeometryDetectorOptions;
+    /**
+     * If set to true, WebXRMeshDetector will convert coordinate systems for meshes.
+     * If not defined, mesh conversions from right handed to left handed coordinate systems won't be conducted.
+     * Right handed mesh data will be available through IWebXRVertexData.xrMesh.
+     */
+    convertCoordinateSystems?: boolean;
+}
+
+/**
+ * A babylon interface for a XR mesh's vertex data.
+ *
+ * Currently not supported by WebXR, available only with BabylonNative
+ */
+export interface IWebXRVertexData {
+    /**
+     * A babylon-assigned ID for this mesh
+     */
+    id: number;
+    /**
+     * Data required for constructing a mesh in Babylon.js.
+     */
+    xrMesh: XRMesh;
+    /**
+     * The node to use to transform the local results to world coordinates.
+     * WorldParentNode will only exist if it was declared in the IWebXRMeshDetectorOptions.
+     */
+    worldParentNode?: TransformNode;
+    /**
+     * An array of vertex positions in babylon space. right/left hand system is taken into account.
+     * Positions will only be calculated if convertCoordinateSystems is set to true in the IWebXRMeshDetectorOptions.
+     */
+    positions?: Float32Array;
+    /**
+     * An array of indices in babylon space. right/left hand system is taken into account.
+     * Indices will only be calculated if convertCoordinateSystems is set to true in the IWebXRMeshDetectorOptions.
+     */
+    indices?: Uint32Array;
+    /**
+     * An array of vertex normals in babylon space. right/left hand system is taken into account.
+     * Normals will not be calculated if convertCoordinateSystems is undefined in the IWebXRMeshDetectorOptions.
+     * Different platforms may or may not support mesh normals when convertCoordinateSystems is set to true.
+     */
+    normals?: Float32Array;
+    /**
+     * A transformation matrix to apply on the mesh that will be built using the meshDefinition.
+     * Local vs. World are decided if worldParentNode was provided or not in the options when constructing the module.
+     * TransformationMatrix will only be calculated if convertCoordinateSystems is set to true in the IWebXRMeshDetectorOptions.
+     */
+    transformationMatrix?: Matrix;
+}
+
+let meshIdProvider = 0;
+
+/**
+ * The mesh detector is used to detect meshes in the real world when in AR
+ */
+export class WebXRMeshDetector extends WebXRAbstractFeature {
+    private _detectedMeshes: Map<XRMesh, IWebXRVertexData> = new Map<XRMesh, IWebXRVertexData>();
+
+    /**
+     * The module's name
+     */
+    public static readonly Name = WebXRFeatureName.MESH_DETECTION;
+    /**
+     * The (Babylon) version of this module.
+     * This is an integer representing the implementation version.
+     * This number does not correspond to the WebXR specs version
+     */
+    public static readonly Version = 1;
+
+    /**
+     * Observers registered here will be executed when a new mesh was added to the session
+     */
+    public onMeshAddedObservable: Observable<IWebXRVertexData> = new Observable();
+    /**
+     * Observers registered here will be executed when a mesh is no longer detected in the session
+     */
+    public onMeshRemovedObservable: Observable<IWebXRVertexData> = new Observable();
+    /**
+     * Observers registered here will be executed when an existing mesh updates
+     */
+    public onMeshUpdatedObservable: Observable<IWebXRVertexData> = new Observable();
+
+    constructor(_xrSessionManager: WebXRSessionManager, private _options: IWebXRMeshDetectorOptions = {}) {
+        super(_xrSessionManager);
+        this.xrNativeFeatureName = "mesh-detection";
+        if (this._xrSessionManager.session) {
+            this._init();
+        } else {
+            this._xrSessionManager.onXRSessionInit.addOnce(() => {
+                this._init();
+            });
+        }
+    }
+
+    public detach(): boolean {
+        if (!super.detach()) {
+            return false;
+        }
+
+        // Only supported by BabylonNative
+        if (!!this._xrSessionManager.isNative &&
+            !!this._xrSessionManager.session.trySetMeshDetectorEnabled) {
+            this._xrSessionManager.session.trySetMeshDetectorEnabled(false);
+        }
+
+        if (!this._options.doNotRemoveMeshesOnSessionEnded) {
+            this._detectedMeshes.forEach((mesh) => {
+                this.onMeshRemovedObservable.notifyObservers(mesh);
+            });
+
+            this._detectedMeshes.clear();
+        }
+
+        return true;
+    }
+
+    public dispose(): void {
+        super.dispose();
+        this.onMeshAddedObservable.clear();
+        this.onMeshRemovedObservable.clear();
+        this.onMeshUpdatedObservable.clear();
+    }
+
+    protected _onXRFrame(frame: XRFrame) {
+        // TODO remove try catch
+        try {
+            if (!this.attached || !frame) {
+                return;
+            }
+
+            const detectedMeshes = frame.worldInformation?.detectedMeshes;
+            if (!!detectedMeshes) {
+                let toRemove = new Set<XRMesh>();
+                this._detectedMeshes.forEach((vertexData, xrMesh) => {
+                    if (!detectedMeshes.has(xrMesh)) {
+                        toRemove.add(xrMesh);
+                    }
+                });
+                toRemove.forEach((xrMesh) => {
+                    const vertexData = this._detectedMeshes.get(xrMesh);
+                    if (!!vertexData) {
+                        this.onMeshRemovedObservable.notifyObservers(vertexData);
+                        this._detectedMeshes.delete(xrMesh);
+                    }
+                });
+
+                // now check for new ones
+                detectedMeshes.forEach((xrMesh) => {
+                    if (!this._detectedMeshes.has(xrMesh)) {
+                        const partialVertexData: Partial<IWebXRVertexData> = {
+                            id: meshIdProvider++,
+                            xrMesh: xrMesh,
+                        };
+                        const vertexData = this._updateVertexDataWithXRMesh(xrMesh, partialVertexData, frame);
+                        this._detectedMeshes.set(xrMesh, vertexData);
+                        this.onMeshAddedObservable.notifyObservers(vertexData);
+                    } else {
+                        // updated?
+                        if (xrMesh.lastChangedTime === this._xrSessionManager.currentTimestamp) {
+                            const vertexData = this._detectedMeshes.get(xrMesh);
+                            if (!!vertexData) {
+                                this._updateVertexDataWithXRMesh(xrMesh, vertexData, frame);
+                                this.onMeshUpdatedObservable.notifyObservers(vertexData);
+                            }
+                        }
+                    }
+                });
+            }
+        } catch (error) {
+            console.log(error.stack);
+        }
+    }
+
+    private _init() {
+        // Only supported by BabylonNative
+        if (!!this._xrSessionManager.isNative) {
+            if (!!this._xrSessionManager.session.trySetMeshDetectorEnabled) {
+                this._xrSessionManager.session.trySetMeshDetectorEnabled(true);
+            }
+
+            if (!!this._options.preferredDetectorOptions &&
+                !!this._xrSessionManager.session.trySetPreferredMeshDetectorOptions) {
+                this._xrSessionManager.session.trySetPreferredMeshDetectorOptions(this._options.preferredDetectorOptions);
+            }
+        }
+    }
+
+    private _updateVertexDataWithXRMesh(xrMesh: XRMesh, mesh: Partial<IWebXRVertexData>, xrFrame: XRFrame): IWebXRVertexData {
+        mesh.xrMesh = xrMesh;
+        mesh.worldParentNode = this._options.worldParentNode;
+
+        if (!!this._options.convertCoordinateSystems) {
+            if (!this._xrSessionManager.scene.useRightHandedSystem) {
+                mesh.positions = new Float32Array(xrMesh.positions.length);
+                for (let i = 0; i < xrMesh.positions.length; i += 3) {
+                    mesh.positions[i] = xrMesh.positions[i];
+                    mesh.positions[i + 1] = xrMesh.positions[i + 1];
+                    mesh.positions[i + 2] = -1 * xrMesh.positions[i + 2];
+                }
+
+                mesh.indices = new Uint32Array(xrMesh.indices.length);
+                for (let i = 0; i < xrMesh.indices.length; i += 3) {
+                    mesh.indices[i] = xrMesh.indices[i];
+                    mesh.indices[i + 1] = xrMesh.indices[i + 2];
+                    mesh.indices[i + 2] = xrMesh.indices[i + 1];
+                }
+
+                if (!!xrMesh.normals) {
+                    mesh.normals = new Float32Array(xrMesh.normals.length);
+                    for (let i = 0; i < xrMesh.normals.length; i += 3) {
+                        mesh.normals[i] = xrMesh.normals[i];
+                        mesh.normals[i + 1] = xrMesh.normals[i + 1];
+                        mesh.normals[i + 2] = -1 * xrMesh.normals[i + 2];
+                    }
+                }
+            }
+            else {
+                mesh.positions = xrMesh.positions;
+                mesh.indices = xrMesh.indices;
+                mesh.normals = xrMesh.normals;
+            }
+
+            // matrix
+            const pose = xrFrame.getPose(xrMesh.meshSpace, this._xrSessionManager.referenceSpace);
+            if (pose) {
+                const mat = mesh.transformationMatrix || new Matrix();
+                Matrix.FromArrayToRef(pose.transform.matrix, 0, mat);
+                if (!this._xrSessionManager.scene.useRightHandedSystem) {
+                    mat.toggleModelMatrixHandInPlace();
+                }
+                mesh.transformationMatrix = mat;
+                if (this._options.worldParentNode) {
+                    mat.multiplyToRef(this._options.worldParentNode.getWorldMatrix(), mat);
+                }
+            }
+        }
+
+        return <IWebXRVertexData>mesh;
+    }
+}
+
+WebXRFeaturesManager.AddWebXRFeature(
+    WebXRMeshDetector.Name,
+    (xrSessionManager, options) => {
+        return () => new WebXRMeshDetector(xrSessionManager, options);
+    },
+    WebXRMeshDetector.Version,
+    false
+);

src/XR/features/WebXRPlaneDetector.ts

@@ -15,12 +15,15 @@ export interface IWebXRPlaneDetectorOptions {
      * The node to use to transform the local results to world coordinates
      */
     worldParentNode?: TransformNode;
-
     /**
      * If set to true a reference of the created planes will be kept until the next session starts
      * If not defined, planes will be removed from the array when the feature is detached or the session ended.
      */
     doNotRemovePlanesOnSessionEnded?: boolean;
+    /**
+     * Preferred detector configuration, not all preferred options will be supported by all platforms.
+     */
+    preferredDetectorOptions?: XRGeometryDetectorOptions;
 }
 
 /**
@@ -176,7 +179,7 @@ export class WebXRPlaneDetector extends WebXRAbstractFeature {
                 } else {
                     // updated?
                     if (xrPlane.lastChangedTime === this._xrSessionManager.currentTimestamp) {
-                        let index = this.findIndexInPlaneArray(xrPlane);
+                        let index = this._findIndexInPlaneArray(xrPlane);
                         const plane = this._detectedPlanes[index];
                         this._updatePlaneWithXRPlane(xrPlane, plane, frame);
                         this.onPlaneUpdatedObservable.notifyObservers(plane);
@@ -194,6 +197,14 @@ export class WebXRPlaneDetector extends WebXRAbstractFeature {
                 this._detectedPlanes.length = 0;
             }
         };
+
+        // Only supported by BabylonNative
+        if (!!this._xrSessionManager.isNative &&
+            !!this._options.preferredDetectorOptions &&
+            !!this._xrSessionManager.session.trySetPreferredPlaneDetectorOptions) {
+            this._xrSessionManager.session.trySetPreferredPlaneDetectorOptions(this._options.preferredDetectorOptions);
+        }
+
         if (!this._xrSessionManager.session.updateWorldTrackingState) {
             // check if this was enabled by a flag
             const alreadyEnabled = (this._xrSessionManager.session as any).worldTrackingState?.planeDetectionState?.enabled;
@@ -225,6 +236,7 @@ export class WebXRPlaneDetector extends WebXRAbstractFeature {
                 mat.multiplyToRef(this._options.worldParentNode.getWorldMatrix(), mat);
             }
         }
+
         return <IWebXRPlane>plane;
     }
 
@@ -232,7 +244,7 @@ export class WebXRPlaneDetector extends WebXRAbstractFeature {
      * avoiding using Array.find for global support.
      * @param xrPlane the plane to find in the array
      */
-    private findIndexInPlaneArray(xrPlane: XRPlane) {
+    private _findIndexInPlaneArray(xrPlane: XRPlane) {
         for (let i = 0; i < this._detectedPlanes.length; ++i) {
             if (this._detectedPlanes[i].xrPlane === xrPlane) {
                 return i;

src/XR/features/index.ts

@@ -9,3 +9,4 @@ export * from "./WebXRControllerPhysics";
 export * from "./WebXRHitTest";
 export * from "./WebXRFeaturePointSystem";
 export * from "./WebXRHandTracking";
+export * from "./WebXRMeshDetector";

src/XR/webXRFeaturesManager.ts

@@ -72,6 +72,10 @@ export class WebXRFeatureName {
      */
     public static readonly HIT_TEST = "xr-hit-test";
     /**
+     * The name of the mesh detection feature
+     */
+    public static readonly MESH_DETECTION = "xr-mesh-detection";
+    /**
      * physics impostors for xr controllers feature
      */
     public static readonly PHYSICS_CONTROLLERS = "xr-physics-controller";

src/XR/webXRSessionManager.ts

@@ -332,6 +332,13 @@ export class WebXRSessionManager implements IDisposable {
         }
     }
 
+    /**
+     * Returns true if Babylon.js is using the BabylonNative backend, otherwise false
+     */
+    public get isNative(): boolean {
+        return this._xrNavigator.xr.native ?? false;
+    }
+
     private _createRenderTargetTexture(width: number, height: number, framebuffer: Nullable<WebGLFramebuffer> = null) {
         // Create internal texture
         var internalTexture = new InternalTexture(this.scene.getEngine(), InternalTextureSource.Unknown, true);

src/XR/webXRTypes.ts

@@ -61,4 +61,4 @@ export interface WebXRRenderTarget extends IDisposable {
      * @returns a promise that will resolve once the XR Layer has been created
      */
     initializeXRLayerAsync(xrSession: XRSession): Promise<XRWebGLLayer>;
-}
+}