Merge pull request #9767 from RaananW/customHandMesh

[XR] Better support for custom meshes

dist/preview release/what's new.md

@@ -69,6 +69,7 @@
 - Added a way to extend the XRSessionInit Object from inside of a feature ([RaananW](https://github.com/RaananW))
 - Added image tracking feature ([RaananW](https://github.com/RaananW))
 - Pointer Events of WebXR controllers have pointerType `xr` ([RaananW](https://github.com/RaananW))
+- better support for custom hand meshes ([RaananW](https://github.com/RaananW))
 
 ### Viewer
 

src/XR/features/WebXRHandTracking.ts

@@ -22,6 +22,7 @@ import { Engine } from "../../Engines/engine";
 import { Tools } from "../../Misc/tools";
 import { Axis } from "../../Maths/math.axis";
 import { TransformNode } from "../../Meshes/transformNode";
+import { Bone } from "../../Bones/bone";
 
 declare const XRHand: XRHand;
 
@@ -83,6 +84,10 @@ export interface IWebXRHandTrackingOptions {
             left: AbstractMesh;
         };
         /**
+         * Are the meshes prepared for a left-handed system. Default hand meshes are right-handed.
+         */
+        leftHandedSystemMeshes?: boolean;
+        /**
          * If a hand mesh was provided, this array will define what axis will update which node. This will override the default hand mesh
          */
         rigMapping?: {
@@ -129,6 +134,8 @@ export class WebXRHand implements IDisposable {
     private _scene: Scene;
     private _defaultHandMesh: boolean = false;
     private _transformNodeMapping: TransformNode[] = [];
+    private _boneMapping: Bone[] = [];
+    private _useBones = false;
     /**
      * Hand-parts definition (key is HandPart)
      */
@@ -161,6 +168,7 @@ export class WebXRHand implements IDisposable {
      * @param _handMesh an optional hand mesh. if not provided, ours will be used
      * @param _rigMapping an optional rig mapping for the hand mesh. if not provided, ours will be used
      * @param disableDefaultHandMesh should the default mesh creation be disabled
+     * @param _leftHandedMeshes are the hand meshes left-handed-system meshes
      */
     constructor(
         /** the controller to which the hand correlates */
@@ -169,12 +177,28 @@ export class WebXRHand implements IDisposable {
         public readonly trackedMeshes: AbstractMesh[],
         private _handMesh?: AbstractMesh,
         private _rigMapping?: string[],
-        disableDefaultHandMesh?: boolean
+        disableDefaultHandMesh?: boolean,
+        private _leftHandedMeshes?: boolean
     ) {
         this.handPartsDefinition = this.generateHandPartsDefinition(xrController.inputSource.hand!);
         this._scene = trackedMeshes[0].getScene();
+        this.onHandMeshReadyObservable.add(() => {
+            // check if we should use bones or transform nodes
+            if (!this._rigMapping) {
+                return;
+            }
+            const transformNode = this._scene.getTransformNodeByName(this._rigMapping[0]);
+            if (!transformNode) {
+                const bone = this._scene.getBoneByName(this._rigMapping[0]);
+                if (bone) {
+                    this._useBones = true;
+                }
+            }
+        });
         if (this._handMesh && this._rigMapping) {
             this._defaultHandMesh = false;
+            this._handMesh.alwaysSelectAsActiveMesh = true;
+            this._handMesh.getChildMeshes().forEach((m) => (m.alwaysSelectAsActiveMesh = true));
             this.onHandMeshReadyObservable.notifyObservers(this);
         } else {
             if (!disableDefaultHandMesh) {
@@ -242,20 +266,36 @@ export class WebXRHand implements IDisposable {
                 const radius = (pose.radius || 0.008) * scaleFactor;
                 mesh.scaling.set(radius, radius, radius);
 
+                // if we are using left-handed meshes, transform before applying to the hand mesh
+                if (this._leftHandedMeshes && !mesh.getScene().useRightHandedSystem) {
+                    mesh.position.z *= -1;
+                    mesh.rotationQuaternion!.z *= -1;
+                    mesh.rotationQuaternion!.w *= -1;
+                }
+
                 // now check for the hand mesh
                 if (this._handMesh && this._rigMapping) {
                     if (this._rigMapping[idx]) {
-                        this._transformNodeMapping[idx] = this._transformNodeMapping[idx] || this._scene.getTransformNodeByName(this._rigMapping[idx]);
-                        if (this._transformNodeMapping[idx]) {
-                            this._transformNodeMapping[idx].position.copyFrom(mesh.position);
-                            this._transformNodeMapping[idx].rotationQuaternion!.copyFrom(mesh.rotationQuaternion!);
-                            // no scaling at the moment
-                            // this._transformNodeMapping[idx].scaling.copyFrom(mesh.scaling).scaleInPlace(20);
-                            mesh.isVisible = false;
+                        if (this._useBones) {
+                            this._boneMapping[idx] = this._boneMapping[idx] || this._scene.getBoneByName(this._rigMapping[idx]);
+                            if (this._boneMapping[idx]) {
+                                this._boneMapping[idx].setPosition(mesh.position);
+                                this._boneMapping[idx].setRotationQuaternion(mesh.rotationQuaternion!);
+                                mesh.isVisible = false;
+                            }
+                        } else {
+                            this._transformNodeMapping[idx] = this._transformNodeMapping[idx] || this._scene.getTransformNodeByName(this._rigMapping[idx]);
+                            if (this._transformNodeMapping[idx]) {
+                                this._transformNodeMapping[idx].position.copyFrom(mesh.position);
+                                this._transformNodeMapping[idx].rotationQuaternion!.copyFrom(mesh.rotationQuaternion!);
+                                // no scaling at the moment
+                                // this._transformNodeMapping[idx].scaling.copyFrom(mesh.scaling).scaleInPlace(20);
+                                mesh.isVisible = false;
+                            }
                         }
                     }
                 }
-                if (!mesh.getScene().useRightHandedSystem) {
+                if (!this._leftHandedMeshes && !mesh.getScene().useRightHandedSystem) {
                     mesh.position.z *= -1;
                     mesh.rotationQuaternion!.z *= -1;
                     mesh.rotationQuaternion!.w *= -1;
@@ -537,7 +577,7 @@ export class WebXRHandTracking extends WebXRAbstractFeature {
         const handedness = xrController.inputSource.handedness === "right" ? "right" : "left";
         const handMesh = this.options.jointMeshes?.handMeshes && this.options.jointMeshes?.handMeshes[handedness];
         const rigMapping = this.options.jointMeshes?.rigMapping && this.options.jointMeshes?.rigMapping[handedness];
-        const webxrHand = new WebXRHand(xrController, trackedMeshes, handMesh, rigMapping, this.options.jointMeshes?.disableDefaultHandMesh);
+        const webxrHand = new WebXRHand(xrController, trackedMeshes, handMesh, rigMapping, this.options.jointMeshes?.disableDefaultHandMesh, this.options.jointMeshes?.leftHandedSystemMeshes);
 
         // get two new meshes
         this._hands[xrController.uniqueId] = {