Merge pull request #9393 from RaananW/handMeshes

default mesh for hand tracking

dist/preview release/what's new.md

@@ -244,6 +244,7 @@
 - Updated Plane Detection API ([RaananW](https://github.com/RaananW))
 - Updated anchor system's promise resolution and API ([#9258](https://github.com/BabylonJS/Babylon.js/issues/9258)) ([RaananW](https://github.com/RaananW))
 - Fixed an issue with teleportation re-attachment ([#9273](https://github.com/BabylonJS/Babylon.js/issues/9273)) ([RaananW](https://github.com/RaananW))
+- Added support and assets for hand meshes on right handed and left handed systems ([RaananW](https://github.com/RaananW))
 
 ### Collisions
 

src/XR/features/WebXRHandTracking.ts

@@ -10,9 +10,18 @@ import { Quaternion } from "../../Maths/math.vector";
 import { Nullable } from "../../types";
 import { PhysicsImpostor } from "../../Physics/physicsImpostor";
 import { WebXRFeaturesManager } from "../webXRFeaturesManager";
-import { IDisposable } from "../../scene";
+import { IDisposable, Scene } from "../../scene";
 import { Observable } from "../../Misc/observable";
 import { InstancedMesh } from "../../Meshes/instancedMesh";
+import { SceneLoader } from "../../Loading/sceneLoader";
+import { Color3 } from "../../Maths/math.color";
+import { NodeMaterial } from "../../Materials/Node/nodeMaterial";
+import { InputBlock } from "../../Materials/Node/Blocks/Input/inputBlock";
+import { Material } from "../../Materials/material";
+import { Engine } from "../../Engines/engine";
+import { Tools } from "../../Misc/tools";
+import { TransformNode } from "../../Meshes";
+import { Axis } from "../../Maths/math.axis";
 
 declare const XRHand: XRHand;
 
@@ -63,9 +72,23 @@ export interface IWebXRHandTrackingOptions {
          */
         physicsProps?: { friction?: number; restitution?: number; impostorType?: number };
         /**
-         * For future use - a single hand-mesh that will be updated according to the XRHand data provided
+         * Should the default hand mesh be disabled. In this case, the spheres will be visible.
          */
-        handMesh?: AbstractMesh;
+        disableDefaultHandMesh?: boolean;
+        /**
+         * a rigged hand-mesh that will be updated according to the XRHand data provided. This will override the default hand mesh
+         */
+        handMeshes?: {
+            right: AbstractMesh;
+            left: AbstractMesh;
+        };
+        /**
+         * If a hand mesh was provided, this array will define what axis will update which node. This will override the default hand mesh
+         */
+        rigMapping?: {
+            right: string[];
+            left: string[];
+        };
     };
 }
 
@@ -103,6 +126,9 @@ export const enum HandPart {
  * Representing a single hand (with its corresponding native XRHand object)
  */
 export class WebXRHand implements IDisposable {
+    private _scene: Scene;
+    private _defaultHandMesh: boolean = false;
+    private _transformNodeMapping: TransformNode[] = [];
     /**
      * Hand-parts definition (key is HandPart)
      */
@@ -127,14 +153,48 @@ export class WebXRHand implements IDisposable {
      * Construct a new hand object
      * @param xrController the controller to which the hand correlates
      * @param trackedMeshes the meshes to be used to track the hand joints
+     * @param handMesh an optional hand mesh. if not provided, ours will be used
+     * @param _rigMapping an optional rig mapping for the hand mesh. if not provided, ours will be used
      */
     constructor(
         /** the controller to which the hand correlates */
         public readonly xrController: WebXRInputSource,
         /** the meshes to be used to track the hand joints */
-        public readonly trackedMeshes: AbstractMesh[]
+        public readonly trackedMeshes: AbstractMesh[],
+        private _handMesh?: AbstractMesh,
+        private _rigMapping?: string[]
     ) {
         this.handPartsDefinition = this.generateHandPartsDefinition(xrController.inputSource.hand!);
+        this._scene = trackedMeshes[0].getScene();
+        if (this._handMesh && this._rigMapping) {
+            this._defaultHandMesh = false;
+        } else {
+            this._generateDefaultHandMesh();
+        }
+
+        // hide the motion controller, if available/loaded
+        if (this.xrController.motionController) {
+            if (this.xrController.motionController.rootMesh) {
+                this.xrController.motionController.rootMesh.setEnabled(false);
+            } else {
+                this.xrController.motionController.onModelLoadedObservable.add((controller) => {
+                    if (controller.rootMesh) {
+                        controller.rootMesh.setEnabled(false);
+                    }
+                });
+            }
+        }
+
+        this.xrController.onMotionControllerInitObservable.add((motionController) => {
+            motionController.onModelLoadedObservable.add((controller) => {
+                if (controller.rootMesh) {
+                    controller.rootMesh.setEnabled(false);
+                }
+            });
+            if (motionController.rootMesh) {
+                motionController.rootMesh.setEnabled(false);
+            }
+        });
     }
 
     /**
@@ -161,14 +221,28 @@ export class WebXRHand implements IDisposable {
                 mesh.position.set(pos.x, pos.y, pos.z);
                 mesh.rotationQuaternion!.set(orientation.x, orientation.y, orientation.z, orientation.w);
                 // left handed system conversion
+                // get the radius of the joint. In general it is static, but just in case it does change we update it on each frame.
+                const radius = (pose.radius || 0.008) * scaleFactor;
+                mesh.scaling.set(radius, radius, radius);
+
+                // now check for the hand mesh
+                if (this._handMesh && this._rigMapping) {
+                    if (this._rigMapping[idx]) {
+                        this._transformNodeMapping[idx] = this._transformNodeMapping[idx] || this._scene.getTransformNodeByName(this._rigMapping[idx]);
+                        if (this._transformNodeMapping[idx]) {
+                            this._transformNodeMapping[idx].position.copyFrom(mesh.position);
+                            this._transformNodeMapping[idx].rotationQuaternion!.copyFrom(mesh.rotationQuaternion!);
+                            // no scaling at the moment
+                            // this._transformNodeMapping[idx].scaling.copyFrom(mesh.scaling).scaleInPlace(20);
+                            mesh.isVisible = false;
+                        }
+                    }
+                }
                 if (!mesh.getScene().useRightHandedSystem) {
                     mesh.position.z *= -1;
                     mesh.rotationQuaternion!.z *= -1;
                     mesh.rotationQuaternion!.w *= -1;
                 }
-                // get the radius of the joint. In general it is static, but just in case it does change we update it on each frame.
-                const radius = (pose.radius || 0.008) * scaleFactor;
-                mesh.scaling.set(radius, radius, radius);
             }
         });
     }
@@ -187,6 +261,89 @@ export class WebXRHand implements IDisposable {
      */
     public dispose() {
         this.trackedMeshes.forEach((mesh) => mesh.dispose());
+        // dispose the hand mesh, if it is the default one
+        if (this._defaultHandMesh && this._handMesh) {
+            this._handMesh.dispose();
+        }
+    }
+
+    private async _generateDefaultHandMesh() {
+        try {
+            const handedness = this.xrController.inputSource.handedness === "right" ? "right" : "left";
+            const filename = `${handedness === "right" ? "r" : "l"}_hand_${this._scene.useRightHandedSystem ? "r" : "l"}hs.glb`;
+            const loaded = await SceneLoader.ImportMeshAsync("", "https://assets.babylonjs.com/meshes/HandMeshes/", filename, this._scene);
+            // shader
+            const handColors = {
+                base: Color3.FromInts(116, 63, 203),
+                fresnel: Color3.FromInts(149, 102, 229),
+                fingerColor: Color3.FromInts(177, 130, 255),
+                tipFresnel: Color3.FromInts(220, 200, 255),
+            };
+
+            const handShader = new NodeMaterial("leftHandShader", this._scene, { emitComments: false });
+            await handShader.loadAsync("https://patrickryanms.github.io/BabylonJStextures/Demos/xrHandMesh/handsShader.json");
+            // build node materials
+            handShader.build(false);
+
+            // depth prepass and alpha mode
+            handShader.needDepthPrePass = true;
+            handShader.transparencyMode = Material.MATERIAL_ALPHABLEND;
+            handShader.alphaMode = Engine.ALPHA_COMBINE;
+
+            const handNodes = {
+                base: handShader.getBlockByName("baseColor") as InputBlock,
+                fresnel: handShader.getBlockByName("fresnelColor") as InputBlock,
+                fingerColor: handShader.getBlockByName("fingerColor") as InputBlock,
+                tipFresnel: handShader.getBlockByName("tipFresnelColor") as InputBlock,
+            };
+
+            handNodes.base.value = handColors.base;
+            handNodes.fresnel.value = handColors.fresnel;
+            handNodes.fingerColor.value = handColors.fingerColor;
+            handNodes.tipFresnel.value = handColors.tipFresnel;
+
+            loaded.meshes[1].material = handShader;
+
+            this._defaultHandMesh = true;
+            this._handMesh = loaded.meshes[0];
+            this._rigMapping = [
+                "wrist_",
+                "thumb_metacarpal_",
+                "thumb_proxPhalanx_",
+                "thumb_distPhalanx_",
+                "thumb_tip_",
+                "index_metacarpal_",
+                "index_proxPhalanx_",
+                "index_intPhalanx_",
+                "index_distPhalanx_",
+                "index_tip_",
+                "middle_metacarpal_",
+                "middle_proxPhalanx_",
+                "middle_intPhalanx_",
+                "middle_distPhalanx_",
+                "middle_tip_",
+                "ring_metacarpal_",
+                "ring_proxPhalanx_",
+                "ring_intPhalanx_",
+                "ring_distPhalanx_",
+                "ring_tip_",
+                "little_metacarpal_",
+                "little_proxPhalanx_",
+                "little_intPhalanx_",
+                "little_distPhalanx_",
+                "little_tip_",
+            ].map((joint) => `${joint}${handedness === "right" ? "R" : "L"}`);
+            // single change for left handed systems
+            const tm = this._scene.getTransformNodeByName(this._rigMapping[0]);
+            if (!tm) {
+                throw new Error("could not find the wrist node");
+            } else {
+                tm.parent && (tm.parent as AbstractMesh).rotate(Axis.Y, Math.PI);
+            }
+        } catch (e) {
+            Tools.Error("error loading hand mesh");
+            console.log(e);
+        }
     }
 }
 
@@ -353,7 +510,10 @@ export class WebXRHandTracking extends WebXRAbstractFeature {
             trackedMeshes.push(newInstance);
         }
 
-        const webxrHand = new WebXRHand(xrController, trackedMeshes);
+        const handedness = xrController.inputSource.handedness === "right" ? "right" : "left";
+        const handMesh = this.options.jointMeshes?.handMeshes && this.options.jointMeshes?.handMeshes[handedness];
+        const rigMapping = this.options.jointMeshes?.rigMapping && this.options.jointMeshes?.rigMapping[handedness];
+        const webxrHand = new WebXRHand(xrController, trackedMeshes, handMesh, rigMapping);
 
         // get two new meshes
         this._hands[xrController.uniqueId] = {