Merge pull request #9620 from rickfromwork/touchGUI

Implementing initial set of MRTK 2.0 touch-ready buttons

dist/preview release/what's new.md

@@ -63,6 +63,7 @@
 - Added `focus()` and `blur()` functions for controls that implement `IFocusableControl` ([Flux159](https://github.com/Flux159))
 - Added `ToggleButton` GUI control ([kintz09](https://github.com/kintz09))
 - Added shorthand methods which set all padding values at once, named `setPadding` and `setPaddingInPixels`, to the control class  ([kintz09](https://github.com/kintz09))
+- Added two touch-enabled GUI controls, `TouchMeshButton3D` and `TouchHolographicButton`, added option on the WebXR hand tracking feature for enabling touch collisions ([rickfromwork](https://github.com/rickfromwork), [satyapoojasama](https://github.com/satyapoojasama))
 
 ### WebXR
 
@@ -72,6 +73,7 @@
 - Pointer Events of WebXR controllers have pointerType `xr` ([RaananW](https://github.com/RaananW))
 - better support for custom hand meshes ([RaananW](https://github.com/RaananW))
 - Allow disabling of the WebXRControllerPointerSelection feature as part of the WebXR Default Experience ([rgerd](https://github.com/rgerd))
+- Added two touch-enabled GUI controls, `TouchMeshButton3D` and `TouchHolographicButton`, added option on the WebXR hand tracking feature for enabling touch collisions ([rickfromwork](https://github.com/rickfromwork), [satyapoojasama](https://github.com/satyapoojasama))
 
 ### Viewer
 

gui/src/3D/controls/control3D.ts

@@ -20,7 +20,7 @@ export class Control3D implements IDisposable, IBehaviorAware<Control3D> {
     private _node: Nullable<TransformNode>;
     private _downCount = 0;
     private _enterCount = -1;
-    private _downPointerIds: { [id: number]: boolean } = {};
+    private _downPointerIds: { [id: number]: number } = {}; // Store number of pointer downs per ID, from near and far interactions
     private _isVisible = true;
 
     /** Gets or sets the control position  in world space */
@@ -306,16 +306,16 @@ export class Control3D implements IDisposable, IBehaviorAware<Control3D> {
 
     /** @hidden */
     public _onPointerEnter(target: Control3D): boolean {
-        if (this._enterCount > 0) {
-            return false;
-        }
-
         if (this._enterCount === -1) { // -1 is for touch input, we are now sure we are with a mouse or pencil
             this._enterCount = 0;
         }
 
         this._enterCount++;
 
+        if (this._enterCount > 1) {
+            return false;
+        }
+
         this.onPointerEnterObservable.notifyObservers(this, -1, target, this);
 
         if (this.pointerEnterAnimation) {
@@ -327,6 +327,12 @@ export class Control3D implements IDisposable, IBehaviorAware<Control3D> {
 
     /** @hidden */
     public _onPointerOut(target: Control3D): void {
+        this._enterCount--;
+
+        if (this._enterCount > 0) {
+            return;
+        }
+
         this._enterCount = 0;
 
         this.onPointerOutObservable.notifyObservers(this, -1, target, this);
@@ -338,14 +344,12 @@ export class Control3D implements IDisposable, IBehaviorAware<Control3D> {
 
     /** @hidden */
     public _onPointerDown(target: Control3D, coordinates: Vector3, pointerId: number, buttonIndex: number): boolean {
-        if (this._downCount !== 0) {
-            this._downCount++;
-            return false;
-        }
-
         this._downCount++;
+        this._downPointerIds[pointerId] = this._downPointerIds[pointerId] + 1 || 1;
 
-        this._downPointerIds[pointerId] = true;
+        if (this._downCount !== 1) {
+            return false;
+        }
 
         this.onPointerDownObservable.notifyObservers(new Vector3WithInfo(coordinates, buttonIndex), -1, target, this);
 
@@ -359,7 +363,11 @@ export class Control3D implements IDisposable, IBehaviorAware<Control3D> {
     /** @hidden */
     public _onPointerUp(target: Control3D, coordinates: Vector3, pointerId: number, buttonIndex: number, notifyClick: boolean): void {
         this._downCount--;
-        delete this._downPointerIds[pointerId];
+        this._downPointerIds[pointerId]--;
+
+        if (this._downPointerIds[pointerId] <= 0) {
+            delete this._downPointerIds[pointerId];
+        }
 
         if (this._downCount < 0) {
             // Handle if forcePointerUp was called prior to this

gui/src/3D/controls/index.ts

@@ -9,4 +9,7 @@ export * from "./planePanel";
 export * from "./scatterPanel";
 export * from "./spherePanel";
 export * from "./stackPanel3D";
+export * from "./touchButton3D";
+export * from "./touchMeshButton3D";
+export * from "./touchHolographicButton";
 export * from "./volumeBasedPanel";

gui/src/3D/controls/touchButton3D.ts

@@ -0,0 +1,367 @@
+// Assumptions: absolute position of button mesh is inside the mesh
+
+import { DeepImmutableObject } from "babylonjs/types";
+import { Vector3, Quaternion } from "babylonjs/Maths/math.vector";
+import { Mesh } from "babylonjs/Meshes/mesh";
+import { AbstractMesh } from "babylonjs/Meshes/abstractMesh";
+import { TransformNode } from "babylonjs/Meshes/transformNode";
+import { Scene } from "babylonjs/scene";
+import { Ray } from "babylonjs/Culling/ray";
+
+import { Button3D } from "./button3D";
+
+/**
+ * Enum for Button States
+ */
+enum ButtonState {
+    /** None */
+    None = 0,
+    /** Pointer Entered */
+    Hover = 1,
+    /** Pointer Down */
+    Press = 2
+}
+
+/**
+ * Class used to create a touchable button in 3D
+ */
+export class TouchButton3D extends Button3D {
+    private _collisionMesh: Mesh;
+    private _collidableFrontDirection: Vector3;
+    private _lastTouchPoint: Vector3;
+    private _tempButtonForwardRay: Ray;
+    private _lastKnownCollidableScale: Vector3;
+
+    private _collidableInitialized = false;
+
+    private _frontOffset = 0;
+    private _backOffset = 0;
+    private _hoverOffset = 0;
+    private _pushThroughBackOffset = 0;
+
+    private _activeInteractions = new Map<number, ButtonState>();
+    private _previousHeight = new Map<number, number>();
+
+    /**
+     * Creates a new touchable button
+     * @param name defines the control name
+     * @param collisionMesh mesh to track collisions with
+     */
+    constructor(name?: string, collisionMesh?: Mesh) {
+        super(name);
+
+        this._tempButtonForwardRay = new Ray(Vector3.Zero(), Vector3.Zero());
+
+        if (collisionMesh) {
+            this.collisionMesh = collisionMesh;
+        }
+    }
+
+    /**
+     * Sets the front-facing direction of the button
+     * @param frontDir the forward direction of the button
+     */
+    public set collidableFrontDirection(frontWorldDir: Vector3) {
+        this._collidableFrontDirection = frontWorldDir.normalize();
+
+        // Zero out the scale to force it to be proplerly updated in _updateDistanceOffsets
+        this._lastKnownCollidableScale = Vector3.Zero();
+
+        this._updateDistanceOffsets();
+    }
+
+    private _getWorldMatrixData(mesh: Mesh) {
+        let translation = Vector3.Zero();
+        let rotation = Quaternion.Identity();
+        let scale = Vector3.Zero();
+
+        mesh.getWorldMatrix().decompose(scale, rotation, translation);
+
+        return {translation: translation, rotation: rotation, scale: scale};
+    }
+
+    /**
+     * Sets the mesh used for testing input collision
+     * @param collisionMesh the new collision mesh for the button
+     */
+    public set collisionMesh(collisionMesh: Mesh) {
+        if (this._collisionMesh) {
+            this._collisionMesh.dispose();
+        }
+
+        // parent the mesh to sync transforms
+        if (!collisionMesh.parent && this.mesh) {
+            collisionMesh.setParent(this.mesh);
+        }
+
+        this._collisionMesh = collisionMesh;
+        this._collisionMesh.metadata = this;
+
+        this.collidableFrontDirection = collisionMesh.forward;
+
+        this._collidableInitialized = true;
+    }
+
+    /*
+     * Given a point, and two points on a line, this returns the distance between
+     * the point and the closest point on the line. The closest point on the line
+     * does not have to be between the two given points.
+     *
+     * Based off the 3D point-line distance equation
+     *
+     * Assumes lineDirection is normalized
+     */
+    private _getShortestDistancePointToLine(point: Vector3, linePoint: Vector3, lineDirection: Vector3) {
+        const pointToLine = linePoint.subtract(point);
+        const cross = lineDirection.cross(pointToLine);
+
+        return cross.length();
+    }
+
+    /*
+     * Checks to see if collidable is in a position to interact with the button
+     *   - check if collidable has a plane height off the button that is within range
+     *   - check that collidable + normal ray intersect the bounding sphere
+     */
+    private _isPrimedForInteraction(collidable: Vector3): boolean {
+        // Check if the collidable has an appropriate planar height
+        const heightFromCenter = this._getHeightFromButtonCenter(collidable);
+
+        if (heightFromCenter > this._hoverOffset || heightFromCenter < this._pushThroughBackOffset) {
+            return false;
+        }
+
+        // Check if the collidable or its hover ray lands within the bounding sphere of the button
+        const distanceFromCenter = this._getShortestDistancePointToLine(this._collisionMesh.getAbsolutePosition(),
+                                                                        collidable,
+                                                                        this._collidableFrontDirection);
+        return distanceFromCenter <= this._collisionMesh.getBoundingInfo().boundingSphere.radiusWorld;
+    }
+
+    /*
+     * Returns a Vector3 of the collidable's projected position on the button
+     * Returns the collidable's position if it is inside the button
+     */
+    private _getPointOnButton(collidable: Vector3): Vector3 {
+        const heightFromCenter = this._getHeightFromButtonCenter(collidable);
+
+        if (heightFromCenter <= this._frontOffset && heightFromCenter >= this._backOffset) {
+            // The collidable is in the button, return its position
+            return collidable;
+        }
+        else if (heightFromCenter > this._frontOffset) {
+            // The collidable is in front of the button, project it to the surface
+            const collidableDistanceToFront = (this._frontOffset - heightFromCenter);
+            return collidable.add(this._collidableFrontDirection.scale(collidableDistanceToFront));
+        }
+        else {
+            // The collidable is behind the button, project it to its back
+            const collidableDistanceToBack = (this._backOffset - heightFromCenter);
+            return collidable.add(this._collidableFrontDirection.scale(collidableDistanceToBack));
+        }
+    }
+
+    /*
+     * Updates the distance values.
+     * Should be called when the front direction changes, or the mesh size changes
+     *
+     * Sets the following values:
+     *    _frontOffset
+     *    _backOffset
+     *    _hoverOffset
+     *    _pushThroughBackOffset
+     *
+     * Requires population of:
+     *    _collisionMesh
+     *    _collidableFrontDirection
+     */
+    private _updateDistanceOffsets() {
+        let worldMatrixData = this._getWorldMatrixData(this._collisionMesh);
+
+        if (!worldMatrixData.scale.equalsWithEpsilon(this._lastKnownCollidableScale)) {
+            const collisionMeshPos = this._collisionMesh.getAbsolutePosition();
+
+            this._tempButtonForwardRay.origin = collisionMeshPos;
+            this._tempButtonForwardRay.direction = this._collidableFrontDirection;
+
+            const frontPickingInfo = this._tempButtonForwardRay.intersectsMesh(this._collisionMesh as DeepImmutableObject<AbstractMesh>);
+            this._tempButtonForwardRay.direction = this._tempButtonForwardRay.direction.negate();
+            const backPickingInfo = this._tempButtonForwardRay.intersectsMesh(this._collisionMesh as DeepImmutableObject<AbstractMesh>);
+
+            this._frontOffset = 0;
+            this._backOffset = 0;
+
+            if (frontPickingInfo.hit && backPickingInfo.hit) {
+                this._frontOffset = this._getDistanceOffPlane(frontPickingInfo.pickedPoint!,
+                                                                  this._collidableFrontDirection,
+                                                                  collisionMeshPos);
+                this._backOffset = this._getDistanceOffPlane(backPickingInfo.pickedPoint!,
+                                                                 this._collidableFrontDirection,
+                                                                 collisionMeshPos);
+            }
+
+            // For now, set the hover height equal to the thickness of the button
+            const buttonThickness = this._frontOffset - this._backOffset;
+
+            this._hoverOffset = this._frontOffset + (buttonThickness * 1.25);
+            this._pushThroughBackOffset = this._backOffset - (buttonThickness * 1.5);
+
+            this._lastKnownCollidableScale = this._getWorldMatrixData(this._collisionMesh).scale;
+        }
+    }
+
+    // Returns the distance in front of the center of the button
+    // Returned value is negative when collidable is past the center
+    private _getHeightFromButtonCenter(collidablePos: Vector3) {
+        return this._getDistanceOffPlane(collidablePos, this._collidableFrontDirection, this._collisionMesh.getAbsolutePosition());
+    }
+
+    // Returns the distance from pointOnPlane to point along planeNormal
+    private _getDistanceOffPlane(point: Vector3, planeNormal: Vector3, pointOnPlane: Vector3) {
+        const d = Vector3.Dot(pointOnPlane, planeNormal);
+        const abc = Vector3.Dot(point, planeNormal);
+
+        return abc - d;
+    }
+
+    // Updates the stored state of the button, and fire pointer events
+    private _updateButtonState(id: number, newState: ButtonState, pointOnButton: Vector3) {
+        const dummyPointerId = 0;
+        const buttonIndex = 0; // Left click
+        const buttonStateForId = this._activeInteractions.get(id) || ButtonState.None;
+
+        // Take into account all inputs interacting with the button to avoid state flickering
+        let previousPushDepth = 0;
+        this._activeInteractions.forEach(function(value, key) {
+            previousPushDepth = Math.max(previousPushDepth, value);
+        });
+
+        if (buttonStateForId != newState) {
+            if (newState == ButtonState.None) {
+                this._activeInteractions.delete(id);
+            }
+            else {
+                this._activeInteractions.set(id, newState);
+            }
+        }
+
+        let newPushDepth = 0;
+        this._activeInteractions.forEach(function(value, key) {
+            newPushDepth = Math.max(newPushDepth, value);
+        });
+
+        if (newPushDepth == ButtonState.Press) {
+            if (previousPushDepth == ButtonState.Hover) {
+                this._onPointerDown(this, pointOnButton, dummyPointerId, buttonIndex);
+            }
+            else if (previousPushDepth == ButtonState.Press) {
+                this._onPointerMove(this, pointOnButton);
+            }
+        }
+        else if (newPushDepth == ButtonState.Hover) {
+            if (previousPushDepth == ButtonState.None) {
+                this._onPointerEnter(this);
+            }
+            else if (previousPushDepth == ButtonState.Press) {
+                this._onPointerUp(this, pointOnButton, dummyPointerId, buttonIndex, false);
+            }
+            else {
+                this._onPointerMove(this, pointOnButton);
+            }
+        }
+        else if (newPushDepth == ButtonState.None) {
+            if (previousPushDepth == ButtonState.Hover) {
+                this._onPointerOut(this);
+            }
+            else if (previousPushDepth == ButtonState.Press) {
+                this._onPointerUp(this, pointOnButton, dummyPointerId, buttonIndex, false);
+                this._onPointerOut(this);
+            }
+        }
+    }
+
+    // Decides whether to change button state based on the planar depth of the input source
+    /** @hidden */
+    public _collisionCheckForStateChange(mesh: AbstractMesh) {
+        if (this._collidableInitialized) {
+            this._updateDistanceOffsets();
+
+            const collidablePosition = mesh.getAbsolutePosition();
+            const inRange = this._isPrimedForInteraction(collidablePosition);
+
+            const uniqueId = mesh.uniqueId;
+
+            let activeInteraction = this._activeInteractions.get(uniqueId);
+            if (inRange) {
+                const pointOnButton = this._getPointOnButton(collidablePosition);
+                const heightFromCenter = this._getHeightFromButtonCenter(collidablePosition);
+                const flickerDelta = 0.003;
+
+                this._lastTouchPoint = pointOnButton;
+
+                const isGreater = function (compareHeight: number) {
+                    return heightFromCenter >= (compareHeight + flickerDelta);
+                };
+
+                const isLower = function (compareHeight: number) {
+                    return heightFromCenter <= (compareHeight - flickerDelta);
+                };
+
+                // Update button state and fire events
+                switch (activeInteraction || ButtonState.None) {
+                    case ButtonState.None:
+                        if (isGreater(this._frontOffset) &&
+                            isLower(this._hoverOffset)) {
+                            this._updateButtonState(uniqueId, ButtonState.Hover, pointOnButton);
+                        }
+
+                        break;
+                    case ButtonState.Hover:
+                        if (isGreater(this._hoverOffset)) {
+                            this._updateButtonState(uniqueId, ButtonState.None, pointOnButton);
+                        }
+                        else if (isLower(this._frontOffset)) {
+                            this._updateButtonState(uniqueId, ButtonState.Press, pointOnButton);
+                        }
+
+                        break;
+                    case ButtonState.Press:
+                        if (isGreater(this._frontOffset)) {
+                            this._updateButtonState(uniqueId, ButtonState.Hover, pointOnButton);
+                        }
+                        else if (isLower(this._pushThroughBackOffset)) {
+                            this._updateButtonState(uniqueId, ButtonState.None, pointOnButton);
+                        }
+
+                        break;
+                }
+
+                this._previousHeight.set(uniqueId, heightFromCenter);
+            }
+            else if ((activeInteraction != undefined) && (activeInteraction != ButtonState.None)) {
+                this._updateButtonState(uniqueId, ButtonState.None, this._lastTouchPoint);
+                this._previousHeight.delete(uniqueId);
+            }
+        }
+    }
+
+    protected _getTypeName(): string {
+        return "TouchButton3D";
+    }
+
+    // Mesh association
+    protected _createNode(scene: Scene): TransformNode {
+        return super._createNode(scene);
+    }
+
+    /**
+     * Releases all associated resources
+     */
+    public dispose() {
+        super.dispose();
+
+        if (this._collisionMesh) {
+            this._collisionMesh.dispose();
+        }
+    }
+}

gui/src/3D/controls/touchHolographicButton.ts

@@ -0,0 +1,364 @@
+import { Nullable } from "babylonjs/types";
+import { Observer } from "babylonjs/Misc/observable";
+import { Vector3 } from "babylonjs/Maths/math.vector";
+import { StandardMaterial } from "babylonjs/Materials/standardMaterial";
+import { TransformNode } from "babylonjs/Meshes/transformNode";
+import { Mesh } from "babylonjs/Meshes/mesh";
+import { PlaneBuilder } from "babylonjs/Meshes/Builders/planeBuilder";
+import { BoxBuilder } from "babylonjs/Meshes/Builders/boxBuilder";
+import { FadeInOutBehavior } from "babylonjs/Behaviors/Meshes/fadeInOutBehavior";
+import { Scene } from "babylonjs/scene";
+
+import { FluentMaterial } from "../materials/fluentMaterial";
+import { StackPanel } from "../../2D/controls/stackPanel";
+import { Image } from "../../2D/controls/image";
+import { TextBlock } from "../../2D/controls/textBlock";
+import { AdvancedDynamicTexture } from "../../2D/advancedDynamicTexture";
+import { Control3D } from "./control3D";
+import { Color3 } from 'babylonjs/Maths/math.color';
+
+import { TouchButton3D } from "./touchButton3D";
+
+/**
+ * Class used to create a holographic button in 3D
+ */
+export class TouchHolographicButton extends TouchButton3D {
+    private _backPlate: Mesh;
+    private _textPlate: Mesh;
+    private _frontPlate: Mesh;
+    private _text: string;
+    private _imageUrl: string;
+    private _shareMaterials = true;
+    private _frontMaterial: FluentMaterial;
+    private _backMaterial: FluentMaterial;
+    private _plateMaterial: StandardMaterial;
+    private _pickedPointObserver: Nullable<Observer<Nullable<Vector3>>>;
+
+    // Tooltip
+    private _tooltipFade: Nullable<FadeInOutBehavior>;
+    private _tooltipTextBlock: Nullable<TextBlock>;
+    private _tooltipTexture: Nullable<AdvancedDynamicTexture>;
+    private _tooltipMesh: Nullable<Mesh>;
+    private _tooltipHoverObserver: Nullable<Observer<Control3D>>;
+    private _tooltipOutObserver: Nullable<Observer<Control3D>>;
+
+    private _disposeTooltip() {
+        this._tooltipFade = null;
+        if (this._tooltipTextBlock) {
+            this._tooltipTextBlock.dispose();
+        }
+        if (this._tooltipTexture) {
+            this._tooltipTexture.dispose();
+        }
+        if (this._tooltipMesh) {
+            this._tooltipMesh.dispose();
+        }
+        this.onPointerEnterObservable.remove(this._tooltipHoverObserver);
+        this.onPointerOutObservable.remove(this._tooltipOutObserver);
+    }
+
+    /**
+     * Rendering ground id of all the mesh in the button
+     */
+    public set renderingGroupId(id: number) {
+        this._backPlate.renderingGroupId = id;
+        this._textPlate.renderingGroupId = id;
+        this._frontPlate.renderingGroupId = id;
+
+        if (this._tooltipMesh) {
+            this._tooltipMesh.renderingGroupId = id;
+        }
+    }
+    public get renderingGroupId(): number {
+        return this._backPlate.renderingGroupId;
+    }
+
+    /**
+     * Text to be displayed on the tooltip shown when hovering on the button. When set to null tooltip is disabled. (Default: null)
+     */
+    public set tooltipText(text: Nullable<string>) {
+        if (!text) {
+            this._disposeTooltip();
+            return;
+        }
+        if (!this._tooltipFade) {
+            // Create tooltip with mesh and text
+            this._tooltipMesh = PlaneBuilder.CreatePlane("", { size: 1 }, this._backPlate._scene);
+            var tooltipBackground = PlaneBuilder.CreatePlane("", { size: 1, sideOrientation: Mesh.DOUBLESIDE }, this._backPlate._scene);
+            var mat = new StandardMaterial("", this._backPlate._scene);
+            mat.diffuseColor = Color3.FromHexString("#212121");
+            tooltipBackground.material = mat;
+            tooltipBackground.isPickable = false;
+            this._tooltipMesh.addChild(tooltipBackground);
+            tooltipBackground.position.z = 0.05;
+            this._tooltipMesh.scaling.y = 1 / 3;
+            this._tooltipMesh.position.y = 0.7;
+            this._tooltipMesh.position.z = -0.15;
+            this._tooltipMesh.isPickable = false;
+            this._tooltipMesh.parent = this._backPlate;
+
+            // Create text texture for the tooltip
+            this._tooltipTexture = AdvancedDynamicTexture.CreateForMesh(this._tooltipMesh);
+            this._tooltipTextBlock = new TextBlock();
+            this._tooltipTextBlock.scaleY = 3;
+            this._tooltipTextBlock.color = "white";
+            this._tooltipTextBlock.fontSize = 130;
+            this._tooltipTexture.addControl(this._tooltipTextBlock);
+
+            // Add hover action to tooltip
+            this._tooltipFade = new FadeInOutBehavior();
+            this._tooltipFade.delay = 500;
+            this._tooltipMesh.addBehavior(this._tooltipFade);
+            this._tooltipHoverObserver = this.onPointerEnterObservable.add(() => {
+                if (this._tooltipFade) {
+                    this._tooltipFade.fadeIn(true);
+                }
+            });
+            this._tooltipOutObserver = this.onPointerOutObservable.add(() => {
+                if (this._tooltipFade) {
+                    this._tooltipFade.fadeIn(false);
+                }
+            });
+        }
+        if (this._tooltipTextBlock) {
+            this._tooltipTextBlock.text = text;
+        }
+    }
+
+    public get tooltipText() {
+        if (this._tooltipTextBlock) {
+            return this._tooltipTextBlock.text;
+        }
+        return null;
+    }
+
+    /**
+     * Gets or sets text for the button
+     */
+    public get text(): string {
+        return this._text;
+    }
+
+    public set text(value: string) {
+        if (this._text === value) {
+            return;
+        }
+
+        this._text = value;
+        this._rebuildContent();
+    }
+
+    /**
+     * Gets or sets the image url for the button
+     */
+    public get imageUrl(): string {
+        return this._imageUrl;
+    }
+
+    public set imageUrl(value: string) {
+        if (this._imageUrl === value) {
+            return;
+        }
+
+        this._imageUrl = value;
+        this._rebuildContent();
+    }
+
+    /**
+     * Gets the back material used by this button
+     */
+    public get backMaterial(): FluentMaterial {
+        return this._backMaterial;
+    }
+
+    /**
+     * Gets the front material used by this button
+     */
+    public get frontMaterial(): FluentMaterial {
+        return this._frontMaterial;
+    }
+
+    /**
+     * Gets the plate material used by this button
+     */
+    public get plateMaterial(): StandardMaterial {
+        return this._plateMaterial;
+    }
+
+    /**
+     * Gets a boolean indicating if this button shares its material with other HolographicButtons
+     */
+    public get shareMaterials(): boolean {
+        return this._shareMaterials;
+    }
+
+    /**
+     * Creates a new button
+     * @param name defines the control name
+     */
+    constructor(name?: string, shareMaterials = true) {
+        super(name);
+
+        this._shareMaterials = shareMaterials;
+
+        // Default animations
+        this.pointerEnterAnimation = () => {
+            if (!this.mesh) {
+                return;
+            }
+            this._frontPlate.setEnabled(true);
+        };
+
+        this.pointerOutAnimation = () => {
+            if (!this.mesh) {
+                return;
+            }
+            this._frontPlate.setEnabled(false);
+        };
+    }
+
+    protected _getTypeName(): string {
+        return "TouchHolographicButton";
+    }
+
+    private _rebuildContent(): void {
+        this._disposeFacadeTexture();
+
+        let panel = new StackPanel();
+        panel.isVertical = true;
+
+        if (this._imageUrl) {
+            let image = new Image();
+            image.source = this._imageUrl;
+            image.paddingTop = "40px";
+            image.height = "180px";
+            image.width = "100px";
+            image.paddingBottom = "40px";
+            panel.addControl(image);
+        }
+
+        if (this._text) {
+            let text = new TextBlock();
+            text.text = this._text;
+            text.color = "white";
+            text.height = "30px";
+            text.fontSize = 24;
+            panel.addControl(text);
+        }
+
+        if (this._frontPlate) {
+            this.content = panel;
+        }
+    }
+
+    // Mesh association
+    protected _createNode(scene: Scene): TransformNode {
+        this._backPlate = BoxBuilder.CreateBox(this.name + "BackMesh", {
+            width: 1.0,
+            height: 1.0,
+            depth: 0.08
+        }, scene);
+
+        this._frontPlate = BoxBuilder.CreateBox(this.name + "FrontMesh", {
+            width: 1.0,
+            height: 1.0,
+            depth: 0.4
+        }, scene);
+
+        this._frontPlate.parent = this._backPlate;
+        this._frontPlate.position.z = -0.08;
+        this._frontPlate.isPickable = false;
+        this._frontPlate.setEnabled(false);
+
+        this._textPlate = <Mesh>super._createNode(scene);
+        this._textPlate.parent = this._backPlate;
+        this._textPlate.position.z = -0.08;
+        this._textPlate.isPickable = false;
+
+        this.collisionMesh = this._frontPlate;
+        this.collidableFrontDirection = this._frontPlate.forward.negate(); // Mesh is facing the wrong way
+
+        return this._backPlate;
+    }
+
+    protected _applyFacade(facadeTexture: AdvancedDynamicTexture) {
+        this._plateMaterial.emissiveTexture = facadeTexture;
+        this._plateMaterial.opacityTexture = facadeTexture;
+    }
+
+    private _createBackMaterial(mesh: Mesh) {
+        this._backMaterial = new FluentMaterial(this.name + "Back Material", mesh.getScene());
+        this._backMaterial.renderHoverLight = true;
+        this._backMaterial.albedoColor = new Color3(0.1, 0.1, 0.4);
+        this._pickedPointObserver = this._host.onPickedPointChangedObservable.add((pickedPoint) => {
+            if (pickedPoint) {
+                this._backMaterial.hoverPosition = pickedPoint;
+                this._backMaterial.hoverColor.a = 1.0;
+            } else {
+                this._backMaterial.hoverColor.a = 0;
+            }
+        });
+    }
+
+    private _createFrontMaterial(mesh: Mesh) {
+        this._frontMaterial = new FluentMaterial(this.name + "Front Material", mesh.getScene());
+        this._frontMaterial.innerGlowColorIntensity = 0; // No inner glow
+        this._frontMaterial.alpha = 0.3; // Additive
+        this._frontMaterial.renderBorders = false;
+    }
+
+    private _createPlateMaterial(mesh: Mesh) {
+        this._plateMaterial = new StandardMaterial(this.name + "Plate Material", mesh.getScene());
+        this._plateMaterial.specularColor = Color3.Black();
+    }
+
+    protected _affectMaterial(mesh: Mesh) {
+        // Back
+        if (this._shareMaterials) {
+            if (!this._host._touchSharedMaterials["backFluentMaterial"]) {
+                this._createBackMaterial(mesh);
+                this._host._touchSharedMaterials["backFluentMaterial"] = this._backMaterial;
+            } else {
+                this._backMaterial = this._host._touchSharedMaterials["backFluentMaterial"] as FluentMaterial;
+            }
+
+            // Front
+            if (!this._host._touchSharedMaterials["frontFluentMaterial"]) {
+                this._createFrontMaterial(mesh);
+                this._host._touchSharedMaterials["frontFluentMaterial"] = this._frontMaterial;
+            } else {
+                this._frontMaterial = this._host._touchSharedMaterials["frontFluentMaterial"] as FluentMaterial;
+            }
+        } else {
+            this._createBackMaterial(mesh);
+            this._createFrontMaterial(mesh);
+        }
+
+        this._createPlateMaterial(mesh);
+        this._backPlate.material = this._backMaterial;
+        this._frontPlate.material = this._frontMaterial;
+        this._textPlate.material = this._plateMaterial;
+
+        this._rebuildContent();
+    }
+
+    /**
+     * Releases all associated resources
+     */
+    public dispose() {
+        super.dispose(); // will dispose main mesh ie. back plate
+
+        this._disposeTooltip();
+
+        if (!this.shareMaterials) {
+            this._backMaterial.dispose();
+            this._frontMaterial.dispose();
+            this._plateMaterial.dispose();
+
+            if (this._pickedPointObserver) {
+                this._host.onPickedPointChangedObservable.remove(this._pickedPointObserver);
+                this._pickedPointObserver = null;
+            }
+        }
+    }
+}

gui/src/3D/controls/touchMeshButton3D.ts

@@ -0,0 +1,81 @@
+import { TransformNode } from "babylonjs/Meshes/transformNode";
+import { AbstractMesh } from "babylonjs/Meshes/abstractMesh";
+import { Mesh } from "babylonjs/Meshes/mesh";
+import { Scene } from "babylonjs/scene";
+
+import { TouchButton3D } from "./touchButton3D";
+
+/**
+ * Class used to create an interactable object. It's a touchable 3D button using a mesh coming from the current scene
+ */
+export class TouchMeshButton3D extends TouchButton3D {
+    /** @hidden */
+    protected _currentMesh: Mesh;
+
+    /**
+     * Creates a new 3D button based on a mesh
+     * @param mesh mesh to become a 3D button
+     * @param collisionMesh mesh to track collisions with
+     * @param name defines the control name
+     */
+    constructor(mesh: Mesh, options: {collisionMesh: Mesh, useDynamicMesh?: boolean}, name?: string) {
+        if (options.useDynamicMesh) {
+            super(name, options.collisionMesh);
+        }
+        else {
+            let newCollisionMesh = options.collisionMesh.clone("", options.collisionMesh.parent);
+            newCollisionMesh.isVisible = false;
+            super(name, newCollisionMesh);
+        }
+
+        this._currentMesh = mesh;
+
+        /**
+         * Provides a default behavior on hover/out & up/down
+         * Override those function to create your own desired behavior specific to your mesh
+         */
+        this.pointerEnterAnimation = () => {
+            if (!this.mesh) {
+                return;
+            }
+            this.mesh.scaling.scaleInPlace(1.1);
+        };
+
+        this.pointerOutAnimation = () => {
+            if (!this.mesh) {
+                return;
+            }
+            this.mesh.scaling.scaleInPlace(1.0 / 1.1);
+        };
+
+        this.pointerDownAnimation = () => {
+            if (!this.mesh) {
+                return;
+            }
+            this.mesh.scaling.scaleInPlace(0.95);
+        };
+
+        this.pointerUpAnimation = () => {
+            if (!this.mesh) {
+                return;
+            }
+            this.mesh.scaling.scaleInPlace(1.0 / 0.95);
+        };
+    }
+
+    protected _getTypeName(): string {
+        return "TouchMeshButton3D";
+    }
+
+    // Mesh association
+    protected _createNode(scene: Scene): TransformNode {
+        this._currentMesh.getChildMeshes().forEach((mesh) => {
+            mesh.metadata = this;
+        });
+
+        return this._currentMesh;
+    }
+
+    protected _affectMaterial(mesh: AbstractMesh) {
+    }
+}

gui/src/3D/gui3DManager.ts

@@ -11,6 +11,7 @@ import { IDisposable, Scene } from "babylonjs/scene";
 
 import { Container3D } from "./controls/container3D";
 import { Control3D } from "./controls/control3D";
+import { TouchButton3D } from "./controls/touchButton3D";
 
 /**
  * Class used to manage 3D user interface
@@ -23,6 +24,7 @@ export class GUI3DManager implements IDisposable {
     private _rootContainer: Container3D;
     private _pointerObserver: Nullable<Observer<PointerInfo>>;
     private _pointerOutObserver: Nullable<Observer<number>>;
+    private _touchableButtons = new Set<TouchButton3D>();
     /** @hidden */
     public _lastPickedControl: Control3D;
     /** @hidden */
@@ -39,6 +41,9 @@ export class GUI3DManager implements IDisposable {
     /** @hidden */
     public _sharedMaterials: { [key: string]: Material } = {};
 
+    /** @hidden */
+    public _touchSharedMaterials:  { [key: string]: Material } = {};
+
     /** Gets the hosting scene */
     public get scene(): Scene {
         return this._scene;
@@ -151,6 +156,19 @@ export class GUI3DManager implements IDisposable {
         return true;
     }
 
+    private _processTouchControls = () => {
+        let utilityLayerScene = this._utilityLayer ? this._utilityLayer.utilityLayerScene : null;
+        if (utilityLayerScene) {
+            const touchMeshes = utilityLayerScene.getMeshesByTags("touchEnabled");
+
+            this._touchableButtons.forEach(function (button: TouchButton3D) {
+                touchMeshes.forEach(function (mesh: AbstractMesh) {
+                    button._collisionCheckForStateChange(mesh);
+                });
+            });
+        }
+    }
+
     /**
      * Gets the root container
      */
@@ -174,6 +192,16 @@ export class GUI3DManager implements IDisposable {
      */
     public addControl(control: Control3D): GUI3DManager {
         this._rootContainer.addControl(control);
+
+        let utilityLayerScene = this._utilityLayer ? this._utilityLayer.utilityLayerScene : null;
+        if (utilityLayerScene && (control instanceof TouchButton3D)) {
+            if (this._touchableButtons.size == 0) {
+                utilityLayerScene.registerBeforeRender(this._processTouchControls);
+            }
+
+            this._touchableButtons.add(control as TouchButton3D);
+        }
+
         return this;
     }
 
@@ -184,6 +212,16 @@ export class GUI3DManager implements IDisposable {
      */
     public removeControl(control: Control3D): GUI3DManager {
         this._rootContainer.removeControl(control);
+
+        let utilityLayerScene = this._utilityLayer ? this._utilityLayer.utilityLayerScene : null;
+        if (utilityLayerScene && (control instanceof TouchButton3D)) {
+            this._touchableButtons.delete(control);
+
+            if (this._touchableButtons.size == 0) {
+                utilityLayerScene.unregisterBeforeRender(this._processTouchControls);
+            }
+        }
+
         return this;
     }
 
@@ -203,6 +241,16 @@ export class GUI3DManager implements IDisposable {
 
         this._sharedMaterials = {};
 
+        for (var materialName in this._touchSharedMaterials) {
+            if (!this._touchSharedMaterials.hasOwnProperty(materialName)) {
+                continue;
+            }
+
+            this._touchSharedMaterials[materialName].dispose();
+        }
+
+        this._touchSharedMaterials = {};
+
         if (this._pointerOutObserver && this._utilityLayer) {
             this._utilityLayer.onPointerOutObservable.remove(this._pointerOutObserver);
             this._pointerOutObserver = null;
@@ -213,6 +261,10 @@ export class GUI3DManager implements IDisposable {
         let utilityLayerScene = this._utilityLayer ? this._utilityLayer.utilityLayerScene : null;
 
         if (utilityLayerScene) {
+            if (this._touchableButtons.size != 0) {
+                utilityLayerScene.unregisterBeforeRender(this._processTouchControls);
+            }
+
             if (this._pointerObserver) {
                 utilityLayerScene.onPointerObservable.remove(this._pointerObserver);
                 this._pointerObserver = null;

src/XR/features/WebXRHandTracking.ts

@@ -22,6 +22,7 @@ import { Engine } from "../../Engines/engine";
 import { Tools } from "../../Misc/tools";
 import { Axis } from "../../Maths/math.axis";
 import { TransformNode } from "../../Meshes/transformNode";
+import { Tags } from "../../Misc/tags";
 import { Bone } from "../../Bones/bone";
 
 declare const XRHand: XRHand;
@@ -94,6 +95,10 @@ export interface IWebXRHandTrackingOptions {
             right: string[];
             left: string[];
         };
+        /**
+         * The utilityLayer scene that contains the 3D UI elements. Passing this in turns on near interactions with the index finger tip
+         */
+        sceneForNearInteraction?: Scene;
     };
 }
 
@@ -106,7 +111,7 @@ export const enum HandPart {
      */
     WRIST = "wrist",
     /**
-     * HandPart - The Thumb
+     * HandPart - The thumb
      */
     THUMB = "thumb",
     /**
@@ -168,6 +173,7 @@ export class WebXRHand implements IDisposable {
      * @param _handMesh an optional hand mesh. if not provided, ours will be used
      * @param _rigMapping an optional rig mapping for the hand mesh. if not provided, ours will be used
      * @param disableDefaultHandMesh should the default mesh creation be disabled
+     * @param _nearInteractionMesh as optional mesh used for near interaction collision checking
      * @param _leftHandedMeshes are the hand meshes left-handed-system meshes
      */
     constructor(
@@ -178,6 +184,7 @@ export class WebXRHand implements IDisposable {
         private _handMesh?: AbstractMesh,
         private _rigMapping?: string[],
         disableDefaultHandMesh?: boolean,
+        private _nearInteractionMesh?: Nullable<AbstractMesh>,
         private _leftHandedMeshes?: boolean
     ) {
         this.handPartsDefinition = this.generateHandPartsDefinition(xrController.inputSource.hand!);
@@ -302,6 +309,11 @@ export class WebXRHand implements IDisposable {
                 }
             }
         });
+        // Update the invisible fingertip collidable
+        if (this._nearInteractionMesh) {
+            const indexTipPose = this.trackedMeshes[hand.INDEX_PHALANX_TIP].position;
+            this._nearInteractionMesh.position.set(indexTipPose.x, indexTipPose.y, indexTipPose.z);
+        }
     }
 
     /**
@@ -318,6 +330,7 @@ export class WebXRHand implements IDisposable {
      */
     public dispose() {
         this.trackedMeshes.forEach((mesh) => mesh.dispose());
+        this._nearInteractionMesh?.dispose();
         this.onHandMeshReadyObservable.clear();
         // dispose the hand mesh, if it is the default one
         if (this._defaultHandMesh && this._handMesh) {
@@ -574,10 +587,17 @@ export class WebXRHandTracking extends WebXRAbstractFeature {
             trackedMeshes.push(newInstance);
         }
 
+        let touchMesh: Nullable<AbstractMesh> = null;
+        if (this.options.jointMeshes?.sceneForNearInteraction) {
+            touchMesh = SphereBuilder.CreateSphere(`${xrController.uniqueId}-handJoint-indexCollidable`, {}, this.options.jointMeshes.sceneForNearInteraction);
+            touchMesh.isVisible = false;
+            Tags.AddTagsTo(touchMesh, "touchEnabled");
+        }
+
         const handedness = xrController.inputSource.handedness === "right" ? "right" : "left";
         const handMesh = this.options.jointMeshes?.handMeshes && this.options.jointMeshes?.handMeshes[handedness];
         const rigMapping = this.options.jointMeshes?.rigMapping && this.options.jointMeshes?.rigMapping[handedness];
-        const webxrHand = new WebXRHand(xrController, trackedMeshes, handMesh, rigMapping, this.options.jointMeshes?.disableDefaultHandMesh, this.options.jointMeshes?.leftHandedSystemMeshes);
+        const webxrHand = new WebXRHand(xrController, trackedMeshes, handMesh, rigMapping, this.options.jointMeshes?.disableDefaultHandMesh, touchMesh, this.options.jointMeshes?.leftHandedSystemMeshes);
 
         // get two new meshes
         this._hands[xrController.uniqueId] = {