Fixing orientation sound update in VR

dist/preview release/babylon.max.js

@@ -17839,6 +17839,10 @@ var BABYLON;
             }
             if (listeningCamera && audioEngine.canUseWebAudio) {
                 audioEngine.audioContext.listener.setPosition(listeningCamera.position.x, listeningCamera.position.y, listeningCamera.position.z);
+                // for VR cameras
+                if (listeningCamera.rigCameras) {
+                    listeningCamera = listeningCamera.rigCameras[0];
+                }
                 var mat = BABYLON.Matrix.Invert(listeningCamera.getViewMatrix());
                 var cameraDirection = BABYLON.Vector3.TransformNormal(new BABYLON.Vector3(0, 0, -1), mat);
                 cameraDirection.normalize();

dist/preview release/loaders/babylon.glTF2FileLoader.d.ts

@@ -214,7 +214,11 @@ declare module BABYLON.GLTF2 {
         translation?: number[];
         weights?: number[];
         index?: number;
-        babylonNode?: Node;
+        babylonMesh?: Mesh;
+        babylonSkinToBones?: {
+            [skin: number]: Bone;
+        };
+        babylonAnimationTargets?: Node[];
     }
     interface IGLTFSampler extends IGLTFChildRootProperty {
         magFilter?: ETextureMagFilter;
@@ -229,6 +233,7 @@ declare module BABYLON.GLTF2 {
         inverseBindMatrices?: number;
         skeleton?: number;
         joints: number[];
+        index?: number;
         babylonSkeleton?: Skeleton;
     }
     interface IGLTFTexture extends IGLTFChildRootProperty {
@@ -292,6 +297,7 @@ declare module BABYLON.GLTF2 {
         private _loadScene(nodeNames);
         private _loadSkin(node);
         private _updateBone(node, parentNode, skin, inverseBindMatrixData);
+        private _createBone(node, skin);
         private _loadMesh(node, parentNode);
         private _loadMeshData(node, mesh, babylonMesh);
         private _loadVertexDataAsync(primitive, onSuccess);

dist/preview release/loaders/babylon.glTF2FileLoader.js

@@ -319,8 +319,8 @@ var BABYLON;
                     if (_this._gltf.nodes) {
                         for (var i = 0; i < _this._gltf.nodes.length; i++) {
                             var node = _this._gltf.nodes[i];
-                            if (node.babylonNode instanceof BABYLON.AbstractMesh) {
-                                meshes.push(node.babylonNode);
+                            if (node.babylonMesh) {
+                                meshes.push(node.babylonMesh);
                             }
                         }
                     }
@@ -386,8 +386,8 @@ var BABYLON;
                 var nodes = this._gltf.nodes;
                 for (var i = 0; i < nodes.length; i++) {
                     var node = nodes[i];
-                    if (node.babylonNode instanceof BABYLON.Mesh) {
-                        node.babylonNode.isVisible = true;
+                    if (node.babylonMesh) {
+                        node.babylonMesh.isVisible = true;
                     }
                 }
             };
@@ -431,17 +431,13 @@ var BABYLON;
             };
             GLTFLoader.prototype._loadSkin = function (node) {
                 var _this = this;
-                if (node.babylonNode) {
-                    return false;
-                }
                 if (node.skin !== undefined) {
                     var skin = this._gltf.skins[node.skin];
                     var skeletonId = "skeleton" + node.skin;
                     skin.babylonSkeleton = new BABYLON.Skeleton(skin.name || skeletonId, skeletonId, this._babylonScene);
+                    skin.index = node.skin;
                     for (var i = 0; i < skin.joints.length; i++) {
-                        var jointIndex = skin.joints[i];
-                        var jointNode = this._gltf.nodes[jointIndex];
-                        jointNode.babylonNode = new BABYLON.Bone(jointNode.name || "bone" + jointIndex, skin.babylonSkeleton);
+                        this._createBone(this._gltf.nodes[skin.joints[i]], skin);
                     }
                     if (skin.skeleton === undefined) {
                         // TODO: handle when skeleton is not defined
@@ -461,30 +457,28 @@ var BABYLON;
             GLTFLoader.prototype._updateBone = function (node, parentNode, skin, inverseBindMatrixData) {
                 var jointIndex = skin.joints.indexOf(node.index);
                 if (jointIndex === -1) {
-                    // TODO: handle non-joint in between two joints
-                    throw new Error("Not implemented");
+                    this._createBone(node, skin);
                 }
-                var babylonBone = node.babylonNode;
+                var babylonBone = node.babylonSkinToBones[skin.index];
                 // TODO: explain the math
-                var matrix = BABYLON.Matrix.FromArray(inverseBindMatrixData, jointIndex * 16);
+                var matrix = jointIndex === -1 ? BABYLON.Matrix.Identity() : BABYLON.Matrix.FromArray(inverseBindMatrixData, jointIndex * 16);
                 matrix.invertToRef(matrix);
                 if (parentNode) {
-                    babylonBone.setParent(parentNode.babylonNode, false);
+                    babylonBone.setParent(parentNode.babylonSkinToBones[skin.index], false);
                     matrix.multiplyToRef(babylonBone.getParent().getInvertedAbsoluteTransform(), matrix);
                 }
                 babylonBone.updateMatrix(matrix);
                 return true;
             };
+            GLTFLoader.prototype._createBone = function (node, skin) {
+                var babylonBone = new BABYLON.Bone(node.name || "bone" + node.index, skin.babylonSkeleton);
+                node.babylonSkinToBones = node.babylonSkinToBones || {};
+                node.babylonSkinToBones[skin.index] = babylonBone;
+                node.babylonAnimationTargets = node.babylonAnimationTargets || [];
+                node.babylonAnimationTargets.push(babylonBone);
+                return babylonBone;
+            };
             GLTFLoader.prototype._loadMesh = function (node, parentNode) {
-                if (node.babylonNode) {
-                    if (node.babylonNode instanceof BABYLON.Bone) {
-                        if (node.mesh !== undefined) {
-                            // TODO: handle mesh attached to bone
-                            throw new Error("Not implemented");
-                        }
-                    }
-                    return false;
-                }
                 var babylonMesh = new BABYLON.Mesh(node.name || "mesh" + node.index, this._babylonScene);
                 babylonMesh.isVisible = false;
                 this._loadTransform(node, babylonMesh);
@@ -492,8 +486,10 @@ var BABYLON;
                     var mesh = this._gltf.meshes[node.mesh];
                     this._loadMeshData(node, mesh, babylonMesh);
                 }
-                babylonMesh.parent = parentNode ? parentNode.babylonNode : null;
-                node.babylonNode = babylonMesh;
+                babylonMesh.parent = parentNode ? parentNode.babylonMesh : null;
+                node.babylonMesh = babylonMesh;
+                node.babylonAnimationTargets = node.babylonAnimationTargets || [];
+                node.babylonAnimationTargets.push(node.babylonMesh);
                 if (node.skin !== undefined) {
                     var skin = this._gltf.skins[node.skin];
                     babylonMesh.skeleton = skin.babylonSkeleton;
@@ -730,7 +726,7 @@ var BABYLON;
                 var channel = animation.channels[channelIndex];
                 var samplerIndex = channel.sampler;
                 var sampler = animation.samplers[samplerIndex];
-                var targetNode = this._gltf.nodes[channel.target.node].babylonNode;
+                var targetNode = this._gltf.nodes[channel.target.node];
                 if (!targetNode) {
                     BABYLON.Tools.Warn("Animation channel target node (" + channel.target.node + ") does not exist");
                     return;
@@ -775,7 +771,7 @@ var BABYLON;
                             return value;
                         },
                         "influence": function () {
-                            var numTargets = targetNode.morphTargetManager.numTargets;
+                            var numTargets = targetNode.babylonMesh.morphTargetManager.numTargets;
                             var value = new Array(numTargets);
                             for (var i = 0; i < numTargets; i++) {
                                 value[i] = outputData[outputBufferOffset++];
@@ -801,9 +797,9 @@ var BABYLON;
                     }
                     animation.targets = animation.targets || [];
                     if (targetPath === "influence") {
-                        var targetMesh = targetNode;
-                        for (var targetIndex = 0; targetIndex < targetMesh.morphTargetManager.numTargets; targetIndex++) {
-                            var morphTarget = targetMesh.morphTargetManager.getTarget(targetIndex);
+                        var morphTargetManager = targetNode.babylonMesh.morphTargetManager;
+                        for (var targetIndex = 0; targetIndex < morphTargetManager.numTargets; targetIndex++) {
+                            var morphTarget = morphTargetManager.getTarget(targetIndex);
                             var animationName = (animation.name || "anim" + animationIndex) + "_" + targetIndex;
                             var babylonAnimation = new BABYLON.Animation(animationName, targetPath, 1, animationType);
                             babylonAnimation.setKeys(keys.map(function (key) { return ({
@@ -820,8 +816,11 @@ var BABYLON;
                         var animationName = animation.name || "anim" + animationIndex;
                         var babylonAnimation = new BABYLON.Animation(animationName, targetPath, 1, animationType);
                         babylonAnimation.setKeys(keys);
-                        targetNode.animations.push(babylonAnimation);
-                        animation.targets.push(targetNode);
+                        for (var i = 0; i < targetNode.babylonAnimationTargets.length; i++) {
+                            var target = targetNode.babylonAnimationTargets[i];
+                            target.animations.push(babylonAnimation.clone());
+                            animation.targets.push(target);
+                        }
                     }
                 };
                 this._loadAccessorAsync(this._gltf.accessors[sampler.input], function (data) {

dist/preview release/loaders/babylon.glTFFileLoader.d.ts

@@ -709,7 +709,11 @@ declare module BABYLON.GLTF2 {
         translation?: number[];
         weights?: number[];
         index?: number;
-        babylonNode?: Node;
+        babylonMesh?: Mesh;
+        babylonSkinToBones?: {
+            [skin: number]: Bone;
+        };
+        babylonAnimationTargets?: Node[];
     }
     interface IGLTFSampler extends IGLTFChildRootProperty {
         magFilter?: ETextureMagFilter;
@@ -724,6 +728,7 @@ declare module BABYLON.GLTF2 {
         inverseBindMatrices?: number;
         skeleton?: number;
         joints: number[];
+        index?: number;
         babylonSkeleton?: Skeleton;
     }
     interface IGLTFTexture extends IGLTFChildRootProperty {
@@ -787,6 +792,7 @@ declare module BABYLON.GLTF2 {
         private _loadScene(nodeNames);
         private _loadSkin(node);
         private _updateBone(node, parentNode, skin, inverseBindMatrixData);
+        private _createBone(node, skin);
         private _loadMesh(node, parentNode);
         private _loadMeshData(node, mesh, babylonMesh);
         private _loadVertexDataAsync(primitive, onSuccess);

dist/preview release/loaders/babylon.glTFFileLoader.js

@@ -2472,8 +2472,8 @@ var BABYLON;
                     if (_this._gltf.nodes) {
                         for (var i = 0; i < _this._gltf.nodes.length; i++) {
                             var node = _this._gltf.nodes[i];
-                            if (node.babylonNode instanceof BABYLON.AbstractMesh) {
-                                meshes.push(node.babylonNode);
+                            if (node.babylonMesh) {
+                                meshes.push(node.babylonMesh);
                             }
                         }
                     }
@@ -2539,8 +2539,8 @@ var BABYLON;
                 var nodes = this._gltf.nodes;
                 for (var i = 0; i < nodes.length; i++) {
                     var node = nodes[i];
-                    if (node.babylonNode instanceof BABYLON.Mesh) {
-                        node.babylonNode.isVisible = true;
+                    if (node.babylonMesh) {
+                        node.babylonMesh.isVisible = true;
                     }
                 }
             };
@@ -2584,17 +2584,13 @@ var BABYLON;
             };
             GLTFLoader.prototype._loadSkin = function (node) {
                 var _this = this;
-                if (node.babylonNode) {
-                    return false;
-                }
                 if (node.skin !== undefined) {
                     var skin = this._gltf.skins[node.skin];
                     var skeletonId = "skeleton" + node.skin;
                     skin.babylonSkeleton = new BABYLON.Skeleton(skin.name || skeletonId, skeletonId, this._babylonScene);
+                    skin.index = node.skin;
                     for (var i = 0; i < skin.joints.length; i++) {
-                        var jointIndex = skin.joints[i];
-                        var jointNode = this._gltf.nodes[jointIndex];
-                        jointNode.babylonNode = new BABYLON.Bone(jointNode.name || "bone" + jointIndex, skin.babylonSkeleton);
+                        this._createBone(this._gltf.nodes[skin.joints[i]], skin);
                     }
                     if (skin.skeleton === undefined) {
                         // TODO: handle when skeleton is not defined
@@ -2614,30 +2610,28 @@ var BABYLON;
             GLTFLoader.prototype._updateBone = function (node, parentNode, skin, inverseBindMatrixData) {
                 var jointIndex = skin.joints.indexOf(node.index);
                 if (jointIndex === -1) {
-                    // TODO: handle non-joint in between two joints
-                    throw new Error("Not implemented");
+                    this._createBone(node, skin);
                 }
-                var babylonBone = node.babylonNode;
+                var babylonBone = node.babylonSkinToBones[skin.index];
                 // TODO: explain the math
-                var matrix = BABYLON.Matrix.FromArray(inverseBindMatrixData, jointIndex * 16);
+                var matrix = jointIndex === -1 ? BABYLON.Matrix.Identity() : BABYLON.Matrix.FromArray(inverseBindMatrixData, jointIndex * 16);
                 matrix.invertToRef(matrix);
                 if (parentNode) {
-                    babylonBone.setParent(parentNode.babylonNode, false);
+                    babylonBone.setParent(parentNode.babylonSkinToBones[skin.index], false);
                     matrix.multiplyToRef(babylonBone.getParent().getInvertedAbsoluteTransform(), matrix);
                 }
                 babylonBone.updateMatrix(matrix);
                 return true;
             };
+            GLTFLoader.prototype._createBone = function (node, skin) {
+                var babylonBone = new BABYLON.Bone(node.name || "bone" + node.index, skin.babylonSkeleton);
+                node.babylonSkinToBones = node.babylonSkinToBones || {};
+                node.babylonSkinToBones[skin.index] = babylonBone;
+                node.babylonAnimationTargets = node.babylonAnimationTargets || [];
+                node.babylonAnimationTargets.push(babylonBone);
+                return babylonBone;
+            };
             GLTFLoader.prototype._loadMesh = function (node, parentNode) {
-                if (node.babylonNode) {
-                    if (node.babylonNode instanceof BABYLON.Bone) {
-                        if (node.mesh !== undefined) {
-                            // TODO: handle mesh attached to bone
-                            throw new Error("Not implemented");
-                        }
-                    }
-                    return false;
-                }
                 var babylonMesh = new BABYLON.Mesh(node.name || "mesh" + node.index, this._babylonScene);
                 babylonMesh.isVisible = false;
                 this._loadTransform(node, babylonMesh);
@@ -2645,8 +2639,10 @@ var BABYLON;
                     var mesh = this._gltf.meshes[node.mesh];
                     this._loadMeshData(node, mesh, babylonMesh);
                 }
-                babylonMesh.parent = parentNode ? parentNode.babylonNode : null;
-                node.babylonNode = babylonMesh;
+                babylonMesh.parent = parentNode ? parentNode.babylonMesh : null;
+                node.babylonMesh = babylonMesh;
+                node.babylonAnimationTargets = node.babylonAnimationTargets || [];
+                node.babylonAnimationTargets.push(node.babylonMesh);
                 if (node.skin !== undefined) {
                     var skin = this._gltf.skins[node.skin];
                     babylonMesh.skeleton = skin.babylonSkeleton;
@@ -2883,7 +2879,7 @@ var BABYLON;
                 var channel = animation.channels[channelIndex];
                 var samplerIndex = channel.sampler;
                 var sampler = animation.samplers[samplerIndex];
-                var targetNode = this._gltf.nodes[channel.target.node].babylonNode;
+                var targetNode = this._gltf.nodes[channel.target.node];
                 if (!targetNode) {
                     BABYLON.Tools.Warn("Animation channel target node (" + channel.target.node + ") does not exist");
                     return;
@@ -2928,7 +2924,7 @@ var BABYLON;
                             return value;
                         },
                         "influence": function () {
-                            var numTargets = targetNode.morphTargetManager.numTargets;
+                            var numTargets = targetNode.babylonMesh.morphTargetManager.numTargets;
                             var value = new Array(numTargets);
                             for (var i = 0; i < numTargets; i++) {
                                 value[i] = outputData[outputBufferOffset++];
@@ -2954,9 +2950,9 @@ var BABYLON;
                     }
                     animation.targets = animation.targets || [];
                     if (targetPath === "influence") {
-                        var targetMesh = targetNode;
-                        for (var targetIndex = 0; targetIndex < targetMesh.morphTargetManager.numTargets; targetIndex++) {
-                            var morphTarget = targetMesh.morphTargetManager.getTarget(targetIndex);
+                        var morphTargetManager = targetNode.babylonMesh.morphTargetManager;
+                        for (var targetIndex = 0; targetIndex < morphTargetManager.numTargets; targetIndex++) {
+                            var morphTarget = morphTargetManager.getTarget(targetIndex);
                             var animationName = (animation.name || "anim" + animationIndex) + "_" + targetIndex;
                             var babylonAnimation = new BABYLON.Animation(animationName, targetPath, 1, animationType);
                             babylonAnimation.setKeys(keys.map(function (key) { return ({
@@ -2973,8 +2969,11 @@ var BABYLON;
                         var animationName = animation.name || "anim" + animationIndex;
                         var babylonAnimation = new BABYLON.Animation(animationName, targetPath, 1, animationType);
                         babylonAnimation.setKeys(keys);
-                        targetNode.animations.push(babylonAnimation);
-                        animation.targets.push(targetNode);
+                        for (var i = 0; i < targetNode.babylonAnimationTargets.length; i++) {
+                            var target = targetNode.babylonAnimationTargets[i];
+                            target.animations.push(babylonAnimation.clone());
+                            animation.targets.push(target);
+                        }
                     }
                 };
                 this._loadAccessorAsync(this._gltf.accessors[sampler.input], function (data) {

src/babylon.scene.ts

@@ -3059,6 +3059,10 @@
 
             if (listeningCamera && audioEngine.canUseWebAudio) {
                 audioEngine.audioContext.listener.setPosition(listeningCamera.position.x, listeningCamera.position.y, listeningCamera.position.z);
+                // for VR cameras
+                if (listeningCamera.rigCameras) {
+                    listeningCamera = listeningCamera.rigCameras[0];
+                }
                 var mat = Matrix.Invert(listeningCamera.getViewMatrix());
                 var cameraDirection = Vector3.TransformNormal(new Vector3(0, 0, -1), mat);
                 cameraDirection.normalize();