Use the custom shadow depth material if defined for a submesh material

src/Lights/Shadows/shadowGenerator.ts

@@ -885,10 +885,23 @@ export class ShadowGenerator implements IShadowGenerator {
             if (this._filter === ShadowGenerator.FILTER_PCF) {
                 engine.setColorWrite(false);
             }
+            if (this._scene.getSceneUniformBuffer().useUbo) {
+                const sceneUBO = this._scene.getSceneUniformBuffer();
+                sceneUBO.updateMatrix("viewProjection", this.getTransformMatrix());
+                sceneUBO.updateMatrix("view", this._viewMatrix);
+                sceneUBO.update();
+            }
         });
 
         // Blur if required afer render.
         this._shadowMap.onAfterUnbindObservable.add(() => {
+            if (this._scene.getSceneUniformBuffer().useUbo) {
+                const sceneUBO = this._scene.getSceneUniformBuffer();
+                sceneUBO.updateMatrix("viewProjection", this._scene.getTransformMatrix());
+                sceneUBO.updateMatrix("view", this._scene.getViewMatrix());
+                sceneUBO.update();
+            }
+
             if (this._filter === ShadowGenerator.FILTER_PCF) {
                 engine.setColorWrite(true);
             }
@@ -1033,57 +1046,70 @@ export class ShadowGenerator implements IShadowGenerator {
 
         var hardwareInstancedRendering = (engine.getCaps().instancedArrays) && (batch.visibleInstances[subMesh._id] !== null) && (batch.visibleInstances[subMesh._id] !== undefined);
         if (this.isReady(subMesh, hardwareInstancedRendering)) {
-            engine.enableEffect(this._effect);
-            renderingMesh._bind(subMesh, this._effect, material.fillMode);
+            const customShadowDepthMat = renderingMesh.material?.customShadowDepthMaterial;
+
+            if (customShadowDepthMat) {
+                subMesh.switchToShadowDepthEffect();
+            }
+
+            const effect = customShadowDepthMat ? subMesh.effect! : this._effect;
+
+            engine.enableEffect(effect);
 
-            this._effect.setFloat3("biasAndScale", this.bias, this.normalBias, this.depthScale);
+            renderingMesh._bind(subMesh, effect, material.fillMode);
+
+            effect.setFloat3("biasAndScaleSM", this.bias, this.normalBias, this.depthScale);
 
-            this._effect.setMatrix("viewProjection", this.getTransformMatrix());
             if (this.getLight().getTypeID() === Light.LIGHTTYPEID_DIRECTIONALLIGHT) {
-                this._effect.setVector3("lightData", this._cachedDirection);
+                effect.setVector3("lightDataSM", this._cachedDirection);
             }
             else {
-                this._effect.setVector3("lightData", this._cachedPosition);
+                effect.setVector3("lightDataSM", this._cachedPosition);
             }
 
             if (scene.activeCamera) {
-                this._effect.setFloat2("depthValues", this.getLight().getDepthMinZ(scene.activeCamera), this.getLight().getDepthMinZ(scene.activeCamera) + this.getLight().getDepthMaxZ(scene.activeCamera));
+                effect.setFloat2("depthValuesSM", this.getLight().getDepthMinZ(scene.activeCamera), this.getLight().getDepthMinZ(scene.activeCamera) + this.getLight().getDepthMaxZ(scene.activeCamera));
             }
 
-            // Alpha test
-            if (material && material.needAlphaTesting()) {
-                var alphaTexture = material.getAlphaTestTexture();
-                if (alphaTexture) {
-                    this._effect.setTexture("diffuseSampler", alphaTexture);
-                    this._effect.setMatrix("diffuseMatrix", alphaTexture.getTextureMatrix() || this._defaultTextureMatrix);
+            if (customShadowDepthMat) {
+                customShadowDepthMat.bindForSubMesh(effectiveMesh.getWorldMatrix(), renderingMesh, subMesh);
+            } else {
+                effect.setMatrix("viewProjection", this.getTransformMatrix());
+                // Alpha test
+                if (material && material.needAlphaTesting()) {
+                    var alphaTexture = material.getAlphaTestTexture();
+                    if (alphaTexture) {
+                        effect.setTexture("diffuseSampler", alphaTexture);
+                        effect.setMatrix("diffuseMatrix", alphaTexture.getTextureMatrix() || this._defaultTextureMatrix);
+                    }
                 }
-            }
 
-            // Bones
-            if (renderingMesh.useBones && renderingMesh.computeBonesUsingShaders && renderingMesh.skeleton) {
-                const skeleton = renderingMesh.skeleton;
+                // Bones
+                if (renderingMesh.useBones && renderingMesh.computeBonesUsingShaders && renderingMesh.skeleton) {
+                    const skeleton = renderingMesh.skeleton;
 
-                if (skeleton.isUsingTextureForMatrices) {
-                    const boneTexture = skeleton.getTransformMatrixTexture(renderingMesh);
+                    if (skeleton.isUsingTextureForMatrices) {
+                        const boneTexture = skeleton.getTransformMatrixTexture(renderingMesh);
 
-                    if (!boneTexture) {
-                        return;
-                    }
+                        if (!boneTexture) {
+                            return;
+                        }
 
-                    this._effect.setTexture("boneSampler", boneTexture);
-                    this._effect.setFloat("boneTextureWidth", 4.0 * (skeleton.bones.length + 1));
-                } else {
-                    this._effect.setMatrices("mBones", skeleton.getTransformMatrices((renderingMesh)));
+                        effect.setTexture("boneSampler", boneTexture);
+                        effect.setFloat("boneTextureWidth", 4.0 * (skeleton.bones.length + 1));
+                    } else {
+                        effect.setMatrices("mBones", skeleton.getTransformMatrices((renderingMesh)));
+                    }
                 }
-            }
 
-            // Morph targets
-            MaterialHelper.BindMorphTargetParameters(renderingMesh, this._effect);
+                // Morph targets
+                MaterialHelper.BindMorphTargetParameters(renderingMesh, effect);
 
-            // Clip planes
-            MaterialHelper.BindClipPlane(this._effect, scene);
+                // Clip planes
+                MaterialHelper.BindClipPlane(effect, scene);
 
-            this._bindCustomEffectForRenderSubMeshForShadowMap(subMesh, this._effect);
+                this._bindCustomEffectForRenderSubMeshForShadowMap(subMesh, effect);
+            }
 
             if (this.forceBackFacesOnly) {
                 engine.setState(true, 0, false, true);
@@ -1091,20 +1117,23 @@ export class ShadowGenerator implements IShadowGenerator {
 
             // Observables
             this.onBeforeShadowMapRenderMeshObservable.notifyObservers(renderingMesh);
-            this.onBeforeShadowMapRenderObservable.notifyObservers(this._effect);
+            this.onBeforeShadowMapRenderObservable.notifyObservers(effect);
 
             // Draw
-            renderingMesh._processRendering(effectiveMesh, subMesh, this._effect, material.fillMode, batch, hardwareInstancedRendering,
-                (isInstance, world) => this._effect.setMatrix("world", world));
+            renderingMesh._processRendering(effectiveMesh, subMesh, effect, material.fillMode, batch, hardwareInstancedRendering,
+                (isInstance, world) => effect.setMatrix("world", world));
 
             if (this.forceBackFacesOnly) {
                 engine.setState(true, 0, false, false);
             }
 
             // Observables
-            this.onAfterShadowMapRenderObservable.notifyObservers(this._effect);
+            this.onAfterShadowMapRenderObservable.notifyObservers(effect);
             this.onAfterShadowMapRenderMeshObservable.notifyObservers(renderingMesh);
 
+            if (customShadowDepthMat) {
+                subMesh.switchToRegularEffect();
+            }
         } else {
             // Need to reset refresh rate of the shadowMap
             if (this._shadowMap) {
@@ -1201,6 +1230,27 @@ export class ShadowGenerator implements IShadowGenerator {
     protected _isReadyCustomDefines(defines: any, subMesh: SubMesh, useInstances: boolean): void {
     }
 
+    private _prepareShadowDefines(subMesh: SubMesh, useInstances: boolean, defines: string[]): string[] {
+        defines.push("#define SM_FLOAT " + (this._textureType !== Constants.TEXTURETYPE_UNSIGNED_INT ? "1" : "0"));
+
+        defines.push("#define SM_ESM " + (this.useExponentialShadowMap || this.useBlurExponentialShadowMap ? "1" : "0"));
+
+        defines.push("#define SM_DEPTHTEXTURE " + (this.usePercentageCloserFiltering || this.useContactHardeningShadow ? "1" : "0"));
+
+        var mesh = subMesh.getMesh();
+
+        // Normal bias.
+        defines.push("#define SM_NORMALBIAS " + (this.normalBias && mesh.isVerticesDataPresent(VertexBuffer.NormalKind) ? "1" : "0"));
+        defines.push("#define SM_DIRECTIONINLIGHTDATA " + (this.getLight().getTypeID() === Light.LIGHTTYPEID_DIRECTIONALLIGHT ? "1" : "0"));
+
+        // Point light
+        defines.push("#define SM_USEDISTANCE " + (this._light.needCube() ? "1" : "0"));
+
+        this._isReadyCustomDefines(defines, subMesh, useInstances);
+
+        return defines;
+    }
+
     /**
      * Determine wheter the shadow generator is ready or not (mainly all effects and related post processes needs to be ready).
      * @param subMesh The submesh we want to render in the shadow map
@@ -1208,182 +1258,183 @@ export class ShadowGenerator implements IShadowGenerator {
      * @returns true if ready otherwise, false
      */
     public isReady(subMesh: SubMesh, useInstances: boolean): boolean {
-        var defines = [];
+        const material = subMesh.getMaterial(),
+              customShadowDepthMaterial = material?.customShadowDepthMaterial;
 
-        if (this._textureType !== Constants.TEXTURETYPE_UNSIGNED_INT) {
-            defines.push("#define FLOAT");
-        }
+        const defines: string[] = [];
 
-        if (this.useExponentialShadowMap || this.useBlurExponentialShadowMap) {
-            defines.push("#define ESM");
-        }
-        else if (this.usePercentageCloserFiltering || this.useContactHardeningShadow) {
-            defines.push("#define DEPTHTEXTURE");
-        }
+        this._prepareShadowDefines(subMesh, useInstances, defines);
 
-        var attribs = [VertexBuffer.PositionKind];
+        if (material && customShadowDepthMaterial) {
+            material.customShadowDepthDefines = defines;
 
-        var mesh = subMesh.getMesh();
-        var material = subMesh.getMaterial();
-
-        // Normal bias.
-        if (this.normalBias && mesh.isVerticesDataPresent(VertexBuffer.NormalKind)) {
-            attribs.push(VertexBuffer.NormalKind);
-            defines.push("#define NORMAL");
-            if (mesh.nonUniformScaling) {
-                defines.push("#define NONUNIFORMSCALING");
+            if (material.customShadowDepthUniforms.length === 0) {
+                material.customShadowDepthUniforms = ["biasAndScaleSM", "depthValuesSM", "lightDataSM"];
             }
-            if (this.getLight().getTypeID() === Light.LIGHTTYPEID_DIRECTIONALLIGHT) {
-                defines.push("#define DIRECTIONINLIGHTDATA");
+
+            subMesh.switchToShadowDepthEffect();
+
+            const isReady = customShadowDepthMaterial.isReadyForSubMesh(subMesh.getRenderingMesh(), subMesh, useInstances);
+
+            subMesh.switchToRegularEffect();
+
+            if (!isReady) {
+                return false;
             }
-        }
+        } else {
+            var attribs = [VertexBuffer.PositionKind];
+
+            var mesh = subMesh.getMesh();
 
-        // Alpha test
-        if (material && material.needAlphaTesting()) {
-            var alphaTexture = material.getAlphaTestTexture();
-            if (alphaTexture) {
-                defines.push("#define ALPHATEST");
-                if (mesh.isVerticesDataPresent(VertexBuffer.UVKind)) {
-                    attribs.push(VertexBuffer.UVKind);
-                    defines.push("#define UV1");
+            // Normal bias.
+            if (this.normalBias && mesh.isVerticesDataPresent(VertexBuffer.NormalKind)) {
+                attribs.push(VertexBuffer.NormalKind);
+                defines.push("#define NORMAL");
+                if (mesh.nonUniformScaling) {
+                    defines.push("#define NONUNIFORMSCALING");
                 }
-                if (mesh.isVerticesDataPresent(VertexBuffer.UV2Kind)) {
-                    if (alphaTexture.coordinatesIndex === 1) {
-                        attribs.push(VertexBuffer.UV2Kind);
-                        defines.push("#define UV2");
+            }
+
+            // Alpha test
+            if (material && material.needAlphaTesting()) {
+                var alphaTexture = material.getAlphaTestTexture();
+                if (alphaTexture) {
+                    defines.push("#define ALPHATEST");
+                    if (mesh.isVerticesDataPresent(VertexBuffer.UVKind)) {
+                        attribs.push(VertexBuffer.UVKind);
+                        defines.push("#define UV1");
+                    }
+                    if (mesh.isVerticesDataPresent(VertexBuffer.UV2Kind)) {
+                        if (alphaTexture.coordinatesIndex === 1) {
+                            attribs.push(VertexBuffer.UV2Kind);
+                            defines.push("#define UV2");
+                        }
                     }
                 }
             }
-        }
 
-        // Bones
-        const fallbacks = new EffectFallbacks();
-        if (mesh.useBones && mesh.computeBonesUsingShaders && mesh.skeleton) {
-            attribs.push(VertexBuffer.MatricesIndicesKind);
-            attribs.push(VertexBuffer.MatricesWeightsKind);
-            if (mesh.numBoneInfluencers > 4) {
-                attribs.push(VertexBuffer.MatricesIndicesExtraKind);
-                attribs.push(VertexBuffer.MatricesWeightsExtraKind);
-            }
-            const skeleton = mesh.skeleton;
-            defines.push("#define NUM_BONE_INFLUENCERS " + mesh.numBoneInfluencers);
-            if (mesh.numBoneInfluencers > 0) {
-                fallbacks.addCPUSkinningFallback(0, mesh);
-            }
+            // Bones
+            const fallbacks = new EffectFallbacks();
+            if (mesh.useBones && mesh.computeBonesUsingShaders && mesh.skeleton) {
+                attribs.push(VertexBuffer.MatricesIndicesKind);
+                attribs.push(VertexBuffer.MatricesWeightsKind);
+                if (mesh.numBoneInfluencers > 4) {
+                    attribs.push(VertexBuffer.MatricesIndicesExtraKind);
+                    attribs.push(VertexBuffer.MatricesWeightsExtraKind);
+                }
+                const skeleton = mesh.skeleton;
+                defines.push("#define NUM_BONE_INFLUENCERS " + mesh.numBoneInfluencers);
+                if (mesh.numBoneInfluencers > 0) {
+                    fallbacks.addCPUSkinningFallback(0, mesh);
+                }
+
+                if (skeleton.isUsingTextureForMatrices) {
+                    defines.push("#define BONETEXTURE");
+                } else {
+                    defines.push("#define BonesPerMesh " + (skeleton.bones.length + 1));
+                }
 
-            if (skeleton.isUsingTextureForMatrices) {
-                defines.push("#define BONETEXTURE");
             } else {
-                defines.push("#define BonesPerMesh " + (skeleton.bones.length + 1));
+                defines.push("#define NUM_BONE_INFLUENCERS 0");
             }
 
-        } else {
-            defines.push("#define NUM_BONE_INFLUENCERS 0");
-        }
-
-        // Morph targets
-        var manager = (<Mesh>mesh).morphTargetManager;
-        let morphInfluencers = 0;
-        if (manager) {
-            if (manager.numInfluencers > 0) {
-                defines.push("#define MORPHTARGETS");
-                morphInfluencers = manager.numInfluencers;
-                defines.push("#define NUM_MORPH_INFLUENCERS " + morphInfluencers);
-                MaterialHelper.PrepareAttributesForMorphTargetsInfluencers(attribs, mesh, morphInfluencers);
+            // Morph targets
+            var manager = (<Mesh>mesh).morphTargetManager;
+            let morphInfluencers = 0;
+            if (manager) {
+                if (manager.numInfluencers > 0) {
+                    defines.push("#define MORPHTARGETS");
+                    morphInfluencers = manager.numInfluencers;
+                    defines.push("#define NUM_MORPH_INFLUENCERS " + morphInfluencers);
+                    MaterialHelper.PrepareAttributesForMorphTargetsInfluencers(attribs, mesh, morphInfluencers);
+                }
             }
-        }
 
-        // ClipPlanes
-        const scene = this._scene;
-        if (scene.clipPlane) {
-            defines.push("#define CLIPPLANE");
-        }
-        if (scene.clipPlane2) {
-            defines.push("#define CLIPPLANE2");
-        }
-        if (scene.clipPlane3) {
-            defines.push("#define CLIPPLANE3");
-        }
-        if (scene.clipPlane4) {
-            defines.push("#define CLIPPLANE4");
-        }
-        if (scene.clipPlane5) {
-            defines.push("#define CLIPPLANE5");
-        }
-        if (scene.clipPlane6) {
-            defines.push("#define CLIPPLANE6");
-        }
+            // ClipPlanes
+            const scene = this._scene;
+            if (scene.clipPlane) {
+                defines.push("#define CLIPPLANE");
+            }
+            if (scene.clipPlane2) {
+                defines.push("#define CLIPPLANE2");
+            }
+            if (scene.clipPlane3) {
+                defines.push("#define CLIPPLANE3");
+            }
+            if (scene.clipPlane4) {
+                defines.push("#define CLIPPLANE4");
+            }
+            if (scene.clipPlane5) {
+                defines.push("#define CLIPPLANE5");
+            }
+            if (scene.clipPlane6) {
+                defines.push("#define CLIPPLANE6");
+            }
 
-        // Instances
-        if (useInstances) {
-            defines.push("#define INSTANCES");
-            MaterialHelper.PushAttributesForInstances(attribs);
-        }
+            // Instances
+            if (useInstances) {
+                defines.push("#define INSTANCES");
+                MaterialHelper.PushAttributesForInstances(attribs);
+            }
 
-        if (this.customShaderOptions) {
-            if (this.customShaderOptions.defines) {
-                for (var define of this.customShaderOptions.defines) {
-                    if (defines.indexOf(define) === -1) {
-                        defines.push(define);
+            if (this.customShaderOptions) {
+                if (this.customShaderOptions.defines) {
+                    for (var define of this.customShaderOptions.defines) {
+                        if (defines.indexOf(define) === -1) {
+                            defines.push(define);
+                        }
                     }
                 }
             }
-        }
-
-        // Point light
-        if (this._light.needCube()) {
-            defines.push("#define USEDISTANCE");
-        }
 
-        this._isReadyCustomDefines(defines, subMesh, useInstances);
-
-        // Get correct effect
-        var join = defines.join("\n");
-        if (this._cachedDefines !== join) {
-            this._cachedDefines = join;
-
-            let shaderName = "shadowMap";
-            let uniforms = ["world", "mBones", "viewProjection", "diffuseMatrix", "lightData", "depthValues", "biasAndScale", "morphTargetInfluences", "boneTextureWidth",
-                            "vClipPlane", "vClipPlane2", "vClipPlane3", "vClipPlane4", "vClipPlane5", "vClipPlane6"];
-            let samplers = ["diffuseSampler", "boneSampler"];
-
-            // Custom shader?
-            if (this.customShaderOptions) {
-                shaderName = this.customShaderOptions.shaderName;
-
-                if (this.customShaderOptions.attributes) {
-                    for (var attrib of this.customShaderOptions.attributes) {
-                        if (attribs.indexOf(attrib) === -1) {
-                            attribs.push(attrib);
+            // Get correct effect
+            var join = defines.join("\n");
+            if (this._cachedDefines !== join) {
+                this._cachedDefines = join;
+
+                let shaderName = "shadowMap";
+                let uniforms = ["world", "mBones", "viewProjection", "diffuseMatrix", "lightDataSM", "depthValuesSM", "biasAndScaleSM", "morphTargetInfluences", "boneTextureWidth",
+                                "vClipPlane", "vClipPlane2", "vClipPlane3", "vClipPlane4", "vClipPlane5", "vClipPlane6"];
+                let samplers = ["diffuseSampler", "boneSampler"];
+
+                // Custom shader?
+                if (this.customShaderOptions) {
+                    shaderName = this.customShaderOptions.shaderName;
+
+                    if (this.customShaderOptions.attributes) {
+                        for (var attrib of this.customShaderOptions.attributes) {
+                            if (attribs.indexOf(attrib) === -1) {
+                                attribs.push(attrib);
+                            }
                         }
                     }
-                }
 
-                if (this.customShaderOptions.uniforms) {
-                    for (var uniform of this.customShaderOptions.uniforms) {
-                        if (uniforms.indexOf(uniform) === -1) {
-                            uniforms.push(uniform);
+                    if (this.customShaderOptions.uniforms) {
+                        for (var uniform of this.customShaderOptions.uniforms) {
+                            if (uniforms.indexOf(uniform) === -1) {
+                                uniforms.push(uniform);
+                            }
                         }
                     }
-                }
 
-                if (this.customShaderOptions.samplers) {
-                    for (var sampler of this.customShaderOptions.samplers) {
-                        if (samplers.indexOf(sampler) === -1) {
-                            samplers.push(sampler);
+                    if (this.customShaderOptions.samplers) {
+                        for (var sampler of this.customShaderOptions.samplers) {
+                            if (samplers.indexOf(sampler) === -1) {
+                                samplers.push(sampler);
+                            }
                         }
                     }
                 }
-            }
 
-            this._effect = this._scene.getEngine().createEffect(shaderName,
-                attribs, uniforms,
-                samplers, join,
-                fallbacks, undefined, undefined, { maxSimultaneousMorphTargets: morphInfluencers });
-        }
+                this._effect = this._scene.getEngine().createEffect(shaderName,
+                    attribs, uniforms,
+                    samplers, join,
+                    fallbacks, undefined, undefined, { maxSimultaneousMorphTargets: morphInfluencers });
+            }
 
-        if (!this._effect.isReady()) {
-            return false;
+            if (!this._effect.isReady()) {
+                return false;
+            }
         }
 
         if (this.useBlurExponentialShadowMap || this.useBlurCloseExponentialShadowMap) {