Add a shadowDepthMaterial property to Material

src/Lights/Shadows/shadowGenerator.ts

@@ -1051,17 +1051,9 @@ export class ShadowGenerator implements IShadowGenerator {
 
         var hardwareInstancedRendering = (engine.getCaps().instancedArrays) && (batch.visibleInstances[subMesh._id] !== null) && (batch.visibleInstances[subMesh._id] !== undefined);
         if (this.isReady(subMesh, hardwareInstancedRendering)) {
-            const customShadowDepthMat = renderingMesh.material?.customShadowDepthMaterial;
+            const shadowDepthMat = renderingMesh.material?.shadowDepthMaterial;
 
-            if (customShadowDepthMat) {
-                subMesh.switchToShadowDepthEffect();
-            }
-
-            let effect = customShadowDepthMat ? subMesh.effect! : this._effect;
-
-            if (!effect && customShadowDepthMat) {
-                effect = customShadowDepthMat.getEffect()!; // ShaderMaterial case where the effect is not stored on the sub mesh
-            }
+            let effect = shadowDepthMat?.getEffect(subMesh) ?? this._effect;
 
             engine.enableEffect(effect);
 
@@ -1080,8 +1072,8 @@ export class ShadowGenerator implements IShadowGenerator {
                 effect.setFloat2("depthValuesSM", this.getLight().getDepthMinZ(scene.activeCamera), this.getLight().getDepthMinZ(scene.activeCamera) + this.getLight().getDepthMaxZ(scene.activeCamera));
             }
 
-            if (customShadowDepthMat) {
-                customShadowDepthMat.bindForSubMesh(effectiveMesh.getWorldMatrix(), renderingMesh, subMesh);
+            if (shadowDepthMat) {
+                material.bindForSubMesh(effectiveMesh.getWorldMatrix(), renderingMesh, subMesh, effect);
             } else {
                 effect.setMatrix("viewProjection", this.getTransformMatrix());
                 // Alpha test
@@ -1139,10 +1131,6 @@ export class ShadowGenerator implements IShadowGenerator {
             // Observables
             this.onAfterShadowMapRenderObservable.notifyObservers(effect);
             this.onAfterShadowMapRenderMeshObservable.notifyObservers(renderingMesh);
-
-            if (customShadowDepthMat) {
-                subMesh.switchToRegularEffect();
-            }
         } else {
             // Need to reset refresh rate of the shadowMap
             if (this._shadowMap) {
@@ -1268,24 +1256,14 @@ export class ShadowGenerator implements IShadowGenerator {
      */
     public isReady(subMesh: SubMesh, useInstances: boolean): boolean {
         const material = subMesh.getMaterial(),
-              customShadowDepthMaterial = material?.customShadowDepthMaterial;
+              shadowDepthMaterial = material?.shadowDepthMaterial;
 
         const defines: string[] = [];
 
         this._prepareShadowDefines(subMesh, useInstances, defines);
 
-        if (material && customShadowDepthMaterial) {
-            material.customShadowDepthDefines = defines;
-
-            if (material.customShadowDepthUniforms.length === 0) {
-                material.customShadowDepthUniforms = ["biasAndScaleSM", "depthValuesSM", "lightDataSM"];
-            }
-
-            subMesh.switchToShadowDepthEffect();
-
-            const isReady = customShadowDepthMaterial.isReadyForSubMesh(subMesh.getRenderingMesh(), subMesh, useInstances);
-
-            subMesh.switchToRegularEffect();
+        if (material && shadowDepthMaterial) {
+            const isReady = shadowDepthMaterial.isReadyForSubMesh(subMesh, defines);
 
             if (!isReady) {
                 return false;

src/Materials/index.ts

@@ -6,7 +6,6 @@ export * from "./effect";
 export * from "./fresnelParameters";
 export * from "./imageProcessingConfiguration";
 export * from "./material";
-export * from "./material.shadowDepth";
 export * from "./materialDefines";
 export * from "./materialHelper";
 export * from "./multiMaterial";
@@ -18,4 +17,5 @@ export * from "./Textures/index";
 export * from "./uniformBuffer";
 export * from "./materialFlags";
 export * from "./Node/index";
-export * from "./effectRenderer";
+export * from "./effectRenderer";
+export * from "./shadowDepthMaterial";

src/Materials/material.ts

@@ -19,6 +19,7 @@ import { Constants } from "../Engines/constants";
 import { Logger } from "../Misc/logger";
 import { IInspectable } from '../Misc/iInspectable';
 import { Plane } from '../Maths/math.plane';
+import { ShadowDepthMaterial } from './shadowDepthMaterial';
 
 declare type Mesh = import("../Meshes/mesh").Mesh;
 declare type Animation = import("../Animations/animation").Animation;
@@ -149,6 +150,11 @@ export class Material implements IAnimatable {
     public customShaderNameResolve: (shaderName: string, uniforms: string[], uniformBuffers: string[], samplers: string[], defines: MaterialDefines | string[], attributes?: string[]) => string;
 
     /**
+     * Custom shadow depth material to use for shadow rendering instead of the in-built one
+     */
+    public shadowDepthMaterial: Nullable<ShadowDepthMaterial> = null;
+
+    /**
      * The ID of the material
      */
     @serialize()

src/Materials/shadowDepthMaterial.ts

@@ -0,0 +1,85 @@
+import { Observer } from "../Misc/observable";
+import { Nullable } from "../types";
+import { Scene } from "../scene";
+import { SubMesh } from "../Meshes/subMesh";
+import { Material } from "../Materials/material";
+import { _TypeStore } from "../Misc/typeStore";
+import { Effect, IEffectCreationOptions } from './effect';
+
+export class ShadowDepthMaterial {
+
+    private _scene: Scene;
+    private _baseMaterial: Material;
+    private _onEffectCreatedObserver: Nullable<Observer<{ effect: Effect, subMesh: Nullable<SubMesh>}>>;
+    private _subMeshToEffect: Map<Nullable<SubMesh>, { origEffect: Effect, depthEffect: Nullable<Effect>, depthDefines: string }>;
+
+    constructor(baseMaterial: Material, scene: Scene) {
+        this._scene = scene;
+        this._baseMaterial = baseMaterial;
+        this._subMeshToEffect = new Map();
+
+        this._onEffectCreatedObserver = this._baseMaterial.onEffectCreatedObservable.add((params: { effect: Effect, subMesh: Nullable<SubMesh> }) => {
+            this._subMeshToEffect.set(params.subMesh, { origEffect: params.effect, depthEffect: null, depthDefines: "" });
+        });
+    }
+
+    public getEffect(subMesh: Nullable<SubMesh>): Nullable<Effect> {
+        return this._subMeshToEffect.get(subMesh)?.depthEffect ?? null;
+    }
+
+    public isReadyForSubMesh(subMesh: SubMesh, defines: string[]): boolean {
+        return this._makeEffect(subMesh, defines)?.isReady() ?? false;
+    }
+
+    public dispose(): void {
+        this._baseMaterial.onEffectCreatedObservable.remove(this._onEffectCreatedObserver);
+        this._onEffectCreatedObserver = null;
+    }
+
+    private _makeEffect(subMesh: Nullable<SubMesh>, defines: string[]): Nullable<Effect> {
+        const params = this._subMeshToEffect.get(subMesh);
+
+        if (!params) {
+            return null;
+        }
+
+        let join = defines.join("\n");
+
+        if (params.depthEffect) {
+            if (join === params.depthDefines) {
+                // we already created the depth effect and it is still up to date
+                return params.depthEffect;
+            }
+        }
+
+        // the depth effect is either out of date or has not been created yet
+        let vertexCode = params.origEffect.vertexSourceCode,
+            fragmentCode = params.origEffect.fragmentSourceCode;
+
+        vertexCode = vertexCode.replace(/void\s+?main/g, Effect.IncludesShadersStore["shadowMapVertexDeclaration"] + "\r\nvoid main");
+        vertexCode = vertexCode.replace(/}\s*$/g, Effect.IncludesShadersStore["shadowMapVertexMetric"] + "\r\n}");
+        vertexCode = vertexCode.replace(/#define SHADER_NAME.*?\n|out vec4 glFragColor;\n/g, "");
+
+        fragmentCode = fragmentCode.replace(/void\s+?main/g, Effect.IncludesShadersStore["shadowMapFragmentDeclaration"] + "\r\nvoid main");
+        fragmentCode = fragmentCode.replace(/}\s*$/g, Effect.IncludesShadersStore["shadowMapFragment"] + "\r\n}");
+        fragmentCode = fragmentCode.replace(/#define SHADER_NAME.*?\n|out vec4 glFragColor;\n/g, "");
+
+        const uniforms = params.origEffect.getUniformNames().slice();
+
+        uniforms.push("biasAndScaleSM", "depthValuesSM", "lightDataSM");
+
+        params.depthEffect = this._scene.getEngine().createEffect({
+            vertexSource: vertexCode,
+            fragmentSource: fragmentCode,
+        }, <IEffectCreationOptions>{
+            attributes: params.origEffect.getAttributesNames(),
+            uniformsNames: uniforms,
+            uniformBuffersNames: params.origEffect.getUniformBuffersNames(),
+            samplers: params.origEffect.getSamplers(),
+            defines: join,
+            indexParameters: params.origEffect.getIndexParameters(),
+        }, this._scene.getEngine());
+
+        return params.depthEffect;
+    }
+}