Add suport for soft transparent shadows

dist/preview release/what's new.md

@@ -17,6 +17,7 @@
 - Added support for `material.disableColorWrite` ([Deltakosh](https://github.com/deltakosh))
 - The Mesh Asset Task also accepts File as sceneInput ([RaananW](https://github.com/RaananW))
 - Added support preserving vert colors for CSG objects ([PirateJC](https://github.com/PirateJC))
+- Added support in `ShadowGenerator` for fast fake soft transparent shadows ([Popov72](https://github.com/Popov72))
 
 ### Engine
 

materialsLibrary/src/custom/customMaterial.ts

@@ -159,9 +159,12 @@ export class CustomMaterial extends StandardMaterial {
             .replace('#define CUSTOM_FRAGMENT_UPDATE_DIFFUSE', (this.CustomParts.Fragment_Custom_Diffuse ? this.CustomParts.Fragment_Custom_Diffuse : ""))
             .replace('#define CUSTOM_FRAGMENT_UPDATE_ALPHA', (this.CustomParts.Fragment_Custom_Alpha ? this.CustomParts.Fragment_Custom_Alpha : ""))
             .replace('#define CUSTOM_FRAGMENT_BEFORE_LIGHTS', (this.CustomParts.Fragment_Before_Lights ? this.CustomParts.Fragment_Before_Lights : ""))
-            .replace('#define CUSTOM_FRAGMENT_BEFORE_FOG', (this.CustomParts.Fragment_Before_Fog ? this.CustomParts.Fragment_Before_Fog : ""))
             .replace('#define CUSTOM_FRAGMENT_BEFORE_FRAGCOLOR', (this.CustomParts.Fragment_Before_FragColor ? this.CustomParts.Fragment_Before_FragColor : ""));
 
+        if (this.CustomParts.Fragment_Before_Fog) {
+            Effect.ShadersStore[name + "PixelShader"] = Effect.ShadersStore[name + "PixelShader"].replace('#define CUSTOM_FRAGMENT_BEFORE_FOG', this.CustomParts.Fragment_Before_Fog);
+        }
+
         this._isCreatedShader = true;
         this._createdShaderName = name;
 

materialsLibrary/src/custom/pbrCustomMaterial.ts

@@ -157,9 +157,12 @@ export class PBRCustomMaterial extends PBRMaterial {
             .replace('#define CUSTOM_FRAGMENT_BEFORE_LIGHTS', (this.CustomParts.Fragment_Before_Lights ? this.CustomParts.Fragment_Before_Lights : ""))
             .replace('#define CUSTOM_FRAGMENT_UPDATE_METALLICROUGHNESS', (this.CustomParts.Fragment_Custom_MetallicRoughness ? this.CustomParts.Fragment_Custom_MetallicRoughness : ""))
             .replace('#define CUSTOM_FRAGMENT_UPDATE_MICROSURFACE', (this.CustomParts.Fragment_Custom_MicroSurface ? this.CustomParts.Fragment_Custom_MicroSurface : ""))
-            .replace('#define CUSTOM_FRAGMENT_BEFORE_FOG', (this.CustomParts.Fragment_Before_Fog ? this.CustomParts.Fragment_Before_Fog : ""))
             .replace('#define CUSTOM_FRAGMENT_BEFORE_FRAGCOLOR', (this.CustomParts.Fragment_Before_FragColor ? this.CustomParts.Fragment_Before_FragColor : ""));
 
+        if (this.CustomParts.Fragment_Before_Fog) {
+            Effect.ShadersStore[name + "PixelShader"] = Effect.ShadersStore[name + "PixelShader"].replace('#define CUSTOM_FRAGMENT_BEFORE_FOG', this.CustomParts.Fragment_Before_Fog);
+        }
+
         this._isCreatedShader = true;
         this._createdShaderName = name;
 

src/Lights/Shadows/shadowGenerator.ts

@@ -23,6 +23,7 @@ import { Constants } from "../../Engines/constants";
 import "../../Shaders/shadowMap.fragment";
 import "../../Shaders/shadowMap.vertex";
 import "../../Shaders/depthBoxBlur.fragment";
+import "../../Shaders/ShadersInclude/shadowMapFragmentSoftTransparentShadow";
 import { Observable } from '../../Misc/observable';
 import { _DevTools } from '../../Misc/devTools';
 import { EffectFallbacks } from '../../Materials/effectFallbacks';
@@ -75,9 +76,10 @@ export interface IShadowGenerator {
      * Determine wheter the shadow generator is ready or not (mainly all effects and related post processes needs to be ready).
      * @param subMesh The submesh we want to render in the shadow map
      * @param useInstances Defines wether will draw in the map using instances
+     * @param isTransparent Indicates that isReady is called for a transparent subMesh
      * @returns true if ready otherwise, false
      */
-    isReady(subMesh: SubMesh, useInstances: boolean): boolean;
+    isReady(subMesh: SubMesh, useInstances: boolean, isTransparent: boolean): boolean;
 
     /**
      * Prepare all the defines in a material relying on a shadow map at the specified light index.
@@ -661,6 +663,15 @@ export class ShadowGenerator implements IShadowGenerator {
         return this;
     }
 
+    /**
+     * Enables or disables shadows with varying strength based on the transparency
+     * When it is enabled, the strength of the shadow is taken equal to mesh.visibility
+     * If you enabled an alpha texture on your material, the alpha value red from the texture is also combined to compute the strength:
+     *          mesh.visibility * alphaTexture.a
+     * Note that by definition transparencyShadow must be set to true for enableSoftTransparentShadow to work!
+     */
+    public enableSoftTransparentShadow: boolean = false;
+
     protected _shadowMap: Nullable<RenderTargetTexture>;
     protected _shadowMap2: Nullable<RenderTargetTexture>;
 
@@ -1017,7 +1028,7 @@ export class ShadowGenerator implements IShadowGenerator {
 
         if (this._transparencyShadow) {
             for (index = 0; index < transparentSubMeshes.length; index++) {
-                this._renderSubMeshForShadowMap(transparentSubMeshes.data[index]);
+                this._renderSubMeshForShadowMap(transparentSubMeshes.data[index], true);
             }
         }
     }
@@ -1040,7 +1051,7 @@ export class ShadowGenerator implements IShadowGenerator {
         effect.setMatrix(matriceNames?.worldView ?? "worldView", tmpMatrix2);
     }
 
-    protected _renderSubMeshForShadowMap(subMesh: SubMesh): void {
+    protected _renderSubMeshForShadowMap(subMesh: SubMesh, isTransparent: boolean = false): void {
         var ownerMesh = subMesh.getMesh();
         var replacementMesh = ownerMesh._internalAbstractMeshDataInfo._actAsRegularMesh ? ownerMesh : null;
         var renderingMesh = subMesh.getRenderingMesh();
@@ -1065,7 +1076,7 @@ export class ShadowGenerator implements IShadowGenerator {
         }
 
         var hardwareInstancedRendering = (engine.getCaps().instancedArrays) && (batch.visibleInstances[subMesh._id] !== null) && (batch.visibleInstances[subMesh._id] !== undefined);
-        if (this.isReady(subMesh, hardwareInstancedRendering)) {
+        if (this.isReady(subMesh, hardwareInstancedRendering, isTransparent)) {
             const shadowDepthWrapper = renderingMesh.material?.shadowDepthWrapper;
 
             let effect = shadowDepthWrapper?.getEffect(subMesh, this) ?? this._effect;
@@ -1089,6 +1100,10 @@ export class ShadowGenerator implements IShadowGenerator {
                 effect.setFloat2("depthValuesSM", this.getLight().getDepthMinZ(scene.activeCamera), this.getLight().getDepthMinZ(scene.activeCamera) + this.getLight().getDepthMaxZ(scene.activeCamera));
             }
 
+            if (isTransparent && this.enableSoftTransparentShadow) {
+                effect.setFloat("softTransparentShadowSM", effectiveMesh.visibility);
+            }
+
             if (shadowDepthWrapper) {
                 subMesh._effectOverride = effect;
                 if (shadowDepthWrapper.standalone) {
@@ -1219,7 +1234,7 @@ export class ShadowGenerator implements IShadowGenerator {
                 return;
             }
 
-            while (this.isReady(subMeshes[currentIndex], localOptions.useInstances)) {
+            while (this.isReady(subMeshes[currentIndex], localOptions.useInstances, subMeshes[currentIndex].getMaterial()?.needAlphaBlendingForMesh(subMeshes[currentIndex].getMesh()) ?? false)) {
                 currentIndex++;
                 if (currentIndex >= subMeshes.length) {
                     if (onCompiled) {
@@ -1250,7 +1265,7 @@ export class ShadowGenerator implements IShadowGenerator {
     protected _isReadyCustomDefines(defines: any, subMesh: SubMesh, useInstances: boolean): void {
     }
 
-    private _prepareShadowDefines(subMesh: SubMesh, useInstances: boolean, defines: string[]): string[] {
+    private _prepareShadowDefines(subMesh: SubMesh, useInstances: boolean, defines: string[], isTransparent: boolean): string[] {
         defines.push("#define SM_FLOAT " + (this._textureType !== Constants.TEXTURETYPE_UNSIGNED_INT ? "1" : "0"));
 
         defines.push("#define SM_ESM " + (this.useExponentialShadowMap || this.useBlurExponentialShadowMap ? "1" : "0"));
@@ -1266,6 +1281,9 @@ export class ShadowGenerator implements IShadowGenerator {
         // Point light
         defines.push("#define SM_USEDISTANCE " + (this._light.needCube() ? "1" : "0"));
 
+        // Soft transparent shadows
+        defines.push("#define SM_SOFTTRANSPARENTSHADOW " + (this.enableSoftTransparentShadow && isTransparent ? "1" : "0"));
+
         this._isReadyCustomDefines(defines, subMesh, useInstances);
 
         return defines;
@@ -1275,15 +1293,16 @@ export class ShadowGenerator implements IShadowGenerator {
      * Determine wheter the shadow generator is ready or not (mainly all effects and related post processes needs to be ready).
      * @param subMesh The submesh we want to render in the shadow map
      * @param useInstances Defines wether will draw in the map using instances
+     * @param isTransparent Indicates that isReady is called for a transparent subMesh
      * @returns true if ready otherwise, false
      */
-    public isReady(subMesh: SubMesh, useInstances: boolean): boolean {
+    public isReady(subMesh: SubMesh, useInstances: boolean, isTransparent: boolean): boolean {
         const material = subMesh.getMaterial(),
               shadowDepthWrapper = material?.shadowDepthWrapper;
 
         const defines: string[] = [];
 
-        this._prepareShadowDefines(subMesh, useInstances, defines);
+        this._prepareShadowDefines(subMesh, useInstances, defines, isTransparent);
 
         if (shadowDepthWrapper) {
             if (!shadowDepthWrapper.isReadyForSubMesh(subMesh, defines, this, useInstances)) {
@@ -1402,7 +1421,7 @@ export class ShadowGenerator implements IShadowGenerator {
 
                 let shaderName = "shadowMap";
                 let uniforms = ["world", "mBones", "viewProjection", "diffuseMatrix", "lightDataSM", "depthValuesSM", "biasAndScaleSM", "morphTargetInfluences", "boneTextureWidth",
-                                "vClipPlane", "vClipPlane2", "vClipPlane3", "vClipPlane4", "vClipPlane5", "vClipPlane6"];
+                                "vClipPlane", "vClipPlane2", "vClipPlane3", "vClipPlane4", "vClipPlane5", "vClipPlane6", "softTransparentShadowSM"];
                 let samplers = ["diffuseSampler", "boneSampler"];
 
                 // Custom shader?

src/Materials/shadowDepthWrapper.ts

@@ -203,6 +203,7 @@ export class ShadowDepthWrapper {
 
         const vertexNormalBiasCode = this._options && this._options.remappedVariables ? `#include<shadowMapVertexNormalBias>(${this._options.remappedVariables.join(",")})` : Effect.IncludesShadersStore["shadowMapVertexNormalBias"],
               vertexMetricCode = this._options && this._options.remappedVariables ? `#include<shadowMapVertexMetric>(${this._options.remappedVariables.join(",")})` : Effect.IncludesShadersStore["shadowMapVertexMetric"],
+              fragmentSoftTransparentShadow = this._options && this._options.remappedVariables ? `#include<shadowMapFragmentSoftTransparentShadow>(${this._options.remappedVariables.join(",")})` : Effect.IncludesShadersStore["shadowMapFragmentSoftTransparentShadow"],
               fragmentBlockCode = Effect.IncludesShadersStore["shadowMapFragment"];
 
         vertexCode = vertexCode.replace(/void\s+?main/g, Effect.IncludesShadersStore["shadowMapVertexDeclaration"] + "\r\nvoid main");
@@ -216,6 +217,7 @@ export class ShadowDepthWrapper {
         vertexCode = vertexCode.replace(/#define SHADER_NAME.*?\n|out vec4 glFragColor;\n/g, "");
 
         fragmentCode = fragmentCode.replace(/void\s+?main/g, Effect.IncludesShadersStore["shadowMapFragmentDeclaration"] + "\r\nvoid main");
+        fragmentCode = fragmentCode.replace(/#define SHADOWDEPTH_SOFTTRANSPARENTSHADOW|#define CUSTOM_FRAGMENT_BEFORE_FOG/g, fragmentSoftTransparentShadow);
         if (fragmentCode.indexOf("#define SHADOWDEPTH_FRAGMENT") !== -1) {
             fragmentCode = fragmentCode.replace(/#define SHADOWDEPTH_FRAGMENT/g, fragmentBlockCode);
         } else {
@@ -225,7 +227,7 @@ export class ShadowDepthWrapper {
 
         const uniforms = origEffect.getUniformNames().slice();
 
-        uniforms.push("biasAndScaleSM", "depthValuesSM", "lightDataSM");
+        uniforms.push("biasAndScaleSM", "depthValuesSM", "lightDataSM", "softTransparentShadowSM");
 
         params.depthEffect = this._scene.getEngine().createEffect({
             vertexSource: vertexCode,

src/Meshes/mesh.ts

@@ -987,7 +987,7 @@ export class Mesh extends AbstractMesh implements IGetSetVerticesData {
 
             if (generator && (!generator.getShadowMap()?.renderList || generator.getShadowMap()?.renderList && generator.getShadowMap()?.renderList?.indexOf(this) !== -1)) {
                 for (var subMesh of this.subMeshes) {
-                    if (!generator.isReady(subMesh, hardwareInstancedRendering)) {
+                    if (!generator.isReady(subMesh, hardwareInstancedRendering, subMesh.getMaterial()?.needAlphaBlendingForMesh(this) ?? false)) {
                         return false;
                     }
                 }

src/Shaders/ShadersInclude/bayerDitherFunctions.fx

@@ -0,0 +1,25 @@
+// from https://www.shadertoy.com/view/Mlt3z8
+
+// Generates the basic 2x2 Bayer permutation matrix:
+//  [1 2]
+//  [3 0]
+// Expects _P in [0,1]
+float bayerDither2(vec2 _P) {
+    return mod(2.0 * _P.y + _P.x + 1.0, 4.0);
+}
+
+// Generates the 4x4 matrix
+// Expects _P any pixel coordinate
+float bayerDither4(vec2 _P) {
+    vec2 P1 = mod(_P, 2.0);              // (P >> 0) & 1
+    vec2 P2 = floor(0.5 * mod(_P, 4.0)); // (P >> 1) & 1
+    return 4.0 * bayerDither2(P1) + bayerDither2(P2);
+}
+
+// Generates the 8x8 matrix
+float bayerDither8(vec2 _P) {
+    vec2 P1 = mod(_P, 2.0);	              // (P >> 0) & 1
+    vec2 P2 = floor(0.5  * mod(_P, 4.0)); // (P >> 1) & 1
+    vec2 P4 = floor(0.25 * mod(_P, 8.0)); // (P >> 2) & 1
+    return 4.0 * (4.0 * bayerDither2(P1) + bayerDither2(P2)) + bayerDither2(P4);
+}

src/Shaders/ShadersInclude/shadowMapFragmentDeclaration.fx

@@ -2,6 +2,12 @@
 	#include<packingFunctions>
 #endif
 
+#if SM_SOFTTRANSPARENTSHADOW == 1
+	#include<bayerDitherFunctions>
+
+    uniform float softTransparentShadowSM;
+#endif
+
 varying float vDepthMetricSM;
 
 #if SM_USEDISTANCE == 1

src/Shaders/ShadersInclude/shadowMapFragmentSoftTransparentShadow.fx

@@ -0,0 +1,3 @@
+#if SM_SOFTTRANSPARENTSHADOW == 1
+    if ((bayerDither8(floor(mod(gl_FragCoord.xy, 8.0)))) / 64.0 >= softTransparentShadowSM * alpha) discard;
+#endif

src/Shaders/shadowMap.fragment.fx

@@ -12,9 +12,18 @@ void main(void)
 #include<clipPlaneFragment>
 
 #ifdef ALPHATEST
-    if (texture2D(diffuseSampler, vUV).a < 0.4)
+    float alphaFromAlphaTexture = texture2D(diffuseSampler, vUV).a;
+    if (alphaFromAlphaTexture < 0.4)
         discard;
 #endif
 
+#if SM_SOFTTRANSPARENTSHADOW == 1
+    #ifdef ALPHATEST
+        if ((bayerDither8(floor(mod(gl_FragCoord.xy, 8.0)))) / 64.0 >= softTransparentShadowSM * alphaFromAlphaTexture) discard;
+    #else
+        if ((bayerDither8(floor(mod(gl_FragCoord.xy, 8.0)))) / 64.0 >= softTransparentShadowSM) discard;
+    #endif
+#endif
+
 #include<shadowMapFragment>
 }