Improved ESM on int textures

src/Lights/Shadows/babylon.shadowGenerator.ts

@@ -305,7 +305,11 @@
 
             this._shadowMap.onClearObservable.add((engine: Engine) => {
                 if (this.useExponentialShadowMap || this.useBlurExponentialShadowMap) {
-                    engine.clear(new Color4(0, 0, 0, 0), true, true, true);
+                    if (this._useFullFloat) {
+                        engine.clear(new Color4(0, 0, 0, 0), true, true, true);
+                    } else {
+                        engine.clear(new Color4(1.0, 1.0, 1.0, 1.0), true, true, true);
+                    }
                 } else {
                     engine.clear(new Color4(1.0, 1.0, 1.0, 1.0), true, true, true);
                 }

src/Shaders/ShadersInclude/shadowsFragmentFunctions.fx

@@ -148,14 +148,6 @@
 		return  min(1.0, visibility + darkness);
 	}
 
-	#ifndef SHADOWFULLFLOAT
-		// Thanks to http://devmaster.net/
-		float unpackHalf(vec2 color)
-		{
-			return color.x + (color.y / 255.0);
-		}
-	#endif
-
 	float computeShadowWithESM(vec4 vPositionFromLight, sampler2D shadowSampler, float darkness)
 	{
 		vec3 clipSpace = vPositionFromLight.xyz / vPositionFromLight.w;
@@ -168,21 +160,21 @@
 			return 1.0;
 		}
 	
+		const float shadowStrength = 30.;
 		#ifndef SHADOWFULLFLOAT
 			float shadowMapSample = unpack(texture2D(shadowSampler, uv));
+			float esm = clamp(exp(-shadowStrength * shadowPixelDepth) * shadowMapSample - darkness, 0., 1.);
 		#else
 			float shadowMapSample = texture2D(shadowSampler, uv).x;
+			float esm = 1.0 - clamp(exp(-shadowStrength * shadowPixelDepth) * shadowMapSample - darkness, 0., 1.);
 		#endif
 
-		const float shadowStrength = 5.;
-		float esm = 1.0 - clamp(exp(-shadowStrength * shadowPixelDepth) * shadowMapSample - darkness, 0., 1.);
-
 		// Apply fade out at frustum edge
-		const float fadeDistance = 0.07;
-		vec2 cs2 = clipSpace.xy * clipSpace.xy; //squarish falloff
-		float mask = smoothstep(1.0, 1.0 - fadeDistance, dot(cs2, cs2));
+		// const float fadeDistance = 0.07;
+		// vec2 cs2 = clipSpace.xy * clipSpace.xy; //squarish falloff
+		// float mask = smoothstep(1.0, 1.0 - fadeDistance, dot(cs2, cs2));
 
-		return mix(1.0, esm, mask);
+		// return mix(1.0, esm, mask);
 
 		return esm;
 	}

src/Shaders/shadowMap.fragment.fx

@@ -9,15 +9,6 @@ vec4 pack(float depth)
 
 	return res;
 }
-
-// Thanks to http://devmaster.net/
-vec2 packHalf(float depth) 
-{ 
-	const vec2 bitOffset = vec2(1.0 / 255., 0.);
-	vec2 color = vec2(depth, fract(depth * 255.));
-
-	return color - (color.yy * bitOffset);
-}
 #endif
 
 varying vec4 vPosition;
@@ -55,7 +46,7 @@ void main(void)
 	depth += bias;
 
 #ifdef ESM
-	const float shadowStrength = 5.0;
+	const float shadowStrength = 30.0;
 	depth = exp(shadowStrength * depth);
 #endif