Added frustum falloff for shadows

materialsLibrary/src/legacyPBR/legacyPbrLightFunctionsCall.fx

@@ -18,20 +18,20 @@
 			#if defined(SHADOWCUBE{X})
 				notShadowLevel = computeShadowWithESMCube(light{X}.vLightData.xyz, shadowSampler{X}, light{X}.shadowsInfo.x, light{X}.shadowsInfo.z, light{X}.depthValues);
 			#else
-				notShadowLevel = computeShadowWithESM(vPositionFromLight{X}, vDepthMetric{X}, shadowSampler{X}, light{X}.shadowsInfo.x, light{X}.shadowsInfo.z);
+				notShadowLevel = computeShadowWithESM(vPositionFromLight{X}, vDepthMetric{X}, shadowSampler{X}, light{X}.shadowsInfo.x, light{X}.shadowsInfo.z, 0.0);
 			#endif
         #else
             #ifdef SHADOWPCF{X}
                 #if defined(SHADOWCUBE{X})
                     notShadowLevel = computeShadowWithPCFCube(light{X}.vLightData.xyz, shadowSampler{X}, light{X}.shadowsInfo.y, light{X}.shadowsInfo.x, light{X}.depthValues);
                 #else
-                    notShadowLevel = computeShadowWithPCF(vPositionFromLight{X}, vDepthMetric{X}, shadowSampler{X}, light{X}.shadowsInfo.y, light{X}.shadowsInfo.x);
+                    notShadowLevel = computeShadowWithPCF(vPositionFromLight{X}, vDepthMetric{X}, shadowSampler{X}, light{X}.shadowsInfo.y, light{X}.shadowsInfo.x, 0.0);
                 #endif
             #else
                 #if defined(SHADOWCUBE{X})
                     notShadowLevel = computeShadowCube(light{X}.vLightData.xyz, shadowSampler{X}, light{X}.shadowsInfo.x, light{X}.depthValues);
                 #else
-                    notShadowLevel = computeShadow(vPositionFromLight{X}, vDepthMetric{X}, shadowSampler{X}, light{X}.shadowsInfo.x);
+                    notShadowLevel = computeShadow(vPositionFromLight{X}, vDepthMetric{X}, shadowSampler{X}, light{X}.shadowsInfo.x, 0.0);
                 #endif
             #endif
         #endif

src/Lights/Shadows/babylon.shadowGenerator.ts

@@ -251,6 +251,12 @@
             return this._shadowMap;
         }
 
+        /**
+		 * Controls the extent to which the shadows fade out at the edge of the frustum
+         * Used only by directionals and spots
+		 */
+        public frustumEdgeFalloff = 0;        
+
         private _light: IShadowLight;
         /**
          * Returns the associated light object.  
@@ -620,7 +626,7 @@
                 effect.setMatrix("lightMatrix" + lightIndex, this.getTransformMatrix());
             } 
             effect.setTexture("shadowSampler" + lightIndex, this.getShadowMapForRendering());
-            light._uniformBuffer.updateFloat3("shadowsInfo", this.getDarkness(), this.blurScale / this.getShadowMap().getSize().width, this.depthScale, lightIndex);
+            light._uniformBuffer.updateFloat4("shadowsInfo", this.getDarkness(), this.blurScale / this.getShadowMap().getSize().width, this.depthScale, this.frustumEdgeFalloff, lightIndex);
             light._uniformBuffer.updateFloat2("depthValues", this.getLight().getDepthMinZ(scene.activeCamera), this.getLight().getDepthMinZ(scene.activeCamera) + this.getLight().getDepthMaxZ(scene.activeCamera), lightIndex);
         }
 

src/Shaders/ShadersInclude/lightFragment.fx

@@ -29,27 +29,27 @@
 			#if defined(SHADOWCUBE{X})
 				shadow = computeShadowWithCloseESMCube(light{X}.vLightData.xyz, shadowSampler{X}, light{X}.shadowsInfo.x, light{X}.shadowsInfo.z, light{X}.depthValues);
 			#else
-				shadow = computeShadowWithCloseESM(vPositionFromLight{X}, vDepthMetric{X}, shadowSampler{X}, light{X}.shadowsInfo.x, light{X}.shadowsInfo.z);
+				shadow = computeShadowWithCloseESM(vPositionFromLight{X}, vDepthMetric{X}, shadowSampler{X}, light{X}.shadowsInfo.x, light{X}.shadowsInfo.z, light{X}.shadowsInfo.w);
 			#endif
 		#else
 			#ifdef SHADOWESM{X}
 				#if defined(SHADOWCUBE{X})
 					shadow = computeShadowWithESMCube(light{X}.vLightData.xyz, shadowSampler{X}, light{X}.shadowsInfo.x, light{X}.shadowsInfo.z, light{X}.depthValues);
 				#else
-					shadow = computeShadowWithESM(vPositionFromLight{X}, vDepthMetric{X}, shadowSampler{X}, light{X}.shadowsInfo.x, light{X}.shadowsInfo.z);
+					shadow = computeShadowWithESM(vPositionFromLight{X}, vDepthMetric{X}, shadowSampler{X}, light{X}.shadowsInfo.x, light{X}.shadowsInfo.z, light{X}.shadowsInfo.w);
 				#endif
 			#else	
 				#ifdef SHADOWPCF{X}
 					#if defined(SHADOWCUBE{X})
 						shadow = computeShadowWithPCFCube(light{X}.vLightData.xyz, shadowSampler{X}, light{X}.shadowsInfo.y, light{X}.shadowsInfo.x, light{X}.depthValues);
 					#else
-						shadow = computeShadowWithPCF(vPositionFromLight{X}, vDepthMetric{X}, shadowSampler{X}, light{X}.shadowsInfo.y, light{X}.shadowsInfo.x);
+						shadow = computeShadowWithPCF(vPositionFromLight{X}, vDepthMetric{X}, shadowSampler{X}, light{X}.shadowsInfo.y, light{X}.shadowsInfo.x, light{X}.shadowsInfo.w);
 					#endif
 				#else
 					#if defined(SHADOWCUBE{X})
 						shadow = computeShadowCube(light{X}.vLightData.xyz, shadowSampler{X}, light{X}.shadowsInfo.x, light{X}.depthValues);
 					#else
-						shadow = computeShadow(vPositionFromLight{X}, vDepthMetric{X}, shadowSampler{X}, light{X}.shadowsInfo.x);
+						shadow = computeShadow(vPositionFromLight{X}, vDepthMetric{X}, shadowSampler{X}, light{X}.shadowsInfo.x, light{X}.shadowsInfo.w);
 					#endif
 				#endif
 			#endif

src/Shaders/ShadersInclude/lightFragmentDeclaration.fx

@@ -16,7 +16,7 @@
 			uniform sampler2D shadowSampler{X};
 			uniform mat4 lightMatrix{X};
 		#endif
-		uniform vec3 shadowsInfo{X};
+		uniform vec4 shadowsInfo{X};
 		uniform vec2 depthValues{X};
 	#endif
 	#ifdef SPOTLIGHT{X}

src/Shaders/ShadersInclude/lightUboDeclaration.fx

@@ -10,7 +10,7 @@
 		#ifdef HEMILIGHT{X}
 			vec3 vLightGround;
 		#endif
-		vec3 shadowsInfo;
+		vec4 shadowsInfo;
 		vec2 depthValues;
 	} light{X};
 

src/Shaders/ShadersInclude/shadowsFragmentFunctions.fx

@@ -7,6 +7,12 @@
 		}
 	#endif
 
+	float computeFallOff(float shadow, vec2 clipSpace, float frustumEdgeFalloff)
+	{
+		float mask = smoothstep(1.0, 1.0 - frustumEdgeFalloff, dot(clipSpace, clipSpace));
+		return mix(1.0, shadow, mask);
+	}
+
 	float computeShadowCube(vec3 lightPosition, samplerCube shadowSampler, float darkness, vec2 depthValues)
 	{
 		vec3 directionToLight = vPositionW - lightPosition;
@@ -106,11 +112,10 @@
 		return esm;
 	}
 
-	float computeShadow(vec4 vPositionFromLight, float depthMetric, sampler2D shadowSampler, float darkness)
+	float computeShadow(vec4 vPositionFromLight, float depthMetric, sampler2D shadowSampler, float darkness, float frustumEdgeFalloff)
 	{
 		vec3 clipSpace = vPositionFromLight.xyz / vPositionFromLight.w;
-		clipSpace = 0.5 * clipSpace + vec3(0.5);
-		vec2 uv = clipSpace.xy;
+		vec2 uv = 0.5 * clipSpace.xy + vec2(0.5);
 
 		if (uv.x < 0. || uv.x > 1.0 || uv.y < 0. || uv.y > 1.0)
 		{
@@ -127,16 +132,15 @@
 
 		if (shadowPixelDepth > shadow)
 		{
-			return darkness;
+			return computeFallOff(darkness, clipSpace.xy, frustumEdgeFalloff);
 		}
 		return 1.;
 	}
 
-	float computeShadowWithPCF(vec4 vPositionFromLight, float depthMetric, sampler2D shadowSampler, float mapSize, float darkness)
+	float computeShadowWithPCF(vec4 vPositionFromLight, float depthMetric, sampler2D shadowSampler, float mapSize, float darkness, float frustumEdgeFalloff)
 	{
 		vec3 clipSpace = vPositionFromLight.xyz / vPositionFromLight.w;
-		clipSpace = 0.5 * clipSpace + vec3(0.5);
-		vec2 uv = clipSpace.xy;
+		vec2 uv = 0.5 * clipSpace.xy + vec2(0.5);
 
 		if (uv.x < 0. || uv.x > 1.0 || uv.y < 0. || uv.y > 1.0)
 		{
@@ -167,14 +171,13 @@
 			if (texture2D(shadowSampler, uv + poissonDisk[3] * mapSize).x < shadowPixelDepth) visibility -= 0.25;
 		#endif
 
-		return  min(1.0, visibility + darkness);
+		return computeFallOff(min(1.0, visibility + darkness), clipSpace.xy, frustumEdgeFalloff);
 	}
 
-	float computeShadowWithESM(vec4 vPositionFromLight, float depthMetric, sampler2D shadowSampler, float darkness, float depthScale)
+	float computeShadowWithESM(vec4 vPositionFromLight, float depthMetric, sampler2D shadowSampler, float darkness, float depthScale, float frustumEdgeFalloff)
 	{
 		vec3 clipSpace = vPositionFromLight.xyz / vPositionFromLight.w;
-		clipSpace = 0.5 * clipSpace + vec3(0.5);
-		vec2 uv = clipSpace.xy;
+		vec2 uv = 0.5 * clipSpace.xy + vec2(0.5);
 
 		if (uv.x < 0. || uv.x > 1.0 || uv.y < 0. || uv.y > 1.0)
 		{
@@ -191,21 +194,13 @@
 		
 		float esm = 1.0 - clamp(exp(min(87., depthScale * shadowPixelDepth)) * shadowMapSample, 0., 1. - darkness);
 
-		// Apply fade out at frustum edge
-		// const float fadeDistance = 0.07;
-		// vec2 cs2 = clipSpace.xy * clipSpace.xy; //squarish falloff
-		// float mask = smoothstep(1.0, 1.0 - fadeDistance, dot(cs2, cs2));
-
-		// esm = mix(1.0, esm, mask);
-
-		return esm;
+		return computeFallOff(esm, clipSpace.xy, frustumEdgeFalloff);
 	}
 
-	float computeShadowWithCloseESM(vec4 vPositionFromLight, float depthMetric, sampler2D shadowSampler, float darkness, float depthScale)
+	float computeShadowWithCloseESM(vec4 vPositionFromLight, float depthMetric, sampler2D shadowSampler, float darkness, float depthScale, float frustumEdgeFalloff)
 	{
 		vec3 clipSpace = vPositionFromLight.xyz / vPositionFromLight.w;
-		clipSpace = 0.5 * clipSpace + vec3(0.5);
-		vec2 uv = clipSpace.xy;
+		vec2 uv = 0.5 * clipSpace.xy + vec2(0.5);
 
 		if (uv.x < 0. || uv.x > 1.0 || uv.y < 0. || uv.y > 1.0)
 		{
@@ -222,13 +217,6 @@
 		
 		float esm = clamp(exp(min(87., -depthScale * (shadowPixelDepth - shadowMapSample))), darkness, 1.);
 
-		// Apply fade out at frustum edge
-		// const float fadeDistance = 0.07;
-		// vec2 cs2 = clipSpace.xy * clipSpace.xy; //squarish falloff
-		// float mask = smoothstep(1.0, 1.0 - fadeDistance, dot(cs2, cs2));
-
-		// esm = mix(1.0, esm, mask);
-
-		return esm;
+		return computeFallOff(esm, clipSpace.xy, frustumEdgeFalloff);
 	}
 #endif