adding depth of field to default pipeline

Tools/Gulp/config.json

@@ -58,6 +58,8 @@
             "additionalPostProcesses",
             "additionalPostProcess_blur",
             "additionalPostProcess_fxaa",
+            "additionalPostProcess_circleOfConfusion",
+            "additionalPostProcess_depthOfFieldMerge",
             "additionalPostProcess_imageProcessing",
             "bones",
             "hdr",
@@ -721,6 +723,28 @@
                 "kernelBlurVertex"
             ]
         },
+        "additionalPostProcess_circleOfConfusion": {
+            "files": [
+                "../../src/PostProcess/babylon.circleOfConfusionPostProcess.js"
+            ],
+            "dependUpon": [
+                "postProcesses"
+            ],
+            "shaders": [
+                "circleOfConfusion.fragment"
+            ]
+        },
+        "additionalPostProcess_depthOfFieldMerge": {
+            "files": [
+                "../../src/PostProcess/babylon.depthOfFieldMergePostProcess.js"
+            ],
+            "dependUpon": [
+                "postProcesses"
+            ],
+            "shaders": [
+                "depthOfFieldMerge.fragment"
+            ]
+        },
         "additionalPostProcess_fxaa": {
             "files": [
                 "../../src/PostProcess/babylon.fxaaPostProcess.js"
@@ -826,7 +850,9 @@
             ],
             "dependUpon": [
                 "renderingPipeline",
-                "additionalPostProcess_fxaa"
+                "additionalPostProcess_fxaa",
+                "additionalPostProcess_circleOfConfusion",
+                "additionalPostProcess_depthOfFieldMerge"
             ]
         },
         "bones": {

src/PostProcess/RenderPipeline/Pipelines/babylon.defaultRenderingPipeline.ts

@@ -8,6 +8,11 @@
         readonly BlurYPostProcessId: string = "BlurYPostProcessEffect";
         readonly CopyBackPostProcessId: string = "CopyBackPostProcessEffect";
         readonly ImageProcessingPostProcessId: string = "ImageProcessingPostProcessEffect";
+        readonly DepthOfFieldPassPostProcessId: string = "DepthOfFieldPassPostProcessId";
+        readonly CircleOfConfusionPostProcessId: string = "CircleOfConfusionPostProcessEffect"; 
+        readonly DepthOfFieldBlurXPostProcessId: string = "DepthOfFieldBlurXPostProcessEffect";
+        readonly DepthOfFieldBlurYPostProcessId: string = "DepthOfFieldBlurYPostProcessEffect";
+        readonly DepthOfFieldMergePostProcessId: string = "DepthOfFieldMergePostProcessEffect";
         readonly FxaaPostProcessId: string = "FxaaPostProcessEffect";
         readonly FinalMergePostProcessId: string = "FinalMergePostProcessEffect";
 
@@ -17,6 +22,11 @@
         public blurX: BlurPostProcess;
         public blurY: BlurPostProcess;
         public copyBack: PassPostProcess;
+        public depthOfFieldPass: PassPostProcess;
+        public circleOfConfusion: CircleOfConfusionPostProcess;
+        public depthOfFieldBlurX: BlurPostProcess;
+        public depthOfFieldBlurY: BlurPostProcess;
+        public depthOfFieldMerge: DepthOfFieldMergePostProcess;
         public fxaa: FxaaPostProcess;
         public imageProcessing: ImageProcessingPostProcess;
         public finalMerge: PassPostProcess;
@@ -26,6 +36,7 @@
 
         // Values       
         private _bloomEnabled: boolean = false;
+        private _depthOfFieldEnabled: boolean = false;
         private _fxaaEnabled: boolean = false;
         private _imageProcessingEnabled: boolean = true;
         private _defaultPipelineTextureType: number;
@@ -92,6 +103,20 @@
             return this._bloomEnabled;
         }
 
+        @serialize()
+        public get depthOfFieldEnabled(): boolean {
+            return this._depthOfFieldEnabled;
+        }   
+        
+        public set depthOfFieldEnabled(enabled: boolean) {
+            if (this._depthOfFieldEnabled === enabled) {
+                return;
+            }
+            this._depthOfFieldEnabled = enabled;
+            
+            this._buildPipeline();
+        }
+
         public set fxaaEnabled(enabled: boolean) {
             if (this._fxaaEnabled === enabled) {
                 return;
@@ -219,6 +244,30 @@
                 this.copyBack.autoClear = false;
             }
 
+            if(this.depthOfFieldEnabled){
+                // Enable and get current depth map
+                var depthMap = this._scene.enableDepthRenderer().getDepthMap();
+                
+                // Circle of confusion value for each pixel is used to determine how much to blur that pixel
+                this.circleOfConfusion = new BABYLON.CircleOfConfusionPostProcess("circleOfConfusion", depthMap, 1, null, BABYLON.Texture.BILINEAR_SAMPLINGMODE, engine, true, this._defaultPipelineTextureType);
+                this.addEffect(new PostProcessRenderEffect(engine, this.CircleOfConfusionPostProcessId, () => { return this.circleOfConfusion; }, true));  
+            
+                // Capture circle of confusion texture
+                this.depthOfFieldPass = new PassPostProcess("depthOfFieldPass", 1.0, null, Texture.BILINEAR_SAMPLINGMODE, engine, false, this._defaultPipelineTextureType);
+                this.addEffect(new PostProcessRenderEffect(engine, this.DepthOfFieldPassPostProcessId, () => { return this.depthOfFieldPass; }, true));
+                
+                // Blur the image but do not blur on sharp far to near distance changes to avoid bleeding artifacts 
+                // See section 2.6.2 http://fileadmin.cs.lth.se/cs/education/edan35/lectures/12dof.pdf
+                this.depthOfFieldBlurY = new BlurPostProcess("verticle blur", new Vector2(0, 1.0), 15, 1.0, null, Texture.BILINEAR_SAMPLINGMODE, engine, false, this._defaultPipelineTextureType, new DepthOfFieldBlurOptions(depthMap, this.circleOfConfusion));
+                this.addEffect(new PostProcessRenderEffect(engine, this.DepthOfFieldBlurYPostProcessId, () => { return this.depthOfFieldBlurY; }, true));
+                this.depthOfFieldBlurX = new BlurPostProcess("horizontal blur", new Vector2(1.0, 0), 15, 1.0, null, Texture.BILINEAR_SAMPLINGMODE, engine, false, this._defaultPipelineTextureType, new DepthOfFieldBlurOptions(depthMap));
+                this.addEffect(new PostProcessRenderEffect(engine, this.DepthOfFieldBlurXPostProcessId, () => { return this.depthOfFieldBlurX; }, true));
+                
+                // Merge blurred images with original image based on circleOfConfusion
+                this.depthOfFieldMerge = new DepthOfFieldMergePostProcess("depthOfFieldMerge", this.circleOfConfusion, this.depthOfFieldPass, 1, null, BABYLON.Texture.BILINEAR_SAMPLINGMODE, engine, true, this._defaultPipelineTextureType);
+                this.addEffect(new PostProcessRenderEffect(engine, this.DepthOfFieldMergePostProcessId, () => { return this.depthOfFieldMerge; }, true));
+            }
+
             if (this._imageProcessingEnabled) {
                 this.imageProcessing = new ImageProcessingPostProcess("imageProcessing", 1.0, null, Texture.BILINEAR_SAMPLINGMODE, engine, false, this._defaultPipelineTextureType);
                 if (this._hdr) {

src/PostProcess/babylon.blurPostProcess.ts

@@ -1,9 +1,14 @@
 module BABYLON {
+	export class DepthOfFieldBlurOptions {		
+		constructor(public depthMap: BaseTexture, public original: Nullable<PostProcess> = null){
+
+		}
+	}
     export class BlurPostProcess extends PostProcess {
 		protected _kernel: number;
 		protected _idealKernel: number;
 		protected _packedFloat: boolean	= false;
-
+		protected depthOfFieldBlurOptions:any;
 		/**
 		 * Sets the length in pixels of the blur sample region
 		 */
@@ -43,11 +48,31 @@
 			return this._packedFloat;
 		}
 
-        constructor(name: string, public direction: Vector2, kernel: number, options: number | PostProcessOptions, camera: Nullable<Camera>, samplingMode: number = Texture.BILINEAR_SAMPLINGMODE, engine?: Engine, reusable?: boolean, textureType: number = Engine.TEXTURETYPE_UNSIGNED_INT) {
-            super(name, "kernelBlur", ["delta", "direction"], null, options, camera, samplingMode, engine, reusable, null, textureType, "kernelBlur", {varyingCount: 0, depCount: 0}, true);
-            this.onApplyObservable.add((effect: Effect) => {
-                effect.setFloat2('delta', (1 / this.width) * this.direction.x, (1 / this.height) * this.direction.y);
-            });
+        constructor(name: string, public direction: Vector2, kernel: number, options: number | PostProcessOptions, camera: Nullable<Camera>, samplingMode: number = Texture.BILINEAR_SAMPLINGMODE, engine?: Engine, reusable?: boolean, textureType: number = Engine.TEXTURETYPE_UNSIGNED_INT, depthOfFieldBlurOptions?:DepthOfFieldBlurOptions) {
+            super(name, "kernelBlur", ["delta", "direction", "near", "far"], ["depthSampler"], options, camera, samplingMode, engine, reusable, null, textureType, "kernelBlur", {varyingCount: 0, depCount: 0}, true);
+			
+			if(depthOfFieldBlurOptions){
+				this.depthOfFieldBlurOptions = depthOfFieldBlurOptions;
+			}
+			
+			this.onApplyObservable.add((effect: Effect) => {
+				if(this.depthOfFieldBlurOptions){
+					// TODO: setTextureFromPostProcess seems to be setting the input texture instead of output of the post process passed in 
+					if(this.depthOfFieldBlurOptions.original != null){
+						effect.setTextureFromPostProcess("textureSampler", this.depthOfFieldBlurOptions.original)
+					}
+					effect.setTexture("depthSampler", this.depthOfFieldBlurOptions.depthMap)
+				}
+				
+				effect.setFloat2('delta', (1 / this.width) * this.direction.x, (1 / this.height) * this.direction.y);
+			
+				// TODO: is there a better way to get camera?
+                var camera = this.getEngine().scenes[0].activeCamera;
+                if(camera){
+                    effect.setFloat('near', camera.minZ);
+                    effect.setFloat('far', camera.maxZ);
+                }
+			});
 
             this.kernel = kernel;
         }
@@ -137,6 +162,10 @@
 				defines += `#define PACKEDFLOAT 1`;
 			}
 
+			if(this.depthOfFieldBlurOptions){
+				defines += `#define DOF 1`;
+			}
+
             this.updateEffect(defines, null, null, {
 				varyingCount: varyingCount,
 				depCount: depCount

src/PostProcess/babylon.circleOfConfusionPostProcess.ts

@@ -0,0 +1,29 @@
+module BABYLON {
+    export class CircleOfConfusionPostProcess extends PostProcess {
+        fStop = 4; // Aperture = focalLength/fStop
+        focusDistance = 15; // in scene units (eg. meter)
+        focalLength = 10; // in scene units/1000 (eg. millimeter)
+
+        
+        constructor(name: string, depthTexture: RenderTargetTexture, options: number | PostProcessOptions, camera: Nullable<Camera>, samplingMode?: number, engine?: Engine, reusable?: boolean, textureType: number = Engine.TEXTURETYPE_UNSIGNED_INT) {
+            super(name, "circleOfConfusion", ["near", "far", "focusDistance", "cocPrecalculation"], ["depthSampler"], options, camera, samplingMode, engine, reusable, null, textureType);
+            this.onApplyObservable.add((effect: Effect) => {
+                effect.setTexture("depthSampler", depthTexture);
+                
+                // Circle of confusion calculation, See https://developer.nvidia.com/gpugems/GPUGems/gpugems_ch23.html
+                var aperture = this.focalLength/this.fStop;
+                var cocPrecalculation = ((aperture * this.focalLength)/((this.focusDistance - this.focalLength)));// * ((this.focusDistance - pixelDistance)/pixelDistance) [This part is done in shader]
+                
+                effect.setFloat('focusDistance', this.focusDistance);
+                effect.setFloat('cocPrecalculation', cocPrecalculation);
+                
+                // TODO: is there a better way to get camera?
+                var camera = this.getEngine().scenes[0].activeCamera;
+                if(camera){
+                    effect.setFloat('near', camera.minZ);
+                    effect.setFloat('far', camera.maxZ);
+                }
+            })
+        }
+    }
+}

src/PostProcess/babylon.depthOfFieldMergePostProcess.ts

@@ -0,0 +1,11 @@
+module BABYLON {
+    export class DepthOfFieldMergePostProcess extends PostProcess {
+        constructor(name: string, original: PostProcess, circleOfConfusion: PostProcess, options: number | PostProcessOptions, camera: Nullable<Camera>, samplingMode?: number, engine?: Engine, reusable?: boolean, textureType: number = Engine.TEXTURETYPE_UNSIGNED_INT) {
+            super(name, "depthOfFieldMerge", [], ["circleOfConfusionSampler", "originalSampler"], options, camera, samplingMode, engine, reusable, null, textureType);
+            this.onApplyObservable.add((effect: Effect) => {
+                effect.setTextureFromPostProcess("circleOfConfusionSampler", circleOfConfusion);
+                effect.setTextureFromPostProcess("originalSampler", original);
+            })
+        }
+    }
+}

src/Shaders/ShadersInclude/kernelBlurFragment.fx

@@ -1,5 +1,14 @@
-#ifdef PACKEDFLOAT
-    blend += unpack(texture2D(textureSampler, sampleCoord{X})) * KERNEL_WEIGHT{X};
+#ifdef DOF
+    sampleDepth = sampleDistance(sampleCoord{X});
+    factor = clamp(1.0-((centerSampleDepth - sampleDepth)/centerSampleDepth),0.0,1.0);
+    computedWeight = KERNEL_WEIGHT{X} * factor;
+    sumOfWeights += computedWeight;
 #else
-    blend += texture2D(textureSampler, sampleCoord{X}) * KERNEL_WEIGHT{X};
+    computedWeight = KERNEL_WEIGHT{X};
+#endif
+
+#ifdef PACKEDFLOAT
+    blend += unpack(texture2D(textureSampler, sampleCoord{X})) * computedWeight;
+#else
+    blend += texture2D(textureSampler, sampleCoord{X}) * computedWeight;
 #endif

src/Shaders/ShadersInclude/kernelBlurFragment2.fx

@@ -1,5 +1,14 @@
-#ifdef PACKEDFLOAT
-    blend += unpack(texture2D(textureSampler, sampleCenter + delta * KERNEL_DEP_OFFSET{X})) * KERNEL_DEP_WEIGHT{X};
+#ifdef DOF
+    sampleDepth = sampleDistance(sampleCoord{X});
+    factor = clamp(1.0-((centerSampleDepth - sampleDepth)/centerSampleDepth),0.0,1.0);
+    computedWeight = KERNEL_DEP_WEIGHT{X} * factor;
+    sumOfWeights += computedWeight;
 #else
-    blend += texture2D(textureSampler, sampleCenter + delta * KERNEL_DEP_OFFSET{X}) * KERNEL_DEP_WEIGHT{X};
+    computedWeight = KERNEL_DEP_WEIGHT{X};
+#endif
+
+#ifdef PACKEDFLOAT
+    blend += unpack(texture2D(textureSampler, sampleCenter + delta * KERNEL_DEP_OFFSET{X})) * computedWeight;
+#else
+    blend += texture2D(textureSampler, sampleCenter + delta * KERNEL_DEP_OFFSET{X}) * computedWeight;
 #endif

src/Shaders/circleOfConfusion.fragment.fx

@@ -0,0 +1,26 @@
+// samplers
+uniform sampler2D depthSampler;
+
+// varyings
+varying vec2 vUV;
+
+// preconputed uniforms (not effect parameters)
+uniform float near;
+uniform float far;
+
+// uniforms
+uniform float focusDistance;
+uniform float cocPrecalculation;
+
+float sampleDistance(const in vec2 offset) {
+    float depth = texture2D(depthSampler, offset).r;	// depth value from DepthRenderer: 0 to 1
+	return near + (far - near)*depth;		            // actual distance from the lens
+}
+
+void main(void)
+{
+    float pixelDistance = sampleDistance(vUV);
+    float coc = abs(cocPrecalculation* ((focusDistance - pixelDistance)/pixelDistance));
+    coc = clamp(coc, 0.0, 1.0);
+    gl_FragColor = vec4(coc, coc, coc, 1.0);
+}

src/Shaders/depthOfFieldMerge.fragment.fx

@@ -0,0 +1,15 @@
+// samplers
+uniform sampler2D textureSampler;
+uniform sampler2D originalSampler;
+uniform sampler2D circleOfConfusionSampler;
+
+// varyings
+varying vec2 vUV;
+
+void main(void)
+{
+    vec4 blurred = texture2D(textureSampler, vUV);
+    vec4 original = texture2D(originalSampler, vUV);
+    float coc = texture2D(circleOfConfusionSampler, vUV).r;
+    gl_FragColor = (original * (1.0-coc))+(blurred*coc);
+}

src/Shaders/kernelBlur.fragment.fx

@@ -4,6 +4,19 @@ uniform vec2 delta;
 
 // Varying
 varying vec2 sampleCenter;
+
+#ifdef DOF
+	uniform sampler2D depthSampler;
+
+	uniform float near;
+	uniform float far;
+
+	float sampleDistance(const in vec2 offset) {
+		float depth = texture2D(depthSampler, offset).r; // depth value from DepthRenderer: 0 to 1 
+		return near + (far - near)*depth; // actual distance from the lens 
+	}
+#endif
+
 #include<kernelBlurVaryingDeclaration>[0..varyingCount]
 
 #ifdef PACKEDFLOAT
@@ -27,11 +40,20 @@ varying vec2 sampleCenter;
 
 void main(void)
 {
-#ifdef PACKEDFLOAT	
-	float blend = 0.;
-#else
-	vec4 blend = vec4(0.);
-#endif
+	#ifdef DOF
+		float sumOfWeights = 0.0; // Since not all values are blended, keep track of sum to devide result by at the end to get an average
+		float sampleDepth = 0.0;
+    	float factor = 0.0;
+		float centerSampleDepth = sampleDistance(sampleCenter);
+	#endif
+
+	float computedWeight = 0.0;
+
+	#ifdef PACKEDFLOAT	
+		float blend = 0.;
+	#else
+		vec4 blend = vec4(0.);
+	#endif
 
 	#include<kernelBlurFragment>[0..varyingCount]
 	#include<kernelBlurFragment2>[0..depCount]
@@ -41,4 +63,8 @@ void main(void)
 	#else
 		gl_FragColor = blend;
 	#endif
+
+	#ifdef DOF
+		gl_FragColor /= sumOfWeights;
+	#endif
 }