multiview webVR post process

src/Cameras/RigModes/vrRigMode.ts

@@ -1,6 +1,6 @@
 import { Camera } from "../camera";
 import { Matrix, Viewport } from "../../Maths/math";
-import { VRDistortionCorrectionPostProcess } from "../../PostProcesses/vrDistortionCorrectionPostProcess";
+import { VRDistortionCorrectionPostProcess, VRMultiviewToSingleview } from "../../PostProcesses/vrDistortionCorrectionPostProcess";
 import { VRCameraMetrics } from "../VR/vrCameraMetrics";
 
 Camera._setVRRigMode = function(camera: Camera, rigParams: any) {
@@ -20,6 +20,13 @@ Camera._setVRRigMode = function(camera: Camera, rigParams: any) {
     camera._rigCameras[1]._cameraRigParams.vrPreViewMatrix = metrics.rightPreViewMatrix;
     camera._rigCameras[1].getProjectionMatrix = camera._rigCameras[1]._getVRProjectionMatrix;
 
+    // For multiview on a webVR camera
+    // First multiview will be rendered to camera._multiviewTexture
+    // Then this postprocess will run on each eye to copy the right texture to each eye
+    if (metrics.multiviewEnabled) {
+        camera._rigPostProcess = new VRMultiviewToSingleview("VRMultiviewToSingleview", camera, 1.0);
+    }
+
     if (metrics.compensateDistortion) {
         camera._rigCameras[0]._rigPostProcess = new VRDistortionCorrectionPostProcess("VR_Distort_Compensation_Left", camera._rigCameras[0], false, metrics);
         camera._rigCameras[1]._rigPostProcess = new VRDistortionCorrectionPostProcess("VR_Distort_Compensation_Right", camera._rigCameras[1], true, metrics);

src/Cameras/VR/vrCameraMetrics.ts

@@ -60,6 +60,11 @@ export class VRCameraMetrics {
     public compensateDistortion = true;
 
     /**
+     * Defines if multiview should be enabled when rendering (Default: false)
+     */
+    public multiviewEnabled = false;
+
+    /**
      * Gets the rendering aspect ratio based on the provided resolutions.
      */
     public get aspectRatio(): number {

src/Cameras/camera.ts

@@ -13,6 +13,7 @@ import { ICullable } from "../Culling/boundingInfo";
 import { Logger } from "../Misc/logger";
 import { _TypeStore } from '../Misc/typeStore';
 import { _DevTools } from '../Misc/devTools';
+import { MultiviewRenderTarget } from '../Materials/Textures/renderTargetTexture';
 
 declare type PostProcess = import("../PostProcesses/postProcess").PostProcess;
 declare type RenderTargetTexture = import("../Materials/Textures/renderTargetTexture").RenderTargetTexture;
@@ -245,6 +246,27 @@ export class Camera extends Node {
     public outputRenderTarget: Nullable<RenderTargetTexture> = null;
 
     /**
+     * @hidden
+     * For cameras that cannot use multiview images to display directly. (e.g. webVR camera will render to multiview texture, then copy to each eye texture and go from there)
+     */
+    public _multiviewTexture: Nullable<RenderTargetTexture> = null;
+
+    /**
+     * @hidden
+     * ensures the multiview texture of the camera exists and has the specified width/height
+     * @param width height to set on the multiview texture
+     * @param height width to set on the multiview texture
+     */
+    public _resizeorCreateMultiviewTexture(width: number, height: number) {
+        if (!this._multiviewTexture) {
+            this._multiviewTexture = new MultiviewRenderTarget(this.getScene(), {width: width, height: height});
+        }else if (this._multiviewTexture.getRenderWidth() != width || this._multiviewTexture.getRenderHeight() != height) {
+            this._multiviewTexture.dispose();
+            this._multiviewTexture = new MultiviewRenderTarget(this.getScene(), {width: width, height: height});
+        }
+    }
+
+    /**
      * Observable triggered when the camera view matrix has changed.
      */
     public onViewMatrixChangedObservable = new Observable<Camera>();

src/Materials/standardMaterial.ts

@@ -794,8 +794,11 @@ export class StandardMaterial extends PushMaterial {
         // Lights
         defines._needNormals = MaterialHelper.PrepareDefinesForLights(scene, mesh, defines, true, this._maxSimultaneousLights, this._disableLighting);
         if (scene.activeCamera) {
+            var previousMultiview = defines.MULTIVIEW;
             defines.MULTIVIEW = (scene.activeCamera.outputRenderTarget !== null && scene.activeCamera.outputRenderTarget.getViewCount() > 1);
-            defines.markAsUnprocessed();
+            if (defines.MULTIVIEW != previousMultiview) {
+                defines.markAsUnprocessed();
+            }
         }
 
         // Textures

src/PostProcesses/vrDistortionCorrectionPostProcess.ts

@@ -53,3 +53,30 @@ export class VRDistortionCorrectionPostProcess extends PostProcess {
         });
     }
 }
+
+/**
+ * VRMultiviewToSingleview used to convert multiview texture arrays to standard textures for scenarios such as webVR
+ * This will not be used for webXR as it supports displaying texture arrays directly
+ */
+export class VRMultiviewToSingleview extends PostProcess {
+    /**
+     * Initializes a VRMultiviewToSingleview
+     * @param name name of the post process
+     * @param camera camera to be applied to
+     * @param scaleFactor scaling factor to the size of the output texture
+     */
+    constructor(name: string, camera: Camera, scaleFactor: number) {
+        super(name, "vrMultiviewToSingleview", ["imageIndex"], ["multiviewSampler"], scaleFactor, camera, Texture.BILINEAR_SAMPLINGMODE);
+
+        this.onSizeChangedObservable.add(() => {
+        });
+        this.onApplyObservable.add((effect: Effect) => {
+            if (camera._scene.activeCamera && camera._scene.activeCamera.name == "_L") {
+                effect.setInt("imageIndex", 0);
+            }else {
+                effect.setInt("imageIndex", 1);
+            }
+            effect.setTexture("multiviewSampler", camera._multiviewTexture);
+        });
+    }
+}

src/Shaders/vrMultiviewToSingleview.fragment.fx

@@ -0,0 +1,12 @@
+#ifdef GL_ES
+	precision mediump sampler2DArray;
+#endif
+
+varying vec2 vUV;
+uniform sampler2DArray multiviewSampler;
+uniform int imageIndex;
+
+void main(void)
+{
+	gl_FragColor = texture(multiviewSampler, vec3(vUV, imageIndex));
+}

src/scene.ts

@@ -4000,7 +4000,9 @@ export class Scene extends AbstractScene implements IAnimatable {
     }
 
     private _bindFrameBuffer() {
-        if (this.activeCamera && this.activeCamera.outputRenderTarget) {
+        if (this.activeCamera && this.activeCamera._multiviewTexture) {
+            this.activeCamera._multiviewTexture._bindFrameBuffer();
+        } else if (this.activeCamera && this.activeCamera.outputRenderTarget) {
             var useMultiview = this.getEngine().getCaps().multiview && this.activeCamera.outputRenderTarget && this.activeCamera.outputRenderTarget.getViewCount() > 1;
             if (useMultiview) {
                 this.activeCamera.outputRenderTarget._bindFrameBuffer();
@@ -4110,7 +4112,7 @@ export class Scene extends AbstractScene implements IAnimatable {
         this.onAfterRenderTargetsRenderObservable.notifyObservers(this);
 
         // Prepare Frame
-        if (this.postProcessManager) {
+        if (this.postProcessManager && !camera._multiviewTexture) {
             this.postProcessManager._prepareFrame();
         }
 
@@ -4130,7 +4132,7 @@ export class Scene extends AbstractScene implements IAnimatable {
         }
 
         // Finalize frame
-        if (this.postProcessManager) {
+        if (this.postProcessManager  && !camera._multiviewTexture) {
             this.postProcessManager._finalizeFrame(camera.isIntermediate);
         }
 
@@ -4148,9 +4150,34 @@ export class Scene extends AbstractScene implements IAnimatable {
             return;
         }
 
-        // rig cameras
-        for (var index = 0; index < camera._rigCameras.length; index++) {
-            this._renderForCamera(camera._rigCameras[index], camera);
+        if (camera._rigCameras[0]._cameraRigParams && camera._rigCameras[0]._cameraRigParams.vrMetrics && camera._rigCameras[0]._cameraRigParams.vrMetrics.multiviewEnabled) {
+            // Multiview is only able to be displayed directly for API's such as webXR
+            // This displays a multiview image by rendering to the multiview image and then
+            // copying the result into the sub cameras instead of rendering them and proceeding as normal from there
+
+            // Render to a multiview texture
+            camera._resizeorCreateMultiviewTexture(this.getEngine().getRenderWidth(true) / 2, this.getEngine().getRenderHeight(true));
+            camera.outputRenderTarget = camera._multiviewTexture;
+            this._renderForCamera(camera);
+            camera.outputRenderTarget = null;
+
+            // Consume the multiview texture through a shader for each eye
+            for (var index = 0; index < camera._rigCameras.length; index++) {
+                var engine = this._engine;
+                this._activeCamera = camera._rigCameras[index];
+                engine.setViewport(this._activeCamera.viewport);
+                if (this.postProcessManager) {
+                    this.postProcessManager._prepareFrame();
+                }
+                if (this.postProcessManager) {
+                    this.postProcessManager._finalizeFrame(this._activeCamera.isIntermediate);
+                }
+            }
+        } else {
+            // rig cameras
+            for (var index = 0; index < camera._rigCameras.length; index++) {
+                this._renderForCamera(camera._rigCameras[index], camera);
+            }
         }
 
         // Use _activeCamera instead of activeCamera to avoid onActiveCameraChanged