move stencil buffer caching/restore to engine, only do certain outline rendering when object has transparency

src/Engines/engine.ts

@@ -1771,6 +1771,40 @@ export class Engine {
         this._depthCullingState.depthFunc = this._gl.LESS;
     }
 
+    private _cachedStencilBuffer: boolean;
+    private _cachedStencilFunction: number;
+    private _cachedStencilMask: number;
+    private _cachedStencilOperationPass: number;
+    private _cachedStencilOperationFail: number;
+    private _cachedStencilOperationDepthFail: number;
+    private _cachedStencilReference: number;
+
+    /**
+     * Caches the the state of the stencil buffer
+     */
+    public cacheStencilState() {
+        this._cachedStencilBuffer = this.getStencilBuffer();
+        this._cachedStencilFunction = this.getStencilFunction();
+        this._cachedStencilMask = this.getStencilMask();
+        this._cachedStencilOperationPass = this.getStencilOperationPass();
+        this._cachedStencilOperationFail = this.getStencilOperationFail();
+        this._cachedStencilOperationDepthFail = this.getStencilOperationDepthFail();
+        this._cachedStencilReference = this.getStencilFunctionReference();
+    }
+
+    /**
+     * Restores the state of the stencil buffer
+     */
+    public restoreStencilState() {
+        this.setStencilFunction(this._cachedStencilFunction);
+        this.setStencilMask(this._cachedStencilMask);
+        this.setStencilBuffer(this._cachedStencilBuffer);
+        this.setStencilOperationPass(this._cachedStencilOperationPass);
+        this.setStencilOperationFail(this._cachedStencilOperationFail);
+        this.setStencilOperationDepthFail(this._cachedStencilOperationDepthFail);
+        this.setStencilFunctionReference(this._cachedStencilReference);
+    }
+
     /**
      * Sets the current depth function to LEQUAL
      */

src/Layers/highlightLayer.ts

@@ -456,13 +456,7 @@ export class HighlightLayer extends EffectLayer {
 
         // Cache
         var engine = this._engine;
-        var previousStencilBuffer = engine.getStencilBuffer();
-        var previousStencilFunction = engine.getStencilFunction();
-        var previousStencilMask = engine.getStencilMask();
-        var previousStencilOperationPass = engine.getStencilOperationPass();
-        var previousStencilOperationFail = engine.getStencilOperationFail();
-        var previousStencilOperationDepthFail = engine.getStencilOperationDepthFail();
-        var previousStencilReference = engine.getStencilFunctionReference();
+        engine.cacheStencilState();
 
         // Stencil operations
         engine.setStencilOperationPass(Constants.REPLACE);
@@ -487,13 +481,7 @@ export class HighlightLayer extends EffectLayer {
         }
 
         // Restore Cache
-        engine.setStencilFunction(previousStencilFunction);
-        engine.setStencilMask(previousStencilMask);
-        engine.setStencilBuffer(previousStencilBuffer);
-        engine.setStencilOperationPass(previousStencilOperationPass);
-        engine.setStencilOperationFail(previousStencilOperationFail);
-        engine.setStencilOperationDepthFail(previousStencilOperationDepthFail);
-        engine.setStencilFunctionReference(previousStencilReference);
+        engine.restoreStencilState();
     }
 
     /**

src/Rendering/outlineRenderer.ts

@@ -278,14 +278,7 @@ export class OutlineRenderer implements ISceneComponent {
         // Outline - step 1
         this._savedDepthWrite = this._engine.getDepthWrite();
         if (mesh.renderOutline) {
-            // Cache stencil state
-            var previousStencilBuffer = this._engine.getStencilBuffer();
-            var previousStencilFunction = this._engine.getStencilFunction();
-            var previousStencilMask = this._engine.getStencilMask();
-            var previousStencilOperationPass = this._engine.getStencilOperationPass();
-            var previousStencilOperationFail = this._engine.getStencilOperationFail();
-            var previousStencilOperationDepthFail = this._engine.getStencilOperationDepthFail();
-            var previousStencilReference = this._engine.getStencilFunctionReference();
+            this._engine.cacheStencilState();
 
             var material = subMesh.getMaterial();
             if (material && material.needAlphaBlending) {
@@ -299,23 +292,17 @@ export class OutlineRenderer implements ISceneComponent {
                 this._engine.setStencilMask(OutlineRenderer._StencilReference);
                 this._engine.setStencilFunctionReference(OutlineRenderer._StencilReference);
                 this.render(subMesh, batch, /* This sets offset to 0 */ true);
+
+                this._engine.setColorWrite(true);
+                this._engine.setStencilFunction(Constants.NOTEQUAL);
             }
 
             // Draw the outline using the above stencil if needed to avoid drawing within the mesh
-            this._engine.setColorWrite(true);
             this._engine.setDepthWrite(false);
-            this._engine.setStencilFunction(Constants.NOTEQUAL);
             this.render(subMesh, batch);
             this._engine.setDepthWrite(this._savedDepthWrite);
 
-            // Restore stencil state
-            this._engine.setStencilFunction(previousStencilFunction);
-            this._engine.setStencilMask(previousStencilMask);
-            this._engine.setStencilBuffer(previousStencilBuffer);
-            this._engine.setStencilOperationPass(previousStencilOperationPass);
-            this._engine.setStencilOperationFail(previousStencilOperationFail);
-            this._engine.setStencilOperationDepthFail(previousStencilOperationDepthFail);
-            this._engine.setStencilFunctionReference(previousStencilReference);
+            this._engine.restoreStencilState();
         }
     }