Render transparent meshes with the same stencil state than the opaque… (#8388)

* Render transparent meshes with the same stencil state than the opaque ones

* Fix outline renderer when using the visibility property of meshes

* Transparent meshes can be rendered in the highlight layer

dist/preview release/what's new.md

@@ -239,3 +239,4 @@
 - HDRCubeTexture default rotation is now similar to the industry one. You might need to add a rotation on y of 90 degrees if you scene changes ([Sebavan](https://github.com/sebavan/))
 - PBRMaterial index of refraction is now defined as index of refraction and not the inverse of it ([Sebavan](https://github.com/sebavan/))
 - `SceneLoaderProgress` class is now `ISceneLoaderProgress` interface ([bghgary](https://github.com/bghgary))
+- Rendering of transparent meshes: stencil state is now set to the value registered in the engine instead of being set to `false` unconditionally ([Popov72](https://github.com/Popov72))

src/Layers/highlightLayer.ts

@@ -513,6 +513,17 @@ export class HighlightLayer extends EffectLayer {
     }
 
     /**
+     * Returns true if the mesh can be rendered, otherwise false.
+     * @param mesh The mesh to render
+     * @param material The material used on the mesh
+     * @returns true if it can be rendered otherwise false
+     */
+    protected _canRenderMesh(mesh: AbstractMesh, material: Material): boolean {
+        // all meshes can be rendered in the highlight layer, even transparent ones
+        return true;
+    }
+
+    /**
      * Adds specific effects defines.
      * @param defines The defines to add specifics to.
      */

src/Rendering/outlineRenderer.ts

@@ -299,7 +299,7 @@ export class OutlineRenderer implements ISceneComponent {
         this._savedDepthWrite = this._engine.getDepthWrite();
         if (mesh.renderOutline) {
             var material = subMesh.getMaterial();
-            if (material && material.needAlphaBlending()) {
+            if (material && material.needAlphaBlendingForMesh(mesh)) {
                 this._engine.cacheStencilState();
                 // Draw only to stencil buffer for the original mesh
                 // The resulting stencil buffer will be used so the outline is not visible inside the mesh when the mesh is transparent
@@ -321,7 +321,7 @@ export class OutlineRenderer implements ISceneComponent {
             this.render(subMesh, batch);
             this._engine.setDepthWrite(this._savedDepthWrite);
 
-            if (material && material.needAlphaBlending()) {
+            if (material && material.needAlphaBlendingForMesh(mesh)) {
                 this._engine.restoreStencilState();
             }
         }

src/Rendering/renderingGroup.ts

@@ -149,6 +149,7 @@ export class RenderingGroup {
 
         // Transparent
         if (this._transparentSubMeshes.length !== 0) {
+            engine.setStencilBuffer(stencilState);
             this._renderTransparent(this._transparentSubMeshes);
             engine.setAlphaMode(Constants.ALPHA_DISABLE);
         }