Show a summary at the end of the validation test process

tests/validation/config.json

@@ -32,7 +32,7 @@
             "title": "Sprite maps",
             "playgroundId": "#ARLADE#23",
             "referenceImage": "sprite-maps.png",            
-            "excludeFromAutomaticTesting": true
+            "excludedEngines": ["webgl1"]
         },              
         {
             "title": "Point light shadows",
@@ -43,8 +43,7 @@
             "title": "Node material 0",
             "playgroundId": "#M5VQE9#20",
             "referenceImage": "node-material0.png",
-            "renderCount": 5,
-            "excludeFromAutomaticTesting": true
+            "renderCount": 5
         },    
         {
             "title": "Node material 1",
@@ -808,8 +807,7 @@
             "scriptToRun": "/Demos/Procedural/proceduralTexture.js",
             "functionToCall": "CreateProceduralTextureTestScene",
             "referenceImage": "procedural.png",
-            "replace": "./land, https://cdn.rawgit.com/BabylonJS/Website/06ecbea7/Demos/Procedural/land",
-            "excludeFromAutomaticTesting": true
+            "replace": "./land, https://cdn.rawgit.com/BabylonJS/Website/06ecbea7/Demos/Procedural/land"
         },
         {
             "title": "Instances",
@@ -818,7 +816,7 @@
             "functionToCall": "CreateInstancesTestScene",
             "referenceImage": "instances.png",
             "replace": "ground., Ground.",
-            "excludeFromAutomaticTesting": true
+            "excludedEngines": ["webgl1"]
         },
         {
             "title": "Instanced Bones",
@@ -882,8 +880,7 @@
             "playgroundId": "#XT1HAS#1",
             "referenceImage": "ssao2.png",
             "renderCount": 200,
-            "excludeFromAutomaticTesting": true,
-            "excludedEngines": ["webgpu"]
+            "excludedEngines": ["webgl1", "webgpu"]
         },
         {
             "title": "DepthRenderer",
@@ -940,24 +937,21 @@
             "playgroundId": "#E5YGEL#2",
             "referenceImage": "motionBlur.png",
             "renderCount": 10,
-            "excludeFromAutomaticTesting": true,
-            "excludedEngines": ["webgpu"]
+            "excludedEngines": ["webgl1", "webgpu"]
         },
         {
             "title": "Screen space reflections",
             "playgroundId": "#PIZ1GK#172",
             "referenceImage": "ssr.png",
             "renderCount": 10,
-            "excludeFromAutomaticTesting": true,
-            "excludedEngines": ["webgpu"]
+            "excludedEngines": ["webgl1", "webgpu"]
         },
         {
             "title": "Geometry buffer renderer",
             "playgroundId": "#PIZ1GK#173",
             "referenceImage": "geometrybufferrenderer.png",
             "renderCount": 10,
-            "excludeFromAutomaticTesting": true,
-            "excludedEngines": ["webgpu"]
+            "excludedEngines": ["webgl1", "webgpu"]
         }
     ]
 }

tests/validation/index.html

@@ -78,6 +78,7 @@
 							runTest(i, function() {
 								i++;
 								if (justOnce || i >= config.tests.length) {
+                                    showResultSummary();
 									return;
 								}
 								recursiveRunTest(i);

tests/validation/validation.js

@@ -6,6 +6,9 @@ var currentScene;
 var config;
 var justOnce;
 var engineName;
+var currentTestName;
+var numTestsOk = 0;
+var failedTests = [];
 
 // Random replacement
 var seed = 1;
@@ -142,6 +145,12 @@ async function evaluate(test, resultCanvas, result, renderImage, waitRing, done)
     engine._fps = 60;
     engine._performanceMonitor = new BABYLON.PerformanceMonitor();
 
+    if (testRes) {
+        numTestsOk++;
+    } else {
+        failedTests.push(currentTestName);
+    }
+
     done(testRes, renderB64);
 }
 
@@ -167,6 +176,7 @@ function processCurrentScene(test, resultCanvas, result, renderImage, index, wai
             }
             catch (e) {
                 console.error(e);
+                failedTests.push(currentTestName);
                 done(false);
             }
         });
@@ -219,6 +229,8 @@ function runTest(index, done, listname) {
 
     console.log("Running " + (listname ? listname + "/" : "") + test.title);
 
+    currentTestName = test.title;
+
     engine.beginFrame();
 
     var resultContext = resultCanvas.getContext("2d");
@@ -243,6 +255,7 @@ function runTest(index, done, listname) {
                 null,
                 function(loadedScene, msg) {
                     console.error(msg);
+                    failedTests.push(currentTestName);
                     done(false);
                 });
         }
@@ -266,6 +279,7 @@ function runTest(index, done, listname) {
                     }, retryTime);
                 }
                 else {
+                    failedTests.push(currentTestName);
                     done(false);
                 }
             }
@@ -365,12 +379,14 @@ function runTest(index, done, listname) {
                     }
                     catch (e) {
                         console.error(e);
+                        failedTests.push(currentTestName);
                         done(false);
                     }
                 }
             };
             request.onerror = function() {
                 console.error("Network error during test load.");
+                failedTests.push(currentTestName);
                 done(false);
             }
 
@@ -441,6 +457,13 @@ function init(_engineName) {
     }
 }
 
+function showResultSummary() {
+    console.log(`${numTestsOk} test(s) succeeded, ${failedTests.length} failed.`);
+    if (failedTests.length > 0) {
+        console.log(`List of failed test(s):\r\n  ${failedTests.join("\r\n  ")}`);
+    }
+}
+
 function dispose() {
     engine.dispose();
     currentScene = null;