Integrate Validation tests on pull requests

.travis.yml

@@ -1,8 +1,13 @@
 language: node_js
 node_js:
   - "6"
+addons:
+  sauce_connect:
+    username: "vandenberghe.sebastien@gmail.com"
+  jwt:
+    secure: kbaJ3o6cVqpB8Gzfd0yDSgGwH+SQXukevtpjMQ8Gip+N6FOumQxihPDqsqYKxc8svvR024nDmkIhbjPjz/YguvC3WUgYW4xk3Za7P7C9cpj2fYdrFuO7pD4sd/fNdUCqvKQ8jcxlIq4eEdBuoBTOHsP9J5KH7Z1M7e58atkx0o8=
 before_script:
   - npm install -g gulp
   - cd ./Tools/Gulp
   - npm install
-script: gulp typescript
+script: gulp

Tools/DevLoader/BabylonLoader.js

@@ -39,7 +39,7 @@ var BABYLONDEVTOOLS;
             dependencies = [];
             callback = null;
             min = (document.location.href.toLowerCase().indexOf('dist=min') > 0);
-            useDist = (min || document.location.href.toLowerCase().indexOf('dist=true') > 0);            
+            useDist = (min || useDist || document.location.href.toLowerCase().indexOf('dist=true') > 0);            
             babylonJSPath = '';
         }
 
@@ -91,6 +91,11 @@ var BABYLONDEVTOOLS;
             return this;
         }
 
+        Loader.prototype.useDist = function() {
+            useDist = true;
+            return this;
+        }
+
         Loader.prototype.dequeue = function () {
             if (queue.length == 0) {
                 console.log('Scripts loaded');
@@ -236,6 +241,9 @@ var BABYLONDEVTOOLS;
             // Modules
             if (loadModules) {
                 for (var i = 0; i< settings.modules.length; i++) {
+                    if (settings.modules[i] === "viewer") {
+                        continue;
+                    }
                     this.loadModule(settings[settings.modules[i]]);
                 }
             }

Tools/Gulp/gulpfile.js

@@ -25,6 +25,8 @@ var config = require("./config.json");
 
 var del = require("del");
 
+var karmaServer = require('karma').Server;
+
 var debug = require("gulp-debug");
 var includeShadersStream;
 var shadersStream;
@@ -418,7 +420,7 @@ var buildExternalLibrary = function (library, settings, watch) {
  * The default task, concat and min the main BJS files.
  */
 gulp.task("default", function (cb) {
-    runSequence("typescript-all", "intellisense", cb);
+    runSequence("typescript-all", "intellisense", "tests-saucelabs", cb);
 });
 
 gulp.task("mainBuild", function (cb) {
@@ -557,6 +559,32 @@ gulp.task("webserver", function () {
 gulp.task("run", ["watch", "webserver"], function () {
 });
 
+
+gulp.task("tests-integration", function (done) {
+    var kamaServerOptions = {
+        configFile: __dirname + "/../../tests/validation/karma.conf.js",
+        singleRun: false
+    };
+
+    var server = new karmaServer(kamaServerOptions, done);
+    server.start();
+});
+
+gulp.task("tests-saucelabs", function (done) {
+    if (!process.env.TRAVIS) {
+        done();
+        return;
+    }
+
+    var kamaServerOptions = {
+        configFile: __dirname + "/../../tests/validation/karma.conf.saucelabs.js",
+        singleRun: true
+    };
+
+    var server = new karmaServer(kamaServerOptions, done);
+    server.start();
+});
+
 gulp.task("clean-JS-MAP", function () {
     return del([
         "../../src/**/*.js.map", "../../src/**/*.js"

Tools/Gulp/package.json

@@ -43,7 +43,18 @@
         "through2": "~0.6.5",
         "ts-loader": "^2.3.7",
         "typescript": "^2.6.2",
-        "webpack-stream": "^4.0.0"
+        "webpack-stream": "^4.0.0",
+
+        "karma": "^2.0.0",
+        "karma-chrome-launcher": "^2.2.0",
+        "karma-sauce-launcher": "^1.2.0",
+
+        "mocha": "^4.0.1",
+        "chai": "^4.1.2",
+        "sinon": "^4.1.3",
+        "karma-mocha": "^1.3.0",
+        "karma-chai":"^0.1.0",
+        "karma-sinon": "^1.0.5"
     },
     "scripts": {
         "install": "npm --prefix ../../Playground/ install ../../Playground/ && gulp typescript-compile && gulp typescript-libraries && gulp deployLocalDev"

tests/validation/config.json

@@ -36,25 +36,25 @@
       "title": "Retail",
       "sceneFolder": "/Scenes/Retail/",
       "sceneFilename": "Retail.babylon",
-      "referenceImage": "Retail.png"
+      "referenceImage": "retail.png"
     },
     {
       "title": "Hill Valley",
       "sceneFolder": "/Scenes/hillvalley/",
       "sceneFilename": "HillValley.incremental.babylon",
-      "referenceImage": "HillValley.png"
+      "referenceImage": "Hillvalley.png"
     },
     {
       "title": "Heart",
       "sceneFolder": "/Scenes/Heart/",
       "sceneFilename": "Heart.babylon",
-      "referenceImage": "heart.png"
+      "referenceImage": "Heart.png"
     },
     {
       "title": "Mansion",
       "sceneFolder": "/Scenes/Mansion/",
       "sceneFilename": "Mansion.babylon",
-      "referenceImage": "Mansion.png"
+      "referenceImage": "mansion.png"
     },
     {
       "title": "SpaceDeK",
@@ -241,7 +241,8 @@
       "renderCount": 20,
       "scriptToRun": "/Demos/GLTF1CesiumMan/index.js",
       "functionToCall": "createScene",
-      "referenceImage": "gltf1CesiumMan.png"
+      "referenceImage": "gltf1CesiumMan.png",
+      "onlyVisual": true
     },
     {
       "title": "GLTF Mesh Primitive Attribute Test",
@@ -263,13 +264,14 @@
       "renderCount": 20,
       "scriptToRun": "/Demos/PBRRough/index.js",
       "functionToCall": "CreatePBRRoughScene",
-      "referenceImage": "PBRRough.png"
+      "referenceImage": "pbrrough.png"
     },
     {
       "title": "GLTF LOD",
       "renderCount": 500,
       "playgroundId": "#TC80RU#41",
-      "referenceImage": "GLTF LOD.png"
+      "referenceImage": "GLTF LOD.png",
+      "onlyVisual": true
     },
     {
       "title": "Reflection probes",
@@ -336,7 +338,8 @@
       "scriptToRun": "/Demos/Procedural/proceduralTexture.js",
       "functionToCall": "CreateProceduralTextureTestScene",
       "referenceImage": "procedural.png",
-      "replace": "./land, https://cdn.rawgit.com/BabylonJS/Website/06ecbea7/Demos/Procedural/land"
+      "replace": "./land, https://cdn.rawgit.com/BabylonJS/Website/06ecbea7/Demos/Procedural/land",
+      "onlyVisual": true
     },
     {
       "title": "Water material (only visual check)",

tests/validation/index.html

@@ -8,7 +8,45 @@
 <body>
 	<script>
         BABYLONDEVTOOLS.Loader.require('validation.js')
-            .load();
+			.load(function() {
+				// Loading tests
+				var xhr = new XMLHttpRequest();
+
+				xhr.open("GET", "config.json", true);
+
+				xhr.addEventListener("load", function () {
+					if (xhr.status === 200) {
+
+						config = JSON.parse(xhr.responseText);
+
+						// Run tests
+						var index = 0;
+						if (window.location.search) {
+							justOnce = true;
+							var title = window.location.search.replace("?", "").replace(/%20/g, " ");
+							for (var index = 0; index < config.tests.length; index++) {
+								if (config.tests[index].title === title) {
+									break;
+								}
+							}
+						}
+
+						var recursiveRunTest = function(i) {
+							runTest(i, function() {
+								i++;
+								if (justOnce || i >= config.tests.length) {
+									return;
+								}
+								recursiveRunTest(i);
+							});
+						}
+
+						recursiveRunTest(index);
+					}
+				}, false);
+
+				xhr.send();
+			});
     </script>	
 </body>
 </html>

tests/validation/integration.js

@@ -0,0 +1,57 @@
+window.__karma__.loaded = function() {};
+
+// Loading tests
+var xhr = new XMLHttpRequest();
+
+xhr.open("GET", "/tests/validation/config.json", true);
+
+xhr.addEventListener("load", function () {
+    if (xhr.status === 200) {
+
+        config = JSON.parse(xhr.responseText);
+
+        describe("Validation Tests", function() {
+            before(function (done) {
+                this.timeout(180000);
+                require = null;
+                BABYLONDEVTOOLS.Loader
+                .require('/tests/validation/validation.js')
+                .useDist()
+                .load(function() {
+                    done();            
+                });
+            });
+        
+            // Run tests
+            for (let index = 0; index < config.tests.length; index++) {
+                var test = config.tests[index];
+                if (test.onlyVisual) {
+                    continue;
+                }
+
+                it(test.title, function(done) {
+                    this.timeout(180000);
+        
+                    try {
+                        runTest(index, function(result) {
+                            try {
+                                expect(result).to.be.true; 
+                                done();
+                            }
+                            catch (e) {
+                                done(e);
+                            }
+                        });                
+                    }
+                    catch (e) {
+                        done(e);
+                    }
+                });
+            };
+        });
+
+        window.__karma__.start();          
+    }
+}, false);
+
+xhr.send();

tests/validation/karma.conf.js

@@ -0,0 +1,36 @@
+module.exports = function (config) {
+    'use strict';
+    config.set({
+
+        basePath: '../../',
+        browserNoActivityTimeout: 1800000,
+
+        urlRoot: '/karma',
+
+        frameworks: ['mocha', 'chai', 'sinon'],
+
+        files: [
+            './Tools/DevLoader/BabylonLoader.js',
+            './tests/validation/index.css',
+            './tests/validation/integration.js',
+            { pattern: '**/*', watched: false, included: false, served: true },
+        ],
+        proxies: {
+            '/': '/base/'
+        },
+
+        reporters: ['progress'],
+
+        port: 1338,
+        colors: true,
+        autoWatch: false,
+        singleRun: false,
+
+        // level of logging
+        // possible values: config.LOG_DISABLE || config.LOG_ERROR || config.LOG_WARN || config.LOG_INFO || config.LOG_DEBUG
+        logLevel: config.LOG_INFO,
+
+        browsers: ['Chrome']
+
+    });
+};

tests/validation/karma.conf.saucelabs.js

@@ -0,0 +1,65 @@
+module.exports = function (config) {
+    'use strict';
+    config.set({
+
+        basePath: '../../',
+        browserNoActivityTimeout: 1800000,
+
+        urlRoot: '/karma',
+
+        frameworks: ['mocha', 'chai', 'sinon'],
+
+        files: [
+            './Tools/DevLoader/BabylonLoader.js',
+            './tests/validation/index.css',
+            './tests/validation/integration.js',
+            { pattern: '**/*', watched: false, included: false, served: true },
+        ],
+        proxies: {
+            '/': '/base/'
+        },
+
+        //reporters: ['progress'],
+
+        port: 1338,
+        colors: true,
+        autoWatch: false,
+        singleRun: false,
+
+        // level of logging
+        // possible values: config.LOG_DISABLE || config.LOG_ERROR || config.LOG_WARN || config.LOG_INFO || config.LOG_DEBUG
+        logLevel: config.LOG_ERROR,
+
+        //browsers: ['Chrome'],
+
+        sauceLabs: {
+            testName: 'Web App Unit Tests'
+        },
+        customLaunchers: {
+            sl_firefox: {
+              base: 'SauceLabs',
+              browserName: 'firefox',
+              platform: 'Windows 10',
+              version: '57'
+            },
+            sl_ie_11: {
+                base: 'SauceLabs',
+                browserName: 'internet explorer',
+                platform: 'Windows 10',
+                version: '11'
+            },
+            sl_android: {
+                base: 'SauceLabs',
+                browserName: 'Browser',
+                platform: 'Android',
+                version: '4.4',
+                deviceName: 'Samsung Galaxy S3 Emulator',
+                deviceOrientation: 'portrait'
+            }
+        },
+        browsers: ['sl_firefox'],
+        reporters: ['dots', 'saucelabs'],
+        singleRun: true
+
+    });
+};

tests/validation/validation.js

@@ -76,16 +76,19 @@ function saveRenderImage(data, canvas) {
     return screenshotCanvas.toDataURL();
 }
 
-function evaluate(test, resultCanvas, result, renderImage, index, waitRing) {
+function evaluate(test, resultCanvas, result, renderImage, index, waitRing, done) {
     var renderData = getRenderData(canvas, engine);
+    var testRes = true;
     if (!test.onlyVisual) {
 
         if (compare(renderData, resultCanvas)) {
             result.classList.add("failed");
             result.innerHTML = "×";
+            testRes = false;
             console.log('%c failed', 'color: red');
         } else {
             result.innerHTML = "✔";
+            testRes = true;
             console.log('%c validated', 'color: green');
         }
     }
@@ -95,12 +98,10 @@ function evaluate(test, resultCanvas, result, renderImage, index, waitRing) {
 
     currentScene.dispose();
 
-    if (!justOnce) {
-        runTest(index + 1);
-    }
+    done(testRes);
 }
 
-function processCurrentScene(test, resultCanvas, result, renderImage, index, waitRing) {
+function processCurrentScene(test, resultCanvas, result, renderImage, index, waitRing, done) {
     currentScene.executeWhenReady(function () {
         var renderCount = test.renderCount || 1;
 
@@ -111,19 +112,16 @@ function processCurrentScene(test, resultCanvas, result, renderImage, index, wai
 
             if (renderCount === 0) {
                 engine.stopRenderLoop();
-                evaluate(test, resultCanvas, result, renderImage, index, waitRing);
+                evaluate(test, resultCanvas, result, renderImage, index, waitRing, done);
             }
         });
 
     });
 }
 
-function
-
-
-runTest(index) {
+function runTest(index, done) {
     if (index >= config.tests.length) {
-        return;
+        done(false);
     }
 
     var test = config.tests[index];
@@ -147,7 +145,7 @@ runTest(index) {
     var waitRing = document.createElement("img");
     waitRing.className = "waitRing";
     titleContainer.appendChild(waitRing);
-    waitRing.src = "loading.gif";
+    waitRing.src = "/tests/integration/loading.gif";
 
     var resultCanvas = document.createElement("canvas");
     resultCanvas.className = "resultImage";
@@ -165,7 +163,7 @@ runTest(index) {
         resultContext.drawImage(img, 0, 0);
     }
 
-    img.src = "ReferenceImages/" + test.referenceImage;
+    img.src = "/tests/integration/ReferenceImages/" + test.referenceImage;
 
     var renderImage = new Image();
     renderImage.className = "renderImage";
@@ -176,12 +174,12 @@ runTest(index) {
     if (test.sceneFolder) {
         BABYLON.SceneLoader.Load(config.root + test.sceneFolder, test.sceneFilename, engine, function (newScene) {
             currentScene = newScene;
-            processCurrentScene(test, resultCanvas, result, renderImage, index, waitRing);
+            processCurrentScene(test, resultCanvas, result, renderImage, index, waitRing, done);
         });
     }
     else if (test.playgroundId) {
         var snippetUrl = "//babylonjs-api2.azurewebsites.net/snippets";
-        var pgRoot = "/playground"
+        var pgRoot = "/Playground"
         var xmlHttp = new XMLHttpRequest();
         xmlHttp.onreadystatechange = function () {
             if (xmlHttp.readyState === 4) {
@@ -194,7 +192,7 @@ runTest(index) {
                     code = code.replace(/"scenes\//g, "\"" + pgRoot + "/scenes/");
 
                     currentScene = eval(code + "\r\ncreateScene(engine)");
-                    processCurrentScene(test, resultCanvas, result, renderImage, index, waitRing);
+                    processCurrentScene(test, resultCanvas, result, renderImage, index, waitRing, done);
                 }
             }
         }
@@ -238,7 +236,7 @@ runTest(index) {
                 }
 
                 currentScene = eval(scriptToRun + test.functionToCall + "(engine)");
-                processCurrentScene(test, resultCanvas, result, renderImage, index, waitRing);
+                processCurrentScene(test, resultCanvas, result, renderImage, index, waitRing, done);
             }
         };
 
@@ -256,31 +254,3 @@ canvas.className = "renderCanvas";
 document.body.appendChild(canvas);
 engine = new BABYLON.Engine(canvas, false);
 engine.setDitheringState(false);
-
-// Loading tests
-var xhr = new XMLHttpRequest();
-
-xhr.open("GET", "config.json", true);
-
-xhr.addEventListener("load", function () {
-    if (xhr.status === 200) {
-
-        config = JSON.parse(xhr.responseText);
-
-        // Run tests
-        var index = 0;
-        if (window.location.search) {
-            justOnce = true;
-            var title = window.location.search.replace("?", "").replace(/%20/g, " ");
-            for (var index = 0; index < config.tests.length; index++) {
-                if (config.tests[index].title === title) {
-                    break;
-                }
-            }
-        }
-        runTest(index);
-
-    }
-}, false);
-
-xhr.send();