Merge pull request #183 from gmetais/develop

v1.11.0

.bowerrc

@@ -1,3 +0,0 @@
-{
-  "directory" : "bower_components"
-}

.gitignore

@@ -1,5 +1,4 @@
 node_modules
-bower_components
 .tmp
 .vagrant
 results/*

.travis.yml

@@ -1,6 +1,18 @@
 language: node_js
+sudo: false
 node_js:
-    - "0.10.33"
+  - "6.2"
+  - "4.0"
+env:
+  - CXX=g++-4.8
+addons:
+  apt:
+    sources:
+      - ubuntu-toolchain-r-test
+    packages:
+      - g++-4.8
 before_install:
-    - "npm install -g grunt-cli"
-install: npm install
+  - "npm install -g npm"
+  - "npm install -g grunt-cli"
+install:
+  - "npm install"

Gruntfile.js

@@ -143,7 +143,7 @@ module.exports = function(grunt) {
                 options: {
                     reporter: 'spec',
                 },
-                src: ['test/core/offendersHelpersTest.js']
+                src: ['test/core/imageOptimizerTest.js']
             },
             coverage: {
                 options: {

README.md

@@ -1,6 +1,6 @@
 <img src="./doc/img/logo.png" alt="Yellow Lab Tools"></img>
 
-Analyzes a webpage and detects **performance** or **front-end code quality** issues. Easy to launch, easy to understand, in-depth details.
+Analyzes a webpage and detects **performance** or **front-end code quality** issues. Free, easy to launch, easy to understand, in-depth details.
 
 
 #  
@@ -75,18 +75,15 @@ You can use [ngrok](https://ngrok.com/), a tool that creates a secure tunnel bet
 If your project is not accessible from outside, or if you want to fork and improve the tool, you can build your own instance. The documentation is [here](https://github.com/gmetais/YellowLabTools/wiki/Install-your-private-server).
 
 
-## Help needed!
-There are so many things left to do, **your help would be greatly appreciated**! Please report bugs, ask for evolutions and come code with me.
-
-
-## License
-Please read the [license](LICENSE). Icons are the property of [IcoMoon.io](https://icomoon.io/)
-
-
 ## Author
-Gaël Métais. I'm a webperf freelance based in Paris. If you understand French, you can visit [my website](http://www.gaelmetais.com).
+Gaël Métais. I'm a webperf freelance. Follow me on Twitter [@gaelmetais](https://twitter.com/gaelmetais), I tweet about Web Performances, Front-end and new versions of YellowLabTools!
+
+If you understand French, you can visit [my website](http://www.gaelmetais.com) (will be soon in English too).
 
 
 ## Contributors
 - Achraf Ben Younes [achrafbenyounes](https://github.com/achrafbenyounes)
+- [camlafit](https://github.com/camlafit)
+- Vincent L. [magikcypress](https://github.com/magikcypress)
+- Ousama Ben Younes [ousamabenyounes](https://github.com/ousamabenyounes)
 

Vagrantfile

@@ -1,17 +1,22 @@
 Vagrant.configure("2") do |config|
 
-  config.vm.box = "precise64"
-  config.vm.box_url = "http://files.vagrantup.com/precise64.box"
+  config.vm.box = "ubuntu/precise64"
 
   config.vm.network :private_network, ip: "10.10.10.10"
   config.ssh.forward_agent = true
 
+  # http://foo-o-rama.com/vagrant--stdin-is-not-a-tty--fix.html
+  config.vm.provision "fix-no-tty", type: "shell" do |s|
+    s.privileged = false
+    s.inline = "sudo sed -i '/tty/!s/mesg n/tty -s \\&\\& mesg n/' /root/.profile"
+  end
+
   config.vm.provider :virtualbox do |vb|
     vb.customize ["modifyvm", :id, "--natdnshostresolver1", "on"]
-    vb.customize ["modifyvm", :id, "--memory", 512]
+    vb.customize ["modifyvm", :id, "--memory", 1024]
     vb.customize ["modifyvm", :id, "--cpus", 2]
   end
 
   config.vm.provision :shell, :path => "server_config/server_install.sh"
 
-end
+end

bin/cli.js

@@ -1,10 +1,11 @@
 #!/usr/bin/env node
 
-var debug = require('debug')('ylt:cli');
-var meow = require('meow');
-var path = require('path');
+var debug       = require('debug')('ylt:cli');
+var meow        = require('meow');
+var path        = require('path');
+var EasyXml     = require('easyxml');
 
-var ylt = require('../lib/index');
+var ylt         = require('../lib/index');
 
 var cli = meow({
     help: [
@@ -14,11 +15,11 @@ var cli = meow({
         'Options:',
         '  --device             Use "phone" or "tablet" to simulate a mobile device (by user-agent and viewport size).',
         '  --screenshot         Will take a screenshot and use this value as the output path. It needs to end with ".png".',
-        '  --js-deep-analysis   When activated, the javascriptExecutionTree will contain sub-requests.',
         '  --wait-for-selector  Once the page is loaded, Phantomas will wait until the given CSS selector matches some elements.',
         '  --cookie             Adds a cookie on the main domain.',
         '  --auth-user          Basic HTTP authentication username.',
         '  --auth-pass          Basic HTTP authentication password.',
+        '  --reporter           The output format: "json" or "xml". Default is "json".',
         ''
     ].join('\n'),
     pkg: '../package.json'
@@ -49,11 +50,6 @@ if (screenshot) {
     options.screenshot = cli.flags.screenshot;
 }
 
-// Deep JS analysis option
-if (cli.flags.jsDeepAnalysis === true || cli.flags.jsDeepAnalysis === 'true') {
-    options.jsDeepAnalysis = true;
-}
-
 // Device simulation
 options.device = cli.flags.device || 'desktop';
 
@@ -67,6 +63,12 @@ options.cookie = cli.flags.cookie || null;
 options.authUser = cli.flags.authUser || null;
 options.authPass = cli.flags.authPass || null;
 
+// Output format
+if (cli.flags.reporter && cli.flags.reporter !== 'json' && cli.flags.reporter !== 'xml') {
+    console.error('Incorrect parameters: reporter has to be "json" or "xml"');
+    process.exit(1);
+}
+
 
 (function execute(url, options) {
     'use strict';
@@ -76,7 +78,14 @@ options.authPass = cli.flags.authPass || null;
         then(function(data) {
 
             debug('Success');
-            console.log(JSON.stringify(data, null, 2));
+            switch(cli.flags.reporter) {
+                case 'xml':
+                    var serializer = new EasyXml();
+                    console.log(serializer.render(data));
+                    break;
+                default:
+                    console.log(JSON.stringify(data, null, 2));
+            }
 
         }).fail(function(err) {
             

bin/server.js

@@ -17,6 +17,7 @@ app.use(cors());
 app.use(wwwRedirectMiddleware);
 app.use(authMiddleware);
 app.use(apiLimitsMiddleware);
+app.use(bodyParser.urlencoded({ extended: true })); // for parsing application/x-www-form-urlencoded
 
 
 // Initialize the controllers
@@ -37,4 +38,4 @@ if (!process.env.GRUNTED) {
     });
 }
 
-module.exports = app;
+module.exports = app;

bower.json

@@ -1,14 +0,0 @@
-{
-  "name": "yellowlabtools",
-  "dependencies": {
-    "angular": "~1.3.15",
-    "angular-route": "~1.3.15",
-    "angular-resource": "~1.3.15",
-    "angular-sanitize": "~1.3.15",
-    "angular-animate": "~1.3.15",
-    "angular-local-storage": "~0.2.0"
-  },
-  "resolutions": {
-    "angular": "~1.3.8"
-  }
-}

front/src/css/dashboard.css

@@ -109,14 +109,18 @@
 .summary .notations .criteria .table {
   width: 100%;
 }
-.summary .notations .criteria .table > div:hover > div {
+.summary .notations .criteria .table > a {
+  text-decoration: none;
+  color: inherit;
+}
+.summary .notations .criteria .table > a:hover > div {
   background: #EBD8E2;
   cursor: pointer;
 }
-.summary .notations .criteria .table > div:hover > div.info {
+.summary .notations .criteria .table > a:hover > div.info {
   background: #FFF;
 }
-.summary .notations .criteria .table > div:hover > div.info .icon-question {
+.summary .notations .criteria .table > a:hover > div.info .icon-question {
   color: #EBD8E2;
 }
 .summary .notations .criteria .grade {

front/src/css/index.css

@@ -33,7 +33,9 @@
 .settings select {
   font-size: 1em;
 }
-.settings input[type=text] {
+.settings input[type=text],
+.settings input[type=password],
+.settings textarea {
   width: 100%;
   min-width: 4em;
 }
@@ -128,6 +130,7 @@
 .advanced > div > div.label {
   width: 25%;
   white-space: nowrap;
+  vertical-align: middle;
 }
 .advanced .subTable {
   display: table;

front/src/css/rule.css

@@ -29,9 +29,16 @@
   border-radius: 0.5em;
   margin: 0 auto 0.5em;
 }
+.rule h3 {
+  margin-bottom: 0em;
+}
+.rule .okThreshold {
+  font-style: italic;
+  font-size: 0.9em;
+}
 .rule .message {
   width: 80%;
-  margin: 0 auto;
+  margin: 1.5em auto;
 }
 .rule .message p {
   margin: 0.5em;
@@ -94,6 +101,7 @@
   margin: 0.2em 0;
   border-radius: 0.4em;
   z-index: 1;
+  cursor: pointer;
 }
 .offendersTable .offenderButton.opens,
 .value .offenderButton.opens {
@@ -150,6 +158,10 @@
   display: block;
   background: #ffe0cc;
 }
+.offendersTable .smallerOffenders,
+.value .smallerOffenders {
+  font-size: 0.9em;
+}
 .offendersHtml {
   display: inline-block;
 }
@@ -164,15 +176,17 @@
   font-style: italic;
   font-weight: normal;
 }
+.checker {
+  /* Checkerboard background */
+  background-color: #ddd;
+  background-image: linear-gradient(45deg, #AAA 25%, transparent 25%, transparent 75%, #AAA 75%, #AAA), linear-gradient(45deg, #AAA 25%, transparent 25%, transparent 75%, #AAA 75%, #AAA);
+  background-size: 1em 1em;
+  background-position: 0 0, 0.5em 0.5em;
+}
 .colorPalette {
   width: 30em;
   border: 2px solid #000;
   text-align: left;
-  /* Checkerboard background */
-  background-color: #ddd;
-  background-image: linear-gradient(45deg, #aaaaaa 25%, transparent 25%, transparent 75%, #aaaaaa 75%, #aaaaaa), linear-gradient(45deg, #aaaaaa 25%, transparent 25%, transparent 75%, #aaaaaa 75%, #aaaaaa);
-  background-size: 1em 1em;
-  background-position: 0 0, 0.5em 0.5em;
 }
 .colorPalette > div {
   display: inline-block;
@@ -205,3 +219,50 @@
   z-index: 2;
   border: 0.2em solid #f1c40f;
 }
+.similarColors {
+  margin: 1em;
+  width: 20em;
+  height: 6em;
+}
+.similarColors > div {
+  display: inline-block;
+  width: 10em;
+  height: 3.5em;
+  padding-top: 2.5em;
+}
+.totalWeightPie {
+  max-width: 39em;
+  margin: 2em auto 4em;
+}
+.totalWeightPie canvas {
+  max-width: inherit;
+}
+.hugeFile {
+  font-weight: bold;
+  color: #e74c3c;
+}
+.imageOffenders {
+  display: table;
+  border-spacing: 3em;
+  width: 90%;
+}
+.imageOffenders > div {
+  display: table-row;
+}
+.imageOffenders > div > div {
+  display: table-cell;
+  vertical-align: middle;
+}
+.imageOffenders img {
+  max-height: 10em;
+  max-width: 40em;
+  border: 1px solid #000;
+  margin-top: 0.5em;
+}
+.smallPreview {
+  display: block;
+  max-height: 4em;
+  max-width: 8em;
+  border: 1px solid #000;
+  margin: 1em auto 0.2em;
+}

front/src/css/timeline.css

@@ -62,34 +62,34 @@
   background: #F04DA7;
 }
 .timeline .domComplete.interval {
-  background: #ede3ff;
+  background: #EDE3FF;
 }
 .timeline .domComplete .color {
-  background: #c2a3ff;
+  background: #C2A3FF;
 }
 .timeline .domContentLoadedEnd.interval {
-  background: #d8f0f0;
+  background: #D8F0F0;
 }
 .timeline .domContentLoadedEnd .color {
-  background: #7ecccc;
+  background: #7ECCCC;
 }
 .timeline .domContentLoaded.interval {
-  background: #e0ffd1;
+  background: #E0FFD1;
 }
 .timeline .domContentLoaded .color {
-  background: #a7e846;
+  background: #A7E846;
 }
 .timeline .domInteractive.interval {
-  background: #fffccc;
+  background: #FFFCCC;
 }
 .timeline .domInteractive .color {
-  background: #ffe433;
+  background: #FFE433;
 }
 .timeline .domCreation.interval {
-  background: #ffe0cc;
+  background: #FFE0CC;
 }
 .timeline .domCreation .color {
-  background: #ff6600;
+  background: #FF6600;
 }
 .timeline .tooltip.detailsOverlay {
   position: absolute;
@@ -152,14 +152,16 @@
   width: 100%;
   border-spacing: 0.25em;
 }
-.table > div {
+.table > div,
+.table > a {
   display: table-row;
 }
 .table > .headers > div {
   font-weight: bold;
   padding: 0.5em 1em;
 }
-.table > div > div {
+.table > div > div,
+.table > a > div {
   padding: 0.1em 1em;
   background: #f2f2f2;
   display: table-cell;
@@ -292,19 +294,19 @@
   white-space: nowrap;
 }
 .table > div > .startTime.domComplete {
-  background: #ede3ff;
+  background: #EDE3FF;
 }
 .table > div > .startTime.domContentLoadedEnd {
-  background: #d8f0f0;
+  background: #D8F0F0;
 }
 .table > div > .startTime.domContentLoaded {
-  background: #e0ffd1;
+  background: #E0FFD1;
 }
 .table > div > .startTime.domInteractive {
-  background: #fffccc;
+  background: #FFFCCC;
 }
 .table > div > .startTime.domCreation {
-  background: #ffe0cc;
+  background: #FFE0CC;
 }
 .execution .icon-warning {
   color: #e74c3c;

front/src/js/app.js

@@ -37,6 +37,11 @@ yltApp.run(['$rootScope', '$location', function($rootScope, $location) {
     $rootScope.$on('$routeChangeSuccess', function(){
         ga('send', 'pageview', {'page': $location.path()});
     });
+
+    // GitHub star button (asynchronously loaded iframe)
+    window.addEventListener('load', function() {
+        window.document.getElementById('ghbtn').src = 'http://ghbtns.com/github-btn.html?user=gmetais&repo=YellowLabTools&type=star&count=true&size=large';
+    });
 }]);
 
 yltApp.config(['$routeProvider', '$locationProvider',
@@ -77,3 +82,7 @@ yltApp.config(['$routeProvider', '$locationProvider',
     }
 ]);
 
+// Disable debugging https://docs.angularjs.org/guide/production
+yltApp.config(['$compileProvider', function ($compileProvider) {
+    $compileProvider.debugInfoEnabled(false);
+}]);

front/src/js/controllers/dashboardCtrl.js

@@ -25,13 +25,11 @@ dashboardCtrl.controller('DashboardCtrl', ['$scope', '$rootScope', '$routeParams
         // By default, Angular sorts object's attributes alphabetically. Countering this problem by retrieving the keys order here.
         $scope.categoriesOrder = Object.keys($scope.result.scoreProfiles.generic.categories);
         
+        $scope.globalScore = Math.max($scope.result.scoreProfiles.generic.globalScore, 0);
+
         $scope.tweetText = 'I\'ve discovered this cool open-source tool that audits the front-end quality of a web page: ';
     }
 
-    $scope.showRulePage = function(ruleName) {
-        $location.path('/result/' + $scope.runId + '/rule/' + ruleName);
-    };
-
     $scope.testAgain = function() {
         API.relaunchTest($scope.result);
     };

front/src/js/controllers/ruleCtrl.js

@@ -1,4 +1,13 @@
-var ruleCtrl = angular.module('ruleCtrl', []);
+var ruleCtrl = angular.module('ruleCtrl', ['chart.js']);
+
+ruleCtrl.config(['ChartJsProvider', function (ChartJsProvider) {
+    // Configure all charts
+    ChartJsProvider.setOptions({
+        animation: false,
+        colours: ['#FF5252', '#FF8A80'],
+        responsive: true
+    });
+}]);
 
 ruleCtrl.controller('RuleCtrl', ['$scope', '$rootScope', '$routeParams', '$location', '$sce', 'Menu', 'Results', 'API', function($scope, $rootScope, $routeParams, $location, $sce, Menu, Results, API) {
     $scope.runId = $routeParams.runId;
@@ -22,6 +31,72 @@ ruleCtrl.controller('RuleCtrl', ['$scope', '$rootScope', '$routeParams', '$locat
 
     function init() {
         $scope.rule = $scope.result.rules[$scope.policyName];
+
+        // Init "Total Weight" chart
+        if ($scope.policyName === 'totalWeight') {
+            $scope.weightLabels = [];
+            $scope.weightColours = ['#7ECCCC', '#A7E846', '#FF944D', '#FFE74A', '#C2A3FF', '#5A9AED', '#FF6452', '#C1C1C1'];
+            $scope.weightData = [];
+
+            var types = ['html', 'css', 'js', 'json', 'image', 'video', 'webfont', 'other'];
+            types.forEach(function(type) {
+                $scope.weightLabels.push(type);
+                $scope.weightData.push(Math.round($scope.rule.offendersObj.list.byType[type].totalWeight / 1024));
+            });
+
+            $scope.weightOptions = {
+                tooltipTemplate: '<%=label%>: <%=value%> KB'
+            };
+        }
+
+        // Init "Breakpoints" chart
+        if ($scope.policyName === 'cssBreakpoints' && $scope.rule.value > 0) {
+
+            // Seek for the biggest breakpoint
+            var max = 0;
+            $scope.rule.offendersObj.forEach(function(offender) {
+                if (offender.pixels > max) {
+                    max = offender.pixels;
+                }
+            });
+            max = Math.max(max + 100, 1400);
+
+            // We group offenders 10px by 10px
+            var GROUP_SIZE = 20;
+
+            // Generate an empty array of values
+            $scope.breakpointsLabels = [];
+            $scope.breakpointsData = [[]];
+            for (var i = 0; i <= max / GROUP_SIZE; i++) {
+                $scope.breakpointsLabels[i] = '';
+                $scope.breakpointsData[0][i] = 0;
+            }
+
+            // Fill it with results
+            $scope.rule.offendersObj.forEach(function(offender) {
+                var group = Math.floor((offender.pixels + 1) / GROUP_SIZE);
+
+                if ($scope.breakpointsLabels[group] !== '') {
+                    $scope.breakpointsLabels[group] += '/';
+                }
+                $scope.breakpointsLabels[group] += offender.breakpoint;
+
+                $scope.breakpointsData[0][group] += offender.count;
+            });
+
+            $scope.breakpointsSeries = ['Number of CSS rules per breakpoint'];
+            $scope.breakpointsColours = ['#9c4274'];
+            $scope.breakpointsOptions = {
+                scaleShowGridLines: false,
+                barShowStroke: false,
+                showTooltips: false,
+                pointDot: false,
+                responsive: true,
+                maintainAspectRatio: true,
+                strokeColor: 'rgba(20, 200, 20, 1)',
+                scaleFontSize: 9
+            };
+        }
     }
 
     $scope.backToDashboard = function() {

front/src/js/controllers/timelineCtrl.js

@@ -44,19 +44,19 @@ timelineCtrl.controller('TimelineCtrl', ['$scope', '$rootScope', '$routeParams',
     }
 
     function initScriptFiltering() {
-        var offenders = $scope.result.rules.jsCount.offendersObj.list;
+        var offenders = $scope.result.rules.totalRequests.offendersObj.list.byType.js;
         $scope.scripts = [];
 
-        offenders.forEach(function(script) {
-            var filePath = script.file;
+        offenders.forEach(function(filePath) {
+            var shortPath = filePath;
 
             if (filePath.length > 100) {
-                filePath = filePath.substr(0, 98) + '...';
+                shortPath = filePath.substr(0, 98) + '...';
             }
 
             var scriptObj = {
-                fullPath: script.file, 
-                shortPath: filePath
+                fullPath: filePath,
+                shortPath: shortPath
             };
 
             $scope.scripts.push(scriptObj);

front/src/js/directives/offendersDirectives.js

@@ -608,7 +608,12 @@
             for (var i = 0 ; i < parsedBacktrace.length ; i++) {
                 html += '<div>';
                 html += '<div>' + (parsedBacktrace[i].fnName || '(anonymous function)') + '</div>';
-                html += '<div class="trace">' + getUrlLink(parsedBacktrace[i].filePath, 40) + ':' + parsedBacktrace[i].line + '</div>';
+                html += '<div class="trace">' + getUrlLink(parsedBacktrace[i].filePath, 40) + '</div>';
+                if (parsedBacktrace[i].column) {
+                    html += '<div>' + parsedBacktrace[i].line + ':' + parsedBacktrace[i].column + '</div>';    
+                } else {
+                    html += '<div>line ' + parsedBacktrace[i].line + '</div>';
+                }
                 html += '</div>';
             }
         }
@@ -630,6 +635,22 @@
                 var fnName = null, fileAndLine;
 
                 var withFnResult = /^([^\s\(]+) \((.+:\d+)\)$/.exec(trace);
+                
+                if (withFnResult === null) {
+                    // Try the PhantomJS 2 format
+                    withFnResult = /^([^\s\(]+) \((.+:\d+:\d+)\)$/.exec(trace);
+                }
+
+                if (withFnResult === null) {
+                    // Yet another PhantomJS 2 format?
+                    withFnResult = /^([^\s\(]+|global code)@(.+:\d+:\d+)$/.exec(trace);
+                }
+
+                if (withFnResult === null) {
+                    // Try the PhantomJS 2 ERROR format
+                    withFnResult = /^([^\s\(]+) (http.+:\d+)$/.exec(trace);
+                }
+
                 if (withFnResult === null) {
                     fileAndLine = trace;
                 } else {
@@ -637,15 +658,26 @@
                     fileAndLine = withFnResult[2];
                 }
 
-                var fileAndLineSplit = /^(.*):(\d+)$/.exec(fileAndLine);
+                // And now the second part
+                var fileAndLineSplit = /^(.*):(\d+):(\d+)$/.exec(fileAndLine);
+
+                if (fileAndLineSplit === null) {
+                    fileAndLineSplit = /^(.*):(\d+)$/.exec(fileAndLine);
+                }
+
                 var filePath = fileAndLineSplit[1];
                 var line = fileAndLineSplit[2];
+                var column = fileAndLineSplit[3];
 
-                out.push({
-                    fnName: fnName,
-                    filePath: filePath,
-                    line: line
-                });
+                // Filter phantomas code
+                if (filePath.indexOf('phantomjs://') === -1) {
+                    out.push({
+                        fnName: fnName,
+                        filePath: filePath,
+                        line: line,
+                        column: column
+                    });
+                }
             });
 
         } catch(e) {
@@ -856,4 +888,34 @@
         };
     });
 
+    offendersDirectives.filter('bytes', function() {
+        return function(bytes) {
+            if (isNaN(parseFloat(bytes)) || !isFinite(bytes)) {
+                return '-';
+            }
+            
+            var kilo = bytes / 1024;
+
+            if (kilo < 1) {
+                return bytes + ' bytes';
+            }
+
+            if (kilo < 100) {
+                return kilo.toFixed(1) + ' KB';
+            }
+
+            if (kilo < 1024) {
+                return kilo.toFixed(0) + ' KB';
+            }
+
+            var mega = kilo / 1024;
+
+            if (mega < 10) {
+                return mega.toFixed(2) + ' MB';
+            }
+
+            return mega.toFixed(1) + ' MB';
+        };
+    });
+
 })();

front/src/js/services/apiService.js

@@ -9,25 +9,26 @@ apiService.factory('API', ['$location', 'Runs', 'Results', function($location, R
                 url: url,
                 waitForResponse: false,
                 screenshot: true,
-                jsTimeline: true,
                 device: settings.device,
                 waitForSelector: settings.waitForSelector,
                 cookie: settings.cookie,
                 authUser: settings.authUser,
                 authPass: settings.authPass,
+                blockDomain: settings.blockDomain,
+                allowedDomains: settings.allowedDomains,
+                noExternals: settings.noExternals
             };
 
-            if (settings.waitForSelector && settings.waitForSelector !== '') {
-                runObject.waitForSelector = settings.waitForSelector;
-            }
-
-            if (settings.cookie && settings.cookie !== '') {
-                runObject.cookie = settings.cookie;
-            }
-
-            if (settings.authUser && settings.authUser !== '' && settings.authPass && settings.authPass !== '') {
-                runObject.authUser = settings.authUser;
-                runObject.authPass = settings.authPass;
+            
+            if (settings.domainsBlackOrWhite === 'black') {
+                runObject.blockDomain = this.parseDomains(settings.domains);
+            } else if (settings.domainsBlackOrWhite === 'white') {
+                var allowedDomains = this.parseDomains(settings.domains);
+                if (allowedDomains.length > 0) {
+                    runObject.allowDomain = allowedDomains;
+                } else {
+                    runObject.noExternals = true;
+                }
             }
 
             Runs.save(runObject, function(data) {
@@ -43,6 +44,17 @@ apiService.factory('API', ['$location', 'Runs', 'Results', function($location, R
 
         relaunchTest: function(result) {
             this.launchTest(result.params.url, result.params.options);
+        },
+
+        parseDomains: function(textareaContent) {
+            var lines = textareaContent.split('\n');
+            
+            function removeEmptyLines (line) {
+                return line.trim() !== '';
+            }
+
+            // Remove empty lines
+            return lines.filter(removeEmptyLines).join(',');
         }
     };
 

front/src/less/dashboard.less

@@ -108,7 +108,11 @@
 
 .summary .notations .criteria .table {
     width: 100%;
-    > div:hover > div {
+    > a {
+        text-decoration: none;
+        color: inherit;
+    }
+    > a:hover > div {
         background: #EBD8E2;
         cursor: pointer;
         &.info {

front/src/less/index.less

@@ -38,7 +38,7 @@
         font-size: 1em;
     }
 
-    input[type=text] {
+    input[type=text], input[type=password], textarea {
         width: 100%;
         min-width: 4em;
     }
@@ -147,6 +147,7 @@
             &.label {
                 width: 25%;
                 white-space: nowrap;
+                vertical-align: middle;
             }
         }
     }

front/src/less/main.less

@@ -132,8 +132,12 @@ a.linkButton {
     background: #000;
 
     > div {
-        overflow: scroll;
         position: relative;
+        overflow: hidden;
+
+        &:hover {
+            overflow: auto;
+        }
     }
 
     .screenshotImage {
@@ -234,14 +238,6 @@ a.linkButton {
     }
 }
 
-
-.star {
-    font-weight: bold;
-    span {
-        font-size: 1.2em;
-    }
-}
-
 .footer {
     padding: 3em;
     color: #413;
@@ -251,6 +247,9 @@ a.linkButton {
     .version {
         font-size: 0.7em;
     }
+    .github {
+        margin: 1em 0 0 0.5em;
+    }
 }
 
 [class^="icon-"]:before, [class*=" icon-"]:before {

front/src/less/rule.less

@@ -32,9 +32,18 @@
     margin: 0 auto 0.5em;
 }
 
+.rule h3 {
+    margin-bottom: 0em;
+}
+
+.rule .okThreshold {
+    font-style: italic;
+    font-size: 0.9em;
+}
+
 .rule .message {
     width: 80%;
-    margin: 0 auto;
+    margin: 1.5em auto;
     p {
         margin: 0.5em;
     }
@@ -103,6 +112,7 @@
         margin: 0.2em 0;
         border-radius: 0.4em;
         z-index: 1;
+        cursor: pointer;
 
         &.opens {
             padding-right: 0.75em;
@@ -157,6 +167,10 @@
             }
         }
     }
+
+    .smallerOffenders {
+        font-size: 0.9em;
+    }
 }
 
 .offendersHtml {
@@ -176,16 +190,18 @@
     }
 }
 
-.colorPalette {
-    width: 30em;
-    border: 2px solid #000;
-    text-align: left;
-
+.checker {
     /* Checkerboard background */
     background-color: #ddd;
     background-image: linear-gradient(45deg, #AAA 25%, transparent 25%, transparent 75%, #AAA 75%, #AAA), linear-gradient(45deg, #AAA 25%, transparent 25%, transparent 75%, #AAA 75%, #AAA);
     background-size:1em 1em;
     background-position:0 0, 0.5em 0.5em;
+}
+
+.colorPalette {
+    width: 30em;
+    border: 2px solid #000;
+    text-align: left;
 
     > div {
         display: inline-block;
@@ -221,4 +237,61 @@
             border: 0.2em solid #f1c40f;
         }
     }
+}
+
+.similarColors {
+    margin: 1em;
+    width: 20em;
+    height: 6em;
+
+    > div {
+        display: inline-block;
+        width: 10em;
+        height: 3.5em;
+        padding-top: 2.5em;
+    }
+}
+
+.totalWeightPie {
+    max-width: 39em;
+    margin: 2em auto 4em;
+
+    canvas {
+        max-width: inherit;
+    }
+}
+
+.hugeFile {
+    font-weight: bold;
+    color: #e74c3c;
+}
+
+.imageOffenders {
+    display: table;
+    border-spacing: 3em;
+    width: 90%;
+
+    > div {
+        display: table-row;
+
+        > div {
+            display: table-cell;
+            vertical-align: middle;
+        }
+    }
+
+    img {
+        max-height: 10em;
+        max-width: 40em;
+        border: 1px solid #000;
+        margin-top: 0.5em;
+    }
+}
+
+.smallPreview {
+    display: block;
+    max-height: 4em;
+    max-width: 8em;
+    border: 1px solid #000;
+    margin: 1em auto 0.2em;
 }

front/src/less/timeline.less

@@ -173,7 +173,8 @@
     border-spacing: 0.25em;
 }
 
-.table > div {
+.table > div,
+.table > a {
     display: table-row;
 }
 
@@ -182,7 +183,8 @@
     padding: 0.5em 1em;
 }
 
-.table > div > div {
+.table > div > div,
+.table > a > div {
     padding: 0.1em 1em;
     background: #f2f2f2;
     display: table-cell;

front/src/main.html

@@ -16,15 +16,36 @@
     <link rel="stylesheet" type="text/css" href="/css/screenshot.css">
     <link rel="stylesheet" type="text/css" href="/css/timeline.css">
     <link rel="stylesheet" type="text/css" href="/css/about.css">
+    <link rel="stylesheet" type="text/css" href="/node_modules/angular-chart.js/dist/angular-chart.css">
     <!-- endbuild -->
 
+    <link rel="preconnect" href="//www.google-analytics.com">
+    <link rel="dns-prefetch" href="//www.google-analytics.com">
+    <link rel="preconnect" href="//ghbtns.com">
+    <link rel="dns-prefetch" href="//ghbtns.com">
+    <link rel="preconnect" href="https://api.github.com">
+    <link rel="dns-prefetch" href="https://api.github.com">
+
+<head>
+
+<body ng-app="YellowLabTools">
+    <div id="header"><h1>Yellow Lab <span class="icon-lab"></span> Tools</h1></div>
+    <div id="body" ng-view autoscroll="true"></div>
+    <div class="footer">
+        <span class="version">@@version</span>
+        <br><a href="/about">More about Yellow Lab Tools</a><br>
+        <div class="github"><iframe id="ghbtn" frameborder="0" scrolling="0" width="145px" height="30px"></iframe></div>
+    </div>
+
     <!-- build:js /js/all.js -->
-    <script src="/bower_components/angular/angular.min.js"></script>
-    <script src="/bower_components/angular-route/angular-route.min.js"></script>
-    <script src="/bower_components/angular-resource/angular-resource.min.js"></script>
-    <script src="/bower_components/angular-sanitize/angular-sanitize.min.js"></script>
-    <script src="/bower_components/angular-animate/angular-animate.min.js"></script>
-    <script src="/bower_components/angular-local-storage/dist/angular-local-storage.min.js"></script>
+    <script src="/node_modules/angular/angular.min.js"></script>
+    <script src="/node_modules/chart.js/Chart.min.js"></script>
+    <script src="/node_modules/angular-route/angular-route.min.js"></script>
+    <script src="/node_modules/angular-resource/angular-resource.min.js"></script>
+    <script src="/node_modules/angular-sanitize/angular-sanitize.min.js"></script>
+    <script src="/node_modules/angular-animate/angular-animate.min.js"></script>
+    <script src="/node_modules/angular-local-storage/dist/angular-local-storage.min.js"></script>
+    <script src="/node_modules/angular-chart.js/dist/angular-chart.min.js"></script>
     <script src="/js/app.js"></script>
     <script src="/js/controllers/indexCtrl.js"></script>
     <script src="/js/controllers/dashboardCtrl.js"></script>
@@ -41,15 +62,6 @@
     <script src="/js/directives/gradeDirective.js"></script>
     <script src="/js/directives/offendersDirectives.js"></script>
     <!-- endbuild -->
-<head>
-
-<body ng-app="YellowLabTools">
-    <div id="header"><h1>Yellow Lab <span class="icon-lab"></span> Tools</h1></div>
-    <div id="body" ng-view autoscroll="true"></div>
-    <div class="footer">
-        <a href="/about">Learn more</a><br>If you like <b>Yellow Lab Tools</b>, <a href="https://github.com/gmetais/YellowLabTools" target="_blank" class="star">give it a <span>&#9733;</span> on GitHub</a>!<br>
-        <span class="version">@@version</span>
-    </div>
 
     <script>
         (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){(i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)})(window,document,'script','//www.google-analytics.com/analytics.js','ga');if('@@googleAnalyticsId'.indexOf('UA-')===0){ga('create','@@googleAnalyticsId','auto');}

front/src/views/dashboard.html

@@ -1,6 +1,13 @@
 <div ng-include="'views/resultSubHeader.html'"></div>
 <div class="summary board">
     
+    <div ng-if="result.blockedRequests">
+        <b><ng-pluralize count="result.blockedRequests.length" when="{'0': 'No blocked request', 'one': '1 blocked request', 'other': '{} blocked requests'}"></ng-pluralize>:</b>
+        <div ng-repeat="request in result.blockedRequests">
+            {{request}}
+        </div>
+    </div>
+
     <div class="globalScore" ng-if="globalScore === 0 || globalScore > 0">
         <div>
             <h2>Global score</h2>
@@ -28,27 +35,28 @@
             <div class="category">{{category.label}}</div>
             <div class="criteria">
                 <div class="table" title="Click to see details">
-                    <div ng-repeat="ruleName in category.rules" ng-if="result.rules[ruleName]" ng-init="rule = result.rules[ruleName]"
-                         ng-class="{'warning': rule.abnormal}" ng-click="showRulePage(ruleName)">
+                    <a ng-repeat="ruleName in category.rules" ng-if="result.rules[ruleName]" ng-init="rule = result.rules[ruleName]"
+                         ng-class="{'warning': rule.abnormal}" href="/result/{{runId}}/rule/{{ruleName}}">
                         <div class="grade">
                             <grade score="rule.score"></grade>
                         </div>
                         <div class="label">{{rule.policy.label}}</div>
                         <div class="result">
-                            {{rule.value}}
+                            <span ng-if="rule.policy.unit == 'bytes'">{{rule.value | bytes}}</span>
+                            <span ng-if="rule.policy.unit != 'bytes'">{{rule.value}}</span>
                             <span ng-if="rule.abnormal" class="icon-warning"></span>
                             <span ng-if="rule.abnormalityScore <= -100" class="icon-warning"></span>
                             <span ng-if="rule.abnormalityScore <= -300" class="icon-warning"></span>
                         </div>
                         <div class="info"><span class="icon-question"></span></div>
-                    </div>
+                    </a>
                 </div>
             </div>
         </div>
     </div>
 
     <div ng-if="!error && !fromSocialShare" class="apiTip">
-        <b>Did you know? Yellow Lab Tools has a brand new <a href="https://github.com/gmetais/grunt-yellowlabtools" target="_blank">Grunt plugin</a>!
+        <b>Did you know?</b> You can <b>bookmark</b> this result page URL or <b>share it</b>, Yellow Lab Tools keeps test results for years!
     </div>
 
     <div class="tweet" ng-if="!error && !fromSocialShare">

front/src/views/index.html

@@ -2,7 +2,7 @@
 <p class="price">Free and open source!</p>
 
 <form ng-submit="launchTest()" >
-    <input type="{{$root.isTouchDevice? 'url' : 'text'}}" name="url" ng-model="url" placeholder="http://www.mysite.com" class="url" />
+    <input type="text" name="url" ng-model="url" placeholder="http://www.mysite.com" class="url" />
     <input type="submit" value="Launch test" class="launchBtn" ng-class="{disabled: !url}" />
     <div class="settings">
         <div class="device">
@@ -42,7 +42,7 @@
                 </div>
                 <div><input type="text" name="cookie" ng-model="settings.cookie" /></div>
             </div>
-            <!--<div>
+            <div>
                 <div class="label">
                     Authent
                     <span class="settingsTooltip">
@@ -57,19 +57,26 @@
                     </div>
                     <div>
                         <div><span>password</div>
-                        <div><input type="text" class="authField" name="authPass" ng-model="settings.authPass" /></div>
+                        <div><input type="password" class="authField" name="authPass" ng-model="settings.authPass" /></div>
                     </div>
                 </div>
-            </div>-->
-            <!--<div>
-                <div class="label">Blocked domains</div>
+            </div>
+            <div>
+                <div class="label">
+                    Block domains
+                    <span class="settingsTooltip">
+                        <span class="icon-question"></span>
+                        <div><b>Block some domains</b><br><br>One line per domain or subdomain.<br><br><i><b>Example:</b><br>google-analytics.com<br>ads.yahoo.com<br>ajax.googleapis.com</i><br><br>An empty whitelist will block all domains except the main domain.</div>
+                    </span>
+                </div>
                 <div>
                     <div>
-                        Blacklist / Whitelist
+                        <input type="radio" name="blackOrWhite" ng-model="settings.domainsBlackOrWhite" value="black" />blacklist
+                        <input type="radio" name="blackOrWhite" ng-model="settings.domainsBlackOrWhite" value="white" />whitelist
                     </div>
-                    <textarea name=""></textarea>
+                    <textarea name="domains" ng-model="settings.domains" rows="5"></textarea>
                 </div>
-            </div>-->
+            </div>
         </div>
     </div>
 </form>

front/src/views/rule.html

@@ -9,7 +9,19 @@
             <div>{{rule.score}}/100</div>
         </div>
         <div class="right">
-            <h3>Value: {{rule.value}}</h3>
+            <h3>
+                Value:
+                <span ng-if="rule.policy.unit == 'bytes'">{{rule.value | bytes}}</span>
+                <span ng-if="rule.policy.unit != 'bytes'">{{rule.value}}</span>
+            </h3>
+            <div class="okThreshold" ng-if="rule.score < 100 && rule.policy.isOkThreshold !== undefined">
+                Have
+                <span ng-if="rule.policy.unit == 'bytes'">{{rule.policy.isOkThreshold | bytes}}</span>
+                <span ng-if="rule.policy.unit != 'bytes'">{{rule.policy.isOkThreshold}}</span>
+                <span ng-if="rule.policy.isOkThreshold > 0 && rule.policy.isOkThreshold < rule.policy.isBadThreshold">or less</span>
+                <span ng-if="rule.policy.isOkThreshold > rule.policy.isBadThreshold">or more</span>
+                to get the 100/100 score.
+            </div>
             <div ng-bind-html="rule.policy.message" class="message"></div>
         </div>
     </div>
@@ -18,7 +30,7 @@
         <p>This rule reached the abnormality threshold, which means there is a real problem you should care about.</p>
     </div>
     <div class="offenders" ng-if="rule.policy.hasOffenders">
-        <h3><ng-pluralize count="rule.offendersObj.count" when="{'0': 'No offenders', 'one': '1 offender', 'other': '{} offenders'}"></ng-pluralize></h3>
+        <h3 ng-if="rule.offendersObj.count >= 0"><ng-pluralize count="rule.offendersObj.count" when="{'0': 'No offenders', 'one': '1 offender', 'other': '{} offenders'}"></ng-pluralize></h3>
 
         <div ng-if="rule.offendersObj.list" class="offendersTable">
             <div ng-repeat="offender in rule.offendersObj.list track by $index">
@@ -84,15 +96,24 @@
                         </span>
                     </div>
 
+                    <div ng-if="policyName === 'cssRules'">
+                        <span ng-if="offender.file === 'inline CSS'">inline CSS</span>
+                        <span ng-if="offender.file !== 'inline CSS'"><url-link url="offender.file" max-length="80"></url-link></span>
+                        : <ng-pluralize count="offender.rules" when="{'0': '0 rule', 'one':'1 rule','other':'{} rules'}"></ng-pluralize>
+                    </div>
+
+                    <div ng-if="policyName === 'similarColors'">
+                        <div class="similarColors checker"><div ng-style="{'background-color': offender.color1, 'color': offender.isDark ? '#FFF' : '#000'}">{{offender.color1}}</div><div ng-style="{'background-color': offender.color2, 'color': offender.isDark ? '#FFF' : '#000'}">{{offender.color2}}</div></div>
+                    </div>
+
                     <div ng-if="policyName === 'cssParsingErrors'">
                         <b>{{offender.error}}</b>
                         <file-and-line file="offender.file" line="offender.line" column="offender.column"></file-and-line>
                         <span ng-if="offender.file">(<a href="http://jigsaw.w3.org/css-validator/validator?profile=css3&usermedium=all&warning=no&uri={{offender.file | encodeURIComponent}}" target="_blank">Check on the W3C validator</a>)</span>
                     </div>
 
-                    <div ng-if="policyName === 'cssComplexSelectors' || policyName === 'cssComplexSelectorsByAttribute' || policyName === 'cssImports' || policyName === 'cssUniversalSelectors' || policyName === 'cssRedundantBodySelectors' || policyName === 'cssRedundantChildNodesSelectors'">
-                        <span ng-if="offender.bolded" ng-bind-html="offender.bolded"></span>
-                        <b ng-if="!offender.bolded">{{offender.css}}</b>
+                    <div ng-if="policyName === 'cssImports'">
+                        {{offender.css}}
                         <file-and-line-button file="offender.file" line="offender.line" column="offender.column"></file-and-line-button>
                     </div>
 
@@ -100,51 +121,28 @@
                         {{offender.rule}} (<b>x{{offender.occurrences}}</b>)
                     </div>
 
-                    <div ng-if="policyName === 'cssDuplicatedProperties'">
-                        Property <b>{{offender.property}}</b> duplicated in <b>{{offender.rule}} { }</b>
-                        <file-and-line-button file="offender.file" line="offender.line" column="offender.column"></file-and-line-button>
-                    </div>
-
-                    <div ng-if="policyName === 'cssEmptyRules'">
-                        <b>{{offender.css}} { }</b>
-                        <file-and-line-button file="offender.file" line="offender.line" column="offender.column"></file-and-line-button>
-                    </div>
-
-                    <div ng-if="policyName === 'cssExpressions'">
-                        {{offender.rule}} {{ '{' + offender.property}}: <b>expression(</b>{{offender.expression}}<b>)</b>}
-                        <file-and-line-button file="offender.file" line="offender.line" column="offender.column"></file-and-line-button>
-                    </div>
-
-                    <div ng-if="policyName === 'cssImportants'">
-                        {{offender.rule}} {{ '{' + offender.property}}: {{offender.value}} <b>!important</b>}
-                        <file-and-line-button file="offender.file" line="offender.line" column="offender.column"></file-and-line-button>
-                    </div>
-
-                    <div ng-if="policyName === 'cssOldIEFixes'">
-                        <span ng-if="offender.browser"><b>{{offender.browser}} fix:</b></span>
-                        <span ng-bind-html="offender.bolded"></span>
-                        <file-and-line-button file="offender.file" line="offender.line" column="offender.column"></file-and-line-button>
-                    </div>
-
                     <div ng-if="policyName === 'cssOldPropertyPrefixes'">
-                        {{offender.rule}} {<b>{{offender.property}}</b>: {{offender.value + '}' }}
-                        <br>
-                        <b>{{offender.message}}</b>
-                        <file-and-line-button file="offender.file" line="offender.line" column="offender.column"></file-and-line-button>
+                        <b>{{offender.property}} {{offender.message}}</b>
+                        <div ng-if="offender.rules.length" ng-click="offender.showMore = !offender.showMore" class="offenderButton">
+                            <span ng-if="!offender.showMore">show</span>
+                            <span ng-if="offender.showMore">hide</span>
+                            <ng-pluralize count="offender.rules.length" when="{'one':'1 offender','other':'{} offenders'}"></ng-pluralize>
+                        </div>
+                        <div ng-if="offender.showMore" class="smallerOffenders">
+                            <div ng-repeat="cssRule in offender.rules">
+                                {{cssRule.rule}} {{'{' + offender.property}}: {{cssRule.value + '}' }}
+                                <file-and-line-button file="cssRule.file" line="cssRule.line" column="cssRule.column"></file-and-line-button>
+                            </div>
+                        </div>
                     </div>
 
-                    <div ng-if="policyName === 'requests' || policyName === 'htmlCount' || policyName === 'jsCount' || policyName === 'cssCount' || policyName === 'imageCount' || policyName === 'webfontCount' || policyName === 'videoCount' || policyName === 'jsonCount' || policyName === 'otherCount' || policyName === 'smallJsFiles' || policyName === 'smallCssFiles' || policyName === 'smallImages'">
-                        <url-link url="offender.file" max-length="100"></url-link>
-                        <span ng-if="offender.size || offender.size === 0">({{offender.size}} kB)</span>
-                    </div>
-
-                    <div ng-if="policyName === 'notFound' || policyName === 'closedConnections' || policyName === 'multipleRequests' || policyName === 'cachingDisabled' || policyName === 'cachingNotSpecified'">
+                    <div ng-if="policyName === 'lazyLoadableImagesBelowTheFold' || policyName === 'hiddenImages'">
+                        <img ng-src="{{offender}}" class="smallPreview checker"></img>
                         <url-link url="offender" max-length="100"></url-link>
                     </div>
 
-                    <div ng-if="policyName === 'assetsNotGzipped'">
-                        <url-link url="offender.file" max-length="100"></url-link>
-                        ({{offender.type}})
+                    <div ng-if="policyName === 'notFound' || policyName === 'emptyRequests' || policyName === 'closedConnections' || policyName === 'multipleRequests' || policyName === 'cachingDisabled' || policyName === 'cachingNotSpecified'">
+                        <url-link url="offender" max-length="100"></url-link>
                     </div>
 
                     <div ng-if="policyName === 'cachingTooShort'">
@@ -157,16 +155,64 @@
                         (<ng-pluralize count="offender.requests" when="{'one':'1 request','other':'{} requests'}"></ng-pluralize>)
                     </div>
 
-                    <div ng-if="policyName === 'globalVariables' || policyName === 'jQueryVersionsLoaded'">
+                    <div ng-if="policyName === 'globalVariables' || policyName === 'jQueryVersionsLoaded' || policyName === 'synchronousXHR'">
                         {{offender}}
                     </div>
-
                 </div>
             </div>
         </div>
 
+        <div ng-repeat="(file, fileDetails) in rule.offendersObj.byFile track by $index">
+            <h3>
+                <ng-pluralize count="fileDetails.count" when="{'one': '1 offender', 'other': '{} offenders'}"></ng-pluralize>
+                in
+                <url-link ng-if="file !== 'Inline CSS'" url="file" max-length="80"></url-link>
+                <span ng-if="file === 'Inline CSS'">inline CSS</span>
+            </h3>
 
-        <div ng-if="!rule.offendersObj.list" class="offendersHtml">
+            <div class="offendersTable">
+                <div ng-repeat="offender in fileDetails.offenders track by $index">
+                    <div ng-if="policyName === 'cssComplexSelectors' || policyName === 'cssComplexSelectorsByAttribute' || policyName === 'cssUniversalSelectors' || policyName === 'cssRedundantBodySelectors' || policyName === 'cssRedundantChildNodesSelectors'">
+                        <span ng-if="offender.bolded" ng-bind-html="offender.bolded"></span>
+                        <b ng-if="!offender.bolded">{{offender.css}}</b>
+                        <span ng-if="offender.line !== null && offender.column !== null"> @ {{offender.line}}:{{offender.column}}</span>
+                    </div>
+
+                    <div ng-if="policyName === 'cssMobileFirst'">
+                        <b>{{offender.query}}</b> for <ng-pluralize count="offender.rules" when="{'one':'1 rule','other':'{} rules'}"></ng-pluralize>
+                        <span ng-if="offender.line !== null && offender.column !== null"> @ {{offender.line}}:{{offender.column}}</span>
+                    </div>
+
+                    <div ng-if="policyName === 'cssDuplicatedProperties'">
+                        Property <b>{{offender.property}}</b> duplicated in <b>{{offender.rule}} { }</b>
+                        <span ng-if="offender.line !== null && offender.column !== null"> @ {{offender.line}}:{{offender.column}}</span>
+                    </div>
+
+                    <div ng-if="policyName === 'cssEmptyRules'">
+                        <b>{{offender.css}} { }</b>
+                        <span ng-if="offender.line !== null && offender.column !== null"> @ {{offender.line}}:{{offender.column}}</span>
+                    </div>
+
+                    <div ng-if="policyName === 'cssExpressions'">
+                        {{offender.rule}} {{ '{' + offender.property}}: <b>expression(</b>{{offender.expression}}<b>)</b>}
+                        <span ng-if="offender.line !== null && offender.column !== null"> @ {{offender.line}}:{{offender.column}}</span>
+                    </div>
+
+                    <div ng-if="policyName === 'cssImportants'">
+                        {{offender.rule}} {{ '{' + offender.property}}: {{offender.value}} <b>!important</b>}
+                        <span ng-if="offender.line !== null && offender.column !== null"> @ {{offender.line}}:{{offender.column}}</span>
+                    </div>
+
+                    <div ng-if="policyName === 'cssOldIEFixes'">
+                        <span ng-if="offender.browser"><b>{{offender.browser}} fix:</b></span>
+                        <span ng-bind-html="offender.bolded"></span>
+                        <span ng-if="offender.line !== null && offender.column !== null"> @ {{offender.line}}:{{offender.column}}</span>
+                    </div>
+                </div>
+            </div>
+        </div>
+
+        <div ng-if="!rule.offendersObj.list && !rule.offendersObj.byFile" class="offendersHtml">
             
             <div ng-if="policyName === 'DOMelementMaxDepth'">
                 <dom-tree tree="rule.offendersObj.tree"></dom-tree>
@@ -187,13 +233,140 @@
 
             <div ng-if="policyName === 'cssColors' && rule.offendersObj.count > 0">
                 <p>This is the colors palette, sized by total occurrences:</p>
-                <div class="colorPalette">
+                <div class="colorPalette checker">
                     <div ng-repeat="offender in rule.offendersObj.palette" style="background-color: {{offender.color}}; width: {{offender.occurrences * 100 / rule.offendersObj.palette[0].occurrences}}%"><div>{{offender.color}} ({{offender.occurrences}} times)</div></div>
                 </div>
             </div>
         </div>
     </div>
 
+    <div ng-if="policyName === 'totalWeight'">
+        <h3>Weight by MIME type</h3>
+        <div class="totalWeightPie">
+            <canvas class="chart chart-doughnut" data="weightData" labels="weightLabels" options="weightOptions" colours="weightColours" legend="true"></canvas>
+        </div>
+        <div ng-repeat="type in weightLabels">
+            <h3>{{rule.offendersObj.list.byType[type].totalWeight | bytes}} of {{type}}</h3>
+            <div class="offendersTable">
+                <div ng-repeat="request in rule.offendersObj.list.byType[type].requests | orderBy:'-weight'" ng-if="request.weight > 0">
+                    <div><url-link url="request.url" max-length="60"></url-link></div>
+                    <div ng-class="{hugeFile: request.weight > 102400}">{{request.weight | bytes}}</div>
+                </div>
+            </div>
+        </div>
+    </div>
+
+    <div ng-if="policyName === 'imageOptimization'">
+        <h3 ng-if="rule.value > 0">{{rule.value | bytes}} could be saved on <ng-pluralize count="rule.offendersObj.list.images.length" when="{'one': '1 image', 'other': '{} images'}"></ng-pluralize></h3>
+        <div class="imageOffenders">
+            <div ng-repeat="image in rule.offendersObj.list.images | orderBy:'-gain'">
+                <div>
+                    Current file: <url-link url="image.url" max-length="50"></url-link>
+                    <div><a href="{{image.url}}" target="_blank"><img ng-src="{{image.url}}" class="checker" /></a></div>
+                </div>
+                <div>
+                    <p ng-if="!image.afterCompression">Current weight: {{image.original | bytes}}</p>
+                    <p ng-if="image.afterCompression">Current weight: {{image.original | bytes}} ({{image.afterCompression | bytes}} gzipped)</p>
+                    <p ng-if="image.lossless && image.afterOptimizationAndCompression">With a lossless optimization:<br/>{{image.afterOptimizationAndCompression | bytes}} gzipped (<b>-{{image.gain | bytes}}</b> gzipped)</p>
+                    <p ng-if="image.lossless && !image.afterOptimizationAndCompression">With a lossless optimization:<br/>{{image.lossless | bytes}} <span ng-if="!image.lossy">(<b>-{{image.gain | bytes}}</b>)</span></p>
+                    <p ng-if="image.lossy && image.afterOptimizationAndCompression">With a lossy optimization:<br/>{{image.afterOptimizationAndCompression | bytes}} gzipped (<b>-{{image.gain | bytes}} gzipped</b>)</p>
+                    <p ng-if="image.lossy && !image.afterOptimizationAndCompression">With a lossy optimization:<br/>{{image.lossy | bytes}} (<b>-{{image.gain | bytes}}</b>)</p>
+                </div>
+            </div>
+        </div>
+    </div>
+
+    <div ng-if="policyName === 'gzipCompression'">
+        <h3 ng-if="rule.value > 0">{{rule.value | bytes}} could be saved on <ng-pluralize count="rule.offendersObj.list.images.length" when="{'one': '1 file', 'other': '{} files'}"></ng-pluralize></h3>
+        <div class="table">
+            <div class="headers">
+                <div>File</div>
+                <div>Current weight</div>
+                <div>Gzipped</div>
+                <div>Gain</div>
+            </div>
+            <div ng-repeat="file in rule.offendersObj.list.files | orderBy:'-gain'">
+                <div>
+                    <url-link url="file.url" max-length="70"></url-link>
+                </div>
+                <div>{{file.original | bytes}}</div>
+                <div>{{file.gzipped | bytes}}</div>
+                <div><b>-{{file.gain | bytes}}</b></div>
+            </div>
+        </div>
+    </div>
+
+    <div ng-if="policyName === 'fileMinification'">
+        <h3 ng-if="rule.value > 0">{{rule.value | bytes}} could be saved on <ng-pluralize count="rule.offendersObj.list.files.length" when="{'one': '1 file', 'other': '{} files'}"></ng-pluralize></h3>
+        <div class="table">
+            <div class="headers">
+                <div>File</div>
+                <div>Current weight</div>
+                <div>Minified</div>
+                <div>Gain</div>
+            </div>
+            <div ng-repeat="file in rule.offendersObj.list.files | orderBy:'-gain'">
+                <div>
+                    <url-link url="file.url" max-length="70"></url-link>
+                </div>
+                <div ng-if="!file.afterCompression">{{file.original | bytes}} (gzipped)</div>
+                <div ng-if="file.afterCompression">{{file.original | bytes}} ({{file.afterCompression | bytes}} gzipped)</div>
+                <div ng-if="!file.afterCompression">{{file.afterOptimizationAndCompression | bytes}} (gzipped)</div>
+                <div ng-if="file.afterCompression">{{file.optimized | bytes}} ({{file.afterOptimizationAndCompression | bytes}} gzipped)</div>
+                <div><b>-{{file.gain | bytes}}</b></div>
+            </div>
+        </div>
+    </div>
+
+    <div ng-if="policyName === 'totalRequests'">
+        <h3>Requests by MIME type</h3>
+        <div ng-repeat="(type, requests) in rule.offendersObj.list.byType">
+            <h3><ng-pluralize count="requests.length" when="{'0': 'No ' + type + ' request', 'one': '1 ' + type + ' request', 'other': '{} ' + type + ' requests'}"></ng-pluralize></h3>
+            <p ng-if="type == 'css' && requests.length > 2">Reduce the number of stylesheets by concatenating them.</p>
+            <p ng-if="type == 'js' && requests.length > 3">Reduce the number of scripts by concatenating them.</p>
+            <p ng-if="type == 'image' && requests.length > 5">Reduce the number of images by lazyloading them or by spriting them.</p>
+            <p ng-if="type == 'webfont' && requests.length > 1">Fonts are generally loaded on the critical path of the head. Load as few as possible.</p>
+            <p ng-if="type == 'other' && requests.length > 0">They can be Flash, XML, music or any undetected format.</p>
+            <div class="offendersTable">
+                <div ng-repeat="request in requests track by $index">
+                    <div><url-link url="request" max-length="100"></url-link></div>
+                </div>
+            </div>
+        </div>
+    </div>
+
+    <div ng-if="policyName === 'identicalFiles'">
+        <div ng-repeat="offender in rule.offendersObj.list track by $index">
+            <h4>A file of {{offender.weight | bytes}} is loaded {{offender.urls.length}} times:</h4>
+            <div class="offendersTable">
+                <div ng-repeat="url in offender.urls">
+                    <div><url-link url="url" max-length="100"></url-link></div>
+                </div>
+            </div>
+        </div>
+    </div>
+
+    <div ng-if="policyName === 'smallRequests'">
+        <div ng-repeat="(type, requests) in rule.offendersObj.list.byType">
+            <h3><ng-pluralize count="requests.length" when="{'0': 'small ' + type + ' file', 'one': '1 small ' + type + ' file', 'other': '{} small ' + type + ' files'}"></ng-pluralize></h3>
+            <p ng-if="type == 'css' && requests.length > 0">Try to inline these styles in the head of the HTML or to merge them with other files.</p>
+            <p ng-if="type == 'js' && requests.length > 0">Try to inline these scripts in the HTML or merge them with other files.</p>
+            <p ng-if="type == 'image' && requests.length > 0">Try to inline these images (with base64 encoding for most image types except SVG that don't need base64 encoding). You can also create sprites.</p>
+            <div class="table">
+                <div class="headers">
+                    <div ng-if="type == 'image'">Preview</div>
+                    <div>File</div>
+                    <div>Weight (bytes)</div>
+                </div>
+                <div ng-repeat="request in requests track by $index">
+                    <div ng-if="type == 'image'"><img ng-src="{{request.url}}" class="smallPreview checker" /></div>
+                    <div><url-link url="request.url" max-length="100"></url-link></div>
+                    <div>{{request.size}}</div>
+                </div>
+            </div>
+        </div>
+    </div>
+
     <div ng-if="policyName === 'DOMaccesses'">
         <h3>{{rule.value}} offenders</h3>
         Please open the <a href="/result/{{runId}}/timeline">JS timeline</a>
@@ -214,6 +387,34 @@
         Please open the <a href="/result/{{runId}}/timeline#filter=eventNotDelegated">JS timeline, filtered by "Events not delegated"</a>
     </div>
 
+    <div ng-if="policyName === 'cssBreakpoints'">
+        <div ng-if="rule.value > 0" class="cssBreakpointsGraph">
+            <h3>Breakpoints distribution graph</h3>
+            <canvas class="chart chart-line" chart-data="breakpointsData" chart-labels="breakpointsLabels" chart-options="breakpointsOptions" chart-legend="true" chart-colours="breakpointsColours" chart-series="breakpointsSeries" width="600" height="250"></canvas>
+            <h3>Breakpoints list</h3>
+            <div class="offendersTable">
+                <div ng-repeat="offender in rule.offendersObj | orderBy:'pixels'">
+                    <div>Breakpoint <b>{{offender.breakpoint}}</b> involves <ng-pluralize count="offender.count" when="{'one': '1 rule', 'other': '{} rules'}"></ng-pluralize></div>
+                </div>
+            </div>
+        </div>
+        <div ng-if="rule.value === 0">
+        No breakpoint
+        </div>
+    </div>
+
+    <div ng-if="policyName === 'http2'">
+        <h3>Protocols advertised by the server</h3>
+        <div class="offendersTable">
+            <div ng-repeat="protocol in rule.offendersObj.list">
+                <div>{{protocol}}</div>
+            </div>
+            <div ng-if="!rule.offendersObj || rule.offendersObj.count == 0">
+                <div>none</div>
+            </div>
+        </div>
+    </div>
+
     <div ng-if="!rule && rule !== null" class="notFound">
         <h2>404</h2>
         Rule "{{policyName}}"" not found

lib/metadata/scoreProfileGeneric.json

@@ -1,9 +1,31 @@
 {
     "categories": {
+        "pageWeight": {
+            "label": "Page weight",
+            "policies": {
+                "totalWeight": 5,
+                "imageOptimization": 2,
+                "gzipCompression": 2,
+                "fileMinification": 1
+            }
+        },
+        "requests": {
+            "label": "Requests",
+            "policies": {
+                "totalRequests": 5,
+                "domains": 3,
+                "notFound": 3,
+                "identicalFiles": 2,
+                "emptyRequests": 3,
+                "smallRequests": 1,
+                "lazyLoadableImagesBelowTheFold": 2,
+                "hiddenImages": 1
+            }
+        },
         "domComplexity": {
             "label": "DOM complexity",
             "policies": {
-                "DOMelementsCount": 1,
+                "DOMelementsCount": 3,
                 "DOMelementMaxDepth": 1,
                 "iframesCount": 1,
                 "DOMidDuplicated": 1
@@ -29,6 +51,7 @@
             "policies": {
                 "jsErrors": 1,
                 "documentWriteCalls": 2,
+                "synchronousXHR": 5,
                 "consoleMessages": 0.5,
                 "globalVariables": 0.5
             }
@@ -48,8 +71,10 @@
             "policies": {
                 "cssRules": 2,
                 "cssComplexSelectors": 2,
-                "cssComplexSelectorsByAttribute": 1.5,
-                "cssColors": 0.5
+                "cssColors": 0.5,
+                "similarColors": 0.5,
+                "cssBreakpoints": 0.5,
+                "cssMobileFirst": 1
             }
         },
         "badCSS": {
@@ -64,58 +89,32 @@
                 "cssImportants": 3,
                 "cssOldIEFixes": 1,
                 "cssOldPropertyPrefixes": 1,
-                "cssUniversalSelectors": 1,
                 "cssRedundantBodySelectors": 1,
                 "cssRedundantChildNodesSelectors": 1
             }
         },
-        "requests": {
-            "label": "Requests number",
+        "serverConfig": {
+            "label": "Server config",
             "policies": {
-                "requests": 5,
-                "htmlCount": 0,
-                "jsCount": 1,
-                "cssCount": 1,
-                "imageCount": 0,
-                "webfontCount": 2,
-                "videoCount": 0,
-                "jsonCount": 0,
-                "otherCount": 0
-            }
-        },
-        "smallRequests": {
-            "label": "Small requests",
-            "policies": {
-                "smallJsFiles": 1,
-                "smallCssFiles": 1,
-                "smallImages": 1
-            }
-        },
-        "network": {
-            "label": "Network",
-            "policies": {
-                "notFound": 2,
-                "assetsNotGzipped": 1.5,
+                "http2": 2,
                 "closedConnections": 2,
-                "multipleRequests": 2,
                 "cachingNotSpecified": 1,
                 "cachingDisabled": 1,
-                "cachingTooShort": 1,
-                "domains": 1
+                "cachingTooShort": 1
             }
         }
     },
     "globalScore": {
-        "domComplexity": 1,
+        "pageWeight": 3,
+        "requests": 3,
+        "domComplexity": 2,
         "domManipulations": 2,
         "scroll": 1,
-        "badJavascript": 1,
+        "badJavascript": 2,
         "jQuery": 1,
         "cssSyntaxError": 1,
         "cssComplexity": 1,
         "badCSS": 1,
-        "requests": 3,
-        "smallRequests": 1,
-        "network": 2
+        "serverConfig": 1
     }
 }

lib/offendersHelpers.js

@@ -113,14 +113,17 @@ var OffendersHelpers = function() {
         if (traceArray) {
             var results = [];
             var parts = null;
+            var obj;
 
             for (var i=0 ; i<traceArray.length ; i++) {
-                parts = /^(([\w$]+) )?\(?([^ ]+):(\d+)\)?$/.exec(traceArray[i]);
+                // Handle the new PhantomJS 2.x syntax
+                parts = /^(([\w$]+)@)?([^ ]+):(\d+):(\d+)$/.exec(traceArray[i]);
 
                 if (parts) {
-                    var obj = {
+                    obj = {
                         file: parts[3],
-                        line: parseInt(parts[4], 10)
+                        line: parseInt(parts[4], 10),
+                        column: parseInt(parts[5], 10)
                     };
 
                     if (parts[2]) {
@@ -128,8 +131,26 @@ var OffendersHelpers = function() {
                     }
 
                     results.push(obj);
+
                 } else {
-                    return null;
+                    // Old syntax
+                    parts = /^(([\w$]+) )?\(?([^ ]+):(\d+)\)?$/.exec(traceArray[i]);
+
+                    if (parts) {
+                        obj = {
+                            file: parts[3],
+                            line: parseInt(parts[4], 10)
+                        };
+
+                        if (parts[2]) {
+                            obj.functionName = parts[2];
+                        }
+
+                        results.push(obj);
+
+                    } else {
+                        return null;
+                    }
                 }
             }
             return results;
@@ -151,6 +172,9 @@ var OffendersHelpers = function() {
     };
 
     this.cssOffenderPattern = function(offender) {
+        // Remove any line breaks
+        offender = offender.replace(/(\r\n|\n|\r)/gm, '');
+
         var parts = /^(.*) (?:<([^ \(]*)>|\[inline CSS\]) @ (\d+):(\d+)$/.exec(offender);
         
         if (!parts) {
@@ -168,7 +192,7 @@ var OffendersHelpers = function() {
     };
 
     this.fileWithSizePattern = function(fileWithSize) {
-        var parts = /^([^ ]*) \((\d+\.\d{2}) kB\)$/.exec(fileWithSize);
+        var parts = /^([^ ]*) \((\d+\.\d{2}|NaN) kB\)$/.exec(fileWithSize);
 
         if (!parts) {
             return {
@@ -182,6 +206,27 @@ var OffendersHelpers = function() {
         }
     };
 
+    this.orderByFile = function(offenders) {
+        var byFile = {};
+
+        offenders.forEach(function(offender) {
+            var file = offender.file || 'Inline CSS';
+            delete offender.file;
+
+            if (!byFile[file]) {
+                byFile[file] = {
+                    count: 0,
+                    offenders: []
+                };
+            }
+
+            byFile[file].count ++;
+            byFile[file].offenders.push(offender);
+        });
+
+        return {byFile: byFile};
+    };
+
 };
 
 module.exports = new OffendersHelpers();

lib/runner.js

@@ -3,6 +3,10 @@ var debug                   = require('debug')('ylt:runner');
 
 var phantomasWrapper        = require('./tools/phantomas/phantomasWrapper');
 var jsExecutionTransformer  = require('./tools/jsExecutionTransformer');
+var colorDiff               = require('./tools/colorDiff');
+var mediaQueriesChecker     = require('./tools/mediaQueriesChecker');
+var isHttp2                 = require('./tools/isHttp2');
+var redownload              = require('./tools/redownload/redownload');
 var rulesChecker            = require('./rulesChecker');
 var scoreCalculator         = require('./scoreCalculator');
 
@@ -19,14 +23,31 @@ var Runner = function(params) {
     };
 
     // Execute Phantomas first
-    phantomasWrapper.execute(data).then(function(phantomasResults) {
+    phantomasWrapper.execute(data)
+
+    .then(function(phantomasResults) {
         data.toolsResults.phantomas = phantomasResults;
 
         // Treat the JS Execution Tree from offenders
         data = jsExecutionTransformer.transform(data);
 
-        // Other tools go here
+        // Compare colors
+        data = colorDiff.compareAllColors(data);
 
+        // Check media queries
+        data = mediaQueriesChecker.analyzeMediaQueries(data);
+
+        // Redownload every file
+        return redownload.recheckAllFiles(data);
+
+    })
+
+    .then(function(data) {
+        // Check if HTTP2
+        return isHttp2.check(data);
+    })
+
+    .then(function(data) {
 
         // Rules checker
         var policies = require('./metadata/policies');
@@ -45,14 +66,17 @@ var Runner = function(params) {
         delete data.toolsResults.phantomas.metrics.scrollExecutionTree;
         delete data.toolsResults.phantomas.offenders.scrollExecutionTree;
 
-        return data;
 
-    }).then(function(data) {
+        if (data.toolsResults.phantomas.offenders.blockedRequests) {
+            data.blockedRequests = data.toolsResults.phantomas.offenders.blockedRequests;
+        }
+
 
         // Finished!
         deferred.resolve(data);
+    })
 
-    }).fail(function(err) {
+    .fail(function(err) {
         debug('Run failed');
         debug(err);
 

lib/screenshotHandler.js

@@ -48,6 +48,8 @@ var screenshotHandler = function() {
 
             })
 
+            .then(this.addWhiteBackground)
+
             .then(this.toBuffer);
     };
 
@@ -92,6 +94,39 @@ var screenshotHandler = function() {
         return deferred.promise;        
     };
 
+    // If the page doesn't set a bg color, the default PhantomJS one is transparent
+    // When transforming PNG to JPG, transparent pixels become black.
+    // This is why we need to add a transparent background.
+    this.addWhiteBackground = function(image) {
+        var deferred = Q.defer();
+
+        // Create a canvas with the same dimensions as your image:
+        lwip.create(image.width(), image.height(), 'white', function(err, canvas){
+            if (err) {
+                debug('Could not create a white canvas');
+                debug(err);
+
+                deferred.reject(err);
+            } else {
+                // Paste original image on top of the canvas
+                canvas.paste(0, 0, image, function(err, image){
+                    if (err) {
+                        debug('Could not paste image on the white canvas');
+                        debug(err);
+
+                        deferred.reject(err);
+                    } else {
+                        // Now image has a white background...
+                        debug('White background correctly added');
+                        deferred.resolve(image);
+                    }
+                });
+            }
+        });
+
+        return deferred.promise;
+    };
+
 
     this.toBuffer = function(image) {
         var deferred = Q.defer();

lib/server/controllers/apiController.js

@@ -6,7 +6,7 @@ var ScreenshotHandler   = require('../../screenshotHandler');
 var RunsQueue           = require('../datastores/runsQueue');
 var RunsDatastore       = require('../datastores/runsDatastore');
 var ResultsDatastore    = require('../datastores/resultsDatastore');
-
+var serverSettings      = require('../../../server_config/settings.json');
 
 var ApiController = function(app) {
     'use strict';
@@ -15,8 +15,6 @@ var ApiController = function(app) {
     var runsDatastore = new RunsDatastore();
     var resultsDatastore = new ResultsDatastore();
 
-
-
     // Create a new run
     app.post('/api/runs', function(req, res) {
 
@@ -33,12 +31,14 @@ var ApiController = function(app) {
                 waitForResponse: req.body.waitForResponse !== false && req.body.waitForResponse !== 'false' && req.body.waitForResponse !== 0,
                 partialResult: req.body.partialResult || null,
                 screenshot: req.body.screenshot || false,
-                jsTimeline: req.body.jsTimeline || false,
                 device: req.body.device || 'desktop',
                 waitForSelector: req.body.waitForSelector || null,
                 cookie: req.body.cookie || null,
                 authUser: req.body.authUser || null,
-                authPass: req.body.authPass || null
+                authPass: req.body.authPass || null,
+                blockDomain: req.body.blockDomain || null,
+                allowDomain: req.body.allowDomain || null,
+                noExternals: req.body.noExternals || false
             }
         };
 
@@ -52,7 +52,6 @@ var ApiController = function(app) {
         debug('Adding test %s to the queue', run.runId);
         var queuePromise = queue.push(run.runId);
 
-
         // Save the run to the datastore
         runsDatastore.add(run, queuePromise.startingPosition);
 
@@ -71,12 +70,15 @@ var ApiController = function(app) {
 
             var runOptions = {
                 screenshot: run.params.screenshot ? screenshot.getTmpFilePath() : false,
-                jsDeepAnalysis: run.params.jsTimeline,
                 device: run.params.device,
                 waitForSelector: run.params.waitForSelector,
                 cookie: run.params.cookie,
                 authUser: run.params.authUser,
-                authPass: run.params.authPass
+                authPass: run.params.authPass,
+                blockDomain: run.params.blockDomain,
+                allowDomain: run.params.allowDomain,
+                noExternals: run.params.noExternals,
+                phantomasEngine: serverSettings.phantomasEngine
             };
 
             return ylt(run.params.url, runOptions);
@@ -95,7 +97,7 @@ var ApiController = function(app) {
             if (run.params.screenshot) {
                 
                 // Replace the empty promise created earlier with Q.resolve()
-                screenshotPromise = screenshot.toThumbnail(400)
+                screenshotPromise = screenshot.toThumbnail(serverSettings.screenshotWidth || 400)
                 
                     // Read screenshot
                     .then(function(screenshotBuffer) {
@@ -128,13 +130,7 @@ var ApiController = function(app) {
                     // Remove uneeded temp screenshot path
                     delete data.params.options.screenshot;
 
-                    // Empty javascriptExecutionTree if not needed
-                    if (!run.params.jsTimeline) {
-                        data.javascriptExecutionTree = {};
-                        data.scrollExecutionTree = {};
-                    }
-
-                    // Remove tools results if not needed
+                    // Here we can remove tools results if not needed
 
                     return resultsDatastore.saveResult(data);
                 })
@@ -331,4 +327,4 @@ var ApiController = function(app) {
 
 };
 
-module.exports = ApiController;
+module.exports = ApiController;

lib/server/controllers/frontController.js

@@ -20,7 +20,7 @@ var FrontController = function(app) {
     app.use('/img', express.static(path.join(__dirname, assetsPath, 'img'), { maxAge: cacheDuration }));
     app.use('/js', express.static(path.join(__dirname, assetsPath, 'js'), { maxAge: cacheDuration }));
     app.use('/views', express.static(path.join(__dirname, assetsPath, 'views'), { maxAge: cacheDuration }));
-    app.use('/bower_components', express.static(path.join(__dirname, '../../../bower_components'), { maxAge: cacheDuration }));
+    app.use('/node_modules', express.static(path.join(__dirname, '../../../node_modules'), { maxAge: cacheDuration }));
 };
 
 module.exports = FrontController;

lib/tools/colorDiff.js

@@ -0,0 +1,92 @@
+var debug   = require('debug')('ylt:colorDiff');
+var diff    = require('color-diff');
+var parse   = require('parse-color');
+
+
+var colorDiff = function() {
+
+    var COLOR_DIFF_THRESHOLD = 0.9;
+
+    this.compareAllColors = function(data) {
+        debug('Starting to compare all colors...');
+
+        var offenders = data.toolsResults.phantomas.offenders.cssColors;
+        var colors = (offenders) ? this.parseAllColors(offenders) : [];
+        var result = [];
+
+        // Compare colors to each other
+        for (var i = 0; i < colors.length - 1 ; i++) {
+            for (var j = i + 1; j < colors.length; j++) {
+                debug('Comparing color %s with color %s', colors[i].original, colors[j].original);
+                if (this.compareTwoColors(colors[i], colors[j])) {
+                    debug('Colors are similar!');
+                    result.push({
+                        color1: colors[i].original,
+                        color2: colors[j].original,
+                        isDark: (colors[i].Lab.L < 60)
+                    });
+                }
+            }
+        }
+
+        data.toolsResults.colorDiff = {
+            metrics: {
+                similarColors: result.length
+            },
+            offenders: {
+                similarColors: result
+            }
+        };
+
+        return data;
+    };
+
+    this.parseAllColors = function(offenders) {
+        var parsedOffenders = offenders.map(this.parseOffender);
+        var deduplicatedColors = {};
+
+        parsedOffenders.forEach(function(color) {
+            if (color !== null) {
+                deduplicatedColors[color] = color;
+            }
+        });
+
+        return Object.keys(deduplicatedColors).map(this.parseColor);
+    };
+
+    this.parseOffender = function(offender) {
+        var regexResult = /^(.*) \(\d+ times\)$/.exec(offender);
+        return regexResult ? regexResult[1] : null;
+    };
+
+    this.parseColor = function(color) {
+        var colorArray = parse(color).rgba;
+
+        var obj = {
+            R: colorArray[0],
+            G: colorArray[1],
+            B: colorArray[2],
+            A: colorArray[3]
+        };
+
+        obj.Lab = diff.rgb_to_lab(obj);
+        obj.original = color;
+
+        return obj;
+    };
+
+    this.compareTwoColors = function(color1, color2) {
+        if (color1.A !== color2.A) {
+            debug('Comparison not possible, because the alpha channel is different');
+            return false;
+        }
+
+        var delta = diff.diff(color1.Lab, color2.Lab);
+        debug('Delta is %d', delta);
+        
+        return delta <= COLOR_DIFF_THRESHOLD;
+    };
+
+};
+
+module.exports = new colorDiff();

lib/tools/isHttp2.js

@@ -0,0 +1,105 @@
+var debug   = require('debug')('ylt:isHttp2');
+var url     = require('url');
+var Q       = require('q');
+var http2   = require('is-http2');
+
+var isHttp2 = function() {
+    'use strict';
+
+    this.check = function(data) {
+        debug('Starting to check for HTTP2 support...');
+
+        return this.checkHttp2(data)
+
+            .then(function(result) {
+
+                if (result.isHttp) {
+                    debug('The website is not even in HTTPS');
+
+                    data.toolsResults.http2 = {
+                        metrics: {
+                            http2: false
+                        }
+                    };
+
+                } else if (result.isHttp2) {
+                    debug('HTTP/2 (or SPDY) is supported');
+
+                    data.toolsResults.http2 = {
+                        metrics: {
+                            http2: true
+                        }
+                    };
+                
+                } else {
+                    debug('HTTP/2 is not supported');
+                    
+                    data.toolsResults.http2 = {
+                        metrics: {
+                            http2: false
+                        }
+                    };
+                }
+
+                // Add the supported protocols as offenders
+                if (result.supportedProtocols) {
+                    debug('Supported protocols: ' + result.supportedProtocols.join(' '));
+                    data.toolsResults.http2.offenders = {
+                        http2: result.supportedProtocols
+                    };
+                }
+
+                debug('End of HTTP2 support check');
+
+                return data;
+            })
+
+            .fail(function() {
+                return data;
+            });
+    };
+
+    this.getParsedUrl = function(data) {
+        return url.parse(data.toolsResults.phantomas.url);
+    };
+
+    this.getProtocol = function(data) {
+        return this.getParsedUrl(data).protocol;
+    };
+
+    this.getDomain = function(data) {
+        return this.getParsedUrl(data).hostname;
+    };
+
+    this.checkHttp2 = function(data) {
+        var deferred = Q.defer();
+
+        // Check if it's HTTPS first
+        if (this.getProtocol(data) === 'http:') {
+            
+            deferred.resolve({
+                isHttp: true
+            });
+
+        } else {
+
+            // To make is-http2 work, you need to have openssl in a version greater than 1.0.0 installed and available in your $path.
+            http2(this.getDomain(data), {includeSpdy: true})
+
+                .then(function(result) {
+                    deferred.resolve(result);
+                })
+                
+                .catch(function(error) {
+                    debug('Error while checking HTTP2 support:');
+                    debug(error);
+                    deferred.reject('Error while checking for HTTP2 support');
+                });
+
+        }
+
+        return deferred.promise;
+    };
+};
+
+module.exports = new isHttp2();

lib/tools/jsExecutionTransformer.js

@@ -38,7 +38,7 @@ var jsExecutionTransformer = function() {
             javascriptExecutionTree = JSON.parse(data.toolsResults.phantomas.offenders.javascriptExecutionTree[0]);
         
             if (javascriptExecutionTree.children) {
-                javascriptExecutionTree.children.forEach(function(node) {
+                javascriptExecutionTree.children.forEach(function(node, index) {
                     
                     var contextLength = (node.data.callDetails && node.data.callDetails.context) ? node.data.callDetails.context.length : null;
 
@@ -90,6 +90,10 @@ var jsExecutionTransformer = function() {
                                 break;
                         }
                     }
+                    // Fix rare bug when domComplete was never triggered
+                    if (index === javascriptExecutionTree.children.length - 1 && !javascriptExecutionTree.data.domComplete) {
+                        javascriptExecutionTree.data.domComplete = node.data.timestamp + 1000;
+                    }
 
                     // Transform domPaths into objects
                     changeListOfDomPaths(node);
@@ -117,24 +121,28 @@ var jsExecutionTransformer = function() {
             debug('JS execution transformation complete');
 
 
-            debug('Starting scroll execution transformation');
-            offenders.DOMaccessesOnScroll = JSON.parse(data.toolsResults.phantomas.offenders.scrollExecutionTree[0]);
-            if (offenders.DOMaccessesOnScroll.children) {
-                offenders.DOMaccessesOnScroll.children.forEach(function(node) {
-                    
-                    // Mark a event flag
-                    if (['documentScroll', 'windowScroll', 'window.onscroll'].indexOf(node.data.type) >= 0) {
-                        node.windowPerformance = true;
-                    }
+            if (data.toolsResults.phantomas.offenders.scrollExecutionTree) {
+                debug('Starting scroll execution transformation');
+                offenders.DOMaccessesOnScroll = JSON.parse(data.toolsResults.phantomas.offenders.scrollExecutionTree[0]);
+                if (offenders.DOMaccessesOnScroll.children) {
+                    offenders.DOMaccessesOnScroll.children.forEach(function(node) {
+                        
+                        // Mark a event flag
+                        if (['documentScroll', 'windowScroll', 'window.onscroll'].indexOf(node.data.type) >= 0) {
+                            node.windowPerformance = true;
+                        }
 
-                    // Transform domPaths into objects
-                    changeListOfDomPaths(node);
-                    
-                    // Count the number of DOM accesses, by counting the tree leafs
-                    metrics.DOMaccessesOnScroll += countTreeLeafs(node);
-                });
+                        // Transform domPaths into objects
+                        changeListOfDomPaths(node);
+                        
+                        // Count the number of DOM accesses, by counting the tree leafs
+                        metrics.DOMaccessesOnScroll += countTreeLeafs(node);
+                    });
+                }
+                debug('Scroll execution transformation complete');
+            } else {
+                debug('Could not parse scrollExecutionTree');
             }
-            debug('Scroll execution transformation complete');
 
         } catch(err) {
             throw err;

lib/tools/mediaQueriesChecker.js

@@ -0,0 +1,168 @@
+var debug   = require('debug')('ylt:mediaQueriesChecker');
+var parseMediaQuery = require('css-mq-parser');
+var offendersHelpers = require('../offendersHelpers');
+
+
+var mediaQueriesChecker = function() {
+    'use strict';
+
+    var MOBILE_MIN_BREAKPOINT = 200;
+    var MOBILE_MAX_BREAKPOINT = 300;
+
+    this.analyzeMediaQueries = function(data) {
+        debug('Starting to check all media queries...');
+
+        var offenders = data.toolsResults.phantomas.offenders.cssMediaQueries;
+        var mediaQueries = (offenders) ? this.parseAllMediaQueries(offenders) : [];
+        
+        var notMobileFirstCount = 0;
+        var notMobileFirstOffenders = [];
+
+        var breakpointsOffenders = {};
+
+        for (var i = 0; i < mediaQueries.length; i++) {
+            var item = mediaQueries[i];
+
+            if (!item) {
+                continue;
+            }
+
+            if (item.isForMobile) {
+                notMobileFirstCount += item.mediaQuery.rules;
+                notMobileFirstOffenders.push(item.mediaQuery);
+            }
+
+            for (var j = 0; j < item.breakpoints.length; j++) {
+                var breakpointString = item.breakpoints[j].string;
+                if (!breakpointsOffenders[breakpointString]) {
+                    breakpointsOffenders[breakpointString] = {
+                        count: 1,
+                        pixels: item.breakpoints[j].pixels
+                    };
+                } else {
+                    breakpointsOffenders[breakpointString].count += 1;
+                }
+            }
+        }
+
+        data.toolsResults.mediaQueriesChecker = {
+            metrics: {
+                cssMobileFirst: notMobileFirstCount,
+                cssBreakpoints: Object.keys(breakpointsOffenders).length
+            },
+            offenders: {
+                cssMobileFirst: notMobileFirstOffenders,
+                cssBreakpoints: breakpointsOffenders
+            }
+        };
+
+        debug('End of media queries check');
+
+        return data;
+    };
+
+    this.parseAllMediaQueries = function(offenders) {
+        return offenders.map(this.parseOneMediaQuery);
+    };
+
+    this.parseOneMediaQuery = function(offender) {
+        var splittedOffender = offendersHelpers.cssOffenderPattern(offender);
+        var parts = /^@media (.*) \((\d+ rules)\)$/.exec(splittedOffender.css);
+
+        if (!parts) {
+            debug('Failed to parse media query ' + offender);
+            return false;
+        }
+
+        var rulesCount = parseInt(parts[2], 10);
+        var query = parts[1];
+
+        var isForMobile = false;
+        var breakpoints = [];
+
+        try {
+
+            var ast = parseMediaQuery(query);
+
+            var min = 0;
+            var max = Infinity;
+            var pixels;
+
+            ast.forEach(function(astItem) {
+                astItem.expressions.forEach(function(expression) {
+                    if (expression.feature === 'width' || expression.feature === 'device-width') {
+                        if (astItem.inverse === false) {
+                            if (expression.modifier === 'max') {
+                                pixels = toPixels(expression.value);
+                                max = Math.min(max, pixels);
+                                breakpoints.push({
+                                    string: expression.value,
+                                    pixels: pixels
+                                });
+                            } else if (expression.modifier === 'min') {
+                                pixels = toPixels(expression.value);
+                                min = Math.max(min, pixels);
+                                breakpoints.push({
+                                    string: expression.value,
+                                    pixels: pixels
+                                });
+                            }
+                        } else if (astItem.inverse === true) {
+                            if (expression.modifier === 'max') {
+                                pixels = toPixels(expression.value);
+                                min = Math.max(min, pixels);
+                                breakpoints.push({
+                                    string: expression.value,
+                                    pixels: pixels
+                                });
+                            } else if (expression.modifier === 'min') {
+                                pixels = toPixels(expression.value);
+                                max = Math.min(max, pixels);
+                                breakpoints.push({
+                                    string: expression.value,
+                                    pixels: pixels
+                                });
+                            }
+                        }
+                    }
+                });
+            });
+
+            isForMobile = (min <= MOBILE_MIN_BREAKPOINT && max >= MOBILE_MAX_BREAKPOINT && max !== Infinity);
+
+        } catch(error) {
+            debug('Failed to parse media query ' + offender);
+        }
+
+        return {
+            mediaQuery: {
+                query: query,
+                rules: rulesCount,
+                file: splittedOffender.file,
+                line: splittedOffender.line,
+                column: splittedOffender.column
+            },
+            isForMobile: isForMobile,
+            breakpoints: breakpoints
+        };
+    };
+
+    // Parses a size in em, pt (or px) and returns it in px
+    function toPixels(size) {
+        var splittedSize = /^([\d\.]+)(.*)/.exec(size);
+        var value = parseFloat(splittedSize[1]);
+        var unit = splittedSize[2];
+
+        if (unit === 'em') {
+            return value * 16;
+        }
+
+        if (unit === 'pt') {
+            return value / 12 * 16;
+        }
+
+        return value;
+    }
+};
+
+module.exports = new mediaQueriesChecker();

lib/tools/phantomas/custom_modules/core/scopeYLT/scopeYLT.js

@@ -18,8 +18,8 @@ exports.module = function(phantomas) {
         responseEndTime = Date.now();
     });
 
-    phantomas.once('init', function() {
-        phantomas.evaluate(function(responseEndTime, deepAnalysis) {
+    phantomas.on('init', function() {
+        phantomas.evaluate(function(responseEndTime) {
             (function(phantomas) {
     
                 // Overwritting phantomas spy function
@@ -49,7 +49,7 @@ exports.module = function(phantomas) {
                             var err;
                             
                             // Before
-                            if (enabled) {
+                            if (enabled && callbackBefore) {
                                 callbackBefore.apply(this, arguments);
                             }
 
@@ -71,7 +71,8 @@ exports.module = function(phantomas) {
 
                                 // After
                                 if (enabled && callbackAfter) {
-                                    callbackAfter.call(this, result, arguments);
+                                    var args = Array.prototype.slice.call(arguments);
+                                    callbackAfter.apply(this, [result].concat(args));
                                 }
 
                                 if (err) {
@@ -106,48 +107,85 @@ exports.module = function(phantomas) {
                     var currentContext = root;
                     var depth = 0;
 
-                    if (deepAnalysis) {
-                        phantomas.log('Entering deep Javascript analysis mode');
-                    }
 
-                    // Add a child but don't enter his context
+                    // Add a child but don't enter its context
                     function pushContext(data) {
-                        if (depth === 0 || deepAnalysis) {
+                        
+                        // Some data is not needed on subchildren
+                        if (depth === 0) {
                             data.timestamp = Date.now() - responseEndTime;
                             data.loadingStep = phantomas.currentStep || '';
-                            currentContext.addChild(data);
                         }
+
+                        // Some data is not needed on subchildren
+                        if (depth > 0) {
+                            if (data.backtrace) {
+                                delete data.backtrace;
+                            }
+                            if (data.resultsNumber) {
+                                delete data.resultsNumber;
+                            }
+                        }
+
+                        currentContext.addChild(data);
                     }
                     
-                    // Add a child to the current context and enter his context
+                    // Add a child to the current context and enter its context
                     function enterContext(data) {
-                        if (depth === 0 || deepAnalysis) {
+                        
+                        // Some data is not needed on subchildren
+                        if (depth === 0) {
                             data.timestamp = Date.now() - responseEndTime;
                             data.loadingStep = phantomas.currentStep || '';
-                            currentContext = currentContext.addChild(data);
                         }
+
+                        // Some data is not needed on subchildren
+                        if (depth > 0) {
+                            if (data.backtrace) {
+                                delete data.backtrace;
+                            }
+                            if (data.resultsNumber) {
+                                delete data.resultsNumber;
+                            }
+                        }
+
+                        currentContext = currentContext.addChild(data);
+
                         depth ++;
                     }
                     
                     // Save given data in the current context and jump change current context to its parent
                     function leaveContext(moreData) {
-                        if (depth === 1 || deepAnalysis) {
+                        
+                        // Some data is not needed on subchildren
+                        if (depth === 1) {
                             currentContext.data.time = Date.now() - currentContext.data.timestamp - responseEndTime;
+                        }
 
-                            // Merge previous data with moreData (ovewrites if exists)
-                            if (moreData) {
-                                for (var key in moreData) {
-                                    currentContext.data[key] = moreData[key];
-                                }
+                        // Some data is not needed on subchildren
+                        if (depth > 1) {
+                            if (moreData && moreData.backtrace) {
+                                delete moreData.backtrace;
                             }
-
-                            var parent = currentContext.parent;
-                            if (parent === null) {
-                                console.error('Error: trying to close root context in ContextTree');
-                            } else {
-                                currentContext = parent;
+                            if (moreData && moreData.resultsNumber) {
+                                delete moreData.resultsNumber;
                             }
                         }
+
+                        // Merge previous data with moreData (ovewrites if exists)
+                        if (moreData) {
+                            for (var key in moreData) {
+                                currentContext.data[key] = moreData[key];
+                            }
+                        }
+
+                        var parent = currentContext.parent;
+                        if (parent === null) {
+                            console.error('Error: trying to close root context in ContextTree');
+                        } else {
+                            currentContext = parent;
+                        }
+
                         depth --;
                     }
                     
@@ -161,7 +199,6 @@ exports.module = function(phantomas) {
                         if (root !== currentContext) {
                             return null;
                         }
-                        var current = currentContext;
 
                         function recusiveRead(node) {
                             if (node.children.length === 0) {
@@ -209,6 +246,6 @@ exports.module = function(phantomas) {
                 })();
 
             })(window.__phantomas);
-        }, responseEndTime, phantomas.getParam('js-deep-analysis'));
+        }, responseEndTime);
     });
 };

lib/tools/phantomas/custom_modules/modules/ajaxReqYLT/ajaxReqYLT.js

@@ -0,0 +1,30 @@
+/**
+ * Analyzes AJAX requests
+ */
+/* global window: true */
+
+exports.version = '0.2.a';
+
+exports.module = function(phantomas) {
+    'use strict';
+
+    phantomas.setMetric('ajaxRequests'); // @desc number of AJAX requests
+    phantomas.setMetric('synchronousXHR'); // @desc number of synchronous 
+
+    phantomas.on('init', function() {
+        phantomas.evaluate(function() {
+            (function(phantomas) {
+                phantomas.spy(window.XMLHttpRequest.prototype, 'open', null, function(result, method, url, async) {
+                    phantomas.incrMetric('ajaxRequests');
+                    phantomas.addOffender('ajaxRequests', '<%s> [%s]', url, method);
+
+                    if (async === false) {
+                        phantomas.incrMetric('synchronousXHR');
+                        phantomas.addOffender('synchronousXHR', url);
+                        phantomas.log('ajaxRequests: synchronous XMLHttpRequest call to <%s>', url);
+                    }
+                }, true);
+            })(window.__phantomas);
+        });
+    });
+};

lib/tools/phantomas/custom_modules/modules/domHiddenYLT/domHiddenYLT.js

@@ -3,21 +3,23 @@
  */
 /* global document: true, Node: true, window: true */
 
-exports.version = '0.1.a';
+exports.version = '1.0.a';
 
 exports.module = function(phantomas) {
     'use strict';
 
     // total length of HTML of hidden elements (i.e. display: none)
     phantomas.setMetric('hiddenContentSize'); // @desc the size of content of hidden elements on the page (with CSS display: none) @offenders
+    phantomas.setMetric('hiddenImages'); // @desc number of hidden images that can be lazy-loaded @offenders
 
     // HTML size
     phantomas.on('report', function() {
         phantomas.evaluate(function() {
             (function(phantomas) {
-                phantomas.spyEnabled(false, 'checking the hiddenContentSize');
+                var runner = new phantomas.nodeRunner(),
+                    lazyLoadableImages = {};
 
-                var runner = new phantomas.nodeRunner();
+                phantomas.spyEnabled(false, 'analyzing hidden content');
 
                 runner.walk(document.body, function(node, depth) {
                     switch (node.nodeType) {
@@ -36,6 +38,31 @@ exports.module = function(phantomas) {
                                     }
                                 }
 
+                                // count hidden images that can be lazy loaded (issue #524)
+                                var images = [];
+                                if (node.tagName === 'IMG') {
+                                    images = [node];
+                                } else if (typeof node.querySelectorAll === 'function') {
+                                    images = node.querySelectorAll('img') || [];
+                                }
+
+                                for (var i = 0, len = images.length; i < len; i++) {
+                                    var src = images[i].src,
+                                        path;
+
+                                    if (src === '' || src.indexOf('data:image') === 0) continue;
+
+                                    if (images[i].width === 1 && images[i].height === 1) continue;
+
+                                    if (!lazyLoadableImages[src]) {
+                                        path = phantomas.getDOMPath(images[i]);
+
+                                        lazyLoadableImages[src] = {
+                                            path: path
+                                        };
+                                    }
+                                }
+
                                 // don't run for child nodes as they're hidden as well
                                 return false;
                             }
@@ -43,6 +70,15 @@ exports.module = function(phantomas) {
                     }
                 });
 
+                Object.keys(lazyLoadableImages).forEach(function(img) {
+                    var entry = lazyLoadableImages[img];
+
+                    phantomas.incrMetric('hiddenImages');
+                    phantomas.addOffender('hiddenImages', img);
+
+                    phantomas.log('hiddenImages: <%s> image (%s) is hidden and can be lazy-loaded', img, entry.path);
+                });
+
                 phantomas.spyEnabled(true);
             }(window.__phantomas));
         });

lib/tools/phantomas/custom_modules/modules/domQYLT/domQYLT.js

@@ -19,7 +19,7 @@ exports.module = function(phantomas) {
     phantomas.setMetric('DOMqueriesAvoidable'); // @desc number of repeated uses of a duplicated query
 
     // fake native DOM functions
-    phantomas.once('init', function() {
+    phantomas.on('init', function() {
         phantomas.evaluate(function() {
             (function(phantomas) {
                 function querySpy(type, query, fnName, context, hasNoResults) {

lib/tools/phantomas/custom_modules/modules/eventYLT/eventYLT.js

@@ -12,7 +12,7 @@ exports.module = function(phantomas) {
     phantomas.setMetric('eventsDispatched'); // @desc number of EventTarget.dispatchEvent calls
     phantomas.setMetric('eventsScrollBound'); // @desc number of scroll event bounds
 
-    phantomas.once('init', function() {
+    phantomas.on('init', function() {
         phantomas.evaluate(function() {
             (function(phantomas) {
                 // spy calls to EventTarget.addEventListener

lib/tools/phantomas/custom_modules/modules/jQYLT/jQYLT.js

@@ -138,10 +138,9 @@ exports.module = function(phantomas) {
     jQueryFunctions = jQueryFunctions.concat(jQueryTraversalFunctions);
 
     // spy calls to jQuery functions
-    phantomas.once('init', function() {
+    phantomas.on('init', function() {
         phantomas.evaluate(function(jQueryFunctions, jQueryTraversalFunctions) {
             (function(phantomas) {
-                var jQuery;
                 var oldJQuery;
 
                 phantomas.spyGlobalVar('jQuery', function(jQuery) {
@@ -371,14 +370,6 @@ exports.module = function(phantomas) {
     });
 
 
-    // store the last resource that was received
-    // try to report where given jQuery version was loaded from
-    phantomas.on('recv', function(entry) {
-        if (entry.isJS) {
-            lastUrl = entry.url;
-        }
-    });
-
     phantomas.on('jQueryLoaded', function(version) {
         phantomas.log('jQuery: loaded v' + version);
         phantomas.setMetric('jQueryVersion', version);
@@ -386,7 +377,5 @@ exports.module = function(phantomas) {
         // report multiple jQuery "instances" (issue #435)
         phantomas.incrMetric('jQueryVersionsLoaded');
         phantomas.addOffender('jQueryVersionsLoaded', 'v%s', version);
-
-        phantomas.log('jQuery: v%s (probably loaded from <%s>)', version, lastUrl);
     });
 };

lib/tools/phantomas/custom_modules/modules/javaScriptBottleYLT/javaScriptBottleYLT.js

@@ -23,7 +23,7 @@ exports.module = function(phantomas) {
         phantomas.log('javaScriptBottlenecks: to spy calls to eval() run phantomas with --spy-eval option');
     }
 
-    phantomas.once('init', function() {
+    phantomas.on('init', function() {
         phantomas.evaluate(function(spyEval) {
             (function(phantomas) {
                 function report(msg, caller, backtrace, metric) {

lib/tools/phantomas/custom_modules/modules/lazyLoadableYLT/lazyLoadableYLT.js

@@ -0,0 +1,79 @@
+/**
+ * Analyzes images and detects which one can be lazy-loaded (are below the fold)
+ *
+ * @see https://github.com/macbre/phantomas/issues/494
+ */
+/* global document: true, window: true */
+
+exports.version = '1.0.a';
+
+exports.module = function(phantomas) {
+    'use strict';
+    
+    phantomas.setMetric('lazyLoadableImagesBelowTheFold'); // @desc number of images displayed below the fold that can be lazy-loaded
+
+    phantomas.on('report', function() {
+        phantomas.log('lazyLoadableImages: analyzing which images can be lazy-loaded...');
+
+        phantomas.evaluate(function() {
+            (function(phantomas) {
+                phantomas.spyEnabled(false, 'analyzing which images can be lazy-loaded');
+
+                var images = document.body.getElementsByTagName('img'),
+                    i,
+                    len = images.length,
+                    offset,
+                    path,
+                    processedImages = {},
+                    src,
+                    viewportHeight = window.innerHeight;
+
+                phantomas.log('lazyLoadableImages: %d image(s) found, assuming %dpx offset to be the fold', len, viewportHeight);
+
+                for (i = 0; i < len; i++) {
+                    // @see https://developer.mozilla.org/en-US/docs/Web/API/Element/getBoundingClientRect
+                    offset = images[i].getBoundingClientRect().top;
+                    src = images[i].src;
+
+                    // ignore base64-encoded images
+                    if (src === '' || /^data:/.test(src)) {
+                        continue;
+                    }
+
+                    path = phantomas.getDOMPath(images[i]);
+
+                    // get the most top position for a given image (deduplicate by src)
+                    if (typeof processedImages[src] === 'undefined') {
+                        processedImages[src] = {
+                            offset: offset,
+                            path: path
+                        };
+                    }
+
+                    // maybe there's the same image loaded above the fold?
+                    if (offset < processedImages[src].offset) {
+                        processedImages[src] = {
+                            offset: offset,
+                            path: path
+                        };
+                    }
+                }
+
+                phantomas.log('lazyLoadableImages: checking %d unique image(s)', Object.keys(processedImages).length);
+
+                Object.keys(processedImages).forEach(function(src) {
+                    var img = processedImages[src];
+
+                    if (img.offset > viewportHeight) {
+                        phantomas.log('lazyLoadableImages: <%s> image (%s) is below the fold (at %dpx)', src, img.path, img.offset);
+
+                        phantomas.incrMetric('lazyLoadableImagesBelowTheFold');
+                        phantomas.addOffender('lazyLoadableImagesBelowTheFold', src);
+                    }
+                });
+
+                phantomas.spyEnabled(true);
+            })(window.__phantomas);
+        });
+    });
+};

lib/tools/phantomas/custom_modules/modules/requestsList/requestsList.js

@@ -0,0 +1,23 @@
+/**
+ * Retries download on every request to get the real file size
+ *
+ */
+
+exports.version = '0.1';
+
+exports.module = function(phantomas) {
+    'use strict';
+
+    phantomas.setMetric('requestsList');
+
+    var requests = [];
+
+    phantomas.on('recv', function(entry, res) {
+        requests.push(entry);
+    });
+
+    phantomas.on('report', function() {
+        phantomas.setMetric('requestsList', true, true);
+        phantomas.addOffender('requestsList', JSON.stringify(requests));
+    });
+};

lib/tools/phantomas/phantomasWrapper.js

@@ -17,13 +17,12 @@ var PhantomasWrapper = function() {
         var deferred = Q.defer();
         var task = data.params;
 
-
         var options = {
             
             // Cusomizable options
-            'timeout': task.options.timeout || 45,
-            'js-deep-analysis': task.options.jsDeepAnalysis || false,
-            'user-agent': (task.options.device === 'desktop') ? 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.110 Safari/537.36' : null,
+            'engine': task.options.phantomasEngine || 'webkit',
+            'timeout': task.options.timeout || 30,
+            'user-agent': (task.options.device === 'desktop') ? 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) YLT Chrome/27.0.1453.110 Safari/537.36' : null,
             'tablet': (task.options.device === 'tablet'),
             'phone': (task.options.device === 'phone'),
             'screenshot': task.options.screenshot || false,
@@ -31,23 +30,28 @@ var PhantomasWrapper = function() {
             'cookie': task.options.cookie,
             'auth-user': task.options.authUser,
             'auth-pass': task.options.authPass,
+            'block-domain': task.options.blockDomain,
+            'allow-domain': task.options.allowDomain,
+            'no-externals': task.options.noExternals,
 
             // Mandatory
             'reporter': 'json:pretty',
             'analyze-css': true,
+            'ignore-ssl-errors': true,
             'skip-modules': [
-                'blockDomains', // not needed
-                'domHiddenContent', // overriden
+                'ajaxRequests', // overridden
+                'domHiddenContent', // overridden
                 'domMutations', // not compatible with webkit
-                'domQueries', // overriden
+                'domQueries', // overridden
                 'events', // overridden
                 'filmStrip', // not needed
                 'har', // not needed for the moment
                 'javaScriptBottlenecks', // needs to be launched after custom module scopeYLT
                 'jQuery', // overridden
                 'jserrors', // overridden
+                'lazyLoadableImages', //overridden
                 'pageSource', // not needed
-                'windowPerformance' // overriden
+                'windowPerformance' // overridden
             ].join(','),
             'include-dirs': [
                 path.join(__dirname, 'custom_modules/core'),
@@ -67,7 +71,7 @@ var PhantomasWrapper = function() {
 
             if (value === true) {
                 optionsString += ' ' + '--' + opt;
-            } else if (value === false || value === null) {
+            } else if (value === false || value === null || value === undefined) {
                 // Nothing
             } else {
                 optionsString += ' ' + '--' + opt + '=' + value;
@@ -112,9 +116,14 @@ var PhantomasWrapper = function() {
 
         // It's time to launch the test!!!
         var triesNumber = 2;
+        var currentTry = 0;
 
         async.retry(triesNumber, function(cb) {
+
+            currentTry ++;
+
             var process = phantomas(task.url, options, function(err, json, results) {
+                var errorCode = err ? parseInt(err.message, 10) : null;
                 
                 if (isKilled) {
                     debug('Process was killed, too late Phantomas, sorry...');
@@ -122,28 +131,33 @@ var PhantomasWrapper = function() {
                 }
 
 
-                debug('Returning from Phantomas');
+                debug('Returning from Phantomas with error %s', errorCode);
 
                 // Adding some YellowLabTools errors here
                 if (json && json.metrics && (!json.metrics.javascriptExecutionTree || !json.offenders.javascriptExecutionTree)) {
-                    err = 1001;
+                    errorCode = 1001;
                 }
 
-                if (!err && (!json || !json.metrics)) {
-                    err = 1002;
+                if (!errorCode && (!json || !json.metrics)) {
+                    errorCode = 1002;
                 }
 
                 // Don't cancel test if it is a timeout and we've got some results
-                if (err === 252 && json) {
+                if (errorCode === 252 && json) {
                     debug('Timeout after ' + options.timeout + ' seconds. But it\'s not a problem, the test is valid.');
-                    err = null;
+                    errorCode = null;
                 }
 
-                if (err) {
-                    debug('Attempt failed. Error code ' + err);
+                if (errorCode) {
+                    debug('Attempt failed. Error code ' + errorCode);
                 }
 
-                cb(err, json);
+                cb(errorCode, json);
+            
+            }).fail(function() {
+                // This function is useless, but the failing promise needs to be handled,
+                // otherwise the module meow writes in the console in case of a timeout (error code 252).
+                debug('Failing promise handled');
             });
             
             phantomasPid = process.pid;

lib/tools/redownload/contentTypeChecker.js

@@ -0,0 +1,188 @@
+var debug   = require('debug')('ylt:contentTypeChecker');
+var Q       = require('q');
+var isJpg   = require('is-jpg');
+var isPng   = require('is-png');
+var isSvg   = require('is-svg');
+var isGif   = require('is-gif');
+var isWoff  = require('is-woff');
+var isWoff2 = require('is-woff2');
+var isOtf   = require('is-otf');
+var isTtf   = require('is-ttf');
+var isEot   = require('is-eot');
+
+var ContentTypeChecker = function() {
+
+    function checkContentType(entry) {
+        var deferred = Q.defer();
+
+        debug('Entering contentTypeChecker');
+        
+        // Ignore very small files as they are generally tracking pixels
+        if (entry.weightCheck && entry.weightCheck.body && entry.weightCheck.bodySize > 100) {
+            var foundType;
+
+            try {
+                foundType = findContentType(entry.weightCheck.body);
+            
+                if (!entry.contentType || entry.contentType === '') {
+                    if (foundType === null) {
+                        debug('ContentType is empty for file %s', entry.url);
+                    } else {
+                        debug('ContentType is empty for file %s. It should be %s.', entry.url, foundType.mimes[0]);
+                        entry.oldContentType = null;
+                        rewriteContentType(entry, foundType);
+                    }
+                } else {
+                    if (foundType !== null && foundType.mimes.indexOf(entry.contentType) === -1) {
+                        debug('ContentType %s is wrong for %s. It should be %s.', entry.contentType, entry.url, foundType.mimes[0]);
+                        entry.oldContentType = entry.contentType;
+                        rewriteContentType(entry, foundType);
+                    }
+                }
+
+            } catch(err) {
+                debug('Error while analyzing the contentType of %s', entry.url);
+                debug(err);
+            }
+        }
+
+        deferred.resolve(entry);
+
+        return deferred.promise;
+    }
+
+    function findContentType(body) {
+        var buffer = new Buffer(body, 'binary');
+
+        if (isJpg(buffer)) {
+            return contentTypes.jpeg;
+        }
+
+        if (isPng(buffer)) {
+            return contentTypes.png;
+        }
+
+        // https://github.com/sindresorhus/is-svg/issues/7
+        if (/<svg/.test(body) && isSvg(body)) {
+            return contentTypes.svg;
+        }
+
+        if (isGif(buffer)) {
+            return contentTypes.gif;
+        }
+
+        if (isWoff(buffer)) {
+            return contentTypes.woff;
+        }
+
+        if (isWoff2(buffer)) {
+            return contentTypes.woff2;
+        }
+
+        if (isOtf(buffer)) {
+            return contentTypes.otf;
+        }
+
+        if (isTtf(buffer)) {
+            return contentTypes.ttf;
+        }
+
+        if (isEot(buffer)) {
+            return contentTypes.eot;
+        }
+
+        return null;
+    }
+
+
+    function rewriteContentType(entry, contentTypeObj) {
+        delete(entry.isHTML);
+        delete(entry.isXML);
+        delete(entry.isCSS);
+        delete(entry.isJS);
+        delete(entry.isJSON);
+        delete(entry.isImage);
+        delete(entry.isSVG);
+        delete(entry.isVideo);
+        delete(entry.isWebFont);
+        delete(entry.isTTF);
+        delete(entry.isFavicon);
+
+        entry.contentType = contentTypeObj.mimes[0];
+        contentTypeObj.updateFn(entry);
+    }
+
+    var contentTypes = {
+        jpeg: {
+            mimes: ['image/jpeg'],
+            updateFn: function(entry) {
+                entry.type = 'image';
+                entry.isImage = true;
+            }
+        },
+        png: {
+            mimes: ['image/png'],
+            updateFn: function(entry) {
+                entry.type = 'image';
+                entry.isImage = true;
+            }
+        },
+        svg: {
+            mimes: ['image/svg+xml'],
+            updateFn: function(entry) {
+                entry.type = 'image';
+                entry.isImage = true;
+                entry.isSVG = true;
+            }
+        },
+        gif: {
+            mimes: ['image/gif'],
+            updateFn: function(entry) {
+                entry.type = 'image';
+                entry.isImage = true;
+            }
+        },
+        woff: {
+            mimes: ['application/x-font-woff', 'application/font-woff', 'font/woff'],
+            updateFn: function(entry) {
+                entry.type = 'webfont';
+                entry.isWebFont = true;
+            }
+        },
+        woff2: {
+            mimes: ['font/woff2', 'application/x-font-woff2', 'application/font-woff2'],
+            updateFn: function(entry) {
+                entry.type = 'webfont';
+                entry.isWebFont = true;
+            }
+        },
+        otf: {
+            mimes: ['application/x-font-otf', 'font/otf', 'font/opentype', 'application/x-font-opentype'],
+            updateFn: function(entry) {
+                entry.type = 'webfont';
+                entry.isWebFont = true;
+            }
+        },
+        ttf: {
+            mimes: ['application/x-font-ttf', 'font/ttf', 'application/x-font-truetype'],
+            updateFn: function(entry) {
+                entry.type = 'webfont';
+                entry.isWebFont = true;
+            }
+        },
+        eot: {
+            mimes: ['application/vnd.ms-fontobject', 'font/eot'],
+            updateFn: function(entry) {
+                entry.type = 'webfont';
+                entry.isWebFont = true;
+            }
+        }
+    };
+    
+    return {
+        checkContentType: checkContentType,
+        findContentType: findContentType
+    };
+};
+
+module.exports = new ContentTypeChecker();

lib/tools/redownload/fileMinifier.js

@@ -0,0 +1,352 @@
+var debug = require('debug')('ylt:fileMinifier');
+
+var Q               = require('q');
+var UglifyJS        = require('uglify-js');
+var CleanCSS        = require('clean-css');
+var Minimize        = require('minimize');
+
+
+var FileMinifier = function() {
+
+    function minifyFile(entry) {
+        var deferred = Q.defer();
+
+        if (!entry.weightCheck || !entry.weightCheck.body) {
+            // No valid file available
+            deferred.resolve(entry);
+            return deferred.promise;
+        }
+
+        var fileSize = entry.weightCheck.uncompressedSize;
+        debug('Let\'s try to optimize %s', entry.url);
+        debug('Current file size is %d', fileSize);
+        var startTime = Date.now();
+
+        if (entry.isJS && !isKnownAsMinified(entry.url) && !looksAlreadyMinified(entry.weightCheck.body)) {
+
+            debug('File is a JS');
+
+            return minifyJs(entry.weightCheck.body)
+
+            .then(function(newFile) {
+                if (!newFile) {
+                    debug('Optimization didn\'t work');
+                    return entry;
+                }
+
+                var endTime = Date.now();
+
+                var newFileSize = newFile.length;
+
+                debug('JS minification complete for %s', entry.url);
+                
+                if (gainIsEnough(fileSize, newFileSize)) {
+                    entry.weightCheck.bodyAfterOptimization = newFile;
+                    entry.weightCheck.optimized = newFileSize;
+                    entry.weightCheck.isOptimized = false;
+                    debug('Filesize is %d bytes smaller (-%d%)', fileSize - newFileSize, Math.round((fileSize - newFileSize) * 100 / fileSize));
+                }
+
+                return entry;
+            })
+
+            .fail(function(err) {
+                return entry;
+            });
+
+        } else if (entry.isCSS) {
+
+            debug('File is a CSS');
+
+            return minifyCss(entry.weightCheck.body)
+
+            .then(function(newFile) {
+                if (!newFile) {
+                    debug('Optimization didn\'t work');
+                    return entry;
+                }
+
+                var endTime = Date.now();
+                debug('CSS minification took %dms', endTime - startTime);
+
+                var newFileSize = newFile.length;
+
+                debug('CSS minification complete for %s', entry.url);
+                
+                if (gainIsEnough(fileSize, newFileSize)) {
+                    entry.weightCheck.bodyAfterOptimization = newFile;
+                    entry.weightCheck.optimized = newFileSize;
+                    entry.weightCheck.isOptimized = false;
+                    debug('Filesize is %d bytes smaller (-%d%)', fileSize - newFileSize, Math.round((fileSize - newFileSize) * 100 / fileSize));
+                }
+
+                return entry;
+            })
+
+            .fail(function(err) {
+                return entry;
+            });
+
+        } else if (entry.isHTML) {
+
+            debug('File is an HTML');
+
+            return minifyHtml(entry.weightCheck.body)
+
+            .then(function(newFile) {
+                if (!newFile) {
+                    debug('Optimization didn\'t work');
+                    return entry;
+                }
+
+                var endTime = Date.now();
+                debug('HTML minification took %dms', endTime - startTime);
+
+                var newFileSize = newFile.length;
+
+                debug('HTML minification complete for %s', entry.url);
+                
+                if (gainIsEnough(fileSize, newFileSize)) {
+                    entry.weightCheck.bodyAfterOptimization = newFile;
+                    entry.weightCheck.optimized = newFileSize;
+                    entry.weightCheck.isOptimized = false;
+                    debug('Filesize is %d bytes smaller (-%d%)', fileSize - newFileSize, Math.round((fileSize - newFileSize) * 100 / fileSize));
+                }
+
+                return entry;
+            })
+
+            .fail(function(err) {
+                return entry;
+            });
+
+        } else {
+            debug('Not minifiable type or already minified');
+            deferred.resolve(entry);
+        }
+
+        return deferred.promise;
+    }
+
+    // The gain is estimated of enough value if it's over 2KB or over 20%,
+    // but it's ignored if is below 400 bytes
+    function gainIsEnough(oldWeight, newWeight) {
+        var gain = oldWeight - newWeight;
+        var ratio = gain / oldWeight;
+        return (gain > 2096 || (ratio > 0.2 && gain > 400));
+    }
+
+    // Uglify
+    function minifyJs(body) {
+        
+        // Splitting the Uglify function because it sometime takes too long (more than 10 seconds)
+        // I hope that, by splitting, it can be a little more asynchronous, so the application doesn't freeze.
+
+        return splittedUglifyStep1(body)
+        .delay(1)
+        .then(splittedUglifyStep2)
+        .delay(1)
+        .then(function(ast) {
+            // Only do the compression step for smaller files
+            // otherwise it can take a very long time compared to the gain
+            if (body.length < 200*1024) {
+                return splittedUglifyStep3(ast);
+            } else {
+                debug('Skipping step 3 because the file is too big (%d bytes)!', body.length);
+                return ast;
+            }
+        })
+        .delay(1)
+        .then(splittedUglifyStep4)
+        .delay(1)
+        .then(splittedUglifyStep5)
+        .delay(1)
+        .then(splittedUglifyStep6)
+        .delay(1)
+        .then(splittedUglifyStep7);
+
+    }
+
+    function splittedUglifyStep1(code) {
+        var deferred = Q.defer();
+        var startTime = Date.now();
+
+        try {
+            var toplevel_ast = UglifyJS.parse(code);
+
+            var endTime = Date.now();
+            debug('Uglify step 1 took %dms', endTime - startTime);
+            deferred.resolve(toplevel_ast);
+
+        } catch(err) {
+            debug('JS syntax error, Uglify\'s parser failed (step 1)');
+            deferred.reject(err);
+        }
+
+        return deferred.promise;
+    }
+
+    function splittedUglifyStep2(toplevel) {
+        var deferred = Q.defer();
+        var startTime = Date.now();
+
+        toplevel.figure_out_scope();
+
+        var endTime = Date.now();
+        debug('Uglify step 2 took %dms', endTime - startTime);
+        deferred.resolve(toplevel);
+        return deferred.promise;
+    }
+
+    function splittedUglifyStep3(toplevel) {
+        var deferred = Q.defer();
+        var startTime = Date.now();
+
+        var compressor = UglifyJS.Compressor({warnings: false});
+        var compressed_ast = toplevel.transform(compressor);
+
+        var endTime = Date.now();
+        debug('Uglify step 3 took %dms', endTime - startTime);
+        deferred.resolve(compressed_ast);
+        return deferred.promise;
+    }
+
+    function splittedUglifyStep4(compressed_ast) {
+        var deferred = Q.defer();
+        var startTime = Date.now();
+
+        compressed_ast.figure_out_scope();
+
+        var endTime = Date.now();
+        debug('Uglify step 4 took %dms', endTime - startTime);
+        deferred.resolve(compressed_ast);
+        return deferred.promise;
+    }
+
+    function splittedUglifyStep5(compressed_ast) {
+        var deferred = Q.defer();
+        var startTime = Date.now();
+
+        compressed_ast.compute_char_frequency();
+
+        var endTime = Date.now();
+        debug('Uglify step 5 took %dms', endTime - startTime);
+        deferred.resolve(compressed_ast);
+        return deferred.promise;
+    }
+
+    function splittedUglifyStep6(compressed_ast) {
+        var deferred = Q.defer();
+        var startTime = Date.now();
+
+        compressed_ast.mangle_names();
+
+        var endTime = Date.now();
+        debug('Uglify step 6 took %dms', endTime - startTime);
+        deferred.resolve(compressed_ast);
+        return deferred.promise;
+    }
+
+    function splittedUglifyStep7(compressed_ast) {
+        var deferred = Q.defer();
+        var startTime = Date.now();
+
+        var code = compressed_ast.print_to_string();
+
+        var endTime = Date.now();
+        debug('Uglify step 7 took %dms', endTime - startTime);
+        deferred.resolve(code);
+        return deferred.promise;
+    }
+
+    // Clear-css
+    function minifyCss(body) {
+        var deferred = Q.defer();
+
+        try {
+            var result = new CleanCSS({compatibility: 'ie8'}).minify(body);
+            deferred.resolve(result.styles);
+        } catch(err) {
+            deferred.reject(err);
+        }
+
+        return deferred.promise;
+    }
+
+    // HTMLMinifier
+    function minifyHtml(body) {
+        var deferred = Q.defer();
+
+        var minimize = new Minimize({
+            empty: true,        // KEEP empty attributes
+            conditionals: true, // KEEP conditional internet explorer comments
+            spare: true         // KEEP redundant attributes
+        });
+
+        minimize.parse(body, function (error, data) {
+            if (error) {
+                deferred.reject(error);
+            } else {
+                deferred.resolve(data);
+            }
+        });
+
+        return deferred.promise;
+    }
+
+    // Avoid loosing time trying to compress some JS libraries known as already compressed
+    function isKnownAsMinified(url) {
+        var result = false;
+
+        // Twitter
+        result = result || /^https?:\/\/platform\.twitter\.com\/widgets\.js/.test(url);
+
+        // Facebook
+        result = result || /^https:\/\/connect\.facebook\.net\/[^\/]*\/(sdk|all)\.js/.test(url);
+
+        // Google +1
+        result = result || /^https:\/\/apis\.google\.com\/js\/plusone\.js/.test(url);
+
+        // jQuery CDN
+        result = result || /^https?:\/\/code\.jquery\.com\/.*\.min.js/.test(url);
+
+        // Google Analytics
+        result = result || /^https?:\/\/(www|ssl)\.google-analytics\.com\/(.*)\.js/.test(url);
+
+        if (result === true) {
+            debug('This file is known as already minified. Skipping minification: %s', url);
+        }
+
+        return result;
+    }
+
+    // Avoid loosing some trying to compress JS files if they alreay look minified
+    // by counting the number of lines compared to the total size.
+    // Less than 2KB per line is suspicious
+    function looksAlreadyMinified(code) {
+        var linesCount = code.split(/\r\n|\r|\n/).length;
+        var linesRatio = code.length / linesCount;
+        var looksMinified = linesRatio > 2 * 1024;
+        
+        debug('Lines ratio is %d bytes per line', Math.round(linesRatio));
+        debug(looksMinified ? 'It looks already minified' : 'It doesn\'t look minified');
+
+        return looksMinified;
+    }
+
+    function entryTypeCanBeMinified(entry) {
+        return entry.isJS || entry.isCSS || entry.isHTML;
+    }
+
+    return {
+        minifyFile: minifyFile,
+        minifyJs: minifyJs,
+        minifyCss: minifyCss,
+        minifyHtml: minifyHtml,
+        gainIsEnough: gainIsEnough,
+        entryTypeCanBeMinified: entryTypeCanBeMinified,
+        isKnownAsMinified: isKnownAsMinified
+    };
+};
+
+module.exports = new FileMinifier();

lib/tools/redownload/gzipCompressor.js

@@ -0,0 +1,102 @@
+var debug = require('debug')('ylt:gzipCompressor');
+
+var Q       = require('q');
+var zlib    = require('zlib');
+
+var GzipCompressor = function() {
+
+    function compressFile(entry) {
+        debug('Entering compressFile');
+        return gzipUncompressedFile(entry)
+
+        .then(gzipOptimizedFile);
+    }
+
+    // Gzip a file if it was not already gziped
+    function gzipUncompressedFile(entry) {
+        var deferred = Q.defer();
+
+        if (entryTypeCanBeGzipped(entry) && entry.weightCheck && !entry.weightCheck.isCompressed && entry.weightCheck.body) {
+            debug('Compression missing, trying to gzip file %s', entry.url);
+
+            var uncompressedSize = entry.weightCheck.uncompressedSize;
+
+            zlib.gzip(new Buffer(entry.weightCheck.body, 'utf8'), function(err, buffer) {
+                if (err) {
+                    debug('Could not compress uncompressed file with gzip');
+                    debug(err);
+
+                    deferred.reject(err);
+                } else {
+                    var compressedSize = buffer.length;
+
+                    if (gainIsEnough(uncompressedSize, compressedSize)) {
+                        debug('File correctly gziped, was %d and is now %d bytes', uncompressedSize, compressedSize);
+
+                        entry.weightCheck.afterCompression = compressedSize;
+                    } else {
+                        debug('Gzip gain is not enough, was %d and is now %d bytes', uncompressedSize, compressedSize);
+                    }
+
+                    deferred.resolve(entry);
+                }
+            });
+        } else {
+            debug('Compression not needed');
+            deferred.resolve(entry);
+        }
+
+        return deferred.promise;
+    }
+
+    // Gzip a file after minification or optimization if this step was successful
+    function gzipOptimizedFile(entry) {
+        var deferred = Q.defer();
+
+        if (entryTypeCanBeGzipped(entry) && entry.weightCheck && entry.weightCheck.isOptimized === false) {
+            debug('Trying to gzip file after minification: %s', entry.url);
+
+            var uncompressedSize = entry.weightCheck.optimized;
+
+            zlib.gzip(new Buffer(entry.weightCheck.bodyAfterOptimization, 'utf8'), function(err, buffer) {
+                if (err) {
+                    debug('Could not compress minified file with gzip');
+                    debug(err);
+
+                    deferred.reject(err);
+                } else {
+                    var compressedSize = buffer.length;
+
+                    debug('Correctly gziped the minified file, was %d and is now %d bytes', uncompressedSize, compressedSize);
+                    entry.weightCheck.afterOptimizationAndCompression = compressedSize;
+
+                    deferred.resolve(entry);
+                }
+            });
+        } else {
+            debug('Compressing optimized file not needed');
+            deferred.resolve(entry);
+        }
+
+        return deferred.promise;
+    }
+
+    // The gain is estimated of enough value if it's over 1KB or over 20%,
+    // but it's ignored if is below 100 bytes
+    function gainIsEnough(oldWeight, newWeight) {
+        var gain = oldWeight - newWeight;
+        var ratio = gain / oldWeight;
+        return (gain > 2048 || (ratio > 0.2 && gain > 100));
+    }
+
+    function entryTypeCanBeGzipped(entry) {
+        return entry.isJS || entry.isCSS || entry.isHTML || entry.isJSON || entry.isSVG || entry.isTTF || entry.isXML || entry.isFavicon;
+    }
+
+    return {
+        compressFile: compressFile,
+        entryTypeCanBeGzipped: entryTypeCanBeGzipped
+    };
+};
+
+module.exports = new GzipCompressor();

lib/tools/redownload/imageOptimizer.js

@@ -0,0 +1,243 @@
+var debug               = require('debug')('ylt:imageOptimizer');
+var Q                   = require('q');
+var imagemin            = require('imagemin');
+var imageminJpegtran    = require('imagemin-jpegtran');
+var imageminJpegoptim   = require('imagemin-jpegoptim');
+var imageminOptipng     = require('imagemin-optipng');
+var imageminSvgo        = require('imagemin-svgo');
+
+var ImageOptimizer = function() {
+
+    var MAX_JPEG_QUALITY = 85;
+    var OPTIPNG_COMPRESSION_LEVEL = 1;
+
+    function optimizeImage(entry) {
+        var deferred = Q.defer();
+
+        if (!entry.weightCheck || !entry.weightCheck.body) {
+            // No valid file available
+            deferred.resolve(entry);
+            return deferred.promise;
+        }
+
+        var fileSize = entry.weightCheck.uncompressedSize;
+        debug('Let\'s try to optimize %s', entry.url);
+        debug('Current file size is %d', fileSize);
+
+        if (isJPEG(entry)) {
+            debug('File is a JPEG');
+
+            // Starting softly with a lossless compression
+            return compressJpegLosslessly(new Buffer(entry.weightCheck.body, 'binary'))
+
+            .then(function(newFile) {
+                if (!newFile) {
+                    debug('Optimization didn\'t work');
+                    return entry;
+                }
+
+                var newFileSize = newFile.length;
+
+                debug('JPEG lossless compression complete for %s', entry.url);
+                
+                if (gainIsEnough(fileSize, newFileSize)) {
+                    entry.weightCheck.lossless = entry.weightCheck.optimized = newFileSize;
+                    entry.weightCheck.isOptimized = false;
+                    debug('Filesize is %d bytes smaller (-%d%)', fileSize - newFileSize, Math.round((fileSize - newFileSize) * 100 / fileSize));
+                }
+
+
+                // Now let's compress lossy to MAX_JPEG_QUALITY
+                return compressJpegLossly(new Buffer(entry.weightCheck.body, 'binary'));
+            })
+            
+            .then(function(newFile) {
+                if (!newFile) {
+                    debug('Optimization didn\'t work');
+                    return entry;
+                }
+
+                var newFileSize = newFile.length;
+
+                debug('JPEG lossy compression complete for %s', entry.url);
+
+                if (entry.weightCheck.lossless && entry.weightCheck.lossless < newFileSize) {
+                    debug('Lossy compression is not as good as lossless compression. Skipping the lossy.');
+                } else if (gainIsEnough(fileSize, newFileSize)) {
+                    
+                    if (entry.weightCheck.isOptimized !== false || newFileSize < entry.weightCheck.lossless) {
+                        entry.weightCheck.optimized = newFileSize;
+                    }
+
+                    entry.weightCheck.lossy = newFileSize;
+                    entry.weightCheck.isOptimized = false;
+                    debug('Filesize is %d bytes smaller (-%d%)', fileSize - newFileSize, Math.round((fileSize - newFileSize) * 100 / fileSize));
+                }
+
+                return entry;
+            })
+
+            .fail(function() {
+                return entry;
+            });
+
+        } else if (isPNG(entry)) {
+
+            debug('File is a PNG');
+
+            // Starting softly with a lossless compression
+            return compressPngLosslessly(new Buffer(entry.weightCheck.body, 'binary'))
+
+            .then(function(newFile) {
+                if (!newFile) {
+                    debug('Optimization didn\'t work');
+                    return entry;
+                }
+                
+                var newFileSize = newFile.length;
+
+                debug('PNG lossless compression complete for %s', entry.url);
+                
+                debug('Old file size: %d', fileSize);
+                debug('New file size: %d', newFileSize);
+                debug('newgainIsEnough: %s', gainIsEnough(fileSize, newFileSize) ? 'true':'false');
+
+                if (gainIsEnough(fileSize, newFileSize)) {
+                    entry.weightCheck.lossless = entry.weightCheck.optimized = newFileSize;
+                    entry.weightCheck.isOptimized = false;
+                    debug('Filesize is %d bytes smaller (-%d%)', fileSize - newFileSize, Math.round((fileSize - newFileSize) * 100 / fileSize));
+                }
+
+                return entry;
+            })
+
+            .fail(function() {
+                return entry;
+            });
+
+        } else if (isSVG(entry)) {
+
+            debug('File is an SVG');
+
+            // Starting softly with a lossless compression
+            return compressSvgLosslessly(new Buffer(entry.weightCheck.body, 'utf8'))
+
+            .then(function(newFile) {
+                if (!newFile) {
+                    debug('Optimization didn\'t work');
+                    return entry;
+                }
+
+                var newFileSize = newFile.length;
+
+                debug('SVG lossless compression complete for %s', entry.url);
+                
+                if (gainIsEnough(fileSize, newFileSize)) {
+                    entry.weightCheck.bodyAfterOptimization = newFile.toString();
+                    entry.weightCheck.lossless = entry.weightCheck.optimized = newFileSize;
+                    entry.weightCheck.isOptimized = false;
+                    debug('Filesize is %d bytes smaller (-%d%)', fileSize - newFileSize, Math.round((fileSize - newFileSize) * 100 / fileSize));
+                }
+
+                return entry;
+            })
+
+            .fail(function() {
+                return entry;
+            });
+
+        } else {
+            debug('File type %s is not an optimizable image', entry.contentType);
+            deferred.resolve(entry);
+        }
+
+        return deferred.promise;
+    }
+
+    // The gain is estimated of enough value if it's over 2KB or over 20%,
+    // but it's ignored if is below 100 bytes
+    function gainIsEnough(oldWeight, newWeight) {
+        var gain = oldWeight - newWeight;
+        var ratio = gain / oldWeight;
+        return (gain > 2048 || (ratio > 0.2 && gain > 100));
+    }
+
+    function isJPEG(entry) {
+        return entry.isImage && entry.contentType === 'image/jpeg';
+    }
+
+    function isPNG(entry) {
+        return entry.isImage && entry.contentType === 'image/png';
+    }
+
+    function isSVG(entry) {
+        return entry.isImage && entry.isSVG;
+    }
+
+    function compressJpegLosslessly(imageBody) {
+        return imageminLauncher(imageBody, 'jpeg', false);
+    }
+
+    function compressJpegLossly(imageBody) {
+        return imageminLauncher(imageBody, 'jpeg', true);
+    }
+
+    function compressPngLosslessly(imageBody) {
+        return imageminLauncher(imageBody, 'png', false);
+    }
+
+    function compressSvgLosslessly(imageBody) {
+        return imageminLauncher(imageBody, 'svg', false);
+    }
+
+    function imageminLauncher(imageBody, type, lossy) {
+        var deferred = Q.defer();
+        var startTime = Date.now();
+
+        debug('Starting %s %s optimization', type, lossy ? 'lossy' : 'lossless');
+
+        var engine;
+        if (type === 'jpeg' && !lossy) {
+            engine = imageminJpegtran({progressive: true});
+        } else if (type === 'jpeg' && lossy) {
+            engine = imageminJpegoptim({progressive: true, max: MAX_JPEG_QUALITY});
+        } else if (type === 'png' && !lossy) {
+            engine = imageminOptipng({optimizationLevel: OPTIPNG_COMPRESSION_LEVEL});
+        } else if (type === 'svg' && !lossy) {
+            engine = imageminSvgo({ plugins: [ { removeUselessDefs: false } ] });
+        } else {
+            deferred.reject('No optimization engine found for imagemin');
+        }
+
+
+        imagemin.buffer(imageBody, {use: engine})
+
+        .then(function(file) {
+            var endTime = Date.now();
+            debug('Optimization for %s took %d ms', type, endTime - startTime);
+            deferred.resolve(file);
+        })
+
+        .catch(function(err) {
+            deferred.reject(file);
+        });
+
+        return deferred.promise;
+    }
+
+    function entryTypeCanBeOptimized(entry) {
+        return isJPEG(entry) || isPNG(entry) || isSVG(entry);
+    }
+
+    return {
+        optimizeImage: optimizeImage,
+        compressJpegLosslessly: compressJpegLosslessly,
+        compressJpegLossly: compressJpegLossly,
+        compressPngLosslessly: compressPngLosslessly,
+        compressSvgLosslessly: compressSvgLosslessly,
+        gainIsEnough: gainIsEnough,
+        entryTypeCanBeOptimized: entryTypeCanBeOptimized
+    };
+};
+
+module.exports = new ImageOptimizer();

lib/tools/redownload/redownload.js

@@ -0,0 +1,631 @@
+/*
+ * Redownloading every files after Phantomas has finished
+ * Checks weight and every kind of compression
+ *
+ */
+
+/*jshint -W069 */
+
+var debug               = require('debug')('ylt:redownload');
+var Q                   = require('q');
+var http                = require('http');
+var zlib                = require('zlib');
+var async               = require('async');
+var request             = require('request');
+var md5                 = require('md5');
+
+var imageOptimizer      = require('./imageOptimizer');
+var fileMinifier        = require('./fileMinifier');
+var gzipCompressor      = require('./gzipCompressor');
+var contentTypeChecker  = require('./contentTypeChecker');
+
+
+var Redownload = function() {
+
+    var MAX_PARALLEL_DOWNLOADS = 10;
+    var REQUEST_TIMEOUT = 15000; // 15 seconds
+
+
+    // This function will re-download every asset and check if it could be optimized
+    function recheckAllFiles(data) {
+        var startTime = Date.now();
+        debug('Redownload started');
+        var deferred = Q.defer();
+
+        var requestsList = JSON.parse(data.toolsResults.phantomas.offenders.requestsList);
+        delete data.toolsResults.phantomas.metrics.requestsList;
+        delete data.toolsResults.phantomas.offenders.requestsList;
+
+        var httpAuth = null;
+        if (data.params && data.params.options && data.params.options.authUser && data.params.options.authPass) {
+            httpAuth = {
+                username: data.params.options.authUser,
+                password: data.params.options.authPass
+            };
+        }
+
+        // Transform every request into a download function with a callback when done
+        var redownloadList = requestsList.map(function(entry) {
+            return function(callback) {
+                
+                redownloadEntry(entry, httpAuth)
+
+                .then(contentTypeChecker.checkContentType)
+
+                .then(imageOptimizer.optimizeImage)
+
+                .then(fileMinifier.minifyFile)
+
+                .then(gzipCompressor.compressFile)
+
+                .then(function(newEntry) {
+                    debug('File %s - Redownloaded, optimized, minified, compressed: done', entry.url);
+                    callback(null, newEntry);
+                })
+
+                .fail(function(err) {
+                    callback(err);
+                });
+            };
+        });
+
+        // Lanch all redownload functions and wait for completion
+        async.parallelLimit(redownloadList, MAX_PARALLEL_DOWNLOADS, function(err, results) {
+            
+            if (err) {
+                debug(err);
+                deferred.reject(err);
+            } else {
+
+                debug('All files checked');
+                endTime = Date.now();
+                debug('Redownload took %d ms', endTime - startTime);
+                
+                var metrics = {};
+                var offenders = {};
+
+                // Count requests
+                offenders.totalRequests = listRequestsByType(results);
+                metrics.totalRequests = offenders.totalRequests.total;
+
+
+                // Remove unwanted responses (redownload failed, about:blank)
+                results = results.filter(function(result) {
+                    return (result !== null && result.weightCheck && result.url.indexOf('about:blank') !== 0);
+                });
+
+
+                // Wrong contentType
+                offenders.incorrectContentTypes = listIncorrectContentTypes(results);
+                metrics.incorrectContentTypes = offenders.incorrectContentTypes.length;
+
+                // Total weight
+                offenders.totalWeight = listRequestWeight(results);
+                metrics.totalWeight = offenders.totalWeight.totalWeight;
+
+                // Empty files
+                offenders.emptyRequests = listEmptyRequests(results);
+                metrics.emptyRequests = offenders.emptyRequests.length;
+
+
+                // Now remove unwanted responses (redirections and empty files)
+                results = results.filter(function(result) {
+                    return ((result.status < 300 || result.status >= 400) && result.weightCheck.bodySize > 0);
+                });
+
+
+                // Image compression
+                offenders.imageOptimization = listImageNotOptimized(results);
+                metrics.imageOptimization = offenders.imageOptimization.totalGain;
+
+                // File minification
+                offenders.fileMinification = listFilesNotMinified(results);
+                metrics.fileMinification = offenders.fileMinification.totalGain;
+
+                // Gzip compression
+                offenders.gzipCompression = listFilesNotGzipped(results);
+                metrics.gzipCompression = offenders.gzipCompression.totalGain;
+
+                // Small requests
+                offenders.smallRequests = listSmallRequests(results);
+                metrics.smallRequests = offenders.smallRequests.total;
+
+                // Detect identical files
+                offenders.identicalFiles = listIdenticalFiles(results);
+                metrics.identicalFiles = offenders.identicalFiles.avoidableRequests;
+
+
+                data.toolsResults.redownload = {
+                    metrics: metrics,
+                    offenders: offenders
+                };
+
+                deferred.resolve(data);
+            }
+        });
+
+        return deferred.promise;
+    }
+
+    function listIncorrectContentTypes(requests) {
+        var results = [];
+        
+        requests.forEach(function(req) {
+            if (req.oldContentType || req.oldContentType === null) {
+                results.push({
+                    url: req.url,
+                    current: req.oldContentType,
+                    correct: req.contentType
+                });
+            }
+        });
+
+        return results;
+    }
+
+    function listEmptyRequests(requests) {
+        var results = [];
+        
+        requests.forEach(function(req) {
+            var weight = req.weightCheck.bodySize || req.contentLength;
+
+            if (weight === 0 && req.method === 'GET' && req.status === 200) {
+                results.push(req.url);
+            }
+        });
+
+        return results;
+    }
+
+    function listRequestWeight(requests) {
+        var results = {
+            totalWeight: 0,
+            byType: {
+                html: {
+                    totalWeight: 0,
+                    requests: []
+                },
+                css: {
+                    totalWeight: 0,
+                    requests: []
+                },
+                js: {
+                    totalWeight: 0,
+                    requests: []
+                },
+                json: {
+                    totalWeight: 0,
+                    requests: []
+                },
+                image: {
+                    totalWeight: 0,
+                    requests: []
+                },
+                video: {
+                    totalWeight: 0,
+                    requests: []
+                },
+                webfont: {
+                    totalWeight: 0,
+                    requests: []
+                },
+                other: {
+                    totalWeight: 0,
+                    requests: []
+                }
+            }
+        };
+
+        requests.forEach(function(req) {
+            var weight = ((typeof req.weightCheck.bodySize === 'number') ? req.weightCheck.bodySize + req.weightCheck.headersSize : req.contentLength) || 0;
+            var type = req.type || 'other';
+            type = (results.byType[type]) ? type : 'other';
+
+            results.totalWeight += weight;
+            results.byType[type].totalWeight += weight;
+
+            results.byType[type].requests.push({
+                url: req.url,
+                weight: weight
+            });
+        });
+
+        return results;
+    }
+
+
+    function listImageNotOptimized(requests) {
+        var results = {
+            totalGain: 0,
+            images: []
+        };
+
+        requests.forEach(function(req) {
+            if (req.weightCheck.bodySize > 0 && imageOptimizer.entryTypeCanBeOptimized(req) && req.weightCheck.isOptimized === false) {
+                var before = req.weightCheck.afterCompression || req.weightCheck.bodySize;
+                var after = req.weightCheck.afterOptimizationAndCompression || req.weightCheck.optimized;
+                var gain = before - after;
+
+                if (gain > 200) {
+                    results.totalGain += gain;
+
+                    results.images.push({
+                        url: req.url,
+                        original: req.weightCheck.bodySize,
+                        isCompressed: req.weightCheck.isCompressed,
+                        afterCompression: req.weightCheck.afterCompression,
+                        afterOptimizationAndCompression: req.weightCheck.afterOptimizationAndCompression,
+                        lossless: req.weightCheck.lossless,
+                        lossy: req.weightCheck.lossy,
+                        gain: gain
+                    });
+                }
+            }
+        });
+
+        return results;
+    }
+
+
+    function listFilesNotMinified(requests) {
+        var results = {
+            totalGain: 0,
+            files: []
+        };
+
+        requests.forEach(function(req) {
+            if (req.weightCheck.bodySize > 0 && fileMinifier.entryTypeCanBeMinified(req) && req.weightCheck.isOptimized === false) {
+                var before = req.weightCheck.afterCompression || req.weightCheck.bodySize;
+                var after = req.weightCheck.afterOptimizationAndCompression || req.weightCheck.optimized;
+                var gain = before - after;
+
+                if (gain > 200) {
+                    results.totalGain += gain;
+
+                    results.files.push({
+                        url: req.url,
+                        original: req.weightCheck.bodySize,
+                        isCompressed: req.weightCheck.isCompressed,
+                        afterCompression: req.weightCheck.afterCompression,
+                        afterOptimizationAndCompression: req.weightCheck.afterOptimizationAndCompression,
+                        optimized: req.weightCheck.optimized,
+                        gain: gain
+                    });
+                }
+            }
+        });
+
+        return results;
+    }
+
+    function listFilesNotGzipped(requests) {
+        var results = {
+            totalGain: 0,
+            files: []
+        };
+
+        requests.forEach(function(req) {
+            if (req.weightCheck.uncompressedSize && req.weightCheck.isCompressed === false && req.weightCheck.afterCompression) {
+                var gain = req.weightCheck.uncompressedSize - req.weightCheck.afterCompression;
+
+                results.totalGain += gain;
+
+                results.files.push({
+                    url: req.url,
+                    original: req.weightCheck.uncompressedSize,
+                    gzipped: req.weightCheck.afterCompression,
+                    gain: gain
+                });
+            }
+        });
+
+        return results;
+    }
+
+    function listRequestsByType(requests) {
+        var results = {
+            total: 0,
+            byType: {
+                html: [],
+                css: [],
+                js: [],
+                json: [],
+                image: [],
+                video: [],
+                webfont: [],
+                other: []
+            }
+        };
+
+        requests.forEach(function(req) {
+            if (req.url !== 'about:blank') {
+                var type = req.type || 'other';
+                type = (results.byType[type]) ? type : 'other';
+                
+                results.byType[type].push(req.url);
+                results.total ++;
+            }
+        });
+
+        return results;
+    }
+
+    function listSmallRequests(requests) {
+        var results = {
+            total: 0,
+            byType: {
+                css: [],
+                js: [],
+                image: []
+            }
+        };
+
+        requests.forEach(function(req) {
+            if (req.weightCheck.bodySize > 0 && req.weightCheck.bodySize < 2048) {
+                if (req.isCSS || req.isJS || req.isImage) {
+                    results.byType[req.type].push({
+                        url: req.url,
+                        size: req.weightCheck.bodySize
+                    });
+                    results.total ++;
+                }
+            }
+        });
+
+        return results;
+    }
+
+    function listIdenticalFiles(requests) {
+        var hashes = {};
+        var list = [];
+        var avoidableRequestsCount = 0;
+
+        requests.forEach(function(req) {
+            var requestHash = md5(req.weightCheck.body);
+            
+            // Try to exclude tracking pixels
+            if (req.weightCheck.bodySize < 80 && req.type === 'image') {
+                return;
+            }
+
+            if (!hashes[requestHash]) {
+                hashes[requestHash] = {
+                    weight: req.weightCheck.bodySize,
+                    urls: []
+                };
+            }
+            
+            if (hashes[requestHash].urls.indexOf(req.url) === -1) {
+                hashes[requestHash].urls.push(req.url);
+            }
+        });
+
+        for (var hash in hashes) {
+            if (hashes[hash].urls.length > 1) {
+                list.push(hashes[hash]);
+                avoidableRequestsCount += hashes[hash].urls.length - 1;
+            }
+        }
+
+        return {
+            avoidableRequests: avoidableRequestsCount,
+            count: list.length,
+            list: list
+        };
+    }
+
+
+    function redownloadEntry(entry, httpAuth) {
+        var deferred = Q.defer();
+        
+        function downloadError(message) {
+            debug('Could not download %s Error: %s', entry.url, message);
+            entry.weightCheck = {
+                message: message
+            };
+            deferred.resolve(entry);
+        }
+
+        // Not downloaded again but will be counted in totalWeight
+        function notDownloadableFile(message) {
+            entry.weightCheck = {
+                message: message
+            };
+            deferred.resolve(entry);
+        }
+
+        // Not counted in totalWeight
+        function unwantedFile(message) {
+            debug(message);
+            deferred.resolve(entry);
+        }
+
+        if (entry.method !== 'GET') {
+            notDownloadableFile('only downloading GET');
+            return deferred.promise;
+        }
+
+        if (entry.status !== 200) {
+            unwantedFile('only downloading requests with status code 200');
+            return deferred.promise;
+        }
+
+        if (entry.url === 'about:blank') {
+            unwantedFile('not downloading about:blank');
+            return deferred.promise;
+        }
+
+
+        debug('Downloading %s', entry.url);
+
+        // Always add a gzip header before sending, in case the server listens to it
+        var reqHeaders = entry.requestHeaders;
+        reqHeaders['Accept-Encoding'] = 'gzip, deflate';
+        reqHeaders['Connection'] = 'keep-alive';
+
+        var requestOptions = {
+            method: entry.method,
+            url: entry.url,
+            headers: reqHeaders,
+            timeout: REQUEST_TIMEOUT
+        };
+
+        // Basic auth
+        if (httpAuth) {
+            requestOptions.auth = {
+                user: httpAuth.username,
+                pass: httpAuth.password,
+                sendImmediately: false // Tries a first time without auth, wait for a 401 error before resending
+            };
+        }
+
+        download(requestOptions, entry.contentType, function(error, result) {
+            if (error) {
+                if (error.code === 'ETIMEDOUT') {
+                    downloadError('timeout after ' + REQUEST_TIMEOUT + 'ms');
+                } else {
+                    downloadError('error while downloading: ' + error.code);
+                }
+                return;
+            }
+                
+            debug('%s downloaded correctly', entry.url);
+
+            entry.weightCheck = result;
+            deferred.resolve(entry);
+        });
+
+        return deferred.promise;
+    }
+
+    // Inspired by https://github.com/cvan/fastHAR-api/blob/10cec585/app.js
+    function download(requestOptions, contentType, callback) {
+
+        var statusCode;
+
+        try {
+
+            request(requestOptions)
+
+            .on('response', function(res) {
+                
+                // Raw headers were added in NodeJS v0.12
+                // (https://github.com/joyent/node/issues/4844), but let's
+                // reconstruct them for backwards compatibility.
+                var rawHeaders = ('HTTP/' + res.httpVersion + ' ' + res.statusCode +
+                                  ' ' + http.STATUS_CODES[res.statusCode] + '\r\n');
+                Object.keys(res.headers).forEach(function(headerKey) {
+                    rawHeaders += headerKey + ': ' + res.headers[headerKey] + '\r\n';
+                });
+                rawHeaders += '\r\n';
+
+                var uncompressedSize = 0;  // size after uncompression
+                var bodySize = 0;  // bytes size over the wire
+                var body = '';  // plain text body (after uncompressing gzip/deflate)
+                var isCompressed = false;
+
+                function tally() {
+
+                    if (statusCode !== 200) {
+                        callback({code: statusCode});
+                        return;
+                    }
+
+                    var result = {
+                        body: body,
+                        headersSize: Buffer.byteLength(rawHeaders, 'utf8'),
+                        bodySize: bodySize,
+                        isCompressed: isCompressed,
+                        uncompressedSize: uncompressedSize
+                    };
+
+                    callback(null, result);
+                }
+
+                switch (res.headers['content-encoding']) {
+                    case 'gzip':
+
+                        var gzip = zlib.createGunzip();
+
+                        gzip.on('data', function (data) {
+                            body += data;
+                            uncompressedSize += data.length;
+                        }).on('end', function () {
+                            isCompressed = true;
+                            tally();
+                        }).on('error', function(err) {
+                            debug('Error while decoding %s', requestOptions.url);
+                            debug(err);
+                            callback(err);
+                        });
+
+                        res.on('data', function (data) {
+                            bodySize += data.length;
+                        }).pipe(gzip);
+
+                        break;
+                    case 'deflate':
+                        res.setEncoding('utf8');
+
+                        var deflate = zlib.createInflate();
+
+                        deflate.on('data', function (data) {
+                            body += data;
+                            uncompressedSize += data.length;
+                        }).on('end', function () {
+                            isCompressed = true;
+                            tally();
+                        }).on('error', function(err) {
+                            debug('Error while decoding %s', requestOptions.url);
+                            debug(err);
+                            callback(err);
+                        });
+
+                        res.on('data', function (data) {
+                            bodySize += data.length;
+                        }).pipe(deflate);
+
+                        break;
+                    default:
+                        if (contentType === 'image/jpeg' || contentType === 'image/png') {
+                            res.setEncoding('binary');
+                        }
+
+                        res.on('data', function (data) {
+                            body += data;
+                            uncompressedSize += data.length;
+                            bodySize += data.length;
+                        }).on('end', function () {
+                            tally();
+                        });
+
+                        break;
+                }
+            })
+
+            .on('response', function(response) {
+                statusCode = response.statusCode;
+            })
+
+            .on('error', function(err) {
+                debug('Error while downloading %s', requestOptions.url);
+                debug(err);
+                callback(err);
+            });
+
+        } catch(err) {
+            debug('Error while downloading %s', requestOptions.url);
+            debug(err);
+            callback(err);
+        }
+    }
+
+    return {
+        recheckAllFiles: recheckAllFiles,
+        listRequestWeight: listRequestWeight,
+        redownloadEntry: redownloadEntry,
+        download: download
+    };
+};
+
+module.exports = new Redownload();

package.json

@@ -1,7 +1,13 @@
 {
   "name": "yellowlabtools",
-  "version": "1.6.1",
+  "version": "1.11.0",
   "description": "Online tool to audit a webpage for performance and front-end quality issues",
+  "license": "GPL-2.0",
+  "author": {
+    "name": "Gaël Métais",
+    "email": "gael@gaelmetais.com",
+    "url": "http://www.gaelmetais.com"
+  },
   "repository": {
     "type": "git",
     "url": "git://github.com/gmetais/YellowLabTools.git"
@@ -9,50 +15,93 @@
   "bin": {
     "yellowlabtools": "./bin/cli.js"
   },
+  "engines": {
+    "node": ">= 4.0"
+  },
   "main": "./lib/index.js",
   "dependencies": {
-    "async": "~1.0.0",
-    "body-parser": "~1.12.4",
-    "compression": "~1.4.4",
-    "cors": "^2.6.0",
-    "debug": "~2.2.0",
-    "express": "~4.12.4",
-    "lwip": "0.0.6",
-    "meow": "^3.1.0",
-    "phantomas": "1.10.2",
-    "ps-node": "0.0.4",
-    "q": "~1.4.1",
-    "rimraf": "~2.3.4",
-    "temporary": "0.0.8"
+    "angular": "1.5.7",
+    "angular-animate": "1.5.7",
+    "angular-chart.js": "0.10.2",
+    "angular-local-storage": "0.2.7",
+    "angular-resource": "1.5.7",
+    "angular-route": "1.5.7",
+    "angular-sanitize": "1.5.7",
+    "async": "1.5.2",
+    "body-parser": "1.15.2",
+    "chart.js": "1.1.1",
+    "clean-css": "3.4.18",
+    "color-diff": "1.0.0",
+    "compression": "1.6.2",
+    "cors": "2.7.1",
+    "css-mq-parser": "0.0.3",
+    "debug": "2.2.0",
+    "easyxml": "2.0.1",
+    "express": "4.14.0",
+    "imagemin": "5.2.2",
+    "imagemin-jpegoptim": "5.0.0",
+    "imagemin-jpegtran": "5.0.2",
+    "imagemin-optipng": "5.1.0",
+    "imagemin-svgo": "5.1.0",
+    "is-eot": "1.0.0",
+    "is-gif": "1.0.0",
+    "is-http2": "1.0.4",
+    "is-jpg": "1.0.0",
+    "is-otf": "0.1.2",
+    "is-png": "1.0.0",
+    "is-svg": "2.0.1",
+    "is-ttf": "0.2.2",
+    "is-woff": "1.0.3",
+    "is-woff2": "1.0.0",
+    "lwip": "0.0.9",
+    "md5": "2.1.0",
+    "meow": "3.7.0",
+    "minimize": "2.0.0",
+    "parse-color": "1.0.0",
+    "phantomas": "1.16.0",
+    "ps-node": "0.1.2",
+    "q": "1.4.1",
+    "request": "2.72.0",
+    "rimraf": "2.5.3",
+    "temporary": "0.0.8",
+    "try-thread-sleep": "1.0.0",
+    "uglify-js": "2.7.0"
   },
   "devDependencies": {
-    "chai": "^2.3.0",
-    "grunt": "^0.4.5",
-    "grunt-blanket": "^0.0.8",
-    "grunt-contrib-clean": "^0.6.0",
-    "grunt-contrib-concat": "^0.5.1",
-    "grunt-contrib-copy": "^0.8.0",
-    "grunt-contrib-cssmin": "^0.12.3",
-    "grunt-contrib-htmlmin": "^0.4.0",
-    "grunt-contrib-jshint": "^0.11.2",
-    "grunt-contrib-less": "^1.0.1",
-    "grunt-contrib-uglify": "^0.9.1",
-    "grunt-env": "^0.4.4",
-    "grunt-express": "^1.4.1",
-    "grunt-filerev": "^2.3.1",
-    "grunt-inline-angular-templates": "^0.1.5",
-    "grunt-line-remover": "^0.0.2",
-    "grunt-mocha-test": "^0.12.7",
-    "grunt-replace": "^0.9.2",
-    "grunt-usemin": "^3.0.0",
-    "grunt-webfont": "^0.5.3",
-    "matchdep": "^0.3.0",
-    "mocha": "^2.2.5",
-    "request": "^2.55.0",
-    "sinon": "^1.14.1",
-    "sinon-chai": "^2.7.0"
+    "chai": "~3.5.0",
+    "grunt": "~0.4.5",
+    "grunt-blanket": "~0.0.10",
+    "grunt-contrib-clean": "~1.0.0",
+    "grunt-contrib-concat": "~1.0.1",
+    "grunt-contrib-copy": "~1.0.0",
+    "grunt-contrib-cssmin": "~1.0.1",
+    "grunt-contrib-htmlmin": "~1.4.0",
+    "grunt-contrib-jshint": "~1.0.0",
+    "grunt-contrib-less": "~1.3.0",
+    "grunt-contrib-uglify": "~1.0.1",
+    "grunt-env": "~0.4.4",
+    "grunt-express": "~1.4.1",
+    "grunt-filerev": "~2.3.1",
+    "grunt-inline-angular-templates": "~0.1.5",
+    "grunt-line-remover": "~0.0.2",
+    "grunt-mocha-test": "~0.12.7",
+    "grunt-replace": "~1.0.1",
+    "grunt-usemin": "~3.1.1",
+    "grunt-webfont": "~1.4.0",
+    "matchdep": "~1.0.1",
+    "mocha": "~2.5.3",
+    "sinon": "~1.17.4",
+    "sinon-chai": "~2.8.0"
   },
   "scripts": {
     "test": "grunt test"
-  }
+  },
+  "keywords": [
+    "performance",
+    "metrics",
+    "webperf",
+    "pagespeed",
+    "budget",
+    "phantomas"
+  ]
 }

server_config/maintenance.js

@@ -0,0 +1,13 @@
+var express                 = require('express');
+var app                     = express();
+var server                  = require('http').createServer(app);
+
+var settings                = require('./settings.json');
+
+app.all('*', function(req, res) {
+    res.status(500).send('YellowLabTools is in maintenance. It should come back soon with a new version!');
+});
+
+server.listen(settings.serverPort, function() {
+    console.log('Maintenance mode started on port %d', server.address().port);
+});

server_config/server_install.sh

@@ -2,17 +2,16 @@
 
 # APT-GET
 sudo apt-get update
-sudo apt-get install lsb-release libfontconfig1 libfreetype6 -y --force-yes
-sudo apt-get install curl git python-software-properties build-essential make g++ -y --force-yes
+sudo apt-get install lsb-release libfontconfig1 libfreetype6 libjpeg-dev -y --force-yes > /dev/null 2>&1
+sudo apt-get install curl git python-software-properties build-essential make g++ -y --force-yes > /dev/null 2>&1
 
 # Installation of NodeJS
-sudo add-apt-repository -y ppa:chris-lea/node.js
-sudo apt-get update
-sudo apt-get install -y nodejs
+curl -sL https://deb.nodesource.com/setup_0.12 | sudo -E bash -
+sudo apt-get install -y nodejs > /dev/null 2>&1
 source ~/.profile
 
 # Installation of some packages globally
-npm install bower forever grunt-cli -g
+npm install forever grunt-cli -g
 source ~/.profile
 
 # Installation of YellowLabTools
@@ -21,8 +20,7 @@ sudo chown $USER /space
 cd /space
 git clone https://github.com/gmetais/YellowLabTools.git --branch master
 cd YellowLabTools
-npm install
-bower install --config.interactive=false --allow-root
+npm install || exit 1
 
 # Front-end compilation
 grunt build

server_config/server_update.sh

@@ -2,23 +2,23 @@
 
 cd /space/YellowLabTools
 
-# Stop the server
+# Stop the server and start the maintenance mode
 forever stopall
+forever start server_config/maintenance.js
 
 # Keep the settings.json file
 git stash
 git pull
 git stash pop
 
-# In case something was added in package.json or bower.json
+# In case something was added in package.json
 rm -rf node_modules
-npm install
-rm -rf bower_components
-bower install --config.interactive=false --allow-root
+npm install || exit 1
 
 # Front-end compilation
 rm -rf front/build
 grunt build
 
-# Restart the server
+# Stop the maintenance mode and restart the server
+forever stopall
 NODE_ENV=production forever start -c "node --stack-size=262000" bin/server.js

server_config/settings-prod.json

@@ -1,6 +1,8 @@
 {
     "serverPort": 80,
+    "phantomasEngine": "webkit",
     "googleAnalyticsId": "",
+    "screenshotWidth": 400,
 
     "authorizedKeys": {
         

server_config/settings.json

@@ -1,6 +1,8 @@
 {
     "serverPort": 8383,
+    "phantomasEngine": "webkit",
     "googleAnalyticsId": "",
+    "screenshotWidth": 400,
 
     "authorizedKeys": {
         

test/api/apiTest.js

@@ -31,6 +31,7 @@ describe('api', function() {
             },
             json: true,
             headers: {
+                'Content-Type': 'application/json',
                 'X-Api-Key': 'invalid'
             }
         }, function(error, response, body) {
@@ -53,6 +54,7 @@ describe('api', function() {
             },
             json: true,
             headers: {
+                'Content-Type': 'application/json',
                 'X-Api-Key': Object.keys(config.authorizedKeys)[0]
             }
         }, function(error, response, body) {
@@ -76,6 +78,7 @@ describe('api', function() {
             },
             json: true,
             headers: {
+                'Content-Type': 'application/json',
                 'X-Api-Key': Object.keys(config.authorizedKeys)[0]
             }
         }, function(error, response, body) {
@@ -99,12 +102,13 @@ describe('api', function() {
                 screenshot: true,
                 device: 'tablet',
                 //waitForSelector: '*',
-                cookie: 'foo=bar',
+                cookie: 'foo=bar;domain=google.com',
                 authUser: 'joe',
                 authPass: 'secret'
             },
             json: true,
             headers: {
+                'Content-Type': 'application/json',
                 'X-Api-Key': Object.keys(config.authorizedKeys)[0]
             }
         }, function(error, response, body) {
@@ -133,6 +137,7 @@ describe('api', function() {
             },
             json: true,
             headers: {
+                'Content-Type': 'application/json',
                 'X-Api-Key': Object.keys(config.authorizedKeys)[0]
             }
         }, function(error, response, body) {
@@ -165,14 +170,13 @@ describe('api', function() {
                 body.should.have.a.property('rules').that.is.an('object');
                 body.should.have.a.property('toolsResults').that.is.an('object');
 
-                // javascriptExecutionTree should only be filled if option jsTimeline is true
                 body.should.have.a.property('javascriptExecutionTree').that.is.an('object');
-                body.javascriptExecutionTree.should.deep.equal({});
+                body.javascriptExecutionTree.should.not.deep.equal({});
 
                 // Check if settings are correctly sent and retrieved
                 body.params.options.should.have.a.property('device').that.equals('tablet');
                 //body.params.options.should.have.a.property('waitForSelector').that.equals('*');
-                body.params.options.should.have.a.property('cookie').that.equals('foo=bar');
+                body.params.options.should.have.a.property('cookie').that.equals('foo=bar;domain=google.com');
                 body.params.options.should.have.a.property('authUser').that.equals('joe');
                 body.params.options.should.have.a.property('authPass').that.equals('secret');
 
@@ -208,6 +212,7 @@ describe('api', function() {
             },
             json: true,
             headers: {
+                'Content-Type': 'application/json',
                 'X-Api-Key': Object.keys(config.authorizedKeys)[0]
             }
         }, function(error, response, body) {
@@ -232,6 +237,7 @@ describe('api', function() {
             url: serverUrl + '/api/runs/' + asyncRunId,
             json: true,
             headers: {
+                'Content-Type': 'application/json',
                 'X-Api-Key': Object.keys(config.authorizedKeys)[0]
             }
         }, function(error, response, body) {

test/core/colorDiffTest.js

@@ -0,0 +1,92 @@
+var should = require('chai').should();
+var colorDiff = require('../../lib/tools/colorDiff');
+
+describe('colorDiff', function() {
+    
+    it('should parse offenders correctly', function() {
+        colorDiff.parseOffender('#000 (1 times)').should.equal('#000');
+        colorDiff.parseOffender('#5bc0de (2 times)').should.equal('#5bc0de');
+        colorDiff.parseOffender('rgba(0,0,0,0.075) (100 times)').should.equal('rgba(0,0,0,0.075)');
+        colorDiff.parseOffender('rgb(91,192,222) (1000 times)').should.equal('rgb(91,192,222)');
+    });
+
+    it('should parse colors correctly', function() {
+        colorDiff.parseColor('#000').should.deep.equal({R: 0, G: 0, B: 0, A: 1, Lab: {L: 0, a: 0, b: 0}, original: '#000'});
+        colorDiff.parseColor('rgba(255,255,255,0.5)').should.deep.equal({R: 255, G: 255, B: 255, A: 0.5, Lab: {L: 100, a: 0.00526049995830391, b: -0.010408184525267927}, original: 'rgba(255,255,255,0.5)'});
+    });
+
+    it('should not compare colors with different alpha', function() {
+        var color1 = colorDiff.parseColor("rgba(0, 0, 0, 1)");
+        var color2 = colorDiff.parseColor("rgba(0, 0, 0, 0.5)");
+
+        colorDiff.compareTwoColors(color1, color2).should.equal(false);
+    });
+
+    it('should find that two colors are similar', function() {
+        var color1 = colorDiff.parseColor("rgba(0, 0, 0, 1)");
+        var color2 = colorDiff.parseColor("rgba(0, 0, 1, 1)");
+
+        colorDiff.compareTwoColors(color1, color2).should.equal(true);
+    });
+
+    it('should find that two colors are similar even with alpha', function() {
+        var color1 = colorDiff.parseColor("rgba(0, 0, 0, 0.3)");
+        var color2 = colorDiff.parseColor("rgba(0, 0, 2, 0.3)");
+
+        colorDiff.compareTwoColors(color1, color2).should.equal(true);
+    });
+
+    it('should find that two colors are different', function() {
+        var color1 = colorDiff.parseColor("rgba(0, 0, 0, 1)");
+        var color2 = colorDiff.parseColor("rgba(99, 99, 99, 1)");
+
+        colorDiff.compareTwoColors(color1, color2).should.equal(false);
+    });
+
+    it('should compare all colors to each other', function() {
+        var colors = [
+            '#000 (1 times)',
+            '#5bc0de (2 times)',
+            'rgba(0,0,0,0.075) (100 times)',
+            'rgb(91,192,222) (1000 times)',
+            'rgba(0,0,2,1) (1 times)',
+            'rgba(99,99,99,1) (1 times)',
+            'rgba(100,100,100,1) (1 times)'
+        ];
+
+        var data = {
+            toolsResults: {
+                phantomas: {
+                    offenders: {
+                        cssColors: colors
+                    }
+                }
+            }
+        };
+
+        newData = colorDiff.compareAllColors(data);
+
+        newData.toolsResults.should.have.a.property('colorDiff');
+        newData.toolsResults.colorDiff.should.have.a.property('metrics');
+        newData.toolsResults.colorDiff.metrics.should.have.a.property('similarColors').that.equals(3);
+        newData.toolsResults.colorDiff.should.have.a.property('offenders');
+        newData.toolsResults.colorDiff.offenders.should.have.a.property('similarColors').that.deep.equals([
+            {
+                color1: '#000',
+                color2: 'rgba(0,0,2,1)',
+                isDark: true
+            },
+            {
+                color1: '#5bc0de',
+                color2: 'rgb(91,192,222)',
+                isDark: false
+            },
+            {
+                color1: 'rgba(99,99,99,1)',
+                color2: 'rgba(100,100,100,1)',
+                isDark: true
+            }
+        ]);
+    });
+
+});

test/core/contentTypeCheckerTest.js

@@ -0,0 +1,20 @@
+var should = require('chai').should();
+var contentTypeChecker = require('../../lib/tools/redownload/contentTypeChecker');
+var fs = require('fs');
+var path = require('path');
+
+describe('contentTypeChecker', function() {
+
+    var jpgImageContent = fs.readFileSync(path.resolve(__dirname, '../www/jpeg-image.jpg'));
+    var pngImageContent = fs.readFileSync(path.resolve(__dirname, '../www/png-image.png'));
+    var svgImageContent = fs.readFileSync(path.resolve(__dirname, '../www/svg-image.svg'));
+    var cssFileContent = fs.readFileSync(path.resolve(__dirname, '../www/unminified-stylesheet.css'));
+    
+    it('detect the right content type', function() {
+        contentTypeChecker.findContentType(jpgImageContent).mimes.should.deep.equal(['image/jpeg']);
+        contentTypeChecker.findContentType(pngImageContent).mimes.should.deep.equal(['image/png']);
+        contentTypeChecker.findContentType(svgImageContent).mimes.should.deep.equal(['image/svg+xml']);
+        should.equal(contentTypeChecker.findContentType(cssFileContent), null);
+    });
+
+});

test/core/customPoliciesTest.js

@@ -183,18 +183,21 @@ describe('customPolicies', function() {
 
         var versions = {
             '1.2.9': 0,
-            '1.3.9': 0,
-            '1.4.4': 10,
-            '1.5.0': 20,
-            '1.6.3': 30,
-            '1.7.0': 40,
-            '1.8.3a': 50,
-            '1.9.2': 70,
-            '1.10.1': 90,
-            '2.0.0-rc1': 90,
-            '1.11.1': 100,
-            '2.1.1-beta1': 100,
-            '3.0.0': 100
+            '1.4.4': 0,
+            '1.5.0': 0,
+            '1.6.3': 10,
+            '1.7.0': 20,
+            '1.8.3a': 30,
+            '1.9.2': 40,
+            '1.10.1': 50,
+            '2.0.0-rc1': 50,
+            '1.11.1': 70,
+            '2.1.1-beta1': 70,
+            '1.12.1': 90,
+            '2.2.1': 90,
+            '3.0.1': 100,
+            '3.1.0': 100,
+            '3.2.1': 100
         };
 
         for (var version in versions) {

test/core/fileMinifierTest.js

@@ -0,0 +1,244 @@
+var should = require('chai').should();
+var fileMinifier = require('../../lib/tools/redownload/fileMinifier');
+var fs = require('fs');
+var path = require('path');
+
+describe('fileMinifier', function() {
+    
+    it('should minify a JS file with minifyJs', function(done) {
+        var fileContent = fs.readFileSync(path.resolve(__dirname, '../www/unminified-script.js'));
+
+        var fileSize = fileContent.length;
+
+        fileMinifier.minifyJs(fileContent.toString()).then(function(newFile) {
+            var newFileSize = newFile.length;
+            newFileSize.should.be.below(fileSize);
+            done();
+        }).fail(function(err) {
+            done(err);
+        });
+    });
+
+    it('should minify a JS file with minifyFile', function(done) {
+        var fileContent = fs.readFileSync(path.resolve(__dirname, '../www/unminified-script.js'));
+        var fileSize = fileContent.length;
+
+        var entry = {
+            method: 'GET',
+            url: 'http://localhost:8388/unminified-script.js',
+            requestHeaders: {
+                'User-Agent': 'something',
+                Referer: 'http://www.google.fr/',
+                Accept: '*/*',
+                'Accept-Encoding': 'gzip, deflate'
+            },
+            status: 200,
+            isJS: true,
+            type: 'js',
+            contentLength: 999,
+            weightCheck: {
+                body: fileContent.toString('utf8'),
+                totalWeight: fileSize + 200,
+                headersSize: 200,
+                bodySize: fileSize,
+                isCompressed: false,
+                uncompressedSize: fileSize
+            }
+        };
+
+        fileMinifier.minifyFile(entry)
+
+        .then(function(newEntry) {
+            newEntry.weightCheck.should.have.a.property('isOptimized').that.equals(false);
+            newEntry.weightCheck.should.have.a.property('optimized').that.is.below(fileSize);
+            newEntry.weightCheck.should.have.a.property('bodyAfterOptimization').that.is.a('string');
+
+            done();
+        })
+
+        .fail(function(err) {
+            done(err);
+        });
+    });
+
+    it('should fail minifying an already minified JS', function(done) {
+        this.timeout(5000);
+
+        var fileContent = fs.readFileSync(path.resolve(__dirname, '../www/jquery1.8.3.js'));
+        var fileSize = fileContent.length;
+
+        var entry = {
+            method: 'GET',
+            url: 'http://localhost:8388/jquery1.8.3.js',
+            requestHeaders: {
+                'User-Agent': 'something',
+                Referer: 'http://www.google.fr/',
+                Accept: '*/*',
+                'Accept-Encoding': 'gzip, deflate'
+            },
+            status: 200,
+            isJS: true,
+            type: 'js',
+            contentLength: 999,
+            weightCheck: {
+                body: fileContent.toString('utf8'),
+                totalWeight: fileSize + 200,
+                headersSize: 200,
+                bodySize: fileSize,
+                isCompressed: false,
+                uncompressedSize: fileSize
+            }
+        };
+
+        fileMinifier.minifyFile(entry)
+
+        .then(function(newEntry) {
+            newEntry.weightCheck.should.not.have.a.property('isOptimized');
+            newEntry.weightCheck.should.not.have.a.property('optimized');
+            newEntry.weightCheck.should.not.have.a.property('bodyAfterOptimization');
+
+            done();
+        })
+
+        .fail(function(err) {
+            done(err);
+        });
+    });
+
+    it('should fail minifying a JS with syntax errors', function(done) {
+        var fileContent = fs.readFileSync(path.resolve(__dirname, '../www/svg-image.svg'));
+        var fileSize = fileContent.length;
+
+        var entry = {
+            method: 'GET',
+            url: 'http://localhost:8388/svg-image.svg',
+            requestHeaders: {
+                'User-Agent': 'something',
+                Referer: 'http://www.google.fr/',
+                Accept: '*/*',
+                'Accept-Encoding': 'gzip, deflate'
+            },
+            status: 200,
+            isJS: true,
+            type: 'js',
+            contentLength: 999,
+            weightCheck: {
+                body: fileContent.toString('utf8'),
+                totalWeight: fileSize + 200,
+                headersSize: 200,
+                bodySize: fileSize,
+                isCompressed: false,
+                uncompressedSize: fileSize
+            }
+        };
+
+        fileMinifier.minifyFile(entry)
+
+        .then(function(newEntry) {
+            newEntry.weightCheck.should.not.have.a.property('isOptimized');
+            newEntry.weightCheck.should.not.have.a.property('optimized');
+            newEntry.weightCheck.should.not.have.a.property('bodyAfterOptimization');
+
+            done();
+        })
+
+        .fail(function(err) {
+            done(err);
+        });
+    });
+
+    it('should minify a CSS file with clean-css', function(done) {
+        var fileContent = fs.readFileSync(path.resolve(__dirname, '../www/unminified-stylesheet.css'));
+
+        var fileSize = fileContent.length;
+
+        fileMinifier.minifyCss(fileContent.toString()).then(function(newFile) {
+            var newFileSize = newFile.length;
+            newFileSize.should.be.below(fileSize);
+            done();
+        }).fail(function(err) {
+            done(err);
+        });
+    });
+
+    it('should minify a CSS file with minifyFile', function(done) {
+        var fileContent = fs.readFileSync(path.resolve(__dirname, '../www/unminified-stylesheet.css'));
+        var fileSize = fileContent.length;
+
+        var entry = {
+            method: 'GET',
+            url: 'http://localhost:8388/unminified-stylesheet.css',
+            requestHeaders: {
+                'User-Agent': 'something',
+                Referer: 'http://www.google.fr/',
+                Accept: '*/*',
+                'Accept-Encoding': 'gzip, deflate'
+            },
+            status: 200,
+            isCSS: true,
+            type: 'css',
+            contentLength: 999,
+            weightCheck: {
+                body: fileContent.toString('utf8'),
+                totalWeight: fileSize + 200,
+                headersSize: 200,
+                bodySize: fileSize,
+                isCompressed: false,
+                uncompressedSize: fileSize
+            }
+        };
+
+        fileMinifier.minifyFile(entry)
+
+        .then(function(newEntry) {
+            newEntry.weightCheck.should.have.a.property('isOptimized').that.equals(false);
+            newEntry.weightCheck.should.have.a.property('optimized').that.is.below(fileSize);
+            newEntry.weightCheck.should.have.a.property('bodyAfterOptimization').that.is.a('string');
+
+            done();
+        })
+
+        .fail(function(err) {
+            done(err);
+        });
+    });
+
+    it('should minify an HTML file with ', function(done) {
+        var fileContent = fs.readFileSync(path.resolve(__dirname, '../www/jquery-page.html'));
+
+        var fileSize = fileContent.length;
+
+        fileMinifier.minifyHtml(fileContent.toString()).then(function(newFile) {
+            var newFileSize = newFile.length;
+            newFileSize.should.be.below(fileSize);
+            done();
+        }).fail(function(err) {
+            done(err);
+        });
+    });
+
+    it('should avoid minifying some JS files known as minified', function() {
+        fileMinifier.isKnownAsMinified('https://platform.twitter.com/widgets.js').should.equal(true);
+        fileMinifier.isKnownAsMinified('http://platform.twitter.com/widgets.js').should.equal(true);
+
+        fileMinifier.isKnownAsMinified('https://connect.facebook.net/fr_FR/sdk.js').should.equal(true);
+        fileMinifier.isKnownAsMinified('https://connect.facebook.net/en_EN/sdk.js').should.equal(true);
+        fileMinifier.isKnownAsMinified('https://connect.facebook.net/fr_FR/all.js').should.equal(true);
+        fileMinifier.isKnownAsMinified('https://connect.facebook.net/en_EN/all.js').should.equal(true);
+
+        fileMinifier.isKnownAsMinified('https://apis.google.com/js/plusone.js').should.equal(true);
+
+        fileMinifier.isKnownAsMinified('https://code.jquery.com/jquery-2.1.4.min.js').should.equal(true);
+        fileMinifier.isKnownAsMinified('http://code.jquery.com/jquery-2.1.4.min.js').should.equal(true);
+        fileMinifier.isKnownAsMinified('https://code.jquery.com/jquery-2.1.4.js').should.equal(false);
+        fileMinifier.isKnownAsMinified('http://code.jquery.com/jquery-2.1.4.js').should.equal(false);
+
+        fileMinifier.isKnownAsMinified('https://ssl.google-analytics.com/ga.js').should.equal(true);
+        fileMinifier.isKnownAsMinified('http://www.google-analytics.com/ga.js').should.equal(true);
+        fileMinifier.isKnownAsMinified('https://www.google-analytics.com/analytics.js').should.equal(true);
+        fileMinifier.isKnownAsMinified('http://www.google-analytics.com/analytics.js').should.equal(true);
+
+        fileMinifier.isKnownAsMinified('http://anydomain.com/anyurl').should.equal(false);
+    });
+
+});

test/core/gzipCompressorTest.js

@@ -0,0 +1,583 @@
+var should = require('chai').should();
+var gzipCompressor = require('../../lib/tools/redownload/gzipCompressor');
+var fileMinifier = require('../../lib/tools/redownload/fileMinifier');
+var fs = require('fs');
+var path = require('path');
+
+describe('gzipCompressor', function() {
+
+    var minifiedJSContent = fs.readFileSync(path.resolve(__dirname, '../www/minified-script.js'));
+    var notMinifiedJSContent = fs.readFileSync(path.resolve(__dirname, '../www/unminified-script.js'));
+    var someTextFileContent = fs.readFileSync(path.resolve(__dirname, '../www/svg-image.svg'));
+
+    
+    it('should gzip a JS file that was not gziped but was minified', function(done) {
+        var fileContent = minifiedJSContent;
+        var fileSize = fileContent.length;
+
+        var entry = {
+            method: 'GET',
+            url: 'http://localhost:8388/minified-script.js',
+            status: 200,
+            isJS: true,
+            type: 'js',
+            contentLength: 999,
+            weightCheck: {
+                body: fileContent.toString('utf8'),
+                totalWeight: fileSize + 200,
+                headersSize: 200,
+                bodySize: fileSize,
+                isCompressed: false,
+                uncompressedSize: fileSize,
+                isOptimized: true
+            }
+        };
+
+        gzipCompressor.compressFile(entry)
+
+        .then(function(newEntry) {
+            newEntry.weightCheck.should.have.a.property('afterCompression').that.is.below(fileSize);
+            newEntry.weightCheck.should.not.have.a.property('afterOptimizationAndCompression');
+
+            done();
+        })
+
+        .fail(function(err) {
+            done(err);
+        });
+    });
+
+    it('should gzip a JS file that was not gziped and not minified', function(done) {
+        /*jshint expr: true*/
+
+        var fileContent = notMinifiedJSContent;
+        var minifiedContent = minifiedJSContent;
+
+        var fileSize = fileContent.length;
+        var minifiedSize = minifiedContent.length;
+
+        var entry = {
+            method: 'GET',
+            url: 'http://localhost:8388/unminified-script.js',
+            status: 200,
+            isJS: true,
+            type: 'js',
+            contentLength: 999,
+            weightCheck: {
+                body: fileContent.toString('utf8'),
+                bodyAfterOptimization: minifiedContent.toString('utf8'),
+                totalWeight: fileSize + 200,
+                headersSize: 200,
+                bodySize: fileSize,
+                isCompressed: false,
+                uncompressedSize: fileSize,
+                isOptimized: false,
+                optimized: minifiedSize
+            }
+        };
+
+        gzipCompressor.compressFile(entry)
+
+        .then(function(newEntry) {
+            newEntry.weightCheck.should.have.a.property('afterCompression').that.is.below(fileSize);
+            newEntry.weightCheck.should.have.a.property('afterOptimizationAndCompression').that.is.not.undefined;
+            newEntry.weightCheck.should.have.a.property('afterOptimizationAndCompression').that.is.below(newEntry.weightCheck.afterCompression);
+
+            done();
+        })
+
+        .fail(function(err) {
+            done(err);
+        });
+    });
+
+    it('should gzip a JS file that is gziped but not minified', function(done) {
+        /*jshint expr: true*/
+
+        var fileContent = notMinifiedJSContent;
+        var minifiedContent = minifiedJSContent;
+        var fileSize = 6436;
+        var gzipedSize = 2646;
+        var minifiedSize = 1954;
+
+        var entry = {
+            method: 'GET',
+            url: 'http://localhost:8388/unminified-script.js',
+            status: 200,
+            isJS: true,
+            type: 'js',
+            contentLength: 999,
+            weightCheck: {
+                body: fileContent.toString('utf8'),
+                bodyAfterOptimization: minifiedContent.toString('utf8'),
+                totalWeight: gzipedSize + 200,
+                headersSize: 200,
+                bodySize: gzipedSize,
+                isCompressed: true,
+                uncompressedSize: fileSize,
+                isOptimized: false,
+                optimized: minifiedSize
+            }
+        };
+
+        gzipCompressor.compressFile(entry)
+
+        .then(function(newEntry) {
+            newEntry.weightCheck.should.have.a.property('afterOptimizationAndCompression').that.is.not.undefined;
+            newEntry.weightCheck.should.have.a.property('afterOptimizationAndCompression').that.is.below(gzipedSize);
+            newEntry.weightCheck.should.have.a.property('afterOptimizationAndCompression').that.is.below(minifiedSize);
+
+            done();
+        })
+
+        .fail(function(err) {
+            done(err);
+        });
+    });
+
+    it('should not gzip a JS file that was gziped and minified', function(done) {
+        /*jshint expr: true*/
+
+        var fileContent = notMinifiedJSContent;
+        var fileSize = 6436;
+        var gzipedSize = 2646;
+
+        var entry = {
+            method: 'GET',
+            url: 'http://localhost:8388/unminified-script.js',
+            status: 200,
+            isJS: true,
+            type: 'js',
+            contentLength: 999,
+            weightCheck: {
+                body: fileContent.toString('utf8'),
+                totalWeight: gzipedSize + 200,
+                headersSize: 200,
+                bodySize: gzipedSize,
+                isCompressed: true,
+                uncompressedSize: fileSize,
+                isOptimized: true
+            }
+        };
+
+        gzipCompressor.compressFile(entry)
+
+        .then(function(newEntry) {
+            newEntry.weightCheck.should.not.have.a.property('minified');
+            newEntry.weightCheck.should.not.have.a.property('bodyAfterOptimization');
+            newEntry.weightCheck.should.not.have.a.property('afterCompression');
+            newEntry.weightCheck.should.not.have.a.property('afterOptimizationAndCompression');
+
+            done();
+        })
+
+        .fail(function(err) {
+            done(err);
+        });
+    });
+
+    it('should gzip a CSS file', function(done) {
+        var fileContent = fs.readFileSync(path.resolve(__dirname, '../www/unminified-stylesheet.css'));
+        var fileSize = fileContent.length;
+
+        var entry = {
+            method: 'GET',
+            url: 'http://localhost:8388/unminified-stylesheet.css',
+            status: 200,
+            isCSS: true,
+            type: 'css',
+            contentLength: 999,
+            weightCheck: {
+                body: fileContent.toString('utf8'),
+                totalWeight: fileSize + 200,
+                headersSize: 200,
+                bodySize: fileSize,
+                isCompressed: false,
+                uncompressedSize: fileSize,
+                isOptimized: true
+            }
+        };
+
+        gzipCompressor.compressFile(entry)
+
+        .then(function(newEntry) {
+            newEntry.weightCheck.should.have.a.property('afterCompression').that.is.below(fileSize);
+            newEntry.weightCheck.should.not.have.a.property('afterOptimizationAndCompression');
+
+            done();
+        })
+
+        .fail(function(err) {
+            done(err);
+        });
+    });
+
+    it('should gzip an HTML file', function(done) {
+        var fileContent = fs.readFileSync(path.resolve(__dirname, '../www/jquery-page.html'));
+        var fileSize = fileContent.length;
+
+        var entry = {
+            method: 'GET',
+            url: 'http://localhost:8388/jquery-page.html',
+            status: 200,
+            isHTML: true,
+            type: 'html',
+            contentLength: 999,
+            weightCheck: {
+                body: fileContent.toString('utf8'),
+                totalWeight: fileSize + 200,
+                headersSize: 200,
+                bodySize: fileSize,
+                isCompressed: false,
+                uncompressedSize: fileSize,
+                isOptimized: true
+            }
+        };
+
+        gzipCompressor.compressFile(entry)
+
+        .then(function(newEntry) {
+            newEntry.weightCheck.should.have.a.property('afterCompression').that.is.below(fileSize);
+            newEntry.weightCheck.should.not.have.a.property('afterOptimizationAndCompression');
+
+            done();
+        })
+
+        .fail(function(err) {
+            done(err);
+        });
+    });
+
+    it('should gzip an SVG file', function(done) {
+        var fileContent = fs.readFileSync(path.resolve(__dirname, '../www/svg-image.svg'));
+        var fileSize = fileContent.length;
+
+        var entry = {
+            method: 'GET',
+            url: 'http://localhost:8388/svg-image.svg',
+            status: 200,
+            isImage: true,
+            isSVG: true,
+            type: 'image',
+            contentLength: 999,
+            weightCheck: {
+                body: fileContent.toString('utf8'),
+                totalWeight: fileSize + 200,
+                headersSize: 200,
+                bodySize: fileSize,
+                isCompressed: false,
+                uncompressedSize: fileSize,
+                isOptimized: true
+            }
+        };
+
+        gzipCompressor.compressFile(entry)
+
+        .then(function(newEntry) {
+            newEntry.weightCheck.should.have.a.property('afterCompression').that.is.below(fileSize);
+            newEntry.weightCheck.should.not.have.a.property('afterOptimizationAndCompression');
+
+            done();
+        })
+
+        .fail(function(err) {
+            done(err);
+        });
+    });
+
+    it('should gzip an XML file', function(done) {
+        var fileContent = someTextFileContent; // it dosn't matter if it's not the correct file type
+        var fileSize = fileContent.length;
+
+        var entry = {
+            method: 'GET',
+            url: 'http://localhost:8388/someTextFile.xml',
+            status: 200,
+            isXML: true,
+            type: 'xml',
+            contentLength: 999,
+            weightCheck: {
+                body: fileContent.toString('utf8'),
+                totalWeight: fileSize + 200,
+                headersSize: 200,
+                bodySize: fileSize,
+                isCompressed: false,
+                uncompressedSize: fileSize,
+                isOptimized: true
+            }
+        };
+
+        gzipCompressor.compressFile(entry)
+
+        .then(function(newEntry) {
+            newEntry.weightCheck.should.have.a.property('afterCompression').that.is.below(fileSize);
+            newEntry.weightCheck.should.not.have.a.property('afterOptimizationAndCompression');
+
+            done();
+        })
+
+        .fail(function(err) {
+            done(err);
+        });
+    });
+
+    it('should gzip a JSON file', function(done) {
+        var fileContent = someTextFileContent; // it dosn't matter if it's not the correct file type
+        var fileSize = fileContent.length;
+
+        var entry = {
+            method: 'GET',
+            url: 'http://localhost:8388/someTextFile.json',
+            status: 200,
+            isJSON: true,
+            type: 'json',
+            contentLength: 999,
+            weightCheck: {
+                body: fileContent.toString('utf8'),
+                totalWeight: fileSize + 200,
+                headersSize: 200,
+                bodySize: fileSize,
+                isCompressed: false,
+                uncompressedSize: fileSize,
+                isOptimized: true
+            }
+        };
+
+        gzipCompressor.compressFile(entry)
+
+        .then(function(newEntry) {
+            newEntry.weightCheck.should.have.a.property('afterCompression').that.is.below(fileSize);
+            newEntry.weightCheck.should.not.have.a.property('afterOptimizationAndCompression');
+
+            done();
+        })
+
+        .fail(function(err) {
+            done(err);
+        });
+    });
+
+    it('should gzip a TTF file', function(done) {
+        var fileContent = someTextFileContent; // it dosn't matter if it's not the correct file type
+        var fileSize = fileContent.length;
+
+        var entry = {
+            method: 'GET',
+            url: 'http://localhost:8388/someTextFile.ttf',
+            status: 200,
+            isWebFont: true,
+            isTTF: true,
+            type: 'webfont',
+            contentLength: 999,
+            weightCheck: {
+                body: fileContent.toString('utf8'),
+                totalWeight: fileSize + 200,
+                headersSize: 200,
+                bodySize: fileSize,
+                isCompressed: false,
+                uncompressedSize: fileSize,
+                isOptimized: true
+            }
+        };
+
+        gzipCompressor.compressFile(entry)
+
+        .then(function(newEntry) {
+            newEntry.weightCheck.should.have.a.property('afterCompression').that.is.below(fileSize);
+            newEntry.weightCheck.should.not.have.a.property('afterOptimizationAndCompression');
+
+            done();
+        })
+
+        .fail(function(err) {
+            done(err);
+        });
+    });
+
+
+    it('should gzip a favicon file', function(done) {
+        var fileContent = someTextFileContent; // it dosn't matter if it's not the correct file type
+        var fileSize = fileContent.length;
+
+        var entry = {
+            method: 'GET',
+            url: 'http://localhost:8388/someTextFile.ico',
+            status: 200,
+            isFavicon: true,
+            type: 'favicon',
+            contentLength: 999,
+            weightCheck: {
+                body: fileContent.toString('utf8'),
+                totalWeight: fileSize + 200,
+                headersSize: 200,
+                bodySize: fileSize,
+                isCompressed: false,
+                uncompressedSize: fileSize,
+                isOptimized: true
+            }
+        };
+
+        gzipCompressor.compressFile(entry)
+
+        .then(function(newEntry) {
+            newEntry.weightCheck.should.have.a.property('afterCompression').that.is.below(fileSize);
+            newEntry.weightCheck.should.not.have.a.property('afterOptimizationAndCompression');
+
+            done();
+        })
+
+        .fail(function(err) {
+            done(err);
+        });
+    });
+
+    it('should not gzip a JPEG file', function(done) {
+        var fileContent = fs.readFileSync(path.resolve(__dirname, '../www/jpeg-image.jpg'));
+        var fileSize = fileContent.length;
+
+        var entry = {
+            method: 'GET',
+            url: 'http://localhost:8388/jpeg-image.jpg',
+            status: 200,
+            isImage: true,
+            type: 'image',
+            contentType: 'image/jpeg',
+            contentLength: 999,
+            weightCheck: {
+                body: fileContent.toString('utf8'),
+                totalWeight: fileSize + 200,
+                headersSize: 200,
+                bodySize: fileSize,
+                isCompressed: false,
+                uncompressedSize: fileSize,
+                isOptimized: true
+            }
+        };
+
+        gzipCompressor.compressFile(entry)
+
+        .then(function(newEntry) {
+            newEntry.weightCheck.should.not.have.a.property('afterCompression');
+            newEntry.weightCheck.should.not.have.a.property('afterOptimizationAndCompression');
+
+            done();
+        })
+
+        .fail(function(err) {
+            done(err);
+        });
+    });
+
+
+    it('should not gzip a PNG file', function(done) {
+        var fileContent = fs.readFileSync(path.resolve(__dirname, '../www/png-image.png'));
+        var fileSize = fileContent.length;
+
+        var entry = {
+            method: 'GET',
+            url: 'http://localhost:8388/png-image.png',
+            status: 200,
+            isImage: true,
+            type: 'image',
+            contentType: 'image/png',
+            contentLength: 999,
+            weightCheck: {
+                body: fileContent.toString('utf8'),
+                totalWeight: fileSize + 200,
+                headersSize: 200,
+                bodySize: fileSize,
+                isCompressed: false,
+                uncompressedSize: fileSize,
+                isOptimized: true
+            }
+        };
+
+        gzipCompressor.compressFile(entry)
+
+        .then(function(newEntry) {
+            newEntry.weightCheck.should.not.have.a.property('afterCompression');
+            newEntry.weightCheck.should.not.have.a.property('afterOptimizationAndCompression');
+
+            done();
+        })
+
+        .fail(function(err) {
+            done(err);
+        });
+    });
+
+    it('should not gzip a GIF file', function(done) {
+        var fileContent = fs.readFileSync(path.resolve(__dirname, '../www/png-image.png')); // Fake gif, don't tell anyone...
+        var fileSize = fileContent.length;
+
+        var entry = {
+            method: 'GET',
+            url: 'http://localhost:8388/gif-image.gif',
+            status: 200,
+            isImage: true,
+            type: 'image',
+            contentType: 'image/gif',
+            contentLength: 999,
+            weightCheck: {
+                body: fileContent.toString('utf8'),
+                totalWeight: fileSize + 200,
+                headersSize: 200,
+                bodySize: fileSize,
+                isCompressed: false,
+                uncompressedSize: fileSize,
+                isOptimized: true
+            }
+        };
+
+        gzipCompressor.compressFile(entry)
+
+        .then(function(newEntry) {
+            newEntry.weightCheck.should.not.have.a.property('afterCompression');
+            newEntry.weightCheck.should.not.have.a.property('afterOptimizationAndCompression');
+
+            done();
+        })
+
+        .fail(function(err) {
+            done(err);
+        });
+    });
+
+    it('should not gzip a WEBP file', function(done) {
+        var fileContent = fs.readFileSync(path.resolve(__dirname, '../www/png-image.png')); // Fake webp, don't tell anyone...
+        var fileSize = fileContent.length;
+
+        var entry = {
+            method: 'GET',
+            url: 'http://localhost:8388/webp-image.webp',
+            status: 200,
+            isImage: true,
+            type: 'image',
+            contentType: 'image/webp',
+            contentLength: 999,
+            weightCheck: {
+                body: fileContent.toString('utf8'),
+                totalWeight: fileSize + 200,
+                headersSize: 200,
+                bodySize: fileSize,
+                isCompressed: false,
+                uncompressedSize: fileSize,
+                isOptimized: true
+            }
+        };
+
+        gzipCompressor.compressFile(entry)
+
+        .then(function(newEntry) {
+            newEntry.weightCheck.should.not.have.a.property('afterCompression');
+            newEntry.weightCheck.should.not.have.a.property('afterOptimizationAndCompression');
+
+            done();
+        })
+
+        .fail(function(err) {
+            done(err);
+        });
+    });
+
+});

test/core/imageOptimizerTest.js

@@ -0,0 +1,321 @@
+var should = require('chai').should();
+var imageOptimizer = require('../../lib/tools/redownload/imageOptimizer');
+var fs = require('fs');
+var path = require('path');
+
+describe('imageOptimizer', function() {
+    
+    it('should optimize a JPEG image losslessly', function(done) {
+        var fileContent = fs.readFileSync(path.resolve(__dirname, '../www/jpeg-image.jpg'));
+
+        var fileSize = fileContent.length;
+
+        imageOptimizer.compressJpegLosslessly(fileContent).then(function(newFile) {
+            var newFileSize = newFile.length;
+            newFileSize.should.be.below(fileSize);
+            done();
+        }).fail(function(err) {
+            done(err);
+        });
+    });
+
+    it('should optimize a JPEG image lossly', function(done) {
+        var fileContent = fs.readFileSync(path.resolve(__dirname, '../www/jpeg-image.jpg'));
+
+        var fileSize = fileContent.length;
+
+        imageOptimizer.compressJpegLossly(fileContent).then(function(newFile) {
+            var newFileSize = newFile.length;
+            newFileSize.should.be.below(fileSize);
+            done();
+        }).fail(function(err) {
+            done(err);
+        });
+    });
+
+    it('should find the best optimization for a jpeg', function(done) {
+        var fileContent = fs.readFileSync(path.resolve(__dirname, '../www/jpeg-image.jpg'));
+        var fileSize = fileContent.length;
+
+        var entry = {
+            method: 'GET',
+            url: 'http://localhost:8388/an-image.jpg',
+            requestHeaders: {
+                'User-Agent': 'something',
+                Referer: 'http://www.google.fr/',
+                Accept: '*/*',
+                'Accept-Encoding': 'gzip, deflate'
+            },
+            status: 200,
+            isImage: true,
+            type: 'image',
+            contentType: 'image/jpeg',
+            contentLength: 999,
+            weightCheck: {
+                body: fileContent,
+                totalWeight: fileSize + 200,
+                headersSize: 200,
+                bodySize: fileSize,
+                isCompressed: false,
+                uncompressedSize: fileSize
+            }
+        };
+
+        imageOptimizer.optimizeImage(entry)
+
+        .then(function(newEntry) {
+            newEntry.weightCheck.should.have.a.property('isOptimized').that.equals(false);
+            newEntry.weightCheck.should.have.a.property('lossless').that.is.below(fileSize);
+            newEntry.weightCheck.should.have.a.property('lossy').that.is.below(newEntry.weightCheck.lossless);
+
+            done();
+        })
+
+        .fail(function(err) {
+            done(err);
+        });
+    });
+
+    it('should optimize a PNG image losslessly', function(done) {
+        var fileContent = fs.readFileSync(path.resolve(__dirname, '../www/png-image.png'));
+
+        var fileSize = fileContent.length;
+
+        imageOptimizer.compressPngLosslessly(fileContent).then(function(newFile) {
+            var newFileSize = newFile.length;
+            newFileSize.should.be.below(fileSize);
+            done();
+        }).fail(function(err) {
+            done(err);
+        });
+    });
+
+    it('should fail to optimize an already optimized PNG', function(done) {
+        var fileContent = fs.readFileSync(path.resolve(__dirname, '../www/logo-large.png'));
+
+        var fileSize = fileContent.length;
+
+        imageOptimizer.compressPngLosslessly(fileContent).then(function(newFile) {
+            var newFileSize = newFile.length;
+            newFileSize.should.equal(fileSize);
+            done();
+        }).fail(function(err) {
+            done(err);
+        });
+    });
+
+    it('should fail to optimize a non-PNG', function(done) {
+        var fileContent = fs.readFileSync(path.resolve(__dirname, '../www/svg-image.svg'));
+
+        var fileSize = fileContent.length;
+
+        imageOptimizer.compressPngLosslessly(fileContent).then(function(newFile) {
+            var newFileSize = newFile.length;
+            newFileSize.should.equal(fileSize);
+            done();
+        }).fail(function(err) {
+            done(err);
+        });
+    });
+
+    it('should optimize a png', function(done) {
+        var fileContent = fs.readFileSync(path.resolve(__dirname, '../www/png-image.png'));
+        var fileSize = fileContent.length;
+
+        var entry = {
+            method: 'GET',
+            url: 'http://localhost:8388/an-image.png',
+            requestHeaders: {
+                'User-Agent': 'something',
+                Referer: 'http://www.google.fr/',
+                Accept: '*/*',
+                'Accept-Encoding': 'gzip, deflate'
+            },
+            status: 200,
+            isImage: true,
+            type: 'image',
+            contentType: 'image/png',
+            contentLength: 999,
+            weightCheck: {
+                body: fileContent,
+                totalWeight: fileSize + 200,
+                headersSize: 200,
+                bodySize: fileSize,
+                isCompressed: false,
+                uncompressedSize: fileSize
+            }
+        };
+
+        imageOptimizer.optimizeImage(entry)
+
+        .then(function(newEntry) {
+            newEntry.weightCheck.should.have.a.property('isOptimized').that.equals(false);
+            newEntry.weightCheck.should.have.a.property('lossless').that.is.below(fileSize);
+
+            done();
+        })
+
+        .fail(function(err) {
+            done(err);
+        });
+    });
+
+    it('should optimize an SVG image losslessly', function(done) {
+        var fileContent = fs.readFileSync(path.resolve(__dirname, '../www/svg-image.svg'));
+
+        var fileSize = fileContent.length;
+
+        imageOptimizer.compressSvgLosslessly(fileContent).then(function(newFile) {
+            var newFileSize = newFile.length;
+            newFileSize.should.be.below(fileSize);
+            done();
+        }).fail(function(err) {
+            done(err);
+        });
+    });
+
+    it('should optimize an SVG', function(done) {
+        var fileContent = fs.readFileSync(path.resolve(__dirname, '../www/svg-image.svg'));
+        var fileSize = fileContent.length;
+
+        var entry = {
+            method: 'GET',
+            url: 'http://localhost:8388/an-image.svg',
+            requestHeaders: {
+                'User-Agent': 'something',
+                Referer: 'http://www.google.fr/',
+                Accept: '*/*',
+                'Accept-Encoding': 'gzip, deflate'
+            },
+            status: 200,
+            isImage: true,
+            isSVG: true,
+            type: 'image',
+            contentType: 'image/svg+xml',
+            contentLength: 999,
+            weightCheck: {
+                body: fileContent,
+                totalWeight: fileSize + 200,
+                headersSize: 200,
+                bodySize: fileSize,
+                isCompressed: false,
+                uncompressedSize: fileSize
+            }
+        };
+
+        imageOptimizer.optimizeImage(entry)
+
+        .then(function(newEntry) {
+            newEntry.weightCheck.should.have.a.property('isOptimized').that.equals(false);
+            newEntry.weightCheck.should.have.a.property('lossless').that.is.below(fileSize);
+            newEntry.weightCheck.should.have.a.property('optimized').that.equals(newEntry.weightCheck.lossless);
+            newEntry.weightCheck.should.have.a.property('bodyAfterOptimization').that.is.a('string');
+
+            done();
+        })
+
+        .fail(function(err) {
+            done(err);
+        });
+    });
+
+
+    it('shouldn\'t fail optimizing a corrupted jpeg', function(done) {
+
+        // In this test, we try to optimize a PNG but with a falsy "image/jpeg" content type
+
+        var fileContent = fs.readFileSync(path.resolve(__dirname, '../www/png-image.png'));
+        var fileSize = fileContent.length;
+
+        var entry = {
+            method: 'GET',
+            url: 'http://localhost:8388/an-image.png',
+            requestHeaders: {
+                'User-Agent': 'something',
+                Referer: 'http://www.google.fr/',
+                Accept: '*/*',
+                'Accept-Encoding': 'gzip, deflate'
+            },
+            status: 200,
+            isImage: true,
+            type: 'image',
+            contentType: 'image/jpeg',
+            contentLength: 999,
+            weightCheck: {
+                body: fileContent,
+                totalWeight: fileSize + 200,
+                headersSize: 200,
+                bodySize: fileSize,
+                isCompressed: false,
+                uncompressedSize: fileSize
+            }
+        };
+
+        imageOptimizer.optimizeImage(entry)
+
+        .then(function(newEntry) {
+            newEntry.weightCheck.should.not.have.a.property('isOptimized');
+            done();
+        })
+
+        .fail(function(err) {
+            done(err);
+        });
+    });
+
+    it('shouldn\'t fail optimizing a corrupted png', function(done) {
+
+        // In this test, we try to optimize a JPEG but with a falsy "image/png" content type
+
+        var fileContent = fs.readFileSync(path.resolve(__dirname, '../www/jpeg-image.jpg'));
+        var fileSize = fileContent.length;
+
+        var entry = {
+            method: 'GET',
+            url: 'http://localhost:8388/an-image.jpg',
+            requestHeaders: {
+                'User-Agent': 'something',
+                Referer: 'http://www.google.fr/',
+                Accept: '*/*',
+                'Accept-Encoding': 'gzip, deflate'
+            },
+            status: 200,
+            isImage: true,
+            type: 'image',
+            contentType: 'image/png',
+            contentLength: 999,
+            weightCheck: {
+                body: fileContent,
+                totalWeight: fileSize + 200,
+                headersSize: 200,
+                bodySize: fileSize,
+                isCompressed: false,
+                uncompressedSize: fileSize
+            }
+        };
+
+        imageOptimizer.optimizeImage(entry)
+
+        .then(function(newEntry) {
+            newEntry.weightCheck.should.not.have.a.property('isOptimized');
+            done();
+        })
+
+        .fail(function(err) {
+            done(err);
+        });
+    });
+
+    it('should determine if gain is enough', function() {
+        imageOptimizer.gainIsEnough(20000, 10000).should.equal(true);
+        imageOptimizer.gainIsEnough(2000, 1000).should.equal(true);
+        imageOptimizer.gainIsEnough(20000, 21000).should.equal(false);
+        imageOptimizer.gainIsEnough(20000, 40000).should.equal(false);
+        imageOptimizer.gainIsEnough(20000, 19500).should.equal(false);
+        imageOptimizer.gainIsEnough(250, 120).should.equal(true);
+        imageOptimizer.gainIsEnough(200, 120).should.equal(false);
+        imageOptimizer.gainIsEnough(2000, 1900).should.equal(false);
+        imageOptimizer.gainIsEnough(200000, 197000).should.equal(true);
+    });
+
+});

test/core/indexTest.js

@@ -101,8 +101,10 @@ describe('index.js', function() {
                 /*jshint expr: true*/
                 console.log.should.not.have.been.called;
 
+                console.log.restore();
                 done();
             }).fail(function(err) {
+                console.log.restore();
                 done(err);
             });
     });

test/core/isHttp2Test.js

@@ -0,0 +1,48 @@
+var should = require('chai').should();
+var isHttp2 = require('../../lib/tools/isHttp2');
+
+describe('isHttp2', function() {
+    
+    it('should parse the protocol correctly', function() {
+        isHttp2.getProtocol({
+            toolsResults: {
+                phantomas: {
+                    url: 'http://www.yahoo.com'
+                }
+            }
+        }).should.equal('http:');
+
+
+        isHttp2.getProtocol({
+            toolsResults: {
+                phantomas: {
+                    url: 'https://www.yahoo.com'
+                }
+            }
+        }).should.equal('https:');
+    });
+
+    it('should parse the domain correctly', function() {
+        isHttp2.getDomain({
+            toolsResults: {
+                phantomas: {
+                    url: 'http://www.yahoo.com'
+                }
+            }
+        }).should.equal('www.yahoo.com');
+
+
+        isHttp2.getDomain({
+            toolsResults: {
+                phantomas: {
+                    url: 'https://www.yahoo.com'
+                }
+            }
+        }).should.equal('www.yahoo.com');
+    });
+
+    it('should have a function checkHttp2', function() {
+        isHttp2.should.have.a.property('checkHttp2').that.is.a('function');
+    });
+
+});

test/core/mediaQueriesCheckerTest.js

@@ -0,0 +1,55 @@
+var should = require('chai').should();
+var mediaQueriesChecker = require('../../lib/tools/mediaQueriesChecker');
+
+describe('mediaQueriesChecker', function() {
+    
+    it('should parse mediaQueryes correctly', function() {
+        mediaQueriesChecker.parseOneMediaQuery('@media screen and (max-width: 1024px) (1 rules) <http://domain.com/css/stylesheet.css> @ 269:1').should.deep.equal({
+            mediaQuery: {
+                query: 'screen and (max-width: 1024px)',
+                rules: 1,
+                file: 'http://domain.com/css/stylesheet.css',
+                line: 269,
+                column: 1
+            },
+            isForMobile: true,
+            breakpoints: [{string: '1024px', pixels: 1024}]
+        });
+    });
+
+    it('should determine if it\'s for mobile correctly', function() {
+        mediaQueriesChecker.parseOneMediaQuery('@media screen and (max-width: 1024px) (1 rules) <file> @ 1:1').isForMobile.should.equal(true);
+        mediaQueriesChecker.parseOneMediaQuery('@media (max-width:1024px) (1 rules) <file> @ 1:1').isForMobile.should.equal(true);
+        mediaQueriesChecker.parseOneMediaQuery('@media screen and (max-width: 320px) (1 rules) <file> @ 1:1').isForMobile.should.equal(true);
+        mediaQueriesChecker.parseOneMediaQuery('@media screen and (max-width: 290px) (1 rules) <file> @ 1:1').isForMobile.should.equal(false);
+        mediaQueriesChecker.parseOneMediaQuery('@media screen and (min-width: 320px) (1 rules) <file> @ 1:1').isForMobile.should.equal(false);
+        mediaQueriesChecker.parseOneMediaQuery('@media screen and (min-width: 600px) (1 rules) <file> @ 1:1').isForMobile.should.equal(false);
+        mediaQueriesChecker.parseOneMediaQuery('@media screen and (max-width: 20em) (1 rules) <file> @ 1:1').isForMobile.should.equal(true);
+        mediaQueriesChecker.parseOneMediaQuery('@media screen and (min-width: 40em) (1 rules) <file> @ 1:1').isForMobile.should.equal(false);
+        mediaQueriesChecker.parseOneMediaQuery('@media (min-width: 600px) and (max-width: 1000px) (1 rules) <file> @ 1:1').isForMobile.should.equal(false);
+        mediaQueriesChecker.parseOneMediaQuery('@media (min-width: 180px) and (max-width: 400px) (1 rules) <file> @ 1:1').isForMobile.should.equal(true);
+        mediaQueriesChecker.parseOneMediaQuery('@media (min-width: 180px) and (max-width: 290px) (1 rules) <file> @ 1:1').isForMobile.should.equal(false);
+        mediaQueriesChecker.parseOneMediaQuery('@media not all and (min-width: 600px) (1 rules) <file> @ 1:1').isForMobile.should.equal(true);
+        mediaQueriesChecker.parseOneMediaQuery('@media not all and (min-width: 180px) (1 rules) <file> @ 1:1').isForMobile.should.equal(false);
+        mediaQueriesChecker.parseOneMediaQuery('@media not all and (min-width: 1000px) and (max-width: 600px) (1 rules) <file> @ 1:1').isForMobile.should.equal(false);
+        mediaQueriesChecker.parseOneMediaQuery('@media not all and (min-width: 400px) and (max-width: 180px) (1 rules) <file> @ 1:1').isForMobile.should.equal(true);
+    });
+
+    it('should count breakpoints correctly', function() {
+        mediaQueriesChecker.parseOneMediaQuery('@media screen and (max-width: 1024px) (1 rules) <file> @ 1:1').breakpoints.should.deep.equal([{string: '1024px', pixels: 1024}]);
+        mediaQueriesChecker.parseOneMediaQuery('@media (max-width:1024px) (1 rules) <file> @ 1:1').breakpoints.should.deep.equal([{string: '1024px', pixels: 1024}]);
+        mediaQueriesChecker.parseOneMediaQuery('@media screen and (max-width: 320px) (1 rules) <file> @ 1:1').breakpoints.should.deep.equal([{string: '320px', pixels: 320}]);
+        mediaQueriesChecker.parseOneMediaQuery('@media (min-width: 600px) and (max-width: 1000px) (1 rules) <file> @ 1:1').breakpoints.should.deep.equal([{string: '600px', pixels: 600}, {string: '1000px', pixels: 1000}]);
+        mediaQueriesChecker.parseOneMediaQuery('@media not all and (min-width: 180px) (1 rules) <file> @ 1:1').breakpoints.should.deep.equal([{string: '180px', pixels: 180}]);
+        mediaQueriesChecker.parseOneMediaQuery('@media not all and (min-width: 1000px) and (max-width: 600px) (1 rules) <file> @ 1:1').breakpoints.should.deep.equal([{string: '1000px', pixels: 1000}, {string: '600px', pixels: 600}]);
+        mediaQueriesChecker.parseOneMediaQuery('@media (max-height:500px) (1 rules) <file> @ 1:1').breakpoints.should.deep.equal([]);
+        mediaQueriesChecker.parseOneMediaQuery('@media screen and (max-width: 100em) (1 rules) <file> @ 1:1').breakpoints.should.deep.equal([{string: '100em', pixels: 1600}]);
+        mediaQueriesChecker.parseOneMediaQuery('@media screen and (max-width: 120pt) (1 rules) <file> @ 1:1').breakpoints.should.deep.equal([{string: '120pt', pixels: 160}]);
+        mediaQueriesChecker.parseOneMediaQuery('@media screen and (max-width: 40.2em) (1 rules) <file> @ 1:1').breakpoints.should.deep.equal([{string: '40.2em', pixels: 643.2}]);
+    });
+
+    it('should fail silently', function() {
+        mediaQueriesChecker.parseOneMediaQuery('@media bad syntax (1 rules) <file> @ 1:1').isForMobile.should.equal(false); 
+    });
+
+});

test/core/offendersHelpersTest.js

@@ -200,6 +200,47 @@ describe('offendersHelpers', function() {
             ]);
         });
 
+        it('should transform a backtrace with the new PhantomJS 2.x syntax into an array', function() {
+            var result = offendersHelpers.backtraceToArray('each@http://m.australia.fr/js/min/vendors.js?20160706185900:4:5365 / f@http://m.australia.fr/js/min/vendors.js?20160706185900:17:82 / http://m.australia.fr/js/min/vendors.js?20160706185900:17:855 / handle@http://m.australia.fr/js/min/vendors.js?20160706185900:5:10871 / report@phantomjs://platform/phantomas.js:535:20 / phantomjs://platform/phantomas.js:524:15');
+
+            result.should.deep.equal([
+                {
+                    functionName: 'each',
+                    file: 'http://m.australia.fr/js/min/vendors.js?20160706185900',
+                    line: 4,
+                    column: 5365
+                },
+                {
+                    functionName: 'f',
+                    file: 'http://m.australia.fr/js/min/vendors.js?20160706185900',
+                    line: 17,
+                    column: 82
+                },
+                {
+                    file: 'http://m.australia.fr/js/min/vendors.js?20160706185900',
+                    line: 17,
+                    column: 855
+                },
+                {
+                    functionName: 'handle',
+                    file: 'http://m.australia.fr/js/min/vendors.js?20160706185900',
+                    line: 5,
+                    column: 10871
+                },
+                {
+                    functionName: 'report',
+                    file: 'phantomjs://platform/phantomas.js',
+                    line: 535,
+                    column: 20
+                },
+                {
+                    file: 'phantomjs://platform/phantomas.js',
+                    line: 524,
+                    column: 15
+                }
+            ]);
+        });
+
         it('should return null if it fails', function() {
             var result = offendersHelpers.backtraceToArray('http://pouet.com/js/jquery.footer-transverse-min-v1.0.20.js:1 /http://pouet.com/js/main.js:1');
 
@@ -299,6 +340,17 @@ describe('offendersHelpers', function() {
             });
         });
 
+        it('should handle line breaks inside the string', function() {
+            var result = offendersHelpers.cssOffenderPattern('.card-mask-wrap { -moz-transform: translate3d(0, \n-288px\n, 0) } // was required by firefox 15 and earlier [inline CSS] @ 29:3');
+
+            result.should.deep.equal({
+                css: '.card-mask-wrap { -moz-transform: translate3d(0, -288px, 0) } // was required by firefox 15 and earlier',
+                file: null,
+                line: 29,
+                column: 3
+            });
+        }); 
+
     });
 
     describe('fileWithSizePattern', function() {

test/core/phantomasWrapperTest.js

@@ -42,18 +42,23 @@ describe('phantomasWrapper', function() {
         phantomasWrapper.execute({
             params: {
                 url: url,
-                options: {}
+                options: {
+                    device: 'desktop'
+                }
             }
         }).then(function(data) {
 
             done('Error: unwanted success');
 
         }).fail(function(err) {
+            try {
+                should.exist(err);
+                err.should.equal(254);
 
-            should.exist(err);
-            err.should.equal(254);
-
-            done();
+                done();
+            } catch(error) {
+                done(error);
+            }
         });
     });
 
@@ -70,16 +75,20 @@ describe('phantomasWrapper', function() {
             }
         }).then(function(data) {
             /*jshint -W030 */
-            
-            data.should.be.an('object');
-            data.should.have.a.property('generator');
-            data.generator.should.contain('phantomas');
-            data.should.have.a.property('url').that.equals(url);
-            data.should.have.a.property('metrics').that.is.an('object').not.empty;
-            data.should.have.a.property('offenders').that.is.an('object').not.empty;
-            data.offenders.should.have.a.property('javascriptExecutionTree').that.is.a('array').not.empty;
 
-            done();
+            try {            
+                data.should.be.an('object');
+                data.should.have.a.property('generator');
+                data.generator.should.contain('phantomas');
+                data.should.have.a.property('url').that.equals(url);
+                data.should.have.a.property('metrics').that.is.an('object').not.empty;
+                data.should.have.a.property('offenders').that.is.an('object').not.empty;
+                data.offenders.should.have.a.property('javascriptExecutionTree').that.is.a('array').not.empty;
+
+                done();
+            } catch(error) {
+                done(error);
+            }
         }).fail(function(err) {
             done(err);
         });

test/core/redownloadTest.js

@@ -0,0 +1,352 @@
+var should = require('chai').should();
+var redownload = require('../../lib/tools/redownload/redownload');
+var fs = require('fs');
+var path = require('path');
+
+describe('redownload', function() {
+    
+    it('should download a list of files', function(done) {
+        this.timeout(10000);
+
+        var requestsList = [
+            {
+                method: 'GET',
+                url: 'http://localhost:8388/simple-page.html',
+                requestHeaders: {
+                    'User-Agent': 'something',
+                   Referer: 'http://www.google.fr/',
+                   Accept: '*/*'
+                },
+                status: 200,
+                isHTML: true,
+                type: 'html'
+            },
+            {
+                method: 'GET',
+                url: 'http://localhost:8388/jquery1.8.3.js',
+                requestHeaders: {
+                    'User-Agent': 'something',
+                   Referer: 'http://www.google.fr/',
+                   Accept: '*/*'
+                },
+                status: 200,
+                isJS: true,
+                type: 'js'
+            },
+            {
+                method: 'GET',
+                url: 'http://localhost:8388/jpeg-image.jpg',
+                requestHeaders: {
+                    'User-Agent': 'something',
+                   Referer: 'http://www.google.fr/',
+                   Accept: '*/*'
+                },
+                status: 200,
+                isImage: true,
+                type: 'image',
+                contentType: 'image/jpeg'
+            },
+            {
+                method: 'GET',
+                url: 'http://localhost:8388/svg-image.svg',
+                requestHeaders: {
+                    'User-Agent': 'something',
+                   Referer: 'http://www.google.fr/',
+                   Accept: '*/*'
+                },
+                status: 200,
+                isImage: true,
+                isSVG: true,
+                type: 'image',
+                contentType: 'image/svg+xml'
+            },
+            {
+                method: 'GET',
+                url: 'http://localhost:8388/unminified-script.js',
+                requestHeaders: {
+                    'User-Agent': 'something',
+                   Referer: 'http://www.google.fr/',
+                   Accept: '*/*'
+                },
+                status: 200,
+                isJS: true,
+                type: 'js'
+            },
+            {
+                method: 'GET',
+                url: 'http://localhost:8388/unminified-stylesheet.css',
+                requestHeaders: {
+                    'User-Agent': 'something',
+                   Referer: 'http://www.google.fr/',
+                   Accept: '*/*'
+                },
+                status: 200,
+                isCSS: true,
+                type: 'css'
+            },
+            {
+                method: 'GET',
+                url: 'http://localhost:8388/xml.xml',
+                requestHeaders: {
+                    'User-Agent': 'something',
+                   Referer: 'http://www.google.fr/',
+                   Accept: '*/*'
+                },
+                status: 200,
+                isXML: true,
+                type: 'xml'
+            },
+            {
+                method: 'GET',
+                url: 'about:blank',
+                requestHeaders: {
+                    'User-Agent': 'something',
+                   Referer: 'http://www.google.fr/',
+                   Accept: '*/*'
+                },
+                status: 200
+            }
+        ];
+
+        var data = {
+            toolsResults: {
+                phantomas: {
+                    metrics: {
+                        requestsList: true
+                    },
+                    offenders: {
+                        requestsList: JSON.stringify(requestsList)
+                    }
+                }
+            }
+        };
+
+        redownload.recheckAllFiles(data)
+
+        .then(function(data) {
+            data.toolsResults.should.have.a.property('redownload');
+            data.toolsResults.redownload.should.have.a.property('metrics');
+            data.toolsResults.redownload.should.have.a.property('offenders');
+
+            data.toolsResults.redownload.offenders.should.have.a.property('totalWeight');
+            data.toolsResults.redownload.offenders.totalWeight.totalWeight.should.be.above(0);
+            data.toolsResults.redownload.offenders.totalWeight.byType.html.requests.length.should.equal(1);
+            data.toolsResults.redownload.offenders.totalWeight.byType.js.requests.length.should.equal(2);
+            data.toolsResults.redownload.offenders.totalWeight.byType.css.requests.length.should.equal(1);
+            data.toolsResults.redownload.offenders.totalWeight.byType.image.requests.length.should.equal(2);
+            data.toolsResults.redownload.offenders.totalWeight.byType.other.requests.length.should.equal(1);
+
+            data.toolsResults.redownload.offenders.should.have.a.property('imageOptimization');
+            data.toolsResults.redownload.offenders.imageOptimization.totalGain.should.be.above(0);
+            data.toolsResults.redownload.offenders.imageOptimization.images.length.should.equal(2);
+
+            data.toolsResults.redownload.offenders.should.have.a.property('gzipCompression');
+            data.toolsResults.redownload.offenders.gzipCompression.totalGain.should.be.above(0);
+            data.toolsResults.redownload.offenders.gzipCompression.files.length.should.equal(5);
+
+            data.toolsResults.redownload.offenders.should.have.a.property('fileMinification');
+            data.toolsResults.redownload.offenders.fileMinification.totalGain.should.be.above(0);
+            data.toolsResults.redownload.offenders.fileMinification.files.length.should.equal(2);
+
+            data.toolsResults.redownload.metrics.should.have.a.property('totalRequests').that.equals(7);
+            data.toolsResults.redownload.offenders.should.have.a.property('totalRequests');
+            data.toolsResults.redownload.offenders.totalRequests.byType.html.length.should.equal(1);
+            data.toolsResults.redownload.offenders.totalRequests.byType.js.length.should.equal(2);
+            data.toolsResults.redownload.offenders.totalRequests.byType.css.length.should.equal(1);
+            data.toolsResults.redownload.offenders.totalRequests.byType.image.length.should.equal(2);
+            data.toolsResults.redownload.offenders.totalRequests.byType.json.length.should.equal(0);
+            data.toolsResults.redownload.offenders.totalRequests.byType.webfont.length.should.equal(0);
+            data.toolsResults.redownload.offenders.totalRequests.byType.video.length.should.equal(0);
+            data.toolsResults.redownload.offenders.totalRequests.byType.other.length.should.equal(1);
+
+            data.toolsResults.redownload.metrics.should.have.a.property('smallRequests').that.equals(0);
+            data.toolsResults.redownload.offenders.should.have.a.property('smallRequests');
+            data.toolsResults.redownload.offenders.smallRequests.byType.js.length.should.equal(0);
+            data.toolsResults.redownload.offenders.smallRequests.byType.css.length.should.equal(0);
+            data.toolsResults.redownload.offenders.smallRequests.byType.image.length.should.equal(0);
+
+            done();
+        })
+
+        .fail(function(err) {
+            done(err);
+        });
+    });
+
+    it('should download a file and measure its size', function(done) {
+        var entry = {
+            method: 'GET',
+            url: 'http://localhost:8388/jquery1.8.3.js',
+            requestHeaders: {
+                'User-Agent': 'something',
+               Referer: 'http://www.google.fr/',
+               Accept: '*/*'
+            },
+            status: 200,
+            isJS: true,
+            type: 'js'
+        };
+
+        redownload.redownloadEntry(entry)
+
+        .then(function(newEntry) {
+
+            newEntry.weightCheck.bodySize.should.equal(93636);
+            newEntry.weightCheck.uncompressedSize.should.equal(newEntry.weightCheck.bodySize);
+            newEntry.weightCheck.isCompressed.should.equal(false);
+            newEntry.weightCheck.headersSize.should.be.above(200).and.below(400);
+            newEntry.weightCheck.body.toString().should.have.string('1.8.3');
+
+            done();
+        })
+
+        .fail(function(err) {
+            done(err);
+        });
+    });
+
+    it('should download a PNG image and find the same body as fs.readFile', function(done) {
+        var fileContent = fs.readFileSync(path.resolve(__dirname, '../www/logo-large.png'));
+
+        var entry = {
+            method: 'GET',
+            url: 'http://localhost:8388/logo-large.png',
+            requestHeaders: {
+                'User-Agent': 'something',
+               Referer: 'http://www.google.fr/',
+               Accept: '*/*'
+            },
+            status: 200,
+            isImage: true,
+            contentType: 'image/png'
+        };
+
+        redownload.redownloadEntry(entry)
+
+        .then(function(newEntry) {
+
+            newEntry.weightCheck.bodySize.should.equal(4193);
+            newEntry.weightCheck.body.should.equal(fileContent.toString('binary'));
+
+            // Opening the image in lwip to check if the format is good
+            var lwip = require('lwip');
+            var buffer = new Buffer(newEntry.weightCheck.body, 'binary');
+            lwip.open(buffer, 'png', function(err, image) {
+                image.width().should.equal(620);
+                image.height().should.equal(104);
+                done(err);
+            });
+            
+        })
+
+        .fail(function(err) {
+            done(err);
+        });
+    });
+
+    it('should fail downloading a file in error', function(done) {
+        var entry = {
+            method: 'GET',
+            url: 'http://localhost:8388/no-file',
+            requestHeaders: {
+                'User-Agent': 'something',
+               Referer: 'http://www.google.fr/',
+               Accept: '*/*'
+            },
+            status: 200,
+            isHTML: true,
+            contentLength: 999
+        };
+
+        redownload.redownloadEntry(entry)
+
+        .then(function(newEntry) {
+            newEntry.weightCheck.should.have.a.property('message').that.equals('error while downloading: 404');
+
+            done();
+        })
+
+        .fail(function(err) {
+            done(err);
+        });
+    });
+
+    it('should not download a non 200 request', function(done) {
+        var entry = {
+            method: 'GET',
+            url: 'http://localhost:8388/no-file',
+            requestHeaders: {
+                'User-Agent': 'something',
+               Referer: 'http://www.google.fr/',
+               Accept: '*/*'
+            },
+            status: 302,
+            isHTML: true,
+            contentLength: 999
+        };
+
+        redownload.redownloadEntry(entry)
+
+        .then(function(newEntry) {
+            newEntry.should.not.have.a.property('weightCheck');
+            done();
+        })
+
+        .fail(function(err) {
+            done(err);
+        });
+    });
+
+    it('should listRequestWeight', function() {
+        var totalWeightObj = redownload.listRequestWeight([{
+            method: 'GET',
+            url: 'http://localhost:8388/jquery1.8.3.js',
+            requestHeaders: {
+                'User-Agent': 'something',
+                Referer: 'http://www.google.fr/',
+                Accept: '*/*',
+                'Accept-Encoding': 'gzip, deflate'
+            },
+            status: 200,
+            isHTML: true,
+            type: 'html',
+            contentLength: 999,
+            weightCheck: {
+                body: 'blabla',
+                headersSize: 200,
+                bodySize: 500,
+                isCompressed: false,
+                uncompressedSize: 500
+            }
+        }]);
+
+        totalWeightObj.totalWeight.should.equal(700);
+        totalWeightObj.byType.html.totalWeight.should.equal(700);
+        totalWeightObj.byType.html.requests.should.have.length(1);
+        totalWeightObj.byType.html.requests[0].should.have.a.property('url').that.equals('http://localhost:8388/jquery1.8.3.js');
+        totalWeightObj.byType.html.requests[0].should.have.a.property('weight').that.equals(700);
+    });
+
+    it('should listRequestWeight even if download failed', function() {
+        var totalWeightObj = redownload.listRequestWeight([{
+            method: 'GET',
+            url: 'http://localhost:8388/jquery1.8.3.js',
+            requestHeaders: {
+                'User-Agent': 'something',
+                Referer: 'http://www.google.fr/',
+                Accept: '*/*',
+                'Accept-Encoding': 'gzip, deflate'
+            },
+            status: 200,
+            isHTML: true,
+            type: 'html',
+            contentLength: 999,
+            weightCheck: {
+                message: 'error while downloading: 404'
+            }
+        }]);
+
+        totalWeightObj.totalWeight.should.equal(999);
+    });
+
+});

test/fixtures/settings.json

@@ -1,6 +1,8 @@
 {
     "serverPort": "8387",
+    "phantomasEngine": "webkit",
     "googleAnalyticsId": "",
+    "screenshotWidth": 400,
     
     "authorizedKeys": {
         "1234567890": "contact@gaelmetais.com"

test/www/jquery-page.html

@@ -190,6 +190,11 @@
         $li.parentsUntil('body', 'div');
         $li.siblings();
         $li.siblings('li');
+
+        $.ajax({
+            url: 'xml.xml',
+            async: false
+        });
     </script>
 </body>
 </html>

test/www/minified-script.js

@@ -0,0 +1,14 @@
+var timelineCtrl=angular.module("timelineCtrl",[])
+timelineCtrl.controller("TimelineCtrl",["$scope","$rootScope","$routeParams","$location","$timeout","Menu","Results","API",function(e,t,n,r,i,a,l,u){function o(){t.loadedResult&&t.loadedResult.runId===n.runId?(e.result=t.loadedResult,c()):l.get({runId:n.runId,exclude:"toolsResults"},function(n){t.loadedResult=n,e.result=n,c()})}function c(){s(),d(),f(),m(),i(p,100)}function s(){var t=r.hash(),n=null
+0===t.indexOf("filter=")&&(n=t.substr(7)),e.warningsFilterOn=null!==n,e.warningsFilters={queryWithoutResults:null===n||"queryWithoutResults"===n,jQueryCallOnEmptyObject:null===n||"jQueryCallOnEmptyObject"===n,eventNotDelegated:null===n||"eventNotDelegated"===n,jsError:null===n||"jsError"===n}}function d(){var t=e.result.rules.jsCount.offendersObj.list
+e.scripts=[],t.forEach(function(t){var n=t.file
+n.length>100&&(n=n.substr(0,98)+"...")
+var r={fullPath:t.file,shortPath:n}
+e.scripts.push(r)})}function f(){var t=e.result.javascriptExecutionTree.children||[],n=t[t.length-1]
+e.endTime=n.data.timestamp+(n.data.time||0),e.executionTree=[],t.forEach(function(t){if(e.selectedScript){if(t.data.backtrace&&-1===t.data.backtrace.indexOf(e.selectedScript.fullPath+":"))return
+if("jQuery loaded"===t.data.type||"jQuery version change"===t.data.type)return}e.executionTree.push(t)})}function m(){var t=199
+e.timelineIntervalDuration=e.endTime/t
+var n=Array.apply(null,Array(e.endTime+1)).map(Number.prototype.valueOf,0)
+e.executionTree.forEach(function(e){if(void 0!==e.data.time)for(var t=Math.min(e.data.time,100)||1,r=e.data.timestamp,i=e.data.timestamp+t;i>r;r++)n[r]|=1}),e.timeline=Array.apply(null,Array(t+1)).map(Number.prototype.valueOf,0),n.forEach(function(t,n){1===t&&(e.timeline[Math.floor(n/e.timelineIntervalDuration)]+=1)}),e.timelineMax=Math.max.apply(Math,e.timeline)}function p(){e.profilerData=e.executionTree}e.runId=n.runId,e.Menu=a.setCurrentPage("timeline",e.runId),e.changeScript=function(){f(),m(),p()},e.findLineIndexByTimestamp=function(t){for(var n=0,r=0;r<e.executionTree.length;r++){var i=e.executionTree[r].data.timestamp-t
+if(i<e.timelineIntervalDuration&&(n=r),i>0)break}return n},e.backToDashboard=function(){r.path("/result/"+e.runId)},e.testAgain=function(){u.relaunchTest(e.result)},o()}]),timelineCtrl.directive("scrollOnClick",["$animate","$timeout",function(e,t){return{restrict:"A",link:function(n,r,i){r.on("click",function(){var r=n.findLineIndexByTimestamp(i.scrollOnClick),a=angular.element(document.getElementById("line_"+r))
+a.addClass("highlight"),t(function(){e.removeClass(a,"highlight"),n.$digest()},50),window.scrollTo(0,a[0].offsetTop)})}}}])

test/www/unminified-script.js

@@ -0,0 +1,190 @@
+var timelineCtrl = angular.module('timelineCtrl', []);
+
+timelineCtrl.controller('TimelineCtrl', ['$scope', '$rootScope', '$routeParams', '$location', '$timeout', 'Menu', 'Results', 'API', function($scope, $rootScope, $routeParams, $location, $timeout, Menu, Results, API) {
+    $scope.runId = $routeParams.runId;
+    $scope.Menu = Menu.setCurrentPage('timeline', $scope.runId);
+
+    function loadResults() {
+        // Load result if needed
+        if (!$rootScope.loadedResult || $rootScope.loadedResult.runId !== $routeParams.runId) {
+            Results.get({runId: $routeParams.runId, exclude: 'toolsResults'}, function(result) {
+                $rootScope.loadedResult = result;
+                $scope.result = result;
+                render();
+            });
+        } else {
+            $scope.result = $rootScope.loadedResult;
+            render();
+        }
+    }
+
+    function render() {
+        initFilters();
+        initScriptFiltering();
+        initExecutionTree();
+        initTimeline();
+        $timeout(initProfiler, 100);
+    }
+
+    function initFilters() {
+        var hash = $location.hash();
+        var filter = null;
+        
+        if (hash.indexOf('filter=') === 0) {
+            filter = hash.substr(7);
+        }
+
+        $scope.warningsFilterOn = (filter !== null);
+        $scope.warningsFilters = {
+            queryWithoutResults: (filter === null || filter === 'queryWithoutResults'),
+            jQueryCallOnEmptyObject: (filter === null || filter === 'jQueryCallOnEmptyObject'),
+            eventNotDelegated: (filter === null || filter === 'eventNotDelegated'),
+            jsError: (filter === null || filter === 'jsError')
+        };
+    }
+
+    function initScriptFiltering() {
+        var offenders = $scope.result.rules.jsCount.offendersObj.list;
+        $scope.scripts = [];
+
+        offenders.forEach(function(script) {
+            var filePath = script.file;
+
+            if (filePath.length > 100) {
+                filePath = filePath.substr(0, 98) + '...';
+            }
+
+            var scriptObj = {
+                fullPath: script.file, 
+                shortPath: filePath
+            };
+
+            $scope.scripts.push(scriptObj);
+        });
+    }
+
+    function initExecutionTree() {
+        var originalExecutions = $scope.result.javascriptExecutionTree.children || [];
+        
+        // Detect the last event of all (before filtering) and read time
+        var lastEvent = originalExecutions[originalExecutions.length - 1];
+        $scope.endTime =  lastEvent.data.timestamp + (lastEvent.data.time || 0);
+
+        // Filter
+        $scope.executionTree = [];
+        originalExecutions.forEach(function(node) {
+            
+            // Filter by script (if enabled)
+            if ($scope.selectedScript) {
+                if (node.data.backtrace && node.data.backtrace.indexOf($scope.selectedScript.fullPath + ':') === -1) {
+                    return;
+                }
+                if (node.data.type === "jQuery loaded" || node.data.type === "jQuery version change") {
+                    return;
+                }
+            }
+
+            $scope.executionTree.push(node);
+        });
+    }
+
+    function initTimeline() {
+
+        // Split the timeline into 200 intervals
+        var numberOfIntervals = 199;
+        $scope.timelineIntervalDuration = $scope.endTime / numberOfIntervals;
+        
+        // Pre-fill array of as many elements as there are milleseconds
+        var millisecondsArray = Array.apply(null, new Array($scope.endTime + 1)).map(Number.prototype.valueOf,0);
+        
+        // Create the milliseconds array from the execution tree
+        $scope.executionTree.forEach(function(node) {
+            if (node.data.time !== undefined) {
+
+                // Ignore artefacts (durations > 100ms)
+                var time = Math.min(node.data.time, 100) || 1;
+
+                for (var i=node.data.timestamp, max=node.data.timestamp + time ; i<max ; i++) {
+                    millisecondsArray[i] |= 1;
+                }
+            }
+        });
+
+        // Pre-fill array of 200 elements
+        $scope.timeline = Array.apply(null, new Array(numberOfIntervals + 1)).map(Number.prototype.valueOf,0);
+
+        // Create the timeline from the milliseconds array
+        millisecondsArray.forEach(function(value, timestamp) {
+            if (value === 1) {
+                $scope.timeline[Math.floor(timestamp / $scope.timelineIntervalDuration)] += 1;
+            }
+        });
+        
+        // Get the maximum value of the array (needed for display)
+        $scope.timelineMax = Math.max.apply(Math, $scope.timeline);
+    }
+
+
+    function initProfiler() {
+        $scope.profilerData = $scope.executionTree;
+    }
+
+    $scope.changeScript = function() {
+        initExecutionTree();
+        initTimeline();
+        initProfiler();
+    };
+
+    $scope.findLineIndexByTimestamp = function(timestamp) {
+        var lineIndex = 0;
+
+        for (var i = 0; i < $scope.executionTree.length; i ++) {
+            var delta = $scope.executionTree[i].data.timestamp - timestamp;
+            
+            if (delta < $scope.timelineIntervalDuration) {
+                lineIndex = i;
+            }
+
+            if (delta > 0) {
+                break;
+            }
+        }
+
+        return lineIndex;
+    };
+
+
+    $scope.backToDashboard = function() {
+        $location.path('/result/' + $scope.runId);
+    };
+
+    $scope.testAgain = function() {
+        API.relaunchTest($scope.result);
+    };
+
+    loadResults();
+
+}]);
+
+timelineCtrl.directive('scrollOnClick', ['$animate', '$timeout', function($animate, $timeout) {
+    return {
+        restrict: 'A',
+        link: function (scope, element, attributes) {            
+            // When the user clicks on the timeline, find the right profiler line and scroll to it
+            element.on('click', function() {
+                var lineIndex = scope.findLineIndexByTimestamp(attributes.scrollOnClick);
+                var lineElement = angular.element(document.getElementById('line_' + lineIndex));
+                
+                // Animate the background color to "flash" the row
+                lineElement.addClass('highlight');
+                $timeout(function() {
+                    $animate.removeClass(lineElement, 'highlight');
+                    scope.$digest();
+                }, 50);
+
+
+                window.scrollTo(0, lineElement[0].offsetTop);
+            });
+        }
+    };
+}]);

test/www/unminified-stylesheet.css

@@ -0,0 +1,772 @@
+/* Timeline colors, related to Window Performances */
+.execution {
+  text-align: center;
+}
+.selectScript {
+  padding-bottom: 2em;
+  font-size: 0.9em;
+}
+.selectScript select {
+  max-width: 30em;
+}
+.selectScript.empty {
+  font-size: 0.8em;
+}
+.selectScript.empty select {
+  width: 10em;
+}
+.timeline {
+  margin: 2em 0 5em;
+}
+.timeline .chart {
+  position: relative;
+  width: 100%;
+  border-bottom: 1px solid #000;
+}
+.timeline .startTime,
+.timeline .endTime {
+  position: absolute;
+  bottom: 0.5em;
+  font-size: 0.8em;
+}
+.timeline .startTime {
+  left: 0em;
+}
+.timeline .endTime {
+  right: 0em;
+}
+.timeline .chartPoints {
+  display: table;
+  height: 100px;
+  width: 99%;
+  margin: 0 auto;
+}
+.timeline .interval {
+  display: table-cell;
+  position: relative;
+  height: 100px;
+  width: 0.5%;
+}
+.timeline .interval .color {
+  position: absolute;
+  bottom: 0;
+  width: 100%;
+}
+.timeline .interval .color.clickable {
+  cursor: pointer;
+}
+.timeline div.interval:hover {
+  background: #9C4274;
+}
+.timeline .interval:hover .color {
+  background: #F04DA7;
+}
+.timeline .domComplete.interval {
+  background: #ede3ff;
+}
+.timeline .domComplete .color {
+  background: #c2a3ff;
+}
+.timeline .domContentLoadedEnd.interval {
+  background: #d8f0f0;
+}
+.timeline .domContentLoadedEnd .color {
+  background: #7ecccc;
+}
+.timeline .domContentLoaded.interval {
+  background: #e0ffd1;
+}
+.timeline .domContentLoaded .color {
+  background: #a7e846;
+}
+.timeline .domInteractive.interval {
+  background: #fffccc;
+}
+.timeline .domInteractive .color {
+  background: #ffe433;
+}
+.timeline .domCreation.interval {
+  background: #ffe0cc;
+}
+.timeline .domCreation .color {
+  background: #ff6600;
+}
+.timeline .tooltip.detailsOverlay {
+  position: absolute;
+  display: none;
+  width: auto;
+  padding: 0.5em 1em;
+  top: -1.5em;
+  right: 1em;
+}
+.timeline .interval:hover .tooltip {
+  display: block;
+}
+.timeline .legend {
+  display: table;
+  width: 100%;
+  margin-top: 1em;
+}
+.timeline .legend > div {
+  display: table-row;
+}
+.timeline .legend > div > div {
+  position: relative;
+  display: table-cell;
+  margin-top: 1em;
+}
+.timeline .titles {
+  font-weight: bold;
+}
+.timeline .titles > div {
+  padding: 0 1em 0 2em;
+}
+.timeline .tips {
+  font-size: 0.7em;
+}
+.timeline .tips > div {
+  padding: 1em 1em 0 0;
+}
+.timeline .legend .color {
+  display: block;
+  position: absolute;
+  left: 0;
+  height: 1.5em;
+  width: 1.5em;
+  border-radius: 0.2em;
+}
+.filters {
+  margin: 1em auto;
+  padding: 0.5em;
+  min-width: 30em;
+  width: 30%;
+  border: 1px dotted #aaa;
+  text-align: left;
+}
+.subFilters {
+  margin-left: 3em;
+  font-size: 0.9em;
+}
+.table {
+  display: table;
+  width: 100%;
+  border-spacing: 0.25em;
+}
+.table > div {
+  display: table-row;
+}
+.table > .headers > div {
+  font-weight: bold;
+  padding: 0.5em 1em;
+}
+.table > div > div {
+  padding: 0.1em 1em;
+  background: #f2f2f2;
+  display: table-cell;
+  text-align: left;
+}
+.table > div.jsError > .type,
+.table > div.jsError > .value {
+  color: #e74c3c;
+  font-weight: bold;
+}
+.table > div.windowPerformance > div,
+.table > div.windowPerformance > div.startTime {
+  background: #EBD8E2;
+}
+.table > div.showingDetails > div {
+  background: #f1c40f;
+}
+.table > div.highlight > div.startTime {
+  background-color: #C0F090;
+}
+.table > div.highlight-remove {
+  transition: 3s;
+}
+.table > div.highlight-remove > div.startTime {
+  transition: background-color 3s ease-in;
+}
+.table > div > .index {
+  color: #bbb;
+  word-break: normal;
+}
+.table > div > .type {
+  white-space: nowrap;
+}
+.table .children {
+  margin-top: 0.2em;
+  font-size: 0.8em;
+  line-height: 1.6em;
+}
+.table .child {
+  margin-left: 0.5em;
+}
+.table .child > .child {
+  margin-left: 1em;
+}
+.table .child:before {
+  content: "↳";
+}
+.table .child .childArgs {
+  display: none;
+}
+.table .child span {
+  position: relative;
+}
+.table .child span:hover {
+  background: #EBD8E2;
+}
+.table .child span:hover div {
+  display: inline-block;
+}
+.table .child span:hover .childArgs {
+  display: block;
+  position: absolute;
+  padding: 0 1em 0 2em;
+  left: 100%;
+  top: 0;
+  background: #EBD8E2;
+  line-height: 1.3em;
+  height: 1.3em;
+  z-index: 2;
+}
+.table .showingDetails .child span:hover {
+  background: inherit;
+}
+.table .showingDetails .child span:hover .childArgs {
+  display: none;
+}
+.table > div > .value {
+  width: 70%;
+  word-break: break-all;
+}
+.table > div > .details {
+  position: relative;
+}
+.table .details .icon-question {
+  color: #f1c40f;
+  cursor: pointer;
+}
+.table .icon-warning {
+  display: inline-block;
+  width: 0.8em;
+}
+.detailsOverlay {
+  display: none;
+  position: absolute;
+  right: 3em;
+  top: -3em;
+  width: 45em;
+  min-height: 1em;
+  padding: 0 1em 1em;
+  background: #fff;
+  border: 2px solid #f1c40f;
+  border-radius: 0.5em;
+  z-index: 2;
+}
+@media screen and (max-width: 1024px) {
+  .detailsOverlay {
+    width: 25em;
+  }
+}
+.showDetails .detailsOverlay {
+  display: block;
+}
+.detailsOverlay .closeBtn {
+  position: absolute;
+  top: 0.5em;
+  right: 0.5em;
+  color: #f1c40f;
+  cursor: pointer;
+}
+.detailsOverlay .advice {
+  color: #e74c3c;
+  font-weight: bold;
+}
+.detailsOverlay .trace {
+  word-break: break-all;
+}
+.table > div > .duration,
+.table > div > .startTime {
+  text-align: center;
+  white-space: nowrap;
+}
+.table > div > .startTime.domComplete {
+  background: #ede3ff;
+}
+.table > div > .startTime.domContentLoadedEnd {
+  background: #d8f0f0;
+}
+.table > div > .startTime.domContentLoaded {
+  background: #e0ffd1;
+}
+.table > div > .startTime.domInteractive {
+  background: #fffccc;
+}
+.table > div > .startTime.domCreation {
+  background: #ffe0cc;
+}
+.execution .icon-warning {
+  color: #e74c3c;
+  cursor: pointer;
+}
+.queryWithoutResultsFilterOn > div {
+  display: none;
+}
+.queryWithoutResultsFilterOn > div.queryWithoutResults {
+  display: table-row;
+}
+.jQueryCallOnEmptyObjectFilterOn > div {
+  display: none;
+}
+.jQueryCallOnEmptyObjectFilterOn > div.jQueryCallOnEmptyObject {
+  display: table-row;
+}
+.eventNotDelegatedFilterOn > div {
+  display: none;
+}
+.eventNotDelegatedFilterOn > div.eventNotDelegated {
+  display: table-row;
+}
+.jsErrorFilterOn > div {
+  display: none;
+}
+.jsErrorFilterOn > div.jsError {
+  display: table-row;
+}
+.testedUrl {
+  color: inherit;
+}
+.summary {
+  text-align: center;
+}
+.summary .globalScore {
+  display: table;
+  width: 60%;
+  margin: 3em auto;
+}
+.summary .globalScore > div {
+  display: table-cell;
+  width: 50%;
+  vertical-align: middle;
+}
+.summary .globalScore .globalGrade {
+  margin: 0.5 auto;
+  width: 2.5em;
+  height: 2.5em;
+  line-height: 2.5em;
+  border-radius: 0.5em;
+  font-size: 3em;
+  font-weight: bold;
+  vertical-align: middle;
+}
+.summary .globalScore .on100 {
+  font-size: 1.2em;
+  font-weight: bold;
+  margin: 0.5em 0 1em;
+}
+.summary .globalScore .screenshotWrapper:hover {
+  opacity: 0.75;
+}
+.summary .globalScore .screenshotWrapper:hover:after {
+  position: absolute;
+  width: 1.25em;
+  height: 1.25em;
+  top: 0.7em;
+  left: 1.55em;
+  font-size: 3em;
+  color: #FFF;
+  background: #000;
+  border-radius: 0.2em;
+  text-align: center;
+  content: "+";
+  opacity: 0.85;
+}
+.summary .globalScore .screenshotWrapper.phone:hover:after {
+  top: 1.7em;
+  left: 0.64em;
+}
+.summary .globalScore .screenshotWrapper.tablet:hover:after {
+  top: 1.5em;
+  left: 0.9em;
+}
+.summary .notations {
+  display: table;
+  width: 80%;
+  margin: 0 10% 1.5em;
+  border-spacing: 1em;
+}
+.summary .notations > div {
+  display: table-row;
+}
+.summary .notations > div > div {
+  display: table-cell;
+  height: 2.5em;
+  vertical-align: middle;
+}
+.summary .notations .category {
+  font-weight: bold;
+  text-align: center;
+  width: 20%;
+}
+.summary .notations .criteria {
+  font-weight: normal;
+  width: 75%;
+}
+.summary .notations .A.categoryScore,
+.summary .notations .B.categoryScore,
+.summary .notations .C.categoryScore,
+.summary .notations .D.categoryScore,
+.summary .notations .E.categoryScore,
+.summary .notations .F.categoryScore,
+.summary .notations .NA.categoryScore {
+  width: 2.5em;
+  max-width: 2.5em;
+  min-width: 2.5em;
+  font-size: 2em;
+  text-align: center;
+  border-radius: 0.5em;
+  font-weight: bold;
+}
+.summary .notations .grade .A,
+.summary .notations .grade .B,
+.summary .notations .grade .C,
+.summary .notations .grade .D,
+.summary .notations .grade .E,
+.summary .notations .grade .F,
+.summary .notations .grade .NA {
+  width: 1em;
+  height: 1em;
+  font-size: 1em;
+  color: transparent;
+  margin: 0 auto;
+  border-radius: 0.5em;
+}
+.summary .notations .criteria .table {
+  width: 100%;
+}
+.summary .notations .criteria .table > div:hover > div {
+  background: #EBD8E2;
+  cursor: pointer;
+}
+.summary .notations .criteria .table > div:hover > div.info {
+  background: #FFF;
+}
+.summary .notations .criteria .table > div:hover > div.info .icon-question {
+  color: #EBD8E2;
+}
+.summary .notations .criteria .grade {
+  width: 10%;
+  padding-left: 0.5em;
+  padding-right: 0.5em;
+  vertical-align: middle;
+}
+.summary .notations .criteria .label {
+  width: 70%;
+}
+.summary .notations .criteria .result {
+  width: 18%;
+  font-weight: bold;
+  white-space: nowrap;
+  text-align: center;
+  vertical-align: middle;
+}
+.summary .notations .warning .label,
+.summary .notations .warning .result,
+.summary .notations .icon-warning {
+  color: #FF1919;
+}
+.summary .notations .criteria .info {
+  width: 2%;
+  text-align: center;
+  vertical-align: middle;
+  background: #FFF;
+  padding-left: 0.1em;
+  padding-right: 0.1em;
+}
+.summary .notations .criteria .icon-question {
+  color: transparent;
+}
+.summary .fromShare {
+  margin-bottom: 3em;
+}
+.summary .apiTip {
+  font-size: 0.8em;
+  margin-bottom: 4em;
+  color: #413;
+}
+.summary .apiTip a {
+  color: inherit;
+}
+.summary .tweet .tweetText {
+  color: #413;
+  background: #F2F2F2;
+  border: none;
+  width: 25em;
+  padding: 0.4em;
+  border-radius: 0.5em;
+  box-shadow: 0.05em 0.1em 0 0 #999;
+}
+.summary .tweet .tweetButton,
+.summary .tweet .linkedinButton {
+  color: #413;
+  background: #F2F2F2;
+  margin-right: 0;
+}
+.summary .tweet .tweetButton:hover,
+.summary .tweet .linkedinButton:hover {
+  color: #F2F2F2;
+  background: #e74c3c;
+}
+.summary .tweet input {
+  font-size: 0.9em;
+}
+
+
+.promess {
+  padding: 0em 2em;
+  margin-bottom: 0.5em;
+  font-weight: normal;
+  font-size: 1.2em;
+}
+.price {
+  padding: 0em 2em 3em;
+  margin-top: 0em;
+  font-size: 0.9em;
+}
+.url {
+  width: 50%;
+}
+.launchBtn {
+  background: #ffa319;
+  color: #fff;
+}
+.launchBtn:focus {
+  background: #e74c3c;
+}
+.launchBtn.disabled {
+  background: #deaca6;
+}
+.launchBtn.disabled:focus {
+  color: #ddd;
+}
+.settings {
+  width: 50%;
+  margin: 0 auto;
+}
+.settings input,
+.settings select {
+  font-size: 1em;
+}
+.settings input[type=text] {
+  width: 100%;
+  min-width: 4em;
+}
+.device {
+  margin-top: 3em;
+}
+.device .item {
+  display: inline-block;
+  margin: 1em 0.75em;
+  width: 5.5em;
+  height: 5.5em;
+  color: #FFF;
+  border: 1px solid #FFF;
+  padding: 1px;
+  border-radius: 0.5em;
+  cursor: pointer;
+  text-decoration: none;
+  font-size: 0.8em;
+}
+                                                                                       
+.device .item.active {                             
+  color: #ffa319;                             
+  border: 2px solid #ffa319;                             
+  padding: 0;                             
+}                             
+.device .item:hover {                             
+  color: #ffa319;                             
+}                             
+.device .item > div {                             
+  margin: 0.2em 0 0.1em;                             
+  font-size: 3em;                             
+}                             
+.settingsTooltip {
+  position: relative;
+}
+.settingsTooltip span {
+  font-size: 0.8em;
+  vertical-align: text-top;
+}
+.settingsTooltip div {
+  display: none;
+  position: absolute;
+  padding: 0.5em;
+  width: 25em;
+  background: #FFF;
+  color: #000;
+  font-size: 0.8em;
+  border-radius: 1em;
+  border: 2px solid #ffa319;
+  white-space: normal;
+  z-index: 2;
+}
+.settingsTooltip:hover div {
+  display: block;
+}
+.showAdvanced {
+  display: inline-block;
+  margin-top: 2em;
+  color: #FFF;
+  text-decoration: none;
+  font-size: 0.9em;
+}
+.showAdvanced:hover {
+  color: #ffa319;
+}
+.currentSettings {
+  font-size: 0.9em;
+}
+.currentSettings span {
+  color: #ffa319;
+}
+.currentSettings span:after {
+  color: #FFF;
+  content: ",";
+}
+.currentSettings span:last-child:after {
+  content: "";
+}
+.advanced {
+  margin: 1em 0 0;
+  display: table;
+  width: 100%;
+  text-align: left;
+  border-spacing: 0.75em;
+}
+.advanced > div {
+  display: table-row;
+}
+.advanced > div > div {
+  display: table-cell;
+  width: 75%;
+}
+.advanced > div > div.label {
+  width: 25%;
+  white-space: nowrap;
+}
+.advanced .subTable {
+  display: table;
+  border-spacing: 0;
+  width: 100%;
+}
+.advanced .subTable > div {
+  display: table-row;
+}
+.advanced .subTable > div > div {
+  display: table-cell;
+  padding: 0 0 0.75em;
+}
+.features {
+  display: table;
+  width: 50%;
+  margin: 6em auto 0;
+  font-size: 0.9em;
+  color: #413;
+}
+.features > div {
+  width: 33.3%;
+  display: table-cell;
+  padding: 0 1.5em;
+}
+input[type=submit],
+input.url {
+  padding: 0 0.5em;
+  margin: 0.5em;
+  font-size: 1.2em;
+  height: 2em;
+  border: 0 solid;
+  border-radius: 0.5em;
+  box-shadow: 0.1em 0.2em 0 0 #5e2846;
+  outline: none;
+}
+input[type=submit]:hover {
+  color: #ddd;
+}
+input[type=submit].clicked {
+  color: #ddd;
+  position: relative;
+  left: 0.1em;
+  top: 0.2em;
+  box-shadow: none;
+}
+/* 
+ *  Core Owl Carousel CSS File
+ *  v1.3.2
+ */
+
+/* clearfix */
+.owl-carousel .owl-wrapper:after {
+  content: ".";
+  display: block;
+  clear: both;
+  visibility: hidden;
+  line-height: 0;
+  height: 0;
+}
+/* display none until init */
+.owl-carousel{
+  display: none;
+  position: relative;
+  width: 100%;
+  -ms-touch-action: pan-y;
+}
+.owl-carousel .owl-wrapper{
+  display: none;
+  position: relative;
+  -webkit-transform: translate3d(0px, 0px, 0px);
+}
+.owl-carousel .owl-wrapper-outer{
+  overflow: hidden;
+  position: relative;
+  width: 100%;
+}
+.owl-carousel .owl-wrapper-outer.autoHeight{
+  -webkit-transition: height 500ms ease-in-out;
+  -moz-transition: height 500ms ease-in-out;
+  -ms-transition: height 500ms ease-in-out;
+  -o-transition: height 500ms ease-in-out;
+  transition: height 500ms ease-in-out;
+}
+  
+.owl-carousel .owl-item{
+  float: left;
+}
+.owl-controls .owl-page,
+.owl-controls .owl-buttons div{
+  cursor: pointer;
+}
+.owl-controls {
+  -webkit-user-select: none;
+  -khtml-user-select: none;
+  -moz-user-select: none;
+  -ms-user-select: none;
+  user-select: none;
+  -webkit-tap-highlight-color: rgba(0, 0, 0, 0);
+}
+
+/* mouse grab icon */
+.grabbing { 
+    cursor:url(grabbing.png) 8 8, move;
+}
+
+/* fix */
+.owl-carousel  .owl-wrapper,
+.owl-carousel  .owl-item{
+  -webkit-backface-visibility: hidden;
+  -moz-backface-visibility:    hidden;
+  -ms-backface-visibility:     hidden;
+  -webkit-transform: translate3d(0,0,0);
+  -moz-transform: translate3d(0,0,0);
+  -ms-transform: translate3d(0,0,0);
+}

test/www/xml.xml

@@ -0,0 +1,80 @@
+<?xml version="1.0"?>
+<catalog>
+   <book id="bk101">
+      <author>Gambardella, Matthew</author>
+      <title>XML Developer's Guide</title>
+      <genre>Computer</genre>
+      <price>44.95</price>
+      <publish_date>2000-10-01</publish_date>
+      <description>An in-depth look at creating applications 
+      with XML.</description>
+   </book>
+   <book id="bk102">
+      <author>Ralls, Kim</author>
+      <title>Midnight Rain</title>
+      <genre>Fantasy</genre>
+      <price>5.95</price>
+      <publish_date>2000-12-16</publish_date>
+      <description>A former architect battles corporate zombies, 
+      an evil sorceress, and her own childhood to become queen 
+      of the world.</description>
+   </book>
+   <book id="bk103">
+      <author>Corets, Eva</author>
+      <title>Maeve Ascendant</title>
+      <genre>Fantasy</genre>
+      <price>5.95</price>
+      <publish_date>2000-11-17</publish_date>
+      <description>After the collapse of a nanotechnology 
+      society in England, the young survivors lay the 
+      foundation for a new society.</description>
+   </book>
+   <book id="bk104">
+      <author>Corets, Eva</author>
+      <title>Oberon's Legacy</title>
+      <genre>Fantasy</genre>
+      <price>5.95</price>
+      <publish_date>2001-03-10</publish_date>
+      <description>In post-apocalypse England, the mysterious 
+      agent known only as Oberon helps to create a new life 
+      for the inhabitants of London. Sequel to Maeve 
+      Ascendant.</description>
+   </book>
+   <book id="bk105">
+      <author>Corets, Eva</author>
+      <title>The Sundered Grail</title>
+      <genre>Fantasy</genre>
+      <price>5.95</price>
+      <publish_date>2001-09-10</publish_date>
+      <description>The two daughters of Maeve, half-sisters, 
+      battle one another for control of England. Sequel to 
+      Oberon's Legacy.</description>
+   </book>
+   <book id="bk106">
+      <author>Randall, Cynthia</author>
+      <title>Lover Birds</title>
+      <genre>Romance</genre>
+      <price>4.95</price>
+      <publish_date>2000-09-02</publish_date>
+      <description>When Carla meets Paul at an ornithology 
+      conference, tempers fly as feathers get ruffled.</description>
+   </book>
+   <book id="bk107">
+      <author>Thurman, Paula</author>
+      <title>Splish Splash</title>
+      <genre>Romance</genre>
+      <price>4.95</price>
+      <publish_date>2000-11-02</publish_date>
+      <description>A deep sea diver finds true love twenty 
+      thousand leagues beneath the sea.</description>
+   </book>
+   <book id="bk108">
+      <author>Knorr, Stefan</author>
+      <title>Creepy Crawlies</title>
+      <genre>Horror</genre>
+      <price>4.95</price>
+      <publish_date>2000-12-06</publish_date>
+      <description>An anthology of horror stories about roaches,
+      centipedes, scorpions  and other insects.</description>
+   </book>
+</catalog>