Merge pull request #183 from gmetais/develop

v1.11.0

.bowerrc

@@ -1,3 +0,0 @@
-{
-  "directory" : "bower_components"
-}

.gitignore

@@ -1,5 +1,4 @@
 node_modules
-bower_components
 .tmp
 .vagrant
 results/*

.travis.yml

@@ -1,8 +1,18 @@
 language: node_js
+sudo: false
 node_js:
-    - "0.12.4"
+  - "6.2"
+  - "4.0"
+env:
+  - CXX=g++-4.8
+addons:
+  apt:
+    sources:
+      - ubuntu-toolchain-r-test
+    packages:
+      - g++-4.8
 before_install:
-    - "npm install -g npm"
-    - "npm install -g grunt-cli"
+  - "npm install -g npm"
+  - "npm install -g grunt-cli"
 install:
-    - "npm install"
+  - "npm install"

README.md

@@ -1,6 +1,6 @@
 <img src="./doc/img/logo.png" alt="Yellow Lab Tools"></img>
 
-Analyzes a webpage and detects **performance** or **front-end code quality** issues. Easy to launch, easy to understand, in-depth details.
+Analyzes a webpage and detects **performance** or **front-end code quality** issues. Free, easy to launch, easy to understand, in-depth details.
 
 
 #  
@@ -75,18 +75,15 @@ You can use [ngrok](https://ngrok.com/), a tool that creates a secure tunnel bet
 If your project is not accessible from outside, or if you want to fork and improve the tool, you can build your own instance. The documentation is [here](https://github.com/gmetais/YellowLabTools/wiki/Install-your-private-server).
 
 
-## Help needed!
-There are so many things left to do, **your help would be greatly appreciated**! Please report bugs, ask for evolutions and come code with me.
-
-
-## License
-Please read the [license](LICENSE). Icons are the property of [IcoMoon.io](https://icomoon.io/)
-
-
 ## Author
-Gaël Métais. I'm a webperf freelance based in Paris. If you understand French, you can visit [my website](http://www.gaelmetais.com).
+Gaël Métais. I'm a webperf freelance. Follow me on Twitter [@gaelmetais](https://twitter.com/gaelmetais), I tweet about Web Performances, Front-end and new versions of YellowLabTools!
+
+If you understand French, you can visit [my website](http://www.gaelmetais.com) (will be soon in English too).
 
 
 ## Contributors
 - Achraf Ben Younes [achrafbenyounes](https://github.com/achrafbenyounes)
+- [camlafit](https://github.com/camlafit)
+- Vincent L. [magikcypress](https://github.com/magikcypress)
+- Ousama Ben Younes [ousamabenyounes](https://github.com/ousamabenyounes)
 

Vagrantfile

@@ -1,17 +1,22 @@
 Vagrant.configure("2") do |config|
 
-  config.vm.box = "precise64"
-  config.vm.box_url = "http://files.vagrantup.com/precise64.box"
+  config.vm.box = "ubuntu/precise64"
 
   config.vm.network :private_network, ip: "10.10.10.10"
   config.ssh.forward_agent = true
 
+  # http://foo-o-rama.com/vagrant--stdin-is-not-a-tty--fix.html
+  config.vm.provision "fix-no-tty", type: "shell" do |s|
+    s.privileged = false
+    s.inline = "sudo sed -i '/tty/!s/mesg n/tty -s \\&\\& mesg n/' /root/.profile"
+  end
+
   config.vm.provider :virtualbox do |vb|
     vb.customize ["modifyvm", :id, "--natdnshostresolver1", "on"]
-    vb.customize ["modifyvm", :id, "--memory", 512]
+    vb.customize ["modifyvm", :id, "--memory", 1024]
     vb.customize ["modifyvm", :id, "--cpus", 2]
   end
 
   config.vm.provision :shell, :path => "server_config/server_install.sh"
 
-end
+end

bin/cli.js

@@ -1,10 +1,11 @@
 #!/usr/bin/env node
 
-var debug = require('debug')('ylt:cli');
-var meow = require('meow');
-var path = require('path');
+var debug       = require('debug')('ylt:cli');
+var meow        = require('meow');
+var path        = require('path');
+var EasyXml     = require('easyxml');
 
-var ylt = require('../lib/index');
+var ylt         = require('../lib/index');
 
 var cli = meow({
     help: [
@@ -14,11 +15,11 @@ var cli = meow({
         'Options:',
         '  --device             Use "phone" or "tablet" to simulate a mobile device (by user-agent and viewport size).',
         '  --screenshot         Will take a screenshot and use this value as the output path. It needs to end with ".png".',
-        '  --js-deep-analysis   When activated, the javascriptExecutionTree will contain sub-requests.',
         '  --wait-for-selector  Once the page is loaded, Phantomas will wait until the given CSS selector matches some elements.',
         '  --cookie             Adds a cookie on the main domain.',
         '  --auth-user          Basic HTTP authentication username.',
         '  --auth-pass          Basic HTTP authentication password.',
+        '  --reporter           The output format: "json" or "xml". Default is "json".',
         ''
     ].join('\n'),
     pkg: '../package.json'
@@ -49,11 +50,6 @@ if (screenshot) {
     options.screenshot = cli.flags.screenshot;
 }
 
-// Deep JS analysis option
-if (cli.flags.jsDeepAnalysis === true || cli.flags.jsDeepAnalysis === 'true') {
-    options.jsDeepAnalysis = true;
-}
-
 // Device simulation
 options.device = cli.flags.device || 'desktop';
 
@@ -67,6 +63,12 @@ options.cookie = cli.flags.cookie || null;
 options.authUser = cli.flags.authUser || null;
 options.authPass = cli.flags.authPass || null;
 
+// Output format
+if (cli.flags.reporter && cli.flags.reporter !== 'json' && cli.flags.reporter !== 'xml') {
+    console.error('Incorrect parameters: reporter has to be "json" or "xml"');
+    process.exit(1);
+}
+
 
 (function execute(url, options) {
     'use strict';
@@ -76,7 +78,14 @@ options.authPass = cli.flags.authPass || null;
         then(function(data) {
 
             debug('Success');
-            console.log(JSON.stringify(data, null, 2));
+            switch(cli.flags.reporter) {
+                case 'xml':
+                    var serializer = new EasyXml();
+                    console.log(serializer.render(data));
+                    break;
+                default:
+                    console.log(JSON.stringify(data, null, 2));
+            }
 
         }).fail(function(err) {
             

bin/server.js

@@ -17,6 +17,7 @@ app.use(cors());
 app.use(wwwRedirectMiddleware);
 app.use(authMiddleware);
 app.use(apiLimitsMiddleware);
+app.use(bodyParser.urlencoded({ extended: true })); // for parsing application/x-www-form-urlencoded
 
 
 // Initialize the controllers
@@ -37,4 +38,4 @@ if (!process.env.GRUNTED) {
     });
 }
 
-module.exports = app;
+module.exports = app;

bower.json

@@ -1,13 +0,0 @@
-{
-  "name": "yellowlabtools",
-  "dependencies": {
-    "angular": "1.3.15",
-    "angular-route": "1.3.15",
-    "angular-resource": "1.3.15",
-    "angular-sanitize": "1.3.15",
-    "angular-animate": "1.3.15",
-    "angular-local-storage": "0.2.2",
-    "angular-chart.js": "0.7.1",
-    "Chart.js": "~1.0.2"
-  }
-}

front/src/css/dashboard.css

@@ -109,14 +109,18 @@
 .summary .notations .criteria .table {
   width: 100%;
 }
-.summary .notations .criteria .table > div:hover > div {
+.summary .notations .criteria .table > a {
+  text-decoration: none;
+  color: inherit;
+}
+.summary .notations .criteria .table > a:hover > div {
   background: #EBD8E2;
   cursor: pointer;
 }
-.summary .notations .criteria .table > div:hover > div.info {
+.summary .notations .criteria .table > a:hover > div.info {
   background: #FFF;
 }
-.summary .notations .criteria .table > div:hover > div.info .icon-question {
+.summary .notations .criteria .table > a:hover > div.info .icon-question {
   color: #EBD8E2;
 }
 .summary .notations .criteria .grade {

front/src/css/index.css

@@ -33,7 +33,9 @@
 .settings select {
   font-size: 1em;
 }
-.settings input[type=text] {
+.settings input[type=text],
+.settings input[type=password],
+.settings textarea {
   width: 100%;
   min-width: 4em;
 }
@@ -128,6 +130,7 @@
 .advanced > div > div.label {
   width: 25%;
   white-space: nowrap;
+  vertical-align: middle;
 }
 .advanced .subTable {
   display: table;

front/src/css/rule.css

@@ -29,9 +29,16 @@
   border-radius: 0.5em;
   margin: 0 auto 0.5em;
 }
+.rule h3 {
+  margin-bottom: 0em;
+}
+.rule .okThreshold {
+  font-style: italic;
+  font-size: 0.9em;
+}
 .rule .message {
   width: 80%;
-  margin: 0 auto;
+  margin: 1.5em auto;
 }
 .rule .message p {
   margin: 0.5em;
@@ -94,6 +101,7 @@
   margin: 0.2em 0;
   border-radius: 0.4em;
   z-index: 1;
+  cursor: pointer;
 }
 .offendersTable .offenderButton.opens,
 .value .offenderButton.opens {
@@ -150,6 +158,10 @@
   display: block;
   background: #ffe0cc;
 }
+.offendersTable .smallerOffenders,
+.value .smallerOffenders {
+  font-size: 0.9em;
+}
 .offendersHtml {
   display: inline-block;
 }
@@ -167,7 +179,7 @@
 .checker {
   /* Checkerboard background */
   background-color: #ddd;
-  background-image: linear-gradient(45deg, #aaaaaa 25%, transparent 25%, transparent 75%, #aaaaaa 75%, #aaaaaa), linear-gradient(45deg, #aaaaaa 25%, transparent 25%, transparent 75%, #aaaaaa 75%, #aaaaaa);
+  background-image: linear-gradient(45deg, #AAA 25%, transparent 25%, transparent 75%, #AAA 75%, #AAA), linear-gradient(45deg, #AAA 25%, transparent 25%, transparent 75%, #AAA 75%, #AAA);
   background-size: 1em 1em;
   background-position: 0 0, 0.5em 0.5em;
 }
@@ -207,6 +219,17 @@
   z-index: 2;
   border: 0.2em solid #f1c40f;
 }
+.similarColors {
+  margin: 1em;
+  width: 20em;
+  height: 6em;
+}
+.similarColors > div {
+  display: inline-block;
+  width: 10em;
+  height: 3.5em;
+  padding-top: 2.5em;
+}
 .totalWeightPie {
   max-width: 39em;
   margin: 2em auto 4em;
@@ -237,8 +260,9 @@
   margin-top: 0.5em;
 }
 .smallPreview {
-  max-height: 1.6em;
-  max-width: 4em;
+  display: block;
+  max-height: 4em;
+  max-width: 8em;
   border: 1px solid #000;
-  margin-top: 0.2em;
+  margin: 1em auto 0.2em;
 }

front/src/css/timeline.css

@@ -62,34 +62,34 @@
   background: #F04DA7;
 }
 .timeline .domComplete.interval {
-  background: #ede3ff;
+  background: #EDE3FF;
 }
 .timeline .domComplete .color {
-  background: #c2a3ff;
+  background: #C2A3FF;
 }
 .timeline .domContentLoadedEnd.interval {
-  background: #d8f0f0;
+  background: #D8F0F0;
 }
 .timeline .domContentLoadedEnd .color {
-  background: #7ecccc;
+  background: #7ECCCC;
 }
 .timeline .domContentLoaded.interval {
-  background: #e0ffd1;
+  background: #E0FFD1;
 }
 .timeline .domContentLoaded .color {
-  background: #a7e846;
+  background: #A7E846;
 }
 .timeline .domInteractive.interval {
-  background: #fffccc;
+  background: #FFFCCC;
 }
 .timeline .domInteractive .color {
-  background: #ffe433;
+  background: #FFE433;
 }
 .timeline .domCreation.interval {
-  background: #ffe0cc;
+  background: #FFE0CC;
 }
 .timeline .domCreation .color {
-  background: #ff6600;
+  background: #FF6600;
 }
 .timeline .tooltip.detailsOverlay {
   position: absolute;
@@ -152,14 +152,16 @@
   width: 100%;
   border-spacing: 0.25em;
 }
-.table > div {
+.table > div,
+.table > a {
   display: table-row;
 }
 .table > .headers > div {
   font-weight: bold;
   padding: 0.5em 1em;
 }
-.table > div > div {
+.table > div > div,
+.table > a > div {
   padding: 0.1em 1em;
   background: #f2f2f2;
   display: table-cell;
@@ -292,19 +294,19 @@
   white-space: nowrap;
 }
 .table > div > .startTime.domComplete {
-  background: #ede3ff;
+  background: #EDE3FF;
 }
 .table > div > .startTime.domContentLoadedEnd {
-  background: #d8f0f0;
+  background: #D8F0F0;
 }
 .table > div > .startTime.domContentLoaded {
-  background: #e0ffd1;
+  background: #E0FFD1;
 }
 .table > div > .startTime.domInteractive {
-  background: #fffccc;
+  background: #FFFCCC;
 }
 .table > div > .startTime.domCreation {
-  background: #ffe0cc;
+  background: #FFE0CC;
 }
 .execution .icon-warning {
   color: #e74c3c;

front/src/js/app.js

@@ -37,6 +37,11 @@ yltApp.run(['$rootScope', '$location', function($rootScope, $location) {
     $rootScope.$on('$routeChangeSuccess', function(){
         ga('send', 'pageview', {'page': $location.path()});
     });
+
+    // GitHub star button (asynchronously loaded iframe)
+    window.addEventListener('load', function() {
+        window.document.getElementById('ghbtn').src = 'http://ghbtns.com/github-btn.html?user=gmetais&repo=YellowLabTools&type=star&count=true&size=large';
+    });
 }]);
 
 yltApp.config(['$routeProvider', '$locationProvider',
@@ -77,3 +82,7 @@ yltApp.config(['$routeProvider', '$locationProvider',
     }
 ]);
 
+// Disable debugging https://docs.angularjs.org/guide/production
+yltApp.config(['$compileProvider', function ($compileProvider) {
+    $compileProvider.debugInfoEnabled(false);
+}]);

front/src/js/controllers/dashboardCtrl.js

@@ -30,10 +30,6 @@ dashboardCtrl.controller('DashboardCtrl', ['$scope', '$rootScope', '$routeParams
         $scope.tweetText = 'I\'ve discovered this cool open-source tool that audits the front-end quality of a web page: ';
     }
 
-    $scope.showRulePage = function(ruleName) {
-        $location.path('/result/' + $scope.runId + '/rule/' + ruleName);
-    };
-
     $scope.testAgain = function() {
         API.relaunchTest($scope.result);
     };

front/src/js/controllers/ruleCtrl.js

@@ -48,6 +48,55 @@ ruleCtrl.controller('RuleCtrl', ['$scope', '$rootScope', '$routeParams', '$locat
                 tooltipTemplate: '<%=label%>: <%=value%> KB'
             };
         }
+
+        // Init "Breakpoints" chart
+        if ($scope.policyName === 'cssBreakpoints' && $scope.rule.value > 0) {
+
+            // Seek for the biggest breakpoint
+            var max = 0;
+            $scope.rule.offendersObj.forEach(function(offender) {
+                if (offender.pixels > max) {
+                    max = offender.pixels;
+                }
+            });
+            max = Math.max(max + 100, 1400);
+
+            // We group offenders 10px by 10px
+            var GROUP_SIZE = 20;
+
+            // Generate an empty array of values
+            $scope.breakpointsLabels = [];
+            $scope.breakpointsData = [[]];
+            for (var i = 0; i <= max / GROUP_SIZE; i++) {
+                $scope.breakpointsLabels[i] = '';
+                $scope.breakpointsData[0][i] = 0;
+            }
+
+            // Fill it with results
+            $scope.rule.offendersObj.forEach(function(offender) {
+                var group = Math.floor((offender.pixels + 1) / GROUP_SIZE);
+
+                if ($scope.breakpointsLabels[group] !== '') {
+                    $scope.breakpointsLabels[group] += '/';
+                }
+                $scope.breakpointsLabels[group] += offender.breakpoint;
+
+                $scope.breakpointsData[0][group] += offender.count;
+            });
+
+            $scope.breakpointsSeries = ['Number of CSS rules per breakpoint'];
+            $scope.breakpointsColours = ['#9c4274'];
+            $scope.breakpointsOptions = {
+                scaleShowGridLines: false,
+                barShowStroke: false,
+                showTooltips: false,
+                pointDot: false,
+                responsive: true,
+                maintainAspectRatio: true,
+                strokeColor: 'rgba(20, 200, 20, 1)',
+                scaleFontSize: 9
+            };
+        }
     }
 
     $scope.backToDashboard = function() {

front/src/js/directives/offendersDirectives.js

@@ -608,7 +608,12 @@
             for (var i = 0 ; i < parsedBacktrace.length ; i++) {
                 html += '<div>';
                 html += '<div>' + (parsedBacktrace[i].fnName || '(anonymous function)') + '</div>';
-                html += '<div class="trace">' + getUrlLink(parsedBacktrace[i].filePath, 40) + ':' + parsedBacktrace[i].line + '</div>';
+                html += '<div class="trace">' + getUrlLink(parsedBacktrace[i].filePath, 40) + '</div>';
+                if (parsedBacktrace[i].column) {
+                    html += '<div>' + parsedBacktrace[i].line + ':' + parsedBacktrace[i].column + '</div>';    
+                } else {
+                    html += '<div>line ' + parsedBacktrace[i].line + '</div>';
+                }
                 html += '</div>';
             }
         }
@@ -630,6 +635,22 @@
                 var fnName = null, fileAndLine;
 
                 var withFnResult = /^([^\s\(]+) \((.+:\d+)\)$/.exec(trace);
+                
+                if (withFnResult === null) {
+                    // Try the PhantomJS 2 format
+                    withFnResult = /^([^\s\(]+) \((.+:\d+:\d+)\)$/.exec(trace);
+                }
+
+                if (withFnResult === null) {
+                    // Yet another PhantomJS 2 format?
+                    withFnResult = /^([^\s\(]+|global code)@(.+:\d+:\d+)$/.exec(trace);
+                }
+
+                if (withFnResult === null) {
+                    // Try the PhantomJS 2 ERROR format
+                    withFnResult = /^([^\s\(]+) (http.+:\d+)$/.exec(trace);
+                }
+
                 if (withFnResult === null) {
                     fileAndLine = trace;
                 } else {
@@ -637,15 +658,26 @@
                     fileAndLine = withFnResult[2];
                 }
 
-                var fileAndLineSplit = /^(.*):(\d+)$/.exec(fileAndLine);
+                // And now the second part
+                var fileAndLineSplit = /^(.*):(\d+):(\d+)$/.exec(fileAndLine);
+
+                if (fileAndLineSplit === null) {
+                    fileAndLineSplit = /^(.*):(\d+)$/.exec(fileAndLine);
+                }
+
                 var filePath = fileAndLineSplit[1];
                 var line = fileAndLineSplit[2];
+                var column = fileAndLineSplit[3];
 
-                out.push({
-                    fnName: fnName,
-                    filePath: filePath,
-                    line: line
-                });
+                // Filter phantomas code
+                if (filePath.indexOf('phantomjs://') === -1) {
+                    out.push({
+                        fnName: fnName,
+                        filePath: filePath,
+                        line: line,
+                        column: column
+                    });
+                }
             });
 
         } catch(e) {

front/src/js/services/apiService.js

@@ -9,25 +9,26 @@ apiService.factory('API', ['$location', 'Runs', 'Results', function($location, R
                 url: url,
                 waitForResponse: false,
                 screenshot: true,
-                jsTimeline: true,
                 device: settings.device,
                 waitForSelector: settings.waitForSelector,
                 cookie: settings.cookie,
                 authUser: settings.authUser,
                 authPass: settings.authPass,
+                blockDomain: settings.blockDomain,
+                allowedDomains: settings.allowedDomains,
+                noExternals: settings.noExternals
             };
 
-            if (settings.waitForSelector && settings.waitForSelector !== '') {
-                runObject.waitForSelector = settings.waitForSelector;
-            }
-
-            if (settings.cookie && settings.cookie !== '') {
-                runObject.cookie = settings.cookie;
-            }
-
-            if (settings.authUser && settings.authUser !== '' && settings.authPass && settings.authPass !== '') {
-                runObject.authUser = settings.authUser;
-                runObject.authPass = settings.authPass;
+            
+            if (settings.domainsBlackOrWhite === 'black') {
+                runObject.blockDomain = this.parseDomains(settings.domains);
+            } else if (settings.domainsBlackOrWhite === 'white') {
+                var allowedDomains = this.parseDomains(settings.domains);
+                if (allowedDomains.length > 0) {
+                    runObject.allowDomain = allowedDomains;
+                } else {
+                    runObject.noExternals = true;
+                }
             }
 
             Runs.save(runObject, function(data) {
@@ -43,6 +44,17 @@ apiService.factory('API', ['$location', 'Runs', 'Results', function($location, R
 
         relaunchTest: function(result) {
             this.launchTest(result.params.url, result.params.options);
+        },
+
+        parseDomains: function(textareaContent) {
+            var lines = textareaContent.split('\n');
+            
+            function removeEmptyLines (line) {
+                return line.trim() !== '';
+            }
+
+            // Remove empty lines
+            return lines.filter(removeEmptyLines).join(',');
         }
     };
 

front/src/less/dashboard.less

@@ -108,7 +108,11 @@
 
 .summary .notations .criteria .table {
     width: 100%;
-    > div:hover > div {
+    > a {
+        text-decoration: none;
+        color: inherit;
+    }
+    > a:hover > div {
         background: #EBD8E2;
         cursor: pointer;
         &.info {

front/src/less/index.less

@@ -38,7 +38,7 @@
         font-size: 1em;
     }
 
-    input[type=text] {
+    input[type=text], input[type=password], textarea {
         width: 100%;
         min-width: 4em;
     }
@@ -147,6 +147,7 @@
             &.label {
                 width: 25%;
                 white-space: nowrap;
+                vertical-align: middle;
             }
         }
     }

front/src/less/main.less

@@ -132,8 +132,12 @@ a.linkButton {
     background: #000;
 
     > div {
-        overflow: scroll;
         position: relative;
+        overflow: hidden;
+
+        &:hover {
+            overflow: auto;
+        }
     }
 
     .screenshotImage {
@@ -234,14 +238,6 @@ a.linkButton {
     }
 }
 
-
-.star {
-    font-weight: bold;
-    span {
-        font-size: 1.2em;
-    }
-}
-
 .footer {
     padding: 3em;
     color: #413;
@@ -251,6 +247,9 @@ a.linkButton {
     .version {
         font-size: 0.7em;
     }
+    .github {
+        margin: 1em 0 0 0.5em;
+    }
 }
 
 [class^="icon-"]:before, [class*=" icon-"]:before {

front/src/less/rule.less

@@ -32,9 +32,18 @@
     margin: 0 auto 0.5em;
 }
 
+.rule h3 {
+    margin-bottom: 0em;
+}
+
+.rule .okThreshold {
+    font-style: italic;
+    font-size: 0.9em;
+}
+
 .rule .message {
     width: 80%;
-    margin: 0 auto;
+    margin: 1.5em auto;
     p {
         margin: 0.5em;
     }
@@ -103,6 +112,7 @@
         margin: 0.2em 0;
         border-radius: 0.4em;
         z-index: 1;
+        cursor: pointer;
 
         &.opens {
             padding-right: 0.75em;
@@ -157,6 +167,10 @@
             }
         }
     }
+
+    .smallerOffenders {
+        font-size: 0.9em;
+    }
 }
 
 .offendersHtml {
@@ -225,6 +239,19 @@
     }
 }
 
+.similarColors {
+    margin: 1em;
+    width: 20em;
+    height: 6em;
+
+    > div {
+        display: inline-block;
+        width: 10em;
+        height: 3.5em;
+        padding-top: 2.5em;
+    }
+}
+
 .totalWeightPie {
     max-width: 39em;
     margin: 2em auto 4em;
@@ -262,8 +289,9 @@
 }
 
 .smallPreview {
-    max-height: 1.6em;
-    max-width: 4em;
+    display: block;
+    max-height: 4em;
+    max-width: 8em;
     border: 1px solid #000;
-    margin-top: 0.2em;
+    margin: 1em auto 0.2em;
 }

front/src/less/timeline.less

@@ -173,7 +173,8 @@
     border-spacing: 0.25em;
 }
 
-.table > div {
+.table > div,
+.table > a {
     display: table-row;
 }
 
@@ -182,7 +183,8 @@
     padding: 0.5em 1em;
 }
 
-.table > div > div {
+.table > div > div,
+.table > a > div {
     padding: 0.1em 1em;
     background: #f2f2f2;
     display: table-cell;

front/src/main.html

@@ -16,18 +16,36 @@
     <link rel="stylesheet" type="text/css" href="/css/screenshot.css">
     <link rel="stylesheet" type="text/css" href="/css/timeline.css">
     <link rel="stylesheet" type="text/css" href="/css/about.css">
-    <link rel="stylesheet" type="text/css" href="/bower_components/angular-chart.js/dist/angular-chart.css">
+    <link rel="stylesheet" type="text/css" href="/node_modules/angular-chart.js/dist/angular-chart.css">
     <!-- endbuild -->
 
+    <link rel="preconnect" href="//www.google-analytics.com">
+    <link rel="dns-prefetch" href="//www.google-analytics.com">
+    <link rel="preconnect" href="//ghbtns.com">
+    <link rel="dns-prefetch" href="//ghbtns.com">
+    <link rel="preconnect" href="https://api.github.com">
+    <link rel="dns-prefetch" href="https://api.github.com">
+
+<head>
+
+<body ng-app="YellowLabTools">
+    <div id="header"><h1>Yellow Lab <span class="icon-lab"></span> Tools</h1></div>
+    <div id="body" ng-view autoscroll="true"></div>
+    <div class="footer">
+        <span class="version">@@version</span>
+        <br><a href="/about">More about Yellow Lab Tools</a><br>
+        <div class="github"><iframe id="ghbtn" frameborder="0" scrolling="0" width="145px" height="30px"></iframe></div>
+    </div>
+
     <!-- build:js /js/all.js -->
-    <script src="/bower_components/angular/angular.min.js"></script>
-    <script src="/bower_components/Chart.js/Chart.min.js"></script>
-    <script src="/bower_components/angular-route/angular-route.min.js"></script>
-    <script src="/bower_components/angular-resource/angular-resource.min.js"></script>
-    <script src="/bower_components/angular-sanitize/angular-sanitize.min.js"></script>
-    <script src="/bower_components/angular-animate/angular-animate.min.js"></script>
-    <script src="/bower_components/angular-local-storage/dist/angular-local-storage.min.js"></script>
-    <script src="/bower_components/angular-chart.js/dist/angular-chart.min.js"></script>
+    <script src="/node_modules/angular/angular.min.js"></script>
+    <script src="/node_modules/chart.js/Chart.min.js"></script>
+    <script src="/node_modules/angular-route/angular-route.min.js"></script>
+    <script src="/node_modules/angular-resource/angular-resource.min.js"></script>
+    <script src="/node_modules/angular-sanitize/angular-sanitize.min.js"></script>
+    <script src="/node_modules/angular-animate/angular-animate.min.js"></script>
+    <script src="/node_modules/angular-local-storage/dist/angular-local-storage.min.js"></script>
+    <script src="/node_modules/angular-chart.js/dist/angular-chart.min.js"></script>
     <script src="/js/app.js"></script>
     <script src="/js/controllers/indexCtrl.js"></script>
     <script src="/js/controllers/dashboardCtrl.js"></script>
@@ -44,15 +62,6 @@
     <script src="/js/directives/gradeDirective.js"></script>
     <script src="/js/directives/offendersDirectives.js"></script>
     <!-- endbuild -->
-<head>
-
-<body ng-app="YellowLabTools">
-    <div id="header"><h1>Yellow Lab <span class="icon-lab"></span> Tools</h1></div>
-    <div id="body" ng-view autoscroll="true"></div>
-    <div class="footer">
-        <a href="/about">Learn more</a><br>If you like <b>Yellow Lab Tools</b>, <a href="https://github.com/gmetais/YellowLabTools" target="_blank" class="star">give it a <span>&#9733;</span> on GitHub</a>!<br>
-        <span class="version">@@version</span>
-    </div>
 
     <script>
         (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){(i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)})(window,document,'script','//www.google-analytics.com/analytics.js','ga');if('@@googleAnalyticsId'.indexOf('UA-')===0){ga('create','@@googleAnalyticsId','auto');}

front/src/views/dashboard.html

@@ -1,6 +1,13 @@
 <div ng-include="'views/resultSubHeader.html'"></div>
 <div class="summary board">
     
+    <div ng-if="result.blockedRequests">
+        <b><ng-pluralize count="result.blockedRequests.length" when="{'0': 'No blocked request', 'one': '1 blocked request', 'other': '{} blocked requests'}"></ng-pluralize>:</b>
+        <div ng-repeat="request in result.blockedRequests">
+            {{request}}
+        </div>
+    </div>
+
     <div class="globalScore" ng-if="globalScore === 0 || globalScore > 0">
         <div>
             <h2>Global score</h2>
@@ -28,8 +35,8 @@
             <div class="category">{{category.label}}</div>
             <div class="criteria">
                 <div class="table" title="Click to see details">
-                    <div ng-repeat="ruleName in category.rules" ng-if="result.rules[ruleName]" ng-init="rule = result.rules[ruleName]"
-                         ng-class="{'warning': rule.abnormal}" ng-click="showRulePage(ruleName)">
+                    <a ng-repeat="ruleName in category.rules" ng-if="result.rules[ruleName]" ng-init="rule = result.rules[ruleName]"
+                         ng-class="{'warning': rule.abnormal}" href="/result/{{runId}}/rule/{{ruleName}}">
                         <div class="grade">
                             <grade score="rule.score"></grade>
                         </div>
@@ -42,14 +49,14 @@
                             <span ng-if="rule.abnormalityScore <= -300" class="icon-warning"></span>
                         </div>
                         <div class="info"><span class="icon-question"></span></div>
-                    </div>
+                    </a>
                 </div>
             </div>
         </div>
     </div>
 
     <div ng-if="!error && !fromSocialShare" class="apiTip">
-        <b>Did you know? Yellow Lab Tools has a brand new <a href="https://github.com/gmetais/grunt-yellowlabtools" target="_blank">Grunt plugin</a>!
+        <b>Did you know?</b> You can <b>bookmark</b> this result page URL or <b>share it</b>, Yellow Lab Tools keeps test results for years!
     </div>
 
     <div class="tweet" ng-if="!error && !fromSocialShare">

front/src/views/index.html

@@ -2,7 +2,7 @@
 <p class="price">Free and open source!</p>
 
 <form ng-submit="launchTest()" >
-    <input type="{{$root.isTouchDevice? 'url' : 'text'}}" name="url" ng-model="url" placeholder="http://www.mysite.com" class="url" />
+    <input type="text" name="url" ng-model="url" placeholder="http://www.mysite.com" class="url" />
     <input type="submit" value="Launch test" class="launchBtn" ng-class="{disabled: !url}" />
     <div class="settings">
         <div class="device">
@@ -42,7 +42,7 @@
                 </div>
                 <div><input type="text" name="cookie" ng-model="settings.cookie" /></div>
             </div>
-            <!--<div>
+            <div>
                 <div class="label">
                     Authent
                     <span class="settingsTooltip">
@@ -57,19 +57,26 @@
                     </div>
                     <div>
                         <div><span>password</div>
-                        <div><input type="text" class="authField" name="authPass" ng-model="settings.authPass" /></div>
+                        <div><input type="password" class="authField" name="authPass" ng-model="settings.authPass" /></div>
                     </div>
                 </div>
-            </div>-->
-            <!--<div>
-                <div class="label">Blocked domains</div>
+            </div>
+            <div>
+                <div class="label">
+                    Block domains
+                    <span class="settingsTooltip">
+                        <span class="icon-question"></span>
+                        <div><b>Block some domains</b><br><br>One line per domain or subdomain.<br><br><i><b>Example:</b><br>google-analytics.com<br>ads.yahoo.com<br>ajax.googleapis.com</i><br><br>An empty whitelist will block all domains except the main domain.</div>
+                    </span>
+                </div>
                 <div>
                     <div>
-                        Blacklist / Whitelist
+                        <input type="radio" name="blackOrWhite" ng-model="settings.domainsBlackOrWhite" value="black" />blacklist
+                        <input type="radio" name="blackOrWhite" ng-model="settings.domainsBlackOrWhite" value="white" />whitelist
                     </div>
-                    <textarea name=""></textarea>
+                    <textarea name="domains" ng-model="settings.domains" rows="5"></textarea>
                 </div>
-            </div>-->
+            </div>
         </div>
     </div>
 </form>

front/src/views/rule.html

@@ -14,6 +14,14 @@
                 <span ng-if="rule.policy.unit == 'bytes'">{{rule.value | bytes}}</span>
                 <span ng-if="rule.policy.unit != 'bytes'">{{rule.value}}</span>
             </h3>
+            <div class="okThreshold" ng-if="rule.score < 100 && rule.policy.isOkThreshold !== undefined">
+                Have
+                <span ng-if="rule.policy.unit == 'bytes'">{{rule.policy.isOkThreshold | bytes}}</span>
+                <span ng-if="rule.policy.unit != 'bytes'">{{rule.policy.isOkThreshold}}</span>
+                <span ng-if="rule.policy.isOkThreshold > 0 && rule.policy.isOkThreshold < rule.policy.isBadThreshold">or less</span>
+                <span ng-if="rule.policy.isOkThreshold > rule.policy.isBadThreshold">or more</span>
+                to get the 100/100 score.
+            </div>
             <div ng-bind-html="rule.policy.message" class="message"></div>
         </div>
     </div>
@@ -22,7 +30,7 @@
         <p>This rule reached the abnormality threshold, which means there is a real problem you should care about.</p>
     </div>
     <div class="offenders" ng-if="rule.policy.hasOffenders">
-        <h3><ng-pluralize count="rule.offendersObj.count" when="{'0': 'No offenders', 'one': '1 offender', 'other': '{} offenders'}"></ng-pluralize></h3>
+        <h3 ng-if="rule.offendersObj.count >= 0"><ng-pluralize count="rule.offendersObj.count" when="{'0': 'No offenders', 'one': '1 offender', 'other': '{} offenders'}"></ng-pluralize></h3>
 
         <div ng-if="rule.offendersObj.list" class="offendersTable">
             <div ng-repeat="offender in rule.offendersObj.list track by $index">
@@ -88,15 +96,24 @@
                         </span>
                     </div>
 
+                    <div ng-if="policyName === 'cssRules'">
+                        <span ng-if="offender.file === 'inline CSS'">inline CSS</span>
+                        <span ng-if="offender.file !== 'inline CSS'"><url-link url="offender.file" max-length="80"></url-link></span>
+                        : <ng-pluralize count="offender.rules" when="{'0': '0 rule', 'one':'1 rule','other':'{} rules'}"></ng-pluralize>
+                    </div>
+
+                    <div ng-if="policyName === 'similarColors'">
+                        <div class="similarColors checker"><div ng-style="{'background-color': offender.color1, 'color': offender.isDark ? '#FFF' : '#000'}">{{offender.color1}}</div><div ng-style="{'background-color': offender.color2, 'color': offender.isDark ? '#FFF' : '#000'}">{{offender.color2}}</div></div>
+                    </div>
+
                     <div ng-if="policyName === 'cssParsingErrors'">
                         <b>{{offender.error}}</b>
                         <file-and-line file="offender.file" line="offender.line" column="offender.column"></file-and-line>
                         <span ng-if="offender.file">(<a href="http://jigsaw.w3.org/css-validator/validator?profile=css3&usermedium=all&warning=no&uri={{offender.file | encodeURIComponent}}" target="_blank">Check on the W3C validator</a>)</span>
                     </div>
 
-                    <div ng-if="policyName === 'cssComplexSelectors' || policyName === 'cssComplexSelectorsByAttribute' || policyName === 'cssImports' || policyName === 'cssUniversalSelectors' || policyName === 'cssRedundantBodySelectors' || policyName === 'cssRedundantChildNodesSelectors'">
-                        <span ng-if="offender.bolded" ng-bind-html="offender.bolded"></span>
-                        <b ng-if="!offender.bolded">{{offender.css}}</b>
+                    <div ng-if="policyName === 'cssImports'">
+                        {{offender.css}}
                         <file-and-line-button file="offender.file" line="offender.line" column="offender.column"></file-and-line-button>
                     </div>
 
@@ -104,45 +121,27 @@
                         {{offender.rule}} (<b>x{{offender.occurrences}}</b>)
                     </div>
 
-                    <div ng-if="policyName === 'cssDuplicatedProperties'">
-                        Property <b>{{offender.property}}</b> duplicated in <b>{{offender.rule}} { }</b>
-                        <file-and-line-button file="offender.file" line="offender.line" column="offender.column"></file-and-line-button>
-                    </div>
-
-                    <div ng-if="policyName === 'cssEmptyRules'">
-                        <b>{{offender.css}} { }</b>
-                        <file-and-line-button file="offender.file" line="offender.line" column="offender.column"></file-and-line-button>
-                    </div>
-
-                    <div ng-if="policyName === 'cssExpressions'">
-                        {{offender.rule}} {{ '{' + offender.property}}: <b>expression(</b>{{offender.expression}}<b>)</b>}
-                        <file-and-line-button file="offender.file" line="offender.line" column="offender.column"></file-and-line-button>
-                    </div>
-
-                    <div ng-if="policyName === 'cssImportants'">
-                        {{offender.rule}} {{ '{' + offender.property}}: {{offender.value}} <b>!important</b>}
-                        <file-and-line-button file="offender.file" line="offender.line" column="offender.column"></file-and-line-button>
-                    </div>
-
-                    <div ng-if="policyName === 'cssOldIEFixes'">
-                        <span ng-if="offender.browser"><b>{{offender.browser}} fix:</b></span>
-                        <span ng-bind-html="offender.bolded"></span>
-                        <file-and-line-button file="offender.file" line="offender.line" column="offender.column"></file-and-line-button>
-                    </div>
-
                     <div ng-if="policyName === 'cssOldPropertyPrefixes'">
-                        {{offender.rule}} {<b>{{offender.property}}</b>: {{offender.value + '}' }}
-                        <br>
-                        <b>{{offender.message}}</b>
-                        <file-and-line-button file="offender.file" line="offender.line" column="offender.column"></file-and-line-button>
+                        <b>{{offender.property}} {{offender.message}}</b>
+                        <div ng-if="offender.rules.length" ng-click="offender.showMore = !offender.showMore" class="offenderButton">
+                            <span ng-if="!offender.showMore">show</span>
+                            <span ng-if="offender.showMore">hide</span>
+                            <ng-pluralize count="offender.rules.length" when="{'one':'1 offender','other':'{} offenders'}"></ng-pluralize>
+                        </div>
+                        <div ng-if="offender.showMore" class="smallerOffenders">
+                            <div ng-repeat="cssRule in offender.rules">
+                                {{cssRule.rule}} {{'{' + offender.property}}: {{cssRule.value + '}' }}
+                                <file-and-line-button file="cssRule.file" line="cssRule.line" column="cssRule.column"></file-and-line-button>
+                            </div>
+                        </div>
                     </div>
 
-                    <div ng-if="policyName === 'lazyLoadableImagesBelowTheFold'">
-                        <url-link url="offender" max-length="100"></url-link>
+                    <div ng-if="policyName === 'lazyLoadableImagesBelowTheFold' || policyName === 'hiddenImages'">
                         <img ng-src="{{offender}}" class="smallPreview checker"></img>
+                        <url-link url="offender" max-length="100"></url-link>
                     </div>
 
-                    <div ng-if="policyName === 'notFound' || policyName === 'closedConnections' || policyName === 'multipleRequests' || policyName === 'cachingDisabled' || policyName === 'cachingNotSpecified'">
+                    <div ng-if="policyName === 'notFound' || policyName === 'emptyRequests' || policyName === 'closedConnections' || policyName === 'multipleRequests' || policyName === 'cachingDisabled' || policyName === 'cachingNotSpecified'">
                         <url-link url="offender" max-length="100"></url-link>
                     </div>
 
@@ -156,15 +155,64 @@
                         (<ng-pluralize count="offender.requests" when="{'one':'1 request','other':'{} requests'}"></ng-pluralize>)
                     </div>
 
-                    <div ng-if="policyName === 'globalVariables' || policyName === 'jQueryVersionsLoaded'">
+                    <div ng-if="policyName === 'globalVariables' || policyName === 'jQueryVersionsLoaded' || policyName === 'synchronousXHR'">
                         {{offender}}
                     </div>
-
                 </div>
             </div>
         </div>
 
-        <div ng-if="!rule.offendersObj.list" class="offendersHtml">
+        <div ng-repeat="(file, fileDetails) in rule.offendersObj.byFile track by $index">
+            <h3>
+                <ng-pluralize count="fileDetails.count" when="{'one': '1 offender', 'other': '{} offenders'}"></ng-pluralize>
+                in
+                <url-link ng-if="file !== 'Inline CSS'" url="file" max-length="80"></url-link>
+                <span ng-if="file === 'Inline CSS'">inline CSS</span>
+            </h3>
+
+            <div class="offendersTable">
+                <div ng-repeat="offender in fileDetails.offenders track by $index">
+                    <div ng-if="policyName === 'cssComplexSelectors' || policyName === 'cssComplexSelectorsByAttribute' || policyName === 'cssUniversalSelectors' || policyName === 'cssRedundantBodySelectors' || policyName === 'cssRedundantChildNodesSelectors'">
+                        <span ng-if="offender.bolded" ng-bind-html="offender.bolded"></span>
+                        <b ng-if="!offender.bolded">{{offender.css}}</b>
+                        <span ng-if="offender.line !== null && offender.column !== null"> @ {{offender.line}}:{{offender.column}}</span>
+                    </div>
+
+                    <div ng-if="policyName === 'cssMobileFirst'">
+                        <b>{{offender.query}}</b> for <ng-pluralize count="offender.rules" when="{'one':'1 rule','other':'{} rules'}"></ng-pluralize>
+                        <span ng-if="offender.line !== null && offender.column !== null"> @ {{offender.line}}:{{offender.column}}</span>
+                    </div>
+
+                    <div ng-if="policyName === 'cssDuplicatedProperties'">
+                        Property <b>{{offender.property}}</b> duplicated in <b>{{offender.rule}} { }</b>
+                        <span ng-if="offender.line !== null && offender.column !== null"> @ {{offender.line}}:{{offender.column}}</span>
+                    </div>
+
+                    <div ng-if="policyName === 'cssEmptyRules'">
+                        <b>{{offender.css}} { }</b>
+                        <span ng-if="offender.line !== null && offender.column !== null"> @ {{offender.line}}:{{offender.column}}</span>
+                    </div>
+
+                    <div ng-if="policyName === 'cssExpressions'">
+                        {{offender.rule}} {{ '{' + offender.property}}: <b>expression(</b>{{offender.expression}}<b>)</b>}
+                        <span ng-if="offender.line !== null && offender.column !== null"> @ {{offender.line}}:{{offender.column}}</span>
+                    </div>
+
+                    <div ng-if="policyName === 'cssImportants'">
+                        {{offender.rule}} {{ '{' + offender.property}}: {{offender.value}} <b>!important</b>}
+                        <span ng-if="offender.line !== null && offender.column !== null"> @ {{offender.line}}:{{offender.column}}</span>
+                    </div>
+
+                    <div ng-if="policyName === 'cssOldIEFixes'">
+                        <span ng-if="offender.browser"><b>{{offender.browser}} fix:</b></span>
+                        <span ng-bind-html="offender.bolded"></span>
+                        <span ng-if="offender.line !== null && offender.column !== null"> @ {{offender.line}}:{{offender.column}}</span>
+                    </div>
+                </div>
+            </div>
+        </div>
+
+        <div ng-if="!rule.offendersObj.list && !rule.offendersObj.byFile" class="offendersHtml">
             
             <div ng-if="policyName === 'DOMelementMaxDepth'">
                 <dom-tree tree="rule.offendersObj.tree"></dom-tree>
@@ -287,6 +335,17 @@
         </div>
     </div>
 
+    <div ng-if="policyName === 'identicalFiles'">
+        <div ng-repeat="offender in rule.offendersObj.list track by $index">
+            <h4>A file of {{offender.weight | bytes}} is loaded {{offender.urls.length}} times:</h4>
+            <div class="offendersTable">
+                <div ng-repeat="url in offender.urls">
+                    <div><url-link url="url" max-length="100"></url-link></div>
+                </div>
+            </div>
+        </div>
+    </div>
+
     <div ng-if="policyName === 'smallRequests'">
         <div ng-repeat="(type, requests) in rule.offendersObj.list.byType">
             <h3><ng-pluralize count="requests.length" when="{'0': 'small ' + type + ' file', 'one': '1 small ' + type + ' file', 'other': '{} small ' + type + ' files'}"></ng-pluralize></h3>
@@ -328,6 +387,34 @@
         Please open the <a href="/result/{{runId}}/timeline#filter=eventNotDelegated">JS timeline, filtered by "Events not delegated"</a>
     </div>
 
+    <div ng-if="policyName === 'cssBreakpoints'">
+        <div ng-if="rule.value > 0" class="cssBreakpointsGraph">
+            <h3>Breakpoints distribution graph</h3>
+            <canvas class="chart chart-line" chart-data="breakpointsData" chart-labels="breakpointsLabels" chart-options="breakpointsOptions" chart-legend="true" chart-colours="breakpointsColours" chart-series="breakpointsSeries" width="600" height="250"></canvas>
+            <h3>Breakpoints list</h3>
+            <div class="offendersTable">
+                <div ng-repeat="offender in rule.offendersObj | orderBy:'pixels'">
+                    <div>Breakpoint <b>{{offender.breakpoint}}</b> involves <ng-pluralize count="offender.count" when="{'one': '1 rule', 'other': '{} rules'}"></ng-pluralize></div>
+                </div>
+            </div>
+        </div>
+        <div ng-if="rule.value === 0">
+        No breakpoint
+        </div>
+    </div>
+
+    <div ng-if="policyName === 'http2'">
+        <h3>Protocols advertised by the server</h3>
+        <div class="offendersTable">
+            <div ng-repeat="protocol in rule.offendersObj.list">
+                <div>{{protocol}}</div>
+            </div>
+            <div ng-if="!rule.offendersObj || rule.offendersObj.count == 0">
+                <div>none</div>
+            </div>
+        </div>
+    </div>
+
     <div ng-if="!rule && rule !== null" class="notFound">
         <h2>404</h2>
         Rule "{{policyName}}"" not found

lib/metadata/policies.js

@@ -150,10 +150,10 @@ var policies = {
     "DOMaccessesOnScroll": {
         "tool": "jsExecutionTransformer",
         "label": "DOM access on scroll",
-        "message": "<p>This rule counts the number of DOM-accessing functions calls on a scroll event, such as queries, readings, writings, bindings and jQuery functions.</p><p>Two scroll events are triggered quickly, one after the other, and only the second one is analyzed so throttled functions are ignored.</p><p>One of the main reasons of a poor scrolling experience is when too much JS is executed on each scroll event. Note that some devices such as smartphones and MacBooks send more scroll events than others.</p><p>Reduce the number of DOM accesses inside scroll listeners. Put DOM queries outside them when possible. Use <a href=\"http://blogorama.nerdworks.in/javascriptfunctionthrottlingan/\" target=\"_blank\">throttling or deboucing</a>.</p>",
+        "message": "<p>This rule counts the number of DOM-accessing functions calls on a scroll event, such as queries, readings, writings, bindings and jQuery functions.</p><p>Two scroll events are triggered quickly, one after the other, and only the second one is analyzed so throttled functions are ignored.</p><p>One of the main reasons of a poor scrolling experience is when too much JS is executed on each scroll event. Note that some devices such as smartphones and MacBooks send more scroll events than others.</p><p>Reduce the number of DOM accesses inside scroll listeners. Put DOM queries outside them when possible. Use <a href=\"http://blogorama.nerdworks.in/javascriptfunctionthrottlingan/\" target=\"_blank\">throttling or debouncing</a>.</p>",
         "isOkThreshold": 1,
-        "isBadThreshold": 12,
-        "isAbnormalThreshold": 25,
+        "isBadThreshold": 20,
+        "isAbnormalThreshold": 35,
         "hasOffenders": true,
         "offendersTransformFn": function(offenders) {
             return offenders;
@@ -195,8 +195,8 @@ var policies = {
         "label": "document.write calls",
         "message": "<p>They slow down the page construction, especially if they are used to insert scripts in the page. Remove them ASAP.</p><p>If you cannot remove them because they come from a third-party script (such as ads), have a look at <a href=\"https://github.com/krux/postscribe\" target=\"_blank\">PostScribe</a>.</p>",
         "isOkThreshold": 0,
-        "isBadThreshold": 10,
-        "isAbnormalThreshold": 20,
+        "isBadThreshold": 5,
+        "isAbnormalThreshold": 10,
         "hasOffenders": true,
         "offendersTransformFn": function(offenders) {
             return {
@@ -241,6 +241,15 @@ var policies = {
             };
         }
     },
+    "synchronousXHR": {
+        "tool": "phantomas",
+        "label": "Synchronous Ajax requests",
+        "message": "<p>Making an XMLHttpRequest with the <i>async</i> option set to <i>false</i> is deprecated due to the negative effect to performances. The browser's main thread needs to stop everything until the response is received.</p>",
+        "isOkThreshold": 0,
+        "isBadThreshold": 1,
+        "isAbnormalThreshold": 1,
+        "hasOffenders": true
+    },
     "consoleMessages": {
         "tool": "phantomas",
         "label": "Console messages",
@@ -254,8 +263,8 @@ var policies = {
         "tool": "phantomas",
         "label": "Global variables",
         "message": "<p>It is a bad practice because they clutter up the global namespace. If two scripts use the same variable name in the global scope, it can cause conflicts and it is generally hard to debug.</p><p>Global variables also take a (very) little bit longer to be accessed than variables in the local scope of a function.</p>",
-        "isOkThreshold": 30,
-        "isBadThreshold": 150,
+        "isOkThreshold": 40,
+        "isBadThreshold": 200,
         "isAbnormalThreshold": 700,
         "hasOffenders": true,
         "offendersTransformFn": function(offenders) {
@@ -267,7 +276,7 @@ var policies = {
     },
     "jQueryVersion": {
         "label": "jQuery version",
-        "message": "<p>Current latest versions of jQuery are 1.11 (with support for old IE versions) and 2.1 (without).</p><p>Each new version of jQuery optimizes performances. Do not keep an old version of jQuery. Updating can sometimes break a few things, but it is generally quite easy to fix them up. So don't hesitate.</p>",
+        "message": "<p>The current latest version of jQuery is 3.0</p><p>Each new version of jQuery optimizes performances. Do not keep an old version of jQuery. Updating can sometimes break a few things, but it is generally quite easy to fix them up. So don't hesitate.</p>",
         "hasOffenders": false,
         "scoreFn": function(data) {
             var differentVersions = data.toolsResults.phantomas.metrics.jQueryVersionsLoaded;
@@ -279,30 +288,31 @@ var policies = {
                 var value = data.toolsResults.phantomas.metrics.jQueryVersion;
                 var score;
 
-                if (value.indexOf('1.11.') === 0 ||
-                    value.indexOf('1.12.') === 0 ||
-                    value.indexOf('2.1.') === 0 ||
-                    value.indexOf('2.2.') === 0 ||
-                    value.indexOf('3.0.') === 0) {
+                if (value.indexOf('3.0.') === 0 ||
+                    value.indexOf('3.1.') === 0 ||
+                    value.indexOf('3.2.') === 0) {
                     score = 100;
+                } else if (value.indexOf('1.12.') === 0 ||
+                           value.indexOf('2.2.') === 0) {
+                    score = 90;
+                } else if (value.indexOf('1.11.') === 0 ||
+                           value.indexOf('2.1.') === 0) {
+                    score = 70;
                 } else if (value.indexOf('1.10.') === 0 ||
                            value.indexOf('2.0.') === 0) {
-                    score = 90;
-                } else if (value.indexOf('1.9.') === 0) {
-                    score = 70;
-                } else if (value.indexOf('1.8.') === 0) {
                     score = 50;
-                } else if (value.indexOf('1.7') === 0) {
+                } else if (value.indexOf('1.9.') === 0) {
                     score = 40;
-                } else if (value.indexOf('1.6') === 0) {
+                } else if (value.indexOf('1.8.') === 0) {
                     score = 30;
-                } else if (value.indexOf('1.5') === 0) {
+                } else if (value.indexOf('1.7') === 0) {
                     score = 20;
-                } else if (value.indexOf('1.4') === 0) {
+                } else if (value.indexOf('1.6') === 0) {
                     score = 10;
-                } else if (value.indexOf('1.3') === 0) {
-                    score = 0;
-                } else if (value.indexOf('1.2') === 0) {
+                } else if (value.indexOf('1.5') === 0 ||
+                           value.indexOf('1.4') === 0 ||
+                           value.indexOf('1.3') === 0 ||
+                           value.indexOf('1.2') === 0) {
                     score = 0;
                 } else {
                     debug('Unknown jQuery version "%s"', value);
@@ -420,51 +430,62 @@ var policies = {
         "isOkThreshold": 750,
         "isBadThreshold": 3000,
         "isAbnormalThreshold": 4500,
-        "hasOffenders": false
+        "hasOffenders": true,
+        "offendersTransformFn": function(offenders) {
+            var hasInline = false;
+            var inlineCount = 0;
+            var files = [];
+
+            offenders.forEach(function(line) {
+                if (line.indexOf('[inline CSS]: ') === 0) {
+                    hasInline = true;
+                    inlineCount += parseInt(line.substr(14));
+                } else {
+                    var parts = /^<(.*)>: (\d+)$/.exec(line);
+
+                    if (parts) {
+                        files.push({
+                            file: parts[1],
+                            rules: parseInt(parts[2], 10)
+                        });
+                    }
+                }
+            });
+
+            if (hasInline) {
+                files.push({
+                    file: 'inline CSS',
+                    rules: inlineCount
+                });
+            }
+
+            return {
+                count: files.length,
+                list: files
+            };
+        }
     },
     "cssComplexSelectors": {
         "tool": "phantomas",
         "label": "Complex selectors",
-        "message": "<p>Complex selectors are CSS selectors with 4 or more expressions, like \"#header ul li .foo\".</p><p>They are adding more work for the browser, and this could be avoided by simplifying selectors.</p>",
+        "message": "<p>Complex selectors are CSS selectors with 4 or more expressions, like \"#header ul li .foo\".</p><p>They are adding more work for the browser, and this could be avoided by simplifying selectors. The <a href=\"http://getbem.com\" target=\"_blank\">B.E.M. methodology</a> is an useful way to simplify your CSS.</p>",
         "isOkThreshold": 0,
         "isBadThreshold": 600,
         "isAbnormalThreshold": 2000,
         "hasOffenders": true,
         "offendersTransformFn": function(offenders) {
-            return {
-                count: offenders.length,
-                list: offenders.map(function(offender) {
-                    var splittedOffender = offendersHelpers.cssOffenderPattern(offender);
-                    return splittedOffender;
-                })
-            };
-        }
-    },
-    "cssComplexSelectorsByAttribute": {
-        "tool": "phantomas",
-        "label": "Complex attributes selector",
-        "message": "<p>Complex attributes selectors are one of these:<ul><li>.foo[type*=bar] (contains bar)</li><li>.foo[type^=bar] (starts with bar)</li><li>.foo[type|=bar] (starts with bar or bar-)</li><li>.foo[type$=bar] (ends with bar)</li><li>.foo[type~=bar baz] (bar or baz)</li></ul></p><p>Their matching process needs more CPU and it has a cost on performances.</p>",
-        "isOkThreshold": 0,
-        "isBadThreshold": 75,
-        "isAbnormalThreshold": 150,
-        "hasOffenders": true,
-        "offendersTransformFn": function(offenders) {
-            return {
-                count: offenders.length,
-                list: offenders.map(function(offender) {
-                    var splittedOffender = offendersHelpers.cssOffenderPattern(offender);
+            var parsedOffenders = offenders.map(function(offender) {
+                var splittedOffender = offendersHelpers.cssOffenderPattern(offender);
+                return splittedOffender;
+            });
 
-                    splittedOffender.bolded = splittedOffender.css.replace(/(\[[^ ]+[~\|\^\$\*]="[^"]+"\])/g, '<b>$1</b>');
-
-                    return splittedOffender;
-                })
-            };
+            return offendersHelpers.orderByFile(parsedOffenders);
         }
     },
     "cssColors": {
         "tool": "phantomas",
-        "label": "Different colors",
-        "message": "<p>This is the number of different colors defined in CSS.</p><p>Your CSS project will be easier to maintain if you keep a small color set.</p>",
+        "label": "Colors count",
+        "message": "<p>This is the number of different colors defined in CSS.</p><p>Your CSS will be easier to maintain if you keep a small color set.</p>",
         "isOkThreshold": 30,
         "isBadThreshold": 150,
         "isAbnormalThreshold": 400,
@@ -507,6 +528,55 @@ var policies = {
             };
         }
     },
+    "similarColors": {
+        "tool": "colorDiff",
+        "label": "Similar colors",
+        "message": "<p>This is the list of colors found in the stylesheets, that are very close to each other. The eye can barely see the difference.</p><p>Use this list to reduce the number of colors in your palette, it will be easier to maintain.</p>",
+        "isOkThreshold": 0,
+        "isBadThreshold": 40,
+        "isAbnormalThreshold": 80,
+        "hasOffenders": true,
+        "offendersTransformFn": function(offenders) {
+            return {
+                count: offenders.length,
+                list: offenders
+            };
+        }
+    },
+    "cssBreakpoints": {
+        "tool": "mediaQueriesChecker",
+        "label": "Breakpoints count",
+        "message": "<p>This is the number of different breakpoints found in the stylesheets' media queries.</p><p>Please note this rule is based on <i>min-width</i>, <i>max-width</i>, <i>min-device-width</i> and <i>max-device-width</i> media queries only.</p><p>Your CSS will be easier to maintain if you keep a reasonable number of breakpoints. Try to make a fluid design - using percents - to avoid the creation of numerous breakpoints.</p>",
+        "isOkThreshold": 6,
+        "isBadThreshold": 40,
+        "isAbnormalThreshold": 60,
+        "hasOffenders": true,
+        "offendersTransformFn": function(offenders) {
+            var offendersTable = [];
+
+            for (var offender in offenders) {
+                offendersTable.push({
+                    breakpoint: offender,
+                    count: offenders[offender].count,
+                    pixels: offenders[offender].pixels
+                });
+            }
+
+            return offendersTable;
+        }
+    },
+    "cssMobileFirst": {
+        "tool": "mediaQueriesChecker",
+        "label": "Not mobile-first media queries",
+        "message": "<p>This is the number of CSS rules inside media queries that address small screens.</p><p>The common good practice, when creating a responsive website, is to write it \"mobile-first\". More explanation in <a href=\"http://www.sitepoint.com/introduction-mobile-first-media-queries\" target=\"_blank\">this great article</a>.</p>",
+        "isOkThreshold": 25,
+        "isBadThreshold": 200,
+        "isAbnormalThreshold": 1000,
+        "hasOffenders": true,
+        "offendersTransformFn": function(offenders) {
+            return offendersHelpers.orderByFile(offenders);
+        }
+    },
     "cssImports": {
         "tool": "phantomas",
         "label": "Uses of @import",
@@ -563,29 +633,28 @@ var policies = {
         "isAbnormalThreshold": 120,
         "hasOffenders": true,
         "offendersTransformFn": function(offenders) {
-            return {
-                count: offenders.length,
-                list: offenders.map(function(offender) {
-                    var splittedOffender = offendersHelpers.cssOffenderPattern(offender);
+            var parsedOffenders = offenders.map(function(offender) {
+                var splittedOffender = offendersHelpers.cssOffenderPattern(offender);
 
-                    var parts = /^([^{]+) {([^ ]+): (.+)}$/.exec(splittedOffender.css);
-
-                    if (!parts) {
-                        debug('cssDuplicatedProperties offenders transform function error with "%s"', offender);
-                        return {
-                            parseError: offender
-                        };
-                    }
+                var parts = /^([^{]+) {([^ ]+): (.+)}$/.exec(splittedOffender.css);
 
+                if (!parts) {
+                    debug('cssDuplicatedProperties offenders transform function error with "%s"', offender);
                     return {
-                        property: parts[2],
-                        rule: parts[1],
-                        file: splittedOffender.file,
-                        line: splittedOffender.line,
-                        column: splittedOffender.column
+                        parseError: offender
                     };
-                })
-            };
+                }
+
+                return {
+                    property: parts[2],
+                    rule: parts[1],
+                    file: splittedOffender.file,
+                    line: splittedOffender.line,
+                    column: splittedOffender.column
+                };
+            });
+
+            return offendersHelpers.orderByFile(parsedOffenders);
         }
     },
     "cssEmptyRules": {
@@ -597,14 +666,8 @@ var policies = {
         "isAbnormalThreshold": 100,
         "hasOffenders": true,
         "offendersTransformFn": function(offenders) {
-            return {
-                count: offenders.length,
-                list: offenders.map(function(offender) {
-                    var splittedOffender = offendersHelpers.cssOffenderPattern(offender);
-
-                    return splittedOffender;
-                })
-            };
+            var parsedOffenders = offenders.map(offendersHelpers.cssOffenderPattern);
+            return offendersHelpers.orderByFile(parsedOffenders);
         }
     },
     "cssExpressions": {
@@ -616,30 +679,29 @@ var policies = {
         "isAbnormalThreshold": 20,
         "hasOffenders": true,
         "offendersTransformFn": function(offenders) {
-            return {
-                count: offenders.length,
-                list: offenders.map(function(offender) {
-                    var splittedOffender = offendersHelpers.cssOffenderPattern(offender);
+            var parsedOffenders = offenders.map(function(offender) {
+                var splittedOffender = offendersHelpers.cssOffenderPattern(offender);
 
-                    var parts = /^(.*) {([^ ]+): expression\((.*)\)}$/.exec(splittedOffender.css);
-
-                    if (!parts) {
-                        debug('cssExpressions offenders transform function error with "%s"', offender);
-                        return {
-                            parseError: offender
-                        };
-                    }
+                var parts = /^(.*) {([^ ]+): expression\((.*)\)}$/.exec(splittedOffender.css);
 
+                if (!parts) {
+                    debug('cssExpressions offenders transform function error with "%s"', offender);
                     return {
-                        rule: parts[1],
-                        property: parts[2],
-                        expression: parts[3],
-                        file: splittedOffender.file,
-                        line: splittedOffender.line,
-                        column: splittedOffender.column
+                        parseError: offender
                     };
-                })
-            };
+                }
+
+                return {
+                    rule: parts[1],
+                    property: parts[2],
+                    expression: parts[3],
+                    file: splittedOffender.file,
+                    line: splittedOffender.line,
+                    column: splittedOffender.column
+                };
+            });
+
+            return offendersHelpers.orderByFile(parsedOffenders);
         }
     },
     "cssImportants": {
@@ -651,30 +713,29 @@ var policies = {
         "isAbnormalThreshold": 200,
         "hasOffenders": true,
         "offendersTransformFn": function(offenders) {
-            return {
-                count: offenders.length,
-                list: offenders.map(function(offender) {
-                    var splittedOffender = offendersHelpers.cssOffenderPattern(offender);
+            var parsedOffenders = offenders.map(function(offender) {
+                var splittedOffender = offendersHelpers.cssOffenderPattern(offender);
 
-                    var parts = /^(.*) {([^ ]+): (.*) ?\!important}$/.exec(splittedOffender.css);
-
-                    if (!parts) {
-                        debug('cssImportants offenders transform function error with "%s"', offender);
-                        return {
-                            parseError: offender
-                        };
-                    }
+                var parts = /^(.*) {([^ ]+): (.*) ?\!important}$/.exec(splittedOffender.css);
 
+                if (!parts) {
+                    debug('cssImportants offenders transform function error with "%s"', offender);
                     return {
-                        rule: parts[1],
-                        property: parts[2],
-                        value: parts[3],
-                        file: splittedOffender.file,
-                        line: splittedOffender.line,
-                        column: splittedOffender.column
+                        parseError: offender
                     };
-                })
-            };
+                }
+
+                return {
+                    rule: parts[1],
+                    property: parts[2],
+                    value: parts[3],
+                    file: splittedOffender.file,
+                    line: splittedOffender.line,
+                    column: splittedOffender.column
+                };
+            });
+
+            return offendersHelpers.orderByFile(parsedOffenders);
         }
     },
     "cssOldIEFixes": {
@@ -686,105 +747,103 @@ var policies = {
         "isAbnormalThreshold": 300,
         "hasOffenders": true,
         "offendersTransformFn": function(offenders) {
-            return {
-                count: offenders.length,
-                list: offenders.map(function(offender) {
-                    var splittedOffender = offendersHelpers.cssOffenderPattern(offender);
+            var parsedOffenders = offenders.map(function(offender) {
+                var splittedOffender = offendersHelpers.cssOffenderPattern(offender);
 
-                    var parts = /^([^{]*)( {([^ ]+): (.*)})?$/.exec(splittedOffender.css);
+                var parts = /^([^{]*)( {([^ ]+): (.*)})?$/.exec(splittedOffender.css);
 
-                    if (!parts) {
-                        debug('cssOldIEFixes offenders transform function error with "%s"', offender);
-                        return {
-                            parseError: offender
-                        };
-                    }
+                if (!parts) {
+                    debug('cssOldIEFixes offenders transform function error with "%s"', offender);
+                    return {
+                        parseError: offender
+                    };
+                }
 
-                    var rule = parts[1];
-                    var property = parts[3];
-                    var value = parts[4];
-                    var browser = null;
+                var rule = parts[1];
+                var property = parts[3];
+                var value = parts[4];
+                var browser = null;
 
-                    if (rule.indexOf('* html') === 0) {
-                        rule = rule.replace(/^\* html/, '<b>* html</b>');
-                        browser = 'IE6';
-                    } else if (rule.indexOf('html>body') === 0) {
-                        rule = rule.replace(/^html>body/, '<b>html>body</b>');
-                        browser = 'IE6';
-                    } else if (property.indexOf('*') === 0) {
-                        property = '<b>' + property + '</b>';
-                        browser = 'IE7';
-                    } else if (value.match(/\!ie$/)) {
-                        value = value.replace(/\!ie$/, '<b>!ie</b>');
-                        browser = 'IE7';
-                    } else if (property === '-ms-filter') {
-                        property = '<b>-ms-filter</b>';
-                        browser = 'IE9';
-                    } else if (value.indexOf('progid:DXImageTransform.Microsoft') >= 0) {
-                        value = value.replace(/progid:DXImageTransform\.Microsoft/, '<b>progid:DXImageTransform.Microsoft</b>');
-                        browser = 'IE9';
-                    }
+                if (rule.indexOf('* html') === 0) {
+                    rule = rule.replace(/^\* html/, '<b>* html</b>');
+                    browser = 'IE6';
+                } else if (rule.indexOf('html>body') === 0) {
+                    rule = rule.replace(/^html>body/, '<b>html>body</b>');
+                    browser = 'IE6';
+                } else if (property.indexOf('*') === 0) {
+                    property = '<b>' + property + '</b>';
+                    browser = 'IE7';
+                } else if (value.match(/\!ie$/)) {
+                    value = value.replace(/\!ie$/, '<b>!ie</b>');
+                    browser = 'IE7';
+                } else if (property === '-ms-filter') {
+                    property = '<b>-ms-filter</b>';
+                    browser = 'IE9';
+                } else if (value.indexOf('progid:DXImageTransform.Microsoft') >= 0) {
+                    value = value.replace(/progid:DXImageTransform\.Microsoft/, '<b>progid:DXImageTransform.Microsoft</b>');
+                    browser = 'IE9';
+                }
 
-                    var propertyAndValue = (property && value) ? ' {' + property + ': ' + value + '}' : '';
-                    splittedOffender.bolded = rule + propertyAndValue;
-                    splittedOffender.browser = browser;
+                var propertyAndValue = (property && value) ? ' {' + property + ': ' + value + '}' : '';
+                splittedOffender.bolded = rule + propertyAndValue;
+                splittedOffender.browser = browser;
 
-                    return splittedOffender;
-                })
-            };
+                return splittedOffender;
+            });
+
+            return offendersHelpers.orderByFile(parsedOffenders);
         }
     },
     "cssOldPropertyPrefixes": {
         "tool": "phantomas",
         "label": "Old prefixes",
-        "message": "<p>Many property prefixes such as -moz- or -webkit- are not needed anymore, or by very few people. You can remove them or replace them with the non-prefixed version. This will help reducing your stylesheets weight.</p>",
+        "message": "<p>Many property prefixes such as -moz- or -webkit- are not needed anymore, or by very few people. Sometimes, they have never even existed. You can remove them or replace them with the non-prefixed version. This will help reducing your stylesheets weight.</p><p>The prefixes database comes from <a href=\"http://caniuse.com/\" target=\"_blank\">Can I Use</a>.</p>",
         "isOkThreshold": 0,
         "isBadThreshold": 75,
         "isAbnormalThreshold": 300,
         "hasOffenders": true,
         "offendersTransformFn": function(offenders) {
-            return {
-                count: offenders.length,
-                list: offenders.map(function(offender) {
-                    var splittedOffender = offendersHelpers.cssOffenderPattern(offender);
+            var properties = {};
+            offenders.forEach(function(offender) {
+                var splittedOffender = offendersHelpers.cssOffenderPattern(offender);
 
-                    var parts = /^([^{]*)(?: ?{ ?([^ ]+): (.*) ?}) \/\/ (.*)$/.exec(splittedOffender.css);
-
-                    if (!parts) {
-                        debug('cssOldPropertyPrefixes offenders transform function error with "%s"', offender);
-                        return {
-                            parseError: offender
-                        };
-                    }
+                var parts = /^([^{]*)(?: ?{ ?([^ ]+): (.*) ?}) \/\/ (.*)$/.exec(splittedOffender.css);
 
+                if (!parts) {
+                    debug('cssOldPropertyPrefixes offenders transform function error with "%s"', offender);
                     return {
-                        rule: parts[1],
-                        property: parts[2],
-                        value: parts[3],
-                        message: parts[4],
-                        file: splittedOffender.file,
-                        line: splittedOffender.line,
-                        column: splittedOffender.column
+                        parseError: offender
                     };
-                })
-            };
-        }
-    },
-    "cssUniversalSelectors": {
-        "tool": "phantomas",
-        "label": "Universal selectors",
-        "message": "<p>Universal selectors are the most expensive CSS selectors.</p><p>More informations <a href=\"http://perfectionkills.com/profiling-css-for-fun-and-profit-optimization-notes/\" target=\"_blank\">here</a>.</p>",
-        "isOkThreshold": 0,
-        "isBadThreshold": 50,
-        "isAbnormalThreshold": 150,
-        "hasOffenders": true,
-        "offendersTransformFn": function(offenders) {
+                }
+
+                var propertyName = parts[2];
+
+                if (!properties[propertyName]) {
+                    properties[propertyName] = {
+                        property: propertyName,
+                        message: parts[4],
+                        rules: []
+                    };
+                }
+
+                properties[propertyName].rules.push({
+                    rule: parts[1],
+                    value: parts[3],
+                    file: splittedOffender.file,
+                    line: splittedOffender.line,
+                    column: splittedOffender.column
+                });
+            });
+
+            // Object to array
+            var list = [];
+            for (var propertyName in properties) {
+                list.push(properties[propertyName]);
+            }
+
             return {
                 count: offenders.length,
-                list: offenders.map(function(offender) {
-                    var splittedOffender = offendersHelpers.cssOffenderPattern(offender);
-                    return splittedOffender;
-                })
+                list: list
             };
         }
     },
@@ -797,56 +856,52 @@ var policies = {
         "isAbnormalThreshold": 200,
         "hasOffenders": true,
         "offendersTransformFn": function(offenders) {
-            return {
-                count: offenders.length,
-                list: offenders.map(function(offender) {
-                    var splittedOffender = offendersHelpers.cssOffenderPattern(offender);
+            var parsedOffenders = offenders.map(function(offender) {
+                var splittedOffender = offendersHelpers.cssOffenderPattern(offender);
+                splittedOffender.bolded = splittedOffender.css.replace(/body/, '<b>body</b>');
+                return splittedOffender;
+            });
 
-                    splittedOffender.bolded = splittedOffender.css.replace(/body/, '<b>body</b>');
-
-                    return splittedOffender;
-                })
-            };
+            return offendersHelpers.orderByFile(parsedOffenders);
         }
     },
     "cssRedundantChildNodesSelectors": {
         "tool": "phantomas",
         "label": "Redundant tags selectors",
-        "message": "<p>Some tags included inside other tags are obvious. For example, when \"ul li\" is specified in a rule, \"ul\" can be removed because the \"li\" element is <b>always</b> inside a \"ul\". Same thing for \"tr td\", \"select option\", ...</p><p>Lowering compexity in CSS selectors can make the page load a little faster.</p>",
+        "message": "<p>Some tags included inside other tags are obvious. For example, when \"ul li\" is specified in a rule, \"ul\" can be removed because the \"li\" tag is nearly always inside an \"ul\" container (the \"ol\" container is quite rare). Same thing for \"tr td\", \"select option\", ...</p><p>Lowering compexity in CSS selectors can make the page load a little faster.</p>",
         "isOkThreshold": 0,
         "isBadThreshold": 60,
         "isAbnormalThreshold": 200,
         "hasOffenders": true,
         "offendersTransformFn": function(offenders) {
-            return {
-                count: offenders.length,
-                list: offenders.map(function(offender) {
-                    var splittedOffender = offendersHelpers.cssOffenderPattern(offender);
+            var parsedOffenders = offenders.map(function(offender) {
+                var splittedOffender = offendersHelpers.cssOffenderPattern(offender);
 
-                    var rule = splittedOffender.css || '';
-                    var redundanters = [
-                        ['ul', 'li'],
-                        ['ol', 'li'],
-                        ['select', 'option'],
-                        ['table', 'tr'],
-                        ['table', 'th'],
-                    ];
+                var rule = splittedOffender.css || '';
+                var redundanters = [
+                    ['ul', 'li'],
+                    ['ol', 'li'],
+                    ['select', 'option'],
+                    ['table', 'tr'],
+                    ['table', 'th'],
+                ];
 
-                    redundanters.forEach(function(couple) {
-                        rule = rule.replace(new RegExp('(^| |>)' + couple[0] + '([^ >]*)?([ >]| > )' + couple[1] + '([^\\w-]|$)', 'g'), '$1<b>' + couple[0] + '</b>$2$3<b>' + couple[1] + '</b>$4');
-                    });
+                redundanters.forEach(function(couple) {
+                    rule = rule.replace(new RegExp('(^| |>)' + couple[0] + '([^ >]*)?([ >]| > )' + couple[1] + '([^\\w-]|$)', 'g'), '$1<b>' + couple[0] + '</b>$2$3<b>' + couple[1] + '</b>$4');
+                });
 
-                    splittedOffender.bolded = rule;
+                splittedOffender.bolded = rule;
 
-                    return splittedOffender;
-                })
-            };
+                return splittedOffender;
+            });
+
+            return offendersHelpers.orderByFile(parsedOffenders);
         }
     },
     "totalWeight": {
-        "tool": "weightChecker",
+        "tool": "redownload",
         "label": "Total weight",
-        "message": "<p>The weight is of course very important if you want the page to load fast. Try to stay under 1MB, which is alreay very long to download over a slow connection.</p>",
+        "message": "<p>The weight is of course very important if you want the page to load fast. Try to stay under 1MB, which is alreay very long to download over a slow connection.</p><p>Please note that Yellow Lab Tools' engine (PhantomJS) is not compatible with image srcset (unless you use a polyfill). This can lead to incorrect page weight.</p>",
         "isOkThreshold": 716800,
         "isBadThreshold": 2097152,
         "isAbnormalThreshold": 3145728,
@@ -854,9 +909,9 @@ var policies = {
         "unit": 'bytes'
     },
     "imageOptimization": {
-        "tool": "weightChecker",
+        "tool": "redownload",
         "label": "Image optimization",
-        "message": "<p>This metric mesures the number of bytes that could be saved by optimizing images.</p><p>Image optimization is generally one of the easiest way to reduce a page weight, and as a result, the page load time. Don't use Photoshop or other image editing tools, they're not very good for optimization. Use specialized tools such as <a href=\"https://kraken.io/\" target=\"_blank\">Kraken.io</a> or the excellent <a href=\"https://imageoptim.com/\" target=\"_blank\">ImageOption</a> on Mac. For SVG images, you can use <a href=\"https://jakearchibald.github.io/svgomg/\" target=\"_blank\">SVGOMG</a></p><p>The tools in use in YellowLabTools are not set to their maximum optimization power, so you might be able to compress even more!</p>",
+        "message": "<p>This metric measures the number of bytes that could be saved by optimizing images.</p><p>Image optimization is generally one of the easiest way to reduce a page weight, and as a result, the page load time. Don't use Photoshop or other image editing tools, they're not very good for optimization. Use specialized tools such as <a href=\"https://kraken.io/\" target=\"_blank\">Kraken.io</a> or the excellent <a href=\"https://imageoptim.com/\" target=\"_blank\">ImageOptim</a> on Mac. For SVG images, you can use <a href=\"https://jakearchibald.github.io/svgomg/\" target=\"_blank\">SVGOMG</a></p><p>The tools in use in YellowLabTools are not set to their maximum optimization power (JPEG quality 85), so you might be able to compress even more!</p><p>Please note that Yellow Lab Tools' engine (PhantomJS) is not compatible with image srcset (unless you use a polyfill). This can lead to incorrect page weight.</p>",
         "isOkThreshold": 10240,
         "isBadThreshold": 122880,
         "isAbnormalThreshold": 307200,
@@ -864,9 +919,9 @@ var policies = {
         "unit": 'bytes'
     },
     "gzipCompression": {
-        "tool": "weightChecker",
+        "tool": "redownload",
         "label": "Gzip compression",
-        "message": "<p>Mesures the number of bytes that could be saved by compressing file transfers.</p><p>Gzip is a powerfull weight reducer and should be enabled on text-based assets in your server's configuration. Note that gzipping small files (< 1 KB) is arguable, and that some assets such as images should not be gzipped as they are already compressed. <a href=\"https://gist.github.com/gmetais/971ce13a1fbeebd88445\" target=\"_blank\">Here</a> is a list of Content-Types that should be gzipped.</p>",
+        "message": "<p>Measures the number of bytes that could be saved by compressing file transfers.</p><p>Gzip is a powerfull weight reducer and should be enabled on text-based assets in your server's configuration. Note that gzipping small files (< 1 KB) is arguable, and that some assets such as images should not be gzipped as they are already compressed. <a href=\"https://gist.github.com/gmetais/971ce13a1fbeebd88445\" target=\"_blank\">Here</a> is a list of Content-Types that should be gzipped.</p>",
         "isOkThreshold": 5125,
         "isBadThreshold": 81920,
         "isAbnormalThreshold": 153600,
@@ -874,7 +929,7 @@ var policies = {
         "unit": 'bytes'
     },
     "fileMinification": {
-        "tool": "weightChecker",
+        "tool": "redownload",
         "label": "File minification",
         "message": "<p>This is the weight that could be saved if all text resources were correctly minified.</p><p>The tools in use here are <b>UglifyJS</b>, <b>clean-css</b> and <b>HTMLMinifier</b>. These tools are so good that some of your minified files can be marked as unminified. Change your tool it this happens :)</p><p>The gains of minification are generally small, but the impact can be high when these text files are loaded on the critical path.</p>",
         "isOkThreshold": 5125,
@@ -884,7 +939,7 @@ var policies = {
         "unit": 'bytes'
     },
     "totalRequests": {
-        "tool": "weightChecker",
+        "tool": "redownload",
         "label": "Requests number",
         "message": "<p>This is one of the most important performance rule. Every request is slowing down the page loading.</p><p>There are several technics to reduce their number:<ul><li>Concatenate JS files</li><li>Concatenate CSS files</li><li>Embed or inline small JS or CSS files in the HTML</li><li>Create sprites or icon fonts</li><li>Base64 encode small images in HTML or stylesheets</li><li>Use lazyloading for images</li></ul></p>",
         "isOkThreshold": 15,
@@ -939,17 +994,29 @@ var policies = {
         "isAbnormalThreshold": 20,
         "hasOffenders": true
     },
-    "multipleRequests": {
-        "tool": "phantomas",
-        "label": "Duplicated requests",
-        "message": "<p>This only happens when the asset has no cache and is requested more than once on the same page. Be very careful about it.</p>",
+    "identicalFiles": {
+        "tool": "redownload",
+        "label": "Identical content",
+        "message": "<p>This is the number of requests that could be avoided, because of downloaded files that have the same content but are loaded from different URLs.</p><p>Try to load them from the same URL.</p>",
         "isOkThreshold": 0,
-        "isBadThreshold": 3,
-        "isAbnormalThreshold": 10,
+        "isBadThreshold": 5,
+        "isAbnormalThreshold": 15,
+        "hasOffenders": true,
+        "offendersTransformFn": function(offenders) {
+            return offenders;
+        }
+    },
+    "emptyRequests": {
+        "tool": "redownload",
+        "label": "Empty requests",
+        "message": "<p>List of GET requests that respond with an empty body. These are probably the easiest requests to remove.</p>",
+        "isOkThreshold": 0,
+        "isBadThreshold": 1,
+        "isAbnormalThreshold": 5,
         "hasOffenders": true
     },
     "smallRequests": {
-        "tool": "weightChecker",
+        "tool": "redownload",
         "label": "Small requests",
         "message": "<p>List of all requests that are less than 2 KB. Try to merge them with other files.</p>",
         "isOkThreshold": 4,
@@ -959,13 +1026,51 @@ var policies = {
     },
     "lazyLoadableImagesBelowTheFold": {
         "tool": "phantomas",
-        "label": "Not lazyloaded images",
+        "label": "Below the fold images",
         "message": "<p>This is the number of images displayed below the fold that could be lazy-loaded. This is an excellent way to accelerate the loading time of an heavy page.</p><p>I recommend using <a href=\"https://github.com/vvo/lazyload\" target=\"_blank\">this lazyloader</a>.</p>",
         "isOkThreshold": 1,
         "isBadThreshold": 12,
         "isAbnormalThreshold": 30,
         "hasOffenders": true
     },
+    "hiddenImages": {
+        "tool": "phantomas",
+        "label": "Hidden images",
+        "message": "<p>List of all images that have a display:none property, or one of their parents. These images are loaded by the browser even if they're not visible. You might be able to find a way to lazy-load them, only when they get visible.</p><p>As images displayed in 1x1 pixels tend to be trackers, they are excluded from this rule.</p>",
+        "isOkThreshold": 1,
+        "isBadThreshold": 12,
+        "isAbnormalThreshold": 30,
+        "hasOffenders": true
+    },
+    "http2": {
+        "label": "HTTP/2 or SPDY",
+        "message": "<p>HTTP/2 is the latest version of the HTTP protocol and is designed to optimize load speed. SPDY is deprecated but still very well supported.</p><p>The latest versions of all major browsers are now compatible. The difficulty is on the server side, where technologies are not quite ready yet.</p>",
+        "hasOffenders": true,
+        "scoreFn": function(data) {
+            if (!data.toolsResults.http2) {
+                return null;
+            }
+
+            var isHttp2 = data.toolsResults.http2.metrics.http2;
+
+            var result = {
+                value: isHttp2 ? 'Yes' : 'No',
+                score: isHttp2 ? 100 : 0,
+                bad: !isHttp2,
+                abnormal: false,
+                abnormalityScore: 0
+            };
+
+            if (data.toolsResults.http2.offenders) {
+                result.offendersObj = {
+                    count: data.toolsResults.http2.offenders.http2.length,
+                    list: data.toolsResults.http2.offenders.http2
+                };
+            }
+
+            return result;
+        }
+    },
     "cachingDisabled": {
         "tool": "phantomas",
         "label": "Caching disabled",

lib/metadata/scoreProfileGeneric.json

@@ -15,9 +15,11 @@
                 "totalRequests": 5,
                 "domains": 3,
                 "notFound": 3,
-                "multipleRequests": 2,
+                "identicalFiles": 2,
+                "emptyRequests": 3,
                 "smallRequests": 1,
-                "lazyLoadableImagesBelowTheFold": 2
+                "lazyLoadableImagesBelowTheFold": 2,
+                "hiddenImages": 1
             }
         },
         "domComplexity": {
@@ -49,6 +51,7 @@
             "policies": {
                 "jsErrors": 1,
                 "documentWriteCalls": 2,
+                "synchronousXHR": 5,
                 "consoleMessages": 0.5,
                 "globalVariables": 0.5
             }
@@ -68,8 +71,10 @@
             "policies": {
                 "cssRules": 2,
                 "cssComplexSelectors": 2,
-                "cssComplexSelectorsByAttribute": 1.5,
-                "cssColors": 0.5
+                "cssColors": 0.5,
+                "similarColors": 0.5,
+                "cssBreakpoints": 0.5,
+                "cssMobileFirst": 1
             }
         },
         "badCSS": {
@@ -84,7 +89,6 @@
                 "cssImportants": 3,
                 "cssOldIEFixes": 1,
                 "cssOldPropertyPrefixes": 1,
-                "cssUniversalSelectors": 1,
                 "cssRedundantBodySelectors": 1,
                 "cssRedundantChildNodesSelectors": 1
             }
@@ -92,6 +96,7 @@
         "serverConfig": {
             "label": "Server config",
             "policies": {
+                "http2": 2,
                 "closedConnections": 2,
                 "cachingNotSpecified": 1,
                 "cachingDisabled": 1,
@@ -105,7 +110,7 @@
         "domComplexity": 2,
         "domManipulations": 2,
         "scroll": 1,
-        "badJavascript": 1,
+        "badJavascript": 2,
         "jQuery": 1,
         "cssSyntaxError": 1,
         "cssComplexity": 1,

lib/offendersHelpers.js

@@ -113,14 +113,17 @@ var OffendersHelpers = function() {
         if (traceArray) {
             var results = [];
             var parts = null;
+            var obj;
 
             for (var i=0 ; i<traceArray.length ; i++) {
-                parts = /^(([\w$]+) )?\(?([^ ]+):(\d+)\)?$/.exec(traceArray[i]);
+                // Handle the new PhantomJS 2.x syntax
+                parts = /^(([\w$]+)@)?([^ ]+):(\d+):(\d+)$/.exec(traceArray[i]);
 
                 if (parts) {
-                    var obj = {
+                    obj = {
                         file: parts[3],
-                        line: parseInt(parts[4], 10)
+                        line: parseInt(parts[4], 10),
+                        column: parseInt(parts[5], 10)
                     };
 
                     if (parts[2]) {
@@ -128,8 +131,26 @@ var OffendersHelpers = function() {
                     }
 
                     results.push(obj);
+
                 } else {
-                    return null;
+                    // Old syntax
+                    parts = /^(([\w$]+) )?\(?([^ ]+):(\d+)\)?$/.exec(traceArray[i]);
+
+                    if (parts) {
+                        obj = {
+                            file: parts[3],
+                            line: parseInt(parts[4], 10)
+                        };
+
+                        if (parts[2]) {
+                            obj.functionName = parts[2];
+                        }
+
+                        results.push(obj);
+
+                    } else {
+                        return null;
+                    }
                 }
             }
             return results;
@@ -151,6 +172,9 @@ var OffendersHelpers = function() {
     };
 
     this.cssOffenderPattern = function(offender) {
+        // Remove any line breaks
+        offender = offender.replace(/(\r\n|\n|\r)/gm, '');
+
         var parts = /^(.*) (?:<([^ \(]*)>|\[inline CSS\]) @ (\d+):(\d+)$/.exec(offender);
         
         if (!parts) {
@@ -182,6 +206,27 @@ var OffendersHelpers = function() {
         }
     };
 
+    this.orderByFile = function(offenders) {
+        var byFile = {};
+
+        offenders.forEach(function(offender) {
+            var file = offender.file || 'Inline CSS';
+            delete offender.file;
+
+            if (!byFile[file]) {
+                byFile[file] = {
+                    count: 0,
+                    offenders: []
+                };
+            }
+
+            byFile[file].count ++;
+            byFile[file].offenders.push(offender);
+        });
+
+        return {byFile: byFile};
+    };
+
 };
 
 module.exports = new OffendersHelpers();

lib/runner.js

@@ -3,7 +3,10 @@ var debug                   = require('debug')('ylt:runner');
 
 var phantomasWrapper        = require('./tools/phantomas/phantomasWrapper');
 var jsExecutionTransformer  = require('./tools/jsExecutionTransformer');
-var weightChecker           = require('./tools/weightChecker/weightChecker');
+var colorDiff               = require('./tools/colorDiff');
+var mediaQueriesChecker     = require('./tools/mediaQueriesChecker');
+var isHttp2                 = require('./tools/isHttp2');
+var redownload              = require('./tools/redownload/redownload');
 var rulesChecker            = require('./rulesChecker');
 var scoreCalculator         = require('./scoreCalculator');
 
@@ -28,9 +31,20 @@ var Runner = function(params) {
         // Treat the JS Execution Tree from offenders
         data = jsExecutionTransformer.transform(data);
 
-        // Redownload every file
-        return weightChecker.recheckAllFiles(data);
+        // Compare colors
+        data = colorDiff.compareAllColors(data);
 
+        // Check media queries
+        data = mediaQueriesChecker.analyzeMediaQueries(data);
+
+        // Redownload every file
+        return redownload.recheckAllFiles(data);
+
+    })
+
+    .then(function(data) {
+        // Check if HTTP2
+        return isHttp2.check(data);
     })
 
     .then(function(data) {
@@ -52,15 +66,14 @@ var Runner = function(params) {
         delete data.toolsResults.phantomas.metrics.scrollExecutionTree;
         delete data.toolsResults.phantomas.offenders.scrollExecutionTree;
 
-        return data;
 
-    })
+        if (data.toolsResults.phantomas.offenders.blockedRequests) {
+            data.blockedRequests = data.toolsResults.phantomas.offenders.blockedRequests;
+        }
 
-    .then(function(data) {
 
         // Finished!
         deferred.resolve(data);
-
     })
 
     .fail(function(err) {

lib/server/controllers/apiController.js

@@ -6,7 +6,7 @@ var ScreenshotHandler   = require('../../screenshotHandler');
 var RunsQueue           = require('../datastores/runsQueue');
 var RunsDatastore       = require('../datastores/runsDatastore');
 var ResultsDatastore    = require('../datastores/resultsDatastore');
-
+var serverSettings      = require('../../../server_config/settings.json');
 
 var ApiController = function(app) {
     'use strict';
@@ -15,8 +15,6 @@ var ApiController = function(app) {
     var runsDatastore = new RunsDatastore();
     var resultsDatastore = new ResultsDatastore();
 
-
-
     // Create a new run
     app.post('/api/runs', function(req, res) {
 
@@ -33,12 +31,14 @@ var ApiController = function(app) {
                 waitForResponse: req.body.waitForResponse !== false && req.body.waitForResponse !== 'false' && req.body.waitForResponse !== 0,
                 partialResult: req.body.partialResult || null,
                 screenshot: req.body.screenshot || false,
-                jsTimeline: req.body.jsTimeline || false,
                 device: req.body.device || 'desktop',
                 waitForSelector: req.body.waitForSelector || null,
                 cookie: req.body.cookie || null,
                 authUser: req.body.authUser || null,
-                authPass: req.body.authPass || null
+                authPass: req.body.authPass || null,
+                blockDomain: req.body.blockDomain || null,
+                allowDomain: req.body.allowDomain || null,
+                noExternals: req.body.noExternals || false
             }
         };
 
@@ -52,7 +52,6 @@ var ApiController = function(app) {
         debug('Adding test %s to the queue', run.runId);
         var queuePromise = queue.push(run.runId);
 
-
         // Save the run to the datastore
         runsDatastore.add(run, queuePromise.startingPosition);
 
@@ -71,12 +70,15 @@ var ApiController = function(app) {
 
             var runOptions = {
                 screenshot: run.params.screenshot ? screenshot.getTmpFilePath() : false,
-                jsDeepAnalysis: run.params.jsTimeline,
                 device: run.params.device,
                 waitForSelector: run.params.waitForSelector,
                 cookie: run.params.cookie,
                 authUser: run.params.authUser,
-                authPass: run.params.authPass
+                authPass: run.params.authPass,
+                blockDomain: run.params.blockDomain,
+                allowDomain: run.params.allowDomain,
+                noExternals: run.params.noExternals,
+                phantomasEngine: serverSettings.phantomasEngine
             };
 
             return ylt(run.params.url, runOptions);
@@ -95,7 +97,7 @@ var ApiController = function(app) {
             if (run.params.screenshot) {
                 
                 // Replace the empty promise created earlier with Q.resolve()
-                screenshotPromise = screenshot.toThumbnail(400)
+                screenshotPromise = screenshot.toThumbnail(serverSettings.screenshotWidth || 400)
                 
                     // Read screenshot
                     .then(function(screenshotBuffer) {
@@ -128,13 +130,7 @@ var ApiController = function(app) {
                     // Remove uneeded temp screenshot path
                     delete data.params.options.screenshot;
 
-                    // Empty javascriptExecutionTree if not needed
-                    if (!run.params.jsTimeline) {
-                        data.javascriptExecutionTree = {};
-                        data.scrollExecutionTree = {};
-                    }
-
-                    // Remove tools results if not needed
+                    // Here we can remove tools results if not needed
 
                     return resultsDatastore.saveResult(data);
                 })
@@ -331,4 +327,4 @@ var ApiController = function(app) {
 
 };
 
-module.exports = ApiController;
+module.exports = ApiController;

lib/server/controllers/frontController.js

@@ -20,7 +20,7 @@ var FrontController = function(app) {
     app.use('/img', express.static(path.join(__dirname, assetsPath, 'img'), { maxAge: cacheDuration }));
     app.use('/js', express.static(path.join(__dirname, assetsPath, 'js'), { maxAge: cacheDuration }));
     app.use('/views', express.static(path.join(__dirname, assetsPath, 'views'), { maxAge: cacheDuration }));
-    app.use('/bower_components', express.static(path.join(__dirname, '../../../bower_components'), { maxAge: cacheDuration }));
+    app.use('/node_modules', express.static(path.join(__dirname, '../../../node_modules'), { maxAge: cacheDuration }));
 };
 
 module.exports = FrontController;

lib/tools/colorDiff.js

@@ -0,0 +1,92 @@
+var debug   = require('debug')('ylt:colorDiff');
+var diff    = require('color-diff');
+var parse   = require('parse-color');
+
+
+var colorDiff = function() {
+
+    var COLOR_DIFF_THRESHOLD = 0.9;
+
+    this.compareAllColors = function(data) {
+        debug('Starting to compare all colors...');
+
+        var offenders = data.toolsResults.phantomas.offenders.cssColors;
+        var colors = (offenders) ? this.parseAllColors(offenders) : [];
+        var result = [];
+
+        // Compare colors to each other
+        for (var i = 0; i < colors.length - 1 ; i++) {
+            for (var j = i + 1; j < colors.length; j++) {
+                debug('Comparing color %s with color %s', colors[i].original, colors[j].original);
+                if (this.compareTwoColors(colors[i], colors[j])) {
+                    debug('Colors are similar!');
+                    result.push({
+                        color1: colors[i].original,
+                        color2: colors[j].original,
+                        isDark: (colors[i].Lab.L < 60)
+                    });
+                }
+            }
+        }
+
+        data.toolsResults.colorDiff = {
+            metrics: {
+                similarColors: result.length
+            },
+            offenders: {
+                similarColors: result
+            }
+        };
+
+        return data;
+    };
+
+    this.parseAllColors = function(offenders) {
+        var parsedOffenders = offenders.map(this.parseOffender);
+        var deduplicatedColors = {};
+
+        parsedOffenders.forEach(function(color) {
+            if (color !== null) {
+                deduplicatedColors[color] = color;
+            }
+        });
+
+        return Object.keys(deduplicatedColors).map(this.parseColor);
+    };
+
+    this.parseOffender = function(offender) {
+        var regexResult = /^(.*) \(\d+ times\)$/.exec(offender);
+        return regexResult ? regexResult[1] : null;
+    };
+
+    this.parseColor = function(color) {
+        var colorArray = parse(color).rgba;
+
+        var obj = {
+            R: colorArray[0],
+            G: colorArray[1],
+            B: colorArray[2],
+            A: colorArray[3]
+        };
+
+        obj.Lab = diff.rgb_to_lab(obj);
+        obj.original = color;
+
+        return obj;
+    };
+
+    this.compareTwoColors = function(color1, color2) {
+        if (color1.A !== color2.A) {
+            debug('Comparison not possible, because the alpha channel is different');
+            return false;
+        }
+
+        var delta = diff.diff(color1.Lab, color2.Lab);
+        debug('Delta is %d', delta);
+        
+        return delta <= COLOR_DIFF_THRESHOLD;
+    };
+
+};
+
+module.exports = new colorDiff();

lib/tools/isHttp2.js

@@ -0,0 +1,105 @@
+var debug   = require('debug')('ylt:isHttp2');
+var url     = require('url');
+var Q       = require('q');
+var http2   = require('is-http2');
+
+var isHttp2 = function() {
+    'use strict';
+
+    this.check = function(data) {
+        debug('Starting to check for HTTP2 support...');
+
+        return this.checkHttp2(data)
+
+            .then(function(result) {
+
+                if (result.isHttp) {
+                    debug('The website is not even in HTTPS');
+
+                    data.toolsResults.http2 = {
+                        metrics: {
+                            http2: false
+                        }
+                    };
+
+                } else if (result.isHttp2) {
+                    debug('HTTP/2 (or SPDY) is supported');
+
+                    data.toolsResults.http2 = {
+                        metrics: {
+                            http2: true
+                        }
+                    };
+                
+                } else {
+                    debug('HTTP/2 is not supported');
+                    
+                    data.toolsResults.http2 = {
+                        metrics: {
+                            http2: false
+                        }
+                    };
+                }
+
+                // Add the supported protocols as offenders
+                if (result.supportedProtocols) {
+                    debug('Supported protocols: ' + result.supportedProtocols.join(' '));
+                    data.toolsResults.http2.offenders = {
+                        http2: result.supportedProtocols
+                    };
+                }
+
+                debug('End of HTTP2 support check');
+
+                return data;
+            })
+
+            .fail(function() {
+                return data;
+            });
+    };
+
+    this.getParsedUrl = function(data) {
+        return url.parse(data.toolsResults.phantomas.url);
+    };
+
+    this.getProtocol = function(data) {
+        return this.getParsedUrl(data).protocol;
+    };
+
+    this.getDomain = function(data) {
+        return this.getParsedUrl(data).hostname;
+    };
+
+    this.checkHttp2 = function(data) {
+        var deferred = Q.defer();
+
+        // Check if it's HTTPS first
+        if (this.getProtocol(data) === 'http:') {
+            
+            deferred.resolve({
+                isHttp: true
+            });
+
+        } else {
+
+            // To make is-http2 work, you need to have openssl in a version greater than 1.0.0 installed and available in your $path.
+            http2(this.getDomain(data), {includeSpdy: true})
+
+                .then(function(result) {
+                    deferred.resolve(result);
+                })
+                
+                .catch(function(error) {
+                    debug('Error while checking HTTP2 support:');
+                    debug(error);
+                    deferred.reject('Error while checking for HTTP2 support');
+                });
+
+        }
+
+        return deferred.promise;
+    };
+};
+
+module.exports = new isHttp2();

lib/tools/jsExecutionTransformer.js

@@ -121,24 +121,28 @@ var jsExecutionTransformer = function() {
             debug('JS execution transformation complete');
 
 
-            debug('Starting scroll execution transformation');
-            offenders.DOMaccessesOnScroll = JSON.parse(data.toolsResults.phantomas.offenders.scrollExecutionTree[0]);
-            if (offenders.DOMaccessesOnScroll.children) {
-                offenders.DOMaccessesOnScroll.children.forEach(function(node) {
-                    
-                    // Mark a event flag
-                    if (['documentScroll', 'windowScroll', 'window.onscroll'].indexOf(node.data.type) >= 0) {
-                        node.windowPerformance = true;
-                    }
+            if (data.toolsResults.phantomas.offenders.scrollExecutionTree) {
+                debug('Starting scroll execution transformation');
+                offenders.DOMaccessesOnScroll = JSON.parse(data.toolsResults.phantomas.offenders.scrollExecutionTree[0]);
+                if (offenders.DOMaccessesOnScroll.children) {
+                    offenders.DOMaccessesOnScroll.children.forEach(function(node) {
+                        
+                        // Mark a event flag
+                        if (['documentScroll', 'windowScroll', 'window.onscroll'].indexOf(node.data.type) >= 0) {
+                            node.windowPerformance = true;
+                        }
 
-                    // Transform domPaths into objects
-                    changeListOfDomPaths(node);
-                    
-                    // Count the number of DOM accesses, by counting the tree leafs
-                    metrics.DOMaccessesOnScroll += countTreeLeafs(node);
-                });
+                        // Transform domPaths into objects
+                        changeListOfDomPaths(node);
+                        
+                        // Count the number of DOM accesses, by counting the tree leafs
+                        metrics.DOMaccessesOnScroll += countTreeLeafs(node);
+                    });
+                }
+                debug('Scroll execution transformation complete');
+            } else {
+                debug('Could not parse scrollExecutionTree');
             }
-            debug('Scroll execution transformation complete');
 
         } catch(err) {
             throw err;

lib/tools/mediaQueriesChecker.js

@@ -0,0 +1,168 @@
+var debug   = require('debug')('ylt:mediaQueriesChecker');
+var parseMediaQuery = require('css-mq-parser');
+var offendersHelpers = require('../offendersHelpers');
+
+
+var mediaQueriesChecker = function() {
+    'use strict';
+
+    var MOBILE_MIN_BREAKPOINT = 200;
+    var MOBILE_MAX_BREAKPOINT = 300;
+
+    this.analyzeMediaQueries = function(data) {
+        debug('Starting to check all media queries...');
+
+        var offenders = data.toolsResults.phantomas.offenders.cssMediaQueries;
+        var mediaQueries = (offenders) ? this.parseAllMediaQueries(offenders) : [];
+        
+        var notMobileFirstCount = 0;
+        var notMobileFirstOffenders = [];
+
+        var breakpointsOffenders = {};
+
+        for (var i = 0; i < mediaQueries.length; i++) {
+            var item = mediaQueries[i];
+
+            if (!item) {
+                continue;
+            }
+
+            if (item.isForMobile) {
+                notMobileFirstCount += item.mediaQuery.rules;
+                notMobileFirstOffenders.push(item.mediaQuery);
+            }
+
+            for (var j = 0; j < item.breakpoints.length; j++) {
+                var breakpointString = item.breakpoints[j].string;
+                if (!breakpointsOffenders[breakpointString]) {
+                    breakpointsOffenders[breakpointString] = {
+                        count: 1,
+                        pixels: item.breakpoints[j].pixels
+                    };
+                } else {
+                    breakpointsOffenders[breakpointString].count += 1;
+                }
+            }
+        }
+
+        data.toolsResults.mediaQueriesChecker = {
+            metrics: {
+                cssMobileFirst: notMobileFirstCount,
+                cssBreakpoints: Object.keys(breakpointsOffenders).length
+            },
+            offenders: {
+                cssMobileFirst: notMobileFirstOffenders,
+                cssBreakpoints: breakpointsOffenders
+            }
+        };
+
+        debug('End of media queries check');
+
+        return data;
+    };
+
+    this.parseAllMediaQueries = function(offenders) {
+        return offenders.map(this.parseOneMediaQuery);
+    };
+
+    this.parseOneMediaQuery = function(offender) {
+        var splittedOffender = offendersHelpers.cssOffenderPattern(offender);
+        var parts = /^@media (.*) \((\d+ rules)\)$/.exec(splittedOffender.css);
+
+        if (!parts) {
+            debug('Failed to parse media query ' + offender);
+            return false;
+        }
+
+        var rulesCount = parseInt(parts[2], 10);
+        var query = parts[1];
+
+        var isForMobile = false;
+        var breakpoints = [];
+
+        try {
+
+            var ast = parseMediaQuery(query);
+
+            var min = 0;
+            var max = Infinity;
+            var pixels;
+
+            ast.forEach(function(astItem) {
+                astItem.expressions.forEach(function(expression) {
+                    if (expression.feature === 'width' || expression.feature === 'device-width') {
+                        if (astItem.inverse === false) {
+                            if (expression.modifier === 'max') {
+                                pixels = toPixels(expression.value);
+                                max = Math.min(max, pixels);
+                                breakpoints.push({
+                                    string: expression.value,
+                                    pixels: pixels
+                                });
+                            } else if (expression.modifier === 'min') {
+                                pixels = toPixels(expression.value);
+                                min = Math.max(min, pixels);
+                                breakpoints.push({
+                                    string: expression.value,
+                                    pixels: pixels
+                                });
+                            }
+                        } else if (astItem.inverse === true) {
+                            if (expression.modifier === 'max') {
+                                pixels = toPixels(expression.value);
+                                min = Math.max(min, pixels);
+                                breakpoints.push({
+                                    string: expression.value,
+                                    pixels: pixels
+                                });
+                            } else if (expression.modifier === 'min') {
+                                pixels = toPixels(expression.value);
+                                max = Math.min(max, pixels);
+                                breakpoints.push({
+                                    string: expression.value,
+                                    pixels: pixels
+                                });
+                            }
+                        }
+                    }
+                });
+            });
+
+            isForMobile = (min <= MOBILE_MIN_BREAKPOINT && max >= MOBILE_MAX_BREAKPOINT && max !== Infinity);
+
+        } catch(error) {
+            debug('Failed to parse media query ' + offender);
+        }
+
+        return {
+            mediaQuery: {
+                query: query,
+                rules: rulesCount,
+                file: splittedOffender.file,
+                line: splittedOffender.line,
+                column: splittedOffender.column
+            },
+            isForMobile: isForMobile,
+            breakpoints: breakpoints
+        };
+    };
+
+    // Parses a size in em, pt (or px) and returns it in px
+    function toPixels(size) {
+        var splittedSize = /^([\d\.]+)(.*)/.exec(size);
+        var value = parseFloat(splittedSize[1]);
+        var unit = splittedSize[2];
+
+        if (unit === 'em') {
+            return value * 16;
+        }
+
+        if (unit === 'pt') {
+            return value / 12 * 16;
+        }
+
+        return value;
+    }
+};
+
+module.exports = new mediaQueriesChecker();

lib/tools/phantomas/custom_modules/core/scopeYLT/scopeYLT.js

@@ -18,8 +18,8 @@ exports.module = function(phantomas) {
         responseEndTime = Date.now();
     });
 
-    phantomas.once('init', function() {
-        phantomas.evaluate(function(responseEndTime, deepAnalysis) {
+    phantomas.on('init', function() {
+        phantomas.evaluate(function(responseEndTime) {
             (function(phantomas) {
     
                 // Overwritting phantomas spy function
@@ -49,7 +49,7 @@ exports.module = function(phantomas) {
                             var err;
                             
                             // Before
-                            if (enabled) {
+                            if (enabled && callbackBefore) {
                                 callbackBefore.apply(this, arguments);
                             }
 
@@ -71,7 +71,8 @@ exports.module = function(phantomas) {
 
                                 // After
                                 if (enabled && callbackAfter) {
-                                    callbackAfter.call(this, result, arguments);
+                                    var args = Array.prototype.slice.call(arguments);
+                                    callbackAfter.apply(this, [result].concat(args));
                                 }
 
                                 if (err) {
@@ -106,48 +107,85 @@ exports.module = function(phantomas) {
                     var currentContext = root;
                     var depth = 0;
 
-                    if (deepAnalysis) {
-                        phantomas.log('Entering deep Javascript analysis mode');
-                    }
 
-                    // Add a child but don't enter his context
+                    // Add a child but don't enter its context
                     function pushContext(data) {
-                        if (depth === 0 || deepAnalysis) {
+                        
+                        // Some data is not needed on subchildren
+                        if (depth === 0) {
                             data.timestamp = Date.now() - responseEndTime;
                             data.loadingStep = phantomas.currentStep || '';
-                            currentContext.addChild(data);
                         }
+
+                        // Some data is not needed on subchildren
+                        if (depth > 0) {
+                            if (data.backtrace) {
+                                delete data.backtrace;
+                            }
+                            if (data.resultsNumber) {
+                                delete data.resultsNumber;
+                            }
+                        }
+
+                        currentContext.addChild(data);
                     }
                     
-                    // Add a child to the current context and enter his context
+                    // Add a child to the current context and enter its context
                     function enterContext(data) {
-                        if (depth === 0 || deepAnalysis) {
+                        
+                        // Some data is not needed on subchildren
+                        if (depth === 0) {
                             data.timestamp = Date.now() - responseEndTime;
                             data.loadingStep = phantomas.currentStep || '';
-                            currentContext = currentContext.addChild(data);
                         }
+
+                        // Some data is not needed on subchildren
+                        if (depth > 0) {
+                            if (data.backtrace) {
+                                delete data.backtrace;
+                            }
+                            if (data.resultsNumber) {
+                                delete data.resultsNumber;
+                            }
+                        }
+
+                        currentContext = currentContext.addChild(data);
+
                         depth ++;
                     }
                     
                     // Save given data in the current context and jump change current context to its parent
                     function leaveContext(moreData) {
-                        if (depth === 1 || deepAnalysis) {
+                        
+                        // Some data is not needed on subchildren
+                        if (depth === 1) {
                             currentContext.data.time = Date.now() - currentContext.data.timestamp - responseEndTime;
+                        }
 
-                            // Merge previous data with moreData (ovewrites if exists)
-                            if (moreData) {
-                                for (var key in moreData) {
-                                    currentContext.data[key] = moreData[key];
-                                }
+                        // Some data is not needed on subchildren
+                        if (depth > 1) {
+                            if (moreData && moreData.backtrace) {
+                                delete moreData.backtrace;
                             }
-
-                            var parent = currentContext.parent;
-                            if (parent === null) {
-                                console.error('Error: trying to close root context in ContextTree');
-                            } else {
-                                currentContext = parent;
+                            if (moreData && moreData.resultsNumber) {
+                                delete moreData.resultsNumber;
                             }
                         }
+
+                        // Merge previous data with moreData (ovewrites if exists)
+                        if (moreData) {
+                            for (var key in moreData) {
+                                currentContext.data[key] = moreData[key];
+                            }
+                        }
+
+                        var parent = currentContext.parent;
+                        if (parent === null) {
+                            console.error('Error: trying to close root context in ContextTree');
+                        } else {
+                            currentContext = parent;
+                        }
+
                         depth --;
                     }
                     
@@ -161,7 +199,6 @@ exports.module = function(phantomas) {
                         if (root !== currentContext) {
                             return null;
                         }
-                        var current = currentContext;
 
                         function recusiveRead(node) {
                             if (node.children.length === 0) {
@@ -209,6 +246,6 @@ exports.module = function(phantomas) {
                 })();
 
             })(window.__phantomas);
-        }, responseEndTime, phantomas.getParam('js-deep-analysis'));
+        }, responseEndTime);
     });
 };

lib/tools/phantomas/custom_modules/modules/ajaxReqYLT/ajaxReqYLT.js

@@ -0,0 +1,30 @@
+/**
+ * Analyzes AJAX requests
+ */
+/* global window: true */
+
+exports.version = '0.2.a';
+
+exports.module = function(phantomas) {
+    'use strict';
+
+    phantomas.setMetric('ajaxRequests'); // @desc number of AJAX requests
+    phantomas.setMetric('synchronousXHR'); // @desc number of synchronous 
+
+    phantomas.on('init', function() {
+        phantomas.evaluate(function() {
+            (function(phantomas) {
+                phantomas.spy(window.XMLHttpRequest.prototype, 'open', null, function(result, method, url, async) {
+                    phantomas.incrMetric('ajaxRequests');
+                    phantomas.addOffender('ajaxRequests', '<%s> [%s]', url, method);
+
+                    if (async === false) {
+                        phantomas.incrMetric('synchronousXHR');
+                        phantomas.addOffender('synchronousXHR', url);
+                        phantomas.log('ajaxRequests: synchronous XMLHttpRequest call to <%s>', url);
+                    }
+                }, true);
+            })(window.__phantomas);
+        });
+    });
+};

lib/tools/phantomas/custom_modules/modules/domHiddenYLT/domHiddenYLT.js

@@ -3,21 +3,23 @@
  */
 /* global document: true, Node: true, window: true */
 
-exports.version = '0.1.a';
+exports.version = '1.0.a';
 
 exports.module = function(phantomas) {
     'use strict';
 
     // total length of HTML of hidden elements (i.e. display: none)
     phantomas.setMetric('hiddenContentSize'); // @desc the size of content of hidden elements on the page (with CSS display: none) @offenders
+    phantomas.setMetric('hiddenImages'); // @desc number of hidden images that can be lazy-loaded @offenders
 
     // HTML size
     phantomas.on('report', function() {
         phantomas.evaluate(function() {
             (function(phantomas) {
-                phantomas.spyEnabled(false, 'checking the hiddenContentSize');
+                var runner = new phantomas.nodeRunner(),
+                    lazyLoadableImages = {};
 
-                var runner = new phantomas.nodeRunner();
+                phantomas.spyEnabled(false, 'analyzing hidden content');
 
                 runner.walk(document.body, function(node, depth) {
                     switch (node.nodeType) {
@@ -36,6 +38,31 @@ exports.module = function(phantomas) {
                                     }
                                 }
 
+                                // count hidden images that can be lazy loaded (issue #524)
+                                var images = [];
+                                if (node.tagName === 'IMG') {
+                                    images = [node];
+                                } else if (typeof node.querySelectorAll === 'function') {
+                                    images = node.querySelectorAll('img') || [];
+                                }
+
+                                for (var i = 0, len = images.length; i < len; i++) {
+                                    var src = images[i].src,
+                                        path;
+
+                                    if (src === '' || src.indexOf('data:image') === 0) continue;
+
+                                    if (images[i].width === 1 && images[i].height === 1) continue;
+
+                                    if (!lazyLoadableImages[src]) {
+                                        path = phantomas.getDOMPath(images[i]);
+
+                                        lazyLoadableImages[src] = {
+                                            path: path
+                                        };
+                                    }
+                                }
+
                                 // don't run for child nodes as they're hidden as well
                                 return false;
                             }
@@ -43,6 +70,15 @@ exports.module = function(phantomas) {
                     }
                 });
 
+                Object.keys(lazyLoadableImages).forEach(function(img) {
+                    var entry = lazyLoadableImages[img];
+
+                    phantomas.incrMetric('hiddenImages');
+                    phantomas.addOffender('hiddenImages', img);
+
+                    phantomas.log('hiddenImages: <%s> image (%s) is hidden and can be lazy-loaded', img, entry.path);
+                });
+
                 phantomas.spyEnabled(true);
             }(window.__phantomas));
         });

lib/tools/phantomas/custom_modules/modules/domQYLT/domQYLT.js

@@ -19,7 +19,7 @@ exports.module = function(phantomas) {
     phantomas.setMetric('DOMqueriesAvoidable'); // @desc number of repeated uses of a duplicated query
 
     // fake native DOM functions
-    phantomas.once('init', function() {
+    phantomas.on('init', function() {
         phantomas.evaluate(function() {
             (function(phantomas) {
                 function querySpy(type, query, fnName, context, hasNoResults) {

lib/tools/phantomas/custom_modules/modules/eventYLT/eventYLT.js

@@ -12,7 +12,7 @@ exports.module = function(phantomas) {
     phantomas.setMetric('eventsDispatched'); // @desc number of EventTarget.dispatchEvent calls
     phantomas.setMetric('eventsScrollBound'); // @desc number of scroll event bounds
 
-    phantomas.once('init', function() {
+    phantomas.on('init', function() {
         phantomas.evaluate(function() {
             (function(phantomas) {
                 // spy calls to EventTarget.addEventListener

lib/tools/phantomas/custom_modules/modules/jQYLT/jQYLT.js

@@ -138,10 +138,9 @@ exports.module = function(phantomas) {
     jQueryFunctions = jQueryFunctions.concat(jQueryTraversalFunctions);
 
     // spy calls to jQuery functions
-    phantomas.once('init', function() {
+    phantomas.on('init', function() {
         phantomas.evaluate(function(jQueryFunctions, jQueryTraversalFunctions) {
             (function(phantomas) {
-                var jQuery;
                 var oldJQuery;
 
                 phantomas.spyGlobalVar('jQuery', function(jQuery) {
@@ -371,14 +370,6 @@ exports.module = function(phantomas) {
     });
 
 
-    // store the last resource that was received
-    // try to report where given jQuery version was loaded from
-    phantomas.on('recv', function(entry) {
-        if (entry.isJS) {
-            lastUrl = entry.url;
-        }
-    });
-
     phantomas.on('jQueryLoaded', function(version) {
         phantomas.log('jQuery: loaded v' + version);
         phantomas.setMetric('jQueryVersion', version);
@@ -386,7 +377,5 @@ exports.module = function(phantomas) {
         // report multiple jQuery "instances" (issue #435)
         phantomas.incrMetric('jQueryVersionsLoaded');
         phantomas.addOffender('jQueryVersionsLoaded', 'v%s', version);
-
-        phantomas.log('jQuery: v%s (probably loaded from <%s>)', version, lastUrl);
     });
 };

lib/tools/phantomas/custom_modules/modules/javaScriptBottleYLT/javaScriptBottleYLT.js

@@ -23,7 +23,7 @@ exports.module = function(phantomas) {
         phantomas.log('javaScriptBottlenecks: to spy calls to eval() run phantomas with --spy-eval option');
     }
 
-    phantomas.once('init', function() {
+    phantomas.on('init', function() {
         phantomas.evaluate(function(spyEval) {
             (function(phantomas) {
                 function report(msg, caller, backtrace, metric) {

lib/tools/phantomas/custom_modules/modules/lazyLoadableYLT/lazyLoadableYLT.js

@@ -0,0 +1,79 @@
+/**
+ * Analyzes images and detects which one can be lazy-loaded (are below the fold)
+ *
+ * @see https://github.com/macbre/phantomas/issues/494
+ */
+/* global document: true, window: true */
+
+exports.version = '1.0.a';
+
+exports.module = function(phantomas) {
+    'use strict';
+    
+    phantomas.setMetric('lazyLoadableImagesBelowTheFold'); // @desc number of images displayed below the fold that can be lazy-loaded
+
+    phantomas.on('report', function() {
+        phantomas.log('lazyLoadableImages: analyzing which images can be lazy-loaded...');
+
+        phantomas.evaluate(function() {
+            (function(phantomas) {
+                phantomas.spyEnabled(false, 'analyzing which images can be lazy-loaded');
+
+                var images = document.body.getElementsByTagName('img'),
+                    i,
+                    len = images.length,
+                    offset,
+                    path,
+                    processedImages = {},
+                    src,
+                    viewportHeight = window.innerHeight;
+
+                phantomas.log('lazyLoadableImages: %d image(s) found, assuming %dpx offset to be the fold', len, viewportHeight);
+
+                for (i = 0; i < len; i++) {
+                    // @see https://developer.mozilla.org/en-US/docs/Web/API/Element/getBoundingClientRect
+                    offset = images[i].getBoundingClientRect().top;
+                    src = images[i].src;
+
+                    // ignore base64-encoded images
+                    if (src === '' || /^data:/.test(src)) {
+                        continue;
+                    }
+
+                    path = phantomas.getDOMPath(images[i]);
+
+                    // get the most top position for a given image (deduplicate by src)
+                    if (typeof processedImages[src] === 'undefined') {
+                        processedImages[src] = {
+                            offset: offset,
+                            path: path
+                        };
+                    }
+
+                    // maybe there's the same image loaded above the fold?
+                    if (offset < processedImages[src].offset) {
+                        processedImages[src] = {
+                            offset: offset,
+                            path: path
+                        };
+                    }
+                }
+
+                phantomas.log('lazyLoadableImages: checking %d unique image(s)', Object.keys(processedImages).length);
+
+                Object.keys(processedImages).forEach(function(src) {
+                    var img = processedImages[src];
+
+                    if (img.offset > viewportHeight) {
+                        phantomas.log('lazyLoadableImages: <%s> image (%s) is below the fold (at %dpx)', src, img.path, img.offset);
+
+                        phantomas.incrMetric('lazyLoadableImagesBelowTheFold');
+                        phantomas.addOffender('lazyLoadableImagesBelowTheFold', src);
+                    }
+                });
+
+                phantomas.spyEnabled(true);
+            })(window.__phantomas);
+        });
+    });
+};

lib/tools/phantomas/phantomasWrapper.js

@@ -17,13 +17,12 @@ var PhantomasWrapper = function() {
         var deferred = Q.defer();
         var task = data.params;
 
-
         var options = {
             
             // Cusomizable options
-            'timeout': task.options.timeout || 45,
-            'js-deep-analysis': task.options.jsDeepAnalysis || false,
-            'user-agent': (task.options.device === 'desktop') ? 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.110 Safari/537.36' : null,
+            'engine': task.options.phantomasEngine || 'webkit',
+            'timeout': task.options.timeout || 30,
+            'user-agent': (task.options.device === 'desktop') ? 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) YLT Chrome/27.0.1453.110 Safari/537.36' : null,
             'tablet': (task.options.device === 'tablet'),
             'phone': (task.options.device === 'phone'),
             'screenshot': task.options.screenshot || false,
@@ -31,23 +30,28 @@ var PhantomasWrapper = function() {
             'cookie': task.options.cookie,
             'auth-user': task.options.authUser,
             'auth-pass': task.options.authPass,
+            'block-domain': task.options.blockDomain,
+            'allow-domain': task.options.allowDomain,
+            'no-externals': task.options.noExternals,
 
             // Mandatory
             'reporter': 'json:pretty',
             'analyze-css': true,
+            'ignore-ssl-errors': true,
             'skip-modules': [
-                'blockDomains', // not needed
-                'domHiddenContent', // overriden
+                'ajaxRequests', // overridden
+                'domHiddenContent', // overridden
                 'domMutations', // not compatible with webkit
-                'domQueries', // overriden
+                'domQueries', // overridden
                 'events', // overridden
                 'filmStrip', // not needed
                 'har', // not needed for the moment
                 'javaScriptBottlenecks', // needs to be launched after custom module scopeYLT
                 'jQuery', // overridden
                 'jserrors', // overridden
+                'lazyLoadableImages', //overridden
                 'pageSource', // not needed
-                'windowPerformance' // overriden
+                'windowPerformance' // overridden
             ].join(','),
             'include-dirs': [
                 path.join(__dirname, 'custom_modules/core'),
@@ -112,9 +116,14 @@ var PhantomasWrapper = function() {
 
         // It's time to launch the test!!!
         var triesNumber = 2;
+        var currentTry = 0;
 
         async.retry(triesNumber, function(cb) {
+
+            currentTry ++;
+
             var process = phantomas(task.url, options, function(err, json, results) {
+                var errorCode = err ? parseInt(err.message, 10) : null;
                 
                 if (isKilled) {
                     debug('Process was killed, too late Phantomas, sorry...');
@@ -122,28 +131,33 @@ var PhantomasWrapper = function() {
                 }
 
 
-                debug('Returning from Phantomas with error %s', err);
+                debug('Returning from Phantomas with error %s', errorCode);
 
                 // Adding some YellowLabTools errors here
                 if (json && json.metrics && (!json.metrics.javascriptExecutionTree || !json.offenders.javascriptExecutionTree)) {
-                    err = 1001;
+                    errorCode = 1001;
                 }
 
-                if (!err && (!json || !json.metrics)) {
-                    err = 1002;
+                if (!errorCode && (!json || !json.metrics)) {
+                    errorCode = 1002;
                 }
 
                 // Don't cancel test if it is a timeout and we've got some results
-                if (err === 252 && json) {
+                if (errorCode === 252 && json) {
                     debug('Timeout after ' + options.timeout + ' seconds. But it\'s not a problem, the test is valid.');
-                    err = null;
+                    errorCode = null;
                 }
 
-                if (err) {
-                    debug('Attempt failed. Error code ' + err);
+                if (errorCode) {
+                    debug('Attempt failed. Error code ' + errorCode);
                 }
 
-                cb(err, json);
+                cb(errorCode, json);
+            
+            }).fail(function() {
+                // This function is useless, but the failing promise needs to be handled,
+                // otherwise the module meow writes in the console in case of a timeout (error code 252).
+                debug('Failing promise handled');
             });
             
             phantomasPid = process.pid;

lib/tools/redownload/contentTypeChecker.js

@@ -0,0 +1,188 @@
+var debug   = require('debug')('ylt:contentTypeChecker');
+var Q       = require('q');
+var isJpg   = require('is-jpg');
+var isPng   = require('is-png');
+var isSvg   = require('is-svg');
+var isGif   = require('is-gif');
+var isWoff  = require('is-woff');
+var isWoff2 = require('is-woff2');
+var isOtf   = require('is-otf');
+var isTtf   = require('is-ttf');
+var isEot   = require('is-eot');
+
+var ContentTypeChecker = function() {
+
+    function checkContentType(entry) {
+        var deferred = Q.defer();
+
+        debug('Entering contentTypeChecker');
+        
+        // Ignore very small files as they are generally tracking pixels
+        if (entry.weightCheck && entry.weightCheck.body && entry.weightCheck.bodySize > 100) {
+            var foundType;
+
+            try {
+                foundType = findContentType(entry.weightCheck.body);
+            
+                if (!entry.contentType || entry.contentType === '') {
+                    if (foundType === null) {
+                        debug('ContentType is empty for file %s', entry.url);
+                    } else {
+                        debug('ContentType is empty for file %s. It should be %s.', entry.url, foundType.mimes[0]);
+                        entry.oldContentType = null;
+                        rewriteContentType(entry, foundType);
+                    }
+                } else {
+                    if (foundType !== null && foundType.mimes.indexOf(entry.contentType) === -1) {
+                        debug('ContentType %s is wrong for %s. It should be %s.', entry.contentType, entry.url, foundType.mimes[0]);
+                        entry.oldContentType = entry.contentType;
+                        rewriteContentType(entry, foundType);
+                    }
+                }
+
+            } catch(err) {
+                debug('Error while analyzing the contentType of %s', entry.url);
+                debug(err);
+            }
+        }
+
+        deferred.resolve(entry);
+
+        return deferred.promise;
+    }
+
+    function findContentType(body) {
+        var buffer = new Buffer(body, 'binary');
+
+        if (isJpg(buffer)) {
+            return contentTypes.jpeg;
+        }
+
+        if (isPng(buffer)) {
+            return contentTypes.png;
+        }
+
+        // https://github.com/sindresorhus/is-svg/issues/7
+        if (/<svg/.test(body) && isSvg(body)) {
+            return contentTypes.svg;
+        }
+
+        if (isGif(buffer)) {
+            return contentTypes.gif;
+        }
+
+        if (isWoff(buffer)) {
+            return contentTypes.woff;
+        }
+
+        if (isWoff2(buffer)) {
+            return contentTypes.woff2;
+        }
+
+        if (isOtf(buffer)) {
+            return contentTypes.otf;
+        }
+
+        if (isTtf(buffer)) {
+            return contentTypes.ttf;
+        }
+
+        if (isEot(buffer)) {
+            return contentTypes.eot;
+        }
+
+        return null;
+    }
+
+
+    function rewriteContentType(entry, contentTypeObj) {
+        delete(entry.isHTML);
+        delete(entry.isXML);
+        delete(entry.isCSS);
+        delete(entry.isJS);
+        delete(entry.isJSON);
+        delete(entry.isImage);
+        delete(entry.isSVG);
+        delete(entry.isVideo);
+        delete(entry.isWebFont);
+        delete(entry.isTTF);
+        delete(entry.isFavicon);
+
+        entry.contentType = contentTypeObj.mimes[0];
+        contentTypeObj.updateFn(entry);
+    }
+
+    var contentTypes = {
+        jpeg: {
+            mimes: ['image/jpeg'],
+            updateFn: function(entry) {
+                entry.type = 'image';
+                entry.isImage = true;
+            }
+        },
+        png: {
+            mimes: ['image/png'],
+            updateFn: function(entry) {
+                entry.type = 'image';
+                entry.isImage = true;
+            }
+        },
+        svg: {
+            mimes: ['image/svg+xml'],
+            updateFn: function(entry) {
+                entry.type = 'image';
+                entry.isImage = true;
+                entry.isSVG = true;
+            }
+        },
+        gif: {
+            mimes: ['image/gif'],
+            updateFn: function(entry) {
+                entry.type = 'image';
+                entry.isImage = true;
+            }
+        },
+        woff: {
+            mimes: ['application/x-font-woff', 'application/font-woff', 'font/woff'],
+            updateFn: function(entry) {
+                entry.type = 'webfont';
+                entry.isWebFont = true;
+            }
+        },
+        woff2: {
+            mimes: ['font/woff2', 'application/x-font-woff2', 'application/font-woff2'],
+            updateFn: function(entry) {
+                entry.type = 'webfont';
+                entry.isWebFont = true;
+            }
+        },
+        otf: {
+            mimes: ['application/x-font-otf', 'font/otf', 'font/opentype', 'application/x-font-opentype'],
+            updateFn: function(entry) {
+                entry.type = 'webfont';
+                entry.isWebFont = true;
+            }
+        },
+        ttf: {
+            mimes: ['application/x-font-ttf', 'font/ttf', 'application/x-font-truetype'],
+            updateFn: function(entry) {
+                entry.type = 'webfont';
+                entry.isWebFont = true;
+            }
+        },
+        eot: {
+            mimes: ['application/vnd.ms-fontobject', 'font/eot'],
+            updateFn: function(entry) {
+                entry.type = 'webfont';
+                entry.isWebFont = true;
+            }
+        }
+    };
+    
+    return {
+        checkContentType: checkContentType,
+        findContentType: findContentType
+    };
+};
+
+module.exports = new ContentTypeChecker();

lib/tools/redownload/fileMinifier.js

@@ -121,7 +121,7 @@ var FileMinifier = function() {
             });
 
         } else {
-            debug('Not minfiable type or already minified');
+            debug('Not minifiable type or already minified');
             deferred.resolve(entry);
         }
 
@@ -146,7 +146,16 @@ var FileMinifier = function() {
         .delay(1)
         .then(splittedUglifyStep2)
         .delay(1)
-        .then(splittedUglifyStep3)
+        .then(function(ast) {
+            // Only do the compression step for smaller files
+            // otherwise it can take a very long time compared to the gain
+            if (body.length < 200*1024) {
+                return splittedUglifyStep3(ast);
+            } else {
+                debug('Skipping step 3 because the file is too big (%d bytes)!', body.length);
+                return ast;
+            }
+        })
         .delay(1)
         .then(splittedUglifyStep4)
         .delay(1)
@@ -311,13 +320,13 @@ var FileMinifier = function() {
         return result;
     }
 
-    // Avoid loosing tome trying to compress JS files if they alreay look minified
+    // Avoid loosing some trying to compress JS files if they alreay look minified
     // by counting the number of lines compared to the total size.
-    // Less than 1000kb per line is suspicious
+    // Less than 2KB per line is suspicious
     function looksAlreadyMinified(code) {
         var linesCount = code.split(/\r\n|\r|\n/).length;
         var linesRatio = code.length / linesCount;
-        var looksMinified = linesRatio > 1024;
+        var looksMinified = linesRatio > 2 * 1024;
         
         debug('Lines ratio is %d bytes per line', Math.round(linesRatio));
         debug(looksMinified ? 'It looks already minified' : 'It doesn\'t look minified');

lib/tools/redownload/gzipCompressor.js

@@ -6,6 +6,7 @@ var zlib    = require('zlib');
 var GzipCompressor = function() {
 
     function compressFile(entry) {
+        debug('Entering compressFile');
         return gzipUncompressedFile(entry)
 
         .then(gzipOptimizedFile);
@@ -41,7 +42,7 @@ var GzipCompressor = function() {
                 }
             });
         } else {
-            
+            debug('Compression not needed');
             deferred.resolve(entry);
         }
 
@@ -73,6 +74,7 @@ var GzipCompressor = function() {
                 }
             });
         } else {
+            debug('Compressing optimized file not needed');
             deferred.resolve(entry);
         }
 

lib/tools/redownload/imageOptimizer.js

@@ -1,8 +1,10 @@
-var debug = require('debug')('ylt:imageOptimizer');
-
-var Q           = require('q');
-var Imagemin    = require('imagemin');
-var jpegoptim   = require('imagemin-jpegoptim');
+var debug               = require('debug')('ylt:imageOptimizer');
+var Q                   = require('q');
+var imagemin            = require('imagemin');
+var imageminJpegtran    = require('imagemin-jpegtran');
+var imageminJpegoptim   = require('imagemin-jpegoptim');
+var imageminOptipng     = require('imagemin-optipng');
+var imageminSvgo        = require('imagemin-svgo');
 
 var ImageOptimizer = function() {
 
@@ -34,7 +36,7 @@ var ImageOptimizer = function() {
                     return entry;
                 }
 
-                var newFileSize = newFile.contents.length;
+                var newFileSize = newFile.length;
 
                 debug('JPEG lossless compression complete for %s', entry.url);
                 
@@ -55,11 +57,13 @@ var ImageOptimizer = function() {
                     return entry;
                 }
 
-                var newFileSize = newFile.contents.length;
+                var newFileSize = newFile.length;
 
                 debug('JPEG lossy compression complete for %s', entry.url);
 
-                if (gainIsEnough(fileSize, newFileSize)) {
+                if (entry.weightCheck.lossless && entry.weightCheck.lossless < newFileSize) {
+                    debug('Lossy compression is not as good as lossless compression. Skipping the lossy.');
+                } else if (gainIsEnough(fileSize, newFileSize)) {
                     
                     if (entry.weightCheck.isOptimized !== false || newFileSize < entry.weightCheck.lossless) {
                         entry.weightCheck.optimized = newFileSize;
@@ -90,7 +94,7 @@ var ImageOptimizer = function() {
                     return entry;
                 }
                 
-                var newFileSize = newFile.contents.length;
+                var newFileSize = newFile.length;
 
                 debug('PNG lossless compression complete for %s', entry.url);
                 
@@ -124,12 +128,12 @@ var ImageOptimizer = function() {
                     return entry;
                 }
 
-                var newFileSize = newFile.contents.length;
+                var newFileSize = newFile.length;
 
                 debug('SVG lossless compression complete for %s', entry.url);
                 
                 if (gainIsEnough(fileSize, newFileSize)) {
-                    entry.weightCheck.bodyAfterOptimization = newFile.contents.toString();
+                    entry.weightCheck.bodyAfterOptimization = newFile.toString();
                     entry.weightCheck.lossless = entry.weightCheck.optimized = newFileSize;
                     entry.weightCheck.isOptimized = false;
                     debug('Filesize is %d bytes smaller (-%d%)', fileSize - newFileSize, Math.round((fileSize - newFileSize) * 100 / fileSize));
@@ -194,35 +198,29 @@ var ImageOptimizer = function() {
 
         var engine;
         if (type === 'jpeg' && !lossy) {
-            engine = Imagemin.jpegtran();
+            engine = imageminJpegtran({progressive: true});
         } else if (type === 'jpeg' && lossy) {
-            engine = jpegoptim({max: MAX_JPEG_QUALITY});
+            engine = imageminJpegoptim({progressive: true, max: MAX_JPEG_QUALITY});
         } else if (type === 'png' && !lossy) {
-            engine = Imagemin.optipng({optimizationLevel: OPTIPNG_COMPRESSION_LEVEL});
+            engine = imageminOptipng({optimizationLevel: OPTIPNG_COMPRESSION_LEVEL});
         } else if (type === 'svg' && !lossy) {
-            engine = Imagemin.svgo();
+            engine = imageminSvgo({ plugins: [ { removeUselessDefs: false } ] });
         } else {
             deferred.reject('No optimization engine found for imagemin');
         }
 
-        try {
 
-            new Imagemin()
-                .src(imageBody)
-                .use(engine)
-                .run(function (err, files) {
-                    if (err) {
-                        deferred.reject(err);
-                    } else {
-                        deferred.resolve(files[0]);
-                        var endTime = Date.now();
-                        debug('Optimization for %s took %d ms', type, endTime - startTime);
-                    }
-                });
+        imagemin.buffer(imageBody, {use: engine})
 
-            } catch(err) {
-                deferred.reject(err);
-            }
+        .then(function(file) {
+            var endTime = Date.now();
+            debug('Optimization for %s took %d ms', type, endTime - startTime);
+            deferred.resolve(file);
+        })
+
+        .catch(function(err) {
+            deferred.reject(file);
+        });
 
         return deferred.promise;
     }

lib/tools/redownload/redownload.js

@@ -4,20 +4,23 @@
  *
  */
 
+/*jshint -W069 */
 
-var debug           = require('debug')('ylt:weightChecker');
-var Q               = require('q');
-var http            = require('http');
-var zlib            = require('zlib');
-var async           = require('async');
-var request         = require('request');
+var debug               = require('debug')('ylt:redownload');
+var Q                   = require('q');
+var http                = require('http');
+var zlib                = require('zlib');
+var async               = require('async');
+var request             = require('request');
+var md5                 = require('md5');
 
-var imageOptimizer  = require('./imageOptimizer');
-var fileMinifier    = require('./fileMinifier');
-var gzipCompressor  = require('./gzipCompressor');
+var imageOptimizer      = require('./imageOptimizer');
+var fileMinifier        = require('./fileMinifier');
+var gzipCompressor      = require('./gzipCompressor');
+var contentTypeChecker  = require('./contentTypeChecker');
 
 
-var WeightChecker = function() {
+var Redownload = function() {
 
     var MAX_PARALLEL_DOWNLOADS = 10;
     var REQUEST_TIMEOUT = 15000; // 15 seconds
@@ -33,11 +36,21 @@ var WeightChecker = function() {
         delete data.toolsResults.phantomas.metrics.requestsList;
         delete data.toolsResults.phantomas.offenders.requestsList;
 
+        var httpAuth = null;
+        if (data.params && data.params.options && data.params.options.authUser && data.params.options.authPass) {
+            httpAuth = {
+                username: data.params.options.authUser,
+                password: data.params.options.authPass
+            };
+        }
+
         // Transform every request into a download function with a callback when done
         var redownloadList = requestsList.map(function(entry) {
             return function(callback) {
                 
-                redownloadEntry(entry)
+                redownloadEntry(entry, httpAuth)
+
+                .then(contentTypeChecker.checkContentType)
 
                 .then(imageOptimizer.optimizeImage)
 
@@ -46,6 +59,7 @@ var WeightChecker = function() {
                 .then(gzipCompressor.compressFile)
 
                 .then(function(newEntry) {
+                    debug('File %s - Redownloaded, optimized, minified, compressed: done', entry.url);
                     callback(null, newEntry);
                 })
 
@@ -74,16 +88,32 @@ var WeightChecker = function() {
                 offenders.totalRequests = listRequestsByType(results);
                 metrics.totalRequests = offenders.totalRequests.total;
 
-                // Remove unwanted requests (redirections, about:blank)
+
+                // Remove unwanted responses (redownload failed, about:blank)
                 results = results.filter(function(result) {
-                    return (result !== null && result.weightCheck && result.weightCheck.bodySize > 0);
+                    return (result !== null && result.weightCheck && result.url.indexOf('about:blank') !== 0);
                 });
 
 
+                // Wrong contentType
+                offenders.incorrectContentTypes = listIncorrectContentTypes(results);
+                metrics.incorrectContentTypes = offenders.incorrectContentTypes.length;
+
                 // Total weight
                 offenders.totalWeight = listRequestWeight(results);
                 metrics.totalWeight = offenders.totalWeight.totalWeight;
 
+                // Empty files
+                offenders.emptyRequests = listEmptyRequests(results);
+                metrics.emptyRequests = offenders.emptyRequests.length;
+
+
+                // Now remove unwanted responses (redirections and empty files)
+                results = results.filter(function(result) {
+                    return ((result.status < 300 || result.status >= 400) && result.weightCheck.bodySize > 0);
+                });
+
+
                 // Image compression
                 offenders.imageOptimization = listImageNotOptimized(results);
                 metrics.imageOptimization = offenders.imageOptimization.totalGain;
@@ -100,7 +130,12 @@ var WeightChecker = function() {
                 offenders.smallRequests = listSmallRequests(results);
                 metrics.smallRequests = offenders.smallRequests.total;
 
-                data.toolsResults.weightChecker = {
+                // Detect identical files
+                offenders.identicalFiles = listIdenticalFiles(results);
+                metrics.identicalFiles = offenders.identicalFiles.avoidableRequests;
+
+
+                data.toolsResults.redownload = {
                     metrics: metrics,
                     offenders: offenders
                 };
@@ -112,6 +147,35 @@ var WeightChecker = function() {
         return deferred.promise;
     }
 
+    function listIncorrectContentTypes(requests) {
+        var results = [];
+        
+        requests.forEach(function(req) {
+            if (req.oldContentType || req.oldContentType === null) {
+                results.push({
+                    url: req.url,
+                    current: req.oldContentType,
+                    correct: req.contentType
+                });
+            }
+        });
+
+        return results;
+    }
+
+    function listEmptyRequests(requests) {
+        var results = [];
+        
+        requests.forEach(function(req) {
+            var weight = req.weightCheck.bodySize || req.contentLength;
+
+            if (weight === 0 && req.method === 'GET' && req.status === 200) {
+                results.push(req.url);
+            }
+        });
+
+        return results;
+    }
 
     function listRequestWeight(requests) {
         var results = {
@@ -155,6 +219,7 @@ var WeightChecker = function() {
         requests.forEach(function(req) {
             var weight = ((typeof req.weightCheck.bodySize === 'number') ? req.weightCheck.bodySize + req.weightCheck.headersSize : req.contentLength) || 0;
             var type = req.type || 'other';
+            type = (results.byType[type]) ? type : 'other';
 
             results.totalWeight += weight;
             results.byType[type].totalWeight += weight;
@@ -275,6 +340,8 @@ var WeightChecker = function() {
         requests.forEach(function(req) {
             if (req.url !== 'about:blank') {
                 var type = req.type || 'other';
+                type = (results.byType[type]) ? type : 'other';
+                
                 results.byType[type].push(req.url);
                 results.total ++;
             }
@@ -308,8 +375,47 @@ var WeightChecker = function() {
         return results;
     }
 
+    function listIdenticalFiles(requests) {
+        var hashes = {};
+        var list = [];
+        var avoidableRequestsCount = 0;
 
-    function redownloadEntry(entry) {
+        requests.forEach(function(req) {
+            var requestHash = md5(req.weightCheck.body);
+            
+            // Try to exclude tracking pixels
+            if (req.weightCheck.bodySize < 80 && req.type === 'image') {
+                return;
+            }
+
+            if (!hashes[requestHash]) {
+                hashes[requestHash] = {
+                    weight: req.weightCheck.bodySize,
+                    urls: []
+                };
+            }
+            
+            if (hashes[requestHash].urls.indexOf(req.url) === -1) {
+                hashes[requestHash].urls.push(req.url);
+            }
+        });
+
+        for (var hash in hashes) {
+            if (hashes[hash].urls.length > 1) {
+                list.push(hashes[hash]);
+                avoidableRequestsCount += hashes[hash].urls.length - 1;
+            }
+        }
+
+        return {
+            avoidableRequests: avoidableRequestsCount,
+            count: list.length,
+            list: list
+        };
+    }
+
+
+    function redownloadEntry(entry, httpAuth) {
         var deferred = Q.defer();
         
         function downloadError(message) {
@@ -355,6 +461,7 @@ var WeightChecker = function() {
         // Always add a gzip header before sending, in case the server listens to it
         var reqHeaders = entry.requestHeaders;
         reqHeaders['Accept-Encoding'] = 'gzip, deflate';
+        reqHeaders['Connection'] = 'keep-alive';
 
         var requestOptions = {
             method: entry.method,
@@ -363,6 +470,15 @@ var WeightChecker = function() {
             timeout: REQUEST_TIMEOUT
         };
 
+        // Basic auth
+        if (httpAuth) {
+            requestOptions.auth = {
+                user: httpAuth.username,
+                pass: httpAuth.password,
+                sendImmediately: false // Tries a first time without auth, wait for a 401 error before resending
+            };
+        }
+
         download(requestOptions, entry.contentType, function(error, result) {
             if (error) {
                 if (error.code === 'ETIMEDOUT') {
@@ -387,107 +503,121 @@ var WeightChecker = function() {
 
         var statusCode;
 
-        request(requestOptions)
+        try {
 
-        .on('response', function(res) {
-            
-            // Raw headers were added in NodeJS v0.12
-            // (https://github.com/joyent/node/issues/4844), but let's
-            // reconstruct them for backwards compatibility.
-            var rawHeaders = ('HTTP/' + res.httpVersion + ' ' + res.statusCode +
-                              ' ' + http.STATUS_CODES[res.statusCode] + '\r\n');
-            Object.keys(res.headers).forEach(function(headerKey) {
-                rawHeaders += headerKey + ': ' + res.headers[headerKey] + '\r\n';
-            });
-            rawHeaders += '\r\n';
+            request(requestOptions)
 
-            var uncompressedSize = 0;  // size after uncompression
-            var bodySize = 0;  // bytes size over the wire
-            var body = '';  // plain text body (after uncompressing gzip/deflate)
-            var isCompressed = false;
+            .on('response', function(res) {
+                
+                // Raw headers were added in NodeJS v0.12
+                // (https://github.com/joyent/node/issues/4844), but let's
+                // reconstruct them for backwards compatibility.
+                var rawHeaders = ('HTTP/' + res.httpVersion + ' ' + res.statusCode +
+                                  ' ' + http.STATUS_CODES[res.statusCode] + '\r\n');
+                Object.keys(res.headers).forEach(function(headerKey) {
+                    rawHeaders += headerKey + ': ' + res.headers[headerKey] + '\r\n';
+                });
+                rawHeaders += '\r\n';
 
-            function tally() {
+                var uncompressedSize = 0;  // size after uncompression
+                var bodySize = 0;  // bytes size over the wire
+                var body = '';  // plain text body (after uncompressing gzip/deflate)
+                var isCompressed = false;
 
-                if (statusCode !== 200) {
-                    callback({code: statusCode});
-                    return;
-                }
+                function tally() {
 
-                var result = {
-                    body: body,
-                    headersSize: Buffer.byteLength(rawHeaders, 'utf8'),
-                    bodySize: bodySize,
-                    isCompressed: isCompressed,
-                    uncompressedSize: uncompressedSize
-                };
-
-                callback(null, result);
-            }
-
-            switch (res.headers['content-encoding']) {
-                case 'gzip':
-
-                    var gzip = zlib.createGunzip();
-
-                    gzip.on('data', function (data) {
-                        body += data;
-                        uncompressedSize += data.length;
-                    }).on('end', function () {
-                        isCompressed = true;
-                        tally();
-                    }).on('error', function(err) {
-                        debug(err);
-                    });
-
-                    res.on('data', function (data) {
-                        bodySize += data.length;
-                    }).pipe(gzip);
-
-                    break;
-                case 'deflate':
-                    res.setEncoding('utf8');
-
-                    var deflate = zlib.createInflate();
-
-                    deflate.on('data', function (data) {
-                        body += data;
-                        uncompressedSize += data.length;
-                    }).on('end', function () {
-                        isCompressed = true;
-                        tally();
-                    }).on('error', function(err) {
-                        debug(err);
-                    });
-
-                    res.on('data', function (data) {
-                        bodySize += data.length;
-                    }).pipe(deflate);
-
-                    break;
-                default:
-                    if (contentType === 'image/jpeg' || contentType === 'image/png') {
-                        res.setEncoding('binary');
+                    if (statusCode !== 200) {
+                        callback({code: statusCode});
+                        return;
                     }
 
-                    res.on('data', function (data) {
-                        body += data;
-                        uncompressedSize += data.length;
-                        bodySize += data.length;
-                    }).on('end', function () {
-                        tally();
-                    });
+                    var result = {
+                        body: body,
+                        headersSize: Buffer.byteLength(rawHeaders, 'utf8'),
+                        bodySize: bodySize,
+                        isCompressed: isCompressed,
+                        uncompressedSize: uncompressedSize
+                    };
 
-                    break;
-            }
-        })
+                    callback(null, result);
+                }
 
-        .on('response', function(response) {
-            statusCode = response.statusCode;
-        })
+                switch (res.headers['content-encoding']) {
+                    case 'gzip':
 
-        .on('error', function(err) {
+                        var gzip = zlib.createGunzip();
+
+                        gzip.on('data', function (data) {
+                            body += data;
+                            uncompressedSize += data.length;
+                        }).on('end', function () {
+                            isCompressed = true;
+                            tally();
+                        }).on('error', function(err) {
+                            debug('Error while decoding %s', requestOptions.url);
+                            debug(err);
+                            callback(err);
+                        });
+
+                        res.on('data', function (data) {
+                            bodySize += data.length;
+                        }).pipe(gzip);
+
+                        break;
+                    case 'deflate':
+                        res.setEncoding('utf8');
+
+                        var deflate = zlib.createInflate();
+
+                        deflate.on('data', function (data) {
+                            body += data;
+                            uncompressedSize += data.length;
+                        }).on('end', function () {
+                            isCompressed = true;
+                            tally();
+                        }).on('error', function(err) {
+                            debug('Error while decoding %s', requestOptions.url);
+                            debug(err);
+                            callback(err);
+                        });
+
+                        res.on('data', function (data) {
+                            bodySize += data.length;
+                        }).pipe(deflate);
+
+                        break;
+                    default:
+                        if (contentType === 'image/jpeg' || contentType === 'image/png') {
+                            res.setEncoding('binary');
+                        }
+
+                        res.on('data', function (data) {
+                            body += data;
+                            uncompressedSize += data.length;
+                            bodySize += data.length;
+                        }).on('end', function () {
+                            tally();
+                        });
+
+                        break;
+                }
+            })
+
+            .on('response', function(response) {
+                statusCode = response.statusCode;
+            })
+
+            .on('error', function(err) {
+                debug('Error while downloading %s', requestOptions.url);
+                debug(err);
+                callback(err);
+            });
+
+        } catch(err) {
+            debug('Error while downloading %s', requestOptions.url);
+            debug(err);
             callback(err);
-        });
+        }
     }
 
     return {
@@ -498,4 +628,4 @@ var WeightChecker = function() {
     };
 };
 
-module.exports = new WeightChecker();
+module.exports = new Redownload();

package.json

@@ -1,7 +1,13 @@
 {
   "name": "yellowlabtools",
-  "version": "1.7.0",
+  "version": "1.11.0",
   "description": "Online tool to audit a webpage for performance and front-end quality issues",
+  "license": "GPL-2.0",
+  "author": {
+    "name": "Gaël Métais",
+    "email": "gael@gaelmetais.com",
+    "url": "http://www.gaelmetais.com"
+  },
   "repository": {
     "type": "git",
     "url": "git://github.com/gmetais/YellowLabTools.git"
@@ -9,55 +15,93 @@
   "bin": {
     "yellowlabtools": "./bin/cli.js"
   },
+  "engines": {
+    "node": ">= 4.0"
+  },
   "main": "./lib/index.js",
   "dependencies": {
-    "async": "1.2.1",
-    "body-parser": "1.13.1",
-    "clean-css": "3.3.0",
-    "compression": "1.5.0",
+    "angular": "1.5.7",
+    "angular-animate": "1.5.7",
+    "angular-chart.js": "0.10.2",
+    "angular-local-storage": "0.2.7",
+    "angular-resource": "1.5.7",
+    "angular-route": "1.5.7",
+    "angular-sanitize": "1.5.7",
+    "async": "1.5.2",
+    "body-parser": "1.15.2",
+    "chart.js": "1.1.1",
+    "clean-css": "3.4.18",
+    "color-diff": "1.0.0",
+    "compression": "1.6.2",
     "cors": "2.7.1",
+    "css-mq-parser": "0.0.3",
     "debug": "2.2.0",
-    "express": "4.12.4",
-    "imagemin": "3.2.0",
-    "imagemin-jpegoptim": "4.0.0",
-    "lwip": "0.0.7",
-    "meow": "3.3.0",
-    "minimize": "1.4.1",
-    "phantomas": "1.11.0",
-    "ps-node": "0.0.4",
+    "easyxml": "2.0.1",
+    "express": "4.14.0",
+    "imagemin": "5.2.2",
+    "imagemin-jpegoptim": "5.0.0",
+    "imagemin-jpegtran": "5.0.2",
+    "imagemin-optipng": "5.1.0",
+    "imagemin-svgo": "5.1.0",
+    "is-eot": "1.0.0",
+    "is-gif": "1.0.0",
+    "is-http2": "1.0.4",
+    "is-jpg": "1.0.0",
+    "is-otf": "0.1.2",
+    "is-png": "1.0.0",
+    "is-svg": "2.0.1",
+    "is-ttf": "0.2.2",
+    "is-woff": "1.0.3",
+    "is-woff2": "1.0.0",
+    "lwip": "0.0.9",
+    "md5": "2.1.0",
+    "meow": "3.7.0",
+    "minimize": "2.0.0",
+    "parse-color": "1.0.0",
+    "phantomas": "1.16.0",
+    "ps-node": "0.1.2",
     "q": "1.4.1",
-    "rimraf": "2.4.0",
+    "request": "2.72.0",
+    "rimraf": "2.5.3",
     "temporary": "0.0.8",
-    "uglify-js": "2.4.23"
+    "try-thread-sleep": "1.0.0",
+    "uglify-js": "2.7.0"
   },
   "devDependencies": {
-    "chai": "^3.0.0",
-    "grunt": "^0.4.5",
-    "grunt-blanket": "^0.0.8",
-    "grunt-contrib-clean": "^0.6.0",
-    "grunt-contrib-concat": "^0.5.1",
-    "grunt-contrib-copy": "^0.8.0",
-    "grunt-contrib-cssmin": "^0.12.3",
-    "grunt-contrib-htmlmin": "^0.4.0",
-    "grunt-contrib-jshint": "^0.11.2",
-    "grunt-contrib-less": "^1.0.1",
-    "grunt-contrib-uglify": "^0.9.1",
-    "grunt-env": "^0.4.4",
-    "grunt-express": "^1.4.1",
-    "grunt-filerev": "^2.3.1",
-    "grunt-inline-angular-templates": "^0.1.5",
-    "grunt-line-remover": "^0.0.2",
-    "grunt-mocha-test": "^0.12.7",
-    "grunt-replace": "^0.9.2",
-    "grunt-usemin": "^3.0.0",
-    "grunt-webfont": "^0.5.3",
-    "matchdep": "^0.3.0",
-    "mocha": "^2.2.5",
-    "request": "^2.58.0",
-    "sinon": "^1.15.3",
-    "sinon-chai": "^2.8.0"
+    "chai": "~3.5.0",
+    "grunt": "~0.4.5",
+    "grunt-blanket": "~0.0.10",
+    "grunt-contrib-clean": "~1.0.0",
+    "grunt-contrib-concat": "~1.0.1",
+    "grunt-contrib-copy": "~1.0.0",
+    "grunt-contrib-cssmin": "~1.0.1",
+    "grunt-contrib-htmlmin": "~1.4.0",
+    "grunt-contrib-jshint": "~1.0.0",
+    "grunt-contrib-less": "~1.3.0",
+    "grunt-contrib-uglify": "~1.0.1",
+    "grunt-env": "~0.4.4",
+    "grunt-express": "~1.4.1",
+    "grunt-filerev": "~2.3.1",
+    "grunt-inline-angular-templates": "~0.1.5",
+    "grunt-line-remover": "~0.0.2",
+    "grunt-mocha-test": "~0.12.7",
+    "grunt-replace": "~1.0.1",
+    "grunt-usemin": "~3.1.1",
+    "grunt-webfont": "~1.4.0",
+    "matchdep": "~1.0.1",
+    "mocha": "~2.5.3",
+    "sinon": "~1.17.4",
+    "sinon-chai": "~2.8.0"
   },
   "scripts": {
     "test": "grunt test"
-  }
+  },
+  "keywords": [
+    "performance",
+    "metrics",
+    "webperf",
+    "pagespeed",
+    "budget",
+    "phantomas"
+  ]
 }

server_config/maintenance.js

@@ -0,0 +1,13 @@
+var express                 = require('express');
+var app                     = express();
+var server                  = require('http').createServer(app);
+
+var settings                = require('./settings.json');
+
+app.all('*', function(req, res) {
+    res.status(500).send('YellowLabTools is in maintenance. It should come back soon with a new version!');
+});
+
+server.listen(settings.serverPort, function() {
+    console.log('Maintenance mode started on port %d', server.address().port);
+});

server_config/server_install.sh

@@ -2,17 +2,16 @@
 
 # APT-GET
 sudo apt-get update
-sudo apt-get install lsb-release libfontconfig1 libfreetype6 -y --force-yes
-sudo apt-get install curl git python-software-properties build-essential make g++ -y --force-yes
+sudo apt-get install lsb-release libfontconfig1 libfreetype6 libjpeg-dev -y --force-yes > /dev/null 2>&1
+sudo apt-get install curl git python-software-properties build-essential make g++ -y --force-yes > /dev/null 2>&1
 
 # Installation of NodeJS
-sudo add-apt-repository -y ppa:chris-lea/node.js
-sudo apt-get update
-sudo apt-get install -y nodejs
+curl -sL https://deb.nodesource.com/setup_0.12 | sudo -E bash -
+sudo apt-get install -y nodejs > /dev/null 2>&1
 source ~/.profile
 
 # Installation of some packages globally
-npm install bower forever grunt-cli -g
+npm install forever grunt-cli -g
 source ~/.profile
 
 # Installation of YellowLabTools
@@ -21,8 +20,7 @@ sudo chown $USER /space
 cd /space
 git clone https://github.com/gmetais/YellowLabTools.git --branch master
 cd YellowLabTools
-npm install
-bower install --config.interactive=false --allow-root
+npm install || exit 1
 
 # Front-end compilation
 grunt build

server_config/server_update.sh

@@ -2,23 +2,23 @@
 
 cd /space/YellowLabTools
 
-# Stop the server
+# Stop the server and start the maintenance mode
 forever stopall
+forever start server_config/maintenance.js
 
 # Keep the settings.json file
 git stash
 git pull
 git stash pop
 
-# In case something was added in package.json or bower.json
+# In case something was added in package.json
 rm -rf node_modules
-npm install
-rm -rf bower_components
-bower install --config.interactive=false --allow-root
+npm install || exit 1
 
 # Front-end compilation
 rm -rf front/build
 grunt build
 
-# Restart the server
+# Stop the maintenance mode and restart the server
+forever stopall
 NODE_ENV=production forever start -c "node --stack-size=262000" bin/server.js

server_config/settings-prod.json

@@ -1,6 +1,8 @@
 {
     "serverPort": 80,
+    "phantomasEngine": "webkit",
     "googleAnalyticsId": "",
+    "screenshotWidth": 400,
 
     "authorizedKeys": {
         

server_config/settings.json

@@ -1,6 +1,8 @@
 {
     "serverPort": 8383,
+    "phantomasEngine": "webkit",
     "googleAnalyticsId": "",
+    "screenshotWidth": 400,
 
     "authorizedKeys": {
         

test/api/apiTest.js

@@ -31,6 +31,7 @@ describe('api', function() {
             },
             json: true,
             headers: {
+                'Content-Type': 'application/json',
                 'X-Api-Key': 'invalid'
             }
         }, function(error, response, body) {
@@ -53,6 +54,7 @@ describe('api', function() {
             },
             json: true,
             headers: {
+                'Content-Type': 'application/json',
                 'X-Api-Key': Object.keys(config.authorizedKeys)[0]
             }
         }, function(error, response, body) {
@@ -76,6 +78,7 @@ describe('api', function() {
             },
             json: true,
             headers: {
+                'Content-Type': 'application/json',
                 'X-Api-Key': Object.keys(config.authorizedKeys)[0]
             }
         }, function(error, response, body) {
@@ -99,12 +102,13 @@ describe('api', function() {
                 screenshot: true,
                 device: 'tablet',
                 //waitForSelector: '*',
-                cookie: 'foo=bar',
+                cookie: 'foo=bar;domain=google.com',
                 authUser: 'joe',
                 authPass: 'secret'
             },
             json: true,
             headers: {
+                'Content-Type': 'application/json',
                 'X-Api-Key': Object.keys(config.authorizedKeys)[0]
             }
         }, function(error, response, body) {
@@ -133,6 +137,7 @@ describe('api', function() {
             },
             json: true,
             headers: {
+                'Content-Type': 'application/json',
                 'X-Api-Key': Object.keys(config.authorizedKeys)[0]
             }
         }, function(error, response, body) {
@@ -165,14 +170,13 @@ describe('api', function() {
                 body.should.have.a.property('rules').that.is.an('object');
                 body.should.have.a.property('toolsResults').that.is.an('object');
 
-                // javascriptExecutionTree should only be filled if option jsTimeline is true
                 body.should.have.a.property('javascriptExecutionTree').that.is.an('object');
-                body.javascriptExecutionTree.should.deep.equal({});
+                body.javascriptExecutionTree.should.not.deep.equal({});
 
                 // Check if settings are correctly sent and retrieved
                 body.params.options.should.have.a.property('device').that.equals('tablet');
                 //body.params.options.should.have.a.property('waitForSelector').that.equals('*');
-                body.params.options.should.have.a.property('cookie').that.equals('foo=bar');
+                body.params.options.should.have.a.property('cookie').that.equals('foo=bar;domain=google.com');
                 body.params.options.should.have.a.property('authUser').that.equals('joe');
                 body.params.options.should.have.a.property('authPass').that.equals('secret');
 
@@ -208,6 +212,7 @@ describe('api', function() {
             },
             json: true,
             headers: {
+                'Content-Type': 'application/json',
                 'X-Api-Key': Object.keys(config.authorizedKeys)[0]
             }
         }, function(error, response, body) {
@@ -232,6 +237,7 @@ describe('api', function() {
             url: serverUrl + '/api/runs/' + asyncRunId,
             json: true,
             headers: {
+                'Content-Type': 'application/json',
                 'X-Api-Key': Object.keys(config.authorizedKeys)[0]
             }
         }, function(error, response, body) {

test/core/colorDiffTest.js

@@ -0,0 +1,92 @@
+var should = require('chai').should();
+var colorDiff = require('../../lib/tools/colorDiff');
+
+describe('colorDiff', function() {
+    
+    it('should parse offenders correctly', function() {
+        colorDiff.parseOffender('#000 (1 times)').should.equal('#000');
+        colorDiff.parseOffender('#5bc0de (2 times)').should.equal('#5bc0de');
+        colorDiff.parseOffender('rgba(0,0,0,0.075) (100 times)').should.equal('rgba(0,0,0,0.075)');
+        colorDiff.parseOffender('rgb(91,192,222) (1000 times)').should.equal('rgb(91,192,222)');
+    });
+
+    it('should parse colors correctly', function() {
+        colorDiff.parseColor('#000').should.deep.equal({R: 0, G: 0, B: 0, A: 1, Lab: {L: 0, a: 0, b: 0}, original: '#000'});
+        colorDiff.parseColor('rgba(255,255,255,0.5)').should.deep.equal({R: 255, G: 255, B: 255, A: 0.5, Lab: {L: 100, a: 0.00526049995830391, b: -0.010408184525267927}, original: 'rgba(255,255,255,0.5)'});
+    });
+
+    it('should not compare colors with different alpha', function() {
+        var color1 = colorDiff.parseColor("rgba(0, 0, 0, 1)");
+        var color2 = colorDiff.parseColor("rgba(0, 0, 0, 0.5)");
+
+        colorDiff.compareTwoColors(color1, color2).should.equal(false);
+    });
+
+    it('should find that two colors are similar', function() {
+        var color1 = colorDiff.parseColor("rgba(0, 0, 0, 1)");
+        var color2 = colorDiff.parseColor("rgba(0, 0, 1, 1)");
+
+        colorDiff.compareTwoColors(color1, color2).should.equal(true);
+    });
+
+    it('should find that two colors are similar even with alpha', function() {
+        var color1 = colorDiff.parseColor("rgba(0, 0, 0, 0.3)");
+        var color2 = colorDiff.parseColor("rgba(0, 0, 2, 0.3)");
+
+        colorDiff.compareTwoColors(color1, color2).should.equal(true);
+    });
+
+    it('should find that two colors are different', function() {
+        var color1 = colorDiff.parseColor("rgba(0, 0, 0, 1)");
+        var color2 = colorDiff.parseColor("rgba(99, 99, 99, 1)");
+
+        colorDiff.compareTwoColors(color1, color2).should.equal(false);
+    });
+
+    it('should compare all colors to each other', function() {
+        var colors = [
+            '#000 (1 times)',
+            '#5bc0de (2 times)',
+            'rgba(0,0,0,0.075) (100 times)',
+            'rgb(91,192,222) (1000 times)',
+            'rgba(0,0,2,1) (1 times)',
+            'rgba(99,99,99,1) (1 times)',
+            'rgba(100,100,100,1) (1 times)'
+        ];
+
+        var data = {
+            toolsResults: {
+                phantomas: {
+                    offenders: {
+                        cssColors: colors
+                    }
+                }
+            }
+        };
+
+        newData = colorDiff.compareAllColors(data);
+
+        newData.toolsResults.should.have.a.property('colorDiff');
+        newData.toolsResults.colorDiff.should.have.a.property('metrics');
+        newData.toolsResults.colorDiff.metrics.should.have.a.property('similarColors').that.equals(3);
+        newData.toolsResults.colorDiff.should.have.a.property('offenders');
+        newData.toolsResults.colorDiff.offenders.should.have.a.property('similarColors').that.deep.equals([
+            {
+                color1: '#000',
+                color2: 'rgba(0,0,2,1)',
+                isDark: true
+            },
+            {
+                color1: '#5bc0de',
+                color2: 'rgb(91,192,222)',
+                isDark: false
+            },
+            {
+                color1: 'rgba(99,99,99,1)',
+                color2: 'rgba(100,100,100,1)',
+                isDark: true
+            }
+        ]);
+    });
+
+});

test/core/contentTypeCheckerTest.js

@@ -0,0 +1,20 @@
+var should = require('chai').should();
+var contentTypeChecker = require('../../lib/tools/redownload/contentTypeChecker');
+var fs = require('fs');
+var path = require('path');
+
+describe('contentTypeChecker', function() {
+
+    var jpgImageContent = fs.readFileSync(path.resolve(__dirname, '../www/jpeg-image.jpg'));
+    var pngImageContent = fs.readFileSync(path.resolve(__dirname, '../www/png-image.png'));
+    var svgImageContent = fs.readFileSync(path.resolve(__dirname, '../www/svg-image.svg'));
+    var cssFileContent = fs.readFileSync(path.resolve(__dirname, '../www/unminified-stylesheet.css'));
+    
+    it('detect the right content type', function() {
+        contentTypeChecker.findContentType(jpgImageContent).mimes.should.deep.equal(['image/jpeg']);
+        contentTypeChecker.findContentType(pngImageContent).mimes.should.deep.equal(['image/png']);
+        contentTypeChecker.findContentType(svgImageContent).mimes.should.deep.equal(['image/svg+xml']);
+        should.equal(contentTypeChecker.findContentType(cssFileContent), null);
+    });
+
+});

test/core/customPoliciesTest.js

@@ -183,18 +183,21 @@ describe('customPolicies', function() {
 
         var versions = {
             '1.2.9': 0,
-            '1.3.9': 0,
-            '1.4.4': 10,
-            '1.5.0': 20,
-            '1.6.3': 30,
-            '1.7.0': 40,
-            '1.8.3a': 50,
-            '1.9.2': 70,
-            '1.10.1': 90,
-            '2.0.0-rc1': 90,
-            '1.11.1': 100,
-            '2.1.1-beta1': 100,
-            '3.0.0': 100
+            '1.4.4': 0,
+            '1.5.0': 0,
+            '1.6.3': 10,
+            '1.7.0': 20,
+            '1.8.3a': 30,
+            '1.9.2': 40,
+            '1.10.1': 50,
+            '2.0.0-rc1': 50,
+            '1.11.1': 70,
+            '2.1.1-beta1': 70,
+            '1.12.1': 90,
+            '2.2.1': 90,
+            '3.0.1': 100,
+            '3.1.0': 100,
+            '3.2.1': 100
         };
 
         for (var version in versions) {

test/core/fileMinifierTest.js

@@ -1,5 +1,5 @@
 var should = require('chai').should();
-var fileMinifier = require('../../lib/tools/weightChecker/fileMinifier');
+var fileMinifier = require('../../lib/tools/redownload/fileMinifier');
 var fs = require('fs');
 var path = require('path');
 

test/core/gzipCompressorTest.js

@@ -1,6 +1,6 @@
 var should = require('chai').should();
-var gzipCompressor = require('../../lib/tools/weightChecker/gzipCompressor');
-var fileMinifier = require('../../lib/tools/weightChecker/fileMinifier');
+var gzipCompressor = require('../../lib/tools/redownload/gzipCompressor');
+var fileMinifier = require('../../lib/tools/redownload/fileMinifier');
 var fs = require('fs');
 var path = require('path');
 

test/core/imageOptimizerTest.js

@@ -1,5 +1,5 @@
 var should = require('chai').should();
-var imageOptimizer = require('../../lib/tools/weightChecker/imageOptimizer');
+var imageOptimizer = require('../../lib/tools/redownload/imageOptimizer');
 var fs = require('fs');
 var path = require('path');
 
@@ -11,7 +11,7 @@ describe('imageOptimizer', function() {
         var fileSize = fileContent.length;
 
         imageOptimizer.compressJpegLosslessly(fileContent).then(function(newFile) {
-            var newFileSize = newFile.contents.length;
+            var newFileSize = newFile.length;
             newFileSize.should.be.below(fileSize);
             done();
         }).fail(function(err) {
@@ -25,7 +25,7 @@ describe('imageOptimizer', function() {
         var fileSize = fileContent.length;
 
         imageOptimizer.compressJpegLossly(fileContent).then(function(newFile) {
-            var newFileSize = newFile.contents.length;
+            var newFileSize = newFile.length;
             newFileSize.should.be.below(fileSize);
             done();
         }).fail(function(err) {
@@ -82,7 +82,7 @@ describe('imageOptimizer', function() {
         var fileSize = fileContent.length;
 
         imageOptimizer.compressPngLosslessly(fileContent).then(function(newFile) {
-            var newFileSize = newFile.contents.length;
+            var newFileSize = newFile.length;
             newFileSize.should.be.below(fileSize);
             done();
         }).fail(function(err) {
@@ -96,7 +96,7 @@ describe('imageOptimizer', function() {
         var fileSize = fileContent.length;
 
         imageOptimizer.compressPngLosslessly(fileContent).then(function(newFile) {
-            var newFileSize = newFile.contents.length;
+            var newFileSize = newFile.length;
             newFileSize.should.equal(fileSize);
             done();
         }).fail(function(err) {
@@ -110,7 +110,7 @@ describe('imageOptimizer', function() {
         var fileSize = fileContent.length;
 
         imageOptimizer.compressPngLosslessly(fileContent).then(function(newFile) {
-            var newFileSize = newFile.contents.length;
+            var newFileSize = newFile.length;
             newFileSize.should.equal(fileSize);
             done();
         }).fail(function(err) {
@@ -166,7 +166,7 @@ describe('imageOptimizer', function() {
         var fileSize = fileContent.length;
 
         imageOptimizer.compressSvgLosslessly(fileContent).then(function(newFile) {
-            var newFileSize = newFile.contents.length;
+            var newFileSize = newFile.length;
             newFileSize.should.be.below(fileSize);
             done();
         }).fail(function(err) {

test/core/isHttp2Test.js

@@ -0,0 +1,48 @@
+var should = require('chai').should();
+var isHttp2 = require('../../lib/tools/isHttp2');
+
+describe('isHttp2', function() {
+    
+    it('should parse the protocol correctly', function() {
+        isHttp2.getProtocol({
+            toolsResults: {
+                phantomas: {
+                    url: 'http://www.yahoo.com'
+                }
+            }
+        }).should.equal('http:');
+
+
+        isHttp2.getProtocol({
+            toolsResults: {
+                phantomas: {
+                    url: 'https://www.yahoo.com'
+                }
+            }
+        }).should.equal('https:');
+    });
+
+    it('should parse the domain correctly', function() {
+        isHttp2.getDomain({
+            toolsResults: {
+                phantomas: {
+                    url: 'http://www.yahoo.com'
+                }
+            }
+        }).should.equal('www.yahoo.com');
+
+
+        isHttp2.getDomain({
+            toolsResults: {
+                phantomas: {
+                    url: 'https://www.yahoo.com'
+                }
+            }
+        }).should.equal('www.yahoo.com');
+    });
+
+    it('should have a function checkHttp2', function() {
+        isHttp2.should.have.a.property('checkHttp2').that.is.a('function');
+    });
+
+});

test/core/mediaQueriesCheckerTest.js

@@ -0,0 +1,55 @@
+var should = require('chai').should();
+var mediaQueriesChecker = require('../../lib/tools/mediaQueriesChecker');
+
+describe('mediaQueriesChecker', function() {
+    
+    it('should parse mediaQueryes correctly', function() {
+        mediaQueriesChecker.parseOneMediaQuery('@media screen and (max-width: 1024px) (1 rules) <http://domain.com/css/stylesheet.css> @ 269:1').should.deep.equal({
+            mediaQuery: {
+                query: 'screen and (max-width: 1024px)',
+                rules: 1,
+                file: 'http://domain.com/css/stylesheet.css',
+                line: 269,
+                column: 1
+            },
+            isForMobile: true,
+            breakpoints: [{string: '1024px', pixels: 1024}]
+        });
+    });
+
+    it('should determine if it\'s for mobile correctly', function() {
+        mediaQueriesChecker.parseOneMediaQuery('@media screen and (max-width: 1024px) (1 rules) <file> @ 1:1').isForMobile.should.equal(true);
+        mediaQueriesChecker.parseOneMediaQuery('@media (max-width:1024px) (1 rules) <file> @ 1:1').isForMobile.should.equal(true);
+        mediaQueriesChecker.parseOneMediaQuery('@media screen and (max-width: 320px) (1 rules) <file> @ 1:1').isForMobile.should.equal(true);
+        mediaQueriesChecker.parseOneMediaQuery('@media screen and (max-width: 290px) (1 rules) <file> @ 1:1').isForMobile.should.equal(false);
+        mediaQueriesChecker.parseOneMediaQuery('@media screen and (min-width: 320px) (1 rules) <file> @ 1:1').isForMobile.should.equal(false);
+        mediaQueriesChecker.parseOneMediaQuery('@media screen and (min-width: 600px) (1 rules) <file> @ 1:1').isForMobile.should.equal(false);
+        mediaQueriesChecker.parseOneMediaQuery('@media screen and (max-width: 20em) (1 rules) <file> @ 1:1').isForMobile.should.equal(true);
+        mediaQueriesChecker.parseOneMediaQuery('@media screen and (min-width: 40em) (1 rules) <file> @ 1:1').isForMobile.should.equal(false);
+        mediaQueriesChecker.parseOneMediaQuery('@media (min-width: 600px) and (max-width: 1000px) (1 rules) <file> @ 1:1').isForMobile.should.equal(false);
+        mediaQueriesChecker.parseOneMediaQuery('@media (min-width: 180px) and (max-width: 400px) (1 rules) <file> @ 1:1').isForMobile.should.equal(true);
+        mediaQueriesChecker.parseOneMediaQuery('@media (min-width: 180px) and (max-width: 290px) (1 rules) <file> @ 1:1').isForMobile.should.equal(false);
+        mediaQueriesChecker.parseOneMediaQuery('@media not all and (min-width: 600px) (1 rules) <file> @ 1:1').isForMobile.should.equal(true);
+        mediaQueriesChecker.parseOneMediaQuery('@media not all and (min-width: 180px) (1 rules) <file> @ 1:1').isForMobile.should.equal(false);
+        mediaQueriesChecker.parseOneMediaQuery('@media not all and (min-width: 1000px) and (max-width: 600px) (1 rules) <file> @ 1:1').isForMobile.should.equal(false);
+        mediaQueriesChecker.parseOneMediaQuery('@media not all and (min-width: 400px) and (max-width: 180px) (1 rules) <file> @ 1:1').isForMobile.should.equal(true);
+    });
+
+    it('should count breakpoints correctly', function() {
+        mediaQueriesChecker.parseOneMediaQuery('@media screen and (max-width: 1024px) (1 rules) <file> @ 1:1').breakpoints.should.deep.equal([{string: '1024px', pixels: 1024}]);
+        mediaQueriesChecker.parseOneMediaQuery('@media (max-width:1024px) (1 rules) <file> @ 1:1').breakpoints.should.deep.equal([{string: '1024px', pixels: 1024}]);
+        mediaQueriesChecker.parseOneMediaQuery('@media screen and (max-width: 320px) (1 rules) <file> @ 1:1').breakpoints.should.deep.equal([{string: '320px', pixels: 320}]);
+        mediaQueriesChecker.parseOneMediaQuery('@media (min-width: 600px) and (max-width: 1000px) (1 rules) <file> @ 1:1').breakpoints.should.deep.equal([{string: '600px', pixels: 600}, {string: '1000px', pixels: 1000}]);
+        mediaQueriesChecker.parseOneMediaQuery('@media not all and (min-width: 180px) (1 rules) <file> @ 1:1').breakpoints.should.deep.equal([{string: '180px', pixels: 180}]);
+        mediaQueriesChecker.parseOneMediaQuery('@media not all and (min-width: 1000px) and (max-width: 600px) (1 rules) <file> @ 1:1').breakpoints.should.deep.equal([{string: '1000px', pixels: 1000}, {string: '600px', pixels: 600}]);
+        mediaQueriesChecker.parseOneMediaQuery('@media (max-height:500px) (1 rules) <file> @ 1:1').breakpoints.should.deep.equal([]);
+        mediaQueriesChecker.parseOneMediaQuery('@media screen and (max-width: 100em) (1 rules) <file> @ 1:1').breakpoints.should.deep.equal([{string: '100em', pixels: 1600}]);
+        mediaQueriesChecker.parseOneMediaQuery('@media screen and (max-width: 120pt) (1 rules) <file> @ 1:1').breakpoints.should.deep.equal([{string: '120pt', pixels: 160}]);
+        mediaQueriesChecker.parseOneMediaQuery('@media screen and (max-width: 40.2em) (1 rules) <file> @ 1:1').breakpoints.should.deep.equal([{string: '40.2em', pixels: 643.2}]);
+    });
+
+    it('should fail silently', function() {
+        mediaQueriesChecker.parseOneMediaQuery('@media bad syntax (1 rules) <file> @ 1:1').isForMobile.should.equal(false); 
+    });
+
+});

test/core/offendersHelpersTest.js

@@ -200,6 +200,47 @@ describe('offendersHelpers', function() {
             ]);
         });
 
+        it('should transform a backtrace with the new PhantomJS 2.x syntax into an array', function() {
+            var result = offendersHelpers.backtraceToArray('each@http://m.australia.fr/js/min/vendors.js?20160706185900:4:5365 / f@http://m.australia.fr/js/min/vendors.js?20160706185900:17:82 / http://m.australia.fr/js/min/vendors.js?20160706185900:17:855 / handle@http://m.australia.fr/js/min/vendors.js?20160706185900:5:10871 / report@phantomjs://platform/phantomas.js:535:20 / phantomjs://platform/phantomas.js:524:15');
+
+            result.should.deep.equal([
+                {
+                    functionName: 'each',
+                    file: 'http://m.australia.fr/js/min/vendors.js?20160706185900',
+                    line: 4,
+                    column: 5365
+                },
+                {
+                    functionName: 'f',
+                    file: 'http://m.australia.fr/js/min/vendors.js?20160706185900',
+                    line: 17,
+                    column: 82
+                },
+                {
+                    file: 'http://m.australia.fr/js/min/vendors.js?20160706185900',
+                    line: 17,
+                    column: 855
+                },
+                {
+                    functionName: 'handle',
+                    file: 'http://m.australia.fr/js/min/vendors.js?20160706185900',
+                    line: 5,
+                    column: 10871
+                },
+                {
+                    functionName: 'report',
+                    file: 'phantomjs://platform/phantomas.js',
+                    line: 535,
+                    column: 20
+                },
+                {
+                    file: 'phantomjs://platform/phantomas.js',
+                    line: 524,
+                    column: 15
+                }
+            ]);
+        });
+
         it('should return null if it fails', function() {
             var result = offendersHelpers.backtraceToArray('http://pouet.com/js/jquery.footer-transverse-min-v1.0.20.js:1 /http://pouet.com/js/main.js:1');
 
@@ -299,6 +340,17 @@ describe('offendersHelpers', function() {
             });
         });
 
+        it('should handle line breaks inside the string', function() {
+            var result = offendersHelpers.cssOffenderPattern('.card-mask-wrap { -moz-transform: translate3d(0, \n-288px\n, 0) } // was required by firefox 15 and earlier [inline CSS] @ 29:3');
+
+            result.should.deep.equal({
+                css: '.card-mask-wrap { -moz-transform: translate3d(0, -288px, 0) } // was required by firefox 15 and earlier',
+                file: null,
+                line: 29,
+                column: 3
+            });
+        }); 
+
     });
 
     describe('fileWithSizePattern', function() {

test/core/phantomasWrapperTest.js

@@ -51,11 +51,14 @@ describe('phantomasWrapper', function() {
             done('Error: unwanted success');
 
         }).fail(function(err) {
+            try {
+                should.exist(err);
+                err.should.equal(254);
 
-            should.exist(err);
-            err.should.equal(254);
-
-            done();
+                done();
+            } catch(error) {
+                done(error);
+            }
         });
     });
 
@@ -72,16 +75,20 @@ describe('phantomasWrapper', function() {
             }
         }).then(function(data) {
             /*jshint -W030 */
-            
-            data.should.be.an('object');
-            data.should.have.a.property('generator');
-            data.generator.should.contain('phantomas');
-            data.should.have.a.property('url').that.equals(url);
-            data.should.have.a.property('metrics').that.is.an('object').not.empty;
-            data.should.have.a.property('offenders').that.is.an('object').not.empty;
-            data.offenders.should.have.a.property('javascriptExecutionTree').that.is.a('array').not.empty;
 
-            done();
+            try {            
+                data.should.be.an('object');
+                data.should.have.a.property('generator');
+                data.generator.should.contain('phantomas');
+                data.should.have.a.property('url').that.equals(url);
+                data.should.have.a.property('metrics').that.is.an('object').not.empty;
+                data.should.have.a.property('offenders').that.is.an('object').not.empty;
+                data.offenders.should.have.a.property('javascriptExecutionTree').that.is.a('array').not.empty;
+
+                done();
+            } catch(error) {
+                done(error);
+            }
         }).fail(function(err) {
             done(err);
         });

test/core/redownloadTest.js

@@ -1,9 +1,9 @@
 var should = require('chai').should();
-var weightChecker = require('../../lib/tools/weightChecker/weightChecker');
+var redownload = require('../../lib/tools/redownload/redownload');
 var fs = require('fs');
 var path = require('path');
 
-describe('weightChecker', function() {
+describe('redownload', function() {
     
     it('should download a list of files', function(done) {
         this.timeout(10000);
@@ -84,6 +84,18 @@ describe('weightChecker', function() {
                 isCSS: true,
                 type: 'css'
             },
+            {
+                method: 'GET',
+                url: 'http://localhost:8388/xml.xml',
+                requestHeaders: {
+                    'User-Agent': 'something',
+                   Referer: 'http://www.google.fr/',
+                   Accept: '*/*'
+                },
+                status: 200,
+                isXML: true,
+                type: 'xml'
+            },
             {
                 method: 'GET',
                 url: 'about:blank',
@@ -109,49 +121,49 @@ describe('weightChecker', function() {
             }
         };
 
-        weightChecker.recheckAllFiles(data)
+        redownload.recheckAllFiles(data)
 
         .then(function(data) {
-            data.toolsResults.should.have.a.property('weightChecker');
-            data.toolsResults.weightChecker.should.have.a.property('metrics');
-            data.toolsResults.weightChecker.should.have.a.property('offenders');
+            data.toolsResults.should.have.a.property('redownload');
+            data.toolsResults.redownload.should.have.a.property('metrics');
+            data.toolsResults.redownload.should.have.a.property('offenders');
 
-            data.toolsResults.weightChecker.offenders.should.have.a.property('totalWeight');
-            data.toolsResults.weightChecker.offenders.totalWeight.totalWeight.should.be.above(0);
-            data.toolsResults.weightChecker.offenders.totalWeight.byType.html.requests.length.should.equal(1);
-            data.toolsResults.weightChecker.offenders.totalWeight.byType.js.requests.length.should.equal(2);
-            data.toolsResults.weightChecker.offenders.totalWeight.byType.css.requests.length.should.equal(1);
-            data.toolsResults.weightChecker.offenders.totalWeight.byType.image.requests.length.should.equal(2);
-            data.toolsResults.weightChecker.offenders.totalWeight.byType.other.requests.length.should.equal(0);
+            data.toolsResults.redownload.offenders.should.have.a.property('totalWeight');
+            data.toolsResults.redownload.offenders.totalWeight.totalWeight.should.be.above(0);
+            data.toolsResults.redownload.offenders.totalWeight.byType.html.requests.length.should.equal(1);
+            data.toolsResults.redownload.offenders.totalWeight.byType.js.requests.length.should.equal(2);
+            data.toolsResults.redownload.offenders.totalWeight.byType.css.requests.length.should.equal(1);
+            data.toolsResults.redownload.offenders.totalWeight.byType.image.requests.length.should.equal(2);
+            data.toolsResults.redownload.offenders.totalWeight.byType.other.requests.length.should.equal(1);
 
-            data.toolsResults.weightChecker.offenders.should.have.a.property('imageOptimization');
-            data.toolsResults.weightChecker.offenders.imageOptimization.totalGain.should.be.above(0);
-            data.toolsResults.weightChecker.offenders.imageOptimization.images.length.should.equal(2);
+            data.toolsResults.redownload.offenders.should.have.a.property('imageOptimization');
+            data.toolsResults.redownload.offenders.imageOptimization.totalGain.should.be.above(0);
+            data.toolsResults.redownload.offenders.imageOptimization.images.length.should.equal(2);
 
-            data.toolsResults.weightChecker.offenders.should.have.a.property('gzipCompression');
-            data.toolsResults.weightChecker.offenders.gzipCompression.totalGain.should.be.above(0);
-            data.toolsResults.weightChecker.offenders.gzipCompression.files.length.should.equal(4);
+            data.toolsResults.redownload.offenders.should.have.a.property('gzipCompression');
+            data.toolsResults.redownload.offenders.gzipCompression.totalGain.should.be.above(0);
+            data.toolsResults.redownload.offenders.gzipCompression.files.length.should.equal(5);
 
-            data.toolsResults.weightChecker.offenders.should.have.a.property('fileMinification');
-            data.toolsResults.weightChecker.offenders.fileMinification.totalGain.should.be.above(0);
-            data.toolsResults.weightChecker.offenders.fileMinification.files.length.should.equal(2);
+            data.toolsResults.redownload.offenders.should.have.a.property('fileMinification');
+            data.toolsResults.redownload.offenders.fileMinification.totalGain.should.be.above(0);
+            data.toolsResults.redownload.offenders.fileMinification.files.length.should.equal(2);
 
-            data.toolsResults.weightChecker.metrics.should.have.a.property('totalRequests').that.equals(6);
-            data.toolsResults.weightChecker.offenders.should.have.a.property('totalRequests');
-            data.toolsResults.weightChecker.offenders.totalRequests.byType.html.length.should.equal(1);
-            data.toolsResults.weightChecker.offenders.totalRequests.byType.js.length.should.equal(2);
-            data.toolsResults.weightChecker.offenders.totalRequests.byType.css.length.should.equal(1);
-            data.toolsResults.weightChecker.offenders.totalRequests.byType.image.length.should.equal(2);
-            data.toolsResults.weightChecker.offenders.totalRequests.byType.json.length.should.equal(0);
-            data.toolsResults.weightChecker.offenders.totalRequests.byType.webfont.length.should.equal(0);
-            data.toolsResults.weightChecker.offenders.totalRequests.byType.video.length.should.equal(0);
-            data.toolsResults.weightChecker.offenders.totalRequests.byType.other.length.should.equal(0);
+            data.toolsResults.redownload.metrics.should.have.a.property('totalRequests').that.equals(7);
+            data.toolsResults.redownload.offenders.should.have.a.property('totalRequests');
+            data.toolsResults.redownload.offenders.totalRequests.byType.html.length.should.equal(1);
+            data.toolsResults.redownload.offenders.totalRequests.byType.js.length.should.equal(2);
+            data.toolsResults.redownload.offenders.totalRequests.byType.css.length.should.equal(1);
+            data.toolsResults.redownload.offenders.totalRequests.byType.image.length.should.equal(2);
+            data.toolsResults.redownload.offenders.totalRequests.byType.json.length.should.equal(0);
+            data.toolsResults.redownload.offenders.totalRequests.byType.webfont.length.should.equal(0);
+            data.toolsResults.redownload.offenders.totalRequests.byType.video.length.should.equal(0);
+            data.toolsResults.redownload.offenders.totalRequests.byType.other.length.should.equal(1);
 
-            data.toolsResults.weightChecker.metrics.should.have.a.property('smallRequests').that.equals(0);
-            data.toolsResults.weightChecker.offenders.should.have.a.property('smallRequests');
-            data.toolsResults.weightChecker.offenders.smallRequests.byType.js.length.should.equal(0);
-            data.toolsResults.weightChecker.offenders.smallRequests.byType.css.length.should.equal(0);
-            data.toolsResults.weightChecker.offenders.smallRequests.byType.image.length.should.equal(0);
+            data.toolsResults.redownload.metrics.should.have.a.property('smallRequests').that.equals(0);
+            data.toolsResults.redownload.offenders.should.have.a.property('smallRequests');
+            data.toolsResults.redownload.offenders.smallRequests.byType.js.length.should.equal(0);
+            data.toolsResults.redownload.offenders.smallRequests.byType.css.length.should.equal(0);
+            data.toolsResults.redownload.offenders.smallRequests.byType.image.length.should.equal(0);
 
             done();
         })
@@ -175,7 +187,7 @@ describe('weightChecker', function() {
             type: 'js'
         };
 
-        weightChecker.redownloadEntry(entry)
+        redownload.redownloadEntry(entry)
 
         .then(function(newEntry) {
 
@@ -209,7 +221,7 @@ describe('weightChecker', function() {
             contentType: 'image/png'
         };
 
-        weightChecker.redownloadEntry(entry)
+        redownload.redownloadEntry(entry)
 
         .then(function(newEntry) {
 
@@ -246,7 +258,7 @@ describe('weightChecker', function() {
             contentLength: 999
         };
 
-        weightChecker.redownloadEntry(entry)
+        redownload.redownloadEntry(entry)
 
         .then(function(newEntry) {
             newEntry.weightCheck.should.have.a.property('message').that.equals('error while downloading: 404');
@@ -273,7 +285,7 @@ describe('weightChecker', function() {
             contentLength: 999
         };
 
-        weightChecker.redownloadEntry(entry)
+        redownload.redownloadEntry(entry)
 
         .then(function(newEntry) {
             newEntry.should.not.have.a.property('weightCheck');
@@ -286,7 +298,7 @@ describe('weightChecker', function() {
     });
 
     it('should listRequestWeight', function() {
-        var totalWeightObj = weightChecker.listRequestWeight([{
+        var totalWeightObj = redownload.listRequestWeight([{
             method: 'GET',
             url: 'http://localhost:8388/jquery1.8.3.js',
             requestHeaders: {
@@ -316,7 +328,7 @@ describe('weightChecker', function() {
     });
 
     it('should listRequestWeight even if download failed', function() {
-        var totalWeightObj = weightChecker.listRequestWeight([{
+        var totalWeightObj = redownload.listRequestWeight([{
             method: 'GET',
             url: 'http://localhost:8388/jquery1.8.3.js',
             requestHeaders: {

test/fixtures/settings.json

@@ -1,6 +1,8 @@
 {
     "serverPort": "8387",
+    "phantomasEngine": "webkit",
     "googleAnalyticsId": "",
+    "screenshotWidth": 400,
     
     "authorizedKeys": {
         "1234567890": "contact@gaelmetais.com"

test/www/jquery-page.html

@@ -190,6 +190,11 @@
         $li.parentsUntil('body', 'div');
         $li.siblings();
         $li.siblings('li');
+
+        $.ajax({
+            url: 'xml.xml',
+            async: false
+        });
     </script>
 </body>
 </html>

test/www/xml.xml

@@ -0,0 +1,80 @@
+<?xml version="1.0"?>
+<catalog>
+   <book id="bk101">
+      <author>Gambardella, Matthew</author>
+      <title>XML Developer's Guide</title>
+      <genre>Computer</genre>
+      <price>44.95</price>
+      <publish_date>2000-10-01</publish_date>
+      <description>An in-depth look at creating applications 
+      with XML.</description>
+   </book>
+   <book id="bk102">
+      <author>Ralls, Kim</author>
+      <title>Midnight Rain</title>
+      <genre>Fantasy</genre>
+      <price>5.95</price>
+      <publish_date>2000-12-16</publish_date>
+      <description>A former architect battles corporate zombies, 
+      an evil sorceress, and her own childhood to become queen 
+      of the world.</description>
+   </book>
+   <book id="bk103">
+      <author>Corets, Eva</author>
+      <title>Maeve Ascendant</title>
+      <genre>Fantasy</genre>
+      <price>5.95</price>
+      <publish_date>2000-11-17</publish_date>
+      <description>After the collapse of a nanotechnology 
+      society in England, the young survivors lay the 
+      foundation for a new society.</description>
+   </book>
+   <book id="bk104">
+      <author>Corets, Eva</author>
+      <title>Oberon's Legacy</title>
+      <genre>Fantasy</genre>
+      <price>5.95</price>
+      <publish_date>2001-03-10</publish_date>
+      <description>In post-apocalypse England, the mysterious 
+      agent known only as Oberon helps to create a new life 
+      for the inhabitants of London. Sequel to Maeve 
+      Ascendant.</description>
+   </book>
+   <book id="bk105">
+      <author>Corets, Eva</author>
+      <title>The Sundered Grail</title>
+      <genre>Fantasy</genre>
+      <price>5.95</price>
+      <publish_date>2001-09-10</publish_date>
+      <description>The two daughters of Maeve, half-sisters, 
+      battle one another for control of England. Sequel to 
+      Oberon's Legacy.</description>
+   </book>
+   <book id="bk106">
+      <author>Randall, Cynthia</author>
+      <title>Lover Birds</title>
+      <genre>Romance</genre>
+      <price>4.95</price>
+      <publish_date>2000-09-02</publish_date>
+      <description>When Carla meets Paul at an ornithology 
+      conference, tempers fly as feathers get ruffled.</description>
+   </book>
+   <book id="bk107">
+      <author>Thurman, Paula</author>
+      <title>Splish Splash</title>
+      <genre>Romance</genre>
+      <price>4.95</price>
+      <publish_date>2000-11-02</publish_date>
+      <description>A deep sea diver finds true love twenty 
+      thousand leagues beneath the sea.</description>
+   </book>
+   <book id="bk108">
+      <author>Knorr, Stefan</author>
+      <title>Creepy Crawlies</title>
+      <genre>Horror</genre>
+      <price>4.95</price>
+      <publish_date>2000-12-06</publish_date>
+      <description>An anthology of horror stories about roaches,
+      centipedes, scorpions  and other insects.</description>
+   </book>
+</catalog>